def test_parse_config_with_files(self):
        temp_dir = tempfile.mkdtemp()
        site_model_input = helpers.touch(dir=temp_dir, content="foo")
        job_config = helpers.touch(dir=temp_dir, content="""
[general]
calculation_mode = classical
[site]
site_model_file = %s
maximum_distance=0
truncation_level=0
random_seed=0
    """ % site_model_input)

        try:
            exp_base_path = os.path.dirname(job_config)

            expected_params = {
                'base_path': exp_base_path,
                'calculation_mode': 'classical',
                'truncation_level': '0',
                'random_seed': '0',
                'maximum_distance': '0',
                'inputs': {'site_model': site_model_input},
            }

            params = engine.parse_config(open(job_config, 'r'))
            self.assertEqual(expected_params, params)
            self.assertEqual(['site_model'], params['inputs'].keys())
            self.assertEqual([site_model_input], params['inputs'].values())
        finally:
            shutil.rmtree(temp_dir)

    def test_load_from_file_with_local_overriding_global(self):
        """
        The config data in the local and global files is loaded correctly.
        The local data will override the global one.
        """
        content = '''
            [A]
            a=1
            b=c

            [B]
            b=2'''
        site_path = touch(content=textwrap.dedent(content))
        os.environ["OQ_SITE_CFG_PATH"] = site_path
        content = '''
            [A]
            a=2
            d=e

            [D]
            c=d-1
            d=4'''
        local_path = touch(content=textwrap.dedent(content))
        os.environ["OQ_LOCAL_CFG_PATH"] = local_path
        config.Config().cfg.clear()
        config.Config()._load_from_file()
        self.assertEqual(["A", "B", "D"],
                         sorted(config.Config().cfg.keys()))
        self.assertEqual({"a": "2", "b": "c", "d": "e"},
                         config.Config().cfg.get("A"))
        self.assertEqual({"b": "2"}, config.Config().cfg.get("B"))
        self.assertEqual({"c": "d-1", "d": "4"}, config.Config().cfg.get("D"))

    def test_load_from_file_with_local_and_global(self):
        """
        The config data in the local and global files is loaded correctly.
        """
        content = """
            [A]
            a=1
            b=c

            [B]
            b=2"""
        site_path = touch(content=textwrap.dedent(content))
        os.environ["OQ_SITE_CFG_PATH"] = site_path
        content = """
            [C]
            c=3
            d=e

            [D]
            d=4"""
        local_path = touch(content=textwrap.dedent(content))
        os.environ["OQ_LOCAL_CFG_PATH"] = local_path
        config.Config().cfg.clear()
        config.Config()._load_from_file()
        self.assertEqual(["A", "B", "C", "D"], sorted(config.Config().cfg.keys()))
        self.assertEqual({"a": "1", "b": "c"}, config.Config().cfg.get("A"))
        self.assertEqual({"b": "2"}, config.Config().cfg.get("B"))
        self.assertEqual({"c": "3", "d": "e"}, config.Config().cfg.get("C"))
        self.assertEqual({"d": "4"}, config.Config().cfg.get("D"))

    def test_risk_mandatory_parameters(self):
        sections = [
            config.RISK_SECTION, config.HAZARD_SECTION, config.GENERAL_SECTION]

        dummy_exposure = helpers.touch()

        params = {}

        validator = config.default_validators(sections, params)
        self.assertFalse(validator.is_valid()[0])

        params = {config.EXPOSURE: dummy_exposure,
                  config.DEPTHTO1PT0KMPERSEC: "33.33",
                  config.VS30_TYPE: "measured"}

        validator = config.default_validators(sections, params)
        self.assertFalse(validator.is_valid()[0])

        params = {config.EXPOSURE: dummy_exposure,
                  config.REGION_GRID_SPACING: '0.5',
                  config.DEPTHTO1PT0KMPERSEC: "33.33",
                  config.VS30_TYPE: "measured"}

        validator = config.default_validators(sections, params)
        self.assertFalse(validator.is_valid()[0])

        params = {config.EXPOSURE: dummy_exposure,
                  config.INPUT_REGION: "1.0, 2.0, 3.0, 4.0, 5.0, 6.0",
                  config.REGION_GRID_SPACING: '0.5',
                  config.DEPTHTO1PT0KMPERSEC: "33.33",
                  config.VS30_TYPE: "measured"}

        validator = config.default_validators(sections, params)
        self.assertTrue(validator.is_valid()[0])

    def test_optimize_source_model(self):
        in_file = helpers.touch(content=self.input_source_model)
        out_file = helpers.touch(content=self.output_source_model)
        area_src_disc = 50.0
        try:
            source_input.optimize_source_model(in_file, area_src_disc,
                                               out_file)
            expected = ElementTree.tostring(
                ElementTree.XML(self.output_source_model))

            actual = ElementTree.tostring(
                ElementTree.XML(open(out_file).read()))
            self.assertEqual(expected, actual)
        finally:
            os.unlink(in_file)
            os.unlink(out_file)

    def test_parse_config_with_sites_csv(self):
        sites_csv = helpers.touch(content='1.0,2.1\n3.0,4.1\n5.0,6.1')
        try:
            source = StringIO.StringIO("""
[general]
calculation_mode = classical
[geometry]
sites_csv = %s
[misc]
maximum_distance=0
truncation_level=3
random_seed=5
""" % sites_csv)
            source.name = 'path/to/some/job.ini'
            exp_base_path = os.path.dirname(
                os.path.join(os.path.abspath('.'), source.name))

            expected_params = {
                'base_path': exp_base_path,
                'sites': 'MULTIPOINT(1.0 2.1, 3.0 4.1, 5.0 6.1)',
                'calculation_mode': 'classical',
                'truncation_level': '3',
                'random_seed': '5',
                'maximum_distance': '0',
                'inputs': {},
            }

            params = engine.parse_config(source)
            self.assertEqual(expected_params, params)
        finally:
            os.unlink(sites_csv)

    def test_prepare_path_parameters(self):
        content = """
            [GENERAL]
            CALCULATION_MODE = Event Based
            OUTPUT_DIR = output

            [HAZARD]
            SOURCE_MODEL_LOGIC_TREE_FILE = source-model.xml
            GMPE_LOGIC_TREE_FILE = gmpe.xml

            [RISK]
            EXPOSURE = /absolute/exposure.xml
            VULNERABILITY = vulnerability.xml
            """
        config_path = helpers.touch(content=textwrap.dedent(content))

        params, sections = _parse_config_file(config_path)
        params, sections = _prepare_config_parameters(params, sections)

        self.assertEqual(
            {
                "BASE_PATH": gettempdir(),
                "OUTPUT_DIR": "output",
                "SOURCE_MODEL_LOGIC_TREE_FILE": "source-model.xml",
                "GMPE_LOGIC_TREE_FILE": "gmpe.xml",
                "EXPOSURE": "/absolute/exposure.xml",
                "VULNERABILITY": "vulnerability.xml",
                "CALCULATION_MODE": "Event Based",
            },
            params,
        )
        self.assertEqual(["GENERAL", "HAZARD", "RISK"], sorted(sections))

    def test_prepare_path_parameters(self):
        content = '''
            [GENERAL]
            CALCULATION_MODE = Event Based
            OUTPUT_DIR = output

            [HAZARD]
            SOURCE_MODEL_LOGIC_TREE_FILE = source-model.xml
            GMPE_LOGIC_TREE_FILE = gmpe.xml

            [RISK]
            EXPOSURE = /absolute/exposure.xml
            VULNERABILITY = vulnerability.xml
            '''
        config_path = helpers.touch(content=textwrap.dedent(content))

        params, sections = _parse_config_file(config_path)
        params, sections = _prepare_config_parameters(params, sections)

        self.assertEqual(
            {'BASE_PATH': gettempdir(),
             'OUTPUT_DIR': 'output',
             'SOURCE_MODEL_LOGIC_TREE_FILE': 'source-model.xml',
             'GMPE_LOGIC_TREE_FILE': 'gmpe.xml',
             'EXPOSURE': '/absolute/exposure.xml',
             'VULNERABILITY': 'vulnerability.xml',
             'CALCULATION_MODE': 'Event Based'},
            params)
        self.assertEqual(['GENERAL', 'HAZARD', 'RISK'], sorted(sections))

 def test_optimize_source_model(self):
     in_file = helpers.touch(content=self.input_source_model)
     out_file = helpers.touch(content=self.output_source_model)
     area_src_disc = 50.0
     try:
         source_input.optimize_source_model(in_file, area_src_disc,
                                            out_file)
         expected = etree.tostring(
             etree.parse(StringIO.StringIO(self.output_source_model)),
             pretty_print=True
         )
         actual = etree.tostring(etree.parse(out_file), pretty_print=True)
         self.assertEqual(expected, actual)
     finally:
         os.unlink(in_file)
         os.unlink(out_file)

    def test_parse_config_with_files(self):
        site_model_input = helpers.touch(content="foo")

        try:
            source = StringIO.StringIO("""
[general]
calculation_mode = classical
[site]
site_model_file = %s
maximum_distance=0
truncation_level=0
random_seed=0
    """ % site_model_input)

            # Add a 'name' to make this look like a real file:
            source.name = 'path/to/some/job.ini'
            exp_base_path = os.path.dirname(
                os.path.join(os.path.abspath('.'), source.name))

            expected_params = {
                'base_path': exp_base_path,
                'calculation_mode': 'classical',
                'truncation_level': '0',
                'random_seed': '0',
                'maximum_distance': '0'
            }

            params, files = engine.parse_config(source)
            self.assertEqual(expected_params, params)
            self.assertEqual(['site_model_file'], files.keys())
            self.assertEqual('acbd18db4cc2f85cedef654fccc4a4d8',
                             files['site_model_file'].digest)
        finally:
            os.unlink(site_model_input)

    def test_parse_config_with_files_no_force_inputs(self):
        site_model_input = helpers.touch(content="foo")

        source = StringIO.StringIO("""
[general]
calculation_mode = classical
[site]
site_model_file = %s
maximum_distance=0
truncation_level=0
random_seed=0
""" % site_model_input)

        # Add a 'name' to make this look like a real file:
        source.name = 'path/to/some/job.ini'
        exp_base_path = os.path.dirname(
            os.path.join(os.path.abspath('.'), source.name))

        expected_params = {
            'base_path': exp_base_path,
            'force_inputs': False,
            'calculation_mode': 'classical',
            'truncation_level': '0',
            'random_seed': '0',
            'maximum_distance': '0'
        }

        # Run first with force_inputs=True to create the new Input.
        params, expected_files = engine2.parse_config(
            source, force_inputs=True)

        # In order for us to reuse the existing input, we need to associate
        # each input with a successful job.
        job = engine2.prepare_job(getpass.getuser())

        job.hazard_calculation = engine2.create_hazard_calculation(
            job.owner, params, expected_files.values())
        job.status = 'complete'
        job.save()

        # Now run twice with force_inputs=False (the default).
        source.seek(0)
        params1, files1 = engine2.parse_config(source)
        source.seek(0)
        params2, files2 = engine2.parse_config(source)

        # Check the params just for sanity.
        self.assertEqual(expected_params, params1)
        self.assertEqual(expected_params, params2)

        # Finally, check that the Input returned by the latest 2 calls matches
        # the input we created above.
        self.assertEqual(len(expected_files), len(files1))
        self.assertEqual(len(expected_files), len(files2))

        self.assertEqual(
            expected_files['site_model_file'].id, files1['site_model_file'].id)
        self.assertEqual(
            expected_files['site_model_file'].id, files2['site_model_file'].id)

    def test_model_content_unknown_content_type(self):
        test_file = helpers.touch()

        params = dict(GMPE_LOGIC_TREE_FILE=test_file, BASE_PATH='/')
        engine._insert_input_files(params, self.job, True)

        [glt] = models.inputs4job(self.job.id, input_type="gsim_logic_tree")
        self.assertEqual('unknown', glt.model_content.content_type)

    def test_parse_config_with_files_force_inputs(self):
        site_model_input = helpers.touch(content="site model")
        sm_lt_input = helpers.touch(content="source model logic tree")
        gsim_lt_input = helpers.touch(content="gsim logic tree")

        source = StringIO.StringIO("""
[hazard_or_whatever]
calculation_mode = classical
gsim_logic_tree_file = %s
source_model_logic_tree_file = %s
site_model_file = %s
not_a_valid_file = foo.xml
""" % (gsim_lt_input, sm_lt_input, site_model_input))

        # Add a 'name' to make this look like a real file:
        source.name = 'path/to/some/job.ini'
        exp_base_path = os.path.dirname(
            os.path.join(os.path.abspath('.'), source.name))

        expected_params = {
            'base_path': exp_base_path,
            'force_inputs': True,
            'calculation_mode': 'classical',
            'not_a_valid_file': 'foo.xml',
        }

        params, files = engine2.parse_config(source, force_inputs=True)

        expected_files = {
            'site_model_file': models.Input.objects.filter(
                input_type='site_model').latest('id'),
            'source_model_logic_tree_file': models.Input.objects.filter(
                input_type='lt_source').latest('id'),
            'gsim_logic_tree_file': models.Input.objects.filter(
                input_type='lt_gsim').latest('id'),
        }

        self.assertEqual(expected_params, params)

        self.assertEqual(len(expected_files), len(files))
        for key in expected_files:
            self.assertEqual(expected_files[key].id, files[key].id)

    def test_parse_config_with_files_no_force_inputs(self):
        site_model_input = helpers.touch(content="foo")

        source = StringIO.StringIO("""
[general]
calculation_mode = classical
[site]
site_model_file = %s
not_a_valid_file = foo.xml
""" % site_model_input)

        # Add a 'name' to make this look like a real file:
        source.name = 'path/to/some/job.ini'
        exp_base_path = os.path.dirname(
            os.path.join(os.path.abspath('.'), source.name))

        expected_params = {
            'base_path': exp_base_path,
            'force_inputs': False,
            'calculation_mode': 'classical',
            'not_a_valid_file': 'foo.xml',
        }

        # Run first with force_inputs=True to create the new Input.
        _, expected_files = engine2.parse_config(source, force_inputs=True)

        # In order for us to reuse the existing input, we need to associate
        # each input with a successful job.
        job = engine2.prepare_job(getpass.getuser())
        job.status = 'complete'
        job.save()
        for inp in expected_files.values():
            i2j = models.Input2job(input=inp, oq_job=job)
            i2j.save()

        # Now run twice with force_inputs=False (the default).
        source.seek(0)
        params1, files1 = engine2.parse_config(source)
        source.seek(0)
        params2, files2 = engine2.parse_config(source)

        # Check the params just for sanity.
        self.assertEqual(expected_params, params1)
        self.assertEqual(expected_params, params2)

        # Finally, check that the Input returned by the latest 2 calls matches
        # the input we created above.
        self.assertEqual(len(expected_files), len(files1))
        self.assertEqual(len(expected_files), len(files2))

        for key in expected_files:
            self.assertEqual(expected_files[key].id, files1[key].id)
            self.assertEqual(expected_files[key].id, files2[key].id)

 def test_get_with_known_section(self):
     """
     get() will correctly return configuration data for known sections.
     """
     content = '''
         [E]
         f=6
         g=h'''
     site_path = touch(content=textwrap.dedent(content))
     os.environ["OQ_SITE_CFG_PATH"] = site_path
     config.Config().cfg.clear()
     config.Config()._load_from_file()
     self.assertEqual({"f": "6", "g": "h"}, config.Config().get("E"))

    def test_model_content_detect_content_type(self):
        # Test detection of the content type (using the file extension).
        test_file = helpers.touch(suffix=".html")

        # We use the gmpe logic tree as our test target because there is no
        # parsing required in the function under test. Thus, we can put
        # whatever test garbage we want in the file, or just use an empty file
        # (which is the case here).
        params = dict(GMPE_LOGIC_TREE_FILE=test_file, BASE_PATH='/')
        engine._insert_input_files(params, self.job, True)

        [glt] = models.inputs4job(self.job.id, input_type="gsim_logic_tree")
        self.assertEqual('html', glt.model_content.content_type)

    def test_parse_file(self):
        content = """
            [GENERAL]
            CALCULATION_MODE = Event Based

            [HAZARD]
            MINIMUM_MAGNITUDE = 5.0
            """
        config_path = helpers.touch(dir=gettempdir(), content=textwrap.dedent(content))

        params, sections = _parse_config_file(config_path)

        self.assertEqual(
            {"BASE_PATH": gettempdir(), "CALCULATION_MODE": "Event Based", "MINIMUM_MAGNITUDE": "5.0"}, params
        )
        self.assertEqual(["GENERAL", "HAZARD"], sorted(sections))

    def test_load_from_file_with_global(self):
        """The config data in the global file is loaded correctly."""
        content = '''
            [A]
            a=1
            b=c

            [B]
            b=2'''
        site_path = touch(content=textwrap.dedent(content))
        os.environ["OQ_SITE_CFG_PATH"] = site_path
        config.Config().cfg.clear()
        config.Config()._load_from_file()
        self.assertEqual(["A", "B"], sorted(config.Config().cfg.keys()))
        self.assertEqual({"a": "1", "b": "c"}, config.Config().cfg.get("A"))
        self.assertEqual({"b": "2"}, config.Config().cfg.get("B"))

    def test_load_from_file_with_local(self):
        """The config data in the local file is loaded correctly."""
        content = """
            [C]
            c=3
            d=e

            [D]
            d=4"""
        local_path = touch(content=textwrap.dedent(content))
        os.environ["OQ_LOCAL_CFG_PATH"] = local_path
        config.Config().cfg.clear()
        config.Config()._load_from_file()
        self.assertEqual(["C", "D"], sorted(config.Config().cfg.keys()))
        self.assertEqual({"c": "3", "d": "e"}, config.Config().cfg.get("C"))
        self.assertEqual({"d": "4"}, config.Config().cfg.get("D"))

    def test_file_path_validation(self):
        # existing file
        params = {'EXPOSURE': helpers.touch(), 'BASE_PATH': '/'}

        validator = config.FilePathValidator(params)
        self.assertTrue(validator.is_valid()[0])

        # non-existing file
        params = {'VULNERABILITY': '/a/b/c',
                  'SOURCE_MODEL_LOGIC_TREE_FILE': '/a/b/c',
                  'BASE_PATH': '/'}

        validator = config.FilePathValidator(params)
        valid, messages = validator.is_valid()

        self.assertFalse(valid)
        self.assertEquals(2, len(messages))