def setUp(self):
        django.setup()

        self.src_file_1 = source_test_utils.create_source()
        self.src_file_2 = source_test_utils.create_source()
        self.src_file_3 = source_test_utils.create_source()
        self.src_file_4 = source_test_utils.create_source()

        self.job_exe_1 = job_test_utils.create_job_exe()
        self.recipe_job_1 = recipe_test_utils.create_recipe_job(job=self.job_exe_1.job)
        self.product_1 = prod_test_utils.create_product(self.job_exe_1, has_been_published=True)
        self.product_2 = prod_test_utils.create_product(self.job_exe_1, has_been_published=True)
        FileAncestryLink.objects.create(ancestor=self.src_file_1, descendant=self.product_1, job_exe=self.job_exe_1,
                                        job=self.job_exe_1.job, recipe=self.recipe_job_1.recipe)
        FileAncestryLink.objects.create(ancestor=self.src_file_1, descendant=self.product_2, job_exe=self.job_exe_1,
                                        job=self.job_exe_1.job, recipe=self.recipe_job_1.recipe)
        FileAncestryLink.objects.create(ancestor=self.src_file_2, descendant=self.product_1, job_exe=self.job_exe_1,
                                        job=self.job_exe_1.job, recipe=self.recipe_job_1.recipe)
        FileAncestryLink.objects.create(ancestor=self.src_file_2, descendant=self.product_2, job_exe=self.job_exe_1,
                                        job=self.job_exe_1.job, recipe=self.recipe_job_1.recipe)

        self.job_exe_2 = job_test_utils.create_job_exe()
        self.recipe_job_2 = recipe_test_utils.create_recipe_job(job=self.job_exe_2.job)
        self.product_3 = prod_test_utils.create_product(self.job_exe_2, has_been_published=True)
        FileAncestryLink.objects.create(ancestor=self.src_file_3, descendant=self.product_3, job_exe=self.job_exe_2,
                                        job=self.job_exe_2.job, recipe=self.recipe_job_2.recipe)
        FileAncestryLink.objects.create(ancestor=self.src_file_4, descendant=self.product_3, job_exe=self.job_exe_2,
                                        job=self.job_exe_2.job, recipe=self.recipe_job_2.recipe)

    def test_no_conditions(self):
        """Tests calling IngestTriggerCondition.is_condition_met() with no conditions"""

        condition = IngestTriggerCondition(None, None)
        source_file = source_test_utils.create_source(media_type='text/plain')
        
        self.assertEqual(condition.is_condition_met(source_file), True)

    def test_media_type_mismatch(self):
        """Tests calling IngestTriggerCondition.is_condition_met() with a mismatched media type"""

        condition = IngestTriggerCondition('application/json', None)
        source_file = source_test_utils.create_source(media_type='text/plain')
        
        self.assertEqual(condition.is_condition_met(source_file), False)

    def test_media_type_match(self):
        """Tests calling IngestTriggerCondition.is_condition_met() with a matching media type"""

        condition = IngestTriggerCondition('text/plain', None)
        source_file = source_test_utils.create_source(media_type='text/plain')
        
        self.assertEqual(condition.is_condition_met(source_file), True)

def create_ingest(file_name='test.txt', status='TRANSFERRING', transfer_started=None, transfer_ended=None,
                  ingest_started=None, ingest_ended=None, data_started=None, data_ended=None, workspace=None,
                  strike=None, source_file=None):
    if not workspace:
        workspace = storage_test_utils.create_workspace()
    if not strike:
        strike = create_strike()
    if not source_file:
        source_file = source_test_utils.create_source(file_name=file_name, data_started=data_started,
                                                      data_ended=data_ended, workspace=workspace)
    if not transfer_started:
        transfer_started = timezone.now()
    if status not in ['QUEUED', 'TRANSFERRING'] and not ingest_started:
        ingest_started = timezone.now()
    if status not in ['QUEUED', 'TRANSFERRING', 'INGESTING'] and not ingest_ended:
        ingest_ended = timezone.now()

    try:
        job_type = Ingest.objects.get_ingest_job_type()
    except:
        job_type = job_utils.create_job_type()
    job = job_utils.create_job(job_type=job_type)
    job_utils.create_job_exe(job=job)

    return Ingest.objects.create(file_name=file_name, file_size=source_file.file_size, status=status, job=job,
                                 bytes_transferred=source_file.file_size, transfer_started=transfer_started,
                                 transfer_ended=transfer_ended, media_type='text/plain', ingest_started=ingest_started,
                                 ingest_ended=ingest_ended, workspace=workspace, strike=strike, source_file=source_file)

    def setUp(self):
        django.setup()

        self.source = source_test_utils.create_source()

        try:
            import ingest.test.utils as ingest_test_utils
            self.ingest = ingest_test_utils.create_ingest(source_file=self.source)
        except:
            self.ingest = None

        try:
            import product.test.utils as product_test_utils
            self.product1 = product_test_utils.create_product(is_superseded=True)

            product_test_utils.create_file_link(ancestor=self.source, descendant=self.product1)
        except:
            self.product1 = None

        try:
            import product.test.utils as product_test_utils
            self.product2 = product_test_utils.create_product()

            product_test_utils.create_file_link(ancestor=self.source, descendant=self.product2)
        except:
            self.product2 = None

    def setUp(self):
        django.setup()

        workspace = storage_test_utils.create_workspace()
        source_file = source_test_utils.create_source(workspace=workspace)
        self.event = trigger_test_utils.create_trigger_event()

        interface_1 = {
            "version": "1.0",
            "command": "test_command",
            "command_arguments": "test_arg",
            "input_data": [{"name": "Test Input 1", "type": "file", "media_types": ["text/plain"]}],
            "output_data": [{"name": "Test Output 1", "type": "files", "media_type": "image/png"}],
        }
        self.job_type_1 = job_test_utils.create_job_type(interface=interface_1)

        interface_2 = {
            "version": "1.0",
            "command": "test_command",
            "command_arguments": "test_arg",
            "input_data": [{"name": "Test Input 2", "type": "files", "media_types": ["image/png", "image/tiff"]}],
            "output_data": [{"name": "Test Output 2", "type": "file"}],
        }
        self.job_type_2 = job_test_utils.create_job_type(interface=interface_2)

        definition = {
            "version": "1.0",
            "input_data": [{"name": "Recipe Input", "type": "file", "media_types": ["text/plain"]}],
            "jobs": [
                {
                    "name": "Job 1",
                    "job_type": {"name": self.job_type_1.name, "version": self.job_type_1.version},
                    "recipe_inputs": [{"recipe_input": "Recipe Input", "job_input": "Test Input 1"}],
                },
                {
                    "name": "Job 2",
                    "job_type": {"name": self.job_type_2.name, "version": self.job_type_2.version},
                    "dependencies": [
                        {"name": "Job 1", "connections": [{"output": "Test Output 1", "input": "Test Input 2"}]}
                    ],
                },
            ],
        }

        recipe_definition = RecipeDefinition(definition)
        recipe_definition.validate_job_interfaces()

        self.recipe_type = recipe_test_utils.create_recipe_type(definition=definition)

        self.data = {
            "version": "1.0",
            "input_data": [{"name": "Recipe Input", "file_id": source_file.id}],
            "workspace_id": workspace.id,
        }

        # Register a fake processor
        self.mock_processor = MagicMock(QueueEventProcessor)
        Queue.objects.register_processor(lambda: self.mock_processor)

    def test_does_not_have_data_types(self):
        """Tests calling IngestTriggerCondition.is_condition_met() with a source file that does not have all required data types"""

        condition = IngestTriggerCondition(None, set(['A', 'B', 'C']))
        source_file = source_test_utils.create_source(media_type='text/plain')
        source_file.add_data_type_tag('A')
        source_file.add_data_type_tag('B')
        
        self.assertEqual(condition.is_condition_met(source_file), False)

    def test_calculate_stats(self):
        '''Tests calculating individual statistics for a metrics entry.'''
        strike = ingest_test_utils.create_strike()
        source_file = source_test_utils.create_source(file_size=200)
        ingest_test_utils.create_ingest(strike=strike, source_file=source_file, status='INGESTED',
                                        transfer_started=datetime.datetime(2015, 1, 1),
                                        transfer_ended=datetime.datetime(2015, 1, 1, 0, 10),
                                        ingest_started=datetime.datetime(2015, 1, 1),
                                        ingest_ended=datetime.datetime(2015, 1, 1, 1))
        ingest_test_utils.create_ingest(strike=strike, status='INGESTED',
                                        transfer_started=datetime.datetime(2015, 1, 1),
                                        transfer_ended=datetime.datetime(2015, 1, 1, 0, 20),
                                        ingest_started=datetime.datetime(2015, 1, 1),
                                        ingest_ended=datetime.datetime(2015, 1, 1, 2))
        ingest_test_utils.create_ingest(strike=strike, status='ERRORED',
                                        transfer_started=datetime.datetime(2015, 1, 1),
                                        transfer_ended=datetime.datetime(2015, 1, 1, 0, 30),
                                        ingest_started=datetime.datetime(2015, 1, 1),
                                        ingest_ended=datetime.datetime(2015, 1, 1, 3))
        ingest_test_utils.create_ingest(strike=strike, status='DEFERRED',
                                        transfer_started=datetime.datetime(2015, 1, 1),
                                        transfer_ended=datetime.datetime(2015, 1, 1, 0, 40),
                                        ingest_started=datetime.datetime(2015, 1, 1),
                                        ingest_ended=datetime.datetime(2015, 1, 1, 4))
        ingest_test_utils.create_ingest(strike=strike, status='DUPLICATE',
                                        transfer_started=datetime.datetime(2015, 1, 1),
                                        transfer_ended=datetime.datetime(2015, 1, 1, 0, 50),
                                        ingest_started=datetime.datetime(2015, 1, 1),
                                        ingest_ended=datetime.datetime(2015, 1, 1, 5))

        MetricsIngest.objects.calculate(datetime.date(2015, 1, 1))

        entries = MetricsIngest.objects.filter(occurred=datetime.date(2015, 1, 1))
        self.assertEqual(len(entries), 1)

        entry = entries.first()
        self.assertEqual(entry.occurred, datetime.date(2015, 1, 1))
        self.assertEqual(entry.deferred_count, 1)
        self.assertEqual(entry.ingested_count, 2)
        self.assertEqual(entry.errored_count, 1)
        self.assertEqual(entry.duplicate_count, 1)
        self.assertEqual(entry.total_count, 5)

        self.assertEqual(entry.file_size_sum, 600)
        self.assertEqual(entry.file_size_min, 100)
        self.assertEqual(entry.file_size_max, 200)
        self.assertEqual(entry.file_size_avg, 120)

        self.assertEqual(entry.transfer_time_sum, 9000)
        self.assertEqual(entry.transfer_time_min, 600)
        self.assertEqual(entry.transfer_time_max, 3000)
        self.assertEqual(entry.transfer_time_avg, 1800)

        self.assertEqual(entry.ingest_time_sum, 10800)
        self.assertEqual(entry.ingest_time_min, 3600)
        self.assertEqual(entry.ingest_time_max, 7200)
        self.assertEqual(entry.ingest_time_avg, 5400)

    def test_media_type_incorrect(self):
        '''Tests calling ParseTriggerCondition.is_condition_met() with a source file that only has the correct data types'''

        condition = ParseTriggerCondition('application/json', set(['A', 'B', 'C']))
        source_file = source_test_utils.create_source(media_type='text/plain')
        source_file.add_data_type_tag('A')
        source_file.add_data_type_tag('B')
        source_file.add_data_type_tag('C')
        
        self.assertEqual(condition.is_condition_met(source_file), False)

    def test_data_types_incorrect(self):
        """Tests calling IngestTriggerCondition.is_condition_met() with a source file that only has the correct media type"""

        condition = IngestTriggerCondition('text/plain', set(['A', 'B', 'C', 'D']))
        source_file = source_test_utils.create_source(media_type='text/plain')
        source_file.add_data_type_tag('A')
        source_file.add_data_type_tag('B')
        source_file.add_data_type_tag('C')
        
        self.assertEqual(condition.is_condition_met(source_file), False)

    def test_both_correct(self):
        """Tests calling IngestTriggerCondition.is_condition_met() with a source file that meets both criteria"""

        condition = IngestTriggerCondition('text/plain', set(['A', 'B', 'C']))
        source_file = source_test_utils.create_source(media_type='text/plain')
        source_file.add_data_type_tag('A')
        source_file.add_data_type_tag('B')
        source_file.add_data_type_tag('C')
        
        self.assertEqual(condition.is_condition_met(source_file), True)

    def test_has_data_types(self):
        '''Tests calling ParseTriggerCondition.is_condition_met() with a source file that has all required data types'''

        condition = ParseTriggerCondition(None, set(['A', 'B', 'C']))
        source_file = source_test_utils.create_source(media_type='text/plain')
        source_file.add_data_type_tag('A')
        source_file.add_data_type_tag('B')
        source_file.add_data_type_tag('C')
        source_file.add_data_type_tag('D')
        source_file.add_data_type_tag('E')
        
        self.assertEqual(condition.is_condition_met(source_file), True)

    def test_uuid_use_input_files(self):
        """Tests setting UUIDs on products with different source input files."""
        source_file2 = source_test_utils.create_source(file_name='input2.txt', workspace=self.workspace)

        products1 = ProductFile.objects.upload_files(self.files, [self.source_file.id], self.job_exe, self.workspace)
        products2 = ProductFile.objects.upload_files(self.files, [source_file2.id], self.job_exe, self.workspace)

        # Make sure the source files are taken into account
        self.assertIsNotNone(products1[0].uuid)
        self.assertIsNotNone(products1[1].uuid)
        self.assertNotEqual(products1[0].uuid, products2[0].uuid)
        self.assertNotEqual(products1[1].uuid, products2[1].uuid)

    def setUp(self):
        django.setup()

        self.source = source_test_utils.create_source()
        self.ancestor = product_test_utils.create_product(file_name='test_ancestor.txt')
        self.descendant = product_test_utils.create_product(file_name='test_descendant.txt')
        self.product = product_test_utils.create_product(file_name='test_product.txt')

        product_test_utils.create_file_link(ancestor=self.source, descendant=self.ancestor)
        product_test_utils.create_file_link(ancestor=self.source, descendant=self.product)
        product_test_utils.create_file_link(ancestor=self.source, descendant=self.descendant)
        product_test_utils.create_file_link(ancestor=self.ancestor, descendant=self.product)
        product_test_utils.create_file_link(ancestor=self.ancestor, descendant=self.descendant)
        product_test_utils.create_file_link(ancestor=self.product, descendant=self.descendant)

    def setUp(self):
        django.setup()

        workspace = storage_test_utils.create_workspace()
        source_file = source_test_utils.create_source(workspace=workspace)

        self.data = {
            'version': '1.0',
            'input_data': [{
                'name': 'Recipe Input',
                'file_id': source_file.id,
            }],
            'workspace_id': workspace.id,
        }

        self.job_type = job_test_utils.create_job_type()

        # Register a fake processor
        self.mock_processor = MagicMock(QueueEventProcessor)
        Queue.objects.register_processor(lambda: self.mock_processor)

    def setUp(self):
        django.setup()

        def upload_files(file_uploads):
            for file_upload in file_uploads:
                file_upload.file.save()

        def delete_files(files):
            for scale_file in files:
                scale_file.save()

        self.workspace = storage_test_utils.create_workspace()
        self.workspace.upload_files = MagicMock(side_effect=upload_files)
        self.workspace.delete_files = MagicMock(side_effect=delete_files)

        self.source_file = source_test_utils.create_source(file_name='input1.txt', workspace=self.workspace)

        self.job_exe = job_test_utils.create_job_exe()
        self.job_exe_no = job_test_utils.create_job_exe()
        with transaction.atomic():
            self.job_exe_no.job.is_operational = False
            self.job_exe_no.job.job_type.is_operational = False
            self.job_exe_no.job.save()
            self.job_exe_no.job.job_type.save()

        self.local_path_1 = os.path.join(SCALE_JOB_EXE_OUTPUT_PATH, 'local/1/file.txt')
        self.local_path_2 = os.path.join(SCALE_JOB_EXE_OUTPUT_PATH, 'local/2/file.json')
        self.local_path_3 = os.path.join(SCALE_JOB_EXE_OUTPUT_PATH, 'local/3/file.h5')

        self.files = [
            (self.local_path_1, 'remote/1/file.txt', None),
            (self.local_path_2, 'remote/2/file.json', 'application/x-custom-json'),
        ]
        self.files_no = [
            (self.local_path_3, 'remote/3/file.h5', 'image/x-hdf5-image'),
        ]

    def setUp(self):
        django.setup()

        self.ingest = ingest_test_utils.create_ingest(status='QUEUED')
        self.source_file = source_test_utils.create_source(workspace=self.ingest.workspace)

    def setUp(self):
        django.setup()

        self.ingest = ingest_test_utils.create_ingest(status='INGESTING')
        self.job_exe_id = JobExecution.objects.get(job_id=self.ingest.job).id
        self.source_file = source_test_utils.create_source(workspace=self.ingest.workspace)

    def setUp(self):
        django.setup()

        workspace = storage_test_utils.create_workspace()
        source_file = source_test_utils.create_source(workspace=workspace)
        self.event = trigger_test_utils.create_trigger_event()

        interface_1 = {
            'version': '1.0',
            'command': 'test_command',
            'command_arguments': 'test_arg',
            'input_data': [{
                'name': 'Test Input 1',
                'type': 'file',
                'media_types': ['text/plain'],
            }],
            'output_data': [{
                'name': 'Test Output 1',
                'type': 'files',
                'media_type': 'image/png',
            }]
        }
        self.job_type_1 = job_test_utils.create_job_type(interface=interface_1)

        interface_2 = {
            'version': '1.0',
            'command': 'test_command',
            'command_arguments': 'test_arg',
            'input_data': [{
                'name': 'Test Input 2',
                'type': 'files',
                'media_types': ['image/png', 'image/tiff'],
            }],
            'output_data': [{
                'name': 'Test Output 2',
                'type': 'file',
            }]
        }
        self.job_type_2 = job_test_utils.create_job_type(interface=interface_2)

        definition = {
            'version': '1.0',
            'input_data': [{
                'name': 'Recipe Input',
                'type': 'file',
                'media_types': ['text/plain'],
            }],
            'jobs': [{
                'name': 'Job 1',
                'job_type': {
                    'name': self.job_type_1.name,
                    'version': self.job_type_1.version,
                },
                'recipe_inputs': [{
                    'recipe_input': 'Recipe Input',
                    'job_input': 'Test Input 1',
                }]
            }, {
                'name': 'Job 2',
                'job_type': {
                    'name': self.job_type_2.name,
                    'version': self.job_type_2.version,
                },
                'dependencies': [{
                    'name': 'Job 1',
                    'connections': [{
                        'output': 'Test Output 1',
                        'input': 'Test Input 2',
                    }]
                }]
            }]
        }

        recipe_definition = RecipeDefinition(definition)
        recipe_definition.validate_job_interfaces()

        self.recipe_type = recipe_test_utils.create_recipe_type(definition=definition)

        data = {
            'version': '1.0',
            'input_data': [{
                'name': 'Recipe Input',
                'file_id': source_file.id,
            }],
            'workspace_id': workspace.id,
        }
        self.data = RecipeData(data)

        # Register a fake processor
        self.mock_processor = MagicMock(QueueEventProcessor)
        Queue.objects.register_processor(lambda: self.mock_processor)