def test_deleted_file(self):
        '''Tests calling ScaleFileManager.download_files() with a deleted file'''

        download_dir = os.path.join('download', 'dir')
        work_dir = os.path.join('work', 'dir')

        workspace_1 = storage_test_utils.create_workspace()
        file_1 = storage_test_utils.create_file(workspace=workspace_1)
        local_path_1 = u'my/local/path/file.txt'
        file_2 = storage_test_utils.create_file(workspace=workspace_1)
        local_path_2 = u'another/local/path/file.txt'
        file_2.is_deleted = True
        file_2.save()
        file_3 = storage_test_utils.create_file(workspace=workspace_1)
        local_path_3 = u'another/local/path/file.json'
        workspace_1.download_files = MagicMock()
        workspace_1_work_dir = ScaleFile.objects._get_workspace_work_dir(work_dir, workspace_1)

        workspace_2 = storage_test_utils.create_workspace()
        file_4 = storage_test_utils.create_file(workspace=workspace_2)
        local_path_4 = u'my/local/path/4/file.txt'
        file_5 = storage_test_utils.create_file(workspace=workspace_2)
        local_path_5 = u'another/local/path/5/file.txt'
        workspace_2.download_files = MagicMock()
        workspace_2_work_dir = ScaleFile.objects._get_workspace_work_dir(work_dir, workspace_2)

        files = [(file_1, local_path_1), (file_2, local_path_2), (file_3, local_path_3), (file_4, local_path_4),
                 (file_5, local_path_5)]
        self.assertRaises(DeletedFile, ScaleFile.objects.download_files, download_dir, work_dir, files)

    def test_success(self, mock_exists, mock_workspaces):
        '''Tests calling ScaleFileManager.cleanup_download_dir() successfully'''

        download_dir = os.path.join('download', 'dir')
        work_dir = os.path.join('work', 'dir')

        workspace_1 = storage_test_utils.create_workspace()
        workspace_1.cleanup_download_dir = MagicMock()
        workspace_1_work_dir = ScaleFile.objects._get_workspace_work_dir(work_dir, workspace_1)

        workspace_2 = storage_test_utils.create_workspace()
        workspace_2.cleanup_download_dir = MagicMock()
        workspace_2_work_dir = ScaleFile.objects._get_workspace_work_dir(work_dir, workspace_2)

        def new_exists(path):
            return path == workspace_1_work_dir
        mock_exists.side_effect = new_exists

        def new_workspaces():
            return [workspace_1, workspace_2]
        mock_workspaces.side_effect = new_workspaces

        ScaleFile.objects.cleanup_download_dir(download_dir, work_dir)

        workspace_1.cleanup_download_dir.assert_called_once_with(download_dir, workspace_1_work_dir)
        # Workspace 2 should not be cleaned up because os.path.exists() returns false
        self.assertFalse(workspace_2.cleanup_download_dir.called)

    def test_success(self):
        """Tests calling ScaleFileManager.download_files() successfully"""

        workspace_1 = storage_test_utils.create_workspace()
        file_1 = storage_test_utils.create_file(workspace=workspace_1)
        local_path_1 = '/my/local/path/file.txt'
        file_2 = storage_test_utils.create_file(workspace=workspace_1)
        local_path_2 = '/another/local/path/file.txt'
        file_3 = storage_test_utils.create_file(workspace=workspace_1)
        local_path_3 = '/another/local/path/file.json'
        workspace_1.setup_download_dir = MagicMock()
        workspace_1.download_files = MagicMock()

        workspace_2 = storage_test_utils.create_workspace()
        file_4 = storage_test_utils.create_file(workspace=workspace_2)
        local_path_4 = '/my/local/path/4/file.txt'
        file_5 = storage_test_utils.create_file(workspace=workspace_2)
        local_path_5 = '/another/local/path/5/file.txt'
        workspace_2.setup_download_dir = MagicMock()
        workspace_2.download_files = MagicMock()

        files = [FileDownload(file_1, local_path_1, False), FileDownload(file_2, local_path_2, False),
                 FileDownload(file_3, local_path_3, False), FileDownload(file_4, local_path_4, False),
                 FileDownload(file_5, local_path_5, False)]
        ScaleFile.objects.download_files(files)

        workspace_1.download_files.assert_called_once_with([FileDownload(file_1, local_path_1, False),
                                                            FileDownload(file_2, local_path_2, False),
                                                            FileDownload(file_3, local_path_3, False)])
        workspace_2.download_files.assert_called_once_with([FileDownload(file_4, local_path_4, False),
                                                            FileDownload(file_5, local_path_5, False)])

    def test_warnings(self):
        """Tests validating a new workspace where the broker type is changed."""

        json_config = {
            'broker': {
                'type': 'host',
                'host_path': '/host/path',
            },
        }
        storage_test_utils.create_workspace(name='ws-test', json_config=json_config)

        json_data = {
            'name': 'ws-test',
            'json_config': {
                'broker': {
                    'type': 'nfs',
                    'nfs_path': 'host:/dir',
                },
            },
        }

        url = rest_util.get_url('/workspaces/validation/')
        response = self.client.generic('POST', url, json.dumps(json_data), 'application/json')
        self.assertEqual(response.status_code, status.HTTP_200_OK, response.content)

        results = json.loads(response.content)
        self.assertEqual(len(results['warnings']), 1)
        self.assertEqual(results['warnings'][0]['id'], 'broker_type')

    def test_deleted_file(self):
        """Tests calling ScaleFileManager.download_files() with a deleted file"""

        workspace_1 = storage_test_utils.create_workspace()
        file_1 = storage_test_utils.create_file(workspace=workspace_1)
        local_path_1 = '/my/local/path/file.txt'
        file_2 = storage_test_utils.create_file(workspace=workspace_1)
        local_path_2 = '/another/local/path/file.txt'
        file_2.is_deleted = True
        file_2.save()
        file_3 = storage_test_utils.create_file(workspace=workspace_1)
        local_path_3 = '/another/local/path/file.json'
        workspace_1.download_files = MagicMock()

        workspace_2 = storage_test_utils.create_workspace()
        file_4 = storage_test_utils.create_file(workspace=workspace_2)
        local_path_4 = '/my/local/path/4/file.txt'
        file_5 = storage_test_utils.create_file(workspace=workspace_2)
        local_path_5 = '/another/local/path/5/file.txt'
        workspace_2.download_files = MagicMock()

        files = [FileDownload(file_1, local_path_1, False), FileDownload(file_2, local_path_2, False),
                 FileDownload(file_3, local_path_3, False), FileDownload(file_4, local_path_4, False),
                 FileDownload(file_5, local_path_5, False)]
        self.assertRaises(DeletedFile, ScaleFile.objects.download_files, files)

    def test_successful(self):
        """Tests calling ParseTriggerRuleConfiguration.validate() successfully"""

        workspace_name = 'Test_Workspace'
        storage_utils.create_workspace(name=workspace_name)
        json_str = '{"condition": {"media_type": "text/plain", "data_types": ["A", "B"]}, "data": {"input_data_name": "my_input", "workspace_name": "%s"}}' % workspace_name
        rule = ParseTriggerRuleConfiguration(PARSE_TYPE, json.loads(json_str))

        rule.validate()

    def test_inactive_workspace(self):
        """Tests calling deleting files from an inactive workspace"""

        workspace_1 = storage_test_utils.create_workspace()
        workspace_1.download_files = MagicMock()
        file_1 = storage_test_utils.create_file(workspace=workspace_1)

        workspace_2 = storage_test_utils.create_workspace(is_active=False)
        file_2 = storage_test_utils.create_file(workspace=workspace_2)

        files = [file_1, file_2]
        self.assertRaises(ArchivedWorkspace, ScaleFile.objects.delete_files, files)

def create_ingest(file_name='test.txt', status='TRANSFERRING', transfer_started=None, transfer_ended=None,
                  ingest_started=None, ingest_ended=None, data_started=None, data_ended=None, workspace=None,
                  strike=None, source_file=None):
    if not workspace:
        workspace = storage_test_utils.create_workspace()
    if not strike:
        strike = create_strike()
    if not source_file:
        source_file = source_test_utils.create_source(file_name=file_name, data_started=data_started,
                                                      data_ended=data_ended, workspace=workspace)
    if not transfer_started:
        transfer_started = timezone.now()
    if status not in ['QUEUED', 'TRANSFERRING'] and not ingest_started:
        ingest_started = timezone.now()
    if status not in ['QUEUED', 'TRANSFERRING', 'INGESTING'] and not ingest_ended:
        ingest_ended = timezone.now()

    try:
        job_type = Ingest.objects.get_ingest_job_type()
    except:
        job_type = job_utils.create_job_type()
    job = job_utils.create_job(job_type=job_type)
    job_utils.create_job_exe(job=job)

    return Ingest.objects.create(file_name=file_name, file_size=source_file.file_size, status=status, job=job,
                                 bytes_transferred=source_file.file_size, transfer_started=transfer_started,
                                 transfer_ended=transfer_ended, media_type='text/plain', ingest_started=ingest_started,
                                 ingest_ended=ingest_ended, workspace=workspace, strike=strike, source_file=source_file)

def create_product(job_exe=None, workspace=None, has_been_published=False, is_published=False, uuid=None,
                   file_name='my_test_file.txt', file_path='/file/path/my_test_file.txt', media_type='text/plain',
                   file_size=100, countries=None, is_superseded=False, superseded=None):
    """Creates a product file model for unit testing

    :returns: The product model
    :rtype: :class:`product.models.ProductFile`
    """

    if not job_exe:
        job_exe = job_utils.create_job_exe()
    if not workspace:
        workspace = storage_utils.create_workspace()

    if not uuid:
        builder = hashlib.md5()
        builder.update(str(job_exe.job.job_type.id))
        builder.update(file_name)
        uuid = builder.hexdigest()

    if is_superseded and not superseded:
        superseded = timezone.now()

    product_file = ProductFile.objects.create(job_exe=job_exe, job=job_exe.job, job_type=job_exe.job.job_type,
                                              has_been_published=has_been_published, is_published=is_published,
                                              uuid=uuid, file_name=file_name, media_type=media_type,
                                              file_size=file_size, file_path=file_path, workspace=workspace,
                                              is_superseded=is_superseded, superseded=superseded)
    if countries:
        product_file.countries = countries
        product_file.save()
    return product_file

    def setUp(self):
        django.setup()

        self.workspace = storage_test_utils.create_workspace()
        self.workspace.upload_files = MagicMock()
        self.workspace.delete_files = MagicMock()

        self.upload_dir = os.path.join('upload', 'dir')
        self.work_dir = os.path.join('work', 'dir')
        self.workspace_work_dir = ScaleFile.objects._get_workspace_work_dir(self.work_dir, self.workspace)

        self.source_file = source_test_utils.create_source(file_name=u'input1.txt',
                                                                       workspace=self.workspace)

        self.job_exe = job_test_utils.create_job_exe()
        self.job_exe_no = job_test_utils.create_job_exe()
        with transaction.atomic():
            self.job_exe_no.job.is_operational = False
            self.job_exe_no.job.job_type.is_operational = False
            self.job_exe_no.job.save()
            self.job_exe_no.job.job_type.save()

        self.files = [
            (u'local/1/file.txt', u'remote/1/file.txt', None),
            (u'local/2/file.json', u'remote/2/file.json', u'application/x-custom-json'),
        ]
        self.files_no = [
            (u'local/3/file.h5', u'remote/3/file.h5', u'image/x-hdf5-image'),
        ]

    def test_url_file_slash(self):
        '''Tests building a URL for a file where the file path URL has a leading slash.'''
        ws = storage_test_utils.create_workspace(name='test', base_url='http://localhost') 
        file = storage_test_utils.create_file(file_name='test.txt', file_path='/file/path/test.txt',
                                                          workspace=ws)

        self.assertEqual(file.url, 'http://localhost/file/path/test.txt')

    def test_fails(self, mock_makedirs, mock_getsize):
        """Tests calling ScaleFileManager.upload_files() when Workspace.upload_files() fails"""
        def new_getsize(path):
            return 100
        mock_getsize.side_effect = new_getsize

        upload_dir = os.path.join('upload', 'dir')
        work_dir = os.path.join('work', 'dir')

        workspace = storage_test_utils.create_workspace()
        file_1 = ScaleFile()
        file_1.media_type = None  # Scale should auto-detect text/plain
        remote_path_1 = 'my/remote/path/file.txt'
        local_path_1 = 'my/local/path/file.txt'
        file_2 = ScaleFile()
        file_2.media_type = 'application/json'
        remote_path_2 = 'my/remote/path/2/file.json'
        local_path_2 = 'my/local/path/2/file.json'
        workspace.upload_files = MagicMock()
        workspace.upload_files.side_effect = Exception
        workspace.delete_files = MagicMock()
        delete_work_dir = os.path.join(work_dir, 'delete', get_valid_filename(workspace.name))

        files = [(file_1, local_path_1, remote_path_1), (file_2, local_path_2, remote_path_2)]
        self.assertRaises(Exception, ScaleFile.objects.upload_files, upload_dir, work_dir, workspace, files)

    def test_success_new(self, mock_mkdir, mock_getsize, mock_execute):
        """Tests calling SourceFileManager.store_file() successfully with a new source file"""
        def new_getsize(path):
            return 100
        mock_getsize.side_effect = new_getsize

        work_dir = 'work'
        workspace = storage_utils.create_workspace()
        remote_path = u'my/remote/path/file.txt'
        local_path = u'my/local/path/file.txt'
        workspace.cleanup_upload_dir = MagicMock()
        workspace.upload_files = MagicMock()
        workspace.setup_upload_dir = MagicMock()
        workspace.delete_files = MagicMock()
        
        wksp_upload_dir = os.path.join(work_dir, 'upload')
        wksp_work_dir = os.path.join(work_dir, 'work', 'workspaces', get_valid_filename(workspace.name))

        src_file = SourceFile.objects.store_file(work_dir, local_path, [], workspace, remote_path)

        workspace.upload_files.assert_called_once_with(wksp_upload_dir, wksp_work_dir, [('file.txt', remote_path)])
        self.assertListEqual(workspace.delete_files.call_args_list, [])

        self.assertEqual(u'file.txt', src_file.file_name)
        self.assertEqual(u'3d8e577bddb17db339eae0b3d9bcf180', src_file.uuid)
        self.assertEqual(remote_path, src_file.file_path)
        self.assertEqual(u'text/plain', src_file.media_type)
        self.assertEqual(workspace.id, src_file.workspace_id)

def create_trigger_rule(name=None, trigger_type='PARSE', configuration=None, is_active=True):
    '''Creates a trigger rule model for unit testing

    :returns: The trigger rule model
    :rtype: :class:`trigger.models.TriggerRule`
    '''

    if not name:
        global RULE_NAME_COUNTER
        name = 'test-name-%i' % RULE_NAME_COUNTER
        RULE_NAME_COUNTER = RULE_NAME_COUNTER + 1

    if not configuration:
        configuration = {
            'version': '1.0',
            'condition': {
                'media_type': 'text/plain',
            },
            'data': {
                'input_data_name': 'input_file',
                'workspace_name': storage_test_utils.create_workspace().name,
            }
        }

    return TriggerRule.objects.create(name=name, type=trigger_type, configuration=configuration, is_active=is_active)

def create_source(file_name='my_test_file.txt', file_size=100, media_type='text/plain',
                  file_path='/file/path/my_test_file.txt', data_started=None, data_ended=None, is_parsed=True,
                  parsed=None, workspace=None, countries=None):
    """Creates a source file model for unit testing

    :returns: The source file model
    :rtype: :class:`source.models.SourceFile`
    """

    if not data_started:
        data_started = timezone.now()
    if not data_ended:
        data_ended = data_started
    if not parsed and is_parsed:
        parsed = timezone.now()
    if not workspace:
        workspace = storage_utils.create_workspace()

    source_file = SourceFile.objects.create(file_name=file_name, media_type=media_type, file_size=file_size,
                                            file_path=file_path, data_started=data_started, data_ended=data_ended,
                                            is_parsed=is_parsed, parsed=parsed, workspace=workspace,
                                            uuid=hashlib.md5(file_name).hexdigest())
    if countries:
        source_file.countries = countries
        source_file.save()
    return source_file

    def setUp(self):
        django.setup()

        self.workspace = storage_test_utils.create_workspace()
        self.recipe_type_1 = RecipeType.objects.create(name='Recipe 1', version='1.0',
                                                       description='Description of Recipe 1', definition='')
        self.recipe_type_2 = RecipeType.objects.create(name='Recipe 2', version='1.0',
                                                       description='Description of Recipe 2', definition='')

    def setUp(self):
        django.setup()

        workspace = storage_test_utils.create_workspace()
        source_file = source_test_utils.create_source(workspace=workspace)
        self.event = trigger_test_utils.create_trigger_event()

        interface_1 = {
            "version": "1.0",
            "command": "test_command",
            "command_arguments": "test_arg",
            "input_data": [{"name": "Test Input 1", "type": "file", "media_types": ["text/plain"]}],
            "output_data": [{"name": "Test Output 1", "type": "files", "media_type": "image/png"}],
        }
        self.job_type_1 = job_test_utils.create_job_type(interface=interface_1)

        interface_2 = {
            "version": "1.0",
            "command": "test_command",
            "command_arguments": "test_arg",
            "input_data": [{"name": "Test Input 2", "type": "files", "media_types": ["image/png", "image/tiff"]}],
            "output_data": [{"name": "Test Output 2", "type": "file"}],
        }
        self.job_type_2 = job_test_utils.create_job_type(interface=interface_2)

        definition = {
            "version": "1.0",
            "input_data": [{"name": "Recipe Input", "type": "file", "media_types": ["text/plain"]}],
            "jobs": [
                {
                    "name": "Job 1",
                    "job_type": {"name": self.job_type_1.name, "version": self.job_type_1.version},
                    "recipe_inputs": [{"recipe_input": "Recipe Input", "job_input": "Test Input 1"}],
                },
                {
                    "name": "Job 2",
                    "job_type": {"name": self.job_type_2.name, "version": self.job_type_2.version},
                    "dependencies": [
                        {"name": "Job 1", "connections": [{"output": "Test Output 1", "input": "Test Input 2"}]}
                    ],
                },
            ],
        }

        recipe_definition = RecipeDefinition(definition)
        recipe_definition.validate_job_interfaces()

        self.recipe_type = recipe_test_utils.create_recipe_type(definition=definition)

        self.data = {
            "version": "1.0",
            "input_data": [{"name": "Recipe Input", "file_id": source_file.id}],
            "workspace_id": workspace.id,
        }

        # Register a fake processor
        self.mock_processor = MagicMock(QueueEventProcessor)
        Queue.objects.register_processor(lambda: self.mock_processor)

    def setUp(self):
        django.setup()

        self.config = {
            'broker': {
                'type': 'host',
                'host_path': '/host/path',
            },
        }

        self.workspace = storage_test_utils.create_workspace(json_config=self.config)

    def test_broker_validation_error(self):
        """Tests that attempting to get a bad broker instance raises an error."""

        config = {
            'version': '1.0',
            'broker': {
                'type': 'host',
            },
        }
        workspace = storage_test_utils.create_workspace(json_config=config)

        # No exceptions indicates success
        self.assertRaises(InvalidBrokerConfiguration, workspace.get_broker)

    def test_broker_validation(self):
        """Tests that getting the broker instance performs validation."""

        config = {
            'version': '1.0',
            'broker': {
                'type': 'host',
                'host_path': '/host/path'
            },
        }
        workspace = storage_test_utils.create_workspace(json_config=config)

        # No exceptions indicates success
        workspace.get_broker()