def test_location(self):
     from tardis.tardis_portal.models import Location
     self.assertEquals(Location.get_default_location().name,
                       'local')
     self.assertEquals(Location.get_location('staging').name,
                       'staging')
     self.assertEquals(len(Location.objects.all()), 6)

 def setUp(self):
     self.user = generate_user('fred')
     Location.force_initialize()
     self.experiment = generate_experiment(users=[self.user])
     self.dataset = generate_dataset(experiments=[self.experiment])
     self.server = SimpleHttpTestServer()
     self.server.start()

 def testReplicaVerify(self):
     from django.conf import settings
     saved = settings.REQUIRE_DATAFILE_CHECKSUMS
     try:
         Location.get_location('test')
         datafile, replica = generate_datafile("1/2/3", self.dataset,
                                               "Hi mum")
         settings.REQUIRE_DATAFILE_CHECKSUMS = True
         self.assertTrue(replica.verify(), 'Replica.verify() failed.')
         replica.datafile.sha512sum = ''
         replica.datafile.md5sum = ''
         self.assertFalse(
             replica.verify(),
             'Replica.verify() succeeded despite no checksum '
             '(settings.REQUIRE_DATAFILE_CHECKSUMS=True).')
         self.assertFalse(replica.verify(allowEmptyChecksums=False),
                          'Replica.verify() succeeded despite no checksum '
                          '(allowEmptyChecksums=False)')
         settings.REQUIRE_DATAFILE_CHECKSUMS = False
         datafile.sha512sum = None
         datafile.md5sum = None
         self.assertTrue(replica.verify(allowEmptyChecksums=True),
                         'Replica.verify() failed wrongly '
                         '(allowEmptyChecksums=True)')
         datafile.sha512sum = None
         datafile.md5sum = None
         self.assertTrue(replica.verify(),
                         'Replica.verify() failed wrongly')
     finally:
         settings.REQUIRE_DATAFILE_CHECKSUMS = saved

    def testMigrateStoreWithSpaces(self):
        dest = Location.get_location('test')
        local = Location.get_location('local')

        datafile, replica = generate_datafile('1/1/Hi Mum', self.dataset,
                                              "Hi mum")
        datafile2, replica2 = generate_datafile('1/1/Hi Dad', self.dataset,
                                                "Hi dad")

        path = datafile.get_absolute_filepath()
        self.assertTrue(os.path.exists(path))
        path2 = datafile.get_absolute_filepath()
        self.assertTrue(os.path.exists(path2))

        # Migrate them
        migrate_replica(replica, dest)
        self.assertFalse(os.path.exists(path))
        migrate_replica(replica2, dest)
        self.assertFalse(os.path.exists(path2))

        # Bring them back
        migrate_replica(datafile.get_preferred_replica(), local)
        self.assertTrue(os.path.exists(path))
        migrate_replica(datafile2.get_preferred_replica(), local)
        self.assertTrue(os.path.exists(path2))

    def setUp(self):
        from tardis.tardis_portal import models
        from tempfile import mkdtemp, mktemp
        from django.conf import settings
        from os import path
        import os

        # Disconnect post_save signal
        from django.db.models.signals import post_save
        from tardis.tardis_portal.models import \
            staging_hook, Dataset_File, Replica, Location
        post_save.disconnect(staging_hook, sender=Replica)

        from django.contrib.auth.models import User
        user = 'tardis_user1'
        pwd = 'secret'
        email = ''
        self.user = User.objects.create_user(user, email, pwd)

        try:
            os.makedirs(settings.GET_FULL_STAGING_PATH_TEST)
        except OSError:
            pass
        self.temp = mkdtemp(dir=settings.GET_FULL_STAGING_PATH_TEST)

        self.file = mktemp(dir=self.temp)
        content = 'test file'
        with open(self.file, "w+b") as f:
            f.write(content)

        Location.force_initialize()

        # make datafile
        exp = models.Experiment(title='test exp1',
                                institution_name='monash',
                                created_by=self.user)
        exp.save()

        # make dataset
        dataset = models.Dataset(description="dataset description...")
        dataset.save()
        dataset.experiments.add(exp)
        dataset.save()

        # create datafile
        df = models.Dataset_File(dataset=dataset, size = len(content),
                                 filename = path.basename(self.file),
                                 md5sum='f20d9f2072bbeb6691c0f9c5099b01f3')
        df.save()

        # create replica
        base_url = 'file://' + settings.GET_FULL_STAGING_PATH_TEST
        location = Location.load_location({
            'name': 'staging-test-yyy', 'url': base_url, 'type': 'external', 
            'priority': 10, 'transfer_provider': 'local'}) 
        replica = models.Replica(datafile=df, url='file://'+self.file,
                                 protocol="staging",location=location)
        replica.verify()
        replica.save()
        self.replica = replica

    def setUp(self):
        # Create test owner without enough details
        username, email, password = ('testuser',
                                     '[email protected]',
                                     'password')
        user = User.objects.create_user(username, email, password)
        profile = UserProfile(user=user, isDjangoAccount=True)
        profile.save()

        Location.force_initialize()

        # Create test experiment and make user the owner of it
        experiment = Experiment(title='Text Experiment',
                                institution_name='Test Uni',
                                created_by=user)
        experiment.save()
        acl = ObjectACL(
            pluginId='django_user',
            entityId=str(user.id),
            content_object=experiment,
            canRead=True,
            canWrite=True,
            canDelete=True,
            isOwner=True,
            aclOwnershipType=ObjectACL.OWNER_OWNED)
        acl.save()

        dataset = Dataset(description='dataset description...')
        dataset.save()
        dataset.experiments.add(experiment)
        dataset.save()

        def create_datafile(filename):
            testfile = path.join(path.dirname(__file__), 'fixtures',
                                 filename)

            size, sha512sum = get_size_and_sha512sum(testfile)

            datafile = Dataset_File(dataset=dataset,
                                    filename=path.basename(testfile),
                                    size=size,
                                    sha512sum=sha512sum)
            datafile.save()
            base_url = 'file://' + path.abspath(path.dirname(testfile))
            location = Location.load_location({
                'name': 'test-grabber', 'url': base_url, 'type': 'external',
                'priority': 10, 'transfer_provider': 'local'})
            replica = Replica(datafile=datafile,
                              url='file://'+path.abspath(testfile),
                              protocol='file',
                              location=location)
            replica.verify()
            replica.save()
            return Dataset_File.objects.get(pk=datafile.pk)

        self.dataset = dataset
        self.datafiles = [create_datafile('data_grabber_test1.admin'),
                          create_datafile('testfile.txt')
                         ] 

 def setUp(self):
     self.user = generate_user('fred')
     Location.force_initialize()
     self.experiment = generate_experiment(
         users=[self.user],
         title='Meanwhile, down in the archives ...',
         url='http://example.com/something')
     self.dataset = generate_dataset(experiments=[self.experiment])

 def setUp(self):
     from django.contrib.auth.models import User
     from tardis.tardis_portal.models import Location
     user = 'tardis_user1'
     pwd = 'secret'
     email = ''
     self.user = User.objects.create_user(user, email, pwd)
     Location.force_initialize()

    def setUp(self):
        from os import path, mkdir
        from tempfile import mkdtemp

        user = 'tardis_user1'
        pwd = 'secret'
        email = ''
        self.user = User.objects.create_user(user, email, pwd)

        self.userProfile = UserProfile(user=self.user).save()

        self.test_dir = mkdtemp()

        Location.force_initialize()

        self.exp = Experiment(title='test exp1',
                institution_name='monash', created_by=self.user)
        self.exp.save()

        acl = ObjectACL(
            pluginId=django_user,
            entityId=str(self.user.id),
            content_object=self.exp,
            canRead=True,
            isOwner=True,
            aclOwnershipType=ObjectACL.OWNER_OWNED,
        )
        acl.save()

        self.dataset = \
            Dataset(description='dataset description...')
        self.dataset.save()
        self.dataset.experiments.add(self.exp)
        self.dataset.save()

        self.experiment_path = path.join(settings.FILE_STORE_PATH,
                str(self.dataset.get_first_experiment().id))

        self.dataset_path = path.join(self.experiment_path,
                                      str(self.dataset.id))

        if not path.exists(self.experiment_path):
            mkdir(self.experiment_path)
        if not path.exists(self.dataset_path):
            mkdir(self.dataset_path)

        # write test file

        self.filename = 'testfile.txt'

        self.f1 = open(path.join(self.test_dir, self.filename), 'w')
        self.f1.write('Test file 1')
        self.f1.close()

        self.f1_size = path.getsize(path.join(self.test_dir,
                                    self.filename))

        self.f1 = open(path.join(self.test_dir, self.filename), 'r')

def _infer_location(path):
    if urlparse.urlparse(path).scheme == '':
        loc = Location.get_default_location()
    else:
        loc = Location.get_location_for_url(path)
    if loc:
        return loc
    else:
        raise Exception('Cannot infer a location for %s' % path)

    def testRemoveExperimentData(self):
        # First with no sharing
        self._build()
        archive_location = Location.get_location('archtest')
        try:
            nos_experiments = Experiment.objects.count()
            nos_datasets = Dataset.objects.count()
            nos_datafiles = Dataset_File.objects.count()
            nos_replicas = Replica.objects.count()
            self.assertTrue(exists(self.replica.get_absolute_filepath()))
            remove_experiment_data(self.experiment, 
                                   'http://example.com/some.tar.gz',
                                   archive_location)
            self.assertEquals(nos_experiments, Experiment.objects.count())
            self.assertEquals(nos_datasets, Dataset.objects.count())
            self.assertEquals(nos_datafiles, Dataset_File.objects.count())
            self.assertEquals(nos_replicas, Replica.objects.count())
            new_replica = self.datafile.get_preferred_replica()
            self.assertTrue(self.replica.id != new_replica.id)
            self.assertFalse(new_replica.stay_remote)
            self.assertTrue(new_replica.verified)
            self.assertEqual(self.replica.protocol, new_replica.protocol)
            self.assertEqual(archive_location.id, new_replica.location.id)
            self.assertEqual('http://example.com/some.tar.gz#1/1/1',
                             new_replica.url)
            self.assertFalse(exists(self.replica.get_absolute_filepath()))
        finally:
            self._clear()

        # (Check that the deletes cascaded ... )
        self.assertEquals(0, Dataset_File.objects.count())
        self.assertEquals(0, Replica.objects.count())
        
        # Repeat, but with the first dataset in 2 experiments.
        self._build()
        self.dataset.experiments.add(self.experiment2)
        archive_location = Location.get_location('archtest')
        try:
            nos_experiments = Experiment.objects.count()
            nos_datasets = Dataset.objects.count()
            nos_datafiles = Dataset_File.objects.count()
            nos_replicas = Replica.objects.count()
            self.assertTrue(exists(self.replica.get_absolute_filepath()))
            remove_experiment_data(self.experiment, 
                                   'http://example.com/some.tar.gz',
                                   archive_location)
            self.assertEquals(nos_experiments, Experiment.objects.count())
            self.assertEquals(nos_datasets, Dataset.objects.count())
            self.assertEquals(nos_datafiles, Dataset_File.objects.count())
            self.assertEquals(nos_replicas, Replica.objects.count())
            new_replica = self.datafile.get_preferred_replica()
            self.assertTrue(self.replica.id == new_replica.id)
            self.assertTrue(exists(self.replica.get_absolute_filepath()))
            self.assertFalse(exists(self.replica2.get_absolute_filepath()))
        finally:
            self._clear()

    def setUpClass(cls):
        cls.priorcwd = os.getcwd()
        os.chdir(os.path.dirname(__file__)+'/atom_test')
        cls.server = TestWebServer()
        cls.server.start()

        Location.force_initialize()
        Location.load_location({
            'name': 'test-atom',
            'transfer_provider': 'http',
            'url': 'http://localhost:4272/files/',
            'type': 'external',
            'priority': 10})
        Location.load_location({
            'name': 'test-atom2', 
            'transfer_provider': 'http',
            'url': 'http://mydatagrabber.cmm.uq.edu.au/files',
            'type': 'external',
            'priority': 10})

        files = path.realpath(path.join(path.dirname(__file__), 
                                        'atom_test', 'files'))
        Location.load_location({
            'name': 'test-atom3',
            'transfer_provider': 'local',
            'url': 'file://' + files,
            'type': 'external',
            'priority': 10})

    def process_enclosure(self, dataset, enclosure):
        filename = getattr(enclosure, 'title', basename(enclosure.href))
        datafile = Dataset_File(filename=filename, dataset=dataset)
        try:
            datafile.mimetype = enclosure.mime
        except AttributeError:
            pass
        try:
            datafile.size = enclosure.length
        except AttributeError:
            pass
        try:
            hash = enclosure.hash
            # Split on white space, then ':' to get tuples to feed into dict
            hashdict = dict([s.partition(':')[::2] for s in hash.split()])
            # Set SHA-512 sum
            datafile.sha512sum = hashdict['sha-512']
        except AttributeError:
            pass
        datafile.save()
        url = enclosure.href
        # This means we will allow the atom feed to feed us any enclosure
        # URL that matches a registered location.  Maybe we should restrict
        # this to a specific location.
        location = Location.get_location_for_url(url)
        if not location:
            logger.error('Rejected ingestion for unknown location %s' % url)
            return

        replica = Replica(datafile=datafile, url=url,
                          location=location)
        replica.protocol = enclosure.href.partition('://')[0]
        replica.save()
        self.make_local_copy(replica)

 def testScoring(self):
     self._setup()
     scorer = MigrationScorer(Location.get_location('local').id)
     self.assertEquals(2.0, scorer.datafile_score(self.df1))
     self.assertEquals(2, get_user_priority(self.user1))
     self.assertEquals(1, get_user_priority(self.user2))
     self.assertEquals(1.0, scorer.user_score(self.user1))
     self.assertEquals(2.0, scorer.user_score(self.user2))
     self.assertEquals(2.0, scorer.experiment_score(self.exp1))
     self.assertEquals(2.0, scorer.dataset_score(self.df1.dataset))
     self.assertEquals(4.0, scorer.score_datafile(self.df1))
     self.assertEquals([(self.df1, self.rep1, 4.0)], 
                       scorer.score_datafiles_in_dataset(self.ds1))
     self.assertEquals([(self.df5, self.rep5, 8.0), 
                        (self.df4, self.rep4, 6.0), 
                        (self.df1, self.rep1, 4.0)],
                       scorer.score_datafiles_in_experiment(self.exp1))
     self.assertEquals([(self.df5, self.rep5, 8.0), 
                        (self.df4, self.rep4, 6.0)],
                       scorer.score_datafiles_in_experiment(self.exp2))
     self.assertEquals([(self.df6, self.rep6, 5.0)],
                       scorer.score_datafiles_in_experiment(self.exp3))
     self.assertEquals([(self.df5, self.rep5, 8.0), 
                        (self.df4, self.rep4, 6.0), 
                        (self.df6, self.rep6, 5.0), 
                        (self.df1, self.rep1, 4.0), 
                        (self.df7, self.rep7, 0.0), 
                        (self.df8, self.rep8, 0.0)],
                       scorer.score_all_datafiles())
     self.assertEquals([(self.df7, self.rep7, 0.0), 
                        (self.df8, self.rep8, 0.0)], 
                       scorer.score_datafiles_in_dataset(self.ds4))

    def testScoringWithTimes(self):
        self._setup()
        scorer = MigrationScorer(
            Location.get_location('local').id, {
                'user_priority_weighting': [5.0, 2.0, 1.0, 0.5, 0.2],
                'file_size_weighting': 1.0,
                'file_access_weighting': 1.0,
                'file_age_weighting': 1.0,
                'file_size_threshold': 0,
                'file_access_threshold': 0,
                'file_age_threshold': 1})
        
        self.assertEquals(0.0, scorer.datafile_score(self.df1))
     
        f = tempfile.NamedTemporaryFile(dir=settings.FILE_STORE_PATH)
        f.write("Hi Mom!!\n")
        rep = Replica.objects.get(pk=self.rep1.pk)
        rep.url = f.name
        rep.save()

        self.assertEquals(2.0, scorer.datafile_score(self.df1))
        
        older = time.time() - (60 * 60 * 24 + 300)
        os.utime(f.name, (older, older))

        self.assertEquals(3.0, scorer.datafile_score(self.df1))

        older = time.time() - (60 * 60 * 24 * 2 + 300)
        os.utime(f.name, (older, older))

        self.assertEquals(5.0, scorer.datafile_score(self.df1))

        f.close()

def remove_experiment_data(exp, archive_url, archive_location):
    """Remove the online Replicas for an Experiment that are not shared with
    other Experiments.  When Replicas are removed, they are replaced with
    offline replicas whose 'url' consists of the archive_url, with the 
    archive pathname for the datafile as a url fragment id.
    """
    for ds in Dataset.objects.filter(experiments=exp):
        if ds.experiments.count() == 1:
            for df in Dataset_File.objects.filter(dataset=ds):
                replicas = Replica.objects.filter(datafile=df, 
                                                  location__type='online')
                if replicas.count() > 0:
                    for replica in replicas:
                        location = Location.get_location(replica.location.name)
                        location.provider.remove_file(replica)
                        if archive_url:
                            old_replica = replicas[0]
                            path_in_archive = '%s/%s/%s' % (
                                exp.id, ds.id, df.filename)
                            new_replica_url = '%s#%s' % (
                                archive_url, quote(path_in_archive))
                            new_replica = Replica(datafile=old_replica.datafile,
                                                  url=new_replica_url,
                                                  protocol=old_replica.protocol,
                                                  verified=True,
                                                  stay_remote=False,
                                                  location=archive_location)
                            new_replica.save()
                    replicas.delete()

    def testLocalFile(self):
        content = urandom(1024)
        cf = ContentFile(content, 'background_task_testfile')

        # Create new Datafile
        datafile = Dataset_File(dataset=self.dataset)
        datafile.filename = cf.name
        datafile.size = len(content)
        datafile.sha512sum = hashlib.sha512(content).hexdigest()
        datafile.save()
        replica = Replica(datafile=datafile,
                          url=write_uploaded_file_to_dataset(self.dataset, cf),
                          location=Location.get_default_location())
        replica.save()

        def get_replica(datafile):
            return Replica.objects.get(datafile=datafile)

        # undo auto-verify:
        replica.verified = False
        replica.save(update_fields=['verified'])

        # Check that it's not currently verified
        expect(get_replica(datafile).verified).to_be(False)

        # Check it verifies
        verify_files()
        expect(get_replica(datafile).verified).to_be(True)

def stage_replica(replica):
    from django.core.files.uploadedfile import TemporaryUploadedFile
    from tardis.tardis_portal.models import Replica, Location
    if not replica.location.type == 'external':
        raise ValueError('Only external replicas can be staged')
    if getattr(settings, "DEEP_DATASET_STORAGE", False):
        relurl = path.relpath(replica.url[7:], settings.SYNC_TEMP_PATH)
        spliturl = relurl.split(os.sep)[1:]
        subdir = path.dirname(path.join(*spliturl))
    else:
        subdir = None
    with TemporaryUploadedFile(replica.datafile.filename,
                               None, None, None) as tf:
        if replica.verify(tempfile=tf.file):
            if not replica.stay_remote:
                tf.file.flush()
                target_replica = {
                    'datafile': replica.datafile,
                    'url': write_uploaded_file_to_dataset(
                        replica.datafile.dataset, tf,
                        subdir=subdir),
                    'location': Location.get_default_location(),
                    'verified': True,
                    'protocol': ''}
                Replica.objects.filter(id=replica.id).update(**target_replica)
            return True
        else:
            return False

    def testMigrateRestore(self):
        dest = Location.get_location('test')
        local = Location.get_location('local')
        datafile, replica = generate_datafile(None, self.dataset, "Hi mum",
                                              verify=False,
                                              verify_checksums_req=True)

        # Attempt to migrate without datafile hashes ... should
        # fail because we can't verify.
        with self.assertRaises(MigrationError):
            migrate_replica(replica, dest)

        # Verify sets hashes ...
        self.assertEquals(replica.verify(allowEmptyChecksums=True), True)
        replica = Replica.objects.get(pk=replica.pk)
        path = datafile.get_absolute_filepath()
        self.assertTrue(os.path.exists(path))
        self.assertTrue(migrate_replica(replica, dest))
        self.assertFalse(os.path.exists(path))

        # Bring it back
        new_replica = datafile.get_preferred_replica()
        url = new_replica.url
        self.assertTrue(migrate_replica(new_replica, local))
        self.assertTrue(os.path.exists(path))
        # Check it was deleted remotely
        with self.assertRaises(TransferError):
            dest.provider.get_length(new_replica)

        # Refresh the datafile object because it is now stale ...
        datafile = Dataset_File.objects.get(id=datafile.id)
        replica = datafile.get_preferred_replica()

        # Repeat the process with 'noRemove'
        self.assertTrue(migrate_replica(replica, dest, noRemove=True))
        new_replica = datafile.get_preferred_replica()
        self.assertTrue(os.path.exists(path))
        self.assertEquals(dest.provider.get_length(new_replica), 6)
        migrate_replica(new_replica, local, noRemove=True)
        newpath = datafile.get_absolute_filepath()
        replica = datafile.get_preferred_replica()
        self.assertTrue(os.path.exists(path))
        self.assertTrue(os.path.exists(newpath))
        self.assertNotEqual(path, newpath)
        self.assertEquals(dest.provider.get_length(new_replica), 6)

    def setUp(self):
        # Create test owner without enough details
        username, email, password = ('testuser',
                                     '[email protected]',
                                     'password')
        user = User.objects.create_user(username, email, password)
        profile = UserProfile(user=user, isDjangoAccount=True)
        profile.save()
        # Need UserAuthentication
        UserAuthentication(userProfile=profile,
                           username=username,
                           authenticationMethod='localdb').save()
        # Create staging dir
        from os import path, makedirs
        staging_dir = path.join(settings.STAGING_PATH, username)
        if not path.exists(staging_dir):
            makedirs(staging_dir)
        # Ensure that staging dir is set up properly
        expect(get_full_staging_path(username)).to_be_truthy()

        Location.force_initialize()

        # Create test experiment and make user the owner of it
        experiment = Experiment(title='Text Experiment',
                                institution_name='Test Uni',
                                created_by=user)
        experiment.save()
        acl = ExperimentACL(
            pluginId=django_user,
            entityId=str(user.id),
            experiment=experiment,\

            canRead=True,
            isOwner=True,
            aclOwnershipType=ExperimentACL.OWNER_OWNED,
            )
        acl.save()

        self.dataset = \
            Dataset(description='dataset description...')
        self.dataset.save()
        self.dataset.experiments.add(experiment)
        self.dataset.save()

        self.username, self.password = (username, password)