def test_store_with_create_or_update_flag():
    project = create_project()

    filepath = utils.make_bogus_binary_file()
    bogus1 = File(filepath, name='Bogus Test File', parent=project)

    bogus1 = syn.store(bogus1, createOrUpdate=True)

    # Create a different file with the same name and parent
    new_filepath = utils.make_bogus_binary_file()
    bogus1.path = new_filepath

    # Expected behavior is that a new version of the first File will be created
    bogus2 = syn.store(bogus1, createOrUpdate=True)

    assert bogus2.id == bogus1.id
    assert bogus2.versionNumber == 2
    assert not filecmp.cmp(bogus2.path, filepath)

    bogus2a = syn.get(bogus2.id)
    assert bogus2a.id == bogus1.id
    assert bogus2a.versionNumber == 2
    assert filecmp.cmp(bogus2.path, bogus2a.path)

    # Create yet another file with the same name and parent
    newer_filepath = utils.make_bogus_binary_file()
    bogus3 = File(newer_filepath, name='Bogus Test File', parent=project)

    # Expected behavior is raising an exception with a 409 error
    assert_raises(requests.exceptions.HTTPError, syn.store, bogus3, createOrUpdate=False)

def test_large_file_upload(file_to_upload_size=11*utils.KB, filepath=None):
    clean_up_file = False

    try:
        project = syn.store(Project("File Upload Load Test " +  datetime.now().strftime("%Y-%m-%d %H%M%S%f")))

        if filepath:
            ## keep a file around so we don't have to regenerate it.
            if not os.path.exists(filepath):
                filepath = utils.make_bogus_binary_file(file_to_upload_size, filepath=filepath, printprogress=True)
                print('Made bogus file: ', filepath)
        else:
            ## generate a temporary file and clean it up when we're done
            clean_up_file = True
            filepath = utils.make_bogus_binary_file(file_to_upload_size, printprogress=True)
            print('Made bogus file: ', filepath)

        try:
            junk = syn.store(File(filepath, parent=project))

            fh = syn._getFileHandle(junk['dataFileHandleId'])
            syn.printEntity(fh)

        finally:
            try:
                if 'junk' in locals():
                    syn.delete(junk)
            except Exception:
                print(traceback.format_exc())
    finally:
        try:
            if 'filepath' in locals() and clean_up_file:
                os.remove(filepath)
        except Exception:
            print(traceback.format_exc())

def test_round_trip():
    fhid = None
    filepath = utils.make_bogus_binary_file(multipart_upload_module.MIN_PART_SIZE + 777771)
    print('Made bogus file: ', filepath)
    try:
        fhid = multipart_upload(syn, filepath)
        print('FileHandle: {fhid}'.format(fhid=fhid))

        # Download the file and compare it with the original
        junk = File(filepath, parent=project, dataFileHandleId=fhid)
        junk.properties.update(syn._createEntity(junk.properties))
        (tmp_f, tmp_path) = tempfile.mkstemp()
        schedule_for_cleanup(tmp_path)
        junk.update(syn._downloadFileEntity(junk, tmp_path))
        assert filecmp.cmp(filepath, junk.path)

    finally:
        try:
            if 'junk' in locals():
                syn.delete(junk)
        except Exception:
            print(traceback.format_exc())
        try:
            os.remove(filepath)
        except Exception:
            print(traceback.format_exc())

def test_round_trip():
    fh = None
    filepath = utils.make_bogus_binary_file(6*MB + 777771, verbose=True)
    print 'Made bogus file: ', filepath
    try:
        fh = syn._chunkedUploadFile(filepath, verbose=False)
        # print 'FileHandle:'
        # syn.printEntity(fh)

        # Download the file and compare it with the original
        junk = File(filepath, parent=project, dataFileHandleId=fh['id'])
        junk.properties.update(syn._createEntity(junk.properties))
        junk.update(syn._downloadFileEntity(junk, filepath))
        assert filecmp.cmp(filepath, junk.path)

    finally:
        try:
            if 'junk' in locals():
                syn.delete(junk)
        except Exception:
            print traceback.format_exc()
        try:
            os.remove(filepath)
        except Exception:
            print traceback.format_exc()
        if fh:
            # print 'Deleting fileHandle', fh['id']
            syn._deleteFileHandle(fh)

def test_store_activity():
    # Create a File and an Activity
    path = utils.make_bogus_binary_file()
    schedule_for_cleanup(path)
    entity = File(path, name='Hinkle horn honking holes', parent=project)
    honking = Activity(name='Hinkle horn honking', 
                       description='Nettlebed Cave is a limestone cave located on the South Island of New Zealand.')
    honking.used('http://www.flickr.com/photos/bevanbfree/3482259379/')
    honking.used('http://www.flickr.com/photos/bevanbfree/3482185673/')

    # This doesn't set the ID of the Activity
    entity = syn.store(entity, activity=honking)

    # But this does
    honking = syn.getProvenance(entity.id)

    # Verify the Activity
    assert honking['name'] == 'Hinkle horn honking'
    assert len(honking['used']) == 2
    assert honking['used'][0]['concreteType'] == 'org.sagebionetworks.repo.model.provenance.UsedURL'
    assert honking['used'][0]['wasExecuted'] == False
    assert honking['used'][0]['url'].startswith('http://www.flickr.com/photos/bevanbfree/3482')
    assert honking['used'][1]['concreteType'] == 'org.sagebionetworks.repo.model.provenance.UsedURL'
    assert honking['used'][1]['wasExecuted'] == False

    # Store another Entity with the same Activity
    entity = File('http://en.wikipedia.org/wiki/File:Nettlebed_cave.jpg', 
                  name='Nettlebed Cave', parent=project, synapseStore=False)
    entity = syn.store(entity, activity=honking)

    # The Activities should match
    honking2 = syn.getProvenance(entity)
    assert honking['id'] == honking2['id']

def test_synStore_sftpIntegration():
    """Creates a File Entity on an sftp server and add the external url. """
    filepath = utils.make_bogus_binary_file(1 * MB - 777771)
    try:
        file = syn.store(File(filepath, parent=project))
        file2 = syn.get(file)
        assert file.externalURL == file2.externalURL and urlparse(file2.externalURL).scheme == "sftp"

        tmpdir = tempfile.mkdtemp()
        schedule_for_cleanup(tmpdir)

        ## test filename override
        file2.fileNameOverride = "whats_new_in_baltimore.data"
        file2 = syn.store(file2)
        ## TODO We haven't defined how filename override interacts with
        ## TODO previously cached files so, side-step that for now by
        ## TODO making sure the file is not in the cache!
        syn.cache.remove(file2.dataFileHandleId, delete=True)
        file3 = syn.get(file, downloadLocation=tmpdir)
        assert os.path.basename(file3.path) == file2.fileNameOverride

        ## test that we got an MD5 à la SYNPY-185
        assert_is_not_none(file3.md5)
        fh = syn._getFileHandle(file3.dataFileHandleId)
        assert_is_not_none(fh["contentMd5"])
        assert_equals(file3.md5, fh["contentMd5"])
    finally:
        try:
            os.remove(filepath)
        except Exception:
            print(traceback.format_exc())

def test_single_thread_upload():
    synapseclient.config.single_threaded = True
    try:
        filepath = utils.make_bogus_binary_file(multipart_upload_module.MIN_PART_SIZE * 2 + 1)
        assert_is_not_none(multipart_upload(syn, filepath))
    finally:
        synapseclient.config.single_threaded = False

def test_store_isRestricted_flag():
    # Store a file with access requirements
    path = utils.make_bogus_binary_file()
    schedule_for_cleanup(path)
    entity = File(path, name='Secret human data', parent=project)
    
    # We don't want to spam ACT with test emails
    with patch('synapseclient.client.Synapse._createAccessRequirementIfNone') as intercepted:
        entity = syn.store(entity, isRestricted=True)
        assert intercepted.called

def test_synStore_sftpIntegration():
    """Creates a File Entity on an sftp server and add the external url. """
    filepath = utils.make_bogus_binary_file(1*MB - 777771)
    try:
        file = syn.store(File(filepath, parent=project))
        file2  = syn.get(file)
        assert file.externalURL==file2.externalURL and urlparse.urlparse(file2.externalURL).scheme=='sftp'
    finally:
        try:
            os.remove(filepath)
        except Exception:
            print traceback.format_exc()

def test_synGet_sftpIntegration():
    #Create file by uploading directly to sftp and creating entity from URL
    serverURL='sftp://ec2-54-212-85-156.us-west-2.compute.amazonaws.com/public/'+str(uuid.uuid1())
    filepath = utils.make_bogus_binary_file(1*MB - 777771)
    print '\n\tMade bogus file: ', filepath
    
    url = syn._sftpUploadFile(filepath, url=serverURL)
    file = syn.store(File(path=url, parent=project, synapseStore=False))

    print '\nDownloading file', os.getcwd(), filepath
    junk = syn.get(file, downloadLocation=os.getcwd(), downloadFile=True)
    filecmp.cmp(filepath, junk.path)

def test_synGet_sftpIntegration():
    # Create file by uploading directly to sftp and creating entity from URL
    serverURL = SFTP_SERVER_PREFIX + SFTP_USER_HOME_PATH + '/test_synGet_sftpIntegration/' + str(uuid.uuid1())
    filepath = utils.make_bogus_binary_file(1*MB - 777771)

    username, password = syn._getUserCredentials(SFTP_SERVER_PREFIX)

    url = SFTPWrapper.upload_file(filepath, url=serverURL, username=username, password=password)
    file = syn.store(File(path=url, parent=project, synapseStore=False))

    junk = syn.get(file, downloadLocation=os.getcwd(), downloadFile=True)
    filecmp.cmp(filepath, junk.path)

def test_chunks():
    # Read a file in chunks, write the chunks out, and compare to the original
    try:
        filepath = utils.make_bogus_binary_file(n=1*MB)
        with tempfile.NamedTemporaryFile(mode='wb', delete=False) as out:
            for i in range(1, nchunks(filepath, chunksize=64*1024)+1):
                out.write(get_chunk(filepath, i, chunksize=64*1024))
        assert filecmp.cmp(filepath, out.name)
    finally:
        if 'filepath' in locals() and filepath:
            os.remove(filepath)
        if 'out' in locals() and out:
            os.remove(out.name)

def test_store_with_force_version_flag():
    project = create_project()

    filepath = utils.make_bogus_binary_file()
    bogus1 = File(filepath, name='Bogus Test File', parent=project)

    # Expect to get version 1 back
    bogus1 = syn.store(bogus1, forceVersion=False)
    assert bogus1.versionNumber == 1

    # Re-store the same thing and don't up the version
    bogus2 = syn.store(bogus1, forceVersion=False)
    assert bogus1.versionNumber == 1
    
    # Create a different file with the same name and parent
    new_filepath = utils.make_bogus_binary_file()
    bogus2.path = new_filepath

    # Expected behavior is that a new version of the first File will be created
    bogus2 = syn.store(bogus2, forceVersion=False)
    assert bogus2.id == bogus1.id
    assert bogus2.versionNumber == 2
    assert not filecmp.cmp(bogus2.path, filepath)

def test_pool_provider_is_used_in__multipart_upload():
    mocked_get_chunk_function = MagicMock(side_effect=[1, 2, 3, 4])
    file_size = 1*MB
    filepath = make_bogus_binary_file(n=file_size)
    md5 = md5_for_file(filepath).hexdigest()
    status = {'partsState': {},
              'uploadId': {},
              'state': 'COMPLETED'}

    pool = MagicMock()
    with patch.object(syn, "restPOST", return_value=status),\
            patch.object(pool_provider, "get_pool", return_value=pool) as mock_provider:
        _multipart_upload(syn, filepath, "application/octet-stream", mocked_get_chunk_function, md5, file_size)
        mock_provider.assert_called()
        pool.map.assert_called()

def test_chunks():
    # Read a file in chunks, write the chunks out, and compare to the original
    try:
        filepath = utils.make_bogus_binary_file()
        with open(filepath, 'rb') as f, tempfile.NamedTemporaryFile(mode='wb', delete=False) as out:
            for chunk in utils.chunks(f):
                buff = chunk.read(4*KB)
                while buff:
                    out.write(buff)
                    buff = chunk.read(4*KB)
        assert filecmp.cmp(filepath, out.name)
    finally:
        if 'filepath' in locals() and filepath:
            os.remove(filepath)
        if 'out' in locals() and out:
            os.remove(out.name)

def test_chunks():
    # Read a file in chunks, write the chunks out, and compare to the original
    try:
        file_size = 1*MB
        filepath = make_bogus_binary_file(n=file_size)
        chunksize = 64*1024
        nchunks = int(math.ceil(float(file_size) / chunksize))
        with tempfile.NamedTemporaryFile(mode='wb', delete=False) as out:
            for i in range(1, nchunks+1):
                out.write(get_file_chunk(filepath, i, chunksize))
        assert_true(filecmp.cmp(filepath, out.name))
    finally:
        if 'filepath' in locals() and filepath:
            os.remove(filepath)
        if 'out' in locals() and out:
            os.remove(out.name)

def test_randomly_failing_parts():
    FAILURE_RATE = 1.0/3.0
    fhid = None
    multipart_upload_module.MIN_PART_SIZE = 5*MB
    multipart_upload_module.MAX_RETRIES = 20

    filepath = utils.make_bogus_binary_file(multipart_upload_module.MIN_PART_SIZE*2 + 777771)
    print('Made bogus file: ', filepath)

    normal_put_chunk = None

    def _put_chunk_or_fail_randomly(url, chunk, verbose=False):
        if random.random() < FAILURE_RATE:
            raise IOError("Ooops! Artificial upload failure for testing.")
        else:
            return normal_put_chunk(url, chunk, verbose)

    ## Mock _put_chunk to fail randomly
    normal_put_chunk = multipart_upload_module._put_chunk
    multipart_upload_module._put_chunk = _put_chunk_or_fail_randomly

    try:
        fhid = multipart_upload(syn, filepath)
        print('FileHandle: {fhid}'.format(fhid=fhid))

        # Download the file and compare it with the original
        junk = File(filepath, parent=project, dataFileHandleId=fhid)
        junk.properties.update(syn._createEntity(junk.properties))
        (tmp_f, tmp_path) = tempfile.mkstemp()
        schedule_for_cleanup(tmp_path)
        junk.update(syn._downloadFileEntity(junk, tmp_path))
        assert filecmp.cmp(filepath, junk.path)

    finally:
        ## Un-mock _put_chunk
        if normal_put_chunk:
            multipart_upload_module._put_chunk = normal_put_chunk

        try:
            if 'junk' in locals():
                syn.delete(junk)
        except Exception:
            print(traceback.format_exc())
        try:
            os.remove(filepath)
        except Exception:
            print(traceback.format_exc())

def test_upload_speed(uploadSize=60 + 777771, threadCount=5):
    import time
    fh = None
    filepath = utils.make_bogus_binary_file(uploadSize*MB)
    print('Made bogus file: ', filepath)
    try:
        t0 = time.time()
        fh = syn._uploadToFileHandleService(filepath, threadCount=threadCount)
        dt =  time.time()-t0
    finally:
        try:
            os.remove(filepath)
        except Exception:
            print(traceback.format_exc())
        if fh:
            syn._deleteFileHandle(fh)
    return dt

def manually_check_retry_on_key_does_not_exist():
    ## This is a manual test -- don't know how to automate this one.
    ## To run: nosetests -vs tests/integration/test_chunked_upload.py:manually_check_retry_on_key_does_not_exist
    ## We're testing the retrying of key-does-not-exist errors from S3.

    ## Expected behavior: Retries several times, getting a error message:
    ## 'The specified key does not exist.', then fails with a stack trace.
    i = 1
    filepath = utils.make_bogus_binary_file(6*MB, verbose=True)

    try:
        token = syn._createChunkedFileUploadToken(filepath, 'application/octet-stream')
        chunkRequest, url = syn._createChunkedFileUploadChunkURL(i, token)
        ## never upload the chunk, so we will get an error 'The specified key does not exist.'
        chunkResult = syn._addChunkToFile(chunkRequest, verbose=True)
    finally:
        os.remove(filepath)

def test_utils_sftp_upload_and_download():
    """Tries to upload a file to an sftp file """
    serverURL = "sftp://ec2-54-212-85-156.us-west-2.compute.amazonaws.com/public/" + str(uuid.uuid1())
    filepath = utils.make_bogus_binary_file(1 * MB - 777771)

    tempdir = tempfile.mkdtemp()

    try:
        print("\n\tMade bogus file: ", filepath)
        url = syn._sftpUploadFile(filepath, url=serverURL)
        print("\tStored URL:", url)
        print("\tDownloading")
        # Get with a specified localpath
        junk = syn._sftpDownloadFile(url, tempdir)
        print("\tComparing:", junk, filepath)
        filecmp.cmp(filepath, junk)
        # Get without specifying path
        print("\tDownloading")
        junk2 = syn._sftpDownloadFile(url)
        print("\tComparing:", junk2, filepath)
        filecmp.cmp(filepath, junk2)
        # Get with a specified localpath as file
        print("\tDownloading")
        junk3 = syn._sftpDownloadFile(url, os.path.join(tempdir, "bar.dat"))
        print("\tComparing:", junk3, filepath)
        filecmp.cmp(filepath, junk3)
    finally:
        try:
            if "junk" in locals():
                os.remove(junk)
            if "junk2" in locals():
                os.remove(junk2)
            if "junk3" in locals():
                os.remove(junk3)
        except Exception:
            print(traceback.format_exc())
        try:
            os.remove(filepath)
        except Exception:
            print(traceback.format_exc())
        try:
            shutil.rmtree(tempdir)
        except Exception:
            print(traceback.format_exc())