def csv_export( request, repo, org, cid, model=None ):
    """
    """
    if (not model) or (not (model in ['entity','file'])):
        raise Http404
    collection = Collection.from_json(Collection.collection_path(request,repo,org,cid))
    things = {'entity':'objects', 'file':'files'}
    csv_path = export_csv_path(collection.path, model)
    csv_filename = os.path.basename(csv_path)
    if model == 'entity':
        file_url = reverse('webui-collection-csv-entities', args=[repo,org,cid])
    elif model == 'file':
        file_url = reverse('webui-collection-csv-files', args=[repo,org,cid])
    # do it
    result = csv_export_model.apply_async( (collection.path,model), countdown=2)
    # add celery task_id to session
    celery_tasks = request.session.get(settings.CELERY_TASKS_SESSION_KEY, {})
    # IMPORTANT: 'action' *must* match a message in webui.tasks.TASK_STATUS_MESSAGES.
    task = {'task_id': result.task_id,
            'action': 'webui-csv-export-model',
            'collection_id': collection.id,
            'collection_url': collection.url(),
            'things': things[model],
            'file_name': csv_filename,
            'file_url': file_url,
            'start': datetime.now().strftime(settings.TIMESTAMP_FORMAT),}
    celery_tasks[result.task_id] = task
    request.session[settings.CELERY_TASKS_SESSION_KEY] = celery_tasks
    return HttpResponseRedirect( reverse('webui-collection', args=[repo,org,cid]) )

def csv_download( request, repo, org, cid, model=None ):
    """Offers CSV file in settings.CSV_TMPDIR for download.
    
    File must actually exist in settings.CSV_TMPDIR and be readable.
    File must be readable by Python csv module.
    If all that is true then it must be a legal CSV file.
    """
    collection = Collection.from_json(Collection.collection_path(request,repo,org,cid))
    path = export_csv_path(collection.path, model)
    filename = os.path.basename(path)
    if not os.path.exists(path):
        raise Http404
    import csv
    # TODO use vars from migrations.densho or put them in settings.
    CSV_DELIMITER = ','
    CSV_QUOTECHAR = '"'
    CSV_QUOTING = csv.QUOTE_ALL
    response = HttpResponse(content_type='text/csv')
    response['Content-Disposition'] = 'attachment; filename="%s"' % filename
    writer = csv.writer(response, delimiter=CSV_DELIMITER, quotechar=CSV_QUOTECHAR, quoting=CSV_QUOTING)
    with open(path, 'rb') as f:
        reader = csv.reader(f, delimiter=CSV_DELIMITER, quotechar=CSV_QUOTECHAR, quoting=CSV_QUOTING)
        for row in reader:
            writer.writerow(row)
    return response

def collections( request ):
    """
    We are displaying collection status vis-a-vis the project Gitolite server.
    It takes too long to run git-status on every repo so, if repo statuses are not
    cached they will be updated by jQuery after page load has finished.
    """
    collections = []
    collection_status_urls = []
    for object_id in gitolite.get_repos_orgs():
        identifier = Identifier(object_id)
        # TODO Identifier: Organization object instead of repo and org
        repo,org = identifier.parts.values()
        collection_paths = Collection.collection_paths(settings.MEDIA_BASE, repo, org)
        colls = []
        for collection_path in collection_paths:
            if collection_path:
                identifier = Identifier(path=collection_path)
                collection = Collection.from_identifier(identifier)
                colls.append(collection)
                gitstatus = collection.gitstatus()
                if gitstatus and gitstatus.get('sync_status'):
                    collection.sync_status = gitstatus['sync_status']
                else:
                    collection_status_urls.append( "'%s'" % collection.sync_status_url())
        collections.append( (object_id,repo,org,colls) )
    # load statuses in random order
    random.shuffle(collection_status_urls)
    return render_to_response(
        'webui/collections/index.html',
        {'collections': collections,
         'collection_status_urls': ', '.join(collection_status_urls),},
        context_instance=RequestContext(request, processors=[])
    )

def signatures(collection_path, git_name, git_mail):
    """Identifies signature files for collection and entities.
    
    @param collection_path: Absolute path to collection repo.
    @param git_name: Username of git committer.
    @param git_mail: Email of git committer.
    @return collection_path: Absolute path to collection.
    """
    gitstatus.lock(settings.MEDIA_BASE, 'collection_signatures')
    collection = Collection.from_identifier(Identifier(path=collection_path))
    updates = signatures.find_updates(collection)
    files_written = signatures.write_updates(updates)

    # TODO move this code to webui.models.Collection
    status,msg = signatures.commit_updates(
        collection,
        files_written,
        git_name, git_mail, agent='ddr-local'
    )
    logger.debug('DONE')
    logger.debug('Updating Elasticsearch')
    if settings.DOCSTORE_ENABLED:
        collection = Collection.from_identifier(Identifier(path=collection_path))
        try:
            collection.post_json()
        except ConnectionError:
            logger.error('Could not update search index')
    
    return collection_path

def collections( request ):
    """
    We are displaying collection status vis-a-vis the project Gitolite server.
    It takes too long to run git-status on every repo so, if repo statuses are not
    cached they will be updated by jQuery after page load has finished.
    """
    collections = []
    collection_status_urls = []
    for o in gitolite.get_repos_orgs():
        repo,org = o.split('-')
        colls = []
        for coll in commands.collections_local(settings.MEDIA_BASE, repo, org):
            if coll:
                coll = os.path.basename(coll)
                c = coll.split('-')
                repo,org,cid = c[0],c[1],c[2]
                collection = Collection.from_json(Collection.collection_path(request,repo,org,cid))
                colls.append(collection)
                gitstatus = collection.gitstatus()
                if gitstatus and gitstatus.get('sync_status'):
                    collection.sync_status = gitstatus['sync_status']
                else:
                    collection_status_urls.append( "'%s'" % collection.sync_status_url())
        collections.append( (o,repo,org,colls) )
    # load statuses in random order
    random.shuffle(collection_status_urls)
    return render_to_response(
        'webui/collections/index.html',
        {'collections': collections,
         'collection_status_urls': ', '.join(collection_status_urls),},
        context_instance=RequestContext(request, processors=[])
    )

def merge( request, repo, org, cid ):
    """
    Decides how to merge the various files in a merge conflict.
    Sends user around to different editors and things until everything is merged.
    """
    collection_path = Collection.collection_path(request,repo,org,cid)
    repository = dvcs.repository(collection_path)
    collection = Collection.from_json(collection_path)
    task_id = collection.locked()
    status = commands.status(collection_path)
    ahead = collection.repo_ahead()
    behind = collection.repo_behind()
    diverged = collection.repo_diverged()
    conflicted = collection.repo_conflicted()
    unmerged = dvcs.list_conflicted(repository)
    staged = dvcs.list_staged(repository)
    if request.method == 'POST':
        form = MergeCommitForm(request.POST)
        if form.is_valid():
            which = form.cleaned_data['which']
            if which == 'merge':
                dvcs.merge_commit(repository)
                committed = 1
            elif which == 'commit':
                dvcs.diverge_commit(repository)
                committed = 1
            else:
                committed = 0
            if committed:
                if task_id:
                    collection.unlock(task_id)
                messages.error(request, 'Merge conflict has been resolved. Please sync to make your changes available to other users.')
                return HttpResponseRedirect( reverse('webui-collection', args=[repo,org,cid]) )
            return HttpResponseRedirect( reverse('webui-merge', args=[repo,org,cid]) )
    else:
        which = 'unknown'
        if conflicted and not unmerged:
            which = 'merge'
        elif diverged and staged:
            which = 'commit'
        form = MergeCommitForm({'path':collection_path, 'which':which,})
    return render_to_response(
        'webui/merge/index.html',
        {'repo': repo,
         'org': org,
         'cid': cid,
         'collection_path': collection_path,
         'collection': collection,
         'status': status,
         'conflicted': conflicted,
         'ahead': ahead,
         'behind': behind,
         'unmerged': unmerged,
         'diverged': diverged,
         'staged': staged,
         'form': form,},
        context_instance=RequestContext(request, processors=[])
    )

def sync_status_ajax( request, repo, org, cid ):
    collection = Collection.from_json(Collection.collection_path(request,repo,org,cid))
    gitstatus = collection.gitstatus()
    if gitstatus:
        sync_status = gitstatus['sync_status']
        if sync_status.get('timestamp',None):
            sync_status['timestamp'] = sync_status['timestamp'].strftime(settings.TIMESTAMP_FORMAT)
        return HttpResponse(json.dumps(sync_status), mimetype="application/json")
    raise Http404

def new_access( request, repo, org, cid, eid, role, sha1 ):
    """Generate a new access file for the specified file.
    
    NOTE: There is no GET for this view.  GET requests will redirect to entity.
    """
    git_name = request.session.get('git_name')
    git_mail = request.session.get('git_mail')
    if not git_name and git_mail:
        messages.error(request, WEBUI_MESSAGES['LOGIN_REQUIRED'])
    collection = Collection.from_json(Collection.collection_path(request,repo,org,cid))
    entity = Entity.from_json(Entity.entity_path(request,repo,org,cid,eid))
    if collection.locked():
        messages.error(request, WEBUI_MESSAGES['VIEWS_COLL_LOCKED'].format(collection.id))
        return HttpResponseRedirect( reverse('webui-entity', args=[repo,org,cid,eid]) )
    collection.repo_fetch()
    if collection.repo_behind():
        messages.error(request, WEBUI_MESSAGES['VIEWS_COLL_BEHIND'].format(collection.id))
        return HttpResponseRedirect( reverse('webui-entity', args=[repo,org,cid,eid]) )
    if entity.locked():
        messages.error(request, WEBUI_MESSAGES['VIEWS_ENT_LOCKED'])
        return HttpResponseRedirect( reverse('webui-entity', args=[repo,org,cid,eid]) )
    file_ = entity.file(repo, org, cid, eid, role, sha1)
    #
    if request.method == 'POST':
        form = NewAccessFileForm(request.POST)
        if form.is_valid():
            src_path = form.cleaned_data['path']
            # start tasks
            result = entity_add_access.apply_async(
                (git_name, git_mail, entity, file_),
                countdown=2)
            result_dict = result.__dict__
            entity.files_log(1,'START task_id %s' % result.task_id)
            entity.files_log(1,'ddrlocal.webui.file.new_access')
            entity.files_log(1,'Locking %s' % entity.id)
            # lock entity
            lockstatus = entity.lock(result.task_id)
            if lockstatus == 'ok':
                entity.files_log(1, 'locked')
            else:
                entity.files_log(0, lockstatus)
            # add celery task_id to session
            celery_tasks = request.session.get(settings.CELERY_TASKS_SESSION_KEY, {})
            # IMPORTANT: 'action' *must* match a message in webui.tasks.TASK_STATUS_MESSAGES.
            task = {'task_id': result.task_id,
                    'action': 'webui-file-new-access',
                    'filename': os.path.basename(src_path),
                    'file_url': file_.url(),
                    'entity_id': entity.id,
                    'start': datetime.now().strftime(settings.TIMESTAMP_FORMAT),}
            celery_tasks[result.task_id] = task
            #del request.session[settings.CELERY_TASKS_SESSION_KEY]
            request.session[settings.CELERY_TASKS_SESSION_KEY] = celery_tasks
            # feedback
            messages.success(request, WEBUI_MESSAGES['VIEWS_FILES_NEWACCESS'] % os.path.basename(src_path))
    # redirect to entity
    return HttpResponseRedirect( reverse('webui-entity', args=[repo,org,cid,eid]) )

def changelog( request, repo, org, cid ):
    collection = Collection.from_json(Collection.collection_path(request,repo,org,cid))
    alert_if_conflicted(request, collection)
    return render_to_response(
        'webui/collections/changelog.html',
        {'repo': repo,
         'org': org,
         'cid': cid,
         'collection': collection,},
        context_instance=RequestContext(request, processors=[])
    )

def detail( request, repo, org, cid ):
    collection = Collection.from_json(Collection.collection_path(request,repo,org,cid))
    alert_if_conflicted(request, collection)
    return render_to_response(
        'webui/collections/detail.html',
        {'repo': repo,
         'org': org,
         'cid': cid,
         'collection': collection,
         'unlock_task_id': collection.locked(),},
        context_instance=RequestContext(request, processors=[])
    )

def edit_ead( request, repo, org, cid ):
    """
    on GET
    - reads contents of EAD.xml
    - puts in form, in textarea
    - user edits XML
    on POST
    - write contents of field to EAD.xml
    - commands.update
    """
    collection = Collection.from_json(Collection.collection_path(request,repo,org,cid))
    if collection.locked():
        messages.error(request, WEBUI_MESSAGES['VIEWS_COLL_LOCKED'].format(collection.id))
        return HttpResponseRedirect( reverse('webui-collection', args=[repo,org,cid]) )
    ead_path_rel = 'ead.xml'
    ead_path_abs = os.path.join(collection.path, ead_path_rel)
    #
    if request.method == 'POST':
        form = UpdateForm(request.POST)
        if form.is_valid():
            git_name = request.session.get('git_name')
            git_mail = request.session.get('git_mail')
            if git_name and git_mail:
                xml = form.cleaned_data['xml']
                # TODO validate XML
                with open(ead_path_abs, 'w') as f:
                    f.write(xml)
                
                exit,status = commands.update(git_name, git_mail,
                                              collection.path, [ead_path_rel],
                                              agent=settings.AGENT)
                
                if exit:
                    messages.error(request, WEBUI_MESSAGES['ERROR'].format(status))
                else:
                    messages.success(request, 'Collection metadata updated')
                    return HttpResponseRedirect( reverse('webui-collection', args=[repo,org,cid]) )
            else:
                messages.error(request, WEBUI_MESSAGES['LOGIN_REQUIRED'])
    else:
        with open(ead_path_abs, 'r') as f:
            xml = f.read()
        form = UpdateForm({'xml':xml,})
    return render_to_response(
        'webui/collections/edit-ead.html',
        {'repo': repo,
         'org': org,
         'cid': cid,
         'collection_uid': collection.id,
         'collection': collection,
         'form': form,},
        context_instance=RequestContext(request, processors=[])
    )

def unlock( request, repo, org, cid, task_id ):
    """Provides a way to remove collection lockfile through the web UI.
    """
    git_name = request.session.get('git_name')
    git_mail = request.session.get('git_mail')
    if not git_name and git_mail:
        messages.error(request, WEBUI_MESSAGES['LOGIN_REQUIRED'])
    collection = Collection.from_json(Collection.collection_path(request,repo,org,cid))
    if task_id and collection.locked() and (task_id == collection.locked()):
        collection.unlock(task_id)
        messages.success(request, 'Collection <b>%s</b> unlocked.' % collection.id)
    return HttpResponseRedirect( reverse('webui-collection', args=[repo,org,cid]) )

def edit( request, repo, org, cid, eid, role, sha1 ):
    git_name = request.session.get('git_name')
    git_mail = request.session.get('git_mail')
    if not git_name and git_mail:
        messages.error(request, WEBUI_MESSAGES['LOGIN_REQUIRED'])
    collection = Collection.from_json(Collection.collection_path(request,repo,org,cid))
    entity = Entity.from_json(Entity.entity_path(request,repo,org,cid,eid))
    if collection.locked():
        messages.error(request, WEBUI_MESSAGES['VIEWS_COLL_LOCKED'].format(collection.id))
        return HttpResponseRedirect( reverse('webui-entity', args=[repo,org,cid,eid]) )
    collection.repo_fetch()
    if collection.repo_behind():
        messages.error(request, WEBUI_MESSAGES['VIEWS_COLL_BEHIND'].format(collection.id))
        return HttpResponseRedirect( reverse('webui-entity', args=[repo,org,cid,eid]) )
    if entity.locked():
        messages.error(request, WEBUI_MESSAGES['VIEWS_ENT_LOCKED'])
        return HttpResponseRedirect( reverse('webui-entity', args=[repo,org,cid,eid]) )
    file_ = entity.file(repo, org, cid, eid, role, sha1)
    #
    if request.method == 'POST':
        form = DDRForm(request.POST, fields=FILE_FIELDS)
        if form.is_valid():
            
            file_.form_post(form)
            file_.dump_json()
            
            # commit files, delete cache, update search index, update git status
            entity_file_edit(request, collection, file_, git_name, git_mail)
            
            return HttpResponseRedirect( file_.url() )
            
    else:
        form = DDRForm(file_.form_prep(), fields=FILE_FIELDS)
    return render_to_response(
        'webui/files/edit-json.html',
        {'repo': file_.repo,
         'org': file_.org,
         'cid': file_.cid,
         'eid': file_.eid,
         'role': file_.role,
         'sha1': file_.sha1,
         'collection': collection,
         'entity': entity,
         'file': file_,
         'form': form,
         },
        context_instance=RequestContext(request, processors=[])
    )

def delete_file( git_name, git_mail, collection_path, entity_id, file_basename, agent='' ):
    """
    @param collection_path: string
    @param entity_id: string
    @param file_basename: string
    @param git_name: Username of git committer.
    @param git_mail: Email of git committer.
    @param agent: (optional) Name of software making the change.
    """
    logger.debug('delete_file(%s,%s,%s,%s,%s,%s)' % (git_name, git_mail, collection_path, entity_id, file_basename, agent))
    gitstatus.lock(settings.MEDIA_BASE, 'delete_file')
    file_id = os.path.splitext(file_basename)[0]
    file_ = DDRFile.from_identifier(Identifier(file_id))
    entity = Entity.from_identifier(Identifier(entity_id))
    collection = Collection.from_identifier(Identifier(path=collection_path))
    logger.debug('delete from repository')
    rm_files,updated_files = entity.prep_rm_file(file_)
    status,message = commands.file_destroy(
        git_name, git_mail,
        collection, entity,
        rm_files, updated_files,
        agent
    )
    logger.debug('delete from search index')
    try:
        docstore.delete(settings.DOCSTORE_HOSTS, settings.DOCSTORE_INDEX, file_.id)
    except ConnectionError:
        logger.error('Could not delete document from Elasticsearch.')
    return status,message,collection_path,file_basename

def git_status( request, repo, org, cid ):
    collection = Collection.from_json(Collection.collection_path(request,repo,org,cid))
    alert_if_conflicted(request, collection)
    gitstatus = collection.gitstatus()
    return render_to_response(
        'webui/collections/git-status.html',
        {'repo': repo,
         'org': org,
         'cid': cid,
         'collection': collection,
         'status': gitstatus['status'],
         'astatus': gitstatus['annex_status'],
         'timestamp': gitstatus['timestamp'],
         },
        context_instance=RequestContext(request, processors=[])
    )

 def after_return(self, status, retval, task_id, args, kwargs, einfo):
     logger.debug('EntityEditTask.after_return(%s, %s, %s, %s, %s)' % (status, retval, task_id, args, kwargs))
     collection_path = args[0]
     collection = Collection.from_json(collection_path)
     lockstatus = collection.unlock(task_id)
     gitstatus.update(settings.MEDIA_BASE, collection_path)
     gitstatus.unlock(settings.MEDIA_BASE, 'entity_edit')

def csv_export( request, cid, model=None ):
    """
    """
    if (not model) or (not (model in ['entity','file'])):
        raise Http404
    collection = Collection.from_identifier(Identifier(cid))
    things = {'entity':'objects', 'file':'files'}
    csv_path = settings.CSV_EXPORT_PATH[model] % collection.id
    csv_filename = os.path.basename(csv_path)
    if model == 'entity':
        file_url = reverse('webui-collection-csv-entities', args=[collection.id])
    elif model == 'file':
        file_url = reverse('webui-collection-csv-files', args=[collection.id])
    # do it
    result = collection_tasks.csv_export_model.apply_async(
        (collection.path,model),
        countdown=2
    )
    # add celery task_id to session
    celery_tasks = request.session.get(settings.CELERY_TASKS_SESSION_KEY, {})
    # IMPORTANT: 'action' *must* match a message in webui.tasks.TASK_STATUS_MESSAGES.
    task = {'task_id': result.task_id,
            'action': 'csv-export-model',
            'collection_id': collection.id,
            'collection_url': collection.absolute_url(),
            'things': things[model],
            'file_name': csv_filename,
            'file_url': file_url,
            'start': converters.datetime_to_text(datetime.now(settings.TZ)),}
    celery_tasks[result.task_id] = task
    request.session[settings.CELERY_TASKS_SESSION_KEY] = celery_tasks
    return HttpResponseRedirect(collection.absolute_url())

def csv_download( request, repo, org, cid, model=None ):
    """Offers CSV file in settings.CSV_TMPDIR for download.
    
    File must actually exist in settings.CSV_EXPORT_PATH and be readable.
    File must be readable by Python csv module.
    If all that is true then it must be a legal CSV file.
    """
    collection = Collection.from_request(request)
    path = settings.CSV_EXPORT_PATH[model] % collection.id
    filename = os.path.basename(path)
    if not os.path.exists(path):
        raise Http404
    import csv
    response = HttpResponse(content_type='text/csv')
    response['Content-Disposition'] = 'attachment; filename="%s"' % filename
    writer = csv.writer(
        response,
        delimiter=fileio.CSV_DELIMITER,
        quotechar=fileio.CSV_QUOTECHAR,
        quoting=fileio.CSV_QUOTING
    )
    with open(path, 'rb') as f:
        reader = csv.reader(
            f,
            delimiter=fileio.CSV_DELIMITER,
            quotechar=fileio.CSV_QUOTECHAR,
            quoting=fileio.CSV_QUOTING
        )
        for row in reader:
            writer.writerow(row)
    return response

 def after_return(self, status, retval, task_id, args, kwargs, einfo):
     logger.debug('EntityNewExpertTask.after_return(%s, %s, %s, %s, %s)' % (status, retval, task_id, args, kwargs))
     collection_path = args[0]
     collection = Collection.from_identifier(Identifier(path=collection_path))
     lockstatus = collection.unlock(task_id)
     gitstatus.update(settings.MEDIA_BASE, collection.path)
     gitstatus.unlock(settings.MEDIA_BASE, 'entity_newexpert')

def save(collection_path, cleaned_data, git_name, git_mail):
    """The time-consuming parts of collection-edit.
    
    @param collection_path: str Absolute path to collection
    @param cleaned_data: dict form.cleaned_data
    @param git_name: Username of git committer.
    @param git_mail: Email of git committer.
    """
    logger.debug('tasks.collection.save(%s,%s,%s)' % (
        git_name, git_mail, collection_path))
    
    collection = Collection.from_identifier(Identifier(path=collection_path))
    gitstatus.lock(settings.MEDIA_BASE, 'collection_edit')
    
    exit,status,updated_files = collection.save(
        git_name, git_mail,
        cleaned_data
    )
    
    dvcs_tasks.gitstatus_update.apply_async(
        (collection_path,),
        countdown=2
    )
    
    return status,collection_path