def fileify(version_pk, commit):
    """
    Create ImportedFile objects for all of a version's files.

    This is a prereq for indexing the docs for search.
    It also causes celery-haystack to kick off an index of the file.
    """
    version = Version.objects.get(pk=version_pk)
    project = version.project

    if not project.cdn_enabled:
        return

    if not commit:
        log.info(
            LOG_TEMPLATE.format(
                project=project.slug,
                version=version.slug,
                msg=("Imported File not being built because no commit " "information"),
            )
        )

    path = project.rtd_build_path(version.slug)
    if path:
        log.info(LOG_TEMPLATE.format(project=version.project.slug, version=version.slug, msg="Creating ImportedFiles"))
        _manage_imported_files(version, path, commit)
    else:
        log.info(LOG_TEMPLATE.format(project=project.slug, version=version.slug, msg="No ImportedFile files"))

def symlink_cnames(version):
    """
    OLD
    Link from HOME/user_builds/cnames/<cname> ->
              HOME/user_builds/<project>/rtd-builds/
    NEW
    Link from HOME/user_builds/cnametoproject/<cname> ->
              HOME/user_builds/<project>/
    """
    try:
        redis_conn = redis.Redis(**settings.REDIS)
        cnames = redis_conn.smembers('rtd_slug:v1:%s' % version.project.slug)
    except redis.ConnectionError:
        log.error(LOG_TEMPLATE.format(project=version.project.slug, version=version.slug, msg='Failed to symlink cnames, Redis error.'), exc_info=True)
        return
    for cname in cnames:
        log.debug(LOG_TEMPLATE.format(project=version.project.slug, version=version.slug, msg="Symlinking CNAME: %s" % cname))
        docs_dir = version.project.rtd_build_path(version.slug)
        # Chop off the version from the end.
        docs_dir = '/'.join(docs_dir.split('/')[:-1])
        # Old symlink location -- Keep this here til we change nginx over
        symlink = version.project.cnames_symlink_path(cname)
        run_on_app_servers('mkdir -p %s' % '/'.join(symlink.split('/')[:-1]))
        run_on_app_servers('ln -nsf %s %s' % (docs_dir, symlink))
        # New symlink location
        new_docs_dir = version.project.doc_path
        new_cname_symlink = os.path.join(getattr(settings, 'SITE_ROOT'), 'cnametoproject', cname)
        run_on_app_servers('mkdir -p %s' % '/'.join(new_cname_symlink.split('/')[:-1]))
        run_on_app_servers('ln -nsf %s %s' % (new_docs_dir, new_cname_symlink))

def symlink_subprojects(version):
    """
    Link from HOME/user_builds/project/subprojects/<project> ->
              HOME/user_builds/<project>/rtd-builds/
    """
    # Subprojects
    if getattr(settings, 'DONT_HIT_DB', True):
        subproject_slugs = [data['slug'] for data in api.project(version.project.pk).subprojects.get()['subprojects']]
    else:
        rels = version.project.subprojects.all()
        subproject_slugs = [rel.child.slug for rel in rels]
    for slug in subproject_slugs:
        slugs = [slug]
        if '_' in slugs[0]:
            slugs.append(slugs[0].replace('_', '-'))
        for subproject_slug in slugs:
            log.debug(LOG_TEMPLATE.format(project=version.project.slug, version=version.slug, msg="Symlinking subproject: %s" % subproject_slug))

            # The directory for this specific subproject
            symlink = version.project.subprojects_symlink_path(subproject_slug)
            run_on_app_servers('mkdir -p %s' % '/'.join(symlink.split('/')[:-1]))

            # Where the actual docs live
            docs_dir = os.path.join(settings.DOCROOT, subproject_slug, 'rtd-builds')
            run_on_app_servers('ln -nsf %s %s' % (docs_dir, symlink))

 def _log_warning(self, msg):
     # :'(
     log.warning(LOG_TEMPLATE.format(
         project=self.project.slug,
         version=self.version.slug,
         msg=msg,
     ))

def record_pdf(api, record, results, state, version):
    if not record or 'sphinx' not in version.project.documentation_type:
        return None
    try:
        if 'pdf' in results:
            pdf_exit = results['pdf'][0]
            pdf_success = pdf_exit == 0
            pdf_output = results['pdf'][1]
            pdf_error = results['pdf'][2]
        else:
            pdf_exit = 999
            pdf_success = False
            pdf_output = pdf_error = "PDF Failed"

        pdf_output = pdf_output.decode('utf-8', 'ignore')
        pdf_error = pdf_error.decode('utf-8', 'ignore')

        api.build.post(dict(
            state=state,
            project='/api/v1/project/%s/' % version.project.pk,
            version='/api/v1/version/%s/' % version.pk,
            success=pdf_success,
            type='pdf',
            output=pdf_output,
            error=pdf_error,
            exit_code=pdf_exit,
        ))
    except Exception:
        log.error(LOG_TEMPLATE.format(project=version.project.slug,
                                      version=version.slug, msg="Unable to post a new build"), exc_info=True)

 def get_client(self):
     """Create Docker client connection."""
     try:
         if self.client is None:
             self.client = APIClient(
                 base_url=self.docker_socket,
                 version=DOCKER_VERSION,
             )
         return self.client
     except DockerException as e:
         log.exception(
             LOG_TEMPLATE.format(
                 project=self.project.slug,
                 version=self.version.slug,
                 msg='Could not connect to Docker API',
             ),
         )
         # We don't raise an error here mentioning Docker, that is a
         # technical detail that the user can't resolve on their own.
         # Instead, give the user a generic failure
         raise BuildEnvironmentError(
             BuildEnvironmentError.GENERIC_WITH_BUILD_ID.format(
                 build_id=self.build['id'],
             )
         )

def symlink_cnames(project):
    """Symlink project CNAME domains

    OLD
    Link from HOME/user_builds/cnames/<cname> ->
              HOME/user_builds/<project>/rtd-builds/
    NEW
    Link from HOME/user_builds/cnametoproject/<cname> ->
              HOME/user_builds/<project>/
    """
    domains = Domain.objects.filter(project=project, cname=True)
    for domain in domains:
        log.debug(LOG_TEMPLATE.format(
            project=project.slug,
            version=project.get_default_version(),
            msg="Symlinking CNAME: %s" % domain.clean_host)
        )
        docs_dir = project.rtd_build_path()
        # Chop off the version from the end.
        docs_dir = '/'.join(docs_dir.split('/')[:-1])
        # Old symlink location -- Keep this here til we change nginx over
        symlink = project.cnames_symlink_path(domain.clean_host)
        run_on_app_servers('mkdir -p %s' % '/'.join(symlink.split('/')[:-1]))
        run_on_app_servers('ln -nsf %s %s' % (docs_dir, symlink))
        # New symlink location
        new_docs_dir = project.doc_path
        new_cname_symlink = os.path.join(
            getattr(settings, 'SITE_ROOT'),
            'cnametoproject',
            domain.clean_host,
        )
        run_on_app_servers('mkdir -p %s' % '/'.join(new_cname_symlink.split('/')[:-1]))
        run_on_app_servers('ln -nsf %s %s' % (new_docs_dir, new_cname_symlink))

    def setup_vcs(self):
        """
        Update the checkout of the repo to make sure it's the latest.

        This also syncs versions in the DB.

        :param build_env: Build environment
        """
        self.setup_env.update_build(state=BUILD_STATE_CLONING)

        self._log(msg='Updating docs from VCS')
        try:
            update_imported_docs(self.version.pk)
            commit = self.project.vcs_repo(self.version.slug).commit
            if commit:
                self.build['commit'] = commit
        except ProjectImportError as e:
            log.error(
                LOG_TEMPLATE.format(project=self.project.slug,
                                    version=self.version.slug,
                                    msg=str(e)),
                exc_info=True,
            )
            raise BuildEnvironmentError('Failed to import project',
                                        status_code=404)

    def setup_base(self):
        conda_env_path = os.path.join(self.project.doc_path, 'conda')
        version_path = os.path.join(conda_env_path, self.version.slug)

        if os.path.exists(version_path):
            # Re-create conda directory each time to keep fresh state
            log.info(
                LOG_TEMPLATE.format(
                    project=self.project.slug,
                    version=self.version.slug,
                    msg='Removing existing conda directory',
                ),
            )
            shutil.rmtree(version_path)
        self.build_env.run(
            'conda',
            'env',
            'create',
            '--quiet',
            '--name',
            self.version.slug,
            '--file',
            self.config.conda.environment,
            bin_path=None,  # Don't use conda bin that doesn't exist yet
            cwd=self.checkout_path,
        )

def email_notification(version, build, email):
    """Send email notifications for build failure

    :param version: :py:cls:`Version` instance that failed
    :param build: :py:cls:`Build` instance that failed
    :param email: Email recipient address
    """
    log.debug(LOG_TEMPLATE.format(project=version.project.slug, version=version.slug,
                                  msg='sending email to: %s' % email))
    context = {'version': version,
               'project': version.project,
               'build': build,
               'build_url': 'https://{0}{1}'.format(
                   getattr(settings, 'PRODUCTION_DOMAIN', 'readthedocs.org'),
                   build.get_absolute_url()),
               'unsub_url': 'https://{0}{1}'.format(
                   getattr(settings, 'PRODUCTION_DOMAIN', 'readthedocs.org'),
                   reverse('projects_notifications', args=[version.project.slug])),
               }

    if build.commit:
        title = _('Failed: {project.name} ({commit})').format(commit=build.commit[:8], **context)
    else:
        title = _('Failed: {project.name} ({version.verbose_name})').format(**context)

    send_email(
        email,
        title,
        template='projects/email/build_failed.txt',
        template_html='projects/email/build_failed.html',
        context=context
    )

def email_notification(version, build, email):
    """Send email notifications for build failure

    :param version: :py:class:`Version` instance that failed
    :param build: :py:class:`Build` instance that failed
    :param email: Email recipient address
    """
    log.debug(
        LOG_TEMPLATE.format(project=version.project.slug, version=version.slug, msg="sending email to: %s" % email)
    )
    context = {
        "version": version,
        "project": version.project,
        "build": build,
        "build_url": "https://{0}{1}".format(
            getattr(settings, "PRODUCTION_DOMAIN", "readthedocs.org"), build.get_absolute_url()
        ),
        "unsub_url": "https://{0}{1}".format(
            getattr(settings, "PRODUCTION_DOMAIN", "readthedocs.org"),
            reverse("projects_notifications", args=[version.project.slug]),
        ),
    }

    if build.commit:
        title = _("Failed: {project.name} ({commit})").format(commit=build.commit[:8], **context)
    else:
        title = _("Failed: {project.name} ({version.verbose_name})").format(**context)

    send_email(
        email,
        title,
        template="projects/email/build_failed.txt",
        template_html="projects/email/build_failed.html",
        context=context,
    )

def setup_vcs(version, build):
    """
    Update the checkout of the repo to make sure it's the latest.
    This also syncs versions in the DB.
    """

    log.info(LOG_TEMPLATE.format(project=version.project.slug,
                                 version=version.slug, msg='Updating docs from VCS'))
    try:
        update_output = update_imported_docs(version.pk)
        commit = version.project.vcs_repo(version.slug).commit
        if commit:
            build['commit'] = commit
    except ProjectImportError:
        log.error(LOG_TEMPLATE.format(project=version.project.slug, version=version.slug,
                                      msg='Failed to import project; skipping build'), exc_info=True)
        raise
    return update_output

def remove_symlink_single_version(version):
    """Remove single_version symlink"""
    log.debug(LOG_TEMPLATE.format(
        project=version.project.slug,
        version=version.project.get_default_version(),
        msg="Removing symlink for single_version")
    )
    symlink = version.project.single_version_symlink_path()
    run_on_app_servers('rm -f %s' % symlink)

def update_static_metadata(project_pk, path=None):
    """Update static metadata JSON file

    Metadata settings include the following project settings:

    version
      The default version for the project, default: `latest`

    language
      The default language for the project, default: `en`

    languages
      List of languages built by linked translation projects.
    """
    project = Project.objects.get(pk=project_pk)
    if not path:
        path = project.static_metadata_path()

    log.info(LOG_TEMPLATE.format(
        project=project.slug,
        version='',
        msg='Updating static metadata',
    ))
    translations = [trans.language for trans in project.translations.all()]
    languages = set(translations)
    # Convert to JSON safe types
    metadata = {
        'version': project.default_version,
        'language': project.language,
        'languages': list(languages),
        'single_version': project.single_version,
    }
    try:
        fh = open(path, 'w+')
        json.dump(metadata, fh)
        fh.close()
        Syncer.copy(path, path, host=socket.gethostname(), file=True)
    except (AttributeError, IOError) as e:
        log.debug(LOG_TEMPLATE.format(
            project=project.slug,
            version='',
            msg='Cannot write to metadata.json: {0}'.format(e)
        ))

 def delete_existing_venv_dir(self):
     venv_dir = self.venv_path()
     # Handle deleting old venv dir
     if os.path.exists(venv_dir):
         log.info(
             LOG_TEMPLATE.format(
                 project=self.project.slug,
                 version=self.version.slug,
                 msg='Removing existing venv directory',
             ),
         )
         shutil.rmtree(venv_dir)

def webhook_notification(version, build, hook_url):
    project = version.project

    data = json.dumps({
        'name': project.name,
        'slug': project.slug,
        'build': {
            'id': build.id,
            'success': build.success,
            'date': build.date.strftime('%Y-%m-%d %H:%M:%S'),
        }
    })
    log.debug(LOG_TEMPLATE.format(project=project.slug, version='', msg='sending notification to: %s' % hook_url))
    requests.post(hook_url, data=data)

 def create_container(self):
     """Create docker container."""
     client = self.get_client()
     try:
         log.info(
             'Creating Docker container: image=%s',
             self.container_image,
         )
         self.container = client.create_container(
             image=self.container_image,
             command=('/bin/sh -c "sleep {time}; exit {exit}"'
                      .format(time=self.container_time_limit,
                              exit=DOCKER_TIMEOUT_EXIT_CODE)),
             name=self.container_id,
             hostname=self.container_id,
             host_config=self.get_container_host_config(),
             detach=True,
             environment=self.environment,
         )
         client.start(container=self.container_id)
     except ConnectionError as e:
         log.exception(
             LOG_TEMPLATE.format(
                 project=self.project.slug,
                 version=self.version.slug,
                 msg=(
                     'Could not connect to the Docker API, '
                     'make sure Docker is running'
                 ),
             ),
         )
         # We don't raise an error here mentioning Docker, that is a
         # technical detail that the user can't resolve on their own.
         # Instead, give the user a generic failure
         raise BuildEnvironmentError(
             BuildEnvironmentError.GENERIC_WITH_BUILD_ID.format(
                 build_id=self.build['id'],
             )
         )
     except DockerAPIError as e:
         log.exception(
             LOG_TEMPLATE
             .format(
                 project=self.project.slug,
                 version=self.version.slug,
                 msg=e.explanation,
             ),
         )
         raise BuildEnvironmentCreationFailed

 def delete_existing_build_dir(self):
     # Handle deleting old build dir
     build_dir = os.path.join(
         self.venv_path(),
         'build',
     )
     if os.path.exists(build_dir):
         log.info(
             LOG_TEMPLATE.format(
                 project=self.project.slug,
                 version=self.version.slug,
                 msg='Removing existing build directory',
             ),
         )
         shutil.rmtree(build_dir)

def symlink_single_version(version):
    """
    Link from HOME/user_builds/<project>/single_version ->
              HOME/user_builds/<project>/rtd-builds/<default_version>/
    """
    default_version = version.project.get_default_version()
    log.debug(LOG_TEMPLATE.format(project=version.project.slug, version=default_version, msg="Symlinking single_version"))

    # The single_version directory
    symlink = version.project.single_version_symlink_path()
    run_on_app_servers('mkdir -p %s' % '/'.join(symlink.split('/')[:-1]))

    # Where the actual docs live
    docs_dir = os.path.join(settings.DOCROOT, version.project.slug, 'rtd-builds', default_version)
    run_on_app_servers('ln -nsf %s %s' % (docs_dir, symlink))

def webhook_notification(version, build, hook_url):
    """Send webhook notification for project webhook

    :param version: Version instance to send hook for
    :param build: Build instance that failed
    :param hook_url: Hook URL to send to
    """
    project = version.project

    data = json.dumps(
        {
            "name": project.name,
            "slug": project.slug,
            "build": {"id": build.id, "success": build.success, "date": build.date.strftime("%Y-%m-%d %H:%M:%S")},
        }
    )
    log.debug(LOG_TEMPLATE.format(project=project.slug, version="", msg="sending notification to: %s" % hook_url))
    requests.post(hook_url, data=data)