def deploy_project():
    """
    Deploy to the project directory in the virtualenv
    """
    
    project_root = '/'.join([deployment_root(),'env',env.project_fullname,'project'])
    local_dir = os.getcwd()
    
    if env.verbosity:
        print env.host,"DEPLOYING project", env.project_fullname
    #Exclude a few things that we don't want deployed as part of the project folder
    rsync_exclude = ['local_settings*','*.pyc','*.log','.*','/build','/dist','/media*','/static*','/www','/public','/template*']

    #make site local settings if they don't already exist
    _make_local_sitesettings()
    created = deploy_files(local_dir, project_root, rsync_exclude=rsync_exclude)
    if not env.patch:
        #hook the project into sys.path
        pyvers = run('python -V').split(' ')[1].split('.')[0:2] #Python x.x.x
        sitepackages = ''.join(['lib/python',pyvers[0],'.',pyvers[1],'/site-packages'])
        link_name = '/'.join([deployment_root(),'env',env.project_fullname,sitepackages,env.project_package_name])
        target = '/'.join([project_root,env.project_package_name])
        run(' '.join(['ln -s',target,link_name]))
        
        #make sure manage.py has exec permissions
        managepy = '/'.join([target,'sitesettings','manage.py'])
        if exists(managepy):
            sudo('chmod ugo+x %s'% managepy)
    
    return created

def deploy_project():
    """
    Deploy to the project directory in the virtualenv
    """
    
    project_root = '/'.join([deployment_root(),'env',env.project_fullname,'project'])
    local_dir = os.getcwd()
    
    if env.verbosity:
        print env.host,"DEPLOYING project", env.project_fullname
    #Exclude a few things that we don't want deployed as part of the project folder
    rsync_exclude = ['local_settings*','*.pyc','*.log','.*','/build','/dist','/media*','/static*','/www','/public','/template*']

    #make site local settings if they don't already exist
    _make_local_sitesettings()
    created = deploy_files(local_dir, project_root, rsync_exclude=rsync_exclude)
    if not env.patch:
        #hook the project into sys.path
        remote_python_version = run('''python -c "import sys; print 'python%d.%d' % (sys.version_info.major, sys.version_info.minor)"''').strip()
        link_name = '/'.join([deployment_root(),'env',env.project_fullname,'lib', remote_python_version, 'site-packages',env.project_package_name])
        target = '/'.join([project_root,env.project_package_name])

        #create any missing dirs
        run('mkdir -p ' + os.path.dirname(link_name))
        run(' '.join(['ln -s',target,link_name]))
        
        #make sure manage.py has exec permissions
        managepy = '/'.join([target,'sitesettings','manage.py'])
        if exists(managepy):
            sudo('chmod ugo+x %s'% managepy)
    
    return created

def migration():
    """
    Integrate with south schema migration
    """

    #activate env        
    with cd('/'.join([deployment_root(),'env',env.project_fullname,'project',env.project_package_name,'sitesettings'])):
        #migrates all or specific env.migration
        venv = '/'.join([deployment_root(),'env',env.project_fullname,'bin','activate'])
        cmdpt1 = ' '.join(['source',venv,'&&'])
        
        sites = _get_django_sites()
        site_ids = sites.keys()
        site_ids.sort()
        for site in site_ids:
            for settings_file in _sitesettings_files():
                site_settings = '.'.join([env.project_package_name,'sitesettings',settings_file.replace('.py','')])
                cmdpt2 = ' '.join(["python manage.py migrate",env.migration])
                if hasattr(env,"fakemigration"):
                    cmdpt2 = ' '.join([cmdpt2,'--fake'])
                cmdpt2 = ''.join([cmdpt2,'--settings=',site_settings])
                if env.verbosity:
                    print " *", cmdpt2
                output = sudo(' '.join([cmdpt1,cmdpt2]),user='site_%s'% site)
            if env.verbosity:
                print output
    return           

def deploy_wsgi():
    """
    deploy python wsgi file(s)
    """
    remote_dir = "/".join([deployment_root(), "env", env.project_fullname, "wsgi"])
    deployed = []
    if env.verbosity:
        print env.host, "DEPLOYING wsgi", remote_dir
    domains = domain_sites()
    for domain in domains:
        deployed += mkdirs(remote_dir)
        with cd(remote_dir):
            u_domain = domain.replace(".", "_")
            filename = "%s.wsgi" % u_domain
            context = {
                "deployment_root": deployment_root(),
                "user": env.user,
                "project_name": env.project_name,
                "u_domain": u_domain,
                "root_domain": env.root_domain,
            }
            upload_template("/".join(["woven", "django-wsgi-template.txt"]), filename, context)
            if env.verbosity:
                print " * uploaded", filename
            # finally set the ownership/permissions
            # We'll use the group to allow www-data execute
            sudo("chown %s:www-data %s" % (env.user, filename))
            run("chmod ug+xr %s" % filename)
    return deployed

def _get_django_sites():
    """
    Get a list of sites as dictionaries {site_id:'domain.name'}

    """
    deployed = version_state('deploy_project')
    if not env.sites and 'django.contrib.sites' in env.INSTALLED_APPS and deployed:
        with cd('/'.join([deployment_root(),'env',env.project_fullname,'project',env.project_package_name,'sitesettings'])):
            venv = '/'.join([deployment_root(),'env',env.project_fullname,'bin','activate'])
            #since this is the first time we run ./manage.py on the server it can be
            #a point of failure for installations
            with settings(warn_only=True):
                output = run(' '.join(['source',venv,'&&',"./manage.py dumpdata sites"]))

                if output.failed:
                    print "ERROR: There was an error running ./manage.py on the node"
                    print "See the troubleshooting docs for hints on how to diagnose deployment issues"
                    if hasattr(output, 'stderr'):
                        print output.stderr
                    sys.exit(1)
            output = output.split('\n')[-1] #ignore any lines prior to the data being dumped
            sites = json.loads(output)
            env.sites = {}
            for s in sites:
                env.sites[s['pk']] = s['fields']['domain']
    return env.sites

def sync_db():
    """
    Runs the django syncdb command
    """
    with cd('/'.join([deployment_root(),'env',env.project_fullname,'project',env.project_name,'sitesettings'])):
        venv = '/'.join([deployment_root(),'env',env.project_fullname,'bin','activate'])
        if env.verbosity:
            print " * python manage.py syncdb --noinput"
        output = run(' '.join(['source',venv,'&&',"./manage.py syncdb --noinput"]))
        if env.verbosity:
            print output

def rmvirtualenv():
    """
    Remove the current or ``env.project_version`` environment and all content in it
    """
    path = '/'.join([deployment_root(),'env',env.project_fullname])
    link = '/'.join([deployment_root(),'env',env.project_name])
    if version_state('mkvirtualenv'):
        sudo(' '.join(['rm -rf',path]))
        sudo(' '.join(['rm -f',link]))
        sudo('rm -f /var/local/woven/%s*'% env.project_fullname)
        set_version_state('mkvirtualenv',delete=True)

def rmvirtualenv():
    """
    Remove the current or ``env.project_version`` environment and all content in it
    """
    path = '/'.join([deployment_root(),'env',env.project_fullname])
    if server_state('mkvirtualenv'):
        sudo(' '.join(['rm -rf',path]))
        set_server_state('mkvirtualenv',delete=True)
    #If there are no further remaining envs we'll delete the home directory to effectively teardown the project
    if not server_state('mkvirtualenv',prefix=True):
        sudo('rm -rf '+deployment_root())        

def deploy_static():
    """
    Deploy static (application) versioned media
    """
    if (not env.STATIC_ROOT and not env.ADMIN_MEDIA_PREFIX) or 'http://' in env.STATIC_URL: return
        
    remote_dir = '/'.join([deployment_root(),'env',env.project_fullname,'static'])
    
    #if app media is not handled by django-staticfiles we can install admin media by default
    if 'django.contrib.admin' in env.INSTALLED_APPS and not env.STATIC_ROOT:
        if env.MEDIA_URL in env.ADMIN_MEDIA_PREFIX:
            print "ERROR: Your ADMIN_MEDIA_PREFIX must not be on the same path as your MEDIA_URL"
            print "for example you cannot use MEDIA_URL = /media/ and ADMIN_MEDIA_PREFIX = /media/admin/"
            sys.exit(1)
        env.STATIC_URL = env.ADMIN_MEDIA_PREFIX    
        admin = AdminMediaHandler('DummyApp')
        local_dir = admin.media_dir
        remote_dir =  ''.join([remote_dir,env.ADMIN_MEDIA_PREFIX])
    else:
        if env.MEDIA_URL in env.STATIC_URL:
            print "ERROR: Your STATIC_URL must not be on the same path as your MEDIA_URL"
            print "for example you cannot use MEDIA_URL = /media/ and STATIC_URL = /media/static/"
            sys.exit(1)
        elif env.STATIC_ROOT:
            local_dir = env.STATIC_ROOT
            static_url = env.STATIC_URL[1:]
            if static_url:
                remote_dir = '/'.join([remote_dir,static_url])
        else: return
    if env.verbosity:
        print env.host,"DEPLOYING static",remote_dir
    return deploy_files(local_dir,remote_dir)

def deploy_templates():
    """ Deploy any templates from your shortest TEMPLATE_DIRS setting. """
    deployed = None
    if not hasattr(env, 'project_template_dir'):
        # The normal pattern would mean the shortest path is the main
        # one. It's probably the last listed.
        length = 1000
        for directory in env.TEMPLATE_DIRS:
            if directory:
                len_dir = len(directory)
                if len_dir < length:
                    length = len_dir
                    env.project_template_dir = directory

    if hasattr(env, 'project_template_dir'):
        remote_dir = '/'.join([
                deployment_root(),
                'env',
                env.project_fullname,
                'templates',
                ])
        if env.verbosity:
            print env.host, "DEPLOYING templates", remote_dir
        deployed = deploy_files(env.project_template_dir, remote_dir)
    return deployed

def deploy_webconf():
    """ Deploy nginx and other wsgi server site configurations to the host """
    deployed = []
    log_dir = '/'.join([deployment_root(),'log'])
    #TODO - incorrect - check for actual package to confirm installation
    if webserver_list():
        if env.verbosity:
            print env.host,"DEPLOYING webconf:"
        if not exists(log_dir):
            run('ln -s /var/log log')
        #deploys confs for each domain based on sites app
        if 'apache2' in get_packages():
            deployed += _deploy_webconf('/etc/apache2/sites-available','django-apache-template.txt')
            deployed += _deploy_webconf('/etc/nginx/sites-available','nginx-template.txt')
        elif 'gunicorn' in get_packages():
            deployed += _deploy_webconf('/etc/nginx/sites-available','nginx-gunicorn-template.txt')
        
        if not exists('/var/www/nginx-default'):
            sudo('mkdir /var/www/nginx-default')
        upload_template('woven/maintenance.html','/var/www/nginx-default/maintenance.html',use_sudo=True)
        sudo('chmod ugo+r /var/www/nginx-default/maintenance.html')
    else:
        print env.host,"""WARNING: Apache or Nginx not installed"""
        
    return deployed

def _make_local_sitesettings(overwrite=False):
    local_settings_dir = os.path.join(os.getcwd(),env.project_package_name,'sitesettings')
    if not os.path.exists(local_settings_dir) or overwrite:
        if overwrite:
            shutil.rmtree(local_settings_dir,ignore_errors=True)
        os.mkdir(local_settings_dir)
        f = open(os.path.join(local_settings_dir,'__init__.py'),"w")
        f.close()

    settings_file_path = os.path.join(local_settings_dir,'settings.py')
    if not os.path.exists(settings_file_path):
        root_domain = _root_domain()    
        u_domain = root_domain.replace('.','_')
        output = render_to_string('woven/sitesettings.txt',
                {"deployment_root":deployment_root(),
                "site_id":"1",
                "project_name": env.project_name,
                "project_fullname": env.project_fullname,
                "project_package_name": env.project_package_name,
                "u_domain":u_domain,
                "domain":root_domain,
                "user":env,
                "MEDIA_URL":env.MEDIA_URL,
                "STATIC_URL":env.STATIC_URL}
            )
                    
        f = open(settings_file_path,"w+")
        f.writelines(output)
        f.close()
        #copy manage.py into that directory
        manage_path = os.path.join(os.getcwd(),env.project_package_name,'manage.py')
        dest_manage_path = os.path.join(os.getcwd(),env.project_package_name,'sitesettings','manage.py')
        shutil.copy(manage_path, dest_manage_path)

    return

def mkvirtualenv():
    """
    Create the virtualenv project environment
    """
    root = '/'.join([deployment_root(), 'env'])
    path = '/'.join([root, env.project_fullname])
    dirs_created = []
    if env.verbosity:
        print env.host, 'CREATING VIRTUALENV', path
    if not exists(root):
        dirs_created += mkdirs(root)
    with cd(root):
        run(' '.join(["virtualenv", env.project_fullname]))
    with cd(path):
        dirs_created += mkdirs('egg_cache')
        sudo('chown -R %s:www-data egg_cache' % env.user)
        sudo('chmod -R g+w egg_cache')
        run(''.join([
                "echo 'cd ",
                path,
                '/',
                'project',
                '/',
                env.project_package_name,
                '/sitesettings',
                "' > bin/postactivate"]))
        sudo('chmod ugo+rwx bin/postactivate')

    #Create a state
    out = State(' '.join([env.host, 'virtualenv', path, 'created']))
    out.object = dirs_created + ['bin', 'lib', 'include']
    out.failed = False
    return out

def deploy_static():
    """
    Deploy static (application) versioned media
    """
    
    if not env.STATIC_URL or 'http://' in env.STATIC_URL: return
    from django.core.servers.basehttp import AdminMediaHandler
    remote_dir = '/'.join([deployment_root(),'env',env.project_fullname,'static'])
    m_prefix = len(env.MEDIA_URL)
    #if app media is not handled by django-staticfiles we can install admin media by default
    if 'django.contrib.admin' in env.INSTALLED_APPS and not 'django.contrib.staticfiles' in env.INSTALLED_APPS:
        
        if env.MEDIA_URL and env.MEDIA_URL == env.ADMIN_MEDIA_PREFIX[:m_prefix]:
            print "ERROR: Your ADMIN_MEDIA_PREFIX (Application media) must not be on the same path as your MEDIA_URL (User media)"
            sys.exit(1)
        admin = AdminMediaHandler('DummyApp')
        local_dir = admin.base_dir
        remote_dir =  ''.join([remote_dir,env.ADMIN_MEDIA_PREFIX])
    else:
        if env.MEDIA_URL and env.MEDIA_URL == env.STATIC_URL[:m_prefix]:
            print "ERROR: Your STATIC_URL (Application media) must not be on the same path as your MEDIA_URL (User media)"
            sys.exit(1)
        elif env.STATIC_ROOT:
            local_dir = env.STATIC_ROOT
            static_url = env.STATIC_URL[1:]
            if static_url:
                remote_dir = '/'.join([remote_dir,static_url])
        else: return
    if env.verbosity:
        print env.host,"DEPLOYING static",remote_dir
    return deploy_files(local_dir,remote_dir)

def deploy_db(rollback=False):
    """
    Deploy a sqlite database from development
    """
    db_name = ''.join([env.project_name,'.db'])
    db_dir = '/'.join([deployment_root(),'database'])
    db_path = '/'.join([db_dir,db_name])
    if not rollback:
        if env.DEFAULT_DATABASE_ENGINE=='django.db.backends.sqlite3' and not exists(db_path):
            if env.verbosity:
                print env.host,"DEPLOYING DEFAULT SQLITE DATABASE",db_path
            if not os.path.exists(env.DEFAULT_DATABASE_NAME) or not env.DEFAULT_DATABASE_NAME:
                print "ERROR: the database does not exist. Run python manage.py syncdb to create your database first."
                sys.exit(1)
            run('mkdir -p '+db_dir)
            put(env.DEFAULT_DATABASE_NAME,db_path)
            #directory and file must be writable by webserver
            sudo("chown -R %s:www-data %s"% (env.user,db_dir))
            sudo("chmod -R ug+w %s"% db_dir)
    elif rollback and env.DEFAULT_DATABASE_ENGINE=='django.db.backends.sqlite3':
        if env.INTERACTIVE:
            delete = confirm('DELETE the database on the host?',default=False)
            if delete:
                run('rm -f '+db_name)
    return

def _deploy_webconf(remote_dir,template):
    
    if not 'http:' in env.MEDIA_URL: media_url = env.MEDIA_URL
    else: media_url = ''
    if not 'http:' in env.STATIC_URL: static_url = env.STATIC_URL
    else: static_url = ''
    if not static_url: static_url = env.ADMIN_MEDIA_PREFIX
    log_dir = '/'.join([deployment_root(),'log'])
    deployed = []
    users_added = []
    
    domains = domain_sites()
    for d in domains:
        u_domain = d.name.replace('.','_')
        wsgi_filename = d.settings.replace('.py','.wsgi')
        site_user = ''.join(['site_',str(d.site_id)])
        filename = ''.join([remote_dir,'/',u_domain,'-',env.project_version,'.conf'])
        context = {"project_name": env.project_name,
                   "deployment_root":deployment_root(),
                    "u_domain":u_domain,
                    "domain":d.name,
                    "root_domain":env.root_domain,
                    "user":env.user,
                    "site_user":site_user,
                    "SITE_ID":d.site_id,
                    "host_ip":socket.gethostbyname(env.host),
                    "wsgi_filename":wsgi_filename,
                    "MEDIA_URL":media_url,
                    "STATIC_URL":static_url,
                    }

        upload_template('/'.join(['woven',template]),
                        filename,
                        context,
                        use_sudo=True)
        if env.verbosity:
            print " * uploaded", filename
            
        #add site users if necessary
        site_users = _site_users()
        if site_user not in users_added and site_user not in site_users:
            add_user(username=site_user,group='www-data',site_user=True)
            users_added.append(site_user)
            if env.verbosity:
                print " * useradded",site_user

    return deployed

def _get_django_sites():
    """
    Get a list of sites as dictionaries {site_id:'domain.name'}

    """
    deployed = server_state("deploy_project")
    if not env.sites and "django.contrib.sites" in env.INSTALLED_APPS and deployed:
        with cd(
            "/".join([deployment_root(), "env", env.project_fullname, "project", env.project_name, "sitesettings"])
        ):
            venv = "/".join([deployment_root(), "env", env.project_fullname, "bin", "activate"])
            output = run(" ".join(["source", venv, "&&", "./manage.py dumpdata sites"]))
            sites = json.loads(output)
            env.sites = {}
            for s in sites:
                env.sites[s["pk"]] = s["fields"]["domain"]
    return env.sites

def deploy_files(local_dir, remote_dir, pattern = '',rsync_exclude=['*.pyc','.*'], use_sudo=False):
    """
    Generic deploy function for cases where one or more files are being deployed to a host.
    Wraps around ``rsync_project`` and stages files locally and/or remotely
    for network efficiency.
    
    ``local_dir`` is the directory that will be deployed.
   
    ``remote_dir`` is the directory the files will be deployed to.
    Directories will be created if necessary.
    
    Note: Unlike other ways of deploying files, all files under local_dir will be
    deployed into remote_dir. This is the equivalent to cp -R local_dir/* remote_dir.

    ``pattern`` finds all the pathnames matching a specified glob pattern relative
    to the local_dir according to the rules used by the Unix shell.
    ``pattern`` enhances the basic functionality by allowing the python | to include
    multiple patterns. eg '*.txt|Django*'
     
    ``rsync_exclude`` as per ``rsync_project``
    
    Returns a list of directories and files created on the host.
    
    """
    #normalise paths
    if local_dir[-1] == os.sep: local_dir = local_dir[:-1]
    if remote_dir[-1] == '/': remote_dir = remote_dir[:-1]
    created_list = []
    staging_dir = local_dir
    
    #resolve pattern into a dir:filename dict
    local_files = _get_local_files(local_dir,pattern)
    #If we are only copying specific files or rendering templates we need to stage locally
    if local_files: staging_dir = _stage_local_files(local_dir, local_files)
    remote_staging_dir = os.path.join(deployment_root(), '.staging')
    if not exists(remote_staging_dir):
        run(' '.join(['mkdir -pv',remote_staging_dir])).split('\n')
        created_list = [remote_staging_dir]
    
    #upload into remote staging
    rsync_project(local_dir=staging_dir,remote_dir=remote_staging_dir,exclude=rsync_exclude,delete=True)

    #create the final destination
    created_dir_list = mkdirs(remote_dir, use_sudo)
    
    if not os.listdir(staging_dir): return created_list

    func = use_sudo and sudo or run
    #cp recursively -R from the staging to the destination and keep a list
    remote_base_path = '/'.join([remote_staging_dir,os.path.basename(local_dir),'*'])
    copy_file_list = func(' '.join(['cp -Ruv',remote_base_path,remote_dir])).split('\n')
    if copy_file_list[0]: created_list += [file.split(' ')[2][1:-1] for file in copy_file_list if file]

    #cleanup any tmp staging dir
    if staging_dir <> local_dir:
        shutil.rmtree(staging_dir,ignore_errors=True)
    
    return created_list

def sync_db():
    """
    Runs the django syncdb command
    """
    with cd('/'.join([deployment_root(),'env',env.project_fullname,'project',env.project_package_name,'sitesettings'])):
        venv = '/'.join([deployment_root(),'env',env.project_fullname,'bin','activate'])
        sites = _get_django_sites()
        site_ids = sites.keys()
        site_ids.sort()
        for site in site_ids:
            for settings_file in _sitesettings_files():
                site_settings = '.'.join([env.project_package_name,'sitesettings',settings_file.replace('.py','')])
                if env.verbosity:
                    print " * python manage.py syncdb --noinput --settings=%s"% site_settings
                output = sudo(' '.join(['source',venv,'&&',"./manage.py syncdb --noinput --settings=%s"% site_settings]),
                              user='site_%s'% site)
                if env.verbosity:
                    print output

 def handle_host(self,*args, **options):
     opts = options.get('options')
     command = args[0]
     root = deployment_root()
     path = '%s/%s/env/%s/project/%s/'% (root,root_domain(),env.project_fullname,env.project_name)
     pythonpath = '%s/%s/env/%s/bin/python'% (root,env.root_domain,env.project_fullname)
     with cd(path):     
         result = run(' '.join([pythonpath,'manage.py',command,opts]))
     if env.verbosity:
         print result