def senatetest():
    '''
    Creates a file sen/schumer.html with schumers contact page
    '''
    # not working - 6-Jan-2011
    notworking = ['hagan', 'corker', 'shelby', 'grassley', 'senate', 'coburn', 'inhofe', 'crapo', 'risch', 'lieberman', 'brown', 'moran', 'roberts']
    
    sendb = get_senate_offices()
    statfile = open("senate-test-out.txt", "w")
    for state in sendb:
        for member in sendb[state]:
            sen = web.lstrips(web.lstrips(web.lstrips(member, 'http://'), 'https://'), 'www.').split('.')[0]
            if sen in WYR_MANUAL: member = WYR_MANUAL[sen]
            #if sen != 'billnelson': continue
            #if sen in working + failure: continue
            print repr(sen)
            q = None

            try:
                q = writerep_general(member, prepare_i(state))
                                
                file('sen/%s.html' % sen, 'w').write('<base href="%s"/>' % member + q)
                #subprocess.Popen(['open', 'sen/%s.html' % sen])
                #subprocess.Popen(['pbcopy'], stdin=subprocess.PIPE).stdin.write(', ' + repr(sen))
                if q.lower().find("thank") >= 0:
                    status = "Thanked"
                else:
                    status = "Failed.  reason unknown."
            except Exception, e:
                print "Caught exception on senator: %s " % member
                status="failed.  exception occurred %s" % e.__str__()
            statfile.write("Member: %s, Status: %s\n" % (member, status))
            statfile.flush()

def dirstrip(f, dir):
    """Strips dir from f.
        >>> dirstrip('a/b/c/d', 'a/b/')
        'c/d'
    """
    f = web.lstrips(f, dir)
    return web.lstrips(f, '/')

def senatetest2(member2email):
    sendb = get_senate_offices()
    for state in sendb:
        for member in sendb[state]:
            sen = web.lstrips(web.lstrips(web.lstrips(member, 'http://'), 'https://'), 'www.').split('.')[0]
            if sen in WYR_MANUAL: member = WYR_MANUAL[sen]
            if sen != member2email : continue
            print repr(sen)
            q = writerep_general(member, prepare_i(state))

            if not q:
                print "Failed to write to %s" % member2email
                import sys
                sys.exit(1)
                
                
            file('sen/%s.html' % sen, 'w').write('<base href="%s"/>' % member + q)

            success=False
            if "thank" in q.lower() or "your message has been submitted" in q.lower() or "your message has been submitted" in q.lower() : 
                #if you're getting thanked, you're probably successful
                success=True
                
            errorString = getError(q)
            print "ErrorString: ", errorString
            
            subprocess.Popen(['open', 'sen/%s.html' % sen])
            subprocess.Popen(['pbcopy'], stdin=subprocess.PIPE).stdin.write(', ' + repr(sen))

            if (success):
                print "Successfully wrote to %s" % member2email
            else:
                print "Failed to write to %s" % member2email
            import sys
            sys.exit(1)

def signin_required():
    f = forms.signin()
    return_to = web.lstrips(web.ctx.fullpath, '/')
    if not jt.site.public_url:
        return_to = web.lstrips(return_to, jt.site.secret_url+'/')
    f.fill(secret_url=jt.site.secret_url, return_to=return_to)
    page_title = 'Please enter the site-wide password'
    return view.render('signin', vars=locals())

def find(path):
    """Find all files in the file hierarchy rooted at path.
        >> find('..../web')
        ['db.py', 'http.py', 'wsgiserver/__init__.py', ....]
    """
    for dirname, dirs, files in os.walk(path):
        dirname = web.lstrips(dirname, path)
        dirname = web.lstrips(dirname, '/')
        for f in files:
            yield os.path.join(dirname, f)

 def getSenatorStateAndContactLink(self, sen2contact):
     '''
     for a given senator, get the contact link
     senators names must match with what appears in their url.
     For example, scottbrown, brown, boxer
     '''
     for state in self.sendb:
         for contactlink in self.sendb.get(state, []):
             sen = web.lstrips(web.lstrips(web.lstrips(contactlink, 'http://'), 'https://'), 'www.').split('.')[0]
             if sen == sen2contact:
                 return (state,contactlink)
     return (None, None)

    def GET(self, emailform=None, loadcontactsform=None):
        i = web.input()
        url = i.get('url', '/')
        title = i.get('title', 'The good government site with teeth')

        user_id = helpers.get_loggedin_userid()
        contacts = get_contacts(user_id)
        sender = helpers.get_user_by_email(helpers.get_loggedin_email() or helpers.get_unverified_email())

        page_or_petition = 'page'    
        if not emailform:
            emailform = forms.emailform()
            track_id, description = None, None
            if url.startswith('/c/') and url != '/c/':
                url = url.rstrip('/')
                pid = web.lstrips(url, '/c/')
                p = get_petition_by_id(pid)
                description = p and p.description
                track_id = helpers.get_trackid(user_id, pid)
                contacts = filter(lambda c: not is_signatory(c.email, pid), contacts)
                page_or_petition = 'petition'

            msg = render_plain.share_mail(title, url, sender, description, track_id)
            emailform.fill(subject=title, body=msg)

        loadcontactsform = loadcontactsform or forms.loadcontactsform()

        msg, msg_type = helpers.get_delete_msg()
        return render.share(title, url, emailform,
                            contacts, loadcontactsform, page_or_petition, msg)

def main():
    assert os.path.exists(ALMANAC_DIR), ALMANAC_DIR
    
    files = glob.glob(ALMANAC_DIR + 'people/*/rep_*.htm') + \
            glob.glob(ALMANAC_DIR + 'people/*/*s[12].htm')
    files.sort()
    for fn in files:
        district = web.storage()
        demog = None
        
        dist = web.lstrips(web.rstrips(fn.split('/')[-1], '.htm'), 'rep_')
        diststate = dist[0:2].upper()
        distnum = dist[-2:]
        distname = tools.fixdist(diststate + '-' + distnum)
        
        d = almanac.scrape_person(fn)
        load_election_results(d, distname)

        if 'demographics' in d:
            demog = d['demographics']
        elif distname[-2:] == '00' or '-' not in distname:   # if -00 then this district is the same as the state.
            #print "Using state file for:", distname
            statefile = ALMANAC_DIR + 'states/%s/index.html' % diststate.lower()
            demog = almanac.scrape_state(statefile).get('state')

        demog_to_dist(demog, district)

        district.almanac = 'http://' + d['filename'][d['filename'].find('nationaljournal.com'):]

        #print 'district:', distname, pformat(district)
        db.update('district', where='name=$distname', vars=locals(), **district)

def dispatch_url(urlpath, modvars, mapping, prefix="/widget"):
    """
    Based on handle in web.py's request.py 
    Simplified to work only for the current module.
    """

    _d, _d, path, _d, query, fragment = urlparse.urlparse(urlpath)

    path = web.lstrips(path, base_path)

    for url, ofn in web.utils.group(mapping, 2):
        fn, result = web.utils.re_subm('^' + prefix + url + '$', ofn, path)
        if result:
            try:
                cls = modvars[fn]
            except KeyError: 
                return "[ERROR 1]"
            
            meth = "GET"
            if not hasattr(cls, meth): 
                return "[ERROR 2]"
            tocall = getattr(cls(), meth)
            args = list(result.groups())
            for d in re.findall(r'\\(\d+)', ofn):
                args.pop(int(d) - 1)

            query = get_query(urlpath)

            return tocall(*([x and urllib.unquote(x) for x in args]), 
                 **{'onlycode': True, 
                    'webinput': query}
            )
    return "[ERROR 3]"

    def get_meta(self, key):
        prefix = self.parse_key(key)[0]
        meta = finddict(SUBJECTS, prefix=prefix)

        meta = web.storage(meta)
        meta.path = web.lstrips(key, meta.prefix)
        return meta

    def GET(self, emailform=None, loadcontactsform=None):
        i = web.input()
        url = i.get("url", "/")
        title = i.get("title", "The good government site with teeth")

        user_id = helpers.get_loggedin_userid()
        contacts = get_contacts(user_id)
        sender = helpers.get_user_by_email(helpers.get_loggedin_email() or helpers.get_unverified_email())

        page_or_petition = "page"
        isdraft = False
        if not emailform:
            emailform = forms.emailform()
            track_id, description = None, None
            if url.startswith("/c/") and url != "/c/":
                url = url.rstrip("/")
                pid = web.lstrips(url, "/c/")
                p = get_petition_by_id(pid)
                isdraft = is_draft(p)
                description = p and p.description
                track_id = helpers.get_trackid(user_id, pid) if not isdraft else None
                contacts = filter(lambda c: not is_signatory(c.email, pid), contacts)
                page_or_petition = "petition"
            msg = render_plain.share_mail(title, url, sender, description, isdraft, track_id)
            emailform.fill(subject=title, body=msg)

        loadcontactsform = loadcontactsform or forms.loadcontactsform()

        msg, msg_type = helpers.get_delete_msg()
        return render.share(title, url, emailform, contacts, loadcontactsform, page_or_petition, msg)

def generate_docs(dir):
    shutil.rmtree(docpath(dir), ignore_errors=True)

    paths = list(find_python_sources(dir))

    submodule_dict = defaultdict(list)

    for path in paths:
        dir = os.path.dirname(path)

        if path.endswith("__init__.py"):
            dir = os.path.dirname(dir)

        submodule_dict[dir].append(path)

    for path in paths:
        dirname = os.path.dirname(path)
        if path.endswith("__init__.py"):
            submodules = [web.lstrips(docpath(s), docpath(dirname) + "/") for s in submodule_dict[dirname]]
        else:
            submodules = []
        submodules.sort()

        mod = modname(path)
        text = str(t(mod, submodules))
        write(docpath(path), text)

        # set the modification time same as the source file
        mtime = os.stat(path).st_mtime
        os.utime(docpath(path), (mtime, mtime))

    def POST_sign(self, pid):
        i = web.input()
        sf = forms.signform()
        tocongress = to_congress(pid)
        p = get_petition_by_id(pid)

        is_new = lambda sid: not isinstance(sid, str)
        get_new = lambda sid: int(web.lstrips(sid, "old_"))
        if tocongress:
            i.pid, i.ptitle, i.msg = pid, p.title, p.description
            wf = forms.wyrform()
            captcha_needed = require_captcha(i)
            wyr_valid = wf.validates(i) and not captcha_needed
            if captcha_needed:
                wf.valid, wf.note = False, "Please fill the captcha below"
        else:
            wf, wyr_valid = None, True

        if sf.validates(i) and wyr_valid:
            uid = auth.assert_login(i)
            signid = save_signature(i, pid, uid)
            if is_new(signid):
                user = helpers.get_user_by_id(uid)
                sendmail_to_signatory(user, pid)
            else:
                signid = get_new(signid)
            if tocongress:
                send_to_congress(uid, i, signid)
            query = urllib.urlencode(dict(url="/c/%s" % pid, title=p.title))
            raise web.seeother("/share?%s" % query, absolute=True)
        else:
            return self.GET(pid, sf=sf, wf=wf)

def unstringify(d):
    """Removes string_ prefix from every key in a dictionary.
    
        >>> unstringify({'string_a': 1, 'string_b': 2})
        {'a': 1, 'b': 2}    
    """
    return dict([(web.lstrips(k, 'string_'), v) for k, v in d.items() if k.startswith('string_')])

def main():
    options, args = parse_args()

    if options.src.startswith("http://"):
        src = OpenLibrary(options.src)
    else:
        src = Disk(options.src)

    if options.dest.startswith("http://"):
        dest = OpenLibrary(options.dest)
        section = "[%s]" % web.lstrips(options.dest, "http://").strip("/")
        if section in read_lines(os.path.expanduser("~/.olrc")):
            dest.autologin()
        else:
            dest.login("admin", "admin123")
    else:
        dest = Disk(options.dest)

    for list_key in options.lists:
        copy_list(src, dest, list_key, comment=options.comment)

    keys = args
    keys = list(expand(src, keys))

    copy(src, dest, keys, comment=options.comment, recursive=options.recursive)

    def process(self):
        self.content.seek(0, 0)
        env = {
          'REMOTE_ADDR': self.client.host,
          'REQUEST_METHOD': self.method,
          'PATH_INFO': self.path,
          'CONTENT_LENGTH': web.intget(self.getHeader('content-length'), 0),
          'wsgi.input': self.content
        }
        if '?' in self.uri:
            env['QUERY_STRING'] = self.uri.split('?', 1)[1]

        for k, v in self.received_headers.iteritems():
            env['HTTP_' + k.upper()] = v
        
        if self.path.startswith('/static/'):
            f = web.lstrips(self.path, '/static/')
            assert '/' not in f
            #@@@ big security hole
            self.write(file('static/' + f).read())
            return self.finish()

        web.webapi._load(env)
        web.ctx.trequest = self
        result = self.actualfunc()
        self.setResponseCode(int(web.ctx.status.split()[0]))
        for (h, v) in web.ctx.headers:
            self.setHeader(h, v)
        self.write(web.ctx.output)
        if not web.ctx.get('persist'):
            self.finish()

 def get_url(self):
     if self.document:
         return self.document.url()
     else:
         if self.key.startswith("subject:"):
             return "/subjects/" + web.lstrips(self.key, "subject:")
         else:
             return "/subjects/" + self.key

    def __call__(self, environ, start_response):
        path = environ.get('PATH_INFO', '')
        path = self.normpath(path)

        for prefix, root_path in self.paths:
            if path.startswith(prefix):
                environ["PATH_INFO"] = web.lstrips(path, prefix)
                return StaticApp(root_path, environ, start_response)
        return self.app(environ, start_response)

    def __call__(self, environ, start_response):
        path = environ.get('PATH_INFO', '')
        path = self.normpath(path)

        if path.startswith(self.prefix):
            environ["PATH_INFO"] = os.path.join(self.root_path, web.lstrips(path, self.prefix))
            return web.httpserver.StaticApp(environ, start_response)
        else:
            return self.app(environ, start_response)

  def translate_path(self, path):
    root = os.getcwd()
    for prefix in self.prefixDict.keys():
      if path.startswith(prefix):
        rootPath = os.path.join(root, self.prefixDict[prefix])
        fullPath = os.path.join(rootPath, web.lstrips(path,prefix))
        return fullPath

    result = web.httpserver.StaticApp.translate_path(self, path)
    return result