def main():
    assert os.path.exists(ALMANAC_DIR), ALMANAC_DIR
    
    files = glob.glob(ALMANAC_DIR + 'people/*/rep_*.htm') + \
            glob.glob(ALMANAC_DIR + 'people/*/*s[12].htm')
    files.sort()
    for fn in files:
        district = web.storage()
        demog = None
        
        dist = web.lstrips(web.rstrips(fn.split('/')[-1], '.htm'), 'rep_')
        diststate = dist[0:2].upper()
        distnum = dist[-2:]
        distname = tools.fixdist(diststate + '-' + distnum)
        
        d = almanac.scrape_person(fn)
        load_election_results(d, distname)

        if 'demographics' in d:
            demog = d['demographics']
        elif distname[-2:] == '00' or '-' not in distname:   # if -00 then this district is the same as the state.
            #print "Using state file for:", distname
            statefile = ALMANAC_DIR + 'states/%s/index.html' % diststate.lower()
            demog = almanac.scrape_state(statefile).get('state')

        demog_to_dist(demog, district)

        district.almanac = 'http://' + d['filename'][d['filename'].find('nationaljournal.com'):]

        #print 'district:', distname, pformat(district)
        db.update('district', where='name=$distname', vars=locals(), **district)

def save_signature(i, pid, uid):
    where = "petition_id=$pid AND user_id=$uid"
    signed = db.select("signatory", where=where, vars=locals())
    share_with = (i.get("share_with", "off") == "on" and "N") or "A"
    update_user_details(i)
    if not signed:
        referrer = get_referrer(pid, uid)
        signid = db.insert(
            "signatory",
            user_id=uid,
            share_with=share_with,
            petition_id=pid,
            comment=i.get("comment"),
            referrer=referrer,
        )
        helpers.set_msg("Thanks for your signing! Why don't you tell your friends about it now?")
        return signid
    else:
        db.update(
            "signatory",
            where="user_id=$uid and petition_id=$pid",
            comment=i.get("comment"),
            deleted=None,
            vars=locals(),
        )
        helpers.set_msg("Your signature has been changed. Why don't you tell your friends about it now?")
        return "old_%s" % signed[0].id

 def POST_unsign(self, pid):
     i = web.input()
     now = datetime.now()
     db.update("signatory", deleted=now, where="petition_id=$pid and user_id=$i.user_id", vars=locals())
     msg = "Your signature has been removed for this petition."
     helpers.set_msg(msg)
     raise web.seeother("/%s" % pid)

 def GET(self, n):
     res = db.select('pastes', where='id=$n', vars=locals())
     db.update('pastes', where='id=$n',vars=locals(), lastview=now_time())
     if res: 
         return render.view(res[0],n)
     else:
         return web.notfound()

def loadbill(fn, maplightid=None):
    bill = xmltramp.load(fn)
    d = bill2dict(bill)
    d.maplightid = maplightid
    
    try:
        bill_id = d.id
        db.insert('bill', seqname=False, **d)
    except IntegrityError:
        bill_id = d.pop('id')
        db.update('bill', where="id=" + web.sqlquote(bill_id), **d)
    
    positions = {}
    for vote in bill.actions['vote':]:
        if not vote().get('roll'): continue
        
        rolldoc = '/us/%s/rolls/%s%s-%s.xml' % (
          d.session, vote('where'), vote('datetime')[:4], vote('roll'))
        roll = xmltramp.load(GOVTRACK_CRAWL + rolldoc)
        for voter in roll['voter':]:
            positions[govtrackp(voter('id'))] = fixvote(voter('vote'))

    if None in positions: del positions[None]
    with db.transaction():
        for p, v in positions.iteritems():
            db.insert('position', seqname=False, 
              bill_id=bill_id, politician_id=p, vote=v)

def load_election_results(d, distname):
    (year, votes, vote_pct) = (0,'0','0')
    if 'name' not in d: 
        print "No name for the congress person for: ",distname
        return
    pname = d['name'].lower()
    if 'electionresults' not in d: 
        print "No election results for %s repsenting %s." % (d['name'],distname)
        return
    for e in d['electionresults']:
        if 'candidate' in e and 'primary' not in e['election'] and \
                pname.replace(' ','') in e['candidate'].lower().replace(' ',''):
            if int(e['election'][0:4]) > year: 
                (year,votes) = (int(e['election'][0:4]), e['totalvotes'])
                if 'percent' in e: vote_pct = e['percent']
    #print year, votes, vote_pct, d['name'], distname
    if year:
        pol=db.select('politician', what='id', 
                where="district_id='"+distname+"' AND "+web.sqlors('lastname ilike ',pname.split(' ')),
                vars=locals()).list()
        if pol and len(pol)==1: 
            polid=pol[0].id
            db.update('politician', where='id=$polid', 
                    n_vote_received=votes.replace(',','').replace('Unopposed','0'),
                    pct_vote_received=vote_pct.replace('%',''), 
                    last_elected_year=year, vars=locals());
        else: print "Couldn't find an id for %s representing %s." % (d['name'], distname)
    else: print "Didn't find a recent election for %s representing %s." %(d['name'], distname) #, pformat(d['electionresults'])

def load_new_pols():
    with db.transaction():
        db.update('congress', where="current_member='t'", current_member=False)
        for polid, pol in new_pols.items():
            district = pol['district_id']
            create_or_update(polid, district)
            update_congress(polid, district)

def load():
    for district in shapes.parse():
        outline = json.dumps({'type': 'MultiPolygon', 'coordinates': district['shapes']})
        if district['district']:
            district = tools.unfips(district['state_fipscode']) + '-' + district['district']
        else:
            district = tools.unfips(district['state_fipscode'])
        db.update('district', where='name=$district', outline=outline, vars=locals())

 def save_and_send_msg(self, i, wyrform, pform=None):
     self.set_pol(i)
     msg_id = self.save_msg(i.ptitle, i.msg)  
     self.set_msg_id(msg_id)
     msg_sent = self.send_msg(i, wyrform, pform)
     if msg_sent == True:
         db.update('wyr', where='id=$msg_id', sent=True, vars=locals())
     return msg_sent    

def load():
    outdb = {}
    done = set()
    with db.transaction():
        db.delete('earmark_sponsor', '1=1')
        db.delete('earmark', '1=1')
        for e in earmarks.parse_file(earmarks.EARMARK_FILE):
            de = dict(e)
            de['id'] = web.intget(de['id'])
            if not de['id'] or de['id'] in done: continue # missing the ID? come on!
            if isinstance(de['house_request'], basestring): continue # CLASSIFIED

            for k in de: de[k] = cleanrow(de[k])
            for x in ['house_member', 'house_state', 'house_party', 'senate_member', 'senate_state', 'senate_party', 'district']:
                de.pop(x)
            
            de['recipient_stem'] = tools.stemcorpname(de['intended_recipient'])
            try:
                db.insert('earmark', seqname=False, **de)
            except:
                pprint(de)
                raise
            done.add(de['id'])
        
    reps_not_found = set()
    for e in earmarks.parse_file(earmarks.EARMARK_FILE):
        for rawRequest, chamber in zip([e.house_request, e.senate_request],[e.house_member, e.senate_member]):
            for rep in chamber:
                if rep.lower() not in lastname2rep:
                    #@@ should work on improving quality
                    reps_not_found.add(rep)
                else:
                    rep = lastname2rep[rep.lower()]
                    if e.id in done: 
                        try:
                            db.insert('earmark_sponsor', seqname=False, earmark_id=e.id, politician_id=rep)
                        except:
                            print "Couldn't add %s as sponsor to earmark %d" %(rep, e.id)
                    outdb.setdefault(rep, {
                      'amt_earmark_requested': 0,
                      'n_earmark_requested': 0,
                      'n_earmark_received': 0,
                      'amt_earmark_received': 0
                    })
                    outdb[rep]['n_earmark_requested'] += 1
                    requested = rawRequest or e.final_amt
                    if not isinstance(requested, float):
                        requested = e.final_amt
                    if requested:
                        outdb[rep]['amt_earmark_requested'] += requested
                    if isinstance(e.final_amt, float) and e.final_amt:
                        outdb[rep]['n_earmark_received'] += 1
                        outdb[rep]['amt_earmark_received'] += e.final_amt
    
    print "Did not find",len(reps_not_found),"reps:", pformat(reps_not_found)
    for rep, d in outdb.iteritems():
        db.update('politician', where='id=$rep', vars=locals(), **d)

 def POST(self):
     i = web.input()
     form = forms.passwordform()
     if form.validates(i):
         password = encrypt_password(i.password)        
         db.update('users', password=password, verified=True, where='email=$i.email', vars=locals())
         helpers.set_msg('Login with your new password.')
         raise web.seeother('/login')
     else:
         return self.GET(form)

def main():
    districts = json.load(file(DISTRICT_TABLE))
    for dist in districts.iterkeys():
        try:
            d = file(GMAPDATA + '/%s-marker.js' % dist.replace('-0', '').replace('-', '')).read()
            x = r_center.findall(d)[0]
            db.update('district', where='name=$dist', vars=locals(), 
              center_lat = x[0], center_lng = x[1], zoom_level=x[2])
        except IOError: # file not found
            continue

 def POST_unsign(self, pid):
     i = web.input()
     now = datetime.now()
     db.update('signatory',
                     deleted=now,
                     where='petition_id=$pid and user_id=$i.user_id',
                     vars=locals())
     msg = 'Your signature has been removed for this petition.'
     helpers.set_msg(msg)
     raise web.seeother('/%s' % pid)

def create_or_update(polid, dist):
    if not db.select('district', where='name=$dist', vars=locals()):
        db.insert('district', seqname=False, name=dist, state_id=dist[:2])

    if db.select('politician', where='id=$polid', vars=locals()):
        db.update('politician', where='id=$polid', district_id=dist, last_elected_year='2008', vars=locals())
    else:
        first, last = id.split('_', 1)
        first, last = first.title(), last.title()
        db.insert('politician', seqname=False, id=polid, firstname=first, lastname=last, last_elected_year='2008', district_id=dist)

 def POST(self):
     i = web.input()
     form = forms.passwordform()
     if form.validates(i):
         password = encrypt_password(i.password)
         db.update('users', password=password, verified=True, where='email=$i.email', vars=locals())
         helpers.set_login_cookie(i.email)
         helpers.set_msg('Password stored')
         raise web.seeother('/c/', absolute=True)
     else:
         return self.GET(form)

def update_user_details(i):
    user = helpers.get_user_by_email(i.get("email"))
    userid = user and user.id
    i["zip5"] = i.get("zipcode")
    i["phone"] = web.numify(i.get("phone"))
    details = ["prefix", "lname", "fname", "addr1", "addr2", "city", "zip5", "zip4", "phone", "state"]

    d = {}
    for (k, v) in i.items():
        if v and (k in details):
            d[k] = v
    db.update("users", where="id=$userid", vars=locals(), **d)

def send_to_congress(i, wyrform, signid=None):
    from utils.writerep import write_your_rep
    
    pform, wyrform = forms.petitionform(), (wyrform or forms.wyrform())
    pform.fill(i), wyrform.fill(i)
    wyr = write_your_rep()
    wyr.set_pol(i)
    if not signid: signid = get_signid(i.pid)
    wyr.set_msg_id(signid, petition=True)
    msg_sent = wyr.send_msg(i, wyrform, pform)
    sent_status = msg_sent and 'S' or 'D'   # Sent or Due for sending
    db.update('signatory', where='id=$signid', sent_to_congress=sent_status, vars=locals())

def update_user_details(i, uid=None):
    if not uid:
        user = helpers.get_user_by_email(i.get('email'))
        uid = user and user.id
    i['phone'] = web.numify(i.get('phone'))
    details = ['prefix', 'lname', 'fname', 'addr1', 'addr2', 'city', 'zip5', 'zip4', 'phone', 'state']
    
    d = {}
    for (k, v) in i.items():
        if v and (k in details):
            d[k] = v
    db.update('users', where='id=$uid', vars=locals(), **d)

def main():
    for pol in voteview.parse():
        if not pol.get('district_id'): continue #@@
    
        if not tools.districtp(pol.district_id) and pol.district_id.endswith('01'):
            pol.district_id = pol.district_id.split('-')[0] + '-' + '00'
    
        watchdog_id = tools.getWatchdogID(pol.district_id, pol.last_name)
        if watchdog_id:
            db.update('politician', where='id=$watchdog_id', vars=locals(),
              icpsrid = pol.icpsr_id, 
              nominate = pol.dim1, 
              predictability = 1 - (pol.n_errs / float(pol.n_votes)))

def main():
    for dist, canl in cans.iteritems():
        dist=dist.replace("-SEN1","").replace("-SEN2","") ## JT: Hack
        for can in canl:
            wid = tools.getWatchdogID(dist,can['lastName'])
            if wid:
                bio = bios[can['candidateId']]['candidate']
                db.update('politician', where='id = $wid', vars=locals(),
                  votesmartid=can['candidateId'], 
                  nickname=can['nickName'],
                  birthplace=bio['birthPlace'],
                  education=bio['education'].replace('\r\n', '\n')
                )