def load_db():
    """ load lines from find.txt to search_dict """
    if not os.path.isfile("find.txt"):
        f = open("find.txt", "w")
        f.write("#test,yano,foobar\n")
        f.close()
    search_file = open("find.txt", "r")
    lines = search_file.readlines()
    search_file.close()
    search_dict = dict()
    for line in lines:
        line = uc.decode(line)
        line = uc.encode(line)
        a = line.replace(r'\n', '')
        new = a.split(r',')
        if len(new) < 3: continue
        channel = new[0]
        nick = new[1]
        if len(new) < 2: continue
        if channel not in search_dict:
            search_dict[channel] = dict()
        if nick not in search_dict[channel]:
            search_dict[channel][nick] = list()
        if len(new) > 3:
            result = ",".join(new[2:])
            result = result.replace('\n','')
        elif len(new) == 3:
            result = new[-1]
            if len(result) > 0:
                result = result[:-1]
        if result:
            search_dict[channel][nick].append(uc.decode(result))
    return search_dict

 def ten(channel, ranking):
     channel = channel.lower()
     if channel not in self.scores_dict:
         return self.STRINGS["nochan"].format(channel)
     q = 0
     top_scores = list()
     if ranking == 'b':
         tob = 'Bottom'
     elif ranking == 't':
         tob= 'Top'
     str_say = "\x02%s 10 (for %s):\x02" % (tob, channel)
     sort = True
     if ranking == 'b':
         sort = False
     scores = sorted(self.scores_dict[channel].iteritems(),
                     key=lambda (k, v): (v[0] - v[1]), reverse=sort)
     for key, value in scores:
         top_scores.append(self.str_score(key, channel))
         if len(scores) == q + 1:
             str_say += ' %s' % (uc.decode(top_scores[q]))
         else:
             str_say += ' %s |' % (uc.decode(top_scores[q]))
         q += 1
         if q > 9:
             break
     return str_say

def c(jenni, input):
    """.c -- Google calculator."""
    if not input.group(2):
        return jenni.reply('Nothing to calculate.')
    q = input.group(2).encode('utf-8')
    q = q.replace('\xcf\x95', 'phi')  # utf-8 U+03D5
    q = q.replace('\xcf\x80', 'pi')  # utf-8 U+03C0
    uri = 'https://www.google.com/ig/calculator?q='
    bytes = web.get(uri + web.urllib.quote(q))
    parts = bytes.split('",')
    equation = [p for p in parts if p.startswith('{lhs: "')][0][7:]
    answer = [p for p in parts if p.startswith('rhs: "')][0][6:]
    if answer and equation:
        answer = answer.decode('unicode-escape')
        answer = ''.join(chr(ord(c)) for c in answer)
        answer = uc.decode(answer)
        answer = answer.replace(u'\xc2\xa0', ',')
        answer = answer.replace('<sup>', '^(')
        answer = answer.replace('</sup>', ')')
        answer = web.decode(answer)
        answer = answer.strip()
        equation = uc.decode(equation)
        equation = equation.strip()
        max_len = 450 - len(answer) - 6
        new_eq = equation
        if len(new_eq) > max_len:
            end = max_len - 10
            new_eq = new_eq[:end]
            new_eq += '[..]'
        output = '\x02' + answer + '\x02' + ' == ' + new_eq
        jenni.say(output)
    else:
        jenni.say('Sorry, no result.')

def e(m):
    entity = m.group()
    if entity.startswith("&#x"):
        cp = int(entity[3:-1], 16)
        meep = unichr(cp)
    elif entity.startswith("&#"):
        cp = int(entity[2:-1])
        meep = unichr(cp)
    else:
        char = name2codepoint[entity[1:-1]]
        meep = unichr(char)
    try:
        return uc.decode(meep)
    except:
        return uc.decode(uc.encode(meep))

def delquote(jenni, input):
    '''.rmquote <number> -- removes a given quote from the database. Can only be done by the owner of the bot.'''
    if not input.owner: return
    text = input.group(2)
    number = int()
    try:
        fn = open('quotes.txt', 'r')
    except:
        return jenni.reply('No quotes to delete.')
    lines = fn.readlines()
    MAX = len(lines)
    fn.close()
    try:
        number = int(text)
    except:
        jenni.reply('Please enter the quote number you would like to delete.')
        return
    newlines = lines[:number-1] + lines[number:]
    fn = open('quotes.txt', 'w')
    for line in newlines:
        txt = uc.decode(line)
        if txt:
            fn.write(txt)
            if txt[-1] != '\n':
                fn.write('\n')
    fn.close()
    jenni.reply('Successfully deleted quote %s.' % (number))

def collectlines(jenni, input):
    """Creates a temporary storage of most recent lines for s///"""
    #channel = (input.sender).encode("utf-8")
    channel = uc.decode(input.sender)
    channel = uc.encode(channel)
    nick = (input.nick).encode("utf-8")
    search_dict = load_db()
    if channel not in search_dict:
        search_dict[channel] = dict()
    if 'last_said' not in search_dict[channel]:
        search_dict[channel]['last_said'] = list()
    if nick not in search_dict[channel]:
        search_dict[channel][nick] = list()
    templist = search_dict[channel][nick]
    last_said_templist = search_dict[channel]['last_said']
    line = input.group()
    try:
        line = (line).encode("utf-8")
    except Exception:
        return
    if line.startswith("s/") or line.startswith('!'):
        return
    elif line.startswith("\x01ACTION"):
        line = line[:-1]
    templist.append(line)
    last_said_templist.append("{}: {}".format(input.nick, line))
    del templist[:-50]
    del last_said_templist[:-50]
    search_dict[channel][nick] = templist
    search_dict[channel]['last_said'] = last_said_templist
    save_db(search_dict)

def short(text):
    """
    This function creates a bitly url for each url in the provided string.
    The return type is a list.
    """

    if not bitly_loaded:
        return list()
    if not text:
        return list()
    bitlys = list()
    try:
        a = re.findall(url_finder, text)
        k = len(a)
        i = 0
        while i < k:
            b = uc.decode(a[i][0])
            ## make sure that it is not already a bitly shortened link
            if not is_bitly(b):
                longer = urllib2.quote(b)
                url = 'https://api-ssl.bitly.com/v3/shorten?login=%s' % (bitly_user)
                url += '&apiKey=%s&longUrl=%s&format=txt' % (bitly_api_key,
                                                             longer)
                #shorter = proxy.get(url)
                shorter = web.get(url)
                shorter.strip()
                bitlys.append([b, shorter])
            else:
                bitlys.append([b, str()])
            i += 1
            return bitlys
    except:
        return
    return bitlys

def collectlines(jenni, input):
    """Creates a temporary storage of most recent lines for s///"""
    # don't log things in PM
    #channel = (input.sender).encode("utf-8")
    channel = uc.decode(input.sender)
    channel = uc.encode(channel)
    nick = (input.nick).encode("utf-8")
    if not channel.startswith('#'): return
    search_dict = load_db()
    if channel not in search_dict:
        search_dict[channel] = dict()
    if nick not in search_dict[channel]:
        search_dict[channel][nick] = list()
    templist = search_dict[channel][nick]
    line = input.group()
    if line.startswith("s/"):
        return
    elif line.startswith("\x01ACTION"):
        line = line[:-1]
        templist.append(line)
    else:
        templist.append(line)
    del templist[:-10]
    search_dict[channel][nick] = templist
    save_db(search_dict)

def get_results(text):
    if not text:
        return list()
    a = re.findall(url_finder, text)
    k = len(a)
    i = 0
    display = list()
    passs = False
    while i < k:
        url = uc.encode(a[i][0])
        url = uc.decode(url)
        url = uc.iriToUri(url)
        url = remove_nonprint(url)
        domain = getTLD(url)
        if '//' in domain:
            domain = domain.split('//')[1]
        if not url.startswith(EXCLUSION_CHAR):
            passs, page_title = find_title(url)
            if bitly_loaded:
                bitly = short(url)
                bitly = bitly[0][1]
            else:
                bitly = url
            display.append([page_title, url, bitly])
        i += 1
    return passs, display

def short(text):
    """
    This function creates a bitly url for each url in the provided string.
    The return type is a list.
    """

    if not bitly_loaded:
        return list()
    if not text:
        return list()
    bitlys = list()
    try:
        a = re.findall(url_finder, text)
        k = len(a)
        i = 0
        while i < k:
            b = uc.decode(a[i][0])
            if not b.startswith('http://bit.ly') and \
                    not b.startswith('http://j.mp/'):
                longer = urllib2.quote(b)
                url = 'http://api.j.mp/v3/shorten?login=%s' % (bitly_user)
                url += '&apiKey=%s&longUrl=%s&format=txt' % (bitly_api_key,
                                                             longer)
                shorter = web.get(url)
                shorter.strip()
                bitlys.append([b, shorter])
            else:
                bitlys.append([b, b])
            i += 1
        return bitlys
    except:
        return

def show_title_auto(jenni, input):
    '''No command - Automatically displays titles for URLs'''
    for each in BLOCKED_MODULES:
        if input.startswith('.%s ' % (each)):
            ## Don't want it to show duplicate titles
            return
    if len(re.findall('\([\d]+\sfiles\sin\s[\d]+\sdirs\)', input)) == 1:
        ## Directory Listing of files
        return
    status, results = get_results(input)

    k = 1

    output_shorts = str()
    results_len = len(results)

    for r in results:
        returned_title = r[0]
        orig = r[1]
        bitly_link = r[2]
        link_pass = r[3]

        if orig and bitly_link and bitly_link != orig and ('bit.ly' in bitly_link or 'j.mp' in bitly_link):
            bitly_link = bitly_link.replace('http:', 'https:')

        if returned_title == 'imgur: the simple image sharer':
            return

        if k > 3:
            break
        k += 1

        useBitLy = doUseBitLy(returned_title, orig)

        reg_format = '[ %s ] - %s'
        special_format = '[ %s ]'
        response = str()

        if status and link_pass:
            if useBitLy and input.sender not in simple_channels and bitly_link:
                response = reg_format % (uc.decode(returned_title), bitly_link)
            else:
                if input.sender in simple_channels:
                    response = special_format % (returned_title)
                else:
                    response = reg_format % (returned_title, orig)
        elif len(orig) > BITLY_TRIGGER_LEN_NOTITLE:
            if useBitLy and bitly_link != orig:
                #response = '%s' % (bitly_link)
                output_shorts += bitly_link + ' '
            else:
                ## Fail silently, link can't be bitly'ed and no title was found
                pass

        if response:
            jenni.say(response)

    if output_shorts:
        jenni.say((output_shorts).strip())

def cs(jenni, input):
    '''.cs <callsign> -- queries qth.com for call sign information'''
    cs = input.group(2).upper()
    try:
        link = "http://www.qth.com/callsign.php?cs=" + uc.decode(web.quote(cs))
    except Exception, e:
        print e
        return jenni.say('Failed to obtain data from qth.com')

 def save(self):
     """ Save to file in comma seperated values """
     os.rename(self.scores_filename, '%s-%s' % (self.scores_filename, str(time.time())))
     scores_file = codecs.open(self.scores_filename, 'w', encoding='utf-8')
     for each_chan in self.scores_dict:
         for each_nick in self.scores_dict[each_chan]:
             line = '{0},{1},{2},{3}\n'.format(each_chan, each_nick, self.scores_dict[each_chan][each_nick][0], self.scores_dict[each_chan][each_nick][1])
             scores_file.write(uc.decode(line))
     scores_file.close()

def find_title(url):
    """
    This finds the title when provided with a string of a URL.
    """
    uri = url

    for item in IGNORE:
        if item in uri:
            return False, 'ignored'

    if not re.search('^((https?)|(ftp))://', uri):
        uri = 'http://' + uri

    if '/#!' in uri:
        uri = uri.replace('/#!', '/?_escaped_fragment_=')

    if 'i.imgur' in uri:
        a = uri.split('.')
        uri = a[0][:-1] + '.'.join(a[1:-1])

    if 'zerobin.net' in uri:
        return True, 'ZeroBin'

    uri = uc.decode(uri)

    ## proxy the lookup of the headers through .py
    def remote_call():
        pyurl = u'https://tumbolia.appspot.com/py/'
        code = 'import simplejson;'
        code += 'opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(),'
        code += 'urllib2.BaseHandler(), urllib2.HTTPHandler(),'
        code += 'urllib2.HTTPRedirectHandler(), urllib2.HTTPErrorProcessor(),'
        code += 'urllib2.UnknownHandler());'
        code += 'urllib2.install_opener(opener);'
        code += "req=urllib2.Request(%s, headers={'Accept':'*/*'});"
        code += "req.add_header('User-Agent', %s);"
        code += "u=urllib2.urlopen(req);"
        code += "rtn=dict();"
        code += "rtn['headers'] = u.headers.dict;"
        code += "contents = u.read(32768);"
        code += "con = str();"
        code += r'''exec "try: con=(contents).decode('utf-8')\n'''
        code += '''except: con=(contents).decode('iso-8859-1')";'''
        code += "rtn['read'] = con;"
        code += "rtn['url'] = u.url;"
        code += "rtn['geturl'] = u.geturl();"
        code += "print simplejson.dumps(rtn)"
        query = code % (repr(uri), repr(USER_AGENT))
        temp = web.quote(query)
        u = web.get(pyurl + temp)

        try:
            useful = json.loads(u)
            return True, useful
        except Exception, e:
            #print "%s -- Failed to parse json from web resource. -- %s" % (time.time(), str(e))
            return False, str(u)

def ep(m):
    entity = m.group()
    if entity.startswith('&#x'):
        cp = int(entity[3:-1], 16)
        meep = unichr(cp)
    elif entity.startswith('&#'):
        cp = int(entity[2:-1])
        meep = unichr(cp)
    else:
        entity_stripped = entity[1:-1]
        try:
            char = name2codepoint[entity_stripped]
            meep = unichr(char)
        except:
            meep = str()
    try:
        return uc.decode(meep)
    except:
        return uc.decode(uc.encode(meep))

def clean_up_answer(answer):
    answer = answer.encode('utf-8')
    answer = answer.decode('utf-8')
    answer = ''.join(chr(ord(c)) for c in answer)
    answer = uc.decode(answer)
    answer = answer.replace('<sup>', '^(')
    answer = answer.replace('</sup>', ')')
    answer = web.decode(answer)
    answer = answer.strip()
    answer += ' [GC]'
    return answer

def ip_lookup(jenni, input):
    txt = input.group(2)
    if not txt:
        return jenni.reply("No search term!")
    txt = uc.encode(txt)
    query = uc.decode(txt)
    response = "[IP/Host Lookup] "
    try:
        page = web.get(base + txt)
    except IOError, err:
        return jenni.say('Could not access given address. (Detailed error: %s)' % (err))

 def e(m):
     entity = m.group()
     if entity.startswith("&#x"):
         cp = int(entity[3:-1], 16)
         meep = unichr(cp)
     elif entity.startswith("&#"):
         cp = int(entity[2:-1])
         meep = unichr(cp)
     else:
         entity_stripped = entity[1:-1]
         try:
             char = name2codepoint[entity_stripped]
             meep = unichr(char)
         except:
             if entity_stripped in HTML_ENTITIES:
                 meep = HTML_ENTITIES[entity_stripped]
             else:
                 meep = str()
     try:
         return uc.decode(meep)
     except:
         return uc.decode(uc.encode(meep))

def quote(s, safe='/'):  # quote('abc def') -> 'abc%20def'
    s = uc.encode(s)
    s = uc.decode(s)
    safe += always_safe
    safe_map = dict()
    for i in range(256):
        c = chr(i)
        safe_map[c] = (c in safe) and c or ('%%%02X' % i)
    try:
        res = map(safe_map.__getitem__, s)
    except:
        return ''
    return ''.join(res)

def find_title(url):
    """
    This finds the title when provided with a string of a URL.
    """
    uri = url

    for item in IGNORE:
        if item in uri:
            return False, 'ignored'

    if not re.search('^((https?)|(ftp))://', uri):
        uri = 'http://' + uri

    if 'twitter.com' in uri:
        uri = uri.replace('#!', '?_escaped_fragment_=')

    if 'i.imgur' in uri:
        a = uri.split('.')
        uri = a[0][:-1] + '.'.join(a[1:-1])

    uri = uc.decode(uri)

    ## proxy the lookup of the headers through .py
    def remote_call():
        pyurl = u'https://tumbolia.appspot.com/py/'
        code = 'import simplejson;'
        code += 'import time; unique = str(time.time());'
        code += "req=urllib2.Request(u'%s', headers={'Accept':'text/html'});"
        code += "req.add_header('User-Agent', 'Mozilla/5.0"
        code += "');"
        code += "u=urllib2.urlopen(req);"
        code += "rtn=dict();"
        code += "rtn['headers'] = u.headers.dict;"
        code += "contents = u.read();"
        code += "con = str();"
        code += r'''exec "try: con=(contents).decode('utf-8')\n'''
        code += '''except: con=(contents).decode('iso-8859-1')";'''
        code += "rtn['read'] = con;"
        code += "rtn['url'] = u.url;"
        code += "rtn['geturl'] = u.geturl();"
        code += r"print simplejson.dumps(rtn)"
        query = code % uri
        temp = web.quote(query)
        u = web.get(pyurl + temp)

        try:
            useful = json.loads(u)
            return True, useful
        except Exception, e:
            #print "%s -- Failed to parse json from web resource. -- %s" % (time.time(), str(e))
            return False, str(u)