def urbandict(phenny, input):
    """.urb <word> - Search Urban Dictionary for a definition."""

    word = input.group(2)
    if not word:
        phenny.say(urbandict.__doc__.strip())
        return

    # create opener
    #opener = urllib.request.build_opener()
    #opener.addheaders = [
    #    ('User-agent', web.Grab().version),
    #    ('Referer', "http://m.urbandictionary.com"),
    #]

    try:
        data = web.get(
            "http://api.urbandictionary.com/v0/define?term={0}".format(
                web.quote(word)))
        data = json.loads(data)
    except:
        raise GrumbleError(
            "Urban Dictionary slemped out on me. Try again in a minute.")

    if data['result_type'] == 'no_results':
        phenny.say("No results found for {0}".format(word))
        return

    result = data['list'][0]
    url = 'http://www.urbandictionary.com/define.php?term={0}'.format(
        web.quote(word))

    response = "{0}: {1} - {2}".format(word, result['definition'].strip()[:256], url)
    phenny.say(response)

def translate(text, input='auto', output='en'): 
    raw = False
    if output.endswith('-raw'): 
        output = output[:-4]
        raw = True

    #opener = urllib.request.build_opener()
    #opener.addheaders = [(
    #    'User-Agent', 'Mozilla/5.0' + 
    #    '(X11; U; Linux i686)' + 
    #    'Gecko/20071127 Firefox/2.0.0.11'
    #)]
    input = web.quote(input)
    output = web.quote(output.encode('utf-8'))
    text = web.quote(text.encode('utf-8'))

    result = web.get('http://translate.google.com/translate_a/t?' +
        ('client=t&hl=en&sl=%s&tl=%s&multires=1' % (input, output)) + 
        ('&otf=1&ssel=0&tsel=0&uptl=en&sc=1&text=%s' % text))

    while ',,' in result: 
        result = result.replace(',,', ',null,')
    result = result.replace('[,', '[null,')
    data = json.loads(result)

    if raw: 
        return str(data), 'en-raw'

    try: language = data[2] # -2][0][0]
    except: language = '?'

    return ''.join(x[0] for x in data[0]), language

def tasteometer(phenny, input):
    input1 = input.group(2)
    if not input1 or len(input1) == 0:
        phenny.say("tasteometer: compares two users' musical taste")
        phenny.say("syntax: .taste user1 user2")
        return
    input2 = input.group(3)
    user1 = resolve_username(input1)
    if not user1:
        user1 = input1
    user2 = resolve_username(input2)
    if not user2:
        user2 = input2
    if not user2 or len(user2) == 0:
        user2 = resolve_username(input.nick)
        if not user2:
            user2 = input.nick
    try:
        req = web.get(
            "%smethod=tasteometer.compare&type1=user&type2=user&value1=%s&value2=%s"
            % (APIURL, web.quote(user1), web.quote(user2))
        )
    except web.HTTPError as e:
        if e.response.status_code == 400:
            phenny.say("uhoh, someone doesn't exist on last.fm, perhaps they need to set user")
            return
        else:
            phenny.say("uhoh. try again later, mmkay?")
            return
    root = etree.fromstring(req.encode("utf-8"))
    score = root.xpath("comparison/result/score")
    if len(score) == 0:
        phenny.say("something isn't right. have those users scrobbled?")
        return

    score = float(score[0].text)
    rating = ""
    if score >= 0.9:
        rating = "Super"
    elif score >= 0.7:
        rating = "Very High"
    elif score >= 0.5:
        rating = "High"
    elif score >= 0.3:
        rating = "Medium"
    elif score >= 0.1:
        rating = "Low"
    else:
        rating = "Very Low"

    artists = root.xpath("comparison/result/artists/artist/name")
    common_artists = ""
    names = []
    if len(artists) == 0:
        common_artists = ". they don't have any artists in common."
    else:
        list(map(lambda a: names.append(a.text), artists))
        common_artists = "and music they have in common includes: %s" % ", ".join(names)

    phenny.say("%s's and %s's musical compatibility rating is %s %s" % (user1, user2, rating, common_artists))

def apertium_generate(phenny, input):
    """Use Apertium APY's generate functionality"""
    lang, text = input.groups()

    opener = urllib.request.build_opener()
    opener.addheaders = headers

    constructed_url = APIanalyseURL + '/generate?lang=' + web.quote(lang)
    constructed_url += '&q=' + web.quote(text.strip())

    try:
        response = opener.open(constructed_url).read()
    except urllib.error.HTTPError as error:
        response = error.read()
        jobj = json.loads(response.decode('utf-8'))
        if 'explanation' in jobj:
            phenny.say('The following error occurred: ' + jobj['explanation'])
        else:
            phenny.say('An error occurred: ' + str(error))
        return

    jobj = json.loads(response.decode('utf-8'))
    messages = []
    for generation, original in jobj:
        messages.append(original + " → " + generation)

    more.add_messages(input.nick, phenny,
                      "\n".join(messages),
                      break_up=lambda x, y: x.split('\n'))

def urbandict(phenny, input):
    """.urb <word> - Search Urban Dictionary for a definition."""

    word = input.group(2)
    if not word:
        phenny.say(urbandict.__doc__.strip())
        return

    try:
        data = web.get(
            "http://api.urbandictionary.com/v0/define?term={0}".format(
                web.quote(word)))
        data = json.loads(data)
    except:
        raise GrumbleError(
            "Urban Dictionary slemped out on me. Try again in a minute.")

    results = data['list']

    if not results:
        phenny.say("No results found for {0}".format(word))
        return

    result = results[0]
    url = 'http://www.urbandictionary.com/define.php?term={0}'.format(
        web.quote(word))

    response = "{0} - {1}".format(result['definition'].strip()[:256], url)
    phenny.say(response)

    def test_analyze_generate(self, mock_addmsgs, mock_open):
        # analyze
        words = ['analyze', 'this']
        anas = [['{0}/{0}<tags>'.format(word), word] for word in words]
        self.input.group.return_value = 'eng ' + ' '.join(words)
        mock_open.return_value.read.return_value = bytes(dumps(anas), 'utf-8')
        apy.apertium_analyse(self.phenny, self.input)
        mock_open.assert_called_once_with('{:s}/analyse?lang={:s}&q={:s}'.format(
            self.phenny.config.APy_analyseURL, 'eng', quote(' '.join(words))))
        msgs = ['{:s}  →  {:s}'.format(orig, ana) for ana, orig in anas]
        self.assertEqual(mock_addmsgs.call_args[0][2], msgs)
        self.reset_mocks(mock_open, mock_addmsgs)

        # generate
        gens = [['generate', '^generate<tags>$']]
        self.input.group.return_value = 'eng ^generate<tags>$'
        mock_open.return_value.read.return_value = bytes(dumps(gens), 'utf-8')
        apy.apertium_generate(self.phenny, self.input)
        mock_open.assert_called_once_with('{:s}/generate?lang={:s}&q={:s}'.format(
            self.phenny.config.APy_analyseURL, 'eng', quote('^generate<tags>$')))
        msgs = ['{:s}  →  {:s}'.format(orig, gen) for gen, orig in gens]
        self.assertEqual(mock_addmsgs.call_args[0][2], msgs)
        self.reset_mocks(mock_open, mock_addmsgs)

        # bad input
        self.check_exceptions([' '.join(words), 'eng'], apy.apertium_analyse)
        self.check_exceptions([' '.join(words), 'eng'], apy.apertium_generate)

def wikipedia_search(query): 
    query = query.replace('!', '')
    query = web.quote(query)
    uri = 'https://en.wikipedia.org/w/api.php?action=query&list=search&continue=&srsearch=%s&format=json' % query
    rec_bytes = web.get(uri)
    jsonstring = json.loads(rec_bytes)
    wihits = jsonstring['query']['searchinfo']['totalhits']
    if wihits > 0:
        wititle = jsonstring['query']['search'][0]['title']
        wiwords = str(jsonstring['query']['search'][0]['wordcount'])
        wisearch = wititle.replace('!', '')
        wisearch = web.quote(wisearch)
        base_url = "https://en.wikipedia.org/wiki/"+wisearch
        return (wititle + " - " + wiwords + " words " + base_url)

def translate(translate_me, input_lang, output_lang='en'): 
	input_lang, output_lang = web.quote(input_lang), web.quote(output_lang)
	translate_me = web.quote(translate_me)
	
	response = get_page('api.apertium.org', '/json/translate?q=%s&langpair=%s|%s' % (translate_me, input_lang, output_lang))
	
	responseArray = json.loads(response)
	if int(responseArray['responseStatus']) != 200:
		raise GrumbleError(APIerrorHttp % (responseArray['responseStatus'], responseArray['responseDetails']))
	if responseArray['responseData']['translatedText'] == []:
		raise GrumbleError(APIerrorData)

	translated_text = responseArray['responseData']['translatedText']
	return translated_text

    def remote_call():
        pyurl = u'https://tumbolia.appspot.com/py/'
        code = 'import simplejson;'
        code += 'opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(),'
        code += 'urllib2.BaseHandler(), urllib2.HTTPHandler(),'
        code += 'urllib2.HTTPRedirectHandler(), urllib2.HTTPErrorProcessor(),'
        code += 'urllib2.UnknownHandler());'
        code += 'urllib2.install_opener(opener);'
        code += "req=urllib2.Request(%s, headers={'Accept':'*/*'});"
        code += "req.add_header('User-Agent', %s);"
        code += "u=urllib2.urlopen(req);"
        code += "rtn=dict();"
        code += "rtn['headers'] = u.headers.dict;"
        code += "contents = u.read(32768);"
        code += "con = str();"
        code += r'''exec "try: con=(contents).decode('utf-8')\n'''
        code += '''except: con=(contents).decode('iso-8859-1')";'''
        code += "rtn['read'] = con;"
        code += "rtn['url'] = u.url;"
        code += "rtn['geturl'] = u.geturl();"
        code += "print simplejson.dumps(rtn)"
        query = code % (repr(uri), repr(USER_AGENT))
        temp = web.quote(query)
        u = web.get(pyurl + temp)

        try:
            useful = json.loads(u)
            return True, useful
        except Exception, e:
            #print "%s -- Failed to parse json from web resource. -- %s" % (time.time(), str(e))
            return False, str(u)

def duck_search(query): 
    query = query.replace('!', '')
    query = web.quote(query)
    uri = 'http://duckduckgo.com/html/?q=%s&kl=uk-en' % query
    bytes = web.get(uri)
    m = r_duck.search(bytes)
    if m: return web.decode(m.group(1))

def rule34(phenny, input):
    """.rule34 <query> - Rule 34: If it exists there is porn of it."""

    q = input.group(2)
    if not q:
        phenny.say(rule34.__doc__.strip())
        return

    try:
        req = web.get("http://rule34.xxx/index.php?page=post&s=list&tags={0}".format(web.quote(q)))
    except:
        raise GrumbleError("THE INTERNET IS FUCKING BROKEN. Please try again later.")

    doc = lxml.html.fromstring(req)
    doc.make_links_absolute('http://rule34.xxx/')
    thumbs = doc.find_class('thumb')
    if len(thumbs) <= 0:
        phenny.reply("You just broke Rule 34! Better start uploading...")
        return

    try:
        link = thumbs[0].find('a').attrib['href']
    except AttributeError:
        raise GrumbleError("THE INTERNET IS FUCKING BROKEN. Please try again later.")

    response = '!!NSFW!! -> {0} <- !!NSFW!!'.format(link)
    phenny.reply(response)

def search(query):
    query = web.quote(query)
    try:
        r = web.get(SEARCH_URL.format(query), verify=False)
    except (web.ConnectionError, web.HTTPError):
        raise GrumbleError("THE INTERNET IS FUCKING BROKEN. Please try again later.")

    # apparently the failure mode if you search for <3 characters is a blank
    # XML page...
    if len(r) <= 0:
        return False

    xml = lxml.etree.fromstring(r.encode('utf-8'))
    results = xml.findall('{0}directory-entries/{0}entry'.format(NS))
    if len(results) <= 0:
        return False

    ret = []
    for entry in results:
        entry_data = {}
        for attr in entry.findall('{0}attr'.format(NS)):
            entry_data[attr.attrib['name']] = attr[0].text
        ret.append(entry_data)

    return ret

def py(phenny, input): 
    query = input.group(2) or ""
    uri = 'http://tumbolia.appspot.com/py/'
    answer = web.get(uri + web.quote(query))
    if answer: 
        phenny.say(answer)
    else: phenny.reply('Sorry, no result.')

def derpibooru_search(query, phenny):
    query = query.replace("!", "")
    query = web.quote(query)
    if hasattr(phenny.config, "derpibooru_key"):
        uri = "https://derpibooru.org/search.json?q=" + query + "&key=" + phenny.config.derpibooru_key
    else:
        uri = "https://derpibooru.org/search.json?q=" + query
    rec_bytes = web.get(uri)
    jsonstring = json.loads(rec_bytes)
    dhits = jsonstring["total"]
    if dhits > 0:
        results = choice(jsonstring["search"])
        url = "https:" + results["image"]
        uploader = results["uploader"]
        uploaded = results["created_at"]
        try:
            import dateutil.parser

            isdateutil = True
            dt = dateutil.parser.parse(uploaded)
            timestamp1 = calendar.timegm(dt.timetuple())
            timestamp1 = time.gmtime(timestamp1)
            uploadedformat = time.strftime("%A %B %d, %Y at %I:%M:%S %p", timestamp1)
        except:
            isdateutil = False
        if isdateutil is True:
            return url + " uploaded by " + uploader + " on " + uploadedformat
        else:
            return url + " uploaded by " + uploader
    else:
        return

def wa(jenni, input):
    """Wolfram Alpha calculator"""
    if not input.group(2):
        return jenni.reply("No search term.")
    query = input.group(2).encode('utf-8')
    uri = 'http://tumbolia.appspot.com/wa/'
    try:
        answer = web.get(uri + web.quote(query.replace('+', '%2B')), 45)
    except timeout as e:
        return jenni.say('[WOLFRAM ERROR] Request timed out')
    if answer:
        answer = answer.decode('string_escape')
        answer = HTMLParser.HTMLParser().unescape(answer)
        #This might not work if there are more than one instance of escaped unicode chars
        #But so far I haven't seen any examples of such output examples from Wolfram Alpha
        match = re.search('\\\:([0-9A-Fa-f]{4})', answer)
        if match is not None:
            char_code = match.group(1)
            char = unichr(int(char_code, 16))
            answer = answer.replace('\:'+char_code, char)
        waOutputArray = string.split(answer, ";")
        if(len(waOutputArray) < 2):
            jenni.say('[WOLFRAM ERROR]'+answer)
        else:
            
            jenni.say('[WOLFRAM] ' + waOutputArray[0]+" = "+waOutputArray[1])
        waOutputArray = []
    else: jenni.reply('Sorry, no result.')

    def remote_call():
        pyurl = u'https://tumbolia.appspot.com/py/'
        code = 'import simplejson;'
        code += "req=urllib2.Request(%s, headers={'Accept':'*/*'});"
        code += "req.add_header('User-Agent', 'Mozilla/5.0');"
        code += "u=urllib2.urlopen(req);"
        code += "rtn=dict();"
        code += "rtn['headers'] = u.headers.dict;"
        code += "contents = u.read();"
        code += "con = str();"
        code += r'''exec "try: con=(contents).decode('utf-8')\n'''
        code += '''except: con=(contents).decode('iso-8859-1')";'''
        code += "rtn['read'] = con;"
        code += "rtn['url'] = u.url;"
        code += "rtn['geturl'] = u.geturl();"
        code += "print simplejson.dumps(rtn)"
        query = code % repr(uri)
        temp = web.quote(query)
        u = web.get(pyurl + temp)

        try:
            useful = json.loads(u)
            return True, useful
        except Exception, e:
            #print "%s -- Failed to parse json from web resource. -- %s" % (time.time(), str(e))
            return False, str(u)

def parse_wiki_page(url, term, section = None):
    try:
        web_url = web.quote(url).replace("%3A", ":", 1)
        html = str(web.get(web_url))
    except:
        return "A wiki page does not exist for that term."
    page = lxml.html.fromstring(html)
    if section is not None:
        text = page.find(".//span[@id='%s']" % section)

        if text is None:
            return "That subsection does not exist."
        text = text.getparent().getnext()

        #a div tag may come before the text
        while text.tag is not None and text.tag != "p":
            text = text.getnext()
        url += "#" + format_term_display(section)
    else:
        #Get first paragraph

        text = page.get_element_by_id('mw-content-text').find('p')

    sentences = text.text_content().split(". ")   
    sentence = '"' + sentences[0] + '"'

    maxlength = 430 - len((' - ' + url).encode('utf-8'))
    if len(sentence.encode('utf-8')) > maxlength: 
        sentence = sentence.encode('utf-8')[:maxlength].decode('utf-8', 'ignore')
        words = sentence[:-5].split(' ')
        words.pop()
        sentence = ' '.join(words) + ' [...]'

    return sentence + ' - ' + url

def derpibooru_search(query, phenny):
    query = query.replace('!', '')
    query = web.quote(query)
    if hasattr(phenny.config, 'derpibooru_key'):
        uri = 'https://derpibooru.org/search.json?q=' + query + '&key=' + phenny.config.derpibooru_key
    else:
        uri = 'https://derpibooru.org/search.json?q=' + query
    rec_bytes = web.get(uri)
    jsonstring = json.loads(rec_bytes)
    dhits = jsonstring['total']
    if dhits > 0:
        results = choice(jsonstring['search'])
        url = 'https:' + results['image']
        uploader = results['uploader']
        uploaded = results['created_at']
        try:
            import dateutil.parser
            isdateutil = True
            dt = dateutil.parser.parse(uploaded)
            timestamp1 = calendar.timegm(dt.timetuple())
            timestamp1 = time.gmtime(timestamp1)
            uploadedformat = time.strftime('%A %B %d, %Y at %I:%M:%S %p',timestamp1)
        except:
            isdateutil = False
        if isdateutil is True:
            return url + ' uploaded by ' + uploader + ' on ' + uploadedformat
        else:
            return url + ' uploaded by ' + uploader
    else:
        return

    def test_perword(self, mock_open):
        # valid perword functions
        words = ['two', 'words']
        funcs = ['tagger', 'morph']
        per = [
            {'input': 'two', 'tagger': ['two<tags>'], 'morph': ['two<tags1>', 'two<tags2>']},
            {'input': 'words', 'tagger': ['words<tags>'], 'morph': ['words<tags1>', 'words<tags2>']}
        ]
        self.input.group.return_value = 'fra ({:s}) {:s}'.format(' '.join(funcs), ' '.join(words))
        mock_open.return_value.read.return_value = bytes(dumps(per), 'utf-8')
        apy.apertium_perword(self.phenny, self.input)
        mock_open.assert_called_once_with('{:s}/perWord?lang={:s}&modes={:s}&q={:s}'.format(
            self.phenny.config.APy_url, 'fra', '+'.join(funcs), quote(' '.join(words))))
        calls = []
        for word in per:
            calls.append(mock.call(word['input'] + ':'))
            for func in funcs:
                calls.append(mock.call('  {:9s}: {:s}'.format(func, ' '.join(word[func]))))
        self.assertEqual(self.phenny.say.call_args_list, calls)
        self.reset_mocks(self.phenny, mock_open)

        # bad input
        self.check_exceptions(['fra (tagger nonfunc) word'], apy.apertium_perword,
                              'invalid perword function')
        self.check_exceptions(['fra', 'fra (tagger)', '(tagger)', 'fra word',
                               '(tagger morph) word'], apy.apertium_perword)

def abbreviate_search(query, phenny): 
    query = query.replace('!', '')
    webquery = web.quote(query)
    uri = 'http://www.nactem.ac.uk/software/acromine/dictionary.py?lf=' + webquery
    rec_bytes = web.get(uri)
    jsonstring = json.loads(rec_bytes)
    try:
        asf = jsonstring[0]['sf']
    except:
        return
    try:
        a1 = jsonstring[0]['sf']
        a2 = jsonstring[1]['sf']
        a3 = jsonstring[2]['sf']
    except:
        try:
            a1 = jsonstring[0]['sf']
            a2 = jsonstring[1]['sf']
        except:
            try:
                a1 = jsonstring[0]['sf']
            except:
                return 'There was an error parsing the json data'
    try:
        return query + ' could be abbreviated as ' + a1 + ' or ' + a2 + ' or ' + a3
    except:
        try:
            return query + ' could be abbreviated as ' + a1 + ' or ' + a2
        except:
            return '1 result for ' + query + ', ' + a1