def _get_goog_urls(query):
     g = pygoogle.pygoogle(query)
     g.pages = 1 
     g.hl = "en"
     self.l.info("Google search result count: %s" % g.get_result_count())
     if g.get_result_count() > 0:
         return g.search_page_wise()[0]
     else:
         g = pygoogle.pygoogle("site:blogspot.com groot")
         g.pages = 1
         g.hl = "en"
         self.l.info("No results for original query, retrying with 'groot'")
         return g.search_page_wise()[0]

def get_impact_factor_from_issn(issn="1475-7516", debug=False):
    """
      For the input ISSN in the format NNNN-NNNN obtain
      the headers and the datasets in a nested list
      equivalent to an array of (# headers)*[4 (years)]
    """
    g = pygoogle("site:http://www.bioxbio.com/if/html " + issn)
    g.pages = 1
    if g.get_urls():
        if_file = urllib.urlopen(g.get_urls()[0])
        html = if_file.read()
        if_file.close()
    else:
        return [], []

    if debug:
        print (html)
    soup = BeautifulSoup(html)
    table = soup.find("table")

    # The first tr contains the field names.
    headings = [th.get_text().strip() for th in table.find("tr").find_all("td")]

    datasets = []
    for row in table.find_all("tr")[1:]:
        dataset = [eval(td.get_text().replace("-", "0")) for td in row.find_all("td") if td.get_text().strip()]
        datasets.append(dataset)

    return headings, datasets

def searchGoogle(searchTerm): 

    print ''
    print 'Searching Google...'
    print ''

    googler = pygoogle(searchTerm) # initialize pygoogle object with search term
    googler.pages = 3 # set max pages

    print '*********************************'
    print 'Google Results'
    print '*********************************'
    print ''

    # display google results in a formatted way
    for keys, values in googler.search().items():
        theKey=keys.replace("'","'")
        theKey=theKey.replace("&","&")
        theValue=values.replace("'","'")
        theValue=theValue.replace("&","&")
        print 'Title: ' + (theKey.encode('ascii', 'ignore'))
        print 'URL: ' + (theValue.encode('ascii', 'ignore'))
        print ''
    print ''
    print '*********************************'
    print ''

    def dork(self,search_term,p,output):
        print YELLOW + "[+] " + END + WHITE + "Searching for " + END + "%s " % search_term
        gs = pygoogle(search_term)
        gs.pages = p
        print YELLOW + "[+] " + END + WHITE + "Results Found : " + END + "%s " % (gs.get_result_count())
        if gs.get_result_count() == 0: print RED + "[-] " + END + WHITE + "No Results Found" + END; time.sleep(1); sys.exit()

        print YELLOW + "[+] " + END + WHITE + "Fetching " + END + "[%s] Results " % (gs.get_result_count())
        url_list = gs.get_urls()

        if len(url_list) == 0:
            print YELLOW + "[!] " + END + WHITE + "Got 0 URLs" + END
            print RED + "[!] " + END + WHITE + "Nothing to save" + END
            time.sleep(1)
            sys.exit()
            
        elif len(url_list) > 1:
            print YELLOW + "[+] " + END + WHITE + "Got " + END + "[%s] URLs" % (len(url_list))
            print YELLOW + "[+] " + END + WHITE + "Writing URLs to " + END + "[%s] " % (output)

            with open(output,'w') as w_file:
                for i in url_list: w_file.write(i+'\n')
            print YELLOW + "[+] " + END + WHITE + "URLs saved to " + END + "[%s] " % (output)

            time.sleep(2)

def googleIt(url):
		db = database()
		source = url
		en = ["it","zh-Hans","fr","nl","es","pt-BR","ca","pa","qu","mr","mo","mn","ne","pcm","nn","or","qu"]
		random.shuffle(en)

		search = pygoogle.pygoogle(hl=en[0],query="site:"+source)
		urlList = search.get_urls()

		print urlList
		sha1 = hashlib.sha1()

		for eachUrl in urlList:
			#Generate hash for url - used as primary key for database
			try:
				eachUrl = u"".join(eachUrl).encode('utf-8').strip()
				sha1.update(eachUrl)
				hash = sha1.hexdigest()
				numTLD = db.countTLD(eachUrl)
				#Persist item in database 
				db.addGoodUrl(source,hash, eachUrl, numTLD)
			except:
				doNothing =0
		print 'Done'
		db.close()

def do_action(lcars, string, case):
	target = string.split(' ', 1)[1]

	g = pygoogle("site:youtube.com " + target)
	g.pages = 1
	urls = g.get_urls()
	if len(urls) == 0:
		reply = "No results found for" + target
		lcars.reply_with(reply)
		return reply
	link = urls[0]

	if "user" in link or "store" in link or "feed" in link or "playlist" in link or "channel" in link:
		for url in urls:
			if "user" in url:
				link = "http://youtube.nestharion.de/" + url.split('/')[-1]
				break
		if not link:
			for url in urls:
				if "store" in url or "feed" in url or "playlist" in url or "channel" in url:
					continue
				else:
					link = url
					break
	if not link:
		link = urls[randint(0, len(urls) - 1)]

	lcars.background(["google-chrome", link])
	reply = "Playing " + target
	lcars.reply_with(reply)
	os.system("sleep 1")
	os.system("xdotool windowactivate --sync $(xdotool search --class Chrome | head -n 1) & sleep 0.3; pkill xdotool")
	os.system("xdotool windowactivate --sync $(xdotool search --class Chrome | tail -n 1) & sleep 0.3; pkill xdotool")

	return reply

def fresh_google_check(link: str, attempt=5, debug=False):
    """Проверяет, индексировался ли уже ресурс гуглом раньше.

    чем за 2 недели до сегодня.
    :param link:
    :param attempt:
    :return:

    """
    if debug:
        return False
    try:
        assert isinstance(link, str)
        today = datetime.date.today()
        date_s = _date_to_julian_day(today - datetime.timedelta(days=365 * 8))
        date_e = _date_to_julian_day(today - datetime.timedelta(days=7 * 2))
        query = u'site:%s daterange:%s-%s' % (link, date_s, date_e, )

        result = False
        for i in range(0, attempt):
            g = pygoogle(query.encode('utf-8'),
                         raise_http_exceptions=True,
                         proxies=settings.PROXIES_FOR_GOOGLING)

            try:
                result = bool(g.get_result_count())
            except PyGoogleHttpException as e:
                renew_connection()
                continue
            break
    except (AssertionError, PyGoogleHttpException, stem.SocketError):
        result = False

    return result

def LinkedIn(linkedinusername):
	from pygoogle import pygoogle
	g = pygoogle("linkedin "+linkedinusername)
	g.pages = 5
	g.get_result_count()
	myURLs = g.get_urls()
	return myURLs

def google_search(search_string):
    g = pygoogle(search_string)
    g.pages = 1
    results = g.get_urls()
    try:
        return results[0]
    except:
        return "That was not the word you're looking for"

def google_first_result(googlestring):
	pygoog = pygoogle(googlestring)
	pygoog.pages = 1
	urls = pygoog.get_urls()
	try:		
		return urls[0]
	except IndexError:
		return "http://www.google.com"

def https_search(url):
   
    string_search = "inurl:https site:"+str(url)

    g = pygoogle(string_search)
    g.pages = 5
    g.hl = "br"

    print string_search 

    results_numbers = 0
    count = 0
    temp = 6 # segundos

    while results_numbers == 0 :
        results_numbers = g.get_result_count()
        print "Resultados:",results_numbers
        print
        if results_numbers == 0:
            time.sleep( temp ) 
            count += temp 
            if count > 60: # segundos
                count = -1
                print "Desisto!"
                break

    desired_results = 5
    search_sites = {}

    if count == -1:
        print "Sem estima dos resultados da pesquisa"
        return 0

    elif results_numbers < desired_results:
        print "Poucos sites!"
        desired_results = results_numbers
    
    while len(search_sites) == 0:
        search_sites = g.search()
        print search_sites
        print
        for key in search_sites.keys():
            #print key, search_sites[key]
            print unicode(key).encode('cp850'), unicode(search_sites[key]).encode('cp850')
        if len(search_sites) == 0 or len(search_sites) < desired_results:
            time.sleep( temp ) 
            count += temp 
            if count > 60: # segundos
                count = -1
                print "Desisto!"
                break
    
    if count == -1:
        print "Possivel bloqueio do Google"
        return 0
   
    
    print "Fim"

def googleSearch ( searchString ):
    g = pygoogle(searchString)
    g.pages = 2
    urls = g.get_urls()
    urls = urls[:10]
    for i in range(len(urls)):
        urls[i]=unicode(urlnorm.norm(urls[i]))

    return urls

def getSearchAndDownloadPaper(textToSearch, fileNameToSave):
	g = pygoogle(textToSearch + ' filetype:pdf')
	g.pages = 1
	try:
		pdfUrl = g.get_urls()[0]
		urllib.urlretrieve(pdfUrl, "../pdfdownloaded/" + fileNameToSave)
		time.sleep(random.randint(30,60))
	except IndexError:
		print fileNameToSave + " " + textToSearch
		time.sleep(180);

def crackedonpygoogle(passhash, plaintext):  #trying this approach
    from pygoogle import pygoogle
    googleresult = pygoogle(passhash)  #default is for moderate safe search.  Probably OK to let this be, since we won't find porn while googling a password hash.  Probably throwing caution (and Rule 34) to the wind here.
    googleresult.pages = 1
    resulturls = googleresult.get_urls()
    for i in range(0,len(resulturls)):
        resulturls[i] = str(resulturls[i])
    if crackedonweb(passhash, plaintext, resulturls):
        return True
    else:
        return False

	def searchtext(self, user_input):
		searchterms = user_input + ' site:stackoverflow.com'
		print "Searching:", searchterms
		g = pygoogle(searchterms)
		g.pages = 1
		urls = g.get_urls()
		#go through search results
		for url in urls[:int(len(urls)/4+0.5)]:
			req = urllib2.Request(url, headers = hdr)
			try:
				self.myParser.answers = 0
				page = urllib2.urlopen(req)
				html = page.read()
				#print html
				html_fixed = html.replace('>', '3cmr93iwm0c9ri3w0')
				html_fixed = html_fixed.replace('<','98jdsf98j3oisdf')
				html_fixed = html_fixed.replace('&','dksljf9w8ejfosidjf')

				#html_fixed = html_fixed.replace('...',' ')
				self.myParser.feed(html_fixed)
				self.snips = self.myParser.snips
				#print self.snips
				for x in self.snips:
					for y in x[0]:
						print url
						answer = sublime.ok_cancel_dialog(y.replace('98jdsf98j3oisdf','<').replace('3cmr93iwm0c9ri3w0','>').replace('dksljf9w8ejfosidjf','&'))
						if answer == 1:
							self.view.insert(self.editor,
								self.view.sel()[0].begin(),y.replace('98jdsf98j3oisdf','<').replace('3cmr93iwm0c9ri3w0','>').replace('dksljf9w8ejfosidjf','&'))
							if self.language in starter:
								self.view.insert(self.editor,
									self.view.sel()[0].begin(),"\n\n"+starter[self.language]+'\n'+x[1].replace('98jdsf98j3oisdf','<').replace('3cmr93iwm0c9ri3w0','>').replace('\t',' ').replace('\n','').replace(starter[self.language],' ').replace(ender[self.language],' ').replace('dksljf9w8ejfosidjf','&')+'\n'+\
									ender[self.language]+"\n\n")
							else:
								self.view.insert(self.editor,
									self.view.sel()[0].begin(),"/*"+x[1].replace('98jdsf98j3oisdf','<').replace('3cmr93iwm0c9ri3w0','>').replace('\t',' ').replace('\n','').replace('dksljf9w8ejfosidjf','&')+\
									'*/'+"\n\n")
							self.myParser.snips = []
							self.myParser.curr_snips = []
							self.myParser.curr_snip = ''
							self.myParser.curr_comment = ''
							self.snips = []
							break
					else: 
						continue
					break
				else:
						self.myParser.snips = []
						continue
				break
			except urllib2.HTTPError,e:
				print e.fp.read()

def fresh_google_check(link):
    '''
    Проверяет, индексировался ли уже ресурс гуглом раньше
    чем за 2 недели до сегодня.
    '''
    sleep(random.random())
    today = datetime.date.today()
    date_s = date_to_julian_day( today - datetime.timedelta(days=365 * 8) )
    date_e = date_to_julian_day( today - datetime.timedelta(days=7 * 2) )
    query = u'site:%s daterange:%s-%s' % (link, date_s, date_e,)
    g = pygoogle(query.encode('utf-8'))
    g.pages = 1
    return bool(g.get_result_count())

 def google(self, args, irc):
   '''(google [search term]) -- 
   Return the top Google result for the term searched.
   '''
   try:
       g = pygoogle(u' '.join(args))
       g.pages = 1
       for title, descr in g.search().iteritems():
           reply = u'{} | {}'.format(descr.strip(), title.strip())
           return reply
   except:
       log.err('[Error]: Google {}'.format(sys.exc_info()[0]))
       return '[Error]: Cannot contact Google API.'

def findUrl(key, numPages, save = False):
    g = pygoogle(key)
    g.pages = numPages
    links = g.get_urls()
    if save:
        try:
            f = open("links.txt","w")
            for link in links:
                f.write(link+"\n")
            f.close()
        except IOError:
            print "cannot open new file"
    else:
        return links

def google_query(query):
	
	g = pygoogle(query) 
	g.pages=1
	g.rsz = 4
	results = {}
	results = g.search()
	rl = results.keys()

	print rl
	

	s = rl[0]
	s.encode('utf-8')
	return s

 def search(self, group, filename, destination):
     movie_name = getTitle(group['library'])
     movienorm = unicodedata.normalize('NFKD', movie_name).encode('ascii','ignore')
     movie_year = group['library']['year']
     searchstring=movienorm+' '+ str(movie_year) +' bande annonce vf HD'
     time.sleep(3)
     log.info('Searching google for: %s', searchstring)
     g = pygoogle(str(searchstring))
     diclist = g.search()
     urllist = g.get_urls()
     cleanlist=[]
     for x in urllist:
         if 'youtube' in x or 'dailymotion' in x:
             cleanlist.append(x)
     if cleanlist:
         bocount=0
         for bo in cleanlist:
             if bocount==0:
                 tempdest=unicodedata.normalize('NFKD', os.path.join(rootDir,filename)).encode('ascii','ignore')+u'.%(ext)s'
                 dest=destination+u'.%(ext)s'
                 log.info('Trying to download : %s to %s ', (bo, tempdest))
                 p=subprocess.Popen([sys.executable, 'youtube_dl/__main__.py', '-o',tempdest,'--newline', bo],cwd=rootDir, shell=False, stdout=subprocess.PIPE,stderr=subprocess.PIPE)
                 while p.poll() is None:
                     l = p.stdout.readline() # This blocks until it receives a newline.
                     lmsg= l.replace('%',' percent')+' '+filename
                     log.info(lmsg)
                 # When the subprocess terminates there might be unconsumed output 
                 # that still needs to be processed.
                 (out, err) = p.communicate()
                 outmsg='Out for '+filename +' : '+out
                 errmsg='Err for '+filename +' : '+err
                 if out:
                     log.info(outmsg)
                 if err:
                     log.info(errmsg)
                     continue
                 else:
                     listetemp=glob.glob(os.path.join(rootDir,'*'))
                     for listfile in listetemp:
                         if unicodedata.normalize('NFKD', filename).encode('ascii','ignore') in listfile:
                             ext=listfile[-4:]
                             finaldest=destination+ext
                             shutil.move(listfile, finaldest)
                             bocount=1
                             log.info('Downloaded trailer for : %s', movienorm)
                             return True
     else:
         return False