def getGooogleResults(query, exclude):
    try:
        print "Searching for {0} but excluding these {1}".format(query, exclude)
        page = 1
        gs = GoogleSearch(query)
        gs.results_per_page = 100
        results = gs.get_results()

def getUrls2( page_num ):
   gs = GoogleSearch('shareholder letter')
   gs.results_per_page = 50
   gs.page = page_num
   results = gs.get_results()
   for item in results:
      print item.url.encode("utf8")

def google(termtosearch, action):
	
	#action = spam or phis
	try:
		gs = GoogleSearch(termtosearch)
		gs.results_per_page = 100
		results = []

		while True:
			tmp = gs.get_results()
			if not tmp:
				break
			results.extend(tmp)

			#TODO switch in this code block
			if action == 'mal':
				for res in results:
					checkAgainstGoogle(res.url.encode('utf8'))
			else:

					if action == 'spam':
						for res in results:
							print '\033[1;34mLooking for SPAM in ......%s\033[1;m' % (res.url.encode('utf8'))
							spam_detect(res.url.encode('utf8'))
					elif action == 'phis':
						for res in results:
							print '\033[1;34mLooking for PHISHING in ......%s\033[1;m' % (res.url.encode('utf8'))
							phishing_detect(res.url.encode('utf8'))
							
					else:
							for res in results:
								print res.url.encode('utf8')

	except SearchError, e:
		print "Search failed: %s" % e

def google(data):  #In this fuction we will do the phrasing of the subject line ourselfs.
   print "Called google"
   tore = ''
   search_string = ''

   if data[1] == 'search':
      for i in range(2,len(data)):
         search_string = search_string + data[i] + ' '
      try:
         
         tore = "Here are the first 25 results from google when \'"+search_string+"\' is queried\n\n"
         gs = GoogleSearch(search_string)
         gs.results_per_page = 25
         results = gs.get_results()
         
         for res in results:
            #print res.title.encode('utf8')
            tore = tore+res.title.encode('utf8')+"\n"
            #print res.desc.encode('utf8')
            tore = tore+res.desc.encode('utf8')+"\n"
            #print res.url.encode('utf8')
            tore = tore+res.url.encode('utf8')+"\n\n--------------------------------------\n"
            print
      except:
         print "Search failed: %s" % e
         tore = "Search failed: %s" % e

   return tore

	def google(self,text):
		
		try:
			print "Trying to search for "+text
		
			g1 = GoogleSearch(text)
		
		
			g1.results_per_page = 25
		
			results = g1.get_results()
			
			if len(results)==0:
				print "No search result!!"
			else:
				print "Results FOund!!"
				print type(results)
				print len(results)
				for res in results[:2]:
					time.sleep(1)
					url = res.url.encode("utf8")
					response = self.search(url)
					if response == "Kgpian":
						self.close()
						break
		except SearchError, e:
			print "Failed Once"

    def getGoogleResults(self,pluginname,latest,cve):
        try:
		gs = GoogleSearch("inurl:'wp-content/plugins/" + pluginname + "'", random_agent=True)
        	gs.results_per_page = 100
                        

		numberOfprocessed = 0	
                self.all_run = []
                
       		for i in range(int(limitForSearch)):
        		results = gs.get_results()
            		if not results:
            		        break

                        # Semaphore for write in order to screen
                        self.checkSimultaneus = threading.Semaphore(int(NumThreats))            
                        # Semaphore for write to file
                        self.writeFile = threading.Semaphore(int(NumThreats)-1)

			for res in results:
			        self.checkSimultaneus.acquire()
				host_name = urlparse(res.url.encode()).hostname
                                # Create thread
                                t = threading.Thread(target=self.__getGoogleResults, args=(host_name,latest,pluginname,cve))
                                self.all_run.append(t)
                                # run thread
                                self.all_run[len(self.all_run)-1].start()
				


				
	except SearchError, e:
  		print "Search failed: %s" % e

	def collect(self):
		gs = GoogleSearch("site:"+self.target)
		while True:
			results = gs.get_results()
			for res in results:
				self.urls.append(res.url)
			if len(results)<10:
				break

    def go(self, query, pages):
        search = GoogleSearch(query)
        search.results_per_page = 10

        for i in range(pages):
            search.page = i
            results = search.get_results()
            for page in results:
                self.scrape(page)

def update_from_web( model, film, year ):
  search = "kinopoisk.ru " + year + " " + film
  print "Search: %s" % search  
  browser=Browser(debug=True)
  gs = GoogleSearch(search)
  gs.results_per_page = 1
  results = gs.get_results()
  try:
    for res in results:
      pageurl = res.url.encode('utf8')
      page = browser.get_page( pageurl )
      soup = BeautifulStoneSoup( page[ page.find("<html"):], convertEntities=BeautifulStoneSoup.HTML_ENTITIES, fromEncoding="windows-1251" )
      print "URL: %s" % pageurl
      rating = soup.find('a',attrs={'class':'continue'})
      if rating:
	  r = strip(rating).split(' ')
          try:
              model.rating = float( r[1] )
              print "Rating: %s" % r[1] 
          except Exception, ex:
              model.rating = 0.0
              print "Can't parse rating"
 
      title = soup.find('h1','moviename-big')
      if title:
          print "Title: %s" % strip(title)
          model.title = strip(title)

      info = soup.find('span','_reachbanner_')
      if info:
          print "Info: %s" % strip(info)
          model.description = strip( info )

      img = soup.find('img', attrs={"width" : "120"})
      if img:
          print "Image: %s" % img['src']
	  model.image = "http://www.kinopoisk.ru%s" % img['src']
   
#getTrailer("t26538","397494/kinopoisk.ru-District-9-36971.mp4","397494/1_36971.jpg","480","270","tr","","");

      import re
      m = re.search("getTrailer\((.*)\)",str(soup))
      if not m:
          pass
      else:
          parts = m.group(1).split('"')
          url = "http://tr.kinopoisk.ru/%s" % parts[3]
	  model.trailer = url
	  image = "http://tr.kinopoisk.ru/%s" % parts[5]
          model.trailer_image = image
          print "Trailer: %s" % url
          print "TrailerImage: %s" % image
     
      break
  
  except Exception,e:
      print "WARNING: %s" % e

 def startSearch(self, domain="", target_keywords=[]):
     gs = GoogleSearch(target_keyword)
     gs.results_per_page = 100
     results = gs.get_results()
     for idx, res in enumerate(results):
         parsed = urlparse(res.url)
         domain = self.__makeUrl(parsed.netloc)
         if domain == target_domain:
             print "Ranking position %d for keyword %s on domain %s" % (idx + 1, target_keyword, target_domain)

def searchPage(textToSearch, page):
    items = []
    gs = GoogleSearch(textToSearch)
    gs.results_per_page = 100
    gs.page = page
    results = gs.get_results()
    for res in results:
        url = res.url.encode('utf8')
        items.append(url);
    return items

def get_number_of_results(term, ajax=False, verbose=True):
    if not ajax:
        gs = GoogleSearch(term)
        page = str(gs._get_results_page())
        match = reg.search(page)
        if match:
            if verbose: print(term, match.groups()[0])
            return int(match.groups()[0].replace(',',''))
        else:
            raw_input((term, page))
    return int(search(term)['responseData']['cursor']['estimatedResultCount'])

def main():
	#the hardcoded search query:
	gs = GoogleSearch("computer")
	gs.result_per_page=10
	results = gs.get_results()

	for r in results:
		Crow(r.url).select("a").to(SqlitePipeline()).async_start(50)

	Crow.run()
	f.close()

def get_results(query):
    gs = GoogleSearch(query);
    gs.results_per_page = 9001;
    results = gs.get_results();
    ret = [];
    for idx, res in enumerate(results):
        domain = mk_nice_domain(res.url);
        domain = domain.replace("pastebin.com/", "pastebin.com/raw.php?i=");
        print 'Found codes at %s' % domain;
        ret.append(domain);
    return ret;

def search_google(term, domain):
    try:
        log.debug('Performing Google search for "{}"'.format(term))
        gs = GoogleSearch(term, tld=domain)
        gs.results_per_page = 10
        results = gs.get_results()
        log.debug('Got {} results'.format(len(results)))
        return [Url(res.url) for res in results[:10]]
    except SearchError as exc:
        log.exception(exc)
        return None

def search( md5hash ):
	urls = []

	gs = GoogleSearch( md5hash )
	gs.results_per_page = 100
	results = gs.get_results()

	for res in results:
		urls.append( res.url.encode('utf8')  )

	return urls

 def scrape(self, keyword, pages=2):
     try:
         gs = GoogleSearch(keyword)
         gs.results_per_page = 10
         gs.page = 0
         results = gs.get_results()
         for res in results:
             url = res.url.encode('utf8')
             Title = res.title
             self.urls.append((url, Title))
     except SearchError, e:
       print "Search failed: %s" % e

def get_hits(term):
	#data = fetch_data("http://api.thriftdb.com/api.hnsearch.com/items/_search?q=" + term)
	#if data[0] is not None:
	#	if loads(data[0])['hits'] > 0:  #loads() dumps a json file which is what the hnsearch api returns
	#		return loads(data[0])['hits']
#		else:
#			return 0.000001
#	else:
#		return data[1]
    gs = GoogleSearch(key)
    gs.results_per_page = 100
    results = gs.get_results()
    return results

def search_by_filename(args):
	args_e=args.encode('utf8')
	try:
		gs = GoogleSearch('"' + args_e + '"')
		gs.results_per_page = 50
		results = gs.get_results()
		for res in results:
			if re_math_sites(allow_sites,res.url.encode('utf8')):
				if re_math_sites(args_e,res.desc.encode('utf8')):
					return clean_result(res.title.encode('utf8'))

	except SearchError, e:
		print "Search failed: %s" % e

def searchInSeretil():
        search_entered =''
        keyboard = xbmc.Keyboard(search_entered, 'הכנס מילות חיפוש כאן')
        keyboard.doModal()
        if keyboard.isConfirmed():
                    search_entered = keyboard.getText()

        if search_entered !='' :
                try:
                  gs = GoogleSearch("site:seretil.me "+ search_entered) 
                  gs.results_per_page = 100
                  results = gs.get_results()
                  for res in results:
                    title=res.title.encode('utf8')
                    url= res.url.encode('utf8')
                    title=title.replace('SERETIL.ME','')
                    title=title.replace('לצפייה ישירה','')
                    title=title.replace('וסדרות','')
                    title=title.replace('תרגום מובנה','')
                    title=title.replace('|','')
                    title=title.replace('.','')
                    title=title.replace('סרטים','')
                    title=title.replace('עם','')
                    title=title.replace('לצפיה','')
                    
                    
                    
                    
                    if 'עונה' in title   :
                                        if not 'page' in url  and not 'tag' in url  and not '?s' in url and not 'search' in url :
                                                addDir(title,url,211,'')
                    else:        
                                    if not 'page' in url  and not 'tag' in url  and not '?s' in url and not 'search' in url:
                                        image=''
                                        req = urllib2.Request(url)
                                        req.add_header('User-Agent', ' Mozilla/5.0 (Windows; U; Windows NT 5.1; en-GB; rv:1.9.0.3) Gecko/2008092417 Firefox/3.0.3')
                                        response = urllib2.urlopen(req)
                                        link3=response.read()
                                        response.close()  
                                        
                                        block= re.compile('<div class="post-wrap post-wrap-single">(.*?)linkwithin_hook',re.M+re.I+re.S).findall(link3)
                                        image=''
                                        images= re.compile('src="http(.*?).?jpg').findall(block[0])
                                        if images:
                                                image='http'+images[0]+'.jpg'
                                        addDir(title,url,5,image)
                
                                    
                except SearchError, e:
                  print "Search failed: %s" % e
                xbmcplugin.setContent(int(sys.argv[1]), 'tvshows')