class BrowserManager:
	def __init__(self):
		self._lock = False
	def bootup(self):
		self._display = Display(visible=0, size=(1024, 768))
		self._display.start()
		profile = {}
		if 'HTTP_PROXY' in os.environ:
			proxy_url = os.environ['HTTP_PROXY']
			proxy_server = proxy_url.split(':')[1][2:]
			proxy_port = proxy_url.split(':')[-1]
			profile['network.proxy.type'] = 1
			profile['network.proxy.http'] = proxy_server
			profile['network.proxy.http_port'] = proxy_port
			profile['network.proxy.https'] = proxy_server
			profile['network.proxy.https_port'] = proxy_port
		self.browser = Browser(profile_preferences=profile)
	def obtain(self,background):
		while self._lock:
			background.wait('Browser lock', 15)
		self._lock = True
		return self.browser
	def release(self,background):
		self._lock = False
	def shutdown(self):
		self.browser.quit()
		self._display.stop()

class Xvfb(object):
    def __init__(self, width=1366, height=768, visible=0):
        self.__virtual_display = None
        self.width = width
        self.height = height
        self.visible = visible

    def __init_display(self):
        if self.__virtual_display is None:
            self.__virtual_display = Display(visible=self.visible, size=(self.width, self.height))
            self.__virtual_display.start()

    def __enter__(self):
        self.__init_display()

    def __exit__(self, exc_type, exc_val, exc_tb):
        self._close_display()

    def _close_display(self):
        if self.__virtual_display:
            try:
                self.__virtual_display.close()
            except:
                pass
        self.__virtual_display = None

    @staticmethod
    def run(func, *args, **kwargs):
        runner = Xvfb()
        with runner:
            return func(*args, **kwargs)

class UITestCase(LiveServerTestCase):
    def use_xvfb(self):
        from pyvirtualdisplay import Display
        self.display = Display('xvfb',
                               visible=1,
                               size=(1280, 1024))
        self.display.start()
        self.driver = WebDriver()

    def setUp(self):
        try:
            self.driver = WebDriver()
            ui_is_not_available = False
        except WebDriverException:
            ui_is_not_available = True

        if ui_is_not_available:
            self.use_xvfb()

        self.driver.implicitly_wait(10)
        super(UITestCase, self).setUp()

    def tearDown(self):
        self.driver.quit()
        if hasattr(self, 'display'):
            self.display.stop()

        super(UITestCase, self).tearDown()

class BCCVLTestCase(unittest.TestCase):
    def setUp(self):
        # acquire URL, username and password from environment variables, or use default values for dev env.
        self.username = os.getenv("BCCVL_TEST_USERNAME", "admin")
        self.password = os.getenv("BCCVL_TEST_PASSWORD", "admin")
        self.url = os.getenv("BCCVL_TEST_URL", "https://192.168.100.200/")

        # The amount of time selenium will potentially wait in searching for elements. This is blocking.
        implicit_wait = int(os.getenv("BCCVL_TEST_IMPLICIT_WAIT", "15"))

        # Run tests in a virtual display (xvfb)
        virtual_display = os.getenv("BCCVL_TEST_VIRTUAL_DISPLAY", "false") == "true"

        # Setup the virtual display
        if virtual_display:
            self.display = Display(visible=0, size=(1920, 1080))
            self.display.start()
        else:
            self.display = None

        # Setup the Firefox Profile and webdriver
        self.driver = webdriver.Firefox()
        self.driver.implicitly_wait(implicit_wait)

        # Maximize the window
        # self.driver.maximize_window()
        self.driver.set_window_size(1200, 800)

        # Go to the bccvl homepage
        self.driver.get(self.url)

    def tearDown(self):
        if self.display:
            self.display.stop()
        self.driver.quit()

def getupc(data, sleeptime):
    display = Display(visible=0, size=(800, 600))
    display.start()
    a = webdriver.Firefox()
    a.get('https://www.google.com/ncr')
    time.sleep(sleeptime)
    search = WebDriverWait(a, 5).until(EC.element_to_be_clickable((By.XPATH, "//input[@type='text']")))
    for i in data:
        ActionChains(a).move_to_element(search).click(search).send_keys(i['name'] + ' upc', Keys.ENTER).perform()
        time.sleep(sleeptime)
        contents = WebDriverWait(a, 5).until(EC.presence_of_all_elements_located((By.XPATH, "//div[@class='g']")))
        try:
            upc = next(
                    (re.split(r'/', href.find_element_by_tag_name('a').get_attribute('href'))[-1] for
                     href in contents if
                     href.find_element_by_tag_name('a').get_attribute('href').startswith(
                             'http://www.upcitemdb.com/upc')))
            i['upc'] = upc
        except StopIteration:
            pass

        search = WebDriverWait(a, 5).until(EC.element_to_be_clickable((By.XPATH, "//input[@type='text']")))
        search.clear()
    a.close()
    display.stop()
    return data

class TestContext(object):
    
    def open_browser(self):

#         if test_config.SELENIUM_USE_REMOTE:
#             dc = getattr(DesiredCapabilities, self.driver.upper())
#             dc['name'] = test_config.SELENIUM_TEST_NAME
#             cmd_exec = test_config.SELENIUM_REMOTE_CMD_EXEC
#             self.browser = webdriver.Remote(desired_capabilities=dc, command_executor=cmd_exec)

        if test_config.SELENIUM_USE_VIRTUALDISPLAY:
            self.virtualdisplay = Display(backend=test_config.SELENIUM_VIRTUALDISPLAY_BACKEND, size=(600, 800)).start()

        self.browser = webdriver.Firefox(firefox_binary=FirefoxBinary(test_config.SELENIUM_FIREFOX_PATH))
        self.browser.implicitly_wait(test_config.SELENIUM_PAGE_WAIT)
        
    def close(self):
        self.browser.quit()
        if hasattr(self, 'virtualdisplay'):
            self.virtualdisplay.stop()
            
    def get(self, url):
        self.browser.get(url)
        self.url = url
    
    def follow_link(self, link):
        link.click()
        self.url = self.browser.current_url
        
    def wait_for(self, by, thing):
        wait = WebDriverWait(self.browser, test_config.SELENIUM_PAGE_WAIT)
        wait.until(EC.presence_of_element_located((by, thing)))

def get_screenshot(site_id, update_id):
    """
    Create a screenshot and save it to the database
    """
    # Get the objects we're working with
    site = Site.objects.get(id=site_id)
    update = Update.objects.get(id=update_id)
    
    # Fire up a headless display to work in
    display = Display(visible=0, size=(1680, 1050))
    display.start()
    
    # Fire up a Selenium browsers
    browser = webdriver.Firefox()
    
    # Set a timeout for the pageload
    seconds = 15
    browser.command_executor._commands['setPageLoadTimeout'] = (
        'POST', '/session/$sessionId/timeouts'
    )
    browser.execute("setPageLoadTimeout", {
        'ms': 1000*seconds,
        'type':'page load'
    })
    
    # Snap a screenshot of the target site
    logger.debug("Opening %s" % site.url)
    timestamp = timezone.now()
    try:
        browser.get(site.url + "?x=" + get_random_string())
        logger.debug("Response received for %s" % site.url)
    except TimeoutException, e:
        logger.error("Request for %s timed out" % site.url)
        pass

	def get_image(self):
		## Uses supplied scrape site to find new pictures
		url = self.scrape_site
		# virtual display for headless runs
		display = Display(visible=0, size=(800, 600))
		display.start()

		with closing(Firefox()) as browser:
			browser.get(url)
			time.sleep(5) # TODO: fix with something less static, but still
			# multipurpose considering scrape_site as a db var
			imgs = browser.find_elements_by_tag_name('img')
			# TODO: fix this temporary workaround that prevents ad server data
			# from reaching the image checks
			no_ad_imgs = [i for i in imgs if 'adsrvr' not in \
				i.get_attribute('src')]
			for img in no_ad_imgs:
				src = img.get_attribute('src')
				alt = img.get_attribute('alt')
				image_id = re.findall("/photo/(.+?)/", src)[0]
				if(self._check_id(image_id) and self._check_ratios(src)):
					self.img_id = image_id
					self.description = alt
					self._save_hd_image()
					break
		display.stop()
		if (self.img_id):
			return
		raise Exception('Failed to find a suitable image: all out or bugged')

def run_selenium(landmark):
	display = Display(visible=0, size=(800, 600))
	display.start()
        logTo(TEST_LOG,'Selenium : Starting Selenium  for '+landmark,'INFO','a')
	interFace=open(HOME_DIR+'/Desktop/one-time-test-suite/iface.txt','r')
	tmp=interFace.readlines()
	iface=tmp[0].split('\n')[0]
	tmpstmp=datetime.now().strftime("%s")
	profile = webdriver.FirefoxProfile()
	profile.update_preferences()
	browser = webdriver.Firefox(firefox_profile=profile) # assign profile to browser
	browser.delete_all_cookies()
	logTo(TEST_LOG,' Selenium : Starting tcpdump .. ','INFO','a')
	tcpcmd='tcpdump -i '+iface+' -w '+EXP_DIR+'/'+'tcpdump_'+landmark.split('.')[0]+'_'+tmpstmp
	args=shlex.split(tcpcmd)
	ptcpdmp=sub.Popen((args))
	time.sleep(10)
	logTo(TEST_LOG,' Selenium : Starting get '+landmark,'INFO','a')
	browser.get('http://www.'+landmark)
	time.sleep(5)
	perfData=browser.execute_script('return window.performance.timing')
	fname=EXP_DIR+'/'+'perfdata_'+landmark.split('/')[0]
	fname=fname.replace('.','-')
	pickle.dump(perfData,open(fname,'wb'))
        logTo(TEST_LOG,'Selenium : Writing done to '+EXP_DIR+'/perfdata_'+landmark,'INFO','a')
	browser.quit()
	display.stop()
	ptcpdmp.terminate()
        logTo(TEST_LOG,'Finished Selenium for '+landmark,'INFO','a')

def get_all_items():
    #list to store alll scraped data
    all_items = list()

    #Display - read about pyvirtualdisplay
    display = Display(visible=0, size=(1024, 768))
    display.start()
    #webdriver - read about selenium.webdriver
    driver = webdriver.Firefox()
    
    #this is a starting page we are scraping
    driver.get("http://www.federalreserve.gov/apps/reportforms/default.aspx")
    #Every element on the HTML page can be located using CSS selectors.
    #Opening the starting page in Chrome, right click on the drop-down menu, click "Inspect" we see a tag on the right highlighted, we copy it's id - MainContent_ddl_ReportForms
    #Knowing the id of dropdown menu, we can locate it with Selenium like this
    main_menu = WebDriverWait(driver,10).until(EC.presence_of_element_located((By.CSS_SELECTOR,"#MainContent_ddl_ReportForms")))
    #Drop down menu is an HTML table of options which can be verified in Chrome browser (Developer Tools, that pop up when you right click and press "Inspect" on an element)
    #Following returns all of the options - rows in that table
    form_options = main_menu.find_elements_by_tag_name("option")
    #We count them
    option_count = len(form_options)
    #Next, we loop over all of them - essentially like we scrolling down the drop down menu and clicking on each every form 
    for form_i in xrange(1,option_count):
        #Get web element corresponding to a form
        form = form_options[form_i]
        #Click as a mouse click-action in browser 
        form.click()
        #Get text, because we need to store the form number
        form_id = form.text
        #Locate a web element corresponding to the submit button. By CSS selector which we found by inspection in Chrome browser (same logic as above)
        submit_button = WebDriverWait(driver,3).until(EC.presence_of_element_located((By.CSS_SELECTOR,"#MainContent_btn_GetForm")))
        #Click as a mouse click-action in browser 
        submit_button.click()      
        #Prepare data structures to store all the info we want to scrape
        a = dict.fromkeys(['Description','OMB','Background','RespondentPanel','Frequency','PublicRelease'])
        #We are on a web page after submit-click, following will search for all items of interest. Or for corresponding
        #web-elements 
        for el in a.keys():
            try:
                item = driver.find_element_by_css_selector("#MainContent_lbl_"+el+"_data") 
                #Once found it will store them in our dictionary, if not it will proceed to "except" section and do nothing
                a[el] = item.text 
            except: 
                #case when there is no such field
                pass
        #we need form number as well
        a['FormNumber'] = form_id
        #keeping them all in one list, which will have a dictionary per Form Number - and later, a row in your excel file per Form number
        all_items.append(a)
    
        #Ok, that part bothers me a little: it looks like I have to refresh "form_options" each time... 
        #Otherwise I get following exception: selenium.common.exceptions.StaleElementReferenceException: Message: Element not found in the cache - perhaps the page has changed since it was looked up
        driver.get("http://www.federalreserve.gov/apps/reportforms/default.aspx")
        main_menu = WebDriverWait(driver,10).until(EC.presence_of_element_located((By.CSS_SELECTOR,"#MainContent_ddl_ReportForms")))
        form_options = main_menu.find_elements_by_tag_name("option")

    driver.close()
    display.stop()

    return all_items

class Spider(scrapy.Spider):
    name = "mayors"
    allowed_domains = ["www.cec.gov.tw"]
    start_urls = ["https://www.cec.gov.tw/pc/zh_TW/IDX/indexC.html",]
    download_delay = 1

    def __init__(self, ad=None, *args, **kwargs):
        super(Spider, self).__init__(*args, **kwargs)
        self.display = Display(visible=0, size=(800, 600))
        self.display.start()
        self.driver = webdriver.Chrome("/var/chromedriver/chromedriver")

    def spider_closed(self, spider):
        self.display.close()

    def parse(self, response):
        self.driver.get(response.url)
        nodes = scrapy.Selector(text=self.driver.page_source).xpath('//a[@target="_top"]')
        for node in nodes:
            county = node.xpath('text()').extract_first()
            print county
            yield response.follow(node, callback=self.parse_list, meta={'meta': county})

    def parse_list(self, response):
        for tr in response.css(u'table.tableT tr.trT'):
            d = {}
            d['type'] = 'mayors'
            d['county'] = response.meta['meta']
            d['constituency'] = 0
            d['elected'] = tr.xpath('td[1]/text()').extract_first().strip()
            d['number'] = int(tr.xpath('td[2]/text()').extract_first())
            d['votes'] = int(re.sub('\D', '', tr.xpath('td[5]/text()').extract_first()))
            d['votes_percentage'] = tr.xpath('td[6]/text()').extract_first()
            yield d

def process_screenshots(app, env):
    if not hasattr(env, 'screenshot_all_screenshots'):
        return

    if not app.config['screenshots_create']:
        print("Not doing screenshots on maggies farm no more")
        return
        
    if 'SPHINX_SS_USE_PVD' in os.environ.keys() and os.environ['SPHINX_SS_USE_PVD'] == "true":
        from pyvirtualdisplay import Display
        # Start a virtual headless display
        display = Display(visible=0, size=(1024, 768))
        display.start()
    else:
        display = None
    
    # Don't bother building screenshots if we're just collecting messages.
    # Just checks if we invoked the build command with "gettext" in there somewhere
    if "gettext" in sys.argv:
        return
    all_args = map(lambda x: x['from_str_arg'], env.screenshot_all_screenshots)
    # If building in a different language, start the server in a different language
    command = SCREENSHOT_COMMAND + SCREENSHOT_COMMAND_OPTS + \
              [re.sub(r"\s", r"", "--from-str={0}".format(json.dumps(all_args)))]
    language = env.config.language
    if language:
        command += ["--lang={0}".format(language)]
    subprocess = Popen(command)
    subprocess.wait()
    try:
        if subprocess.returncode:
            raise Exception("Screenshot process had nonzero return code: {0}".format(subprocess.returncode))
    finally:
        if display:
            display.stop()

def openurl(companyname=first_arg):
    display = Display(visible=0, size=(1024, 768))
    display.start()
    browser = webdriver.Firefox()
    time.sleep(randint(8,10))
    try:
        browser.get('http://www.google.com')
        time.sleep(5)
        search = browser.find_element_by_name('q')
        input_text = companyname + str(" crunchbase")
        search.send_keys(input_text)
        time.sleep(randint(10,15))
        search.send_keys(Keys.RETURN)
        time.sleep(randint(10,15))
        gn = browser.find_element_by_tag_name('h3').text
        gnc = str(gn).split(' | ')[0].replace(" ","")
        output_file = '0515' + gnc + '.html'
        browser.find_element_by_link_text(gn).click()
        time.sleep(randint(55,60))
        company_html = browser.page_source
        time.sleep(randint(5,10))
        with open("smallname.txt", 'a') as myfile:
            json.dump(output_file,myfile)
        with open(output_file, 'a+') as myfile:
            myfile.write(company_html)
    except:
        company_html = 'none'        
        with open("missedname.txt", "a") as myfile:
            json.dump(companyname,myfile)            
    time.sleep(1)
    browser.close()
    time.sleep(1)
    display.stop()
    return company_html

class SeleniumRunner(object):
    def __call__(self, f):
        @functools.wraps(f)
        def decorated(_self, *args, **kwargs):
            with self as driver:
                return f(_self, driver, *args, **kwargs)
        return decorated

    def __enter__(self):
        self.display = Display(visible=0, size=(800, 600))
        self.display.start()
        self.driver = webdriver.Chrome()
        return self.driver

    def __exit__(self, *args, **kwargs):
        try:
            self.driver.quit()
        except (AttributeError,) as e:
            # Someone has messed with our browser
            pass
        try:
            self.display.stop()
        except (AttributeError,) as e:
            # Someone has messed with our display
            pass

    def process_install_form (self):
        if (self.args.xvfb):
	    print "Omeka is being installed in: " + self.folder_name
            display = Display(visible=0, size=(800, 600))
            display.start()
        driver = webdriver.Firefox()
        driver.get("http://localhost/omeka/" + self.folder_name + "/install")
        inputElement = driver.find_element_by_name("username")
        inputElement.send_keys(self.omeka_user)
        inputElement = driver.find_element_by_name("password")
        inputElement.send_keys(self.omeka_passwd)
        inputElement = driver.find_element_by_name("password_confirm")
        inputElement.send_keys(self.omeka_passwd)
        inputElement = driver.find_element_by_name("super_email")
        inputElement.send_keys("[email protected]")
        inputElement = driver.find_element_by_name("administrator_email")
        inputElement.send_keys("[email protected]")
        inputElement = driver.find_element_by_name("site_title")
        inputElement.send_keys(self.omeka_title)
        inputElement.submit()
        try:
            WebDriverWait(driver, 10).until(
                lambda driver : driver.find_element_by_partial_link_text("Tableau"))
        finally:
            driver.quit()

class AdminTestCase(LiveServerTestCase):

    def setUp(self):
        self.display = Display(visible=0, size=(800, 600))
        self.display.start()

        self.selenium = webdriver.Firefox()

        super(AdminTestCase, self).setUp()

    def tearDown(self):
        self.selenium.quit()
        self.display.stop()
        super(AdminTestCase, self).tearDown()

    def test_payment(self):
        """
        payment will be successful.
        """

        self.selenium.get("%s/pay" % self.live_server_url)
        self.selenium.implicitly_wait(20)
        self.selenium.maximize_window()

        self.selenium.find_element_by_name("amount").send_keys("100000")

        pay_button = self.selenium \
            .find_element_by_xpath('//input[@value="pay"]')
        pay_button.click()

        return_to_site_button = self.selenium.find_element_by_id("btn3")

        return_to_site_button.click()

        self.assertIn("successful", self.selenium.page_source)

def main(param):

    if len(param) != 2:
        sys.exit(-9)
    if len(param[1]) <= 0:
        sys.exit(-8)
    paths = param[0]
    shotsdir = paths.get('path', 'output.shotsdir').lstrip('"').rstrip('"')
    targets = param[1]

    display = Display(visible=0, size=(800, 600))
    display.start()

    binary = FirefoxBinary('/opt/firefox/firefox')
    browser = webdriver.Firefox(firefox_binary=binary)

    tgt_len = len(targets)
    for i, tgt in enumerate(targets):
        browser.get(tgt[0])
        browser.save_screenshot(shotsdir+'/'+tgt[1]+'.png')
        print '( %3d / %3d ) Took %s.png' % (i+1, tgt_len, tgt[1])

    browser.quit()

    display.stop()

def work():

    logging.info("start weeklys screenshot work")
    print ("start ... ")

    if not DISPLAY:
        print ("hide display ... ")
        display = Display(visible=0, size=(1366, 768))
        display.start()

    config = getConfigObj()
    if config == None:
        return False
    userName = config.get("USER", "UserName")
    userPWD = config.get("USER", "userPWD")

    ret = getTowerWeeklyScreenshot(userName, userPWD, DEFAULT_SAVE_PATH)

    if not ret:
        print ('Error, abort. Please check the log file "%s"' % LOG_FILE)
        return False

    logging.info("finish all work, exit.")

    if not DISPLAY:
        display.stop()

    return True

def rzhd():
    directions=[create_url(),]

    while raw_input('Want to add more directions? y/n ')=='y':
        directions.append(create_url())
        print "------------------"
    # n=raw_input('Check tickets every ...(seconds)? ')
    n = 60

    place=choose_place()
    i = 0
    display = Display(visible=0, size=(5, 5))
    display.start() # Запускаем вирутальный дисплей
    while len(directions)!=0:
        i+=1
        print
        print "----------------->Searching for PLATSKART<-----------------"

        print "try #",i
        print time.asctime()
        print

        for url in directions:
            if find_train(url, place)==True:
                send_email('[email protected]', url)
                if raw_input('Did you buy ticket? y/n ')=='y':
                    directions.remove(url)
                    if len(directions) == 0:
                        print "Successfully bought all tickets!"
                        return True                
            print str(n)+" seconds until next try..."
            time.sleep(float(n)) # Дадим браузеру корректно завершиться
    display.stop() # Закрываем виртуальный дисплей

def get_news():
    if check_wlan():
        from pyvirtualdisplay import Display
        import re

        display = Display(visible=0, size=(800, 600))
        display.start()

        driver = webdriver.Firefox()
        url = "http://www.deutschlandfunk.de/"
        driver.get(url)
        source = driver.find_element_by_xpath('//*[@id="wrapper"]/div/section[2]/div[1]').get_attribute('innerHTML')

        n_articles = source.count('<article')
        print(str(n_articles) + " articles found.")

        lst = re.findall('<h3>(.+)</h3>', source)
        result = lst

        driver.close()

        display.stop()
        return result
    else:
        print("Error: Not connected to the internet")