def main():
    global startTrail, prevTrail,driver,conn,models
    try:
        startTrail = [int(re.search(r'\d+', _).group(0)) for _ in sys.argv[1:]]
        ulog('startTrail=%s'%startTrail)
        conn=sqlite3.connect('zyxel.sqlite3')
        sql("CREATE TABLE IF NOT EXISTS TFiles("
            "id INTEGER NOT NULL,"
            "model TEXT," # NSA320
            "prod_name TEXT," # 2-Bay Power Media Server
            "fw_ver TEXT," # 4.70(AFO.0)C0
            "rel_date DATE," # '07-18-2014' or 'Jul 18, 2014'
            "file_size INTEGER," # 
            "page_url TEXT," 
            # http://www.zyxel.com/us/en/support/DownloadLandingSR.shtml?c=us&l=en&kbid=MD09138&md=NSA320#searchZyxelTab4
            # http://www.zyxel.com/support/DownloadLandingSR.shtml?c=gb&l=en&kbid=MD09138&md=NSA320
            "file_url TEXT," # data-filelink="ftp://ftp2.zyxel.com/NSA320/firmware/NSA320_4.70(AFO.1)C0.zip"
            "tree_trail TEXT," # [26, 2, 1, 0, 0]
            "file_sha1 TEXT," # 
            "PRIMARY KEY (id)"
            "UNIQUE(model,fw_ver)"
            ")")
        driver=harvest_utils.getFirefox()
        # driver.implicitly_wait(2.0)
        harvest_utils.driver=driver
        prevTrail=[]
        getAllModels()
        modelWalker()
        driver.quit()
        conn.close()
    except Exception as ex:
        ipdb.set_trace()
        traceback.print_exc()
        driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_excep.png')

def main():
    global startTrail, prevTrail,driver,conn
    try:
        startTrail = [int(re.search(r'\d+', _).group(0)) for _ in sys.argv[1:]]
        ulog('startTrail=%s'%startTrail)
        conn=sqlite3.connect('asus.sqlite3')
        sql("CREATE TABLE IF NOT EXISTS TFiles("
            "id INTEGER NOT NULL,"
            "model TEXT," # RT-N10D1
            "fw_ver TEXT," # 2.1.1.1.92
            "rel_date DATE," # 2015/01/19
            "file_size INTEGER," # 3.62 MBytes
            "fw_desc TEXT," # '- Fixed infosvr vulnerability.'
            "page_url TEXT," # http://www.asus.com/support/Download/11/2/0/93/RvfuXsVTKYqBfjU7/8/
            "file_url TEXT," # http://dlcdnet.asus.com/pub/ASUS/wireless/RT-N10_D1/FW_RT_N10D1_211192.zip
            "on_click TEXT,"
            "tree_trail TEXT," # [26, 2, 1, 0, 0]
            "file_sha1 TEXT," # 5d3bc16eec2f6c34a5e46790b513093c28d8924a
            "PRIMARY KEY (id)"
            "UNIQUE(model,fw_ver)"
            ")")
        driver=harvest_utils.getFirefox()
        # driver.implicitly_wait(2.0)
        harvest_utils.driver=driver
        prevTrail=[]
        modelEnumer()
        driver.quit()
        conn.close()
    except Exception as ex:
        ipdb.set_trace()
        traceback.print_exc()
        driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_excep.png')

def storeFile(modelName, fileItem):
    global driver, prevTrail
    try:
        try:
            fileUrl = fileItem.get_attribute('data-durl')
            if fileUrl is None:
                fileUrl = fileItem.get_attribute('href')
        except Exception as ex:
            fileUrl = fileItem.get_attribute('href')
        _, ext = path.splitext(fileUrl)
        if ext in ['.html', '.htm']:
            return
        fileName = fileItem.text.strip()
        vendor='Netgear'
        try:
            fwVer=re.search(r'(?<=Version\ )\d+(\.\d+)+',fileName, flags=re.I)\
                .group(0)
        except Exception as ex:
            fwVer=None
        pageUrl = driver.current_url
        rev=""
        trailStr=str(prevTrail)
        sql("INSERT OR REPLACE INTO TFiles (vendor, model,revision,"
            "fw_ver, file_name, "
            "page_url, file_url, tree_trail) VALUES"
            "(:vendor, :modelName, :rev,"
            ":fwVer,:fileName,"
            ":pageUrl,:fileUrl,:trailStr)",locals())
        ulog('UPSERT "%(modelName)s", '
            ' "%(fileName)s", %(fileUrl)s'%locals())
    except Exception as ex:
        ipdb.set_trace(); traceback.print_exc()
        driver.save_screenshot('netgear_exc.png')

def main():
    global startTrail, prevTrail, driver, conn
    try:
        startTrail = [int(re.search(r"\d+", _).group(0)) for _ in sys.argv[1:]]
        ulog("startTrail=%s" % startTrail)
        conn = sqlite3.connect("asus.sqlite3")
        sql(
            "CREATE TABLE IF NOT EXISTS TFiles("
            "id INTEGER NOT NULL,"
            "product TEXT,"  # Wireless, Networking
            "series TEXT,"  #  "AP/Router"
            "model TEXT,"  # F9K1104
            "rel_date DATE,"  # Post Date: 06/20/2012
            "fw_ver TEXT,"  # Download version: 1.00.23
            "file_size INTEGER,"  # Size: 3.74 MB
            "fw_desc TEXT,"
            "page_url TEXT,"  # http://belkin.force.com/Articles/articles/en_US/Download/7371
            "file_url TEXT,"  # http://nextnet.belkin.com/update/files/F9K1104/v1/WW/F9K1104_WW_1.0.23.bin
            "on_click TEXT,"
            "tree_trail TEXT,"  # [26, 2, 1, 0, 0]
            "file_sha1 TEXT,"  # 5d3bc16eec2f6c34a5e46790b513093c28d8924a
            "PRIMARY KEY (id)"
            "UNIQUE(model,fw_ver)"
            ")"
        )
        driver = harvest_utils.getFirefox()
        harvest_utils.driver = driver
        prevTrail = []
        productWalker()
        driver.quit()
        conn.close()
    except Exception as ex:
        ipdb.set_trace()
        traceback.print_exc()
        driver.save_screenshot(getScriptName() + "_" + getFuncName() + "_excep.png")

def upsertOneModel():
    global driver, prevTrail, modelName
    try:
        prodName = [_.text for _ in getElems('div.sectionTitle p.hidden-xs') if _.text.strip()]
        if prodName:
            assert len(prodName)==1
            prodName=prodName[0]
        else:
            prodName=None
        pageUrl=driver.current_url
        try:
            imgUrl=waitVisible('.productPic img.img-responsive',4,1).get_attribute('src')
        except TimeoutException:
            imgUrl=None
        assert imgUrl is None or imgUrl.startswith('http')
        trailStr=str(prevTrail)
        sql("INSERT OR REPLACE INTO TFiles(model,prod_name,page_url,"
            "image_url,tree_trail) VALUES(:modelName, :prodName, :pageUrl,"
            ":imgUrl,:trailStr)", glocals())
        ulog('UPSERT "%(modelName)s" "%(prodName)s",%(trailStr)s,'
            '%(pageUrl)s' %glocals())
    except Exception as ex:
        ipdb.set_trace()
        traceback.print_exc()
        driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_excep.png')

def modelEnumer():
    global driver, prevTrail, modelName
    rootUrl='http://www.asus.com/Networking/AllProducts/'
    CSSs=driver.find_elements_by_css_selector
    CSS=driver.find_element_by_css_selector
    numElm=lambda c:len(CSSs(c))
    try:
        goToUrl(rootUrl)
        # retryStable(lambda:numElm('#list-table-area a'), 30,2)
        models = getElems('#list-table-area a')
        modelNames1 = [getElemText(_) for _ in models]
        numModels=len(models)
        ulog('numModels=%s'%numModels)
        startIdx = getStartIdx()
        for idx in range(startIdx, numModels):
            model = models[idx]
            modelName = retryA(lambda: model.text)
            ulog('click %s "%s"'%(idx, modelName))
            prevTrail+=[idx]
            retryA(lambda:model.click())
            fileEnumer()
            prevTrail.pop()
            goToUrl(rootUrl)
            models = getElems('#list-table-area a')
            if len(models) != numModels:
                modelNames2 = [getElemText(_) for _ in models]
                modelNamesD = set(modelNames1) - set(modelNames2)
                assert len(models)==numModels
    except Exception as ex:
        ipdb.set_trace()
        traceback.print_exc()
        driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_excep.png')

def main():
    global startTrail, prevTrail,conn
    try:
        startTrail = [int(re.search(r'\d+', _).group(0)) for _ in sys.argv[1:]]
        ulog('startTrail=%s'%startTrail)
        conn=sqlite3.connect('speedguide.sqlite3')
        sql("CREATE TABLE IF NOT EXISTS TFiles("
            "id INTEGER NOT NULL,"
            "brand TEXT," # "Asus"
            "model TEXT," # "RT-AC3100"
            "prod_name TEXT," # "Dual-band wireless-AC3100 gigabit router"
            "category TEXT," # "Wireless Router"
            "default_user_name TEXT,"
            "default_password TEXT,"
            "wifi_proto TEXT," # "abgn+ac"
            "availability TEXT,"
            "description TEXT,"
            "product_page TEXT,"
            "hw_fla1_amount TEXT," # '128 Mb'
            "hw_ram1_amount TEXT," # '512 Mb'
            "image_url TEXT,"
            "props_hstore TEXT," # 'a=>1,b=>"hello world",c=>True'::hstore
            "tree_trail TEXT," # [26, 2]
            "PRIMARY KEY (id),"
            "UNIQUE(brand,model)"
            ")")
        prevTrail=[]
        pageWalker()
        conn.close()
    except Exception as ex:
        ipdb.set_trace()
        traceback.print_exc()
        driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_excep.png')

def pageWalker():
    global prevTrail
    try:
        curUrl = "http://www.speedguide.net/broadband-list.php"
        startIdx = getStartIdx()
        for idx in range(0, startIdx):
            ulog('idx=%s'%idx); ulog('curUrl= '+curUrl)
            d = pq(url=curUrl)
            nextPage = d('img[alt^="Next page"]')[0]
            curUrl = nextPage.getparent().attrib['href']
            curUrl = urlChangePath(d.base_url,curUrl)

        for idx in itertools.count(startIdx):
            ulog('idx=%s'%idx); ulog('curUrl= '+curUrl)
            prevTrail+=[idx]
            routerWalker(curUrl)
            prevTrail.pop()
            d = pq(url=curUrl)
            try:
                nextPage = d('img[alt^="Next page"]')[0]
            except IndexError:
                ulog('Last page')
                break
            curUrl = nextPage.getparent().attrib['href']
            curUrl = urlChangePath(d.base_url,curUrl)
    except Exception as ex:
        ipdb.set_trace()
        traceback.print_exc()

def marketWalker():
    global driver,prevTrail
    try:
        showEol=waitVisible('#showEndLife')
        if not showEol.is_selected():
            showEol.click()
        selMkt= Select(waitVisible('.product-cat-box select:nth-child(1)'))
        startIdx=getStartIdx()
        numMkts = len(selMkt.options)
        curSel = selMkt.all_selected_options[0].text
        ulog('current Selected="%s"'%curSel)
        for idx in range(startIdx, numMkts):
            selMkt.select_by_index(idx)
            nextSel=selMkt.options[idx].text
            ulog('gonna select "%s"'%nextSel)
            btn = waitVisible('button.round-button.go')
            with UntilTextChanged('.content-box',9,0.4):
                btn.click()
            prevTrail+=[idx]
            modelWalker()
            prevTrail.pop()
            showEol=waitVisible('#showEndLife')
            if not showEol.is_selected():
                showEol.click()
            selMkt= Select(waitVisible('.product-cat-box select:nth-child(1)'))
    except Exception as ex:
        ipdb.set_trace()
        traceback.print_exc()
        driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_exc.png')

def seriesWalker():
    keywords = [
        "Router",
        "NAS",
        "Range Extender",
        "IP Camera",
        "Internet Camera",
        "LAN Switch",
        "PowerLine",
        "VPN/Firewall",
    ]
    try:
        series = getElems("#dl_series a")
        startIdx = getStartIdx()
        numSeries = len(series)
        for idx in range(startIdx, numSeries):
            seriesTxt = series[idx].text
            if not (seriesTxt / inin / keywords):
                continue
            ulog('idx=%s, click "%s"' % (idx, seriesTxt))
            prevTrail += [idx]
            series[idx].click()
            modelWalker()
            prevTrail.pop()
            series = getElems("#dl_series a")
    except Exception as ex:
        ipdb.set_trace()
        traceback.print_exc()
        driver.save_screenshot(getScriptName() + "_" + getFuncName() + "_excep.png")

def modelWalker(baseUrl):
    global prevTrail
    try:
        ulog('baseUrl= '+baseUrl)
        d = pq(url=baseUrl)
        models = d('.prodContainer:not(#JapaneseProd)')
        startIdx = getStartIdx()
        numModels = len(models)
        ulog('numModels= %s'%numModels)
        for idx in range(startIdx, numModels):
            ulog('idx=%s'%idx)

            try:
                modelName = [_.strip() for _ in models[idx].text_content().splitlines() if _.strip()][1]
            except IndexError:
                ulog('No model name')
                continue
            ulog('modelName="%s"'%modelName)

            onclick = models[idx].attrib['onclick']
            href = re.search(r"'(.+)(?<!\\)'", onclick).group(1)
            prevTrail+=[idx]
            detailScraper(urlChangePath(d.base_url, href))
            prevTrail.pop()
    except Exception as ex:
        ipdb.set_trace()
        traceback.print_exc()

def main():
    global startTrail, prevTrail,driver,conn
    try:
        startTrail = [int(re.search(r'\d+', _).group(0)) for _ in sys.argv[1:]]
        ulog('startTrail=%s'%startTrail)
        conn=sqlite3.connect('zyxel.sqlite3')
        sql("CREATE TABLE IF NOT EXISTS TFiles("
            "id INTEGER NOT NULL,"
            "model TEXT," # NBG5715
            "prod_name TEXT," # NBG5715
            "fw_ver TEXT," # V1.00(AAAG.8)C0
            "rel_date DATE," # 06-18-2015
            "image_url TEXT," # http://www2.zyxel.com/uploads/images/img_nbg5715_p_01_380.gif
            "page_url TEXT," # http://www2.zyxel.com/us/en/support/DownloadLandingSR.shtml?c=us&l=en&kbid=M-00022&md=NBG5715#searchZyxelTab4
            "file_url TEXT," # ftp://ftp2.zyxel.com/NBG5715/firmware/NBG5715_V1.00(AAAG.8)C0.zip
            "tree_trail TEXT," # [26, 2, 1, 0, 0]
            "file_size INTEGER," # 
            "file_sha1 TEXT," # 5d3bc16eec2f6c34a5e46790b513093c28d8924a
            "PRIMARY KEY (id)"
            "UNIQUE(model,fw_ver)"
            ")")
        driver=harvest_utils.getFirefox()
        # driver.implicitly_wait(2.0)
        harvest_utils.driver=driver
        prevTrail=[]
        goToUrl(rootUrl)
        getAllModels()
        modelWalker()
        driver.quit()
        conn.close()
    except Exception as ex:
        ipdb.set_trace()
        traceback.print_exc()
        driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_excep.png')

def enterFrame(iframeId:str):
    global driver
    prev_url=driver.current_url
    url=waitVisible('iframe[id=%s]'%iframeId).get_attribute('src')
    ulog('%s => %s'%(prev_url,url))
    driver.get(url)
    try:
        retryUntilTrue(isReadyState, 10, 2)
    except TimeoutException as ex:
        print(ex)
        pass

def convertUserPassword(s:str)->str:
    """
    'Default admin password: |  (blank)'
    """
    if s.isalnum():
        return s
    elif s=='(blank)':
        return ''
    elif s=='n/a':
        return ''
    else:
        ulog('s="%s"'%s)
        return s

def faqScraper(baseUrl, model, image_url, dev_desc, dev_hstore):
    # http://arris.force.com/consumers/articles/Drivers_and_Firmware/2247-N8-10NA-9-1-1h0d34-Firmware-Upgrade
    global prevTrail
    try:
        ulog('baseUrl= '+baseUrl)
        d = pq(url=baseUrl)
        files = [_ for _ in d('a') if _.text_content()/cicontains/'Firmware']
        numFiles= len(files)
        ulog('numFiles=%s'%numFiles)
        startIdx = getStartIdx()
        for idx in range(startIdx, numFiles):
            ulog('idx=%s'%idx)
            f=files[idx]
            file_name = f.text_content().strip()
            ulog('file_name="%s"'%file_name)
            try:
                fw_ver = re.search(r"\d+\.([\w\.\-]+)", file_name, re.I).group(0)
            except IndexError:
                ipdb.set_trace()
            file_url = f.attrib['href']
            tree_trail = str(prevTrail+[idx])
            sql("INSERT OR REPLACE INTO TFiles (model, image_url, dev_desc, dev_hstore, fw_ver, page_url, file_url, tree_trail) VALUES (:model, :image_url, :dev_desc, :dev_hstore, :fw_ver, :baseUrl, :file_url, :tree_trail)", locals())
            uprint('UPSERT "%(model)s", "%(fw_ver)s", %(tree_trail)s, %(file_url)s'%locals())

    except Exception as ex:
        ipdb.set_trace()
        traceback.print_exc()

def retryStable(cond, timeOut:float=5, pollFreq:float=0.5):
    oldCond = cond()
    ulog('oldCond=%s'%oldCond)
    timeElapsed=0.0
    while timeElapsed<timeOut:
        beginTime=time.time()
        newCond=cond()
        if newCond != oldCond:
            ulog('newCond=%s'%newCond)
            oldCond=newCond
            timeElpased=0
        else:
            time.sleep(pollFreq)
            timeElapsed += time.time()-beginTime

def modelWalker():
    global driver, prevTrail, modelName
    rootUrl='http://www2.zyxel.com/us/en/support/DownloadLandingSR.shtml?c=us&l=en&md='
    try:
        startIdx = getStartIdx()
        ulog('len(allModels)=%d'%len(allModels))
        for idx in range(startIdx,len(allModels)):
            ulog('idx=%s'%idx)
            modelName=allModels[idx]
            ulog('modelName="%s"'%modelName)
            goToUrl(rootUrl+parse.quote(modelName))
            waitUntil(isReadyState)

            # click "Firmware" tab
            tab = elemWithText('li.resp-tab-item','Firmware')
            if not tab:
                ulog('no Firmware tab,bypass!')
                prevTrail+=[idx]
                upsertOneModel()
                prevTrail.pop()
                continue
            tab.click()
            time.sleep(0.1)
            prevTrail+=[idx]
            versionWalker()
            prevTrail.pop()
    except Exception as ex:
        ipdb.set_trace()
        traceback.print_exc()
        driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_excep.png')

def guessFileSize(txt:str)->int:
    """ txt='6.56 MB'
    """
    try:
        m = re.search(r'(\d+\.?\d+)\s*(GB|MB|KB)', txt, re.I)
        if not m:
            ulog('error txt="%s"'%txt)
            return 0
        unitDic=dict(MB=1024**2,KB=1024,GB=1024**3)
        unitTxt = m.group(2).upper()
        return int(float(m.group(1)) * unitDic[unitTxt] )
    except Exception as ex:
        ipdb.set_trace()
        print('txt=',txt)

def walkProd():
    global driver, prevTrail
    try:
        # click overlay advertisement popup left button "No Thanks"
        try:
            driver.find_element_by_css_selector("a.btn.close.fl-left").\
                    click()
        except (NoSuchElementException):
            pass

        zpath = ('#ctl00_ctl00_ctl00_mainContent_localizedContent_bodyCenter'+
                 '_adsPanel_lbProduct')
        waitTextChanged(zpath)
        curSel = Select(css(zpath))
        numProds = len(curSel.options)
        ulog("numProds=%d"%numProds)

        startIdx = getStartIdx()
        for idx in range(startIdx, numProds):
            curSel = Select(css(zpath))
            ulog("idx=%s"%idx)
            ulog('select "%s"'%curSel.options[idx].text)
            curSel.select_by_index(idx)
            prevTrail+=[idx]
            while True:
                ret = walkFile()
                if ret != TRY_AGAIN:
                    break
            if ret== PROC_GIVE_UP:
                ulog('"%s" is GIVE UP'% curSel.options[idx].text)
            prevTrail.pop()
        return PROC_OK
    except Exception as ex:
        traceback.print_exc(); ipdb.set_trace()
        driver.save_screenshot('netgear_exc.png')

def guessFileSize(txt:str)->int:
    """ txt='27.29 MBytes'
    """
    try:
        m = re.search(r'(\d+\.?\d+)\s*(MB|KB)', txt, re.I)
        if not m:
            ulog('error txt="%s"'%txt)
            return 0
        unitDic=dict(MB=1024**2,KB=1024)
        unitTxt = m.group(2).upper()
        return int(float(m.group(1)) * unitDic[unitTxt] )
    except Exception as ex:
        ipdb.set_trace()
        traceback.print_exc()