本文整理汇总了Python中mechanize.build_opener函数的典型用法代码示例。如果您正苦于以下问题：Python build_opener函数的具体用法？Python build_opener怎么用？Python build_opener使用的例子？那么恭喜您, 这里精选的函数代码示例或许可以为您提供帮助。
在下文中一共展示了build_opener函数的20个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于我们的系统推荐出更棒的Python代码示例。
示例1: retrieve_product_data
def retrieve_product_data(self, product_link):
cookies = mechanize.CookieJar()
opener = mechanize.build_opener(mechanize.HTTPCookieProcessor(cookies))
opener.addheaders = [('User-agent', 'Mozilla/5.0 (MyProgram/0.1)'),
('From', '[email protected]')]
mechanize.install_opener(opener)
browser = mechanize.Browser()
product_data = browser.open(product_link).get_data()
soup = BeautifulSoup(product_data)
product_name = soup.find('title').string.encode('ascii', 'ignore')
product_prices = soup.find('div', 'price').contents
try:
cash_price = int(clean_price_string(product_prices[4]))
product_data = ProductData()
product_data.custom_name = product_name
product_data.price = cash_price
product_data.url = product_link
product_data.comparison_field = product_link
return product_data
except IndexError:
return None
开发者ID:vkhemlan，项目名称:solotodo，代码行数:26，代码来源:global_mac.py
示例2: _retrieve_product
def _retrieve_product(cls, url):
cookies = mechanize.CookieJar()
opener = mechanize.build_opener(mechanize.HTTPCookieProcessor(cookies))
opener.addheaders = [('User-agent', 'Mozilla/5.0 (MyProgram/0.1)'),
('From', '[email protected]')]
mechanize.install_opener(opener)
browser = mechanize.Browser()
product_data = browser.open(url).get_data()
soup = BeautifulSoup(product_data)
product_name = soup.find('h1').string.encode('ascii', 'ignore')
product_price = soup.find('span', {'id': 'product_price'})
product_price = Decimal(clean_price_string(product_price.string))
payment_methods = ['cash', 'deposit', 'wire_transfer']
additional_data = soup.find('td', 'descr').findAll('h3')
if not additional_data:
payment_methods.extend(['debit_card', 'credit_card'])
elif additional_data[0].string and 'Contado' not in \
additional_data[0].string:
payment_methods.extend(['debit_card', 'credit_card'])
prices = {}
for p in payment_methods:
prices[p] = product_price
return [product_name, prices]
开发者ID:SoloTodo，项目名称:storescrapper，代码行数:29，代码来源:global_mac.py
示例3: customizeUserAgent
def customizeUserAgent():
import mechanize
cookies = mechanize.CookieJar()
opener = mechanize.build_opener(mechanize.HTTPCookieProcessor(cookies))
# Pretend to be Chrome to avoid getting the mobile site.
opener.addheaders = [("User-agent", "Chrome/16.0.912.63")]
mechanize.install_opener(opener)
开发者ID:georgevdd，项目名称:bank-account-reporting，代码行数:7，代码来源:Halifax.py
示例4: retrieve_product_links
def retrieve_product_links(self):
cookies = mechanize.CookieJar()
opener = mechanize.build_opener(mechanize.HTTPCookieProcessor(cookies))
opener.addheaders = [('User-agent', 'Mozilla/5.0 (MyProgram/0.1)'),
('From', '[email protected]')]
mechanize.install_opener(opener)
url_base = 'http://www.globalmac.cl/'
browser = mechanize.Browser()
url_extensions = [
['Distribuidor-Apple-Chile/MacBook-Air', 'Notebook'],
['Distribuidor-Apple-Chile/MacBook-Pro', 'Notebook'],
['Hardware-Mac-PC/Discos-Duros-Notebook-SATA-2.5', 'StorageDrive'],
['Hardware-Mac-PC/Discos-Duros-SATA-3.5', 'StorageDrive'],
['Hardware-Mac-PC/Discos-Duros-SSD-SATA-2.5', 'StorageDrive'],
]
product_links = []
for url_extension, ptype in url_extensions:
url = url_base + url_extension
base_data = browser.open(url).get_data()
soup = BeautifulSoup(base_data)
for item in soup.findAll('div', 'name'):
product_links.append([item.find('a')['href'], ptype])
return product_links
开发者ID:vkhemlan，项目名称:solotodo，代码行数:29，代码来源:global_mac.py
示例5: uploadFileToAquaforum
def uploadFileToAquaforum(uploadFilename, requestedFileName):
'''
returns response page
'''
# build opener. Can be extended to handle cookies/proxies
opener = mechanize.build_opener()
# goto upload page
request3 = mechanize.Request(FORUM_UPLOAD_URL)
response3 = opener.open(request3)
# parse form on upload page and add file
forms = mechanize.ParseResponse(response3, backwards_compat=False)
form = forms[0]
filectr = form.find_control("imgfile")
# filectr.add_file(open('/home/jasper/avatar.jpg'),"image/jpeg","avatar.jpg")
theFile = file(uploadFilename, 'rb')
filectr.add_file(theFile, "image/jpeg", os.path.split(
requestedFileName)[-1])
# obtain form data
request4 = form.click() # urllib2.Request object
theFile.close()
request4.add_header('Referer', response3.geturl())
response4 = opener.open(request4)
return response4.read()
开发者ID:labordus，项目名称:aquaf，代码行数:25，代码来源:diversen.py
示例6: constructUploadName
def constructUploadName(loginname, requestedfilename):
# construct name
import random
filename = os.path.split(requestedfilename)[1]
filename = filename[:string.find(filename, ".")] + ".jpg" # construct jpg extension
resultName = string.lower(loginname + "_" + APP_VERSION_STR + "_" + filename) # prepend loginname
resultName = string.replace(resultName, " ", "_") # replace spaces
resultName = string.replace(resultName, "'", "_") # replace '
# resultName = urllib.quote(resultName) #make safe url
theResult = ""
for theChar in resultName:
if theChar in ALLOWED_CHARS:
theResult += theChar
resultName = theResult
# check whether ok
# build opener. Can be extended to handle cookies/proxies
opener = mechanize.build_opener()
# goto upload page
request3 = mechanize.Request(FORUM_UPLOAD_URL)
response3 = opener.open(request3)
# page = string.lower(response3.read())
response3.close()
random.seed()
# while not name ok, permutate
for _i in range(6):
resultName = str(random.random())[-1] + resultName # prepend with random number
# while string.find(page,resultName)<>-1:
return resultName
开发者ID:labordus，项目名称:aquaf，代码行数:28，代码来源:diversen.py
示例7: readUrl
def readUrl(inUrl):
tryCount = 0
while tryCount < 5 :
# print "Create CookieJar"
cookies = mechanize.CookieJar()
# print "Build Opener"
opener = mechanize.build_opener(mechanize.HTTPCookieProcessor(cookies))
# print "Add Headers"
opener.addheaders = [("User-agent", "Mozilla/5.0 (compatible; MyProgram/0.1)"),("From", "[email protected]")]
# print "Install Opener"
mechanize.install_opener(opener)
try:
# print "Open URL"
response = mechanize.urlopen(inUrl)
tryCount = 99
except:
tryCount += 1
print "******** Error on urlopen ***********"
print "URL: ", inUrl
print "Trying Again....", tryCount
# print response.read()
# html = urllib.urlopen(inUrl).read()
# print "Reading Response"
html = response.read()
# print "Response Read:", html[0:100]
root = lxml.html.fromstring(html)
# print "Root created: ", root
return root
开发者ID:carriercomm，项目名称:scraperwiki-scraper-vault，代码行数:31，代码来源:gptest02_1.py
示例8: __init__
def __init__(self, username="RP\\12345", password="abcdef"):
self.username = "RP\\"+username
self.password = password
self.password_manager = urllib2.HTTPPasswordMgrWithDefaultRealm()
ntlm_auth = HTTPNtlmAuthHandler.HTTPNtlmAuthHandler(self.password_manager)
opener = mechanize.build_opener(ntlm_auth)
mechanize.install_opener(opener)
开发者ID:Dino198，项目名称:PythonLeo，代码行数:7，代码来源:PythonLeo.py
示例9: __callRequest
def __callRequest(self):
cookieJar = mechanize.LWPCookieJar()
try: #TODO ohne try evtl.
cookieJar.load(self._cookiePath, self.__bIgnoreDiscard, self.__bIgnoreExpired)
except Exception as e:
logger.info(e)
sParameters = urllib.urlencode(self.__aParameters)
opener = mechanize.build_opener(SmartRedirectHandler,
mechanize.HTTPEquivProcessor,
mechanize.HTTPRefreshProcessor)
if (len(sParameters) > 0):
oRequest = mechanize.Request(self.__sUrl, sParameters)
else:
oRequest = mechanize.Request(self.__sUrl)
for aHeader in self.__aHeaderEntries:
for sHeaderKey, sHeaderValue in aHeader.items():
oRequest.add_header(sHeaderKey, sHeaderValue)
cookieJar.add_cookie_header(oRequest)
if self.caching and self.cacheTime > 0:
sContent = self.readCache(self.getRequestUri())
if sContent:
return sContent
try:
oResponse = opener.open(oRequest,timeout = 60)
except mechanize.HTTPError, e:
if not self.ignoreErrors:
xbmcgui.Dialog().ok('xStream','Fehler beim Abrufen der Url:',self.__sUrl, str(e))
logger.error("HTTPError "+str(e)+" Url: "+self.__sUrl)
return ''
else:
oResponse = e
开发者ID:AlexBander，项目名称:plugin.video.xstream，代码行数:34，代码来源:requestHandler.py
示例10: _parsingRobotsFile
def _parsingRobotsFile(self, url):
"""
Setup internal state after downloading robots.txt
If urlopen.code in (401, 403), all user-agent is disallowed. -> 삭제
If urlopen.code >= 400, all user-agent is allowd.
"""
domain_name = urlparse.urlparse(url)[1]
rules = RobotsTextRules()
#getLogger().debug("++Trying to download: %s", url)
opener = mechanize.build_opener(mechanize.HTTPRefreshProcessor,)
rq = mechanize.Request(url)
rq.add_header("User-agent", "Mozilla/5.0 (compatible; Windows NT 6.1?; ZumBot/1.0; http://help.zum.com/inquiry)")
#shttp.setRequestHeader(user_agent = USER_AGENT)
rs = None
try:
rs = opener.open(rq)
header = rs.info()
rules.return_code = rs.code
except Exception, msg:
try:
if not url.startswith("http://www."):
t_url = url.replace("http://", "http://www.")
rq = mechanize.Request(t_url )
rq.add_header("User-agent", "Mozilla/5.0 (compatible; Windows NT 6.1?; ZumBot/1.0; http://help.zum.com/inquiry)")
rs = opener.open(rq)
header = rs.info()
rules.return_code = rs.code
except Exception, msg:
return rules
开发者ID:Shiwoo-Park，项目名称:glance，代码行数:33，代码来源:robots_validator.py
示例11: __init__
def __init__(self, feed_id, apikey):
self._version = "1.0.0"
self._feed_id = feed_id
self._opener = mechanize.build_opener()
self._opener.addheaders = [('X-ApiKey',apikey)]
self._data = []
self._payload = {}
开发者ID:joelbyrnes，项目名称:home_monitoring，代码行数:7，代码来源:lib_cosm.py
示例12: init
def init(self):
br = mechanize.Browser()
br.set_handle_robots(False)
self.cj = mechanize.LWPCookieJar()
br.set_cookiejar(self.cj)
br.set_handle_refresh(mechanize._http.HTTPRefreshProcessor(),
max_time=1)
br.open("https://www.tumblr.com/login")
br.select_form(nr=0)
br['user[email]'] = ""
br['user[password]'] = ""
url, data, hdrs = br.form.click_request_data()
br.open("https://www.tumblr.com/login", data)
self.nf = 0
opener = mechanize.build_opener(
mechanize.HTTPCookieProcessor(self.cj))
mechanize.install_opener(opener)
self._fetch()
开发者ID:non117，项目名称:boxnya-plugin，代码行数:25，代码来源:tumblrnotify.py
示例13: _checkStoredInjections
def _checkStoredInjections(self):
for r in self.results:
# At this state injections in Result obj are not
# compacted yet so it will only be 1st injected param
url, data = r.target.getPayloadedUrl(r.first_param, "")
# In case of proxy
if self.engine.getOption('http-proxy') is not None:
proxy = ProxyHandler({'http': self.engine.getOption('http-proxy')})
opener = build_opener(proxy)
install_opener(opener)
# Some headers
if self.engine.getOption('ua') is not None:
if self.engine.getOption('ua') is "RANDOM":
headers = {'User-Agent': random.choice(USER_AGENTS)}
else:
headers = {'User-Agent': self.engine.getOption('ua')}
else:
headers = {}
if self.engine.getOption("cookie") is not None:
headers["Cookie"] = self.engine.getOption("cookie")
# Build the request
req = Request(url, data, headers)
try:
to = 10 if self.engine.getOption('http-proxy') is None else 20
response = urlopen(req, timeout=to)
except HTTPError, e:
self._addError(e.code, r.target.getAbsoluteUrl())
continue
except URLError, e:
self._addError(e.reason, r.target.getAbsoluteUrl())
continue
开发者ID:Drx51，项目名称:Framework，代码行数:34，代码来源:scanner.py
示例14: _performInjections
def _performInjections(self, target):
# Check every parameter
for k, v in target.params.iteritems():
pl = Payload(taint=True)
url, data = target.getPayloadedUrl(k, pl.payload)
# In case of proxy
if self.engine.getOption('http-proxy') is not None:
proxy = ProxyHandler({'http': self.engine.getOption('http-proxy')})
opener = build_opener(proxy)
install_opener(opener)
# Some headers
if self.engine.getOption('ua') is not None:
if self.engine.getOption('ua') is "RANDOM":
headers = {'User-Agent': random.choice(USER_AGENTS)}
else:
headers = {'User-Agent': self.engine.getOption('ua')}
else:
headers = {}
if self.engine.getOption("cookie") is not None:
headers["Cookie"] = self.engine.getOption("cookie")
# Build the request
req = Request(url, data, headers)
try:
to = 10 if self.engine.getOption('http-proxy') is None else 20
response = urlopen(req, timeout=to)
except HTTPError, e:
self._addError(e.code, target.getAbsoluteUrl())
return
except URLError, e:
self._addError(e.reason, target.getAbsoluteUrl())
return
开发者ID:Drx51，项目名称:Framework，代码行数:33，代码来源:scanner.py
示例15: slurp_with_login_and_pwd
def slurp_with_login_and_pwd():
import sys
import mechanize
# sys.path.append('ClientCookie-1.0.3')
# from mechanize import ClientCookie
# sys.path.append('ClientForm-0.1.17')
# import ClientForm
# Create special URL opener (for User-Agent) and cookieJar
cookieJar = mechanize.CookieJar()
opener = mechanize.build_opener(mechanize.HTTPCookieProcessor(cookieJar))
opener.addheaders = [("User-agent","Mozilla/5.0 (compatible)")]
mechanize.install_opener(opener)
fp = mechanize.urlopen("http://login.yahoo.com")
forms = mechanize.ParseResponse(fp)
fp.close()
# print forms on this page
for form in forms:
print "***************************"
print form
form = forms[0]
form["login"] = "yahoo-user-id" # use your userid
form["passwd"] = "password" # use your password
fp = mechanize.urlopen(form.click())
fp.close()
fp = mechanize.urlopen("https://class.coursera.org/ml-003/lecture/download.mp4?lecture_id=1") # use your group
fp.readlines()
fp.close()
开发者ID:yz-，项目名称:ut，代码行数:31，代码来源:slurping.py
示例16: GetHtml
def GetHtml(url):
opener = mechanize.build_opener()
opener.addheaders = [("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:2.0) Gecko/20100101 Firefox/4.0")]
mechanize.install_opener(opener)
request = mechanize.urlopen(url)
html = request.read()
request.close()
return html
开发者ID:yanigisawa，项目名称:PyComicSyndicator，代码行数:8，代码来源:htmlFetch.py
示例17: post
def post(self,URL,QueryString,headers=None):
opener = mechanize.build_opener(mechanize.HTTPCookieProcessor(self.cookieJar))
if headers:
opener.addheaders = headers
qs = urllib.urlencode(QueryString)
response = opener.open(URL,qs)
return response
开发者ID:odinshorse，项目名称:arbitrage，代码行数:8，代码来源:bot.py
示例18: test_retrieve_to_named_file
def test_retrieve_to_named_file(self):
url = urljoin(self.uri, "/mechanize/")
test_filename = os.path.join(self.make_temp_dir(), "python.html")
opener = mechanize.build_opener()
verif = CallbackVerifier(self)
filename, headers = opener.retrieve(url, test_filename, verif.callback)
self.assertEqual(filename, test_filename)
self._check_retrieve(url, filename, headers)
self.assert_(os.path.isfile(filename))
开发者ID:Almad，项目名称:Mechanize，代码行数:9，代码来源:functional_tests.py
示例19: setup_mechanize
def setup_mechanize():
"""
Set up user agent for all mechanize calls.
"""
cookies = mechanize.CookieJar()
opener = mechanize.build_opener(mechanize.HTTPCookieProcessor(cookies))
homepage = "http://github.com/aszlig/picfetcher"
opener.addheaders = [("User-agent", "PicFetcher/0.1.0 (+%s)" % homepage)]
mechanize.install_opener(opener)
开发者ID:aszlig，项目名称:picfetcher，代码行数:9，代码来源:picturefetch.py
示例20: __init__
def __init__(self, **kwargs):
#self._opener = kwargs.get('opener',None)
self._username = unicode(kwargs.get('user', RedditSession._username))
self._passwd = unicode(kwargs.get('passwd', RedditSession._password))
self._cookies = mechanize.CookieJar()
self._opener = mechanize.build_opener(mechanize.HTTPCookieProcessor(self._cookies))
self._do_login()
开发者ID:instanceoftom，项目名称:pyreddit，代码行数:9，代码来源:core.py
注：本文中的mechanize.build_opener函数示例由纯净天空整理自Github/MSDocs等源码及文档管理平台，相关代码片段筛选自各路编程大神贡献的开源项目，源码版权归原作者所有，传播和使用请参考对应项目的License；未经允许，请勿转载。 |
客服电话
APP下载
官方微信
请发表评论