本文整理汇总了Python中requests.utils.get_encoding_from_headers函数的典型用法代码示例。如果您正苦于以下问题:Python get_encoding_from_headers函数的具体用法?Python get_encoding_from_headers怎么用?Python get_encoding_from_headers使用的例子?那么恭喜您, 这里精选的函数代码示例或许可以为您提供帮助。
在下文中一共展示了get_encoding_from_headers函数的20个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于我们的系统推荐出更棒的Python代码示例。
示例1: find_encoding
def find_encoding(content, headers=None):
# content is unicode
if isinstance(content, unicode):
return 'unicode'
encoding = None
# Try charset from content-type
if headers:
encoding = get_encoding_from_headers(headers)
if encoding == 'ISO-8859-1':
encoding = None
# Try charset from content
if not encoding:
encoding = get_encodings_from_content(content)
encoding = encoding and encoding[0] or None
# Fallback to auto-detected encoding.
if not encoding and chardet is not None:
encoding = chardet.detect(content)['encoding']
if encoding and encoding.lower() == 'gb2312':
encoding = 'gb18030'
return encoding or 'latin_1'
开发者ID:Crooky,项目名称:qiandao,代码行数:26,代码来源:utils.py
示例2: _receive_response
def _receive_response(self, task, response):
"""
Called by the delegate when a response has been received.
This call is expected only on background threads, and thus may not do
anything that is not Python-thread-safe. This means that, for example,
it is safe to grab things from the _tasks dictionary, but it is not
safe to make other method calls on this object unless they explicitly
state that they are safe in background threads.
"""
queue, request = self._tasks[task]
resp = Response()
resp.status_code = getKey(response, 'statusCode')
resp.reason = ''
# TODO: Why do I have to do this?
raw_headers = getKey(response, 'allHeaderFields')
resp.headers = CaseInsensitiveDict(raw_headers)
resp.encoding = get_encoding_from_headers(resp.headers)
# TODO: This needs to point to an object that we can use to provide
# the various raw things that requests needs.
resp.raw = None
if isinstance(request.url, bytes):
resp.url = request.url.decode('utf-8')
else:
resp.url = request.url
resp.request = request
resp.connection = self
# Put this response on the queue.
queue.put_nowait(resp)
开发者ID:Lukasa,项目名称:requests-darwin,代码行数:35,代码来源:adapter.py
示例3: encoding
def encoding(self):
if hasattr(self, '_encoding'):
return self._encoding
# content is unicode
if isinstance(self.content, unicode):
return 'unicode'
# Try charset from content-type
encoding = get_encoding_from_headers(self.headers)
if encoding == 'ISO-8859-1':
encoding = None
# Try charset from content
if not encoding:
encoding = get_encodings_from_content(self.content)
encoding = encoding and encoding[0] or None
# Fallback to auto-detected encoding.
if not encoding and chardet is not None:
encoding = chardet.detect(self.content)['encoding']
if encoding and encoding.lower() == 'gb2312':
encoding = 'gb18030'
self._encoding = encoding or 'utf-8'
return self._encoding
开发者ID:5aket,项目名称:pyspider,代码行数:27,代码来源:response.py
示例4: get_unicode_from_request
def get_unicode_from_request(r):
"""Returns the requested content back in unicode.
:param r: Request object to get unicode content from.
Try:
1. charset from content-type
2. fall back and assume utf-8
3. latin-1 replacing unicode chars
"""
if isinstance(r.body, unicode):
return r.body
tried_encodings = []
# Try charset from content-type
# i.e. "Content-type: text/plain; charset=us-ascii"
encoding = get_encoding_from_headers(CaseInsensitiveDict(r.headers))
if encoding:
try:
return unicode(r.body, encoding)
except UnicodeError:
tried_encodings.append(encoding)
# workaround if encoding is not specified, assume utf-8, then latin-1
try:
return unicode(r.body, 'utf-8')
except:
tried_encodings.append('utf-8')
return unicode(r.body, 'latin-1', errors='replace')
开发者ID:rusenask,项目名称:mirage,代码行数:31,代码来源:__init__.py
示例5: build_response
def build_response(request,
status_code=200,
headers={},
content='(none)'):
"""
Build a :class:`requests.Response` object on the basis of the passed
parameters.
"""
response = Response()
response.status_code = status_code
response.reason = responses[status_code]
response.headers = CaseInsensitiveDict(headers)
# Pretend that we've already read from the socket
response._content = content
response.encoding = get_encoding_from_headers(response.headers)
response.url = request.url
response.raw = MockRawResponse()
# Give the Response some context.
response.request = request
response.connection = MockConnection()
return response
开发者ID:alphagov,项目名称:ghtools,代码行数:26,代码来源:requestmocker.py
示例6: build_response
def build_response(self, req, resp):
"""Builds a :class:`Response <requests.Response>` object from a urllib3
response. This should not be called from user code, and is only exposed
for use when subclassing the
:class:`HTTPAdapter <requests.adapters.HTTPAdapter>`
:param req: The :class:`PreparedRequest <PreparedRequest>` used to generate the response.
:param resp: The urllib3 response object.
"""
response = Response()
# Fallback to None if there's no status_code, for whatever reason.
response.status_code = getattr(resp, 'status', None)
# Make headers case-insensitive.
response.headers = CaseInsensitiveDict(getattr(resp, 'headers', {}))
# Set encoding.
response.encoding = get_encoding_from_headers(response.headers)
response.raw = resp
response.reason = response.raw.reason
if isinstance(req.url, bytes):
response.url = req.url.decode('utf-8')
else:
response.url = req.url
# Add new cookies from the server.
extract_cookies_to_jar(response.cookies, req, resp)
# Give the Response some context.
response.request = req
response.connection = self
return response
开发者ID:pcreech,项目名称:pulp,代码行数:35,代码来源:adapters.py
示例7: send
def send(self, request, **kwargs):
url = urlparse(request.url)
if url.scheme != 'https':
raise Exception('Only HTTPS is supported!')
ctx = self._make_context()
conn = httpslib.HTTPSConnection(
url.hostname, url.port or 443, ssl_context=ctx)
conn.request(request.method, url.path, request.body, request.headers)
resp = conn.getresponse()
response = Response()
# Fallback to None if there's no status_code, for whatever reason.
response.status_code = getattr(resp, 'status', None)
# Make headers case-insensitive.
response.headers = CaseInsensitiveDict(getattr(resp, 'headers', {}))
# Set encoding.
response.encoding = get_encoding_from_headers(response.headers)
response.raw = resp
response.reason = response.raw.reason
if isinstance(request.url, bytes):
response.url = request.url.decode('utf-8')
else:
response.url = request.url
# Give the Response some context.
response.request = request
response.connection = self
return response
开发者ID:mcrute,项目名称:dev_urandom,代码行数:35,代码来源:pkcs11-adapter.py
示例8: start_response
def start_response(status, headers):
response.status_code = int(status.split(' ')[0])
response.reason = responses.get(response.status_code, 'Unknown Status Code')
response.headers = CaseInsensitiveDict(headers)
response.encoding = get_encoding_from_headers(response.headers)
response.elapsed = datetime.datetime.utcnow() - start
self._log(response)
开发者ID:seanbrant,项目名称:requests-wsgi-adapter,代码行数:7,代码来源:wsgiadapter.py
示例9: encoding
def encoding(rsp):
"""
encoding of Response.content.
if Response.encoding is None, encoding will be guessed
by header or content or chardet if avaibable.
"""
# content is unicode
if isinstance(rsp.content, six.text_type):
return 'unicode'
# Try charset from content-type
encoding = get_encoding_from_headers(rsp.headers)
if encoding == 'ISO-8859-1':
encoding = None
# Try charset from content
if not encoding and get_encodings_from_content:
encoding = get_encodings_from_content(rsp.content)
encoding = encoding and encoding[0] or None
# Fallback to auto-detected encoding.
if not encoding and chardet is not None:
encoding = chardet.detect(rsp.content)['encoding']
if encoding and encoding.lower() == 'gb2312':
encoding = 'gb18030'
encoding = encoding or 'utf-8'
return encoding
开发者ID:zymtech,项目名称:parse_newspage,代码行数:30,代码来源:parserstandalone.py
示例10: request
def request(method, url, **kwargs):
if 'data' in kwargs:
kwargs['params'] = kwargs.pop('data')
elif 'params' in kwargs and kwargs['params'] is None:
kwargs.pop('params')
auth = None
if 'auth' in kwargs:
auth = kwargs.pop('auth')
for i in ['auth', 'allow_redirects', 'stream']:
if i in kwargs:
kwargs.pop(i)
if app.app.registry.api_url in url:
if auth:
authorization = api.authorization
api.authorization = ('Basic', auth)
resp = api._gen_request(method.upper(), url, expect_errors=True, **kwargs)
if auth:
api.authorization = authorization
else:
resp = app._gen_request(method.upper(), url, expect_errors=True, **kwargs)
response = Response()
response.status_code = resp.status_int
response.headers = CaseInsensitiveDict(getattr(resp, 'headers', {}))
response.encoding = get_encoding_from_headers(response.headers)
response.raw = resp
response._content = resp.body
response.reason = resp.status
if isinstance(url, bytes):
response.url = url.decode('utf-8')
else:
response.url = url
response.request = resp.request
return response
开发者ID:Leits,项目名称:openprocurement.chronograph,代码行数:33,代码来源:base.py
示例11: guess_response_encoding
def guess_response_encoding(resp):
'''
Guess the content encoding of a requests response.
Note: there's a performance issue due to chardet.
'''
# first try the encoding supplied by responce header and content
encs = get_encodings_from_content(resp.content) or []
for enc in encs:
try:
resp.content.decode(enc)
LOG.info('Detected encoding %s from response content.', enc)
return enc
except UnicodeDecodeError:
LOG.debug('Encoding from response content doesn\'t work.')
enc = get_encoding_from_headers(resp.headers)
if enc:
try:
resp.content.decode(enc)
LOG.info('Detected encoding %s from response header.', enc)
return enc
except UnicodeDecodeError:
LOG.debug('Encoding from response header doesn\'t work.')
# neither encoding works, we have to go the hard way.
start = clock()
g = detect(resp.content)
LOG.info('Detected encoding %s with cofidence of %g in %gs.' % (g['encoding'], g['confidence'], clock() - start))
return g['encoding']
开发者ID:dirtysalt,项目名称:dirtysalt.github.io,代码行数:30,代码来源:utils.py
示例12: build_response
def build_response(self, request, resp):
"""
Builds a Requests' response object. This emulates most of the logic of
the standard fuction but deals with the lack of the ``.headers``
property on the HTTP20Response object.
"""
response = Response()
response.status_code = resp.status
response.headers = CaseInsensitiveDict(resp.getheaders())
response.raw = resp
response.reason = resp.reason
response.encoding = get_encoding_from_headers(response.headers)
extract_cookies_to_jar(response.cookies, request, response)
if isinstance(request.url, bytes):
response.url = request.url.decode('utf-8')
else:
response.url = request.url
response.request = request
response.connection = self
# One last horrible patch: Requests expects its raw responses to have a
# release_conn method, which I don't. We should monkeypatch a no-op on.
resp.release_conn = lambda: None
return response
开发者ID:lifuzu,项目名称:hyper,代码行数:29,代码来源:contrib.py
示例13: create_response
def create_response(request, **kwargs):
"""
:param int status_code: The status code to return upon a successful
match. Defaults to 200.
:param HTTPResponse raw: A HTTPResponse object to return upon a
successful match.
:param io.IOBase body: An IO object with a read() method that can
return a body on successful match.
:param bytes content: A byte string to return upon a successful match.
:param unicode text: A text string to return upon a successful match.
:param object json: A python object to be converted to a JSON string
and returned upon a successful match.
:param dict headers: A dictionary object containing headers that are
returned upon a successful match.
:param CookieJar cookies: A cookie jar with cookies to set on the
response.
"""
connection = kwargs.pop('connection', _FakeConnection())
_check_body_arguments(**kwargs)
raw = kwargs.pop('raw', None)
body = kwargs.pop('body', None)
content = kwargs.pop('content', None)
text = kwargs.pop('text', None)
json = kwargs.pop('json', None)
headers = kwargs.pop('headers', {})
encoding = None
if content is not None and not isinstance(content, six.binary_type):
raise TypeError('Content should be binary data')
if text is not None and not isinstance(text, six.string_types):
raise TypeError('Text should be string data')
if json is not None:
text = jsonutils.dumps(json)
if text is not None:
encoding = get_encoding_from_headers(headers) or 'utf-8'
content = text.encode(encoding)
if content is not None:
body = _IOReader(content)
if not raw:
raw = HTTPResponse(status=kwargs.get('status_code', _DEFAULT_STATUS),
headers=headers,
reason=kwargs.get('reason'),
body=body or _IOReader(six.b('')),
decode_content=False,
preload_content=False,
original_response=compat._fake_http_response)
response = _http_adapter.build_response(request, raw)
response.connection = connection
if encoding and not response.encoding:
response.encoding = encoding
_extract_cookies(request, response, kwargs.get('cookies'))
return response
开发者ID:jamielennox,项目名称:requests-mock,代码行数:59,代码来源:response.py
示例14: __init__
def __init__(self, response, method):
self._response = response
self._method = method
self.status_code = response.code
self.headers = CaseInsensitiveDict((
(header, ', '.join(values)) for header, values in
response.headers.getAllRawHeaders()))
self.encoding = get_encoding_from_headers(self.headers) or 'ISO-8859-1'
self._waiting_for_content = []
开发者ID:gurteshwar,项目名称:treq,代码行数:9,代码来源:response.py
示例15: get_unicode_from_response
def get_unicode_from_response(response):
"""Return the requested content back in unicode.
This will first attempt to retrieve the encoding from the response
headers. If that fails, it will use
:func:`requests_toolbelt.utils.deprecated.get_encodings_from_content`
to determine encodings from HTML elements.
.. code-block:: python
import requests
from requests_toolbelt.utils import deprecated
r = requests.get(url)
text = deprecated.get_unicode_from_response(r)
:param response: Response object to get unicode content from.
:type response: requests.models.Response
"""
tried_encodings = set()
# Try charset from content-type
encoding = utils.get_encoding_from_headers(response.headers)
if encoding:
try:
return str(response.content, encoding)
except UnicodeError:
tried_encodings.add(encoding.lower())
encodings = get_encodings_from_content(response.content)
for _encoding in encodings:
_encoding = _encoding.lower()
if _encoding in tried_encodings:
continue
try:
return str(response.content, _encoding)
except UnicodeError:
tried_encodings.add(_encoding)
# Fall back:
if encoding:
try:
return str(response.content, encoding, errors='replace')
except TypeError:
pass
return response.text
开发者ID:vanstoner,项目名称:xld-mule-mc-plugin,代码行数:48,代码来源:deprecated.py
示例16: procdata_getencoding
def procdata_getencoding(seed,headers,content):
code = utils.get_encoding_from_headers(headers)
if code:
if code.lower() == 'gbk' or code.lower() == 'gb2312':
code = 'gbk'
elif code.lower() == 'utf-8':
code = 'utf-8'
else:
code = None
if code == None:
code = utils.get_encodings_from_content(content)
print "content",seed,code
if code:
code = code[0]
if code.lower() == 'gbk' or code.lower() == 'gb2312':
code = 'gbk'
return code
开发者ID:salmonx,项目名称:fengbei,代码行数:20,代码来源:daemon.py
示例17: filter_encoding
def filter_encoding(self,seed, headers,content):
code = utils.get_encoding_from_headers(headers)
if code:
if code.lower() == 'gbk' or code.lower() == 'gb2312':
code = 'gbk'
return True
elif code.lower() == 'utf-8' or code.lower() == 'utf8':
code = 'utf8'
# as for utf8, we should check the content
else: # 'ISO-8859-1' and so on,
code = None
# chinese website may also miss the content-encoding header, so detect the content
if code == None:
codes = utils.get_encodings_from_content(content)
if codes:
for code in codes:
if code.lower() in [ 'gbk','gb2312']:
return True
elif code.lower() == 'utf8' or code.lower() == 'utf-8':
code = 'utf8'
break
if code != 'utf8':
return False
# here handle utf8
# to detect any chinese char win
try:
ucon = content.decode('utf8')
for uchar in ucon:
i = ord(uchar)
if i >= 0x4e00 and i <= 0x9fa5:
return True
except Exception, e:
print url, e
pass
开发者ID:salmonx,项目名称:fengbei,代码行数:38,代码来源:worker_filter.py
示例18: encoding
def encoding(self):
"""
encoding of Response.content.
if Response.encoding is None, encoding will be guessed
by header or content or chardet if available.
"""
if hasattr(self, '_encoding'):
return self._encoding
# content is unicode
if isinstance(self.content, six.text_type):
return 'unicode'
# Try charset from content-type
encoding = get_encoding_from_headers(self.headers)
if encoding == 'ISO-8859-1':
encoding = None
# Try charset from content
if not encoding and get_encodings_from_content:
if six.PY3:
encoding = get_encodings_from_content(utils.pretty_unicode(self.content[:100]))
else:
encoding = get_encodings_from_content(self.content)
encoding = encoding and encoding[0] or None
# Fallback to auto-detected encoding.
if not encoding and chardet is not None:
encoding = chardet.detect(self.content[:600])['encoding']
if encoding and encoding.lower() == 'gb2312':
encoding = 'gb18030'
self._encoding = encoding or 'utf-8'
return self._encoding
开发者ID:01jiagnwei01,项目名称:pyspider,代码行数:36,代码来源:response.py
示例19: build_response
def build_response(self, req, resp):
response = Response()
response.status_code = resp.status_code
response.headers = CaseInsensitiveDict((k, v) for k, v in resp.items())
response.encoding = get_encoding_from_headers(response.headers)
response.raw = StringIO(resp.content)
response.reason = None
if isinstance(req.url, bytes):
response.url = req.url.decode('utf-8')
else:
response.url = req.url
# Convert from django's SimpleCookie to request's CookieJar
cookiejar_from_dict(resp.cookies, response.cookies)
# context
response.request = req
response.connection = self
response = dispatch_hook('response', req.hooks, response)
return response
开发者ID:miing,项目名称:mci_migo,代码行数:24,代码来源:apiutils.py
示例20: start_response
def start_response(status, headers):
response.status_code = int(status.split(' ')[0])
response.headers = CaseInsensitiveDict(headers)
response.encoding = get_encoding_from_headers(response.headers)
开发者ID:gitter-badger,项目名称:api-star,代码行数:4,代码来源:test.py
注:本文中的requests.utils.get_encoding_from_headers函数示例由纯净天空整理自Github/MSDocs等源码及文档管理平台,相关代码片段筛选自各路编程大神贡献的开源项目,源码版权归原作者所有,传播和使用请参考对应项目的License;未经允许,请勿转载。 |
请发表评论