本文整理汇总了Python中wpull.application.builder.Builder类的典型用法代码示例。如果您正苦于以下问题:Python Builder类的具体用法?Python Builder怎么用?Python Builder使用的例子?那么恭喜您, 这里精选的类代码示例或许可以为您提供帮助。
在下文中一共展示了Builder类的20个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于我们的系统推荐出更棒的Python代码示例。
示例1: test_save_cookie
def test_save_cookie(self):
arg_parser = AppArgumentParser()
with tempfile.NamedTemporaryFile() as in_file:
in_file.write(b"# Kittens\n")
in_file.write(b"localhost.local")
in_file.write(b"\tFALSE\t/\tFALSE\t9999999999\tisloggedin\t1\n")
in_file.write(b"\tFALSE\t/\tFALSE\t\tadmin\t1\n")
in_file.flush()
args = arg_parser.parse_args(
[
self.get_url("/some_page/"),
"--load-cookies",
in_file.name,
"--tries",
"1",
"--save-cookies",
"wpull_test_cookies.txt",
]
)
builder = Builder(args, unit_test=True)
app = builder.build()
exit_code = yield from app.run()
self.assertEqual(0, exit_code)
self.assertEqual(1, builder.factory["Statistics"].files)
with open("wpull_test_cookies.txt", "rb") as saved_file:
cookie_data = saved_file.read()
self.assertIn(b"isloggedin\t1", cookie_data)
self.assertNotIn(b"admin\t1", cookie_data)
开发者ID:pombredanne,项目名称:wpull,代码行数:34,代码来源:http_app_test.py
示例2: test_app_python_plugin_script
def test_app_python_plugin_script(self):
arg_parser = AppArgumentParser()
filename = os.path.join(os.path.dirname(__file__),
'sample_user_scripts', 'extensive.plugin.py')
args = arg_parser.parse_args([
self.get_url('/'),
self.get_url('/some_page'),
self.get_url('/mordor'),
'localhost:1/wolf',
'--plugin-script', filename,
'--page-requisites',
'--reject-regex', '/post/',
'--wait', '12',
'--retry-connrefused', '--tries', '1'
])
builder = Builder(args, unit_test=True)
app = builder.build()
exit_code = yield from app.run()
print(list(os.walk('.')))
self.assertEqual(42, exit_code)
engine = builder.factory['PipelineSeries']
self.assertEqual(2, engine.concurrency)
stats = builder.factory['Statistics']
self.assertEqual(3, stats.files)
# duration should be virtually 0 but account for slowness on travis ci
self.assertGreater(10.0, stats.duration)
开发者ID:Super-Rad,项目名称:wpull,代码行数:32,代码来源:script_test.py
示例3: test_session_cookie
def test_session_cookie(self):
arg_parser = AppArgumentParser()
with tempfile.NamedTemporaryFile() as in_file:
in_file.write(b"# Kittens\n")
in_file.write(b"localhost.local")
# session cookie, Python style
in_file.write(b"\tFALSE\t/\tFALSE\t\ttest\tno\n")
# session cookie, Firefox/Wget/Curl style
in_file.write(b"\tFALSE\t/\tFALSE\t0\tsessionid\tboxcat\n")
in_file.flush()
args = arg_parser.parse_args(
[
self.get_url("/cookie"),
"--load-cookies",
in_file.name,
"--tries",
"1",
"--save-cookies",
"wpull_test_cookies.txt",
"--keep-session-cookies",
]
)
builder = Builder(args, unit_test=True)
app = builder.build()
callback_called = False
def callback(pipeline):
nonlocal callback_called
if callback_called:
return
callback_called = True
self.assertEqual(2, len(builder.factory["CookieJar"]))
app.event_dispatcher.add_listener(Application.Event.pipeline_end, callback)
exit_code = yield from app.run()
self.assertTrue(callback_called)
self.assertEqual(0, exit_code)
self.assertEqual(1, builder.factory["Statistics"].files)
cookies = list(sorted(builder.factory["CookieJar"], key=lambda cookie: cookie.name))
_logger.debug("{0}".format(cookies))
self.assertEqual(2, len(cookies))
self.assertEqual("sessionid", cookies[0].name)
self.assertEqual("boxcat", cookies[0].value)
self.assertEqual("test", cookies[1].name)
self.assertEqual("yes", cookies[1].value)
with open("wpull_test_cookies.txt", "rb") as saved_file:
cookie_data = saved_file.read()
self.assertIn(b"test\tyes", cookie_data)
开发者ID:pombredanne,项目名称:wpull,代码行数:60,代码来源:http_app_test.py
示例4: test_app_args_post_data
def test_app_args_post_data(self):
arg_parser = AppArgumentParser()
args = arg_parser.parse_args([self.get_url("/post/"), "--post-data", "text=hi"])
builder = Builder(args, unit_test=True)
app = builder.build()
exit_code = yield from app.run()
self.assertEqual(0, exit_code)
开发者ID:pombredanne,项目名称:wpull,代码行数:7,代码来源:http_app_test.py
示例5: test_timestamping_hit_orig
def test_timestamping_hit_orig(self):
arg_parser = AppArgumentParser()
args = arg_parser.parse_args([self.get_url("/lastmod"), "--timestamping"])
filename = os.path.join(self.temp_dir.name, "lastmod")
filename_orig = os.path.join(self.temp_dir.name, "lastmod")
with open(filename, "wb") as out_file:
out_file.write(b"HI")
with open(filename_orig, "wb") as out_file:
out_file.write(b"HI")
os.utime(filename_orig, (631152000, 631152000))
builder = Builder(args, unit_test=True)
app = builder.build()
exit_code = yield from app.run()
self.assertEqual(0, exit_code)
with open(filename, "rb") as in_file:
self.assertEqual(b"HI", in_file.read())
with open(filename_orig, "rb") as in_file:
self.assertEqual(b"HI", in_file.read())
开发者ID:pombredanne,项目名称:wpull,代码行数:26,代码来源:http_app_test.py
示例6: main
def main(exit=True, install_tornado_bridge=True, use_signals=True):
if install_tornado_bridge:
tornado.platform.asyncio.AsyncIOMainLoop().install()
arg_parser = AppArgumentParser()
args = arg_parser.parse_args()
builder = Builder(args)
application = builder.build()
if use_signals:
application.setup_signal_handlers()
if args.debug_manhole:
import manhole
import wpull
wpull.wpull_builder = builder
manhole.install()
exit_code = application.run_sync()
if exit:
sys.exit(exit_code)
else:
return exit_code
开发者ID:charygao,项目名称:wpull,代码行数:26,代码来源:main.py
示例7: test_ssl_bad_certificate
def test_ssl_bad_certificate(self):
arg_parser = AppArgumentParser()
args = arg_parser.parse_args([
self.get_url('/'),
'--no-robots',
'--no-check-certificate',
'--tries', '1'
])
builder = Builder(args, unit_test=True)
class MockWebSession(WebSession):
@asyncio.coroutine
def start(self):
raise SSLVerificationError('A very bad certificate!')
class MockWebClient(builder.factory.class_map['WebClient']):
def session(self, request):
return MockWebSession(request, self._http_client, self._redirect_tracker_factory(), Request)
builder.factory.class_map['WebClient'] = MockWebClient
app = builder.build()
exit_code = yield from app.run()
self.assertEqual(7, exit_code)
self.assertEqual(0, builder.factory['Statistics'].files)
开发者ID:Super-Rad,项目名称:wpull,代码行数:26,代码来源:https_test.py
示例8: test_app_args_warc_dedup
def test_app_args_warc_dedup(self):
arg_parser = AppArgumentParser()
with open('dedup.cdx', 'wb') as out_file:
out_file.write(b' CDX a k u\n')
out_file.write(
self.get_url('/static/my_file.txt').encode('ascii')
)
out_file.write(b' KQ4IUKATKL63FT5GMAE2YDRV3WERNL34')
out_file.write(b' <under-the-deer>\n')
args = arg_parser.parse_args([
self.get_url('/static/my_file.txt'),
'--no-parent',
'--warc-file', 'test',
'--no-warc-compression',
'-4',
'--no-robots',
'--warc-dedup', 'dedup.cdx',
])
builder = Builder(args, unit_test=True)
app = builder.build()
exit_code = yield from app.run()
with open('test.warc', 'rb') as in_file:
data = in_file.read()
self.assertIn(b'KQ4IUKATKL63FT5GMAE2YDRV3WERNL34', data)
self.assertIn(b'Type: revisit', data)
self.assertIn(b'<under-the-deer>', data)
self.assertEqual(0, exit_code)
self.assertGreaterEqual(builder.factory['Statistics'].files, 1)
开发者ID:Super-Rad,项目名称:wpull,代码行数:34,代码来源:warc_test.py
示例9: test_app_args_warc
def test_app_args_warc(self):
arg_parser = AppArgumentParser()
args = arg_parser.parse_args([
self.get_url('/'),
'--no-parent',
'--recursive',
'--page-requisites',
'--warc-file', 'test',
'-4',
'--no-robots',
'--no-warc-digests',
])
builder = Builder(args, unit_test=True)
app = builder.build()
exit_code = yield from app.run()
self.assertTrue(os.path.exists('test.warc.gz'))
with gzip.GzipFile('test.warc.gz') as in_file:
data = in_file.read()
self.assertIn(b'FINISHED', data)
self.assertEqual(0, exit_code)
self.assertGreaterEqual(builder.factory['Statistics'].files, 1)
开发者ID:Super-Rad,项目名称:wpull,代码行数:25,代码来源:warc_test.py
示例10: test_big_payload
def test_big_payload(self):
hash_obj = hashlib.sha1(b"foxfoxfox")
payload_list = []
for dummy in range(10000):
data = hash_obj.digest()
hash_obj.update(data)
payload_list.append(data)
data = hash_obj.digest()
payload_list.append(data)
expected_payload = b"".join(payload_list)
arg_parser = AppArgumentParser()
args = arg_parser.parse_args([self.get_url("/big_payload")])
builder = Builder(args, unit_test=True)
app = builder.build()
exit_code = yield from app.run()
self.assertTrue(os.path.exists("big_payload"))
with open("big_payload", "rb") as in_file:
self.assertEqual(expected_payload, in_file.read())
self.assertEqual(0, exit_code)
self.assertEqual(1, builder.factory["Statistics"].files)
开发者ID:pombredanne,项目名称:wpull,代码行数:26,代码来源:http_app_test.py
示例11: test_immediate_robots_forbidden
def test_immediate_robots_forbidden(self):
arg_parser = AppArgumentParser()
args = arg_parser.parse_args([self.get_url("/forbidden"), "--recursive"])
builder = Builder(args, unit_test=True)
app = builder.build()
exit_code = yield from app.run()
self.assertEqual(0, exit_code)
self.assertEqual(0, builder.factory["Statistics"].files)
开发者ID:pombredanne,项目名称:wpull,代码行数:10,代码来源:http_app_test.py
示例12: test_session_timeout
def test_session_timeout(self):
arg_parser = AppArgumentParser()
args = arg_parser.parse_args([self.get_url("/sleep_long"), "--tries=1", "--session-timeout=0.1"])
builder = Builder(args, unit_test=True)
app = builder.build()
exit_code = yield from app.run()
self.assertEqual(4, exit_code)
self.assertEqual(0, builder.factory["Statistics"].files)
开发者ID:pombredanne,项目名称:wpull,代码行数:10,代码来源:http_app_test.py
示例13: test_misc_urls
def test_misc_urls(self):
arg_parser = AppArgumentParser()
args = arg_parser.parse_args(["http://[0:0:0:0:0:ffff:a00:0]/", "--tries", "1", "--timeout", "0.5", "-r"])
builder = Builder(args, unit_test=True)
app = builder.build()
exit_code = yield from app.run()
self.assertEqual(4, exit_code)
开发者ID:pombredanne,项目名称:wpull,代码行数:10,代码来源:http_app_test.py
示例14: test_no_content
def test_no_content(self):
arg_parser = AppArgumentParser()
args = arg_parser.parse_args([self.get_url("/no_content"), "--tries=1"])
builder = Builder(args, unit_test=True)
app = builder.build()
exit_code = yield from app.run()
self.assertEqual(0, exit_code)
self.assertEqual(1, builder.factory["Statistics"].files)
开发者ID:pombredanne,项目名称:wpull,代码行数:10,代码来源:http_app_test.py
示例15: test_non_http_redirect
def test_non_http_redirect(self):
arg_parser = AppArgumentParser()
args = arg_parser.parse_args([self.get_url("/non_http_redirect"), "--recursive", "--no-robots"])
builder = Builder(args, unit_test=True)
app = builder.build()
exit_code = yield from app.run()
self.assertEqual(0, exit_code)
self.assertEqual(0, builder.factory["Statistics"].files)
开发者ID:pombredanne,项目名称:wpull,代码行数:10,代码来源:http_app_test.py
示例16: test_ignore_length
def test_ignore_length(self):
arg_parser = AppArgumentParser()
args = arg_parser.parse_args([self.get_url("/underrun"), "--ignore-length", "--no-robots"])
builder = Builder(args, unit_test=True)
app = builder.build()
exit_code = yield from app.run()
self.assertEqual(0, exit_code)
self.assertEqual(1, builder.factory["Statistics"].files)
开发者ID:pombredanne,项目名称:wpull,代码行数:10,代码来源:http_app_test.py
示例17: test_strip_session_id
def test_strip_session_id(self):
arg_parser = AppArgumentParser()
args = arg_parser.parse_args([self.get_url("/forum/"), "-r", "--strip-session-id"])
builder = Builder(args, unit_test=True)
app = builder.build()
exit_code = yield from app.run()
self.assertEqual(0, exit_code)
self.assertEqual(1, builder.factory["Statistics"].files)
开发者ID:pombredanne,项目名称:wpull,代码行数:10,代码来源:http_app_test.py
示例18: test_referer_option
def test_referer_option(self):
arg_parser = AppArgumentParser()
args = arg_parser.parse_args([self.get_url("/referrer/"), "-r", "--referer", "http://left.shark/"])
builder = Builder(args, unit_test=True)
app = builder.build()
exit_code = yield from app.run()
self.assertEqual(0, exit_code)
self.assertEqual(2, builder.factory["Statistics"].files)
开发者ID:pombredanne,项目名称:wpull,代码行数:10,代码来源:http_app_test.py
示例19: test_basic_auth_fail
def test_basic_auth_fail(self):
arg_parser = AppArgumentParser()
args = arg_parser.parse_args([self.get_url("/basic_auth"), "--user", "root", "--password", "toothless"])
builder = Builder(args, unit_test=True)
app = builder.build()
exit_code = yield from app.run()
self.assertEqual(0, exit_code)
self.assertEqual(0, builder.factory["Statistics"].files)
开发者ID:pombredanne,项目名称:wpull,代码行数:10,代码来源:http_app_test.py
示例20: test_escaped_fragment_recursive
def test_escaped_fragment_recursive(self):
arg_parser = AppArgumentParser()
args = arg_parser.parse_args([self.get_url("/escape_from_fragments/"), "-r", "--escaped-fragment"])
builder = Builder(args, unit_test=True)
app = builder.build()
exit_code = yield from app.run()
self.assertEqual(0, exit_code)
self.assertEqual(2, builder.factory["Statistics"].files)
开发者ID:pombredanne,项目名称:wpull,代码行数:10,代码来源:http_app_test.py
注:本文中的wpull.application.builder.Builder类示例由纯净天空整理自Github/MSDocs等源码及文档管理平台,相关代码片段筛选自各路编程大神贡献的开源项目,源码版权归原作者所有,传播和使用请参考对应项目的License;未经允许,请勿转载。 |
请发表评论