本文整理汇总了Python中mrjob.conf.dump_mrjob_conf函数的典型用法代码示例。如果您正苦于以下问题：Python dump_mrjob_conf函数的具体用法？Python dump_mrjob_conf怎么用？Python dump_mrjob_conf使用的例子？那么恭喜您, 这里精选的函数代码示例或许可以为您提供帮助。
在下文中一共展示了dump_mrjob_conf函数的20个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于我们的系统推荐出更棒的Python代码示例。
示例1: test_round_trip
def test_round_trip(self):
conf = {'runners': {'foo': {'qux': 'quux'}}}
conf_path = os.path.join(self.tmp_dir, 'mrjob.conf')
dump_mrjob_conf(conf, open(conf_path, 'w'))
with no_handlers_for_logger('mrjob.conf'):
self.assertEqual(conf, load_mrjob_conf(conf_path=conf_path))
开发者ID:icio，项目名称:mrjob，代码行数:7，代码来源:test_conf.py
示例2: test_duplicate_conf_path
def test_duplicate_conf_path(self):
conf_path = os.path.join(self.tmp_dir, "mrjob.conf")
with open(conf_path, "w") as f:
dump_mrjob_conf({}, f)
self.assertEqual(load_opts_from_mrjob_confs("foo", [conf_path, conf_path]), [(conf_path, {})])
开发者ID:kartheek6，项目名称:mrjob，代码行数:7，代码来源:test_conf.py
示例3: test_include_relative_to_real_path
def test_include_relative_to_real_path(self):
os.mkdir(os.path.join(self.tmp_dir, 'conf'))
base_conf_path = os.path.join(self.tmp_dir, 'conf', 'mrjob.base.conf')
real_base_conf_path = os.path.realpath(base_conf_path)
conf_path = os.path.join(self.tmp_dir, 'conf', 'mrjob.conf')
conf_symlink_path = os.path.join(self.tmp_dir, 'mrjob.conf')
with open(base_conf_path, 'w') as f:
dump_mrjob_conf({}, f)
with open(conf_path, 'w') as f:
dump_mrjob_conf({'include': 'mrjob.base.conf'}, f)
os.symlink(os.path.join('conf', 'mrjob.conf'), conf_symlink_path)
self.assertEqual(
load_opts_from_mrjob_conf('foo', conf_path),
[(real_base_conf_path, {}), (conf_path, {})])
# relative include should work from the symlink even though
# it's not in the same directory as mrjob.base.conf
self.assertEqual(
load_opts_from_mrjob_conf('foo', conf_symlink_path),
[(real_base_conf_path, {}), (conf_symlink_path, {})])
开发者ID:Affirm，项目名称:mrjob，代码行数:26，代码来源:test_conf.py
示例4: test_can_turn_off_bootstrap_mrjob
def test_can_turn_off_bootstrap_mrjob(self):
# track the dir we're loading mrjob from rather than the full path
# to deal with edge cases where we load from the .py file,
# and the script loads from the .pyc compiled from that .py file.
our_mrjob_dir = os.path.dirname(os.path.realpath(mrjob.__file__))
self.mrjob_conf_path = os.path.join(self.tmp_dir, 'mrjob.conf')
dump_mrjob_conf({'runners': {'local': {'bootstrap_mrjob': False}}},
open(self.mrjob_conf_path, 'w'))
mr_job = MRJobWhereAreYou(['-c', self.mrjob_conf_path])
mr_job.sandbox()
with mr_job.make_runner() as runner:
# sanity check
self.assertEqual(runner.get_opts()['bootstrap_mrjob'], False)
runner.run()
output = list(runner.stream_output())
self.assertEqual(len(output), 1)
# script should load mrjob from the same place our test does
_, script_mrjob_dir = mr_job.parse_output_line(output[0])
self.assertEqual(our_mrjob_dir, script_mrjob_dir)
开发者ID:vdt，项目名称:mrjob，代码行数:25，代码来源:test_local.py
示例5: make_mrjob_conf
def make_mrjob_conf(self):
_, self.mrjob_conf_path = tempfile.mkstemp(prefix='mrjob.conf.')
dump_mrjob_conf({'runners': {'emr': {
'check_emr_status_every': 0.01,
's3_scratch_uri': 's3://walrus/tmp',
's3_sync_wait_time': 0.01,
}}}, open(self.mrjob_conf_path, 'w'))
开发者ID:boursier，项目名称:mrjob，代码行数:7，代码来源:emr_test.py
示例6: make_tmp_dir_and_mrjob_conf
def make_tmp_dir_and_mrjob_conf(self):
self.tmp_dir = tempfile.mkdtemp()
self.mrjob_conf_path = os.path.join(self.tmp_dir, 'mrjob.conf')
dump_mrjob_conf({'runners': {'emr': {
'check_emr_status_every': 0.01,
's3_sync_wait_time': 0.01,
}}}, open(self.mrjob_conf_path, 'w'))
开发者ID:boursier，项目名称:mrjob，代码行数:7，代码来源:emr_test.py
示例7: _test_round_trip
def _test_round_trip(self, conf):
conf_path = os.path.join(self.tmp_dir, "mrjob.conf")
with open(conf_path, "w") as f:
dump_mrjob_conf(conf, f)
with no_handlers_for_logger("mrjob.conf"):
self.assertEqual(conf, load_mrjob_conf(conf_path=conf_path))
开发者ID:kartheek6，项目名称:mrjob，代码行数:7，代码来源:test_conf.py
示例8: _test_round_trip
def _test_round_trip(self, conf):
conf_path = os.path.join(self.tmp_dir, 'mrjob.conf')
with open(conf_path, 'w') as f:
dump_mrjob_conf(conf, f)
self.assertEqual(conf, load_mrjob_conf(conf_path=conf_path))
开发者ID:Affirm，项目名称:mrjob，代码行数:7，代码来源:test_conf.py
示例9: test_recursive_include
def test_recursive_include(self):
conf_path = os.path.join(self.tmp_dir, 'mrjob.conf')
with open(conf_path, 'w') as f:
dump_mrjob_conf({'include': conf_path}, f)
self.assertEqual(
load_opts_from_mrjob_conf('foo', conf_path),
[(conf_path, {})])
开发者ID:Affirm，项目名称:mrjob，代码行数:8，代码来源:test_conf.py
示例10: test_using_json_and_not_yaml
def test_using_json_and_not_yaml(self):
conf = {"runners": {"foo": {"qux": "quux"}}}
conf_path = os.path.join(self.tmp_dir, "mrjob.conf")
dump_mrjob_conf(conf, open(conf_path, "w"))
with open(conf_path) as f:
contents = f.read()
self.assertEqual(contents.replace(" ", "").replace("\n", ""), '{"runners":{"foo":{"qux":"quux"}}}')
开发者ID:nyccto，项目名称:mrjob，代码行数:9，代码来源:test_conf.py
示例11: make_tmp_dir_and_mrjob_conf
def make_tmp_dir_and_mrjob_conf(self):
self.tmp_dir = tempfile.mkdtemp()
self.mrjob_conf_path = os.path.join(self.tmp_dir, 'mrjob.conf')
dump_mrjob_conf({'runners': {'emr': {
'check_emr_status_every': 0.01,
's3_sync_wait_time': 0.01,
'aws_availability_zone': 'PUPPYLAND',
'additional_emr_info': {'key': 'value'},
}}}, open(self.mrjob_conf_path, 'w'))
开发者ID:Jyrsa，项目名称:mrjob，代码行数:9，代码来源:emr_test.py
示例12: test_using_json_and_not_yaml
def test_using_json_and_not_yaml(self):
conf = {'runners': {'foo': {'qux': 'quux'}}}
conf_path = os.path.join(self.tmp_dir, 'mrjob.conf')
dump_mrjob_conf(conf, open(conf_path, 'w'))
contents = open(conf_path).read()
assert_equal(contents.replace(' ', '').replace('\n', ''),
'{"runners":{"foo":{"qux":"quux"}}}')
开发者ID:gimlids，项目名称:LTPM，代码行数:9，代码来源:conf_test.py
示例13: test_doubly_recursive_include
def test_doubly_recursive_include(self):
conf_path_1 = os.path.join(self.tmp_dir, "mrjob.1.conf")
conf_path_2 = os.path.join(self.tmp_dir, "mrjob.2.conf")
with open(conf_path_1, "w") as f:
dump_mrjob_conf({"include": conf_path_2}, f)
with open(conf_path_2, "w") as f:
dump_mrjob_conf({"include": conf_path_1}, f)
self.assertEqual(load_opts_from_mrjob_conf("foo", conf_path_1), [(conf_path_2, {}), (conf_path_1, {})])
开发者ID:kartheek6，项目名称:mrjob，代码行数:11，代码来源:test_conf.py
示例14: test_recurse
def test_recurse(self):
path = os.path.join(self.tmp_dir, 'LOL.conf')
recurse_conf = dict(include=path)
with open(path, 'w') as f:
dump_mrjob_conf(recurse_conf, f)
stderr = StringIO()
with no_handlers_for_logger():
log_to_stream('mrjob.conf', stderr)
InlineMRJobRunner(conf_path=path)
self.assertIn('%s tries to recursively include %s!' % (path, path),
stderr.getvalue())
开发者ID:nyccto，项目名称:mrjob，代码行数:12，代码来源:test_runner.py
示例15: test_relative_include
def test_relative_include(self):
base_conf_path = os.path.join(self.tmp_dir, "mrjob.base.conf")
real_base_conf_path = os.path.realpath(base_conf_path)
conf_path = os.path.join(self.tmp_dir, "mrjob.conf")
with open(base_conf_path, "w") as f:
dump_mrjob_conf({}, f)
with open(conf_path, "w") as f:
dump_mrjob_conf({"include": "mrjob.base.conf"}, f)
self.assertEqual(load_opts_from_mrjob_conf("foo", conf_path), [(real_base_conf_path, {}), (conf_path, {})])
开发者ID:kartheek6，项目名称:mrjob，代码行数:13，代码来源:test_conf.py
示例16: test_doubly_recursive_include
def test_doubly_recursive_include(self):
conf_path_1 = os.path.join(self.tmp_dir, 'mrjob.1.conf')
conf_path_2 = os.path.join(self.tmp_dir, 'mrjob.2.conf')
with open(conf_path_1, 'w') as f:
dump_mrjob_conf({'include': conf_path_2}, f)
with open(conf_path_2, 'w') as f:
dump_mrjob_conf({'include': conf_path_1}, f)
self.assertEqual(
load_opts_from_mrjob_conf('foo', conf_path_1),
[(conf_path_2, {}), (conf_path_1, {})])
开发者ID:Affirm，项目名称:mrjob，代码行数:13，代码来源:test_conf.py
示例17: test_conf_path_order_beats_include
def test_conf_path_order_beats_include(self):
conf_path_1 = os.path.join(self.tmp_dir, 'mrjob.1.conf')
conf_path_2 = os.path.join(self.tmp_dir, 'mrjob.2.conf')
with open(conf_path_1, 'w') as f:
dump_mrjob_conf({}, f)
with open(conf_path_2, 'w') as f:
dump_mrjob_conf({}, f)
# shouldn't matter that conf_path_1 includes conf_path_2
self.assertEqual(
load_opts_from_mrjob_confs('foo', [conf_path_1, conf_path_2]),
[(conf_path_1, {}), (conf_path_2, {})])
开发者ID:Affirm，项目名称:mrjob，代码行数:14，代码来源:test_conf.py
示例18: test_nested_include
def test_nested_include(self):
conf_path = os.path.join(self.tmp_dir, 'mrjob.conf')
conf_path_1 = os.path.join(self.tmp_dir, 'mrjob.1.conf')
conf_path_2 = os.path.join(self.tmp_dir, 'mrjob.2.conf')
conf_path_3 = os.path.join(self.tmp_dir, 'mrjob.3.conf')
# accidentally reversed the order of nested includes when
# trying to make precedence work; this test would catch that
with open(conf_path, 'w') as f:
dump_mrjob_conf({'include': conf_path_1}, f)
with open(conf_path_1, 'w') as f:
dump_mrjob_conf({'include': [conf_path_2, conf_path_3]}, f)
with open(conf_path_2, 'w') as f:
dump_mrjob_conf({}, f)
with open(conf_path_3, 'w') as f:
dump_mrjob_conf({}, f)
self.assertEqual(
load_opts_from_mrjob_conf('foo', conf_path),
[(conf_path_2, {}),
(conf_path_3, {}),
(conf_path_1, {}),
(conf_path, {})])
开发者ID:Affirm，项目名称:mrjob，代码行数:27，代码来源:test_conf.py
示例19: test_relative_include
def test_relative_include(self):
base_conf_path = os.path.join(self.tmp_dir, 'mrjob.base.conf')
real_base_conf_path = os.path.realpath(base_conf_path)
conf_path = os.path.join(self.tmp_dir, 'mrjob.conf')
with open(base_conf_path, 'w') as f:
dump_mrjob_conf({}, f)
with open(conf_path, 'w') as f:
dump_mrjob_conf({'include': 'mrjob.base.conf'}, f)
self.assertEqual(
load_opts_from_mrjob_conf('foo', conf_path),
[(real_base_conf_path, {}), (conf_path, {})])
开发者ID:Affirm，项目名称:mrjob，代码行数:15，代码来源:test_conf.py
示例20: test_multiple_config_files
def test_multiple_config_files(self):
tmp_dir = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, tmp_dir)
path_left = os.path.join(tmp_dir, 'left.yaml')
path_right = os.path.join(tmp_dir, 'right.yaml')
with open(path_left, 'w') as f:
dump_mrjob_conf({'runners': {'inline': {'jobconf': {'x': 1}}}}, f)
with open(path_right, 'w') as f:
dump_mrjob_conf({'runners': {'inline': {'jobconf': {'y': 2}}}}, f)
mr_job = MRCustomBoringJob(args=['-r', 'inline',
'-c', path_left, '-c', path_right])
with mr_job.make_runner() as r:
self.assertEqual(r._opts['jobconf']['x'], 1)
self.assertEqual(r._opts['jobconf']['y'], 2)
开发者ID:AnthonyNystrom，项目名称:mrjob，代码行数:15，代码来源:test_job.py
注：本文中的mrjob.conf.dump_mrjob_conf函数示例由纯净天空整理自Github/MSDocs等源码及文档管理平台，相关代码片段筛选自各路编程大神贡献的开源项目，源码版权归原作者所有，传播和使用请参考对应项目的License；未经允许，请勿转载。 |
客服电话
APP下载
官方微信
请发表评论