本文整理汇总了Python中mrjob.inline.InlineMRJobRunner类的典型用法代码示例。如果您正苦于以下问题:Python InlineMRJobRunner类的具体用法?Python InlineMRJobRunner怎么用?Python InlineMRJobRunner使用的例子?那么恭喜您, 这里精选的类代码示例或许可以为您提供帮助。
在下文中一共展示了InlineMRJobRunner类的20个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于我们的系统推荐出更棒的Python代码示例。
示例1: test_owner_and_label_kwargs
def test_owner_and_label_kwargs(self):
runner = InlineMRJobRunner(conf_paths=[],
owner='ads', label='ads_chain')
match = _JOB_KEY_RE.match(runner.get_job_key())
self.assertEqual(match.group(1), 'ads_chain')
self.assertEqual(match.group(2), 'ads')
开发者ID:anirudhreddy92,项目名称:mrjob,代码行数:7,代码来源:test_runner.py
示例2: test_auto_owner
def test_auto_owner(self):
os.environ['USER'] = 'mcp'
runner = InlineMRJobRunner(conf_paths=[])
match = _JOB_KEY_RE.match(runner.get_job_key())
self.assertEqual(match.group(1), 'no_script')
self.assertEqual(match.group(2), 'mcp')
开发者ID:anirudhreddy92,项目名称:mrjob,代码行数:7,代码来源:test_runner.py
示例3: test_owner_and_label_kwargs
def test_owner_and_label_kwargs(self):
runner = InlineMRJobRunner(conf_path=False,
owner='ads', label='ads_chain')
match = JOB_NAME_RE.match(runner.get_job_name())
self.assertEqual(match.group(1), 'ads_chain')
self.assertEqual(match.group(2), 'ads')
开发者ID:BrandonHaynes,项目名称:mrjob,代码行数:7,代码来源:test_runner.py
示例4: test_empty_no_user
def test_empty_no_user(self):
self.getuser_should_fail = True
runner = InlineMRJobRunner(conf_paths=[])
match = JOB_NAME_RE.match(runner.get_job_name())
self.assertEqual(match.group(1), "no_script")
self.assertEqual(match.group(2), "no_user")
开发者ID:pyzen,项目名称:mrjob,代码行数:7,代码来源:test_runner.py
示例5: test_stream_output
def test_stream_output(self):
a_dir_path = os.path.join(self.tmp_dir, 'a')
b_dir_path = os.path.join(self.tmp_dir, 'b')
l_dir_path = os.path.join(self.tmp_dir, '_logs')
os.mkdir(a_dir_path)
os.mkdir(b_dir_path)
os.mkdir(l_dir_path)
a_file_path = os.path.join(a_dir_path, 'part-00000')
b_file_path = os.path.join(b_dir_path, 'part-00001')
c_file_path = os.path.join(self.tmp_dir, 'part-00002')
x_file_path = os.path.join(l_dir_path, 'log.xml')
y_file_path = os.path.join(self.tmp_dir, '_SUCCESS')
with open(a_file_path, 'w') as f:
f.write('A')
with open(b_file_path, 'w') as f:
f.write('B')
with open(c_file_path, 'w') as f:
f.write('C')
with open(x_file_path, 'w') as f:
f.write('<XML XML XML/>')
with open(y_file_path, 'w') as f:
f.write('I win')
runner = InlineMRJobRunner(conf_paths=[], output_dir=self.tmp_dir)
self.assertEqual(sorted(runner.stream_output()),
[b'A', b'B', b'C'])
开发者ID:anirudhreddy92,项目名称:mrjob,代码行数:32,代码来源:test_runner.py
示例6: test_stream_output
def test_stream_output(self):
a_dir_path = os.path.join(self.tmp_dir, "a")
b_dir_path = os.path.join(self.tmp_dir, "b")
l_dir_path = os.path.join(self.tmp_dir, "_logs")
os.mkdir(a_dir_path)
os.mkdir(b_dir_path)
os.mkdir(l_dir_path)
a_file_path = os.path.join(a_dir_path, "part-00000")
b_file_path = os.path.join(b_dir_path, "part-00001")
c_file_path = os.path.join(self.tmp_dir, "part-00002")
x_file_path = os.path.join(l_dir_path, "log.xml")
y_file_path = os.path.join(self.tmp_dir, "_SUCCESS")
with open(a_file_path, "w") as f:
f.write("A")
with open(b_file_path, "w") as f:
f.write("B")
with open(c_file_path, "w") as f:
f.write("C")
with open(x_file_path, "w") as f:
f.write("<XML XML XML/>")
with open(y_file_path, "w") as f:
f.write("I win")
runner = InlineMRJobRunner(conf_paths=[], output_dir=self.tmp_dir)
self.assertEqual(sorted(runner.stream_output()), ["A", "B", "C"])
开发者ID:pyzen,项目名称:mrjob,代码行数:31,代码来源:test_runner.py
示例7: test_auto_owner
def test_auto_owner(self):
os.environ['USER'] = 'mcp'
runner = InlineMRJobRunner(conf_path=False)
match = JOB_NAME_RE.match(runner.get_job_name())
self.assertEqual(match.group(1), 'no_script')
self.assertEqual(match.group(2), 'mcp')
开发者ID:BrandonHaynes,项目名称:mrjob,代码行数:7,代码来源:test_runner.py
示例8: test_getattr_forward
def test_getattr_forward(self):
with no_handlers_for_logger():
r = InlineMRJobRunner(conf_path=False)
store = r._opts
self.assertIsInstance(store, InlineRunnerOptionStore)
a = r.get_default_opts()
self.assertEqual(a, store.default_options())
开发者ID:icio,项目名称:mrjob,代码行数:7,代码来源:test_conf.py
示例9: test_doesnt_actually_create_archive
def test_doesnt_actually_create_archive(self):
archive_dir = self.makedirs('archive')
runner = InlineMRJobRunner()
archive_path = runner._dir_archive_path(archive_dir)
self.assertFalse(os.path.exists(archive_path))
开发者ID:okomestudio,项目名称:mrjob,代码行数:7,代码来源:test_runner.py
示例10: test_auto_owner
def test_auto_owner(self):
os.environ["USER"] = "mcp"
runner = InlineMRJobRunner(conf_paths=[])
match = JOB_NAME_RE.match(runner.get_job_name())
self.assertEqual(match.group(1), "no_script")
self.assertEqual(match.group(2), "mcp")
开发者ID:pyzen,项目名称:mrjob,代码行数:7,代码来源:test_runner.py
示例11: test_empty_no_user
def test_empty_no_user(self):
self.getuser_should_fail = True
runner = InlineMRJobRunner(conf_paths=[])
match = _JOB_KEY_RE.match(runner.get_job_key())
self.assertEqual(match.group(1), 'no_script')
self.assertEqual(match.group(2), 'no_user')
开发者ID:anirudhreddy92,项目名称:mrjob,代码行数:7,代码来源:test_runner.py
示例12: test_trailing_slash
def test_trailing_slash(self):
archive_dir = self.makedirs('archive') + os.sep
runner = InlineMRJobRunner()
archive_path = runner._dir_archive_path(archive_dir)
self.assertEqual(os.path.basename(archive_path), 'archive.tar.gz')
开发者ID:okomestudio,项目名称:mrjob,代码行数:7,代码来源:test_runner.py
示例13: test_same_dir_twice
def test_same_dir_twice(self):
archive_dir = self.makedirs('archive')
runner = InlineMRJobRunner()
archive_path_1 = runner._dir_archive_path(archive_dir)
archive_path_2 = runner._dir_archive_path(archive_dir)
self.assertEqual(os.path.basename(archive_path_1), 'archive.tar.gz')
self.assertEqual(archive_path_1, archive_path_2)
开发者ID:okomestudio,项目名称:mrjob,代码行数:9,代码来源:test_runner.py
示例14: UpdateJobConfForHadoopVersionTestCase
class UpdateJobConfForHadoopVersionTestCase(TestCase):
# jobconf with strange mix of Hadoop 1 and Hadoop 2 variables
JOBCONF = {
'foo.bar': 'baz', # unknown jobconf
'mapred.jar': 'a.jar', # Hadoop 1 jobconf
'mapreduce.job.user.name': 'dave', # Hadoop 2 jobconf
}
def setUp(self):
self.runner = InlineMRJobRunner(conf_paths=[])
def updated_and_warnings(self, jobconf, hadoop_version):
jobconf = jobconf.copy()
with no_handlers_for_logger('mrjob.runner'):
stderr = StringIO()
log_to_stream('mrjob.runner', stderr)
self.runner._update_jobconf_for_hadoop_version(
jobconf, hadoop_version)
return jobconf, stderr.getvalue()
def test_no_version(self):
updated, warnings = self.updated_and_warnings(
self.JOBCONF, None)
self.assertEqual(updated, self.JOBCONF)
self.assertEqual(warnings, '')
def test_hadoop_1(self):
updated, warnings = self.updated_and_warnings(
self.JOBCONF, '1.0')
self.assertEqual(updated,
combine_dicts(self.JOBCONF, {'user.name': 'dave'}))
self.assertIn('do not match hadoop version', warnings)
self.assertIn('mapreduce.job.user.name: user.name', warnings)
def test_hadoop_2(self):
updated, warnings = self.updated_and_warnings(
self.JOBCONF, '2.0')
self.assertEqual(updated,
combine_dicts(self.JOBCONF,
{'mapreduce.job.jar': 'a.jar'}))
self.assertIn('do not match hadoop version', warnings)
self.assertIn('mapred.jar: mapreduce.job.jar', warnings)
def test_dont_overwrite(self):
# this jobconf contains two versions of the same variable
jobconf = {'mapred.jar': 'a.jar', 'mapreduce.job.jar': 'b.jar'}
updated, warnings = self.updated_and_warnings(jobconf, '1.0')
self.assertEqual(updated, jobconf)
self.assertEqual(warnings, '')
开发者ID:Milkigit,项目名称:mrjob,代码行数:56,代码来源:test_runner.py
示例15: test_mrjob_zip_compiles
def test_mrjob_zip_compiles(self):
runner = InlineMRJobRunner()
with no_handlers_for_logger('mrjob.runner'):
mrjob_zip = runner._create_mrjob_zip()
ZipFile(mrjob_zip).extractall(self.tmp_dir)
self.assertTrue(
compileall.compile_dir(os.path.join(self.tmp_dir, 'mrjob'),
quiet=1))
开发者ID:okomestudio,项目名称:mrjob,代码行数:10,代码来源:test_runner.py
示例16: test_output_dir_not_considered_hidden
def test_output_dir_not_considered_hidden(self):
output_dir = os.path.join(self.tmp_dir, '_hidden', '_output_dir')
self.makefile(os.path.join(output_dir, 'part-00000'),
b'cats\n')
runner = InlineMRJobRunner(conf_paths=[], output_dir=output_dir)
self.assertEqual(sorted(to_lines(runner.stream_output())),
[b'cats\n'])
开发者ID:Affirm,项目名称:mrjob,代码行数:10,代码来源:test_runner.py
示例17: test_dirs_with_same_name
def test_dirs_with_same_name(self):
foo_archive = self.makedirs(os.path.join('foo', 'archive'))
bar_archive = self.makedirs(os.path.join('bar', 'archive'))
runner = InlineMRJobRunner()
foo_archive_path = runner._dir_archive_path(foo_archive)
bar_archive_path = runner._dir_archive_path(bar_archive)
self.assertEqual(os.path.basename(foo_archive_path),
'archive.tar.gz')
self.assertNotEqual(foo_archive_path, bar_archive_path)
开发者ID:okomestudio,项目名称:mrjob,代码行数:11,代码来源:test_runner.py
示例18: test_empty_dir
def test_empty_dir(self):
runner = InlineMRJobRunner()
empty_dir = self.makedirs('empty')
tar_gz_path = runner._dir_archive_path(empty_dir)
self.assertEqual(os.path.basename(tar_gz_path), 'empty.tar.gz')
runner._create_dir_archive(empty_dir)
with tarfile.open(tar_gz_path, 'r:gz') as tar_gz:
self.assertEqual(sorted(tar_gz.getnames()), [])
开发者ID:okomestudio,项目名称:mrjob,代码行数:12,代码来源:test_runner.py
示例19: test_archive
def test_archive(self):
runner = InlineMRJobRunner()
tar_gz_path = runner._dir_archive_path(self._to_archive)
self.assertEqual(os.path.basename(tar_gz_path), 'archive.tar.gz')
runner._create_dir_archive(self._to_archive)
tar_gz = tarfile.open(tar_gz_path, 'r:gz')
try:
self.assertEqual(sorted(tar_gz.getnames()),
[os.path.join('bar', 'baz'), 'foo'])
finally:
tar_gz.close()
开发者ID:okomestudio,项目名称:mrjob,代码行数:14,代码来源:test_runner.py
示例20: test_deprecated_stream_output
def test_deprecated_stream_output(self):
self.makefile('part-00000', contents=b'1\n2')
self.makefile('part-00001', contents=b'3\n4\n')
runner = InlineMRJobRunner(conf_paths=[], output_dir=self.tmp_dir)
log = self.start(patch('mrjob.runner.log'))
# should group output into lines, but not join across files
self.assertEqual(sorted(runner.stream_output()),
[b'1\n', b'2', b'3\n', b'4\n'])
# should issue deprecation warning
self.assertEqual(log.warning.call_count, 1)
开发者ID:okomestudio,项目名称:mrjob,代码行数:14,代码来源:test_runner.py
注:本文中的mrjob.inline.InlineMRJobRunner类示例由纯净天空整理自Github/MSDocs等源码及文档管理平台,相关代码片段筛选自各路编程大神贡献的开源项目,源码版权归原作者所有,传播和使用请参考对应项目的License;未经允许,请勿转载。 |
请发表评论