• 设为首页
  • 点击收藏
  • 手机版
    手机扫一扫访问
    迪恩网络手机版
  • 关注官方公众号
    微信扫一扫关注
    迪恩网络公众号
登陆 注册
描述文字

Python inline.InlineMRJobRunner类代码示例

原作者: [db:作者] 来自: [db:来源] 收藏 邀请

本文整理汇总了Python中mrjob.inline.InlineMRJobRunner类的典型用法代码示例。如果您正苦于以下问题：Python InlineMRJobRunner类的具体用法？Python InlineMRJobRunner怎么用？Python InlineMRJobRunner使用的例子？那么恭喜您, 这里精选的类代码示例或许可以为您提供帮助。



在下文中一共展示了InlineMRJobRunner类的20个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于我们的系统推荐出更棒的Python代码示例。

示例1: test_owner_and_label_kwargs

    def test_owner_and_label_kwargs(self):
        runner = InlineMRJobRunner(conf_paths=[],
                                   owner='ads', label='ads_chain')
        match = _JOB_KEY_RE.match(runner.get_job_key())

        self.assertEqual(match.group(1), 'ads_chain')
        self.assertEqual(match.group(2), 'ads')
开发者ID:anirudhreddy92，项目名称:mrjob，代码行数:7，代码来源:test_runner.py


示例2: test_auto_owner

    def test_auto_owner(self):
        os.environ['USER'] = 'mcp'
        runner = InlineMRJobRunner(conf_paths=[])
        match = _JOB_KEY_RE.match(runner.get_job_key())

        self.assertEqual(match.group(1), 'no_script')
        self.assertEqual(match.group(2), 'mcp')
开发者ID:anirudhreddy92，项目名称:mrjob，代码行数:7，代码来源:test_runner.py


示例3: test_owner_and_label_kwargs

    def test_owner_and_label_kwargs(self):
        runner = InlineMRJobRunner(conf_path=False,
                                  owner='ads', label='ads_chain')
        match = JOB_NAME_RE.match(runner.get_job_name())

        self.assertEqual(match.group(1), 'ads_chain')
        self.assertEqual(match.group(2), 'ads')
开发者ID:BrandonHaynes，项目名称:mrjob，代码行数:7，代码来源:test_runner.py


示例4: test_empty_no_user

    def test_empty_no_user(self):
        self.getuser_should_fail = True
        runner = InlineMRJobRunner(conf_paths=[])
        match = JOB_NAME_RE.match(runner.get_job_name())

        self.assertEqual(match.group(1), "no_script")
        self.assertEqual(match.group(2), "no_user")
开发者ID:pyzen，项目名称:mrjob，代码行数:7，代码来源:test_runner.py


示例5: test_stream_output

    def test_stream_output(self):
        a_dir_path = os.path.join(self.tmp_dir, 'a')
        b_dir_path = os.path.join(self.tmp_dir, 'b')
        l_dir_path = os.path.join(self.tmp_dir, '_logs')
        os.mkdir(a_dir_path)
        os.mkdir(b_dir_path)
        os.mkdir(l_dir_path)

        a_file_path = os.path.join(a_dir_path, 'part-00000')
        b_file_path = os.path.join(b_dir_path, 'part-00001')
        c_file_path = os.path.join(self.tmp_dir, 'part-00002')
        x_file_path = os.path.join(l_dir_path, 'log.xml')
        y_file_path = os.path.join(self.tmp_dir, '_SUCCESS')

        with open(a_file_path, 'w') as f:
            f.write('A')

        with open(b_file_path, 'w') as f:
            f.write('B')

        with open(c_file_path, 'w') as f:
            f.write('C')

        with open(x_file_path, 'w') as f:
            f.write('<XML XML XML/>')

        with open(y_file_path, 'w') as f:
            f.write('I win')

        runner = InlineMRJobRunner(conf_paths=[], output_dir=self.tmp_dir)
        self.assertEqual(sorted(runner.stream_output()),
                         [b'A', b'B', b'C'])
开发者ID:anirudhreddy92，项目名称:mrjob，代码行数:32，代码来源:test_runner.py


示例6: test_stream_output

    def test_stream_output(self):
        a_dir_path = os.path.join(self.tmp_dir, "a")
        b_dir_path = os.path.join(self.tmp_dir, "b")
        l_dir_path = os.path.join(self.tmp_dir, "_logs")
        os.mkdir(a_dir_path)
        os.mkdir(b_dir_path)
        os.mkdir(l_dir_path)

        a_file_path = os.path.join(a_dir_path, "part-00000")
        b_file_path = os.path.join(b_dir_path, "part-00001")
        c_file_path = os.path.join(self.tmp_dir, "part-00002")
        x_file_path = os.path.join(l_dir_path, "log.xml")
        y_file_path = os.path.join(self.tmp_dir, "_SUCCESS")

        with open(a_file_path, "w") as f:
            f.write("A")

        with open(b_file_path, "w") as f:
            f.write("B")

        with open(c_file_path, "w") as f:
            f.write("C")

        with open(x_file_path, "w") as f:
            f.write("<XML XML XML/>")

        with open(y_file_path, "w") as f:
            f.write("I win")

        runner = InlineMRJobRunner(conf_paths=[], output_dir=self.tmp_dir)
        self.assertEqual(sorted(runner.stream_output()), ["A", "B", "C"])
开发者ID:pyzen，项目名称:mrjob，代码行数:31，代码来源:test_runner.py


示例7: test_auto_owner

    def test_auto_owner(self):
        os.environ['USER'] = 'mcp'
        runner = InlineMRJobRunner(conf_path=False)
        match = JOB_NAME_RE.match(runner.get_job_name())

        self.assertEqual(match.group(1), 'no_script')
        self.assertEqual(match.group(2), 'mcp')
开发者ID:BrandonHaynes，项目名称:mrjob，代码行数:7，代码来源:test_runner.py


示例8: test_getattr_forward

 def test_getattr_forward(self):
     with no_handlers_for_logger():
         r = InlineMRJobRunner(conf_path=False)
     store = r._opts
     self.assertIsInstance(store, InlineRunnerOptionStore)
     a = r.get_default_opts()
     self.assertEqual(a, store.default_options())
开发者ID:icio，项目名称:mrjob，代码行数:7，代码来源:test_conf.py


示例9: test_doesnt_actually_create_archive

    def test_doesnt_actually_create_archive(self):
        archive_dir = self.makedirs('archive')

        runner = InlineMRJobRunner()
        archive_path = runner._dir_archive_path(archive_dir)

        self.assertFalse(os.path.exists(archive_path))
开发者ID:okomestudio，项目名称:mrjob，代码行数:7，代码来源:test_runner.py


示例10: test_auto_owner

    def test_auto_owner(self):
        os.environ["USER"] = "mcp"
        runner = InlineMRJobRunner(conf_paths=[])
        match = JOB_NAME_RE.match(runner.get_job_name())

        self.assertEqual(match.group(1), "no_script")
        self.assertEqual(match.group(2), "mcp")
开发者ID:pyzen，项目名称:mrjob，代码行数:7，代码来源:test_runner.py


示例11: test_empty_no_user

    def test_empty_no_user(self):
        self.getuser_should_fail = True
        runner = InlineMRJobRunner(conf_paths=[])
        match = _JOB_KEY_RE.match(runner.get_job_key())

        self.assertEqual(match.group(1), 'no_script')
        self.assertEqual(match.group(2), 'no_user')
开发者ID:anirudhreddy92，项目名称:mrjob，代码行数:7，代码来源:test_runner.py


示例12: test_trailing_slash

    def test_trailing_slash(self):
        archive_dir = self.makedirs('archive') + os.sep

        runner = InlineMRJobRunner()
        archive_path = runner._dir_archive_path(archive_dir)

        self.assertEqual(os.path.basename(archive_path), 'archive.tar.gz')
开发者ID:okomestudio，项目名称:mrjob，代码行数:7，代码来源:test_runner.py


示例13: test_same_dir_twice

    def test_same_dir_twice(self):
        archive_dir = self.makedirs('archive')

        runner = InlineMRJobRunner()
        archive_path_1 = runner._dir_archive_path(archive_dir)
        archive_path_2 = runner._dir_archive_path(archive_dir)

        self.assertEqual(os.path.basename(archive_path_1), 'archive.tar.gz')
        self.assertEqual(archive_path_1, archive_path_2)
开发者ID:okomestudio，项目名称:mrjob，代码行数:9，代码来源:test_runner.py


示例14: UpdateJobConfForHadoopVersionTestCase

class UpdateJobConfForHadoopVersionTestCase(TestCase):

    # jobconf with strange mix of Hadoop 1 and Hadoop 2 variables
    JOBCONF = {
        'foo.bar': 'baz',                   # unknown jobconf
        'mapred.jar': 'a.jar',              # Hadoop 1 jobconf
        'mapreduce.job.user.name': 'dave',  # Hadoop 2 jobconf
    }

    def setUp(self):
        self.runner = InlineMRJobRunner(conf_paths=[])

    def updated_and_warnings(self, jobconf, hadoop_version):
        jobconf = jobconf.copy()
        with no_handlers_for_logger('mrjob.runner'):
            stderr = StringIO()
            log_to_stream('mrjob.runner', stderr)
            self.runner._update_jobconf_for_hadoop_version(
                jobconf, hadoop_version)

        return jobconf, stderr.getvalue()

    def test_no_version(self):
        updated, warnings = self.updated_and_warnings(
            self.JOBCONF, None)

        self.assertEqual(updated, self.JOBCONF)
        self.assertEqual(warnings, '')

    def test_hadoop_1(self):
        updated, warnings = self.updated_and_warnings(
            self.JOBCONF, '1.0')

        self.assertEqual(updated,
                         combine_dicts(self.JOBCONF, {'user.name': 'dave'}))
        self.assertIn('do not match hadoop version', warnings)
        self.assertIn('mapreduce.job.user.name: user.name', warnings)

    def test_hadoop_2(self):
        updated, warnings = self.updated_and_warnings(
            self.JOBCONF, '2.0')

        self.assertEqual(updated,
                         combine_dicts(self.JOBCONF,
                                       {'mapreduce.job.jar': 'a.jar'}))
        self.assertIn('do not match hadoop version', warnings)
        self.assertIn('mapred.jar: mapreduce.job.jar', warnings)

    def test_dont_overwrite(self):
        # this jobconf contains two versions of the same variable
        jobconf = {'mapred.jar': 'a.jar', 'mapreduce.job.jar': 'b.jar'}

        updated, warnings = self.updated_and_warnings(jobconf, '1.0')

        self.assertEqual(updated, jobconf)
        self.assertEqual(warnings, '')
开发者ID:Milkigit，项目名称:mrjob，代码行数:56，代码来源:test_runner.py


示例15: test_mrjob_zip_compiles

    def test_mrjob_zip_compiles(self):
        runner = InlineMRJobRunner()
        with no_handlers_for_logger('mrjob.runner'):
            mrjob_zip = runner._create_mrjob_zip()

        ZipFile(mrjob_zip).extractall(self.tmp_dir)

        self.assertTrue(
            compileall.compile_dir(os.path.join(self.tmp_dir, 'mrjob'),
                                   quiet=1))
开发者ID:okomestudio，项目名称:mrjob，代码行数:10，代码来源:test_runner.py


示例16: test_output_dir_not_considered_hidden

    def test_output_dir_not_considered_hidden(self):
        output_dir = os.path.join(self.tmp_dir, '_hidden', '_output_dir')

        self.makefile(os.path.join(output_dir, 'part-00000'),
                      b'cats\n')

        runner = InlineMRJobRunner(conf_paths=[], output_dir=output_dir)

        self.assertEqual(sorted(to_lines(runner.stream_output())),
                         [b'cats\n'])
开发者ID:Affirm，项目名称:mrjob，代码行数:10，代码来源:test_runner.py


示例17: test_dirs_with_same_name

    def test_dirs_with_same_name(self):
        foo_archive = self.makedirs(os.path.join('foo', 'archive'))
        bar_archive = self.makedirs(os.path.join('bar', 'archive'))

        runner = InlineMRJobRunner()
        foo_archive_path = runner._dir_archive_path(foo_archive)
        bar_archive_path = runner._dir_archive_path(bar_archive)

        self.assertEqual(os.path.basename(foo_archive_path),
                         'archive.tar.gz')
        self.assertNotEqual(foo_archive_path, bar_archive_path)
开发者ID:okomestudio，项目名称:mrjob，代码行数:11，代码来源:test_runner.py


示例18: test_empty_dir

    def test_empty_dir(self):
        runner = InlineMRJobRunner()

        empty_dir = self.makedirs('empty')

        tar_gz_path = runner._dir_archive_path(empty_dir)
        self.assertEqual(os.path.basename(tar_gz_path), 'empty.tar.gz')

        runner._create_dir_archive(empty_dir)

        with tarfile.open(tar_gz_path, 'r:gz') as tar_gz:
            self.assertEqual(sorted(tar_gz.getnames()), [])
开发者ID:okomestudio，项目名称:mrjob，代码行数:12，代码来源:test_runner.py


示例19: test_archive

    def test_archive(self):
        runner = InlineMRJobRunner()

        tar_gz_path = runner._dir_archive_path(self._to_archive)
        self.assertEqual(os.path.basename(tar_gz_path), 'archive.tar.gz')

        runner._create_dir_archive(self._to_archive)

        tar_gz = tarfile.open(tar_gz_path, 'r:gz')
        try:
            self.assertEqual(sorted(tar_gz.getnames()),
                             [os.path.join('bar', 'baz'), 'foo'])
        finally:
            tar_gz.close()
开发者ID:okomestudio，项目名称:mrjob，代码行数:14，代码来源:test_runner.py


示例20: test_deprecated_stream_output

    def test_deprecated_stream_output(self):
        self.makefile('part-00000', contents=b'1\n2')
        self.makefile('part-00001', contents=b'3\n4\n')

        runner = InlineMRJobRunner(conf_paths=[], output_dir=self.tmp_dir)

        log = self.start(patch('mrjob.runner.log'))

        # should group output into lines, but not join across files
        self.assertEqual(sorted(runner.stream_output()),
                         [b'1\n', b'2', b'3\n', b'4\n'])

        # should issue deprecation warning
        self.assertEqual(log.warning.call_count, 1)
开发者ID:okomestudio，项目名称:mrjob，代码行数:14，代码来源:test_runner.py



注：本文中的mrjob.inline.InlineMRJobRunner类示例由纯净天空整理自Github/MSDocs等源码及文档管理平台，相关代码片段筛选自各路编程大神贡献的开源项目，源码版权归原作者所有，传播和使用请参考对应项目的License；未经允许，请勿转载。

鲜花
握手
雷人
路过
鸡蛋
该文章已有0人参与评论

请发表评论

全部评论

专题导读
More+
上一篇：
Python job.MRJob类代码示例发布时间：2022-05-27
下一篇：
Python hadoop.HadoopJobRunner类代码示例发布时间：2022-05-27
热门推荐
More+
热门话题
More+
阅读排行榜

扫描微信二维码

查看手机版网站

随时了解更新最新资讯

139-2527-9053

在线客服（服务时间 9:00～18:00）

在线QQ客服
地址：深圳市南山区西丽大学城创智工业园
电邮：jeky_zhao#qq.com
移动电话：139-2527-9053

Powered by 互联科技 X3.4© 2001-2213 极客世界.|Sitemap