def test_format_protoclaims_ref_adds_column(self):
        claim_1 = pywikibot.Claim(self.repo, 'P123')
        claim_1.setTarget('1')
        ref_1 = Reference(claim_1)
        itis_1 = Statement('foo')
        itis_2 = Statement('bar').add_reference(ref_1)

        self.preview_item.protoclaims = {'P123': [itis_1, itis_2]}
        expected = (
            "{| class='wikitable'\n"
            "|-\n"
            "! Property\n"
            "! Value\n"
            "! Qualifiers\n"
            "! References\n"
            '|-\n'
            '| wd_template_1 \n'
            '| formatted_itis_1 \n'
            '|  \n'
            '|  \n'
            '|-\n'
            '| wd_template_1 \n'
            '| formatted_itis_2 \n'
            '|  \n'
            '| \nformatted_reference_1 \n'
            "|}"
        )
        self.assertEqual(self.preview_item.format_protoclaims(), expected)
        self.mock_wd_template.assert_called_once_with('P123')
        self.mock_format_itis.assert_has_calls([
            mock.call(itis_1),
            mock.call(itis_2)
        ])
        self.mock_format_qual.assert_not_called()
        self.mock_format_ref.assert_called_once_with(ref_1)

    def test_two_index_pages(self, mock_cache, mock_requests_get):
        index_pages = [['book link href="http://host/foo.zip"',
                        'next page href="local_page.html" link'],
                       ['book link href="http://host/bar.zip"',
                        'last page, no link']]
        mock_requests_get.return_value.iter_lines.side_effect = index_pages
        expected_cache_requests = [call('http://host/foo.zip'),
                                   call('http://host/bar.zip')]
        text1_lines = ['Title',
                       'The text of a book.']
        text2_lines = ['A short book.']
        mock_cache.return_value.fetch_lines.side_effect = [text1_lines,
                                                           text2_lines]
        expected_words = ['Title',
                          'The', 'text', 'of', 'a', 'book.',
                          'A', 'short', 'book.']

        words = list(fetch_all_words())

        self.assertEqual(expected_words, words)
        self.assertEqual(expected_cache_requests,
                         mock_cache.return_value.fetch_lines.mock_calls)
        mock_requests_get.assert_called_with(
            'http://www.gutenberg.org/robot/local_page.html',
            stream=True)

 def test_format_protoclaims_multple_different_prop(self):
     itis_1 = Statement('foo')
     itis_2 = Statement('bar')
     protoclaims = {'P123': itis_1, 'P321': itis_2}
     self.preview_item.protoclaims = OrderedDict(
         sorted(protoclaims.items(), key=lambda t: int(t[0][1:])))
     expected = (
         "{| class='wikitable'\n"
         "|-\n"
         "! Property\n"
         "! Value\n"
         "! Qualifiers\n"
         '|-\n'
         '| wd_template_1 \n'
         '| formatted_itis_1 \n'
         '|  \n'
         '|-\n'
         '| wd_template_2 \n'
         '| formatted_itis_2 \n'
         '|  \n'
         "|}"
     )
     self.assertEqual(self.preview_item.format_protoclaims(), expected)
     self.mock_wd_template.assert_has_calls([
         mock.call('P123'),
         mock.call('P321')],
         any_order=True
     )
     self.mock_format_itis.assert_has_calls([
         mock.call(itis_1),
         mock.call(itis_2)],
         any_order=True
     )
     self.mock_format_qual.assert_not_called()
     self.mock_format_ref.assert_not_called()

 def test_format_protoclaims_single_with_multiple_qual(self):
     itis = Statement('dummy')
     qual_1 = Qualifier('P321', 'qual_dummy')
     qual_2 = Qualifier('P213', 'qual_dummy')
     itis._quals.add(qual_1)
     itis._quals.add(qual_2)
     self.preview_item.protoclaims = {'P123': itis}
     expected = (
         "{| class='wikitable'\n"
         "|-\n"
         "! Property\n"
         "! Value\n"
         "! Qualifiers\n"
         '|-\n'
         '| wd_template_1 \n'
         '| formatted_itis_1 \n'
         '| * formatted_qual_1 \n'
         '* formatted_qual_2 \n'
         "|}"
     )
     self.assertEqual(self.preview_item.format_protoclaims(), expected)
     self.mock_wd_template.assert_called_once_with('P123')
     self.mock_format_itis.assert_called_once_with(itis)
     self.mock_format_qual.assert_has_calls([
         mock.call(qual_1),
         mock.call(qual_2)],
         any_order=True
     )
     self.mock_format_ref.assert_not_called()

  def test_do_cleanup_all(self, do_erase_packages_method, do_delete_users_method,
                      do_erase_dir_silent_method,
                      do_erase_files_silent_method, do_kill_processes_method,
                      get_os_type_method, find_repo_files_for_repos_method,
                      do_erase_alternatives_method, get_additional_dirs_method, clear_cache_mock):
    out = StringIO.StringIO()
    sys.stdout = out
    get_additional_dirs_method.return_value = ['/tmp/hadoop-yarn','/tmp/hsperfdata_007']
    propertyMap = {PACKAGE_SECTION:['abcd', 'pqrst'], USER_SECTION:['abcd', 'pqrst'],
                   REPO_SECTION:['abcd', 'pqrst'], DIR_SECTION:['abcd', 'pqrst'],
                   PROCESS_SECTION:['abcd', 'pqrst'],
                   ALT_SECTION:{ALT_KEYS[0]:['alt1','alt2'], ALT_KEYS[1]:[
                     'dir1']}, USER_HOMEDIR_SECTION:['decf']}
    get_os_type_method.return_value = 'redhat'
    find_repo_files_for_repos_method.return_value = ['abcd', 'pqrst']

    self.hostcleanup.do_cleanup(propertyMap)

    self.assertTrue(do_delete_users_method.called)
    self.assertTrue(do_erase_dir_silent_method.called)
    self.assertTrue(do_erase_files_silent_method.called)
    self.assertTrue(do_erase_packages_method.called)
    self.assertTrue(do_kill_processes_method.called)
    self.assertTrue(do_erase_alternatives_method.called)
    calls = [call(['decf']), call(['abcd', 'pqrst']), call(['/tmp/hadoop-yarn','/tmp/hsperfdata_007'])]
    do_erase_dir_silent_method.assert_has_calls(calls)
    do_erase_packages_method.assert_called_once_with(['abcd', 'pqrst'])
    do_erase_files_silent_method.assert_called_once_with(['abcd', 'pqrst'])
    do_delete_users_method.assert_called_once_with(['abcd', 'pqrst'])
    do_kill_processes_method.assert_called_once_with(['abcd', 'pqrst'])
    do_erase_alternatives_method.assert_called_once_with({ALT_KEYS[0]:['alt1',
                                              'alt2'], ALT_KEYS[1]:['dir1']})

    sys.stdout = sys.__stdout__

 def test_format_protoclaims_multple_same_prop(self):
     itis_1 = Statement('foo')
     itis_2 = Statement('bar')
     self.preview_item.protoclaims = {'P123': [itis_1, itis_2]}
     expected = (
         "{| class='wikitable'\n"
         "|-\n"
         "! Property\n"
         "! Value\n"
         "! Qualifiers\n"
         '|-\n'
         '| wd_template_1 \n'
         '| formatted_itis_1 \n'
         '|  \n'
         '|-\n'
         '| wd_template_1 \n'
         '| formatted_itis_2 \n'
         '|  \n'
         "|}"
     )
     self.assertEqual(self.preview_item.format_protoclaims(), expected)
     self.mock_wd_template.assert_called_once_with('P123')
     self.mock_format_itis.assert_has_calls([
         mock.call(itis_1),
         mock.call(itis_2)
     ])
     self.mock_format_qual.assert_not_called()
     self.mock_format_ref.assert_not_called()

    def run_test(self, mock_field_map, mock_build_sql_query,
                 mock_ext_output, mock_projection):
        mock_field_map.return_value = [u'id_cfpb', u'name']
        mock_build_sql_query.return_value = 'SELECT blah blah'

        mock_cursor_attrs = {'execute.return_value': True}
        mock_cursor = MagicMock(**mock_cursor_attrs)
        mock_cursor_enter_return = MagicMock(
            return_value=["row1", "row2", "row3"], **mock_cursor_attrs)
        mock_cursor_enter_return.__iter__.return_value = [
            'row1', 'row2', 'row3']
        mock_cursor.__enter__ = Mock(return_value=mock_cursor_enter_return)
        mock_connect_attrs = {'cursor.return_value': mock_cursor}
        mock_connect = Mock(**mock_connect_attrs)
        mock_extractor_attrs = {'connect.return_value': mock_connect}
        mock_extractor = Mock(table="table", **mock_extractor_attrs)
        mock_ext_output.return_value = mock_extractor

        mock_projection.return_value = "1"
        format = Format("mapping_file", "docs_file", "table",
                        sql_filter="WHERE id_cfpb is not NULL", marker_table=True)
        format.run()

        mock_field_map.assert_called_once()
        assert_equal(mock_projection.call_count, 3)
        mock_projection.assert_has_calls([
            mock.call(mock_field_map.return_value, "row1"),
            mock.call(mock_field_map.return_value, "row2"),
            mock.call(mock_field_map.return_value, "row3")
        ])

  def test_do_cleanup_with_skip(self, do_erase_packages_method,
                      do_delete_users_method,
                      do_erase_dir_silent_method,
                      do_erase_files_silent_method, do_kill_processes_method,
                      get_os_type_method, find_repo_files_for_repos_method, clear_cache_mock):

    out = StringIO.StringIO()
    sys.stdout = out
    propertyMap = {PACKAGE_SECTION:['abcd', 'pqrst'], USER_SECTION:['abcd', 'pqrst'],
                   REPO_SECTION:['abcd', 'pqrst'], DIR_SECTION:['abcd', 'pqrst'],
                   PROCESS_SECTION:['abcd', 'pqrst']}
    get_os_type_method.return_value = 'redhat'
    find_repo_files_for_repos_method.return_value = ['abcd', 'pqrst']
    HostCleanup.SKIP_LIST = [PACKAGE_SECTION, REPO_SECTION]

    self.hostcleanup.do_cleanup(propertyMap)

    self.assertTrue(do_delete_users_method.called)
    self.assertTrue(do_erase_dir_silent_method.called)
    self.assertFalse(do_erase_files_silent_method.called)
    self.assertFalse(do_erase_packages_method.called)
    self.assertTrue(do_kill_processes_method.called)
    calls = [call(None), call(['abcd', 'pqrst'])]
    do_erase_dir_silent_method.assert_has_calls(calls)
    do_delete_users_method.assert_called_once_with(['abcd', 'pqrst'])
    do_kill_processes_method.assert_called_once_with(['abcd', 'pqrst'])

    sys.stdout = sys.__stdout__

 def test_save_mr_mapping(self, option_parser_mock, curl_mock, json_loads_mock, write_mapping_mock,
                          backup_file_mock, file_handler_mock):
   file_handler_mock.return_value = logging.FileHandler('') # disable creating real file
   opm = option_parser_mock.return_value
   options = self.get_mock_options()
   args = ["save-mr-mapping"]
   opm.parse_args.return_value = (options, args)
   curl_mock.side_effect = ['"href" : "', '"href" : "', '"href" : "']
   json_loads_mock.return_value = {"host_components": [{"HostRoles": {"host_name": "host1"}}]}
   UpgradeHelper_HDP2.main()
   expected_curl_calls = [
     call(False, "-u", "admin:admin",
          "http://localhost:8080/api/v1/clusters/c1/services/MAPREDUCE/components/MAPREDUCE_CLIENT"),
     call(False, "-u", "admin:admin",
          "http://localhost:8080/api/v1/clusters/c1/services/MAPREDUCE/components/TASKTRACKER"),
     call(False, "-u", "admin:admin",
          "http://localhost:8080/api/v1/clusters/c1/services/MAPREDUCE/components/JOBTRACKER")]
   curl_mock.assert_has_calls(expected_curl_calls, any_order=True)
   self.assertTrue(write_mapping_mock.called)
   write_call = write_mapping_mock.call_args
   args, kargs = write_call
   self.assertTrue('MAPREDUCE_CLIENT' in args[0].keys())
   self.assertTrue(["host1"] == args[0]['MAPREDUCE_CLIENT'])
   self.assertTrue('TASKTRACKER' in args[0].keys())
   self.assertTrue('TASKTRACKER' in args[0].keys())
   pass

 def test_delete_mr(self, option_parser_mock, curl_mock,
                    backup_file_mock, file_handler_mock, read_mapping_mock, get_yn_mock):
   file_handler_mock.return_value = logging.FileHandler('') # disable creating real file
   opm = option_parser_mock.return_value
   options = self.get_mock_options()
   args = ["delete-mr"]
   opm.parse_args.return_value = (options, args)
   curl_mock.return_value = ''
   get_yn_mock.return_value = True
   read_mapping_mock.return_value = {
     "TASKTRACKER": ["c6401", "c6402"],
     "JOBTRACKER": ["c6401"],
     "MAPREDUCE_CLIENT": ["c6401"]}
   UpgradeHelper_HDP2.main()
   expected_curl_calls = [
     call(False, "-u", "admin:admin", "-X", "PUT", "-d", """{"HostRoles": {"state": "MAINTENANCE"}}""",
          "http://localhost:8080/api/v1/clusters/c1/hosts/c6401/host_components/TASKTRACKER"),
     call(False, "-u", "admin:admin", "-X", "PUT", "-d", """{"HostRoles": {"state": "MAINTENANCE"}}""",
          "http://localhost:8080/api/v1/clusters/c1/hosts/c6402/host_components/TASKTRACKER"),
     call(False, "-u", "admin:admin", "-X", "PUT", "-d", """{"HostRoles": {"state": "MAINTENANCE"}}""",
          "http://localhost:8080/api/v1/clusters/c1/hosts/c6401/host_components/JOBTRACKER"),
     call(False, "-u", "admin:admin", "-X", "DELETE",
          "http://localhost:8080/api/v1/clusters/c1/services/MAPREDUCE")]
   curl_mock.assert_has_calls(expected_curl_calls, any_order=True)
   pass

  def test_execute_retryable_command_with_time_lapse(self, CustomServiceOrchestrator_mock,
                                     read_stack_version_mock, sleep_mock, time_mock
  ):
    CustomServiceOrchestrator_mock.return_value = None
    dummy_controller = MagicMock()
    actionQueue = ActionQueue(AmbariConfig(), dummy_controller)
    python_execution_result_dict = {
      'exitcode': 1,
      'stdout': 'out',
      'stderr': 'stderr',
      'structuredOut': '',
      'status': 'FAILED'
    }
    time_mock.side_effect = [4, 8, 10, 14, 18, 22]

    def side_effect(command, tmpoutfile, tmperrfile, override_output_files=True, retry=False):
      return python_execution_result_dict

    command = copy.deepcopy(self.retryable_command)
    with patch.object(CustomServiceOrchestrator, "runCommand") as runCommand_mock:
      runCommand_mock.side_effect = side_effect
      actionQueue.execute_command(command)

    #assert that python executor start
    self.assertTrue(runCommand_mock.called)
    self.assertEqual(2, runCommand_mock.call_count)
    self.assertEqual(1, sleep_mock.call_count)
    sleep_mock.assert_has_calls([call(2)], False)
    runCommand_mock.assert_has_calls([
      call(command, '/tmp/ambari-agent/output-19.txt', '/tmp/ambari-agent/errors-19.txt', override_output_files=True, retry=False),
      call(command, '/tmp/ambari-agent/output-19.txt', '/tmp/ambari-agent/errors-19.txt', override_output_files=False, retry=True)])

 def test_namespace_deletion(self, execute_command, unify):
     execute_command.side_effect = ['namespace', None]
     namespace = Namespace('namespace')
     namespace.delete()
     self.assertTrue(execute_command.call_count, 2)
     self.assertTrue(call('ip netns list') in execute_command.mock_calls)
     self.assertTrue(call('ip netns del namespace') in execute_command.mock_calls)

 def test_action_install_pattern_suse(self, shell_mock, call_mock):
   call_mock.side_effect=[(0, None), (0, "Loading repository data...\nReading installed packages...\n\nS | Name\n--+-----\n  | Pack")]
   with Environment('/') as env:
     Package("some_package*",
             )
   call_mock.assert_has_calls([call("installed_pkgs=`rpm -qa 'some_package*'` ; [ ! -z \"$installed_pkgs\" ]"),
                               call("zypper --non-interactive search --type package --uninstalled-only --match-exact 'some_package*'")])
   shell_mock.assert_called_with(['/usr/bin/zypper', '--quiet', 'install', '--auto-agree-with-licenses', '--no-confirm', 'some_package*'], logoutput=False, sudo=True)

 def test_action_install_pattern_rhel(self, shell_mock, call_mock):
   call_mock.side_effect=[(0, None), (1, "Some text")]
   with Environment('/') as env:
     Package("some_package*",
     )
   call_mock.assert_has_calls([call("installed_pkgs=`rpm -qa 'some_package*'` ; [ ! -z \"$installed_pkgs\" ]"),
                               call("! yum list available 'some_package*'")])
   shell_mock.assert_called_with(['/usr/bin/yum', '-d', '0', '-e', '0', '-y', 'install', 'some_package*'], logoutput=False, sudo=True)

 def test_action_install_pattern_installed_rhel(self, shell_mock, call_mock):
   call_mock.side_effect=[(0, None), (0, "Some text")]
   with Environment('/') as env:
     Package("some_package*",
     )
   call_mock.assert_has_calls([call("installed_pkgs=`rpm -qa 'some_package*'` ; [ ! -z \"$installed_pkgs\" ]"),
                               call("! yum list available 'some_package*'")])
   self.assertEqual(shell_mock.call_count, 0, "shell.checked_call shouldn't be called")

 def test_action_install_pattern_suse(self, shell_mock, call_mock):
   call_mock.side_effect=[(0, None), (0, "Loading repository data...\nReading installed packages...\nNo packages found.\n")]
   with Environment('/') as env:
     Package("some_package*",
             )
   call_mock.assert_has_calls([call("installed_pkgs=`rpm -qa 'some_package*'` ; [ ! -z \"$installed_pkgs\" ]"),
                               call("zypper --non-interactive search --type package --uninstalled-only --match-exact 'some_package*'")])
   self.assertEqual(shell_mock.call_count, 0, "shell.checked_call shouldn't be called")

 def test_action_install_ubuntu(self, shell_mock, call_mock):
   call_mock.side_effect = [(1, None), (0, None)]
   with Environment('/') as env:
     Package("some_package",
     )
   call_mock.assert_has_calls([call("dpkg --get-selections | grep -v deinstall | awk '{print $1}' | grep '^some-package$'"),
                               call(['/usr/bin/apt-get', '-q', '-o', 'Dpkg::Options::=--force-confdef', '--allow-unauthenticated', '--assume-yes', 'install', 'some-package'], logoutput=False, sudo=True, env={'DEBIAN_FRONTEND': 'noninteractive'})])
   
   self.assertEqual(shell_mock.call_count, 0, "shell.checked_call shouldn't be called")

    def test_scan_dict_values_executes_value_handler_for_all_matching_prefixes(self):
        dictionary = {'a': 'foo123', 'b': {'c': 'foo234'}}
        handler = Mock()
        handler.return_value = "foo"

        result = CloudFormationTemplateTransformer.scan_dict_values(dictionary, handler)
        expected_calls = [mock.call('foo123'), mock.call('foo234')]
        six.assertCountEqual(self, expected_calls, handler.mock_calls)
        six.assertCountEqual(self, result, {'a': 'foo', 'b': {'c': 'foo'}})

 def test_do_delete_by_owner(self, listdir_mock, join_mock, stat_mock, do_erase_dir_silent_method):
   listdir_mock.return_value = ["k", "j"]
   join_mock.return_value = "path"
   response = MagicMock()
   response.st_uid = 1
   stat_mock.return_value = response
   self.hostcleanup.do_delete_by_owner([1, 2], ["a"])
   self.assertTrue(do_erase_dir_silent_method.called)
   calls = [call(["path"]), call(["path"])]
   do_erase_dir_silent_method.assert_has_calls(calls)

 def test_action_install_ubuntu(self, shell_mock, call_mock):
   call_mock.side_effect = [(1, None), (0, None)]
   with Environment('/') as env:
     Package("some_package",
     )
   call_mock.assert_has_calls([call("dpkg --get-selections | grep ^some-package$ | grep -v deinstall"),
                               call("DEBIAN_FRONTEND=noninteractive /usr/bin/apt-get -q -o Dpkg::Options::='--force-confdef'"
                                     " --allow-unauthenticated --assume-yes install some-package")
                             ])
   
   self.assertEqual(shell_mock.call_count, 0, "shell.checked_call shouldn't be called")