def setUp(self):
     dirname = os.path.dirname(os.path.realpath(__file__))
     fixture = os.path.join(dirname, 'fixtures', self.filename)
     provider = {'name': 'Wenn'}
     with open(fixture) as f:
         self.file = f.read()
         etree.fromstring(self.file)
         self.items = WENNParser().parse_message(etree.fromstring(self.file), provider)

 def setUp(self):
     dirname = os.path.dirname(os.path.realpath(__file__))
     fixture = os.path.normpath(os.path.join(dirname, '../fixtures', self.filename))
     provider = {'name': 'Wenn'}
     with open(fixture, 'rb') as f:
         self.file = f.read()
         etree.fromstring(self.file)
         self.items = WENNFeedParser().parse(etree.fromstring(self.file), provider)

 def setUp(self):
     dirname = os.path.dirname(os.path.realpath(__file__))
     fixture = os.path.join(dirname, 'fixtures', self.filename)
     provider = {'name': 'Test'}
     with open(fixture) as f:
         self.nitf = f.read()
         self.item = NITFParser().parse_message(etree.fromstring(self.nitf), provider)

 def setUp(self):
     dirname = os.path.dirname(os.path.realpath(__file__))
     fixture = os.path.normpath(os.path.join(dirname, '../fixtures', self.filename))
     provider = {'name': 'Test'}
     with open(fixture) as f:
         self.scoop = f.read()
         self.item = ScoopNewsMLTwoFeedParser().parse(etree.fromstring(self.scoop), provider)

    def _update(self, provider):
        self.provider = provider
        self.path = provider.get('config', {}).get('path', None)

        if not self.path:
            return []

        for filename in get_sorted_files(self.path, sort_by=FileSortAttributes.created):
            try:
                filepath = os.path.join(self.path, filename)
                if os.path.isfile(filepath):
                    stat = os.lstat(filepath)
                    last_updated = datetime.fromtimestamp(stat.st_mtime, tz=utc)
                    if self.is_latest_content(last_updated, provider.get('last_updated')):
                        with open(filepath, 'r') as f:
                            item = self.parser.parse_message(etree.fromstring(f.read()), provider)
                            self.move_file(self.path, filename, provider=provider, success=True)
                            yield [item]
                    else:
                        self.move_file(self.path, filename, provider=provider, success=True)
            except etreeParserError as ex:
                logger.exception("Ingest Type: AAP - File: {0} could not be processed".format(filename))
                self.move_file(self.path, filename, provider=provider, success=False)
                raise ParserError.nitfParserError(ex, provider)
            except ParserError as ex:
                self.move_file(self.path, filename, provider=provider, success=False)
            except Exception as ex:
                self.move_file(self.path, filename, provider=provider, success=False)
                raise ProviderError.ingestError(ex, provider)

        push_notification('ingest:update')

    def _generate_auth_token(self, provider):
        """
        Generates Authentication Token as per the configuration in Ingest Provider.

        :param provider: dict - Ingest provider details to which the current directory has been configured
        :type provider: dict :py:class: `superdesk.io.ingest_provider_model.IngestProviderResource`
        :return: token details if successfully authenticated
        :rtype: str
        :raises: IngestApiError.apiGeneralError() if auth_url is missing in the Ingest Provider configuration
        """
        session = requests.Session()
        session.mount('https://', SSLAdapter())

        auth_url = provider.get('config', {}).get('auth_url', None)
        if not auth_url:
            raise IngestApiError.apiGeneralError(provider=provider,
                                                 exception=KeyError(
                                                     '''
                                                     Ingest Provider {} is missing Authentication URL.
                                                     Please check the configuration.
                                                     '''.format(provider['name']))
                                                 )

        payload = {
            'username': provider.get('config', {}).get('username', ''),
            'password': provider.get('config', {}).get('password', ''),
        }

        response = session.get(auth_url, params=payload, verify=False, timeout=30)
        if response.status_code < 200 or response.status_code >= 300:
            raise IngestApiError.apiAuthError(provider=provider)

        tree = etree.fromstring(response.content)  # workaround for http mock lib
        return tree.text

 def _parse_file(self, filename):
     dirname = os.path.dirname(os.path.realpath(__file__))
     fixture = os.path.normpath(os.path.join(dirname, '../fixtures', filename))
     provider = {'name': 'Test'}
     with open(fixture, 'rb') as f:
         self.root_elt = etree.fromstring(f.read())
         self.item = RitzauFeedParser().parse(self.root_elt, provider)

 def setUpFixture(self, filename):
     dirname = os.path.dirname(os.path.realpath(__file__))
     fixture = os.path.join(dirname, 'fixtures', filename)
     with open(fixture) as f:
         self.tree = etree.fromstring(f.read().encode('utf-8'))
     parser = newsml_2_0.NewsMLTwoParser()
     self.item = parser.parse_message(self.tree)[0]

 def setUp(self):
     config.NITF_MAPPING = self.mapping
     dirname = os.path.dirname(os.path.realpath(__file__))
     fixture = os.path.normpath(os.path.join(dirname, '../fixtures', self.filename))
     provider = {'name': 'Test'}
     with open(fixture) as f:
         self.nitf = f.read()
         self.item = NITFFeedParser().parse(etree.fromstring(self.nitf), provider)

 def setUp(self):
     with self.app.app_context():
         self.app.data.insert('vocabularies', self.vocab)
     dirname = os.path.dirname(os.path.realpath(__file__))
     fixture = os.path.normpath(os.path.join(dirname, '../fixtures', self.filename))
     provider = {'name': 'Test'}
     with open(fixture, 'rb') as f:
         self.root_elt = etree.fromstring(f.read())
         self.items = NewsMLTwoFeedParser().parse(self.root_elt, provider)

 def test_get_subjects_with_invalid_qcode(self):
     xml = ('<?xml version="1.0" encoding="UTF-8"?>'
            '<nitf><head>'
            '<tobject tobject.type="News">'
            '<tobject.property tobject.property.type="Current" />'
            '<tobject.subject tobject.subject.refnum="00000000" '
            'tobject.subject.type="Justice" tobject.subject.matter="Police" />'
            '</tobject></head></nitf>')
     subjects = NITFFeedParser().get_subjects(etree.fromstring(xml))
     self.assertEqual(len(subjects), 0)

 def test_fixture_list_with_no_dates(self):
     filename = 'aap_cricket.xml'
     dirname = os.path.dirname(os.path.realpath(__file__))
     fixture = os.path.normpath(os.path.join(dirname, '../fixtures', filename))
     with open(fixture, 'rb') as f:
         self.xml = f.read()
         fixture = {'fixture_xml': etree.fromstring(self.xml), 'sport_id': '3',
                    'sport_name': 'Cricket', 'comp_name': 'Champions Trophy in England/Wales', 'comp_id': 'dom-436'}
         items = AAPSportsFixturesParser().parse(fixture, None)
         self.assertTrue(len(items) == 2)

 def test_fixtures(self):
     filename = 'aap_soccer.xml'
     dirname = os.path.dirname(os.path.realpath(__file__))
     fixture = os.path.normpath(os.path.join(dirname, '../fixtures', filename))
     with open(fixture, 'rb') as f:
         self.xml = f.read()
         fixture = {'fixture_xml': etree.fromstring(self.xml), 'sport_id': '4',
                    'sport_name': 'Soccer', 'comp_name': 'Qualifiers', 'comp_id': 'int-314'}
         items = AAPSportsFixturesParser().parse(fixture, None)
         self.assertTrue(len(items) == 5)

    def _get_tree(self, endpoint, payload=None):
        """Get xml response for given API endpoint and payload.

        :param: endpoint
        :type endpoint: str
        :param: payload
        :type payload: str
        """

        if payload is None:
            payload = {}

        payload['token'] = self._get_auth_token(self.provider, update=True)
        url = self._get_absolute_url(endpoint)

        if not self.session:
            self.session = requests.Session()

        retries = 0
        while True:
            try:
                response = self.session.get(url, params=payload, timeout=(30, 15))
            except requests.exceptions.Timeout as ex:
                if retries < 3:
                    logger.warn('Reuters API timeout retrying, retries {}'.format(retries))
                    retries += 1
                    continue
                raise IngestApiError.apiTimeoutError(ex, self.provider)
            except requests.exceptions.TooManyRedirects as ex:
                # Tell the user their URL was bad and try a different one
                raise IngestApiError.apiRedirectError(ex, self.provider)
            except requests.exceptions.RequestException as ex:
                # catastrophic error. bail.
                raise IngestApiError.apiRequestError(ex, self.provider)
            except Exception as error:
                traceback.print_exc()
                raise IngestApiError.apiGeneralError(error, self.provider)

            if response.status_code == 404:
                raise LookupError('Not found %s' % payload)

            break

        try:
            return etree.fromstring(response.content)  # workaround for http mock lib
        except UnicodeEncodeError as error:
            traceback.print_exc()
            raise IngestApiError.apiUnicodeError(error, self.provider)
        except ParseError as error:
            traceback.print_exc()
            raise IngestApiError.apiParseError(error, self.provider)
        except Exception as error:
            traceback.print_exc()
            raise IngestApiError.apiGeneralError(error, self.provider)

 def test_fixture_list_with_dates(self):
     filename = 'aap_golf.xml'
     dirname = os.path.dirname(os.path.realpath(__file__))
     fixture = os.path.normpath(os.path.join(dirname, '../fixtures', filename))
     with open(fixture, 'rb') as f:
         self.xml = f.read()
         fixture = {'fixture_xml': etree.fromstring(self.xml), 'sport_id': '8',
                    'sport_name': 'Golf', 'comp_name': 'D+D Real Czech Masters', 'comp_id': 'int-26669'}
         items = AAPSportsFixturesParser().parse(fixture, None)
         self.assertTrue(len(items) == 1)
         self.assertEqual(items[0].get('calendars')[0].get('qcode'), 'sport')
         self.assertEqual(items[0].get('calendars')[1].get('qcode'), 'sportgeneral')

 def test_get_subjects(self):
     xml = ('<?xml version="1.0" encoding="UTF-8"?>'
            '<nitf><head>'
            '<tobject tobject.type="News">'
            '<tobject.property tobject.property.type="Current" />'
            '<tobject.subject tobject.subject.refnum="02003000" '
            'tobject.subject.type="Justice" tobject.subject.matter="Police" />'
            '</tobject></head></nitf>')
     subjects = NITFFeedParser().get_subjects(etree.fromstring(xml))
     self.assertEqual(len(subjects), 2)
     self.assertIn({'qcode': '02000000', 'name': 'Justice'}, subjects)
     self.assertIn({'qcode': '02003000', 'name': 'Police'}, subjects)

 def __init__(self, methodname):
     super().__init__(methodname)
     dirname = os.path.dirname(os.path.realpath(__file__))
     fixture = os.path.normpath(os.path.join(dirname, '../fixtures', self.filename))
     provider = {'name': 'Test'}
     with open(fixture, 'rb') as f:
         buf = f.read()
     self.ori_file = buf
     buf = buf.replace(b'\r', b'
')
     parser = etree.XMLParser(recover=True)
     parsed = etree.fromstring(buf, parser)
     self.articles = wordpress_wxr.WPWXRFeedParser().parse(parsed, provider)

    def parse_file(self, filename, provider):
        try:
            self.path = provider.get('config', {}).get('path', None)

            if not self.path:
                return []

            with open(os.path.join(self.path, filename), 'r') as f:
                item = self.parser.parse_message(etree.fromstring(f.read()), provider)

            return [item]
        except Exception as ex:
            self.move_file(self.path, filename, provider=provider, success=False)
            raise ParserError.parseFileError('AAP', filename, ex, provider)

 def test_get_subjects(self):
     xml = (
         '<?xml version="1.0" encoding="UTF-8"?>'
         "<nitf><head>"
         '<tobject tobject.type="News">'
         '<tobject.property tobject.property.type="Current" />'
         '<tobject.subject tobject.subject.refnum="02003000" '
         'tobject.subject.type="Justice" tobject.subject.matter="Police" />'
         "</tobject></head></nitf>"
     )
     subjects = get_subjects(etree.fromstring(xml))
     self.assertEqual(len(subjects), 2)
     self.assertIn({"qcode": "02000000", "name": "Justice"}, subjects)
     self.assertIn({"qcode": "02003000", "name": "Police"}, subjects)

def fetch_token_from_api(provider):
    session = requests.Session()
    session.mount("https://", SSLAdapter())

    url = "https://commerce.reuters.com/rmd/rest/xml/login"
    payload = {
        "username": provider.get("config", {}).get("username", ""),
        "password": provider.get("config", {}).get("password", ""),
    }

    response = session.get(url, params=payload, stream=False, verify=False)
    # workaround for httmock lib
    # tree = etree.fromstring(response.text)
    tree = etree.fromstring(response.content)
    return tree.text