def _fetch_data(self, config, provider):
        """Fetch the latest feed data.

        :param dict config: RSS resource configuration
        :param provider: data provider instance, needed as an argument when
            raising ingest errors
        :return: fetched RSS data
        :rtype: str

        :raises IngestApiError: if fetching data fails for any reason
            (e.g. authentication error, resource not found, etc.)
        """
        url = config["url"]

        if config.get("auth_required", False):
            auth = (config.get("username"), config.get("password"))
        else:
            auth = None

        response = requests.get(url, auth=auth)

        if response.ok:
            return response.content
        else:
            if response.status_code in (401, 403):
                raise IngestApiError.apiAuthError(Exception(response.reason), provider)
            elif response.status_code == 404:
                raise IngestApiError.apiNotFoundError(Exception(response.reason), provider)
            else:
                raise IngestApiError.apiGeneralError(Exception(response.reason), provider)

    def _generate_auth_token(self, provider):
        """
        Generates Authentication Token as per the configuration in Ingest Provider.

        :param provider: dict - Ingest provider details to which the current directory has been configured
        :type provider: dict :py:class: `superdesk.io.ingest_provider_model.IngestProviderResource`
        :return: token details if successfully authenticated
        :rtype: str
        :raises: IngestApiError.apiGeneralError() if auth_url is missing in the Ingest Provider configuration
        """
        session = requests.Session()
        session.mount('https://', SSLAdapter())

        auth_url = provider.get('config', {}).get('auth_url', None)
        if not auth_url:
            raise IngestApiError.apiGeneralError(provider=provider,
                                                 exception=KeyError(
                                                     '''
                                                     Ingest Provider {} is missing Authentication URL.
                                                     Please check the configuration.
                                                     '''.format(provider['name']))
                                                 )

        payload = {
            'username': provider.get('config', {}).get('username', ''),
            'password': provider.get('config', {}).get('password', ''),
        }

        response = session.get(auth_url, params=payload, verify=False, timeout=30)
        if response.status_code < 200 or response.status_code >= 300:
            raise IngestApiError.apiAuthError(provider=provider)

        tree = etree.fromstring(response.content)  # workaround for http mock lib
        return tree.text

    def _fetch_data(self, config, provider):
        url = config['url']
        api_key = config['api_key']

        last_update = provider.get('last_updated', utcfromtimestamp(0)).strftime('%Y-%m-%dT%H:%M:%S')

        # Results are pagified so we'll read this many at a time
        offset_jump = 10

        params = {'start': last_update, 'limit': offset_jump}
        headers = {'apikey': api_key}

        items = []

        offset = 0
        while True:
            params['offset'] = offset

            try:
                response = requests.get(url, params=params, headers=headers, timeout=30)
            except requests.exceptions.ConnectionError as err:
                raise IngestApiError.apiConnectionError(exception=err)

            if response.ok:
                # The total number of results are given to us in json, get them
                # via a regex to read the field so we don't have to convert the
                # whole thing to json pointlessly
                item_ident = re.search('\"total\": *[0-9]*', response.text).group()
                results_str = re.search('[0-9]+', item_ident).group()

                if results_str is None:
                    raise IngestApiError.apiGeneralError(
                        Exception(response.text), provider)

                num_results = int(results_str)

                if num_results > 0:
                    items.append(response.text)

                if offset >= num_results:
                    return items

                offset += offset_jump
            else:
                if re.match('Error: No API Key provided', response.text):
                    raise IngestApiError.apiAuthError(
                        Exception(response.text), provider)
                elif response.status_code == 404:
                    raise IngestApiError.apiNotFoundError(
                        Exception(response.reason), provider)
                else:
                    raise IngestApiError.apiGeneralError(
                        Exception(response.reason), provider)

        return items

    def _update(self, provider, update):
        try:
            config = provider['config']
            user = config['username']
            password = config['password']
            id_list = config['idList']
            # before "products" was hardcoded as value for "idListType"
            id_list_type = config.get('idListType', 'products')
            if not user.strip() or not password.strip() or not id_list.strip():
                raise KeyError
        except KeyError:
            raise SuperdeskIngestError.notConfiguredError(Exception('username, password and idList are needed'))

        # we remove spaces and empty values from id_list to do a clean list
        id_list = ','.join([id_.strip() for id_ in id_list.split(',') if id_.strip()])

        params = {'idList': id_list,
                  'idListType': id_list_type,
                  'format': '5',
                  'maxItems': '25',
                  'sortOrder': 'chronological'}
        try:
            min_date_time = provider['private']['min_date_time']
            sequence_number = provider['private']['sequence_number']
        except KeyError:
            pass
        else:
            params['minDateTime'] = min_date_time
            params['sequenceNumber'] = sequence_number

        try:
            r = requests.get(URL, auth=(user, password), params=params)
        except Exception:
            raise IngestApiError.apiRequestError(Exception('error while doing the request'))

        try:
            root_elt = etree.fromstring(r.content)
        except Exception:
            raise IngestApiError.apiRequestError(Exception('error while doing the request'))

        parser = self.get_feed_parser(provider)
        items = parser.parse(root_elt, provider)

        try:
            min_date_time = root_elt.xpath('//iptc:timestamp[@role="minDateTime"]/text()', namespaces=NS)[0].strip()
            sequence_number = root_elt.xpath('//iptc:transmitId/text()', namespaces=NS)[0].strip()
        except IndexError:
            raise IngestApiError.apiRequestError(Exception('missing minDateTime or transmitId'))
        else:
            update.setdefault('private', {})
            update['private']['min_date_time'] = min_date_time
            update['private']['sequence_number'] = sequence_number

        return [items]

    def _fetch_data(self, config, provider):
        """Fetch the latest feed data.

        :param dict config: RSS resource configuration
        :param provider: data provider instance, needed as an argument when
            raising ingest errors
        :return: fetched RSS data
        :rtype: str

        :raises IngestApiError: if fetching data fails for any reason
            (e.g. authentication error, resource not found, etc.)
        """
        url = config['url']

        if config.get('auth_required', False):
            auth = (config.get('username'), config.get('password'))
            self.auth_info = {
                'username': config.get('username', ''),
                'password': config.get('password', '')
            }
        else:
            auth = None

        try:
            response = requests.get(url, auth=auth, timeout=30)
        except requests.exceptions.ConnectionError as err:
            raise IngestApiError.apiConnectionError(exception=err, provider=provider)
        except requests.exceptions.RequestException as err:
            raise IngestApiError.apiURLError(exception=err, provider=provider)

        if response.ok:
            return response.content
        else:
            if response.status_code in (401, 403):
                raise IngestApiError.apiAuthError(
                    Exception(response.reason), provider)
            elif response.status_code == 404:
                raise IngestApiError.apiNotFoundError(
                    Exception(response.reason), provider)
            else:
                raise IngestApiError.apiGeneralError(
                    Exception(response.reason), provider)

    def _request(self, url):
        try:
            response = requests.get(url, params={}, timeout=120)
        except requests.exceptions.Timeout as ex:
            # Maybe set up for a retry, or continue in a retry loop
            raise IngestApiError.apiTimeoutError(ex, self.provider)
        except requests.exceptions.TooManyRedirects as ex:
            # Tell the user their URL was bad and try a different one
            raise IngestApiError.apiRedirectError(ex, self.provider)
        except requests.exceptions.RequestException as ex:
            # catastrophic error. bail.
            raise IngestApiError.apiRequestError(ex, self.provider)
        except Exception as error:
            traceback.print_exc()
            raise IngestApiError.apiGeneralError(error, self.provider)

        if response.status_code == 404:
            raise LookupError('Not found')

        return response.content

 def _update(self, provider, update):
     user = provider['config']['wufoo_username']
     wufoo_data = {
         "url": WUFOO_URL.format(subdomain=user),
         "user": user,
         "api_key": provider['config']['wufoo_api_key'],
         "form_query_entries_tpl": WUFOO_QUERY_FORM + WUFOO_QUERY_ENTRIES,
         "update": update}
     try:
         parser = self.get_feed_parser(provider, None)
     except requests.exceptions.Timeout as ex:
         raise IngestApiError.apiTimeoutError(ex, provider)
     except requests.exceptions.TooManyRedirects as ex:
         raise IngestApiError.apiRedirectError(ex, provider)
     except requests.exceptions.RequestException as ex:
         raise IngestApiError.apiRequestError(ex, provider)
     except Exception as error:
         traceback.print_exc()
         raise IngestApiError.apiGeneralError(error, self.provider)
     items = parser.parse(wufoo_data, provider)
     return [items]

    def _get_tree(self, endpoint, payload=None):
        """Get xml response for given API endpoint and payload.

        :param: endpoint
        :type endpoint: str
        :param: payload
        :type payload: str
        """

        if payload is None:
            payload = {}

        payload['token'] = self._get_auth_token(self.provider, update=True)
        url = self._get_absolute_url(endpoint)

        if not self.session:
            self.session = requests.Session()

        retries = 0
        while True:
            try:
                response = self.session.get(url, params=payload, timeout=(30, 15))
            except requests.exceptions.Timeout as ex:
                if retries < 3:
                    logger.warn('Reuters API timeout retrying, retries {}'.format(retries))
                    retries += 1
                    continue
                raise IngestApiError.apiTimeoutError(ex, self.provider)
            except requests.exceptions.TooManyRedirects as ex:
                # Tell the user their URL was bad and try a different one
                raise IngestApiError.apiRedirectError(ex, self.provider)
            except requests.exceptions.RequestException as ex:
                # catastrophic error. bail.
                raise IngestApiError.apiRequestError(ex, self.provider)
            except Exception as error:
                traceback.print_exc()
                raise IngestApiError.apiGeneralError(error, self.provider)

            if response.status_code == 404:
                raise LookupError('Not found %s' % payload)

            break

        try:
            return etree.fromstring(response.content)  # workaround for http mock lib
        except UnicodeEncodeError as error:
            traceback.print_exc()
            raise IngestApiError.apiUnicodeError(error, self.provider)
        except ParseError as error:
            traceback.print_exc()
            raise IngestApiError.apiParseError(error, self.provider)
        except Exception as error:
            traceback.print_exc()
            raise IngestApiError.apiGeneralError(error, self.provider)

 def test_raise_apiNotFoundError(self):
     with assert_raises(IngestApiError) as error_context:
         ex = Exception("Testing apiNotFoundError")
         raise IngestApiError.apiNotFoundError(ex, self.provider)
     exception = error_context.exception
     self.assertTrue(exception.code == 4006)
     self.assertTrue(exception.message == "API service not found(404) error")
     self.assertIsNotNone(exception.system_exception)
     self.assertEqual(exception.system_exception.args[0], "Testing apiNotFoundError")
     self.assertEqual(len(self.mock_logger_handler.messages['error']), 1)
     self.assertEqual(self.mock_logger_handler.messages['error'][0],
                      "IngestApiError Error 4006 - API service not found(404) error: "
                      "Testing apiNotFoundError on channel TestProvider")

 def test_raise_apiRedirectError(self):
     with assert_raises(IngestApiError) as error_context:
         ex = Exception("Testing apiRedirectError")
         raise IngestApiError.apiRedirectError(ex, self.provider)
     exception = error_context.exception
     self.assertTrue(exception.code == 4002)
     self.assertTrue(exception.message == "API ingest has too many redirects")
     self.assertIsNotNone(exception.system_exception)
     self.assertEqual(exception.system_exception.args[0], "Testing apiRedirectError")
     self.assertEqual(len(self.mock_logger_handler.messages['error']), 1)
     self.assertEqual(self.mock_logger_handler.messages['error'][0],
                      "IngestApiError Error 4002 - API ingest has too many redirects: "
                      "Testing apiRedirectError on channel TestProvider")

    def _test(self, provider):
        config = provider.get('config', {})
        url = config['url']
        api_key = config['api_key']

        # limit the data to a single article and filter out all article fields
        # to save bandwidth
        params = {'limit': 1, 'fields': 'id'}
        headers = {'apikey': api_key}

        try:
            response = requests.get(url, params=params, headers=headers, timeout=30)
        except requests.exceptions.ConnectionError as err:
            raise IngestApiError.apiConnectionError(exception=err)

        if not response.ok:
            if response.status_code == 404:
                raise IngestApiError.apiNotFoundError(
                    Exception(response.reason), provider)
            else:
                raise IngestApiError.apiGeneralError(
                    Exception(response.reason), provider)

 def test_raise_apiParseError(self):
     with assert_raises(IngestApiError) as error_context:
         ex = Exception("Testing apiParseError")
         raise IngestApiError.apiParseError(ex, self.provider)
     exception = error_context.exception
     self.assertTrue(exception.code == 4005)
     self.assertTrue(exception.message == "API ingest xml parse error")
     self.assertIsNotNone(exception.system_exception)
     self.assertEquals(exception.system_exception.args[0], "Testing apiParseError")
     self.assertEqual(len(self.mock_logger_handler.messages["error"]), 1)
     self.assertEqual(
         self.mock_logger_handler.messages["error"][0],
         "IngestApiError Error 4005 - API ingest xml parse error: " "Testing apiParseError on channel TestProvider",
     )

 def test_raise_apiUnicodeError(self):
     with assert_raises(IngestApiError) as error_context:
         try:
             ex = Exception("Testing apiUnicodeError")
             raise ex
         except Exception:
             raise IngestApiError.apiUnicodeError(ex, self.provider)
     exception = error_context.exception
     self.assertTrue(exception.code == 4004)
     self.assertTrue(exception.message == "API ingest Unicode Encode Error")
     self.assertIsNotNone(exception.system_exception)
     self.assertEqual(exception.system_exception.args[0], "Testing apiUnicodeError")
     self.assertEqual(len(self.mock_logger_handler.messages['error']), 1)
     self.assertEqual(self.mock_logger_handler.messages['error'][0],
                      "IngestApiError Error 4004 - API ingest Unicode Encode Error: "
                      "Testing apiUnicodeError on channel TestProvider")

    def _update(self, provider, update):
        try:
            config = provider['config']
            user = config['username']
            password = config['password']
        except KeyError:
            SuperdeskIngestError.notConfiguredError(Exception('username and password are needed'))

        url_override = config.get('url', '').strip()
        if not url_override.startswith('http'):
            SuperdeskIngestError.notConfiguredError(Exception('if URL is set, it must be a valid http link'))

        if url_override:
            params = {'user': user, 'password': password, 'maksAntal': 50}
        else:
            params = {'user': user, 'password': password, 'maksAntal': 50, 'waitAcknowledge': 'true'}

        try:
            r = requests.get(url_override or URL, params=params)
        except Exception:
            raise IngestApiError.apiRequestError(Exception('error while doing the request'))

        try:
            root_elt = etree.fromstring(r.text)
        except Exception:
            raise IngestApiError.apiRequestError(Exception('error while parsing the request answer'))

        try:
            if root_elt.xpath('(//error/text())[1]')[0] != '0':
                err_msg = root_elt.xpath('(//errormsg/text())[1]')[0]
                raise IngestApiError.apiRequestError(Exception('error code returned by API: {msg}'.format(msg=err_msg)))
        except IndexError:
            raise IngestApiError.apiRequestError(Exception('Invalid XML, <error> element not found'))

        parser = self.get_feed_parser(provider)
        items = []
        for elt in root_elt.xpath('//RBNews'):
            item = parser.parse(elt, provider)
            items.append(item)
            if not url_override:
                try:
                    queue_id = elt.xpath('.//ServiceQueueId/text()')[0]
                except IndexError:
                    raise IngestApiError.apiRequestError(Exception('missing ServiceQueueId element'))
                ack_params = {'user': user, 'password': password, 'servicequeueid': queue_id}
                try:
                    requests.get(URL_ACK, params=ack_params)
                except Exception:
                    raise IngestApiError.apiRequestError(Exception('error while doing the request'))

        return [items]

    def test_raise_apiGeneralError(self):
        with assert_raises(IngestApiError) as error_context:
            ex = Exception("Testing general API error")
            raise IngestApiError.apiGeneralError(ex, self.provider)

        exception = error_context.exception
        self.assertEqual(exception.code, 4000)
        self.assertEqual(exception.message, "Unknown API ingest error")
        self.assertEqual(exception.provider_name, "TestProvider")

        self.assertIsNotNone(exception.system_exception)
        self.assertEqual(exception.system_exception.args[0], "Testing general API error")

        self.assertEqual(len(self.mock_logger_handler.messages["error"]), 1)
        self.assertEqual(
            self.mock_logger_handler.messages["error"][0],
            "IngestApiError Error 4000 - Unknown API ingest error: "
            "Testing general API error on channel TestProvider",
        )

    def _get_tree(self, endpoint, payload=None):
        """
        Get xml response for given API endpoint and payload.
        :param: endpoint
        :type endpoint: str
        :param: payload
        :type payload: str
        """

        if payload is None:
            payload = {}

        payload['token'] = self._get_auth_token(self.provider, update=True)
        url = self._get_absolute_url(endpoint)

        try:
            response = requests.get(url, params=payload, timeout=15)
        except requests.exceptions.Timeout as ex:
            # Maybe set up for a retry, or continue in a retry loop
            raise IngestApiError.apiTimeoutError(ex, self.provider)
        except requests.exceptions.TooManyRedirects as ex:
            # Tell the user their URL was bad and try a different one
            raise IngestApiError.apiRedirectError(ex, self.provider)
        except requests.exceptions.RequestException as ex:
            # catastrophic error. bail.
            raise IngestApiError.apiRequestError(ex, self.provider)
        except Exception as error:
            traceback.print_exc()
            raise IngestApiError.apiGeneralError(error, self.provider)

        if response.status_code == 404:
            raise LookupError('Not found %s' % payload)

        try:
            return etree.fromstring(response.content)  # workaround for http mock lib
        except UnicodeEncodeError as error:
            traceback.print_exc()
            raise IngestApiError.apiUnicodeError(error, self.provider)
        except ParseError as error:
            traceback.print_exc()
            raise IngestApiError.apiParseError(error, self.provider)
        except Exception as error:
            traceback.print_exc()
            raise IngestApiError.apiGeneralError(error, self.provider)

    def test_raise_apiAuthError(self):
        with assert_raises(IngestApiError) as error_context:
            ex = Exception("Testing API authorization error")
            raise IngestApiError.apiAuthError(ex, self.provider)

        exception = error_context.exception
        self.assertEqual(exception.code, 4007)
        self.assertEqual(
            exception.message, "API authorization error")
        self.assertEqual(exception.provider_name, "TestProvider")

        self.assertIsNotNone(exception.system_exception)
        self.assertEqual(
            exception.system_exception.args[0],
            "Testing API authorization error")

        self.assertEqual(len(self.mock_logger_handler.messages['error']), 1)
        self.assertEqual(
            self.mock_logger_handler.messages['error'][0],
            "IngestApiError Error 4007 - API authorization error: "
            "Testing API authorization error on channel TestProvider")

    def _update(self, provider, update):
        # Each update run will retrieve the data for a single "market"
        market_index = provider.get('private', {}).get('market_index', 0)
        markets = json.loads(provider.get('config', {}).get('market_definitions', []).replace('\'', '"'))
        market = markets[market_index]
        logger.info('Retrieving fuel data for the {} market'.format(market.get('market')))

        try:
            self.session_token = self._get_token(provider).get('id')
            prices = self._get_prices(provider, market)
            self._save(prices, market)
        except Exception as ex:
            raise IngestApiError.apiGeneralError(ex, self.provider)
        finally:
            # Save the next market to process
            market_index = (market_index + 1) % len(markets)
            get_resource_service('ingest_providers').system_update(provider.get('_id'),
                                                                   {'private': {'market_index': market_index}},
                                                                   provider)

        return None

    def _update(self, provider, update):

        def convert_date(epoch):
            dt = local_to_utc(config.DEFAULT_TIMEZONE, datetime.fromtimestamp(int(str(epoch)[:10])))
            return dt

        username = provider.get('config', {}).get('username')
        password = provider.get('config', {}).get('password')
        url = provider.get('config', {}).get('api_url')

        try:
            response = requests.get(url, auth=(username, password))
            response.raise_for_status()
        except Exception as ex:
            raise IngestApiError.apiGeneralError(ex, self.provider)

        data = json.loads(response.content.decode('UTF-8'))

        service = get_resource_service('traffic_incidents')
        incidents = []
        for feature in data.get('features', []):
            props = feature.get('properties', {})
            incident = {
                'guid': int(props.get('id')),
                'start_date': convert_date(props.get('startDate')),
                'end_date': convert_date(props.get('endDate')),
                'incident_type': props.get('type'),
                'incident_description': props.get('description'),
                'city': props.get('city'),
                'state': props.get('state'),
                'from_street_name': props.get('fromStreetName'),
                'from_cross_street_name': props.get('fromCrossStreetName'),
                'to_street_name': props.get('toStreetName'),
                'to_cross_street_name': props.get('toCrossStreetName'),
                'geometry': feature.get('geometry')
            }
            incident.get('geometry').pop('crs')
            incidents.append(incident)
        service.delete(lookup={})
        service.post(incidents)

    def get_tree(self, endpoint, payload=None):
        """Get xml response for given API endpoint and payload."""
        if payload is None:
            payload = {}
        payload['token'] = self.get_token()
        url = self.get_url(endpoint)

        try:
            response = requests.get(url, params=payload, timeout=21.0)
        except requests.exceptions.Timeout as ex:
            # Maybe set up for a retry, or continue in a retry loop
            raise IngestApiError.apiTimeoutError(ex, self.provider)
        except requests.exceptions.TooManyRedirects as ex:
            # Tell the user their URL was bad and try a different one
            raise IngestApiError.apiRedirectError(ex, self.provider)
        except requests.exceptions.RequestException as ex:
            # catastrophic error. bail.
            raise IngestApiError.apiRequestError(ex, self.provider)
        except Exception as error:
            traceback.print_exc()
            raise IngestApiError(error, self.provider)

        if response.status_code == 404:
            raise IngestApiError.apiNotFoundError(LookupError('Not found %s' % payload), self.provider)

        try:
            # workaround for httmock lib
            # return etree.fromstring(response.text.encode('utf-8'))
            return etree.fromstring(response.content)
        except UnicodeEncodeError as error:
            traceback.print_exc()
            raise IngestApiError.apiUnicodeError(error, self.provider)
        except ParseError as error:
            traceback.print_exc()
            raise IngestApiError.apiParseError(error, self.provider)
        except Exception as error:
            traceback.print_exc()
            raise IngestApiError(error, self.provider)