def test_cookieredirect():
    """
    Test cookie redirection security
    """
    rurl = choice(REDIRECTS2)

    b = BaseBrowser()
    r = b.location(HTTPBIN + 'cookies')
    assert len(json.loads(r.text)['cookies']) == 0

    # add a cookie to the redirection service domain (not the target!)
    cookie = b.cookies.build('k', 'v1', rurl)
    b.cookies.set(cookie)
    r = b.location(rurl)
    assert r.url == HTTPBIN + 'cookies'
    # the cookie was not forwarded; it's for another domain
    # this is important for security reasons,
    # and because python-requests tries to do it by default!
    assert len(json.loads(r.text)['cookies']) == 0

    # add a cookie for the target
    cookie = b.cookies.build('k', 'v2', HTTPBIN)
    b.cookies.set(cookie)
    r = b.location(rurl)
    assert r.url == HTTPBIN + 'cookies'
    assert len(json.loads(r.text)['cookies']) == 1
    assert json.loads(r.text)['cookies']['k'] == 'v2'

    # check all cookies sent in the request chain
    assert r.cookies == {'k': 'v2'}
    assert r.history[0].cookies['k'] == 'v1'  # some services add other cookies

def test_cookienav():
    """
    Test browsing while getting new cookies
    """
    b = BaseBrowser()
    r = b.location(HTTPBIN + 'cookies')
    assert len(json.loads(r.text)['cookies']) == 0

    r = b.location(HTTPBIN + 'cookies/set/hello/world')
    assert len(json.loads(r.text)['cookies']) == 1
    assert json.loads(r.text)['cookies']['hello'] == 'world'
    r = b.location(HTTPBIN + 'cookies/set/hello2/world2')
    assert len(json.loads(r.text)['cookies']) == 2
    assert json.loads(r.text)['cookies']['hello2'] == 'world2'

    r = b.location(REQUESTBIN)
    assert 'session' in r.cookies  # requestbin should give this by default
    assert 'hello' not in r.cookies  # we didn't send the wrong cookie
    # return to httpbin, check we didn't give the wrong cookie
    r = b.location(HTTPBIN + 'cookies')
    assert 'session' not in json.loads(r.text)['cookies']

    # override cookies temporarily
    r = b.location(HTTPBIN + 'cookies', cookies={'bla': 'bli'})
    assert len(json.loads(r.text)['cookies']) == 1
    assert json.loads(r.text)['cookies']['bla'] == 'bli'
    # reload, the "fake" cookie should not be there
    r = b.location(HTTPBIN + 'cookies')
    assert len(json.loads(r.text)['cookies']) == 2
    assert 'bla' not in json.loads(r.text)['cookies']

    def search_housings(self, type, cities, nb_rooms, area_min, area_max, cost_min, cost_max, house_types):

        if type not in self.TYPES:
            raise TypeNotSupported

        self.update_header()
        result = self.form_item.open(data="{'rubrique': '%s'}" % self.TYPES.get(type))
        biens = json.loads(json.loads(result.content)['d'])

        for house_type in house_types:
            id_type = self.RET[type].get(house_type, '1')

            data = {}
            data['rubrique'] = self.TYPES.get(type)
            data['ach_id'] = None
            data['FromMoteur'] = "true"

            for bien in biens:
                if bien['Idchoix'] == int(id_type):
                    data['lstSSTbien'] = bien['SsTypebien']
                    data['lstTbien'] = bien['TypeBien']
                    data['Caracteristique'] = bien['Idchoix']

            data['OrigineAlerte'] = "SaveSearchMoteurHome"
            data['pays'] = "fra"
            data['prix_min'] = cost_min if cost_min and cost_min > 0 else None
            data['prix_max'] = cost_max if cost_max and cost_max > 0 else None
            data['lstThemes'] = ""

            min_rooms = nb_rooms if nb_rooms else None
            if not min_rooms:
                data['lstNbPieces'] = 0
            else:
                data['lstNbPieces'] = ','.join('%s' % n for n in range(min_rooms, 6))

            data['lstNbChambres'] = None
            data['surface_min'] = area_min if area_min else None
            # var localisationType = { "all": -1, "ville": 5, "region": 2, "departement": 4, "pays": 1, "regionUsuelle": 3 };
            data['localisationType'] = 5
            data['reference'] = ''
            data['rayon'] = 0
            data['localisation_id_rayon'] = None
            data['lstLocalisationId'] = ','.join(cities)
            data['photos'] = 0
            data['colocation'] = ''
            data['meuble'] = ''
            data['pageNumber'] = 1
            data['order_by'] = 1
            data['sort_order'] = 1
            data['top'] = 25
            data['SaveSearch'] = "false"
            data['EmailUser'] = ""
            data['GSMUser'] = ""

            self.search.go(data="{'p_SearchParams':'%s', 'forcealerte':'0'}" % json.dumps(data))

            data = '{pageIndex: 1,source:"undefined"}'
            for item in self.search_result.go(data=data).iter_housings():
                yield item

def test_redirects():
    """
    Check redirects are followed
    """
    b = BaseBrowser()
    b.location(HTTPBIN + 'redirect/1')
    assert b.url == HTTPBIN + 'get'

    r = b.location(HTTPBIN + 'redirect/1')
    assert json.loads(r.text)['headers'].get('Referer') == HTTPBIN + 'redirect/1'
    assert r.url == HTTPBIN + 'get'

    # Normal redirect chain
    b.url = None
    r = b.location(HTTPBIN + 'redirect/4')
    assert json.loads(r.text)['headers'].get('Referer') == HTTPBIN + 'redirect/1'
    assert len(r.history) == 4
    assert r.history[3].request.url == HTTPBIN + 'redirect/1'
    assert r.history[3].request.headers.get('Referer') == HTTPBIN + 'redirect/2'
    assert r.history[2].request.url == HTTPBIN + 'redirect/2'
    assert r.history[2].request.headers.get('Referer') == HTTPBIN + 'redirect/3'
    assert r.history[1].request.url == HTTPBIN + 'redirect/3'
    assert r.history[1].request.headers.get('Referer') == HTTPBIN + 'redirect/4'
    assert r.history[0].request.url == HTTPBIN + 'redirect/4'
    assert r.history[0].request.headers.get('Referer') is None
    assert r.url == HTTPBIN + 'get'

    # Disable all referers
    r = b.location(HTTPBIN + 'redirect/2', referrer=False)
    assert json.loads(r.text)['headers'].get('Referer') is None
    assert len(r.history) == 2
    assert r.history[1].request.headers.get('Referer') is None
    assert r.history[0].request.headers.get('Referer') is None
    assert r.url == HTTPBIN + 'get'

    # Only overrides first referer
    r = b.location(HTTPBIN + 'redirect/2', referrer='http://example.com/')
    assert json.loads(r.text)['headers'].get('Referer') == HTTPBIN + 'redirect/1'
    assert len(r.history) == 2
    assert r.history[1].request.headers.get('Referer') == HTTPBIN + 'redirect/2'
    assert r.history[0].request.headers.get('Referer') == 'http://example.com/'
    assert r.url == HTTPBIN + 'get'

    # Don't follow
    r = b.location(HTTPBIN + 'redirect/2', allow_redirects=False)
    assert len(r.history) == 0
    assert r.url == HTTPBIN + 'redirect/2'
    assert r.status_code == 302

    def confirm(self):
        form = self.get_form(id='authentification')

        url = self.browser.BASEURL + '//sec/vkm/gen_crypto?estSession=0'
        infos_data = self.browser.open(url).text
        infos_data = re.match('^_vkCallback\((.*)\);$', infos_data).group(1)
        infos = json.loads(infos_data.replace("'", '"'))
        infos['grid'] = self.decode_grid(infos)
        url = self.browser.BASEURL + '/sec/vkm/gen_ui?modeClavier=0&cryptogramme=' + infos["crypto"]
        content = self.browser.open(url).content
        img = Captcha(BytesIO(content), infos)

        try:
            img.build_tiles()
        except TileError as err:
            error("Error: %s" % err)
            if err.tile:
                err.tile.display()

        pwd = img.get_codes(self.browser.password[:6])
        t = pwd.split(',')
        newpwd = ','.join(t[self.strange_map[j]] for j in range(6))
        form['codsec'] = newpwd
        form['cryptocvcs'] = infos["crypto"].encode('iso-8859-1')
        form['vkm_op'] = 'sign'
        form.submit()

    def get_video_by_quality(self, url, quality):
        _url = url \
            + '/' + quality \
            + '.json'

        response = self.openurl(_url)
        return simplejson.loads(response.read(), self.ENCODING)

    def api0_request(self, command, action, parameter='', data=None, nologin=False):
        if data is None:
            # Always do POST requests.
            data = ''
        elif isinstance(data, (list,tuple,dict)):
            data = urlencode(data)
        elif isinstance(data, unicode):
            data = data.encode('utf-8')

        url = self.buildurl('http://api.adopteunmec.com/api.php',
                                                     S=self.APIKEY,
                                                     C=command,
                                                     A=action,
                                                     P=parameter,
                                                     O='json')

        buf = self.openurl(url, data).read()

        try:
            r = json.loads(buf[buf.find('{'):])
        except ValueError:
            raise ValueError(buf)

        if 'errors' in r and r['errors'] != '0' and len(r['errors']) > 0:
            code = r['errors'][0]
            if code in (u'0.0.2', u'1.1.1', u'1.1.2'):
                if not nologin:
                    self.login()
                    return self.api0_request(command, action, parameter, data, nologin=True)
                else:
                    raise BrowserIncorrectPassword(AuMException.ERRORS[code])
            else:
                raise AuMException(code)

        return r

    def get_history(self):
        txt = self.get_from_js('ListeMvts_data = new Array(', ');')

        if txt is None:
            no_trans = self.get_from_js('js_noMvts = new Ext.Panel(', ')')
            if no_trans is not None:
                # there is no transactions for this account, this is normal.
                return
            else:
                # No history on this account
                return

        data = json.loads('[%s]' % txt.replace('"', '\\"').replace("'", '"'))

        for line in data:
            t = Transaction(line[self.COL_ID])

            if self.is_coming is not None:
                t.type = t.TYPE_CARD
                date = self.parser.strip(line[self.COL_DEBIT_DATE])
            else:
                date = self.parser.strip(line[self.COL_DATE])
            raw = self.parser.strip(line[self.COL_LABEL])

            t.parse(date, raw)
            t.set_amount(line[self.COL_VALUE])

            if t.date is NotAvailable:
                continue

            if self.set_coming(t):
                continue

            yield t

    def get_location(self, ipaddr):
        res = self.browser.location('http://ip-api.com/json/%s' % ipaddr.encode('utf-8'))
        jres = json.loads(res.text)

        if "status" in jres and jres["status"] == "fail":
            raise Exception("IPAPI failure : %s" % jres["message"])

        iploc = IpLocation(ipaddr)
        iploc.city = u'%s'%jres['city']
        iploc.region = u'%s'%jres['regionName']
        iploc.zipcode = u'%s'%jres['zip']
        iploc.country = u'%s'%jres['country']
        if jres['lat'] != '':
            iploc.lt = float(jres['lat'])
        else:
            iploc.lt = 0.0
        if jres['lon'] != '':
            iploc.lg = float(jres['lon'])
        else:
            iploc.lg = 0.0
        #iploc.host = 'NA'
        #iploc.tld = 'NA'
        if 'isp' in jres:
            iploc.isp = u'%s'%jres['isp']

        return iploc

        def parse(self, el):
            item = XPath(u'//script[@type="application/ld+json"]')(self)

            json_content = CleanText(u'.',
                                     replace=[('//<![CDATA[ ', ''),
                                              (' //]]>', '')])(item[0])
            self.el = json.loads(json_content)

    def load_async(self, time):
        total = 0
        restart = True
        while restart:
            restart = False

            # load content of loading divs.
            lst = self.doc.xpath('//input[@type="hidden" and starts-with(@id, "asynch")]')
            if len(lst) > 0:
                params = {}
                for i, input in enumerate(lst):
                    params['key%s' % i] = input.attrib['name']
                    params['div%s' % i] = input.attrib['value']
                params['time'] = time

                r = self.browser.open('/AsynchAjax', params=params)
                data = json.loads(r.content)

                for i, d in enumerate(data['data']):
                    div = self.doc.xpath('//div[@id="%s"]' % d['key'])[0]
                    html = d['flux']
                    div.clear()
                    div.attrib['id'] = d['key'] # needed because clear removes also all attributes
                    div.insert(0, etree.fromstring(html, parser=etree.HTMLParser()))

                if 'time' in data:
                    wait = float(data['time'])/1000.0
                    self.logger.debug('should wait %f more seconds', wait)
                    total += wait
                    if total > 120:
                        raise BrowserUnavailable('too long time to wait')

                    sleep(wait)
                    restart = True

    def get_list(self):
        accounts = []
        previous_account = None

        noaccounts = self.get_from_js('_js_noMvts =', ';')
        if noaccounts is not None:
            assert 'avez aucun compte' in noaccounts
            return []

        txt = self.get_from_js('_data = new Array(', ');', is_list=True)

        if txt is None:
            raise BrowserUnavailable('Unable to find accounts list in scripts')

        data = json.loads('[%s]' % txt.replace("'", '"'))

        for line in data:
            a = Account()
            a.id = line[self.COL_ID].replace(' ', '')

            if re.match(r'Classement=(.*?):::Banque=(.*?):::Agence=(.*?):::SScompte=(.*?):::Serie=(.*)', a.id):
                a.id = str(CleanDecimal().filter(a.id))

            a._acc_nb = a.id.split('_')[0] if len(a.id.split('_')) > 1 else None
            a.label = MyStrip(line[self.COL_LABEL], xpath='.//div[@class="libelleCompteTDB"]')
            # This account can be multiple life insurance accounts
            if a.label == 'ASSURANCE VIE-BON CAPI-SCPI-DIVERS *':
                continue

            a.balance = Decimal(FrenchTransaction.clean_amount(line[self.COL_BALANCE]))
            a.currency = a.get_currency(line[self.COL_BALANCE])
            a.type = self.get_account_type(a.label)

            # The parent account must be created right before
            if a.type == Account.TYPE_CARD:
                # duplicate
                if find_object(accounts, id=a.id):
                    self.logger.warning('Ignoring duplicate card %r', a.id)
                    continue
                a.parent = previous_account

            if line[self.COL_HISTORY] == 'true':
                a._inv = False
                a._link = self.get_history_link()
                a._args = self.make__args_dict(line)
            else:
                a._inv = True
                a._args = {'_ipc_eventValue':  line[self.COL_ID],
                           '_ipc_fireEvent':   line[self.COL_FIRE_EVENT],
                          }
                a._link = self.doc.xpath('//form[@name="changePageForm"]')[0].attrib['action']

            if a.type is Account.TYPE_CARD:
                a.coming = a.balance
                a.balance = Decimal('0.0')

            accounts.append(a)
            previous_account = a

        return accounts

    def iter_persons(self, pattern):
        params = [("partner", self.PARTNER_KEY), ("q", pattern), ("format", "json"), ("filter", "person")]

        res = self.__do_request("search", params)
        if res is None:
            return
        jres = json.loads(res)
        if "person" not in jres["feed"]:
            return
        for p in jres["feed"]["person"]:
            thumbnail_url = NotAvailable
            if "picture" in p:
                thumbnail_url = unicode(p["picture"]["href"])
            person = Person(p["code"], unicode(p["name"]))
            desc = u""
            if "birthDate" in p:
                desc += "(%s), " % p["birthDate"]
            if "activity" in p:
                for a in p["activity"]:
                    desc += "%s, " % a["$"]
            person.real_name = NotLoaded
            person.birth_place = NotLoaded
            person.birth_date = NotLoaded
            person.death_date = NotLoaded
            person.gender = NotLoaded
            person.nationality = NotLoaded
            person.short_biography = NotLoaded
            person.short_description = desc.strip(", ")
            person.roles = NotLoaded
            person.thumbnail_url = thumbnail_url
            yield person

    def iter_movies(self, pattern):
        params = [("partner", self.PARTNER_KEY), ("q", pattern), ("format", "json"), ("filter", "movie")]

        res = self.__do_request("search", params)
        if res is None:
            return
        jres = json.loads(res)
        if "movie" not in jres["feed"]:
            return
        for m in jres["feed"]["movie"]:
            tdesc = u""
            if "title" in m:
                tdesc += "%s" % m["title"]
            if "productionYear" in m:
                tdesc += " ; %s" % m["productionYear"]
            elif "release" in m:
                tdesc += " ; %s" % m["release"]["releaseDate"]
            if "castingShort" in m and "actors" in m["castingShort"]:
                tdesc += " ; %s" % m["castingShort"]["actors"]
            short_description = tdesc.strip("; ")
            thumbnail_url = NotAvailable
            if "poster" in m:
                thumbnail_url = unicode(m["poster"]["href"])
            movie = Movie(m["code"], unicode(m["originalTitle"]))
            movie.other_titles = NotLoaded
            movie.release_date = NotLoaded
            movie.duration = NotLoaded
            movie.short_description = short_description
            movie.pitch = NotLoaded
            movie.country = NotLoaded
            movie.note = NotLoaded
            movie.roles = NotLoaded
            movie.all_release_dates = NotLoaded
            movie.thumbnail_url = thumbnail_url
            yield movie

    def get_person_biography(self, id):
        params = [
            ("partner", self.PARTNER_KEY),
            ("code", id),
            ("profile", "large"),
            ("mediafmt", "mp4-lc"),
            ("filter", "movie"),
            ("striptags", "biography,biographyshort"),
            ("format", "json"),
        ]

        res = self.__do_request("person", params)
        if res is not None:
            jres = json.loads(res)
            if "person" in jres:
                jres = jres["person"]
            else:
                return None
        else:
            return None

        biography = NotAvailable
        if "biography" in jres:
            biography = unicode(jres["biography"])

        return biography

    def get_transactions(self):
        data = []
        for script in self.doc.xpath('//script'):
            txt = script.text
            if txt is None:
                continue

            start = txt.find(self.JSON_PREFIX)
            if start < 0:
                continue
            
            txt = txt[start+len(self.JSON_PREFIX)
                      :start+txt[start:].find(';')]
            data = json.loads(txt)
                        
            break

        for tr in data:
            t = Transaction(0)
            text = tr[self.ROW_TEXT]
            t.parse(tr[self.ROW_DATE], text)
            if "+" in tr[self.ROW_DEBIT]:
                t.set_amount(credit=tr[self.ROW_DEBIT])
            else:
                t.set_amount(debit=tr[self.ROW_DEBIT])
            yield t

    def login(self, login, password):
        DOMAIN_LOGIN = self.browser.DOMAIN_LOGIN
        DOMAIN = self.browser.DOMAIN

        url_login = 'https://' + DOMAIN_LOGIN + '/index.html'

        base_url = 'https://' + DOMAIN
        url = base_url + '//sec/vk/gen_crypto?estSession=0'
        headers = {
                 'Referer': url_login
                  }
        request = self.browser.request_class(url, None, headers)
        infos_data = self.browser.readurl(request)

        infos_data = re.match('^_vkCallback\((.*)\);$', infos_data).group(1)

        infos = json.loads(infos_data.replace("'", '"'))

        url = base_url + '//sec/vk/gen_ui?modeClavier=0&cryptogramme=' + infos["crypto"]
        img = Captcha(self.browser.openurl(url), infos)

        try:
            img.build_tiles()
        except TileError, err:
            error("Error: %s" % err)
            if err.tile:
                err.tile.display()

    def get_video(self, video=None):
        if video is None:
            video = DailymotionVideo(self.group_dict['id'])

        div = self.parser.select(self.document.getroot(), 'div#content', 1)

        video.title = unicode(self.parser.select(div, 'span.title', 1).text).strip()
        video.author = unicode(self.parser.select(div, 'a.name, span.name, a[rel=author]', 1).text).strip()
        try:
            video.description = html2text(self.parser.tostring(self.parser.select(div, 'div#video_description', 1))).strip() or unicode()
        except BrokenPageError:
            video.description = u''

        embed_page = self.browser.readurl('http://www.dailymotion.com/embed/video/%s' % video.id)

        m = re.search('var info = ({.*?}),[^{"]', embed_page)
        if not m:
            raise BrokenPageError('Unable to find information about video')

        info = json.loads(m.group(1))
        for key in ['stream_h264_hd1080_url','stream_h264_hd_url',
                    'stream_h264_hq_url','stream_h264_url',
                    'stream_h264_ld_url']:
            if info.get(key):#key in info and info[key]:
                max_quality = key
                break
        else:
            raise BrokenPageError(u'Unable to extract video URL')
        video.url = info[max_quality]

        video.set_empty_fields(NotAvailable)

        return video

    def build_token(self, token):
        """
        These fucking faggots have introduced a new protection on the token.

        Each time there is a call to SAB (selectActionButton), the token
        available in the form is modified with a key available in JS:

        ipsff(function(){TW().ipthk([12, 25, 17, 5, 23, 26, 15, 30, 6]);});

        Each value of the array is an index for the current token to append the
        char at this position at the end of the token.
        """
        table = None
        for script in self.document.xpath('//script'):
            if script.text is None:
                continue
            m = re.search(r'ipthk\(([^\)]+)\)', script.text, flags=re.MULTILINE)
            if m:
                table = json.loads(m.group(1))
        if table is None:
            return token

        for i in table:
            token += token[i]
        return token

    def login(self, login, password):
        infos_data = self.browser.open('/sec/vk/gen_crypto?estSession=0').text
        infos_data = re.match('^_vkCallback\((.*)\);$', infos_data).group(1)
        infos = json.loads(infos_data.replace("'", '"'))

        url = '/sec/vk/gen_ui?modeClavier=0&cryptogramme=' + infos["crypto"]
        img = Captcha(BytesIO(self.browser.open(url).content), infos)

        try:
            img.build_tiles()
        except TileError as err:
            error("Error: %s" % err)
            if err.tile:
                err.tile.display()

        form = self.get_form(name=self.browser.LOGIN_FORM)
        form['user_id'] = login
        form['codsec'] = img.get_codes(password[:6])
        form['cryptocvcs'] = infos['crypto']
        form['vk_op'] = 'auth'
        form.url = '/authent.html'
        try:
            form.pop('button')
        except KeyError:
            pass
        form.submit()