def search(self, keyword, start=1, count=10):
        # construct search URL
        keyword.replace(" ", "%20")
        url = self.SEARCH_BASE + keyword + "&start=" + str(start) + "&count=" + str(count)
        # get http response
        response = get_response(url)
        # convert to json object
        json_res = json.loads(response)
        if "items" not in json_res:
            return []
        items = json_res["items"]

        result = []
        for item in items:
            # filter out the freelencer profile posts
            if item['title'].find("Hire") == -1:
                i = Item(item["link"], item["title"])
                result.append(i)

        # return at least 10 search results
        while len(result) < 10:
            start += count
            url = url = self.SEARCH_BASE + keyword + "&start=" + str(start) + "&count=" + str(count)
            response = get_response(url)
            json_res = json.loads(response)
            items = json_res["items"]
            for item in items:
                # filter out the freelencer profile posts
                if item['title'].find("Hire") == -1:
                    i = Item(item["link"], item["title"])
                    result.append(i)
        return result

def getRegionChildren(state,county,city,childtype):

    search_params = {
        "city": city,
        "state": state,
        "childtype": childtype,
        'county': county,
        "zws_id": Zillow_API_key
    }

    region_tags =  (('id'),
              ('name'),
              ('zindex'),
              ('latitude'),
              ('longitude'))
    
    region_cols = ['id', 'name','zindex','latitude', 'longitude']

    ##Get starting home data##
    r = utils.get_response(api = 'regionChildren', params = search_params)

    home = utils.parse_response(response = r,
                                api = 'regionChildren',
                                tags = region_tags,
                                cols = region_cols)

    #print(home)
    return home

def getSearchKeywordsRecommendation(keywords):
    if not keywords: print "enter a keyword"
    response = get_response("%(url)s&keywords=%(kw)s" % {
            "url" : url("getSearchKeywordsRecommendation"),
            "kw" : keywords,
            })
    return response

def get_member_points():
     email = request.args.get('email')
     attendance = get_attendance(email)
     eids = [x['event_id'] for x in attendance]
     events = get_events_by_id(eids)
     points = sum([x['points'] for x in events if 'points' in x.keys()])
     p = {'points':points, 'attendance':events}
     return utils.get_response(p)

def schedule():
    myEmail = utils.get_email_from_token(request.args.get('token'))
    params = {'where':json.dumps({'member_email': myEmail})}
    schedule = ParseDriver.make_parse_get_request('/1/classes/Commitments', params)['results']
    if len(schedule) == 0:
         return 'no schedule found', 404
    schedule = schedule[0]
    return utils.get_response(schedule['commitments'])

 def setUp(self):
     self.response = get_response('test/data/commune_2009_account.html', encoding='windows-1252')
     self.data = {
         'home_tax_basis': 137402 * 1e3,
         'home_tax_rate': 0.2099,
         'home_tax_value': 28841 * 1e3,
         'home_tax_cuts_on_deliberation': 30475 * 1e3,
         'business_tax_value': 0,
         'business_tax_rate': 0,
     }

 def setUp(self):
     self.response = get_response('test/data/department_2012_account.html',
                                  encoding='windows-1252')
     self.data = {
         'property_tax_basis': 445315000,
         'property_tax_value': 141253000,
         'property_tax_rate': 0.3172,
         'business_profit_contribution_basis': 0,
         'business_profit_contribution_value': 40288000,
         'business_network_tax_value': 974000,
     }

 def setUp(self):
     self.response = get_response('test/data/epci_2010_account.html', encoding='windows-1252')
     self.data = {
         'name': 'GFP : CC MONTAGNE BOURBONNAISE',
         'population': 6843,
         'operating_revenues': 606000,
         'compensation_2010_value': 26000,
         'business_property_contribution_additionnal_value': 8000,
         'business_property_contribution_uniq_value': 0,
         'business_property_contribution_eolien_value': 0,
     }

    def setUp(self):
        self.response = get_response('test/data/region_2012_account.html', encoding='windows-1252')

        self.data = {
            'name': 'REGION BASSE-NORMANDIE',
            'population': 1470880,
            'operating_revenues': 572356000,
            'tipp': 113678000,
            'business_profit_contribution_value': 64681000,
            'business_profit_contribution_cuts_on_deliberation': 288000,
            'business_network_tax_value': 13299000,
        }

def recent_golinks():
     email = utils.get_email_from_token(request.args.get('token'))
     page = int(request.args.get('page', '0'))
     params = {}
     params['order'] = '-createdAt'
     params['skip']=page*100
     r = ParseDriver.make_parse_get_request('/1/classes/ParseGoLink', params)
     results = r['results']
     for x in results:
          if 'num_clicks' not in x.keys():
                x['num_clicks']=0
     return utils.get_response(results)

    def setUp(self):
        self.response = get_response('test/data/region_2015_account.html', encoding='windows-1252')

        self.data = {
            'name': 'REGION BASSE-NORMANDIE',
            'population': 1477209,
            'local_tax': 82652000,
            'operating_revenues': 571984000,
            'tipp': 117483000,
            'business_profit_contribution_value': 69076000,
            'business_profit_contribution_cuts_on_deliberation': 94000,
            'business_network_tax_value': 13611000,
        }

def popular_golinks():
     email = utils.get_email_from_token(request.args.get('token'))
     page = int(request.args.get('page', '0'))
     params = {}
     params['order'] = '-num_clicks'
     params['skip']=page*100
     params['where']= json.dumps({'member_email': email})
     r = ParseDriver.make_parse_get_request('/1/classes/ParseGoLink', params)
     results = r['results']
     for x in results:
          if 'num_clicks' not in x.keys():
                x['num_clicks']=0
     return utils.get_response(results)

def attendance():
     requesterEmail = utils.get_email_from_token(request.args.get('token'))
     me = scripts.load_pickle_key('member_email_hash')[requesterEmail]
     emails = scripts.load_pickle_key('committee_members_hash')[me['committee']]
     emails = [x['email'] for x in emails]
     params = {'limit':sys.maxint, 'where': json.dumps({'member_email': {'$in':emails}})}
     event_members = ParseDriver.make_parse_get_request('/1/classes/ParseEventMember', params)['results']
     # return a dictionary with keys emails, values list of attended events
     h = {}
     seen = []
     for em in event_members:
          if em['member_email'] not in seen:
                seen.append(em['member_email'])
                h[em['member_email']] = []
          h[em['member_email']].append({'event_id': em['event_id'], 'type': em['type']})
     return utils.get_response(h)

 def setUp(self):
     self.response = get_response('test/data/epci_2014_account.html', encoding='windows-1252')
     self.data = {
         'name': 'GFP : CC MONTAGNE BOURBONNAISE',
         'population': 6897,
         'operating_revenues': 753000,
         'home_tax_basis': 6975000,
         'home_tax_rate': 0.0112,
         'home_tax_value': 78000,
         'additionnal_land_property_tax_value': 0,
         'business_property_contribution_additionnal_value': 11000,
         'business_property_contribution_uniq_value': 0,
         'business_property_contribution_eolien_value': 0,
         'business_profit_contribution_value': 5000,
         'other_tax': 942000,
         'fiscal_repayment': -860000,
     }

 def setUp(self):
     self.response = get_response('test/data/department_2014_account.html',
                                  encoding='windows-1252')
     self.data = {
         'operating_revenues': 543861000,
         'local_tax': 190228000,
         'other_tax': 150642000,
         'advertisement_tax': 42176000,
         'allocation': 108938000,
         'working_capital': 26133000,
         'property_tax_basis': 464872000,
         'property_tax_value': 147459000,
         'property_tax_rate': 0.3172,
         'property_tax_cuts_on_deliberation': 204000,
         'business_profit_contribution_basis': 0,
         'business_profit_contribution_value': 40139000,
         'business_profit_contribution_cuts_on_deliberation': 34000,
         'business_network_tax_value': 1074000,
     }

        tree = parse_response_soup(body, encoding)

    return tree


if __name__ == '__main__':

    parser = argparse.ArgumentParser(
        description="Parse the url with lxml and show the diff")
    parser.add_argument('url')
    parser.add_argument('--encoding', default='utf-8')
    parser.add_argument('--nosoup', default=False, action='store_true')
    parser.add_argument('--soupthreshold', default=0.9, type=float)
    args = parser.parse_args()

    response = get_response(args.url, {})
    original_body = response.body

    allow_soup = not args.nosoup
    tree = parse_response(body=original_body,
                          encoding=args.encoding,
                          allow_soup=allow_soup,
                          soupthreshold=args.soupthreshold)
    parsed_body = etree.tostring(tree, method='html', encoding=args.encoding)

    original_body_lines = original_body.decode('utf-8').splitlines()
    parsed_body_lines = parsed_body.decode('utf-8').splitlines()

    output_file = '{fn}.diff.html'.format(fn=slug_from_url(args.url))
    with(open(output_file, 'w')) as diff_file:

def dummy_view():
    response = get_response(request)
    return jsonify(response)