def GET(self):
        web.header("Content-Type", "text/html; charset=utf-8")
        db = utils.get_mongo_database()
        gstats_db = db.goal_stats
        goal_stats = list(gstats_db.find())

        n = db.games.count()

        ret = standard_heading("CouncilRoom.com: Goal Stats")
        ret += '<span class="subhead">Goal Stats</span>\n<p>\n'
        ret += '<table width="50%">'
        ret += '<tr><td><th>Goal Name<th width="1%">Total Times Achieved<th width="1%">% Occurrence<th>Description<th>Leaders'
        for g in sorted(goal_stats, key=lambda k: k['count'], reverse=True):
            goal_name = g['_id']
            ret += '<tr><td>'
            ret += '<img src="%s" alt="%s"/>' % (goals.GetGoalImageFilename(goal_name), goal_name)
            ret += '<th>%s<td align="right">%d<td align="right">%.2f<td align="center">%s' % (goal_name, g['count'], g['count']*100./n, goals.GetGoalDescription(goal_name))
            rank = 1
            ret += '<td>'
            for (players, count) in g['top']:
                if len(players)==1:
                    ret += "%d) %s (%d)<br />" % (rank, game.PlayerDeck.PlayerLink(players[0]), count)
                else:
                    ret += "%d) %d tied with %d (" % (rank, len(players), count)
                    links = [game.PlayerDeck.PlayerLink(player) for player in players]
                    ret += ', '.join(links)
                    ret += ')<br/>'
                rank += len(players)

        return ret

 def get(self, request, *args, **kwargs):
     db = get_mongo_database()
     mongocars = db.cars
     print(request.GET)
     form = SearchForm(request.GET)
     res = []
     context = {'form': form}
     if request.method == 'GET':
         if len(request.GET) > 0:
             searching_parametr = get_params_from_request(request.GET)
             print(searching_parametr)
             a = {'mpg': {'$gt': '10', '$lt': '20'}}
             print(a)
             for car in mongocars.find(searching_parametr):
                 car_result = Car.objects.get(id=car.get('sql_id'))
                 res.append(car_result)
             # Pagination
     paginator = Paginator(res, 10)
     page = self.request.GET.get('page')
     try:
         cars = paginator.page(page)
     except PageNotAnInteger:
         cars = paginator.page(1)
     except EmptyPage:
         cars = paginator.page(paginator.num_pages)
     context.update({
         'search_res': cars,
     })
     return render(request, 'search.html', context)

def main(parsed_args):
    db = utils.get_mongo_database()
    goal_db = db.goals
    gstats_db = db.goal_stats
    all_goals = goals.goal_check_funcs.keys()
    total_pcount = collections.defaultdict(int)
    goal_scanner = incremental_scanner.IncrementalScanner('goals', db)
    stat_scanner = incremental_scanner.IncrementalScanner('goal_stats', db)

    if not parsed_args.incremental:
        log.warning('resetting scanner and db')
        stat_scanner.reset()
        gstats_db.remove()

    log.info("Starting run: %s", stat_scanner.status_msg())

    # TODO: The following logic doesn't work now that goal calculation doesn't happen with a scanner.
    # if goal_scanner.get_max_game_id() == stat_scanner.get_max_game_id():
    #     log.info("Stats already set! Skip")
    #     exit(0)

    log.info('all_goals %s', all_goals)
    for goal_name in all_goals:
        log.info("Working on %s", goal_name)
        found_goals_cursor = goal_db.find({'goals.goal_name': goal_name},
                                          {'goals.player': 1, '_id': 0})
        total = found_goals_cursor.count()
        log.info("Found %d instances of %s", total, goal_name)

        pcount = collections.defaultdict(int)
        for goal in found_goals_cursor:
            player = goal['goals'][0]['player']
            pcount[player] += 1
            total_pcount[player] += 1

        psorted = sorted(pcount.iteritems(), key=operator.itemgetter(1), 
                         reverse=True)
        top = []
        leaders = 0
        i = 0
        while leaders < 3 and i < len(psorted):
            (player, count) = psorted[i]
            players = []
            if player not in AIs.names:
                players = [player]
            i += 1
            while i < len(psorted) and psorted[i][1] == count:
                if psorted[i][0] not in AIs.names:
                    players.append(psorted[i][0])
                i += 1
            leaders += len(players)
            if len(players) > 0:
                top.append((players, count))
			
        mongo_val = {'_id': goal_name, 'count': total, 'top': top}
        gstats_db.save(mongo_val)

    stat_scanner.set_max_game_id(goal_scanner.get_max_game_id())
    stat_scanner.save()
    log.info("Ending run: %s", stat_scanner.status_msg())

def main(parsed_args):
    """ Scan and update buy data"""
    start = time.time()
    db = utils.get_mongo_database()
    games = db.games
    output_db = db

    overall_stats = DeckBuyStats()

    scanner = incremental_scanner.IncrementalScanner(BUYS_COL_NAME, output_db)
    buy_collection = output_db[BUYS_COL_NAME]

    if not parsed_args.incremental:
        log.warning('resetting scanner and db')
        scanner.reset()
        buy_collection.drop()

    start_size = scanner.get_num_games()
    log.info("Starting run: %s", scanner.status_msg())
    do_scan(scanner, games, overall_stats, parsed_args.max_games)
    log.info("Ending run: %s", scanner.status_msg())
    end_size = scanner.get_num_games()

    if parsed_args.incremental:
        existing_overall_data = DeckBuyStats()
        utils.read_object_from_db(existing_overall_data, buy_collection, '')
        overall_stats.merge(existing_overall_data)
        def deck_freq(data_set):
            return data_set[dominioncards.Estate].available.frequency()
        log.info('existing %s decks', deck_freq(existing_overall_data))
        log.info('after merge %s decks', deck_freq(overall_stats))

    utils.write_object_to_db(overall_stats, buy_collection, '')

    scanner.save()

    def test_rawgames_integration(self):
        """Validate IsotropicScraper for raw games"""

        # Careful, as right now this touches bits of production, such
        # as the S3 buckets.

        # TODO: Figure out how to get this pointed at a database for
        # use in integration tests.

        iso = isotropic.IsotropicScraper(utils.get_mongo_database(), 'unittest_raw_games')

        self.assertTrue(iso.is_rawgames_in_s3(datetime.date(2010, 10, 15)))

        self.assertRaisesRegexp(HTTPError, 'HTTP Error 404: Not Found',
                                iso.copy_rawgames_to_s3, datetime.date(2009, 10, 15))

        content = iso.get_rawgames_from_s3(datetime.date(2010, 10, 15))
        self.assertEquals(content[0:7], 'BZh91AY', "Tar file signature")

        # TODO: Figure out how to run the lengthy tests that hit
        # actual files on Isotropic only in certain limited modes
        if None:
            iso.copy_rawgames_to_s3(datetime.date(2012, 10, 3))

            # Scrape and insert a whole gameday
            count = iso.scrape_and_store_rawgames(datetime.date(2012, 9, 15))
            self.assertEquals(count, 9999, "Inserted expected number of rawgames")

def main(args):
    db = utils.get_mongo_database()
    games = db.games
    game_stats = db.game_stats

    for player_name in args.players:
        log.debug("Processing top level player name %s", player_name)
        norm_target_player = norm_name(player_name)
        games_coll = games.find({keys.PLAYERS: norm_target_player})

        calculate_game_stats(list(games_coll), game_stats)

    def GET(self):
        web.header("Content-Type", "text/html; charset=utf-8")
        web.header("Access-Control-Allow-Origin", "*")
        query_dict = dict(urlparse.parse_qsl(web.ctx.env['QUERY_STRING']))
        # params:
        # targets, opt
        # cond1, opt
        # cond2, opt
        # format? json, csv?
        db = utils.get_mongo_database()
        targets = query_dict.get('targets', '').split(',')
        if sum(len(t) for t in targets) == 0:
            targets = card_info.card_names()

        # print targets
        def str_card_index(card_name):
            title = card_info.sane_title(card_name)
            if title:
                return str(card_info.card_index(title))
            return ''
        target_inds = map(str_card_index, targets)
        # print targets, target_inds

        cond1 = str_card_index(query_dict.get('cond1', ''))
        cond2 = str_card_index(query_dict.get('cond2', ''))
        
        if cond1 < cond2:
            cond1, cond2 = cond2, cond1

        card_stats = {}
        for target_ind in target_inds:
            key = target_ind + ';'
            if cond1:
                key += cond1
            if cond2:
                key += ',' + cond2

            db_val = db.card_supply.find_one({'_id': key})
            if db_val:
                small_gain_stat = SmallGainStat()
                small_gain_stat.from_primitive_object(db_val['vals'])
                card_name = card_info.card_names()[int(target_ind)]
                card_stats[card_name] = small_gain_stat
        
        format = query_dict.get('format', 'json')
        if format == 'json':
            readable_card_stats = {}
            for card_name, card_stat in card_stats.iteritems():
                readable_card_stats[card_name] = (
                    card_stat.to_readable_primitive_object())
            return json.dumps(readable_card_stats)
        return 'unsupported format ' + format

def retrieve_test_game(game_id):
    """Store the raw game for the passed game id in the test data dir.
    """
    db = utils.get_mongo_database()
    raw_games_col = db.raw_games
    rawgame = raw_games_col.find_one({'_id': game_id})

    if rawgame is None:
        print('could not find game ' + game_id)
    else:
        contents = bz2.decompress(rawgame['text']).decode('utf-8')
        with codecs.open('testing/testdata/'+game_id, encoding='utf-8', mode='w') as f:
            f.write(contents)

    def GET(self):
        web.header("Content-Type", "text/html; charset=utf-8")  
        query_dict = dict(urlparse.parse_qsl(web.ctx.env['QUERY_STRING']))
        db = utils.get_mongo_database()
        selected_card = ''

        if 'card' in query_dict:
            selected_card = query_dict['card']

        results = db.trueskill_openings.find({'_id': {'$regex': '^open:'}})
        openings = list(results)
        card_list = dominioncards.opening_cards()
        def split_opening(o):
            ret = o['_id'][len('open:'):].split('+')
            if ret == ['']: return []

            # Convert the __repr__() representation stored in the
            # database to the singular version of the card name.
            return [dominioncards.get_card(card).singular for card in ret]

        if selected_card not in ('All cards', ''):
            openings = [o for o in openings if selected_card in 
                        split_opening(o)]
                        
        openings = [o for o in openings if split_opening(o)]
        for opening in openings:
            floor = opening['mu'] - opening['sigma'] * 3
            ceil = opening['mu'] + opening['sigma'] * 3
            opening['level_key'] = make_level_key(floor, ceil)
            opening['level_str'] = make_level_str(floor, ceil)
            opening['skill_str'] = skill_str(opening['mu'], opening['sigma'])
            opening['cards'] = split_opening(opening)
            opening['cards'].sort()
            opening['cards'].sort(key=lambda card: dominioncards.get_card(card).cost, reverse=True)
            costs = [str(dominioncards.get_card(card).cost) for card in opening['cards']]
            while len(costs) < 2:
                costs.append('-')
            opening['cost'] = '/'.join(costs)

        openings.sort(key=lambda opening: opening['level_key'])
        openings.reverse()
        if selected_card == '':
            openings = [op for op in openings
                        if op['level_key'][0] != 0
                        or op['_id'] == ['Silver', 'Silver']]

        render = web.template.render('')
        return render.openings_template(openings, card_list, selected_card)

def main(args):
    """ Update analysis statistics.  By default, do so incrementally, unless
    --noincremental argument is given."""

    commit_after = 25000

    database = utils.get_mongo_database()
    games = database.games

    output_collection_name = 'analysis'
    output_collection = database[output_collection_name]
    game_analysis = GamesAnalysis()

    scanner = incremental_scanner.IncrementalScanner(output_collection_name,
                                                     database)
 
    if args.incremental:
        utils.read_object_from_db(game_analysis, output_collection, '')
    else:
        log.warning('resetting scanner and db')
        scanner.reset()

    output_file_name = 'static/output/all_games_card_stats.js'

    if not os.path.exists('static/output'):
        os.makedirs('static/output')

    log.info("Starting run: %s", scanner.status_msg())

    for idx, raw_game in enumerate(utils.progress_meter(scanner.scan(games, {}))):
        try:
            game_analysis.analyze_game(Game(raw_game))

            if args.max_games >= 0 and idx >= args.max_games:
                log.info("Reached max_games of %d", args.max_games)
                break

            if idx % commit_after == 0 and idx > 0:
                start = time.time()
                game_analysis.max_game_id = scanner.get_max_game_id()
                game_analysis.num_games = scanner.get_num_games()
                utils.write_object_to_db(game_analysis, output_collection, '')
                scanner.save()
                log.info("Committed calculations to the DB in %5.2fs", time.time() - start)

        except int, exception:
            log.exception('Exception occurred for %s in raw game %s', Game(raw_game).isotropic_url(), raw_game)
            raise 

def save_to_mongodb(sender, instance, **kwargs):
    db = get_mongo_database()
    mongocars = db.cars
    mongocars.remove({"sql_id": instance.id})
    mongocar = {'name': instance.name,
                'mpg': instance.mpg,
                'cylinders': instance.cylinders,
                'displacement': instance.displacement,
                'horsepower': instance.horsepower,
                'weight': instance.weight,
                'acceleration': instance.acceleration,
                'year': instance.year,
                'price': instance.price,
                'origin': instance.origin,
                'sql_id': instance.id}
    mongocars.insert(mongocar)

def summarize_game_stats_for_days(days):
    """Examines games and determines if need to be summarized.

    Takes a list of one or more days in the format "YYYYMMDD" or
    datetime.date, and generates tasks to summarize each of the
    individual games that occurred on those days.

    Skips days where there are no games available.

    Skips games that are already present in the games_stats
    collection.

    Returns the number of individual games referred for summarizing.
    """
    game_count = 0
    db = utils.get_mongo_database()
    games_col = db.games
    game_stats_col = db.game_stats
    games_col.ensure_index('game_date')

    for day in days:
        if type(day) is datetime.date:
            day = day.strftime('%Y%m%d')
        games_to_process = games_col.find({'game_date': day}, {'_id': 1})

        if games_to_process.count() < 1:
            log.info('No games available to summarize on %s', day)
            continue

        log.info('%s games to summarize on %s', games_to_process.count(), day)

        chunk = []
        for game in games_to_process:
            if len(chunk) >= SUMMARIZE_GAMES_CHUNK_SIZE:
                summarize_games.delay(chunk, day)
                chunk = []

            # Is this really slow? Does it need to be fixed? 
            #if game_stats_col.find({'_id.game_id': game['_id']}).count() == 0:
            chunk.append(game['_id'])
            game_count += 1

        if len(chunk) > 0:
            summarize_games.delay(chunk, day)

    return game_count

def summarize_games(game_ids, day):
    """Summarize the passed list of games"""
    log.info("Summarizing %d games from %s", len(game_ids), day)

    db = utils.get_mongo_database()
    games_col = db.games
    game_stats_col = db.game_stats

    games = []
    for game_id in game_ids:
        game = games_col.find_one({'_id': game_id})
        if game:
            games.append(game)
        else:
            log.warning('Found nothing for game id %s', game_id)

    return game_stats.calculate_game_stats(games, game_stats_col)

def calc_goals_for_days(days):
    """Examines games and determines if any goals were achieved, storing them in the DB.


    Takes a list of one or more days in the format "YYYYMMDD" or
    datetime.date, and generates tasks to calculate the goals achieved
    in each of the individual games that occurred on those days.

    Skips days where there are no games available.

    Skips games that are already present in the goal collection.

    Returns the number of individual games referred for searching.
    """
    game_count = 0
    db = utils.get_mongo_database()
    games_col = db.games
    goals_col = db.goals
    games_col.ensure_index('game_date')

    for day in days:
        if type(day) is datetime.date:
            day = day.strftime('%Y%m%d')
        games_to_process = games_col.find({'game_date': day}, {'_id': 1})

        if games_to_process.count() < 1:
            log.info('no games to search for goals on %s', day)
            continue

        log.info('%s games to search for goals on %s', games_to_process.count(), day)

        chunk = []
        for game in games_to_process:
            if len(chunk) >= CALC_GOALS_CHUNK_SIZE:
                calc_goals.delay(chunk, day) 
                chunk = []

            if goals_col.find({'_id': game['_id']}).count() == 0:
                chunk.append(game['_id'])
                game_count += 1

        if len(chunk) > 0:
            calc_goals.delay(chunk, day) 

    return game_count

    def GET(self):
        web.header("Content-Type", "text/html; charset=utf-8")  
        query_dict = dict(urlparse.parse_qsl(web.ctx.env['QUERY_STRING']))
        db = utils.get_mongo_database()
        selected_card = ''

        if 'card' in query_dict:
            selected_card = query_dict['card']

        if selected_card not in ('All cards', ''):
            query = db.trueskill_openings.find({'cards': selected_card})
        else:
            query = db.trueskill_openings.find({})

        #offset = db.trueskill_openings.find_one({'name':
        #    'open:Silver+Silver'})['mu']
        offset = 0
        openings = list(query)
        card_list = card_info.OPENING_CARDS
        for opening in openings:
            for stat in ('mu', 'floor', 'ceil'):
                opening[stat] -= offset

            floor = opening['floor']
            ceil = opening['ceil']
            opening['level_key'] = make_level_key(floor, ceil)
            opening['level_str'] = make_level_str(floor, ceil)
            opening['skill_str'] = skill_str(opening['mu'], opening['sigma'])
            opening['cards'].sort()
            opening['cards'].sort(key=lambda card: (card_info.Cost(card)),
                reverse=True)
            costs = [str(card_info.Cost(card)) for card in opening['cards']]
            while len(costs) < 2:
                costs.append('-')
            opening['cost'] = '/'.join(costs)

        openings.sort(key=lambda opening: opening['level_key'])
        openings.reverse()
        if selected_card == '':
            openings = [op for op in openings
                        if op['level_key'][0] != 0
                        or op['cards'] == ['Silver', 'Silver']]
    
        render = web.template.render('')
        return render.openings_template(openings, card_list, selected_card)

def parse_days(days):
    """Parses rawgames into games records and stores them in the DB.

    Takes a list of one or more days in the format "YYYYMMDD" or
    datetime.date, and generates tasks to parse the games that
    occurred on those days.

    Skips days where there are no rawgames available.

    Skips days where the parsed game collection has more than 95% of
    the quantity of rawgames, as this suggests the date has already
    been parsed.

    Returns the number of individual games referred for parsing.
    """
    game_count = 0
    db = utils.get_mongo_database()
    raw_games_col = db.raw_games
    games_col = db.games
    raw_games_col.ensure_index('game_date')

    for day in days:
        if type(day) is datetime.date:
            day = day.strftime('%Y%m%d')
        games_to_parse = raw_games_col.find({'game_date': day}, {'_id': 1})

        raw_games_qty = games_to_parse.count()
        if raw_games_qty < 1:
            log.info('no games to parse in %s', day)
            continue

        parsed_games_qty = games_col.find({'game_date': day}).count()
        if float(parsed_games_qty) / float(raw_games_qty) > 0.85:
            log.info('Looks like raw games for %s have already been parsed. Found %5.2f%% in games collection.',
                     day, 100.0 * parsed_games_qty / raw_games_qty)
            continue

        game_count += games_to_parse.count()
        log.info('%s games to parse in %s', games_to_parse.count(), day)
        for chunk in utils.segments([x['_id'] for x in games_to_parse], PARSE_GAMES_CHUNK_SIZE):
            parse_games.delay(chunk, day) 

    return game_count

def main(args):
    commit_after = 25000
    database = utils.get_mongo_database()
    games = database.games
    collection = database.optimal_card_ratios
    db_tracker = None

    scanner = incremental_scanner.IncrementalScanner('optimal_card_ratios', database)

    if not args.incremental:
        log.warning('resetting scanner and db')
        scanner.reset()

    log.info("Starting run: %s", scanner.status_msg())

    for ind, game in enumerate(
        utils.progress_meter(scanner.scan(games, {}))):
        if not db_tracker:
            log.debug("Initializing db tracker manager")
            db_tracker = DBCardRatioTrackerManager(collection, args.incremental)
            log.debug("DB tracker manager initialized")

        result = process_game(Game(game))
        for final_ratio_dict, progressive_ratio_dict, win_points in result:
            db_tracker.integrate_results('final', final_ratio_dict, win_points)
            db_tracker.integrate_results('progressive', progressive_ratio_dict, win_points)

        if args.max_games >= 0 and ind >= args.max_games:
            log.info("Reached max_games of %d", args.max_games)
            break

        if ind % commit_after == 0 and ind > 0:
            start = time.time()
            db_tracker.save()
            scanner.save()
            log.info("Committed calculations to the DB in %5.2fs", time.time() - start)

    log.info("Ending run: %s", scanner.status_msg())

    if db_tracker:
        db_tracker.save()
    scanner.save()

def scrape_raw_games(date):
    """Download the specified raw game archive, store it in S3, and load it into MongoDB.

    date is a datetime.date object
    """
    db = utils.get_mongo_database()

    scraper = isotropic.IsotropicScraper(db)

    try:
        inserted = scraper.scrape_and_store_rawgames(date)
        if inserted > 0:
            # Also need to parse the raw games for the days where we
            # inserted new records.
            parse_days.delay([date])
        return inserted

    except isotropic.ScrapeError:
        log.info("Data for %s is not yet available", date)
        return None

    def GET(self):
        web.header("Content-Type", "text/html; charset=utf-8")
        db = utils.get_mongo_database()

        goal_freq = collections.defaultdict(int)
        attainments_by_player = collections.defaultdict(
            lambda: collections.defaultdict(int))

        for goal_doc in db.goals.find():
            for goal in goal_doc['goals']:
                player = goal['player']
                goal_name = goal['goal_name']

                goal_freq[goal_name] += 1
                attainments_by_player[player][goal_name] += 1

        player_scores = {}
        tot_games = float(db.games.count())
        for player in attainments_by_player:
            score = 0
            player_goal_freqs = attainments_by_player[player]
            for goal in player_goal_freqs:
                global_rareness = tot_games / goal_freq[goal]
                player_goal_freq = player_goal_freqs[goal]
                score += global_rareness / (1 + math.exp(-player_goal_freq))
            player_scores[player] = score

        goal_freq = goal_freq.items()
        goal_freq.sort(key = lambda x: x[1])

        ret = ''
        for goal, freq in goal_freq:
            ret += goal + ' ' + str(freq) + '<br>'

        ret += '<br>'
        player_scores = player_scores.items()
        player_scores.sort(key = lambda x: -x[1])
        for player, score in player_scores[:10]:
            ret += player + ' ' + '%.3f' % score + '<br>'

        return ret

def main():
    db = utils.get_mongo_database()
    game_summaries = []
    output_fn = open('margin.txt', 'w')
    for idx, raw_game in enumerate(
        utils.progress_meter(db.games.find({}), 1000)):
        g = game.Game(raw_game)
        if g.dubious_quality():
            continue
        if len(g.get_player_decks()) != 2:
            continue
        try:
            joined_names = ','.join(
                p.Name().encode('utf-8') for p in 
                g.get_player_decks(sort_by_turn_order=True))
            output_fn.write('%f:%s:%s\n' % (
                    first_player_margin(g), 
                    joined_names,
                    ','.join(g.get_supply())))
        except UnicodeDecodeError:
            pass