def extend_record(record):
    record = vault.extend_record(record)
    _extend_record_common_fields(record)

    if record['record_type'] == 'commit':
        record['branches'] = ','.join(record['branches'])
        if 'correction_comment' not in record:
            record['correction_comment'] = ''
        record['message'] = make_commit_message(record)
        if record['commit_date']:
            record['commit_date_str'] = format_datetime(record['commit_date'])
    elif record['record_type'] == 'mark':
        review = vault.get_memory_storage().get_record_by_primary_key(
            record['review_id'])
        patch = vault.get_memory_storage().get_record_by_primary_key(
            utils.get_patch_id(record['review_id'], record['patch']))
        if not review or not patch:
            return None

        _extend_by_parent_info(record, review, 'parent_')
        _extend_by_parent_info(record, patch, 'patch_')
    elif record['record_type'] == 'patch':
        review = vault.get_memory_storage().get_record_by_primary_key(
            record['review_id'])
        _extend_by_parent_info(record, review, 'parent_')
    elif record['record_type'] == 'email':
        record['email_link'] = record.get('email_link') or ''
        record['blueprint_links'] = []
        for bp_id in record.get('blueprint_id', []):
            bp_module, bp_name = bp_id.split(':')
            record['blueprint_links'].append(
                make_blueprint_link(bp_module, bp_name))
    elif record['record_type'] in ['bpd', 'bpc']:
        record['summary'] = utils.format_text(record['summary'])
        if record.get('mention_count'):
            record['mention_date_str'] = format_datetime(
                record['mention_date'])
        record['blueprint_link'] = make_blueprint_link(record['module'],
                                                       record['name'])
    elif record['record_type'] in ['bugr', 'bugf']:
        record['number'] = record['web_link'].split('/')[-1]
        record['title'] = filter_bug_title(record['title'])
        record['status_class'] = re.sub('\s+', '', record['status'])

    elif record['record_type'] == 'tr':
        record['date_str'] = format_date(record['date'])  # no need for hours

    return record

def open_reviews(module):
    memory_storage_inst = vault.get_memory_storage()
    time_now = int(time.time())

    module_id_index = vault.get_vault()['module_id_index']
    module = module.lower()
    if module not in module_id_index:
        flask.abort(404)

    modules = module_id_index[module]['modules']

    review_ids = (memory_storage_inst.get_record_ids_by_modules(modules) &
                  memory_storage_inst.get_record_ids_by_types(['review']))

    waiting_on_reviewer = []
    total_open = 0

    for review in memory_storage_inst.get_records(review_ids):
        if review.status == 'NEW':
            total_open += 1
            if review.value in [1, 2]:
                waiting_on_reviewer.append(vault.extend_record(review))

    return {
        'module': module,
        'total_open': total_open,
        'waiting_on_reviewer': len(waiting_on_reviewer),
        'waiting_on_submitter': total_open - len(waiting_on_reviewer),
        'latest_revision': _process_stat(
            waiting_on_reviewer, 'updated_on', time_now),
        'first_revision': _process_stat(waiting_on_reviewer, 'date', time_now),
    }

def get_single_plus_two_reviews_report(records, **kwargs):
    memory_storage_inst = vault.get_memory_storage()
    plus_twos = collections.defaultdict(list)
    for record in records:
        if record['record_type'] != 'mark':
            continue

        if (record['branch'] == 'master' and
                record['type'] == 'Code-Review' and record['value'] == +2):
            review_id = record['review_id']
            review = memory_storage_inst.get_record_by_primary_key(review_id)
            if review and review['status'] == 'MERGED':
                plus_twos[review_id].append(record)

    response = []
    for review_id in plus_twos.keys():
        if len(plus_twos[review_id]) < 2:
            mark = plus_twos[review_id][0]
            review = memory_storage_inst.get_record_by_primary_key(
                mark['review_id'])
            response.append({'review_by': review['user_id'],
                             'mark_by': mark['user_id'],
                             'subject': review['subject'],
                             'url': review['url'],
                             'project': review['project']})

    return response

def get_users_json(record_ids, **kwargs):
    core_in = parameters.get_single_parameter(kwargs, 'core_in') or None
    valid_modules = set()
    if core_in:
        core_in = set(core_in.split(','))
        valid_modules = vault.resolve_project_types(
            kwargs['_params']['project_type'])
        valid_modules = set(m[0] for m in vault.resolve_modules(
            valid_modules, kwargs['_params']['release']))

    user_ids = vault.get_memory_storage().get_index_keys_by_record_ids(
        'user_id', record_ids)
    if kwargs['_params']['user_id']:
        user_ids.add(kwargs['_params']['user_id'][0])

    result = []
    for user_id in user_ids:
        user = vault.get_user_from_runtime_storage(user_id)
        r = {'id': user_id, 'text': user['user_name']}

        add_flag = not core_in
        if core_in and user.get('core'):
            core_modules = [module_branch[0] for module_branch in user['core']
                            if (module_branch[1] in core_in and
                                module_branch[0] in valid_modules)]
            if core_modules:
                r['core'] = core_modules
                if user['companies']:
                    r['company_name'] = user['companies'][-1]['company_name']
                add_flag = True
        if add_flag:
            result.append(r)

    result.sort(key=lambda x: x['text'])
    return result

def blueprint_summary(module, blueprint_name):
    blueprint_id = utils.get_blueprint_id(module, blueprint_name)
    bpd = vault.get_memory_storage().get_record_by_primary_key(
        'bpd:' + blueprint_id)
    if not bpd:
        flask.abort(404)
        return

    bpd = helpers.extend_record(bpd)
    record_ids = vault.get_memory_storage().get_record_ids_by_blueprint_ids(
        [blueprint_id])
    activity = [helpers.extend_record(record) for record in
                vault.get_memory_storage().get_records(record_ids)]
    activity.sort(key=lambda x: x['date'], reverse=True)

    return {'blueprint': bpd, 'activity': activity}

def get_modules_json(record_ids, **kwargs):
    module_id_index = vault.get_vault()['module_id_index']

    tags = parameters.get_parameter({}, 'tag', 'tags')

    # all modules mentioned in records
    module_ids = vault.get_memory_storage().get_index_keys_by_record_ids(
        'module', record_ids)

    add_modules = set([])
    for module in six.itervalues(module_id_index):
        if set(module['modules']) & module_ids:
            add_modules.add(module['id'])
    module_ids |= add_modules

    # keep only modules with specified tags
    if tags:
        module_ids = set(module_id for module_id in module_ids
                         if ((module_id in module_id_index) and
                             (module_id_index[module_id].get('tag') in tags)))

    result = []
    for module_id in module_ids:
        module = module_id_index[module_id]
        result.append({'id': module['id'],
                       'text': module['module_group_name'],
                       'tag': module['tag']})

    return sorted(result, key=operator.itemgetter('text'))

def record(record_id):
    memory_storage_inst = vault.get_memory_storage()
    record_obj = memory_storage_inst.get_record_by_primary_key(record_id)
    if not record_obj:
        flask.abort(404)

    result = dict(record=helpers.get_activity([record_obj], 0, 1)[0])
    return result

def company_activity(company):
    memory_storage_inst = vault.get_memory_storage()
    original_name = memory_storage_inst.get_original_company_name(company)

    result = _get_activity_summary(
        memory_storage_inst.get_record_ids_by_companies([original_name]))
    result['company_name'] = original_name

    return result

def get_company(company_name, **kwargs):
    memory_storage_inst = vault.get_memory_storage()
    for company in memory_storage_inst.get_companies():
        if company.lower() == company_name.lower():
            return {
                'id': company_name,
                'text': memory_storage_inst.get_original_company_name(
                    company_name)
            }
    flask.abort(404)

def generate_records_for_person_day(record_ids):
    memory_storage_inst = vault.get_memory_storage()
    id_dates = []
    for record in memory_storage_inst.get_records(record_ids):
        id_dates.append((record.date, record.record_id))

    id_dates.sort(key=operator.itemgetter(0))
    for record in memory_storage_inst.get_records(
            record_id for date, record_id in id_dates):
        yield record

def _validate_params(params):
    vault_inst = vault.get_vault()
    memory_storage_inst = vault.get_memory_storage()

    _check_param_in(params, 'release', vault_inst['releases'], True)
    _check_param_in(params, 'project_type', vault_inst['project_types_index'])
    _check_param_in(params, 'module', vault_inst['module_id_index'])
    _check_param_in(params, 'company',
                    memory_storage_inst.get_companies_lower())
    _check_param_in(params, 'user_id', memory_storage_inst.get_user_ids())
    _check_param_in(params, 'metric', parameters.METRIC_TO_RECORD_TYPE, True)

def open_reviews(module):
    memory_storage_inst = vault.get_memory_storage()
    time_now = int(time.time())

    module_id_index = vault.get_vault()['module_id_index']
    module = module.lower()
    if module not in module_id_index:
        flask.abort(404)

    modules = module_id_index[module]['modules']

    review_ids = (memory_storage_inst.get_record_ids_by_modules(modules) &
                  memory_storage_inst.get_record_ids_by_types(['review']))

    waiting_on_reviewer = []
    waiting_on_submitter = []
    total_open = 0

    for review in memory_storage_inst.get_records(review_ids):
        if review.status == 'NEW':
            total_open += 1

            # review.value is minimum from votes made for the latest patch
            if review.value in [1, 2]:
                # CI or engineer liked this change request, waiting for someone
                # to merge or to put dislike
                waiting_on_reviewer.append(helpers.extend_record(review))
            elif review.value in [-1, -2]:
                # CI or reviewer do not like this, waiting for submitter to fix
                waiting_on_submitter.append(helpers.extend_record(review))
            else:
                # new requests without votes, waiting for CI
                pass

    limit = int(flask.request.args.get('limit') or CONF.report_default_limit)
    if limit < 0:
        limit = None

    return {
        'module': module,
        'total_open': total_open,
        'waiting_on_reviewer': len(waiting_on_reviewer),
        'waiting_on_submitter': len(waiting_on_submitter),
        'waiting_on_ci': (total_open - len(waiting_on_reviewer) -
                          len(waiting_on_submitter)),
        'reviewer_latest_revision': _process_stat(
            waiting_on_reviewer, 'updated_on', time_now, limit),
        'reviewer_first_revision': _process_stat(
            waiting_on_reviewer, 'date', time_now, limit),
        'submitter_latest_revision': _process_stat(
            waiting_on_submitter, 'updated_on', time_now, limit),
        'submitter_first_revision': _process_stat(
            waiting_on_submitter, 'date', time_now, limit),
    }

def get_companies_json(record_ids, **kwargs):
    memory_storage = vault.get_memory_storage()
    companies = memory_storage.get_index_keys_by_record_ids(
        'company_name', record_ids)
    if kwargs['_params']['company']:
        companies.add(kwargs['_params']['company'][0])

    result = [memory_storage.get_original_company_name(company)
              for company in companies]

    return [{'id': utils.safe_encode(c.lower()), 'text': c}
            for c in sorted(result)]

def get_companies_json(record_ids, **kwargs):
    memory_storage = vault.get_memory_storage()
    companies = set(company
                    for company in memory_storage.get_index_keys_by_record_ids(
                        'company_name', record_ids))

    if kwargs['_params']['company']:
        companies.add(memory_storage.get_original_company_name(
            kwargs['_params']['company'][0]))

    return [{'id': c.lower().replace('&', ''), 'text': c}
            for c in sorted(companies)]

def user_activity(user_id):
    user = vault.get_user_from_runtime_storage(user_id)
    if not user:
        flask.abort(404)
    user = helpers.extend_user(user)

    memory_storage_inst = vault.get_memory_storage()
    result = _get_activity_summary(
        memory_storage_inst.get_record_ids_by_user_ids([user_id]))
    result['user'] = user

    return result

def _get_activity_summary(record_ids):
    memory_storage_inst = vault.get_memory_storage()

    record_ids_by_type = memory_storage_inst.get_record_ids_by_types(
        ['mark', 'patch', 'email', 'bpd', 'bpc', 'ci'])

    record_ids &= record_ids_by_type
    punch_card_data = _get_punch_card_data(
        memory_storage_inst.get_records(record_ids))

    return {
        'punch_card_data': punch_card_data,
    }

def get_users_json(record_ids, **kwargs):
    user_ids = vault.get_memory_storage().get_index_keys_by_record_ids(
        'user_id', record_ids)
    if kwargs['_params']['user_id']:
        user_ids.add(kwargs['_params']['user_id'][0])

    result = [{'id': user_id,
               'text': (vault.get_user_from_runtime_storage(user_id)
                        ['user_name'])}
              for user_id in user_ids]

    result.sort(key=lambda x: x['text'])
    return result

def extend_record(record):
    record = vault.extend_record(record)
    _extend_record_common_fields(record)

    if record['record_type'] == 'commit':
        record['branches'] = ','.join(record['branches'])
        if 'correction_comment' not in record:
            record['correction_comment'] = ''
        record['message'] = make_commit_message(record)
        if record['commit_date']:
            record['commit_date_str'] = format_datetime(record['commit_date'])
    elif record['record_type'] == 'mark':
        parent = vault.get_memory_storage().get_record_by_primary_key(
            record['review_id'])
        if not parent:
            return None

        _extend_by_parent_info(record, parent)
    elif record['record_type'] == 'patch':
        parent = vault.get_memory_storage().get_record_by_primary_key(
            record['review_id'])
        _extend_by_parent_info(record, parent)
    elif record['record_type'] == 'email':
        record['email_link'] = record.get('email_link') or ''
        record['blueprint_links'] = []
        for bp_id in record.get('blueprint_id', []):
            bp_module, bp_name = bp_id.split(':')
            record['blueprint_links'].append(
                make_blueprint_link(bp_module, bp_name))
    elif record['record_type'] in ['bpd', 'bpc']:
        record['summary'] = utils.format_text(record['summary'])
        if record.get('mention_count'):
            record['mention_date_str'] = format_datetime(
                record['mention_date'])
        record['blueprint_link'] = make_blueprint_link(record['module'],
                                                       record['name'])

    return record

def _get_activity_summary(record_ids):
    memory_storage_inst = vault.get_memory_storage()

    types = ['mark', 'patch', 'email', 'bpd', 'bpc']
    record_ids_by_type = set()
    for t in types:
        record_ids_by_type |= memory_storage_inst.get_record_ids_by_type(t)

    record_ids &= record_ids_by_type
    punch_card_data = _get_punch_card_data(
        memory_storage_inst.get_records(record_ids))

    return {
        'punch_card_data': punch_card_data,
    }

def get_engineers(records, metric_filter, finalize_handler, **kwargs):
    modules_names = parameters.get_parameter({}, 'module', 'modules')
    modules = set([m for m, r in vault.resolve_modules(modules_names, [''])])

    def postprocessing(record):
        if finalize_handler:
            record = finalize_handler(record)
        user = vault.get_user_from_runtime_storage(record['id'])
        record['core'] = get_core_engineer_branch(user, modules)
        return record

    return _get_aggregated_stats(records, metric_filter,
                                 vault.get_memory_storage().get_user_ids(),
                                 'user_id', 'author_name',
                                 finalize_handler=postprocessing)