def mendelian_variant_search_spec(request):

    project, family = get_project_and_family_for_user(request.user, request.GET)

    # TODO: use form

    search_hash = request.GET.get('search_hash')
    search_spec_dict, variants = cache_utils.get_cached_results(project.project_id, search_hash)
    search_spec = MendelianVariantSearchSpec.fromJSON(search_spec_dict)
    if variants is None:
        variants = api_utils.calculate_mendelian_variant_search(search_spec, family.xfamily())
    else:
        variants = [Variant.fromJSON(v) for v in variants]
    add_extra_info_to_variants_family(get_reference(), family, variants)
    return_type = request.GET.get('return_type')
    if return_type == 'json' or not return_type:
        return JSONResponse({
            'is_error': False,
            'variants': [v.toJSON() for v in variants],
            'search_spec': search_spec_dict,
        })
    elif request.GET.get('return_type') == 'csv':
        response = HttpResponse(content_type='text/csv')
        response['Content-Disposition'] = 'attachment; filename="xbrowse_results_{}.csv"'.format(search_hash)
        writer = csv.writer(response)
        indiv_ids = family.indiv_ids_with_variant_data()
        headers = xbrowse_displays.get_variant_display_headers(indiv_ids)
        writer.writerow(headers)
        for variant in variants:
            fields = xbrowse_displays.get_display_fields_for_variant(variant, get_reference(), indiv_ids)
            writer.writerow(fields)
        return response

def calculate_mendelian_variant_search(search_spec, xfamily):

    variants = None

    if search_spec.search_mode == 'standard_inheritance':

        variants = list(get_variants_with_inheritance_mode(
            get_mall(),
            xfamily,
            search_spec.inheritance_mode,
            variant_filter=search_spec.variant_filter,
            quality_filter=search_spec.genotype_quality_filter,
        ))

    elif search_spec.search_mode == 'custom_inheritance':

        variants = list(get_variants_family(
            get_datastore(),
            xfamily,
            genotype_filter=search_spec.genotype_inheritance_filter,
            variant_filter=search_spec.variant_filter,
            quality_filter=search_spec.genotype_quality_filter,
        ))

    elif search_spec.search_mode == 'gene_burden':

        gene_stream = get_genes_family(
            get_datastore(),
            get_reference(),
            xfamily,
            burden_filter=search_spec.gene_burden_filter,
            variant_filter=search_spec.variant_filter,
            quality_filter=search_spec.genotype_quality_filter,
        )

        variants = list(stream_utils.gene_stream_to_variant_stream(gene_stream, get_reference()))

    elif search_spec.search_mode == 'allele_count':

        variants = list(get_variants_allele_count(
            get_datastore(),
            xfamily,
            search_spec.allele_count_filter,
            variant_filter=search_spec.variant_filter,
            quality_filter=search_spec.genotype_quality_filter,
        ))

    elif search_spec.search_mode == 'all_variants':
        variants = list(get_variants_family(
            get_datastore(),
            xfamily,
            variant_filter=search_spec.variant_filter,
            quality_filter=search_spec.genotype_quality_filter,
        ))

    return variants

def gene_info(request, gene_str):

    real_gene_id = get_gene_id_from_str(gene_str, get_reference())
    gene = get_reference().get_gene(real_gene_id)
    gene['expression'] = get_reference().get_tissue_expression_display_values(real_gene_id)
    gene_json = json.dumps(gene)

    return render(request, 'gene_info.html', {
        'gene_json': gene_json,
        'gene_symbol': gene['symbol'],
    })

    def handle(self, *args, **options):

        if settings.CUSTOM_ANNOTATOR_SETTINGS is not None:
            print("Load dbNSFP.. ")

            # note that you could use mall.get_custom_annotator() here too
            custom_annotator = CustomAnnotator(settings.CUSTOM_ANNOTATOR_SETTINGS)
            custom_annotator.load()

        get_reference().load()
        mall.get_annotator().load()

def calculate_mendelian_variant_search(search_spec, xfamily):
    sys.stderr.write("     mendelian_variant_search for %s - search mode: %s  %s\n" % (xfamily.project_id, search_spec.search_mode, search_spec.__dict__))

    variants = None
    if search_spec.search_mode == 'standard_inheritance':
        variants = list(get_variants_with_inheritance_mode(
            get_mall(xfamily.project_id),
            xfamily,
            search_spec.inheritance_mode,
            variant_filter=search_spec.variant_filter,
            quality_filter=search_spec.quality_filter,
        ))

    elif search_spec.search_mode == 'custom_inheritance':
        variants = list(get_variants_family(
            get_datastore(xfamily.project_id),
            xfamily,
            genotype_filter=search_spec.genotype_inheritance_filter,
            variant_filter=search_spec.variant_filter,
            quality_filter=search_spec.quality_filter,
        ))

    elif search_spec.search_mode == 'gene_burden':
        gene_stream = get_genes_family(
            get_datastore(xfamily.project_id),
            get_reference(),
            xfamily,
            burden_filter=search_spec.gene_burden_filter,
            variant_filter=search_spec.variant_filter,
            quality_filter=search_spec.quality_filter,
        )

        variants = list(stream_utils.gene_stream_to_variant_stream(gene_stream, get_reference()))

    elif search_spec.search_mode == 'allele_count':
        variants = list(get_variants_allele_count(
            get_datastore(xfamily.project_id),
            xfamily,
            search_spec.allele_count_filter,
            variant_filter=search_spec.variant_filter,
            quality_filter=search_spec.quality_filter,
        ))

    elif search_spec.search_mode == 'all_variants':
        variants = list(get_variants_family(
            get_datastore(xfamily.project_id),
            xfamily,
            variant_filter=search_spec.variant_filter,
            quality_filter=search_spec.quality_filter,
            indivs_to_consider=xfamily.indiv_id_list(),
        ))

    return variants

def gene_info(request, gene_id):

    gene = get_reference().get_gene(gene_id)
    gene['expression'] = get_reference().get_tissue_expression_display_values(gene_id)

    ret = {
        'gene': gene,
        'is_error': False, 
        'found_gene': gene is not None, 
    }

    return JSONResponse(ret)

def gene_info(request, gene_str):

    real_gene_id = get_gene_id_from_str(gene_str, get_reference())
    gene = get_reference().get_gene(real_gene_id)
    gene['expression'] = get_reference().get_tissue_expression_display_values(real_gene_id)
    add_notes_to_genes([gene], request.user)
    gene_json = json.dumps(gene)

    return render(request, 'gene_info.html', {
        'gene_json': gene_json,
        'gene_symbol': gene['symbol'],
        'new_page_url': '/gene_info/{}'.format(gene_str),
    })

def family_coverage_gene(request, family, gene_id):

    project_id = family.project.project_id
    gene = get_reference().get_gene(gene_id)
    gene_structure = get_reference().get_gene_structure(gene_id)
    individuals = family.get_individuals()
    indiv_ids = [i.indiv_id for i in individuals]
    num_individuals = len(indiv_ids)

    coding_regions = []
    for c in get_coding_regions_from_gene_structure(gene_id, gene_structure):
        coding_region = {}
        coding_region['start'] = genomeloc.get_chr_pos(c.xstart)[1]
        coding_region['stop'] = genomeloc.get_chr_pos(c.xstop)[1]
        coding_region['gene_id'] = c.gene_id
        coding_region['size'] = c.xstop-c.xstart+1
        coding_regions.append(coding_region)

    coverages = {}
    for individual in individuals:
        coverages[individual.indiv_id] = get_coverage_store().get_coverage_for_gene(
            str(individual.pk),
            gene['gene_id']
        )

    whole_gene = Counter({'callable': 0, 'low_coverage': 0, 'poor_mapping': 0})
    for coverage_spec in coverages.values():
        whole_gene['callable'] += coverage_spec['gene_totals']['callable']
        whole_gene['low_coverage'] += coverage_spec['gene_totals']['low_coverage']
        whole_gene['poor_mapping'] += coverage_spec['gene_totals']['poor_mapping']
    gene_coding_size = 0
    for c in coding_regions:
        gene_coding_size += c['stop']-c['start']+1
    totalsize = gene_coding_size*num_individuals
    whole_gene['ratio_callable'] = whole_gene['callable'] / float(totalsize)
    whole_gene['ratio_low_coverage'] = whole_gene['low_coverage'] / float(totalsize)
    whole_gene['ratio_poor_mapping'] = whole_gene['poor_mapping'] / float(totalsize)
    whole_gene['gene_coding_size'] = gene_coding_size

    return render(request, 'coverage/family_coverage_gene.html', {
        'project': family.project,
        'family': family,
        'gene': gene,
        'coverages_json': json.dumps(coverages),
        'whole_gene_json': json.dumps(whole_gene),
        'coding_regions_json': json.dumps(coding_regions),
        'indiv_ids_json': json.dumps(indiv_ids),
        'individuals': individuals,
        'whole_gene': whole_gene,
    })

def cohort_gene_search_variants(request):

    # TODO: this view not like the others - refactor to forms

    error = None

    project, cohort = get_project_and_cohort_for_user(request.user, request.GET)
    if not project.can_view(request.user):
        return PermissionDenied

    form = api_forms.CohortGeneSearchVariantsForm(request.GET)
    if form.is_valid():
        gene_id = form.cleaned_data['gene_id']
        inheritance_mode = form.cleaned_data['inheritance_mode']
        variant_filter = form.cleaned_data['variant_filter']
        quality_filter = form.cleaned_data['quality_filter']
    else:
        error = server_utils.form_error_string(form)

    if not error:

        indivs_with_inheritance, gene_variation = cohort_search.get_individuals_with_inheritance_in_gene(
            get_datastore(project.project_id),
            get_reference(),
            cohort.xcohort(),
            inheritance_mode,
            gene_id,
            variant_filter=variant_filter,
            quality_filter=quality_filter
        )

        relevant_variants = gene_variation.get_relevant_variants_for_indiv_ids(cohort.indiv_id_list())

        api_utils.add_extra_info_to_variants_family(get_reference(), cohort, relevant_variants)

        ret = {
            'is_error': False, 
            'variants': [v.toJSON() for v in relevant_variants],
            'gene_info': get_reference().get_gene(gene_id),
        }
        return JSONResponse(ret)

    else: 
        ret = {
            'is_error': True, 
            'error': error
        }
        return JSONResponse(ret)

def combine_mendelian_families_spec(request):

    project, family_group = utils.get_project_and_family_group_for_user(request.user, request.GET)
    if not project.can_view(request.user):
        return HttpResponse('unauthorized')

    search_hash = request.GET.get('search_hash')
    search_spec, genes = cache_utils.get_cached_results(project.project_id, search_hash)
    if genes is None:
        genes = api_utils.calculate_combine_mendelian_families(family_group, search_spec)
    api_utils.add_extra_info_to_genes(project, get_reference(), genes)

    if request.GET.get('return_type', '') != 'csv':
        return JSONResponse({
            'is_error': False,
            'genes': genes,
            'search_spec': search_spec,
        })
    else:
        response = HttpResponse(content_type='text/csv')
        response['Content-Disposition'] = 'attachment; filename="family_group_results_{}.csv"'.format(search_hash)
        writer = csv.writer(response)
        writer.writerow(["gene", "# families", "family list", "chrom", "start", "end"])
        for gene in genes:
            family_id_list = [family_id for (project_id, family_id) in gene["family_id_list"]]
            writer.writerow(map(str, [gene["gene_name"], len(family_id_list), " ".join(family_id_list), gene["chr"], gene["start"], gene["end"], ""]))
        return response

def family_coverage_gene_list(request, family, gene_list):
    """
    Table of summary coverages for each gene in the gene list
    """
    sample_id_list = [str(individual.pk) for individual in family.get_individuals()]

    cache_key = ('family_coverage_gene_list', family.project.project_id, family.family_id, gene_list.slug)
    cached_results = server_utils.get_cached_results(cache_key)
    if cached_results:
        gene_coverages = cached_results
    else:
        gene_coverages = []
        for gene_id in gene_list.gene_id_list():
            d = {
                'gene_id': gene_id,
                'gene_name': get_reference().get_gene_symbol(gene_id),
                'totals': get_coverage_store().get_coverage_totals_for_gene(gene_id, sample_id_list)
            }
            d['coding_size'] = sum(d['totals'].values())
            try:
                d['percent'] = float(d['totals']['callable'])*100 / d['coding_size']
            except ZeroDivisionError:
                d['percent'] = 0
            gene_coverages.append(d)
        server_utils.save_results_cache(cache_key, gene_coverages)

    return render(request, 'coverage/family_coverage_gene_list.html', {
        'project': family.project,
        'family': family,
        'gene_list': gene_list,
        'gene_coverages': gene_coverages,
    })

def control_comparison(control_cohort, gene_id, sample_hits, sample_size, inheritance_mode, variant_filter, quality_filter):
    """
    Compare the results of num_hits, total against the reference population
    Return dict of 'num_hits', 'fisher_2sided_palue',
    """
    cohort = get_population_datastore().get_control_cohort(control_cohort)
    indivs_with_inheritance, gene_variation = get_individuals_with_inheritance_in_gene(
        get_population_datastore(),
        get_reference(),
        cohort,
        inheritance_mode,
        gene_id,
        variant_filter=variant_filter,
        quality_filter=quality_filter
        )
    control_hits = len(indivs_with_inheritance)
    fisher_results = fisher.pvalue(
        sample_hits,
        sample_size,
        control_hits,
        get_population_datastore().get_control_cohort_size(settings.DEFAULT_CONTROL_COHORT)
    )
    return {
        'control_hits': control_hits,
        'fisher_2sided_pvalue': fisher_results.two_tail,
    }

def breakpoint_search(request, project_id, family_id):

    log.info("Showing main breakpoint search page")

    project = get_object_or_404(Project, project_id=project_id)
    family = get_object_or_404(Family, project=project, family_id=family_id)
    if not project.can_view(request.user):
        raise PermissionDenied

    if not family.has_data('breakpoints'):
        return render(request, 'analysis_unavailable.html', {
            'reason': 'This family does not have any breakpoint data.'
        })

    gene_lists = project.get_gene_list_map()

    gene_list_json = dict([ (get_reference().get_gene_symbol(g),[ gl.name for gl in gll ]) for g,gll in gene_lists.iteritems() ])

    bam_file_paths = dict((ind.indiv_id, os.path.join(settings.READ_VIZ_BAM_PATH,ind.bam_file_path)) for ind in family.get_individuals())

    return render(request, 'breakpoint_search.html', {
        'project': project,
        'family': family,
        'gene_lists_json': json.dumps(gene_list_json),
        'bam_files_json': json.dumps(bam_file_paths),
    })

def cohort_variant_search(request):

    project, cohort = get_project_and_cohort_for_user(request.user, request.GET)
    if not project.can_view(request.user):
        return HttpResponse('unauthorized')

    form = api_forms.CohortVariantSearchForm(request.GET)
    if form.is_valid():
        search_spec = form.cleaned_data['search_spec']
        search_spec.family_id = cohort.cohort_id

        variants = api_utils.calculate_mendelian_variant_search(search_spec, cohort.xfamily())
        search_hash = cache_utils.save_results_for_spec(project.project_id, search_spec.toJSON(), [v.toJSON() for v in variants])
        api_utils.add_extra_info_to_variants_cohort(get_reference(), cohort, variants)

        return JSONResponse({
            'is_error': False,
            'variants': [v.toJSON() for v in variants],
            'search_hash': search_hash,
        })

    else:
        return JSONResponse({
            'is_error': True,
            'error': server_utils.form_error_string(form)
        })

def saved_variants(request, project_id):

    project = get_object_or_404(Project, project_id=project_id)
    if not project.can_view(request.user):
        return HttpResponse('unauthorized')
    
    variants = get_all_saved_variants_for_project(project)
    if 'family' in request.GET:
        requested_family_id = request.GET.get('family')
        variants = filter(lambda v: v.extras['family_id'] == requested_family_id, variants)
        
    variants = sorted(variants, key=lambda v: (v.extras['family_id'], v.xpos))
    grouped_variants = itertools.groupby(variants, key=lambda v: v.extras['family_id'])
    for family_id, family_variants in grouped_variants:
        family = Family.objects.get(project=project, family_id=family_id)
        family_variants = list(family_variants)

        add_extra_info_to_variants_family(get_reference(), family, family_variants)

    return render(request, 'project/saved_variants.html', {
        'project': project,
        'tag': None,
        'variants_json': json.dumps([v.toJSON() for v in variants]),
        'families_json': json.dumps({family.family_id: family.get_json_obj() for family in project.get_families()})
    })

def fetch_project_individuals_data(project_id):
    """
    Notes:
      1. ONLY project-authorized user has access to this individual
    """
    project = get_object_or_404(Project, project_id=project_id)
    variants = get_causal_variants_for_project(project)
    variants = sorted(variants, key=lambda v: (v.extras['family_id'], v.xpos))
    grouped_variants = itertools.groupby(variants, key=lambda v: v.extras['family_id'])
    for family_id, family_variants in grouped_variants:
        family = Family.objects.get(project=project, family_id=family_id)
        family_variants = list(family_variants)
        add_extra_info_to_variants_family(get_reference(), family, family_variants)

    family_data = [v.toJSON() for v in variants]
    variant_data = {family.family_id: family.get_json_obj() for family in project.get_families()}

    phenotype_entry_counts = gather_phenotype_data_for_project(project_id, variant_data)

    status_description_map = {}
    for abbrev, details in ANALYSIS_STATUS_CHOICES:
        status_description_map[abbrev] = details[0]
    families_json = json_displays.family_list(project.get_families())
    family_statuses = {}
    for f in families_json:
        family_statuses[f['family_id']] = status_description_map[f['analysis_status']['status']]

    return family_data, variant_data, phenotype_entry_counts, family_statuses

def diagnostic_search(request):

    project, family = utils.get_project_and_family_for_user(request.user, request.GET)
    if not project.can_view(request.user):
        raise PermissionDenied

    form = api_forms.DiagnosticSearchForm(family, request.GET)
    if form.is_valid():

        search_spec = form.cleaned_data['search_spec']
        search_spec.family_id = family.family_id

        gene_list = form.cleaned_data['gene_list']
        diagnostic_info_list = []
        for gene_id in gene_list.gene_id_list():
            diagnostic_info = get_gene_diangostic_info(family, gene_id, search_spec.variant_filter)
            add_extra_info_to_variants_family(get_reference(), family, diagnostic_info._variants)
            diagnostic_info_list.append(diagnostic_info)



        return JSONResponse({
            'is_error': False,
            'gene_diagnostic_info_list': [d.toJSON() for d in diagnostic_info_list],
            'gene_list_info': gene_list.toJSON(details=True),
            'data_summary': family.get_data_summary(),
        })

    else:
        return JSONResponse({
            'is_error': True,
            'error': server_utils.form_error_string(form)
        })

def mendelian_variant_search(request):

    # TODO: how about we move project getter into the form, and just test for authX here?
    # esp because error should be described in json, not just 404
    project, family = get_project_and_family_for_user(request.user, request.GET)

    form = api_forms.MendelianVariantSearchForm(request.GET)
    if form.is_valid():

        search_spec = form.cleaned_data['search_spec']
        search_spec.family_id = family.family_id

        variants = api_utils.calculate_mendelian_variant_search(search_spec, family.xfamily())
        search_hash = cache_utils.save_results_for_spec(project.project_id, search_spec.toJSON(), [v.toJSON() for v in variants])
        add_extra_info_to_variants_family(get_reference(), family, variants)

        return_type = request.GET.get('return_type', 'json')
        if return_type == 'json':
            return JSONResponse({
                'is_error': False,
                'variants': [v.toJSON() for v in variants],
                'search_hash': search_hash,
            })
        elif return_type == 'csv':
            return ''
        else:
            return HttpResponse("Return type not implemented")

    else:
        return JSONResponse({
            'is_error': True,
            'error': server_utils.form_error_string(form)
        })

def combine_mendelian_families_variants(request):

    project, family_group = utils.get_project_and_family_group_for_user(request.user, request.GET)

    form = api_forms.CombineMendelianFamiliesVariantsForm(request.GET)
    if form.is_valid():
        variants_grouped = get_variants_by_family_for_gene(
            get_mall(project.project_id),
            [f.xfamily() for f in form.cleaned_data['families']],
            form.cleaned_data['inheritance_mode'],
            form.cleaned_data['gene_id'],
            variant_filter=form.cleaned_data['variant_filter'],
            quality_filter=form.cleaned_data['quality_filter']
        )
        variants_by_family = []
        for family in form.cleaned_data['families']:
            variants = variants_grouped[(family.project.project_id, family.family_id)]
            add_extra_info_to_variants_family(get_reference(), family, variants)
            variants_by_family.append({
                'project_id': family.project.project_id,
                'family_id': family.family_id,
                'family_name': str(family),
                'variants': [v.toJSON() for v in variants],
            })
        return JSONResponse({
            'is_error': False,
            'variants_by_family': variants_by_family,
        })

    else:
        return JSONResponse({
            'is_error': True,
            'error': server_utils.form_error_string(form)
        })

def cohort_gene_search(request):

    project, cohort = get_project_and_cohort_for_user(request.user, request.GET)
    sys.stderr.write("cohort_gene_search %s  %s: starting ... \n" % (project.project_id, cohort.cohort_id))
    form = api_forms.CohortGeneSearchForm(request.GET)
    if form.is_valid():
        search_spec = form.cleaned_data['search_spec']
        search_spec.cohort_id = cohort.cohort_id
        sys.stderr.write("cohort_gene_search %s  %s: search spec: %s \n" % (project.project_id, cohort.cohort_id, str(search_spec.toJSON())))
        genes = api_utils.calculate_cohort_gene_search(cohort, search_spec)
        sys.stderr.write("cohort_gene_search %s  %s: get %s genes \n" % (project.project_id, cohort.cohort_id, len(genes)))
        search_hash = cache_utils.save_results_for_spec(project.project_id, search_spec.toJSON(), genes)
        api_utils.add_extra_info_to_genes(project, get_reference(), genes)
        sys.stderr.write("cohort_gene_search %s  %s: done adding extra info \n" % (project.project_id, cohort.cohort_id))
        return JSONResponse({
            'is_error': False,
            'genes': genes,
            'search_hash': search_hash,
        })

    else:
        return JSONResponse({
            'is_error': True,
            'error': server_utils.form_error_string(form)
        })