def get_variants_from_note_tuples(project, note_tuples):
    variants = []
    for note_t in note_tuples:
        variant = get_datastore(project.project_id).get_single_variant(
            project.project_id, note_t[3], note_t[0], note_t[1], note_t[2]
        )
        if not variant:
            variant = Variant(note_t[0], note_t[1], note_t[2])
            get_annotator().annotate_variant(variant, project.get_reference_population_slugs())
            # variant.annotation = get_annotator().get_variant(note_t[0], note_t[1], note_t[2])
        variant.set_extra("family_id", note_t[3])
        variant.set_extra("project_id", project.project_id)
        variants.append(variant)
    return variants

def mendelian_variant_search_spec(request):

    project, family = get_project_and_family_for_user(request.user, request.GET)

    # TODO: use form

    search_hash = request.GET.get('search_hash')
    search_spec_dict, variants = cache_utils.get_cached_results(project.project_id, search_hash)
    search_spec = MendelianVariantSearchSpec.fromJSON(search_spec_dict)
    if variants is None:
        variants = api_utils.calculate_mendelian_variant_search(search_spec, family.xfamily())
    else:
        variants = [Variant.fromJSON(v) for v in variants]
    add_extra_info_to_variants_family(get_reference(), family, variants)
    return_type = request.GET.get('return_type')
    if return_type == 'json' or not return_type:
        return JSONResponse({
            'is_error': False,
            'variants': [v.toJSON() for v in variants],
            'search_spec': search_spec_dict,
        })
    elif request.GET.get('return_type') == 'csv':
        response = HttpResponse(content_type='text/csv')
        response['Content-Disposition'] = 'attachment; filename="results_{}.csv"'.format(search_hash)
        writer = csv.writer(response)
        indiv_ids = family.indiv_ids_with_variant_data()
        headers = xbrowse_displays.get_variant_display_headers(get_mall(project.project_id), project, indiv_ids)
        writer.writerow(headers)
        for variant in variants:
            fields = xbrowse_displays.get_display_fields_for_variant(get_mall(project.project_id), project, variant, indiv_ids)
            writer.writerow(fields)
        return response

    def get_single_variant(self, project_id, family_id, xpos, ref, alt):

        collection = self._get_family_collection(project_id, family_id)
        variant = collection.find_one({'xpos': xpos, 'ref': ref, 'alt': alt})
        if variant:
            return Variant.fromJSON(variant)
        else:
            return None

    def get_variants(self, project_id, family_id, genotype_filter=None, variant_filter=None):

        db_query = _make_db_query(genotype_filter, variant_filter)
        collection = self._get_family_collection(project_id, family_id)
        for variant_dict in collection.find(db_query).sort('xpos'):
            variant = Variant.fromJSON(variant_dict)
            if passes_variant_filter(variant, variant_filter)[0]:
                yield variant

 def get_variants_in_range(self, project_id, family_id, xpos_start, xpos_end):
     collection = self._get_family_collection(project_id, family_id)
     if not collection:
         raise ValueError("Family not found: " + str(family_id))
     for variant_dict in collection.find({'$and': [{'xpos': {'$gte': xpos_start}}, {'xpos': {'$lte': xpos_end}}]}):
         variant = Variant.fromJSON(variant_dict)
         self.add_annotations_to_variant(variant, project_id)
         yield variant

def get_variants_from_variant_tuples(project, variant_tuples, user=None):
    datastore = get_datastore(project)
    population_slugs = project.get_reference_population_slugs()

    variant_tuples_by_family_id = {}
    for xpos, ref, alt, family_id in variant_tuples:
        if family_id not in variant_tuples_by_family_id:
            variant_tuples_by_family_id[family_id] = []
        variant_tuples_by_family_id[family_id].append((xpos, ref, alt))

    variants = []
    for family_id, variant_tuples in variant_tuples_by_family_id.items():
        variants_for_family = datastore.get_multiple_variants(
            project.project_id,
            family_id,
            variant_tuples,
            user=user
        )
        for (xpos, ref, alt), variant in zip(variant_tuples, variants_for_family):
            if not variant:
                variant = Variant(xpos, ref, alt)
                get_annotator().annotate_variant(variant, population_slugs)
                variant.set_extra('created_variant', True)

            variant.set_extra('family_id', family_id)
            variant.set_extra('project_id', project.project_id)
            variants.append(variant)

    return variants

    def get_variants_cohort(self, project_id, cohort_id, variant_filter=None):

        db_query = self._make_db_query(None, variant_filter)
        collection = self._get_family_collection(project_id, cohort_id)
        for i, variant in enumerate(collection.find(db_query).sort('xpos').limit(settings.VARIANT_QUERY_RESULTS_LIMIT+5)):
            if i > settings.VARIANT_QUERY_RESULTS_LIMIT:
                raise Exception("ERROR: this search exceeded the %s variant result size limit. Please set additional filters and try again." % settings.VARIANT_QUERY_RESULTS_LIMIT)

            yield Variant.fromJSON(variant)

def get_variants_from_variant_tuples(project, variant_tuples):
    variants = []
    for t in variant_tuples:
        variant = get_datastore(project.project_id).get_single_variant(
            project.project_id,
            t[3],
            t[0],
            t[1],
            t[2]
        )
        if not variant:
            variant = Variant(t[0], t[1], t[2])
            get_annotator().annotate_variant(variant, project.get_reference_population_slugs())
            
        variant.set_extra('family_id', t[3])
        variant.set_extra('project_id', project.project_id)
        variants.append(variant)
    return variants

 def get_single_variant(self, project_id, family_id, xpos, ref, alt, user=None):
     collection = self._get_family_collection(project_id, family_id)
     if not collection:
         return None
     variant_dict = collection.find_one({'xpos': xpos, 'ref': ref, 'alt': alt})
     if variant_dict:
         variant = Variant.fromJSON(variant_dict)
         self.add_annotations_to_variants([variant], project_id, family_id=family_id)
         return variant
     else:
         return None

    def get_variants_in_range(self, project_id, family_id, xpos_start, xpos_end):
        collection = self._get_family_collection(project_id, family_id)
        if not collection:
            raise ValueError("Family not found: " + str(family_id))
        for i, variant_dict in enumerate(collection.find({'$and': [{'xpos': {'$gte': xpos_start}}, {'xpos': {'$lte': xpos_end}}]}).limit(MONGO_QUERY_RESULTS_LIMIT+5)):
            if i > MONGO_QUERY_RESULTS_LIMIT:
                raise Exception("ERROR: this search exceeded the %s variant result size limit. Please set additional filters and try again." % MONGO_QUERY_RESULTS_LIMIT)

            variant = Variant.fromJSON(variant_dict)
            self.add_annotations_to_variant(variant, project_id)
            yield variant

    def get_variants(self, project_id, family_id, genotype_filter=None, variant_filter=None):

        db_query = self._make_db_query(genotype_filter, variant_filter)
        collection = self._get_family_collection(project_id, family_id)
        if not collection:
            print("Error: mongodb collection not found for project %s family %s " % (project_id, family_id))
            return
        for variant_dict in collection.find(db_query).sort('xpos'):
            variant = Variant.fromJSON(variant_dict)
            self.add_annotations_to_variant(variant, project_id)
            if passes_variant_filter(variant, variant_filter)[0]:
                yield variant

    def get_variants(self, project_id, variant_filter=None):

        variant_filter_t = VariantFilter(**(variant_filter if variant_filter else {}))

        db_query = self._make_db_query(None, variant_filter)
        collection = self._get_project_collection(project_id)
        for variant_dict in collection.find(db_query).sort('xpos'):
            variant = Variant.fromJSON(variant_dict)
            if variant_filter is None:
                yield variant
            if passes_variant_filter(variant, variant_filter_t)[0]:
                yield variant

    def get_variants(self, project_id, family_id, genotype_filter=None, variant_filter=None):

        db_query = self._make_db_query(genotype_filter, variant_filter)
        collection = self._get_family_collection(project_id, family_id)
        if not collection:
            print("Error: mongodb collection not found for project %s family %s " % (project_id, family_id))
            return
        for i, variant_dict in enumerate(collection.find(db_query).sort('xpos').limit(MONGO_QUERY_RESULTS_LIMIT+5)):
            if i >= MONGO_QUERY_RESULTS_LIMIT:
                raise Exception("ERROR: this search exceeded the %s variant result size limit. Please set additional filters and try again." % MONGO_QUERY_RESULTS_LIMIT)

            variant = Variant.fromJSON(variant_dict)
            self.add_annotations_to_variant(variant, project_id)
            if passes_variant_filter(variant, variant_filter)[0]:
                yield variant

    def get_variants(self, project_id, family_id, genotype_filter=None, variant_filter=None, quality_filter=None, indivs_to_consider=None, user=None):
        db_query = self._make_db_query(genotype_filter, variant_filter)
        collection = self._get_family_collection(project_id, family_id)
        if not collection:
            logger.error("Error: mongodb collection not found for project %s family %s " % (project_id, family_id))
            return
        for i, variant_dict in enumerate(collection.find({'$and' : [{k: v} for k, v in db_query.items()]}).sort('xpos').limit(settings.VARIANT_QUERY_RESULTS_LIMIT+5)):
            if i >= settings.VARIANT_QUERY_RESULTS_LIMIT:
                raise Exception("ERROR: this search exceeded the %s variant result size limit. Please set additional filters and try again." % settings.VARIANT_QUERY_RESULTS_LIMIT)

            variant = Variant.fromJSON(variant_dict)
            variant.set_extra('project_id', project_id)
            variant.set_extra('family_id', family_id)
            self.add_annotations_to_variants([variant], project_id)

            if passes_variant_filter(variant, variant_filter)[0]:
                yield variant

    def get_variants_in_gene(self, project_id, gene_id, variant_filter=None):

        if variant_filter is None:
            modified_variant_filter = VariantFilter()
        else:
            modified_variant_filter = copy.deepcopy(variant_filter)
        modified_variant_filter.add_gene(gene_id)

        db_query = self._make_db_query(None, modified_variant_filter)
        collection = self._get_project_collection(project_id)

        variants = []
        for variant_dict in collection.find(db_query).hint([('gene_ids', pymongo.ASCENDING), ('xpos', pymongo.ASCENDING)]):
            variant = Variant.fromJSON(variant_dict)
            if passes_variant_filter(variant, modified_variant_filter):
                variants.append(variant)
        variants = sorted(variants, key=lambda v: v.unique_tuple())
        return variants

def cohort_variant_search_spec(request):

    project, cohort = get_project_and_cohort_for_user(request.user, request.GET)

    # TODO: use form

    search_spec_dict, variants = cache_utils.get_cached_results(project.project_id, request.GET.get('search_hash'))
    search_spec = MendelianVariantSearchSpec.fromJSON(search_spec_dict)
    if variants is None:
        variants = api_utils.calculate_mendelian_variant_search(search_spec, cohort.xfamily())
    else:
        variants = [Variant.fromJSON(v) for v in variants]
    api_utils.add_extra_info_to_variants_cohort(get_reference(), cohort, variants)

    return JSONResponse({
        'is_error': False,
        'variants': [v.toJSON() for v in variants],
        'search_spec': search_spec.toJSON(),
    })

    def get_project_variants_in_gene(self, project_id, gene_id, variant_filter=None):

        if variant_filter is None:
            modified_variant_filter = VariantFilter()
        else:
            modified_variant_filter = copy.deepcopy(variant_filter)
        modified_variant_filter.add_gene(gene_id)

        db_query = self._make_db_query(None, modified_variant_filter)
        logger.info("Project Gene Search: " + str(project_id) + " all variants query: " + str(db_query))
        collection = self._get_project_collection(project_id)
        # we have to collect list in memory here because mongo can't sort on xpos,
        # as result size can get too big.
        # need to find a better way to do this.
        variants = [Variant.fromJSON(variant_dict) for variant_dict in collection.find(db_query).hint([('db_gene_ids', pymongo.ASCENDING), ('xpos', pymongo.ASCENDING)])]
        self.add_annotations_to_variants(variants, project_id)
        variants = filter(lambda variant: passes_variant_filter(variant, modified_variant_filter), variants)
        variants = sorted(variants, key=lambda v: v.unique_tuple())
        return variants

    def get_variants_in_gene(self, project_id, family_id, gene_id, genotype_filter=None, variant_filter=None):

        if variant_filter is None:
            modified_variant_filter = VariantFilter()
        else:
            modified_variant_filter = copy.deepcopy(variant_filter)
        modified_variant_filter.add_gene(gene_id)

        db_query = _make_db_query(genotype_filter, modified_variant_filter)
        collection = self._get_family_collection(project_id, family_id)

        # we have to collect list in memory here because mongo can't sort on xpos,
        # as result size can get too big.
        # need to find a better way to do this.
        variants = []
        for variant_dict in collection.find(db_query).hint([('gene_ids', pymongo.ASCENDING), ('xpos', pymongo.ASCENDING)]):
            variant = Variant.fromJSON(variant_dict)
            if passes_variant_filter(variant, modified_variant_filter):
                variants.append(variant)
        variants = sorted(variants, key=lambda v: v.unique_tuple())
        for v in variants:
            yield v

    def get_elasticsearch_variants(
            self,
            project_id,
            family_id=None,
            variant_filter=None,
            genotype_filter=None,
            variant_id_filter=None,
            quality_filter=None,
            indivs_to_consider=None,
            include_all_consequences=False,
            user=None,
            max_results_limit=settings.VARIANT_QUERY_RESULTS_LIMIT,
        ):
        from xbrowse_server.base.models import Project, Family, Individual
        from seqr.models import Sample
        from seqr.utils.es_utils import _liftover_grch38_to_grch37
        from xbrowse_server.mall import get_reference

        redis_client = None
        if settings.REDIS_SERVICE_HOSTNAME:
            try:
                redis_client = redis.StrictRedis(host=settings.REDIS_SERVICE_HOSTNAME, socket_connect_timeout=3)
                redis_client.ping()
            except redis.exceptions.TimeoutError as e:
                logger.warn("Unable to connect to redis host: {}".format(settings.REDIS_SERVICE_HOSTNAME) + str(e))
                redis_client = None

        cache_key = "Variants___%s___%s___%s" % (
            project_id,
            family_id,
            json.dumps([
                variant_filter.toJSON() if variant_filter else None,
                genotype_filter,
                quality_filter,
                variant_id_filter,
                indivs_to_consider,
                include_all_consequences,
            ])
        )

        cached_results = redis_client and redis_client.get(cache_key)
        if cached_results is not None:
            variant_results = json.loads(cached_results)
            return [Variant.fromJSON(variant_json) for variant_json in variant_results]

        if family_id is None:
            project = Project.objects.get(project_id=project_id)
            elasticsearch_index = project.get_elasticsearch_index()
            logger.info("Searching in project elasticsearch index: " + str(elasticsearch_index))
        else:
            family = Family.objects.get(project__project_id=project_id, family_id=family_id)
            elasticsearch_index = family.get_elasticsearch_index()
            project = family.project
            logger.info("Searching in family elasticsearch index: " + str(elasticsearch_index))

        if indivs_to_consider is None and genotype_filter and not family_id:
            indivs_to_consider = genotype_filter.keys()

        individuals = Individual.objects.filter(family__project__project_id=project_id).only("indiv_id", "seqr_individual")
        if indivs_to_consider:
            individuals = individuals.filter(indiv_id__in=indivs_to_consider)
        if family_id is not None:
            individuals = individuals.filter(family__family_id=family_id)
            if not indivs_to_consider:
                indivs_to_consider = [i.indiv_id for i in individuals]
        prefetch_related_objects(individuals, "seqr_individual")

        es_indices = [index.rstrip('*') for index in elasticsearch_index.split(',')]

        samples = Sample.objects.filter(
            individual__in=[i.seqr_individual for i in individuals if i.seqr_individual],
            dataset_type=Sample.DATASET_TYPE_VARIANT_CALLS,
            sample_status=Sample.SAMPLE_STATUS_LOADED,
            elasticsearch_index__startswith=es_indices[0],
            loaded_date__isnull=False,
        ).order_by('-loaded_date')
        prefetch_related_objects(samples, "individual")

        family_individual_ids_to_sample_ids = {}
        for i in individuals:
            indiv_id = i.indiv_id
            sample_id = None
            if i.seqr_individual:
                sample_id = next((
                    sample.sample_id for sample in samples
                    if sample.individual == i.seqr_individual and sample.elasticsearch_index.startswith(tuple(es_indices))
                ), None)
            family_individual_ids_to_sample_ids[indiv_id] = sample_id or indiv_id

        query_json = self._make_db_query(genotype_filter, variant_filter)

        es_client = elasticsearch.Elasticsearch(host=settings.ELASTICSEARCH_SERVICE_HOSTNAME, timeout=30)
        mapping = es_client.indices.get_mapping(str(elasticsearch_index) + "*")
        index_fields = {}
        is_parent_child = False
        is_nested = False
        if elasticsearch_index in mapping and 'join_field' in mapping[elasticsearch_index]["mappings"]["variant"]["properties"]:
            # Nested indices are not sharded so all samples are in the single index
            logger.info("matching indices: " + str(elasticsearch_index))
            is_parent_child = True
#.........这里部分代码省略.........

def add_or_edit_variant_note(request):
    """Add a variant note"""
    family = None
    if 'family_id' in request.GET:
        project, family = get_project_and_family_for_user(request.user, request.GET)
    else:
        project = utils.get_project_for_user(request.user, request.GET)

    form = api_forms.VariantNoteForm(project, request.GET)
    if not form.is_valid():
        return JSONResponse({
            'is_error': True,
            'error': server_utils.form_error_string(form)
        })

    variant = get_datastore(project.project_id).get_single_variant(
        project.project_id,
        family.family_id,
        form.cleaned_data['xpos'],
        form.cleaned_data['ref'],
        form.cleaned_data['alt'],
    )

    if not variant:
        variant = Variant.fromJSON({
            'xpos' : form.cleaned_data['xpos'], 'ref': form.cleaned_data['ref'], 'alt': form.cleaned_data['alt'],
            'genotypes': {}, 'extras': {},
        })

    if 'note_id' in form.cleaned_data and form.cleaned_data['note_id']:
        event_type = "edit_variant_note"

        notes = VariantNote.objects.filter(
            id=form.cleaned_data['note_id'],
            project=project,
            xpos=form.cleaned_data['xpos'],
            ref=form.cleaned_data['ref'],
            alt=form.cleaned_data['alt'],
        )
        if not notes:
            return JSONResponse({
                'is_error': True,
                'error': 'note id %s not found' % form.cleaned_data['note_id']
            })

        note = notes[0]
        note.user = request.user
        note.note = form.cleaned_data['note_text']
        note.date_saved = timezone.now()
        if family:
            note.family = family
        note.save()
    else:
        event_type = "add_variant_note"

        VariantNote.objects.create(
            user=request.user,
            project=project,
            xpos=form.cleaned_data['xpos'],
            ref=form.cleaned_data['ref'],
            alt=form.cleaned_data['alt'],
            note=form.cleaned_data['note_text'],
            date_saved=timezone.now(),
            family=family,
        )

    add_extra_info_to_variants_family(get_reference(), family, [variant,])

    try:
        settings.EVENTS_COLLECTION.insert({
            'event_type': event_type,
            'date': timezone.now(),
            'project_id': ''.join(project.project_id),
            'family_id': family.family_id,
            'note': form.cleaned_data['note_text'],

            'xpos':form.cleaned_data['xpos'],
            'pos':variant.pos,
            'chrom': variant.chr,
            'ref':form.cleaned_data['ref'],
            'alt':form.cleaned_data['alt'],
            'gene_names': ", ".join(variant.extras['gene_names'].values()),
            'username': request.user.username,
            'email': request.user.email,
        })
    except Exception as e:
        logging.error("Error while logging %s event: %s" % (event_type, e))


    return JSONResponse({
        'is_error': False,
        'variant': variant.toJSON(),
    })