def get_udfc_search_fields(self, user):
        from treemap.models import InstanceUser
        from treemap.udf import UDFModel
        from treemap.util import to_object_name, leaf_models_of_class
        from treemap.lib.perms import udf_write_level, READ, WRITE

        try:
            iu = self.instanceuser_set.get(user__pk=user.pk)
        except InstanceUser.DoesNotExist:
            iu = None

        data = DotDict({"models": set(), "udfc": {}})
        for clz in leaf_models_of_class(UDFModel):
            model_name = clz.__name__
            for k, v in clz.collection_udf_settings.items():
                udfds = (u for u in udf_defs(self, model_name) if u.name == k)
                for udfd in udfds:
                    if udf_write_level(iu, udfd) in (READ, WRITE):
                        _base_nest_path = "udfc.%s." % (to_object_name(k))
                        ids_nest_path = "%sids.%s" % (_base_nest_path, to_object_name(model_name))
                        models_nest_path = "%smodels.%s" % (_base_nest_path, to_object_name(model_name))
                        data[ids_nest_path] = udfd.pk
                        data[models_nest_path] = {"udfd": udfd, "fields": udfd.datatype_dict[0]["choices"]}
                        p = "udfc.%s." % to_object_name(k)
                        data[p + "action_verb"] = v["action_verb"]
                        data[p + "range_field_key"] = v["range_field_key"]
                        data[p + "action_field_key"] = v["action_field_key"]
                        data["models"] |= {clz}

        return data

    def get_udfc_search_fields(self, user):
        from treemap.models import InstanceUser
        from treemap.udf import UDFModel
        from treemap.util import to_object_name, leaf_models_of_class
        from treemap.lib.perms import udf_write_level, READ, WRITE

        try:
            iu = self.instanceuser_set.get(user__pk=user.pk)
        except InstanceUser.DoesNotExist:
            iu = None

        data = DotDict({'models': set(), 'udfc': {}})
        for clz in (leaf_models_of_class(UDFModel)):
            model_name = clz.__name__
            items = (
                (k, v) for k, v
                in getattr(clz, 'collection_udf_settings', {}).iteritems())
            for k, v in items:
                udfds = (u for u in udf_defs(self, model_name) if u.name == k)
                for udfd in udfds:
                    if udf_write_level(iu, udfd) in (READ, WRITE):
                        nest_path = ('udfc.%s.models.%s' %
                                     (to_object_name(k),
                                      to_object_name(model_name)))
                        data[nest_path] = {
                            'udfd': udfd,
                            'fields': udfd.datatype_dict[0]['choices']
                        }
                        p = 'udfc.%s.' % to_object_name(k)
                        data[p + 'action_verb'] = v['action_verb']
                        data[p + 'range_field_key'] = v['range_field_key']
                        data[p + 'action_field_key'] = v['action_field_key']
                        data['models'] |= {clz}

        return data

def udf_list(request, instance):
    editable_udf_models = instance.editable_udf_models()
    udf_models = \
        sorted([{'name': clz.__name__,
                 'display_name': clz.display_name(instance)}
                for clz in editable_udf_models['core']],
               key=lambda model: model['name'],
               reverse=True) + \
        sorted([{'name': clz.__name__,
                 'display_name': clz.display_name(instance)}
                for clz in editable_udf_models['gsi']],
               key=lambda model: model['name'])

    editable_gsi_models = [clz.__name__ for clz in editable_udf_models['gsi']]

    udf_model_names = sorted([model['name'] for model in udf_models])

    udfs = sorted([udf for udf in udf_defs(instance)
                   if udf.model_type in udf_model_names],
                  key=lambda udf: (udf.model_type, udf.iscollection,
                                   udf.name))

    def dict_update(d1, d2):
        d1.update(d2)
        return d1

    udf_models = [dict_update(model, {
        'specs': [{'udf': udf, 'datatype': _get_type_display(udf)}
                  for udf in udfs if udf.model_type == model['name']]
        }) for model in udf_models]

    return {
        "udf_models": udf_models,
        "editable_gsi_models": editable_gsi_models
    }

    def _commit_tree_data(self, data, plot, tree, tree_edited):
        for tree_attr, field_name in TreeImportRow.TREE_MAP.iteritems():
            value = data.get(field_name, None)
            if value:
                tree_edited = True
                if tree is None:
                    tree = Tree(instance=plot.instance)
                setattr(tree, tree_attr, value)

        ie = self.import_event
        tree_udf_defs = udf_defs(ie.instance, 'Tree')
        for udf_def in tree_udf_defs:
            udf_column_name = ie.get_udf_column_name(udf_def)
            value = data.get(udf_column_name, None)
            # Legitimate values could be falsey
            if value is not None:
                tree_edited = True
                if tree is None:
                    tree = Tree(instance=plot.instance)
                tree.udfs[udf_def.name] = \
                    self._import_value_to_udf_value(udf_def, value)

        if tree_edited:
            tree.plot = plot
            tree.save_with_system_user_bypass_auth()
            tree.plot.update_updated_fields(ie.owner)

    def _get_udfc_search_fields(self):
        from treemap.util import to_object_name

        empty_udfc = {to_object_name(n_k):
                      {to_object_name(m_k): {'fields': [], 'udfd': None}
                       for m_k in UDFC_MODELS}
                      for n_k in UDFC_NAMES}

        udfds = []
        for model_name in UDFC_MODELS:
            for udfd in udf_defs(self, model_name):
                if udfd.name in UDFC_NAMES:
                    udfds.append(udfd)

        udfc = deepcopy(empty_udfc)

        for udfd in udfds:
            udfd_info = {
                'udfd': udfd,
                'fields': udfd.datatype_dict[0]['choices']
            }
            name_dict = udfc[to_object_name(udfd.name)]
            name_dict[to_object_name(udfd.model_type)] = udfd_info

        return udfc

    def alert_identifiers():
        def identifier(udf):
            model_name = udf.model_type.lower()
            return 'udf:%(model)s:%(pk)s.Status' % {
                'model': model_name, 'pk': udf.pk}

        return [identifier(udf) for udf in udf_defs(instance)
                if udf.iscollection and udf.name == 'Alerts']

def _values_for_model(instance, job, table, model, select, select_params, prefix=None):
    if prefix:
        prefix += "__"
    else:
        prefix = ""

    prefixed_names = []
    model_class = safe_get_model_class(model)
    dummy_instance = model_class()

    for field_name in (
        perm.field_name
        for perm in field_permissions(job.user, instance, model)
        if perm.permission_level >= FieldPermission.READ_ONLY
    ):
        prefixed_name = prefix + field_name

        if field_name.startswith("udf:"):
            name = field_name[4:]
            if name in model_class.collection_udf_settings.keys():
                field_definition_id = None
                for udfd in udf_defs(instance, model):
                    if udfd.iscollection and udfd.name == name:
                        field_definition_id = udfd.id

                if field_definition_id is None:
                    continue

                select[
                    prefixed_name
                ] = """
                    WITH formatted_data AS (
                        SELECT concat('(', data, ')') as fdata
                        FROM %s
                        WHERE field_definition_id = %s and model_id = %s.id
                    )

                    SELECT array_to_string(array_agg(fdata), ', ', '*')
                    FROM formatted_data
                    """ % (
                    UserDefinedCollectionValue._meta.db_table,
                    field_definition_id,
                    table,
                )
            else:
                select[prefixed_name] = "{0}.udfs->%s".format(table)
                select_params.append(name)
        else:
            if not model_hasattr(dummy_instance, field_name):
                # Exception will be raised downstream if you look for
                # a field on a model that no longer exists but still
                # has a stale permission record. Here we check for that
                # case and don't include the field if it does not exist.
                continue

        prefixed_names.append(prefixed_name)

    return prefixed_names

def extra_select_and_values_for_model(
        instance, job, table, model, prefix=None):
    if prefix:
        prefix += '__'
    else:
        prefix = ''

    perms = permissions(job.user, instance, model)

    extra_select = {}
    prefixed_names = []
    dummy_instance = safe_get_model_class(model)()

    for perm in (perm for perm in perms
                 if perm.permission_level >= FieldPermission.READ_ONLY):
        field_name = perm.field_name
        prefixed_name = prefix + field_name

        if field_name in _UDFC_FIELDS:

            field_definition_id = None
            for udfd in udf_defs(instance, model):
                if udfd.iscollection and udfd.name == field_name[4:]:
                    field_definition_id = udfd.id

            if field_definition_id is None:
                continue

            extra_select[prefixed_name] = (
                """
                WITH formatted_data AS (
                    SELECT concat('(', data, ')') as fdata
                    FROM %s
                    WHERE field_definition_id = %s and model_id = %s.id
                )

                SELECT array_to_string(array_agg(fdata), ', ', '*')
                FROM formatted_data
                """
                % (UserDefinedCollectionValue._meta.db_table,
                   field_definition_id, table))
        elif field_name.startswith('udf:'):
            name = field_name[4:]
            extra_select[prefixed_name] = "%s.udfs->'%s'" % (table, name)
        else:
            if not model_hasattr(dummy_instance, field_name):
                # Exception will be raised downstream if you look for
                # a field on a model that no longer exists but still
                # has a stale permission record. Here we check for that
                # case and don't include the field if it does not exist.
                continue

        prefixed_names.append(prefixed_name)

    return (extra_select, prefixed_names)

 def validate_user_defined_fields(self):
     ie = self.import_event
     for udf_def in udf_defs(ie.instance):
         column_name = ie.get_udf_column_name(udf_def)
         value = self.datadict.get(column_name, None)
         if value:
             try:
                 udf_def.clean_value(value)
                 self.cleaned[column_name] = value
             except ValidationError as ve:
                 self.append_error(
                     errors.INVALID_UDF_VALUE, column_name, str(ve))

    def info(group):
        group['fields'] = [
            (field, labels.get(field),
             templates.get(field, "treemap/field/tr.html"))
            for field in group.get('field_keys', [])
        ]
        group['collection_udfs'] = [
            next(udf for udf in udf_defs(instance)
                 if udf.full_name == udf_name)
            for udf_name in group.get('collection_udf_keys', [])
        ]

        return group

    def get_disabled_cudfs(group):
        if 'model' in group:
            models = (to_model_name(group['model']), )
        else:
            models = ('Tree', 'Plot')

        udfs = {udf.full_name for udf in udf_defs(instance)
                if udf.iscollection and udf.model_type in models and
                (group['header'] not in collection_groups
                 or udf.name == group['header'])}

        disabled_cudfs = udfs - set(group['collection_udf_keys'])
        return sorted(disabled_cudfs)

def _get_fields(instance, model_name):
    Model = safe_get_model_class(model_name)
    mobj = Model(instance=instance)
    udfs = {udf.canonical_name
            for udf in udf_defs(instance, model_name)
            if not udf.iscollection}
    concrete_fields = {
        f.name for f in mobj._meta.get_fields(include_parents=False)
        if _should_show_field(Model, f.name) and not f.is_relation}
    model_fields = concrete_fields | udfs
    model_fields = {'%s.%s' % (to_object_name(model_name), f)
                    for f in model_fields}
    return model_fields

def get_alert_field_info(identifier, instance):
    from treemap.util import get_model_for_instance
    alert_match = ALERT_IDENTIFIER_PATTERN.match(identifier)
    if alert_match:
        model_name, pk = alert_match.groups()
        Model = get_model_for_instance(model_name, instance)
        udf_def = next(udf for udf in udf_defs(instance) if udf.pk == int(pk))
        display_name = force_text(Model.terminology(instance)['singular'])
        return {
            'identifier': identifier,
            'search_type': 'DEFAULT',
            'default_identifier': udf_def.full_name,
            'label': 'Open %(model)s Alerts' % {'model': display_name},
        }
    return None

 def validate_user_defined_fields(self):
     ie = self.import_event
     for udf_def in udf_defs(ie.instance):
         column_name = ie.get_udf_column_name(udf_def)
         value = self.datadict.get(column_name, None)
         if value:
             try:
                 udf_def.clean_value(value)
                 self.cleaned[column_name] = value
             except ValidationError as ve:
                 message = str(ve)
                 if isinstance(ve.message_dict, dict):
                     message = '\n'.join(
                         [unicode(m) for m in ve.message_dict.values()])
                 self.append_error(
                     errors.INVALID_UDF_VALUE, column_name, message)

 def get_display_model_name(audit_name, instance=None):
     if audit_name.startswith('udf:'):
         try:
             # UDF Collections store their model names in the audit table as
             # udf:<pk of UserDefinedFieldDefinition>
             pk = int(audit_name[4:])
             if not instance:
                 udf_def = UserDefinedFieldDefinition.objects.get(pk=pk)
                 return udf_def.name
             else:
                 for udf_def in udf_defs(instance):
                     if udf_def.pk == pk:
                         return udf_def.name
         except (ValueError, UserDefinedFieldDefinition.DoesNotExist):
             pass  # If something goes wrong, just use the defaults
     return audit_name

    def delete(self, *args, **kwargs):

        if self.iscollection:
            UserDefinedCollectionValue.objects.filter(field_definition=self).delete()

            Audit.objects.filter(instance=self.instance).filter(model="udf:%s" % self.pk).delete()

            if "mobile_api_fields" in self.instance.config:
                for group in self.instance.mobile_api_fields:
                    if self.full_name in group.get("collection_udf_keys", []):
                        # If this is the only collection UDF with this name,
                        # we remove the entire group, since there would be no
                        # eligible items to go *in* the group after deletion
                        if len([udf for udf in udf_defs(self.instance) if udf.name == self.name]) == 1:
                            self.instance.mobile_api_fields.remove(group)
                        # Otherwise, just remove this UDF from the group
                        else:
                            group["collection_udf_keys"].remove(self.full_name)
                    self.instance.save()
        else:
            Model = safe_get_udf_model_class(self.model_type)
            objects_with_udf_data = Model.objects.filter(instance=self.instance).filter(udfs__contains=[self.name])

            for obj in objects_with_udf_data:
                del obj.udfs[self.name]
                # save_base instead of save_with_user,
                # we delete the audits anyways
                obj.save_base()

            Audit.objects.filter(instance=self.instance).filter(model=self.model_type).filter(
                field=self.canonical_name
            ).delete()

            # If there is no mobile_api_field in the config, that means the
            # instance is using the default, which should not mutate
            if "mobile_api_fields" in self.instance.config:
                for group in self.instance.mobile_api_fields:
                    if self.full_name in group.get("field_keys", []):
                        group["field_keys"].remove(self.full_name)
                        self.instance.save()

        # remove field permissions for this udf
        FieldPermission.objects.filter(
            model_name=self.model_type, field_name=self.canonical_name, instance=self.instance
        ).delete()

        super(UserDefinedFieldDefinition, self).delete(*args, **kwargs)

def udf_bulk_update(request, instance):
    '''
    udf_bulk_update(request, instance)

    'instance': a treemap instance
    'request': an HTTP request object whose body is a JSON representation
               of a dict containing the key 'choice_changes'.

    choice_changes is a list of directives per choice-type
    UserDefinedFieldDefinition.  Each directive is a dict, defined as follows:
    {
        'id': id of a UserDefinedFieldDefinition,
        'changes': a list of changes to make to the UserDefinedFieldDefinition
                   with that id.
    }

    Each change is either a delete, rename, or add request pertaining to
    one choice of the UserDefinedFieldDefinition.

    There should be no more than one change per choice,
    and the list should be ordered as deletes, then renames, then adds.
    See the docstring for `_udf_update_choice` for the structure of each
    choice change parameter.
    '''
    params = json.loads(request.body)
    choice_changes = params.get('choice_changes', None)

    if choice_changes:
        choice_map = {int(param['id']): param['changes']
                      for param in choice_changes}
        udfds = [udf for udf in udf_defs(instance)
                 if udf.pk in choice_map.keys()]

        # Update one at a time rather than doing bulk_update.
        # There won't be that many of them, and we need to go through
        # all the UDF machinery to update models and audit records.

        # Also, assume that the frontend will not send more than one change
        # (rename or delete) for the same choice,
        # or changes (rename or delete) for any new choices.
        for udf in udfds:
            for params in choice_map[udf.pk]:
                _udf_update_choice(udf, instance, params)

    return HttpResponse(_('Updated Custom Fields'))

    def _commit_plot_data(self, data, plot):
        plot_edited = False
        for plot_attr, field_name in TreeImportRow.PLOT_MAP.iteritems():
            value = data.get(field_name, None)
            if value:
                plot_edited = True
                setattr(plot, plot_attr, value)

        ie = self.import_event
        plot_udf_defs = udf_defs(ie.instance, 'Plot')
        for udf_def in plot_udf_defs:
            udf_column_name = ie.get_udf_column_name(udf_def)
            value = data.get(udf_column_name, None)
            if value:
                plot_edited = True
                plot.udfs[udf_def.name] = value

        if plot_edited:
            plot.save_with_system_user_bypass_auth()

def get_udfc_search_fields(instance, user):
    from treemap.models import InstanceUser
    from treemap.udf import UDFModel
    from treemap.util import to_object_name, leaf_models_of_class
    from treemap.lib.perms import udf_write_level, READ, WRITE

    try:
        iu = instance.instanceuser_set.get(user__pk=user.pk)
    except InstanceUser.DoesNotExist:
        iu = None

    data = DotDict({'models': set(), 'udfc': {}})
    for clz in leaf_models_of_class(UDFModel):
        model_name = clz.__name__
        if model_name not in ['Tree'] + instance.map_feature_types:
            continue
        for k, v in clz.collection_udf_settings.items():
            udfds = (u for u in udf_defs(instance, model_name) if u.name == k)
            for udfd in udfds:
                if udf_write_level(iu, udfd) in (READ, WRITE):
                    _base_nest_path = 'udfc.%s.' % (to_object_name(k))
                    ids_nest_path = ('%sids.%s'
                                     % (_base_nest_path,
                                        to_object_name(model_name)))
                    models_nest_path = ('%smodels.%s' %
                                        (_base_nest_path,
                                         to_object_name(model_name)))
                    data[ids_nest_path] = udfd.pk
                    data[models_nest_path] = {
                        'udfd': udfd,
                        'fields': udfd.datatype_dict[0]['choices']
                    }
                    p = 'udfc.%s.' % to_object_name(k)
                    data[p + 'action_verb'] = v['action_verb']
                    data[p + 'range_field_key'] = v['range_field_key']
                    data[p + 'action_field_key'] = v['action_field_key']
                    data['models'] |= {clz}

    return data

 def get_user_defined_fields(self):
     if hasattr(self, 'instance'):
         return udf_defs(self.instance, self._model_name)
     else:
         return []