def get_collectionlist(choice):

	# Object is called to update in "addcollection" part
	colllist_obj = CollectionList.objects.first()

	colllist = colllist_obj["collectionlist"]

	if(choice == "info"):
		len_coll = []
		len_unlabeled = []
		len_labeled= []

		for coll in colllist:
	
			model = get_document(coll)
			len_unlbld = model.objects(label__exists = False).count()
			len_lbld = model.objects(label__exists = True).count()
	
			len_coll.append(len_unlbld + len_lbld)	
			len_unlabeled.append(len_unlbld)
			len_labeled.append(len_lbld)


		collectionlist = zip(colllist, len_coll, len_unlabeled, len_labeled)

		return collectionlist

	elif(choice == "update"):

		return colllist_obj, colllist

    def _from_son(cls, son, _auto_dereference=True, only_fields=None, created=False):
        """Create an instance of a Document (subclass) from a PyMongo SON.
        """
        if not only_fields:
            only_fields = []

        # get the class name from the document, falling back to the given
        # class if unavailable
        class_name = son.get('_cls', cls._class_name)
        data = dict(("%s" % key, value) for key, value in son.iteritems())

        # Return correct subclass for document type
        if class_name != cls._class_name:
            cls = get_document(class_name)

        changed_fields = []
        errors_dict = {}

        fields = cls._fields
        if not _auto_dereference:
            fields = copy.copy(fields)

        for field_name, field in fields.iteritems():
            field._auto_dereference = _auto_dereference
            if field.db_field in data:
                value = data[field.db_field]
                try:
                    data[field_name] = (value if value is None
                                        else field.to_python(value))
                    if field_name != field.db_field:
                        del data[field.db_field]
                except (AttributeError, ValueError), e:
                    errors_dict[field_name] = e

    def __expand_dynamic_values(self, name, value):
        """Expand any dynamic values to their correct types / values."""
        if not isinstance(value, (dict, list, tuple)):
            return value

        # If the value is a dict with '_cls' in it, turn it into a document
        is_dict = isinstance(value, dict)
        if is_dict and '_cls' in value:
            cls = get_document(value['_cls'])
            return cls(**value)

        if is_dict:
            value = {
                k: self.__expand_dynamic_values(k, v)
                for k, v in value.items()
            }
        else:
            value = [self.__expand_dynamic_values(name, v) for v in value]

        # Convert lists / values so we can watch for any changes on them
        EmbeddedDocumentListField = _import_class('EmbeddedDocumentListField')
        if (isinstance(value, (list, tuple)) and
                not isinstance(value, BaseList)):
            if issubclass(type(self), EmbeddedDocumentListField):
                value = EmbeddedDocumentList(value, self, name)
            else:
                value = BaseList(value, self, name)
        elif isinstance(value, dict) and not isinstance(value, BaseDict):
            value = BaseDict(value, self, name)

        return value

    def __expand_dynamic_values(self, name, value):
        """expand any dynamic values to their correct types / values"""
        if not isinstance(value, (dict, list, tuple)):
            return value

        is_list = False
        if not hasattr(value, 'items'):
            is_list = True
            value = dict([(k, v) for k, v in enumerate(value)])

        if not is_list and '_cls' in value:
            cls = get_document(value['_cls'])
            return cls(**value)

        data = {}
        for k, v in value.items():
            key = name if is_list else k
            data[k] = self.__expand_dynamic_values(key, v)

        if is_list:  # Convert back to a list
            data_items = sorted(data.items(), key=operator.itemgetter(0))
            value = [v for k, v in data_items]
        else:
            value = data

        # Convert lists / values so we can watch for any changes on them
        if (isinstance(value, (list, tuple)) and
           not isinstance(value, BaseList)):
            value = BaseList(value, self, name)
        elif isinstance(value, dict) and not isinstance(value, BaseDict):
            value = BaseDict(value, self, name)

        return value

    def create(self, validated_data):
        class_name = validated_data.pop('_class_name', None)
        if class_name:
            cls = get_document(class_name)
        else:
            cls = self.Meta.model

        return self.get_serializer(cls).create(validated_data)

    def _from_son(cls, son, _auto_dereference=True, only_fields=None, created=False):
        """Create an instance of a Document (subclass) from a PyMongo SON.
        """
        if not only_fields:
            only_fields = []

        # get the class name from the document, falling back to the given
        # class if unavailable
        class_name = son.get('_cls', cls._class_name)
        data = dict(("%s" % key, value) for key, value in son.items())

        # Return correct subclass for document type
        if class_name != cls._class_name:
            cls = get_document(class_name)

        changed_fields = []
        errors_dict = {}

        fields = cls._fields
        if not _auto_dereference:
            fields = copy.copy(fields)

        for field_name, field in fields.items():
            field._auto_dereference = _auto_dereference
            if field.db_field in data:
                value = data[field.db_field]
                try:
                    data[field_name] = (value if value is None
                                        else field.to_python(value))
                    if field_name != field.db_field:
                        del data[field.db_field]
                except (AttributeError, ValueError) as e:
                    errors_dict[field_name] = e
            elif field.default:
                default = field.default
                if isinstance(default, collections.Callable):
                    default = default()
                if isinstance(default, BaseDocument):
                    changed_fields.append(field_name)
                elif not only_fields or field_name in only_fields:
                    changed_fields.append(field_name)

        if errors_dict:
            errors = "\n".join(["%s - %s" % (k, v)
                                for k, v in list(errors_dict.items())])
            msg = ("Invalid data to create a `%s` instance.\n%s"
                   % (cls._class_name, errors))
            raise InvalidDocumentError(msg)

        if cls.STRICT:
            data = dict((k, v)
                        for k, v in data.items() if k in cls._fields)
        obj = cls(__auto_convert=False, _created=created, __only_fields=only_fields, **data)
        obj._changed_fields = changed_fields
        if not _auto_dereference:
            obj._fields = fields

        return obj

	def post(self, request, collname):

			if "confirmpass" in request.POST:				

				user_pass = request.POST['user_pass']

				reset_pass = config.get('dataset', 'reset_pass')

				if(user_pass == reset_pass):

					client_address = get_client_ip(request)

					logging.info('RESET : ' + client_address + ' requested to reset all labels on '   + collname )

					backup_json(collname)
	
					model = get_document(collname)
					
					model.objects.update(unset__label=1)

					logging.info('RESET : Reset done for all labels on ' + collname + ', by ' + client_address)

					sbj = "Reset All Labels"
					msg = sbj + '\n\nCollection : ' + collname + '\nIP address : ' + client_address + "\n\nThis mail sent to : ebasar"

					send_mail(sbj, msg, "ebasar")

					if HOSTNAME[:9] == "applejack":	 
						msg2 = msg + ", hurrial"
						send_mail(sbj, msg2, "hurrial")
						send_mail(sbj, msg2, "ebasar")
					else:
						send_mail(sbj, msg, "ebasar")

					confirmed = True

					result = 'All labels are successfully removed from '

					return render(request, 'confirmpass.html', {	
							'thing' : collname,
							'confirmed' : confirmed,
							'result' : result,
					})

				else:

					confirmed = False
					action = 'resetlabels'
					event = 'reset all labels for '
					denied_msg = "Wrong password. Please try again."

					return render(request, 'confirmpass.html', {
							'action': action,	
							'event' : event,
							'thing' : collname,
							'confirmed' : confirmed,
							'denied_msg' : denied_msg,
					})

    def _from_son(cls, son, _auto_dereference=True, only_fields=None, created=False):
        """Create an instance of a Document (subclass) from a PyMongo
        SON.
        """
        if not only_fields:
            only_fields = []

        if son and not isinstance(son, dict):
            raise ValueError("The source SON object needs to be of type 'dict'")

        # Get the class name from the document, falling back to the given
        # class if unavailable
        class_name = son.get('_cls', cls._class_name)

        # Convert SON to a dict, making sure each key is a string
        data = {str(key): value for key, value in son.iteritems()}

        # Return correct subclass for document type
        if class_name != cls._class_name:
            cls = get_document(class_name)

        changed_fields = []
        errors_dict = {}

        fields = cls._fields
        if not _auto_dereference:
            fields = copy.copy(fields)

        for field_name, field in fields.iteritems():
            field._auto_dereference = _auto_dereference
            if field.db_field in data:
                value = data[field.db_field]
                try:
                    data[field_name] = (value if value is None
                                        else field.to_python(value))
                    if field_name != field.db_field:
                        del data[field.db_field]
                except (AttributeError, ValueError) as e:
                    errors_dict[field_name] = e

        if errors_dict:
            errors = '\n'.join(['%s - %s' % (k, v)
                                for k, v in errors_dict.items()])
            msg = ('Invalid data to create a `%s` instance.\n%s'
                   % (cls._class_name, errors))
            raise InvalidDocumentError(msg)

        # In STRICT documents, remove any keys that aren't in cls._fields
        if cls.STRICT:
            data = {k: v for k, v in data.iteritems() if k in cls._fields}

        obj = cls(__auto_convert=False, _created=created, __only_fields=only_fields, **data)
        obj._changed_fields = changed_fields
        if not _auto_dereference:
            obj._fields = fields

        return obj

    def _from_son(cls, son, _auto_dereference=False):
        # get the class name from the document, falling back to the given
        # class if unavailable
        class_name = son.get('_cls', cls._class_name)

        # Return correct subclass for document type
        if class_name != cls._class_name:
            cls = get_document(class_name)

        return cls(_son=son)

    def _from_son(cls, son, _auto_dereference=True):
        """Create an instance of a Document (subclass) from a PyMongo SON.
        """

        # get the class name from the document, falling back to the given
        # class if unavailable
        class_name = son.get('_cls', cls._class_name)
        data = dict(("%s" % key, value) for key, value in son.items())
        if not UNICODE_KWARGS:
            # python 2.6.4 and lower cannot handle unicode keys
            # passed to class constructor example: cls(**data)
            to_str_keys_recursive(data)

        # Return correct subclass for document type
        if class_name != cls._class_name:
            cls = get_document(class_name)

        changed_fields = []
        errors_dict = {}

        fields = cls._fields
        if not _auto_dereference:
            fields = copy.copy(fields)

        for field_name, field in fields.items():
            field._auto_dereference = _auto_dereference
            if field.db_field in data:
                value = data[field.db_field]
                try:
                    data[field_name] = (value if value is None
                                        else field.to_python(value))
                    if field_name != field.db_field:
                        del data[field.db_field]
                except (AttributeError, ValueError) as e:
                    errors_dict[field_name] = e
            elif field.default:
                default = field.default
                if isinstance(default, collections.Callable):
                    default = default()
                if isinstance(default, BaseDocument):
                    changed_fields.append(field_name)

        if errors_dict:
            errors = "\n".join(["%s - %s" % (k, v)
                     for k, v in list(errors_dict.items())])
            msg = ("Invalid data to create a `%s` instance.\n%s"
                   % (cls._class_name, errors))
            raise InvalidDocumentError(msg)

        obj = cls(__auto_convert=False, **data)
        obj._changed_fields = changed_fields
        obj._created = False
        if not _auto_dereference:
            obj._fields = fields
        return obj

    def ajax_reference(self, class_name, id):
        oid = ObjectId(id)
        doc = get_document(class_name)
        obj = doc.objects.get(id=oid)

        return Response(json.dumps({
            'label': str(obj),
            'id': str(obj.id),
            'collection': obj.to_dbref().collection,
            'url': url_for(obj.view_endpoint + ".edit_view", id=obj.id)
        }), content_type='application/json')

    def cls_by_son(cls, son):
        """Return class for the bson.son.SON instance provided. Use `_cls` as a class name by default. Override to customize the instance creation (known children handling)"""

        try:
            class_name = son['_cls']
        except (TypeError, KeyError, ValueError):
            class_name = cls._class_name

        # Return correct subclass for document type
        if class_name != cls._class_name:
            cls = get_document(class_name)

        return cls

def backup_json(collname):

	currDate = datetime.now().strftime("%y%m%d-%H:%M") 

	filename = collname + "_" + currDate + ".json"

	model = get_document(collname)
		
	with open("data/backups/" + filename, 'w') as f:
		f.write(model.objects.to_json() + '\n')

	logging.info('BACKUP : Backup to ' + filename)

	return 0

def _resolve_model(obj):
    """
    Inherited from rest_framework.utils.model_meta
    Overridden for MongoDB compability
    """
    if isinstance(obj, six.string_types) and len(obj.split('.')) == 2:
        app_name, model_name = obj.split('.')
        resolved_model = get_document(model_name)
        if resolved_model is None:
            msg = "Mongoengine did not return a model for {0}.{1}"
            raise ImproperlyConfigured(msg.format(app_name, model_name))
        return resolved_model
    elif inspect.isclass(obj) and issubclass(obj, mongoengine.BaseDocument):
        return obj
    raise ValueError("{0} is not a MongoDB Document".format(obj))

    def to_internal_value(self, data):
        """
        Dict of native values <- Dict of primitive datatypes.
        """
        if not isinstance(data, dict):
            message = self.error_messages['invalid'].format(
                datatype=type(data).__name__
            )
            raise drf_fields.ValidationError({
                api_settings.NON_FIELD_ERRORS_KEY: [message]
            })

        ret = OrderedDict()
        errors = OrderedDict()

        if data['_cls']:
            cls = get_document(data['_cls'])
            if not issubclass(cls, self.Meta.model):
                #not playing the 'pass anything in, and we'll construct it for you' game
                cls = self.Meta.model
        else:
            cls = self.Meta.model

        fields = self.chainmap[cls]

        fields = [field for name, field in fields.items() if not field.read_only]

        for field in fields:
            validate_method = getattr(self, 'validate_' + field.field_name, None)
            primitive_value = field.get_value(data)
            try:
                validated_value = field.run_validation(primitive_value)
                if validate_method is not None:
                    validated_value = validate_method(validated_value)
            except drf_fields.ValidationError as exc:
                errors[field.field_name] = exc.detail
            except drf_fields.DjangoValidationError as exc:
                errors[field.field_name] = list(exc.messages)
            except SkipField:
                pass
            else:
                drf_fields.set_value(ret, field.source_attrs, validated_value)

        if errors:
            raise drf_fields.ValidationError(errors)

        return ret

    def _from_son(cls, son, _auto_dereference=True):
        """Create an instance of a Document (subclass) from a PyMongo SON.
        """

        # get the class name from the document, falling back to the given
        # class if unavailable
        class_name = son.get('_cls', cls._class_name)
        data = dict(("%s" % key, value) for key, value in son.iteritems())
        if not UNICODE_KWARGS:
            # python 2.6.4 and lower cannot handle unicode keys
            # passed to class constructor example: cls(**data)
            to_str_keys_recursive(data)

        # Return correct subclass for document type
        if class_name != cls._class_name:
            cls = get_document(class_name)

        #Apply the traits
        traits = son.get('_traits', [])
        if traits:
            cls = apply_traits(cls, traits)

        changed_fields = []
        errors_dict = {}

        fields = cls._fields
        if not _auto_dereference:
            fields = copy.copy(fields)

        for field_name, field in fields.iteritems():
            field._auto_dereference = _auto_dereference
            if field.db_field in data:
                value = data[field.db_field]
                try:
                    data[field_name] = (value if value is None
                                        else field.to_python(value))
                    if field_name != field.db_field:
                        del data[field.db_field]
                except (AttributeError, ValueError), e:
                    errors_dict[field_name] = e
            elif field.default:
                default = field.default
                if callable(default):
                    default = default()
                if isinstance(default, BaseDocument):
                    changed_fields.append(field_name)

    def __expand_dynamic_values(self, name, value):
        """expand any dynamic values to their correct types / values"""
        if not isinstance(value, (dict, list, tuple)):
            return value

        EmbeddedDocumentListField = _import_class('EmbeddedDocumentListField')

        is_list = False
        if not hasattr(value, 'items'):
            is_list = True
            value = dict([(k, v) for k, v in enumerate(value)])

        if not is_list and '_cls' in value:
            doc_cls = get_document(value['_cls'])
            if '_ref' in value and isinstance(value['_ref'], DBRef):
                reference = value['_ref']
                doc = doc_cls._get_db().dereference(reference)
                if doc is not None:
                    doc = doc_cls._from_son(doc)
                return doc
            else:
                return doc_cls(**value)

        data = {}
        for k, v in value.items():
            key = name if is_list else k
            data[k] = self.__expand_dynamic_values(key, v)

        if is_list:  # Convert back to a list
            data_items = sorted(data.items(), key=operator.itemgetter(0))
            value = [v for k, v in data_items]
        else:
            value = data

        # Convert lists / values so we can watch for any changes on them
        if (isinstance(value, (list, tuple)) and
                not isinstance(value, BaseList)):
            if issubclass(type(self), EmbeddedDocumentListField):
                value = EmbeddedDocumentList(value, self, name)
            else:
                value = BaseList(value, self, name)
        elif isinstance(value, dict) and not isinstance(value, BaseDict):
            value = BaseDict(value, self, name)

        return value

def get_labels(collname):

	model = get_document(collname)

	all_labels = model.objects(label__exists = True).only("label")

	label_set = []
	for lbl in all_labels:
		if(lbl["label"] not in label_set):
			label_set.append(lbl["label"])

	num_of_cl = []
	for labl in label_set:
		num_of_cl.append(model.objects(label = labl).count())

	labellist = zip(label_set, num_of_cl)	

	return labellist

def get_step_data(collname, num, page=None):

	model = get_document(collname)

	tweets = []

	if(page == "Cluster_Them"):
		for item in model.objects[5:num+5]:
			for i in item["ctweettuplelist"][:10]:
					tweets.append(i[2])
			tweets.append("------------------")

	else:
		for item in model.objects[5:num+5]:
			tweets.append(item["text"])

	
	length = model.objects.count()

	return tweets, length

    def __init__(self, native_data=None, data=None, many=False):
        """
        :param object native_data: Object or list of objects to serialize
        :param dict data: Optional serialized data
        :param bool many: Define if native_data is an object or a list of objects
        :raises AttributeError: If model is not defined in a subclassed instance
        """

        if self.model is None:
            raise AttributeError('model must be defined in serializer class')
        else:
            self.model = get_document(self.model)
        if not hasattr(self, 'fields') or self.fields is None:
            self.fields = self.model._fields
        self.data = data
        self.native_data = native_data
        self.many = many

        if self.native_data is not None:
            self.data = self.serialize()