def find_neglected_prereg_within_reminder_limit():

    queue_data = QueuedMail.objects.filter(
        email_type=PREREG_REMINDER_TYPE,
        send_at__gte=timezone.now() - settings.PREREG_AGE_LIMIT,
        data__draft_id__isnull=False
    ).values_list('data', flat=True)

    already_queued = [entry['draft_id'] for entry in queue_data]

    ## If project has been deleted since last time the script ran remove reminder email from queue.
    drafts_for_deleted_projects = list(DraftRegistration.objects
                                   .filter(_id__in=already_queued)
                                   .exclude(branched_from__is_deleted=False)
                                   .values_list('_id', flat=True))

    QueuedMail.objects.filter(
        email_type=PREREG_REMINDER_TYPE,
        data__draft_id__in=drafts_for_deleted_projects).delete()

    return DraftRegistration.objects.filter(
        deleted__isnull=True,
        registered_node=None,
        branched_from__is_deleted=False,
        registration_schema=get_prereg_schema(),
        datetime_initiated__lte=timezone.now()-settings.PREREG_WAIT_TIME,
        datetime_initiated__gte=timezone.now()-settings.PREREG_AGE_LIMIT,
    ).exclude(_id__in = already_queued)

 def _email_template_context(self, user, node, is_authorizer=False, urls=None):
     registration = self._get_registration()
     prereg_schema = prereg_utils.get_prereg_schema()
     if prereg_schema in registration.registered_schema:
         return {"custom_message": " as part of the Preregistration Challenge (https://cos.io/prereg)"}
     else:
         return {}

def migrate_registrations():
    PREREG_CHALLENGE_METASCHEMA = get_prereg_schema()
    registrations = Node.find(
        Q('is_registration', 'eq', True) &
        Q('registered_schema', 'eq', PREREG_CHALLENGE_METASCHEMA)
    )
    count = 0
    for reg in registrations:
        data = reg.registered_meta[PREREG_CHALLENGE_METASCHEMA._id]
        migrated = False
        logger.debug('Reading preregistration with id: {0}'.format(reg._id))
        for question in data.values():
            if isinstance(question.get('value'), dict):
                for value in question['value'].values():
                    migrated_one = migrate_file_meta(value)
                    if migrated_one and not migrated:
                        migrated = True
            else:
                migrated_one = migrate_file_meta(question)
                if migrated_one and not migrated:
                    migrated = True
        if migrated:
            reg.save()
            count += 1
            logger.info('Migrated preregistration with id: {0}'.format(reg._id))
    logger.info('Done with {0} prereg registrations migrated.'.format(count))

def migrate_drafts(dry):
    PREREG_CHALLENGE_METASCHEMA = get_prereg_schema()
    draft_registrations = DraftRegistration.find(
        Q('registration_schema', 'eq', PREREG_CHALLENGE_METASCHEMA)
    )
    count = 0
    for r in draft_registrations:
        # NOTE: We don't query Q('approval', 'eq', None) just in case
        # approval is set but the fk doesn't exist in the database
        if r.approval or r.registered_node:
            continue
        logger.debug('Reading draft with id: {0}'.format(r._id))
        data = r.registration_metadata
        migrated = False
        for q, ans in data.iteritems():
            if isinstance(ans.get('value'), dict):
                for value in ans['value'].values():
                    migrated = migrate_file_meta(value)
            else:
                migrated = migrate_file_meta(ans)
        if migrated:
            count += 1
            logger.info('Migrated draft with id: {0}'.format(r._id))
        if not dry:
            r.save()
    logger.info('Done with {0} drafts migrated.'.format(count))

    def setUp(self):
        super(TestPreregFiles, self).setUp()
        self.prereg_user = AuthUserFactory()
        self.user = AuthUserFactory()
        self.node = ProjectFactory(creator=self.user)

        ensure_schemas()
        prereg_schema = get_prereg_schema()
        self.d_of_qs = {
            'q7': OsfStorageFileNode(node=self.node, name='7'),
            'q11': OsfStorageFileNode(node=self.node, name='11'),
            'q16': OsfStorageFileNode(node=self.node, name='16'),
            'q12': OsfStorageFileNode(node=self.node, name='12'),
            'q13': OsfStorageFileNode(node=self.node, name='13'),
            'q19': OsfStorageFileNode(node=self.node, name='19'),
            'q26': OsfStorageFileNode(node=self.node, name='26')
        }
        data = {}
        for q, f in self.d_of_qs.iteritems():
            guid = f.get_guid(create=True)._id
            f.save()
            if q == 'q26':
                data[q] = {
                    'comments': [],
                    'value': '26',
                    'extra': [
                        {
                            'data': {
                                'provider': 'osfstorage',
                                'path': f.path,
                            },
                            'fileId': guid,
                            'nodeId': self.node._id,
                        }
                    ]
                }
                continue
            data[q] = {
                'value': {
                    'uploader': {
                        'extra': [
                            {
                                'data': {
                                    'provider': 'osfstorage',
                                    'path': f.path,
                                },
                                'fileId': guid,
                                'nodeId': self.node._id,
                            }
                        ]
                    }
                }
            }
        self.draft = DraftRegistrationFactory(
            initiator=self.user,
            registration_schema=prereg_schema,
            registration_metadata=data
        )
        self.prereg_user.save()
        self.admin_user = UserFactory(osf_id=self.prereg_user.pk)

 def get_queryset(self):
     return DraftRegistration.objects.filter(
         registration_schema=get_prereg_schema(),
     ).exclude(
         approval__isnull=True,
         registered_node__isnull=True,
     ).order_by(self.get_ordering())

    def _send_rejection_email(self, user, draft):
        schema = draft.registration_schema
        prereg_schema = prereg_utils.get_prereg_schema()

        if schema._id == prereg_schema._id:
            mails.send_mail(user.username, mails.PREREG_CHALLENGE_REJECTED, user=user, draft_url=draft.absolute_url)
        else:
            raise NotImplementedError("TODO: add a generic email template for registration approvals")

def submit_draft_for_review(auth, node, draft, *args, **kwargs):
    """Submit for approvals and/or notifications

    :return: serialized registration
    :rtype: dict
    :raises: HTTPError if embargo end date is invalid
    """
    data = request.get_json()
    meta = {}
    registration_choice = data.get('registrationChoice', 'immediate')
    validate_registration_choice(registration_choice)
    if registration_choice == 'embargo':
        # Initiate embargo
        end_date_string = data['embargoEndDate']
        validate_embargo_end_date(end_date_string, node)
        meta['embargo_end_date'] = end_date_string
    meta['registration_choice'] = registration_choice

    if draft.registered_node and not draft.registered_node.is_deleted:
        raise HTTPError(http.BAD_REQUEST, data=dict(message_long='This draft has already been registered, if you wish to '
                                                              'register it again or submit it for review please create '
                                                              'a new draft.'))

    # Don't allow resubmission unless submission was rejected
    if draft.approval and draft.approval.state != Sanction.REJECTED:
        raise HTTPError(http.CONFLICT, data=dict(message_long='Cannot resubmit previously submitted draft.'))

    draft.submit_for_review(
        initiated_by=auth.user,
        meta=meta,
        save=True
    )

    if prereg_utils.get_prereg_schema() == draft.registration_schema:

        node.add_log(
            action=NodeLog.PREREG_REGISTRATION_INITIATED,
            params={'node': node._primary_key},
            auth=auth,
            save=False
        )
        node.save()

    push_status_message(language.AFTER_SUBMIT_FOR_REVIEW,
                        kind='info',
                        trust=False)
    return {
        'status': 'initiated',
        'urls': {
            'registrations': node.web_url_for('node_registrations')
        }
    }, http.ACCEPTED

def drafts_for_user(user):
    from website import models  # noqa

    user_projects = models.Node.find(
        Q('is_deleted', 'eq', False) &
        Q('permissions.{0}'.format(user._id), 'in', [permissions.ADMIN])
    )
    PREREG_CHALLENGE_METASCHEMA = get_prereg_schema()
    return models.DraftRegistration.find(
        Q('registration_schema', 'eq', PREREG_CHALLENGE_METASCHEMA) &
        Q('approval', 'eq', None) &
        Q('registered_node', 'eq', None) &
        Q('branched_from', 'in', [p._id for p in user_projects])
    )

 def get_queryset(self):
     query = (
         Q('registration_schema', 'eq', get_prereg_schema()) &
         Q('approval', 'ne', None)
     )
     ordering = self.get_ordering()
     if 'initiator' in ordering:
         return DraftRegistration.find(query).sort(ordering)
     if ordering == SORT_BY['title']:
         return DraftRegistration.find(query).sort(
             'registration_metadata.q1.value')
     if ordering == SORT_BY['n_title']:
         return DraftRegistration.find(query).sort(
             '-registration_metadata.q1.value')
     return sort_drafts(DraftRegistration.find(query), ordering)

    def _notify_initiator(self):
        from website.project.model import DraftRegistration

        registration = self._get_registration()
        prereg_schema = prereg_utils.get_prereg_schema()

        draft = DraftRegistration.find_one(Q("registered_node", "eq", registration))

        if prereg_schema in registration.registered_schema:
            mails.send_mail(
                draft.initiator.username,
                mails.PREREG_CHALLENGE_ACCEPTED,
                user=draft.initiator,
                registration_url=registration.absolute_url,
                mimetype="html",
            )

    def _send_rejection_email(self, user, draft):
        schema = draft.registration_schema
        prereg_schema = prereg_utils.get_prereg_schema()

        if schema._id == prereg_schema._id:
            mails.send_mail(
                user.username,
                mails.PREREG_CHALLENGE_REJECTED,
                user=user,
                draft_url=draft.absolute_url,
                can_change_preferences=False,
                logo=osf_settings.OSF_PREREG_LOGO
            )
        else:
            raise NotImplementedError(
                'TODO: add a generic email template for registration approvals'
            )

def find_neglected_prereg_within_reminder_limit():

    queue_data = QueuedMail.objects.filter(
        email_type=PREREG_REMINDER_TYPE,
        send_at__gte=timezone.now() - settings.PREREG_AGE_LIMIT,
        data__draft_id__isnull=False
    ).values_list('data', flat=True)

    already_queued = [entry['draft_id'] for entry in queue_data]

    return DraftRegistration.objects.filter(
        deleted__isnull=True,
        registered_node=None,
        registration_schema=get_prereg_schema(),
        datetime_initiated__lte=timezone.now()-settings.PREREG_WAIT_TIME,
        datetime_initiated__gte=timezone.now()-settings.PREREG_AGE_LIMIT,
    ).exclude(_id__in = already_queued)

def main(dry_run=False, test=False):
    init_app(set_backends=True, routes=False)
    prereg_schema = get_prereg_schema()
    count = 0
    with TokuTransaction():
        prereg_drafts = DraftRegistration.find(
            Q('registration_schema', 'eq', prereg_schema)
        )
        for draft in prereg_drafts:
            # only migrate unapproved drafts
            if draft.is_approved:
                continue
            changed = migrate_draft_metadata(draft, test)
            if changed:
                count += 1
        logger.info('Migrated {} drafts'.format(count))
        if dry_run:
            raise RuntimeError('Dry run, rolling back transaction')

def submit_draft_for_review(auth, node, draft, *args, **kwargs):
    """Submit for approvals and/or notifications

    :return: serialized registration
    :rtype: dict
    :raises: HTTPError if embargo end date is invalid
    """
    data = request.get_json()
    meta = {}
    registration_choice = data.get('registrationChoice', 'immediate')
    validate_registration_choice(registration_choice)
    if registration_choice == 'embargo':
        # Initiate embargo
        end_date_string = data['embargoEndDate']
        validate_embargo_end_date(end_date_string, node)
        meta['embargo_end_date'] = end_date_string
    meta['registration_choice'] = registration_choice
    draft.submit_for_review(
        initiated_by=auth.user,
        meta=meta,
        save=True
    )

    if prereg_utils.get_prereg_schema() == draft.registration_schema:

        node.add_log(
            action=NodeLog.PREREG_REGISTRATION_INITIATED,
            params={'node': node._primary_key},
            auth=auth,
            save=False
        )
        node.save()

    push_status_message(language.AFTER_SUBMIT_FOR_REVIEW,
                        kind='info',
                        trust=False)
    return {
        'status': 'initiated',
        'urls': {
            'registrations': node.web_url_for('node_registrations')
        }
    }, http.ACCEPTED

def migrate_drafts(dry):

    PREREG_CHALLENGE_METASCHEMA = get_prereg_schema()
    draft_registrations = DraftRegistration.find(
        Q('registration_schema', 'eq', PREREG_CHALLENGE_METASCHEMA) &
        Q('approval', 'eq', None) &
        Q('registered_node', 'eq', None)
    )
    count = 0
    for r in draft_registrations:
        data = r.registration_metadata
        migrated = False
        for q, ans in data.iteritems():
            if isinstance(ans.get('value'), dict):
                for value in ans['value'].values():
                    migrated = migrate_file_meta(value)
            else:
                migrated = migrate_file_meta(ans)
        if migrated:
            count += 1
            logger.info('Migrated draft with id: {0}'.format(r._id))
        if not dry:
            r.save()
    logger.info('Done with {0} drafts migrated.'.format(count))

def migrate_drafts():
    PREREG_CHALLENGE_METASCHEMA = get_prereg_schema()
    draft_registrations = DraftRegistration.find(
        Q('registration_schema', 'eq', PREREG_CHALLENGE_METASCHEMA)
    )
    count = 0
    for draft in draft_registrations:
        migrated = False
        logger.debug('Reading preregistration draft with id: {0}'.format(draft._id))
        for answer in draft.registration_metadata.values():
            if isinstance(answer.get('value'), dict):
                for value in answer['value'].values():
                    migrated_one = migrate_file_meta(value)
                    if migrated_one and not migrated:
                        migrated = True
            else:
                migrated_one = migrate_file_meta(answer)
                if migrated_one and not migrated:
                    migrated = True
        if migrated:
            draft.save()
            count += 1
            logger.info('Migrated preregistration draft {0}'.format(draft._id))
    logger.info('Done with {0} prereg drafts migrated.'.format(count))

def get_submitted_preregistrations(order='-approval__initiation_date'):
    return DraftRegistration.objects.filter(
        registration_schema=get_prereg_schema(),
        approval__isnull=False
    ).order_by(order).select_related('initiator', 'registration_schema', 'approval')

    def setUp(self):
        super(TestMigrateRegistrationExtra, self).setUp()
        self.user = UserFactory()
        self.node = ProjectFactory(creator=self.user)
        self.file = self._get_test_file()
        MetaSchema.remove()

        self.file_ans = {
            'file': {
                'data':{
                    'kind':'file',
                    'extra':{
                        'checkout': None,
                        'hashes':{
                            'sha256':'1fffe6116ecfa778f9938060d5caab923ba4b8db60bd2dd57f16a72e5ef06292'
                        },
                        'downloads':0,
                        'version':1
                    },
                    'modified':'2016-04-15T18:10:48',
                    'name':'file.txt',
                    'provider':'osfstorage',
                }
            }
        }
        self.complex_metadata = {
            'q1': {
                'value': 'Answer 1',
                'extra': []
            },
            'q2': {
                'value': 'Answer 2',
                'extra': {}
            },
            'q3': {
                'value': 'Answer 3',
                'extra': self.file_ans
            },
            'q4': {
                'value': {
                    'question': {
                        'value': 'Answer 4',
                        'extra': {}
                    },
                    'uploader': {
                        'value': '',
                        'extra': {}
                    }
                },
            },
            'q5': {
                'value': 'Answer 5',
                'extra': [
                    {
                        'viewUrl': '/project/abcdef/files/osfstorage/5723787136b74e1a953d9612/',
                        'hasSelectedFile': True,
                        'selectedFileName': 'file.txt'
                    }
                ]
            }
        }
        self.simple_metadata = {
            'Summary': 'Some airy'
        }
        self.schema = get_prereg_schema()
        self.draft1 = DraftRegistrationFactory(
            registration_metadata=self.complex_metadata,
            registration_schema=self.schema,
            approval=None,
            registered_node=None

        )
        self.draft2 = DraftRegistrationFactory(
            registration_metadata=self.simple_metadata
        )

 def test_get_prereg_schema_can_return_erpc_metaschema(self):
     schema = get_prereg_schema('erpc')
     assert_is_instance(schema, MetaSchema)
     assert_equal(schema.name, 'Election Research Preacceptance Competition')