def _kwargs_to_nodes(kwargs):
    """Retrieve project and component objects from keyword arguments.

    :param dict kwargs: Dictionary of keyword arguments
    :return: Tuple of project and component

    """
    project = kwargs.get('project') or Node.load(kwargs.get('pid', kwargs.get('nid')))
    if not project:
        raise HTTPError(http.NOT_FOUND)
    if project.category != 'project':
        raise HTTPError(http.BAD_REQUEST)
    if project.is_deleted:
        raise HTTPError(http.GONE)

    if kwargs.get('nid') or kwargs.get('node'):
        node = kwargs.get('node') or Node.load(kwargs.get('nid'))
        if not node:
            raise HTTPError(http.NOT_FOUND)
        if node.is_deleted:
            raise HTTPError(http.GONE)
    else:
        node = None

    return project, node

def find_dashboard(user):
    dashboard_folder = Node.find_for_user(user, subquery=Q("is_dashboard", "eq", True))

    if dashboard_folder.count() == 0:
        new_dashboard(user)
        dashboard_folder = Node.find_for_user(user, Q("is_dashboard", "eq", True))
    return dashboard_folder[0]

def search_projects_by_title(**kwargs):
    """ Search for nodes by title. Can pass in arguments from the URL to modify the search
    :arg term: The substring of the title.
    :arg category: Category of the node.
    :arg isDeleted: yes, no, or either. Either will not add a qualifier for that argument in the search.
    :arg isFolder: yes, no, or either. Either will not add a qualifier for that argument in the search.
    :arg isRegistration: yes, no, or either. Either will not add a qualifier for that argument in the search.
    :arg includePublic: yes or no. Whether the projects listed should include public projects.
    :arg includeContributed: yes or no. Whether the search should include projects the current user has
        contributed to.
    :arg ignoreNode: a list of nodes that should not be included in the search.
    :return: a list of dictionaries of projects

    """
    # TODO(fabianvf): At some point, it would be nice to do this with elastic search
    user = kwargs['auth'].user

    term = request.args.get('term', '')
    max_results = int(request.args.get('maxResults', '10'))
    category = request.args.get('category', 'project').lower()
    is_deleted = request.args.get('isDeleted', 'no').lower()
    is_folder = request.args.get('isFolder', 'no').lower()
    is_registration = request.args.get('isRegistration', 'no').lower()
    include_public = request.args.get('includePublic', 'yes').lower()
    include_contributed = request.args.get('includeContributed', 'yes').lower()
    ignore_nodes = request.args.getlist('ignoreNode', [])

    matching_title = (
        Q('title', 'icontains', term) &  # search term (case insensitive)
        Q('category', 'eq', category)  # is a project
    )

    matching_title = conditionally_add_query_item(matching_title, 'is_deleted', is_deleted)
    matching_title = conditionally_add_query_item(matching_title, 'is_folder', is_folder)
    matching_title = conditionally_add_query_item(matching_title, 'is_registration', is_registration)

    if len(ignore_nodes) > 0:
        for node_id in ignore_nodes:
            matching_title = matching_title & Q('_id', 'ne', node_id)

    my_projects = []
    my_project_count = 0
    public_projects = []

    if include_contributed == "yes":
        my_projects = Node.find(
            matching_title &
            Q('contributors', 'eq', user._id)  # user is a contributor
        ).limit(max_results)
        my_project_count = my_project_count

    if my_project_count < max_results and include_public == "yes":
        public_projects = Node.find(
            matching_title &
            Q('is_public', 'eq', True)  # is public
        ).limit(max_results - my_project_count)

    results = list(my_projects) + list(public_projects)
    ret = process_project_search_results(results, **kwargs)
    return ret

 def create(self, validated_data):
     node = Node(**validated_data)
     try:
         node.save()
     except ValidationValueError as e:
         raise InvalidModelValueError(detail=e.message)
     return node

def set_tag_many_to_many_on_nodes(page_size=10000):
    print 'Starting {}...'.format(sys._getframe().f_code.co_name)
    node_count = 0
    m2m_count = 0
    start = datetime.now()
    total = MODMNode.find(build_query(m2m_tag_fields, MODMNode)).count()
    print '{} Nodes'.format(total)
    while node_count < total:
        with transaction.atomic():
            for modm_node in MODMNode.find(build_query(
                    m2m_tag_fields, MODMNode)).sort('-date_modified')[
                        node_count:page_size + node_count]:
                django_node = Node.objects.get(
                    pk=modm_to_django[modm_node._id])
                for m2m_tag_field in m2m_tag_fields:
                    try:
                        attr = getattr(django_node, m2m_tag_field)
                    except AttributeError as ex:
                        # node field doesn't exist on node
                        pass
                    else:
                        # node field exists, do the stuff
                        django_pks = []
                        for modm_m2m_value in getattr(modm_node, m2m_tag_field,
                                                      []):
                            suffix = 'system' if m2m_tag_field == 'system_tags' else 'not_system'
                            if isinstance(modm_m2m_value, MODMTag):
                                django_pks.append(modm_to_django[
                                    '{}:{}'.format(modm_m2m_value._id,
                                                   suffix)])
                            elif isinstance(modm_m2m_value, basestring):
                                django_pks.append(modm_to_django[
                                    '{}:{}'.format(modm_m2m_value, suffix)])
                            elif modm_m2m_value is None:
                                print 'Tag of None found on Node {}'.format(
                                    modm_node._id)
                            else:
                                # wth
                                print '\a'  # bells!
                                print '\a'
                                print '\a'
                                print '\a'
                                print '\a'
                                print '\a'
                                print '\a'
                                import bpdb

                                bpdb.set_trace()

                        if len(django_pks) > 0:
                            attr.add(*django_pks)
                        m2m_count += len(django_pks)
                node_count += 1
                if node_count % page_size == 0 or node_count == total:
                    print 'Through {} nodes and {} m2m'.format(node_count,
                                                               m2m_count)
    print 'Done with {} in {} seconds...'.format(
        sys._getframe().f_code.co_name,
        (datetime.now() - start).total_seconds())

 def create(self, validated_data):
     node = Node(**validated_data)
     node.is_folder = True
     node.category = ''
     try:
         node.save()
     except ValidationValueError as e:
         raise InvalidModelValueError(detail=e.message)
     return node

 def create(self, validated_data):
     node = Node(**validated_data)
     node.is_collection = True
     node.category = ''
     try:
         node.save()
     except ValidationValueError as e:
         raise InvalidModelValueError(detail=e.message)
     except NodeStateError:
         raise ser.ValidationError('Each user cannot have more than one Bookmark collection.')
     return node

def get_dashboard_nodes(auth):
    """Get summary information about the current user's dashboard nodes.

    :param-query no_components: Exclude components from response.
        NOTE: By default, components will only be shown if the current user
        is contributor on a comonent but not its parent project. This query
        parameter forces ALL components to be excluded from the request.
    :param-query permissions: Filter upon projects for which the current user
        has the specified permissions. Examples: 'write', 'admin'
    """
    user = auth.user

    nodes = Node.find_for_user(
        user,
        subquery=(
            Q("category", "eq", "project")
            & Q("is_deleted", "eq", False)
            & Q("is_registration", "eq", False)
            & Q("is_folder", "eq", False)
        ),
    )

    if request.args.get("no_components") not in [True, "true", "True", "1", 1]:
        comps = Node.find_for_user(  # NOTE - this used to be a find on nodes above. Does this mess it up?
            user,
            (
                # components only
                Q("category", "ne", "project")
                &
                # exclude deleted nodes
                Q("is_deleted", "eq", False)
                &
                # exclude registrations
                Q("is_registration", "eq", False)
            ),
        )
    else:
        comps = []

    nodes = list(nodes) + list(comps)
    if request.args.get("permissions"):
        perm = request.args["permissions"].strip().lower()
        if perm not in permissions.PERMISSIONS:
            raise HTTPError(
                http.BAD_REQUEST,
                dict(
                    message_short="Invalid query parameter",
                    message_long="{0} is not in {1}".format(perm, permissions.PERMISSIONS),
                ),
            )
        response_nodes = [node for node in nodes if node.has_permission(user, permission=perm)]
    else:
        response_nodes = nodes
    return _render_nodes(response_nodes, auth)

def migrate_nodes(index):
    logger.info('Migrating nodes to index: {}'.format(index))
    query = Q('is_public', 'eq', True) & Q('is_deleted', 'eq', False)
    total = Node.find(query).count()
    increment = 200
    total_pages = (total // increment) + 1
    pages = paginated(Node, query=query, increment=increment, each=False)
    for page_number, page in enumerate(pages):
        logger.info('Updating page {} / {}'.format(page_number + 1, total_pages))
        Node.bulk_update_search(page, index=index)
        Node._clear_caches()

    logger.info('Nodes migrated: {}'.format(total))

def set_node_many_to_many_on_nodes(page_size=5000):
    print 'Starting {}...'.format(sys._getframe().f_code.co_name)
    node_count = 0
    m2m_count = 0
    start = datetime.now()
    total = MODMNode.find(
        build_query(m2m_node_fields, MODMNode),
        allow_institution=True).count()
    print '{} Nodes'.format(total)
    while node_count < total:
        with transaction.atomic():
            for modm_node in MODMNode.find(
                    build_query(m2m_node_fields, MODMNode),
                    allow_institution=True).sort('-date_modified')[
                        node_count:page_size + node_count]:
                django_node = Node.objects.get(
                    pk=modm_to_django[modm_node._id])
                for m2m_node_field in m2m_node_fields:
                    attr = getattr(django_node, m2m_node_field)
                    django_pks = []
                    for modm_m2m_value in getattr(modm_node, m2m_node_field,
                                                  []):
                        if isinstance(modm_m2m_value, MODMNode):
                            django_pks.append(modm_to_django[
                                modm_m2m_value._id])
                        elif isinstance(modm_m2m_value, basestring):
                            django_pks.append(modm_to_django[modm_m2m_value])
                        elif isinstance(modm_m2m_value, Pointer):
                            django_pks.append(modm_to_django[
                                modm_m2m_value.node._id])
                        else:
                            # wth
                            print '\a'
                            print '\a'
                            print '\a'
                            print '\a'
                            print '\a'
                            print '\a'
                            print '\a'
                            import bpdb
                            bpdb.set_trace()
                    if len(django_pks) > 0:
                        attr.add(*django_pks)
                    m2m_count += len(django_pks)
                node_count += 1
                if node_count % page_size == 0 or node_count == total:
                    print 'Through {} nodes and {} m2m'.format(node_count,
                                                               m2m_count)
    print 'Done with {} in {} seconds...'.format(
        sys._getframe().f_code.co_name,
        (datetime.now() - start).total_seconds())

def save_bare_nodes(page_size=20000):
    print 'Starting {}...'.format(sys._getframe().f_code.co_name)
    count = 0
    start = datetime.now()
    total = MODMNode.find(allow_institution=True).count()
    while count < total:
        with transaction.atomic():
            nids = []
            for modm_node in MODMNode.find(
                    allow_institution=True).sort('-date_modified')[
                        count:count + page_size]:
                guid = Guid.objects.get(guid=modm_node._id)
                node_fields = dict(_guid_id=guid.pk, **modm_node.to_storage())

                # remove fields not yet implemented
                cleaned_node_fields = {key: node_fields[key]
                                       for key in node_fields
                                       if key not in node_key_blacklist}

                # make datetimes not naive
                for k, v in cleaned_node_fields.iteritems():
                    if isinstance(v, datetime):
                        cleaned_node_fields[k] = pytz.utc.localize(v)

                # remove null fields, postgres hate null fields
                cleaned_node_fields = {k: v
                                       for k, v in
                                       cleaned_node_fields.iteritems()
                                       if v is not None}
                nids.append(Node(**cleaned_node_fields))
                count += 1
                if count % page_size == 0 or count == total:
                    then = datetime.now()
                    print 'Saving nodes {} through {}...'.format(
                        count - page_size, count)
                    woot = Node.objects.bulk_create(nids)
                    for wit in woot:
                        modm_to_django[wit._guid.guid] = wit.pk
                    now = datetime.now()
                    print 'Done with {} nodes in {} seconds...'.format(
                        len(woot), (now - then).total_seconds())
                    nids = []
                    trash = gc.collect()
                    print 'Took out {} trashes'.format(trash)

    print 'Modm Nodes: {}'.format(total)
    print 'django Nodes: {}'.format(Node.objects.all().count())
    print 'Done with {} in {} seconds...'.format(
        sys._getframe().f_code.co_name,
        (datetime.now() - start).total_seconds())

def get_projects_forked():
    projects_forked = Node.find(
        Q('parent_node', 'eq', None) &
        Q('is_fork', 'eq', True) &
        CONTENT_NODE_QUERY
    )
    return projects_forked

def get_projects():
    # This count includes projects, forks, and registrations
    projects = Node.find(
        Q('parent_node', 'eq', None) &
        CONTENT_NODE_QUERY
    )
    return projects

def project_generate_private_link_post(auth, node, **kwargs):
    """ creata a new private link object and add it to the node and its selected children"""

    node_ids = request.json.get('node_ids', [])
    name = request.json.get('name', '')
    anonymous = request.json.get('anonymous', False)

    if node._id not in node_ids:
        node_ids.insert(0, node._id)

    nodes = [Node.load(node_id) for node_id in node_ids]

    has_public_node = any(node.is_public for node in nodes)

    new_link = new_private_link(
        name=name, user=auth.user, nodes=nodes, anonymous=anonymous
    )

    if anonymous and has_public_node:
        status.push_status_message(
            'Anonymized view-only links <b>DO NOT</b> '
            'anonymize contributors of public project or component.'
        )

    return new_link

def add_pointer(auth):
    """Add a single pointer to a node using only JSON parameters

    """
    to_node_id = request.json.get('toNodeID')
    pointer_to_move = request.json.get('pointerID')

    if not (to_node_id and pointer_to_move):
        raise HTTPError(http.BAD_REQUEST)

    pointer = Node.load(pointer_to_move)
    to_node = Node.load(to_node_id)
    try:
        _add_pointers(to_node, [pointer], auth)
    except ValueError:
        raise HTTPError(http.BAD_REQUEST)

    def test_fix_templated(self):
        assert_equal(
            2,
            fix_nodes(get_broken_templated())
        )

        Node._clear_caches()

        broken_nodes = list(get_broken_templated())

        assert_equal(0, len(broken_nodes))
        assert_is_none(self.bad_template_project.piwik_site_id)
        assert_is_none(self.bad_template_component.piwik_site_id)

        assert_is_not_none(self.template_project.piwik_site_id)
        assert_is_not_none(self.template_component.piwik_site_id)

    def test_bulk_creates_children_and_sanitizes_html_logged_in_owner(self):
        title = '<em>Cool</em> <strong>Project</strong>'
        description = 'An <script>alert("even cooler")</script> child'

        res = self.app.post_json_api(self.url, {
            'data': [{
                'type': 'nodes',
                'attributes': {
                    'title': title,
                    'description': description,
                    'category': 'project',
                    'public': True
                }
            }]
        }, auth=self.user.auth, bulk=True)
        child_id = res.json['data'][0]['id']
        assert_equal(res.status_code, 201)
        url = '/{}nodes/{}/'.format(API_BASE, child_id)

        res = self.app.get(url, auth=self.user.auth)
        assert_equal(res.json['data']['attributes']['title'], strip_html(title))
        assert_equal(res.json['data']['attributes']['description'], strip_html(description))
        assert_equal(res.json['data']['attributes']['category'], 'project')

        self.project.reload()
        child_id = res.json['data']['id']
        assert_equal(child_id, self.project.nodes[0]._id)
        assert_equal(Node.load(child_id).logs[0].action, NodeLog.PROJECT_CREATED)

 def setUp(self, *args, **kwargs):
     OsfTestCase.setUp(self, *args, **kwargs)
     if not self.kind:
         return
     self.sanction = self.Factory()
     self.reg = Node.find_one(Q(self.Model.SHORT_NAME, 'eq', self.sanction))
     self.user = self.reg.creator

    def test_POST_register_embargo_does_not_make_project_or_children_public(self, mock_enqueue):
        public_project = ProjectFactory(creator=self.user, is_public=True)
        component = NodeFactory(creator=self.user, parent=public_project, title="Component", is_public=True)
        subproject = ProjectFactory(creator=self.user, parent=public_project, title="Subproject", is_public=True)
        subproject_component = NodeFactory(creator=self.user, parent=subproject, title="Subcomponent", is_public=True)
        res = self.app.post(
            public_project.api_url_for("node_register_template_page_post", template=u"Open-Ended_Registration"),
            self.valid_embargo_payload,
            content_type="application/json",
            auth=self.user.auth,
        )
        public_project.reload()
        assert_equal(res.status_code, 201)

        # Last node directly registered from self.project
        registration = Node.load(public_project.node__registrations[-1])

        assert_true(registration.is_registration)
        assert_false(registration.is_public)
        assert_true(registration.is_pending_embargo_for_existing_registration)
        assert_is_not_none(registration.embargo)

        for node in registration.get_descendants_recursive():
            assert_true(node.is_registration)
            assert_false(node.is_public)

 def load(cls, key):
     from website.models import Node
     try:
         node = Node.find_one(Q('institution_id', 'eq', key), allow_institution=True)
         return cls(node)
     except NoResultsFound:
         return None