def command(self):
     from allura import model as M
     main_session_classes = [M.main_orm_session, M.repository_orm_session,
             M.task_orm_session]
     if asbool(self.config.get('activitystream.recording.enabled', False)):
         from activitystream.storage.mingstorage import activity_orm_session
         main_session_classes.append(activity_orm_session)
     self.basic_setup()
     main_indexes = defaultdict(lambda: defaultdict(list))  # by db, then collection name
     project_indexes = defaultdict(list)  # by collection name
     base.log.info('Collecting indexes...')
     for m in Mapper.all_mappers():
         mgr = m.collection.m
         cname = mgr.collection_name
         cls = m.mapped_class
         if cname is None:
             base.log.info('... skipping abstract class %s', cls)
             continue
         base.log.info('... for class %s', cls)
         if session(cls) in main_session_classes:
             idx = main_indexes[session(cls)][cname]
         else:
             idx = project_indexes[cname]
         idx.extend(mgr.indexes)
     base.log.info('Updating indexes for main DB')
     for odm_session, db_indexes in main_indexes.iteritems():
         db = odm_session.impl.db
         for name, indexes in db_indexes.iteritems():
             self._update_indexes(db[name], indexes)
     base.log.info('Updating indexes for project DB')
     db = M.project_doc_session.db
     base.log.info('... DB: %s', db)
     for name, indexes in project_indexes.iteritems():
         self._update_indexes(db[name], indexes)
     base.log.info('Done updating indexes')

 def install_app(self, ep_name, mount_point=None, mount_label=None, ordinal=None, **override_options):
     App = g.entry_points['tool'][ep_name]
     with h.push_config(c, project=self):
         try:
             mount_point = v.MountPointValidator(App).to_python(mount_point)
         except fe.Invalid as e:
             raise exceptions.ToolError(str(e))
     if ordinal is None:
         ordinal = int(self.ordered_mounts(include_hidden=True)
                       [-1]['ordinal']) + 1
     options = App.default_options()
     options['mount_point'] = mount_point
     options[
         'mount_label'] = mount_label or App.default_mount_label or mount_point
     options['ordinal'] = int(ordinal)
     options.update(override_options)
     cfg = AppConfig(
         project_id=self._id,
         tool_name=ep_name.lower(),
         options=options)
     app = App(self, cfg)
     with h.push_config(c, project=self, app=app):
         session(cfg).flush()
         app.install(self)
     return app

 def commit(self):
     ss = VersionedArtifact.commit(self)
     session(self).flush()
     if self.version > 1:
         v1 = self.get_version(self.version - 1)
         v2 = self
         la = [line + '\n' for line in v1.text.splitlines()]
         lb = [line + '\n' for line in v2.text.splitlines()]
         diff = ''.join(difflib.unified_diff(
             la, lb,
             'v%d' % v1.version,
             'v%d' % v2.version))
         description = '<pre>' + diff + '</pre>'
         if v1.title != v2.title:
             subject = '%s renamed page %s to %s' % (
                 context.user.username, v1.title, v2.title)
         else:
             subject = '%s modified page %s' % (
                 context.user.username, self.title)
     else:
         description = self.text
         subject = '%s created page %s' % (
             context.user.username, self.title)
     Feed.post(self, title=None, description=description)
     Notification.post(
         artifact=self, topic='metadata', text=description, subject=subject)
     return ss

 def next_ticket_num(self):
     gbl = Globals.query.find_and_modify(
         query=dict(app_config_id=self.app_config_id),
         update={'$inc': { 'last_ticket_num': 1}},
         new=True)
     session(gbl).expunge(gbl)
     return gbl.last_ticket_num

 def commit(self, update_stats=True):
     '''Save off a snapshot of the artifact and increment the version #'''
     self.version += 1
     try:
         ip_address = request.headers.get('X_FORWARDED_FOR', request.remote_addr)
         ip_address = ip_address.split(',')[0].strip()
     except:
         ip_address = '0.0.0.0'
     data = dict(
         artifact_id=self._id,
         artifact_class='%s.%s' % (
             self.__class__.__module__,
             self.__class__.__name__),
         version=self.version,
         author=dict(
             id=c.user._id,
             username=c.user.username,
             display_name=c.user.get_pref('display_name'),
             logged_ip=ip_address),
         timestamp=datetime.utcnow(),
         data=state(self).clone())
     ss = self.__mongometa__.history_class(**data)
     session(ss).insert_now(ss, state(ss))
     log.info('Snapshot version %s of %s',
              self.version, self.__class__)
     if update_stats:
         if self.version > 1:
             g.statsUpdater.modifiedArtifact(
                 self.type_s, self.mod_date, self.project, c.user)
         else :
             g.statsUpdater.newArtifact(
                 self.type_s, self.mod_date, self.project, c.user)
     return ss

    def deliver(cls, nid, artifact_index_ids, topic):
        '''Called in the notification message handler to deliver notification IDs
        to the appropriate mailboxes.  Atomically appends the nids
        to the appropriate mailboxes.
        '''

        artifact_index_ids.append(None)  # get tool-wide ("None") and specific artifact subscriptions
        d = {
            'project_id': c.project._id,
            'app_config_id': c.app.config._id,
            'artifact_index_id': {'$in': artifact_index_ids},
            'topic': {'$in': [None, topic]}
        }
        mboxes = cls.query.find(d).all()
        log.debug('Delivering notification %s to mailboxes [%s]', nid, ', '.join([str(m._id) for m in mboxes]))
        for mbox in mboxes:
            try:
                mbox.query.update(
                    # _id is automatically specified by ming's "query", so this matches the current mbox
                    {'$push': dict(queue=nid),
                     '$set': dict(last_modified=datetime.utcnow(),
                                  queue_empty=False),
                     })
                # Make sure the mbox doesn't stick around to be flush()ed
                session(mbox).expunge(mbox)
            except:
                # log error but try to keep processing, lest all the other eligible
                # mboxes for this notification get skipped and lost forever
                log.exception(
                    'Error adding notification: %s for artifact %s on project %s to user %s',
                    nid, artifact_index_ids, c.project._id, mbox.user_id)

    def test_events(self, post_event):
        setup_trove_categories()

        # Create event
        cfg = {"trovecategories.enableediting": "true"}
        with h.push_config(config, **cfg):
            r = self.app.post("/categories/create/", params=dict(categoryname="test"))

        category_id = post_event.call_args[0][1]
        assert_true(isinstance(category_id, int))
        assert_equals(post_event.call_args[0][0], "trove_category_created")
        category = M.TroveCategory.query.get(trove_cat_id=category_id)

        # Update event
        category.fullname = "test2"
        session(M.TroveCategory).flush()
        edited_category_id = post_event.call_args[0][1]
        assert_true(isinstance(edited_category_id, int))
        assert_equals(edited_category_id, category_id)
        assert_equals(post_event.call_args[0][0], "trove_category_updated")

        # Delete event
        M.TroveCategory.delete(category)
        session(M.TroveCategory).flush()
        deleted_category_id = post_event.call_args[0][1]
        assert_true(isinstance(deleted_category_id, int))
        assert_equals(deleted_category_id, category_id)
        assert_equals(post_event.call_args[0][0], "trove_category_deleted")

 def install_app(self, ep_name, mount_point=None, mount_label=None, ordinal=None, **override_options):
     App = g.entry_points["tool"][ep_name]
     if not mount_point:
         base_mount_point = mount_point = App.default_mount_point
         for x in range(10):
             if self.app_instance(mount_point) is None:
                 break
             mount_point = base_mount_point + "-%d" % x
     if not h.re_path_portion.match(mount_point):
         raise exceptions.ToolError, 'Mount point "%s" is invalid' % mount_point
     # HACK: reserved url components
     if mount_point in ("feed", "index", "icon", "_nav.json"):
         raise exceptions.ToolError, ('Mount point "%s" is reserved' % mount_point)
     if self.app_instance(mount_point) is not None:
         raise exceptions.ToolError, ('Mount point "%s" is already in use' % mount_point)
     assert self.app_instance(mount_point) is None
     if ordinal is None:
         ordinal = int(self.ordered_mounts(include_hidden=True)[-1]["ordinal"]) + 1
     options = App.default_options()
     options["mount_point"] = mount_point
     options["mount_label"] = mount_label or App.default_mount_label or mount_point
     options["ordinal"] = int(ordinal)
     options.update(override_options)
     cfg = AppConfig(project_id=self._id, tool_name=ep_name, options=options)
     app = App(self, cfg)
     with h.push_config(c, project=self, app=app):
         session(cfg).flush()
         app.install(self)
     return app

 def set_api_ticket(self, expire=None):
     if not expire:
         expire = timedelta(days=1)
     api_ticket = M.ApiTicket(user_id=self.user._id, capabilities={'import': ['Projects','test']},
                              expires=datetime.utcnow() + expire)
     session(api_ticket).flush()
     self.set_api_token(api_ticket)

    def prepare_context(self, context):
        full_timeline = g.director.get_timeline(
            self.user, page=0, limit=100,
            actor_only=True,
        )
        filtered_timeline = list(islice(ifilter(perm_check(c.user), full_timeline),
                                        0, 8))
        for activity in filtered_timeline:
            # Get the project for the activity.obj so we can use it in the
            # template. Expunge first so Ming doesn't try to flush the attr
            # we create to temporarily store the project.
            #
            # The get_activity_object() calls are cheap, pulling from
            # the session identity map instead of mongo since identical
            # calls are made by perm_check() above.
            session(activity).expunge(activity)
            activity_obj = get_activity_object(activity.obj)
            activity.obj.project = getattr(activity_obj, 'project', None)

        context.update({
            'follow_toggle': W.follow_toggle,
            'following': g.director.is_connected(c.user, self.user),
            'timeline': filtered_timeline,
            'activity_app': self.activity_app,
        })
        g.register_js('activity_js/follow.js')
        return context

 def track_login(self, req):
     user_ip = utils.ip_address(req)
     user_agent = req.headers.get('User-Agent')
     self.last_access['login_date'] = datetime.utcnow()
     self.last_access['login_ip'] = user_ip
     self.last_access['login_ua'] = user_agent
     session(self).flush(self)

def migrate_project_database(project):
    c.project = project
    target_uri = M.Project.default_database_uri(project.shortname)
    target_db = target_uri.rsplit('/')[-1]
    if project.database_uri == target_uri:
        log.info('Project %s is already migrated to %s', project.shortname, project.database_uri)
        return 2
    conn = M.session.main_doc_session.db.connection
    host = '%s:%s' % (conn.host, conn.port)
    dirname = os.tempnam()
    try:
        log.info('Backing up %s to %s', project.shortname, dirname)
        db_uri = project.database_uri
        db = db_uri.rsplit('/')[-1]
        assert 0 == os.system('%s --host %s --db %s -o %s' % (
                MONGO_DUMP, host, db, dirname))
        assert 0 == os.system('%s --host %s --db %s %s/%s ' % (
                MONGO_RESTORE, host, target_db, dirname, db))
        for p in M.Project.query.find(dict(database_uri=db_uri)):
            p.database_uri = M.Project.default_database_uri(project.shortname)
        session(project).flush()
        conn.drop_database(db)
    finally:
        if os.path.exists(dirname):
            shutil.rmtree(dirname)
    return 0

def main():
    args = arguments()

    c.user = M.User.query.get(username='root')

    with h.push_context(args.shortname, args.mountpt, neighborhood='Projects'):

        tool = c.project.app_config_by_tool_type(args.mountpt)

        # create tons of topics
        discussion = Forum.query.get(
            app_config_id=tool._id,
            shortname=args.forumname)

        for i in range(5000):
            subject = 'fake topic {}'.format(str(i))
            thd = discussion.thread_class()(discussion_id=discussion._id, subject=subject)
            # subj = str(uuid.uuid4())[:8]
            p = thd.post(subject, 'a new topic 2')

            for j in range(randint(1, 5)):
                new_post = {'text':'comment text'}
                # post = thd.add_post(**new_post)
                post = thd.add_post(text='comment text for real', subject="test subject")

            if i % 1000:
                session(p).flush()

 def next_ticket_num(cls):
     gbl = cls.query.find_and_modify(
         query=dict(app_config_id=c.app.config._id),
         update={'$inc': { 'last_ticket_num': 1}},
         new=True)
     session(cls).expunge(gbl)
     return gbl.last_ticket_num

 def test_existing_lcd_unchained(self):
     commit1 = self._add_commit('Commit 1', ['file1', 'dir1/file1'])
     commit2 = self._add_commit(
         'Commit 2', ['file1', 'dir1/file1', 'dir1/file2'], ['dir1/file2'], [commit1])
     commit3 = self._add_commit(
         'Commit 3', ['file1', 'dir1/file1', 'dir1/file2'], ['file1'], [commit2])
     prev_lcd = M.repo.LastCommit(
         path='dir1',
         commit_id=commit2._id,
         entries=[
             dict(
                 name='file1',
                 commit_id=commit1._id),
             dict(
                 name='file2',
                 commit_id=commit2._id),
         ],
     )
     session(prev_lcd).flush()
     tree = self._build_tree(commit3, '/dir1', ['file1', 'file2'])
     lcd = M.repo.LastCommit.get(tree)
     self.assertEqual(lcd._id, prev_lcd._id)
     self.assertEqual(
         self.repo._commits[lcd.commit_id].message, commit2.message)
     self.assertEqual(lcd.path, 'dir1')
     self.assertEqual(lcd.entries, prev_lcd.entries)

 def deliver(cls, nid, artifact_index_id, topic):
     """Called in the notification message handler to deliver notification IDs
     to the appropriate mailboxes.  Atomically appends the nids
     to the appropriate mailboxes.
     """
     d = {
         "project_id": c.project._id,
         "app_config_id": c.app.config._id,
         "artifact_index_id": {"$in": [None, artifact_index_id]},
         "topic": {"$in": [None, topic]},
     }
     mboxes = cls.query.find(d).all()
     log.debug("Delivering notification %s to mailboxes [%s]", nid, ", ".join([str(m._id) for m in mboxes]))
     for mbox in mboxes:
         try:
             mbox.query.update(
                 {"$push": dict(queue=nid), "$set": dict(last_modified=datetime.utcnow(), queue_empty=False)}
             )
             # Make sure the mbox doesn't stick around to be flush()ed
             session(mbox).expunge(mbox)
         except:
             # log error but try to keep processing, lest all the other eligible
             # mboxes for this notification get skipped and lost forever
             log.exception(
                 "Error adding notification: %s for artifact %s on project %s to user %s",
                 nid,
                 artifact_index_id,
                 c.project._id,
                 mbox.user_id,
             )

 def clone_from(self, source_url):
     '''Initialize a repo as a clone of another'''
     self._repo.status = 'cloning'
     session(self._repo).flush(self._repo)
     log.info('Initialize %r as a clone of %s',
              self._repo, source_url)
     try:
         fullname = self._setup_paths(create_repo_dir=False)
         if os.path.exists(fullname):
             shutil.rmtree(fullname)
         if self.can_hotcopy(source_url):
             shutil.copytree(source_url, fullname)
             post_receive = os.path.join(self._repo.full_fs_path, 'hooks', 'post-receive')
             if os.path.exists(post_receive):
                 os.rename(post_receive, post_receive + '-user')
             repo = git.Repo(fullname)
         else:
             repo = git.Repo.clone_from(
                 source_url,
                 to_path=fullname,
                 bare=True)
         self.__dict__['_git'] = repo
         self._setup_special_files(source_url)
     except:
         self._repo.status = 'ready'
         session(self._repo).flush(self._repo)
         raise

def main():
    test = sys.argv[-1] == 'test'
    projects = M.Project.query.find().all()
    log.info('Fixing tracker fields')
    for p in projects:
        if p.parent_id:
            continue
        c.project = p
        q = TM.Globals.query.find()
        if not q.count():
            continue
        for g in q:
            if g.open_status_names:
                continue
            if g.status_names is None:
                old_names = ['open', 'closed']
            else:
                old_names = g.status_names.split() or ['open', 'closed']
            if g.open_status_names is None:
                g.open_status_names = ' '.join(
                    name for name in old_names if name != 'closed')
            if g.closed_status_names is None:
                g.closed_status_names = 'closed'
        if test:
            log.info('... would fix tracker(s) in %s', p.shortname)
        else:
            log.info('... fixing tracker(s) in %s', p.shortname)
            session(g).flush()
        session(g).clear()

 def request_token(self, **kw):
     req = oauth.Request.from_request(
         request.method,
         request.url.split('?')[0],
         headers=request.headers,
         parameters=dict(request.params),
         query_string=request.query_string
     )
     consumer_token = M.OAuthConsumerToken.query.get(
         api_key=req['oauth_consumer_key'])
     if consumer_token is None:
         log.error('Invalid consumer token')
         raise exc.HTTPForbidden
     consumer = consumer_token.consumer
     try:
         self.server.verify_request(req, consumer, None)
     except:
         log.error('Invalid signature')
         raise exc.HTTPForbidden
     req_token = M.OAuthRequestToken(
         consumer_token_id=consumer_token._id,
         callback=req.get('oauth_callback', 'oob')
     )
     session(req_token).flush()
     log.info('Saving new request token with key: %s', req_token.api_key)
     return req_token.to_string()

 def __call__(self, restore_context=True):
     '''Call the task function with its context.  If restore_context is True,
     c.project/app/user will be restored to the values they had before this
     function was called.
     '''
     from allura import model as M
     self.time_start = datetime.utcnow()
     session(self).flush(self)
     log.info('starting %r', self)
     old_cproject = getattr(c, 'project', None)
     old_capp = getattr(c, 'app', None)
     old_cuser = getattr(c, 'user', None)
     try:
         func = self.function
         c.project = M.Project.query.get(_id=self.context.project_id)
         c.app = None
         if c.project:
             c.project.notifications_disabled = self.context.get('notifications_disabled', False)
             app_config = M.AppConfig.query.get(_id=self.context.app_config_id)
             if app_config:
                 c.app = c.project.app_instance(app_config)
         c.user = M.User.query.get(_id=self.context.user_id)
         with log_output(log):
             self.result = func(*self.args, **self.kwargs)
         self.state = 'complete'
         return self.result
     except Exception, exc:
         log.info('Error on job %r, re-raising it', self)
         self.state = 'error'
         if hasattr(exc, 'format_error'):
             self.result = exc.format_error()
             log.error(self.result)
         else:
             self.result = traceback.format_exc()
         raise