def _get_authz_info(self):
     try:
         mtime = os.path.getmtime(self.authz_file)
     except OSError as e:
         if self._authz is not None:
             self.log.error('Error accessing authz file: %s',
                            exception_to_unicode(e))
         self._mtime = mtime = 0
         self._authz = None
         self._users = set()
     if mtime != self._mtime:
         self._mtime = mtime
         rm = RepositoryManager(self.env)
         modules = set(repos.reponame
                       for repos in rm.get_real_repositories())
         if '' in modules and self.authz_module_name:
             modules.add(self.authz_module_name)
         modules.add('')
         self.log.info('Parsing authz file: %s', self.authz_file)
         try:
             self._authz = parse(read_file(self.authz_file), modules)
             self._users = set(user for paths in self._authz.itervalues()
                               for path in paths.itervalues()
                               for user, result in path.iteritems()
                               if result)
         except Exception as e:
             self._authz = None
             self._users = set()
             self.log.error('Error parsing authz file: %s',
                            exception_to_unicode(e))
     return self._authz, self._users

 def onecmd(self, line):
     """`line` may be a `str` or an `unicode` object"""
     try:
         if isinstance(line, str):
             if self.interactive:
                 encoding = sys.stdin.encoding
             else:
                 encoding = getpreferredencoding() # sys.argv
             line = to_unicode(line, encoding)
         if self.interactive:
             line = line.replace('\\', '\\\\')
         rv = cmd.Cmd.onecmd(self, line) or 0
     except SystemExit:
         raise
     except AdminCommandError as e:
         printerr(_("Error: %(msg)s", msg=to_unicode(e)))
         if e.show_usage:
             print()
             self.do_help(e.cmd or self.arg_tokenize(line)[0])
         rv = 2
     except TracError as e:
         printerr(exception_to_unicode(e))
         rv = 2
     except Exception as e:
         printerr(exception_to_unicode(e))
         rv = 2
         if self.env_check():
             self.env.log.error("Exception in trac-admin command: %s",
                                exception_to_unicode(e, traceback=True))
     if not self.interactive:
         return rv

    def _provider_failure(self, exc, req, ep, current_filters, all_filters):
        """Raise a TracError exception explaining the failure of a provider.

        At the same time, the message will contain a link to the timeline
        without the filters corresponding to the guilty event provider `ep`.
        """
        self.log.error('Timeline event provider failed: %s',
                       exception_to_unicode(exc, traceback=True))

        ep_kinds = dict((f[0], f[1])
                        for f in ep.get_timeline_filters(req) or [])
        ep_filters = set(ep_kinds.keys())
        current_filters = set(current_filters)
        other_filters = set(current_filters) - ep_filters
        if not other_filters:
            other_filters = set(all_filters) -  ep_filters
        args = [(a, req.args.get(a)) for a in ('from', 'format', 'max',
                                               'daysback')]
        href = req.href.timeline(args + [(f, 'on') for f in other_filters])
        # TRANSLATOR: ...want to see the 'other kinds of events' from... (link)
        other_events = tag.a(_('other kinds of events'), href=href)
        raise TracError(tag(
            tag.p(tag_("Event provider %(name)s failed for filters "
                       "%(kinds)s: ",
                       name=tag.tt(ep.__class__.__name__),
                       kinds=', '.join('"%s"' % ep_kinds[f] for f in
                                       current_filters & ep_filters)),
                  tag.b(exception_to_unicode(exc)), class_='message'),
            tag.p(tag_("You may want to see the %(other_events)s from the "
                       "Timeline or notify your Trac administrator about the "
                       "error (detailed information was written to the log).",
                       other_events=other_events))))

    def _do_authz_raw(self, req):
        # get default authz file from trac.ini
        authz_file = self.config.get('trac', 'authz_file')

        # test if authz file exists and is writable
        if not os.access(authz_file,os.W_OK|os.R_OK):
            raise TracError("Can't access authz file %s" % authz_file)

        # evaluate forms
        if req.method == 'POST':
            current=req.args.get('current').strip().replace('\r', '')

            # encode to utf-8
            current = current.encode('utf-8')

            # parse and validate authz file with a config parser
            from ConfigParser import ConfigParser
            from StringIO import StringIO
            cp = ConfigParser()
            try:
                cp.readfp(StringIO(current))
            except Exception, e:
                raise TracError("Invalid Syntax: %s" % exception_to_unicode(e))

            # write to disk
            try:
                fp = open(authz_file, 'wb')
                current = fp.write(current)
                fp.close()
            except Exception, e:
                raise TracError("Can't write authz file: %s" % exception_to_unicode(e))

 def replace(self, old_uid, new_uid):
     try:
         self.env.db_transaction("""
             DELETE FROM %s WHERE %s=%%s
             """ % (self.table, self.column), (new_uid,))
     except _get_db_exc(self.env), e:
         result = exception_to_unicode(e)
         msg = 'failed: %s' % exception_to_unicode(e, traceback=True)
         self.log.debug(self.msg(old_uid, new_uid, self.table,
                                 self.column, result=msg))
         return dict(error={(self.table, self.column, None): result})

def _dispatch_request(req, env, env_error):
    resp = []

    # fixup env.abs_href if `[trac] base_url` was not specified
    if env and not env.abs_href.base:
        env._abs_href = req.abs_href

    try:
        if not env and env_error:
            raise HTTPInternalError(env_error)
        try:
            dispatcher = RequestDispatcher(env)
            dispatcher.dispatch(req)
        except RequestDone:
            pass
        resp = req._response or []

    except HTTPException, e:
        # This part is a bit more complex than it should be.
        # See trac/web/api.py for the definition of HTTPException subclasses.
        if env:
            env.log.warn(exception_to_unicode(e))
        try:
            # We try to get localized error messages here, 
            # but we should ignore secondary errors
            title = _('Error')
            if e.reason:
                if title.lower() in e.reason.lower():
                    title = e.reason
                else:
                    title = _('Error: %(message)s', message=e.reason)
        except:
            title = 'Error'
        # The message is based on the e.detail, which can be an Exception
        # object, but not a TracError one: when creating HTTPException,
        # a TracError.message is directly assigned to e.detail
        if isinstance(e.detail, Exception): # not a TracError
            message = exception_to_unicode(e.detail)
        elif isinstance(e.detail, Fragment): # markup coming from a TracError
            message = e.detail
        else:
            message = to_unicode(e.detail)
        data = {'title': title, 'type': 'TracError', 'message': message,
                'frames': [], 'traceback': None}
        if e.code == 403 and req.authname == 'anonymous':
            # TRANSLATOR: ... not logged in, you may want to 'do so' now (link)
            do_so = tag.a(_("do so"), href=req.href.login())
            req.chrome['notices'].append(
                tag_("You are currently not logged in. You may want to "
                     "%(do_so)s now.", do_so=do_so))
        try:
            req.send_error(sys.exc_info(), status=e.code, env=env, data=data)
        except RequestDone:
            pass

 def _log_error(item, e):
     ue = exception_to_unicode(e)
     if isinstance(e, DistributionNotFound):
         env.log.debug('Skipping "%s": ("%s" not found)', item, ue)
     elif isinstance(e, VersionConflict):
         env.log.error('Skipping "%s": (version conflict "%s")',
                       item, ue)
     elif isinstance(e, UnknownExtra):
         env.log.error('Skipping "%s": (unknown extra "%s")', item, ue)
     else:
         env.log.error('Skipping "%s": %s', item,
                       exception_to_unicode(e, traceback=True))

def do_upgrade(env, version, cursor):
    """Move attachments from the `attachments` directory into `files`, hashing
    the filenames in the process."""
    path = env.path
    old_dir = os.path.join(path, 'attachments')
    if not os.path.exists(old_dir):
        return
    old_stat = os.stat(old_dir)
    new_dir = os.path.join(path, 'files', 'attachments')
    if not os.path.exists(new_dir):
        os.makedirs(new_dir)

    cursor.execute("""
        SELECT type, id, filename FROM attachment ORDER BY type, id
        """)
    for row in cursor:
        move_attachment_file(env, *row)

    # Try to preserve permissions and ownerships of the attachments
    # directory for $ENV/files
    for dir, dirs, files in os.walk(os.path.join(path, 'files')):
        try:
            if hasattr(os, 'chmod'):
                os.chmod(dir, old_stat.st_mode)
            if hasattr(os, 'chflags') and hasattr(old_stat, 'st_flags'):
                os.chflags(dir, old_stat.st_flags)
            if hasattr(os, 'chown'):
                os.chown(dir, old_stat.st_uid, old_stat.st_gid)
        except OSError:
            pass

    # Remove empty directory hierarchy
    try:
        for dir, dirs, files in os.walk(old_dir, topdown=False):
            os.rmdir(dir)
    except OSError, e:
        env.log.warning("Can't delete old attachments directory %s: %s",
                         old_dir, exception_to_unicode(e))
        # TRANSLATOR: Wrap message to 80 columns
        printerr(_("""\
The upgrade of attachments was successful, but the old attachments directory:

  %(src_dir)s

couldn't be removed, possibly due to the presence of files that weren't
referenced in the database. The error was:

  %(exception)s

This error can be ignored, but for keeping your environment clean you should
backup any remaining files in that directory and remove it manually.
""", src_dir=old_dir, exception=exception_to_unicode(e)))

    def process_request(self, req):
        # First check for the expected X-Gitlab-Event header
        event = req.get_header('X-Gitlab-Event')
        if not event and event.lower() == 'merge request hook':
            self.log.warn('GitLab webhook request event missing or '
                          'not handled: {}'.format(event))
            req.send_response(422)
            req.end_headers()
            return

        token = req.get_header('X-Gitlab-Token')
        if not self._verify_token(token):
            self.log.warn('GitLab webhook request security token missing '
                          'or not valid')
            req.send_response(401)
            req.end_headers()
            return

        try:
            hook_data = json.load(req)
        except Exception as exc:
            self.log.warn(
                'Gitlab webhook failed to parse the JSON request '
                'data: {}'.format(exc))
            return req.send_no_content()

        self.log.debug('GitLab webhook received event payload:\n' +
                pformat(hook_data))

        if hook_data['object_attributes']['state'] == 'closed':
            # Do not update tickets/branches for closed merged requests
            return req.send_no_content()

        try:
            synced_branch = self._sync_branch(hook_data)
        except Exception as exc:
            self.log.warn(
                'Gitlab webhook failed to sync the downstream '
                'branch: {}'.format(exception_to_unicode(exc, True)))
            synced_branch = False

        try:
            self._create_or_update_ticket(hook_data, synced_branch)
        except Exception as exc:
            self.log.warn(
                'Gitlab webhook failed to create or update the '
                'ticket for this merge request: {}'.format(
                    exception_to_unicode(exc, True)))

        req.send_no_content()

def _dispatch_request(req, env, env_error):
    resp = []

    # fixup env.abs_href if `[trac] base_url` was not specified
    if env and not env.abs_href.base:
        env._abs_href = req.abs_href

    try:
        if not env and env_error:
            raise HTTPInternalError(env_error)
        try:
            dispatcher = RequestDispatcher(env)
            dispatcher.dispatch(req)
        except RequestDone:
            pass
        resp = req._response or []

    except HTTPException, e:
        # This part is a bit more complex than it should be.
        # See trac/web/api.py for the definition of HTTPException subclasses.
        if env:
            env.log.warn(exception_to_unicode(e))
        title = 'Error'
        if e.reason:
            if 'error' in e.reason.lower():
                title = e.reason
            else:
                title = 'Error: %s' % e.reason
        # The message is based on the e.detail, which can be an Exception
        # object, but not a TracError one: when creating HTTPException,
        # a TracError.message is directly assigned to e.detail
        if isinstance(e.detail, Exception): # not a TracError
            message = exception_to_unicode(e.detail)
        elif isinstance(e.detail, Fragment): # markup coming from a TracError
            message = e.detail
        else:
            message = to_unicode(e.detail)
        data = {'title': title, 'type': 'TracError', 'message': message,
                'frames': [], 'traceback': None}
        if e.code == 403 and req.authname == 'anonymous':
            req.chrome['notices'].append(Markup(
                _('You are currently not logged in. You may want to '
                  '<a href="%(href)s">do so</a> now.',
                  href=req.href.login())))
        try:
            req.send_error(sys.exc_info(), status=e.code, env=env, data=data)
        except RequestDone:
            pass

    def _close_mr(self, ticket_id, proj_id, mr_id, resolution):
        if not self.gitlab_api_token:
            self.log.warn(
                "GitLab API token not configured; GitLab webhook can't "
                "update the downstream merge request")
            return

        self.log.debug('Trying to close merge request {} since ticket {} '
                       'was closed.'.format(mr_id, ticket_id))

        headers = {'Private-Token': self.gitlab_api_token}
        url = '{}/api/v4/projects/{}/merge_requests/{}'.format(
                self.gitlab_url.rstrip('/'), proj_id, mr_id)
        try:
            r = requests.put(url, data={'state_event': 'close'},
                             headers=headers,
                             timeout=10)
        except Exception as exc:
            self.log.error(
                    'Error updating merge request: {}'.format(
                        exception_to_unicode(exc, True)))
            return

        text = ("Downstream ticket [Trac#{}]({}) was closed as {}, so I "
                "closed this merge request.  If you feel this was in error "
                "feel free to reopen.".format(
                    ticket_id, self.env.abs_href.ticket(ticket_id),
                    resolution))

        self._post_comment_to_mr(proj_id, mr_id, text)

        self.log.info('Successfully closed merge request {}'.format(mr_id))

    def render_admin_panel(self, req, cat, page, version):
        self.log.debug("cat: %s page: %s", cat, page)
        req.perm.require('TRAC_ADMIN')

        options = (
            'api_base_url',
            'api_token',
            'room_id',
            'only_owner_changed',
            'notify_symbol',
            'api_token_field_name')

        self.log.debug("method: %s", req.method)
        if req.method == 'POST':
            for option in options:
                self.config.set(SECTION_NAME, option, req.args.get(option))

            try:
                self.config.save()
                self.log.debug('config saved.')
                add_notice(req, 'Your changes have been saved.')
            except Exception, e:
                self.log.error("Error writing to trac.ini: %s", exception_to_unicode(e))
                add_warning(req, 'Error writing to trac.ini.')

            req.redirect(req.href.admin(cat, page))

    def _do_delete(self, req, milestone):
        req.perm(milestone.resource).require('MILESTONE_DELETE')

        retarget_to = req.args.get('target') or None
        # Don't translate ticket comment (comment:40:ticket:5658)
        retargeted_tickets = \
            milestone.move_tickets(retarget_to, req.authname,
                "Ticket retargeted after milestone deleted")
        milestone.delete(author=req.authname)
        add_notice(req, _('The milestone "%(name)s" has been deleted.',
                          name=milestone.name))
        if retargeted_tickets:
            add_notice(req, _('The tickets associated with milestone '
                              '"%(name)s" have been retargeted to milestone '
                              '"%(retarget)s".', name=milestone.name,
                              retarget=retarget_to))
            new_values = {'milestone': retarget_to}
            comment = _("Tickets retargeted after milestone deleted")
            tn = BatchTicketNotifyEmail(self.env)
            try:
                tn.notify(retargeted_tickets, new_values, comment, None,
                          req.authname)
            except Exception, e:
                self.log.error("Failure sending notification on ticket batch "
                               "change: %s", exception_to_unicode(e))
                add_warning(req, tag_("The changes have been saved, but an "
                                      "error occurred while sending "
                                      "notifications: %(message)s",
                                      message=to_unicode(e)))

 def safe_wiki_to_html(context, text):
     try:
         return format_to_html(self.env, context, text)
     except Exception, e:
         self.log.error('Unable to render component documentation: %s',
                        exception_to_unicode(e, traceback=True))
         return tag.pre(text)

    def backup(self, dest_file):
        from subprocess import Popen, PIPE
        db_url = self.env.config.get('trac', 'database')
        scheme, db_prop = _parse_db_str(db_url)
        db_params = db_prop.setdefault('params', {})
        db_name = os.path.basename(db_prop['path'])

        args = [self.pg_dump_path, '-C', '--inserts', '-x', '-Z', '8']
        if 'user' in db_prop:
            args.extend(['-U', db_prop['user']])
        if 'host' in db_params:
            host = db_params['host']
        else:
            host = db_prop.get('host')
        if host:
            args.extend(['-h', host])
            if '/' not in host:
                args.extend(['-p', str(db_prop.get('port', '5432'))])

        if 'schema' in db_params:
            args.extend(['-n', '"%s"' % db_params['schema']])

        dest_file += ".gz"
        args.extend(['-f', dest_file, db_name])

        environ = os.environ.copy()
        if 'password' in db_prop:
            environ['PGPASSWORD'] = str(db_prop['password'])
        try:
            p = Popen(args, env=environ, stderr=PIPE, close_fds=close_fds)
        except OSError, e:
            raise TracError(_("Unable to run %(path)s: %(msg)s",
                              path=self.pg_dump_path,
                              msg=exception_to_unicode(e)))

    def __init__(self, path, params, log):
        self.log = log
        self.pool = Pool()

        # Remove any trailing slash or else subversion might abort
        if isinstance(path, unicode):
            path_utf8 = path.encode('utf-8')
        else: # note that this should usually not happen (unicode arg expected)
            path_utf8 = to_unicode(path).encode('utf-8')

        path_utf8 = core.svn_path_canonicalize(
                                os.path.normpath(path_utf8).replace('\\', '/'))
        self.path = path_utf8.decode('utf-8')

        root_path_utf8 = repos.svn_repos_find_root_path(path_utf8, self.pool())
        if root_path_utf8 is None:
            raise TracError(_("%(path)s does not appear to be a Subversion "
                              "repository.", path=to_unicode(path_utf8)))

        try:
            self.repos = repos.svn_repos_open(root_path_utf8, self.pool())
        except core.SubversionException, e:
            raise TracError(_("Couldn't open Subversion repository %(path)s: "
                              "%(svn_error)s", path=to_unicode(path_utf8),
                              svn_error=exception_to_unicode(e)))

    def __init__(self, path, params, log,
                 persistent_cache=False,
                 git_bin='git',
                 git_fs_encoding='utf-8',
                 shortrev_len=7,
                 rlookup_uid=lambda _: None,
                 use_committer_id=False,
                 use_committer_time=False,
                 ):

        self.logger = log
        self.gitrepo = path
        self.params = params
        self.shortrev_len = max(4, min(shortrev_len, 40))
        self.rlookup_uid = rlookup_uid
        self.use_committer_time = use_committer_time
        self.use_committer_id = use_committer_id

        try:
            self.git = PyGIT.StorageFactory(path, log, not persistent_cache,
                                            git_bin=git_bin,
                                            git_fs_encoding=git_fs_encoding) \
                            .getInstance()
        except PyGIT.GitError, e:
            log.error(exception_to_unicode(e))
            raise TracError("%s does not appear to be a Git "
                            "repository." % path)

    def __init__(self, path, authz, log, options={}):
        self.log = log
        self.options = options
        self.pool = Pool()

        # Remove any trailing slash or else subversion might abort
        if isinstance(path, unicode):
            self.path = path
            path_utf8 = path.encode("utf-8")
        else:  # note that this should usually not happen (unicode arg expected)
            self.path = to_unicode(path)
            path_utf8 = self.path.encode("utf-8")
        path_utf8 = os.path.normpath(path_utf8).replace("\\", "/")
        root_path_utf8 = repos.svn_repos_find_root_path(path_utf8, self.pool())
        if root_path_utf8 is None:
            raise TracError(_("%(path)s does not appear to be a Subversion " "repository.", path=to_unicode(path_utf8)))

        try:
            self.repos = repos.svn_repos_open(root_path_utf8, self.pool())
        except core.SubversionException, e:
            raise TracError(
                _(
                    "Couldn't open Subversion repository %(path)s: " "%(svn_error)s",
                    path=to_unicode(path_utf8),
                    svn_error=exception_to_unicode(e),
                )
            )

 def expand_macro(self, formatter, name, content):
     db = self.env.get_db_cnx()
     cursor = db.cursor()
     try:
         cursor.execute(content)
     except Exception, e:
         return system_message(_("Invalid SQL"), exception_to_unicode(e))

    def _parse_arg_list(self):
        """Parse the supplied request parameters into a list of
        `(name, value)` tuples.
        """
        fp = self.environ['wsgi.input']

        # Avoid letting cgi.FieldStorage consume the input stream when the
        # request does not contain form data
        ctype = self.get_header('Content-Type')
        if ctype:
            ctype, options = cgi.parse_header(ctype)
        if ctype not in ('application/x-www-form-urlencoded',
                         'multipart/form-data'):
            fp = StringIO('')

        # Python 2.6 introduced a backwards incompatible change for
        # FieldStorage where QUERY_STRING is no longer ignored for POST
        # requests. We'll keep the pre 2.6 behaviour for now...
        if self.method == 'POST':
            qs_on_post = self.environ.pop('QUERY_STRING', '')
        fs = _FieldStorage(fp, environ=self.environ, keep_blank_values=True)
        if self.method == 'POST':
            self.environ['QUERY_STRING'] = qs_on_post

        args = []
        for value in fs.list or ():
            try:
                name = unicode(value.name, 'utf-8')
                if not value.filename:
                    value = unicode(value.value, 'utf-8')
            except UnicodeDecodeError, e:
                raise HTTPBadRequest(
                    _("Invalid encoding in form data: %(msg)s",
                      msg=exception_to_unicode(e)))
            args.append((name, value))