def main():
logging.config.fileConfig(path_to_cfg)
start_utc = datetime.utcnow()
start_time = time()
global r
try:
r = reddit.Reddit(user_agent=cfg_file.get('reddit', 'user_agent'))
logging.info('Logging in as %s', cfg_file.get('reddit', 'username'))
r.login(cfg_file.get('reddit', 'username'),
cfg_file.get('reddit', 'password'))
except Exception as e:
logging.error(' ERROR: %s', e)
mod_subreddit = r.get_subreddit('mod')
#
# Do actions on individual subreddits
#
# get subreddit list
subreddits = Subreddit.query.filter(Subreddit.enabled == True).all()
sr_dict = dict()
for subreddit in subreddits:
sr_dict[subreddit.name.lower()] = subreddit
# do actions on subreddits
do_subreddits(mod_subreddit, sr_dict, start_utc)
#
# Do actions on networks
#
mods_checked = 0
# get network list
networks = Network.query.filter(Network.enabled == True).all()
# do actions on each network
for network in networks:
# get subreddits in network
network_subs = Subreddit.query.filter(Subreddit.network == network.id, Subreddit.enabled == True).all()
network_sr_dict = dict()
for subreddit in network_subs:
network_sr_dict[subreddit.name.lower()] = subreddit
# do subreddit actions on subreddits
do_subreddits(mod_subreddit, network_sr_dict, start_utc)
# check network mods
logging.info('Checking network moderators')
for subreddit in network_sr_dict.itervalues():
# only check subs in networks
if subreddit.network:
mods_checked += check_network_moderators(network, network_sr_dict)
logging.info(' Checked %s networks, added %s moderators', len(networks), mods_checked)
logging.info('Completed full run in %s', elapsed_since(start_time))
def main():
global r
logging.config.fileConfig(path_to_cfg)
# the below only works with re2
# re.set_fallback_notification(re.FALLBACK_EXCEPTION)
# which queues to check and the function to call
queue_funcs = {'report': 'get_reports',
'spam': 'get_mod_queue',
'submission': 'get_new',
'comment': 'get_comments'}
while True:
try:
r = praw.Reddit(user_agent=cfg_file.get('reddit', 'user_agent'))
logging.info('Logging in as {0}'
.format(cfg_file.get('reddit', 'username')))
r.login(cfg_file.get('reddit', 'username'),
cfg_file.get('reddit', 'password'))
sr_dict, cond_dict = initialize(queue_funcs.keys())
break
except Exception as e:
logging.error('ERROR: {0}'.format(e))
run_counter = 0
while True:
run_counter += 1
try:
# only check reports every 10 runs
# sleep afterwards in case ^C is needed
if run_counter % 10 == 0:
check_queues(queue_funcs, sr_dict, cond_dict)
Condition.clear_standard_cache()
if process_messages():
sr_dict, cond_dict = initialize(queue_funcs.keys(),
reload_mod_subs=False)
logging.info('Sleeping ({0})'.format(datetime.now()))
sleep(5)
run_counter = 0
else:
check_queues({q: queue_funcs[q]
for q in queue_funcs
if q != 'report'},
sr_dict, cond_dict)
if process_messages():
sr_dict, cond_dict = initialize(queue_funcs.keys(),
reload_mod_subs=False)
except (praw.errors.ModeratorRequired,
praw.errors.ModeratorOrScopeRequired,
HTTPError) as e:
if not isinstance(e, HTTPError) or e.response.status_code == 403:
logging.info('Re-initializing due to {0}'.format(e))
sr_dict, cond_dict = initialize(queue_funcs.keys())
except KeyboardInterrupt:
raise
except Exception as e:
logging.error('ERROR: {0}'.format(e))
session.rollback()
def main():
global r
logging.config.fileConfig(path_to_cfg)
while True:
try:
r = praw.Reddit(user_agent=cfg_file.get('reddit', 'user_agent'))
logging.info('Logging in as {0}'
.format(cfg_file.get('reddit', 'username')))
r.login(cfg_file.get('reddit', 'username'),
cfg_file.get('reddit', 'password'))
sr_dict = get_enabled_subreddits()
settings_dict = {subreddit: update_from_wiki(sr, cfg_file.get('reddit', 'owner_username')) for subreddit, sr in sr_dict.iteritems()}
break
except Exception as e:
logging.error('ERROR: {0}'.format(e))
traceback.print_exc(file=sys.stdout)
while True:
try:
bans_to_remove = session.query(Ban).filter(Ban.unban_after <= datetime.utcnow()).all()
logging.debug("\nChecking due bans")
for ban in bans_to_remove:
logging.debug(" Unbanning /u/{0} from /r/{1}".format(ban.user, ban.subreddit))
sr = sr_dict[ban.subreddit]
sr.remove_ban(ban.user)
session.add(Log(ban.user, ban.subreddit, 'unban'))
session.delete(ban)
sleep(5)
logging.info("\nLOOP\n")
updated_srs = process_messages(sr_dict, settings_dict)
if updated_srs:
if any(subreddit not in sr_dict.keys() for subreddit in updated_srs):
# settings and mod subs out of sync, reload everything
settings_dict = sr_dict.copy()
sr_dict = get_enabled_subreddits(reload_mod_subs=True)
else:
sr_dict = get_enabled_subreddits(reload_mod_subs=False)
settings_dict.update(updated_srs)
except (praw.errors.ModeratorRequired,
praw.errors.ModeratorOrScopeRequired,
praw.requests.HTTPError) as e:
if not isinstance(e, praw.requests.HTTPError) or e.response.status_code == 403:
logging.info('Re-initializing due to {0}'.format(e))
sr_dict = get_enabled_subreddits()
except KeyboardInterrupt:
raise
except Exception as e:
logging.error('ERROR: {0}'.format(e))
import traceback
traceback.print_exc()
def main():
# we get cfg_file from models.py
# see import at the top
username = cfg_file.get('reddit', 'username')
password = cfg_file.get('reddit', 'password')
print "Logging in..."
comments = Monitor(username, password)
print " Success!"
comments.monitor_comments()
def build_message(self, text, item, match,
disclaimer=False, permalink=False, intro=False):
"""Builds a message/comment for the bot to post or send."""
if intro:
message = cfg_file.get('reddit', 'intro')
message = message + " " + text
else:
message = text
if disclaimer:
message = message+'\n\n'+cfg_file.get('reddit', 'disclaimer')
if permalink and '{{permalink}}' not in message:
message = '{{permalink}}\n\n'+message
message = replace_placeholders(message, item, match)
return message
def check_queues(queue_funcs, sr_dict, cond_dict):
"""Checks all the queues for new items to process."""
global r
for queue in queue_funcs:
subreddits = [s for s in sr_dict if len(cond_dict[s][queue]) > 0]
if len(subreddits) == 0:
continue
multireddits = build_multireddit_groups(subreddits)
# fetch and process the items for each multireddit
for multi in multireddits:
if queue == 'report':
limit = cfg_file.get('reddit', 'report_backlog_limit_hours')
stop_time = datetime.utcnow() - timedelta(hours=int(limit))
else:
stop_time = max(getattr(sr, 'last_'+queue)
for sr in sr_dict.values()
if sr.name in multi)
queue_subreddit = r.get_subreddit('+'.join(multi))
if queue_subreddit:
queue_func = getattr(queue_subreddit, queue_funcs[queue])
items = queue_func(limit=None)
check_items(queue, items, stop_time, sr_dict, cond_dict)
def respond_to_modmail(modmail, start_time):
"""Responds to modmail if any submitters sent one before approval."""
cache = list()
# respond to any modmail sent in the configured window of time
time_window = timedelta(minutes=int(cfg_file.get('reddit',
'modmail_response_window_mins')))
approvals = session.query(ActionLog).filter(
and_(ActionLog.action == 'approve',
ActionLog.action_time >= start_time - time_window)
).all()
for item in approvals:
found = None
done = False
for i in cache:
if datetime.utcfromtimestamp(i.created_utc) < item.created_utc:
done = True
break
if (i.dest.lower() == '#'+item.subreddit.name.lower() and
i.author.name == item.user and
not i.replies):
found = i
break
if not found and not done:
for i in modmail:
cache.append(i)
if datetime.utcfromtimestamp(i.created_utc) < item.created_utc:
break
if (i.dest.lower() == '#'+item.subreddit.name.lower() and
i.author.name == item.user and
not i.replies):
found = i
break
if found:
found.reply('Your submission has been approved automatically by '+
cfg_file.get('reddit', 'username')+'. For future submissions '
'please wait at least '+cfg_file.get('reddit',
'modmail_response_window_mins')+' minutes before messaging '
'the mods, this post would have been approved automatically '
'even without you sending this message.')
log_request('modmail')
log_request('modmail_listing', len(cache) / 100 + 1)
def send_error_message(user, sr_name, error):
"""Sends an error message to the user if a wiki update failed."""
global r
r.send_message(user,
'Error updating from wiki in /r/{0}'.format(sr_name),
'### Error updating from [wiki configuration in /r/{0}]'
'(http://www.reddit.com/r/{0}/wiki/{1}):\n\n---\n\n'
'{2}\n\n---\n\n[View configuration documentation](https://'
'github.com/Deimos/AutoModerator/wiki/Wiki-Configuration)'
.format(sr_name,
cfg_file.get('reddit', 'wiki_page_name'),
error))
def get_user_info(username, condition):
"""Gets user info from cache, or from reddit if not cached or expired."""
global r
try:
cache_row = (session.query(UserCache)
.filter(UserCache.user == username)
.one())
# see if the condition includes a check that expires
if (condition.is_gold or
condition.link_karma or
condition.comment_karma or
condition.combined_karma):
expiry = timedelta(hours=int(cfg_file.get('reddit',
'user_cache_expiry_hours')))
else:
expiry = None
# if not past the expiry, return cached data
if (not expiry or
datetime.utcnow() - cache_row.info_last_check < expiry):
cached = r.get_redditor(username, fetch=False)
cached.is_gold = cache_row.is_gold
cached.created_utc = timegm(cache_row.created_utc.timetuple())
cached.link_karma = cache_row.link_karma
cached.comment_karma = cache_row.comment_karma
return cached
except NoResultFound:
cache_row = UserCache()
cache_row.user = username
session.add(cache_row)
# fetch the user's info from reddit
try:
user = r.get_redditor(username)
log_request('user')
# save to cache
cache_row.is_gold = user.is_gold
cache_row.created_utc = datetime.utcfromtimestamp(user.created_utc)
cache_row.link_karma = user.link_karma
cache_row.comment_karma = user.comment_karma
cache_row.info_last_check = datetime.utcnow()
session.commit()
except urllib2.HTTPError as e:
if e.code == 404:
# weird case where the user is deleted but API still shows username
return None
else:
raise
return user
def check_queues(sr_dict, cond_dict):
"""Checks all the queues for new items to process."""
global r
for queue in QUEUES:
subreddits = get_subreddits_for_queue(sr_dict, cond_dict, queue)
if not subreddits:
continue
if queue == 'report':
report_backlog_limit = timedelta(hours=int(cfg_file.get('reddit',
'report_backlog_limit_hours')))
stop_time = datetime.utcnow() - report_backlog_limit
else:
last_attr = getattr(Subreddit, 'last_'+queue)
stop_time = (session.query(func.max(last_attr))
.filter(Subreddit.enabled == True).one()[0])
# issues with request being too long at multireddit of ~3000 chars
# so split into multiple checks if it's longer than that
# split comment checks into groups of max 40 subreddits as well
multireddits = []
current_multi = []
current_len = 0
for sub in subreddits:
if (current_len > 3000 or
queue == 'comment' and len(current_multi) >= 40):
multireddits.append('+'.join(current_multi))
current_multi = []
current_len = 0
current_multi.append(sub)
current_len += len(sub) + 1
multireddits.append('+'.join(current_multi))
# fetch and process the items for each multireddit
for multi in multireddits:
queue_subreddit = r.get_subreddit(multi)
if queue_subreddit:
queue_method = getattr(queue_subreddit, QUEUES[queue])
items = queue_method(limit=1000)
check_items(queue, items, sr_dict, cond_dict, stop_time)
def respond_to_modmail(modmail, start_time):
"""Responds to modmail if any submitters sent one before approval."""
cache = list()
approvals = ActionLog.query.filter(
and_(ActionLog.action == 'approve',
ActionLog.action_time >= start_time)).all()
for item in approvals:
found = None
done = False
for i in cache:
if datetime.utcfromtimestamp(i.created_utc) < item.created_utc:
done = True
break
if (i.dest.lower() == '#'+item.subreddit.name.lower() and
i.author.name == item.user and
not i.replies):
found = i
break
if not found and not done:
for i in modmail:
cache.append(i)
if datetime.utcfromtimestamp(i.created_utc) < item.created_utc:
break
if (i.dest.lower() == '#'+item.subreddit.name.lower() and
i.author.name == item.user and
not i.replies):
found = i
break
if found:
found.reply('Your submission has been approved automatically by '+
cfg_file.get('reddit', 'username')+'. For future submissions '
'please wait at least 5 minutes before messaging the mods, '
'this post would have been approved automatically even '
'without you sending this message.')
def main():
global r
logging.config.fileConfig(path_to_cfg)
# which queues to check and the function to call
queue_funcs = {'report': 'get_reports',
'spam': 'get_mod_queue',
'submission': 'get_new',
'comment': 'get_comments'}
while True:
try:
r = praw.Reddit(user_agent=cfg_file.get('reddit', 'user_agent'))
logging.info('Logging in as {0}'
.format(cfg_file.get('reddit', 'username')))
r.login(cfg_file.get('reddit', 'username'),
cfg_file.get('reddit', 'password'))
sr_dict = get_enabled_subreddits()
Condition.update_standards()
cond_dict = load_all_conditions(sr_dict, queue_funcs.keys())
break
except Exception as e:
logging.error('ERROR: {0}'.format(e))
reports_mins = int(cfg_file.get('reddit', 'reports_check_period_mins'))
reports_check_period = timedelta(minutes=reports_mins)
last_reports_check = time()
while True:
try:
# if the standard conditions have changed, reinit all conditions
if Condition.update_standards():
logging.info('Updating standard conditions from database')
cond_dict = load_all_conditions(sr_dict, queue_funcs.keys())
# check reports if past checking period
if elapsed_since(last_reports_check) > reports_check_period:
last_reports_check = time()
check_queues({'report': queue_funcs['report']},
sr_dict, cond_dict)
check_queues({q: queue_funcs[q]
for q in queue_funcs
if q != 'report'},
sr_dict, cond_dict)
updated_srs = process_messages()
if updated_srs:
if any(sr not in sr_dict for sr in updated_srs):
sr_dict = get_enabled_subreddits(reload_mod_subs=True)
else:
sr_dict = get_enabled_subreddits(reload_mod_subs=False)
for sr in updated_srs:
update_conditions_for_sr(cond_dict,
queue_funcs.keys(),
sr_dict[sr])
except (praw.errors.ModeratorRequired,
praw.errors.ModeratorOrScopeRequired,
HTTPError) as e:
if not isinstance(e, HTTPError) or e.response.status_code == 403:
logging.info('Re-initializing due to {0}'.format(e))
sr_dict = get_enabled_subreddits()
except KeyboardInterrupt:
raise
except Exception as e:
logging.error('ERROR: {0}'.format(e))
session.rollback()
def update_from_wiki(subreddit, requester):
"""Updates conditions from the subreddit's wiki."""
global r
username = cfg_file.get('reddit', 'username')
try:
page = subreddit.get_wiki_page(cfg_file.get('reddit', 'wiki_page_name'))
except Exception:
send_error_message(requester, subreddit.display_name,
'The wiki page could not be accessed. Please ensure the page '
'http://www.reddit.com/r/{0}/wiki/{1} exists and that {2} '
'has the "wiki" mod permission to be able to access it.'
.format(subreddit.display_name,
cfg_file.get('reddit', 'wiki_page_name'),
username))
return False
html_parser = HTMLParser.HTMLParser()
page_content = html_parser.unescape(page.content_md)
# check that all the conditions are valid yaml
condition_defs = yaml.safe_load_all(page_content)
condition_num = 1
try:
for cond_def in condition_defs:
condition_num += 1
except Exception as e:
indented = ''
for line in str(e).split('\n'):
indented += ' {0}\n'.format(line)
send_error_message(requester, subreddit.display_name,
'Error when reading conditions from wiki - '
'Syntax invalid in section #{0}:\n\n{1}'
.format(condition_num, indented))
return False
# reload and actually process the conditions
condition_defs = yaml.safe_load_all(page_content)
condition_num = 1
kept_sections = []
for cond_def in condition_defs:
# ignore any non-dict sections (can be used as comments, etc.)
if not isinstance(cond_def, dict):
continue
cond_def = lowercase_keys_recursively(cond_def)
try:
check_condition_valid(cond_def)
except ValueError as e:
send_error_message(requester, subreddit.display_name,
'Invalid condition in section #{0} - {1}'
.format(condition_num, e))
return False
# create a condition for final checks
condition = Condition(cond_def)
# test to make sure that the final regex(es) are valid
for pattern in condition.match_patterns.values():
try:
re.compile(pattern)
except Exception as e:
send_error_message(requester, subreddit.display_name,
'Generated an invalid regex from section #{0} - {1}'
.format(condition_num, e))
return False
condition_num += 1
kept_sections.append(cond_def)
# Update the subreddit, or add it if necessary
try:
db_subreddit = (session.query(Subreddit)
.filter(Subreddit.name == subreddit.display_name.lower())
.one())
except NoResultFound:
db_subreddit = Subreddit()
db_subreddit.name = subreddit.display_name.lower()
db_subreddit.last_submission = datetime.utcnow() - timedelta(days=1)
db_subreddit.last_spam = datetime.utcnow() - timedelta(days=1)
db_subreddit.last_comment = datetime.utcnow() - timedelta(days=1)
session.add(db_subreddit)
db_subreddit.conditions_yaml = page_content
session.commit()
r.send_message(requester,
'{0} conditions updated'.format(username),
"{0}'s conditions were successfully updated for /r/{1}"
.format(username, subreddit.display_name))
return True
def main():
logging.config.fileConfig(path_to_cfg)
start_utc = datetime.utcnow()
start_time = time()
global r
mod_subreddit = None
try:
r = praw.Reddit(user_agent=cfg_file.get('reddit', 'user_agent'))
logging.info('Logging in as %s', cfg_file.get('reddit', 'username'))
r.login(cfg_file.get('reddit', 'username'),
cfg_file.get('reddit', 'password'))
subreddits = Subreddit.query.filter(Subreddit.enabled == True).all()
sr_dict = dict()
for subreddit in subreddits:
sr_dict[subreddit.name.lower()] = subreddit
# force population of _mod_subs and build multi-reddit
list(r.get_subreddit('mod').get_spam(limit=1))
mod_multi = '+'.join([s.name for s in subreddits
if s.name.lower() in r.user._mod_subs])
mod_subreddit = r.get_subreddit(mod_multi)
except Exception as e:
logging.error(' ERROR: %s', e)
# check if we got a subreddit to use
if mod_subreddit == None:
logging.error('AutoModerator has no subreddits to run on')
return
# check reports
items = mod_subreddit.get_reports(limit=1000)
stop_time = datetime.utcnow() - REPORT_BACKLOG_LIMIT
check_items('report', items, sr_dict, stop_time)
# check spam
items = mod_subreddit.get_modqueue(limit=1000)
stop_time = (db.session.query(func.max(Subreddit.last_spam))
.filter(Subreddit.enabled == True).one()[0])
check_items('spam', items, sr_dict, stop_time)
# check new submissions
items = mod_subreddit.get_new_by_date(limit=1000)
stop_time = (db.session.query(func.max(Subreddit.last_submission))
.filter(Subreddit.enabled == True).one()[0])
check_items('submission', items, sr_dict, stop_time)
# check new comments
comment_multi = '+'.join([s.name for s in subreddits
if not s.reported_comments_only])
if comment_multi:
comment_multi_sr = r.get_subreddit(comment_multi)
items = comment_multi_sr.get_comments(limit=1000)
stop_time = (db.session.query(func.max(Subreddit.last_comment))
.filter(Subreddit.enabled == True).one()[0])
check_items('comment', items, sr_dict, stop_time)
# respond to modmail
try:
respond_to_modmail(r.user.get_modmail(), start_utc)
except Exception as e:
logging.error(' ERROR: %s', e)
logging.info('Completed full run in %s', elapsed_since(start_time))
def update_from_wiki(sr, requester=None):
"""Returns updated settings object from the subreddit's wiki."""
global r
username = cfg_file.get('reddit', 'username')
if not requester:
requester = '/r/{0}'.format(sr.display_name)
logging.info('Updating from wiki in /r/{0}'.format(sr.display_name))
try:
page = sr.get_wiki_page(cfg_file.get('reddit', 'wiki_page_name'))
except Exception:
send_error_message(requester, sr.display_name,
'The wiki page could not be accessed. Please ensure the page '
'http://www.reddit.com/r/{0}/wiki/{1} exists and that {2} '
'has the "wiki" mod permission to be able to access it.'
.format(sr.display_name,
cfg_file.get('reddit', 'wiki_page_name'),
username))
return False
html_parser = HTMLParser.HTMLParser()
page_content = html_parser.unescape(page.content_md)
# check that all the settings are valid yaml
settings_defs = [def_group for def_group in yaml.safe_load_all(page_content)]
if len(settings_defs) == 1:
settings = settings_defs[0]
else:
send_error_message(requester, sr.display_name,
'Error when reading settings from wiki - '
'/u/{0} requires a single configuration section, multiple sections found.'
.format(username))
return False
if not isinstance(settings, dict):
send_error_message(requester, sr.display_name,
'Error when reading settings from wiki - '
'no settings found.')
return False
if len(settings) > 0:
settings = lowercase_keys_recursively(settings)
# init = defaults.copy()
# init = {name: value
# for name, value in settings
# if name in init
# (init, settings)
for setting, value in settings.iteritems():
# only keep settings values that we have defined
if setting not in defaults:
send_error_message(requester, sr.display_name,
'Error while updating from wiki - '
'unknown configuration directive `{0}` encountered.'.format(setting))
return False
# only keep proper value types
if type(value) is not settings_values[setting]:
send_error_message(requester, sr.display_name,
'Error while updating from wiki - '
'`{0}` may not be type {1}.'
.format(value, type(value)))
return False
if setting == 'default_ban_duration':
if value == '' or value == 'forever' or value == None:
settings[setting] = None
else:
settings[setting] = str_to_timedelta(value)
else:
# everything checks out
settings[setting] = value
r.send_message(requester,
'{0} settings updated'.format(username),
"{0}'s settings were successfully updated for /r/{1}"
.format(username, sr.display_name))
return settings
def check_queues(sr_dict, cond_dict):
"""Checks all the queues for new items to process."""
global r
for queue in QUEUES:
subreddits = get_subreddits_for_queue(sr_dict, cond_dict, queue)
if not subreddits:
continue
multireddits = []
if queue == 'comment':
# split comment checks into groups, using max size from cfg
# create the number of empty multireddits that we'll need
num_multis = int(ceil(len(subreddits) / float(cfg_file.get('reddit',
'comment_multireddit_size'))))
for i in range(num_multis):
multireddits.append([])
multi_avg_comments = Counter()
for sub in subreddits:
# find the index with the lowest total
lowest_index = 0
lowest_sum = multi_avg_comments[0]
for i in range(1, num_multis):
if multi_avg_comments[i] < lowest_sum:
lowest_index = i
lowest_sum = multi_avg_comments[i]
# add this subreddit to that multi
multireddits[lowest_index].append(sub)
multi_avg_comments[lowest_index] += sr_dict[sub.lower()].avg_comments
else:
# issues with request being too long at multireddit of ~3000 chars
# so split into multiple checks if it's longer than that
current_multi = []
current_len = 0
for sub in subreddits:
if current_len > 3000:
multireddits.append(current_multi)
current_multi = []
current_len = 0
current_multi.append(sub)
current_len += len(sub) + 1
multireddits.append(current_multi)
# fetch and process the items for each multireddit
for multi in multireddits:
if queue == 'report':
report_backlog_limit = timedelta(
hours=int(cfg_file.get('reddit',
'report_backlog_limit_hours')))
stop_time = datetime.utcnow() - report_backlog_limit
else:
stop_time = max([getattr(sr, 'last_'+queue)
for sr in sr_dict.values()
if sr.name in multi])
queue_subreddit = r.get_subreddit('+'.join(multi))
if queue_subreddit:
queue_method = getattr(queue_subreddit, QUEUES[queue])
items = queue_method(limit=1000)
check_items(queue, items, sr_dict, cond_dict, stop_time)
客服电话
APP下载
官方微信
请发表评论