本文整理汇总了Python中retask.queue.Queue类的典型用法代码示例。如果您正苦于以下问题:Python Queue类的具体用法?Python Queue怎么用?Python Queue使用的例子?那么恭喜您, 这里精选的类代码示例或许可以为您提供帮助。
在下文中一共展示了Queue类的20个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于我们的系统推荐出更棒的Python代码示例。
示例1: notifier
def notifier(self):
""" Connect to the instruction queue and notify bugyou to create a queue
for the plugin and start pushing the fedmsg messags.
"""
queue = Queue('instruction')
queue.connect()
for plugin in self.plugins:
try:
topic = self.config.get(plugin, 'topic')
except ConfigParser.NoOptionError:
log.error("Config does not exists")
if topic is None:
log.info("Config does not exists")
continue
payload = {
'type': 'create',
'queue_name': plugin,
'topic': topic,
}
task = Task(payload)
queue.enqueue(task)
if plugin in self.active_plugins:
Plugin = self.active_plugins[plugin]
plugin_obj = Plugin()
plugin_obj.initialize()
开发者ID:sayanchowdhury,项目名称:bugyou_plugins,代码行数:27,代码来源:cntrl.py
示例2: main
def main():
jobqueue = Queue('jobqueue')
jobqueue.connect()
while True:
task = jobqueue.wait()
task_data = task.data
pos, num_images = task_data['pos']
compose_details = task_data['compose']
if pos == 1:
session = init_model()
compose_id = compose_details['id']
compose_obj = session.query(ComposeDetails).filter_by(
compose_id=compose_id).first()
compose_obj.status = u'r'
session.commit()
params = copy.deepcopy(compose_details)
params.update({'status': 'running'})
publish_to_fedmsg(topic='compose.running', **params)
result, running_status = auto_job(task_data)
开发者ID:gobindaakhuli,项目名称:fedora-infra-ansible,代码行数:26,代码来源:autocloud_job.py
示例3: main
def main():
jobqueue = Queue('jobqueue')
jobqueue.connect()
while True:
task = jobqueue.wait()
log.debug("%s", task.data)
auto_job(task.data)
开发者ID:TridevGuha,项目名称:autocloud,代码行数:7,代码来源:autocloud_job.py
示例4: compile
def compile(request):
"""
Enqueue the task to Queue
"""
filename = request.POST.get('filename', False)
text = request.POST.get('text', False)
if filename is False:
return HttpResponse(json.dumps({'output':'Invalid filename'}),
content_type="application/json")
if text is False:
return HttpResponse(json.dumps({'output':'Empty file'}),
content_type="application/json")
try:
queue = Queue('rcc')
queue.connect()
task = Task({'filename':filename, 'text':text})
job = queue.enqueue(task)
except:
return HttpResponse(json.dumps({'output':'Error creating Job'}),
content_type="application/json")
while True:
if job.result is None:
continue
break
return HttpResponse(json.dumps({'output' : job.result}),
content_type="application/json")
开发者ID:birajkarmakar,项目名称:rcc,代码行数:31,代码来源:views.py
示例5: produce_jobs
def produce_jobs(infox):
""" Queue the jobs into jobqueue
:args infox: list of dictionaries contains the image url and the buildid
"""
jobqueue = Queue('jobqueue')
jobqueue.connect()
session = init_model()
timestamp = datetime.datetime.now()
for info in infox:
jd = JobDetails(
taskid=info['buildid'],
status='q',
created_on=timestamp,
user='admin',
last_updated=timestamp)
session.add(jd)
session.commit()
job_details_id = jd.id
log.info('Save {jd_id} to database'.format(jd_id=job_details_id))
info.update({'job_id': jd.id})
task = Task(info)
jobqueue.enqueue(task)
log.info('Enqueue {jd_id} to redis'.format(jd_id=job_details_id))
publish_to_fedmsg(topic='image.queued', image_url=info['image_url'],
image_name=info['name'], status='queued',
buildid=info['buildid'], job_id=info['job_id'],
release=info['release'])
开发者ID:ralphbean,项目名称:autocloud,代码行数:31,代码来源:__init__.py
示例6: main
def main():
jobqueue = Queue('jobqueue')
jobqueue.connect()
while True:
task = jobqueue.wait()
task_data = task.data
pos, num_images = task_data['pos']
compose_details = task_data['compose']
if pos == 1:
session = init_model()
compose_id = compose_details['id']
compose_obj = session.query(ComposeDetails).filter_by(
compose_id=compose_id).first()
compose_status = compose_obj.status.code
# Here the check if the compose_status has completed 'c' is for
# failsafe. This condition is never to be hit. This is to avoid
# sending message to fedmsg.
if compose_status in ('r', 'c'):
log.info("Compose %s already running. Skipping sending to \
fedmsg" % compose_id)
else:
compose_obj.status = u'r'
session.commit()
params = copy.deepcopy(compose_details)
params.update({'status': 'running'})
publish_to_fedmsg(topic='compose.running', **params)
result, running_status = auto_job(task_data)
开发者ID:abhishekg785,项目名称:autocloud,代码行数:35,代码来源:autocloud_job.py
示例7: Worker
class Worker(object):
""" Represents the worker process. Waits for tasks to come in from the
webapp and then acts on them.
"""
def __init__(self):
self.queue = Queue('commits')
self.queue.connect()
# TODO -- set both of these with the config file.
# Use pyramid tools to load config.
self.sleep_interval = 1
self.scratch_dir = "/home/threebean/scratch/pep8bot-scratch"
try:
os.makedirs(self.scratch_dir)
except OSError:
pass # Assume that the scratch_dir already exists.
def run(self):
while True:
time.sleep(self.sleep_interval)
print "Waking"
if self.queue.length == 0:
continue
task = self.queue.dequeue()
data = task.data
url = data['repository']['url']
# TODO -- don't clone this url. But fork and clone our url.
name = data['repository']['name']
owner = data['repository']['owner']['name']
self.working_dir = tempfile.mkdtemp(
prefix=owner + '-' + name,
dir=self.scratch_dir,
)
print "** Cloning to", self.working_dir
print sh.git.clone(url, self.working_dir)
print "** Processing files."
for root, dirs, files in os.walk(self.working_dir):
if '.git' in root:
continue
for filename in files:
if filename.endswith(".py"):
infile = root + "/" + filename
print "** Tidying", infile
tmpfile = infile + ".bak"
script = os.path.expanduser(
"~/devel/PythonTidy/PythonTidy.py"
)
sh.python(script, infile, tmpfile)
shutil.move(tmpfile, infile)
with directory(self.working_dir):
print sh.pwd()
print sh.git.status()
开发者ID:pep8bot,项目名称:pep8bot,代码行数:58,代码来源:worker.py
示例8: _get_queue
def _get_queue(self, bgroup):
if not bgroup in self.build_queues:
q_id = "copr-be-{0}".format(bgroup)
q = Queue(q_id)
if not q.connect():
# As we already connected to jg_control_message, this should
# be also OK.
raise Exception("can't connect to redis, should never happen!")
return q
return self.build_queues[bgroup]
开发者ID:danvratil,项目名称:copr,代码行数:11,代码来源:jobgrabcontrol.py
示例9: connect_queues
def connect_queues(self):
"""
Connects to the retask queues. One queue per builders group.
"""
for group in self.opts.build_groups:
queue = Queue("copr-be-{0}".format(group["id"]))
queue.connect()
self.task_queues_by_group[group["name"]] = queue
for arch in group["archs"]:
self.task_queues_by_arch[arch] = queue
开发者ID:evilkost,项目名称:copr,代码行数:11,代码来源:job_grab.py
示例10: produce_jobs
def produce_jobs(infox):
""" Queue the jobs into jobqueue
:args infox: list of dictionaries contains the image url and the buildid
"""
jobqueue = Queue('jobqueue')
jobqueue.connect()
family_mapping = {
'Cloud_Base': 'b',
'Atomic': 'a'
}
session = init_model()
timestamp = datetime.datetime.now()
for info in infox:
image_name = info['path'].split('.x86_64')[0].split('/')[-1]
jd = ComposeJobDetails(
arch=info['arch'],
compose_id=info['compose']['id'],
created_on=timestamp,
family=family_mapping[info['subvariant']],
image_url=info['absolute_path'],
last_updated=timestamp,
release=info['compose']['release'],
status='q',
subvariant=info['subvariant'],
user='admin',
image_format=info['format'],
image_type=info['type'],
image_name=image_name,
)
session.add(jd)
session.commit()
job_details_id = jd.id
log.info('Save {jd_id} to database'.format(jd_id=job_details_id))
info.update({'job_id': jd.id})
task = Task(info)
jobqueue.enqueue(task)
log.info('Enqueue {jd_id} to redis'.format(jd_id=job_details_id))
publish_to_fedmsg(topic='image.queued',
compose_url=info['absolute_path'],
compose_id=info['compose']['id'],
image_name=image_name,
status='queued',
job_id=info['job_id'],
release=info['compose']['release'],
family=jd.family.value,
type=info['type'])
session.close()
开发者ID:AdamWill,项目名称:autocloud,代码行数:53,代码来源:__init__.py
示例11: update
def update(data):
'''
Updates the git repo for the given user
:arg user: github username
:arg repo: Code repo name
'''
queue = Queue('puluupdates')
if not queue.connect():
return
task = Task(data=data, raw=True)
queue.enqueue(task)
开发者ID:kushaldas,项目名称:pulu,代码行数:13,代码来源:utils.py
示例12: BasePlugin
class BasePlugin(object):
__metaclass__ = abc.ABCMeta
def __init__(self, *args, **kwargs):
self.config = load_config(PLUGINS_CONFIG_FILEPATH)
self.active_services = get_active_services()
self.services = []
def initialize(self):
self.init_retask_connection()
self.load_services()
self.init_worker()
def init_retask_connection(self):
""" Connect to the retask queue for the plugin """
self.queue = Queue(self.plugin_name)
conn = self.queue.connect()
log.info("Initializing redis conection: %s" % self.plugin_name)
if not conn:
log.error("Could not connect to %s queue" % self.plugin_name)
return False
def consume(self):
while True:
task = self.queue.wait()
if task:
log.debug("Processing Message: %s" % task.data['msg']['body']['msg_id'])
self.process(task.data['msg'])
def init_worker(self):
""" Create a process and start consuming the messages """
process = multiprocessing.Process(target=self.consume)
process.start()
def load_services(self):
""" Load the services for the plugin """
services = self.config.get(self.plugin_name, 'services').split(',')
log.info("Start loading services")
for service in services:
self.services.append(self.active_services[service].load())
log.info("Complete loading services %s" % self.services)
@abc.abstractmethod
def process(self):
""" Consumes the messages from retask """
return
@abc.abstractmethod
def do_pagure(self):
""" Override to do activity related to pagure """
return
开发者ID:sayanchowdhury,项目名称:bugyou_plugins,代码行数:51,代码来源:base.py
示例13: upload_file
def upload_file():
if request.method == "POST":
file = request.files["file"]
if file and allowed_file(file.filename):
filename = secure_filename(file.filename)
file.save(os.path.join(APP.config["UPLOAD_FOLDER"], filename))
# Now add the information in the queue for processing
t = Task({"filename": filename})
queue = Queue("incoming_files")
queue.connect()
queue.enqueue(t)
return "Log uploaded successfully."
return """
开发者ID:kushaldas,项目名称:teenspirit,代码行数:14,代码来源:app.py
示例14: LogBot
class LogBot(irc.IRCClient):
"""A logging IRC bot."""
nickname = 'pyconsprints'
def __init__(self, channel):
self.chn = '#'+channel
self.qs_queue = []
self.logger = None
self.q = Queue('bug-messages')
self.q.connect()
self.channel_admin = ['kushal',]
def connectionMade(self):
irc.IRCClient.connectionMade(self)
self.islogging = False
self._namescallback = {}
def connectionLost(self, reason):
irc.IRCClient.connectionLost(self, reason)
self.islogging = False
def signedOn(self):
"""Called when bot has succesfully signed on to server."""
self.join(self.factory.channel)
def privmsg(self, user, channel, msg):
"""This will get called when the bot receives a message."""
user = user.split('!', 1)[0]
if user == BOTNAME:
print '[[%s]]' % msg
task = Task(msg)
self.q.enqueue(task)
user_cond = user in self.channel_admin
if msg == '#masters' and user_cond:
self.msg(self.chn, "My current masters are: %s" % ",".join(self.channel_admin))
if msg.startswith('#add:') and user_cond:
try:
name = msg.split()[1]
print name
self.channel_admin.append(name)
self.msg(self.chn,'%s is a master now.' % name)
except Exception, err:
print err
if msg.startswith('#test:') and user_cond:
bugid = msg.split()[1]
msg = 'dummy/issue%s' % bugid
task = Task(msg)
self.q.enqueue(task)
开发者ID:kushaldas,项目名称:pypatcher,代码行数:49,代码来源:patcherbot.py
示例15: init_task_queues
def init_task_queues(self):
"""
Connect to the retask.Queue for each group_id. Remove old tasks from queues.
"""
try:
for group in self.opts.build_groups:
group_id = group["id"]
queue = Queue("copr-be-{0}".format(group_id))
queue.connect()
self.task_queues[group_id] = queue
except ConnectionError:
raise CoprBackendError(
"Could not connect to a task queue. Is Redis running?")
self.clean_task_queues()
开发者ID:evilkost,项目名称:copr,代码行数:15,代码来源:backend.py
示例16: __init__
def __init__(self, channel):
self.chn = '#'+channel
self.qs_queue = []
self.logger = None
self.q = Queue('bug-messages')
self.q.connect()
self.channel_admin = ['kushal',]
开发者ID:kushaldas,项目名称:pypatcher,代码行数:7,代码来源:patcherbot.py
示例17: __init__
def __init__(self, opts, events, worker_num, group_id,
callback=None, lock=None):
# base class initialization
multiprocessing.Process.__init__(self, name="worker-builder")
self.opts = opts
# job management stuff
self.task_queue = Queue("copr-be-{0}".format(str(group_id)))
self.task_queue.connect()
# event queue for communicating back to dispatcher
self.events = events
self.worker_num = worker_num
self.group_id = group_id
self.kill_received = False
self.lock = lock
self.frontend_callback = FrontendClient(opts, events)
self.callback = callback
if not self.callback:
log_name = "worker-{0}-{1}.log".format(
self.group_name,
self.worker_num)
self.logfile = os.path.join(self.opts.worker_logdir, log_name)
self.callback = WorkerCallback(logfile=self.logfile)
self.vm_name = None
self.vm_ip = None
self.callback.log("creating worker: dynamic ip")
开发者ID:1dot75cm,项目名称:Copr,代码行数:31,代码来源:dispatcher.py
示例18: init_retask_connection
def init_retask_connection(self):
""" Connect to the retask queue for the plugin """
self.queue = Queue(self.plugin_name)
conn = self.queue.connect()
log.info("Initializing redis conection: %s" % self.plugin_name)
if not conn:
log.error("Could not connect to %s queue" % self.plugin_name)
return False
开发者ID:sayanchowdhury,项目名称:bugyou_plugins,代码行数:8,代码来源:base.py
示例19: RedisQueue
class RedisQueue(object):
def __init__(self, host, name, port=6379, password=None):
self.super_queue = Queue(
name,
{
'host': host,
'port': port,
'db': 0,
'password': password,
})
self.super_queue.connect()
def get(self):
return self.super_queue.wait()
def put(self, data):
self.super_queue.enqueue(Task(data))
开发者ID:rossdylan,项目名称:netcrawl,代码行数:17,代码来源:RedisQueue.py
示例20: __init__
def __init__(self, opts, log=None):
self.log = log
self.opts = opts
# channel for Backend <--> JobGrabber communication
self.jg_start = Queue("jg_control_start")
# channel for JobGrabber <--> [[Builders]] communication
self.build_queues = dict()
while not self.jg_start.connect():
wait_log(self.log, "waiting for redis", 5)
开发者ID:danvratil,项目名称:copr,代码行数:9,代码来源:jobgrabcontrol.py
注:本文中的retask.queue.Queue类示例由纯净天空整理自Github/MSDocs等源码及文档管理平台,相关代码片段筛选自各路编程大神贡献的开源项目,源码版权归原作者所有,传播和使用请参考对应项目的License;未经允许,请勿转载。 |
请发表评论