• 设为首页
  • 点击收藏
  • 手机版
    手机扫一扫访问
    迪恩网络手机版
  • 关注官方公众号
    微信扫一扫关注
    迪恩网络公众号

Python storm.emit函数代码示例

原作者: [db:作者] 来自: [db:来源] 收藏 邀请

本文整理汇总了Python中storm.emit函数的典型用法代码示例。如果您正苦于以下问题:Python emit函数的具体用法?Python emit怎么用?Python emit使用的例子?那么恭喜您, 这里精选的函数代码示例或许可以为您提供帮助。



在下文中一共展示了emit函数的20个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于我们的系统推荐出更棒的Python代码示例。

示例1: nextTuple

 def nextTuple(self):
     try:
         input_tuple = None
         try:
             if self.counter >= self.script._num_iterations:
                 return
         except:
             try:
                 input_tuple = self.script._static_input.pop(0)
             except AttributeError:
                 # there is no static input
                 pass
             except IndexError:
                 # static input is empty - no more processing
                 return
         outputs = self.script.process(input_tuple)
         if outputs is None:
             return
         for streamname, output in outputs.iteritems():
             result = output if isinstance(output, list) else [output]
             storm.emit(result, stream=streamname, id=self.counter)
             storm.log("Dispel4Py ------> %s: emitted tuple %s to stream %s" % (self.script.id, result, streamname))
             self.counter += 1
     except:
         # logging the error but it should be passed to client somehow
         storm.log("Dispel4Py ------> %s: %s" % (self.scriptname, traceback.format_exc(), ))
开发者ID:akrause2014,项目名称:dispel4py,代码行数:26,代码来源:source_wrapper.py


示例2: process

 def process(self, tup):
     #TO DO: Add check for empty values
     if tup.values[0]:
         words = tup.values[0].split(" ")
         if words:
             for word in words:
                 storm.emit([word])
开发者ID:EmilyZhou11,项目名称:Real-Time-Analytics-with-Apache-Storm,代码行数:7,代码来源:splitsentence.py


示例3: process

 def process(self,tup):
     if tup.values:
         words = tup.values[0]
         if words:
             storm.emit([words])
     else:
         pass
开发者ID:gggopi,项目名称:Storm-CompanyAbout-Crawler,代码行数:7,代码来源:splitsentences.py


示例4: process

 def process(self, tup):
   text = tup.values[1]
   #language = langid.classify(text)[0]
   #l = LangID()
   #l.train()
   language = self.l.classify(text)
   storm.emit([tup.values[0], language])
开发者ID:gr33ndata,项目名称:storm-starter,代码行数:7,代码来源:getlanguage-dysl.py


示例5: process

    def process(self, tup):
        segments = tup.values[1].rstrip('/').rsplit('/', 1)
        path = segments[0] if self.match(segments[-1]) else '/'.join(segments)

        event = dict(
            timestamp=tup.values[0],
            path=path
            )

        kwargs = dict(
            id=tup.values[2]
            )

        try:
            # TODO: Retrieve users from all indicies.
            events = self.es.get(self.index, kwargs['id'], 'user',
                                 preference='_primary')
            kwargs['version'] = events['_version']
            body = {'events': events['_source']['events'] + [event]}
        except NotFoundError:
            kwargs['op_type'] = 'create'
            body = {'events': [event]}

        try:
            body['rank'] = math.log10(len(body['events'])) / 2
            self.es.index(self.index, 'user', body, **kwargs)
            paths = list(event['path'] for event in body['events'])
            emit([kwargs['id'], paths])
            ack(tup)
        except TransportError:
            fail(tup)
开发者ID:cutoffthetop,项目名称:recommender,代码行数:31,代码来源:user.py


示例6: process

 def process(self, tup):
     url = tup.values[0]
     storm.log("HARing "+url)
     output = self.get_har_with_image(url)
     if output is not "FAIL":
         storm.emit(output, anchors=[tup])
         storm.ack(tup)
开发者ID:ukwa,项目名称:wren,代码行数:7,代码来源:webrenderer.py


示例7: process

    def process(self, tup):
        sentence = tup.values[0]
        sentence = re.sub(r"[,.;!\?]", "", sentence) # get rid of punctuation and num
        words = jieba.cut(sentence, cut_all=True)

        for word in words:
            storm.emit([word])
开发者ID:4everer,项目名称:stormTasks,代码行数:7,代码来源:splitsentence.py


示例8: emit

  def emit(self):

    if not self.entity:
      self.entity = self.getLastEntity()

    # We always emit tuple = (entity, "{json string}")
    storm.emit([self.entity,json.dumps(self.d)])
开发者ID:sinfonier-project,项目名称:sinfonier-backend,代码行数:7,代码来源:basesinfonierbolt.py


示例9: nextTuple

    def nextTuple(self):
        try: 
            html = urllib2.urlopen(self.url_animals_tl).read()
            soup = BeautifulSoup(html)
            scripts = soup.find_all('script')
            code = scripts[len(scripts) - 1]

            content = code.contents[0].strip()
            prefix = 'P.start.start('
            i = content.find(prefix)
            if i != -1:
                json_str = content[i + len(prefix) : len(content) - 2]
                json_obj = json.loads(json_str)
                pins = json_obj['tree']['children'][3]['children'][0]['children'][0]['children']
                
                for pin in pins:
                    pin_id = pin['options']['pin_id']
                    if 'module' in pin['children'][1]['options']:
                        module = pin['children'][1]['options']['module']
                    orig_link = pin['data']['link']
                    orig_host = urlparse(pin['data']['link']).hostname
                    images = pin['data']['images']
                    if 'orig' in images:
                        pass
                    storm.emit([pin_id, orig_link, orig_host, json.dumps(pin, indent=4, sort_keys=True), self.category])
                
            time.sleep(2)
        except StopIteration:
            pass
        except urllib2.HTTPError, err:
            if err.code == 404:
                pass
开发者ID:gaohao,项目名称:pinterest-topology,代码行数:32,代码来源:pintr_spout.py


示例10: process

	def process(self, tup):
		sentence = tup.values[0].lower()
		words = sentence.split(" ")
		bow = numpy.zeros((self._n,))
		for w in words:
			bow[self.map[w]]+=1   	
		storm.emit([bow.tolist()])
开发者ID:srifai,项目名称:storm-starter,代码行数:7,代码来源:tfidf.py


示例11: process

 def process(self, tup):
     word = tup.values[0];
     if (random() < 0.75):
         storm.emit([word + 'lalala'], anchors=[tup])
         storm.ack(tup)
     else:
         storm.log(word + ' randomly skipped!')
开发者ID:ChongerJin,项目名称:storm,代码行数:7,代码来源:tester_bolt.py


示例12: process

 def process(self,tup):
     # storm.log("Dispel4Py ------> %s: Received block." % (self.script.id, ))
     try:
         inputname = self.inputmapping[tup.component][tup.stream]
         storm.log("Dispel4Py ------> %s: Received block at input '%s'" % (self.script.id, inputname, ))
         # inputs = tup.values
         inputs = decode_types(tup.values)
         outputs = self.script.process( { inputname : inputs })
         # storm.log("Dispel4Py ------> %s: Processing complete." % self.scriptname)
         
         if outputs is None:
             return
         for streamname, output in outputs.iteritems():
             result = output if isinstance(output, list) else [output]
             try:
                 storm.emit(result, stream=streamname)
                 storm.log("Dispel4Py ------> %s: Emitted to stream %s: %s" % (self.script.id, streamname, str(result)[:200]))
             except TypeError:
                 # encode manually
                 encoded = encode_types(result)
                 storm.emit(encoded, stream=streamname)
                 storm.log("Dispel4Py ------> %s: Emitted to stream %s" % (self.script.id, streamname))
             # except:
             #     storm.log("%s: %s" % (self.script.id, traceback.format_exc()))
     except:
         storm.log("Dispel4Py ------> %s: %s" % (self.script.id, traceback.format_exc(), ))
开发者ID:akrause2014,项目名称:dispel4py,代码行数:26,代码来源:simple_wrapper.py


示例13: nextTuple

 def nextTuple(self):
     docs = self.get_docs()
     uuid = docs[0]['uuid']
     tup = [docs[0]['href'][18:]]
     self.buffer[uuid] = (tup, 0)
     emit(tup, id=uuid)
     sleep(1.0)
开发者ID:cutoffthetop,项目名称:recommender,代码行数:7,代码来源:zonapi.py


示例14: nextTuple

 def nextTuple(self):
     id = "periodic_%s" % str(uuid4())
     body = json.dumps({'message_id': CHECK_METRIC_ALARM_MSG_ID})
     message = "Periodic monitoring message sent [%s] %s"
     self.log(message % (id, body))
     emit([None, body], id=id)
     time.sleep(60)
开发者ID:eglynn,项目名称:synaps,代码行数:7,代码来源:check_spout.py


示例15: process

    def process(self, tuple):
        '''
        Must fulfil the following contract expressed in the Java wrapper:

        declarer.declare(new Fields(TopologyFields.AUTHOR_SCREEN_NAME, TopologyFields.CREATED_AT,
                TopologyFields.FAV_COUNT, TopologyFields.HASHTAGS_TEXTS, TopologyFields.IN_REPLY_TO_SCREEN_NAME, 
                TopologyFields.LANG, TopologyFields.RETWEET_COUNT, TopologyFields.RETWEETED, 
                TopologyFields.SOURCE, TopologyFields.PLACE, TopologyFields.POSSIBLY_SENSITIVE,
                TopologyFields.TEXT, TopologyFields.TOPIC_NAME));
        '''
        place, topic_name, query = tuple.values
        try: 
            tweets = list(get_tweets.get_tweets_for_trends(self._twitter_api, [{"query" : query}], popular = True, tweet_processor = self._storm_tweet_processor))[0]["tweets"]
        except tweepy.TweepError as te:
            # We have hit the REST API Rate limit for Twitter https://dev.twitter.com/docs/rate-limiting/1.1, no more tweets for some time
            log_tweeter_error(te, sleep_time=self._rate_limit_sleep_time)
            return 

        for pt in tweets:
            # Here we add the trending topic name, and take the place name from those
            # used internally by get_tweets, instead of the from place names returned by twitter
            tup = [pt['author_screen_name'], pt['created_at'], 
                   pt['favorite_count'], pt['hashtags_texts'], pt['in_reply_to_screen_name'],
                   pt['lang'], pt['retweet_count'], pt['retweeted'],
                   pt['source'], place, pt['possibly_sensitive'],
                   pt['text'], topic_name]
            storm.emit(tup)
开发者ID:juanrh,项目名称:data42,代码行数:27,代码来源:twitter_components.py


示例16: process

 def process(self, tup):
     # Split the inbound sentence at spaces
     words = tup.values[0].split(" ")
     # Loop over words and emit
     for word in words:
       storm.logInfo("Emitting %s" % word)
       storm.emit([word])
开发者ID:byron1655,项目名称:hdinsight-python-storm-wordcount,代码行数:7,代码来源:splitbolt.py


示例17: fail

 def fail(self, cnt_id):
     tup, retries = self.buffer[cnt_id]
     if retries >= 5:
         del self.buffer[cnt_id]
         log('[ZonAPISpout] Message %s failed for good.' % cnt_id)
     else:
         self.buffer[cnt_id] = (tup, retries + 1)
         emit(tup, id=cnt_id)
开发者ID:cutoffthetop,项目名称:recommender,代码行数:8,代码来源:zonapi.py


示例18: process

 def process(self, tup):
     print "SplitSentenceBolt.process:tup " + str(tup)
     print "SplitSentenceBolt.process:tup.values[0] " + str(tup.values[0])
     words = tup.values[0].split(" ")
     print "SplitSentenceBolt.process:words " + str(words)
     for word in words:
       storm.emit([word])
       print "SplitSentenceBolt.process:emit " + str(word)
开发者ID:luoyan,项目名称:miniprog,代码行数:8,代码来源:splitsentence.py


示例19: fail

 def fail(self, msg_id):
     tup, retries = self.buffer[msg_id]
     if retries >= 5:
         del self.buffer[msg_id]
         log('[RabbitMQSpout] Message %s failed for good.' % msg_id)
     else:
         self.buffer[msg_id] = (tup, retries + 1)
         emit(tup, id=msg_id)
开发者ID:cutoffthetop,项目名称:recommender,代码行数:8,代码来源:rabbitmq.py


示例20: process

 def process(self, tup):
     '''We serialize the input and output by json for convenience.'''
     try:
         data = array(json.loads(tup.values[1]))
         result = self.model.predict(data)
         storm.emit([tup.values[0], json.dumps(result.tolist())])
     except:
         traceback.print_exc(file=open('/tmp/trace_svm_bolt.txt', 'a'))
开发者ID:jimmylai,项目名称:slideshare,代码行数:8,代码来源:svm_bolt.py



注:本文中的storm.emit函数示例由纯净天空整理自Github/MSDocs等源码及文档管理平台,相关代码片段筛选自各路编程大神贡献的开源项目,源码版权归原作者所有,传播和使用请参考对应项目的License;未经允许,请勿转载。


鲜花

握手

雷人

路过

鸡蛋
该文章已有0人参与评论

请发表评论

全部评论

专题导读
上一篇:
Python storm.log函数代码示例发布时间:2022-05-27
下一篇:
Python common.get_params_from_request函数代码示例发布时间:2022-05-27
热门推荐
阅读排行榜

扫描微信二维码

查看手机版网站

随时了解更新最新资讯

139-2527-9053

在线客服(服务时间 9:00~18:00)

在线QQ客服
地址:深圳市南山区西丽大学城创智工业园
电邮:jeky_zhao#qq.com
移动电话:139-2527-9053

Powered by 互联科技 X3.4© 2001-2213 极客世界.|Sitemap