本文整理汇总了Golang中github.com/henrylee2cn/pholcus/crawl/downloader/context.Request类的典型用法代码示例。如果您正苦于以下问题:Golang Request类的具体用法?Golang Request怎么用?Golang Request使用的例子?那么恭喜您, 这里精选的类代码示例或许可以为您提供帮助。
在下文中一共展示了Request类的5个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于我们的系统推荐出更棒的Golang代码示例。
示例1: Push
func (self *SrcManage) Push(req *context.Request) {
if spiderId, ok := req.GetSpiderId(); ok {
priority := int(req.GetPriority())
if priority > MAX_PRIORITY {
priority = MAX_PRIORITY
}
for i, x := 0, priority+1-len(self.queue[spiderId]); i < x; i++ {
self.queue[spiderId] = append(self.queue[spiderId], []*context.Request{})
}
self.queue[spiderId][priority] = append(self.queue[spiderId][priority], req)
}
}
开发者ID:Huangyan9188,项目名称:pholcus,代码行数:14,代码来源:src_manage.go
示例2: Push
func (self *SrcManage) Push(req *context.Request) {
spiderId, ok := req.GetSpiderId()
if !ok {
return
}
// 初始化该蜘蛛的队列
if _, ok := self.queue[spiderId]; !ok {
self.mutex[spiderId] = new(sync.Mutex)
self.queue[spiderId] = make(map[int][]*context.Request)
}
priority := req.GetPriority()
// 登记该蜘蛛下该优先级队列
if _, ok := self.queue[spiderId][priority]; !ok {
self.uIndex(spiderId, priority)
}
// 添加请求到队列
self.queue[spiderId][priority] = append(self.queue[spiderId][priority], req)
}
开发者ID:npk,项目名称:pholcus-1,代码行数:22,代码来源:src_manage.go
示例3: Push
// 添加请求到队列
func (self *scheduler) Push(req *context.Request) {
pushMutex.Lock()
defer func() {
pushMutex.Unlock()
}()
if self.status == status.STOP {
return
}
is := self.Compare(req.GetUrl() + req.GetMethod())
// 有重复则返回
if is {
return
}
// 留作未来分发请求用
// if pholcus.Self.GetRunMode() == config.SERVER || req.CanOutsource() {
// return
// }
self.SrcManage.Push(req)
}
开发者ID:Huangyan9188,项目名称:pholcus,代码行数:23,代码来源:scheduler.go
示例4: Download
func (self *Surfer) Download(cReq *context.Request) *context.Response {
cResp := context.NewResponse(nil)
resp, err := self.download.Download(cReq.GetMethod(), cReq.GetUrl(), cReq.GetReferer(), cReq.GetPostData(), cReq.GetHeader(), cReq.GetCookies())
cResp.SetRequest(cReq)
cResp.SetResponse(resp)
if err != nil {
cResp.SetStatus(true, err.Error())
return cResp
}
cResp.SetStatus(false, "")
return cResp
}
开发者ID:jumping,项目名称:pholcus,代码行数:17,代码来源:downloader_surfer.go
示例5: Download
func (self *Surfer) Download(cReq *context.Request) *context.Response {
cResp := context.NewResponse(nil)
resp, err := self.download.Download(cReq.GetMethod(), cReq.GetUrl(), cReq.GetReferer(), cReq.GetPostData(), cReq.GetHeader(), cReq.GetCookies())
cResp.SetRequest(cReq)
if err != nil {
cResp.SetStatus(true, err.Error())
return cResp
}
// get converter to utf-8
body := self.changeCharsetEncodingAuto(resp.Body, resp.Header.Get("Content-Type"))
//fmt.Printf("utf-8 body %v \r\n", bodyStr)
defer resp.Body.Close()
cResp.SetText(body)
cResp.SetStatus(false, "")
return cResp
}
开发者ID:Huangyan9188,项目名称:pholcus,代码行数:20,代码来源:downloader_surfer.go
注:本文中的github.com/henrylee2cn/pholcus/crawl/downloader/context.Request类示例整理自Github/MSDocs等源码及文档管理平台,相关代码片段筛选自各路编程大神贡献的开源项目,源码版权归原作者所有,传播和使用请参考对应项目的License;未经允许,请勿转载。 |
请发表评论