本文整理汇总了Golang中github.com/PuerkitoBio/goquery.NewDocumentFromResponse函数的典型用法代码示例。如果您正苦于以下问题:Golang NewDocumentFromResponse函数的具体用法?Golang NewDocumentFromResponse怎么用?Golang NewDocumentFromResponse使用的例子?那么恭喜您, 这里精选的函数代码示例或许可以为您提供帮助。
在下文中一共展示了NewDocumentFromResponse函数的20个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于我们的系统推荐出更棒的Golang代码示例。
示例1: TestSubmittingAComment
func TestSubmittingAComment(t *testing.T) {
// Given a server
_, server, _, err := LoginUser(t)
if err != nil {
t.Fatal(err)
}
// When an authenticated client requests /item?id=1
id := 1
res, err := http.Get(fmt.Sprintf(server.URL+"/item?id=%d", id))
if err != nil {
t.Fatal(err)
}
defer res.Body.Close()
// It should respond with status 200
if want, got := 200, res.StatusCode; want != got {
t.Fatalf("status : want '%v' got '%v' ", want, got)
}
doc, err := goquery.NewDocumentFromResponse(res)
csrf, ok := doc.Find("input[name='comment_csrf']").First().Attr("value")
if !ok {
t.Fatalf("csrf value not found for comment form")
}
initialCommentNumber := doc.Find(".comment").Length()
formValues := url.Values{
"comment_content": {"this is a new comment"},
"comment_csrf": {csrf},
"comment_submit": {"submit"},
"comment_parent_id": {"0"},
"comment_goto": {fmt.Sprintf("/item?id=%d", id)},
"comment_thread_id": {fmt.Sprintf("%d", id)},
}
// when an authenicated client submits a valid comment
res, err = http.Post(server.URL+gonews.Route{}.Reply(), "application/x-www-form-urlencoded", strings.NewReader(formValues.Encode()))
if err != nil {
t.Fatal(err)
}
defer res.Body.Close()
// It should respond with status 200
if want, got := 200, res.StatusCode; want != got {
t.Fatalf("Status : want '%v' got '%v' ", want, got)
}
// The number of comments on the story page should have increased by one
doc, err = goquery.NewDocumentFromResponse(res)
if err != nil {
t.Fatal(err)
}
if want, got := initialCommentNumber+1, doc.Find(".comment").Length(); want != got {
t.Fatalf(".comment length : want '%v' got '%v' ", want, got)
}
}
开发者ID:Mparaiso,项目名称:gonews,代码行数:52,代码来源:app_test.go
示例2: main
func main() {
login()
for _, v := range cookies {
fmt.Println(v)
}
var bash_str string = "#!/bin/bash \n"
baseMap := map[string]string{}
doc, _ := goquery.NewDocumentFromResponse(getResultHtml(spider_base_url))
doc.Find(".lesson-info-h2").Each(func(i int, contentSelection *goquery.Selection) {
selection := contentSelection.Find("a")
base_href, _ := selection.Attr("href")
dir_name := selection.Text()
bash_str += "mkdir \"" + dir_name + "\"\n"
baseMap[dir_name] = base_href
fmt.Println(dir_name, "-->", base_href)
})
downloadList := []DownloadBean{}
for k, v := range baseMap {
doc, _ := goquery.NewDocumentFromResponse(getResultHtml(v))
doc.Find(".lessonvideo-list dd h2").Each(func(i int, contentSelection *goquery.Selection) {
selection := contentSelection.Find("a")
href, _ := selection.Attr("href")
filename := selection.Text()
fmt.Println(k, "-->", filename, "-->", href)
bean := DownloadBean{dirname: k, href: href, filename: filename}
downloadList = append(downloadList, bean)
})
}
for _, v := range downloadList {
doc, _ := goquery.NewDocumentFromResponse(getResultHtml(v.href))
doc.Find("source").Each(func(i int, contentSelection *goquery.Selection) {
download_url, _ := contentSelection.Attr("src")
one_file := "wget " + download_url + " -O \"./" + v.dirname + "/" + v.filename + ".mp4\"\n"
bash_str += one_file
fmt.Println(one_file)
})
}
err := ioutil.WriteFile("./download.sh", []byte(bash_str), 0x777)
if err != nil {
panic(err.Error())
}
fmt.Println("写入脚本完成")
}
开发者ID:cwen-coder,项目名称:crawler,代码行数:50,代码来源:login5.go
示例3: GetResources
// GetResources retrieves resource info
func GetResources(resp *http.Response) Resources {
doc, _ := goquery.NewDocumentFromResponse(resp)
resWrapperSelector := doc.Find("div#resWrap")
lumber := resWrapperSelector.Find("td#l4")
clay := resWrapperSelector.Find("td#l3")
iron := resWrapperSelector.Find("td#l2")
crop := resWrapperSelector.Find("td#l1")
lumberProd, lumberStored, lumberCapa := parseResource(lumber)
clayProd, clayStored, _ := parseResource(clay)
ironProd, ironStored, _ := parseResource(iron)
cropProd, cropStored, cropCapa := parseResource(crop)
return Resources{
Granary{
Capacity: cropCapa,
Crop: cropStored,
},
Warehouse{
Capacity: lumberCapa,
Lumber: lumberStored,
Clay: clayStored,
Iron: ironStored,
},
Production{
lumberProd,
clayProd,
ironProd,
cropProd,
},
}
}
开发者ID:hojgr,项目名称:travian,代码行数:35,代码来源:resources.go
示例4: Parser
//解析函数定义,声明必须相同,下同
func Parser(httpRes *http.Response) ([]string, []basic.Item) {
//两个需要返回的列表
linklist := make([]string, 0) //下一步需要请求的链接
itemlist := make([]basic.Item, 0) //保存的数据类型为 map[string]interface{}
//自定义部分
//抓取所有链接
doc, _ := goquery.NewDocumentFromResponse(httpRes)
doc.Find("a").Each(func(i int, s *goquery.Selection) {
link, exits := s.Attr("href")
if exits {
link = basic.CheckLink(link)
if link != "" {
linklist = append(linklist, link)
}
}
})
//保存每个页面的标题
title := strings.TrimSpace(doc.Find("head title").Text())
if title != "" {
item := make(map[string]interface{})
item["标题"] = title
itemlist = append(itemlist, item)
}
return linklist, itemlist
}
开发者ID:gooo000,项目名称:scrago,代码行数:28,代码来源:main.go
示例5: DoRequest
func (api *ExternalAPIEngine) DoRequest(
arg APIArg, req *http.Request, restype int) (
ar *ExternalAPIRes, hr *ExternalHTMLRes, tr *ExternalTextRes, err error) {
var resp *http.Response
var jw *jsonw.Wrapper
resp, jw, err = doRequestShared(api, arg, req, (restype == XAPIResJSON))
if err != nil {
return
}
switch restype {
case XAPIResJSON:
ar = &ExternalAPIRes{resp.StatusCode, jw}
case XAPIResHTML:
var goq *goquery.Document
goq, err = goquery.NewDocumentFromResponse(resp)
if err == nil {
hr = &ExternalHTMLRes{resp.StatusCode, goq}
}
case XAPIResText:
var buf bytes.Buffer
_, err := buf.ReadFrom(resp.Body)
defer resp.Body.Close()
if err == nil {
tr = &ExternalTextRes{resp.StatusCode, string(buf.Bytes())}
}
default:
err = fmt.Errorf("unknown restype to DoRequest")
}
return
}
开发者ID:polluks,项目名称:client,代码行数:33,代码来源:api.go
示例6: harvest_board_indices
func harvest_board_indices(board_url string, board_name string) []BoardIndexPage {
var ret []BoardIndexPage
doc, err := goquery.NewDocumentFromResponse(ptt_get(board_url))
if err != nil {
log.Fatal(err)
}
re := regexp.MustCompile("/bbs/" + board_name + "/index([0-9]+)\\.html")
doc.Find("a[href^='/bbs/" + board_name + "/index']").Each(func(_ int, s *goquery.Selection) {
href, exists := s.Attr("href")
if !exists {
return
}
matched := re.FindStringSubmatch(href)
if len(matched) == 0 {
return
}
pn, err := strconv.Atoi(matched[1])
if err != nil {
log.Fatal(err)
}
ret = append(ret, BoardIndexPage{pn, href})
})
if ret[0].page_number > ret[1].page_number {
ret[0], ret[1] = ret[1], ret[0]
}
for i := ret[0].page_number + 1; i < ret[1].page_number; i++ {
ret = append(ret, BoardIndexPage{i, "/bbs/" + board_name + "/index" + strconv.Itoa(i) + ".html"})
}
return ret
}
开发者ID:gugod,项目名称:go-ptt-crawler,代码行数:35,代码来源:crawl-board.go
示例7: main
func main() {
flag.Parse()
if flag.NArg() != 1 {
fmt.Fprintln(os.Stderr, "yomikata [word]")
os.Exit(1)
}
word := flag.Arg(0)
resp, err := http.Get("http://yomikata.org/word/" + word)
if err != nil {
fmt.Fprintln(os.Stderr, os.Args[0]+":", err)
os.Exit(1)
}
doc, err := goquery.NewDocumentFromResponse(resp)
if err != nil {
fmt.Fprintln(os.Stderr, os.Args[0]+":", err)
os.Exit(1)
}
if doc.Find("#word").Text() == "" {
fmt.Fprintln(os.Stderr, "わかりません")
os.Exit(1)
}
if *exact {
fmt.Println(doc.Find(".spAns .psAns").First().Text())
} else {
doc.Find(".spAns").Each(func(_ int, s *goquery.Selection) {
fmt.Printf("%s (%s)\n", s.Find(".psAns").Text(), s.Find(".psPt").Text())
})
}
}
开发者ID:mattn,项目名称:yomikata,代码行数:30,代码来源:main.go
示例8: GetStatistics
// GetStatistics retrieves statistics
func GetStatistics(resp *http.Response) Statistics {
doc, _ := goquery.NewDocumentFromResponse(resp)
s := doc.Find("tr.hl").First()
positionStr := strings.TrimRight(s.Find(".ra").Text(), ".")
position, _ := strconv.Atoi(positionStr)
username := strings.Trim(s.Find(".pla").Find("a").Text(), " ")
alliance := s.Find("al").Text()
popStr := s.Find(".pop").Text()
pop, _ := strconv.Atoi(popStr)
villageCountStr := s.Find(".vil").Text()
villageCount, _ := strconv.Atoi(villageCountStr)
stats := Statistics{
position,
username,
alliance,
pop,
villageCount,
}
return stats
}
开发者ID:hojgr,项目名称:travian,代码行数:28,代码来源:statistics.go
示例9: FetchCwzb
// FetchRzrqSumData 抓取数据
func FetchCwzb(code, ctype string) ([]*Cwzb, error) {
ctypeStr := "K"
if ctype == "sz" {
ctypeStr = "J"
}
formt := "http://quotes.cnfol.com/new/f10/cwzb/%s%s.html"
resp, err := wget.Get(fmt.Sprintf(formt, code, ctypeStr))
if err != nil {
return nil, err
}
doc, err := goquery.NewDocumentFromResponse(resp)
if err != nil {
return nil, err
}
doc.Find("table tr").Each(func(i int, s *goquery.Selection) {
td := s.Find("td")
if td.Length() < 5 {
return
}
td.Each(func(i int, s *goquery.Selection) {
if i == 0 {
return
}
fmt.Println(s.Text())
})
})
return nil, nil
}
开发者ID:jiorry,项目名称:gotock,代码行数:34,代码来源:cwzb.go
示例10: mustParseResponse
func mustParseResponse(resp *http.Response) *goquery.Document {
gq, err := goquery.NewDocumentFromResponse(resp)
if err != nil {
log.Fatal(err)
}
return gq
}
开发者ID:freman,项目名称:hipclean,代码行数:7,代码来源:util.go
示例11: Search
func (a PlayStore) Search(sr SearchRequest) ([]App, error) {
playurl, post := a.searchUrl(sr)
body := strings.NewReader(post)
req, err := http.NewRequest("POST", playurl, body)
if err != nil {
return nil, err
}
req.Header.Set("Content-Type", "application/x-www-form-urlencoded; param=value")
req.Header.Add("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.57 Safari/537.36")
client := sr.Client
resp, err := client.Do(req)
if err != nil {
return nil, err
}
doc, err := goquery.NewDocumentFromResponse(resp)
if err != nil {
return nil, err
}
cards := doc.Find(".card")
apps := make([]App, cards.Length())
cards.Each(func(i int, s *goquery.Selection) {
if val, ok := s.Attr("data-docid"); ok {
apps[i].Package = val
}
if val, ok := s.Find(".title").Attr("title"); ok {
apps[i].Name = val
}
if val, ok := s.Find(".subtitle").Attr("title"); ok {
apps[i].PublisherName = val
}
if val, ok := s.Find(".card-click-target").Attr("href"); ok {
apps[i].StoreUrl = "https://play.google.com" + val
}
if val, ok := s.Find(".cover-image").Attr("src"); ok {
apps[i].IconUrl = val
}
if val, ok := s.Find(".current-rating").Attr("style"); ok {
rating := strings.Split(val, " ")
if len(rating) >= 2 {
if rating_perc, err := strconv.ParseFloat(strings.TrimRight(rating[1], "%;"), 32); err == nil {
apps[i].Rating = float32(rating_perc * 5.0 / 100.0)
}
}
}
})
return apps, nil
}
开发者ID:hobbs,项目名称:appcrawl,代码行数:60,代码来源:playstore.go
示例12: Result
// Get the collection of the ProxyList by parse the http response
func (this *XiCiDaiLi) Result(response *http.Response) ([]contract.ProxyList, error) {
doc, err := goquery.NewDocumentFromResponse(response)
if err != nil {
return nil, err
}
var proxyList []contract.ProxyList
proxyTable := doc.Find("table#ip_list")
proxyRows := proxyTable.Find("tr:nth-child(n+2)")
proxyRows.Each(func(i int, s *goquery.Selection) {
country := s.Find("td:nth-child(2) img").AttrOr("alt", "n/a")
ip := s.Find("td:nth-child(3)").Text()
port := s.Find("td:nth-child(4)").Text()
protocol := s.Find("td:nth-child(7)").Text()
if ip != "" && port != "" {
proxyList = append(proxyList, contract.ProxyList{
Ip: ip,
Port: port,
Protocol: protocol,
Country: strings.ToLower(country),
})
}
})
return proxyList, nil
}
开发者ID:zeuxisoo,项目名称:go-delega,代码行数:31,代码来源:xi_ci_dai_li.go
示例13: GetMajorList
func GetMajorList(client *http.Client) []string {
banlist := map[string]string{"110431": "", "110432": "", "110511": "", "110521": "", "113110": "", "113111": ""}
getmajorlist := []string{}
res, err := client.Get(DHUHostUrl + DHUGetMajorURL)
if err != nil {
fmt.Println(err)
fmt.Println("Something Wrong In GetMajorList")
} else {
doc, _ := goquery.NewDocumentFromResponse(res)
doc.Find("select").Eq(1).Find("option").Each(func(i int, s *goquery.Selection) {
sss, ok := s.Attr("value")
if ok {
_, testmap := banlist[sss]
if testmap {
return
} else {
getmajorlist = append(getmajorlist, sss)
}
} else {
fmt.Println("Something Wrong")
}
})
return getmajorlist
}
return nil
}
开发者ID:XingLong9630,项目名称:DHUCourseSelection,代码行数:26,代码来源:CrawlerForTeachSchema.go
示例14: GetAllLessons
func GetAllLessons(res *http.Response) []CourseList {
doc, err := goquery.NewDocumentFromResponse(res)
if err != nil {
fmt.Println(err)
return nil
} else {
newlist := []CourseList{}
dec := mahonia.NewDecoder("GB18030")
doc.Find("tr").Each(func(i int, s *goquery.Selection) {
selectid := s.Find("td").Eq(0).Text()
_, err := strconv.Atoi(selectid)
if err == nil {
_, teachername, _ := dec.Translate([]byte(s.Find("td").Eq(6).Text()), true)
newinfo := []CourseInfo{}
s.Find("td").Eq(7).Find("tr").Each(func(i int, s *goquery.Selection) {
_, weektime, _ := dec.Translate([]byte(s.Find("td").Eq(0).Text()), true)
_, daytime, _ := dec.Translate([]byte(s.Find("td").Eq(1).Text()), true)
info := CourseInfo{string(weektime), string(daytime)}
newinfo = append(newinfo, info)
})
courselist := CourseList{selectid, string(teachername), newinfo}
newlist = append(newlist, courselist)
} else {
return
}
})
return newlist
}
}
开发者ID:XingLong9630,项目名称:DHUCourseSelection,代码行数:29,代码来源:CrawlerForCourseTable.go
示例15: getHtml
func (this *Object) getHtml() (string, error) {
res, err := request("GET", this.TargetUrl, "")
if err != nil {
return "", err
}
if res.StatusCode != 200 {
return "", errors.New(fmt.Sprintf("http status not 200, it is %d", res.StatusCode))
}
doc, err := goquery.NewDocumentFromResponse(res)
if err != nil {
return "", err
}
if err != nil {
return "", err
}
html, err := this.parse(doc)
if err != nil {
return "", err
}
this.setArgs(doc)
if err := this.setPageLimit(doc); err != nil {
return "", err
}
return html, nil
}
开发者ID:coseyo,项目名称:getsale,代码行数:30,代码来源:object.go
示例16: ParsePage
func (c Boot24Crawler) ParsePage(res *http.Response) *goquery.Document {
doc, err := goquery.NewDocumentFromResponse(res)
if err != nil {
log.Fatal(err)
}
return doc
}
开发者ID:cod3hulk,项目名称:go-boat-data-crawler,代码行数:7,代码来源:main.go
示例17: FetchStockData
// FetchStockData 抓取数据
func FetchStockData(code string) (*StockIndustry, *StockICB, error) {
// http://stockData.stock.hexun.com/600028.shtml
formt := "http://stockData.stock.hexun.com/%s.shtml"
resp, err := wget.Get(fmt.Sprintf(formt, code))
if err != nil {
return nil, nil, gos.DoError(err)
}
doc, err := goquery.NewDocumentFromResponse(resp)
if err != nil {
return nil, nil, gos.DoError(err)
}
tr := doc.Find("#list3 table.box6 tr")
stockIndustry := &StockIndustry{}
stockICB := &StockICB{}
stockIndustry.Name, err = iconv.ConvertString(tr.Eq(7).Find("td").Eq(1).Text(), "gb2312", "utf-8")
if err != nil {
return nil, nil, gos.DoError(err)
}
stockICB.Name, err = iconv.ConvertString(tr.Eq(8).Find("td").Eq(1).Text(), "gb2312", "utf-8")
if err != nil {
return nil, nil, gos.DoError(err)
}
return stockIndustry, stockICB, nil
}
开发者ID:jiorry,项目名称:gotock,代码行数:31,代码来源:stockdata.go
示例18: get_site
func get_site(uri string, r *http.Request) ([]temp_item, error) {
c := appengine.NewContext(r)
client := urlfetch.Client(c)
resp, err := client.Get(uri)
if err != nil {
return nil, err
}
doc, err := goquery.NewDocumentFromResponse(resp)
if err != nil {
return nil, err
}
var list_items []temp_item
doc.Find("#bodyContent li").Not("#bodyContent #toc li").Each(func(i int, s *goquery.Selection) {
listitem := s.Text()
c.Infof("ITEM:\n" + listitem + "\n\n")
q, err := filter_question(listitem)
c.Infof("%v", err)
if err == nil {
list_items = append(list_items, q)
}
})
//c.Infof("%+v\n", list_items)
return list_items, nil
}
开发者ID:f4rez,项目名称:Cronos,代码行数:32,代码来源:question_crawler.go
示例19: GetRecipeIds
func GetRecipeIds(categoryId int, page int) ([]int, error) {
url := GetCategoryPageURL(categoryId, page)
resp, err := http.Get(url)
if err != nil {
return nil, fmt.Errorf("failed to Get %s: %v", url, err)
}
defer resp.Body.Close()
doc, err := goquery.NewDocumentFromResponse(resp)
if err != nil {
return nil, fmt.Errorf("failed to parse response: %v", err)
}
recipeIds := []int{}
doc.Find(".info").Each(func(i int, s *goquery.Selection) {
s.Find(".name").Each(func(i int, s *goquery.Selection) {
if href, ok := s.Find("a").Attr("href"); ok {
id, err := strconv.Atoi(strings.Split(href, "/")[2])
if err != nil {
glog.Error("Failed to extract recipe id: %v", err)
} else {
recipeIds = append(recipeIds, id)
}
}
})
})
return recipeIds, nil
}
开发者ID:uronce-cc,项目名称:weat,代码行数:28,代码来源:recipe.go
示例20: BingDomain
// BingDomain uses bing's 'domain:' search operator and scrapes the HTML to find ips and hostnames for a domain.
func BingDomain(domain, server string) (string, Results, error) {
task := "bing"
results := Results{}
resp, err := http.Get("http://www.bing.com/search?q=domain:" + domain)
if err != nil {
return task, results, err
}
doc, err := goquery.NewDocumentFromResponse(resp)
if err != nil {
return task, results, err
}
doc.Selection.Find("cite").Each(func(_ int, s *goquery.Selection) {
u, err := url.Parse(s.Text())
if err != nil || u.Host == "" {
return
}
ip, err := LookupName(u.Host, server)
if err != nil || ip == "" {
cfqdn, err := LookupCname(u.Host, server)
if err != nil || cfqdn == "" {
return
}
ip, err = LookupName(cfqdn, server)
if err != nil || ip == "" {
return
}
}
results = append(results, Result{Source: task, IP: ip, Hostname: u.Host})
})
return task, results, err
}
开发者ID:intfrr,项目名称:blacksheepwall,代码行数:33,代码来源:bing.go
注:本文中的github.com/PuerkitoBio/goquery.NewDocumentFromResponse函数示例由纯净天空整理自Github/MSDocs等源码及文档管理平台,相关代码片段筛选自各路编程大神贡献的开源项目,源码版权归原作者所有,传播和使用请参考对应项目的License;未经允许,请勿转载。 |
请发表评论