func TestSubmittingAComment(t *testing.T) {

	// Given a server
	_, server, _, err := LoginUser(t)
	if err != nil {
		t.Fatal(err)
	}
	// When an authenticated client requests /item?id=1
	id := 1
	res, err := http.Get(fmt.Sprintf(server.URL+"/item?id=%d", id))
	if err != nil {
		t.Fatal(err)
	}
	defer res.Body.Close()
	// It should respond with status 200

	if want, got := 200, res.StatusCode; want != got {
		t.Fatalf("status : want '%v' got '%v' ", want, got)
	}
	doc, err := goquery.NewDocumentFromResponse(res)
	csrf, ok := doc.Find("input[name='comment_csrf']").First().Attr("value")
	if !ok {
		t.Fatalf("csrf value not found for comment form")
	}
	initialCommentNumber := doc.Find(".comment").Length()
	formValues := url.Values{
		"comment_content":   {"this is a new comment"},
		"comment_csrf":      {csrf},
		"comment_submit":    {"submit"},
		"comment_parent_id": {"0"},
		"comment_goto":      {fmt.Sprintf("/item?id=%d", id)},
		"comment_thread_id": {fmt.Sprintf("%d", id)},
	}
	// when an authenicated client submits a valid comment
	res, err = http.Post(server.URL+gonews.Route{}.Reply(), "application/x-www-form-urlencoded", strings.NewReader(formValues.Encode()))
	if err != nil {
		t.Fatal(err)
	}
	defer res.Body.Close()
	// It should respond with status 200
	if want, got := 200, res.StatusCode; want != got {
		t.Fatalf("Status : want '%v' got '%v' ", want, got)
	}
	// The number of comments on the story page should have increased by one
	doc, err = goquery.NewDocumentFromResponse(res)
	if err != nil {
		t.Fatal(err)
	}
	if want, got := initialCommentNumber+1, doc.Find(".comment").Length(); want != got {
		t.Fatalf(".comment length : want '%v' got '%v' ", want, got)
	}
}

func main() {

	login()
	for _, v := range cookies {
		fmt.Println(v)
	}
	var bash_str string = "#!/bin/bash \n"
	baseMap := map[string]string{}
	doc, _ := goquery.NewDocumentFromResponse(getResultHtml(spider_base_url))
	doc.Find(".lesson-info-h2").Each(func(i int, contentSelection *goquery.Selection) {
		selection := contentSelection.Find("a")
		base_href, _ := selection.Attr("href")
		dir_name := selection.Text()
		bash_str += "mkdir \"" + dir_name + "\"\n"
		baseMap[dir_name] = base_href
		fmt.Println(dir_name, "-->", base_href)

	})
	downloadList := []DownloadBean{}

	for k, v := range baseMap {
		doc, _ := goquery.NewDocumentFromResponse(getResultHtml(v))
		doc.Find(".lessonvideo-list dd h2").Each(func(i int, contentSelection *goquery.Selection) {
			selection := contentSelection.Find("a")
			href, _ := selection.Attr("href")
			filename := selection.Text()
			fmt.Println(k, "-->", filename, "-->", href)
			bean := DownloadBean{dirname: k, href: href, filename: filename}
			downloadList = append(downloadList, bean)
		})
	}

	for _, v := range downloadList {
		doc, _ := goquery.NewDocumentFromResponse(getResultHtml(v.href))
		doc.Find("source").Each(func(i int, contentSelection *goquery.Selection) {
			download_url, _ := contentSelection.Attr("src")
			one_file := "wget " + download_url + "  -O \"./" + v.dirname + "/" + v.filename + ".mp4\"\n"
			bash_str += one_file
			fmt.Println(one_file)
		})
	}

	err := ioutil.WriteFile("./download.sh", []byte(bash_str), 0x777)
	if err != nil {
		panic(err.Error())
	}

	fmt.Println("写入脚本完成")

}

// GetResources retrieves resource info
func GetResources(resp *http.Response) Resources {
	doc, _ := goquery.NewDocumentFromResponse(resp)

	resWrapperSelector := doc.Find("div#resWrap")

	lumber := resWrapperSelector.Find("td#l4")
	clay := resWrapperSelector.Find("td#l3")
	iron := resWrapperSelector.Find("td#l2")
	crop := resWrapperSelector.Find("td#l1")

	lumberProd, lumberStored, lumberCapa := parseResource(lumber)
	clayProd, clayStored, _ := parseResource(clay)
	ironProd, ironStored, _ := parseResource(iron)
	cropProd, cropStored, cropCapa := parseResource(crop)

	return Resources{
		Granary{
			Capacity: cropCapa,
			Crop:     cropStored,
		},
		Warehouse{
			Capacity: lumberCapa,
			Lumber:   lumberStored,
			Clay:     clayStored,
			Iron:     ironStored,
		},
		Production{
			lumberProd,
			clayProd,
			ironProd,
			cropProd,
		},
	}
}

//解析函数定义，声明必须相同，下同
func Parser(httpRes *http.Response) ([]string, []basic.Item) {
	//两个需要返回的列表
	linklist := make([]string, 0)     //下一步需要请求的链接
	itemlist := make([]basic.Item, 0) //保存的数据类型为 map[string]interface{}

	//自定义部分
	//抓取所有链接
	doc, _ := goquery.NewDocumentFromResponse(httpRes)
	doc.Find("a").Each(func(i int, s *goquery.Selection) {
		link, exits := s.Attr("href")
		if exits {
			link = basic.CheckLink(link)
			if link != "" {
				linklist = append(linklist, link)
			}
		}
	})
	//保存每个页面的标题
	title := strings.TrimSpace(doc.Find("head title").Text())
	if title != "" {
		item := make(map[string]interface{})
		item["标题"] = title
		itemlist = append(itemlist, item)
	}

	return linklist, itemlist
}

func (api *ExternalAPIEngine) DoRequest(
	arg APIArg, req *http.Request, restype int) (
	ar *ExternalAPIRes, hr *ExternalHTMLRes, tr *ExternalTextRes, err error) {

	var resp *http.Response
	var jw *jsonw.Wrapper

	resp, jw, err = doRequestShared(api, arg, req, (restype == XAPIResJSON))
	if err != nil {
		return
	}

	switch restype {
	case XAPIResJSON:
		ar = &ExternalAPIRes{resp.StatusCode, jw}
	case XAPIResHTML:
		var goq *goquery.Document
		goq, err = goquery.NewDocumentFromResponse(resp)
		if err == nil {
			hr = &ExternalHTMLRes{resp.StatusCode, goq}
		}
	case XAPIResText:
		var buf bytes.Buffer
		_, err := buf.ReadFrom(resp.Body)
		defer resp.Body.Close()
		if err == nil {
			tr = &ExternalTextRes{resp.StatusCode, string(buf.Bytes())}
		}
	default:
		err = fmt.Errorf("unknown restype to DoRequest")
	}
	return
}

func harvest_board_indices(board_url string, board_name string) []BoardIndexPage {
	var ret []BoardIndexPage

	doc, err := goquery.NewDocumentFromResponse(ptt_get(board_url))
	if err != nil {
		log.Fatal(err)
	}

	re := regexp.MustCompile("/bbs/" + board_name + "/index([0-9]+)\\.html")
	doc.Find("a[href^='/bbs/" + board_name + "/index']").Each(func(_ int, s *goquery.Selection) {
		href, exists := s.Attr("href")
		if !exists {
			return
		}
		matched := re.FindStringSubmatch(href)
		if len(matched) == 0 {
			return
		}
		pn, err := strconv.Atoi(matched[1])
		if err != nil {
			log.Fatal(err)
		}
		ret = append(ret, BoardIndexPage{pn, href})
	})

	if ret[0].page_number > ret[1].page_number {
		ret[0], ret[1] = ret[1], ret[0]
	}

	for i := ret[0].page_number + 1; i < ret[1].page_number; i++ {
		ret = append(ret, BoardIndexPage{i, "/bbs/" + board_name + "/index" + strconv.Itoa(i) + ".html"})
	}

	return ret
}

func main() {
	flag.Parse()

	if flag.NArg() != 1 {
		fmt.Fprintln(os.Stderr, "yomikata [word]")
		os.Exit(1)
	}
	word := flag.Arg(0)
	resp, err := http.Get("http://yomikata.org/word/" + word)
	if err != nil {
		fmt.Fprintln(os.Stderr, os.Args[0]+":", err)
		os.Exit(1)
	}
	doc, err := goquery.NewDocumentFromResponse(resp)
	if err != nil {
		fmt.Fprintln(os.Stderr, os.Args[0]+":", err)
		os.Exit(1)
	}
	if doc.Find("#word").Text() == "" {
		fmt.Fprintln(os.Stderr, "わかりません")
		os.Exit(1)
	}
	if *exact {
		fmt.Println(doc.Find(".spAns .psAns").First().Text())
	} else {
		doc.Find(".spAns").Each(func(_ int, s *goquery.Selection) {
			fmt.Printf("%s (%s)\n", s.Find(".psAns").Text(), s.Find(".psPt").Text())
		})
	}
}

// GetStatistics retrieves statistics
func GetStatistics(resp *http.Response) Statistics {
	doc, _ := goquery.NewDocumentFromResponse(resp)

	s := doc.Find("tr.hl").First()

	positionStr := strings.TrimRight(s.Find(".ra").Text(), ".")
	position, _ := strconv.Atoi(positionStr)

	username := strings.Trim(s.Find(".pla").Find("a").Text(), " ")
	alliance := s.Find("al").Text()

	popStr := s.Find(".pop").Text()
	pop, _ := strconv.Atoi(popStr)

	villageCountStr := s.Find(".vil").Text()
	villageCount, _ := strconv.Atoi(villageCountStr)

	stats := Statistics{
		position,
		username,
		alliance,
		pop,
		villageCount,
	}

	return stats
}

// FetchRzrqSumData 抓取数据
func FetchCwzb(code, ctype string) ([]*Cwzb, error) {
	ctypeStr := "K"
	if ctype == "sz" {
		ctypeStr = "J"
	}
	formt := "http://quotes.cnfol.com/new/f10/cwzb/%s%s.html"

	resp, err := wget.Get(fmt.Sprintf(formt, code, ctypeStr))
	if err != nil {
		return nil, err
	}

	doc, err := goquery.NewDocumentFromResponse(resp)
	if err != nil {
		return nil, err
	}

	doc.Find("table tr").Each(func(i int, s *goquery.Selection) {
		td := s.Find("td")
		if td.Length() < 5 {
			return
		}

		td.Each(func(i int, s *goquery.Selection) {
			if i == 0 {
				return
			}
			fmt.Println(s.Text())
		})
	})

	return nil, nil
}

func mustParseResponse(resp *http.Response) *goquery.Document {
	gq, err := goquery.NewDocumentFromResponse(resp)
	if err != nil {
		log.Fatal(err)
	}
	return gq
}

func (a PlayStore) Search(sr SearchRequest) ([]App, error) {

	playurl, post := a.searchUrl(sr)

	body := strings.NewReader(post)
	req, err := http.NewRequest("POST", playurl, body)
	if err != nil {
		return nil, err
	}
	req.Header.Set("Content-Type", "application/x-www-form-urlencoded; param=value")
	req.Header.Add("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.57 Safari/537.36")
	client := sr.Client
	resp, err := client.Do(req)

	if err != nil {
		return nil, err
	}

	doc, err := goquery.NewDocumentFromResponse(resp)

	if err != nil {
		return nil, err
	}
	cards := doc.Find(".card")
	apps := make([]App, cards.Length())
	cards.Each(func(i int, s *goquery.Selection) {
		if val, ok := s.Attr("data-docid"); ok {
			apps[i].Package = val
		}
		if val, ok := s.Find(".title").Attr("title"); ok {
			apps[i].Name = val
		}

		if val, ok := s.Find(".subtitle").Attr("title"); ok {
			apps[i].PublisherName = val
		}

		if val, ok := s.Find(".card-click-target").Attr("href"); ok {
			apps[i].StoreUrl = "https://play.google.com" + val
		}

		if val, ok := s.Find(".cover-image").Attr("src"); ok {
			apps[i].IconUrl = val
		}

		if val, ok := s.Find(".current-rating").Attr("style"); ok {
			rating := strings.Split(val, " ")
			if len(rating) >= 2 {
				if rating_perc, err := strconv.ParseFloat(strings.TrimRight(rating[1], "%;"), 32); err == nil {
					apps[i].Rating = float32(rating_perc * 5.0 / 100.0)
				}

			}

		}

	})

	return apps, nil
}

// Get the collection of the ProxyList by parse the http response
func (this *XiCiDaiLi) Result(response *http.Response) ([]contract.ProxyList, error) {
	doc, err := goquery.NewDocumentFromResponse(response)

	if err != nil {
		return nil, err
	}

	var proxyList []contract.ProxyList

	proxyTable := doc.Find("table#ip_list")
	proxyRows := proxyTable.Find("tr:nth-child(n+2)")

	proxyRows.Each(func(i int, s *goquery.Selection) {
		country := s.Find("td:nth-child(2) img").AttrOr("alt", "n/a")
		ip := s.Find("td:nth-child(3)").Text()
		port := s.Find("td:nth-child(4)").Text()
		protocol := s.Find("td:nth-child(7)").Text()

		if ip != "" && port != "" {
			proxyList = append(proxyList, contract.ProxyList{
				Ip:       ip,
				Port:     port,
				Protocol: protocol,
				Country:  strings.ToLower(country),
			})
		}
	})

	return proxyList, nil
}

func GetMajorList(client *http.Client) []string {
	banlist := map[string]string{"110431": "", "110432": "", "110511": "", "110521": "", "113110": "", "113111": ""}
	getmajorlist := []string{}
	res, err := client.Get(DHUHostUrl + DHUGetMajorURL)
	if err != nil {
		fmt.Println(err)
		fmt.Println("Something Wrong In GetMajorList")
	} else {
		doc, _ := goquery.NewDocumentFromResponse(res)
		doc.Find("select").Eq(1).Find("option").Each(func(i int, s *goquery.Selection) {
			sss, ok := s.Attr("value")
			if ok {
				_, testmap := banlist[sss]
				if testmap {
					return
				} else {
					getmajorlist = append(getmajorlist, sss)
				}
			} else {
				fmt.Println("Something Wrong")
			}
		})
		return getmajorlist
	}
	return nil
}

func GetAllLessons(res *http.Response) []CourseList {
	doc, err := goquery.NewDocumentFromResponse(res)
	if err != nil {
		fmt.Println(err)
		return nil
	} else {
		newlist := []CourseList{}
		dec := mahonia.NewDecoder("GB18030")
		doc.Find("tr").Each(func(i int, s *goquery.Selection) {
			selectid := s.Find("td").Eq(0).Text()
			_, err := strconv.Atoi(selectid)
			if err == nil {
				_, teachername, _ := dec.Translate([]byte(s.Find("td").Eq(6).Text()), true)
				newinfo := []CourseInfo{}
				s.Find("td").Eq(7).Find("tr").Each(func(i int, s *goquery.Selection) {
					_, weektime, _ := dec.Translate([]byte(s.Find("td").Eq(0).Text()), true)
					_, daytime, _ := dec.Translate([]byte(s.Find("td").Eq(1).Text()), true)
					info := CourseInfo{string(weektime), string(daytime)}
					newinfo = append(newinfo, info)
				})
				courselist := CourseList{selectid, string(teachername), newinfo}
				newlist = append(newlist, courselist)
			} else {
				return
			}
		})
		return newlist
	}
}

func (this *Object) getHtml() (string, error) {

	res, err := request("GET", this.TargetUrl, "")
	if err != nil {
		return "", err
	}
	if res.StatusCode != 200 {
		return "", errors.New(fmt.Sprintf("http status not 200, it is %d", res.StatusCode))
	}

	doc, err := goquery.NewDocumentFromResponse(res)
	if err != nil {
		return "", err
	}

	if err != nil {
		return "", err
	}
	html, err := this.parse(doc)
	if err != nil {
		return "", err
	}

	this.setArgs(doc)
	if err := this.setPageLimit(doc); err != nil {
		return "", err
	}

	return html, nil
}

func (c Boot24Crawler) ParsePage(res *http.Response) *goquery.Document {
	doc, err := goquery.NewDocumentFromResponse(res)
	if err != nil {
		log.Fatal(err)
	}
	return doc
}

// FetchStockData 抓取数据
func FetchStockData(code string) (*StockIndustry, *StockICB, error) {
	// http://stockData.stock.hexun.com/600028.shtml
	formt := "http://stockData.stock.hexun.com/%s.shtml"
	resp, err := wget.Get(fmt.Sprintf(formt, code))
	if err != nil {
		return nil, nil, gos.DoError(err)
	}

	doc, err := goquery.NewDocumentFromResponse(resp)
	if err != nil {
		return nil, nil, gos.DoError(err)
	}

	tr := doc.Find("#list3 table.box6 tr")

	stockIndustry := &StockIndustry{}
	stockICB := &StockICB{}

	stockIndustry.Name, err = iconv.ConvertString(tr.Eq(7).Find("td").Eq(1).Text(), "gb2312", "utf-8")
	if err != nil {
		return nil, nil, gos.DoError(err)
	}

	stockICB.Name, err = iconv.ConvertString(tr.Eq(8).Find("td").Eq(1).Text(), "gb2312", "utf-8")
	if err != nil {
		return nil, nil, gos.DoError(err)
	}

	return stockIndustry, stockICB, nil
}

func get_site(uri string, r *http.Request) ([]temp_item, error) {
	c := appengine.NewContext(r)
	client := urlfetch.Client(c)
	resp, err := client.Get(uri)
	if err != nil {
		return nil, err
	}

	doc, err := goquery.NewDocumentFromResponse(resp)
	if err != nil {
		return nil, err
	}

	var list_items []temp_item

	doc.Find("#bodyContent li").Not("#bodyContent #toc li").Each(func(i int, s *goquery.Selection) {
		listitem := s.Text()

		c.Infof("ITEM:\n" + listitem + "\n\n")

		q, err := filter_question(listitem)

		c.Infof("%v", err)

		if err == nil {
			list_items = append(list_items, q)
		}
	})

	//c.Infof("%+v\n", list_items)
	return list_items, nil
}

func GetRecipeIds(categoryId int, page int) ([]int, error) {
	url := GetCategoryPageURL(categoryId, page)
	resp, err := http.Get(url)
	if err != nil {
		return nil, fmt.Errorf("failed to Get %s: %v", url, err)
	}
	defer resp.Body.Close()

	doc, err := goquery.NewDocumentFromResponse(resp)
	if err != nil {
		return nil, fmt.Errorf("failed to parse response: %v", err)
	}

	recipeIds := []int{}
	doc.Find(".info").Each(func(i int, s *goquery.Selection) {
		s.Find(".name").Each(func(i int, s *goquery.Selection) {
			if href, ok := s.Find("a").Attr("href"); ok {
				id, err := strconv.Atoi(strings.Split(href, "/")[2])
				if err != nil {
					glog.Error("Failed to extract recipe id: %v", err)
				} else {
					recipeIds = append(recipeIds, id)
				}
			}
		})
	})
	return recipeIds, nil
}

// BingDomain uses bing's 'domain:' search operator and scrapes the HTML to find ips and hostnames for a domain.
func BingDomain(domain, server string) (string, Results, error) {
	task := "bing"
	results := Results{}
	resp, err := http.Get("http://www.bing.com/search?q=domain:" + domain)
	if err != nil {
		return task, results, err
	}
	doc, err := goquery.NewDocumentFromResponse(resp)
	if err != nil {
		return task, results, err
	}
	doc.Selection.Find("cite").Each(func(_ int, s *goquery.Selection) {
		u, err := url.Parse(s.Text())
		if err != nil || u.Host == "" {
			return
		}
		ip, err := LookupName(u.Host, server)
		if err != nil || ip == "" {
			cfqdn, err := LookupCname(u.Host, server)
			if err != nil || cfqdn == "" {
				return
			}
			ip, err = LookupName(cfqdn, server)
			if err != nil || ip == "" {
				return
			}

		}
		results = append(results, Result{Source: task, IP: ip, Hostname: u.Host})
	})
	return task, results, err
}