Golang mahonia.NewDecoder函数代码示例

原作者: [db:作者] 来自: [db:来源] 收藏邀请

本文整理汇总了Golang中code/google/com/p/mahonia.NewDecoder函数的典型用法代码示例。如果您正苦于以下问题：Golang NewDecoder函数的具体用法？Golang NewDecoder怎么用？Golang NewDecoder使用的例子？那么恭喜您, 这里精选的函数代码示例或许可以为您提供帮助。

在下文中一共展示了NewDecoder函数的20个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于我们的系统推荐出更棒的Golang代码示例。

示例1: ConvertToString

//转码
func ConvertToString(src string, srcCode string, tagCode string) string {
	srcCoder := mahonia.NewDecoder(srcCode)
	srcResult := srcCoder.ConvertString(src)
	tagCoder := mahonia.NewDecoder(tagCode)
	_, cdata, _ := tagCoder.Translate([]byte(srcResult), true)
	result := string(cdata)
	return result
}

开发者ID:slygo，项目名称:360baosdk，代码行数:9，代码来源:node.go

示例2: ProcessFile

// Function ProcessFile process the given file and incorporate the information
// into the NGramGenerator g for future N-Gram model generation.
func (g *NGramGenerator) ProcessFile(filename string) error {
	var decoder mahonia.Decoder
	if g.charset != "" {
		decoder = mahonia.NewDecoder(g.charset)
	}
	lineProcessor := func(line string) (bool, error) {
		line = strings.Trim(line, " \t\n\f\b\r")
		if decoder != nil {
			line = decoder.ConvertString(line)
		}
		tokens := strings.Split(line, " ")
		var prevToken string
		for i, t := range tokens {
			//Monogram frequency
			g.uniGram[t]++
			g.uniGramCount++
			//Bigram frequency
			var key BiGramKey
			if i == 0 {
				key = BiGramKey{SentenceStartTag, t}
			} else {
				key = BiGramKey{prevToken, t}
			}
			fmt.Printf("%v\n", key)
			g.biGram[key]++
			g.biGramCount++
			prevToken = t
		}
		return true, nil
	}
	return util.ForEachLineInFile(filename, lineProcessor)
}

开发者ID:postfix，项目名称:LanModel，代码行数:34，代码来源:ngram_generator.go

示例3: NewSegCNCorpus

func NewSegCNCorpus(source_charset string) *SegCNCorpus {
	if source_charset != "" {
		return &SegCNCorpus{nil, 0, mahonia.NewDecoder(source_charset)}
	}
	return &SegCNCorpus{nil, 0, nil}

}

开发者ID:postfix，项目名称:LanModel，代码行数:7，代码来源:model_evaluator.go

示例4: XmlCharsetReader

func XmlCharsetReader(charset string, input io.Reader) (io.Reader, error) {
	decoder := mahonia.NewDecoder(charset)
	if decoder == nil {
		return nil, fmt.Errorf("Unknown charset %s", charset)
	}
	return decoder.NewReader(input), nil
}

开发者ID:lanior，项目名称:upc，代码行数:7，代码来源:xml.go

示例5: Grep

func (self *Grepper) Grep() {
	for arg := range self.In {

		fh, err := os.Open(arg.Path)
		f := bufio.NewReader(fh)
		if err != nil {
			panic(err)
		}
		buf := make([]byte, 1024)
		decoder := mahonia.NewDecoder(arg.Encode)

		m := make([]*print.Match, 0)

		var lineNum = 1
		for {
			buf, _, err = f.ReadLine()
			if err != nil {
				break
			}

			s := string(buf)
			if decoder != nil && arg.Encode != file.UTF8 && arg.Encode != file.ASCII {
				s = decoder.ConvertString(s)
			}
			if strings.Contains(s, arg.Pattern) {
				m = append(m, &print.Match{lineNum, s})
			}
			lineNum++
		}
		self.Out <- &print.Params{arg.Pattern, arg.Path, m}
		fh.Close()

	}
	close(self.Out)
}

开发者ID:hnakamur，项目名称:the_platinum_searcher，代码行数:35，代码来源:grep.go

示例6: main

func main() {
	content, err := ioutil.ReadFile("studygolang.xml")
	if err != nil {
		log.Fatal(err)
	}
	var result Result
	err = xml.Unmarshal(content, &result)
	if err != nil {
		log.Fatal(err)
	}
	log.Println(result)
	log.Println(result.Persons[0].Name)

	//input := "<?xml version=\"1.0\" encoding=\"GBK\"?><request><head><h_exch_code>800101</h_exch_code><h_bank_no>1111</h_bank_no><h_user_id>1038738897</h_user_id><h_branch_id>B00008211</h_branch_id><h_fact_date>20110321</h_fact_date><h_fact_time>16:28:30</h_fact_time><h_exch_date>20130929</h_exch_date><h_serial_no>123456</h_serial_no><h_rsp_code>hj123545</h_rsp_code><h_rsp_msg>ar</h_rsp_msg></head><body><record><user_pwd>54f9b3396fe28c208d525db21588965c</user_pwd></record></body></request>"
	//inputReader := strings.NewReader(input)
	content, err = ioutil.ReadFile("request.xml")
	decoder := mahonia.NewDecoder("gb18030")
	r := bufio.NewReader(decoder.NewReader(content))
	if err != nil {
		log.Fatal(err)
	}
	log.Println(content)
	var request Request_800101
	err = xml.Unmarshal(content, &request)
	if err != nil {
		log.Fatal(err)
	}
	log.Println(request)
}

开发者ID:gaohanejie，项目名称:test，代码行数:29，代码来源:testxml2.go

示例7: UTF8Reader

func (ur *utf8reader) UTF8Reader(charset string, body io.Reader) (r io.Reader, err error) {
	alias := strings.ToLower(charset)
	newname, ok := charsetaliases[alias]
	if ok {
		charset = newname
	}
	r = mahonia.NewDecoder(charset).NewReader(body)
	return
}

开发者ID:sunfmin，项目名称:mimemail，代码行数:9，代码来源:reader_test.go

示例8: convert

func convert(s string) string {
	var dec mahonia.Decoder
	dec = mahonia.NewDecoder("gbk")
	if ret, ok := dec.ConvertStringOK(s); ok {
		fmt.Println("GBK to UTF-8: ", ret)
		return ret
	}

	return s
}

开发者ID:cccqcn，项目名称:go，代码行数:10，代码来源:fileDataServer.go

示例9: getNormalizedFieldName

func (dt *DbfTable) getNormalizedFieldName(name string) (s string) {
	e := mahonia.NewEncoder(dt.fileEncoding)
	b := []byte(e.ConvertString(name))

	if len(b) > 10 {
		b = b[0:10]
	}

	d := mahonia.NewDecoder(dt.fileEncoding)
	s = d.ConvertString(string(b))

	return
}

开发者ID:jvehent，项目名称:go-dbf，代码行数:13，代码来源:dbfreader.go

示例10: Find

func (this *QQwry) Find(ip string) {
	if this.filepath == "" {
		return
	}

	file, err := os.OpenFile(this.filepath, os.O_RDONLY, 0400)
	defer file.Close()
	if err != nil {
		return
	}
	this.file = file

	this.Ip = ip
	offset := this.searchIndex(binary.BigEndian.Uint32(net.ParseIP(ip).To4()))
	// log.Println("loc offset:", offset)
	if offset <= 0 {
		return
	}

	var country []byte
	var area []byte

	mode := this.readMode(offset + 4)
	// log.Println("mode", mode)
	if mode == REDIRECT_MODE_1 {
		countryOffset := this.readUInt24()
		mode = this.readMode(countryOffset)
		// log.Println("1 - mode", mode)
		if mode == REDIRECT_MODE_2 {
			c := this.readUInt24()
			country = this.readString(c)
			countryOffset += 4
		} else {
			country = this.readString(countryOffset)
			countryOffset += uint32(len(country) + 1)
		}
		area = this.readArea(countryOffset)
	} else if mode == REDIRECT_MODE_2 {
		countryOffset := this.readUInt24()
		country = this.readString(countryOffset)
		area = this.readArea(offset + 8)
	} else {
		country = this.readString(offset + 4)
		area = this.readArea(offset + uint32(5+len(country)))
	}

	enc := mahonia.NewDecoder("gbk")
	this.Country = enc.ConvertString(string(country))
	this.City = enc.ConvertString(string(area))

}

开发者ID:RouGang，项目名称:qqwry，代码行数:51，代码来源:qqwry.go

示例11: toUtf8

func toUtf8(content []byte) (error, string) {
	detector := chardet.NewTextDetector()
	result, err := detector.DetectBest(content)
	if err != nil {
		return err, ""
	}

	if result.Charset == "utf8" {
		return nil, string(content)
	}

	decoder := mahonia.NewDecoder(result.Charset)
	return nil, decoder.ConvertString(string(content))
}

开发者ID:hogehoge999，项目名称:gogs，代码行数:14，代码来源:view.go

示例12: SelfPage

func SelfPage(cururl string) {

	x, _ := goquery.NewDocument(cururl)
	title := x.Find("title").Text()

	if url, b := x.Find("#content a img").Attr("src"); b == true {

		if alt, b := x.Find("#content a img").Attr("alt"); b == true {
			title = alt
		}

		enc := mahonia.NewDecoder("gbk")
		if _, e := AddSpiderData(url, enc.ConvertString(title)); e != nil {
			fmt.Println(e)
		}
	}

}

开发者ID:rose312，项目名称:mzr，代码行数:18，代码来源:mzs2.go

示例13: GetTotalPhysicalMemory

func GetTotalPhysicalMemory() (int64, error) {
	d, err := exec.Command("cmd", "/c", "wmic memphysical get MaxCapacity").Output()
	if err != nil {
		return -1, err
	}

	dec := mahonia.NewDecoder("GBK")

	s := dec.ConvertString(string(d))

	r := strings.NewReader(s)

	var tmp string
	var n int64
	fmt.Fscanln(r, &tmp)
	fmt.Fscanln(r, &n)

	return n, nil
}

开发者ID:CalvinNeo，项目名称:EasyMLPlatform，代码行数:19，代码来源:get_mem_used.go

示例14: Test_quick

func Test_quick(t *testing.T) {
	dbfHandle := DBFOpen("../map/bou2_4p.dbf", "rb")
	defer DBFClose(dbfHandle)

	decoder := mahonia.NewDecoder("gbk")

	for i := 0; i < DBFGetRecordCount(dbfHandle); i++ {
		for j := 0; j < DBFGetFieldCount(dbfHandle); j++ {
			name, type_, _, _ := DBFGetFieldInfo(dbfHandle, j)
			switch type_ {
			case String:
				p(name, decoder.ConvertString(string(DBFReadStringAttribute(dbfHandle, i, j))))
			case Integer, Logical:
				p(name, DBFReadIntegerAttribute(dbfHandle, i, j))
			case Double:
				p(name, DBFReadDoubleAttribute(dbfHandle, i, j))
			}
		}
	}
}

开发者ID:h12w，项目名称:go-shapefile，代码行数:20，代码来源:quick_test.go

示例15: Login

func (ac *Account) Login() error {
	var pld PreLoginData
	err := pld.Load(ac.UserName)
	if err != nil {
		return err
	}
	client := &http.Client{}
	data := `entry=weibo&gateway=1&from=&savestate=7&useticket=1&pagerefer=http%3A%2F%2Fweibo.com%2Fa%2Fdownload&vsnf=1&su=` + ac.UserName + `&service=miniblog&servertime=` + strconv.FormatInt(pld.Servertime, 10) + `&nonce=` + pld.Nonce + `&pwencode=rsa2&rsakv=` + pld.Rsakv + `&sp=` + pld.GetPwd(ac.Password) + `&encoding=UTF-8&prelt=415&url=http%3A%2F%2Fweibo.com%2Fajaxlogin.php%3Fframelogin%3D1%26callback%3Dparent.sinaSSOController.feedBackUrlCallBack&returntype=META`
	fmt.Printf("form data: %v\n", data)
	req, err := http.NewRequest("POST", SSO_LOGIN_URL+"?"+data, nil) // bytes.NewBuffer([]byte(data)))
	if err != nil {
		return err
	}
	SetHeader(req)
	resp, err := client.Do(req)
	if err != nil {
		return err
	}
	fmt.Printf("resp: %v\n", resp)
	defer resp.Body.Close()
	fmt.Println("sso login cookie: ", resp.Header["Set-Cookie"])
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}
	enc := mahonia.NewDecoder("GBK")
	bodyString := enc.ConvertString(string(body))
	sIdx := strings.Index(bodyString, `replace("`)
	if sIdx > 0 {
		ac.loginUrl = bodyString[sIdx+9:]
		sIdx = strings.Index(ac.loginUrl, `")`)
		if sIdx > 0 {
			ac.loginUrl = ac.loginUrl[:sIdx]
			fmt.Println("loginUrl:", ac.loginUrl)
			if strings.Contains(ac.loginUrl, "ticket") {
				return ac.saveCookie()
			}
		}
	}
	return errors.New(fmt.Sprintf("fail to login, loginUrl: %s", ac.loginUrl))
}

开发者ID:pangkunyi，项目名称:comics，代码行数:41，代码来源:weibo-login.go

示例16: reparseHtml

func reparseHtml(s string) (*xmlpath.Node, error) {
	content := mahonia.NewDecoder("cp932").ConvertString(s)

	doc, err := xhtml.Parse(strings.NewReader(content))
	if err != nil {
		return nil, fmt.Errorf("could not parse HTML for %s ...(snip): %v",
			content[:30], err)
	}

	var b bytes.Buffer
	xhtml.Render(&b, doc)
	fixed := strings.NewReader(b.String())

	root, err := xmlpath.ParseHTML(fixed)
	if err != nil {
		return nil, fmt.Errorf("could not rebuild HTML for %s ...(snip): %v",
			content[:30], err)
	}

	return root, nil
}

开发者ID:80nashi，项目名称:windmon，代码行数:21，代码来源:titus.go

示例17: readFile

func readFile(fileName string) (deals []StockDeal) {
	s := []StockDeal{}
	file, err := os.Open(fileName)
	if err != nil {
		fmt.Println("open File Error!")
		return s
	}
	defer file.Close()
	rd := bufio.NewReader(file)
	decoder := mahonia.NewDecoder("gbk")
	if decoder == nil {
		fmt.Println("编码不存在!")
		return s
	}

	for {
		str, err := rd.ReadString('\n')

		str = decoder.ConvertString(str)
		fmt.Println(str)
		if strings.HasPrefix(str, "20") {
			strs := strings.Fields(str)
			fmt.Println(strs)
			var deal StockDeal
			deal.date = strs[0]      //日期
			deal.time = strs[1]      //时间
			deal.code = strs[2]      //代码
			deal.name = strs[3]      //名称
			deal.deal = strs[4]      //买入或者卖出
			deal.callPrice = strs[5] //成交价
			s = append(s, deal)

		}
		if err == io.EOF || err != nil {
			fmt.Println("Read Over")
			break
		}
	}
	return s
}

开发者ID:JamesJiangCHN，项目名称:XueQiuStock，代码行数:40，代码来源:stock.go

示例18: readWebPageSource

func readWebPageSource(url string, encode ETargetEncodeType) (string, error) {
	for i := 1; i <= HttpTryCnt; i++ {

		rsp, err := http.Get(url)
		if err != nil {
			continue
		}
		defer rsp.Body.Close()

		if rsp.StatusCode == http.StatusOK {
			decoder := mahonia.NewDecoder(string(encode))
			data, e := ioutil.ReadAll(decoder.NewReader(rsp.Body))
			if e != nil {
				continue
			}

			return string(data), e
		}
	}

	return "", fmt.Errorf("read web page source timeout.")
}

开发者ID:aiwuTech，项目名称:goAcq，代码行数:22，代码来源:read.go

示例19: FieldValue

func (dt *DbfTable) FieldValue(row int, fieldIndex int) (value string) {

	// create decoder to convert bytes to utf-8
	d := mahonia.NewDecoder(dt.fileEncoding)

	offset := int(dt.numberOfBytesInHeader)
	lengthOfRecord := int(dt.lengthOfEachRecord)

	offset = offset + (row * lengthOfRecord)

	recordOffset := 1

	for i := 0; i < len(dt.fields); i++ {
		if i == fieldIndex {
			break
		} else {
			recordOffset += int(dt.fields[i].fieldLength)
		}
	}

	temp := dt.dataStore[(offset + recordOffset):((offset + recordOffset) + int(dt.fields[fieldIndex].fieldLength))]

	for i := 0; i < len(temp); i++ {
		if temp[i] == 0x00 {
			temp = temp[0:i]
			break
		}
	}

	s := d.ConvertString(string(temp))
	//fmt.Printf("utf-8 value:[%#v]\n", s)

	value = strings.TrimSpace(s)

	//fmt.Printf("raw value:[%#v]\n", dt.dataStore[(offset + recordOffset):((offset + recordOffset) + int(dt.Fields[fieldIndex].fieldLength))])
	//fmt.Printf("utf-8 value:[%#v]\n", []byte(s))
	//value = string(dt.dataStore[(offset + recordOffset):((offset + recordOffset) + int(dt.Fields[fieldIndex].fieldLength))])
	return
}

开发者ID:jvehent，项目名称:go-dbf，代码行数:39，代码来源:dbfreader.go

示例20: DecodeResponseBody

func (requestInfo RequestInformation) DecodeResponseBody(body io.Reader) (*SaltResponse, error) {
	charset := mahonia.NewDecoder(requestInfo.Charset)
	if charset == nil {
		return nil, errors.New("charset is null.")
	}

	r := charset.NewReader(body)
	decoder := xml.NewDecoder(r)

	decoder.CharsetReader = CharsetReader

	for {
		token, err := decoder.Token()
		if err == io.EOF {
			break
		} else if err != nil {
			return nil, err
		}
		switch startElement := token.(type) {
		case xml.StartElement:
			if startElement.Name.Space == "http://schemas.xmlsoap.org/soap/envelope/" && startElement.Name.Local == "Body" {
				//nextElementIsBody = true
				responseBody := SaltResponse{}
				err = decoder.DecodeElement(&responseBody, &startElement)

				if err != nil {

					return nil, err

				}
				return &responseBody, nil
			}
		}
	}

	return nil, errors.New("Did not find SOAP body element")
}

开发者ID:pallat，项目名称:saltMultiRecord，代码行数:37，代码来源:salt.go

注：本文中的code/google/com/p/mahonia.NewDecoder函数示例整理自Github/MSDocs等源码及文档管理平台，相关代码片段筛选自各路编程大神贡献的开源项目，源码版权归原作者所有，传播和使用请参考对应项目的License；未经允许，请勿转载。