本文整理汇总了Golang中code/google/com/p/mahonia.NewDecoder函数的典型用法代码示例。如果您正苦于以下问题:Golang NewDecoder函数的具体用法?Golang NewDecoder怎么用?Golang NewDecoder使用的例子?那么恭喜您, 这里精选的函数代码示例或许可以为您提供帮助。
在下文中一共展示了NewDecoder函数的20个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于我们的系统推荐出更棒的Golang代码示例。
示例1: ConvertToString
//转码
func ConvertToString(src string, srcCode string, tagCode string) string {
srcCoder := mahonia.NewDecoder(srcCode)
srcResult := srcCoder.ConvertString(src)
tagCoder := mahonia.NewDecoder(tagCode)
_, cdata, _ := tagCoder.Translate([]byte(srcResult), true)
result := string(cdata)
return result
}
开发者ID:slygo,项目名称:360baosdk,代码行数:9,代码来源:node.go
示例2: ProcessFile
// Function ProcessFile process the given file and incorporate the information
// into the NGramGenerator g for future N-Gram model generation.
func (g *NGramGenerator) ProcessFile(filename string) error {
var decoder mahonia.Decoder
if g.charset != "" {
decoder = mahonia.NewDecoder(g.charset)
}
lineProcessor := func(line string) (bool, error) {
line = strings.Trim(line, " \t\n\f\b\r")
if decoder != nil {
line = decoder.ConvertString(line)
}
tokens := strings.Split(line, " ")
var prevToken string
for i, t := range tokens {
//Monogram frequency
g.uniGram[t]++
g.uniGramCount++
//Bigram frequency
var key BiGramKey
if i == 0 {
key = BiGramKey{SentenceStartTag, t}
} else {
key = BiGramKey{prevToken, t}
}
fmt.Printf("%v\n", key)
g.biGram[key]++
g.biGramCount++
prevToken = t
}
return true, nil
}
return util.ForEachLineInFile(filename, lineProcessor)
}
开发者ID:postfix,项目名称:LanModel,代码行数:34,代码来源:ngram_generator.go
示例3: NewSegCNCorpus
func NewSegCNCorpus(source_charset string) *SegCNCorpus {
if source_charset != "" {
return &SegCNCorpus{nil, 0, mahonia.NewDecoder(source_charset)}
}
return &SegCNCorpus{nil, 0, nil}
}
开发者ID:postfix,项目名称:LanModel,代码行数:7,代码来源:model_evaluator.go
示例4: XmlCharsetReader
func XmlCharsetReader(charset string, input io.Reader) (io.Reader, error) {
decoder := mahonia.NewDecoder(charset)
if decoder == nil {
return nil, fmt.Errorf("Unknown charset %s", charset)
}
return decoder.NewReader(input), nil
}
开发者ID:lanior,项目名称:upc,代码行数:7,代码来源:xml.go
示例5: Grep
func (self *Grepper) Grep() {
for arg := range self.In {
fh, err := os.Open(arg.Path)
f := bufio.NewReader(fh)
if err != nil {
panic(err)
}
buf := make([]byte, 1024)
decoder := mahonia.NewDecoder(arg.Encode)
m := make([]*print.Match, 0)
var lineNum = 1
for {
buf, _, err = f.ReadLine()
if err != nil {
break
}
s := string(buf)
if decoder != nil && arg.Encode != file.UTF8 && arg.Encode != file.ASCII {
s = decoder.ConvertString(s)
}
if strings.Contains(s, arg.Pattern) {
m = append(m, &print.Match{lineNum, s})
}
lineNum++
}
self.Out <- &print.Params{arg.Pattern, arg.Path, m}
fh.Close()
}
close(self.Out)
}
开发者ID:hnakamur,项目名称:the_platinum_searcher,代码行数:35,代码来源:grep.go
示例6: main
func main() {
content, err := ioutil.ReadFile("studygolang.xml")
if err != nil {
log.Fatal(err)
}
var result Result
err = xml.Unmarshal(content, &result)
if err != nil {
log.Fatal(err)
}
log.Println(result)
log.Println(result.Persons[0].Name)
//input := "<?xml version=\"1.0\" encoding=\"GBK\"?><request><head><h_exch_code>800101</h_exch_code><h_bank_no>1111</h_bank_no><h_user_id>1038738897</h_user_id><h_branch_id>B00008211</h_branch_id><h_fact_date>20110321</h_fact_date><h_fact_time>16:28:30</h_fact_time><h_exch_date>20130929</h_exch_date><h_serial_no>123456</h_serial_no><h_rsp_code>hj123545</h_rsp_code><h_rsp_msg>ar</h_rsp_msg></head><body><record><user_pwd>54f9b3396fe28c208d525db21588965c</user_pwd></record></body></request>"
//inputReader := strings.NewReader(input)
content, err = ioutil.ReadFile("request.xml")
decoder := mahonia.NewDecoder("gb18030")
r := bufio.NewReader(decoder.NewReader(content))
if err != nil {
log.Fatal(err)
}
log.Println(content)
var request Request_800101
err = xml.Unmarshal(content, &request)
if err != nil {
log.Fatal(err)
}
log.Println(request)
}
开发者ID:gaohanejie,项目名称:test,代码行数:29,代码来源:testxml2.go
示例7: UTF8Reader
func (ur *utf8reader) UTF8Reader(charset string, body io.Reader) (r io.Reader, err error) {
alias := strings.ToLower(charset)
newname, ok := charsetaliases[alias]
if ok {
charset = newname
}
r = mahonia.NewDecoder(charset).NewReader(body)
return
}
开发者ID:sunfmin,项目名称:mimemail,代码行数:9,代码来源:reader_test.go
示例8: convert
func convert(s string) string {
var dec mahonia.Decoder
dec = mahonia.NewDecoder("gbk")
if ret, ok := dec.ConvertStringOK(s); ok {
fmt.Println("GBK to UTF-8: ", ret)
return ret
}
return s
}
开发者ID:cccqcn,项目名称:go,代码行数:10,代码来源:fileDataServer.go
示例9: getNormalizedFieldName
func (dt *DbfTable) getNormalizedFieldName(name string) (s string) {
e := mahonia.NewEncoder(dt.fileEncoding)
b := []byte(e.ConvertString(name))
if len(b) > 10 {
b = b[0:10]
}
d := mahonia.NewDecoder(dt.fileEncoding)
s = d.ConvertString(string(b))
return
}
开发者ID:jvehent,项目名称:go-dbf,代码行数:13,代码来源:dbfreader.go
示例10: Find
func (this *QQwry) Find(ip string) {
if this.filepath == "" {
return
}
file, err := os.OpenFile(this.filepath, os.O_RDONLY, 0400)
defer file.Close()
if err != nil {
return
}
this.file = file
this.Ip = ip
offset := this.searchIndex(binary.BigEndian.Uint32(net.ParseIP(ip).To4()))
// log.Println("loc offset:", offset)
if offset <= 0 {
return
}
var country []byte
var area []byte
mode := this.readMode(offset + 4)
// log.Println("mode", mode)
if mode == REDIRECT_MODE_1 {
countryOffset := this.readUInt24()
mode = this.readMode(countryOffset)
// log.Println("1 - mode", mode)
if mode == REDIRECT_MODE_2 {
c := this.readUInt24()
country = this.readString(c)
countryOffset += 4
} else {
country = this.readString(countryOffset)
countryOffset += uint32(len(country) + 1)
}
area = this.readArea(countryOffset)
} else if mode == REDIRECT_MODE_2 {
countryOffset := this.readUInt24()
country = this.readString(countryOffset)
area = this.readArea(offset + 8)
} else {
country = this.readString(offset + 4)
area = this.readArea(offset + uint32(5+len(country)))
}
enc := mahonia.NewDecoder("gbk")
this.Country = enc.ConvertString(string(country))
this.City = enc.ConvertString(string(area))
}
开发者ID:RouGang,项目名称:qqwry,代码行数:51,代码来源:qqwry.go
示例11: toUtf8
func toUtf8(content []byte) (error, string) {
detector := chardet.NewTextDetector()
result, err := detector.DetectBest(content)
if err != nil {
return err, ""
}
if result.Charset == "utf8" {
return nil, string(content)
}
decoder := mahonia.NewDecoder(result.Charset)
return nil, decoder.ConvertString(string(content))
}
开发者ID:hogehoge999,项目名称:gogs,代码行数:14,代码来源:view.go
示例12: SelfPage
func SelfPage(cururl string) {
x, _ := goquery.NewDocument(cururl)
title := x.Find("title").Text()
if url, b := x.Find("#content a img").Attr("src"); b == true {
if alt, b := x.Find("#content a img").Attr("alt"); b == true {
title = alt
}
enc := mahonia.NewDecoder("gbk")
if _, e := AddSpiderData(url, enc.ConvertString(title)); e != nil {
fmt.Println(e)
}
}
}
开发者ID:rose312,项目名称:mzr,代码行数:18,代码来源:mzs2.go
示例13: GetTotalPhysicalMemory
func GetTotalPhysicalMemory() (int64, error) {
d, err := exec.Command("cmd", "/c", "wmic memphysical get MaxCapacity").Output()
if err != nil {
return -1, err
}
dec := mahonia.NewDecoder("GBK")
s := dec.ConvertString(string(d))
r := strings.NewReader(s)
var tmp string
var n int64
fmt.Fscanln(r, &tmp)
fmt.Fscanln(r, &n)
return n, nil
}
开发者ID:CalvinNeo,项目名称:EasyMLPlatform,代码行数:19,代码来源:get_mem_used.go
示例14: Test_quick
func Test_quick(t *testing.T) {
dbfHandle := DBFOpen("../map/bou2_4p.dbf", "rb")
defer DBFClose(dbfHandle)
decoder := mahonia.NewDecoder("gbk")
for i := 0; i < DBFGetRecordCount(dbfHandle); i++ {
for j := 0; j < DBFGetFieldCount(dbfHandle); j++ {
name, type_, _, _ := DBFGetFieldInfo(dbfHandle, j)
switch type_ {
case String:
p(name, decoder.ConvertString(string(DBFReadStringAttribute(dbfHandle, i, j))))
case Integer, Logical:
p(name, DBFReadIntegerAttribute(dbfHandle, i, j))
case Double:
p(name, DBFReadDoubleAttribute(dbfHandle, i, j))
}
}
}
}
开发者ID:h12w,项目名称:go-shapefile,代码行数:20,代码来源:quick_test.go
示例15: Login
func (ac *Account) Login() error {
var pld PreLoginData
err := pld.Load(ac.UserName)
if err != nil {
return err
}
client := &http.Client{}
data := `entry=weibo&gateway=1&from=&savestate=7&useticket=1&pagerefer=http%3A%2F%2Fweibo.com%2Fa%2Fdownload&vsnf=1&su=` + ac.UserName + `&service=miniblog&servertime=` + strconv.FormatInt(pld.Servertime, 10) + `&nonce=` + pld.Nonce + `&pwencode=rsa2&rsakv=` + pld.Rsakv + `&sp=` + pld.GetPwd(ac.Password) + `&encoding=UTF-8&prelt=415&url=http%3A%2F%2Fweibo.com%2Fajaxlogin.php%3Fframelogin%3D1%26callback%3Dparent.sinaSSOController.feedBackUrlCallBack&returntype=META`
fmt.Printf("form data: %v\n", data)
req, err := http.NewRequest("POST", SSO_LOGIN_URL+"?"+data, nil) // bytes.NewBuffer([]byte(data)))
if err != nil {
return err
}
SetHeader(req)
resp, err := client.Do(req)
if err != nil {
return err
}
fmt.Printf("resp: %v\n", resp)
defer resp.Body.Close()
fmt.Println("sso login cookie: ", resp.Header["Set-Cookie"])
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return err
}
enc := mahonia.NewDecoder("GBK")
bodyString := enc.ConvertString(string(body))
sIdx := strings.Index(bodyString, `replace("`)
if sIdx > 0 {
ac.loginUrl = bodyString[sIdx+9:]
sIdx = strings.Index(ac.loginUrl, `")`)
if sIdx > 0 {
ac.loginUrl = ac.loginUrl[:sIdx]
fmt.Println("loginUrl:", ac.loginUrl)
if strings.Contains(ac.loginUrl, "ticket") {
return ac.saveCookie()
}
}
}
return errors.New(fmt.Sprintf("fail to login, loginUrl: %s", ac.loginUrl))
}
开发者ID:pangkunyi,项目名称:comics,代码行数:41,代码来源:weibo-login.go
示例16: reparseHtml
func reparseHtml(s string) (*xmlpath.Node, error) {
content := mahonia.NewDecoder("cp932").ConvertString(s)
doc, err := xhtml.Parse(strings.NewReader(content))
if err != nil {
return nil, fmt.Errorf("could not parse HTML for %s ...(snip): %v",
content[:30], err)
}
var b bytes.Buffer
xhtml.Render(&b, doc)
fixed := strings.NewReader(b.String())
root, err := xmlpath.ParseHTML(fixed)
if err != nil {
return nil, fmt.Errorf("could not rebuild HTML for %s ...(snip): %v",
content[:30], err)
}
return root, nil
}
开发者ID:80nashi,项目名称:windmon,代码行数:21,代码来源:titus.go
示例17: readFile
func readFile(fileName string) (deals []StockDeal) {
s := []StockDeal{}
file, err := os.Open(fileName)
if err != nil {
fmt.Println("open File Error!")
return s
}
defer file.Close()
rd := bufio.NewReader(file)
decoder := mahonia.NewDecoder("gbk")
if decoder == nil {
fmt.Println("编码不存在!")
return s
}
for {
str, err := rd.ReadString('\n')
str = decoder.ConvertString(str)
fmt.Println(str)
if strings.HasPrefix(str, "20") {
strs := strings.Fields(str)
fmt.Println(strs)
var deal StockDeal
deal.date = strs[0] //日期
deal.time = strs[1] //时间
deal.code = strs[2] //代码
deal.name = strs[3] //名称
deal.deal = strs[4] //买入或者卖出
deal.callPrice = strs[5] //成交价
s = append(s, deal)
}
if err == io.EOF || err != nil {
fmt.Println("Read Over")
break
}
}
return s
}
开发者ID:JamesJiangCHN,项目名称:XueQiuStock,代码行数:40,代码来源:stock.go
示例18: readWebPageSource
func readWebPageSource(url string, encode ETargetEncodeType) (string, error) {
for i := 1; i <= HttpTryCnt; i++ {
rsp, err := http.Get(url)
if err != nil {
continue
}
defer rsp.Body.Close()
if rsp.StatusCode == http.StatusOK {
decoder := mahonia.NewDecoder(string(encode))
data, e := ioutil.ReadAll(decoder.NewReader(rsp.Body))
if e != nil {
continue
}
return string(data), e
}
}
return "", fmt.Errorf("read web page source timeout.")
}
开发者ID:aiwuTech,项目名称:goAcq,代码行数:22,代码来源:read.go
示例19: FieldValue
func (dt *DbfTable) FieldValue(row int, fieldIndex int) (value string) {
// create decoder to convert bytes to utf-8
d := mahonia.NewDecoder(dt.fileEncoding)
offset := int(dt.numberOfBytesInHeader)
lengthOfRecord := int(dt.lengthOfEachRecord)
offset = offset + (row * lengthOfRecord)
recordOffset := 1
for i := 0; i < len(dt.fields); i++ {
if i == fieldIndex {
break
} else {
recordOffset += int(dt.fields[i].fieldLength)
}
}
temp := dt.dataStore[(offset + recordOffset):((offset + recordOffset) + int(dt.fields[fieldIndex].fieldLength))]
for i := 0; i < len(temp); i++ {
if temp[i] == 0x00 {
temp = temp[0:i]
break
}
}
s := d.ConvertString(string(temp))
//fmt.Printf("utf-8 value:[%#v]\n", s)
value = strings.TrimSpace(s)
//fmt.Printf("raw value:[%#v]\n", dt.dataStore[(offset + recordOffset):((offset + recordOffset) + int(dt.Fields[fieldIndex].fieldLength))])
//fmt.Printf("utf-8 value:[%#v]\n", []byte(s))
//value = string(dt.dataStore[(offset + recordOffset):((offset + recordOffset) + int(dt.Fields[fieldIndex].fieldLength))])
return
}
开发者ID:jvehent,项目名称:go-dbf,代码行数:39,代码来源:dbfreader.go
示例20: DecodeResponseBody
func (requestInfo RequestInformation) DecodeResponseBody(body io.Reader) (*SaltResponse, error) {
charset := mahonia.NewDecoder(requestInfo.Charset)
if charset == nil {
return nil, errors.New("charset is null.")
}
r := charset.NewReader(body)
decoder := xml.NewDecoder(r)
decoder.CharsetReader = CharsetReader
for {
token, err := decoder.Token()
if err == io.EOF {
break
} else if err != nil {
return nil, err
}
switch startElement := token.(type) {
case xml.StartElement:
if startElement.Name.Space == "http://schemas.xmlsoap.org/soap/envelope/" && startElement.Name.Local == "Body" {
//nextElementIsBody = true
responseBody := SaltResponse{}
err = decoder.DecodeElement(&responseBody, &startElement)
if err != nil {
return nil, err
}
return &responseBody, nil
}
}
}
return nil, errors.New("Did not find SOAP body element")
}
开发者ID:pallat,项目名称:saltMultiRecord,代码行数:37,代码来源:salt.go
注:本文中的code/google/com/p/mahonia.NewDecoder函数示例整理自Github/MSDocs等源码及文档管理平台,相关代码片段筛选自各路编程大神贡献的开源项目,源码版权归原作者所有,传播和使用请参考对应项目的License;未经允许,请勿转载。 |
请发表评论