Golang transform.NewReader函数代码示例

原作者: [db:作者] 来自: [db:来源] 收藏邀请

本文整理汇总了Golang中golang.org/x/text/transform.NewReader函数的典型用法代码示例。如果您正苦于以下问题：Golang NewReader函数的具体用法？Golang NewReader怎么用？Golang NewReader使用的例子？那么恭喜您, 这里精选的函数代码示例或许可以为您提供帮助。

在下文中一共展示了NewReader函数的20个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于我们的系统推荐出更棒的Golang代码示例。

示例1: NewReader

// NewReader returns a reader which decode from the given encoding, to utf8.
//
// If enc is nil, then only an utf8-enforcing replacement reader
// (see http://godoc.org/code.google.com/p/go.text/encoding#pkg-variables)
// is used.
func NewReader(r io.Reader, enc encoding.Encoding) io.Reader {
	if enc == nil || enc == encoding.Replacement {
		return transform.NewReader(r, encoding.Replacement.NewEncoder())
	}
	return transform.NewReader(r,
		transform.Chain(enc.NewDecoder(), encoding.Replacement.NewEncoder()))
}

开发者ID:xingskycn，项目名称:go，代码行数:12，代码来源:reader.go

示例2: main

// Encoding UTF-8 to Cp1251 and back
func main() {
	var err error

	encoder := charmap.Windows1251.NewEncoder()
	decoder := charmap.Windows1251.NewDecoder()

	inUtf8 := "Ёжики пушистые 好 ἱερογλυφικὰ γράμματ"
	// inUtf8 := "Ёжики пушистые"

	sr := strings.NewReader(inUtf8)
	tr := transform.NewReader(sr, encoder)
	inCp1251, err := ioutil.ReadAll(tr)
	if err != nil {
		fmt.Println("Encoding error: ", err)
	}

	srBack := bytes.NewReader(inCp1251)
	trBack := transform.NewReader(srBack, decoder)
	outUtf8, err := ioutil.ReadAll(trBack)
	if err != nil {
		fmt.Println("Decoding error: ", err)
	}

	fmt.Println("Source UTF8:", inUtf8)
	fmt.Println("CP1251:", inCp1251, string(inCp1251))
	fmt.Println("Result UTF8:", string(outUtf8))

	fmt.Println(strings.Repeat("=", 80))
	fmt.Println("Test https://github.com/fiam/gounidecode")
	fmt.Println("Original: ", inUtf8)
	fmt.Println("Translit: ", unidecode.Unidecode(inUtf8))
}

开发者ID:plumbum，项目名称:go-samples，代码行数:33，代码来源:main.go

示例3: newDecodeReader

func newDecodeReader(r io.Reader, encoding int) io.Reader {
	switch encoding {
	case EUCJP:
		return transform.NewReader(r, japanese.EUCJP.NewDecoder())
	case SHIFTJIS:
		return transform.NewReader(r, japanese.ShiftJIS.NewDecoder())
	}
	return nil
}

开发者ID:t-suwa，项目名称:the_platinum_searcher，代码行数:9，代码来源:decoder.go

示例4: decodeTransfer

// DecodeTransfer decodes base64, quoted-printable or plain text.
func decodeTransfer(r io.Reader, label string) io.Reader {
	switch strings.ToLower(label) {
	case "base64":
		return base64.NewDecoder(base64.StdEncoding, transform.NewReader(r, nonASCIITransformer{}))
	case "quoted-printable":
		return quotedprintable.NewReader(transform.NewReader(r, transform.Chain(nonASCIITransformer{}, newlineAppendTransformer{})))
	case "", "7bit", "8bit", "binary":
		return r
	default:
		return failReader{fmt.Errorf("unsupported transfer encoding: %v", label)}
	}
}

开发者ID:kaey，项目名称:mail，代码行数:13，代码来源:mail.go

示例5: New

func New(name string) *Type2 {
	var typeable bool = true
	fi, err := os.Stat(name)
	if err != nil && os.IsNotExist(err) {
		typeable = false
	}
	if err == nil && fi.IsDir() {
		typeable = false
	}
	if !typeable {
		return &Type2{
			Name:     name,
			Typeable: false,
		}
	}

	file, err := os.Open(name)
	if err != nil {
		typeable = false
	}
	buf, err := ioutil.ReadAll(file)
	if err != nil {
		typeable = false
	}
	_, err = file.Seek(0, 0)
	if err != nil {
		typeable = false
	}
	if !typeable {
		return &Type2{
			Name:     name,
			Typeable: false,
		}
	}

	var reader io.Reader
	e := guess_jp(buf)
	switch e {
	case "Shift_JIS":
		reader = transform.NewReader(file, japanese.ShiftJIS.NewDecoder())
	case "EUC-JP":
		reader = transform.NewReader(file, japanese.EUCJP.NewDecoder())
	default:
		reader = file
	}
	return &Type2{
		Name:     name,
		Typeable: typeable,
		File:     reader,
	}
}

开发者ID:kuked，项目名称:type2，代码行数:51，代码来源:type2.go

示例6: Reader

// Reader returns a new UTF-8 io.Reader for the response body.
func (res *Response) Reader() (io.Reader, error) {
	enc, err := res.Encoding()
	if err != nil {
		return nil, err
	}
	return transform.NewReader(bytes.NewReader(res.Body), enc.NewDecoder()), nil
}

开发者ID:moomerman，项目名称:whois，代码行数:8，代码来源:response.go

示例7: to_utf8

// Shift-JIS -> UTF-8
func to_utf8(str string) (string, error) {
	body, err := ioutil.ReadAll(transform.NewReader(strings.NewReader(str), japanese.ShiftJIS.NewEncoder()))
	if err != nil {
		return "", err
	}

	var f []byte
	encodings := []string{"sjis", "utf-8"}
	for _, enc := range encodings {
		if enc != "" {
			ee, _ := charset.Lookup(enc)
			if ee == nil {
				continue
			}
			var buf bytes.Buffer
			ic := transform.NewWriter(&buf, ee.NewDecoder())
			_, err := ic.Write(body)
			if err != nil {
				continue
			}
			err = ic.Close()
			if err != nil {
				continue
			}
			f = buf.Bytes()
			break
		}
	}
	return string(f), nil
}

开发者ID:sharkattack51，项目名称:go_Shift-JIS_to_UTF-8，代码行数:31，代码来源:main.go

示例8: decode_utf8

func decode_utf8(fixedHtml string) string {
	e := charmap.ISO8859_15
	reader := strings.NewReader(fixedHtml)
	rInUTF8 := transform.NewReader(reader, e.NewDecoder())

	return reader_to_str(rInUTF8)
}

开发者ID:peondusud，项目名称:go.scrap.lbc，代码行数:7，代码来源:lbc_utils.go

示例9: decode

func decode(charset string, input io.Reader) (io.Reader, error) {
	if charset != "cp1251" {
		return nil, fmt.Errorf("unsupported charset")
	}

	return transform.NewReader(input, charmap.Windows1251.NewDecoder()), nil
}

开发者ID:gwutz，项目名称:xmlrpc，代码行数:7，代码来源:decoder_test.go

示例10: TestNonRepertoire

// TestNonRepertoire tests that codes outside of an Encoding's repertoire are
// converted:
//   - to the Unicode replacement character '\ufffd' when decoding to UTF-8,
//   - to the ASCII substitute character '\x1a' when encoding from UTF-8.
func TestNonRepertoire(t *testing.T) {
	testCases := []struct {
		e          encoding.Encoding
		dSrc, eSrc string
	}{
		{charmap.Windows1252, "\x81", "갂"},
		{japanese.EUCJP, "\xfe\xfc", "갂"},
		{japanese.ISO2022JP, "\x1b$B\x7e\x7e", "갂"},
		{japanese.ShiftJIS, "\xef\xfc", "갂"},
		{korean.EUCKR, "\xfe\xfe", "א"},
		{simplifiedchinese.GBK, "\xfe\xfe", "갂"},
		{simplifiedchinese.HZGB2312, "~{z~", "갂"},
		{traditionalchinese.Big5, "\x81\x40", "갂"},
	}
	for _, tc := range testCases {
		for _, direction := range []string{"Decode", "Encode"} {
			enc, want, src := (transform.Transformer)(nil), "", ""
			if direction == "Decode" {
				enc, want, src = tc.e.NewDecoder(), "\ufffd", tc.dSrc
			} else {
				enc, want, src = tc.e.NewEncoder(), "\x1a", tc.eSrc
			}

			dst, err := ioutil.ReadAll(transform.NewReader(strings.NewReader(src), enc))
			if err != nil {
				t.Errorf("%s %v: %v", direction, tc.e, err)
				continue
			}
			if got := string(dst); got != want {
				t.Errorf("%s %v:\ngot  %q\nwant %q", direction, tc.e, got, want)
				continue
			}
		}
	}
}

开发者ID:davidsoloman，项目名称:beats，代码行数:39，代码来源:encoding_test.go

示例11: getReader

func getReader(file *os.File) io.Reader {
	encoding := getEncoding()
	if encoding == nil {
		return file
	}
	return transform.NewReader(file, encoding.NewDecoder())
}

开发者ID:ryoon，项目名称:cli-kintone，代码行数:7，代码来源:import.go

示例12: EucToUtf8

// EucToUtf8 convert euc encoded string to utf-8 encoded string
func EucToUtf8(data string) (string, error) {
	in := bytes.NewBufferString(data)
	out := new(bytes.Buffer)
	reader := transform.NewReader(in, japanese.EUCJP.NewDecoder())
	_, e := io.Copy(out, reader)
	return out.String(), e
}

开发者ID:DoG-peer，项目名称:bouSaga2.0，代码行数:8，代码来源:utils.go

示例13: TestReader

func TestReader(t *testing.T) {
	for _, tc := range sniffTestCases {
		content, err := ioutil.ReadFile("testdata/" + tc.filename)
		if err != nil {
			t.Errorf("%s: error reading file: %v", tc.filename, err)
			continue
		}

		r, err := NewReader(bytes.NewReader(content), tc.declared)
		if err != nil {
			t.Errorf("%s: error creating reader: %v", tc.filename, err)
			continue
		}

		got, err := ioutil.ReadAll(r)
		if err != nil {
			t.Errorf("%s: error reading from charset.NewReader: %v", tc.filename, err)
			continue
		}

		e, _ := Lookup(tc.want)
		want, err := ioutil.ReadAll(transform.NewReader(bytes.NewReader(content), e.NewDecoder()))
		if err != nil {
			t.Errorf("%s: error decoding with hard-coded charset name: %v", tc.filename, err)
			continue
		}

		if !bytes.Equal(got, want) {
			t.Errorf("%s: got %q, want %q", tc.filename, got, want)
			continue
		}
	}
}

开发者ID:jelmersnoeck，项目名称:go.net，代码行数:33，代码来源:charset_test.go

示例14: TestScramNormalizesPassword

func (s *SASLSuite) TestScramNormalizesPassword(c *C) {
	// From: libidn-1.9/tests/tst_stringprep.c
	// See RFC 4013, section 3
	testCases := []struct {
		raw        string
		normalized string
	}{
		{"I\xC2\xADX", "IX"},
		{"user", "user"},
		{"USER", "USER"},
		{"\xC2\xAA", "a"},
		{"x\xC2\xADy", "xy"},
		{"\xE2\x85\xA3", "IV"},
		{"\xE2\x85\xA8", "IX"},
		//They should error because they have forbidden chars
		//{"\x07", ""},      //should error
		//{"\xD8\xA71", ""}, //shold error
	}

	for _, test := range testCases {
		t := transform.NewReader(strings.NewReader(test.raw), Stringprep)
		r := bufio.NewReader(t)
		normalized, _, err := r.ReadLine()

		c.Check(err, IsNil)
		c.Check(normalized, DeepEquals, []byte(test.normalized))
	}
}

开发者ID:PMaynard，项目名称:coyim，代码行数:28，代码来源:stringprep_test.go

示例15: getReader

func getReader(reader io.Reader) io.Reader {
	encoding := getEncoding()
	if encoding == nil {
		return reader
	}
	return transform.NewReader(reader, encoding.NewDecoder())
}

开发者ID:kintone，项目名称:cli-kintone，代码行数:7，代码来源:import.go

示例16: getGbkDoc

func getGbkDoc(client *http.Client, url string) (*goquery.Document, error) {
	retry := 3
get:
	resp, err := client.Get(url)
	if err != nil {
		if retry > 0 {
			retry--
			goto get
		} else {
			return nil, me(err, "get")
		}
	}
	defer resp.Body.Close()
	r := transform.NewReader(resp.Body, simplifiedchinese.GBK.NewDecoder())
	doc, err := goquery.NewDocumentFromReader(r)
	if err != nil {
		if retry > 0 {
			retry--
			goto get
		} else {
			return nil, me(err, "new document from response")
		}
	}
	return doc, nil
}

开发者ID:reusee，项目名称:jd-analysis，代码行数:25，代码来源:utils.go

示例17: TestScramNormalizesPassword

func (s *SASLSuite) TestScramNormalizesPassword(c *C) {
	// From: libidn-1.9/tests/tst_stringprep.c
	// See RFC 4013, section 3
	testCases := []struct {
		raw        string
		normalized string
	}{
		{"I\xC2\xADX", "IX"},
		{"user", "user"},
		{"USER", "USER"},
		{"user\u200B", "user "},
		{"user\u2002", "user "},
		{"\xC2\xAA", "a"},
		{"x\xC2\xADy", "xy"},
		{"\xE2\x85\xA3", "IV"},
		{"\xE2\x85\xA8", "IX"},
		{"\u034F\u1806\u180Bb\u180C\u180Dy\u200Ct\u200D\u2060\uFE00e\uFE01\uFE02\uFE03\uFE04\uFE05\uFE06\uFE07\uFE08\uFE09\uFE0A\uFE0B\uFE0C\uFE0D\uFE0E\uFE0F\uFEFF", "byte"},
		//They should error because they have forbidden chars
		//{"\x07", ""},      //should error
		//{"\xD8\xA71", ""}, //shold error
	}

	for _, test := range testCases {
		t := transform.NewReader(strings.NewReader(test.raw), Stringprep)
		r := bufio.NewReader(t)
		normalized, _, err := r.ReadLine()

		c.Check(err, IsNil)
		c.Check(string(normalized), DeepEquals, test.normalized)
	}
}

开发者ID:grayleonard，项目名称:coyim，代码行数:31，代码来源:stringprep_test.go

示例18: NewReaderLabel

// NewReaderLabel returns a reader that converts from the specified charset to
// UTF-8. It uses Lookup to find the encoding that corresponds to label, and
// returns an error if Lookup returns nil. It is suitable for use as
// encoding/xml.Decoder's CharsetReader function.
func NewReaderLabel(label string, input io.Reader) (io.Reader, error) {
	e, _ := Lookup(label)
	if e == nil {
		return nil, fmt.Errorf("unsupported charset: %q", label)
	}
	return transform.NewReader(input, e.NewDecoder()), nil
}

开发者ID:Clarifai，项目名称:kubernetes，代码行数:11，代码来源:charset.go

示例19: main

func main() {
	t := transform.Chain(norm.NFD, transform.RemoveFunc(isMn), norm.NFC)
	r := transform.NewReader(os.Stdin, t)
	if _, err := io.Copy(os.Stdout, r); err != nil {
		log.Fatal(err)
	}
}

开发者ID:Roman2K，项目名称:transliterate，代码行数:7，代码来源:main.go

示例20: defaultCharsetReader

func defaultCharsetReader(cs string, input io.Reader) (io.Reader, error) {
	e, _ := charset.Lookup(cs)
	if e == nil {
		return nil, fmt.Errorf("cannot decode charset %v", cs)
	}
	return transform.NewReader(input, e.NewDecoder()), nil
}

开发者ID:kissthink，项目名称:goread，代码行数:7，代码来源:utils.go

注：本文中的golang.org/x/text/transform.NewReader函数示例由纯净天空整理自Github/MSDocs等源码及文档管理平台，相关代码片段筛选自各路编程大神贡献的开源项目，源码版权归原作者所有，传播和使用请参考对应项目的License；未经允许，请勿转载。