本文整理汇总了Golang中code/google/com/p/snappy-go/snappy.Encode函数的典型用法代码示例。如果您正苦于以下问题:Golang Encode函数的具体用法?Golang Encode怎么用?Golang Encode使用的例子?那么恭喜您, 这里精选的函数代码示例或许可以为您提供帮助。
在下文中一共展示了Encode函数的20个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于我们的系统推荐出更棒的Golang代码示例。
示例1: snappify
func snappify(b []byte) ([]byte, error) {
b[4] |= (2 << 4) // set the document type to '2' (incr snappy)
optHeaderLength, optHeaderSize := varintdecode(b[5:])
optHeaderLength += optHeaderSize
// XXX this could be more efficient! I'm creating a new buffer to
// store the compressed document, which isn't necessary. You
// could probably write directly to the slice after the header
// and after the varint holding the length
compressed, err := snappy.Encode(nil, b[5+optHeaderLength:])
if err != nil {
return nil, err
}
// XXX I'm sure that this could be using a slice of b rather than nil
// so we don't need to copy, but my Go-fu is too low to do it.
compressedLength := varint(nil, uint(len(compressed)))
copy(b[5+optHeaderLength:], compressedLength)
bytesCopied := copy(b[5+optHeaderLength+len(compressedLength):], compressed)
// XXX should we verify that bytesCopied == len(compressed)?
return b[0:bytesCopied], nil
}
开发者ID:hoelzro,项目名称:Sereal,代码行数:25,代码来源:pack.go
示例2: _send
func (cn *connection) _send(opcode byte, body []byte, compression bool) error {
if cn.c == nil {
return driver.ErrBadConn
}
var flags byte = 0x00
if compression {
var err error
body, err = snappy.Encode(nil, body)
if err != nil {
return err
}
flags = flagCompressed
}
frame := make([]byte, len(body)+8)
frame[0] = protoRequest
frame[1] = flags
frame[2] = 0
frame[3] = opcode
binary.BigEndian.PutUint32(frame[4:8], uint32(len(body)))
copy(frame[8:], body)
if _, err := cn.c.Write(frame); err != nil {
return err
}
return nil
}
开发者ID:vuleetu,项目名称:gocql,代码行数:25,代码来源:gocql.go
示例3: Forward
func (this *SnappyCodec) Forward(src, dst []byte) (uint, uint, error) {
if src == nil {
return uint(0), uint(0), errors.New("Invalid null source buffer")
}
if dst == nil {
return uint(0), uint(0), errors.New("Invalid null destination buffer")
}
if kanzi.SameByteSlices(src, dst, false) {
return 0, 0, errors.New("Input and output buffers cannot be equal")
}
count := this.size
if this.size == 0 {
count = uint(len(src))
}
if n := snappy.MaxEncodedLen(int(count)); len(dst) < n {
return 0, 0, fmt.Errorf("Output buffer is too small - size: %d, required %d", len(dst), n)
}
res, err := snappy.Encode(dst, src[0:count])
if err != nil {
return 0, 0, fmt.Errorf("Encoding error: %v", err)
}
return count, uint(len(res)), nil
}
开发者ID:lygstate,项目名称:kanzi,代码行数:31,代码来源:SnappyCodec.go
示例4: TestSnappyCompressor
func TestSnappyCompressor(t *testing.T) {
c := SnappyCompressor{}
if c.Name() != "snappy" {
t.Fatalf("expected name to be 'snappy', got %v", c.Name())
}
str := "My Test String"
//Test Encoding
if expected, err := snappy.Encode(nil, []byte(str)); err != nil {
t.Fatalf("failed to encode '%v' with error %v", str, err)
} else if res, err := c.Encode([]byte(str)); err != nil {
t.Fatalf("failed to encode '%v' with error %v", str, err)
} else if bytes.Compare(expected, res) != 0 {
t.Fatal("failed to match the expected encoded value with the result encoded value.")
}
val, err := c.Encode([]byte(str))
if err != nil {
t.Fatalf("failed to encode '%v' with error '%v'", str, err)
}
//Test Decoding
if expected, err := snappy.Decode(nil, val); err != nil {
t.Fatalf("failed to decode '%v' with error %v", val, err)
} else if res, err := c.Decode(val); err != nil {
t.Fatalf("failed to decode '%v' with error %v", val, err)
} else if bytes.Compare(expected, res) != 0 {
t.Fatal("failed to match the expected decoded value with the result decoded value.")
}
}
开发者ID:simudream,项目名称:resourced-master,代码行数:30,代码来源:compressor_test.go
示例5: compressedChunkGreaterN
// compressedChunkGreaterN like compressedChunk produces a single, compressed,
// snappy-framed block. The returned block will have decoded length at most n
// and encoded length greater than n.
func compressedChunkGreaterN(t *testing.T, n int) []byte {
decoded := make([]byte, n)
var numTries int
var encoded []byte
for len(encoded) <= n && numTries < 3 {
numTries++
nrd, err := io.ReadFull(rand.Reader, decoded)
if err != nil {
t.Errorf("crypto/rand: %v", err)
return nil
}
if nrd != n {
t.Errorf("crypto/rand: bad read (%d bytes)", nrd)
return nil
}
encoded, err = snappy.Encode(encoded[:cap(encoded)], decoded)
if err != nil {
t.Errorf("snappy: %v", err)
return nil
}
}
if len(encoded) <= n {
t.Error(errNotEnoughEntropy)
return nil
}
return compressedChunk(t, decoded)
}
开发者ID:hailocab,项目名称:go-snappystream,代码行数:31,代码来源:reader_test.go
示例6: Compress
//*********COMMPRESSION*************************
func Compress(src []byte) ([]byte, bool) {
dst, err := snappy.Encode(nil, src)
if err != nil {
return nil, false
}
return dst, true
}
开发者ID:KunBetter,项目名称:GridSearch,代码行数:8,代码来源:Utils.go
示例7: compressor
func compressor(fw *Writer, toCompress <-chan *writerBlock, toWrite chan<- *writerBlock) {
switch fw.CompressionCodec {
case CompressionDeflate:
bb := new(bytes.Buffer)
comp, _ := flate.NewWriter(bb, flate.DefaultCompression)
for block := range toCompress {
_, block.err = comp.Write(block.encoded.Bytes())
block.err = comp.Close()
if block.err == nil {
block.compressed = bb.Bytes()
toWrite <- block
}
bb = new(bytes.Buffer)
comp.Reset(bb)
}
case CompressionNull:
for block := range toCompress {
block.compressed = block.encoded.Bytes()
toWrite <- block
}
case CompressionSnappy:
for block := range toCompress {
block.compressed, block.err = snappy.Encode(block.compressed, block.encoded.Bytes())
if block.err != nil {
block.err = fmt.Errorf("cannot compress: %v", block.err)
}
toWrite <- block
}
}
close(toWrite)
}
开发者ID:ahawks,项目名称:goavro,代码行数:31,代码来源:ocf_writer.go
示例8: store
func store(b []byte) string {
encoded := make([]byte, len(b))
encoded, err := snappy.Encode(encoded, b)
if err != nil {
log.Fatal(err)
}
return fmt.Sprintf("%x", sha1.Sum(encoded))
}
开发者ID:pombredanne,项目名称:snp,代码行数:8,代码来源:snp.go
示例9: main
func main() {
f := file{Name: "", Dir: false}
f.Name = opt.Args[0]
if f.Name != "" {
//Determine if fname is a file or directory
fi, err := os.Stat(f.Name)
if err != nil {
fmt.Printf("%s does not exit!\n", f.Name)
os.Exit(2)
}
mode := fi.Mode()
if mode&os.ModeSymlink == os.ModeSymlink {
fmt.Printf("%s is a symbolic link\n", f.Name)
os.Exit(2)
}
if fi.IsDir() {
f.Dir = true
}
oldfile, err := os.Open(f.Name)
if err != nil {
fmt.Printf("Error: %s\n", err.Error())
os.Exit(2)
}
buffer := make([]byte, 100)
//for n, e := oldfile.Read(buffer); e == nil; n, e = oldfile.Read(buffer) {
// if n > 0 {
// os.Stdout.Write(buffer[0:n])
// }
//}
_, err = oldfile.Read(buffer)
if err != nil {
fmt.Printf("Error reading file %s: %s", f.Name, err)
os.Exit(2)
}
newfile := make([]byte, 100)
_, err = snappy.Encode(buffer, newfile)
if err != nil {
fmt.Printf("Errer compressing file: %s", f.Name)
os.Exit(2)
}
f.EncName = f.Name + ".sz"
export, err := os.OpenFile(f.EncName, os.O_WRONLY|os.O_CREATE, 0644) //Should keep
//original perms
export.Write(newfile)
export.Close()
os.Exit(0)
} else {
//Should never get here but just in-case
fmt.Println("No arguments given!")
os.Exit(2)
}
}
开发者ID:vendion,项目名称:gsnappy,代码行数:58,代码来源:gsnappy.go
示例10: GetSnappyBlock
func (d *DFS) GetSnappyBlock(blockSize int, reply *DataChunk) error {
err := d.GetBlock(blockSize, reply)
handleError(err)
//
reply.Chunk, err = snappy.Encode(reply.Chunk, reply.Chunk)
handleError(err)
//
return nil
}
开发者ID:BradMcDanel,项目名称:gorpc-tests,代码行数:9,代码来源:dfs.go
示例11: saveBlock
func (w *Writer) saveBlock() error {
// Calculate hash of uncompressed data for ref.
ref := calculateRef(w.h, w.buf[:w.n])
//TODO check if this block exists on disk.
if blockExistsOnDisk(ref) {
// Append ref to list.
w.refs = append(w.refs, ref)
w.n = 0
w.blockCount++
return nil
}
// Compress.
compressedData, err := snappy.Encode(w.cdata[headerSize:], w.buf[:w.n])
if err != nil {
return err
}
dataLen := headerSize + len(compressedData)
// Pad with zeroes so that the encrypted box is multiple of PadSize.
var paddedLen int
if dataLen == 0 {
paddedLen = PadSize - nonceSize - secretbox.Overhead
} else {
paddedLen = (((dataLen + nonceSize + secretbox.Overhead) + (PadSize - 1)) / PadSize) * PadSize
paddedLen -= nonceSize + secretbox.Overhead
}
for i := dataLen; i < paddedLen; i++ {
w.cdata[i] = 0
}
plainBlock := w.cdata[:paddedLen]
// Set block kind.
plainBlock[0] = w.kind
// Store compressed length.
binary.BigEndian.PutUint32(plainBlock[1:], uint32(len(compressedData)))
// Encrypt.
var nonce [24]byte
if err := generateNonce(&nonce); err != nil {
return err
}
//TODO avoid allocation
fullBox := make([]byte, len(nonce)+len(plainBlock)+secretbox.Overhead)
copy(fullBox, nonce[:])
secretbox.Seal(fullBox[len(nonce):len(nonce)], plainBlock, &nonce, &config.Keys.BlockEnc)
// Save to disk.
if err := writeBlockToDisk(ref, fullBox); err != nil {
return err
}
// Append ref to list.
w.refs = append(w.refs, ref)
w.n = 0
w.blockCount++
return nil
}
开发者ID:dchest,项目名称:hesfic,代码行数:57,代码来源:block.go
示例12: writeResponse
func writeResponse(w io.Writer, id uint64, serr string, response proto.Message) (err error) {
// clean response if error
if serr != "" {
response = nil
}
// marshal response
pbResponse := []byte{}
if response != nil {
pbResponse, err = proto.Marshal(response)
if err != nil {
return err
}
}
// compress serialized proto data
compressedPbResponse, err := snappy.Encode(nil, pbResponse)
if err != nil {
return err
}
// generate header
header := &wire.ResponseHeader{
Id: id,
Error: serr,
RawResponseLen: uint32(len(pbResponse)),
SnappyCompressedResponseLen: uint32(len(compressedPbResponse)),
Checksum: crc32.ChecksumIEEE(compressedPbResponse),
}
// check header size
pbHeader, err := proto.Marshal(header)
if err != err {
return
}
if uint32(len(pbHeader)) > wire.Default_Const_MaxHeaderLen {
return fmt.Errorf("protorpc.writeResponse: header larger than max_header_len: %d.",
len(pbHeader),
)
}
// send header (more)
if err = sendFrame(w, pbHeader); err != nil {
return
}
// send body (end)
if err = sendFrame(w, compressedPbResponse); err != nil {
return
}
return nil
}
开发者ID:josephwinston,项目名称:cockroach,代码行数:53,代码来源:wire.go
示例13: NewSnappyEncodeOp
func NewSnappyEncodeOp() stream.Operator {
generator := func() interface{} {
fn := func(in []byte) [][]byte {
compressed, err := snappy.Encode(nil, in)
if err != nil {
log.Printf("Error in snappy compression %v", err)
}
return [][]byte{compressed}
}
return fn
}
return mapper.NewOpFactory(generator, "NewSnappyEncodeOp")
}
开发者ID:adriansoghoian,项目名称:go-stream,代码行数:13,代码来源:snappy.go
示例14: finishBlock
// finishBlock finishes the current block and returns its block handle, which is
// its offset and length in the table.
func (w *Writer) finishBlock() (blockHandle, error) {
// Write the restart points to the buffer.
if w.nEntries == 0 {
// Every block must have at least one restart point.
w.restarts = w.restarts[:1]
w.restarts[0] = 0
}
tmp4 := w.tmp[:4]
for _, x := range w.restarts {
binary.LittleEndian.PutUint32(tmp4, x)
w.buf.Write(tmp4)
}
binary.LittleEndian.PutUint32(tmp4, uint32(len(w.restarts)))
w.buf.Write(tmp4)
// Compress the buffer, discarding the result if the improvement
// isn't at least 12.5%.
b := w.buf.Bytes()
w.tmp[0] = noCompressionBlockType
if w.compression == db.SnappyCompression {
compressed, err := snappy.Encode(w.compressedBuf, b)
if err != nil {
return blockHandle{}, err
}
w.compressedBuf = compressed[:cap(compressed)]
if len(compressed) < len(b)-len(b)/8 {
w.tmp[0] = snappyCompressionBlockType
b = compressed
}
}
// Calculate the checksum.
checksum := crc.New(b).Update(w.tmp[:1]).Value()
binary.LittleEndian.PutUint32(w.tmp[1:5], checksum)
// Write the bytes to the file.
if _, err := w.writer.Write(b); err != nil {
return blockHandle{}, err
}
if _, err := w.writer.Write(w.tmp[:5]); err != nil {
return blockHandle{}, err
}
bh := blockHandle{w.offset, uint64(len(b))}
w.offset += uint64(len(b)) + blockTrailerLen
// Reset the per-block state.
w.buf.Reset()
w.nEntries = 0
w.restarts = w.restarts[:0]
return bh, nil
}
开发者ID:xushiwei,项目名称:leveldb-go,代码行数:53,代码来源:writer.go
示例15: write
// write attempts to encode p as a block and write it to the underlying writer.
// The returned int may not equal p's length if compression below
// MaxBlockSize-4 could not be achieved.
func (w *writer) write(p []byte) (int, error) {
var err error
if len(p) > MaxBlockSize {
return 0, errors.New(fmt.Sprintf("block too large %d > %d", len(p), MaxBlockSize))
}
w.dst = w.dst[:cap(w.dst)] // Encode does dumb resize w/o context. reslice avoids alloc.
w.dst, err = snappy.Encode(w.dst, p)
if err != nil {
return 0, err
}
block := w.dst
n := len(p)
compressed := true
// check for data which is better left uncompressed. this is determined if
// the encoded content is longer than the source.
if len(w.dst) >= len(p) {
compressed = false
block = p[:n]
}
if !w.sentStreamID {
_, err := w.writer.Write(streamID)
if err != nil {
return 0, err
}
w.sentStreamID = true
}
// set the block type
if compressed {
writeHeader(w.hdr, blockCompressed, block, p[:n])
} else {
writeHeader(w.hdr, blockUncompressed, block, p[:n])
}
_, err = w.writer.Write(w.hdr)
if err != nil {
return 0, err
}
_, err = w.writer.Write(block)
if err != nil {
return 0, err
}
return n, nil
}
开发者ID:carriercomm,项目名称:api-1,代码行数:53,代码来源:writer.go
示例16: write
func (w *writer) write(p []byte) (int, error) {
var err error
if len(p) > MaxBlockSize {
return 0, errors.New(fmt.Sprintf("block too large %d > %d", len(p), MaxBlockSize))
}
w.dst, err = snappy.Encode(w.dst, p)
if err != nil {
return 0, err
}
if !w.sentStreamID {
_, err := w.writer.Write(streamID)
if err != nil {
return 0, err
}
w.sentStreamID = true
}
length := uint32(len(w.dst)) + 4 // +4 for checksum
w.hdr[0] = 0x00 // compressed frame ID
// 3 byte little endian length
w.hdr[1] = byte(length)
w.hdr[2] = byte(length >> 8)
w.hdr[3] = byte(length >> 16)
// 4 byte little endian CRC32 checksum
checksum := maskChecksum(crc32.Checksum(p, crcTable))
w.hdr[4] = byte(checksum)
w.hdr[5] = byte(checksum >> 8)
w.hdr[6] = byte(checksum >> 16)
w.hdr[7] = byte(checksum >> 24)
_, err = w.writer.Write(w.hdr)
if err != nil {
return 0, err
}
_, err = w.writer.Write(w.dst)
if err != nil {
return 0, err
}
return len(p), nil
}
开发者ID:nulpunt,项目名称:nulpunt,代码行数:48,代码来源:writer.go
示例17: encode
func (m *Message) encode(pe packetEncoder) error {
pe.push(&crc32Field{})
pe.putInt8(messageFormat)
attributes := int8(m.Codec) & compressionCodecMask
pe.putInt8(attributes)
err := pe.putBytes(m.Key)
if err != nil {
return err
}
var payload []byte
if m.compressedCache != nil {
payload = m.compressedCache
m.compressedCache = nil
} else {
switch m.Codec {
case CompressionNone:
payload = m.Value
case CompressionGZIP:
var buf bytes.Buffer
writer := gzip.NewWriter(&buf)
writer.Write(m.Value)
writer.Close()
m.compressedCache = buf.Bytes()
payload = m.compressedCache
case CompressionSnappy:
tmp, err := snappy.Encode(nil, m.Value)
if err != nil {
return err
}
m.compressedCache = tmp
payload = m.compressedCache
default:
return EncodingError
}
}
err = pe.putBytes(payload)
if err != nil {
return err
}
return pe.pop()
}
开发者ID:senseobservationsystems,项目名称:sarama,代码行数:48,代码来源:message.go
示例18: Put
// Put adds the package documentation to the database.
func (db *Database) Put(pdoc *doc.Package, nextCrawl time.Time) error {
c := db.Pool.Get()
defer c.Close()
score := documentScore(pdoc)
terms := documentTerms(pdoc, score)
var gobBuf bytes.Buffer
if err := gob.NewEncoder(&gobBuf).Encode(pdoc); err != nil {
return err
}
// Truncate large documents.
if gobBuf.Len() > 700000 {
pdocNew := *pdoc
pdoc = &pdocNew
pdoc.Truncated = true
pdoc.Vars = nil
pdoc.Funcs = nil
pdoc.Types = nil
pdoc.Consts = nil
pdoc.Examples = nil
gobBuf.Reset()
if err := gob.NewEncoder(&gobBuf).Encode(pdoc); err != nil {
return err
}
}
gobBytes, err := snappy.Encode(nil, gobBuf.Bytes())
if err != nil {
return err
}
kind := "p"
switch {
case pdoc.Name == "":
kind = "d"
case pdoc.IsCmd:
kind = "c"
}
t := int64(0)
if !nextCrawl.IsZero() {
t = nextCrawl.Unix()
}
_, err = putScript.Do(c, pdoc.ImportPath, pdoc.Synopsis, score, gobBytes, strings.Join(terms, " "), pdoc.Etag, kind, t)
return err
}
开发者ID:AlekSi,项目名称:gddo,代码行数:49,代码来源:database.go
示例19: writeRequest
func writeRequest(w io.Writer, id uint64, method string, request proto.Message) error {
// marshal request
pbRequest := []byte{}
if request != nil {
var err error
pbRequest, err = proto.Marshal(request)
if err != nil {
return err
}
}
// compress serialized proto data
compressedPbRequest, err := snappy.Encode(nil, pbRequest)
if err != nil {
return err
}
// generate header
header := &wire.RequestHeader{
Id: id,
Method: method,
RawRequestLen: uint32(len(pbRequest)),
SnappyCompressedRequestLen: uint32(len(compressedPbRequest)),
Checksum: crc32.ChecksumIEEE(compressedPbRequest),
}
// check header size
pbHeader, err := proto.Marshal(header)
if err != err {
return err
}
if uint32(len(pbHeader)) > wire.Default_Const_MaxHeaderLen {
return fmt.Errorf("protorpc.writeRequest: header larger than max_header_len: %d.", len(pbHeader))
}
// send header (more)
if err := sendFrame(w, pbHeader); err != nil {
return err
}
// send body (end)
if err := sendFrame(w, compressedPbRequest); err != nil {
return err
}
return nil
}
开发者ID:josephwinston,项目名称:cockroach,代码行数:47,代码来源:wire.go
示例20: KVWrite
func KVWrite(db *rpcplus.Client, table, uid string, value interface{}) (result bool, err error) {
//ts("KVWrite", table, uid)
//defer te("KVWrite", table, uid)
var buf []byte
switch value.(type) {
case gp.Message:
buf, err = gp.Marshal(value.(gp.Message))
if err != nil {
logger.Error("KVWrite Error On Marshal %s : %s (%s)", table, uid, err.Error())
return
}
case []byte:
buf = value.([]byte)
default:
logger.Error("KVWrite args type error %v", value)
return
}
dst, err := snappy.Encode(nil, buf)
if err != nil {
logger.Error("KVWrite Error On snappy.Encode %s : %s (%s)", table, uid, err.Error())
return
}
var reply protobuf.DBWriteResult
err = db.Call("DBServer.Write", protobuf.DBWrite{table, uid, dst}, &reply)
if err != nil {
logger.Error("KVWrite Error On Create %s: %s (%s)", table, uid, err.Error())
return
}
if reply.Code != protobuf.Ok {
logger.Error("KVWrite Error On Create %s: %s code (%d)", table, uid, reply.Code)
return
}
result = true
return
}
开发者ID:bodogbo,项目名称:OSG-Server-Go,代码行数:47,代码来源:kv2db.go
注:本文中的code/google/com/p/snappy-go/snappy.Encode函数示例整理自Github/MSDocs等源码及文档管理平台,相关代码片段筛选自各路编程大神贡献的开源项目,源码版权归原作者所有,传播和使用请参考对应项目的License;未经允许,请勿转载。 |
请发表评论