func TestOldUnoM(t *testing.T) {
	popr := math_rand.New(math_rand.NewSource(time.Now().UnixNano()))
	older := NewPopulatedOldUnoM(popr, true)
	// need optional field to be always initialized, to check it's lost in this test
	older.Field1 = proto.String(randStringUnrecognized(popr))
	data1, err := proto.Marshal(older)
	if err != nil {
		panic(err)
	}

	newer := &UnoM{}
	if err = proto.Unmarshal(data1, newer); err != nil {
		panic(err)
	}
	data2, err := proto.Marshal(newer)
	if err != nil {
		panic(err)
	}

	older2 := &OldUnoM{}
	if err := proto.Unmarshal(data2, older2); err != nil {
		panic(err)
	}

	// check that Field1 is lost
	if older2.Field1 != nil {
		t.Fatalf("field must be lost, but it's not, older: %#v, older2: %#v", older, older2)
	}

	// now restore Field1 and messages should be equal now
	older2.Field1 = older.Field1
	if err := older.VerboseEqual(older2); err != nil {
		t.Fatalf("%#v !VerboseProto %#v, since %v", older, older2, err)
	}
}

func TestFloatingPointProto(t *testing.T) {
	seed := time.Now().UnixNano()
	popr := math_rand.New(math_rand.NewSource(seed))
	p := NewPopulatedFloatingPoint(popr, false)
	dAtA, err := github_com_gogo_protobuf_proto.Marshal(p)
	if err != nil {
		t.Fatalf("seed = %d, err = %v", seed, err)
	}
	msg := &FloatingPoint{}
	if err := github_com_gogo_protobuf_proto.Unmarshal(dAtA, msg); err != nil {
		t.Fatalf("seed = %d, err = %v", seed, err)
	}
	littlefuzz := make([]byte, len(dAtA))
	copy(littlefuzz, dAtA)
	for i := range dAtA {
		dAtA[i] = byte(popr.Intn(256))
	}
	if err := p.VerboseEqual(msg); err != nil {
		t.Fatalf("seed = %d, %#v !VerboseProto %#v, since %v", seed, msg, p, err)
	}
	if !p.Equal(msg) {
		t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p)
	}
	if len(littlefuzz) > 0 {
		fuzzamount := 100
		for i := 0; i < fuzzamount; i++ {
			littlefuzz[popr.Intn(len(littlefuzz))] = byte(popr.Intn(256))
			littlefuzz = append(littlefuzz, byte(popr.Intn(256)))
		}
		// shouldn't panic
		_ = github_com_gogo_protobuf_proto.Unmarshal(littlefuzz, msg)
	}
}

func (c codec) Decode(data []byte) (runtime.Object, error) {
	unknown := &runtime.Unknown{}
	if err := proto.Unmarshal(data, unknown); err != nil {
		return nil, err
	}
	obj, err := c.creater.New(unknown.APIVersion, unknown.Kind)
	if err != nil {
		return nil, err
	}
	pobj, ok := obj.(proto.Message)
	if !ok {
		return nil, fmt.Errorf("runtime object is not a proto.Message: %v", reflect.TypeOf(obj))
	}
	if err := proto.Unmarshal(unknown.RawJSON, pobj); err != nil {
		return nil, err
	}
	if unknown.APIVersion != c.outputVersion {
		out, err := c.convertor.ConvertToVersion(obj, c.outputVersion)
		if err != nil {
			return nil, err
		}
		obj = out
	}
	return obj, nil
}

func (o *OortFS) Lookup(ctx context.Context, parent []byte, name string) (string, *pb.Attr, error) {
	// Get the id
	b, err := o.comms.ReadGroupItem(ctx, parent, []byte(name))
	if store.IsNotFound(err) {
		return "", &pb.Attr{}, nil
	} else if err != nil {
		return "", &pb.Attr{}, err
	}
	d := &pb.DirEntry{}
	err = proto.Unmarshal(b, d)
	if err != nil {
		return "", &pb.Attr{}, err
	}
	if d.Tombstone != nil {
		return "", &pb.Attr{}, nil
	}
	// Get the Inode entry
	b, err = o.GetChunk(ctx, d.Id)
	if err != nil {
		return "", &pb.Attr{}, err
	}
	n := &pb.InodeEntry{}
	err = proto.Unmarshal(b, n)
	if err != nil {
		return "", &pb.Attr{}, err
	}
	return d.Name, n.Attr, nil
}

func TestMessageProto(t *testing.T) {
	popr := math_rand.New(math_rand.NewSource(time.Now().UnixNano()))
	p := NewPopulatedMessage(popr, false)
	data, err := github_com_gogo_protobuf_proto.Marshal(p)
	if err != nil {
		panic(err)
	}
	msg := &Message{}
	if err := github_com_gogo_protobuf_proto.Unmarshal(data, msg); err != nil {
		panic(err)
	}
	littlefuzz := make([]byte, len(data))
	copy(littlefuzz, data)
	for i := range data {
		data[i] = byte(popr.Intn(256))
	}
	if err := p.VerboseEqual(msg); err != nil {
		t.Fatalf("%#v !VerboseProto %#v, since %v", msg, p, err)
	}
	if !p.Equal(msg) {
		t.Fatalf("%#v !Proto %#v", msg, p)
	}
	if len(littlefuzz) > 0 {
		fuzzamount := 100
		for i := 0; i < fuzzamount; i++ {
			littlefuzz[popr.Intn(len(littlefuzz))] = byte(popr.Intn(256))
			littlefuzz = append(littlefuzz, byte(popr.Intn(256)))
		}
		// shouldn't panic
		_ = github_com_gogo_protobuf_proto.Unmarshal(littlefuzz, msg)
	}
}

func TestIndexQueryProto(t *testing.T) {
	seed := time.Now().UnixNano()
	popr := math_rand.New(math_rand.NewSource(seed))
	p := NewPopulatedIndexQuery(popr, false)
	data, err := github_com_gogo_protobuf_proto.Marshal(p)
	if err != nil {
		t.Fatalf("seed = %d, err = %v", seed, err)
	}
	msg := &IndexQuery{}
	if err := github_com_gogo_protobuf_proto.Unmarshal(data, msg); err != nil {
		t.Fatalf("seed = %d, err = %v", seed, err)
	}
	littlefuzz := make([]byte, len(data))
	copy(littlefuzz, data)
	for i := range data {
		data[i] = byte(popr.Intn(256))
	}
	if !p.Equal(msg) {
		t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p)
	}
	if len(littlefuzz) > 0 {
		fuzzamount := 100
		for i := 0; i < fuzzamount; i++ {
			littlefuzz[popr.Intn(len(littlefuzz))] = byte(popr.Intn(256))
			littlefuzz = append(littlefuzz, byte(popr.Intn(256)))
		}
		// shouldn't panic
		_ = github_com_gogo_protobuf_proto.Unmarshal(littlefuzz, msg)
	}
}

func (r *rpc) executeMessage(conn net.Conn, rpcType internal.RPCType, buf []byte) (internal.RPCType, proto.Message, error) {
	switch rpcType {
	case internal.RPCType_FetchData:
		var req internal.FetchDataRequest
		if err := proto.Unmarshal(buf, &req); err != nil {
			return internal.RPCType_Error, nil, fmt.Errorf("fetch request unmarshal: %v", err)
		}
		resp, err := r.handleFetchData(&req)
		return rpcType, resp, err
	case internal.RPCType_Join:
		var req internal.JoinRequest
		if err := proto.Unmarshal(buf, &req); err != nil {
			return internal.RPCType_Error, nil, fmt.Errorf("join request unmarshal: %v", err)
		}
		resp, err := r.handleJoinRequest(&req)
		return rpcType, resp, err
	case internal.RPCType_PromoteRaft:
		var req internal.PromoteRaftRequest
		if err := proto.Unmarshal(buf, &req); err != nil {
			return internal.RPCType_Error, nil, fmt.Errorf("promote to raft request unmarshal: %v", err)
		}
		resp, err := r.handlePromoteRaftRequest(&req)
		return rpcType, resp, err
	default:
		return internal.RPCType_Error, nil, fmt.Errorf("unknown rpc type:%v", rpcType)
	}
}

func (c codec) DecodeInto(data []byte, obj runtime.Object) error {
	version, kind, err := c.typer.ObjectVersionAndKind(obj)
	if err != nil {
		return err
	}
	unknown := &runtime.Unknown{}
	if err := proto.Unmarshal(data, unknown); err != nil {
		return err
	}
	if unknown.APIVersion == version && unknown.Kind == kind {
		pobj, ok := obj.(proto.Message)
		if !ok {
			return fmt.Errorf("runtime object is not a proto.Message: %v", reflect.TypeOf(obj))
		}

		return proto.Unmarshal(unknown.RawJSON, pobj)
	}

	versioned, err := c.creater.New(unknown.APIVersion, unknown.Kind)
	if err != nil {
		return err
	}

	pobj, ok := versioned.(proto.Message)
	if !ok {
		return fmt.Errorf("runtime object is not a proto.Message: %v", reflect.TypeOf(obj))
	}

	if err := proto.Unmarshal(unknown.RawJSON, pobj); err != nil {
		return err
	}
	return c.convertor.Convert(versioned, obj)
}

func (e *envelopeReceiver) Match(actual interface{}) (success bool, err error) {
	envelope := &events.Envelope{}
	input, ok := actual.(chan []byte)
	if !ok {
		return false, errors.New("Envelope receiver: expected a channel of byte slices")
	}
	var msgBytes []byte
	select {
	case msgBytes = <-input:
	default:
		return false, nil
	}

	if err := proto.Unmarshal(msgBytes, envelope); err != nil {
		// Try again, stripping out the length prefix
		if err := proto.Unmarshal(msgBytes[4:], envelope); err != nil {
			return false, nil
		}
		e.hadPrefix = true
	}
	if e.mustHavePrefix && !e.hadPrefix {
		return false, nil
	}
	e.foundEnvelope = envelope
	if e.matcher != nil {
		return e.matcher.Match(envelope)
	}
	return true, nil
}

// TestEngineMerge tests that the passing through of engine merge operations
// to the goMerge function works as expected. The semantics are tested more
// exhaustively in the merge tests themselves.
func TestEngineMerge(t *testing.T) {
	runWithAllEngines(func(engine Engine, t *testing.T) {
		testcases := []struct {
			testKey  proto.EncodedKey
			merges   [][]byte
			expected []byte
		}{
			{
				proto.EncodedKey("haste not in life"),
				[][]byte{
					appender("x"),
					appender("y"),
					appender("z"),
				},
				appender("xyz"),
			},
			{
				proto.EncodedKey("timeseriesmerged"),
				[][]byte{
					timeSeriesInt(testtime, 1000, []tsIntSample{
						{1, 1, 5, 5, 5},
					}...),
					timeSeriesInt(testtime, 1000, []tsIntSample{
						{2, 1, 5, 5, 5},
						{1, 2, 10, 7, 3},
					}...),
					timeSeriesInt(testtime, 1000, []tsIntSample{
						{10, 1, 5, 5, 5},
					}...),
					timeSeriesInt(testtime, 1000, []tsIntSample{
						{5, 1, 5, 5, 5},
						{3, 1, 5, 5, 5},
					}...),
				},
				timeSeriesInt(testtime, 1000, []tsIntSample{
					{1, 3, 15, 7, 3},
					{2, 1, 5, 5, 5},
					{3, 1, 5, 5, 5},
					{5, 1, 5, 5, 5},
					{10, 1, 5, 5, 5},
				}...),
			},
		}
		for _, tc := range testcases {
			for i, update := range tc.merges {
				if err := engine.Merge(tc.testKey, update); err != nil {
					t.Fatalf("%d: %v", i, err)
				}
			}
			result, _ := engine.Get(tc.testKey)
			var resultV, expectedV proto.MVCCMetadata
			gogoproto.Unmarshal(result, &resultV)
			gogoproto.Unmarshal(tc.expected, &expectedV)
			if !reflect.DeepEqual(resultV, expectedV) {
				t.Errorf("unexpected append-merge result: %v != %v", resultV, expectedV)
			}
		}
	}, t)
}

func compareMergedValues(t *testing.T, result, expected []byte) bool {
	var resultV, expectedV MVCCMetadata
	if err := proto.Unmarshal(result, &resultV); err != nil {
		t.Fatal(err)
	}
	if err := proto.Unmarshal(expected, &expectedV); err != nil {
		t.Fatal(err)
	}
	return reflect.DeepEqual(resultV, expectedV)
}

func (o *OortFS) ReadDirAll(ctx context.Context, id []byte) (*pb.ReadDirAllResponse, error) {
	v, err := o.validateIP(ctx)
	if err != nil {
		return nil, err
	}
	if !v {
		return nil, errors.New("Unknown or unauthorized FS use")
	}
	// Get the keys from the group
	items, err := o.comms.LookupGroup(ctx, id)
	log.Println("ITEMS: ", items)
	if err != nil {
		// TODO: Needs beter error handling
		log.Println("Error looking up group: ", err)
		return &pb.ReadDirAllResponse{}, err
	}
	// Iterate over each item, getting the ID then the Inode Entry
	e := &pb.ReadDirAllResponse{}
	dirent := &pb.DirEntry{}
	for _, item := range items {
		// lookup the item in the group to get the id
		b, err := o.comms.ReadGroupItemByKey(ctx, id, item.ChildKeyA, item.ChildKeyB)
		if err != nil {
			// TODO: Needs beter error handling
			log.Println("Error with lookup: ", err)
			continue
		}
		err = proto.Unmarshal(b, dirent)
		if err != nil {
			return &pb.ReadDirAllResponse{}, err
		}
		if dirent.Tombstone != nil {
			// Skip deleted entries
			continue
		}
		// get the inode entry
		b, err = o.GetChunk(ctx, dirent.Id)
		if err != nil {
			continue
		}
		n := &pb.InodeEntry{}
		err = proto.Unmarshal(b, n)
		if err != nil {
			continue
		}
		if n.IsDir {
			e.DirEntries = append(e.DirEntries, &pb.DirEnt{Name: dirent.Name, Attr: n.Attr})
		} else {
			e.FileEntries = append(e.FileEntries, &pb.DirEnt{Name: dirent.Name, Attr: n.Attr})
		}
	}
	sort.Sort(ByDirent(e.DirEntries))
	sort.Sort(ByDirent(e.FileEntries))
	return e, nil
}

func FromProtobufferMessage(data []byte) (appId string, err error) {
	receivedMessage := &logmessage.LogMessage{}
	err = proto.Unmarshal(data, receivedMessage)
	if err == nil {
		return *receivedMessage.AppId, nil
	}

	receivedEnvelope := &logmessage.LogEnvelope{}
	err = proto.Unmarshal(data, receivedEnvelope)
	if err != nil {
		err = errors.New(fmt.Sprintf("Log data could not be unmarshaled to message or envelope. Dropping it... Error: %v. Data: %v", err, data))
		return "", err
	}
	return *receivedEnvelope.RoutingKey, nil
}

// CopyFrom copies all the cached results from the originRangeID
// response cache into this one. Note that the cache will not be
// locked while copying is in progress. Failures decoding individual
// cache entries return an error. The copy is done directly using the
// engine instead of interpreting values through MVCC for efficiency.
func (rc *ResponseCache) CopyFrom(e engine.Engine, originRangeID proto.RangeID) error {
	prefix := keys.ResponseCacheKey(originRangeID, nil) // response cache prefix
	start := engine.MVCCEncodeKey(prefix)
	end := engine.MVCCEncodeKey(prefix.PrefixEnd())

	return e.Iterate(start, end, func(kv proto.RawKeyValue) (bool, error) {
		// Decode the key into a cmd, skipping on error. Otherwise,
		// write it to the corresponding key in the new cache.
		cmdID, err := rc.decodeResponseCacheKey(kv.Key)
		if err != nil {
			return false, util.Errorf("could not decode a response cache key %s: %s",
				proto.Key(kv.Key), err)
		}
		key := keys.ResponseCacheKey(rc.rangeID, &cmdID)
		encKey := engine.MVCCEncodeKey(key)
		// Decode the value, update the checksum and re-encode.
		meta := &engine.MVCCMetadata{}
		if err := gogoproto.Unmarshal(kv.Value, meta); err != nil {
			return false, util.Errorf("could not decode response cache value %s [% x]: %s",
				proto.Key(kv.Key), kv.Value, err)
		}
		meta.Value.Checksum = nil
		meta.Value.InitChecksum(key)
		_, _, err = engine.PutProto(e, encKey, meta)
		return false, err
	})
}

// Entries implements the raft.Storage interface. Note that maxBytes is advisory
// and this method will always return at least one entry even if it exceeds
// maxBytes. Passing maxBytes equal to zero disables size checking.
// TODO(bdarnell): consider caching for recent entries, if rocksdb's builtin caching
// is insufficient.
func (r *Range) Entries(lo, hi, maxBytes uint64) ([]raftpb.Entry, error) {
	// Scan over the log to find the requested entries in the range [lo, hi),
	// stopping once we have enough.
	var ents []raftpb.Entry
	size := uint64(0)
	var ent raftpb.Entry
	scanFunc := func(kv proto.KeyValue) (bool, error) {
		err := gogoproto.Unmarshal(kv.Value.GetBytes(), &ent)
		if err != nil {
			return false, err
		}
		size += uint64(ent.Size())
		ents = append(ents, ent)
		return maxBytes > 0 && size > maxBytes, nil
	}

	_, err := engine.MVCCIterate(r.rm.Engine(),
		keys.RaftLogKey(r.Desc().RaftID, lo),
		keys.RaftLogKey(r.Desc().RaftID, hi),
		proto.ZeroTimestamp, true /* consistent */, nil /* txn */, scanFunc)

	if err != nil {
		return nil, err
	}

	// If neither the number of entries nor the size limitations had an
	// effect, we weren't able to supply everything the client wanted.
	if len(ents) != int(hi-lo) && (maxBytes == 0 || size < maxBytes) {
		return nil, raft.ErrUnavailable
	}

	return ents, nil
}

// DecodeBooleanPoint reads from the underlying reader and unmarshals into p.
func (dec *BooleanPointDecoder) DecodeBooleanPoint(p *BooleanPoint) error {
	for {
		// Read length.
		var sz uint32
		if err := binary.Read(dec.r, binary.BigEndian, &sz); err != nil {
			return err
		}

		// Read point data.
		buf := make([]byte, sz)
		if _, err := io.ReadFull(dec.r, buf); err != nil {
			return err
		}

		// Unmarshal into point.
		var pb internal.Point
		if err := proto.Unmarshal(buf, &pb); err != nil {
			return err
		}

		// If the point contains stats then read stats and retry.
		if pb.Stats != nil {
			dec.stats = decodeIteratorStats(pb.Stats)
			continue
		}

		// Decode into point object.
		*p = *decodeBooleanPoint(&pb)

		return nil
	}
}

// Since HTTPTransporter.Recv() is already buffered, so we don't need a 'recvLoop' here.
func (m *MesosMessenger) decodeLoop() {
	for {
		select {
		case <-m.stop:
			return
		default:
		}
		msg, err := m.tr.Recv()
		if err != nil {
			if err == discardOnStopError {
				log.V(1).Info("exiting decodeLoop, transport shutting down")
				return
			} else {
				panic(fmt.Sprintf("unexpected transport error: %v", err))
			}
		}
		log.V(2).Infof("Receiving message %v from %v\n", msg.Name, msg.UPID)
		msg.ProtoMessage = reflect.New(m.installedMessages[msg.Name]).Interface().(proto.Message)
		if err := proto.Unmarshal(msg.Bytes, msg.ProtoMessage); err != nil {
			log.Errorf("Failed to unmarshal message %v: %v\n", msg, err)
			continue
		}
		// TODO(yifan): Catch panic.
		m.installedHandlers[msg.Name](msg.UPID, msg.ProtoMessage)
	}
}

func dbGetProto(rdb *C.DBEngine, key MVCCKey,
	msg proto.Message) (ok bool, keyBytes, valBytes int64, err error) {
	if len(key.Key) == 0 {
		err = emptyKeyError()
		return
	}
	var result C.DBString
	if err = statusToError(C.DBGet(rdb, goToCKey(key), &result)); err != nil {
		return
	}
	if result.len <= 0 {
		msg.Reset()
		return
	}
	ok = true
	if msg != nil {
		// Make a byte slice that is backed by result.data. This slice
		// cannot live past the lifetime of this method, but we're only
		// using it to unmarshal the roachpb.
		data := cSliceToUnsafeGoBytes(C.DBSlice(result))
		err = proto.Unmarshal(data, msg)
	}
	C.free(unsafe.Pointer(result.data))
	keyBytes = int64(key.EncodedSize())
	valBytes = int64(result.len)
	return
}

func TestFloatingPointMarshalTo(t *testing.T) {
	popr := math_rand.New(math_rand.NewSource(time.Now().UnixNano()))
	p := NewPopulatedFloatingPoint(popr, false)
	size := p.Size()
	data := make([]byte, size)
	for i := range data {
		data[i] = byte(popr.Intn(256))
	}
	_, err := p.MarshalTo(data)
	if err != nil {
		panic(err)
	}
	msg := &FloatingPoint{}
	if err := github_com_gogo_protobuf_proto.Unmarshal(data, msg); err != nil {
		panic(err)
	}
	for i := range data {
		data[i] = byte(popr.Intn(256))
	}
	if err := p.VerboseEqual(msg); err != nil {
		t.Fatalf("%#v !VerboseProto %#v, since %v", msg, p, err)
	}
	if !p.Equal(msg) {
		t.Fatalf("%#v !Proto %#v", msg, p)
	}
}

func Generate(req *plugin.CodeGeneratorRequest) *plugin.CodeGeneratorResponse {
	// Begin by allocating a generator. The request and response structures are stored there
	// so we can do error handling easily - the response structure contains the field to
	// report failure.
	g := generator.New()
	g.Request = req

	g.CommandLineParameters(g.Request.GetParameter())

	// Create a wrapped version of the Descriptors and EnumDescriptors that
	// point to the file that defines them.
	g.WrapTypes()

	g.SetPackageNames()
	g.BuildTypeNameMap()

	g.GenerateAllFiles()

	gtest := generator.New()

	data, err := proto.Marshal(req)
	if err != nil {
		g.Error(err, "failed to marshal modified proto")
	}
	if err := proto.Unmarshal(data, gtest.Request); err != nil {
		g.Error(err, "parsing modified proto")
	}

	if len(gtest.Request.FileToGenerate) == 0 {
		gtest.Fail("no files to generate")
	}

	gtest.CommandLineParameters(gtest.Request.GetParameter())

	// Create a wrapped version of the Descriptors and EnumDescriptors that
	// point to the file that defines them.
	gtest.WrapTypes()

	gtest.SetPackageNames()
	gtest.BuildTypeNameMap()

	gtest.GeneratePlugin(testgen.NewPlugin())

	for i := 0; i < len(gtest.Response.File); i++ {
		if strings.Contains(*gtest.Response.File[i].Content, `//These tests are generated by github.com/gogo/protobuf/plugin/testgen`) {
			gtest.Response.File[i].Name = proto.String(strings.Replace(*gtest.Response.File[i].Name, ".pb.go", "pb_test.go", -1))
			g.Response.File = append(g.Response.File, gtest.Response.File[i])
		}
	}

	for i := 0; i < len(g.Response.File); i++ {
		formatted, err := format.Source([]byte(g.Response.File[i].GetContent()))
		if err != nil {
			g.Error(err, "go format error")
		}
		fmts := string(formatted)
		g.Response.File[i].Content = &fmts
	}
	return g.Response
}