func TestValidatePrevTimestamp(t *testing.T) {
	gun := "docker.com/notary"
	repo, cs, err := testutils.EmptyRepo(gun)
	require.NoError(t, err)

	r, tg, sn, ts, err := testutils.Sign(repo)
	require.NoError(t, err)
	root, targets, snapshot, timestamp, err := getUpdates(r, tg, sn, ts)
	require.NoError(t, err)

	updates := []storage.MetaUpdate{root, targets, snapshot}

	store := storage.NewMemStorage()
	store.UpdateCurrent(gun, timestamp)

	serverCrypto := testutils.CopyKeys(t, cs, data.CanonicalTimestampRole)
	updates, err = validateUpdate(serverCrypto, gun, updates, store)
	require.NoError(t, err)

	// we generated our own timestamp, and did not take the other timestamp,
	// but all other metadata should come from updates
	var foundTimestamp bool
	for _, update := range updates {
		if update.Role == data.CanonicalTimestampRole {
			foundTimestamp = true
			oldTimestamp, newTimestamp := &data.SignedTimestamp{}, &data.SignedTimestamp{}
			require.NoError(t, json.Unmarshal(timestamp.Data, oldTimestamp))
			require.NoError(t, json.Unmarshal(update.Data, newTimestamp))
			require.Equal(t, oldTimestamp.Signed.Version+1, newTimestamp.Signed.Version)
		}
	}
	require.True(t, foundTimestamp)
}

// GetOrCreateTimestamp returns the current timestamp for the gun. This may mean
// a new timestamp is generated either because none exists, or because the current
// one has expired. Once generated, the timestamp is saved in the store.
// Additionally, if we had to generate a new snapshot for this timestamp,
// it is also saved in the store
func GetOrCreateTimestamp(gun string, store storage.MetaStore, cryptoService signed.CryptoService) (
	*time.Time, []byte, error) {

	updates := []storage.MetaUpdate{}

	lastModified, timestampJSON, err := store.GetCurrent(gun, data.CanonicalTimestampRole)
	if err != nil {
		logrus.Debug("error retrieving timestamp: ", err.Error())
		return nil, nil, err
	}

	prev := &data.SignedTimestamp{}
	if err := json.Unmarshal(timestampJSON, prev); err != nil {
		logrus.Error("Failed to unmarshal existing timestamp")
		return nil, nil, err
	}
	snapChecksums, err := prev.GetSnapshot()
	if err != nil || snapChecksums == nil {
		return nil, nil, err
	}
	snapshotSha256Bytes, ok := snapChecksums.Hashes[notary.SHA256]
	if !ok {
		return nil, nil, data.ErrMissingMeta{Role: data.CanonicalSnapshotRole}
	}
	snapshotSha256Hex := hex.EncodeToString(snapshotSha256Bytes[:])
	snapshotTime, snapshot, err := snapshot.GetOrCreateSnapshot(gun, snapshotSha256Hex, store, cryptoService)
	if err != nil {
		logrus.Debug("Previous timestamp, but no valid snapshot for GUN ", gun)
		return nil, nil, err
	}
	snapshotRole := &data.SignedSnapshot{}
	if err := json.Unmarshal(snapshot, snapshotRole); err != nil {
		logrus.Error("Failed to unmarshal retrieved snapshot")
		return nil, nil, err
	}

	// If the snapshot was generated, we should write it with the timestamp
	if snapshotTime == nil {
		updates = append(updates, storage.MetaUpdate{Role: data.CanonicalSnapshotRole, Version: snapshotRole.Signed.Version, Data: snapshot})
	}

	if !timestampExpired(prev) && !snapshotExpired(prev, snapshot) {
		return lastModified, timestampJSON, nil
	}

	tsUpdate, err := createTimestamp(gun, prev, snapshot, store, cryptoService)
	if err != nil {
		logrus.Error("Failed to create a new timestamp")
		return nil, nil, err
	}
	updates = append(updates, *tsUpdate)

	c := time.Now()

	// Write the timestamp, and potentially snapshot
	if err = store.UpdateMany(gun, updates); err != nil {
		return nil, nil, err
	}
	return &c, tsUpdate.Data, nil
}

// UnmarshalTrusted unmarshals and verifies signatures only, not metadata, for a
// given role's metadata
func UnmarshalTrusted(b []byte, v interface{}, role string, db *keys.KeyDB) error {
	s := &data.Signed{}
	if err := json.Unmarshal(b, s); err != nil {
		return err
	}
	if err := VerifySignatures(s, role, db); err != nil {
		return err
	}
	return json.Unmarshal(s.Signed, v)
}

// Unmarshal unmarshals and verifys the raw bytes for a given role's metadata
func Unmarshal(b []byte, v interface{}, role string, minVersion int, db *keys.KeyDB) error {
	s := &data.Signed{}
	if err := json.Unmarshal(b, s); err != nil {
		return err
	}
	if err := Verify(s, role, minVersion, db); err != nil {
		return err
	}
	return json.Unmarshal(s.Signed, v)
}

func TestHTTPStoreGetMeta(t *testing.T) {
	handler := func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte(testRoot))
	}
	server := httptest.NewServer(http.HandlerFunc(handler))
	defer server.Close()
	store, err := NewHTTPStore(
		server.URL,
		"metadata",
		"txt",
		"key",
		&http.Transport{},
	)
	if err != nil {
		t.Fatal(err)
	}
	j, err := store.GetMeta("root", 4801)
	if err != nil {
		t.Fatal(err)
	}
	p := &data.Signed{}
	err = json.Unmarshal(j, p)
	if err != nil {
		t.Fatal(err)
	}
	rootKey, err := base64.StdEncoding.DecodeString(testRootKey)
	require.NoError(t, err)
	k := data.NewPublicKey("ecdsa-x509", rootKey)

	sigBytes := p.Signatures[0].Signature
	if err != nil {
		t.Fatal(err)
	}
	var decoded map[string]interface{}
	if err := json.Unmarshal(*p.Signed, &decoded); err != nil {
		t.Fatal(err)
	}
	msg, err := json.MarshalCanonical(decoded)
	if err != nil {
		t.Fatal(err)
	}
	method := p.Signatures[0].Method
	err = signed.Verifiers[method].Verify(k, sigBytes, msg)
	if err != nil {
		t.Fatal(err)
	}

}

// ExpireMetadata expires the metadata, which would make it invalid - don't do anything if
// we don't have the timestamp key
func (m *MetadataSwizzler) ExpireMetadata(role string) error {
	signedThing, err := signedFromStore(m.MetadataCache, role)
	if err != nil {
		return err
	}

	var unmarshalled map[string]interface{}
	if err := json.Unmarshal(*signedThing.Signed, &unmarshalled); err != nil {
		return err
	}

	unmarshalled["expires"] = time.Now().AddDate(-1, -1, -1)

	metaBytes, err := json.MarshalCanonical(unmarshalled)
	if err != nil {
		return err
	}
	signedThing.Signed = (*json.RawMessage)(&metaBytes)

	pubKeys, err := getPubKeys(m.CryptoService, signedThing, role)
	if err == nil {
		metaBytes, err = serializeMetadata(m.CryptoService, signedThing, role, pubKeys...)
	}

	if err != nil {
		return err
	}
	return m.MetadataCache.Set(role, metaBytes)
}

// OffsetMetadataVersion updates the metadata version
func (m *MetadataSwizzler) OffsetMetadataVersion(role string, offset int) error {
	signedThing, err := signedFromStore(m.MetadataCache, role)
	if err != nil {
		return err
	}

	var unmarshalled map[string]interface{}
	if err := json.Unmarshal(*signedThing.Signed, &unmarshalled); err != nil {
		return err
	}

	oldVersion, ok := unmarshalled["version"].(float64)
	if !ok {
		oldVersion = float64(0) // just ignore the error and set it to 0
	}
	unmarshalled["version"] = int(oldVersion) + offset

	metaBytes, err := json.MarshalCanonical(unmarshalled)
	if err != nil {
		return err
	}
	signedThing.Signed = (*json.RawMessage)(&metaBytes)

	pubKeys, err := getPubKeys(m.CryptoService, signedThing, role)
	if err == nil {
		metaBytes, err = serializeMetadata(m.CryptoService, signedThing, role, pubKeys...)
	}

	if err != nil {
		return err
	}
	return m.MetadataCache.Set(role, metaBytes)
}

// SetInvalidMetadataType unmarshallable, but has the wrong metadata type (not
// actually a metadata type)
func (m *MetadataSwizzler) SetInvalidMetadataType(role string) error {
	signedThing, err := signedFromStore(m.MetadataCache, role)
	if err != nil {
		return err
	}

	var unmarshalled map[string]interface{}
	if err := json.Unmarshal(*signedThing.Signed, &unmarshalled); err != nil {
		return err
	}

	unmarshalled["_type"] = "not_real"

	metaBytes, err := json.MarshalCanonical(unmarshalled)
	if err != nil {
		return err
	}
	signedThing.Signed = (*json.RawMessage)(&metaBytes)

	pubKeys, err := getPubKeys(m.CryptoService, signedThing, role)
	if err == nil {
		metaBytes, err = serializeMetadata(m.CryptoService, signedThing, role, pubKeys...)
	}

	if err != nil {
		return err
	}
	return m.MetadataCache.Set(role, metaBytes)
}

// SetInvalidSignedMeta corrupts the metadata into something that is unmarshallable
// as a Signed object, but not unmarshallable into a SignedMeta object
func (m *MetadataSwizzler) SetInvalidSignedMeta(role string) error {
	signedThing, err := signedFromStore(m.MetadataCache, role)
	if err != nil {
		return err
	}

	pubKeys, err := getPubKeys(m.CryptoService, signedThing, role)
	if err != nil {
		return err
	}

	var unmarshalled map[string]interface{}
	if err := json.Unmarshal(*signedThing.Signed, &unmarshalled); err != nil {
		return err
	}

	unmarshalled["_type"] = []string{"not a string"}
	unmarshalled["version"] = "string not int"
	unmarshalled["expires"] = "cannot be parsed as time"

	metaBytes, err := json.MarshalCanonical(unmarshalled)
	if err != nil {
		return err
	}
	signedThing.Signed = (*json.RawMessage)(&metaBytes)

	metaBytes, err = serializeMetadata(m.CryptoService, signedThing, role, pubKeys...)
	if err != nil {
		return err
	}
	return m.MetadataCache.Set(role, metaBytes)
}

// GetOrCreateSnapshotKey either creates a new snapshot key, or returns
// the existing one. Only the PublicKey is returned. The private part
// is held by the CryptoService.
func GetOrCreateSnapshotKey(gun string, store storage.MetaStore, crypto signed.CryptoService, createAlgorithm string) (data.PublicKey, error) {
	_, rootJSON, err := store.GetCurrent(gun, data.CanonicalRootRole)
	if err != nil {
		// If the error indicates we couldn't find the root, create a new key
		if _, ok := err.(storage.ErrNotFound); !ok {
			logrus.Errorf("Error when retrieving root role for GUN %s: %v", gun, err)
			return nil, err
		}
		return crypto.Create(data.CanonicalSnapshotRole, gun, createAlgorithm)
	}

	// If we have a current root, parse out the public key for the snapshot role, and return it
	repoSignedRoot := new(data.SignedRoot)
	if err := json.Unmarshal(rootJSON, repoSignedRoot); err != nil {
		logrus.Errorf("Failed to unmarshal existing root for GUN %s to retrieve snapshot key ID", gun)
		return nil, err
	}

	snapshotRole, err := repoSignedRoot.BuildBaseRole(data.CanonicalSnapshotRole)
	if err != nil {
		logrus.Errorf("Failed to extract snapshot role from root for GUN %s", gun)
		return nil, err
	}

	// We currently only support single keys for snapshot and timestamp, so we can return the first and only key in the map if the signer has it
	for keyID := range snapshotRole.Keys {
		if pubKey := crypto.GetKey(keyID); pubKey != nil {
			return pubKey, nil
		}
	}
	logrus.Debugf("Failed to find any snapshot keys in cryptosigner from root for GUN %s, generating new key", gun)
	return crypto.Create(data.CanonicalSnapshotRole, gun, createAlgorithm)
}

// MutateTimestamp takes a function that mutates the timestamp metadata - once done, it
// serializes the timestamp again
func (m *MetadataSwizzler) MutateTimestamp(mutate func(*data.Timestamp)) error {
	signedThing, err := signedFromStore(m.MetadataCache, data.CanonicalTimestampRole)
	if err != nil {
		return err
	}

	var timestamp data.Timestamp
	if err := json.Unmarshal(*signedThing.Signed, &timestamp); err != nil {
		return err
	}

	mutate(&timestamp)

	sTimestamp := &data.SignedTimestamp{Signed: timestamp, Signatures: signedThing.Signatures}
	signedThing, err = sTimestamp.ToSigned()
	if err != nil {
		return err
	}

	pubKeys, err := getPubKeys(m.CryptoService, signedThing, data.CanonicalTimestampRole)
	if err != nil {
		return err
	}

	metaBytes, err := serializeMetadata(m.CryptoService, signedThing, data.CanonicalTimestampRole, pubKeys...)
	if err != nil {
		return err
	}
	return m.MetadataCache.Set(data.CanonicalTimestampRole, metaBytes)
}

// MutateSnapshot takes a function that mutates the snapshot metadata - once done, it
// serializes the snapshot again
func (m *MetadataSwizzler) MutateSnapshot(mutate func(*data.Snapshot)) error {
	signedThing, err := signedFromStore(m.MetadataCache, data.CanonicalSnapshotRole)
	if err != nil {
		return err
	}

	var snapshot data.Snapshot
	if err := json.Unmarshal(*signedThing.Signed, &snapshot); err != nil {
		return err
	}

	mutate(&snapshot)

	sSnapshot := &data.SignedSnapshot{Signed: snapshot, Signatures: signedThing.Signatures}
	signedThing, err = sSnapshot.ToSigned()
	if err != nil {
		return err
	}

	pubKeys, err := getPubKeys(m.CryptoService, signedThing, data.CanonicalSnapshotRole)
	if err != nil {
		return err
	}

	metaBytes, err := serializeMetadata(m.CryptoService, signedThing, data.CanonicalSnapshotRole, pubKeys...)
	if err != nil {
		return err
	}
	return m.MetadataCache.Set(data.CanonicalSnapshotRole, metaBytes)
}

// CreateTimestamp creates a new timestamp. If a prev timestamp is provided, it
// is assumed this is the immediately previous one, and the new one will have a
// version number one higher than prev. The store is used to lookup the current
// snapshot, this function does not save the newly generated timestamp.
func CreateTimestamp(gun string, prev *data.SignedTimestamp, snapshot []byte, store storage.MetaStore, cryptoService signed.CryptoService) (*data.Signed, int, error) {
	algorithm, public, err := store.GetKey(gun, data.CanonicalTimestampRole)
	if err != nil {
		// owner of gun must have generated a timestamp key otherwise
		// we won't proceed with generating everything.
		return nil, 0, err
	}
	key := data.NewPublicKey(algorithm, public)
	sn := &data.Signed{}
	err = json.Unmarshal(snapshot, sn)
	if err != nil {
		// couldn't parse snapshot
		return nil, 0, err
	}
	ts, err := data.NewTimestamp(sn)
	if err != nil {
		return nil, 0, err
	}
	if prev != nil {
		ts.Signed.Version = prev.Signed.Version + 1
	}
	sgndTs, err := json.MarshalCanonical(ts.Signed)
	if err != nil {
		return nil, 0, err
	}
	out := &data.Signed{
		Signatures: ts.Signatures,
		Signed:     sgndTs,
	}
	err = signed.Sign(cryptoService, out, key)
	if err != nil {
		return nil, 0, err
	}
	return out, ts.Signed.Version, nil
}

func TestGetTimestampOldTimestampExpired(t *testing.T) {
	store := storage.NewMemStorage()
	repo, crypto, err := testutils.EmptyRepo("gun")
	require.NoError(t, err)

	meta, err := testutils.SignAndSerialize(repo)
	require.NoError(t, err)

	// create an expired timestamp
	_, err = repo.SignTimestamp(time.Now().AddDate(-1, -1, -1))
	require.True(t, repo.Timestamp.Signed.Expires.Before(time.Now()))
	require.NoError(t, err)
	timestampJSON, err := json.Marshal(repo.Timestamp)
	require.NoError(t, err)

	// set all the metadata
	require.NoError(t, store.UpdateCurrent("gun",
		storage.MetaUpdate{Role: data.CanonicalRootRole, Version: 0, Data: meta[data.CanonicalRootRole]}))
	require.NoError(t, store.UpdateCurrent("gun",
		storage.MetaUpdate{Role: data.CanonicalSnapshotRole, Version: 0, Data: meta[data.CanonicalSnapshotRole]}))
	require.NoError(t, store.UpdateCurrent("gun",
		storage.MetaUpdate{Role: data.CanonicalTimestampRole, Version: 1, Data: timestampJSON}))

	_, gottenTimestamp, err := GetOrCreateTimestamp("gun", store, crypto)
	require.NoError(t, err, "GetTimestamp errored")

	require.False(t, bytes.Equal(timestampJSON, gottenTimestamp),
		"Timestamp was not regenerated when old one was expired")

	signedMeta := &data.SignedMeta{}
	require.NoError(t, json.Unmarshal(gottenTimestamp, signedMeta))
	// the new metadata is not expired
	require.True(t, signedMeta.Signed.Expires.After(time.Now()))
}

func TestHTTPStoreGetSized(t *testing.T) {
	handler := func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte(testRoot))
	}
	server := httptest.NewServer(http.HandlerFunc(handler))
	defer server.Close()
	store, err := NewHTTPStore(
		server.URL,
		"metadata",
		"txt",
		"key",
		&http.Transport{},
	)
	require.NoError(t, err)
	j, err := store.GetSized("root", 4801)
	require.NoError(t, err)
	require.Equal(t, testRoot, string(j))
	p := &data.Signed{}
	err = json.Unmarshal(j, p)
	require.NoError(t, err)

	// if there is a network error, it gets translated to NetworkError
	store, err = NewHTTPStore(
		server.URL,
		"metadata",
		"txt",
		"key",
		failRoundTripper{},
	)
	require.NoError(t, err)
	_, err = store.GetSized("root", 4801)
	require.IsType(t, NetworkError{}, err)
	require.Equal(t, "FAIL", err.Error())
}

// MutateTargets takes a function that mutates the targets metadata - once done, it
// serializes the targets again
func (m *MetadataSwizzler) MutateTargets(mutate func(*data.Targets)) error {
	signedThing, err := signedFromStore(m.MetadataCache, data.CanonicalTargetsRole)
	if err != nil {
		return err
	}

	var targets data.Targets
	if err := json.Unmarshal(*signedThing.Signed, &targets); err != nil {
		return err
	}

	mutate(&targets)

	sTargets := &data.SignedTargets{Signed: targets, Signatures: signedThing.Signatures}
	signedThing, err = sTargets.ToSigned()
	if err != nil {
		return err
	}

	pubKeys, err := getPubKeys(m.CryptoService, signedThing, data.CanonicalTargetsRole)
	if err != nil {
		return err
	}

	metaBytes, err := serializeMetadata(m.CryptoService, signedThing, data.CanonicalTargetsRole, pubKeys...)
	if err != nil {
		return err
	}
	return m.MetadataCache.Set(data.CanonicalTargetsRole, metaBytes)
}

// generateSnapshot generates a new snapshot from the previous one in the store - this assumes all
// the other roles except timestamp have already been set on the repo, and will set the generated
// snapshot on the repo as well
func generateSnapshot(gun string, builder tuf.RepoBuilder, store storage.MetaStore) (*storage.MetaUpdate, error) {
	var prev *data.SignedSnapshot
	_, currentJSON, err := store.GetCurrent(gun, data.CanonicalSnapshotRole)
	if err == nil {
		prev = new(data.SignedSnapshot)
		if err = json.Unmarshal(currentJSON, prev); err != nil {
			logrus.Error("Failed to unmarshal existing snapshot for GUN ", gun)
			return nil, err
		}
	}

	if _, ok := err.(storage.ErrNotFound); !ok && err != nil {
		return nil, err
	}

	meta, ver, err := builder.GenerateSnapshot(prev)

	switch err.(type) {
	case nil:
		return &storage.MetaUpdate{
			Role:    data.CanonicalSnapshotRole,
			Version: ver,
			Data:    meta,
		}, nil
	case signed.ErrInsufficientSignatures, signed.ErrNoKeys, signed.ErrRoleThreshold:
		// If we cannot sign the snapshot, then we don't have keys for the snapshot,
		// and the client should have submitted a snapshot
		return nil, validation.ErrBadHierarchy{
			Missing: data.CanonicalSnapshotRole,
			Msg:     "no snapshot was included in update and server does not hold current snapshot key for repository"}
	default:
		return nil, validation.ErrValidation{Msg: err.Error()}
	}
}

// ChangeRootKey swaps out the root key with a new key, and re-signs the metadata
// with the new key
func (m *MetadataSwizzler) ChangeRootKey() error {
	key, err := CreateKey(m.CryptoService, m.Gun, data.CanonicalRootRole, data.ECDSAKey)
	if err != nil {
		return err
	}

	b, err := m.MetadataCache.GetSized(data.CanonicalRootRole, store.NoSizeLimit)
	if err != nil {
		return err
	}

	signedRoot := &data.SignedRoot{}
	if err := json.Unmarshal(b, signedRoot); err != nil {
		return err
	}

	signedRoot.Signed.Keys[key.ID()] = key
	signedRoot.Signed.Roles[data.CanonicalRootRole].KeyIDs = []string{key.ID()}

	var signedThing *data.Signed
	if signedThing, err = signedRoot.ToSigned(); err != nil {
		return err
	}

	var metaBytes []byte
	pubKeys, err := getPubKeys(m.CryptoService, signedThing, data.CanonicalRootRole)
	if err == nil {
		metaBytes, err = serializeMetadata(m.CryptoService, signedThing, data.CanonicalRootRole, pubKeys...)
	}

	if err != nil {
		return err
	}
	return m.MetadataCache.Set(data.CanonicalRootRole, metaBytes)
}

func getPubKeys(cs signed.CryptoService, s *data.Signed, role string) ([]data.PublicKey, error) {
	var pubKeys []data.PublicKey
	if role == data.CanonicalRootRole {
		// if this is root metadata, we have to get the keys from the root because they
		// are certs
		root := &data.Root{}
		if err := json.Unmarshal(*s.Signed, root); err != nil {
			return nil, err
		}
		rootRole, ok := root.Roles[data.CanonicalRootRole]
		if !ok || rootRole == nil {
			return nil, tuf.ErrNotLoaded{}
		}
		for _, pubKeyID := range rootRole.KeyIDs {
			pubKeys = append(pubKeys, root.Keys[pubKeyID])
		}
	} else {
		pubKeyIDs := cs.ListKeys(role)
		for _, pubKeyID := range pubKeyIDs {
			pubKey := cs.GetKey(pubKeyID)
			if pubKey != nil {
				pubKeys = append(pubKeys, pubKey)
			}
		}
	}
	return pubKeys, nil
}

// UnmarshalPublicKey is used to parse individual public keys in JSON
func UnmarshalPublicKey(data []byte) (PublicKey, error) {
	var parsed tufKey
	err := json.Unmarshal(data, &parsed)
	if err != nil {
		return nil, err
	}
	return typedPublicKey(parsed), nil
}