func TestDeleteInTransaction(t *testing.T) {
	c, closeFunc := NewContext(t)
	defer closeFunc()

	type testEntity struct {
		Val int
	}

	key := datastore.NewKey(c, "TestEntity", "", 1, nil)
	if _, err := nds.Put(c, key, &testEntity{2}); err != nil {
		t.Fatal(err)
	}

	// Prime cache.
	if err := nds.Get(c, key, &testEntity{}); err != nil {
		t.Fatal(err)
	}

	if err := nds.RunInTransaction(c, func(tc context.Context) error {
		return nds.DeleteMulti(tc, []*datastore.Key{key})
	}, nil); err != nil {
		t.Fatal(err)
	}

	if err := nds.Get(c, key, &testEntity{}); err == nil {
		t.Fatal("expected no entity")
	} else if err != datastore.ErrNoSuchEntity {
		t.Fatal(err)
	}
}

func TestProcess(t *testing.T) {

	ctx, done, _ := aetest.NewContext()
	defer done()

	smallTest := strings.NewReader(`{
		"User/jsmith": {
			"Name": "John Smith",
			"LotteryNumbers": [1,2,3,4,5],
			"CreatedAt": {
				"Type": "time",
				"Value": "1993-05-01T12:31:00.000Z"
			}
		},
		"User/jdoe": {
			"Name": "Jane Doe",
			"LotteryNumbers": [2,4,6,8,10],
			"CreatedAt": {
				"Type": "time",
				"Value": "1992-01-30T08:01:00.000Z"
			}
		}
	}`)

	if err := Process(ctx, smallTest); err != nil {
		t.Errorf("Unexpected error %s", err)
	}

	key1 := datastore.NewKey(ctx, "User", "jdoe", 0, nil)
	key2 := datastore.NewKey(ctx, "User", "jsmith", 0, nil)

	value1 := fakeUser{}
	value2 := fakeUser{}

	if err := nds.Get(ctx, key1, &value1); err != nil {
		t.Errorf("Unexpected error %s retrieving value1", err)
	} else if err := nds.Get(ctx, key2, &value2); err != nil {
		t.Errorf("Unexpected error %s retrieving value2", err)
	}

	value1.CreatedAt = value1.CreatedAt.UTC()
	value2.CreatedAt = value2.CreatedAt.UTC()

	if !reflect.DeepEqual(value1, fakeUser{
		Name:           "Jane Doe",
		LotteryNumbers: []int64{2, 4, 6, 8, 10},
		CreatedAt:      time.Date(1992, 1, 30, 8, 1, 0, 0, time.UTC),
	}) {
		t.Errorf("Unexpected value in value1: %+v", value1)
	}

	if !reflect.DeepEqual(value2, fakeUser{
		Name:           "John Smith",
		LotteryNumbers: []int64{1, 2, 3, 4, 5},
		CreatedAt:      time.Date(1993, 5, 1, 12, 31, 0, 0, time.UTC),
	}) {
		t.Errorf("Unexpected value in value2: %+v", value1)
	}

}

func TestGetArgs(t *testing.T) {
	c, closeFunc := NewContext(t)
	defer closeFunc()

	type testEntity struct {
		IntVal int64
	}

	if err := nds.Get(c, nil, &testEntity{}); err == nil {
		t.Fatal("expected error for nil key")
	}

	key := datastore.NewKey(c, "Entity", "", 1, nil)
	if err := nds.Get(c, key, nil); err != datastore.ErrInvalidEntityType {
		t.Fatal("expected ErrInvalidEntityType for nil value")
	}

	if err := nds.Get(c, key, datastore.PropertyList{}); err == nil {
		t.Fatal("expected error for datastore.PropertyList")
	}

	if err := nds.Get(c, key, testEntity{}); err == nil {
		t.Fatal("expected error for struct")
	}

	rte := newReaderTestEntity()
	if err := nds.Get(c, key, rte); err == nil {
		t.Fatal("expected error for interface")
	}
}

// TestGetNamespacedKey ensures issue https://goo.gl/rXU8nK is fixed so that
// memcache uses the namespace from the key instead of the context.
func TestGetNamespacedKey(t *testing.T) {
	c, closeFunc := NewContext(t)
	defer closeFunc()

	const intVal = int64(12)
	type testEntity struct {
		IntVal int64
	}

	namespacedCtx, err := appengine.Namespace(c, "keyNamespace")
	if err != nil {
		t.Fatal(err)
	}

	key := datastore.NewKey(c, "Entity", "", 1, nil)
	namespacedKey := datastore.NewKey(namespacedCtx,
		"Entity", "", key.IntID(), nil)
	entity := &testEntity{intVal}

	if namespacedKey, err = nds.Put(c, namespacedKey, entity); err != nil {
		t.Fatal(err)
	}

	// Prime cache.
	if err := nds.Get(namespacedCtx, namespacedKey, &testEntity{}); err != nil {
		t.Fatal(err)
	}

	// Ensure that we get a value back from the cache by checking if the
	// datastore is called at all.
	entityFromCache := true
	nds.SetDatastoreGetMulti(func(c context.Context,
		keys []*datastore.Key, vals interface{}) error {
		if len(keys) != 0 {
			entityFromCache = false
		}
		return nil
	})
	if err := nds.Get(c, namespacedKey, &testEntity{}); err != nil {
		t.Fatal(err)
	}
	nds.SetDatastoreGetMulti(datastore.GetMulti)

	if !entityFromCache {
		t.Fatal("entity not obtained from cache")
	}

	if err := nds.Delete(namespacedCtx, namespacedKey); err != nil {
		t.Fatal(err)
	}

	entity = &testEntity{}
	if err := nds.Get(c, namespacedKey, entity); err == nil {
		t.Fatalf("expected no such entity error but got %+v", entity)
	} else if err != datastore.ErrNoSuchEntity {
		t.Fatal(err)
	}
}

// New creates and returns a new blank account. It returns an error if an account
// with the specified email address already exists.
func New(ctx context.Context, email, password string) (*Account, error) {

	account := new(Account)
	account.Email = email
	account.CreatedAt = time.Now()
	if err := account.SetPassword(password); err != nil {
		return nil, err
	}

	err := nds.RunInTransaction(ctx, func(txCtx context.Context) error {

		dsKey := account.Key(txCtx)
		if err := nds.Get(txCtx, dsKey, account); err == nil {
			return ErrAccountExists
		} else if err != datastore.ErrNoSuchEntity {
			return err
		}

		_, err := nds.Put(txCtx, dsKey, account)
		return err

	}, nil)

	if err != nil {
		return nil, err
	}

	account.flag = camethroughus
	account.originalEmail = email
	return account, nil

}

func TestMemcacheNamespace(t *testing.T) {

	c, closeFunc := NewContext(t)
	defer closeFunc()

	type testEntity struct {
		IntVal int
	}

	// Illegal namespace chars.
	nds.SetMemcacheNamespace("£££")

	key := datastore.NewKey(c, "Entity", "", 1, nil)
	if err := nds.Get(c, key, &testEntity{}); err == nil {
		t.Fatal("expected namespace error")
	}

	if _, err := nds.Put(c, key, &testEntity{}); err == nil {
		t.Fatal("expected namespace error")
	}

	if err := nds.Delete(c, key); err == nil {
		t.Fatal("expected namespace error")
	}

	if err := nds.RunInTransaction(c, func(tc context.Context) error {
		return nil
	}, nil); err == nil {
		t.Fatal("expected namespace error")
	}

	nds.SetMemcacheNamespace("")
}

// Next processes the next item
func (x *example6) Next(c context.Context, counters mapper.Counters, key *datastore.Key) error {
	// we need to load the entity ourselves
	photo := new(Photo)
	if err := nds.Get(c, key, photo); err != nil {
		return err
	}
	photo.ID = key.IntID()

	suffix := photo.Taken.Format("20060102")
	_, err := x.bq.Tabledata.InsertAll(x.appID, "datasetName", "tableName", &bigquery.TableDataInsertAllRequest{
		TemplateSuffix: suffix,
		Rows: []*bigquery.TableDataInsertAllRequestRows{
			{
				Json: map[string]bigquery.JsonValue{
					"id":    photo.ID,
					"taken": photo.Taken,
					"photographer": map[string]bigquery.JsonValue{
						"id":   photo.Photographer.ID,
						"name": photo.Photographer.Name,
					},
				},
			},
		},
	}).Context(c).Do()
	return err
}

// TestClearNamespacedLocks tests to make sure that locks are cleared when
// RunInTransaction is using a namespace.
func TestClearNamespacedLocks(t *testing.T) {
	c, closeFunc := NewContext(t, nil)
	defer closeFunc()

	c, err := appengine.Namespace(c, "testnamespace")
	if err != nil {
		t.Fatal(err)
	}

	type testEntity struct {
		Val int
	}

	key := datastore.NewKey(c, "TestEntity", "", 1, nil)

	// Prime cache.
	if err := nds.Get(c, key, &testEntity{}); err == nil {
		t.Fatal("expected no such entity")
	} else if err != datastore.ErrNoSuchEntity {
		t.Fatal(err)
	}

	if err := nds.RunInTransaction(c, func(tc context.Context) error {

		if err := nds.Get(tc, key, &testEntity{}); err == nil {
			return errors.New("expected no such entity")
		} else if err != datastore.ErrNoSuchEntity {
			return err
		}

		if _, err := nds.Put(tc, key, &testEntity{3}); err != nil {
			return err
		}
		return nil
	}, nil); err != nil {
		t.Fatal(err)
	}

	entity := &testEntity{}
	if err := nds.Get(c, key, entity); err != nil {
		t.Fatal(err)
	}

	if entity.Val != 3 {
		t.Fatal("incorrect val")
	}
}

// retrieve obtains the application configuration as a datastore.PropertyList.
func retrieve(ctx context.Context) (datastore.PropertyList, error) {

	p := datastore.PropertyList(make([]datastore.Property, 0, 8))
	key := datastore.NewKey(ctx, Entity, Entity, 0, nil)
	err := nds.Get(ctx, key, &p)
	return p, err

}

// Next processes the next item
func (x *example1) Next(c context.Context, counters mapper.Counters, key *datastore.Key) error {
	// we need to load the entity ourselves
	photo := new(Photo)
	if err := nds.Get(c, key, photo); err != nil {
		return err
	}

	counters.Increment(photo.Photographer.Name, 1)
	return nil
}

// HasChanged checks the current state of an account in the datastore. It returns
// true if the saved version of the account has diverged from the state of the account
// as described in account.
func HasChanged(ctx context.Context, account *Account) (bool, error) {

	var currentState Account
	key := account.Key(ctx)
	if err := nds.Get(ctx, key, &currentState); err != nil {
		return false, err
	} else {
		return reflect.DeepEqual(*account, currentState), nil
	}

}

// Get retrieves the account identified by email and stores it in
// the value pointed to by account.
func Get(ctx context.Context, email string, account *Account) error {

	if err := nds.Get(ctx, datastore.NewKey(ctx, Entity, email, 0, nil), account); err != nil {
		return err
	} else {
		account.flag = camethroughus
		account.originalEmail = account.Email
		return nil
	}

}

// Next processes the next item
func (x *example3) Next(c context.Context, counters mapper.Counters, key *datastore.Key) error {
	photo := new(Photo)
	if err := nds.Get(c, key, photo); err != nil {
		log.Errorf(c, err.Error())
		return err
	}
	photo.ID = key.IntID()

	counters.Increment(photo.Photographer.Name, 1)
	return nil
}

// Save changes the application configuration to
// the values in conf. All HTTP requests subsequent to this one
// are guaranteed to use the new values in their configuration.
//
// Note that subsequent calls to Get with the same request context
// will continue to retrieve the old version of the configuration.
//
// As a special case, calling Save with a *config.Config will replace
// the entire contents of the configuration with the contents of Config.
func Save(ctx context.Context, conf interface{}) error {

	if typedConfig, ok := conf.(*Config); ok {
		pl := datastore.PropertyList(*typedConfig)
		replaceKey := datastore.NewKey(ctx, Entity, Entity, 0, nil)
		_, replaceErr := nds.Put(ctx, replaceKey, &pl)
		return replaceErr
	}

	return datastore.RunInTransaction(ctx, func(txCtx context.Context) error {

		props := datastore.PropertyList{}

		key := datastore.NewKey(txCtx, Entity, Entity, 0, nil)
		if err := nds.Get(txCtx, key, &props); err != nil && err != datastore.ErrNoSuchEntity {
			return err
		}

		// merge existing config with the new values
		if newProps, err := datastore.SaveStruct(conf); err != nil {
			return err
		} else {

			for _, newProp := range newProps {
				newProp.NoIndex = true
				replacing := false
				for _, prop := range props {
					// make sure NoIndex is set
					prop.NoIndex = true
					if prop.Name == newProp.Name {
						replacing = true
						prop.Value = newProp.Value
						break
					}
				}
				if !replacing {
					// append
					props = append(props, newProp)
				}

			}

		}

		_, err := nds.Put(txCtx, key, &props)
		return err

	}, nil)

}

func TestGetSliceProperty(t *testing.T) {
	c, closeFunc := NewContext(t)
	defer closeFunc()

	type testEntity struct {
		IntVals []int64
	}

	key := datastore.NewKey(c, "Entity", "", 1, nil)
	intVals := []int64{0, 1, 2, 3}
	val := &testEntity{intVals}

	if _, err := nds.Put(c, key, val); err != nil {
		t.Fatal(err)
	}

	// Get from datastore.
	newVal := &testEntity{}
	if err := nds.Get(c, key, newVal); err != nil {
		t.Fatal(err)
	}

	if !reflect.DeepEqual(val.IntVals, intVals) {
		t.Fatal("slice properties not equal", val.IntVals)
	}

	// Get from memcache.
	newVal = &testEntity{}
	if err := nds.Get(c, key, newVal); err != nil {
		t.Fatal(err)
	}

	if !reflect.DeepEqual(val.IntVals, intVals) {
		t.Fatal("slice properties not equal", val.IntVals)
	}
}

// IncrementX is the version of Increment you should call if you wish to increment a
// counter inside a transaction.
func (c *Counter) IncrementX(txCtx context.Context, delta int64) error {

	val := count{}

	// pick a key at random and alter its value by delta
	key := datastore.NewKey(txCtx, c.entity, c.shardKeys[rand.Int63()%int64(c.shardCount)], 0, nil)

	if err := nds.Get(txCtx, key, &val); err != nil && err != datastore.ErrNoSuchEntity {
		return err
	}

	val.C += delta

	if _, err := nds.Put(txCtx, key, &val); err != nil {
		return err
	}

	return nil

}

func warmupHandler(c *echo.Context) error {
	if appengine.IsDevAppServer() {
		k := datastore.NewKey(c, "photo", "", 1, nil)
		p := new(Photo)
		err := nds.Get(c, k, p)
		if err != datastore.ErrNoSuchEntity {
			return c.NoContent(http.StatusOK)
		}

		photographers := []Photographer{
			{1, "Mr Canon"},
			{2, "Miss Nikon"},
			{3, "Mrs Pentax"},
			{4, "Ms Sony"},
		}

		// create some dummy data
		var id int64
		for m := 1; m <= 12; m++ {
			for d := 1; d < 28; d++ {
				taken := time.Date(2015, time.Month(m), d, 12, 0, 0, 0, time.UTC)
				photographer := photographers[rand.Int31n(4)]
				p = &Photo{
					Photographer: photographer,
					Uploaded:     time.Now().UTC(),
					Width:        8000,
					Height:       6000,
					Taken:        taken,
				}
				id++
				k = datastore.NewKey(c, "photo", "", id, nil)
				nds.Put(c, k, p)
			}
		}
	}
	return c.NoContent(http.StatusOK)
}

// LoadConfig retrieves the most recent state of the configuration
// from the datastore into the value pointed to by conf.
func LoadConfig(ctx context.Context, conf interface{}) error {
	return nds.Get(ctx, datastore.NewKey(ctx, config.Entity, config.Entity, 0, nil), conf)
}

func TestPutGetDelete(t *testing.T) {
	c, closeFunc := NewContext(t)
	defer closeFunc()

	type testEntity struct {
		IntVal int
	}

	// Check we set memcahce, put datastore and delete memcache.
	seq := make(chan string, 3)
	nds.SetMemcacheSetMulti(func(c context.Context,
		items []*memcache.Item) error {
		seq <- "memcache.SetMulti"
		return memcache.SetMulti(c, items)
	})
	nds.SetDatastorePutMulti(func(c context.Context,
		keys []*datastore.Key, vals interface{}) ([]*datastore.Key, error) {
		seq <- "datastore.PutMulti"
		return datastore.PutMulti(c, keys, vals)
	})
	nds.SetMemcacheDeleteMulti(func(c context.Context,
		keys []string) error {
		seq <- "memcache.DeleteMulti"
		close(seq)
		return memcache.DeleteMulti(c, keys)
	})

	incompleteKey := datastore.NewIncompleteKey(c, "Entity", nil)
	key, err := nds.Put(c, incompleteKey, &testEntity{43})
	if err != nil {
		t.Fatal(err)
	}

	nds.SetMemcacheSetMulti(memcache.SetMulti)
	nds.SetDatastorePutMulti(datastore.PutMulti)
	nds.SetMemcacheDeleteMulti(memcache.DeleteMulti)

	if s := <-seq; s != "memcache.SetMulti" {
		t.Fatal("memcache.SetMulti not", s)
	}
	if s := <-seq; s != "datastore.PutMulti" {
		t.Fatal("datastore.PutMulti not", s)
	}
	if s := <-seq; s != "memcache.DeleteMulti" {
		t.Fatal("memcache.DeleteMulti not", s)
	}
	// Check chan is closed.
	<-seq

	if key.Incomplete() {
		t.Fatal("Key is incomplete")
	}

	te := &testEntity{}
	if err := nds.Get(c, key, te); err != nil {
		t.Fatal(err)
	}

	if te.IntVal != 43 {
		t.Fatal("te.Val != 43", te.IntVal)
	}

	// Get from cache.
	te = &testEntity{}
	if err := nds.Get(c, key, te); err != nil {
		t.Fatal(err)
	}

	if te.IntVal != 43 {
		t.Fatal("te.Val != 43", te.IntVal)
	}

	// Change value.
	if _, err := nds.Put(c, key, &testEntity{64}); err != nil {
		t.Fatal(err)
	}

	// Get from cache.
	te = &testEntity{}
	if err := nds.Get(c, key, te); err != nil {
		t.Fatal(err)
	}

	if te.IntVal != 64 {
		t.Fatal("te.Val != 64", te.IntVal)
	}

	if err := nds.Delete(c, key); err != nil {
		t.Fatal(err)
	}

	if err := nds.Get(c, key, &testEntity{}); err != datastore.ErrNoSuchEntity {
		t.Fatal("expected datastore.ErrNoSuchEntity")
	}
}

func processPhotos(c context.Context, processor PhotoProcessor) error {
	// use the full 10 minutes allowed (assuming front-end instance type)
	c, _ = context.WithTimeout(c, time.Duration(10)*time.Minute)

	r := processor.Start(c)
	log.Debugf(c, "processPhotos from %s to %s cursor %s", r.From.Format(dateFormat), r.To.Format(dateFormat), r.Start)

	// TODO: describe pros & cons of different querying + continuation strategies

	q := datastore.NewQuery("photo")
	q = q.Filter("taken >=", r.From)
	q = q.Filter("taken <", r.To)
	q = q.Order("taken")

	// I use keys only because it saves on cost - entities come from memcache if possible
	q = q.KeysOnly()

	var cursor *datastore.Cursor
	if r.Start != "" {
		newCursor, err := datastore.DecodeCursor(r.Start)
		if err != nil {
			log.Errorf(c, "get start cursor error %s", err.Error())
			return err
		}
		cursor = &newCursor
	}

	// only one entity is loaded at a time
	p := new(Photo)

	timeout := make(chan bool, 1)
	timer := time.AfterFunc(r.Timeout, func() {
		timeout <- true
	})
	defer timer.Stop()

Loop:
	for {
		// check if we've timed out or whether to keep going
		select {
		case <-timeout:
			break Loop
		default:
		}

		processed := 0

		q = q.Limit(r.Size)
		if cursor != nil {
			q = q.Start(*cursor)
		}
		it := q.Run(c)
		for {
			// if not using keys only then we would load the actual entity here using
			// key, err := it.Next(p)
			key, err := it.Next(nil)
			if err == datastore.Done {
				break
			}
			if err != nil {
				log.Errorf(c, "get key error %s", err.Error())
				return err
			}

			// loads the actual entity from memcache / datastore
			err = nds.Get(c, key, p)
			if err != nil {
				log.Errorf(c, "get photo error %s", err.Error())
				return err
			}

			// call the processor with the entity
			p.ID = key.IntID()
			processor.Process(c, p)

			processed++
		}

		// did we process a full batch? if so, there may be more
		if processed == r.Size {
			newCursor, err := it.Cursor()
			if err != nil {
				log.Errorf(c, "get next cursor error %s", err.Error())
				return err
			}
			cursor = &newCursor
		} else {
			// otherwise we're finished
			cursor = nil
			break
		}
	}

	// let the processor write any aggregation entries / tasks etc...
	processor.Complete(c)

	// if we didn't complete everything then continue from the cursor
	if cursor != nil {
		r.Start = cursor.String()
		processPhotosFunc.Call(c, processor)
//.........这里部分代码省略.........