func (suite *sessionRecoverySuite) TestExpiredCacheRecovery() {
	scope := New().(*realScope)
	scope.userCache = newTestCache()

	mock := multiclient.NewMock()
	stub := &multiclient.Stub{
		Service:  loginService,
		Endpoint: readSessionEndpoint,
		Response: &sessreadproto.Response{
			SessId: proto.String(testSessId),
			Token:  proto.String(testToken),
		},
	}
	mock.Stub(stub)
	multiclient.SetCaller(mock.Caller())

	u, err := FromSessionToken(testSessId, testToken)
	suite.Assertions.Equal(testSessId, u.SessId)
	suite.Assertions.NoError(err)
	suite.Assertions.NotNil(u)
	u.RenewTs = time.Now().Add(-2 * time.Minute)
	u.ExpiryTs = u.RenewTs.Add(1 * time.Minute)
	suite.Assertions.NoError(scope.userCache.Store(u))
	suite.Assertions.Equal(0, stub.CountCalls())

	// As this session has expired, the login service should be called to recover (and renew) it
	suite.Assertions.False(scope.IsAuth())
	suite.Assertions.NoError(scope.RecoverSession(testSessId))
	suite.Assertions.True(scope.IsAuth())
	u = scope.AuthUser()
	suite.Assertions.NotNil(u)
	suite.Assertions.Equal(1, stub.CountCalls())
}

// ToProtobuf takes a Error and returns a protobuf error
func ToProtobuf(err Error) *pe.PlatformError {
	return &pe.PlatformError{
		Type:        pe.PlatformError_ErrorType(pe.PlatformError_ErrorType_value[err.Type()]).Enum(),
		Code:        proto.String(err.Code()),
		Description: proto.String(err.Description()),
		Context:     err.Context(),
		HttpCode:    proto.Uint32(err.HttpCode()),
	}
}

func healthCheckSampleToProto(hc *HealthCheck, sample *Sample) *hcproto.HealthCheck {
	return &hcproto.HealthCheck{
		Timestamp:        proto.Int64(sample.At.Unix()),
		HealthCheckId:    proto.String(hc.Id),
		ServiceName:      proto.String(hc.ServiceName),
		ServiceVersion:   proto.Uint64(hc.ServiceVersion),
		Hostname:         proto.String(hc.Hostname),
		InstanceId:       proto.String(hc.InstanceId),
		IsHealthy:        proto.Bool(sample.IsHealthy),
		ErrorDescription: proto.String(sample.ErrorDescription),
		Measurements:     mapToProto(sample.Measurements),
		Priority:         hcproto.HealthCheck_Priority(hc.Priority).Enum(),
	}
}

func mapToProto(m map[string]string) []*hcproto.HealthCheck_KeyValue {
	if m == nil {
		return []*hcproto.HealthCheck_KeyValue{}
	}
	ret := make([]*hcproto.HealthCheck_KeyValue, len(m))
	i := 0
	for k, v := range m {
		ret[i] = &hcproto.HealthCheck_KeyValue{
			Key:   proto.String(k),
			Value: proto.String(v),
		}
		i++
	}
	return ret
}

// SignOut destroys the current session so that it cannot be used again
func (s *realScope) SignOut(user *User) error {
	cl := multiclient.New().DefaultScopeFrom(s.getRpcScope())

	cl.AddScopedReq(&multiclient.ScopedReq{
		Uid:      "deletesess",
		Service:  loginService,
		Endpoint: deleteSessionEndpoint,
		Req: &sessdelproto.Request{
			SessId: proto.String(user.SessId),
		},
		Rsp: &sessdelproto.Response{},
	})

	if cl.Execute().AnyErrors() {
		return cl.Succeeded("deletesess")
	}

	if err := s.userCache.Purge(user.SessId); err != nil {
		log.Errorf("[Auth] Error purging session cache: %v", err)
	}

	s.Lock()
	defer s.Unlock()

	s.authUser = nil
	s.triedAuth = false

	return nil
}

// jsonschemaHandler returns all registered endpoints in json schema format as per ITF draft4
// http://json-schema.org/latest/json-schema-core.html
func jsonschemaHandler(req *Request) (proto.Message, errors.Error) {
	// Get all endpoints
	request := req.Data().(*jsonschemaproto.Request)
	endpoint := request.GetEndpoint()

	endpoints := reg.iterate()
	schemas := make([]*jsonschema.JsonSchema, 0)
	for _, ep := range endpoints {
		if endpoint != "" && endpoint != ep.GetName() {
			continue
		}
		schema, err := marshalEndpoint(ep)
		if err == nil && schema != nil {
			schemas = append(schemas, schema)
		}
	}

	rsp, err := json.Marshal(schemas)
	if err != nil {
		return nil, errors.InternalServerError("com.hailocab.kernel.marshal.error", fmt.Sprintf("Unable to unmarshal response data: %v", err.Error()))
	}

	return &jsonschemaproto.Response{
		Jsonschema: proto.String(string(rsp)),
	}, nil
}

func (s *realScope) doAuth(mech, device string, creds map[string]string) (*User, error) {
	reqProto := &authproto.Request{
		Mech:       proto.String(mech),
		DeviceType: proto.String(device),
		Meta:       make([]*loginproto.KeyValue, 0),
	}
	for k, v := range creds {
		switch k {
		case "username":
			reqProto.Username = proto.String(v)
		case "password":
			reqProto.Password = proto.String(v)
		case "newPassword":
			reqProto.NewPassword = proto.String(v)
		case "application":
			reqProto.Application = proto.String(v)
		default:
			// Add additional fields to Meta, such as DeviceId, osVersion, appVersion
			reqProto.Meta = append(reqProto.Meta, &loginproto.KeyValue{
				Key:   proto.String(k),
				Value: proto.String(v),
			})
		}
	}

	cl := multiclient.New().DefaultScopeFrom(s.getRpcScope())
	rsp := &authproto.Response{}
	cl.AddScopedReq(&multiclient.ScopedReq{
		Uid:      "auth",
		Service:  loginService,
		Endpoint: authEndpoint,
		Req:      reqProto,
		Rsp:      rsp,
		Options:  client.Options{"retries": 0},
	})

	if cl.Execute().AnyErrors() {
		// specfically map out bad credentials error
		err := cl.Succeeded("auth")
		if err.Code() == badCredentialsErrCode {
			return nil, BadCredentialsError
		}
		return nil, err
	}

	// recover this user
	u, err := FromSessionToken(rsp.GetSessId(), rsp.GetToken())
	if err != nil {
		return nil, err
	}

	if err := s.userCache.Store(u); err != nil {
		log.Errorf("[Auth] Error caching session: %v", err)
	}

	return u, nil
}

func (s *stats) endpoint(ep Endpoint, t endpointTimers) *pstats.EndpointStats {
	return &pstats.EndpointStats{
		EndpointName: proto.String(ep.GetName()),
		Sla:          endpointSLA(ep),
		Success:      endpointStat(t.success),
		Error:        endpointStat(t.errors),
	}
}

// publishFailure publishes a failure/panic event to be monitored.
func publishFailure(r interface{}) {
	var p string

	switch r.(type) {
	case string:
		p = r.(string)
	case error:
		p = fmt.Sprintf("%v", r.(error))
	default:
		p = "Unknown panic"
	}

	b := make([]byte, 1024)
	runtime.Stack(b, true)

	if err := client.Pub("com.hailocab.monitor.failure", &fproto.Failure{
		ServiceName:    proto.String(Name),
		ServiceVersion: proto.Uint64(Version),
		AzName:         proto.String(az),
		Hostname:       proto.String(hostname),
		InstanceId:     proto.String(InstanceID),
		Timestamp:      proto.Int64(time.Now().Unix()),
		Uptime:         proto.Int64(int64(time.Since(serviceStarted).Seconds())),
		Type:           proto.String("PANIC"),
		Reason:         proto.String(p),
		Stack:          proto.String(string(b)),
	}); err != nil {
		log.Errorf("[Server] Failed to publish failure event: %v", err)
	}
}

func TestResponder(t *testing.T) {
	stub := &Stub{
		Service:  mockFooService,
		Endpoint: mockHealthEndpoint,
		Responder: func(invocation int, req *client.Request) (proto.Message, errors.Error) {
			if invocation == 1 {
				return &hcproto.Response{
					Healthchecks: []*hcproto.HealthCheck{
						&hcproto.HealthCheck{
							Timestamp:      proto.Int64(1403629015),
							ServiceName:    proto.String("foo"),
							ServiceVersion: proto.Uint64(1403629015),
							Hostname:       proto.String("localhost"),
							InstanceId:     proto.String("foobar"),
							HealthCheckId:  proto.String("boom"),
							IsHealthy:      proto.Bool(true),
						},
					},
				}, nil
			}
			return nil, errors.InternalServerError("only.one.allowed", "First call only works")
		},
	}
	mock := NewMock().Stub(stub)

	caller := mock.Caller()
	req, _ := client.NewRequest(mockFooService, mockHealthEndpoint, &hcproto.Request{})
	rsp := &hcproto.Response{}
	e := caller(req, rsp)

	assert.Nil(t, e,
		"Expecting our mocked call to be intercepted and stubbed response returned, got err: %v", e)

	assert.Len(t, rsp.GetHealthchecks(), 1,
		"Response does not contain our mocked content: no healthchecks")

	// now repeat, and we SHOULD get an error
	e = caller(req, rsp)
	assert.NotNil(t, e,
		"Expecting our mocked call to be intercepted and error response returned on 2nd call")

	assert.Equal(t, e.Code(), "only.one.allowed",
		"Expecting code 'only.one.allowed', got '%s'", e.Code())
}

// doRecoverSession is the meat and veg for RecoverSession
func (s *realScope) doRecoverSession(sessId string) (*User, error) {
	// Check cache; ignore errors (will have impact on service performance, but not functionality)
	queryLogin := false
	u, hit, err := s.userCache.Fetch(sessId)
	if err != nil {
		log.Warnf("[Auth] Error fetching session from cache (will call login service): %v", err)
		queryLogin = true
	} else if u != nil && u.ExpiryTs.Before(time.Now()) && u.CanAutoRenew() { // Cached token has expired
		log.Infof("[Auth] Cache-recovered token has expired (%s); will call login service", u.ExpiryTs.String())
		queryLogin = true
	} else {
		queryLogin = u == nil && !hit
	}

	if queryLogin {
		cl := multiclient.New().DefaultScopeFrom(s.getRpcScope())
		rsp := &sessreadproto.Response{}
		cl.AddScopedReq(&multiclient.ScopedReq{
			Uid:      "readsess",
			Service:  loginService,
			Endpoint: readSessionEndpoint,
			Req: &sessreadproto.Request{
				SessId: proto.String(sessId),
			},
			Rsp: rsp,
		})

		if cl.Execute().AnyErrorsIgnoring([]string{errors.ErrorNotFound}, nil) {
			err := cl.Succeeded("readsess")
			log.Errorf("[Auth] Auth scope recovery error [%s: %s] %v", err.Type(), err.Code(), err.Description())
			return nil, err
		}

		// found a session?
		if rsp.GetSessId() == "" && rsp.GetToken() == "" {
			log.Debugf("[Auth] Session '%s' not found (not valid) when trying to recover from login service", sessId)
			// @todo we could cache this (at least for a short time) to prevent repeated hammering of login service
		} else {
			u, err = FromSessionToken(rsp.GetSessId(), rsp.GetToken())
			if err != nil {
				log.Errorf("[Auth] Error getting user from session: %v", err)
			} else {
				log.Tracef("[Auth] Auth scope - recovered user '%s' from session '%s'", u.Id, rsp.GetSessId())
			}
		}

		// ignore errors; just means we have no user
		if u != nil {
			s.userCache.Store(u)
		}
	}

	return u, nil
}

// TestRecoverSessionHappyFound tests happy case (no service call failures) when we do find a user
func (suite *sessionRecoverySuite) TestRecoverSessionHappyFound() {
	scope := New().(*realScope)
	scope.userCache = newTestCache()

	mock := multiclient.NewMock()
	stub := &multiclient.Stub{
		Service:  loginService,
		Endpoint: readSessionEndpoint,
		Response: &sessreadproto.Response{
			SessId: proto.String(testSessId),
			Token:  proto.String(testToken),
		},
	}
	mock.Stub(stub)
	multiclient.SetCaller(mock.Caller())

	err := scope.RecoverSession(testSessId)
	suite.Assertions.NoError(err, "Unexpected recovery error")
	suite.Assertions.True(scope.IsAuth())
	suite.Assertions.True(scope.HasTriedAuth())
	suite.Assertions.Equal(1, stub.CountCalls(), "Expecting 1 call to readsession")
	req := &sessreadproto.Request{}
	err = stub.Request(0).Unmarshal(req)
	suite.Assertions.NoError(err)
	suite.Assertions.Equal(testSessId, req.GetSessId())
	suite.Assertions.False(req.GetNoRenew())
	u := scope.authUser
	suite.Assertions.NotNil(u)
	suite.Assertions.Equal("dave", u.Id)

	// Clean out scope
	scope.Clean()
	suite.Assertions.False(scope.IsAuth(), "Expecting scope to be IsAuth==false after Clean()")
	suite.Assertions.False(scope.HasTriedAuth(), "Expecting scope to have HasTriedAuth()==false after Clean()")
	suite.Assertions.Nil(scope.AuthUser(), "Expecting AuthUser()==nil after Clean()")

	// recover AGAIN -- this time it shoud be cached
	suite.Assertions.NoError(scope.RecoverSession(testSessId), "Unexpected recovery error")
	suite.Assertions.True(scope.IsAuth())
	suite.Assertions.Equal(1, stub.CountCalls(), "Expecting 1 call to readsession (should be cached now)")
}

// reloadSlas loads timeouts from discovery service for all services we know about (have tried to call)
func (t *Timeout) reloadSlas() {
	replacement := make(map[string]map[string]time.Duration)

	for service := range t.endpoints {
		// load from discovery service
		log.Debugf("[Client] Loading SLAs from discovery service for %v...", service)
		req, err := NewRequest("com.hailocab.kernel.discovery", "endpoints", &eps.Request{
			Service: proto.String(service),
		})
		if err != nil {
			log.Warnf("[Client] Failed to create proto request to get endpoints for service: %s", service)
			continue
		}
		rsp := &eps.Response{}
		// explicitly define timeout since we're in no rush
		if err := t.client.Req(req, rsp, Options{"retries": 0, "timeout": time.Second * 5}); err != nil {
			log.Warnf("[Client] Trouble getting endpoint response back from discovery-service for service: %s", service)
			continue
		}

		for _, ep := range rsp.GetEndpoints() {
			endpoint := strings.TrimLeft(strings.TrimPrefix(ep.GetFqName(), service), ".")
			if _, ok := replacement[service]; !ok {
				replacement[service] = make(map[string]time.Duration)
			}
			replacement[service][endpoint] = msToDuration(ep.GetUpper95())
		}
	}

	// double check we have all the things we started with -- if not, but back the "last known" (probably defaults)
	for service, serviceEndpoints := range t.endpoints {
		for endpoint, timeout := range serviceEndpoints {
			if _, ok := replacement[service]; !ok {
				replacement[service] = make(map[string]time.Duration)
			}
			if _, ok := replacement[service][endpoint]; !ok {
				log.Debugf("[Client] Failed to find SLA for %s.%s, falling back to %v", service, endpoint, timeout)
				replacement[service][endpoint] = timeout
			}
		}
	}

	// SLAs changed? if not, don't bother switching+logging
	if hashSlas(replacement) == t.hashEndpoints() {
		return
	}

	t.Lock()
	defer t.Unlock()
	t.endpoints = replacement

	log.Infof("[Client] Loaded new SLAs from discovery service: %v", t.endpoints)
}

// traceIn traces a request inbound to a service to handle
func traceIn(req *Request) {
	if req.shouldTrace() {
		go trace.Send(&traceproto.Event{
			Timestamp:         proto.Int64(time.Now().UnixNano()),
			TraceId:           proto.String(req.TraceID()),
			Type:              traceproto.Event_IN.Enum(),
			MessageId:         proto.String(req.MessageID()),
			ParentMessageId:   proto.String(req.ParentMessageID()),
			From:              proto.String(req.From()),
			To:                proto.String(fmt.Sprintf("%v.%v", req.Service(), req.Endpoint())),
			Hostname:          proto.String(hostname),
			Az:                proto.String(az),
			Payload:           proto.String(""), // @todo
			HandlerInstanceId: proto.String(InstanceID),
			PersistentTrace:   proto.Bool(req.TraceShouldPersist()),
		})
	}
}

// traceReq decides if we want to trigger a trace event (when sending a request) and if so deals with it
func (c *client) traceReq(req *Request) {
	if req.shouldTrace() {
		trace.Send(&traceproto.Event{
			Timestamp:       proto.Int64(time.Now().UnixNano()),
			TraceId:         proto.String(req.TraceID()),
			Type:            traceproto.Event_REQ.Enum(),
			MessageId:       proto.String(req.MessageID()),
			ParentMessageId: proto.String(req.ParentMessageID()),
			From:            proto.String(req.From()),
			FromEndpoint:    proto.String(req.FromEndpoint()),
			To:              proto.String(fmt.Sprintf("%v.%v", req.Service(), req.Endpoint())),
			Hostname:        proto.String(c.hostname),
			Az:              proto.String(c.az),
			Payload:         proto.String(""), // @todo
			PersistentTrace: proto.Bool(req.TraceShouldPersist()),
		})
	}
}

func Foo(req h2.Request) (proto.Message, h2.Error) {
	request := req.Data().(*foo.Request)
	log.Debugf("Received bar=%v", request.GetBar())

	s := services.Response{}
	err := h2.Call("com.hailocab.kernel.discovery", "services", &services.Request{}, &s)
	if err != nil {
		log.Warnf("Ouch... the discovery services seems to be down ): %v", err)
	}
	rsp := &foo.Response{
		Baz: proto.String(fmt.Sprintf("There are %v services running on your h2 cluster", len(s.GetServices()))),
	}
	return rsp, nil
}

// load config via login service
func (s *serviceToService) load() error {
	svc := s.getService()
	if svc == "" {
		log.Debug("[Auth] Skipping loading service-to-service auth rules (no service defined)")
		return nil
	}

	log.Tracef("[Auth] Loading service-to-service auth rules for %s", svc)
	reqProto := &endpointauth.Request{
		Service: proto.String(svc),
	}
	req, err := client.NewRequest("com.hailocab.service.login", "endpointauth", reqProto)
	if err != nil {
		return err
	}
	// scope it
	req.SetFrom(svc)
	rsp := &endpointauth.Response{}
	if err := client.Req(req, rsp); err != nil {
		return err
	}

	newEndpoints := make(map[string]grantedServices)

	for _, ep := range rsp.GetEndpoints() {
		name := ep.GetEndpoint()
		if _, ok := newEndpoints[name]; !ok {
			newEndpoints[name] = make(grantedServices)
		}
		// add in the granted services to this endpoint
		for _, gs := range ep.GetGranted() {
			newEndpoints[name][gs.GetName()] = role(gs.GetRole())
		}
	}

	// check if changed - to avoid locking/changing/logging if not
	if hashEndpoints(newEndpoints) == s.hash() {
		return nil
	}

	// switch in config
	s.Lock()
	defer s.Unlock()
	s.endpoints = newEndpoints

	log.Debugf("[Auth] Loaded service-to-service auth rules: %#v", s.endpoints)

	return nil
}

func Endpoint(name string, configStruct interface{}) *service.Endpoint {
	handler := func(req *server.Request) (proto.Message, errors.Error) {
		return &schemaProto.Response{
			Schema: proto.String(schema.Of(configStruct).String()),
		}, nil
	}

	return &server.Endpoint{
		Name:       name,
		Mean:       200,
		Upper95:    400,
		Handler:    handler,
		Authoriser: service.OpenToTheWorldAuthoriser(),
	}
}

// Hello provides a func to process an incoming 'hello' request.
// A handler requires a request parameter and will return either a response or an error
func Hello(req h2.Request) (proto.Message, h2.Error) {
	request := req.Data().(*protoHello.Request)

	// Error example
	if len(request.GetName()) == 0 {
		// We specify multi error types in the platform layer repository in the 'errors'
		// package. See: https://godoc.org/github.com/hailocab/platform-layer/errors
		return nil, errors.BadRequest(HelloEndpoint, "You didn't specify a name in the request")
	}

	// Return response with the message set to `Hello <name>`
	return &protoHello.Response{
		Message: proto.String(fmt.Sprintf("Hello %q", request.GetName())),
	}, nil
}

func TestMockCallerPopulatesResponse(t *testing.T) {
	req, _ := client.NewRequest(mockFooService, mockHealthEndpoint, &hcproto.Request{})
	stub := &Stub{
		Service:  mockFooService,
		Endpoint: mockHealthEndpoint,
		Response: &hcproto.Response{
			Healthchecks: []*hcproto.HealthCheck{
				&hcproto.HealthCheck{
					Timestamp:      proto.Int64(1403629015),
					ServiceName:    proto.String("foo"),
					ServiceVersion: proto.Uint64(1403629015),
					Hostname:       proto.String("localhost"),
					InstanceId:     proto.String("foobar"),
					HealthCheckId:  proto.String("boom"),
					IsHealthy:      proto.Bool(true),
				},
			},
		},
	}
	mock := NewMock().Stub(stub)

	caller := mock.Caller()
	rsp := &hcproto.Response{}
	e := caller(req, rsp)
	assert.Nil(t, e,
		"Expecting our mocked call to be intercepted and stubbed response returned, got err: %v", e)

	// ensure stub has what we expect
	assert.Len(t, stub.matched, 1,
		"Expecting 1 match payload to be stored after execution")

	assert.Equal(t, stub.CountCalls(), 1, "CountCalls should return 1 too")

	assert.Len(t, rsp.GetHealthchecks(), 1,
		"Response does not contain our mocked content: no healthchecks")
}