func NewGRPCClientDialOpt(c *Config) (grpc.DialOption, error) {
	var opt grpc.DialOption
	if c.InsecureSkipVerify {
		return grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{InsecureSkipVerify: true})), nil
	}
	tlsConf, err := NewClientTLSConfig(c)
	if err != nil {
		return opt, err
	}
	return grpc.WithTransportCredentials(credentials.NewTLS(tlsConf)), nil
}

func (t *grpcTransportListener) Accept(fn func(transport.Socket)) error {
	var opts []grpc.ServerOption

	// setup tls if specified
	if t.secure || t.tls != nil {
		config := t.tls
		if config == nil {
			var err error
			addr := t.listener.Addr().String()
			config, err = getTLSConfig(addr)
			if err != nil {
				return err
			}
		}

		creds := credentials.NewTLS(config)
		opts = append(opts, grpc.Creds(creds))
	}

	// new service
	srv := grpc.NewServer(opts...)

	// register service
	pb.RegisterTransportServer(srv, &microTransport{fn: fn})

	// start serving
	return srv.Serve(t.listener)
}

// CheckFlags checks that the required config values are set.
func (c *Config) CheckFlags(required RequiredFlags) error {
	var missing []string
	if c.CertFile != "" {
		b, err := ioutil.ReadFile(c.CertFile)
		if err != nil {
			return fmt.Errorf("Failed to load certificates from %s: %v", c.CertFile, err)
		}

		cp := x509.NewCertPool()
		if !cp.AppendCertsFromPEM(b) {
			return fmt.Errorf("Failed to append certificates from %s", c.CertFile)
		}

		c.TLSCreds = credentials.NewTLS(&tls.Config{RootCAs: cp})
	}
	if required != NoneRequired {
		c.SetFromGcloud()
	}
	if required&ProjectRequired != 0 && c.Project == "" {
		missing = append(missing, "-project")
	}
	if required&InstanceRequired != 0 && c.Instance == "" {
		missing = append(missing, "-instance")
	}
	if len(missing) > 0 {
		return fmt.Errorf("Missing %s", strings.Join(missing, " and "))
	}
	return nil
}

func (s *SRVLoader) getConfig() (*pb.NodeConfig, error) {
	nconfig := &pb.NodeConfig{}
	var opts []grpc.DialOption
	var creds credentials.TransportAuthenticator
	creds = credentials.NewTLS(&tls.Config{
		InsecureSkipVerify: true,
	})
	opts = append(opts, grpc.WithTransportCredentials(creds))
	conn, err := grpc.Dial(s.SyndicateURL, opts...)
	if err != nil {
		return nconfig, fmt.Errorf("Failed to dial ring server for config: %s", err)
	}
	defer conn.Close()

	client := pb.NewSyndicateClient(conn)

	ctx, _ := context.WithTimeout(context.Background(), 10*time.Second)

	rr := &pb.RegisterRequest{}
	rr.Hostname, _ = os.Hostname()
	addrs, _ := net.InterfaceAddrs()
	for k, _ := range addrs {
		rr.Addrs = append(rr.Addrs, addrs[k].String())
	}
	rr.Hardware, err = GetHardwareProfile()
	if err != nil {
		return nconfig, err
	}
	rr.Tiers = []string{rr.Hostname}

	nconfig, err = client.RegisterNode(ctx, rr)
	return nconfig, err
}

func newClient(cfg *Config) (*Client, error) {
	if cfg == nil {
		cfg = &Config{RetryDialer: dialEndpointList}
	}
	var creds *credentials.TransportAuthenticator
	if cfg.TLS != nil {
		c := credentials.NewTLS(cfg.TLS)
		creds = &c
	}
	// use a temporary skeleton client to bootstrap first connection
	ctx, cancel := context.WithCancel(context.TODO())
	conn, err := cfg.RetryDialer(&Client{cfg: *cfg, creds: creds, ctx: ctx})
	if err != nil {
		return nil, err
	}
	client := &Client{
		conn:   conn,
		cfg:    *cfg,
		creds:  creds,
		ctx:    ctx,
		cancel: cancel,
	}
	client.Cluster = NewCluster(client)
	client.KV = NewKV(client)
	client.Lease = NewLease(client)
	client.Watcher = NewWatcher(client)
	client.Auth = NewAuth(client)
	client.Maintenance = &maintenance{c: client}

	return client, nil
}

func (cc *CCServer) Serve() error {
	l, err := net.Listen("tcp", cc.cfg.ListenAddress)
	if err != nil {
		return err
	}
	var opts []grpc.ServerOption
	if cc.cfg.UseTLS {
		if cc.cfg.MutualTLS {
			mtc, err := mutualTLS(*cc.cfg)
			if err != nil {
				return err
			}
			creds := credentials.NewTLS(mtc)
			opts = []grpc.ServerOption{grpc.Creds(creds)}
		} else {
			creds, err := credentials.NewServerTLSFromFile(cc.cfg.CertFile, cc.cfg.KeyFile)
			if err != nil {
				return err
			}
			opts = []grpc.ServerOption{grpc.Creds(creds)}
		}
	}
	s := grpc.NewServer(opts...)
	pb.RegisterCmdCtrlServer(s, cc)
	return s.Serve(l)
}

func GroupWrite(c *GroupClientConfig) {
	defer c.wg.Done()
	var err error
	var opts []grpc.DialOption
	var creds credentials.TransportAuthenticator
	creds = credentials.NewTLS(&tls.Config{
		InsecureSkipVerify: true,
	})
	opts = append(opts, grpc.WithTransportCredentials(creds))
	conn, err := grpc.Dial(c.addr, opts...)
	if err != nil {
		log.Fatalln(fmt.Sprintf("Failed to dial server: %s", err))
	}
	defer conn.Close()
	client := gp.NewGroupStoreClient(conn)
	w := &gp.WriteRequest{
		Value: *c.value,
	}
	empty := []byte("")
	for i, _ := range c.wm {
		ctx, _ := context.WithTimeout(context.Background(), 10*time.Second)
		c.wm[i].Value = *c.value
		w.TimestampMicro = brimtime.TimeToUnixMicro(time.Now())
		res, err := client.Write(ctx, c.wm[i])
		if err != nil {
			log.Println("Client", c.id, ":", err)
		}
		if res.TimestampMicro > w.TimestampMicro {
			log.Printf("TSM is newer than attempted, Key %d-%d Got %s, Sent: %s", c.id, i, brimtime.UnixMicroToTime(res.TimestampMicro), brimtime.UnixMicroToTime(w.TimestampMicro))
		}
		c.wm[i].Value = empty
	}
}

func GroupStreamRead(c *GroupClientConfig) {
	defer c.wg.Done()
	var err error
	var opts []grpc.DialOption
	var creds credentials.TransportAuthenticator
	creds = credentials.NewTLS(&tls.Config{
		InsecureSkipVerify: true,
	})
	opts = append(opts, grpc.WithTransportCredentials(creds))
	conn, err := grpc.Dial(c.addr, opts...)
	if err != nil {
		log.Fatalln(fmt.Sprintf("Failed to dial server: %s", err))
	}
	defer conn.Close()
	client := gp.NewGroupStoreClient(conn)
	stream, err := client.StreamRead(context.Background())
	for i, _ := range c.rm {
		if err := stream.Send(c.rm[i]); err != nil {
			log.Println(err)
			continue
		}
		_, err := stream.Recv()
		if err == io.EOF {
			return
		}
		if err != nil {
			log.Println(err)
			continue
		}
	}
	stream.CloseSend()
}

// serverWithAuth builds a gRPC server, possibly with authentication if key / cert files are given.
func serverWithAuth(keyFile, certFile, caCertFile string) *grpc.Server {
	if keyFile == "" {
		return grpc.NewServer(grpc.MaxMsgSize(maxMsgSize)) // No auth.
	}
	log.Debug("Loading x509 key pair from key: %s cert: %s", keyFile, certFile)
	cert, err := tls.LoadX509KeyPair(certFile, keyFile)
	if err != nil {
		log.Fatalf("Failed to load x509 key pair: %s", err)
	}
	config := tls.Config{
		Certificates: []tls.Certificate{cert},
		ClientAuth:   tls.RequestClientCert,
	}
	if caCertFile != "" {
		cert, err := ioutil.ReadFile(caCertFile)
		if err != nil {
			log.Fatalf("Failed to read CA cert file: %s", err)
		}
		config.ClientCAs = x509.NewCertPool()
		if !config.ClientCAs.AppendCertsFromPEM(cert) {
			log.Fatalf("Failed to find any PEM certificates in CA cert")
		}
	}
	return grpc.NewServer(grpc.Creds(credentials.NewTLS(&config)), grpc.MaxMsgSize(maxMsgSize))
}

// NewServer loads various TLS certificates and creates a
// gRPC Server that verifies the client certificate was
// issued by the provided issuer certificate and presents a
// a server TLS certificate.
func NewServer(c *cmd.GRPCServerConfig, stats metrics.Scope) (*grpc.Server, net.Listener, error) {
	cert, err := tls.LoadX509KeyPair(c.ServerCertificatePath, c.ServerKeyPath)
	if err != nil {
		return nil, nil, err
	}
	clientIssuerBytes, err := ioutil.ReadFile(c.ClientIssuerPath)
	if err != nil {
		return nil, nil, err
	}
	clientCAs := x509.NewCertPool()
	if ok := clientCAs.AppendCertsFromPEM(clientIssuerBytes); !ok {
		return nil, nil, errors.New("Failed to parse client issuer certificates")
	}
	servConf := &tls.Config{
		Certificates: []tls.Certificate{cert},
		ClientAuth:   tls.RequireAndVerifyClientCert,
		ClientCAs:    clientCAs,
	}
	creds := credentials.NewTLS(servConf)
	l, err := net.Listen("tcp", c.Address)
	if err != nil {
		return nil, nil, err
	}
	si := &serverInterceptor{stats, clock.Default()}
	return grpc.NewServer(grpc.Creds(creds), grpc.UnaryInterceptor(si.intercept)), l, nil
}

// GetRemoteCA returns the remote endpoint's CA certificate
func GetRemoteCA(ctx context.Context, d digest.Digest, picker *picker.Picker) (RootCA, error) {
	// We need a valid picker to be able to Dial to a remote CA
	if picker == nil {
		return RootCA{}, fmt.Errorf("valid remote address picker required")
	}

	// This TLS Config is intentionally using InsecureSkipVerify. Either we're
	// doing TOFU, in which case we don't validate the remote CA, or we're using
	// a user supplied hash to check the integrity of the CA certificate.
	insecureCreds := credentials.NewTLS(&tls.Config{InsecureSkipVerify: true})
	opts := []grpc.DialOption{
		grpc.WithTransportCredentials(insecureCreds),
		grpc.WithBackoffMaxDelay(10 * time.Second),
		grpc.WithPicker(picker)}

	firstAddr, err := picker.PickAddr()
	if err != nil {
		return RootCA{}, err
	}

	conn, err := grpc.Dial(firstAddr, opts...)
	if err != nil {
		return RootCA{}, err
	}
	defer conn.Close()

	client := api.NewCAClient(conn)
	response, err := client.GetRootCACertificate(ctx, &api.GetRootCACertificateRequest{})
	if err != nil {
		return RootCA{}, err
	}

	if d != "" {
		verifier, err := digest.NewDigestVerifier(d)
		if err != nil {
			return RootCA{}, fmt.Errorf("unexpected error getting digest verifier: %v", err)
		}

		io.Copy(verifier, bytes.NewReader(response.Certificate))

		if !verifier.Verified() {
			return RootCA{}, fmt.Errorf("remote CA does not match fingerprint. Expected: %s", d.Hex())

		}
	}

	// Check the validity of the remote Cert
	_, err = helpers.ParseCertificatePEM(response.Certificate)
	if err != nil {
		return RootCA{}, err
	}

	// Create a Pool with our RootCACertificate
	pool := x509.NewCertPool()
	if !pool.AppendCertsFromPEM(response.Certificate) {
		return RootCA{}, fmt.Errorf("failed to append certificate to cert pool")
	}

	return RootCA{Cert: response.Certificate, Pool: pool}, nil
}

func (s *OortGroupStore) ListenAndServe() {
	go func(s *OortGroupStore) {
		s.grpcStopping = false
		for {
			var err error
			listenAddr := s.oort.GetListenAddr()
			if listenAddr == "" {
				log.Fatalln("No listen address specified in ring at address2")
			}
			l, err := net.Listen("tcp", listenAddr)
			if err != nil {
				log.Fatalln("Unable to bind to address:", err)
			}
			log.Println("GroupStore bound to:", listenAddr)
			var opts []grpc.ServerOption
			creds := credentials.NewTLS(s.serverTLSConfig)
			opts = []grpc.ServerOption{grpc.Creds(creds)}
			s.grpc = grpc.NewServer(opts...)
			groupproto.RegisterGroupStoreServer(s.grpc, s)
			err = s.grpc.Serve(l)
			if err != nil && !s.grpcStopping {
				log.Println("GroupStore Serve encountered error:", err, "will attempt to restart")
			} else if err != nil && s.grpcStopping {
				log.Println("GroupStore got error but halt is in progress:", err)
				l.Close()
				break
			} else {
				log.Println("GroupStore Serve exited without error, quiting")
				l.Close()
				break
			}
		}
	}(s)
}

// RegisterGRPCGateway starts the gateway (i.e. reverse proxy) that proxies
// HTTP requests to the appropriate gRPC endpoints.
func (s *adminServer) RegisterGRPCGateway(serverCtx *Context) error {
	// Setup HTTP<->gRPC handlers.
	var opts []grpc.DialOption
	if serverCtx.Insecure {
		opts = append(opts, grpc.WithInsecure())
	} else {
		tlsConfig, err := serverCtx.GetClientTLSConfig()
		if err != nil {
			return err
		}
		opts = append(
			opts,
			// TODO(tamird): remove this timeout. It is currently necessary because
			// GRPC will not actually bail on a bad certificate error - it will just
			// retry indefinitely. See https://github.com/grpc/grpc-go/issues/622.
			grpc.WithTimeout(time.Second),
			grpc.WithBlock(),
			grpc.WithTransportCredentials(credentials.NewTLS(tlsConfig)),
		)
	}
	if err := RegisterAdminHandlerFromEndpoint(s.gwCtx, s.gwMux, serverCtx.Addr, opts); err != nil {
		return util.Errorf("error constructing grpc-gateway: %s. are your certificates valid?", err)
	}

	// Pass all requests for gRPC-based API endpoints to the gateway mux.
	s.ServeMux.Handle(apiEndpoint, s.gwMux)
	return nil
}

func (n *Node) initManagerConnection(ctx context.Context, ready chan<- struct{}) error {
	opts := []grpc.DialOption{}
	insecureCreds := credentials.NewTLS(&tls.Config{InsecureSkipVerify: true})
	opts = append(opts, grpc.WithTransportCredentials(insecureCreds))
	// Using listen address instead of advertised address because this is a
	// local connection.
	addr := n.config.ListenControlAPI
	opts = append(opts, grpc.WithDialer(
		func(addr string, timeout time.Duration) (net.Conn, error) {
			return net.DialTimeout("unix", addr, timeout)
		}))
	conn, err := grpc.Dial(addr, opts...)
	if err != nil {
		return err
	}
	client := api.NewHealthClient(conn)
	for {
		resp, err := client.Check(ctx, &api.HealthCheckRequest{Service: "ControlAPI"})
		if err != nil {
			return err
		}
		if resp.Status == api.HealthCheckResponse_SERVING {
			break
		}
		time.Sleep(500 * time.Millisecond)
	}
	n.setControlSocket(conn)
	if ready != nil {
		close(ready)
	}
	return nil
}

func benchmarkEchoGRPC(b *testing.B, size int) {
	var conn *grpc.ClientConn
	var client EchoClient
	benchmarkEcho(b, size, listenAndServeGRPC,
		func(addr net.Addr) {
			var err error
			conn, err = grpc.Dial(addr.String(), grpc.WithTransportCredentials(credentials.NewTLS(clientTLSConfig)))
			if err != nil {
				b.Fatal(err)
			}
			client = NewEchoClient(conn)
		},
		func() {
			if err := conn.Close(); err != nil {
				b.Fatal(err)
			}
		},
		func(echoMsg string) string {
			resp, err := client.Echo(context.Background(), &EchoRequest{Msg: echoMsg})
			if err != nil {
				b.Fatal(err)
			}
			return resp.Msg
		},
	)
}

func New(addr string, certFile string, keyFile string) (_ *Manager, err error) {
	c := &Manager{}

	cert, err := tls.LoadX509KeyPair(certFile, keyFile)
	if err != nil {
		return nil, err
	}
	cert.Leaf, err = x509.ParseCertificate(cert.Certificate[0])
	if err != nil {
		return nil, err
	}

	c.Cert = &cert
	c.Self, err = NewPeerInfo("", cert.Certificate[0])

	c.tlsConfig = &tls.Config{
		Certificates:       []tls.Certificate{cert},
		ClientAuth:         tls.RequestClientCert,
		InsecureSkipVerify: true,
	}

	c.Listener, err = net.Listen("tcp", addr)
	if err != nil {
		return nil, err
	}

	serverTls := c.tlsConfig
	serverTls.ServerName = addr
	c.Server = grpc.NewServer(grpc.Creds(credentials.NewTLS(serverTls)))
	go c.Server.Serve(c.Listener)
	return c, nil
}

func (n *Node) initManagerConnection(ctx context.Context, ready chan<- struct{}) error {
	opts := []grpc.DialOption{}
	insecureCreds := credentials.NewTLS(&tls.Config{InsecureSkipVerify: true})
	opts = append(opts, grpc.WithTransportCredentials(insecureCreds))
	// Using listen address instead of advertised address because this is a
	// local connection.
	addr := n.config.ListenControlAPI
	opts = append(opts, grpc.WithDialer(
		func(addr string, timeout time.Duration) (net.Conn, error) {
			return net.DialTimeout("unix", addr, timeout)
		}))
	conn, err := grpc.Dial(addr, opts...)
	if err != nil {
		return err
	}
	state := grpc.Idle
	for {
		s, err := conn.WaitForStateChange(ctx, state)
		if err != nil {
			n.setControlSocket(nil)
			return err
		}
		if s == grpc.Ready {
			n.setControlSocket(conn)
			if ready != nil {
				close(ready)
				ready = nil
			}
		} else if state == grpc.Shutdown {
			n.setControlSocket(nil)
		}
		state = s
	}
}

func main() {
	flag.Parse()

	var err error

	tls, err := createTLS("127.0.0.1")
	if err != nil {
		fmt.Printf("Could not create TLS config %s\n", err)
	}

	creds := credentials.NewTLS(tls)

	conn, err = grpc.Dial(*clientAddress, grpc.WithTransportCredentials(creds))
	if err != nil {
		fmt.Printf("Could not dial client %s\n", err)
	}
	defer conn.Close()

	client = pb.NewTestServiceClient(conn)

	router := mux.NewRouter()
	router.StrictSlash(true)
	router.HandleFunc("/", indexHandler)
	router.HandleFunc("/hello/", indexHandler)
	router.HandleFunc("/hello/{name}", helloHandler).Methods("GET")
	fmt.Printf("Starting server at %s\n", *bind)
	err = http.ListenAndServe(*bind, router)
	if err != nil {
		fmt.Println(err)
	}
}

// NewGRPCConnection is a convenience method that returns GRPC Client Connection given a hostname, endpoint, and TLS options
func NewGRPCConnection(hostname string, port string, tlsConfig *tls.Config) (*grpc.ClientConn, error) {
	var opts []grpc.DialOption
	netAddr := net.JoinHostPort(hostname, port)
	creds := credentials.NewTLS(tlsConfig)
	opts = append(opts, grpc.WithTransportCredentials(creds))
	return grpc.Dial(netAddr, opts...)
}

func (c *Client) dialTarget(endpoint string) (proto string, host string, creds *credentials.TransportCredentials) {
	proto = "tcp"
	host = endpoint
	creds = c.creds
	url, uerr := url.Parse(endpoint)
	if uerr != nil || !strings.Contains(endpoint, "://") {
		return
	}
	// strip scheme:// prefix since grpc dials by host
	host = url.Host
	switch url.Scheme {
	case "unix":
		proto = "unix"
	case "http":
		creds = nil
	case "https":
		if creds != nil {
			break
		}
		tlsconfig := &tls.Config{}
		emptyCreds := credentials.NewTLS(tlsconfig)
		creds = &emptyCreds
	default:
		return "", "", nil
	}
	return
}