func (cc *CCServer) Serve() error {
	l, err := net.Listen("tcp", cc.cfg.ListenAddress)
	if err != nil {
		return err
	}
	var opts []grpc.ServerOption
	if cc.cfg.UseTLS {
		if cc.cfg.MutualTLS {
			mtc, err := mutualTLS(*cc.cfg)
			if err != nil {
				return err
			}
			creds := credentials.NewTLS(mtc)
			opts = []grpc.ServerOption{grpc.Creds(creds)}
		} else {
			creds, err := credentials.NewServerTLSFromFile(cc.cfg.CertFile, cc.cfg.KeyFile)
			if err != nil {
				return err
			}
			opts = []grpc.ServerOption{grpc.Creds(creds)}
		}
	}
	s := grpc.NewServer(opts...)
	pb.RegisterCmdCtrlServer(s, cc)
	return s.Serve(l)
}

func (rs *RingSyndicates) launchSyndicates(k int) {
	rs.Syndics[k].Lock()
	defer rs.waitGroup.Done()
	rs.waitGroup.Add(1)
	l, err := net.Listen("tcp", fmt.Sprintf(":%d", rs.Syndics[k].config.Port))
	if err != nil {
		log.Fatalln(err)
		return
	}
	var opts []grpc.ServerOption
	creds, err := credentials.NewServerTLSFromFile(rs.Syndics[k].config.CertFile, rs.Syndics[k].config.KeyFile)
	if err != nil {
		log.Fatalln("Error load cert or key:", err)
	}
	opts = []grpc.ServerOption{grpc.Creds(creds)}
	rs.Syndics[k].gs = grpc.NewServer(opts...)

	if rs.Syndics[k].config.Master {
		pb.RegisterSyndicateServer(rs.Syndics[k].gs, rs.Syndics[k].server)
		log.Println("Master starting up on", rs.Syndics[k].config.Port)
		rs.Syndics[k].gs.Serve(l)
	} else {
		//pb.RegisterRingDistServer(s, newRingDistServer())
		//log.Printf("Starting ring slave up on %d...\n", cfg.Port)
		//s.Serve(l)
		log.Fatalln("Syndicate slaves not implemented yet")
	}
	rs.Syndics[k].Unlock()
}

func main() {
	// 2. 解析flags
	flag.Parse()

	// 3. 监听端口
	lis, err := net.Listen("tcp", fmt.Sprintf(":%d", *port))
	if err != nil {
		grpclog.Fatalf("failed to listen: %v", err)
	}

	// 4. 指定TLS
	var opts []grpc.ServerOption
	if *tls {
		creds, err := credentials.NewServerTLSFromFile(*certFile, *keyFile)
		if err != nil {
			grpclog.Fatalf("Failed to generate credentials %v", err)
		}
		opts = []grpc.ServerOption{grpc.Creds(creds)}
	}

	// 关联Server和Handler
	grpcServer := grpc.NewServer(opts...)
	pb.RegisterRouteGuideServer(grpcServer, newServer())
	grpcServer.Serve(lis)
}

// Test the execution of an invalid transaction.
func TestExecuteInvokeInvalidTransaction(t *testing.T) {
	testDBWrapper.CleanDB(t)
	var opts []grpc.ServerOption
	if viper.GetBool("peer.tls.enabled") {
		creds, err := credentials.NewServerTLSFromFile(viper.GetString("peer.tls.cert.file"), viper.GetString("peer.tls.key.file"))
		if err != nil {
			grpclog.Fatalf("Failed to generate credentials %v", err)
		}
		opts = []grpc.ServerOption{grpc.Creds(creds)}
	}
	grpcServer := grpc.NewServer(opts...)
	viper.Set("peer.fileSystemPath", "/var/hyperledger/test/tmpdb")

	//use a different address than what we usually use for "peer"
	//we override the peerAddress set in chaincode_support.go
	peerAddress := "0.0.0.0:21212"

	lis, err := net.Listen("tcp", peerAddress)
	if err != nil {
		t.Fail()
		t.Logf("Error starting peer listener %s", err)
		return
	}

	getPeerEndpoint := func() (*pb.PeerEndpoint, error) {
		return &pb.PeerEndpoint{ID: &pb.PeerID{Name: "testpeer"}, Address: peerAddress}, nil
	}

	ccStartupTimeout := time.Duration(chaincodeStartupTimeoutDefault) * time.Millisecond
	pb.RegisterChaincodeSupportServer(grpcServer, NewChaincodeSupport(DefaultChain, getPeerEndpoint, false, ccStartupTimeout, nil))

	go grpcServer.Serve(lis)

	var ctxt = context.Background()

	url := "github.com/hyperledger/fabric/examples/chaincode/go/chaincode_example02"
	chaincodeID := &pb.ChaincodeID{Path: url}

	//FAIL, FAIL!
	args := []string{"x", "-1"}
	err = invokeExample02Transaction(ctxt, chaincodeID, args, false)

	//this HAS to fail with expectedDeltaStringPrefix
	if err != nil {
		errStr := err.Error()
		t.Logf("Got error %s\n", errStr)
		t.Logf("InvalidInvoke test passed")
		GetChain(DefaultChain).Stop(ctxt, &pb.ChaincodeDeploymentSpec{ChaincodeSpec: &pb.ChaincodeSpec{ChaincodeID: chaincodeID}})

		closeListenerAndSleep(lis)
		return
	}

	t.Fail()
	t.Logf("Error invoking transaction %s", err)

	GetChain(DefaultChain).Stop(ctxt, &pb.ChaincodeDeploymentSpec{ChaincodeSpec: &pb.ChaincodeSpec{ChaincodeID: chaincodeID}})

	closeListenerAndSleep(lis)
}

//Start Start
func Start(port int, tls bool, certFile, keyFile string, ch chan string) {

	log.Println("demo start")
	sid = utils.Randstr20()

	s := new(grpcService)
	lis, err := net.Listen("tcp", fmt.Sprintf(":%d", port))
	if err != nil {
		grpclog.Fatalf("failed to listen: %v", err)
	}
	var opts []grpc.ServerOption
	if tls {
		creds, err := credentials.NewServerTLSFromFile(certFile, keyFile)
		if err != nil {
			grpclog.Fatalf("Failed to generate credentials %v", err)
		}
		opts = []grpc.ServerOption{grpc.Creds(creds)}
	}

	grpcServer := grpc.NewServer(opts...)
	demo.RegisterRPCServer(grpcServer, s)
	go func() {
		<-ch
		grpcServer.Stop()
	}()
	grpcServer.Serve(lis)

	log.Println("demo end")
}

func createGRPCLayer(port int) (*grpc.Server, net.Listener, grpc.DialOption) {
	var s *grpc.Server
	var ll net.Listener
	var err error
	var serverOpts []grpc.ServerOption
	var dialOpts grpc.DialOption

	keyFileName := fmt.Sprintf("key.%d.pem", time.Now().UnixNano())
	certFileName := fmt.Sprintf("cert.%d.pem", time.Now().UnixNano())

	defer os.Remove(keyFileName)
	defer os.Remove(certFileName)

	err = generateCertificates(keyFileName, certFileName)
	if err == nil {
		var creds credentials.TransportCredentials
		creds, err = credentials.NewServerTLSFromFile(certFileName, keyFileName)
		serverOpts = append(serverOpts, grpc.Creds(creds))
		ta := credentials.NewTLS(&tls.Config{
			InsecureSkipVerify: true,
		})
		dialOpts = grpc.WithTransportCredentials(&authCreds{tlsCreds: ta})
	} else {
		dialOpts = grpc.WithInsecure()
	}

	listenAddress := fmt.Sprintf("%s:%d", "", port)
	ll, err = net.Listen("tcp", listenAddress)
	if err != nil {
		panic(err)
	}

	s = grpc.NewServer(serverOpts...)
	return s, ll, dialOpts
}

// serverWithAuth builds a gRPC server, possibly with authentication if key / cert files are given.
func serverWithAuth(keyFile, certFile, caCertFile string) *grpc.Server {
	if keyFile == "" {
		return grpc.NewServer(grpc.MaxMsgSize(maxMsgSize)) // No auth.
	}
	log.Debug("Loading x509 key pair from key: %s cert: %s", keyFile, certFile)
	cert, err := tls.LoadX509KeyPair(certFile, keyFile)
	if err != nil {
		log.Fatalf("Failed to load x509 key pair: %s", err)
	}
	config := tls.Config{
		Certificates: []tls.Certificate{cert},
		ClientAuth:   tls.RequestClientCert,
	}
	if caCertFile != "" {
		cert, err := ioutil.ReadFile(caCertFile)
		if err != nil {
			log.Fatalf("Failed to read CA cert file: %s", err)
		}
		config.ClientCAs = x509.NewCertPool()
		if !config.ClientCAs.AppendCertsFromPEM(cert) {
			log.Fatalf("Failed to find any PEM certificates in CA cert")
		}
	}
	return grpc.NewServer(grpc.Creds(credentials.NewTLS(&config)), grpc.MaxMsgSize(maxMsgSize))
}

func main() {
	sig := runOutput()

	on := fmt.Sprintf(":%d", comm.Port())

	cert, err := tls.X509KeyPair(comm.Cert(), comm.Key())
	if err != nil {
		log.Fatal("failed to load cert and key", err)
	}

	creds := credentials.NewServerTLSFromCert(&cert)
	s := grpc.NewServer(
		grpc.Creds(creds),
	)

	svr := &server{sig: sig}
	comm.RegisterGarageServer(s, svr)

	listener, err := net.Listen("tcp", on)
	if err != nil {
		log.Fatal("failed to listen", err)
	}

	err = s.Serve(listener)
	if err != nil {
		log.Fatal("failed to serve", err)
	}
}

func TestDropRequestFailedNonFailFast(t *testing.T) {
	// Start a backend.
	beLis, err := net.Listen("tcp", "localhost:0")
	if err != nil {
		t.Fatalf("Failed to listen %v", err)
	}
	beAddr := strings.Split(beLis.Addr().String(), ":")
	bePort, err := strconv.Atoi(beAddr[1])
	backends := startBackends(t, besn, beLis)
	defer stopBackends(backends)

	// Start a load balancer.
	lbLis, err := net.Listen("tcp", "localhost:0")
	if err != nil {
		t.Fatalf("Failed to create the listener for the load balancer %v", err)
	}
	lbCreds := &serverNameCheckCreds{
		sn: lbsn,
	}
	lb := grpc.NewServer(grpc.Creds(lbCreds))
	if err != nil {
		t.Fatalf("Failed to generate the port number %v", err)
	}
	be := &lbpb.Server{
		IpAddress:        []byte(beAddr[0]),
		Port:             int32(bePort),
		LoadBalanceToken: lbToken,
		DropRequest:      true,
	}
	var bes []*lbpb.Server
	bes = append(bes, be)
	sl := &lbpb.ServerList{
		Servers: bes,
	}
	ls := newRemoteBalancer(sl)
	lbpb.RegisterLoadBalancerServer(lb, ls)
	go func() {
		lb.Serve(lbLis)
	}()
	defer func() {
		ls.stop()
		lb.Stop()
	}()
	creds := serverNameCheckCreds{
		expected: besn,
	}
	ctx, _ := context.WithTimeout(context.Background(), 10*time.Second)
	cc, err := grpc.DialContext(ctx, besn, grpc.WithBalancer(Balancer(&testNameResolver{
		addr: lbLis.Addr().String(),
	})), grpc.WithBlock(), grpc.WithTransportCredentials(&creds))
	if err != nil {
		t.Fatalf("Failed to dial to the backend %v", err)
	}
	helloC := hwpb.NewGreeterClient(cc)
	ctx, _ = context.WithTimeout(context.Background(), 10*time.Millisecond)
	if _, err := helloC.SayHello(ctx, &hwpb.HelloRequest{Name: "grpc"}, grpc.FailFast(false)); grpc.Code(err) != codes.DeadlineExceeded {
		t.Fatalf("%v.SayHello(_, _) = _, %v, want _, %s", helloC, err, codes.DeadlineExceeded)
	}
	cc.Close()
}

func (t *grpcTransportListener) Accept(fn func(transport.Socket)) error {
	var opts []grpc.ServerOption

	// setup tls if specified
	if t.secure || t.tls != nil {
		config := t.tls
		if config == nil {
			var err error
			addr := t.listener.Addr().String()
			config, err = getTLSConfig(addr)
			if err != nil {
				return err
			}
		}

		creds := credentials.NewTLS(config)
		opts = append(opts, grpc.Creds(creds))
	}

	// new service
	srv := grpc.NewServer(opts...)

	// register service
	pb.RegisterTransportServer(srv, &microTransport{fn: fn})

	// start serving
	return srv.Serve(t.listener)
}

func main() {
	flag.Parse()
	lis, err := net.Listen("tcp", *serverAddr)
	if err != nil {
		grpclog.Fatalf("failed to listen: %v", err)
	}
	var opts []grpc.ServerOption
	if *tls {
		creds, err := credentials.NewServerTLSFromFile(*certFile, *keyFile)
		if err != nil {
			grpclog.Fatalf("Failed to generate credentials %v", err)
		}
		opts = append(opts, grpc.Creds(creds))
	}
	grpcServer := grpc.NewServer(opts...)
	oidcClient, err := util.GetOIDCClient(*clientID, *clientSecret, *discovery, *redirectURL)
	if err != nil {
		grpclog.Fatalf("unable to get oidc client: %s", err)
	}
	s, err := server.NewRoloServer(oidcClient, *policyFile)
	if err != nil {
		grpclog.Fatalln("unable to create ca from parent:", err)
	}
	pb.RegisterRoloServer(grpcServer, s)
	grpclog.Println("serving at", *serverAddr)
	grpcServer.Serve(lis)
}

func (s *Server) ListenGRPC() error {
	grpcPort := s.Config.GetGrpcPortString()
	//Listen
	lis, err := net.Listen("tcp", grpcPort)
	if err != nil {
		return fmt.Errorf("server.go: failed to listen: %v", err)
	}

	var opts []grpc.ServerOption
	if s.Config.TlsCertFile != "" && s.Config.TlsKeyFile != "" {
		creds, err := credentials.NewServerTLSFromFile(s.Config.TlsCertFile, s.Config.TlsKeyFile)
		if err != nil {
			log.Fatalf("server.go: Failed to generate credentials %v", err)
		}
		opts = []grpc.ServerOption{grpc.Creds(creds)}
		s.tlsCheck = tlscheck.New(s.Config.TlsCertFile, s.Config.TlsKeyFile, time.Hour*24*16)
	}

	h := health.New(s, s.Storage)
	grpcServer := grpc.NewServer(opts...)
	catalogGrpc := &catalogGrpcServer{
		server: s,
	}
	pb.RegisterCatalogServiceServer(grpcServer, catalogGrpc)
	grpc_health_v1.RegisterHealthServer(grpcServer, h)

	go http.ListenAndServe(s.Config.GetHealthPortString(), h)

	log.Infof("server.go: Binding %s for grpc", grpcPort)
	//Serve
	return grpcServer.Serve(lis)
}

func createEventHubServer() (net.Listener, *grpc.Server, error) {
	var lis net.Listener
	var grpcServer *grpc.Server
	var err error
	if viper.GetBool("peer.validator.enabled") {
		lis, err = net.Listen("tcp", viper.GetString("peer.validator.events.address"))
		if err != nil {
			return nil, nil, fmt.Errorf("failed to listen: %v", err)
		}

		//TODO - do we need different SSL material for events ?
		var opts []grpc.ServerOption
		if viper.GetBool("peer.tls.enabled") {
			creds, err := credentials.NewServerTLSFromFile(viper.GetString("peer.tls.cert.file"), viper.GetString("peer.tls.key.file"))
			if err != nil {
				return nil, nil, fmt.Errorf("Failed to generate credentials %v", err)
			}
			opts = []grpc.ServerOption{grpc.Creds(creds)}
		}

		grpcServer = grpc.NewServer(opts...)
		ehServer := producer.NewOpenchainEventsServer(uint(viper.GetInt("peer.validator.events.buffersize")), viper.GetInt("peer.validator.events.timeout"))
		pb.RegisterOpenchainEventsServer(grpcServer, ehServer)
	}
	return lis, grpcServer, err
}

func startPKI() {
	var opts []grpc.ServerOption
	if viper.GetBool("peer.pki.tls.enabled") {
		// TLS configuration
		creds, err := credentials.NewServerTLSFromFile(
			filepath.Join(viper.GetString("server.rootpath"), "tlsca.cert"),
			filepath.Join(viper.GetString("server.rootpath"), "tlsca.priv"),
		)
		if err != nil {
			panic("Failed creating credentials for OBC-CA: " + err.Error())
		}
		opts = []grpc.ServerOption{grpc.Creds(creds)}
	}

	fmt.Printf("open socket...\n")
	sockp, err := net.Listen("tcp", viper.GetString("server.port"))
	if err != nil {
		panic("Cannot open port: " + err.Error())
	}
	fmt.Printf("open socket...done\n")

	server = grpc.NewServer(opts...)

	eca.Start(server)
	tca.Start(server)
	tlsca.Start(server)

	fmt.Printf("start serving...\n")
	server.Serve(sockp)
}

// set up the GRPC server
func setupGRPCServer(signerConfig signer.Config) (*grpc.Server, net.Listener, error) {

	//RPC server setup
	kms := &api.KeyManagementServer{
		CryptoServices: signerConfig.CryptoServices,
	}
	ss := &api.SignerServer{
		CryptoServices: signerConfig.CryptoServices,
	}
	hs := ghealth.NewServer()

	lis, err := net.Listen("tcp", signerConfig.GRPCAddr)
	if err != nil {
		return nil, nil, fmt.Errorf("grpc server failed to listen on %s: %v",
			signerConfig.GRPCAddr, err)
	}

	creds := credentials.NewTLS(signerConfig.TLSConfig)
	opts := []grpc.ServerOption{grpc.Creds(creds)}
	grpcServer := grpc.NewServer(opts...)

	pb.RegisterKeyManagementServer(grpcServer, kms)
	pb.RegisterSignerServer(grpcServer, ss)
	healthpb.RegisterHealthServer(grpcServer, hs)

	// Set status for both of the grpc service "KeyManagement" and "Signer", these are
	// the only two we have at present, if we add more grpc service in the future,
	// we should add a new line for that service here as well.
	hs.SetServingStatus(notary.HealthCheckKeyManagement, healthpb.HealthCheckResponse_SERVING)
	hs.SetServingStatus(notary.HealthCheckSigner, healthpb.HealthCheckResponse_SERVING)

	return grpcServer, lis, nil
}

func main() {

	f, err := os.OpenFile(logFile, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
	if err != nil {
		log.Fatalf("error opening file: %v", err)
	}
	log.SetOutput(io.MultiWriter(f, os.Stdout))

	http.HandleFunc("/", fronthandler)
	http.HandleFunc("/_ah/health", healthhandler)
	go http.ListenAndServe(httpport, nil)

	ce, err := credentials.NewServerTLSFromFile("server_crt.pem", "server_key.pem")
	if err != nil {
		log.Fatalf("Failed to generate credentials %v", err)
	}

	lis, err := net.Listen("tcp", grpcport)
	if err != nil {
		log.Fatalf("failed to listen: %v", err)
	}

	sopts := []grpc.ServerOption{grpc.MaxConcurrentStreams(10)}
	sopts = append(sopts, grpc.Creds(ce))
	s := grpc.NewServer(sopts...)

	echo.RegisterEchoServerServer(s, &server{})
	log.Printf("Starting gRPC server on port %v", grpcport)

	s.Serve(lis)
}

func (s *OortGroupStore) ListenAndServe() {
	go func(s *OortGroupStore) {
		s.grpcStopping = false
		for {
			var err error
			listenAddr := s.oort.GetListenAddr()
			if listenAddr == "" {
				log.Fatalln("No listen address specified in ring at address2")
			}
			l, err := net.Listen("tcp", listenAddr)
			if err != nil {
				log.Fatalln("Unable to bind to address:", err)
			}
			log.Println("GroupStore bound to:", listenAddr)
			var opts []grpc.ServerOption
			creds := credentials.NewTLS(s.serverTLSConfig)
			opts = []grpc.ServerOption{grpc.Creds(creds)}
			s.grpc = grpc.NewServer(opts...)
			groupproto.RegisterGroupStoreServer(s.grpc, s)
			err = s.grpc.Serve(l)
			if err != nil && !s.grpcStopping {
				log.Println("GroupStore Serve encountered error:", err, "will attempt to restart")
			} else if err != nil && s.grpcStopping {
				log.Println("GroupStore got error but halt is in progress:", err)
				l.Close()
				break
			} else {
				log.Println("GroupStore Serve exited without error, quiting")
				l.Close()
				break
			}
		}
	}(s)
}

// NewNode creates a new raft node to use for tests
func NewNode(t *testing.T, clockSource *fakeclock.FakeClock, tc *cautils.TestCA, opts ...raft.NodeOptions) *TestNode {
	l, err := net.Listen("tcp", "127.0.0.1:0")
	require.NoError(t, err, "can't bind to raft service port")
	wrappedListener := NewWrappedListener(l)

	securityConfig, err := tc.NewNodeConfig(ca.ManagerRole)
	require.NoError(t, err)

	serverOpts := []grpc.ServerOption{grpc.Creds(securityConfig.ServerTLSCreds)}
	s := grpc.NewServer(serverOpts...)

	cfg := raft.DefaultNodeConfig()

	stateDir, err := ioutil.TempDir("", "test-raft")
	require.NoError(t, err, "can't create temporary state directory")

	keyRotator := NewSimpleKeyRotator(raft.EncryptionKeys{CurrentDEK: []byte("current")})
	newNodeOpts := raft.NodeOptions{
		ID:             securityConfig.ClientTLSCreds.NodeID(),
		Addr:           l.Addr().String(),
		Config:         cfg,
		StateDir:       stateDir,
		ClockSource:    clockSource,
		TLSCredentials: securityConfig.ClientTLSCreds,
		KeyRotator:     keyRotator,
	}

	if len(opts) > 1 {
		panic("more than one optional argument provided")
	}
	if len(opts) == 1 {
		newNodeOpts.JoinAddr = opts[0].JoinAddr
		if opts[0].Addr != "" {
			newNodeOpts.Addr = opts[0].Addr
		}
	}

	n := raft.NewNode(newNodeOpts)

	healthServer := health.NewHealthServer()
	api.RegisterHealthServer(s, healthServer)
	raft.Register(s, n)

	go func() {
		// After stopping, we should receive an error from Serve
		assert.Error(t, s.Serve(wrappedListener))
	}()

	healthServer.SetServingStatus("Raft", api.HealthCheckResponse_SERVING)

	return &TestNode{
		Node:           n,
		Listener:       wrappedListener,
		SecurityConfig: securityConfig,
		Address:        newNodeOpts.Addr,
		StateDir:       newNodeOpts.StateDir,
		Server:         s,
		KeyRotator:     keyRotator,
	}
}

func startTLSCA(t *testing.T) {
	CacheConfiguration() // Cache configuration
	ecaS = NewECA(nil)
	tlscaS = NewTLSCA(ecaS)

	var opts []grpc.ServerOption
	creds, err := credentials.NewServerTLSFromFile(viper.GetString("server.tls.cert.file"), viper.GetString("server.tls.key.file"))
	if err != nil {
		t.Logf("Failed creating credentials for TLS-CA service: %s", err)
		t.Fail()
	}

	opts = []grpc.ServerOption{grpc.Creds(creds)}

	srv = grpc.NewServer(opts...)

	ecaS.Start(srv)
	tlscaS.Start(srv)

	sock, err := net.Listen("tcp", viper.GetString("server.port"))
	if err != nil {
		t.Logf("Failed to start TLS-CA service: %s", err)
		t.Fail()
	}

	srv.Serve(sock)
}

func startTLSCA(t *testing.T) {
	LogInit(ioutil.Discard, os.Stdout, os.Stdout, os.Stderr, os.Stdout)

	eca_s = NewECA()
	tlsca_s = NewTLSCA(eca_s)

	var opts []grpc.ServerOption
	creds, err := credentials.NewServerTLSFromFile(viper.GetString("server.tls.certfile"), viper.GetString("server.tls.keyfile"))
	if err != nil {
		t.Logf("Failed creating credentials for TLS-CA service: %s", err)
		t.Fail()
	}

	opts = []grpc.ServerOption{grpc.Creds(creds)}

	srv = grpc.NewServer(opts...)

	eca_s.Start(srv)
	tlsca_s.Start(srv)

	sock, err := net.Listen("tcp", viper.GetString("server.port"))
	if err != nil {
		t.Logf("Failed to start TLS-CA service: %s", err)
		t.Fail()
	}

	srv.Serve(sock)
}