func (n *GitLabClient) RegisterRunner(runner RunnerCredentials, description, tags string) *RegisterRunnerResponse {
	// TODO: pass executor
	request := RegisterRunnerRequest{
		Info:        n.getRunnerVersion(RunnerConfig{}),
		Token:       runner.Token,
		Description: description,
		Tags:        tags,
	}

	var response RegisterRunnerResponse
	result, statusText := n.do(runner, "POST", "runners/register.json", 201, &request, &response)
	shortToken := helpers.ShortenToken(runner.Token)

	switch result {
	case 201:
		logrus.Println(shortToken, "Registering runner...", "succeeded")
		return &response
	case 403:
		logrus.Errorln(shortToken, "Registering runner...", "forbidden (check registration token)")
		return nil
	case clientError:
		logrus.Errorln(shortToken, "Registering runner...", "error", statusText)
		return nil
	default:
		logrus.Errorln(shortToken, "Registering runner...", "failed", statusText)
		return nil
	}
}

func (n *GitLabClient) GetBuild(config RunnerConfig) (*GetBuildResponse, bool) {
	request := GetBuildRequest{
		Info:  n.getRunnerVersion(config),
		Token: config.Token,
	}

	var response GetBuildResponse
	result, statusText := n.do(config.RunnerCredentials, "POST", "builds/register.json", 201, &request, &response)

	switch result {
	case 201:
		logrus.Println(config.ShortDescription(), "Checking for builds...", "received")
		return &response, true
	case 403:
		logrus.Errorln(config.ShortDescription(), "Checking for builds...", "forbidden")
		return nil, false
	case 404:
		logrus.Debugln(config.ShortDescription(), "Checking for builds...", "nothing")
		return nil, true
	case clientError:
		logrus.Errorln(config.ShortDescription(), "Checking for builds...", "error:", statusText)
		return nil, false
	default:
		logrus.Warningln(config.ShortDescription(), "Checking for builds...", "failed:", statusText)
		return nil, true
	}
}

func (p *TenantService) TenantDetail(token string, tenantId string) (ret interface{}, errorCode string, err error) {
	if !bson.IsObjectIdHex(tenantId) {
		logrus.Errorln("invalid object id for getTenantDetail: ", tenantId)
		err = errors.New("invalid object id for getTenantDetail")
		return nil, TENANT_ERROR_CREATE, err
	}
	code, err := GetTokenService().TokenValidate(token)
	if err != nil {
		return nil, code, err
	}

	if authorized := GetAuthService().Authorize("get_tenant", token, tenantId, p.collectionName); !authorized {
		logrus.Errorln("required opertion is not allowed!")
		return nil, COMMON_ERROR_UNAUTHORIZED, errors.New("Required opertion is not authorized!")
	}

	selector := bson.M{}
	selector["_id"] = bson.ObjectIdHex(tenantId)

	ret = entity.Tenant{}
	queryStruct := dao.QueryStruct{
		CollectionName: p.collectionName,
		Selector:       selector,
		Skip:           0,
		Limit:          0,
		Sort:           ""}

	err = dao.HandleQueryOne(&ret, queryStruct)
	return
}

func RegisterRunner(url, token, description, tags string) *RegisterRunnerResponse {
	// TODO: pass executor
	request := RegisterRunnerRequest{
		Info:        GetRunnerVersion(RunnerConfig{}),
		Token:       token,
		Description: description,
		Tags:        tags,
	}

	var response RegisterRunnerResponse
	result, statusText := postJSON(getURL(url, "runners/register.json"), 201, &request, &response)
	shortToken := helpers.ShortenToken(token)

	switch result {
	case 201:
		log.Println(shortToken, "Registering runner...", "succeeded")
		return &response
	case 403:
		log.Errorln(shortToken, "Registering runner...", "forbidden (check registration token)")
		return nil
	default:
		log.Errorln(shortToken, "Registering runner...", "failed", statusText)
		return nil
	}
}

func compressResponse(resp *http.Response, compressionLevel int) error {

	// Establish a new pipe.
	pipeReader, pipeWriter := io.Pipe()

	// In a seperate Go routine, compress the request body and copy it to the
	// pipe.
	go func(r io.ReadCloser) {

		// Defer the closing of both the reader and writer.
		defer closeLogError(r)
		defer closeLogError(pipeWriter)

		// Create a new gzip writer, wrapping the original writer,
		// and defer its closing.
		gzipWriter, err := gzip.NewWriterLevel(pipeWriter, compressionLevel)
		if err != nil {
			log.Errorln(err)
			return
		}
		defer closeLogError(gzipWriter)

		// Copy the response body to the gzip writer.
		if _, err := io.Copy(gzipWriter, r); err != nil {
			log.Errorln(err)
		}
	}(resp.Body)

	resp.Header.Set("content-encoding", "gzip")
	resp.Header.Del("content-length")
	resp.Body = pipeReader
	return nil
}

//return a query language by authenticate
//this will be used for listAll, list and deleteAll interface
func (p *AuthService) BuildQueryByAuth(operation string, tokenId string) (query bson.M, err error) {
	logrus.Infoln("build query object by auth")

	token, err := GetTokenService().GetTokenById(tokenId)
	if err != nil {
		logrus.Errorln("get token error:", err)
		return nil, errors.New("get token by id error!")
	}

	authParam := &AuthParam{auth_token: token}

	policyValue, exist := p.authdata[operation]
	if !exist {
		logrus.Infoln("no auth policy for specific operation, operation:", operation)
		query = bson.M{}
		return
	}

	switch checkType := policyValue.(type) {
	case entity.OrCheck:
		orresult, orsuccess := p.orCheck_query(checkType.Basechecks, checkType.Andchecks, checkType.Orchecks, authParam)
		if !orsuccess {
			logrus.Warnln("build auth query error")
			return nil, errors.New("build auth query error")
		}
		query, err = format(orresult)
		if err != nil {
			logrus.Warnln("format query result to bson error %v", err)
			return nil, err
		}
		return
	case entity.AndCheck:
		andresult, andsuccess := p.andCheck_query(checkType.Basechecks, checkType.Andchecks, checkType.Orchecks, authParam)
		if !andsuccess {
			logrus.Warnln("build auth query error")
			return nil, errors.New("build auth query error")
		}
		query, err = format(andresult)
		if err != nil {
			logrus.Warnln("format query result to bson error %v", err)
			return nil, err
		}
		return
	case entity.BaseCheck:
		baseresult, basesuccess := p.baseCheck_query(checkType.Checktype, checkType.Value, authParam)
		if !basesuccess {
			logrus.Warnln("build auth query error")
			return nil, errors.New("build auth query error")
		}
		query, err = format(baseresult)
		if err != nil {
			logrus.Warnln("format query result to bson error %v", err)
			return nil, err
		}
		return
	default:
		logrus.Errorln("unkonwn check type:", checkType)
		return nil, errors.New("unknown check type")
	}
}

func (sn *Node) StartGossip() {
	go func() {
		t := time.Tick(GOSSIP_TIME)
		for {
			select {
			case <-t:
				sn.viewmu.Lock()
				c := sn.HostListOn(sn.ViewNo)
				sn.viewmu.Unlock()
				if len(c) == 0 {
					log.Errorln(sn.Name(), "StartGossip: none in hostlist for view: ", sn.ViewNo, len(c))
					continue
				}
				sn.randmu.Lock()
				from := c[sn.Rand.Int()%len(c)]
				sn.randmu.Unlock()
				log.Errorln("Gossiping with: ", from)
				sn.CatchUp(int(atomic.LoadInt64(&sn.LastAppliedVote)+1), from)
			case <-sn.closed:
				log.Warnln("stopping gossip: closed")
				return
			}
		}
	}()
}

func exportEnvironmentsList(envsList []models.EnvironmentItemModel) error {
	log.Info("[BITRISE_CLI] - Exporting workflow environments")

	for _, env := range envsList {
		envKey, envValue, err := env.GetKeyValue()
		if err != nil {
			log.Errorln("[BITRISE_CLI] - Failed to get environment key-value pair from env:", env)
			return err
		}
		if envValue != "" {
			expand := true
			if env["is_expand"] != "" {
				boolValue, err := goinp.ParseBool(env["is_expand"])
				if err != nil {
					log.Error("Failed to parse bool:", err)
					return err
				}
				expand = boolValue
			}
			if err := bitrise.RunEnvmanAdd(envKey, envValue, expand); err != nil {
				log.Errorln("[BITRISE_CLI] - Failed to run envman add")
				return err
			}
		}
	}
	return nil
}

//terminate specified hosts of a cluster
// Request
// URL:
// 	PUT /v1/cluster/<CLUSTER_ID>/hosts
// Header:
// 	X-Auth-Token
// Except Body:
//{
//    "host_ids":["568e23655d5c3d173019f1ba","568e2be45d5c3d173019f1bb","568e2bfd5d5c3d173019f1bc","568e2c335d5c3d173019f1bd"]
//}
//
// Response:
//{
//  "success": true
//}
//
func (p *Resource) HostsDeleteHandler(req *restful.Request, resp *restful.Response) {
	logrus.Infof("HostsDeleteHandler is called!")

	x_auth_token := req.HeaderParameter("X-Auth-Token")
	code, err := services.TokenValidation(x_auth_token)
	if err != nil {
		logrus.Errorln("token validation error is %v", err)
		response.WriteStatusError(code, err, resp)
		return
	}

	clusterId := req.PathParameter(ParamID)
	body := TerminateHostsRequestBody{}
	err = json.NewDecoder(req.Request.Body).Decode(&body)

	errorCode, err := services.GetClusterService().TerminateHosts(clusterId, body.HostIds, x_auth_token)
	if err != nil {
		logrus.Errorln("terminate hosts error is %v", err)
		response.WriteStatusError(errorCode, err, resp)
		return
	}

	// Write success response
	response.WriteSuccess(resp)
	return
}

func (s *SlaveSpider) Base() {
	method := "SlaveSpider Base"
	logrus.Println("SlaveSpider", method)

	for {
		if len(s.Urls) == 0 {
			// logrus.Infoln(method, "Send to Master New urls:", len(s.NewUrls) /*s.NewUrls*/)
			send := len(s.NewUrls)

			masterToSlave, err := s.Request()
			if err != nil {
				logrus.Errorln(method, err)
				continue
			}

			s.NewUrls = []string{}
			s.Match = []string{}

			logrus.Infoln(method, "slaveToMaster url:", send, "get Task:", len(masterToSlave.Task) /*, masterToSlave.Task*/)
			s.Urls = masterToSlave.Task
			time.Sleep(time.Second * 2)
		} else {
			err := s.MatchOnce()
			if err != nil {
				logrus.Errorln(method, err)
				continue
			}
		}

	}

	return
}

func (p *AuthService) baseCheck_query(checktype string, value string, authParam *AuthParam) (map[string]interface{}, bool) {
	basequery := make(map[string]interface{})

	if strings.EqualFold(checktype, "role") {
		if strings.EqualFold(value, authParam.auth_token.Role.Rolename) {
			return basequery, true
		} else {
			return nil, false
		}
	} else if strings.EqualFold(checktype, "field") {
		valueArrays := strings.Split(value, "=")
		if len(valueArrays) != 2 {
			logrus.Errorln("a field policy format error! value is :", value)
			return nil, false
		}

		//handle first value
		value1 := valueArrays[0]

		// handle second value(can be a value or from token)
		value2 := valueArrays[1]
		if strings.HasPrefix(value2, "%") {
			key2 := strings.TrimPrefix(value2, "%")
			value2 = p.getValueFromToken(authParam.auth_token, key2)
		}

		basequery[value1] = value2

		return basequery, true

	} else {
		logrus.Errorln("unsupported checktype:", checktype)
		return nil, false
	}
}

func (sr *Runner) checkCleared(check stalker.Check) {
	log.Debugln("check cleared")
	log.Infof("%s %s detected as cleared", check.Hostname, check.Check)
	query := map[string]string{"hostname": check.Hostname, "check": check.Check}
	cursor, err := r.Db(STALKERDB).Table("notifications").Filter(query).Run(sr.rsess)
	if err != nil {
		log.Errorln("Error checking for existing notification:", err.Error())
		return
	}
	defer cursor.Close()
	result := stalker.Notification{}
	cursor.One(&result)
	if result.Active == false {
		log.Infoln("No notification to clear")
		return
	}
	_, err = r.Db(STALKERDB).Table("notifications").Filter(query).Delete().RunWrite(sr.rsess)
	if err != nil {
		log.Errorln("Error deleting notification entry:", err.Error())
		return
	}
	sr.emitClear(check)
	return

}

func (n *GitLabClient) VerifyRunner(runner RunnerCredentials) bool {
	request := VerifyRunnerRequest{
		Token: runner.Token,
	}

	// HACK: we use non-existing build id to check if receive forbidden or not found
	result, statusText := n.do(runner, "PUT", fmt.Sprintf("builds/%d", -1), 200, &request, nil)
	shortToken := helpers.ShortenToken(runner.Token)

	switch result {
	case 404:
		// this is expected due to fact that we ask for non-existing job
		logrus.Println(shortToken, "Veryfing runner...", "is alive")
		return true
	case 403:
		logrus.Errorln(shortToken, "Veryfing runner...", "is removed")
		return false
	case clientError:
		logrus.Errorln(shortToken, "Veryfing runner...", "error", statusText)
		return false
	default:
		logrus.Errorln(shortToken, "Veryfing runner...", "failed", statusText)
		return true
	}
}

func userLoginParamCheck(doc interface{}) (username string, password string, paraErr error) {
	var document interface{}
	document, paraErr = mejson.Marshal(doc)
	if paraErr != nil {
		logrus.Errorf("marshal user err is %v", paraErr)
		return
	}

	docJson := document.(map[string]interface{})
	usernameDoc := docJson["username"]
	if usernameDoc == nil {
		logrus.Errorln("invalid parameter ! username can not be null")
		paraErr = errors.New("Invalid parameter!")
		return
	} else {
		username = usernameDoc.(string)
	}

	passwordDoc := docJson["password"]
	if passwordDoc == nil {
		logrus.Errorln("invalid parameter ! password can not be null")
		paraErr = errors.New("Invalid parameter!")
		return
	} else {
		password = passwordDoc.(string)
	}
	return
}

// step steps through the pipeline to head.next
func (p *Pipeline) step() {
	if p.head == p.tail {
		go func() {
			defer func() {
				if r := recover(); r != nil {
					logrus.Errorln("recover executing step function", r)
				}
			}()

			// stop all containers
			for _, id := range p.containers {
				p.engine.ContainerStop(id)
			}

			// wait for all logs to terminate
			// p.wait.Done() // this is for the ambassador
			p.wait.Wait()

			// signal completion
			p.done <- nil
		}()
	} else {
		go func() {
			defer func() {
				if r := recover(); r != nil {
					logrus.Errorln("recover executing step to head function", r)
				}
			}()

			p.head = p.head.next
			p.next <- nil
		}()
	}
}

func (d NetworkDriver) DeleteEndpoint(request *network.DeleteEndpointRequest) error {
	logutils.JSONMessage("DeleteEndpoint", request)
	log.Debugf("Removing endpoint %v\n", request.EndpointID)

	hostname, err := osutils.GetHostname()
	if err != nil {
		err = errors.Wrap(err, "Hostname fetching error")
		log.Errorln(err)
		return err
	}

	if err = d.client.WorkloadEndpoints().Delete(
		api.WorkloadEndpointMetadata{
			Name:         request.EndpointID,
			Node:         hostname,
			Orchestrator: d.orchestratorID,
			Workload:     d.containerName}); err != nil {
		err = errors.Wrapf(err, "Endpoint %v removal error", request.EndpointID)
		log.Errorln(err)
		return err
	}

	logutils.JSONMessage("DeleteEndpoint response JSON=%v", map[string]string{})

	return err
}

func (p *TokenService) TokenReGenerate(token string, userId string, tenantId string) (ret TokenId, errorCode string, err error) {
	if len(userId) == 0 || len(tenantId) == 0 {
		logrus.Errorf("user and tenant id can not be null!")
		return ret, "E12002", errors.New("invalid parameter!user and tenant id can not be null")
	}

	code, err := p.TokenValidate(token)
	if err != nil {
		return ret, code, err
	}

	if authorized := GetAuthService().Authorize("regenerate_token", token, "", p.collectionName); !authorized {
		logrus.Errorln("required opertion is not allowed!")
		return ret, "E12004", errors.New("required opertion is not authorized!")
	}

	user_name, tenant_name, err := getNamesById(userId, tenantId)
	if err != nil {
		logrus.Errorln("failed to get user and tenant by id, error is %v", err)
		return ret, TOKEN_ERROR_CREATE, err
	}

	newtoken, err := p.checkAndGenerateToken(user_name, "", tenant_name, false)
	if err != nil {
		logrus.Errorf("failed to generate token, error is %s", err)
		return ret, TOKEN_ERROR_CREATE, err
	}

	tokenId := TokenId{Id: newtoken}

	return tokenId, "", nil
}

func (h *hook) updateKeys() error {
	resp, err := http.Get(h.cfg.JWKSetURL)
	if err != nil {
		log.Errorln("failed to fetch JWK Set: " + err.Error())
		return err
	}

	var parsedJWKs gojwk.Key
	err = json.NewDecoder(resp.Body).Decode(&parsedJWKs)
	if err != nil {
		resp.Body.Close()
		log.Errorln("failed to decode JWK JSON: " + err.Error())
		return err
	}
	resp.Body.Close()

	keys := map[string]crypto.PublicKey{}
	for _, parsedJWK := range parsedJWKs.Keys {
		publicKey, err := parsedJWK.DecodePublicKey()
		if err != nil {
			log.Errorln("failed to decode JWK into public key: " + err.Error())
			return err
		}
		keys[parsedJWK.Kid] = publicKey
	}
	h.publicKeys = keys

	log.Debug("successfully fetched JWK Set")
	return nil
}

func (p *UserService) UserDetail(token string, userId string) (ret interface{}, errorCode string, err error) {
	code, err := GetTokenService().TokenValidate(token)
	if err != nil {
		return nil, code, err
	}

	if authorized := GetAuthService().Authorize("get_user", token, userId, p.userCollectionName); !authorized {
		logrus.Errorln("required opertion is not allowed!")
		return nil, COMMON_ERROR_UNAUTHORIZED, errors.New("Required opertion is not authorized!")
	}

	selector := bson.M{}
	selector["_id"] = bson.ObjectIdHex(userId)

	ret = new(entity.User)
	queryStruct := dao.QueryStruct{
		CollectionName: p.userCollectionName,
		Selector:       selector,
		Skip:           0,
		Limit:          0,
		Sort:           ""}

	err = dao.HandleQueryOne(ret, queryStruct)
	logrus.Errorln(ret)
	return
}

func configureLogging(v *viper.Viper) {
	level, err := log.ParseLevel(v.GetString("log_level"))
	if err != nil {
		log.Fatalln(err)
	}
	log.SetLevel(level)

	if v.GetString("log_format") == "text" {
		log.SetFormatter(&log.TextFormatter{DisableColors: true, FullTimestamp: true})
	} else if v.GetString("log_format") == "json" {
		log.SetFormatter(&log.JSONFormatter{})
	} else {
		log.Errorln("Error: log_type invalid, defaulting to text")
		log.SetFormatter(&log.TextFormatter{})
	}
	switch v.GetString("log_target") {
	case "stdout":
		log.SetOutput(os.Stdout)
	case "stderr":
		log.SetOutput(os.Stderr)
	default:
		log.Errorln("Error: log_target invalid, defaulting to Stdout")
		log.SetOutput(os.Stdout)
	}
}