func setCredURLFromNetrc(cfg *config.Configuration, req *http.Request) bool {
	hostname := req.URL.Host
	var host string

	if strings.Contains(hostname, ":") {
		var err error
		host, _, err = net.SplitHostPort(hostname)
		if err != nil {
			tracerx.Printf("netrc: error parsing %q: %s", hostname, err)
			return false
		}
	} else {
		host = hostname
	}

	machine, err := cfg.FindNetrcHost(host)
	if err != nil {
		tracerx.Printf("netrc: error finding match for %q: %s", hostname, err)
		return false
	}

	if machine == nil {
		return false
	}

	setRequestAuth(cfg, req, machine.Login, machine.Password)
	return true
}

// Initialise custom adapters based on current config
func configureCustomAdapters(cfg *config.Configuration, m *Manifest) {
	pathRegex := regexp.MustCompile(`lfs.customtransfer.([^.]+).path`)
	for k, v := range cfg.AllGitConfig() {
		match := pathRegex.FindStringSubmatch(k)
		if match == nil {
			continue
		}

		name := match[1]
		path := v
		// retrieve other values
		args, _ := cfg.Git.Get(fmt.Sprintf("lfs.customtransfer.%s.args", name))
		concurrent := cfg.Git.Bool(fmt.Sprintf("lfs.customtransfer.%s.concurrent", name), true)
		direction, _ := cfg.Git.Get(fmt.Sprintf("lfs.customtransfer.%s.direction", name))
		if len(direction) == 0 {
			direction = "both"
		} else {
			direction = strings.ToLower(direction)
		}

		// Separate closure for each since we need to capture vars above
		newfunc := func(name string, dir Direction) TransferAdapter {
			return newCustomAdapter(name, dir, path, args, concurrent)
		}

		if direction == "download" || direction == "both" {
			m.RegisterNewTransferAdapterFunc(name, Download, newfunc)
		}
		if direction == "upload" || direction == "both" {
			m.RegisterNewTransferAdapterFunc(name, Upload, newfunc)
		}
	}
}

// LogHttpStats is intended to be called after all HTTP operations for the
// commmand have finished. It dumps k/v logs, one line per httpTransfer into
// a log file with the current timestamp.
func LogHttpStats(cfg *config.Configuration) {
	if !cfg.IsLoggingStats {
		return
	}

	file, err := statsLogFile()
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error logging http stats: %s\n", err)
		return
	}

	fmt.Fprintf(file, "concurrent=%d batch=%v time=%d version=%s\n", cfg.ConcurrentTransfers(), cfg.BatchTransfer(), time.Now().Unix(), config.Version)

	for key, responses := range httpTransferBuckets {
		for _, response := range responses {
			stats := httpTransfers[response]
			fmt.Fprintf(file, "key=%s reqheader=%d reqbody=%d resheader=%d resbody=%d restime=%d status=%d url=%s\n",
				key,
				stats.requestStats.HeaderSize,
				stats.requestStats.BodySize,
				stats.responseStats.HeaderSize,
				stats.responseStats.BodySize,
				stats.responseStats.Stop.Sub(stats.responseStats.Start).Nanoseconds(),
				response.StatusCode,
				response.Request.URL)
		}
	}

	fmt.Fprintf(os.Stderr, "HTTP Stats logged to file %s\n", file.Name())
}

func setRequestAuthFromUrl(cfg *config.Configuration, req *http.Request, u *url.URL) bool {
	if !cfg.NtlmAccess(GetOperationForRequest(req)) && u.User != nil {
		if pass, ok := u.User.Password(); ok {
			fmt.Fprintln(os.Stderr, "warning: current Git remote contains credentials")
			setRequestAuth(cfg, req, u.User.Username(), pass)
			return true
		}
	}

	return false
}

func ConfigureManifest(m *Manifest, cfg *config.Configuration) *Manifest {
	m.basicTransfersOnly = cfg.BasicTransfersOnly()

	configureBasicDownloadAdapter(m)
	configureBasicUploadAdapter(m)
	if cfg.TusTransfersAllowed() {
		configureTusAdapter(m)
	}
	configureCustomAdapters(cfg, m)
	return m
}

func skipCredsCheck(cfg *config.Configuration, req *http.Request) bool {
	if cfg.NtlmAccess(GetOperationForRequest(req)) {
		return false
	}

	if len(req.Header.Get("Authorization")) > 0 {
		return true
	}

	q := req.URL.Query()
	return len(q["token"]) > 0
}

func determineIncludeExcludePaths(config *config.Configuration, includeArg, excludeArg *string) (include, exclude []string) {
	if includeArg == nil {
		include = config.FetchIncludePaths()
	} else {
		include = tools.CleanPaths(*includeArg, ",")
	}
	if excludeArg == nil {
		exclude = config.FetchExcludePaths()
	} else {
		exclude = tools.CleanPaths(*excludeArg, ",")
	}
	return
}

func setRequestAuth(cfg *config.Configuration, req *http.Request, user, pass string) {
	if cfg.NtlmAccess(GetOperationForRequest(req)) {
		return
	}

	if len(user) == 0 && len(pass) == 0 {
		return
	}

	token := fmt.Sprintf("%s:%s", user, pass)
	auth := "Basic " + strings.TrimSpace(base64.StdEncoding.EncodeToString([]byte(token)))
	req.Header.Set("Authorization", auth)
}

// BatchOrLegacy calls the Batch API and falls back on the Legacy API
// This is for simplicity, legacy route is not most optimal (serial)
// TODO LEGACY API: remove when legacy API removed
func BatchOrLegacy(cfg *config.Configuration, objects []*ObjectResource, operation string, transferAdapters []string) (objs []*ObjectResource, transferAdapter string, e error) {
	if !cfg.BatchTransfer() {
		objs, err := Legacy(cfg, objects, operation)
		return objs, "", err
	}
	objs, adapterName, err := Batch(cfg, objects, operation, transferAdapters)
	if err != nil {
		if errors.IsNotImplementedError(err) {
			git.Config.SetLocal("", "lfs.batch", "false")
			objs, err := Legacy(cfg, objects, operation)
			return objs, "", err
		}
		return nil, "", err
	}
	return objs, adapterName, nil
}

// Internal http request management
func doHttpRequest(cfg *config.Configuration, req *http.Request, creds auth.Creds) (*http.Response, error) {
	var (
		res   *http.Response
		cause string
		err   error
	)

	if cfg.NtlmAccess(auth.GetOperationForRequest(req)) {
		cause = "ntlm"
		res, err = doNTLMRequest(cfg, req, true)
	} else {
		cause = "http"
		res, err = NewHttpClient(cfg, req.Host).Do(req)
	}

	if res == nil {
		res = &http.Response{
			StatusCode: 0,
			Header:     make(http.Header),
			Request:    req,
			Body:       ioutil.NopCloser(bytes.NewBufferString("")),
		}
	}

	if err != nil {
		if errors.IsAuthError(err) {
			SetAuthType(cfg, req, res)
			doHttpRequest(cfg, req, creds)
		} else {
			err = errors.Wrap(err, cause)
		}
	} else {
		err = handleResponse(cfg, res, creds)
	}

	if err != nil {
		if res != nil {
			SetErrorResponseContext(cfg, err, res)
		} else {
			setErrorRequestContext(cfg, err, req)
		}
	}

	return res, err
}

// doApiBatchRequest runs the request to the LFS batch API. If the API returns a
// 401, the repo will be marked as having private access and the request will be
// re-run. When the repo is marked as having private access, credentials will
// be retrieved.
func DoBatchRequest(cfg *config.Configuration, req *http.Request) (*http.Response, *batchResponse, error) {
	res, err := DoRequest(req, cfg.PrivateAccess(auth.GetOperationForRequest(req)))

	if err != nil {
		if res != nil && res.StatusCode == 401 {
			return res, nil, errors.NewAuthError(err)
		}
		return res, nil, err
	}

	resp := &batchResponse{}
	err = httputil.DecodeResponse(res, resp)

	if err != nil {
		httputil.SetErrorResponseContext(cfg, err, res)
	}

	return res, resp, err
}

// NewHttpClient returns a new HttpClient for the given host (which may be "host:port")
func NewHttpClient(c *config.Configuration, host string) *HttpClient {
	httpClientsMutex.Lock()
	defer httpClientsMutex.Unlock()

	if httpClients == nil {
		httpClients = make(map[string]*HttpClient)
	}
	if client, ok := httpClients[host]; ok {
		return client
	}

	dialtime := c.Git.Int("lfs.dialtimeout", 30)
	keepalivetime := c.Git.Int("lfs.keepalive", 1800) // 30 minutes
	tlstime := c.Git.Int("lfs.tlstimeout", 30)

	tr := &http.Transport{
		Proxy: ProxyFromGitConfigOrEnvironment(c),
		Dial: (&net.Dialer{
			Timeout:   time.Duration(dialtime) * time.Second,
			KeepAlive: time.Duration(keepalivetime) * time.Second,
		}).Dial,
		TLSHandshakeTimeout: time.Duration(tlstime) * time.Second,
		MaxIdleConnsPerHost: c.ConcurrentTransfers(),
	}

	tr.TLSClientConfig = &tls.Config{}
	if isCertVerificationDisabledForHost(c, host) {
		tr.TLSClientConfig.InsecureSkipVerify = true
	} else {
		tr.TLSClientConfig.RootCAs = getRootCAsForHost(c, host)
	}

	client := &HttpClient{
		Config: c,
		Client: &http.Client{Transport: tr, CheckRedirect: CheckRedirect},
	}
	httpClients[host] = client

	return client
}

func SshAuthenticate(cfg *config.Configuration, operation, oid string) (SshAuthResponse, config.Endpoint, error) {
	// This is only used as a fallback where the Git URL is SSH but server doesn't support a full SSH binary protocol
	// and therefore we derive a HTTPS endpoint for binaries instead; but check authentication here via SSH

	endpoint := cfg.Endpoint(operation)
	res := SshAuthResponse{}
	if len(endpoint.SshUserAndHost) == 0 {
		return res, endpoint, nil
	}

	tracerx.Printf("ssh: %s git-lfs-authenticate %s %s %s",
		endpoint.SshUserAndHost, endpoint.SshPath, operation, oid)

	exe, args := sshGetExeAndArgs(cfg, endpoint)
	args = append(args,
		fmt.Sprintf("git-lfs-authenticate %s %s %s", endpoint.SshPath, operation, oid))

	cmd := exec.Command(exe, args...)

	// Save stdout and stderr in separate buffers
	var outbuf, errbuf bytes.Buffer
	cmd.Stdout = &outbuf
	cmd.Stderr = &errbuf

	// Execute command
	err := cmd.Start()
	if err == nil {
		err = cmd.Wait()
	}

	// Processing result
	if err != nil {
		res.Message = errbuf.String()
	} else {
		err = json.Unmarshal(outbuf.Bytes(), &res)
	}

	return res, endpoint, err
}

func getCredURLForAPI(cfg *config.Configuration, req *http.Request) (*url.URL, error) {
	operation := GetOperationForRequest(req)
	apiUrl, err := url.Parse(cfg.Endpoint(operation).Url)
	if err != nil {
		return nil, err
	}

	// if the LFS request doesn't match the current LFS url, don't bother
	// attempting to set the Authorization header from the LFS or Git remote URLs.
	if req.URL.Scheme != apiUrl.Scheme ||
		req.URL.Host != apiUrl.Host {
		return req.URL, nil
	}

	if setRequestAuthFromUrl(cfg, req, apiUrl) {
		return nil, nil
	}

	credsUrl := apiUrl
	if len(cfg.CurrentRemote) > 0 {
		if u := cfg.GitRemoteUrl(cfg.CurrentRemote, operation == "upload"); u != "" {
			gitRemoteUrl, err := url.Parse(u)
			if err != nil {
				return nil, err
			}

			if gitRemoteUrl.Scheme == apiUrl.Scheme &&
				gitRemoteUrl.Host == apiUrl.Host {

				if setRequestAuthFromUrl(cfg, req, gitRemoteUrl) {
					return nil, nil
				}

				credsUrl = gitRemoteUrl
			}
		}
	}
	return credsUrl, nil
}

func ntlmClientSession(c *config.Configuration, creds auth.Creds) (ntlm.ClientSession, error) {
	if c.NtlmSession != nil {
		return c.NtlmSession, nil
	}
	splits := strings.Split(creds["username"], "\\")

	if len(splits) != 2 {
		errorMessage := fmt.Sprintf("Your user name must be of the form DOMAIN\\user. It is currently %s", creds["username"])
		return nil, errors.New(errorMessage)
	}

	session, err := ntlm.CreateClientSession(ntlm.Version2, ntlm.ConnectionOrientedMode)

	if err != nil {
		return nil, err
	}

	session.SetUserInfo(splits[1], creds["password"], strings.ToUpper(splits[0]))
	c.NtlmSession = session
	return session, nil
}

func determineIncludeExcludePaths(config *config.Configuration, includeArg, excludeArg string) (include, exclude []string) {
	return tools.CleanPathsDefault(includeArg, ",", config.FetchIncludePaths()),
		tools.CleanPathsDefault(excludeArg, ",", config.FetchExcludePaths())
}

func Environ(cfg *config.Configuration, manifest *transfer.Manifest) []string {
	osEnviron := os.Environ()
	env := make([]string, 0, len(osEnviron)+7)

	dltransfers := manifest.GetDownloadAdapterNames()
	sort.Strings(dltransfers)
	ultransfers := manifest.GetUploadAdapterNames()
	sort.Strings(ultransfers)

	fetchPruneConfig := cfg.FetchPruneConfig()

	env = append(env,
		fmt.Sprintf("LocalWorkingDir=%s", config.LocalWorkingDir),
		fmt.Sprintf("LocalGitDir=%s", config.LocalGitDir),
		fmt.Sprintf("LocalGitStorageDir=%s", config.LocalGitStorageDir),
		fmt.Sprintf("LocalMediaDir=%s", LocalMediaDir()),
		fmt.Sprintf("LocalReferenceDir=%s", config.LocalReferenceDir),
		fmt.Sprintf("TempDir=%s", TempDir()),
		fmt.Sprintf("ConcurrentTransfers=%d", cfg.ConcurrentTransfers()),
		fmt.Sprintf("TusTransfers=%v", cfg.TusTransfersAllowed()),
		fmt.Sprintf("BasicTransfersOnly=%v", cfg.BasicTransfersOnly()),
		fmt.Sprintf("BatchTransfer=%v", cfg.BatchTransfer()),
		fmt.Sprintf("SkipDownloadErrors=%v", cfg.SkipDownloadErrors()),
		fmt.Sprintf("FetchRecentAlways=%v", fetchPruneConfig.FetchRecentAlways),
		fmt.Sprintf("FetchRecentRefsDays=%d", fetchPruneConfig.FetchRecentRefsDays),
		fmt.Sprintf("FetchRecentCommitsDays=%d", fetchPruneConfig.FetchRecentCommitsDays),
		fmt.Sprintf("FetchRecentRefsIncludeRemotes=%v", fetchPruneConfig.FetchRecentRefsIncludeRemotes),
		fmt.Sprintf("PruneOffsetDays=%d", fetchPruneConfig.PruneOffsetDays),
		fmt.Sprintf("PruneVerifyRemoteAlways=%v", fetchPruneConfig.PruneVerifyRemoteAlways),
		fmt.Sprintf("PruneRemoteName=%s", fetchPruneConfig.PruneRemoteName),
		fmt.Sprintf("AccessDownload=%s", cfg.Access("download")),
		fmt.Sprintf("AccessUpload=%s", cfg.Access("upload")),
		fmt.Sprintf("DownloadTransfers=%s", strings.Join(dltransfers, ",")),
		fmt.Sprintf("UploadTransfers=%s", strings.Join(ultransfers, ",")),
	)
	if len(cfg.FetchExcludePaths()) > 0 {
		env = append(env, fmt.Sprintf("FetchExclude=%s", strings.Join(cfg.FetchExcludePaths(), ", ")))
	}
	if len(cfg.FetchIncludePaths()) > 0 {
		env = append(env, fmt.Sprintf("FetchInclude=%s", strings.Join(cfg.FetchIncludePaths(), ", ")))
	}
	for _, ext := range cfg.Extensions() {
		env = append(env, fmt.Sprintf("Extension[%d]=%s", ext.Priority, ext.Name))
	}

	for _, e := range osEnviron {
		if !strings.Contains(e, "GIT_") {
			continue
		}
		env = append(env, e)
	}

	return env
}

func SetAuthType(cfg *config.Configuration, req *http.Request, res *http.Response) {
	authType := GetAuthType(res)
	operation := auth.GetOperationForRequest(req)
	cfg.SetAccess(operation, authType)
	tracerx.Printf("api: http response indicates %q authentication. Resubmitting...", authType)
}

func setErrorRequestContext(cfg *config.Configuration, err error, req *http.Request) {
	errors.SetContext(err, "Endpoint", cfg.Endpoint(auth.GetOperationForRequest(req)).Url)
	errors.SetContext(err, "URL", TraceHttpReq(req))
	setErrorHeaderContext(err, "Response", req.Header)
}