func makeRequest(testCase plan.TestCase) ([]byte, error) {
	callURL, err := url.Parse(fmt.Sprintf("http://%v:8080/", testCase.Client))
	if err != nil {
		return []byte(""), err
	}

	args := url.Values{}
	for k, v := range testCase.Arguments {
		args.Add(k, v)
	}
	callURL.RawQuery = args.Encode()

	ctx, _ := context.WithTimeout(
		context.Background(), testCase.Plan.Config.CallTimeout*time.Second)
	resp, err := ctxhttp.Get(ctx, nil, callURL.String())
	if err != nil {
		return []byte(""), err
	}

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return []byte(""), err
	}

	if resp.StatusCode != 200 {
		return []byte(""), fmt.Errorf("wanted status code 200, got %v", resp.StatusCode)
	}

	return body, nil
}

// Lookup returns rectangles for the given address. Currently the only
// implementation is the Google geocoding service.
func Lookup(ctx context.Context, address string) ([]Rect, error) {
	mu.RLock()
	rects, ok := cache[address]
	mu.RUnlock()
	if ok {
		return rects, nil
	}

	rectsi, err := sf.Do(address, func() (interface{}, error) {
		// TODO: static data files from OpenStreetMap, Wikipedia, etc?
		urlStr := "https://maps.googleapis.com/maps/api/geocode/json?address=" + url.QueryEscape(address) + "&sensor=false"
		res, err := ctxhttp.Get(ctx, ctxutil.Client(ctx), urlStr)
		if err != nil {
			return nil, err
		}
		defer res.Body.Close()
		rects, err := decodeGoogleResponse(res.Body)
		log.Printf("Google geocode lookup (%q) = %#v, %v", address, rects, err)
		if err == nil {
			mu.Lock()
			cache[address] = rects
			mu.Unlock()
		}
		return rects, err
	})
	if err != nil {
		return nil, err
	}
	return rectsi.([]Rect), nil
}

// ContentWithContext is a helper to abstract the location concept (not thread safe)
func (obj *Object) ContentWithContext(ctx context.Context) ([]byte, error) {
	if obj == nil {
		return nil, nil
	}
	if len(obj.Blob) > 0 || obj.Location == "" {
		return obj.Blob, nil
	}
	resp, err := ctxhttp.Get(ctx, nil, obj.Location)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		return nil, errors.New(resp.Status)
	}
	ct := resp.Header.Get("Content-Type")
	if ct != "" {
		obj.ContentType = ct
	}
	blob, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	obj.Blob = blob
	return blob, nil
}

func (ds *Datasource) fetchIntervalState(ctx context.Context, interval string, n int) (State, error) {
	var url string
	if n != 0 {
		url = ds.baseIntervalURL(interval, n) + ".state.txt"
	} else {
		url = fmt.Sprintf("%s/replication/%s/state.txt", ds.baseURL(), interval)
	}

	resp, err := ctxhttp.Get(ctx, ds.client(), url)
	if err != nil {
		return State{}, err
	}
	defer resp.Body.Close()

	if resp.StatusCode != 200 {
		return State{}, fmt.Errorf("incorrect status code: %v", resp.StatusCode)
	}

	data, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return State{}, err
	}

	return decodeIntervalState(data, interval)
}

// PluginGetURL fetches the content of a URL, which could be static or dynamic (passed in).
// It only works with an api with a simple Text/Text signature.
func PluginGetURL(static bool, uri string, model interface{}) Plugin {
	if static {
		if uri == "" {
			return nil
		}
	}
	return func(ctx context.Context, in interface{}) (out interface{}, err error) {
		inb, err := TextBytes(in)
		if err != nil {
			return nil, err
		}
		ins := string(inb)
		if static {
			ins = uri
		}
		resp, err := ctxhttp.Get(ctx, http.DefaultClient, ins) // handles context.Done() correctly
		if err != nil {
			return nil, err
		}
		byts, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return nil, err // unable to create a simple test case for this error
		}
		err = resp.Body.Close()
		if err != nil {
			return nil, err // unable to create a simple test case for this error
		}
		return TextConvert(byts, model)
	}
}

// Login takes a Hover username and password and returns the login cookie value
func Login(ctx context.Context, hc *http.Client, username, password string) (*http.Cookie, error) {
	v := make(url.Values)
	v.Set("username", username)
	v.Set("password", password)

	r, err := ctxhttp.Get(ctx, hc, fmt.Sprintf("%s/login?%s", defaultURL, v.Encode()))
	if err != nil {
		return nil, err
	}
	defer r.Body.Close()

	if r.StatusCode != 200 {
		return nil, InvalidLogin(fmt.Sprintf("login HTTP status code was %d", r.StatusCode))
	}
	var c *http.Cookie
	for _, cook := range r.Cookies() {
		if cook.Name == "hoverauth" && cook.Value != "" {
			c = cook
			break
		}
	}
	if c == nil {
		return nil, InvalidLogin("unable to find 'hoverauth' cookie with data in response")
	}
	return c, nil
}

// queryCAA sends the query request to the GPD API. If the return code is
// dns.RcodeSuccess the 'Answer' section is parsed for CAA records, otherwise
// an error is returned. Unlike bdns.DNSResolver.LookupCAA it will not repeat
// failed queries if the context has not expired as we expect to be running
// multiple queries in parallel and only need a M of N quorum (we also expect
// GPD to have quite good availability)
func (cdr *CAADistributedResolver) queryCAA(ctx context.Context, url string, ic *http.Client) ([]*dns.CAA, error) {
	apiResp, err := ctxhttp.Get(ctx, ic, url)
	if err != nil {
		return nil, err
	}
	defer func() {
		_ = apiResp.Body.Close()
	}()
	body, err := ioutil.ReadAll(&io.LimitedReader{R: apiResp.Body, N: 1024})
	if err != nil {
		return nil, err
	}
	if apiResp.StatusCode != http.StatusOK {
		if string(body) != "" {
			return nil, fmt.Errorf("Unexpected HTTP status code %d, body: %s", apiResp.StatusCode, body)
		}
		return nil, fmt.Errorf("Unexpected HTTP status code %d", apiResp.StatusCode)
	}
	var respObj core.GPDNSResponse
	err = json.Unmarshal(body, &respObj)
	if err != nil {
		return nil, err
	}
	if respObj.Status != dns.RcodeSuccess {
		if respObj.Comment != "" {
			return nil, fmt.Errorf("Query failed with %s: %s", dns.RcodeToString[respObj.Status], respObj.Comment)
		}
		return nil, fmt.Errorf("Query failed wtih %s", dns.RcodeToString[respObj.Status])
	}

	return parseAnswer(respObj.Answer)
}

func testOnGCE() bool {
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	resc := make(chan bool, 2)

	// Try two strategies in parallel.
	// See https://github.com/GoogleCloudPlatform/gcloud-golang/issues/194
	go func() {
		res, err := ctxhttp.Get(ctx, metaClient, "http://"+metadataIP)
		if err != nil {
			resc <- false
			return
		}
		defer res.Body.Close()
		resc <- res.Header.Get("Metadata-Flavor") == "Google"
	}()

	go func() {
		addrs, err := net.LookupHost("metadata.google.internal")
		if err != nil || len(addrs) == 0 {
			resc <- false
			return
		}
		resc <- strsContains(addrs, metadataIP)
	}()

	return <-resc
}

func TestContextCancel(t *testing.T) {
	// server that doesn't reply before the timeout
	wg := sync.WaitGroup{}
	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		time.Sleep(2 * time.Second)
		fmt.Fprint(w, r.URL.Path)
		wg.Done()
	}))
	defer srv.Close()

	tr := NewTransport(nil, RetryAll(RetryMaxRetries(1), RetryStatusInterval(500, 600)), ConstDelay(0))
	c := &http.Client{
		Transport: tr,
	}

	ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(time.Second))
	defer cancelFn()

	wg.Add(1)
	res, err := ctxhttp.Get(ctx, c, srv.URL+"/test")
	require.Nil(t, res)

	assert.Equal(t, context.DeadlineExceeded, err)
	wg.Wait()
}

// NewProvider uses the OpenID Connect discovery mechanism to construct a Provider.
//
// The issuer is the URL identifier for the service. For example: "https://accounts.google.com"
// or "https://login.salesforce.com".
func NewProvider(ctx context.Context, issuer string) (*Provider, error) {
	wellKnown := strings.TrimSuffix(issuer, "/") + "/.well-known/openid-configuration"
	resp, err := ctxhttp.Get(ctx, clientFromContext(ctx), wellKnown)
	if err != nil {
		return nil, err
	}
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("%s: %s", resp.Status, body)
	}
	defer resp.Body.Close()
	var p providerJSON
	if err := json.Unmarshal(body, &p); err != nil {
		return nil, fmt.Errorf("oidc: failed to decode provider discovery object: %v", err)
	}
	if p.Issuer != issuer {
		return nil, fmt.Errorf("oidc: issuer did not match the issuer returned by provider, expected %q got %q", issuer, p.Issuer)
	}
	return &Provider{
		issuer:       p.Issuer,
		authURL:      p.AuthURL,
		tokenURL:     p.TokenURL,
		userInfoURL:  p.UserInfoURL,
		rawClaims:    body,
		remoteKeySet: newRemoteKeySet(ctx, p.JWKSURL, time.Now),
	}, nil
}

func TestContextCancelOnRetry(t *testing.T) {
	callCnt := int32(0)
	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		cnt := atomic.AddInt32(&callCnt, 1)
		switch cnt {
		case 1:
			w.WriteHeader(500)
		default:
			time.Sleep(2 * time.Second)
			fmt.Fprint(w, r.URL.Path)
		}
	}))
	defer srv.Close()

	// cancel while waiting on retry response
	tr := NewTransport(nil, RetryAll(RetryMaxRetries(1), RetryStatusInterval(500, 600)), ConstDelay(0))
	c := &http.Client{
		Transport: tr,
	}

	ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(time.Second))
	defer cancelFn()

	res, err := ctxhttp.Get(ctx, c, srv.URL+"/test")
	require.Nil(t, res)

	assert.Equal(t, context.DeadlineExceeded, err)
	assert.Equal(t, int32(2), atomic.LoadInt32(&callCnt))

	// cancel while waiting on delay
	atomic.StoreInt32(&callCnt, 0)
	tr = NewTransport(nil, RetryAll(RetryMaxRetries(1), RetryStatusInterval(500, 600)), ConstDelay(2*time.Second))
	c = &http.Client{
		Transport: tr,
	}

	ctx, cancelFn = context.WithDeadline(context.Background(), time.Now().Add(time.Second))
	defer cancelFn()

	res, err = ctxhttp.Get(ctx, c, srv.URL+"/test")
	require.Nil(t, res)

	assert.Equal(t, context.DeadlineExceeded, err)
	assert.Equal(t, int32(1), atomic.LoadInt32(&callCnt))
}

func NewTHttpClientWithCtx(urlstr string, ctx context.Context) (TTransport, error) {
	parsedURL, err := url.Parse(urlstr)
	if err != nil {
		return nil, err
	}

	response, err := ctxhttp.Get(ctx, nil, urlstr)
	if err != nil {
		return nil, err
	}
	return &THttpClient{response: response, url: parsedURL, ctx: ctx}, nil
}

func (c *ArtifactStoreClient) getAPI(path string) (io.ReadCloser, *ArtifactsError) {
	url := c.server + path
	ctx, cancel := context.WithTimeout(c.ctx, c.timeout)
	defer cancel()

	if resp, err := ctxhttp.Get(ctx, nil, url); err != nil {
		return nil, NewRetriableError(err.Error())
	} else {
		if resp.StatusCode != http.StatusOK {
			return nil, determineResponseError(resp, url, "POST")
		}
		return resp.Body, nil
	}
}

// WaitAuthorization polls an authorization at the given URL
// until it is in one of the final states, StatusValid or StatusInvalid,
// or the context is done.
//
// It returns a non-nil Authorization only if its Status is StatusValid.
// In all other cases WaitAuthorization returns an error.
// If the Status is StatusInvalid, the returned error is ErrAuthorizationFailed.
func (c *Client) WaitAuthorization(ctx context.Context, url string) (*Authorization, error) {
	var count int
	sleep := func(v string, inc int) error {
		count += inc
		d := backoff(count, 10*time.Second)
		d = retryAfter(v, d)
		wakeup := time.NewTimer(d)
		defer wakeup.Stop()
		select {
		case <-ctx.Done():
			return ctx.Err()
		case <-wakeup.C:
			return nil
		}
	}

	for {
		res, err := ctxhttp.Get(ctx, c.HTTPClient, url)
		if err != nil {
			return nil, err
		}
		retry := res.Header.Get("retry-after")
		if res.StatusCode != http.StatusOK && res.StatusCode != http.StatusAccepted {
			res.Body.Close()
			if err := sleep(retry, 1); err != nil {
				return nil, err
			}
			continue
		}
		var raw wireAuthz
		err = json.NewDecoder(res.Body).Decode(&raw)
		res.Body.Close()
		if err != nil {
			if err := sleep(retry, 0); err != nil {
				return nil, err
			}
			continue
		}
		if raw.Status == StatusValid {
			return raw.authorization(url), nil
		}
		if raw.Status == StatusInvalid {
			return nil, ErrAuthorizationFailed
		}
		if err := sleep(retry, 0); err != nil {
			return nil, err
		}
	}
}

func (f Fetcher) GetAll(ctx context.Context, urls []string) ([]*FetcherResponse, error) {
	defer metrics.MeasureSince([]string{"fn.FetchRemoteData"}, time.Now())

	fetches := make([]*FetcherResponse, len(urls))

	var wg sync.WaitGroup
	wg.Add(len(urls))

	// TODO: add thruput here..

	for i, urlStr := range urls {
		fetches[i] = &FetcherResponse{}

		go func(fetch *FetcherResponse) {
			defer wg.Done()

			url, err := urlx.Parse(urlStr)
			if err != nil {
				fetch.Err = err
				return
			}
			fetch.URL = url

			lg.Infof("Fetching %s", url.String())

			resp, err := ctxhttp.Get(ctx, f.client(), url.String())
			if err != nil {
				lg.Warnf("Error fetching %s because %s", url.String(), err)
				fetch.Err = err
				return
			}
			defer resp.Body.Close()

			fetch.Status = resp.StatusCode

			body, err := ioutil.ReadAll(resp.Body)
			if err != nil {
				fetch.Err = err
				return
			}
			fetch.Data = body
			fetch.Err = nil

		}(fetches[i])
	}

	wg.Wait()
	return fetches, nil
}

// GetChallenge retrieves the current status of an challenge.
//
// A client typically polls a challenge status using this method.
func (c *Client) GetChallenge(ctx context.Context, url string) (*Challenge, error) {
	res, err := ctxhttp.Get(ctx, c.HTTPClient, url)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()
	if res.StatusCode != http.StatusOK && res.StatusCode != http.StatusAccepted {
		return nil, responseError(res)
	}
	v := wireChallenge{URI: url}
	if err := json.NewDecoder(res.Body).Decode(&v); err != nil {
		return nil, fmt.Errorf("acme: invalid response: %v", err)
	}
	return v.challenge(), nil
}

// GetAuthorization retrieves an authorization identified by the given URL.
//
// If a caller needs to poll an authorization until its status is final,
// see the WaitAuthorization method.
func (c *Client) GetAuthorization(ctx context.Context, url string) (*Authorization, error) {
	res, err := ctxhttp.Get(ctx, c.HTTPClient, url)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()
	if res.StatusCode != http.StatusOK && res.StatusCode != http.StatusAccepted {
		return nil, responseError(res)
	}
	var v wireAuthz
	if err := json.NewDecoder(res.Body).Decode(&v); err != nil {
		return nil, fmt.Errorf("acme: invalid response: %v", err)
	}
	return v.authorization(url), nil
}

func (h *HttpProbe) Probe(c ctx.Context) error {
	resp, err := http.Get(c, nil, h.url)
	if err != nil {
		return ErrNotReachable
	} else {
		go func() {
			defer resp.Body.Close()
			io.Copy(ioutil.Discard, resp.Body)
		}()
		if resp.StatusCode != 200 {
			return fmt.Errorf("%d: %s", resp.StatusCode, h.url)
		} else {
			return nil
		}
	}
}

func handleAddBookmark(ctx context.Context, w http.ResponseWriter, r *http.Request) {
	var input struct {
		Url string `json:"url"`
	}
	if err := json.NewDecoder(r.Body).Decode(&input); err != nil {
		web.JSONErr(w, err.Error(), http.StatusBadRequest)
		return
	}

	resp, err := ctxhttp.Get(ctx, &crawler, input.Url)
	if err != nil {
		log.Error("cannot crawl",
			"url", input.Url,
			"error", err.Error())
		web.StdJSONResp(w, http.StatusInternalServerError)
		return
	}
	defer resp.Body.Close()

	body := make([]byte, 1024*20)
	if n, err := resp.Body.Read(body); err != nil && err != io.EOF {
		log.Error("cannot read crawler response",
			"url", input.Url,
			"error", err.Error())
		web.StdJSONResp(w, http.StatusInternalServerError)
		return
	} else {
		body = body[:n]
	}

	title := pageTitle(body)

	var b Bookmark
	err = pg.DB(ctx).Get(&b, `
		INSERT INTO bookmarks (title, url, created)
		VALUES ($1, $2, $3)
		ON CONFLICT DO NOTHING
		RETURNING *
	`, title, input.Url, time.Now())
	if err != nil {
		log.Error("cannot create bookmark", "error", err.Error())
		web.StdJSONResp(w, http.StatusInternalServerError)
		return
	}

	web.JSONResp(w, b, http.StatusCreated)
}

// changesetReader will return a ReadCloser with the data from the changeset.
// It will be gzip compressed, so the caller must decompress.
// It is the caller's responsibility to call Close on the Reader when done.
func (ds *Datasource) changesetReader(ctx context.Context, n ChangesetSeqNum) (io.ReadCloser, error) {
	url := ds.changesetURL(n)
	resp, err := ctxhttp.Get(ctx, ds.client(), url)
	if err != nil {
		return nil, err
	}

	if resp.StatusCode != 200 {
		resp.Body.Close()
		return nil, &UnexpectedStatusCodeError{
			Code: resp.StatusCode,
			URL:  url,
		}
	}

	return resp.Body, nil
}