// NewDocSetForIssue creates a new DocSet patched to the latest patch level of
// the given issue.
//
// The returned DocSet is not periodically refreshed.
func NewDocSetForIssue(workDir, repo string, issue int64) (*DocSet, error) {
	// Only do pull and patch if directory doesn't exist.
	issueInfo, err := rc.Issue(issue)
	if err != nil {
		return nil, fmt.Errorf("Failed to load issue info: %s", err)
	}
	patchset := issueInfo.Patchsets[len(issueInfo.Patchsets)-1]
	addr, err := mail.ParseAddress(issueInfo.OwnerEmail)
	if err != nil {
		return nil, fmt.Errorf("CL contains invalid author email: %s", err)
	}
	domain := strings.Split(addr.Address, "@")[1]
	if !util.In(domain, config.WHITELIST) {
		return nil, fmt.Errorf("User is not authorized to test docset CLs.")
	}
	var d *DocSet
	repoDir := filepath.Join(workDir, "patches", fmt.Sprintf("%d-%d", issue, patchset))
	if _, err := os.Stat(repoDir); os.IsNotExist(err) {
		d, err = newDocSet(repoDir, repo, issue, patchset, false)
		if err != nil {
			if err := os.RemoveAll(repoDir); err != nil {
				glog.Errorf("Failed to remove %q: %s", repoDir, err)
			}
			return nil, fmt.Errorf("Failed to create new doc set: %s", err)
		}
	} else {
		d = &DocSet{
			repoDir: repoDir,
		}
		d.BuildNavigation()
	}
	return d, nil
}

// addResultToTile adds the Digests from the DMResults to the tile at the given offset.
func addResultToTile(res *DMResults, tile *tiling.Tile, offset int, counter metrics.Counter) {
	res.ForEach(func(traceID, digest string, params map[string]string) {
		if ext, ok := params["ext"]; !ok || ext != "png" {
			return // Skip non-PNG results they are be converted to PNG by a separate process.
		}

		var trace *types.GoldenTrace
		var ok bool
		needsUpdate := false
		if tr, ok := tile.Traces[traceID]; !ok {
			trace = types.NewGoldenTrace()
			tile.Traces[traceID] = trace
			needsUpdate = true
		} else {
			trace = tr.(*types.GoldenTrace)
			if !util.MapsEqual(params, tile.Traces[traceID].Params()) {
				needsUpdate = true
			}
		}
		trace.Params_ = params

		if needsUpdate {
			// Update the Tile's ParamSet with any new keys or values we see.
			for k, v := range params {
				if _, ok = tile.ParamSet[k]; !ok {
					tile.ParamSet[k] = []string{v}
				} else if !util.In(v, tile.ParamSet[k]) {
					tile.ParamSet[k] = append(tile.ParamSet[k], v)
				}
			}
		}
		trace.Values[offset] = digest
		counter.Inc(1)
	})
}

// Merge adds in new info from the passed in ClusterSummary.
func (c *ClusterSummary) Merge(from *ClusterSummary) {
	for _, k := range from.Keys {
		if !util.In(k, c.Keys) {
			c.Keys = append(c.Keys, k)
		}
	}
}

// Get returns the ShortCommit info for the given branch, target, and buildID.
func (i *info) Get(branch, target, buildID string) (*gitinfo.ShortCommit, error) {
	// Get the list of targets and confirm that this target is in it, otherwise add it to the list of targets.
	branchtargets := i.branchtargets()
	branchtarget := fmt.Sprintf("%s:%s", branch, target)
	if !util.In(branchtarget, branchtargets) {
		// If we aren't currently scanning results for this (branch, target) pair
		// then add it to the list.
		branchtargets = append(branchtargets, branchtarget)
		err := i.db.Put([]byte(TARGETS_KEY), []byte(strings.Join(branchtargets, " ")), nil)
		if err != nil {
			glog.Errorf("Failed to add new target %s: %s", branchtarget, err)
		}
		// Always try to fetch the information from the Android Build API directly if
		// we don't have it yet.
		return i.single_get(branch, target, buildID)
	} else {
		key, err := toKey(branch, target, buildID)
		if err != nil {
			return nil, fmt.Errorf("Can't Get with an invalid build ID %q: %s", buildID, err)
		}
		// Scan backwards through the build info until we find a buildID that is equal to or
		// comes before the buildID we are looking for.
		iter := i.db.NewIterator(lutil.BytesPrefix([]byte(toPrefix(branch, target))), nil)
		defer iter.Release()
		if found := iter.Seek([]byte(key)); found {
			value := &gitinfo.ShortCommit{}
			if err := json.Unmarshal(iter.Value(), value); err != nil {
				return nil, fmt.Errorf("Unable to deserialize value: %s", err)
			}
			return value, nil
		} else {
			return i.single_get(branch, target, buildID)
		}
	}
}

// CombineClusters combines freshly found clusters with existing clusters.
//
//  Algorithm:
//    Run clustering and pick out the "Interesting" clusters.
//    Compare all the Interesting clusters to all the existing relevant clusters,
//      where "relevant" clusters are ones whose Hash/timestamp of the step
//      exists in the current tile.
//    Start with an empty "list".
//    For each cluster:
//      For each relevant existing cluster:
//        Take the top 20 keys from the existing cluster and count how many appear
//        in the cluster.
//      If there are no matches then this is a new cluster, add it to the "list".
//      If there are matches, possibly to multiple existing clusters, find the
//      existing cluster with the most matches.
//        Take the cluster (old/new) with the most members, or the best fit if
//        they have the same number of matches.
//    Return all the updated clusters.
func CombineClusters(freshSummaries, oldSummaries []*types.ClusterSummary) []*types.ClusterSummary {
	ret := []*types.ClusterSummary{}

	stillFresh := []*types.ClusterSummary{}
	// If two cluster summaries have the same hash and same Regression direction
	// then they are the same, merge them together.
	for _, fresh := range freshSummaries {
		for _, old := range oldSummaries {
			if fresh.Hash == old.Hash && math.Signbit(fresh.StepFit.Regression) == math.Signbit(old.StepFit.Regression) {
				old.Merge(fresh)
				ret = append(ret, old)
				break
			}
		}
		stillFresh = append(stillFresh, fresh)
	}

	// Even if a summary has a different hash it might still be the same event if
	// there is an overlap in the traces each summary contains.
	for _, fresh := range stillFresh {
		var bestMatch *types.ClusterSummary = nil
		bestMatchHits := 0
		for _, old := range oldSummaries {
			hits := 0
			for _, key := range util.AtMost(old.Keys, 20) {
				if util.In(key, fresh.Keys) {
					hits += 1
				}
			}
			if hits > bestMatchHits {
				bestMatchHits = hits
				bestMatch = old
			}
		}
		if bestMatch != nil {
			keysLengthEqual := len(fresh.Keys) == len(bestMatch.Keys)
			regressionInSameDirection := math.Signbit(fresh.StepFit.Regression) == math.Signbit(bestMatch.StepFit.Regression)
			freshHasBetterFit := math.Abs(fresh.StepFit.Regression) > math.Abs(bestMatch.StepFit.Regression)
			freshHasMoreKeys := len(fresh.Keys) > len(bestMatch.Keys)
			if freshHasMoreKeys || (keysLengthEqual && regressionInSameDirection && freshHasBetterFit) {
				fresh.Status = bestMatch.Status
				fresh.Message = bestMatch.Message
				fresh.ID = bestMatch.ID
				fresh.Bugs = bestMatch.Bugs
				ret = append(ret, fresh)
				// Find the bestMatch in oldSummaries and replace it with fresh.
				for i, oldBest := range oldSummaries {
					if oldBest == bestMatch {
						oldSummaries[i] = fresh
						break
					}
				}
			}
		} else {
			ret = append(ret, fresh)
		}
	}
	return ret
}

// Matches returns true if the given Trace matches the given query.
func Matches(tr Trace, query url.Values) bool {
	for k, values := range query {
		if _, ok := tr.Params()[k]; !ok || !util.In(tr.Params()[k], values) {
			return false
		}
	}
	return true
}

// makeRollResult determines what the result of a roll should be, given that
// it is going to be closed.
func (r *AutoRoller) makeRollResult(roll *autoroll.AutoRollIssue) string {
	if util.In(roll.Result, autoroll.DRY_RUN_RESULTS) {
		if roll.Result == autoroll.ROLL_RESULT_DRY_RUN_IN_PROGRESS {
			return autoroll.ROLL_RESULT_DRY_RUN_FAILURE
		} else {
			return roll.Result
		}
	}
	return autoroll.ROLL_RESULT_FAILURE
}

// scoreBuild returns the current score for the given commit/builder pair. The
// details on how scoring works are described in the doc for NewBuildQueue.
func scoreBuild(commit *vcsinfo.LongCommit, build *buildbot.Build, now time.Time, timeLambda float64) float64 {
	s := -1.0
	if build != nil {
		if build.GotRevision == commit.Hash {
			s = 1.0
		} else if util.In(commit.Hash, build.Commits) {
			s = 1.0 / float64(len(build.Commits))
		}
	}
	return s * timeFactor(now, commit.Timestamp, timeLambda)
}

// Finds the paramSet for the given slice of traces.
func GetParamSet(traces map[string]Trace, paramSet map[string][]string) {
	for _, trace := range traces {
		for k, v := range trace.Params() {
			if _, ok := paramSet[k]; !ok {
				paramSet[k] = []string{v}
			} else if !util.In(v, paramSet[k]) {
				paramSet[k] = append(paramSet[k], v)
			}
		}
	}
}

// differences returns all strings that appear in server but not local.
func differences(server, local []string) ([]string, []string) {
	newPackages := []string{}
	installedPackages := []string{}
	for _, s := range server {
		if util.In(s, local) {
			installedPackages = append(installedPackages, s)
		} else {
			newPackages = append(newPackages, s)
		}
	}
	return newPackages, installedPackages
}

// setStatus sets the current reporting status of the bot.
func (r *AutoRoller) setStatus(s string, lastError error) error {
	r.mtx.Lock()
	defer r.mtx.Unlock()
	if !util.In(string(s), VALID_STATUSES) {
		return fmt.Errorf("Invalid status: %s", s)
	}
	if s == STATUS_ERROR {
		if lastError == nil {
			return fmt.Errorf("Cannot set error status without an error!")
		}
	} else if lastError != nil {
		return fmt.Errorf("Cannot be in any status other than error when an error occurred.")
	}
	r.status = s
	r.lastError = lastError
	return nil
}

// polyListTestsHandler returns a JSON list with high level information about
// each test.
//
// Takes two query parameters:
//  include - True if ignored digests should be included. (true, false)
//  query   - A query to restrict the responses to, encoded as a URL encoded paramset.
//  head    - True if only digest that appear at head should be included.
//
// The return format looks like:
//
//  [
//    {
//      "name": "01-original",
//      "diameter": 123242,
//      "untriaged": 2,
//      "num": 2
//    },
//    ...
//  ]
//
func polyListTestsHandler(w http.ResponseWriter, r *http.Request) {
	if err := r.ParseForm(); err != nil {
		util.ReportError(w, r, err, "Failed to parse form data.")
		return
	}
	// If the query only includes source_type parameters, and include==false, then we can just
	// filter the response from summaries.Get(). If the query is broader than that, or
	// include==true, then we need to call summaries.CalcSummaries().
	if err := r.ParseForm(); err != nil {
		util.ReportError(w, r, err, "Invalid request.")
		return
	}
	q, err := url.ParseQuery(r.FormValue("query"))
	if err != nil {
		util.ReportError(w, r, err, "Invalid query in request.")
		return
	}
	_, hasSourceType := q["source_type"]
	sumSlice := []*summary.Summary{}
	if r.FormValue("include") == "false" && r.FormValue("head") == "true" && len(q) == 1 && hasSourceType {
		sumMap := summaries.Get()
		corpus := q["source_type"]
		for _, s := range sumMap {
			if util.In(s.Corpus, corpus) {
				sumSlice = append(sumSlice, s)
			}
		}
	} else {
		glog.Infof("%q %q %q", r.FormValue("query"), r.FormValue("include"), r.FormValue("head"))
		sumMap, err := summaries.CalcSummaries(nil, r.FormValue("query"), r.FormValue("include") == "true", r.FormValue("head") == "true")
		if err != nil {
			util.ReportError(w, r, err, "Failed to calculate summaries.")
			return
		}
		for _, s := range sumMap {
			sumSlice = append(sumSlice, s)
		}
	}
	sort.Sort(SummarySlice(sumSlice))
	w.Header().Set("Content-Type", "application/json")
	enc := json.NewEncoder(w)
	if err := enc.Encode(sumSlice); err != nil {
		glog.Errorf("Failed to write or encode result: %s", err)
	}
}

// Validate returns an error iff there is some problem with the issue.
func (i *AutoRollIssue) Validate() error {
	if i.Closed {
		if i.Result == ROLL_RESULT_IN_PROGRESS {
			return fmt.Errorf("AutoRollIssue cannot have a Result of %q if it is Closed.", ROLL_RESULT_IN_PROGRESS)
		}
		if i.CommitQueue {
			return errors.New("AutoRollIssue cannot be marked CommitQueue if it is Closed.")
		}
	} else {
		if i.Committed {
			return errors.New("AutoRollIssue cannot be Committed without being Closed.")
		}
		if !util.In(i.Result, OPEN_ROLL_VALID_RESULTS) {
			return fmt.Errorf("AutoRollIssue which is not Closed must have as a Result one of: %v", OPEN_ROLL_VALID_RESULTS)
		}
	}
	return nil
}

// changeHandler changes the status of a service.
//
// Takes the following query parameters:
//
//   name - The name of the service.
//   action - The action to perform. One of ["start", "stop", "restart"].
//
// The response is of the form:
//
//   {
//     "result": "started"
//   }
//
func changeHandler(w http.ResponseWriter, r *http.Request) {
	var err error

	if err := r.ParseForm(); err != nil {
		util.ReportError(w, r, err, "Failed to parse form.")
		return
	}
	action := r.Form.Get("action")
	if !util.In(action, ACTIONS) {
		util.ReportError(w, r, fmt.Errorf("Not a valid action: %s", action), "Invalid action.")
		return
	}
	name := r.Form.Get("name")
	if name == "" {
		util.ReportError(w, r, fmt.Errorf("Not a valid service name: %s", name), "Invalid service name.")
		return
	}
	if *local {
		w.Header().Set("Content-Type", "application/json")
		if err := json.NewEncoder(w).Encode(ChangeResult{"started"}); err != nil {
			glog.Errorf("Failed to write or encode output: %s", err)
		}
		return
	}
	ch := make(chan string)
	switch action {
	case "start":
		_, err = dbc.StartUnit(name, "replace", ch)
	case "stop":
		_, err = dbc.StopUnit(name, "replace", ch)
	case "restart":
		_, err = dbc.RestartUnit(name, "replace", ch)
	}
	if err != nil {
		util.ReportError(w, r, err, "Action failed.")
		return
	}
	res := ChangeResult{}
	res.Result = <-ch
	w.Header().Set("Content-Type", "application/json")
	if err := json.NewEncoder(w).Encode(res); err != nil {
		glog.Errorf("Failed to write or encode output: %s", err)
	}
}

// Add inserts a new ModeChange.
func (mh *ModeHistory) Add(m, user, message string) error {
	if !util.In(m, VALID_MODES) {
		return fmt.Errorf("Invalid mode: %s", m)
	}

	modeChange := &ModeChange{
		Message: message,
		Mode:    m,
		Time:    time.Now(),
		User:    user,
	}

	mh.mtx.Lock()
	defer mh.mtx.Unlock()
	if err := mh.db.SetMode(modeChange); err != nil {
		return err
	}
	return mh.refreshHistory()
}

// SkpCommits returns the indices for all the commits that contain SKP updates.
func (g *GitInfo) SkpCommits(tile *tiling.Tile) ([]int, error) {
	// Executes a git log command that looks like:
	//
	//   git log --format=format:%H  32956400b4d8f33394e2cdef9b66e8369ba2a0f3..e7416bfc9858bde8fc6eb5f3bfc942bc3350953a SKP_VERSION
	//
	// The output should be a \n separated list of hashes that match.
	first, last := tile.CommitRange()
	output, err := exec.RunCwd(g.dir, "git", "log", "--format=format:%H", first+".."+last, "SKP_VERSION")
	if err != nil {
		return nil, fmt.Errorf("SkpCommits: Failed to find git log of SKP_VERSION: %s", err)
	}
	hashes := strings.Split(output, "\n")

	ret := []int{}
	for i, c := range tile.Commits {
		if c.CommitTime != 0 && util.In(c.Hash, hashes) {
			ret = append(ret, i)
		}
	}
	return ret, nil
}

// updateCurrentRoll retrieves updated information about the current DEPS roll.
func (r *AutoRoller) updateCurrentRoll() error {
	r.mtx.Lock()
	defer r.mtx.Unlock()

	currentRoll := r.recent.CurrentRoll()
	if currentRoll == nil {
		return nil
	}
	currentResult := currentRoll.Result

	updated, err := r.retrieveRoll(currentRoll.Issue)
	if err != nil {
		return err
	}

	// We have to rely on data we store for the dry run case.
	if !updated.Closed && util.In(currentResult, autoroll.DRY_RUN_RESULTS) {
		updated.Result = currentResult
	}

	// If the current roll succeeded, we need to make sure we update the
	// repo so that we see the roll commit. This can take some time, so
	// we have to repeatedly update until we see the commit.
	if updated.Committed {
		glog.Infof("Roll succeeded (%d); syncing the repo until it lands.", currentRoll.Issue)
		for {
			glog.Info("Syncing...")
			if err := r.rm.ForceUpdate(); err != nil {
				return err
			}
			if r.rm.RolledPast(currentRoll.RollingTo) {
				break
			}
			time.Sleep(10 * time.Second)
		}
	}
	return r.recent.Update(updated)
}

// addBenchDataToTile adds BenchData to a Tile.
//
// See the description at the top of this file for how the mapping works.
func addBenchDataToTile(benchData *BenchData, tile *tiling.Tile, offset int, counter metrics.Counter) {

	// cb is the anonymous closure we'll pass over all the trace values found in benchData.
	cb := func(key string, value float64, params map[string]string) {
		needsUpdate := false
		var trace *types.PerfTrace
		if tr, ok := tile.Traces[key]; !ok {
			trace = types.NewPerfTrace()
			tile.Traces[key] = trace
			needsUpdate = true
		} else {
			trace = tr.(*types.PerfTrace)
			if !util.MapsEqual(params, tile.Traces[key].Params()) {
				needsUpdate = true
			}
		}
		trace.Params_ = params
		trace.Values[offset] = value
		counter.Inc(1)

		if needsUpdate {
			// Update the Tile's ParamSet with any new keys or values we see.
			//
			// TODO(jcgregorio) Maybe defer this until we are about to Put the Tile
			// back to disk and rebuild ParamSet from scratch over all the Traces.
			for k, v := range params {
				if _, ok := tile.ParamSet[k]; !ok {
					tile.ParamSet[k] = []string{v}
				} else if !util.In(v, tile.ParamSet[k]) {
					tile.ParamSet[k] = append(tile.ParamSet[k], v)
				}
			}
		}
	}

	benchData.ForEach(cb)
}

// updateRepo syncs the given repo and returns a set of BuildCandidates for it.
func (q *BuildQueue) updateRepo(repoUrl string, repo *gitinfo.GitInfo, now time.Time) (map[string][]*BuildCandidate, error) {
	from := now.Add(-q.period)
	if q.period == PERIOD_FOREVER {
		from = time.Unix(0, 0)
	}
	recentCommits := repo.From(from)
	commitDetails := map[string]*gitinfo.LongCommit{}
	for _, c := range recentCommits {
		details, err := repo.Details(c)
		if err != nil {
			return nil, err
		}
		commitDetails[c] = details
	}

	// Get all builds associated with the recent commits.
	buildsByCommit, err := buildbot.GetBuildsForCommits(recentCommits, map[int]bool{})
	if err != nil {
		return nil, err
	}

	// Find the sets of all bots and masters, organize builds by
	// commit/builder and builder/number.
	masters := map[string]string{}
	builds := map[string]map[string]*buildbot.Build{}
	buildsByBuilderAndNum := map[string]map[int]*buildbot.Build{}
	for commit, buildsForCommit := range buildsByCommit {
		builds[commit] = map[string]*buildbot.Build{}
		for _, build := range buildsForCommit {
			if !util.In(build.Builder, q.botWhitelist) {
				continue
			}
			masters[build.Builder] = build.Master
			builds[commit][build.Builder] = build
			if _, ok := buildsByBuilderAndNum[build.Builder]; !ok {
				buildsByBuilderAndNum[build.Builder] = map[int]*buildbot.Build{}
			}
			buildsByBuilderAndNum[build.Builder][build.Number] = build
		}
	}
	allBots := make([]string, 0, len(masters))
	for builder, _ := range masters {
		allBots = append(allBots, builder)
	}

	// Find the current scores for each commit/builder pair.
	currentScores := map[string]map[string]float64{}
	for _, commit := range recentCommits {
		myBuilds, ok := builds[commit]
		if !ok {
			myBuilds = map[string]*buildbot.Build{}
		}
		currentScores[commit] = map[string]float64{}
		for _, builder := range allBots {
			currentScores[commit][builder] = scoreBuild(commitDetails[commit], myBuilds[builder], now, q.timeLambda)
		}
	}

	// For each commit/builder pair, determine the score increase obtained
	// by running a build at that commit.
	scoreIncrease := map[string]map[string]float64{}
	for _, commit := range recentCommits {
		scoreIncrease[commit] = map[string]float64{}
		for _, builder := range allBots {
			// Shortcut: Don't bisect builds with a huge number
			// of commits.  This saves lots of time and only affects
			// the first successful build for a bot.
			if _, ok := builds[commit][builder]; ok {
				if len(builds[commit][builder].Commits) > NO_BISECT_COMMIT_LIMIT {
					glog.Warningf("Skipping %s on %s; previous build has too many commits.", commit[0:7], builder)
					scoreIncrease[commit][builder] = 0.0
					continue
				}
			}

			newScores := map[string]float64{}
			// Pretend to create a new Build at the given commit.
			newBuild := buildbot.Build{
				Builder:     builder,
				Master:      masters[builder],
				Number:      math.MaxInt32,
				GotRevision: commit,
				Repository:  repoUrl,
			}
			commits, stealFrom, stolen, err := buildbot.FindCommitsForBuild(&newBuild, q.repos)
			if err != nil {
				return nil, err
			}
			// Re-score all commits in the new build.
			newBuild.Commits = commits
			for _, c := range commits {
				if _, ok := currentScores[c]; !ok {
					// If this build has commits which are outside of our window,
					// insert them into currentScores to account for them.
					score := scoreBuild(commitDetails[commit], builds[commit][builder], now, q.timeLambda)
					currentScores[c] = map[string]float64{
						builder: score,
					}
				}
//.........这里部分代码省略.........

// isFinished returns true if the Build has finished running.
func isFinished(b *androidbuildinternal.Build) bool {
	return util.In(b.BuildAttemptStatus, terminal_build_status)
}