// Add, see the diff.LRUCache interface for details.
func (c *RedisLRUCache) Add(key, value interface{}) bool {
	prefixedKey, rawKey, err := c.encodeKey(key)
	if err != nil {
		glog.Errorf("Unable to create redis key: %s", err)
		return false
	}

	byteVal, err := c.encodeVal(value)
	if err != nil {
		glog.Errorf("Unable to create redis value: %s", err)
		return false
	}

	conn := c.pool.Get()
	defer util.Close(conn)

	util.LogErr(conn.Send("MULTI"))
	util.LogErr(conn.Send("SET", prefixedKey, byteVal))
	util.LogErr(conn.Send("SADD", c.indexSetKey, rawKey))
	_, err = conn.Do("EXEC")
	if err != nil {
		glog.Errorf("Unable to add key: %s", err)
		return false
	}
	return true
}

// DownloadBinarySeedFiles downloads the seed skp files stored in Google
// Storage to be used by afl-fuzz.  It places them in
// config.Generator.FuzzSamples after cleaning the folder out.
// It returns an error on failure.
func DownloadBinarySeedFiles(storageClient *storage.Client) error {
	if err := os.RemoveAll(config.Generator.FuzzSamples); err != nil && !os.IsNotExist(err) {
		return fmt.Errorf("Could not clean binary seed path %s: %s", config.Generator.FuzzSamples, err)
	}
	if err := os.MkdirAll(config.Generator.FuzzSamples, 0755); err != nil {
		return fmt.Errorf("Could not create binary seed path %s: %s", config.Generator.FuzzSamples, err)
	}

	err := gs.AllFilesInDir(storageClient, config.GS.Bucket, "skp_samples", func(item *storage.ObjectAttrs) {
		name := item.Name
		// skip the parent folder
		if name == "skp_samples/" {
			return
		}
		content, err := gs.FileContentsFromGS(storageClient, config.GS.Bucket, name)
		if err != nil {
			glog.Errorf("Problem downloading %s from Google Storage, continuing anyway", item.Name)
			return
		}
		fileName := filepath.Join(config.Generator.FuzzSamples, strings.SplitAfter(name, "skp_samples/")[1])
		if err = ioutil.WriteFile(fileName, content, 0644); err != nil && !os.IsExist(err) {
			glog.Errorf("Problem creating binary seed file %s, continuing anyway", fileName)
		}
	})
	return err
}

func _main(ts db.DB) {
	week := time.Hour * 24 * 7
	commits, err := ts.List(time.Now().Add(-week), time.Now())
	if err != nil {
		glog.Errorf("Failed to load commits: %s", err)
		return
	}
	if len(commits) > 50 {
		commits = commits[:50]
	}

	begin := time.Now()
	_, _, err = ts.TileFromCommits(commits)
	if err != nil {
		glog.Errorf("Failed to load Tile: %s", err)
		return
	}
	glog.Infof("Time to load tile: %v", time.Now().Sub(begin))
	// Now load a second time.
	begin = time.Now()
	_, _, err = ts.TileFromCommits(commits)
	if err != nil {
		glog.Errorf("Failed to load Tile: %s", err)
		return
	}
	glog.Infof("Time to load tile the second time: %v", time.Now().Sub(begin))
}

func getPreviousState() (map[string]fileState, map[string]int64, map[string]int64, time.Time, error) {
	if _, err := os.Stat(*stateFile); os.IsNotExist(err) {
		// State file does not exist, return empty values.
		return map[string]fileState{}, map[string]int64{}, map[string]int64{}, time.Time{}, nil
	}
	f, err := os.Open(*stateFile)
	if err != nil {
		glog.Errorf("Failed to open old state file %s for reading: %s", *stateFile, err)
		// Delete it and return empty values.
		if err := os.Remove(*stateFile); err != nil {
			return nil, nil, nil, time.Time{}, fmt.Errorf("Could not delete old state file %s: %s", *stateFile, err)
		}
		glog.Errorf("Deleted old state file %s", *stateFile)
		return map[string]fileState{}, map[string]int64{}, map[string]int64{}, time.Time{}, nil
	}
	defer util.Close(f)
	state := &logserverState{}
	dec := gob.NewDecoder(f)
	if err := dec.Decode(state); err != nil {
		glog.Errorf("Failed to decode old state file %s: %s", *stateFile, err)
		// Delete it and return empty values.
		if err := os.Remove(*stateFile); err != nil {
			return nil, nil, nil, time.Time{}, fmt.Errorf("Could not delete old state file %s: %s", *stateFile, err)
		}
		glog.Errorf("Deleted old state file %s", *stateFile)
		return map[string]fileState{}, map[string]int64{}, map[string]int64{}, time.Time{}, nil
	}
	return state.FilesToState, state.AppLogLevelToSpace, state.AppLogLevelToCount, state.LastCompletedRun, nil
}

func (e *EventBus) publishEvent(topic string, arg interface{}, globally bool) {
	e.mutex.Lock()
	defer e.mutex.Unlock()
	if th, ok := e.handlers[topic]; ok {
		for _, callback := range th.callbacks {
			th.wg.Add(1)
			go func(callback CallbackFn) {
				defer th.wg.Done()
				callback(arg)
			}(callback)
		}
	}

	if globally {
		if eventCodec, ok := globalEvents[topic]; ok {
			go func() {
				byteData, err := eventCodec.Encode(arg)
				if err != nil {
					glog.Errorf("Unable to encode event data for topic %s:  %s", topic, err)
					return
				}

				if err := e.globalEventBus.Publish(topic, byteData); err != nil {
					glog.Errorf("Unable to publish global event for topic %s:  %s", topic, err)
				}
			}()
		}
	}
}

func (p FileInfoModifiedSlice) Less(i, j int) bool {
	iFileInfo := p.fileInfos[i]
	iModTime := iFileInfo.ModTime()
	if iFileInfo.Mode()&os.ModeSymlink == os.ModeSymlink {
		destFileInfo, err := getSymlinkFileInfo(iFileInfo)
		if err != nil {
			glog.Errorf("Could not follow %s: %s", iFileInfo.Name(), err)
		} else {
			iModTime = destFileInfo.ModTime()
		}
	}

	jFileInfo := p.fileInfos[j]
	jModTime := jFileInfo.ModTime()
	if jFileInfo.Mode()&os.ModeSymlink == os.ModeSymlink {
		destFileInfo, err := getSymlinkFileInfo(jFileInfo)
		if err != nil {
			glog.Errorf("Could not follow %s: %s", jFileInfo.Name(), err)
		} else {
			jModTime = destFileInfo.ModTime()
		}
	}

	if p.reverseSort {
		return iModTime.After(jModTime)
	} else {
		return iModTime.Before(jModTime)
	}
}

// performAnalysis waits for files that need to be analyzed (from forAnalysis) and makes a copy of
// them in config.Aggregator.BinaryFuzzPath with their hash as a file name.
// It then analyzes it using the supplied AnalysisPackage and then signals the results should be
// uploaded. If any unrecoverable errors happen, this method terminates.
func performAnalysis(identifier int, analysisPackage AnalysisPackage, forAnalysis <-chan string, forUpload chan<- uploadPackage, terminated chan<- string) {
	glog.Infof("Spawning analyzer %d", identifier)
	prepareForExit := func(err error) {
		glog.Errorf("Analyzer %d terminated due to error: %s", identifier, err)
		terminated <- fmt.Sprintf("analyzer%d", identifier)
	}
	// our own unique working folder
	executableDir := filepath.Join(config.Aggregator.ExecutablePath, fmt.Sprintf("analyzer%d", identifier))

	if err := analysisPackage.Setup(executableDir); err != nil {
		prepareForExit(err)
		return
	}

	for {
		badBinaryPath := <-forAnalysis
		hash, data, err := calculateHash(badBinaryPath)
		if err != nil {
			prepareForExit(err)
			return
		}
		newFuzzPath := filepath.Join(config.Aggregator.BinaryFuzzPath, hash)
		if err := ioutil.WriteFile(newFuzzPath, data, 0644); err != nil {
			prepareForExit(err)
			return
		}
		if upload, err := analysisPackage.Analyze(executableDir, hash); err != nil {
			glog.Errorf("Problem analyzing %s", newFuzzPath)
			prepareForExit(err)
			return
		} else {
			forUpload <- upload
		}
	}
}

// processTileStream processes the first tile instantly and starts a background
// process to recalculate the blame lists as tiles and expectations change.
func (b *Blamer) processTileStream() error {
	expChanges := make(chan []string)
	b.storages.EventBus.SubscribeAsync(expstorage.EV_EXPSTORAGE_CHANGED, func(e interface{}) {
		expChanges <- e.([]string)
	})
	tileStream := b.storages.GetTileStreamNow(2*time.Minute, false)

	lastTile := <-tileStream
	if err := b.updateBlame(lastTile); err != nil {
		return err
	}

	// Schedule a background process to keep updating the blame lists.
	go func() {
		for {
			select {
			case tile := <-tileStream:
				if err := b.updateBlame(tile); err != nil {
					glog.Errorf("Error updating blame lists: %s", err)
				} else {
					lastTile = tile
				}
			case <-expChanges:
				storage.DrainChangeChannel(expChanges)
				if err := b.updateBlame(lastTile); err != nil {
					glog.Errorf("Error updating blame lists: %s", err)
				}
			}
		}
	}()

	return nil
}

func (h *historian) backFillDigestInfo(tilesToBackfill int) {
	go func() {
		// Get the first tile and determine the tile id of the first tile
		var err error
		lastTile, err := h.storages.TileStore.Get(0, -1)
		if err != nil {
			glog.Errorf("Unable to retrieve last tile. Quiting backfill. Error: %s", err)
			return
		}

		var tile *tiling.Tile
		firstTileIndex := util.MaxInt(lastTile.TileIndex-tilesToBackfill+1, 0)
		for idx := firstTileIndex; idx <= lastTile.TileIndex; idx++ {
			if tile, err = h.storages.TileStore.Get(0, idx); err != nil {
				glog.Errorf("Unable to read tile %d from tile store. Quiting backfill. Error: %s", idx, err)
				return
			}

			// Process the tile, but giving higher priority to digests from the
			// latest tile.
			if err = h.processTile(tile); err != nil {
				glog.Errorf("Error processing tile: %s", err)
			}

			// Read the last tile, just to make sure it has not changed.
			if lastTile, err = h.storages.TileStore.Get(0, -1); err != nil {
				glog.Errorf("Unable to retrieve last tile. Quiting backfill. Error: %s", err)
				return
			}
		}
	}()
}

// polyAllHashesHandler returns the list of all hashes we currently know about regardless of triage status.
//
// Endpoint used by the Android buildbots to avoid transferring already known images.
func polyAllHashesHandler(w http.ResponseWriter, r *http.Request) {
	unavailableDigests := storages.DiffStore.UnavailableDigests()

	byTest := tallies.ByTest()
	hashes := map[string]bool{}
	for _, test := range byTest {
		for k, _ := range test {
			if _, ok := unavailableDigests[k]; !ok {
				hashes[k] = true
			}
		}
	}

	w.Header().Set("Content-Type", "text/plain")
	for k, _ := range hashes {
		if _, err := w.Write([]byte(k)); err != nil {
			glog.Errorf("Failed to write or encode result: %s", err)
			return
		}
		if _, err := w.Write([]byte("\n")); err != nil {
			glog.Errorf("Failed to write or encode result: %s", err)
			return
		}
	}
}

func writeOutAllSourceImages() {
	// Pull all the source images from the db and write them out to inout.
	rows, err := db.Query("SELECT id, image, create_ts FROM source_images ORDER BY create_ts DESC")
	if err != nil {
		glog.Errorf("Failed to open connection to SQL server: %q\n", err)
		panic(err)
	}
	defer util.Close(rows)
	for rows.Next() {
		var id int
		var image []byte
		var create_ts time.Time
		if err := rows.Scan(&id, &image, &create_ts); err != nil {
			glog.Errorf("failed to fetch from database: %q", err)
			continue
		}
		filename := fmt.Sprintf(filepath.Join(config.Fiddle.InoutPath, "image-%d.png"), id)
		if _, err := os.Stat(filename); os.IsExist(err) {
			glog.Infof("Skipping write since file exists: %q", filename)
			continue
		}
		if err := ioutil.WriteFile(filename, image, 0666); err != nil {
			glog.Errorf("failed to write image file: %q", err)
		}
	}
}

// scanForNewCandidates runs scanHelper once every config.Aggregator.RescanPeriod, which scans the
// config.Generator.AflOutputPath for new fuzzes.
// If scanHelper returns an error, this method will terminate.
func (agg *BinaryAggregator) scanForNewCandidates() {
	defer agg.monitoringWaitGroup.Done()

	alreadyFoundBinaries := &SortedStringSlice{}
	// time.Tick does not fire immediately, so we fire it manually once.
	if err := agg.scanHelper(alreadyFoundBinaries); err != nil {
		glog.Errorf("Scanner terminated due to error: %v", err)
		return
	}
	glog.Infof("Sleeping for %s, then waking up to find new crashes again", config.Aggregator.RescanPeriod)

	t := time.Tick(config.Aggregator.RescanPeriod)
	for {
		select {
		case <-t:
			if err := agg.scanHelper(alreadyFoundBinaries); err != nil {
				glog.Errorf("Aggregator scanner terminated due to error: %v", err)
				return
			}
			glog.Infof("Sleeping for %s, then waking up to find new crashes again", config.Aggregator.RescanPeriod)
		case <-agg.monitoringShutdown:
			glog.Info("Aggregator scanner got signal to shut down")
			return
		}

	}
}

func AttemptLoadCertFromMetadata() {
	if *certChainFile == "" {
		// Try loading from GCE project level metadata.
		certChainContents, err := metadata.ProjectGet("ca_cert_chain")
		if err != nil {
			glog.Errorf("Failed to load ca_cert_chain from metadata: %s", err)
			return
		}
		keyContents, err := metadata.ProjectGet("ca_key")
		if err != nil {
			glog.Errorf("Failed to load ca_key from metadata: %s", err)
			return
		}
		fullCertChainFilename := filepath.Join(*workDir, CA_CERT_CHAIN_FILENAME)
		fullKeyFilename := filepath.Join(*workDir, CA_KEY_FILENAME)
		if err := ioutil.WriteFile(fullCertChainFilename, []byte(certChainContents), 0600); err != nil {
			glog.Errorf("Failed to write %s: %s", fullCertChainFilename, err)
			return
		}
		if err := ioutil.WriteFile(fullKeyFilename, []byte(keyContents), 0600); err != nil {
			glog.Errorf("Failed to write %s: %s", fullKeyFilename, err)
			return
		}
		*keyFile = fullKeyFilename
		*certChainFile = fullCertChainFilename
		glog.Infof("SUCCESS: Loaded cert from metadata.")
	}
}

func main() {
	defer common.LogPanic()
	common.Init()

	args := flag.Args()
	if len(args) != 3 {
		glog.Errorf("Expected arguments: branch target buildID")
		glog.Errorf("i.e.:  git_master-skia razor-userdebug 1772442")
		os.Exit(1)
	}

	// Set the arguments necessary to lookup the git hash.
	branch := args[0]
	target := args[1]
	buildID := args[2]
	glog.Infof("Branch, target, buildID: %s, %s, %s", branch, target, buildID)

	// Set up the oauth client.
	var client *http.Client
	var err error

	// In this case we don't want a backoff transport since the Apiary backend
	// seems to fail a lot, so we basically want to fall back to polling if a
	// call fails.
	transport := &http.Transport{
		Dial: util.DialTimeout,
	}

	if *local {
		// Use a local client secret file to load data.
		client, err = auth.InstalledAppClient(OAUTH_CACHE_FILEPATH, CLIENT_SECRET_FILEPATH,
			transport,
			androidbuildinternal.AndroidbuildInternalScope,
			storage.CloudPlatformScope)
		if err != nil {
			glog.Fatalf("Unable to create installed app oauth client:%s", err)
		}
	} else {
		// Use compute engine service account.
		client = auth.GCEServiceAccountClient(transport)
	}

	f, err := androidbuild.New("/tmp/android-gold-ingest", client)
	if err != nil {
		glog.Fatalf("Failed to construct client: %s", err)
	}
	for {
		r, err := f.Get(branch, target, buildID)
		if err != nil {
			glog.Errorf("Failed to get requested info: %s", err)
			time.Sleep(1 * time.Minute)
			continue
		}
		if r != nil {
			glog.Infof("Successfully found: %#v", *r)
		}

		time.Sleep(1 * time.Minute)
	}
}

// buildDM builds the test harness for parsing skp (and other) files.
// First it creates a hard link for the gyp and cpp files. The gyp file is linked into Skia's gyp folder and the cpp file is linked into SKIA_ROOT/../fuzzer_cache/src, which is where the gyp file is configured to point.
// Then it activates Skia's gyp command, which creates the build (ninja) files.
// Finally, it runs those build files.
// If any step fails in unexpected ways, it returns an error.
func buildDM(buildType string, isClean bool) error {
	glog.Infof("Building %s dm", buildType)

	// Change directory to skia folder
	if err := os.Chdir(config.Generator.SkiaRoot); err != nil {
		return err
	}

	// clean previous build if specified
	buildLocation := filepath.Join("out", buildType)
	if isClean {
		if err := os.RemoveAll(buildLocation); err != nil {
			return err
		}
	}

	// run gyp
	if message, err := exec.RunSimple("./gyp_skia"); err != nil {
		glog.Errorf("Failed gyp message: %s", message)
		return err
	}

	// run ninja
	cmd := fmt.Sprintf("ninja -C %s dm", buildLocation)
	if message, err := exec.RunSimple(cmd); err != nil {
		glog.Errorf("Failed ninja message: %s", message)
		return err
	}
	return nil
}

// StartCleaner is a process that periodically checks the status of every issue
// that has been previewed and removes all the local files for closed issues.
func StartCleaner(workDir string) {
	glog.Info("Starting Cleaner")
	c := reitveld.NewClient()
	for _ = range time.Tick(config.REFRESH) {
		matches, err := filepath.Glob(workDir + "/patches/*")
		glog.Infof("Matches: %v", matches)
		if err != nil {
			glog.Errorf("Failed to retrieve list of patched checkouts: %s", err)
			continue
		}
		for _, filename := range matches {
			_, file := filepath.Split(filename)
			glog.Info(file)
			m := issueAndPatch.FindStringSubmatch(file)
			if len(m) < 2 {
				continue
			}
			issue, err := strconv.ParseInt(m[1], 10, 64)
			if err != nil {
				glog.Errorf("Failed to parse %q as int: %s", m[1], err)
				continue
			}
			issueInfo, err := c.Issue(issue)
			if err != nil {
				glog.Errorf("Failed to retrieve issue status %d: %s", issue, err)
			}
			if issueInfo.Closed {
				if err := os.RemoveAll(filename); err != nil {
					glog.Errorf("Failed to remove %q: %s", filename, err)
				}
			}
		}
	}
}

// waitForAnalysis waits for files that need to be analyzed (from forAnalysis) and makes a copy of
// them in config.Aggregator.BinaryFuzzPath with their hash as a file name.
// It then analyzes it using the supplied AnalysisPackage and then signals the results should be
// uploaded. If any unrecoverable errors happen, this method terminates.
func (agg *BinaryAggregator) waitForAnalysis(identifier int, analysisPackage AnalysisPackage) {
	defer agg.pipelineWaitGroup.Done()
	defer go_metrics.GetOrRegisterCounter("analysis_process_count", go_metrics.DefaultRegistry).Dec(int64(1))
	glog.Infof("Spawning analyzer %d", identifier)

	// our own unique working folder
	executableDir := filepath.Join(config.Aggregator.ExecutablePath, fmt.Sprintf("analyzer%d", identifier))
	if err := analysisPackage.Setup(executableDir); err != nil {
		glog.Errorf("Analyzer %d terminated due to error: %s", identifier, err)
		return
	}
	for {
		select {
		case badBinaryPath := <-agg.forAnalysis:
			atomic.AddInt64(&agg.analysisCount, int64(1))
			err := agg.analysisHelper(executableDir, badBinaryPath, analysisPackage)
			if err != nil {
				glog.Errorf("Analyzer %d terminated due to error: %s", identifier, err)
				return
			}
		case <-agg.pipelineShutdown:
			glog.Infof("Analyzer %d recieved shutdown signal", identifier)
			return
		}
	}
}

func runBenchmark(fileInfoName, pathToPagesets, pathToPyFiles, localOutputDir, chromiumBuildName, chromiumBinary, runID, browserExtraArgs string) error {
	pagesetBaseName := filepath.Base(fileInfoName)
	if pagesetBaseName == util.TIMESTAMP_FILE_NAME || filepath.Ext(pagesetBaseName) == ".pyc" {
		// Ignore timestamp files and .pyc files.
		return nil
	}

	// Convert the filename into a format consumable by the run_benchmarks
	// binary.
	pagesetName := strings.TrimSuffix(pagesetBaseName, filepath.Ext(pagesetBaseName))
	pagesetPath := filepath.Join(pathToPagesets, fileInfoName)

	glog.Infof("===== Processing %s for %s =====", pagesetPath, runID)

	skutil.LogErr(os.Chdir(pathToPyFiles))
	args := []string{
		util.BINARY_RUN_BENCHMARK,
		fmt.Sprintf("%s.%s", *benchmarkName, util.BenchmarksToPagesetName[*benchmarkName]),
		"--page-set-name=" + pagesetName,
		"--page-set-base-dir=" + pathToPagesets,
		"--also-run-disabled-tests",
	}

	// Need to capture output for all benchmarks.
	outputDirArgValue := filepath.Join(localOutputDir, pagesetName)
	args = append(args, "--output-dir="+outputDirArgValue)
	// Figure out which browser should be used.
	if *targetPlatform == util.PLATFORM_ANDROID {
		if err := installChromeAPK(chromiumBuildName); err != nil {
			return fmt.Errorf("Error while installing APK: %s", err)
		}
		args = append(args, "--browser=android-chrome-shell")
	} else {
		args = append(args, "--browser=exact", "--browser-executable="+chromiumBinary)
	}
	// Split benchmark args if not empty and append to args.
	if *benchmarkExtraArgs != "" {
		for _, benchmarkArg := range strings.Split(*benchmarkExtraArgs, " ") {
			args = append(args, benchmarkArg)
		}
	}
	// Add the number of times to repeat.
	args = append(args, fmt.Sprintf("--page-repeat=%d", *repeatBenchmark))
	// Add browserArgs if not empty to args.
	if browserExtraArgs != "" {
		args = append(args, "--extra-browser-args="+browserExtraArgs)
	}
	// Set the PYTHONPATH to the pagesets and the telemetry dirs.
	env := []string{
		fmt.Sprintf("PYTHONPATH=%s:%s:%s:$PYTHONPATH", pathToPagesets, util.TelemetryBinariesDir, util.TelemetrySrcDir),
		"DISPLAY=:0",
	}
	timeoutSecs := util.PagesetTypeToInfo[*pagesetType].RunChromiumPerfTimeoutSecs
	if err := util.ExecuteCmd("python", args, env, time.Duration(timeoutSecs)*time.Second, nil, nil); err != nil {
		glog.Errorf("Run benchmark command failed with: %s", err)
		glog.Errorf("Killing all running chrome processes in case there is a non-recoverable error.")
		skutil.LogErr(util.ExecuteCmd("pkill", []string{"-9", "chrome"}, []string{}, 5*time.Minute, nil, nil))
	}
	return nil
}

// tileHandler accepts URIs like /tiles/0/1
// where the URI format is /tiles/<tile-scale>/<tile-number>
//
// It returns JSON of the form:
//
//  {
//    tiles: [20],
//    scale: 0,
//    paramset: {
//      "os": ["Android", "ChromeOS", ..],
//      "arch": ["Arm7", "x86", ...],
//    },
//    commits: [
//      {
//        "commit_time": 140329432,
//        "hash": "0e03478100ea",
//        "author": "[email protected]",
//        "commit_msg": "The subject line of the commit.",
//      },
//      ...
//    ],
//    ticks: [
//      [1.5, "Mon"],
//      [3.5, "Tue"]
//    ],
//    skps: [
//      5, 13, 24
//    ]
//  }
//
//  Where skps are the commit indices where the SKPs were updated.
//
func tileHandler(w http.ResponseWriter, r *http.Request) {
	glog.Infof("Tile Handler: %q\n", r.URL.Path)
	handlerStart := time.Now()
	match := tileHandlerPath.FindStringSubmatch(r.URL.Path)
	if r.Method != "GET" || match == nil || len(match) != 3 {
		http.NotFound(w, r)
		return
	}
	tileScale, err := strconv.ParseInt(match[1], 10, 0)
	if err != nil {
		util.ReportError(w, r, err, "Failed parsing tile scale.")
		return
	}
	tileNumber, err := strconv.ParseInt(match[2], 10, 0)
	if err != nil {
		util.ReportError(w, r, err, "Failed parsing tile number.")
		return
	}
	glog.Infof("tile: %d %d", tileScale, tileNumber)
	tile, err := getTile(int(tileScale), int(tileNumber))
	if err != nil {
		util.ReportError(w, r, err, "Failed retrieving tile.")
		return
	}

	guiTile := tiling.NewTileGUI(tile.Scale, tile.TileIndex)
	guiTile.Commits = tile.Commits
	guiTile.ParamSet = tile.ParamSet
	// SkpCommits goes out to the git repo, add caching if this turns out to be
	// slow.
	if skps, err := git.SkpCommits(tile); err != nil {
		guiTile.Skps = []int{}
		glog.Errorf("Failed to calculate skps: %s", err)
	} else {
		guiTile.Skps = skps
	}

	ts := []int64{}
	for _, c := range tile.Commits {
		if c.CommitTime != 0 {
			ts = append(ts, c.CommitTime)
		}
	}
	glog.Infof("%#v", ts)
	guiTile.Ticks = human.FlotTickMarks(ts)

	// Marshal and send
	marshaledResult, err := json.Marshal(guiTile)
	if err != nil {
		util.ReportError(w, r, err, "Failed to marshal JSON.")
		return
	}
	w.Header().Set("Content-Type", "application/json")
	_, err = w.Write(marshaledResult)
	if err != nil {
		glog.Errorf("Failed to write or encode output: %s", err)
	}
	glog.Infoln("Total handler time: ", time.Since(handlerStart).Nanoseconds())
}

// UpdateCommitInfo finds all the new commits since the last time we ran and
// adds them to the tiles, creating new tiles if necessary.
func (i *Ingester) UpdateCommitInfo(pull bool) error {
	glog.Infof("Ingest %s: Starting UpdateCommitInfo", i.datasetName)
	if err := i.git.Update(pull, false); err != nil {
		return fmt.Errorf("Ingest %s: Failed git pull for during UpdateCommitInfo: %s", i.datasetName, err)
	}

	// Compute Git CL number for each Git hash.
	allHashes := i.git.From(time.Time(BEGINNING_OF_TIME))
	hashToNumber := map[string]int{}
	for i, h := range allHashes {
		hashToNumber[h] = i
	}
	i.hashToNumber = hashToNumber

	// Find the time of the last Commit seen.
	ts := time.Time(BEGINNING_OF_TIME)
	lastTile, err := i.tileStore.Get(0, -1)
	if err == nil && lastTile != nil {
		ts = i.lastCommitTimeInTile(lastTile)
	} else {
		// Boundary condition; just started making Tiles and none exist.
		newTile := tiling.NewTile()
		newTile.Scale = 0
		newTile.TileIndex = 0
		if err := i.tileStore.Put(0, 0, newTile); err != nil {
			return fmt.Errorf("Ingest %s: UpdateCommitInfo: Failed to write new tile: %s", i.datasetName, err)
		}
	}
	glog.Infof("Ingest %s: UpdateCommitInfo: Last commit timestamp: %s", i.datasetName, ts)

	// Find all the Git hashes that are new to us.
	newHashes := i.git.From(ts)

	glog.Infof("Ingest %s: len(newHashes): from %d", i.datasetName, len(newHashes))

	// Add Commit info to the Tiles for each new hash.
	tt := NewTileTracker(i.tileStore, i.hashToNumber)
	for _, hash := range newHashes {
		if err := tt.Move(hash); err != nil {
			glog.Errorf("UpdateCommitInfo Move(%s) failed with: %s", hash, err)
			continue
		}
		details, err := i.git.Details(hash, true)
		if err != nil {
			glog.Errorf("Failed to get details for hash: %s: %s", hash, err)
			continue
		}
		tt.Tile().Commits[tt.Offset(hash)] = &tiling.Commit{
			CommitTime: details.Timestamp.Unix(),
			Hash:       hash,
			Author:     details.Author,
		}
	}
	glog.Infof("Ingest %s: Starting to flush tile.", i.datasetName)
	tt.Flush()

	glog.Infof("Ingest %s: Finished UpdateCommitInfo", i.datasetName)
	return nil
}