// Update the database if the lastPullTime is more than 6 hours before the current time
func updateDB(db *sql.DB, ctx context.Context, lastPullTime int64) int64 {
	// If the last pull was more than 6 hours ago
	if lastPull < time.Now().Add(-1*timeout).Unix() {
		log.Infof(ctx, "Updating database")

		// Remove deleted questions from the database
		log.Infof(ctx, "Removing deleted questions from db")
		if err := backend.RemoveDeletedQuestions(db, ctx); err != nil {
			log.Warningf(ctx, "Error removing deleted questions: %v", err.Error())
			return lastPullTime
		}

		// Setting time frame to get new questions.
		toDate := time.Now()
		fromDate := time.Unix(lastPull, 0)

		// Collect new questions from SO
		questions, err := backend.GetNewQns(fromDate, toDate)
		if err != nil {
			log.Warningf(ctx, "Error getting new questions: %v", err.Error())
			return lastPullTime
		}

		// Add new questions to database
		log.Infof(ctx, "Adding new questions to db")
		if err := backend.AddQuestions(db, ctx, questions); err != nil {
			log.Warningf(ctx, "Error adding new questions: %v", err.Error())
			return lastPullTime
		}

		lastPullTime = time.Now().Unix()
		log.Infof(ctx, "New questions added")
	}
	return lastPullTime
}

func deleteData(res http.ResponseWriter, req *http.Request) {
	ctx := appengine.NewContext(req)
	u := user.Current(ctx)
	keyVal := req.FormValue("keyVal")
	key, err := datastore.DecodeKey(keyVal)
	if err != nil {
		http.Error(res, "Invalid data", http.StatusBadRequest)
		log.Warningf(ctx, err.Error())
		return
	}
	var l list
	err = datastore.Get(ctx, key, &l)
	if err != nil {
		http.Error(res, "Invalid data", http.StatusBadRequest)
		log.Warningf(ctx, err.Error())
		return
	}
	if l.Owner != u.Email {
		http.Error(res, "Not authorized to delete this entry", http.StatusUnauthorized)
		log.Warningf(ctx, err.Error())
		return
	}
	err = datastore.Delete(ctx, key)
	if err != nil {
		http.Error(res, "Server Error", http.StatusInternalServerError)
		log.Errorf(ctx, err.Error())
		return
	}
}

//Function called when the /viewTags request is made
//Retrieves all distinct tags and the number of questions saved in the db with that tag
func readTagsFromDb(ctx context.Context) []tagData {
	var tempData []tagData
	var (
		tag   sql.NullString
		count sql.NullInt64
	)

	rows, err := db.Query("SELECT tag, COUNT(tag) FROM question_tag GROUP BY tag")
	if err != nil {
		log.Warningf(ctx, "Tag query failed: %v", err.Error())
		return tempData
	}

	defer rows.Close()
	for rows.Next() {
		err := rows.Scan(&tag, &count)
		if err != nil {
			log.Warningf(ctx, "Tag Scan failed: %v", err.Error())
			continue
		}
		currentTag := tagData{tag.String, int(count.Int64)}
		tempData = append(tempData, currentTag)
	}

	return tempData
}

// determine whether old request has ended according to logs
func (l *Locker) previousRequestEnded(c context.Context, requestID string) bool {
	q := &log.Query{
		RequestIDs: []string{requestID},
	}
	results := q.Run(c)
	record, err := results.Next()
	if err == log.Done {
		// no record found so it hasn't ended
		if l.LogVerbose {
			log.Warningf(c, "no log found for previous request %s", requestID)
		}
		return false
	}
	if err != nil {
		// Managed VMs do not have access to the logservice API
		if l.LogVerbose {
			log.Warningf(c, "err getting log for previous request %s %v", requestID, err)
		}
		return false
	}
	if l.LogVerbose {
		log.Debugf(c, "found previous request log %v", record)
	}
	return record.Finished
}

// Handle wraps a task handler with task / lock processing
func (l *Locker) Handle(handler TaskHandler, factory EntityFactory) http.Handler {
	fn := func(w http.ResponseWriter, r *http.Request) {
		c := appengine.NewContext(r)

		// ensure request is a task request
		if r.Method != "POST" || r.Header.Get("X-Appengine-TaskName") == "" {
			log.Warningf(c, "non task request")
			w.WriteHeader(http.StatusBadRequest)
			return
		}

		// use the same queue name for any tasks scheduled by this handler
		queue := r.Header.Get("X-Appengine-QueueName")
		c = WithQueue(c, queue)

		key, seq, err := l.Parse(c, r)
		if err != nil {
			log.Warningf(c, "parse failed: %v", err)
			w.WriteHeader(http.StatusBadRequest)
			return
		}

		entity := factory()
		err = l.Aquire(c, key, entity, seq)
		if err != nil {
			log.Warningf(c, "lock failed: %v", err)
			// if we have a lock error, it provides the http response to use
			if lerr, ok := err.(Error); ok {
				w.WriteHeader(lerr.Response)
			} else {
				w.WriteHeader(http.StatusInternalServerError)
			}
			return
		}

		// TODO: explore having handler return something to indicate
		// if the task needs to continue with the next seq or be completed
		err = handler(c, r, key, entity)
		if err != nil {
			log.Warningf(c, "handler failed: %v", err)
			// clear the lock to allow the next retry
			if err := l.clearLock(c, key, entity); err != nil {
				log.Warningf(c, "clearLock failed: %v", err)
				// if we have a lock error, it provides the http response to use
				if lerr, ok := err.(Error); ok {
					w.WriteHeader(lerr.Response)
				} else {
					w.WriteHeader(http.StatusInternalServerError)
				}
			} else {
				w.WriteHeader(http.StatusInternalServerError)
			}
			return
		}

		w.WriteHeader(http.StatusOK)
	}

	return http.HandlerFunc(fn)
}

func adminUpdateTask(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)

	tid := r.FormValue("taskKey")

	k, err := datastore.DecodeKey(tid)
	if err != nil {
		panic(err)
	}

	task := &Task{}
	if err := datastore.Get(c, k, task); err != nil {
		log.Warningf(c, "Error getting task %v: %v", k, err)
		http.Error(w, err.Error(), 404)
		return
	}

	task.Name = r.FormValue("name")
	task.Description = r.FormValue("description")
	task.Value = asInt(r.FormValue("value"))
	task.Period = asInt(r.FormValue("period"))
	task.Disabled = mightParseBool(r.FormValue("disabled"))
	task.Automatic = mightParseBool(r.FormValue("automatic"))
	task.Assignee = r.FormValue("assignee")

	if _, err := datastore.Put(c, k, task); err != nil {
		log.Warningf(c, "Error storing task %v, %+v: %v", k, task, err)
		http.Error(w, err.Error(), 500)
		return
	}
	w.WriteHeader(204)
}

// Handler for pulling questions from Stack Overflow manually, based on a given ID
// Request is parsed to find the supplied ID
// A check is completed to see if the question is already in the system
// If so, it retrieves that question, and returns it to be viewed, along with a message
// Makes a new backend request to retrieve new questions
// Parses the returned data into a new page, which can be inserted into the template.
func newQnHandler(w http.ResponseWriter, r *http.Request, ctx context.Context) {
	id, _ := strconv.Atoi(r.FormValue("id"))

	res, err := backend.CheckForExistingQuestion(db, id)
	if err != nil {
		log.Infof(ctx, "QUERY FAILED, %v", err)
	}

	if res == 1 {

		existingQn := backend.PullQnByID(db, ctx, id)
		if err != nil {
			log.Warningf(ctx, err.Error())
		}
		w.Write(existingQn)
	} else {

		intArray := []int{id}
		questions, err := backend.GetQuestions(ctx, intArray)
		if err != nil {
			log.Warningf(ctx, err.Error())
		} else {
			questions.Items[0].Body = backend.StripTags(questions.Items[0].Body)
			qnJson, err := json.Marshal(questions.Items[0])
			if err != nil {
				log.Warningf(ctx, err.Error())
			}
			w.Write(qnJson)
		}
	}
}

// PrimaryPublicCertificates returns primary's PublicCertificates.
func PrimaryPublicCertificates(c context.Context, primaryURL string) (*PublicCertificates, error) {
	cacheKey := fmt.Sprintf("pub_certs:%s", primaryURL)
	var pubCerts []byte
	setCache := false
	item, err := memcache.Get(c, cacheKey)
	if err != nil {
		setCache = true
		if err != memcache.ErrCacheMiss {
			log.Warningf(c, "failed to get cert from cache: %v", err)
		}
		pubCerts, err = downloadCert(urlfetch.Client(c), primaryURL)
		if err != nil {
			log.Errorf(c, "failed to download cert: %v", err)
			return nil, err
		}
	} else {
		pubCerts = item.Value
	}
	pc := &PublicCertificates{}
	if err = json.Unmarshal(pubCerts, pc); err != nil {
		log.Errorf(c, "failed to unmarshal cert: %v %v", string(pubCerts), err)
		return nil, err
	}
	if setCache {
		err = memcache.Set(c, &memcache.Item{
			Key:        cacheKey,
			Value:      pubCerts,
			Expiration: time.Hour,
		})
		if err != nil {
			log.Warningf(c, "failed to set cert to cache: %v", err)
		}
	}
	return pc, nil
}

func handleTweet(res http.ResponseWriter, req *http.Request) {
	ctx := appengine.NewContext(req)
	u := getCurrentUser(req)
	if u == nil {
		http.Error(res, "Incorrect login", http.StatusUnauthorized)
		log.Warningf(ctx, "Incorrect login from: %s\n", req.RemoteAddr)
		return
	}
	if req.Method != "POST" {
		http.Error(res, "Unknown method", http.StatusMethodNotAllowed)
		log.Warningf(ctx, "Incorrect method on tweet.json from %s", req.RemoteAddr)
		return
	}
	buffer := make([]byte, tweetSize)
	n, err := req.Body.Read(buffer)
	if err != nil && err != io.EOF {
		http.Error(res, "Bad Request", http.StatusBadRequest)
		log.Warningf(ctx, "Bad request: %s\n", err.Error())
		return
	}
	msg := string(buffer[:n])
	t := tweet{
		Username:   u.Username,
		Message:    msg,
		SubmitTime: time.Now(),
	}
	err = postTweet(ctx, &t, u.Email)
	if err != nil {
		http.Error(res, "Server error!", http.StatusInternalServerError)
		log.Errorf(ctx, "Put Tweet Error: %s\n", err.Error())
		return
	}
}

func adminAutoPay(w http.ResponseWriter, r *http.Request) {
	now := time.Now()
	c := appengine.NewContext(r)
	q := datastore.NewQuery("Task").
		Filter("Disabled =", false).
		Filter("Automatic = ", true).
		Filter("Next < ", now)

	tasks := []*Task{}
	if err := fillKeyQuery(c, q, &tasks); err != nil {
		log.Warningf(c, "Error finding automatic things: %v", err)
		w.WriteHeader(500)
		return
	}

	if len(tasks) == 0 {
		log.Infof(c, "No automatic tasks.")
		w.WriteHeader(204)
		return
	}

	storeKeys := make([]*datastore.Key, 0, 2*len(tasks))
	vals := []interface{}{}

	for i := range tasks {
		log.Infof(c, "Recording automatic task %q for %v at %s", tasks[i].Name,
			tasks[i].Assignee, moneyFmt(tasks[i].Value))

		su, err := getUserByEmail(c, tasks[i].Assignee)
		if err != nil {
			log.Warningf(c, "Failed to look up user %v: %v", tasks[i].Assignee, err)
			w.WriteHeader(500)
			return
		}

		tasks[i].updateTime()
		tasks[i].Prev = now
		storeKeys = append(storeKeys, tasks[i].Key)
		vals = append(vals, tasks[i])

		storeKeys = append(storeKeys,
			datastore.NewIncompleteKey(c, "LoggedTask", nil))
		vals = append(vals, &LoggedTask{
			Task:      tasks[i].Key,
			User:      su.Key,
			Completed: now,
			Who:       su.Name,
			Name:      tasks[i].Name,
			Amount:    tasks[i].Value,
		})
	}

	if _, err := datastore.PutMulti(c, storeKeys, vals); err != nil {
		panic(err)
	}

	w.WriteHeader(204)
}

// Returns the current user requesting the page
func getUser(w http.ResponseWriter, r *http.Request, ctx context.Context) stackongo.User {
	guest := stackongo.User{
		Display_name: "Guest",
	}

	// Collect userId from browser cookie
	username, err := r.Cookie("user_name")
	if err == nil && username.Value != "" && username.Value != "Guest" {
		user := readUserFromDb(ctx, username.Value)
		updateLoginTime(ctx, user.User_id)
		return user
	}

	// If user_id cookie is not set, look for code in url request to collect access token.
	// If code is not available, return guest user
	code := r.FormValue("code")
	if code == "" {
		log.Infof(ctx, "Returning guest user")
		guest.User_type = "no_code"
		return guest
	}

	queries := r.URL.Query()
	queries.Del("code")
	r.URL.RawQuery = queries.Encode()
	// Collect access token using the recieved code
	access_tokens, err := backend.ObtainAccessToken(code, r.URL.String())
	if err != nil {
		log.Warningf(ctx, "Access token not obtained: %v", err.Error())
		guest.User_type = "no_access_token"
		return guest
	}

	// Get the authenticated user with the collected access token
	user, err := backend.AuthenticatedUser(map[string]string{}, access_tokens["access_token"])
	if err != nil {
		log.Warningf(ctx, "User not authenticated: %v", err)
		guest.User_type = "not_on_SO"
		return guest
	}

	// Add user to db if not already in
	addUserToDB(ctx, user)

	//zhu li do the thing
	updateLoginTime(ctx, user.User_id)
	return user
}

func recaptchaCheck(ctx context.Context, response, ip string) (bool, error) {
	if appengine.IsDevAppServer() {
		return true, nil
	}

	form := url.Values{}
	form.Add("secret", os.Getenv("SECRET"))
	form.Add("response", response)
	form.Add("remoteip", ip)
	req, err := http.NewRequest("POST", recaptchaURL, strings.NewReader(form.Encode()))
	if err != nil {
		return false, err
	}

	cli := urlfetch.Client(ctx)

	req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
	resp, err := cli.Do(req)
	if err != nil {
		return false, err
	}

	var recaptcha recaptchaResponse
	if err := json.NewDecoder(resp.Body).Decode(&recaptcha); err != nil {
		return false, err
	}

	if !recaptcha.Success {
		log.Warningf(ctx, "%+v", recaptcha)
		return false, nil
	}

	return true, nil
}

//Display a list of tags that are logged in the database
//User can either click on a tag to view any questions containing that tag
//Format array of tags into another array, to be easier formatted on the page into a table in the template
//An array of tagData arrays of size 4
func viewTagsHandler(w http.ResponseWriter, r *http.Request, ctx context.Context, pageNum int, user stackongo.User) {
	query := readTagsFromDb(ctx)
	var tagArray [][]tagData
	var tempTagArray []tagData
	i := 0
	for _, t := range query {
		tempTagArray = append(tempTagArray, t)
		i++
		if i == 4 {
			tagArray = append(tagArray, tempTagArray)
			i = 0
			//clear the temp array.
			tempTagArray = nil
		}
	}
	tagArray = append(tagArray, tempTagArray)
	page := template.Must(template.ParseFiles("public/viewTags.html"))
	first := (pageNum - 1) * 5
	last := pageNum * 5
	lastPage := len(tagArray) / 5
	if len(tagArray)%5 != 0 {
		lastPage++
	}
	if last > len(tagArray) {
		last = len(tagArray)
	}
	if err := page.Execute(w, queryReply{user, mostRecentUpdate, pageNum, lastPage, tagArray[first:last]}); err != nil {
		log.Warningf(ctx, "%v", err.Error())
	}

}

// Handler for adding a new question to the database upon submission
// It is returned as a stringified JSON object in the request body
// The string is unmarshalled into a stackongo.Question type, and added to an array
// to be added into the database using the AddQuestions function in backend/databasing.go
func addNewQuestionToDatabaseHandler(w http.ResponseWriter, r *http.Request, ctx context.Context) {
	body, err := ioutil.ReadAll(r.Body)
	if err != nil {
		log.Infof(ctx, "%v", err)
	}
	var f interface{}
	err = json.Unmarshal(body, &f)
	if err != nil {
		log.Infof(ctx, "%v", err)
	}
	m := f.(map[string]interface{})
	question := m["Question"]
	state := m["State"]
	if err != nil {
		log.Infof(ctx, "%v", err)
	}
	var qn stackongo.Question
	json.Unmarshal([]byte(question.(string)), &qn)
	log.Infof(ctx, "%v", qn)

	user := getUser(w, r, ctx)
	log.Infof(ctx, "%v", user.User_id)

	if err := backend.AddSingleQuestion(db, qn, state.(string), user.User_id); err != nil {
		log.Warningf(ctx, "Error adding new question to db:\t", err)
	}
	backend.UpdateTableTimes(db, ctx, "question")
}

// Handler for adding new question page
func addQuestionHandler(w http.ResponseWriter, r *http.Request, ctx context.Context,
	pageNum int, user stackongo.User) {
	page := template.Must(template.ParseFiles("public/addQuestion.html"))
	if err := page.Execute(w, queryReply{user, mostRecentUpdate, pageNum, 0, nil}); err != nil {
		log.Warningf(ctx, "%v", err.Error())
	}
}

func createProfile(res http.ResponseWriter, req *http.Request) {
	ctx := appengine.NewContext(req)
	u := user.Current(ctx)

	if req.Method == "POST" {
		username := req.FormValue("username")
		// check if name is taken
		if !confirmCreateProfile(username) {
			http.Error(res, "Invalid input!", http.StatusBadRequest)
			log.Warningf(ctx, "Invalid profile information from %s\n", req.RemoteAddr)
			return
		}

		key := datastore.NewKey(ctx, "profile", u.Email, 0, nil)
		p := profile{
			Username: username,
			Email:    u.Email,
		}
		_, err := datastore.Put(ctx, key, &p)
		if err != nil {
			http.Error(res, "server error!", http.StatusInternalServerError)
			log.Errorf(ctx, "Create profile Error: &s\n", err.Error())
			return
		}
	}
	err := tpl.ExecuteTemplate(res, "createProfile.gohtml", nil)
	if err != nil {
		http.Error(res, "Serever error!", http.StatusInternalServerError)
		log.Errorf(ctx, "Template Parse Error: %s\n", err.Error())
		return
	}
}

// Handler to find all questions answered/being answered by the user in URL
func userHandler(w http.ResponseWriter, r *http.Request, ctx context.Context, pageNum int, user stackongo.User) {
	userID_string := r.FormValue("id")
	userID_int, _ := strconv.Atoi(userID_string)
	log.Infof(ctx, "current user id=%s", userID_string)

	// Create a new webData struct
	tempData, updateTime, err := readFromDb(ctx, "state='unanswered' OR questions.user="+userID_string)
	if err != nil {
		log.Errorf(ctx, "Error reading from db: %v", err.Error())
	} else {
		mostRecentUpdate = updateTime
	}

	// Parse the html template to serve to the page
	page := getTemplate("template.html")

	var Query = []string{
		"user",
		tempData.Users[userID_int].User_info.Display_name,
	}
	log.Infof(ctx, "Query = %v", Query)
	if err := page.Execute(w, writeResponse(user, tempData, pageNum, Query)); err != nil {
		log.Warningf(ctx, "%v", err.Error())
	}
}

func (sink *LogSink) writeLogStorage(r *wcg.LogRecord) error {
	c := sink.ctx
	formatter := sink.formatter
	if c == nil {
		c = NewContext(r.Request)
	}
	var text string
	if r.SourceStack != nil && len(r.SourceStack) > 0 {
		text = fmt.Sprintf(
			"%s\n-- Stack Trace --\n%s",
			string(formatter.Format(r)),
			strings.Join(r.SourceStack, "\n"),
		)
	} else {
		text = string(formatter.Format(r))
	}

	switch r.Level {
	case wcg.LogLevelTrace:
		log.Debugf(c, "%s", text) // app engine does not support trace level
	case wcg.LogLevelDebug:
		log.Debugf(c, "%s", text)
	case wcg.LogLevelInfo:
		log.Infof(c, "%s", text)
	case wcg.LogLevelWarn:
		log.Warningf(c, "%s", text)
	case wcg.LogLevelError:
		log.Errorf(c, "%s", text)
	case wcg.LogLevelFatal:
		log.Criticalf(c, "%s", text)
	default:
		// nothing
	}
	return nil
}

func adminNewTask(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)

	asInt := func(s string) int {
		i, err := strconv.Atoi(s)
		if err != nil {
			panic(err)
		}
		return i
	}

	task := Task{
		Name:        r.FormValue("name"),
		Description: r.FormValue("description"),
		Assignee:    r.FormValue("assignee"),
		RType:       r.FormValue("rtype"),
		Automatic:   mightParseBool(r.FormValue("automatic")),
		Period:      asInt(r.FormValue("period")),
		Value:       asInt(r.FormValue("value")),
		Next:        time.Now().UTC(),
	}

	k, err := datastore.Put(c,
		datastore.NewIncompleteKey(c, "Task", nil), &task)
	if err != nil {
		log.Warningf(c, "Error storing task:  %v", err)
		panic(err)
	}
	log.Infof(c, "Stored new thing with key %v", k)

	http.Redirect(w, r, "/admin/tasks/", 307)
}

func submitAddHandler(w http.ResponseWriter, r *http.Request) {
	ctx := appengine.NewContext(r)
	if err := r.ParseForm(); err != nil {
		serveErr(ctx, err, w)
		return
	}

	success, err := recaptchaCheck(ctx, r.FormValue("g-recaptcha-response"), r.RemoteAddr)
	if err != nil {
		serveErr(ctx, err, w)
		return
	}

	if !success {
		log.Warningf(ctx, "reCAPTCHA check failed")
		failedTmpl.ExecuteTemplate(w, "base", nil)
		return
	}

	sub := Submission{
		URL:       template.URL(strings.TrimSpace(r.FormValue("url"))),
		Submitted: time.Now(),
	}

	if _, err := datastore.Put(ctx, datastore.NewIncompleteKey(ctx, "Submission", nil), &sub); err != nil {
		serveErr(ctx, err, w)
		return
	}

	thanksTmpl.ExecuteTemplate(w, "base", nil)
}