func grabData() {
	resp, err := http.Get(redditURL)
	if err != nil {
		return
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		return
	}
	r := new(redditResponse)
	err = json.NewDecoder(resp.Body).Decode(r)
	if err != nil {
		return
	}
	data = make([]item, len(r.Data.Children))
	for i, child := range r.Data.Children {
		if !govalidator.IsURL(child.Data.URL) {
			continue
		}
		if !govalidator.IsURL(child.Data.Thumbnail) {
			child.Data.Thumbnail = ""
		}
		data[i] = child.Data
	}
}

func encodeHandler(response http.ResponseWriter, request *http.Request, db Database, baseURL string) {
	decoder := json.NewDecoder(request.Body)
	var data struct {
		URL string `json:"url"`
	}
	err := decoder.Decode(&data)
	if err != nil {
		http.Error(response, `{"error": "Unable to parse json"}`, http.StatusBadRequest)
		return
	}

	if !govalidator.IsURL(data.URL) {
		http.Error(response, `{"error": "Not a valid URL"}`, http.StatusBadRequest)
		return
	}

	id, err := db.Save(data.URL)
	if err != nil {
		log.Println(err)
		return
	}
	resp := map[string]string{"url": strings.Replace(path.Join(baseURL, encode(id)), ":/", "://", 1), "id": encode(id), "error": ""}
	jsonData, _ := json.Marshal(resp)
	response.Write(jsonData)

}

// ConvertToModel implements the FieldType interface
func (fieldType SimpleType) ConvertToModel(value interface{}) (interface{}, error) {
	if value == nil {
		return nil, nil
	}
	valueType := reflect.TypeOf(value)
	switch fieldType.GetKind() {
	case KindString, KindUser, KindIteration, KindArea:
		if valueType.Kind() != reflect.String {
			return nil, fmt.Errorf("value %v should be %s, but is %s", value, "string", valueType.Name())
		}
		return value, nil
	case KindURL:
		if valueType.Kind() == reflect.String && govalidator.IsURL(value.(string)) {
			return value, nil
		}
		return nil, fmt.Errorf("value %v should be %s, but is %s", value, "URL", valueType.Name())
	case KindFloat:
		if valueType.Kind() != reflect.Float64 {
			return nil, fmt.Errorf("value %v should be %s, but is %s", value, "float64", valueType.Name())
		}
		return value, nil
	case KindInteger, KindDuration:
		if valueType.Kind() != reflect.Int {
			return nil, fmt.Errorf("value %v should be %s, but is %s", value, "int", valueType.Name())
		}
		return value, nil
	case KindInstant:
		// instant == milliseconds
		if !valueType.Implements(timeType) {
			return nil, fmt.Errorf("value %v should be %s, but is %s", value, "time.Time", valueType.Name())
		}
		return value.(time.Time).UnixNano(), nil
	case KindWorkitemReference:
		if valueType.Kind() != reflect.String {
			return nil, fmt.Errorf("value %v should be %s, but is %s", value, "string", valueType.Name())
		}
		idValue, err := strconv.Atoi(value.(string))
		return idValue, errors.WithStack(err)
	case KindList:
		if (valueType.Kind() != reflect.Array) && (valueType.Kind() != reflect.Slice) {
			return nil, fmt.Errorf("value %v should be %s, but is %s,", value, "array/slice", valueType.Kind())
		}
		return value, nil
	case KindEnum:
		// to be done yet | not sure what to write here as of now.
		return value, nil
	case KindMarkup:
		// 'markup' is just a string in the API layer for now:
		// it corresponds to the MarkupContent.Content field. The MarkupContent.Markup is set to the default value
		switch value.(type) {
		case rendering.MarkupContent:
			markupContent := value.(rendering.MarkupContent)
			return markupContent.ToMap(), nil
		default:
			return nil, errors.Errorf("value %v should be %s, but is %s", value, "MarkupContent", valueType)
		}
	default:
		return nil, errors.Errorf("unexpected type constant: '%s'", fieldType.GetKind())
	}
}

func (site Site) saveShort(url string) (shortest string, err error) {
	if !govalidator.IsURL(url) {
		return "", errors.New("invalid url")
	}

	redisdb := site.redisdb()
	defer redisdb.Close()

	hash := fmt.Sprintf("%x", md5.Sum([]byte(url)))

	similar, _ := redis.String(redisdb.Do("GET", "i:"+hash))
	if similar != "" {
		return site.Host + similar, nil
	}

	for hashShortestLen := 1; hashShortestLen <= 32; hashShortestLen++ {
		s, _ := redisdb.Do("GET", hash[0:hashShortestLen])
		if s == nil {
			shortest = hash[0:hashShortestLen]
			break
		}
	}
	if shortest == "" {
		return "", errors.New("url shortening failed")
	}

	redisdb.Do("SET", shortest, url)
	redisdb.Do("SET", "i:"+hash, shortest)
	return site.Host + shortest, nil
}

// Spawn initializes the HTTP component
func (httpsender *HTTPSender) Spawn(id int) utils.Composer {
	s := *httpsender
	s.id = id

	if httpsender.Config.Logger == nil {
		s.logger = logrus.NewEntry(logrus.New())
		s.logger.Logger.Out = ioutil.Discard
	} else {
		s.logger = httpsender.Config.Logger.WithFields(logrus.Fields{
			"worker": id,
		})
	}

	if httpsender.Debug {
		s.logger.Logger.Level = logrus.DebugLevel
	}

	s.logger.Debugf("Spawning worker")

	if govalidator.IsURL(s.URL) {
		s.Client = new(http.Client)

		if httpsender.Config.Insecure {
			s.Client.Transport = &http.Transport{
				TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
			}
		}
	} else {
		s.err = errors.New("Invalid URL")
	}

	return &s
}

func parsePrometheusURL() error {
	if cfg.prometheusURL == "" {
		hostname, err := os.Hostname()
		if err != nil {
			return err
		}
		_, port, err := net.SplitHostPort(cfg.web.ListenAddress)
		if err != nil {
			return err
		}
		cfg.prometheusURL = fmt.Sprintf("http://%s:%s/", hostname, port)
	}

	if ok := govalidator.IsURL(cfg.prometheusURL); !ok {
		return fmt.Errorf("Invalid Prometheus URL: %s", cfg.prometheusURL)
	}

	promURL, err := url.Parse(cfg.prometheusURL)
	if err != nil {
		return err
	}
	cfg.web.ExternalURL = promURL

	ppref := strings.TrimRight(cfg.web.ExternalURL.Path, "/")
	if ppref != "" && !strings.HasPrefix(ppref, "/") {
		ppref = "/" + ppref
	}
	cfg.web.ExternalURL.Path = ppref
	return nil
}

func shitbucketImportHandler(w http.ResponseWriter, r *http.Request) error {
	if r.Method == "GET" {
		ctx := context.Get(r, TemplateContext).(map[string]interface{})
		ctx["Title"] = "Import"
		return renderTemplate(w, "shitbucket-import", ctx)
	}

	if err := r.ParseForm(); err != nil {
		return err
	}

	url := r.PostForm["url"][0]

	session, err := store.Get(r, "flashes")
	if !govalidator.IsURL(url) {
		if err != nil {
			return err
		}
		if url == "" {
			session.AddFlash("URL cannot be blank", "danger")
		} else {
			session.AddFlash(fmt.Sprintf("%s is not a valid URL", url), "danger")
		}
		session.Save(r, w)
	}

	res, err := http.Get(url)
	if err != nil {
		return err
	}

	if res.StatusCode != http.StatusOK {
		session.AddFlash(fmt.Sprintf("%s did not return a 200 status code", url), "danger")
		session.Save(r, w)
		http.Redirect(w, r, reverse("shitbucket-import"), http.StatusSeeOther)
		return nil
	}

	defer res.Body.Close()
	content, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return err
	}

	count, err := shitbucketImporter(content)
	if err != nil {
		session.AddFlash(fmt.Sprintf("There was an error importing: %s", err), "danger")
		session.Save(r, w)
		http.Redirect(w, r, reverse("shitbucket-import"), http.StatusSeeOther)
		return nil
	}

	session.AddFlash(fmt.Sprintf("Successfully added %d URLs from %s", count, url), "success")
	session.Save(r, w)

	http.Redirect(w, r, reverse("shitbucket-import"), http.StatusSeeOther)
	return nil
}

func validateURLs(urls []string) string {
	for _, curr := range urls {
		if !govalidator.IsURL(curr) {
			return curr
		}
	}

	return ""
}

func (n *NewChannel) Decode() error {
	n.URL = strings.Trim(n.URL, " ")
	if n.URL == "" || !govalidator.IsURL(n.URL) {
		return Errors{
			"url": "Valid URL is required",
		}
	}
	return nil
}

// CanHandle tells if the URL can be handled by this resolver
func (gh *GithubFetcher) CanHandle(kubeware string) bool {
	if !govalidator.IsURL(kubeware) {
		return false
	}
	kubewareURL, err := url.Parse(kubeware)
	if err != nil {
		return false
	}
	return gh.canHandleURL(kubewareURL)
}

func urlSubmitHandler(w http.ResponseWriter, r *http.Request) error {
	if err := r.ParseForm(); err != nil {
		return err
	}

	uschema := &URLSchema{}
	decoder := schema.NewDecoder()

	if err := decoder.Decode(uschema, r.PostForm); err != nil {
		return err
	}

	urlstring := uschema.URL
	tagsstring := uschema.Tags
	private := uschema.Private
	session, err := store.Get(r, "flashes")
	if err != nil {
		return err
	}
	if !govalidator.IsURL(urlstring) {
		errormessage := "URL is required"
		if urlstring != "" {
			errormessage = fmt.Sprintf("URL \"%s\" is not valid", urlstring)
		}
		session.AddFlash(errormessage, "danger")
		session.Save(r, w)
		http.Redirect(w, r, reverse("url-new"), http.StatusSeeOther)
		return nil
	}

	title, err := getPageTitle(urlstring)
	if err != nil {
		// <strike>Add flash about title not being fetchable</strike>
		// or alternatively add logic for detecting content type because it might be
		// an image or PDF
		session.AddFlash("Sorry! Could not fetch the page title!", "danger")
		session.Save(r, w)
	}

	url := &URL{
		URL:       urlstring,
		Title:     title,
		Private:   private,
		CreatedAt: time.Now(),
		UpdatedAt: time.Now(),
	}

	err = url.SaveWithTags(tagsstring)
	if err != nil {
		return err
	}

	http.Redirect(w, r, reverse("url-view", "id", url.ID), http.StatusSeeOther)
	return nil
}

//validCallBackURL the Mpesa *sys does not check for this* added as a convinience
func validCallBackURL(url string) validator {
	return func() *ProcessCheckoutResponse {
		if !govalidator.IsURL(url) {
			resp := new(ProcessCheckoutResponse)
			resp.ReturnCode = missingParameters
			resp.Description = "Invalid URL"
			resp.TransactionID = bson.NewObjectId().Hex()
			return resp
		}
		return nil
	}

}

func (ui *tatui) setTatWebUIURL(str string) {
	str = strings.Replace(str, "/set-tatwebui-url ", "", 1)
	if str == "" {
		return
	}
	validURL := govalidator.IsURL(str)
	if !validURL {
		ui.msg.Text = "You entered an invalid URL"
		ui.render()
		return
	}
	viper.Set("tatwebui-url", str)
	ui.saveConfig()
}

func buildCatalogCreationCheckRequest(repo string, branch string, number int, token string) *http.Request {
	url := fmt.Sprintf("https://api.github.com/repos/%s/contents/templates/%s/%d", repo, branch, number)
	if !govalidator.IsURL(url) {
		return nil
	}
	request, err := http.NewRequest("GET", url, nil)
	if err != nil {
		fmt.Print(err.Error())
		return nil
	}
	request.SetBasicAuth(token, "x-oauth-basic")
	request.Close = true
	return request
}

// Edits an existing Website in the database.
func ApiWebsitesEdit(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	if !lib.IsLoggedIn(r) {
		SendJsonMessage(w, http.StatusUnauthorized, false, "Unauthorized.")
		return
	}

	// Get data from Request
	r.ParseForm()
	oldUrl := ps.ByName("url")
	name := r.Form.Get("name")
	protocol := r.Form.Get("protocol")
	url := r.Form.Get("url")
	method := r.Form.Get("checkMethod")

	// Simple Validation
	if oldUrl == "" || name == "" || protocol == "" || url == "" || method == "" {
		SendJsonMessage(w, http.StatusBadRequest, false, "Unable to process your Request: Submit valid values.")
		return
	}
	if protocol != "http" && protocol != "https" {
		SendJsonMessage(w, http.StatusBadRequest, false, "Unable to process your Request: Submit a valid protocol.")
		return
	}
	if !govalidator.IsURL(protocol + "://" + url) {
		SendJsonMessage(w, http.StatusBadRequest, false, "Unable to process your Request: Submit a valid url.")
		return
	}
	if method != "HEAD" && method != "GET" {
		SendJsonMessage(w, http.StatusBadRequest, false, "Unable to process your Request: Submit a valid check method.")
		return
	}

	// Update Database
	db := lib.GetDatabase()
	res, err := db.Exec("UPDATE websites SET name = ?, protocol = ?, url = ?, checkMethod = ? WHERE url = ?;", name, protocol, url, method, oldUrl)
	if err != nil {
		logging.MustGetLogger("").Error("Unable to edit Website: ", err)
		SendJsonMessage(w, http.StatusInternalServerError, false, "Unable to process your Request: "+err.Error())
		return
	}

	// Check if exactly one Website has been edited
	rowsAffected, _ := res.RowsAffected()
	if rowsAffected == 1 {
		SendJsonMessage(w, http.StatusOK, true, "")
	} else {
		SendJsonMessage(w, http.StatusBadRequest, false, "Unable to process your Request: Could not edit Website.")
	}
}

func validateAlertmanagerURL() error {
	if cfg.notifier.AlertmanagerURL == "" {
		return nil
	}
	if ok := govalidator.IsURL(cfg.notifier.AlertmanagerURL); !ok {
		return fmt.Errorf("invalid Alertmanager URL: %s", cfg.notifier.AlertmanagerURL)
	}
	url, err := url.Parse(cfg.notifier.AlertmanagerURL)
	if err != nil {
		return err
	}
	if url.Scheme == "" {
		return fmt.Errorf("missing scheme in Alertmanager URL: %s", cfg.notifier.AlertmanagerURL)
	}
	return nil
}

func validateAlertmanagerURL(u string) error {
	if u == "" {
		return nil
	}
	if ok := govalidator.IsURL(u); !ok {
		return fmt.Errorf("invalid Alertmanager URL: %s", u)
	}
	url, err := url.Parse(u)
	if err != nil {
		return err
	}
	if url.Scheme == "" {
		return fmt.Errorf("missing scheme in Alertmanager URL: %s", u)
	}
	return nil
}

func parseInfluxdbURL() error {
	if cfg.influxdbURL == "" {
		return nil
	}

	if ok := govalidator.IsURL(cfg.influxdbURL); !ok {
		return fmt.Errorf("Invalid InfluxDB URL: %s", cfg.influxdbURL)
	}

	url, err := url.Parse(cfg.influxdbURL)
	if err != nil {
		return err
	}

	cfg.remote.InfluxdbURL = url
	return nil
}

// parseSearchString accepts a raw string and generates a searchKeyword object
func parseSearchString(rawSearchString string) (searchKeyword, error) {
	// TODO remove special characters and exclaimations if any
	rawSearchString = strings.Trim(rawSearchString, "/") // get rid of trailing slashes
	rawSearchString = strings.Trim(rawSearchString, "\"")
	parts := strings.Fields(rawSearchString)
	var res searchKeyword
	for _, part := range parts {
		// QueryUnescape is required in case of encoded url strings.
		// And does not harm regular search strings
		// but this processing is required because at this moment, we do not know if
		// search input is a regular string or a URL

		part, err := url.QueryUnescape(part)
		if err != nil {
			log.Warn(nil, map[string]interface{}{
				"pkg":  "search",
				"part": part,
			}, "unable to escape url!")
		}
		// IF part is for search with id:1234
		// TODO: need to find out the way to use ID fields.
		if strings.HasPrefix(part, "id:") {
			res.id = append(res.id, strings.TrimPrefix(part, "id:")+":*A")
		} else if strings.HasPrefix(part, "type:") {
			typeName := strings.TrimPrefix(part, "type:")
			if len(typeName) == 0 {
				return res, errors.NewBadParameterError("Type name must not be empty", part)
			}
			res.workItemTypes = append(res.workItemTypes, typeName)
		} else if govalidator.IsURL(part) {
			part := strings.ToLower(part)
			part = trimProtocolFromURLString(part)
			searchQueryFromURL := getSearchQueryFromURLString(part)
			res.words = append(res.words, searchQueryFromURL)
		} else {
			part := strings.ToLower(part)
			part = sanitizeURL(part)
			res.words = append(res.words, part+":*")
		}
	}
	return res, nil
}

// Inserts a new Website into the database.
func ApiWebsitesAdd(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	if !lib.IsLoggedIn(r) {
		SendJsonMessage(w, http.StatusUnauthorized, false, "Unauthorized.")
		return
	}

	// Get data from Request
	r.ParseForm()
	name := r.Form.Get("name")
	protocol := r.Form.Get("protocol")
	url := ps.ByName("url")
	method := r.Form.Get("checkMethod")

	// Simple Validation
	if name == "" || protocol == "" || url == "" || method == "" {
		SendJsonMessage(w, http.StatusBadRequest, false, "Unable to process your Request: Submit valid values.")
		return
	}
	if protocol != "http" && protocol != "https" {
		SendJsonMessage(w, http.StatusBadRequest, false, "Unable to process your Request: Submit a valid protocol.")
		return
	}
	if !govalidator.IsURL(protocol + "://" + url) {
		SendJsonMessage(w, http.StatusBadRequest, false, "Unable to process your Request: Submit a valid url.")
		return
	}
	if method != "HEAD" && method != "GET" {
		SendJsonMessage(w, http.StatusBadRequest, false, "Unable to process your Request: Submit a valid check method.")
		return
	}

	// Insert into Database
	db := lib.GetDatabase()
	_, err := db.Exec("INSERT INTO websites (name, protocol, url, checkMethod) VALUES (?, ?, ?, ?);", name, protocol, url, method)
	if err != nil {
		logging.MustGetLogger("").Error("Unable to add Website: ", err)
		SendJsonMessage(w, http.StatusInternalServerError, false, "Unable to process your Request: "+err.Error())
		return
	}

	SendJsonMessage(w, http.StatusOK, true, "")
}