func v1GetTags(repoSlice []RepoType) (tagSlice []TagInfo, e error) {
	var client *http.Client
	if *RegistryTLSNoVerify {
		tr := &http.Transport{
			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
		}
		client = &http.Client{Transport: tr}
	} else {
		client = &http.Client{}
	}
	for _, repo := range repoSlice {
		// get tags for one repo
		var response []byte
		response, e = RegistryQuery(client, RegistryAPIURL+"/v1/repositories/"+string(repo)+"/tags")
		if e != nil {
			except.Error(e)
			if s, ok := e.(*HTTPStatusCodeError); ok {
				except.Error("Skipping Repo: %s, tag lookup status code %d", string(repo), s.StatusCode)
				continue
			}
			return
		}
		//parse JSON output
		var m map[TagType]ImageIDType
		if e = json.Unmarshal(response, &m); e != nil {
			return nil, e
		}
		var t TagInfo
		t.Repo = repo
		t.TagMap = m
		tagSlice = append(tagSlice, t)
	}
	return
}

// ValidRepoName verifies that the name of a repo is in a legal format.
// A valid name can optionally include a wildcard "*" but only as the last character.
func ValidRepoName(name string) bool {
	if len(name) == 0 {
		return false
	}
	if len(name) > 256 {
		except.Error("Invalid repo name, too long: %s", name)
		return false
	}
	for i, c := range name {
		switch {
		case c >= 'a' && c <= 'z':
			continue
		case c >= 'A' && c <= 'Z':
			continue
		case c >= '0' && c <= '9':
			continue
		case c == '/' || c == '_' || c == '-' || c == '.':
			continue
		case c == '*' && i == len(name)-1:
			continue
		default:
			except.Error("Invalid repo name %s", name)
			return false
		}
	}
	return true
}

func runAllScripts(imageID ImageIDType) (outMap map[string]interface{}, err error) {
	//script name -> either byte array, or known types (e.g., ImageDataInfo)
	outMap = make(map[string]interface{})
	scripts := getScriptsToRun()
	for _, script := range scripts {
		//run script
		output, err := script.Run(imageID)
		if err != nil {
			except.Error(err, ": Error in running script: ", script.Name())
			continue //continue trying to run other scripts
		}

		//analyze script output
		switch script.Name() {
		case PKGEXTRACTSCRIPT:
			imageDataInfo, err := parsePkgExtractOutput(output, imageID)
			if err != nil {
				except.Error(err, ": Error in parsing PkgExtractOuput")
				return nil, err
			}
			outMap[script.Name()] = imageDataInfo
		default:
			//script name -> byte array
			outMap[script.Name()] = output
		}
	}

	return
}

func dockerImageID(regspec string, metadata *ImageMetadataInfo) (ID string, err error) {
	matchRepo := string(metadata.Repo)
	if regspec != config.DockerHub {
		matchRepo = regspec + "/" + matchRepo
	}
	matchTag := string(metadata.Tag)
	if strings.HasPrefix(matchRepo, "library/") {
		matchRepo = strings.Replace(matchRepo, "library/", "", 1)
	}
	// verify the image ID of the pulled image matches the expected metadata.
	imageMap, err := GetLocalImages(false, false)
	if err != nil {
		except.Error(err, ":unable to list local images")
		return
	}
	for imageID, repotagSlice := range imageMap {
		for _, repotag := range repotagSlice {
			if string(repotag.Repo) == matchRepo && string(repotag.Tag) == matchTag {
				ID = string(imageID)
				return
			}
		}
	}
	err = errors.New("Failed to find local image ID for " + metadata.Repo + ":" + metadata.Tag)
	except.Error(err)
	return
}

// RegistryRequestWithToken queries a Docker Registry that uses v1 Token Auth, e.g., Docker Hub.
func RegistryRequestWithToken(client *http.Client, URL string, dockerToken string) (response []byte, e error) {
	var req *http.Request
	req, e = http.NewRequest("GET", URL, nil)
	if e != nil {
		except.Error(e)
		return
	}
	req.Header.Set("Authorization", "Token "+dockerToken)
	var r *http.Response
	r, e = client.Do(req)
	if e != nil {
		except.Error(e)
		return
	}
	defer r.Body.Close()
	if r.StatusCode != 200 {
		e = &HTTPStatusCodeError{StatusCode: r.StatusCode}
		return
	}
	response, e = ioutil.ReadAll(r.Body)
	if e != nil {
		except.Error(e)
		return
	}
	return
}

func v2GetMetadata(client *http.Client, repo, tag string) (metadata ImageMetadataInfo, e error) {
	response, err := RegistryQueryV2(client, RegistryAPIURL+"/v2/"+repo+"/manifests/"+tag)
	if err != nil {
		except.Error(err)
		if s, ok := err.(*HTTPStatusCodeError); ok {
			except.Error("Skipping Repo: %s, tag lookup status code %d", string(repo), s.StatusCode)
			e = err
		}
		return
	}

	metadata.Repo = repo
	metadata.Tag = tag
	metadata.Image = ""
	// Recent versions of Docker daemon (1.8.3+?) have complex code that calculates
	// the image ID from the V2 manifest.
	// This seems to be in flux as Docker moves toward content-addressable images in 1.10+,
	// and as the registry image manifest schema itself is still evolving.
	// As a temporary measure until Docker converges to a more stable state, collector
	// will calculate its own hash over the V2 manifest and use the calculated
	// value to try to filter out images that have previously been processed.
	// The Docker-calculated image ID will get added to the metadata struct
	// after the image is pulled.
	sum := sha256.Sum256(response)
	metadata.ManifestHash = hex.EncodeToString(sum[:])
	var m ManifestV2Schema1
	b := bytes.NewBuffer(response)
	if e = json.NewDecoder(b).Decode(&m); e != nil {
		blog.Warn("Failed to parse manifest")
		return
	}
	if m.SchemaVersion != 1 {
		blog.Warn("Manifest schema version %d is not yet supported\n", m.SchemaVersion)
		return
	}
	if len(m.History) == 0 {
		e = errors.New("repo " + repo + ":" + tag + " no images found in history")
		return
	}
	var image ImageStruct
	if e = json.Unmarshal([]byte(m.History[0].V1Compatibility), &image); e != nil {
		blog.Warn("Failed to parse ImageStruct")
		return
	}
	var creationTime time.Time
	if creationTime, e = time.Parse(time.RFC3339Nano, image.Created); e != nil {
		blog.Warn("Failed to parse creation time")
		return
	}
	metadata.Datetime = creationTime
	metadata.Size = image.Size
	metadata.Author = image.Author
	metadata.Checksum = image.Checksum
	metadata.Comment = image.Comment
	metadata.Parent = image.Parent
	return
}

// DockerAPI performs an HTTP GET,POST,DELETE operation to the Docker daemon.
func DockerAPI(tr *http.Transport, operation, apipath string, jsonString []byte,
	XRegistryAuth string) (resp []byte, e error) {
	switch operation {
	case "GET", "POST", "DELETE":
		break
	default:
		e = errors.New("Operation " + operation + " not supported")
		return
	}
	// for unix socket, URL (host.domain) is needed but can be anything
	var host string
	HTTP := "http://"
	if DockerProto == "unix" {
		host = dummydomain
	} else {
		host = DockerAddr
		if DockerTLSVerify {
			HTTP = "https://"
		}
	}
	URL := HTTP + host + apipath
	blog.Info("DockerAPI %s", URL)
	req, e := http.NewRequest(operation, URL, bytes.NewBuffer(jsonString))
	if e != nil {
		except.Error(e, ":DockerAPI failed to create http request")
		return
	}
	req.Header.Add("Content-Type", "application/json")
	if XRegistryAuth != "" {
		req.Header.Add("X-Registry-Auth", XRegistryAuth)
	}

	//req.Header.Set("Authorization", "Bearer "+authToken)
	client := &http.Client{Transport: tr, Timeout: DockerTimeout}
	r, e := client.Do(req)
	if e != nil {
		except.Error(e, ":DockerAPI URL", URL, "client request failed")
		return
	}
	defer r.Body.Close()
	resp, e = ioutil.ReadAll(r.Body)
	if e != nil {
		except.Error(e, ":DockerAPI URL", URL, "invalid response body")
		return
	}
	if r.StatusCode < 200 || r.StatusCode > 299 {
		e = errors.New("DockerAPI URL: " + URL + " status code: " + strconv.Itoa(r.StatusCode) +
			"error: " + string(resp))
		return
	}
	return
}

// RemoveImages removes least recently pulled docker images from the local docker host.
func RemoveImages(PulledImages []ImageMetadataInfo) {
	numRemoved := 0
	imageMap, err := GetLocalImages(false, false)
	if err != nil {
		except.Error(err, ": RemoveImages unable to list local images")
	}
	for _, metadata := range PulledImages {
		if strings.HasPrefix(metadata.Repo, "library/") {
			metadata.Repo = strings.Replace(metadata.Repo, "library/", "", 1)
		}
		imageID := ImageIDType(metadata.Image)
		if metadata.Image == "" {
			// unknown image ID. Search the repotags for a match
			var err error
			imageID, err = imageMap.Image(RepoType(metadata.Repo), TagType(metadata.Tag))
			if err != nil {
				except.Error(err, ": RemoveImages unable to find image ID")
				break
			}
		}

		// Get all repo:tags associated with the image
		repoTagSlice := imageMap.RepoTags(imageID)
		if len(repoTagSlice) == 0 {
			except.Error("RemoveImages unable to find expected repo:tag " + metadata.Repo +
				":" + metadata.Tag + " for image ID=" + string(imageID))
			except.Error("imageMap is %v", imageMap)
			continue
		}
		for _, repotag := range repoTagSlice {
			// basespec := RegistrySpec + "/" + string(t.Repo) + ":"
			if ExcludeRepo[RepoType(repotag.Repo)] {
				continue
			}
			apipath := "/images/" + string(repotag.Repo) + ":" + string(repotag.Tag)
			blog.Info("RemoveImages %s", apipath)
			config.BanyanUpdate("Remove", apipath)
			_, err := DockerAPI(DockerTransport, "DELETE", apipath, []byte{}, "")
			if err != nil {
				except.Error(err, "RemoveImages Repo:Tag", repotag.Repo, repotag.Tag,
					"image", metadata.Image)
			}
			numRemoved++
		}
	}

	blog.Info("Number of repo/tags removed this time around: %d", numRemoved)

	RemoveDanglingImages()
	return
}

func copy(src, dest string) {
	// Read all content of src to data
	data, err := ioutil.ReadFile(src)
	if err != nil {
		except.Error(err, ": Error in reading from file: ", src)
		return
	}
	// Write data to dest
	err = ioutil.WriteFile(dest, data, 0755)
	if err != nil {
		except.Error(err, ": Error in writing to file: ", dest)
		return
	}
}

func jsonifyAndWriteToFile(filenamePath string, data interface{}) (err error) {
	b, err := json.MarshalIndent(data, "", "\t")
	if err != nil {
		except.Error(err, ": Error in marshaling json")
		return err
	}

	err = ioutil.WriteFile(filenamePath, b, 0644)
	if err != nil {
		except.Error(err, ": Error in writing to file: ", filenamePath)
		return err
	}

	return nil
}

// LogsContainer makes a docker remote API call to get logs from a container.
func LogsContainer(containerID string) (output []byte, err error) {
	apipath := "/containers/" + containerID + "/logs?stdout=1"
	resp, err := DockerAPI(DockerTransport, "GET", apipath, []byte{}, "")
	if err != nil {
		except.Error(err, ": Error in Remote Docker API call: ", apipath)
		return
	}
	blog.Debug("Response from docker remote API call for logs: " + string(resp))
	for {
		if len(resp) < 8 {
			break
		}
		header := resp[0:8]
		var size int32
		buf := bytes.NewBuffer(header[4:8])
		binary.Read(buf, binary.BigEndian, &size)
		payload := resp[8:(8 + size)]
		// blog.Info(string(frame))
		resp = resp[(8 + size):]
		if header[0] == uint8(1) {
			// 1=stdout: return only the stdout log
			output = append(output, payload...)
		}
	}
	return
}

// getRepos queries the Docker registry for the list of the repositories it is currently hosting.
// However, if the user specified a list of repositories, then getRepos() just returns that list
// of specified repositories and does not query the Docker registry.
func getRepos() (repoSlice []RepoType, err error) {
	if len(ReposToProcess) > 0 {
		for repo := range ReposToProcess {
			repoSlice = append(repoSlice, repo)
		}
		return
	}

	if *RegistryProto == "v2" {
		except.Error("v2 registry search/catalog interface not yet supported in collector")
		return
	}

	// a query with an empty query string returns all the repos
	var client *http.Client
	if *RegistryTLSNoVerify {
		tr := &http.Transport{
			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
		}
		client = &http.Client{Transport: tr}
	} else {
		client = &http.Client{}
	}
	return registrySearchV1(client, "")
}

// getReposTokenAuthV1 validates the user-specified list of repositories against an index server, e.g., Docker Hub.
// It returns a list of IndexInfo structs with index info for each validated repository.
func getReposTokenAuthV1(repo RepoType, client *http.Client) (indexInfo IndexInfo, e error) {
	_, _, BasicAuth, XRegistryAuth = GetRegistryURL()
	URL := RegistryAPIURL + "/v1/repositories/" + string(repo) + "/images"
	req, e := http.NewRequest("GET", URL, nil)
	req.Header.Set("X-Docker-Token", "true")
	if BasicAuth != "" {
		req.Header.Set("Authorization", "Basic "+BasicAuth)
	}
	r, e := client.Do(req)
	if e != nil {
		except.Error(e, ":getReposTokenAuthV1 HTTP request failed")
		return
	}
	defer r.Body.Close()
	if r.StatusCode != 200 {
		e = &HTTPStatusCodeError{StatusCode: r.StatusCode}
		return
	}
	dockerToken := r.Header.Get("X-Docker-Token")
	if dockerToken == "" {
		e = errors.New("lookup error for repo " + string(repo))
		return
	}
	registryURL := r.Header.Get("X-Docker-Endpoints")
	arr := strings.Split(registryURL, ",")
	if len(arr) == 0 {
		registryURL = ""
		e = errors.New("lookup error for repo " + string(repo))
		return
	}
	registryURL = strings.TrimSpace(arr[0])
	indexInfo = IndexInfo{Repo: repo, DockerToken: dockerToken, RegistryURL: registryURL}
	return
}

// WriteImageAllData writes image (pkg and other) data into file
func (f *FileWriter) WriteImageAllData(outMapMap map[string]map[string]interface{}) {
	blog.Info("Writing image (pkg and other) data into file...")

	for imageID, scriptMap := range outMapMap {
		for scriptName, out := range scriptMap {
			scriptDir := f.dir + "/" + trimExtension(scriptName)
			err := fsutil.CreateDirIfNotExist(scriptDir)
			if err != nil {
				except.Error(err, ": Error creating script dir: ", scriptDir)
				continue
			}
			image := string(imageID)
			if len(image) < 12 {
				except.Warn("Weird...Haven't seen imageIDs so small -- possibly a test?")
			} else {
				image = string(imageID)[0:12]
			}
			filenamePath := scriptDir + "/" + image
			if _, ok := out.([]byte); ok {
				f.format = "txt"
				filenamePath += "-miscdata"
			} else {
				// by default it is json. But f.format could get overwritten at any point
				// in the for loop if the output type is []byte, hence the (re)assignment
				f.format = "json"
				// NOTE: If we start using json for output other than imageData, change this
				filenamePath += "-pkgdata"
			}
			f.writeFileInFormat(filenamePath, &out)
		}
	}
	return
}

func CreateDirIfNotExist(dir string) (err error) {
	exists, err := DirExists(dir)
	if err != nil {
		except.Error(err, ": Error while querying dir: ", dir)
		return err
	}
	if !exists {
		err = os.MkdirAll(dir, 0755)
		if err != nil {
			except.Error(err, ": Error in creating dir: ", dir)
			return err
		}
	}

	return nil
}

// lookupMetadataTokenAuthV1 takes as input the imageID, and Docker Hub auth/index info,
// and it returns ImageMetadataInfo for that image by querying the indexed registry.
func lookupMetadataTokenAuthV1(imageID ImageIDType, client *http.Client, indexInfo IndexInfo) (
	metadata ImageMetadataInfo, e error) {

	blog.Info("Get Metadata for Image: %s", string(imageID))
	URL := "https://" + indexInfo.RegistryURL + "/v1/images/" + string(imageID) + "/json"
	response, e := RegistryRequestWithToken(client, URL, indexInfo.DockerToken)
	if e != nil {
		except.Error(e, "Unable to query metadata for image: "+string(imageID))
		return
	}
	// log.Print("metadata query response: " + string(response))
	var m ImageStruct
	if e = json.Unmarshal(response, &m); e != nil {
		return
	}
	var creationTime time.Time
	metadata.Image = string(imageID)
	if creationTime, e = time.Parse(time.RFC3339Nano, m.Created); e != nil {
		return
	}
	metadata.Datetime = creationTime
	metadata.Size = m.Size
	metadata.Author = m.Author
	metadata.Checksum = m.Checksum
	metadata.Comment = m.Comment
	metadata.Parent = m.Parent
	return
}

func getTagsTokenAuthV1(repo RepoType, client *http.Client, indexInfo IndexInfo) (tagSlice []TagInfo, e error) {
	tagSlice, e = lookupTagsTokenAuthV1(client, indexInfo)
	if e != nil {
		except.Error(e, ": Error in looking up tags in dockerhub")
	}
	return
}

// Run handles running of a script inside an image
func (sh ScriptInfo) Run(imageID ImageIDType) (b []byte, err error) {
	jsonString, err := createCmd(imageID, sh.name, sh.staticBinary, sh.dirPath)
	if err != nil {
		except.Error(err, ": Error in creating command")
		return
	}
	blog.Debug("Container spec: %s", string(jsonString))
	containerID, err := CreateContainer(jsonString)
	if err != nil {
		except.Error(err, ": Error in creating container")
		return
	}
	blog.Debug("New container ID: %s", containerID)

	defer RemoveContainer(containerID)

	jsonString, err = StartContainer(containerID)
	if err != nil {
		except.Error(err, ": Error in starting container")
		return
	}
	blog.Debug("Response from StartContainer: %s", string(jsonString))
	statusCode, err := WaitContainer(containerID)
	if err != nil {
		except.Error(err, ": Error in waiting for container to stop")
		return
	}
	if statusCode != 0 {
		err = errors.New("Bash script exit status: " + strconv.Itoa(statusCode))
		return
	}
	b, err = LogsContainer(containerID)
	if err != nil {
		except.Error(err, ":Error in extracting output from container")
		return
	}
	/*
		_, err = removeContainer(containerID)
		if err != nil {
			except.Error(err, ":Error in removing container for image", containerID)
			return
		}
	*/
	return
}

// RemoveImageByID calls Docker to remove an image specified by ID.
func RemoveImageByID(image ImageIDType) (resp []byte, err error) {
	apipath := "/images/" + string(image)
	resp, err = DockerAPI(DockerTransport, "DELETE", apipath, []byte{}, "")
	if err != nil {
		except.Error(err, "RemoveImageByID")
		return
	}
	return
}

func getMetadataTokenAuthV1(repotag TagInfo, metadataMap ImageToMetadataMap, client *http.Client,
	indexInfo IndexInfo) (metadataSlice []ImageMetadataInfo, e error) {

	// for each tag, generate the current Image Metadata Info
	repo := repotag.Repo
	tagmap := repotag.TagMap
	for tag, imageID := range tagmap {
		if metadataMap.Exists(imageID) {
			continue
		}

		var metadata ImageMetadataInfo
		metadata, e = lookupMetadataTokenAuthV1(imageID, client, indexInfo)
		if e != nil {
			if s, ok := e.(*HTTPStatusCodeError); ok {
				except.Error("Registry returned HTTP status code %d, skipping %s:%s image %s",
					s.StatusCode, string(repo), string(tag), string(imageID))
				continue
			}
			// some other error (network broken?), so give up
			except.Error(e, "Unable to lookup metadata for",
				repo, ":", tag, string(imageID))
			return
		}
		metadata.Repo = string(repo)
		metadata.Tag = string(tag)
		metadataMap.Insert(ImageIDType(metadata.Image), metadata)
	}

	for tag, imageID := range tagmap {
		var curr ImageMetadataInfo
		if metadataMap.Exists(imageID) {
			// copy previous entry and fill in this repo/tag
			curr, _ = metadataMap.Metadata(imageID)
			curr.Repo = string(repo)
			curr.Tag = string(tag)
			metadataSlice = append(metadataSlice, curr)
		} else {
			e = errors.New("Missing metadata for image ID " + string(imageID))
			return
		}
	}
	return
}