func runQueryExtended(engine EngineI, query string, c *C, appendPoints bool, expectedSeries string) {

	var result []*protocol.Series
	err := engine.RunQuery(nil, "", query, func(series *protocol.Series) error {
		if appendPoints && result != nil {
			result[0].Points = append(result[0].Points, series.Points...)
		} else {
			result = append(result, series)
		}
		return nil
	})

	c.Assert(err, IsNil)

	series, err := common.StringToSeriesArray(expectedSeries)
	c.Assert(err, IsNil)

	if !reflect.DeepEqual(result, series) {
		resultData, _ := json.MarshalIndent(result, "", "  ")
		seriesData, _ := json.MarshalIndent(series, "", "  ")

		fmt.Fprintf(os.Stderr,
			"===============\nThe two series aren't equal.\nExpected: %s\nActual: %s\n===============\n",
			seriesData, resultData)
	}

	c.Assert(result, DeepEquals, series)
}

func GetOS(w http.ResponseWriter, r *http.Request) {
	var os_query libocit.OS
	os_query = GetOSQuery(r)

	ids := GetResourceList(os_query)

	var ret libocit.HttpRet
	if len(ids) < 1 {
		ret.Status = "Failed"
		ret.Message = "Cannot find the avaliable OS"
	} else {
		ret.Status = "OK"
		ret.Message = "Find the avaliable OS"
		var oss []libocit.OS
		for index := 0; index < len(ids); index++ {
			oss = append(oss, *(store[ids[index]]))
		}

		data, _ := json.MarshalIndent(oss, "", "\t")
		ret.Data = string(data)
	}

	body, _ := json.MarshalIndent(ret, "", "\t")
	w.Write([]byte(body))
}

// ServeHTTP shows the current plans in the query cache.
func (plr *Planner) ServeHTTP(response http.ResponseWriter, request *http.Request) {
	if err := acl.CheckAccessHTTP(request, acl.DEBUGGING); err != nil {
		acl.SendError(response, err)
		return
	}
	if request.URL.Path == "/debug/query_plans" {
		keys := plr.plans.Keys()
		response.Header().Set("Content-Type", "text/plain")
		response.Write([]byte(fmt.Sprintf("Length: %d\n", len(keys))))
		for _, v := range keys {
			response.Write([]byte(fmt.Sprintf("%#v\n", v)))
			if plan, ok := plr.plans.Peek(v); ok {
				if b, err := json.MarshalIndent(plan, "", "  "); err != nil {
					response.Write([]byte(err.Error()))
				} else {
					response.Write(b)
				}
				response.Write(([]byte)("\n\n"))
			}
		}
	} else if request.URL.Path == "/debug/vschema" {
		response.Header().Set("Content-Type", "application/json; charset=utf-8")
		b, err := json.MarshalIndent(plr.VSchema().Keyspaces, "", " ")
		if err != nil {
			response.Write([]byte(err.Error()))
			return
		}
		buf := bytes.NewBuffer(nil)
		json.HTMLEscape(buf, b)
		response.Write(buf.Bytes())
	} else {
		response.WriteHeader(http.StatusNotFound)
	}
}

// Saves either of the two config types to the specified file with the specified permissions.
func SaveConfig(fileout string, config interface{}, perms os.FileMode) error {

	//check to see if we got a struct or a map (minimal or extended config, respectively)
	v := reflect.ValueOf(config)
	if v.Kind() == reflect.Struct {
		config := config.(Config)

		//Parse the nicely formatted security section, and set the raw values for JSON marshalling
		newSecurity := make([]interface{}, 0)
		if config.Security.NoFiles {
			newSecurity = append(newSecurity, "nofiles")
		}
		setuser := make(map[string]interface{})
		setuser["setuser"] = config.Security.SetUser
		newSecurity = append(newSecurity, setuser)
		config.RawSecurity = newSecurity

		jsonout, err := json.MarshalIndent(config, "", "    ")
		if err != nil {
			return err
		}
		return ioutil.WriteFile(fileout, jsonout, perms)
	} else if v.Kind() == reflect.Map {
		jsonout, err := json.MarshalIndent(config, "", "    ")
		if err != nil {
			return err
		}
		return ioutil.WriteFile(fileout, jsonout, perms)
	}
	return fmt.Errorf("Something very bad happened")
}

func TestUnmarshal(t *testing.T) {
	tests := []struct {
		filename string
		typ      interface{}
		expected string
	}{
		// ASCII
		{"ascii.mof", MSFT_DSCConfigurationStatus{}, asciiMOFExpected},

		// UTF-16, little-endian
		{"utf16-le.mof", MSFT_DSCConfigurationStatus{}, utf16leMOFExpected},
	}
	for _, test := range tests {
		b, err := ioutil.ReadFile(test.filename)
		if err != nil {
			t.Fatal(err)
		}
		var v interface{}
		if err := Unmarshal(b, &v); err != nil {
			t.Fatal(err)
		}
		s := test.typ
		if err := Unmarshal(b, &s); err != nil {
			t.Fatal(err)
		}
		vb, _ := json.MarshalIndent(&v, "", "  ")
		sb, _ := json.MarshalIndent(&s, "", "  ")
		if string(vb) != test.expected {
			t.Errorf("%v: unexpected interface value", test.filename)
		}
		if string(sb) != test.expected {
			t.Errorf("%v: unexpected struct value", test.filename)
		}
	}
}

func equal(got, want string) error {
	var v1, v2 interface{}

	if err := json.Unmarshal([]byte(got), &v1); err != nil {
		return err
	}

	if err := json.Unmarshal([]byte(want), &v2); err != nil {
		return err
	}

	stripNondeterministicResources(v1)
	stripNondeterministicResources(v2)

	if !reflect.DeepEqual(v1, v2) {
		p1, err := json.MarshalIndent(v1, "", "\t")
		if err != nil {
			panic(err)
		}

		p2, err := json.MarshalIndent(v2, "", "\t")
		if err != nil {
			panic(err)
		}

		return fmt.Errorf("got:\n%s\nwant:\n%s\n", p1, p2)
	}

	return nil
}

func (o TxnManager) pubTxnCollectionIfNotExist(txnId int, collection string) {
	query := map[string]interface{}{
		"_id": txnId,
		"collections": map[string]interface{}{
			"$nin": []string{collection},
		},
	}
	update := map[string]interface{}{
		"$push": map[string]interface{}{
			"collections": collection,
		},
	}
	updateByte, err := json.MarshalIndent(&update, "", "\t")
	if err != nil {
		panic(err)
	}
	queryByte, err := json.MarshalIndent(&query, "", "\t")
	if err != nil {
		panic(err)
	}
	log.Println("pubTxnCollectionIfNotExist,update Transactions update:" + string(updateByte) + ", query:" + string(queryByte))
	if err := o.DB.C("Transactions").Update(query, update); err != nil {
		if err != mgo.ErrNotFound {
			panic(err)
		}
	}
}

func TestInterfaceSchema(t *testing.T) {
	var config struct {
		Field interface{}
	}

	schema := Of(config)
	value, _ := json.MarshalIndent(schema, "", "    ")
	t.Log(string(value))

	value, _ = json.MarshalIndent(schema, "\t\t", "    ")
	assert.Equal(t, string(value), `{
		    "schema": {
		        "type": "object",
		        "properties": {
		            "Field": {
		                "type": "string",
		                "required": true,
		                "title": "Field",
		                "propertyOrder": 1
		            }
		        },
		        "required": true,
		        "title": ""
		    }
		}`)
}

func ExampleMessage() {
	client := wit.NewClient(os.Getenv("WIT_ACCESS_TOKEN"))

	// Process a text message
	request := &wit.MessageRequest{}
	request.Query = "Hello world"
	result, err := client.Message(request)
	if err != nil {
		log.Println(err)
		os.Exit(-1)
	}
	log.Println(result)
	data, _ := json.MarshalIndent(result, "", "    ")
	log.Println(string(data[:]))

	// Process an audio/wav message
	request = &wit.MessageRequest{}
	request.File = "../audio_sample/helloWorld.wav"
	request.ContentType = "audio/wav;rate=8000"
	result, err = client.AudioMessage(request)
	if err != nil {
		log.Println(err)
		os.Exit(-1)
	}
	log.Println(result)
	data, _ = json.MarshalIndent(result, "", "    ")
	log.Println(string(data[:]))
}

func TestCmdInspectFormat(t *testing.T) {
	actual, host := runInspectCommand(t, []string{"test-a"})
	expected, _ := json.MarshalIndent(host, "", "    ")
	assert.Equal(t, string(expected), actual)

	actual, _ = runInspectCommand(t, []string{"--format", "{{.DriverName}}", "test-a"})
	assert.Equal(t, "none", actual)

	actual, _ = runInspectCommand(t, []string{"--format", "{{json .DriverName}}", "test-a"})
	assert.Equal(t, "\"none\"", actual)

	actual, _ = runInspectCommand(t, []string{"--format", "{{prettyjson .Driver}}", "test-a"})
	type ExpectedDriver struct {
		CaCertPath     string
		IPAddress      string
		MachineName    string
		PrivateKeyPath string
		SSHPort        int
		SSHUser        string
		SwarmDiscovery string
		SwarmHost      string
		SwarmMaster    bool
		URL            string
	}
	expected, err := json.MarshalIndent(&ExpectedDriver{MachineName: "test-a", URL: "unix:///var/run/docker.sock"}, "", "    ")
	assert.NoError(t, err)
	assert.Equal(t, string(expected), actual)
}

func processRefreshTokenFlow(authRequest mongodb.Document) (responseContent []byte) {

	var err error
	var user mongodb.Document
	var token mongodb.Document
	var accessToken mongodb.Document
	var refreshToken mongodb.Document

	token, err = getToken(authRequest[REFRESH_TOKEN].(string), authRequest[CLIENT_ID].(string), REFRESH_TOKEN_COLLECTION)

	if err == nil {
		//check if user exists
		user, err = getUser(mongodb.Document{ID: token[USER_ID]}) //get user from db
		if err == nil {
			//create tokens
			accessToken, err = generateToken(user, authRequest, ACCESS_TOKEN_COLLECTION)
			if err == nil {
				refreshToken, err = generateToken(user, authRequest, REFRESH_TOKEN_COLLECTION)
			}
		}
	}

	if err != nil {
		mwError := bson.M{"errorCode": "500", "errorMessage": err.Error()}
		responseContent, _ = json.MarshalIndent(mwError, "", "  ")
	} else {
		//success reponse with toekn info.
		tokenInfo := bson.M{ACCESS_TOKEN: accessToken[TOKEN], REFRESH_TOKEN: refreshToken[TOKEN]}
		responseContent, _ = json.MarshalIndent(tokenInfo, "", "  ")
	}

	return responseContent
}

func TestLex(t *testing.T) {
	for _, test := range lexTests {
		for i, _ := range test.ast.Classes {
			test.ast.Classes[i].Filename = "test.manifest"
			normalizeBlock(&test.ast.Classes[i].Block, "test.manifest")
		}
		for i, _ := range test.ast.Defines {
			test.ast.Defines[i].Filename = "test.manifest"
			normalizeBlock(&test.ast.Defines[i].Block, "test.manifest")
		}
		for i, _ := range test.ast.Nodes {
			test.ast.Nodes[i].Filename = "test.manifest"
			normalizeBlock(&test.ast.Nodes[i].Block, "test.manifest")
		}

		ast := NewAST()
		if err := Parse(ast, "test.manifest", strings.NewReader(test.manifest)); err != nil {
			t.Log(test.manifest)
			t.Error(err)
		} else {
			if !equalsAsJson(ast, test.ast) {
				t.Logf("%#v", test.ast)
				t.Logf("%#v", ast)
				js2, _ := json.MarshalIndent(test.ast, "", "  ")
				t.Log(string(js2))
				js, _ := json.MarshalIndent(ast, "", "  ")
				t.Log(string(js))
				t.Error("Expected manifest", test.manifest)
				if ast != nil {
					t.Log("Read manifest", ast.String())
				}
			}
		}
	}
}

func (s *Site) logFiles() {
	log := make([]Event, len(s.L))
	i := 0
	for k := range s.L {
		log[i] = k
		i++
	}
	slog, err := json.MarshalIndent(log, "", "\t")
	if err != nil {
		fmt.Printf("%+v\n", err)
	} else {
		ioutil.WriteFile("log.json", slog, 0666)
	}

	cal := make([]Appointment, len(s.V))
	i = 0
	for _, v := range s.V {
		cal[i] = v
		i++
	}
	scal, err := json.MarshalIndent(cal, "", "\t")
	if err != nil {
		fmt.Printf("%+v\n", err)
	} else {
		ioutil.WriteFile("cal.json", scal, 0666)
	}
}

// ToJSON returns a JSON representation of the object.
func (n *ActionNode) ToJSON() (string, error) {
	data, err := json.MarshalIndent(n, "", "  ")
	if err != nil {
		return "", fmt.Errorf("cannot JSON-marshal node: %v", err)
	}
	result := string(data) + "\n"
	if n.Args == nil {
		result += "{}\n"
	} else {
		data, err := json.MarshalIndent(n.Args, "", "  ")
		if err != nil {
			return "", fmt.Errorf("cannot JSON-marshal node args: %v", err)
		}
		result += string(data) + "\n"
	}
	if n.Reply == nil {
		result += "{}\n"
	} else {
		data, err := json.MarshalIndent(n.Reply, "", "  ")
		if err != nil {
			return "", fmt.Errorf("cannot JSON-marshal node reply: %v", err)
		}
		result += string(data) + "\n"
	}
	return result, nil
}

func testFlatten(inputFile, outputFile, contextFile, base string,
	compactArrays bool, t *testing.T) {
	inputJson, jsonErr := ReadJSONFromFile(test_dir + inputFile)
	if !isNil(jsonErr) {
		t.Error("Could not open input file")
		return
	}
	outputJson, jsonErr := ReadJSONFromFile(test_dir + outputFile)
	if !isNil(jsonErr) {
		t.Error("Could not open output file")
		return
	}
	contextJson, jsonErr := ReadJSONFromFile(test_dir + contextFile)
	if !isNil(jsonErr) {
		contextJson = nil
	}
	options := &Options{
		Base:           base,
		CompactArrays:  compactArrays,
		ExpandContext:  nil,
		DocumentLoader: NewDocumentLoader(),
	}
	flattenedJson, flattenErr := Flatten(inputJson, contextJson, options)
	if !isNil(flattenErr) {
		t.Error("Compaction failed with error ", flattenErr.Error())
		return
	}
	flattenedString, _ := json.MarshalIndent(flattenedJson, "", "    ")
	outputString, _ := json.MarshalIndent(outputJson, "", "    ")
	if !reflect.DeepEqual(flattenedJson, outputJson) {
		t.Error("Expected:\n", string(outputString), "\nGot:\n",
			string(flattenedString))
	}
}

// PrettyString returns RockerImageData as a printable string
func (data *RockerImageData) PrettyString() string {
	prettyVars, err := json.MarshalIndent(data.Vars, "", "  ")
	if err != nil {
		log.Fatal(err)
	}
	prettyProps, err := json.MarshalIndent(data.Properties, "", "  ")
	if err != nil {
		log.Fatal(err)
	}
	green := color.New(color.FgGreen).SprintfFunc()
	yellow := color.New(color.FgYellow).SprintfFunc()
	sep := "=======================================================\n"

	res := fmt.Sprintf("%s%s\n", green(sep),
		green("Image: %s", data.ImageName.String()))

	if !data.Created.IsZero() {
		res = fmt.Sprintf("%sCreated: %s\n", res, data.Created.Format(time.RFC850))
	}

	if data.Properties != nil {
		res = fmt.Sprintf("%sProperties: %s\n", res, prettyProps)
	}

	if data.Vars != nil {
		res = fmt.Sprintf("%sVars: %s\n", res, prettyVars)
	}

	if data.Rockerfile != "" {
		res = fmt.Sprintf("%s%s\n%s\n%s\n%s", res, yellow("Rockerfile:"), yellow(sep), data.Rockerfile, yellow(sep))
	}

	return res
}

func UUIDRender(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)

	// call mongo and lookup the redirection to use...
	session, err := services.GetMongoCon()
	if err != nil {
		panic(err)
	}
	defer session.Close()

	// Optional. Switch the session to a monotonic behavior.
	session.SetMode(mgo.Monotonic, true)
	c := session.DB("test").C("schemaorg")
	c2 := session.DB("test").C("csvwmeta")

	// Get the schema.org data
	URI := fmt.Sprintf("http://opencoredata.org/id/dataset/%s", vars["UUID"])
	result := SchemaOrgMetadata{}
	err = c.Find(bson.M{"url": URI}).One(&result)
	if err != nil {
		log.Printf("URL lookup error: %v", err)
	}

	// context setting hack
	// result.Context = ` "opencore": "http://opencoredata.org/voc/1/", "glview": "http://geolink.org/view/1/", "schema": "http://schema.org/"`
	result.Context = "http://schema.org"

	jsonldtext, _ := json.MarshalIndent(result, "", " ") // results as embeddale JSON-LD

	// Get the CSVW  data
	result2 := CSVWMeta{}
	err = c2.Find(bson.M{"url": URI}).One(&result2)
	if err != nil {
		log.Printf("URL lookup error: %v", err)
	}

	// result.Context = ` "opencore": "http://opencoredata.org/voc/1/", "glview": "http://geolink.org/view/1/", "schema": "http://schema.org/"`
	// needs to be:     "@context": ["http://www.w3.org/ns/csvw", {"@language": "en"}],
	result2.Context = "http://www.w3.org/ns/csvw"

	csvwtext, _ := json.MarshalIndent(result2, "", " ") // results as embeddale JSON-LD

	ht, err := template.New("some template").ParseFiles("templates/jrso_dataset_new.html") //open and parse a template text file
	if err != nil {
		log.Printf("template parse failed: %s", err)
	}

	// need a simple function call to extract the "janus" keyword from the keyword string and toLower it and
	// pass it in this struct to use in the data types web component
	measureString := getJanusKeyword(result.Keywords)
	dataForTemplate := TemplateForDoc{Schema: result, CSVW: result2, Schemastring: template.JS(string(jsonldtext)), Csvwstring: template.JS(string(csvwtext)), MeasureType: measureString, UUID: vars["UUID"]}

	err = ht.ExecuteTemplate(w, "T", dataForTemplate) //substitute fields in the template 't', with values from 'user' and write it out to 'w' which implements io.Writer
	if err != nil {
		log.Printf("htemplate execution failed: %s", err)
	}

	// go get the CSVW metadata and inject the whole package and the rendered table

}

func SaveShopEvent(action ActionShop, info *Info, err error, description string) {
	debug.Log("Action", string(action))
	event := NewEvent()
	if info != nil {
		event.Info = info
	}

	event.Info.Caller = trace.WhoCalledMe(4)
	if err != nil {
		event.Type = EventTypeError
		event.Error = err.Error()
	} else {
		event.Type = EventTypeSuccess
	}

	event.Action = string(action)
	event.Info.Description = description

	if !saveShopEventDB(event) {
		jsonBytes, err := json.MarshalIndent(event, "", "	")
		if err != nil {
			log.Println("Could not jsonMarshal event")
		}
		log.Println("Saving Shop Event failed! ", string(jsonBytes))
	}
	jsonBytes, _ := json.MarshalIndent(event, "", "	")
	debug.Log("Saved Shop Event! ", string(jsonBytes))
}

func Equal(got, want string, fn func(string) string) error {
	var v1, v2 interface{}

	if err := json.Unmarshal([]byte(got), &v1); err != nil {
		return fmt.Errorf(`failed to parse "got" JSON: %s`, err)
	}

	if err := json.Unmarshal([]byte(want), &v2); err != nil {
		return fmt.Errorf(`failed to parse "want" JSON: %s`, err)
	}

	if err := mask(v1, fn); err != nil {
		return err
	}

	if err := mask(v2, fn); err != nil {
		return err
	}

	if !reflect.DeepEqual(v1, v2) {
		p1, err := json.MarshalIndent(v1, "", "\t")
		if err != nil {
			panic(err)
		}

		p2, err := json.MarshalIndent(v2, "", "\t")
		if err != nil {
			panic(err)
		}

		return fmt.Errorf("got:\n%s\nwant:\n%s\n", p1, p2)
	}

	return nil
}

func main() {
	client := usergrid.Client{Organization: ORGNAME, Application: APPNAME, Uri: API}
	var objmap interface{}
	var results = make([]interface{}, 0)
	messages := make(chan interface{})
	done := make(chan bool, 1)
	if len(CLIENT_ID) > 0 && len(CLIENT_SECRET) > 0 {
		err := client.OrgLogin(CLIENT_ID, CLIENT_SECRET)
		if err != nil {
			log.Printf(err.Error())
			return
		}
	}
	go func() {
		for {
			select {
			case v := <-messages:
				results = append(results, v)
			case <-time.After(time.Second * 10):
				fmt.Println("Timeout!")
			}
		}
	}()
	client.Get("", nil, usergrid.JSONResponseHandler(&objmap))
	str, _ := json.MarshalIndent(objmap, "", "  ")
	log.Printf("RESPONSE: %s", str)
	go FetchAll(client, ENDPOINT, messages, done, "")
	<-done
	entities, _ := json.MarshalIndent(results, "", "  ")
	fmt.Printf("%s\n", entities)
	log.Printf("Done. %d requests and %d responses", REQUESTS, RESPONSES)
	return
}