func ProcessFile(file string) ([]byte, error) {

	ext := filepath.Ext(file)
	isYaml := ext == ".yaml" || ext == ".yml"
	isJson := ext == ".json"

	if !isYaml && !isJson {
		return nil, nil
	}

	bytes, err := ioutil.ReadFile(file)
	if err != nil {
		return nil, kerr.Wrap("NMWROTKPLJ", err)
	}

	if isYaml {
		j, err := yaml.YAMLToJSON(bytes)
		if err != nil {
			return nil, kerr.Wrap("FAFJCYESRH", err)
		}
		bytes = j
	}

	return bytes, nil
}

// ScanFiles takes a chanel of files
func ScanFilesToBytes(ctx context.Context, in chan File) chan Content {

	out := make(chan Content)

	go func() {

		defer close(out)

		for {
			select {
			case value, open := <-in:
				if !open {
					return
				}
				if value.Err != nil {
					out <- Content{"", nil, kerr.Wrap("PQUCOUYLJE", value.Err)}
					return
				}
				bytes, err := ProcessFile(value.File)
				// process returns Bytes == nil for non json files, so we should skip them
				if bytes != nil || err != nil {
					out <- Content{value.File, bytes, err}
				}
			case <-ctx.Done():
				out <- Content{"", nil, kerr.Wrap("AFBJCTFOKX", ctx.Err())}
				return
			}
		}

	}()

	return out

}

// Generate generates the source code for type structs, and writes the
// generated.go to the filesystem.
func Generate(ctx context.Context, env *envctx.Env) error {

	wgctx.Add(ctx, "Generate")
	defer wgctx.Done(ctx, "Generate")

	cmd := cmdctx.FromContext(ctx)

	cmd.Printf("Generating types for %s... ", env.Path)

	outputDir := env.Dir
	filename := "generated.go"
	source, err := generate.Structs(ctx, env)
	if err != nil {
		return kerr.Wrap("XFNESBLBTQ", err)
	}

	// We only backup in the system structs and types files because they are
	// the only generated files we ever need to roll back
	backup := env.Path == "kego.io/system"

	if err = save(outputDir, source, filename, backup); err != nil {
		return kerr.Wrap("UONJTTSTWW", err)
	} else {
		cmd.Println("OK.")
	}

	return nil
}

func walkFile(path string) error {

	b, err := ioutil.ReadFile(path)
	if err != nil {
		return kerr.Wrap("FGAHVRNUPV", err)
	}

	fset := token.NewFileSet()
	file, err := parser.ParseFile(fset, path, nil, parser.ParseComments)
	if err != nil {
		return kerr.Wrap("ALRIEYJBJE", err)
	}

	// visitor implements ast.Visitor
	v := &visitor{b, false}
	ast.Walk(v, file)

	// if we made a replacement, re-write the modified file
	if v.Found {
		fmt.Println(path)
		f, err := os.Create(path)
		if err != nil {
			return kerr.Wrap("QFPDQRTIRS", err)
		}
		defer f.Close()
		if err := printer.Fprint(f, fset, file); err != nil {
			return kerr.Wrap("VEPVDWFWEF", err)
		}
	}

	return nil
}

func mutateRestoreNode(ctx context.Context, n *node.Node, p *node.Node, b *node.Node) error {
	n.Restore(ctx, b)
	if p == nil {
		return nil
	}
	switch p.Type.NativeJsonType(ctx) {
	case system.J_MAP:
		// don't have to call n.InitialiseMapItem because the node is already
		// initialized
		if err := n.AddToMap(ctx, p, n.Key, true); err != nil {
			return kerr.Wrap("TOPLOONYCL", err)
		}
	case system.J_ARRAY:
		// don't have to call n.InitialiseArrayItem because the node is already
		// initialized
		if err := n.AddToArray(ctx, p, n.Index, true); err != nil {
			return kerr.Wrap("WFXSQYOEAY", err)
		}
	case system.J_OBJECT:
		// don't have to call n.InitialiseObjectField because the node is
		// already initialized
		if err := n.AddToObject(ctx, p, n.Rule, n.Key, true); err != nil {
			return kerr.Wrap("QMBJQMLOCY", err)
		}
	}
	return nil
}

func getInfo(ctx context.Context, dir string) (info *generate.InfoStruct, found bool, err error) {
	f, err := os.Open(filepath.Join(dir, "generated.go"))
	if err != nil {
		if os.IsNotExist(err) {
			return nil, false, nil
		}
		return nil, false, kerr.Wrap("TLFTCRNBKK", err)
	}
	defer f.Close()

	scanner := bufio.NewScanner(f)
	scanner.Scan()
	if err := scanner.Err(); err != nil {
		return nil, false, kerr.Wrap("HLDYEAPLEQ", err)
	}

	if !strings.HasPrefix(scanner.Text(), "// info:{") {
		return nil, false, nil
	}

	data := []byte(scanner.Text()[8:])

	var i generate.InfoStruct
	if err := json.Unmarshal(data, &i); err != nil {
		return nil, false, kerr.Wrap("UJXKJVLXHG", err)
	}
	return &i, true, nil
}

func extractFields(ctx context.Context, fields map[string]*system.Field, t *system.Type) error {
	for t.Alias != nil {
		t = system.WrapRule(ctx, t.Alias).Parent
	}
	if !t.Basic && !t.Interface {
		// All types apart from Basic types embed system:object
		ob, ok := system.GetTypeFromCache(ctx, "kego.io/system", "object")
		if !ok {
			return kerr.New("YRFWOTIGFT", "Type system:object not found in sys ctx")
		}
		if err := extractFields(ctx, fields, ob); err != nil {
			return kerr.Wrap("DTQEFALIMM", err)
		}
	}
	for _, embedRef := range t.Embed {
		embed, ok := system.GetTypeFromCache(ctx, embedRef.Package, embedRef.Name)
		if !ok {
			return kerr.New("SLIRILCARQ", "Type %s not found in sys ctx", embedRef)
		}
		if err := extractFields(ctx, fields, embed); err != nil {
			return kerr.Wrap("JWAPCVIYBJ", err)
		}
	}
	for name, rule := range t.Fields {
		if _, ok := fields[name]; ok {
			return kerr.New("BARXPFXQNB", "Duplicate field %s", name)
		}
		fields[name] = &system.Field{Name: name, Rule: rule, Origin: t.Id}
	}
	return nil
}

func (n *Node) SetValue(ctx context.Context, value interface{}) error {
	switch n.JsonType {
	case system.J_STRING:
		val := value.(string)
		if n.ValueString == val {
			// ignore the change if there's no change to the value
			return nil
		}
		if err := n.SetValueString(ctx, val); err != nil {
			return kerr.Wrap("NCIMXDORED", err)
		}
	case system.J_BOOL:
		val := value.(bool)
		if n.ValueBool == val {
			// ignore the change if there's no change to the value
			return nil
		}
		if err := n.SetValueBool(ctx, val); err != nil {
			return kerr.Wrap("HKFEEMFRHR", err)
		}
	case system.J_NUMBER:
		val := value.(float64)
		if n.ValueNumber == val {
			// ignore the change if there's no change to the value
			return nil
		}
		if err := n.SetValueNumber(ctx, val); err != nil {
			return kerr.Wrap("LBEBBNFJVG", err)
		}
	}
	return nil
}

func buildRulesCollectionChildren(ctx context.Context, n *node.Node, cache map[*node.Node][]system.RuleInterface) error {

	rules := n.Rule.Interface.(system.ObjectInterface).GetObject(nil).Rules

	// If we have additional rules on the main items rule, we should add them to rules
	items, err := n.Rule.ItemsRule()
	if err != nil {
		return kerr.Wrap("YFNERJIKWF", err)
	}
	itemsRuleOb := items.Interface.(system.ObjectInterface).GetObject(nil)
	if len(itemsRuleOb.Rules) > 0 {
		rules = append(rules, itemsRuleOb.Rules...)
	}

	for _, child := range n.Array {
		err := buildRulesObject(ctx, child, cache, rules)
		if err != nil {
			return kerr.Wrap("FWKXQRMPQJ", err)
		}
	}
	for _, child := range n.Map {
		err := buildRulesObject(ctx, child, cache, rules)
		if err != nil {
			return kerr.Wrap("MKYHIDMORV", err)
		}
	}
	return nil
}

func (n *Node) setZero(ctx context.Context, null bool, missing bool) error {

	if missing && !null {
		return kerr.New("NYQULBBBHO", "If missing, must also be null")
	}

	if missing && (n.Parent == nil || n.Parent.JsonType != system.J_OBJECT) {
		return kerr.New("XRYLQWRNPH", "Parent must be J_OBJECT")
	}

	if n.Type == nil {
		return kerr.New("ABXFQOYCBA", "Can't set value without a type")
	}

	if n.Type.IsNativeCollection() && n.Rule == nil {
		return kerr.New("VGKTIRMDTJ", "Can't create collection zero value without a rule")
	}

	n.Missing = missing
	n.Null = null
	n.ValueString = ""
	n.ValueNumber = 0.0
	n.ValueBool = false
	n.Array = []*Node{}
	n.Map = map[string]*Node{}

	if null {
		n.JsonType = system.J_NULL
	} else {
		// if this node was previously null, we must reset the json type
		n.JsonType = n.Type.NativeJsonType(ctx)
	}

	var rv reflect.Value
	if n.Type.IsNativeCollection() {
		var err error
		if rv, err = n.Rule.ZeroValue(null); err != nil {
			return kerr.Wrap("WNQLTRJRBD", err)
		}
	} else {
		// this is for both objects and native values
		var err error
		if rv, err = n.Type.ZeroValue(ctx, null); err != nil {
			return kerr.Wrap("UDBVTIDRIK", err)
		}
	}
	n.Value = rv.Interface()
	n.setVal(rv)

	if !null && n.Type.IsNativeObject() {
		if err := n.initialiseFields(ctx, nil, true); err != nil {
			return kerr.Wrap("VSAXCHGCOG", err)
		}
		if err := n.setCorrectTypeField(ctx); err != nil {
			return kerr.Wrap("CUCJDNBBSU", err)
		}
	}

	return nil
}

func (n *Node) initialiseCollectionItem(ctx context.Context, parent *Node, key string, index int) error {
	n.resetAllValues()
	n.Parent = parent
	n.Index = index
	n.Key = key
	n.Missing = false

	var collectionRule *system.RuleWrapper
	if n.Parent.Type.Alias != nil {
		collectionRule = system.WrapRule(ctx, n.Parent.Type.Alias)
	} else {
		collectionRule = parent.Rule
	}
	rule, err := collectionRule.ItemsRule()
	if err != nil {
		return kerr.Wrap("SBJVMGUOOA", err)
	}
	n.Rule = rule

	t, err := extractType(ctx, system.Pack(nil), rule)
	if err != nil {
		return kerr.Wrap("EQNRHQWXFJ", err)
	}

	if err := n.setType(ctx, t); err != nil {
		return kerr.Wrap("UPAQMUGDNH", err)
	}

	return nil
}

func MarshalIndent(ctx context.Context, v interface{}, prefix, indent string) ([]byte, error) {
	i, err := system.Repack(ctx, v)
	if err != nil {
		return nil, kerr.Wrap("QNHVVOBKFO", err)
	}
	b, err := json.MarshalIndent(i, prefix, indent)
	if err != nil {
		return nil, kerr.Wrap("QXXEBOMOFG", err)
	}
	return b, nil
}

func Marshal(ctx context.Context, v interface{}) ([]byte, error) {
	i, err := system.Repack(ctx, v)
	if err != nil {
		return nil, kerr.Wrap("JVGOWMVMXN", err)
	}
	b, err := json.Marshal(i)
	if err != nil {
		return nil, kerr.Wrap("LXDTUOBQPD", err)
	}
	return b, nil
}

func Open(ctx context.Context, filename string) (value interface{}, err error) {
	bytes, err := scanner.ProcessFile(filename)
	if err != nil {
		return nil, kerr.Wrap("HPWXWFTKWA", err)
	}
	err = system.Unmarshal(ctx, bytes, &value)
	if err != nil {
		return nil, kerr.Wrap("CXIULJCEBE", err)
	}
	return
}

func (n *Node) SetValueZero(ctx context.Context, null bool, t *system.Type) error {
	if t != nil {
		if err := n.setType(ctx, t); err != nil {
			return kerr.Wrap("TGPRRSUSMQ", err)
		}
	}
	if err := n.setZero(ctx, null, false); err != nil {
		return kerr.Wrap("JWCKCAJXUB", err)
	}
	return nil
}

func validateBytes(ctx context.Context, bytes []byte) (errors []ValidationError, err error) {
	n, err := node.Unmarshal(ctx, bytes)
	if err != nil {
		return nil, kerr.Wrap("QIVNOQKCQF", err)
	}
	errors, err = ValidateNode(ctx, n)
	if err != nil {
		return nil, kerr.Wrap("RVKNMWKQHD", err)
	}
	return errors, nil
}

func getInfo(infoBase64 string) (info *shared.Info, err error) {
	info = &shared.Info{}
	infoBytes, err := base64.StdEncoding.DecodeString(infoBase64)
	if err != nil {
		return nil, kerr.Wrap("UTKDDLYKKH", err)
	}
	buf := bytes.NewBuffer(infoBytes)
	if err := gob.NewDecoder(buf).Decode(info); err != nil {
		return nil, kerr.Wrap("AAFXLQRUEW", err)
	}
	return info, nil
}

func Repack(ctx context.Context, data interface{}) (interface{}, error) {

	if re, ok := data.(Repacker); ok {
		p, pkg, name, jt, err := re.Repack(ctx)
		if err != nil {
			return nil, kerr.Wrap("ISAOPMMEYI", err)
		}
		if jt == J_OBJECT {
			return p, nil
		}
		typRef := NewReference(pkg, name)
		typeVal, err := typRef.ValueContext(ctx)
		if err != nil {
			return nil, kerr.Wrap("IWSCKOWIMW", err)
		}
		out := map[string]interface{}{}
		out["type"] = typeVal
		out["value"] = p
		return out, nil
	}

	switch data.(type) {
	case string, float64, bool:
		return data, nil
	}

	val := reflect.ValueOf(data)
	switch val.Kind() {
	case reflect.Slice:
		out := []interface{}{}
		for i := 0; i < val.Len(); i++ {
			c, err := Repack(ctx, val.Index(i).Interface())
			if err != nil {
				return nil, kerr.Wrap("KSHOKVIBRS", err)
			}
			out = append(out, c)
		}
		return out, nil
	case reflect.Map:
		out := map[string]interface{}{}
		for _, key := range val.MapKeys() {
			c, err := Repack(ctx, val.MapIndex(key).Interface())
			if err != nil {
				return nil, kerr.Wrap("SOLGDVLWSK", err)
			}
			out[key.Interface().(string)] = c
		}
		return out, nil
	}

	return nil, kerr.New("POKFWVHEDY", "%T does can't be repacked", data)
}

func (n *Node) setCorrectTypeField(ctx context.Context) error {
	typeField, ok := n.Map["type"]
	if !ok {
		return kerr.New("DQKGYKFQKJ", "type field not found")
	}
	typeString, err := n.Type.Id.ValueContext(ctx)
	if err != nil {
		return kerr.Wrap("MRHEBUUXBB", err)
	}
	if err := typeField.SetValueString(ctx, typeString); err != nil {
		return kerr.Wrap("CURDKCQLGS", err)
	}
	return nil
}

func Marshal(ctx context.Context, data interface{}) ([]byte, error) {

	p, err := Repack(ctx, data)
	if err != nil {
		return nil, kerr.Wrap("OMFSDKIBCY", err)
	}

	b, err := json.Marshal(p)
	if err != nil {
		return nil, kerr.Wrap("YCEBRGOCGO", err)
	}

	return b, nil
}