func lexNextToken(input string, scanners []scannerItem) (*token, int, error) {
	for _, scanner := range scanners {
		if scanner.regex.MatchString(input) {
			idx := scanner.regex.FindStringIndex(input)
			if idx[0] == 0 {
				if scanner.typ == S_EXPR && strings.Count(input[idx[0]:idx[1]], "(") != strings.Count(input[idx[0]:idx[1]], ")") {
					terminated := false
					var curr int
					for curr = idx[1]; curr <= len(input[idx[0]:]); curr++ {
						if strings.Count(input[idx[0]:curr], "(") == strings.Count(input[idx[0]:curr], ")") {
							terminated = true
							break
						}
					}
					if terminated == false {
						return nil, len(input), kerr.New("MMTRYGGLNE", "Unterminated expression: %s", input[idx[0]:curr-1])
					} else {
						idx[1] = curr
					}
				}
				token := getToken(
					scanner.typ,
					input[idx[0]:idx[1]],
				)
				return &token, idx[1], nil
			}
		}
	}
	return nil, len(input), kerr.New("DPJDASCGQX", "Selector parsing error at %s", input)
}

func (n *Node) setZero(ctx context.Context, null bool, missing bool) error {

	if missing && !null {
		return kerr.New("NYQULBBBHO", "If missing, must also be null")
	}

	if missing && (n.Parent == nil || n.Parent.JsonType != system.J_OBJECT) {
		return kerr.New("XRYLQWRNPH", "Parent must be J_OBJECT")
	}

	if n.Type == nil {
		return kerr.New("ABXFQOYCBA", "Can't set value without a type")
	}

	if n.Type.IsNativeCollection() && n.Rule == nil {
		return kerr.New("VGKTIRMDTJ", "Can't create collection zero value without a rule")
	}

	n.Missing = missing
	n.Null = null
	n.ValueString = ""
	n.ValueNumber = 0.0
	n.ValueBool = false
	n.Array = []*Node{}
	n.Map = map[string]*Node{}

	if null {
		n.JsonType = system.J_NULL
	} else {
		// if this node was previously null, we must reset the json type
		n.JsonType = n.Type.NativeJsonType(ctx)
	}

	var rv reflect.Value
	if n.Type.IsNativeCollection() {
		var err error
		if rv, err = n.Rule.ZeroValue(null); err != nil {
			return kerr.Wrap("WNQLTRJRBD", err)
		}
	} else {
		// this is for both objects and native values
		var err error
		if rv, err = n.Type.ZeroValue(ctx, null); err != nil {
			return kerr.Wrap("UDBVTIDRIK", err)
		}
	}
	n.Value = rv.Interface()
	n.setVal(rv)

	if !null && n.Type.IsNativeObject() {
		if err := n.initialiseFields(ctx, nil, true); err != nil {
			return kerr.Wrap("VSAXCHGCOG", err)
		}
		if err := n.setCorrectTypeField(ctx); err != nil {
			return kerr.Wrap("CUCJDNBBSU", err)
		}
	}

	return nil
}

func (v *Mappy) Unpack(ctx context.Context, in Packed, iface bool) error {

	if in == nil || in.Type() == J_NULL {
		return nil
	}

	if iface {
		if in.Type() != J_MAP {
			return kerr.New("DTLQPHOJDT", "Mappy.Unpack: %s must by J_MAP", in.Type())
		}
		in = in.Map()["value"]
	}

	if in.Type() != J_MAP {
		return kerr.New("KYPSXYBLNC", "Mappy.Unpack: %s must by J_MAP", in.Type())
	}

	for key, value := range in.Map() {
		ob, err := UnpackString(ctx, value)
		if err != nil {
			return kerr.Wrap("HOLNALWYBA", err)
		}
		(*v)[key] = ob
	}

	return nil
}

func (v *StructView) Render() *vecty.HTML {
	if v.model == nil {
		return elem.Div(vecty.Text("Struct (nil)"))
	}

	if v.model.Node.Type.Basic {
		// This view is only for types that embed system:object
		panic(kerr.New("QHDQMXTNIH", "Basic type %s not supported by StructView", v.model.Node.Type.Id.String()))
	}

	out := vecty.List{}

	// Always show the editor for system:object first
	objectEditor, ok := clientctx.FromContext(v.Ctx).Get("kego.io/system:object")
	if !ok {
		panic(kerr.New("BJRMXESSUV", "Can't find editor for system:object"))
	}
	out = append(out, objectEditor.EditorView(v.Ctx, v.model.Node, editable.Block))

	out = append(out, NewStructFragmentView(v.Ctx, v.model.Node, v.model.Node.Type.Id))

	return elem.Div(
		out,
	)

}

// ItemsRule returns Items rule for a collection Rule.
func (r *RuleWrapper) ItemsRule() (*RuleWrapper, error) {
	if !r.IsCollection() {
		return nil, kerr.New("VPAGXSTQHM", "%s is not a collection", r.Parent.Id.Value())
	}
	// I don't think this should be here:
	/*
		if r.Parent.Alias != nil {
			aw, err := WrapRule(r.Ctx, r.Parent.Alias)
			if err != nil {
				return nil, kerr.Wrap("PVCNTDVGWA", err)
			}
			if aw.IsCollection() {
				ir, err := aw.ItemsRule()
				if err != nil {
					return nil, kerr.Wrap("UIGQFXJLJE", err)
				}
				return ir, nil
			}
		}
	*/
	c, ok := r.Interface.(CollectionRule)
	if !ok {
		return nil, kerr.New("TNRVQVJIFH", "%T is not a CollectionRule", r.Interface)
	}
	ir := c.GetItemsRule()
	if ir == nil {
		return nil, kerr.New("SUJLYBXPYS", "%s has nil items rule", r.Parent.Id.Value())
	}
	w := WrapRule(r.Ctx, ir)
	return w, nil
}

func extractFields(ctx context.Context, fields map[string]*system.Field, t *system.Type) error {
	for t.Alias != nil {
		t = system.WrapRule(ctx, t.Alias).Parent
	}
	if !t.Basic && !t.Interface {
		// All types apart from Basic types embed system:object
		ob, ok := system.GetTypeFromCache(ctx, "kego.io/system", "object")
		if !ok {
			return kerr.New("YRFWOTIGFT", "Type system:object not found in sys ctx")
		}
		if err := extractFields(ctx, fields, ob); err != nil {
			return kerr.Wrap("DTQEFALIMM", err)
		}
	}
	for _, embedRef := range t.Embed {
		embed, ok := system.GetTypeFromCache(ctx, embedRef.Package, embedRef.Name)
		if !ok {
			return kerr.New("SLIRILCARQ", "Type %s not found in sys ctx", embedRef)
		}
		if err := extractFields(ctx, fields, embed); err != nil {
			return kerr.Wrap("JWAPCVIYBJ", err)
		}
	}
	for name, rule := range t.Fields {
		if _, ok := fields[name]; ok {
			return kerr.New("BARXPFXQNB", "Duplicate field %s", name)
		}
		fields[name] = &system.Field{Name: name, Rule: rule, Origin: t.Id}
	}
	return nil
}

func (r *RuleWrapper) GetReflectType() (reflect.Type, error) {

	if r.Struct != nil && r.Struct.Interface {
		typ, ok := r.Parent.GetReflectInterface(r.Ctx)
		if !ok {
			return nil, kerr.New("QGUVEUTXAN", "Type interface for %s not found", r.Parent.Id.Value())
		}
		return typ, nil
	}

	if c, ok := r.Interface.(CollectionRule); ok {
		itemsRule := c.GetItemsRule()
		if itemsRule != nil {
			items := WrapRule(r.Ctx, itemsRule)
			itemsType, err := items.GetReflectType()
			if err != nil {
				return nil, kerr.Wrap("LMKEHHWHKL", err)
			}
			if r.Parent.NativeJsonType(r.Ctx) == J_MAP {
				return reflect.MapOf(reflect.TypeOf(""), itemsType), nil
			}
			return reflect.SliceOf(itemsType), nil
		}
	}

	typ, ok := r.Parent.GetReflectType(r.Ctx)
	if !ok {
		return nil, kerr.New("DLAJJPJDPL", "Type %s not found", r.Parent.Id.Value())
	}

	return typ, nil

}

func (t *Type) ZeroValue(ctx context.Context, null bool) (reflect.Value, error) {
	if t.IsNativeCollection() {
		return reflect.Value{}, kerr.New("PGUHCGBJWE", "ZeroValue must not be used with collection type")
	}
	rt, ok := t.GetReflectType(ctx)
	if !ok {
		return reflect.Value{}, kerr.New("RSWTEOTNBD", "Type not found for %s", t.Id)
	}
	return zeroValue(rt, null), nil
}

func GetReferencePartsFromTypeString(ctx context.Context, typeString string) (path string, name string, err error) {

	env := envctx.FromContext(ctx)

	if strings.Contains(typeString, "/") {
		// If the type name contains a slash, I'm assuming it's a fully
		// qualified type name of the form "kego.io/system:type".
		// TODO: Improve this with a regex?
		parts := strings.Split(typeString, ":")

		// We hard-code system and json to prevent them having to always be
		// specified in the aliases
		if parts[0] == "kego.io/system" {
			return "kego.io/system", parts[1], nil
		} else if parts[0] == "kego.io/json" {
			return "kego.io/json", parts[1], nil
		}

		_, found := findKey(env.Aliases, parts[0])
		if !found && parts[0] != env.Path {
			return "", "", UnknownPackageError{
				Struct:         kerr.New("KJSOXDESFD", "Unknown package %s", parts[0]),
				UnknownPackage: parts[0],
			}
		}
		return parts[0], parts[1], nil
	} else if strings.Contains(typeString, ":") {
		// If the type name contains a colon, I'm assuming it's an abreviated
		// qualified type name of the form "system:type". We should look the
		// package name up in the aliases map.
		// TODO: Improve this with a regex?
		parts := strings.Split(typeString, ":")

		// We hard-code system and json to prevent them having to always be
		// specified in the aliases
		if parts[0] == "system" {
			return "kego.io/system", parts[1], nil
		} else if parts[0] == "json" {
			return "kego.io/json", parts[1], nil
		}

		packagePath, ok := env.Aliases[parts[0]]
		if !ok {
			return "", "", UnknownPackageError{
				Struct:         kerr.New("DKKFLKDKYI", "Unknown package %s", parts[0]),
				UnknownPackage: parts[0],
			}
		}
		return packagePath, parts[1], nil
	} else {
		return env.Path, typeString, nil
	}
}

func GetEmbedEditable(ctx context.Context, node *node.Node, embed *system.Reference) (editable.Editable, error) {

	if node == nil || node.Null || node.Missing {
		return nil, nil
	}

	if *node.Type.Id == *embed {
		return GetEditable(ctx, node), nil
	}

	jcache := jsonctx.FromContext(ctx)
	nf, df, ok := jcache.GetNewFunc(embed.Package, embed.Name)
	if !ok {
		return nil, kerr.New("DGWDERFPVV", "Can't find %s in jsonctx", embed.String())
	}
	t := reflect.TypeOf(nf())
	if df != nil {
		t = t.Elem()
	}

	v := node.Val
	for v.Kind() == reflect.Ptr || v.Kind() == reflect.Interface {
		v = v.Elem()
	}

	var field reflect.Value
	for i := 0; i < v.Type().NumField(); i++ {
		f := v.Type().Field(i)
		if f.Anonymous && f.Type == t {
			field = v.Field(i)
			break
		}
	}
	if field == (reflect.Value{}) {
		return nil, kerr.New("UDBOWYUBER", "Can't find %s field in struct", t)
	}

	// This is the recommended method of presenting an custom editor.
	if ed, ok := field.Interface().(editable.Editable); ok {
		return ed, nil
	}

	editors := clientctx.FromContext(ctx)

	// Don't do this. Implement the Editable interface instead. We can't do this
	// for system types so we use this method instead.
	if e, ok := editors.Get(embed.String()); ok {
		return e, nil
	}

	return nil, nil

}

func scanForTypesAndExports(ctx context.Context, env *envctx.Env, cache *sysctx.SysPackageInfo, hash *PackageHasher) error {

	// While we're scanning for types, we should use a custom unpacking env,
	// because the env from the context is the one of the local package.

	files := scanner.ScanDirToFiles(ctx, env.Dir, env.Recursive)
	bytes := scanner.ScanFilesToBytes(ctx, files)
	localContext := envctx.NewContext(ctx, env)
	for b := range bytes {
		if b.Err != nil {
			return kerr.Wrap("JACKALTIGG", b.Err)
		}

		o := &system.ObjectStub{}
		if err := system.Unmarshal(localContext, b.Bytes, o); err != nil {
			return kerr.Wrap("HCYGNBDFFA", err)
		}
		if o.Type == nil {
			return kerr.New("NUKWIHYFMQ", "%s has no type", b.File)
		}
		if o.Id == nil && *o.Type != *system.NewReference("kego.io/system", "package") {
			// we tolerate missing ID only for system:package
			return kerr.New("DLLMKTDYFW", "%s has no id", b.File)
		}
		relativeFile, err := filepath.Rel(env.Dir, b.File)
		if err != nil {
			return kerr.Wrap("AWYRJSCYQS", err)
		}
		switch *o.Type {
		case *system.NewReference("kego.io/system", "type"):
			if err := ProcessTypeFileBytes(ctx, env, relativeFile, b.Bytes, cache, hash); err != nil {
				return kerr.Wrap("IVEFDDSKHE", err)
			}
		case *system.NewReference("kego.io/system", "package"):
			cache.PackageBytes = b.Bytes
			cache.PackageFilename = relativeFile
		default:
			cache.Globals.Set(o.Id.Name, relativeFile)
			if o.Export {
				cache.Exports.Set(o.Id.Name, o.Type.Name, o.Type.Package, b.Bytes)
				if hash != nil {
					hash.Exports[o.Id.Name+" "+o.Type.Name+" "+o.Type.Package] = cityhash.CityHash64(b.Bytes, uint32(len(b.Bytes)))
				}
			}
		}

	}
	return nil
}

// FromContext returns the User value stored in ctx, if any.
func FromContext(ctx context.Context) *Env {
	e, ok := ctx.Value(envKey).(*Env)
	if !ok {
		panic(kerr.New("WYDYAGVLCR", "No env in ctx").Error())
	}
	return e
}

// FromContext returns the Cache value stored in ctx, and panics if it's not found.
func FromContext(ctx context.Context) *JsonCache {
	e, ok := ctx.Value(jsonKey).(*JsonCache)
	if !ok {
		panic(kerr.New("XUTUUVDMMX", "No json cache in ctx").Error())
	}
	return e
}

// FromContext returns the User value stored in ctx, if any.
func FromContext(ctx context.Context) *sync.WaitGroup {
	wg, ok := ctx.Value(wgKey).(*sync.WaitGroup)
	if !ok {
		panic(kerr.New("UNYGADKEGY", "No wg in ctx").Error())
	}
	return wg
}

func addNewFile(ctx context.Context, app *stores.App, all bool) {

	var types []*system.Type
	if all {
		rt := reflect.TypeOf((*system.ObjectInterface)(nil)).Elem()
		typesAll := system.GetAllTypesThatImplementReflectInterface(ctx, rt)

		// TODO: Work out a more elegant way of doing this!
		rule := reflect.TypeOf((*system.RuleInterface)(nil)).Elem()
		for _, t := range typesAll {
			if t.Id.Package == "kego.io/system" {
				// none of the system types should be added as a global
				continue
			}
			if t.Implements(ctx, rule) {
				// rules should never be added as a global
				continue
			}
			types = append(types, t)
		}

	} else {
		syscache := sysctx.FromContext(ctx)
		t, ok := syscache.GetType("kego.io/system", "type")
		if !ok {
			panic(kerr.New("NNFSJEXNKF", "Can't find system:type in sys ctx").Error())
		}
		types = []*system.Type{t.(*system.Type)}
	}

	app.Dispatch(&actions.OpenAddPopup{
		Types: types,
	})

}

func WrapRule(ctx context.Context, r RuleInterface) *RuleWrapper {

	ob, ok := r.(ObjectInterface)
	if !ok {
		panic(kerr.New("VKFNPJDNVB", "%T does not implement ObjectInterface", r).Error())
	}

	// ob.GetObject(nil).Type will be a @ rule type, so we change to the type,
	// which will be the parent
	t, ok := ob.GetObject(nil).Type.ChangeToType().GetType(ctx)
	if !ok {
		panic(kerr.New("KYCTDXKFYR", "GetType: type %v not found", ob.GetObject(nil).Type.ChangeToType().Value()).Error())
	}

	return &RuleWrapper{Ctx: ctx, Interface: r, Parent: t, Struct: r.GetRule(nil)}
}

func (r Reference) ValueContext(ctx context.Context) (string, error) {

	env := envctx.FromContextOrNil(ctx)
	if env == nil || env.Path == "" {
		return r.Value(), nil
	}

	if r.Package == "" && r.Name == "" {
		return "", nil
	}
	if r.Package == env.Path {
		return r.Name, nil
	}
	if r.Package == "kego.io/json" {
		return fmt.Sprintf("json:%s", r.Name), nil
	}
	if r.Package == "kego.io/system" {
		return fmt.Sprintf("system:%s", r.Name), nil
	}
	for alias, pkg := range env.Aliases {
		if pkg == r.Package {
			return fmt.Sprintf("%s:%s", alias, r.Name), nil
		}
	}
	return "", kerr.New("WGCDQQCFAD", "Package %s not found in aliases", r.Package)
}

func (n *Node) ReorderArrayChild(from, to int) error {

	if n.JsonType != system.J_ARRAY {
		return kerr.New("MHEXGBUQOL", "Must be J_ARRAY")
	}

	a := n.Array

	// remove the item we're moving
	item := a[from]

	a = append(
		a[:from],
		a[from+1:]...)

	// insert it back in the correct place
	n.Array = append(
		a[:to],
		append([]*Node{item}, a[to:]...)...)

	// correct the indexes
	for i, c := range n.Array {
		c.Index = i
	}

	val := n.Val.Index(from).Interface()
	deleteFromSlice(n.Val, from)
	insertIntoSlice(n.Val, to, reflect.ValueOf(val))
	return nil
}

func FromContext(ctx context.Context) *EditorCache {
	ec, ok := ctx.Value(ctxKey).(*EditorCache)
	if !ok {
		panic(kerr.New("BIUVXISEMA", "No editors in ctx").Error())
	}
	return ec
}

func (n *Node) setType(ctx context.Context, t *system.Type) error {
	if t == nil {
		n.Type = nil
		n.JsonType = system.J_NULL
		n.UnderlyingRule = nil
		n.UnderlyingInnerType = nil
		return nil
	}
	if t.Interface {
		return kerr.New("VHOSYBMDQL", "Can't set type to an interface - must be concrete type.")
	}
	n.Type = t
	n.JsonType = t.NativeJsonType(ctx)

	if t.Alias != nil {
		rw := system.WrapRule(ctx, t.Alias)
		n.UnderlyingRule = rw

		uit := rw.InnerType(ctx)
		for uit.Alias != nil {
			uit = system.WrapRule(ctx, uit.Alias).InnerType(ctx)
		}
		n.UnderlyingInnerType = uit
	} else {
		n.UnderlyingRule = system.WrapEmptyRule(ctx, t)
		n.UnderlyingInnerType = t
	}

	return nil
}