func CPUProfile(cpuProfile string) func() {
	errors.Logf("DEBUG", "starting cpu profile: %v", cpuProfile)
	f, err := os.Create(cpuProfile)
	if err != nil {
		log.Fatal(err)
	}
	err = pprof.StartCPUProfile(f)
	if err != nil {
		log.Fatal(err)
	}
	sigs := make(chan os.Signal, 1)
	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
	go func() {
		sig := <-sigs
		errors.Logf("DEBUG", "closing cpu profile")
		pprof.StopCPUProfile()
		err := f.Close()
		errors.Logf("DEBUG", "closed cpu profile, err: %v", err)
		panic(errors.Errorf("caught signal: %v", sig))
	}()
	return func() {
		errors.Logf("DEBUG", "closing cpu profile")
		pprof.StopCPUProfile()
		err := f.Close()
		errors.Logf("DEBUG", "closed cpu profile, err: %v", err)
	}
}

func (w *Walker) probabilities(lat *lattice.Lattice) ([]int, error) {
	P := make([]int, len(lat.V))
	for i, node := range lat.V {
		count, err := node.ChildCount()
		if err != nil {
			return nil, err
		}
		if count < len(lat.Children(i)) {
			errors.Logf("WARNING", "count < lat.Children count %v < %v", count, len(lat.Children(i)))
			count = len(lat.Children(i))
		}
		if count > 0 && len(lat.Children(i)) == 0 {
			errors.Logf("WARNING", "count > 0 && lat.Children == 0 : %v > 0 lat.Children == %v", count, len(lat.Children(i)))
		}
		if i+1 == len(lat.V) {
			P[i] = -1
		} else if count == 0 {
			P[i] = 1
			errors.Logf("WARNING", "0 count for %v, using %v", node, P[i])
		} else {
			P[i] = count
		}
	}
	return P, nil
}

func (w *Walker) Mine(dt lattice.DataType, rptr miners.Reporter, fmtr lattice.Formatter) error {
	errors.Logf("INFO", "Customize creation")

	pConf := w.Config.Copy()
	pConf.Samples = 1000
	pConf.Unique = false
	premine := walker.NewWalker(pConf, ospace.MakeUniformWalk(.02, false))
	premine.Reject = false
	collector := &reporters.Collector{make([]lattice.Node, 0, 10)}
	uniq, err := reporters.NewUnique(pConf, fmtr, collector, "")
	if err != nil {
		return err
	}
	pRptr := &reporters.Skip{
		Skip:     10,
		Reporter: &reporters.Chain{[]miners.Reporter{reporters.NewLog(fmtr, false, "DEBUG", "premining"), uniq}},
	}

	err = premine.Mine(dt, pRptr, fmtr)
	if err != nil {
		return err
	}

	w.Teleports = collector.Nodes
	errors.Logf("INFO", "teleports %v", len(w.Teleports))

	return (w.Walker).Mine(dt, rptr, fmtr)
}

func (r *File) Report(n lattice.Node) error {
	err := r.fmtr.FormatPattern(r.patterns, n)
	if err != nil {
		return err
	}
	err = r.fmtr.FormatEmbeddings(r.embeddings, n)
	if err != nil {
		return err
	}
	_, err = fmt.Fprintf(r.names, "%v\n", r.fmtr.PatternName(n))
	if err != nil {
		return err
	}
	if r.prfmtr != nil {
		matrices, err := r.prfmtr.Matrices(n)
		if err == nil {
			r.prfmtr.FormatMatrices(r.matrices, r.fmtr, n, matrices)
		}
		if err != nil {
			fmt.Fprintf(r.matrices, "ERR: %v\n", err)
			errors.Logf("ERROR", "Pr Matrices Computation Error: vs", err)
		} else if r.prfmtr.CanComputeSelPr(n, matrices) {
			pr, err := r.prfmtr.SelectionProbability(n, matrices)
			if err != nil {
				fmt.Fprintf(r.prs, "ERR: %v\n", err)
				errors.Logf("ERROR", "PrComputation Error: %v", err)
			} else {
				fmt.Fprintf(r.prs, "%g, %v\n", pr, r.fmtr.PatternName(n))
			}
		} else {
			fmt.Fprintf(r.prs, "SKIPPED\n")
		}
	}
	return nil
}

func countReporter(rptrs map[string]Reporter, argv []string, fmtr lattice.Formatter, conf *config.Config) (miners.Reporter, []string) {
	args, optargs, err := getopt.GetOpt(
		argv,
		"hf:",
		[]string{
			"help",
			"filename=",
		},
	)
	if err != nil {
		errors.Logf("ERROR", "%v", err)
		Usage(ErrorCodes["opts"])
	}
	filename := "count"
	for _, oa := range optargs {
		switch oa.Opt() {
		case "-h", "--help":
			Usage(0)
		case "-f", "--filename":
			filename = oa.Arg()
		default:
			errors.Logf("ERROR", "Unknown flag '%v'\n", oa.Opt())
			Usage(ErrorCodes["opts"])
		}
	}
	r, err := reporters.NewCount(conf, filename)
	if err != nil {
		errors.Logf("ERROR", "There was error creating output files\n")
		errors.Logf("ERROR", "%v", err)
		os.Exit(1)
	}
	return r, args
}

func (w *Walker) RejectingWalk(samples chan lattice.Node, terminate chan bool) chan lattice.Node {
	accepted := make(chan lattice.Node)
	go func() {
		i := 0
		seen := set.NewSortedSet(w.Config.Samples)
		for sampled := range samples {
			accept := false
			if !w.Reject || w.Dt.Acceptable(sampled) {
				label := types.ByteSlice(sampled.Pattern().Label())
				if !w.Config.Unique || !seen.Has(label) {
					if w.Config.Unique {
						seen.Add(label)
					}
					accept = true
					i++
				} else {
					errors.Logf("DEBUG", "duplicate %v", sampled)
				}
			} else {
				errors.Logf("DEBUG", "rejected %v", sampled)
			}
			if i >= w.Config.Samples {
				terminate <- true
			} else {
				terminate <- false
			}
			if accept {
				accepted <- sampled
			}
		}
		close(accepted)
		close(terminate)
	}()
	return accepted
}

func (l *IntLoader) items(input lattice.Input) func(do func(tx, item int32) error) error {
	return func(do func(tx, item int32) error) error {
		in, closer := input()
		defer closer()
		scanner := bufio.NewScanner(in)
		tx := int32(0)
		for scanner.Scan() {
			if tx%1000 == 0 {
				errors.Logf("INFO", "line %d", tx)
			}
			line := scanner.Text()
			for _, col := range strings.Split(line, " ") {
				if col == "" {
					continue
				}
				item, err := strconv.Atoi(col)
				if err != nil {
					errors.Logf("WARN", "input line %d contained non int '%s'", tx, col)
					continue
				}
				err = do(tx, int32(item))
				if err != nil {
					errors.Logf("ERROR", "%v", err)
					return err
				}
			}
			tx += 1
		}
		if err := scanner.Err(); err != nil {
			return err
		}
		return nil
	}
}

func (w *Walker) Mine(dt lattice.DataType, rptr miners.Reporter, fmtr lattice.Formatter) error {
	err := w.Init(dt, rptr)
	if err != nil {
		return err
	}
	errors.Logf("INFO", "finished initialization, starting walk")
	samples, terminate, errs := w.Walk(w)
	samples = w.RejectingWalk(samples, terminate)
loop:
	for {
		select {
		case sampled, open := <-samples:
			if sampled != nil {
				if err := w.Rptr.Report(sampled); err != nil {
					return err
				}
			}
			if !open {
				break loop
			}
		case err := <-errs:
			if err != nil {
				return err
			}
		}
	}
	errors.Logf("INFO", "exiting walker Mine")
	return nil
}

func fileReporter(rptrs map[string]Reporter, argv []string, fmtr lattice.Formatter, conf *config.Config) (miners.Reporter, []string) {
	args, optargs, err := getopt.GetOpt(
		argv,
		"hp:e:n:",
		[]string{
			"help",
			"patterns=",
			"embeddings=",
			"names=",
			"matrices=",
			"probabilities=",
			"show-pr",
		},
	)
	if err != nil {
		errors.Logf("ERROR", "%v", err)
		Usage(ErrorCodes["opts"])
	}
	patterns := "patterns"
	embeddings := "embeddings"
	names := "names.txt"
	matrices := "matrices.json"
	probabilities := "probabilities.prs"
	showPr := false
	for _, oa := range optargs {
		switch oa.Opt() {
		case "-h", "--help":
			Usage(0)
		case "-p", "--patterns":
			patterns = oa.Arg()
		case "-e", "--embeddings":
			embeddings = oa.Arg()
		case "-n", "--names":
			names = oa.Arg()
		case "--matrices":
			matrices = oa.Arg()
		case "--probabilites":
			probabilities = oa.Arg()
		case "--show-pr":
			showPr = true
		default:
			errors.Logf("ERROR", "Unknown flag '%v'\n", oa.Opt())
			Usage(ErrorCodes["opts"])
		}
	}
	fr, err := reporters.NewFile(conf, fmtr, showPr, patterns, embeddings, names, matrices, probabilities)
	if err != nil {
		errors.Logf("ERROR", "There was error creating output files\n")
		errors.Logf("ERROR", "%v\n", err)
		os.Exit(1)
	}
	return fr, args
}

func dbscanReporter(rptrs map[string]Reporter, argv []string, fmtr lattice.Formatter, conf *config.Config) (miners.Reporter, []string) {
	args, optargs, err := getopt.GetOpt(
		argv,
		"hf:e:g:a:",
		[]string{
			"help",
			"filename=",
			"epsilon=",
			"gamma=",
			"attr=",
		},
	)
	if err != nil {
		errors.Logf("ERROR", "%v", err)
		Usage(ErrorCodes["opts"])
	}
	clusters := "clusters"
	metrics := "metrics"
	attr := ""
	epsilon := 0.2
	gamma := 0.2
	for _, oa := range optargs {
		switch oa.Opt() {
		case "-h", "--help":
			Usage(0)
		case "-c", "--clusters-name":
			clusters = oa.Arg()
		case "-m", "--metrics-name":
			metrics = oa.Arg()
		case "-a", "--attr":
			attr = oa.Arg()
		case "-e", "--epsilon":
			epsilon = ParseFloat(oa.Arg())
		case "-g", "--gamma":
			gamma = ParseFloat(oa.Arg())
		default:
			errors.Logf("ERROR", "Unknown flag '%v'\n", oa.Opt())
			Usage(ErrorCodes["opts"])
		}
	}
	if attr == "" {
		errors.Logf("ERROR", "You must supply --attr=<attr> to dbscan")
		Usage(ErrorCodes["opts"])
	}
	r, err := reporters.NewDbScan(conf, fmtr, clusters, metrics, attr, epsilon, gamma)
	if err != nil {
		errors.Logf("ERROR", "There was error creating output files\n")
		errors.Logf("ERROR", "%v", err)
		os.Exit(1)
	}
	return r, args
}

func (m *Miner) Mine(dt lattice.DataType, rptr miners.Reporter, fmtr lattice.Formatter) error {
	err := m.Init(dt, rptr)
	if err != nil {
		return err
	}
	errors.Logf("INFO", "finished initialization, starting walk")
	err = m.mine()
	if err != nil {
		return err
	}
	errors.Logf("INFO", "exiting Mine")
	return nil
}

func (w *Walker) walkFrom(v lattice.Node) (path []lattice.Node, err error) {
	weight := func(_, a lattice.Node) (float64, error) {
		// kids, err := a.CanonKids()
		// if err != nil {
		// 	return 0, err
		// }
		// return float64(len(kids)), nil
		return 1, nil
	}
	transition := func(c lattice.Node) (lattice.Node, error) {
		errors.Logf("EST-WALK-DEBUG", "walk cur %v", c)
		kids, err := c.CanonKids()
		if err != nil {
			return nil, err
		}
		// errors.Logf("EST-WALK-DEBUG", "cur %v len(kids) %v", c, len(kids))
		_, next, err := walker.Transition(c, kids, weight, false)
		return next, err
	}
	c := v
	n, err := transition(c)
	if err != nil {
		return nil, err
	}
	path = append(path, c)
	for n != nil {
		c = n
		path = append(path, c)
		n, err = transition(c)
		if err != nil {
			return nil, err
		}
	}
	return path, nil
}

func Next(ctx interface{}, cur lattice.Node) (lattice.Node, error) {
	w := ctx.(*Walker)
	if ismax, err := cur.Maximal(); err != nil {
		return nil, err
	} else if ismax && w.Dt.Acceptable(cur) {
		w.teleportAllowed = true
		errors.Logf("INFO", "ALLOWING TELEPORTS")
	}
	if w.teleportAllowed && rand.Float64() < w.TeleportProbability {
		w.teleportAllowed = false
		next := w.Teleports[rand.Intn(len(w.Teleports))]
		errors.Logf("INFO", "TELEPORT\n    from %v\n      to %v", cur, next)
		return next, nil
	}
	return musk.Next(ctx, cur)
}

func extendNode(dt *Digraph, n Node, debug bool) (*hashtable.LinearHash, error) {
	if debug {
		errors.Logf("DEBUG", "n.SubGraph %v", n.SubGraph())
	}
	sg := n.SubGraph()
	b := subgraph.Build(len(sg.V), len(sg.E)).From(sg)
	extPoints, err := n.Extensions()
	if err != nil {
		return nil, err
	}
	patterns := hashtable.NewLinearHash()
	for _, ep := range extPoints {
		bc := b.Copy()
		bc.Extend(ep)
		if len(bc.V) > dt.MaxVertices {
			continue
		}
		vord, eord := bc.CanonicalPermutation()
		ext := bc.BuildFromPermutation(vord, eord)
		if !patterns.Has(ext) {
			patterns.Put(ext, &extInfo{ep, vord})
		}
	}

	return patterns, nil
}

func (l *IntLoader) indices(items itemsIter) (idx, inv index, err error) {
	max_tx, max_item, err := l.max(items)
	if err != nil {
		return nil, nil, err
	}
	counts := make([]int, max_item+1)
	err = items(func(tx, item int32) error {
		counts[item]++
		return nil
	})
	if err != nil {
		return nil, nil, err
	}
	errors.Logf("DEBUG", "max tx : %v, max item : %v", max_tx, max_item)
	idx = make(index, max_tx+1)
	inv = make(index, max_item+1)
	err = items(func(tx, item int32) error {
		if counts[item] > l.sets.Support() {
			idx = idx.add(tx, item)
			inv = inv.add(item, tx)
		}
		return nil
	})
	if err != nil {
		return nil, nil, err
	}
	return idx, inv, nil
}

func uniqueReporter(reports map[string]Reporter, argv []string, fmtr lattice.Formatter, conf *config.Config) (miners.Reporter, []string) {
	args, optargs, err := getopt.GetOpt(
		argv,
		"h",
		[]string{
			"help",
			"histogram=",
		},
	)
	if err != nil {
		errors.Logf("ERROR", "%v", err)
		Usage(ErrorCodes["opts"])
	}
	histogram := ""
	for _, oa := range optargs {
		switch oa.Opt() {
		case "-h", "--help":
			Usage(0)
		case "--histogram":
			histogram = oa.Arg()
		default:
			errors.Logf("ERROR", "Unknown flag '%v'", oa.Opt())
			Usage(ErrorCodes["opts"])
		}
	}
	var rptr miners.Reporter
	if len(args) == 0 {
		errors.Logf("ERROR", "You must supply an inner reporter to unique")
		fmt.Fprintln(os.Stderr, "try: unique file")
		Usage(ErrorCodes["opts"])
	} else if _, has := reports[args[0]]; !has {
		errors.Logf("ERROR", "Unknown reporter '%v'", args[0])
		fmt.Fprintln(os.Stderr, "Reporters:")
		for k := range reports {
			fmt.Fprintln(os.Stderr, "  ", k)
		}
		Usage(ErrorCodes["opts"])
	} else {
		rptr, args = reports[args[0]](reports, args[1:], fmtr, conf)
	}
	uniq, err := reporters.NewUnique(conf, fmtr, rptr, histogram)
	if err != nil {
		errors.Logf("ERROR", "Error creating unique reporter '%v'\n", err)
		Usage(ErrorCodes["opts"])
	}
	return uniq, args
}

func NewIndices(b *Builder, minSupport int) *Indices {
	errors.Logf("DEBUG", "About to build indices %v %v", len(b.V), len(b.E))
	i := &Indices{
		ColorIndex:     make(map[int][]int, len(b.VertexColors)),
		SrcIndex:       make(map[IdColorColor][]int, len(b.V)),
		TargIndex:      make(map[IdColorColor][]int, len(b.V)),
		EdgeIndex:      make(map[Edge]*Edge, len(b.E)),
		EdgeCounts:     make(map[Colors]int, len(b.EdgeColors)),
		FreqEdges:      make([]Colors, 0, len(b.EdgeColors)),
		EdgesFromColor: make(map[int][]Colors, len(b.VertexColors)),
		EdgesToColor:   make(map[int][]Colors, len(b.VertexColors)),
		VertexColors:   b.VertexColors,
		EdgeColors:     b.EdgeColors,
	}
	i.G = b.Build(
		func(u *Vertex) {
			if b.VertexColors[u.Color] < minSupport {
				return
			}
			if i.ColorIndex[u.Color] == nil {
				i.ColorIndex[u.Color] = make([]int, 0, b.VertexColors[u.Color])
			}
			i.ColorIndex[u.Color] = append(i.ColorIndex[u.Color], u.Idx)
		},
		func(e *Edge) {
			edge := Edge{Src: e.Src, Targ: e.Targ, Color: e.Color}
			srcKey := IdColorColor{e.Src, e.Color, b.V[e.Targ].Color}
			targKey := IdColorColor{e.Targ, e.Color, b.V[e.Src].Color}
			colorKey := Colors{b.V[e.Src].Color, b.V[e.Targ].Color, e.Color}
			if i.SrcIndex[srcKey] == nil {
				i.SrcIndex[srcKey] = make([]int, 0, len(b.Adj[e.Src]))
			}
			if i.TargIndex[targKey] == nil {
				i.TargIndex[targKey] = make([]int, 0, len(b.Adj[e.Targ]))
			}
			i.EdgeIndex[edge] = e
			i.SrcIndex[srcKey] = append(i.SrcIndex[srcKey], e.Targ)
			i.TargIndex[targKey] = append(i.TargIndex[targKey], e.Src)
			i.EdgeCounts[colorKey] += 1
			// only add to frequent edges exactly when this colorKey has
			// surpassed min_support.
			if i.EdgeCounts[colorKey] == minSupport {
				if i.EdgesFromColor[e.Color] == nil {
					i.EdgesFromColor[e.Color] = make([]Colors, 0, 10)
				}
				if i.EdgesToColor[e.Color] == nil {
					i.EdgesToColor[e.Color] = make([]Colors, 0, 10)
				}
				i.FreqEdges = append(i.FreqEdges, colorKey)
				i.EdgesFromColor[colorKey.SrcColor] = append(
					i.EdgesFromColor[colorKey.SrcColor],
					colorKey)
				i.EdgesToColor[colorKey.TargColor] = append(
					i.EdgesToColor[colorKey.TargColor],
					colorKey)
			}
		})
	return i
}

func chainReporter(reports map[string]Reporter, argv []string, fmtr lattice.Formatter, conf *config.Config) (miners.Reporter, []string) {
	args, optargs, err := getopt.GetOpt(
		argv,
		"h",
		[]string{
			"help",
		},
	)
	if err != nil {
		errors.Logf("ERROR", "%v", err)
		Usage(ErrorCodes["opts"])
	}
	for _, oa := range optargs {
		switch oa.Opt() {
		case "-h", "--help":
			Usage(0)
		default:
			errors.Logf("ERROR", "Unknown flag '%v'", oa.Opt())
			Usage(ErrorCodes["opts"])
		}
	}
	rptrs := make([]miners.Reporter, 0, 10)
	for len(args) >= 1 {
		if args[0] == "endchain" {
			args = args[1:]
			break
		}
		if _, has := reports[args[0]]; !has {
			errors.Logf("ERROR", "Unknown reporter '%v'\n", args[0])
			fmt.Fprintln(os.Stderr, "Reporters:")
			for k := range reports {
				fmt.Fprintln(os.Stderr, "  ", k)
			}
			Usage(ErrorCodes["opts"])
		}
		var rptr miners.Reporter
		rptr, args = reports[args[0]](reports, args[1:], fmtr, conf)
		rptrs = append(rptrs, rptr)
	}
	if len(rptrs) == 0 {
		errors.Logf("ERROR", "Empty chain")
		fmt.Fprintln(os.Stderr, "try: chain log file")
		Usage(ErrorCodes["opts"])
	}
	return &reporters.Chain{rptrs}, args
}

func NewWalker(conf *config.Config, teleportProbability float64) *Walker {
	errors.Logf("INFO", "teleport probability %v", teleportProbability)
	miner := &Walker{
		TeleportProbability: teleportProbability,
	}
	miner.Walker = *walker.NewWalker(conf, musk.MakeMaxUniformWalk(Next, miner))
	return miner
}

func Run(dt lattice.DataType, fmtr lattice.Formatter, mode miners.Miner, rptr miners.Reporter) int {
	errors.Logf("INFO", "loaded data, about to start mining")
	mineErr := mode.Mine(dt, rptr, fmtr)

	code := 0
	if e := mode.Close(); e != nil {
		errors.Logf("ERROR", "error closing %v", e)
		code++
	}
	if mineErr != nil {
		fmt.Fprintf(os.Stderr, "There was error during the mining process\n")
		fmt.Fprintf(os.Stderr, "%v\n", mineErr)
		code++
	} else {
		errors.Logf("INFO", "Done!")
	}
	return code
}