func main() {
	filecsv := `C:\Users\yejianfeng\Desktop\mxm\skill.csv`
	file, err := os.Open(filecsv)
	if err != nil {
		panic(err)
	}
	defer file.Close()

	reader := csv.NewReader(file)
	roleFolder := `C:\Users\yejianfeng\Desktop\mxm\skill\`
	reader.Read()
	for {
		fields, err := reader.Read()
		if err == io.EOF {
			break
		}

		picurl := fields[15]

		jpg := roleFolder + picurl + ".jpg"
		_, err = os.Open(jpg)
		if err == nil || os.IsExist(err) {
			continue
		}

		png := roleFolder + picurl + ".png"
		_, err = os.Open(png)
		if err == nil || os.IsExist(err) {
			continue
		}

		fmt.Println(fields[0])
	}
}

func loadCsvFile(name, urlpath string, timestamp time.Time, force bool) (*csv.Reader, error) {
	filename := filepath.Join(csvDir, name+timestamp.Format(".2006-01-02")+".csv")
	if !force {
		file, err := os.Open(filename)
		if err != nil {
			if !os.IsNotExist(err) {
				return nil, err
			}
		} else {
			return csv.NewReader(file), nil
		}
	}
	log.Info("Downloading %s.csv from Google Spreadsheets", name)
	resp, err := http.Get(csvUrlPrefix + urlpath + csvUrlSuffix)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	err = ioutil.WriteFile(filename, body, 0644)
	if err != nil {
		return nil, err
	}
	return csv.NewReader(bytes.NewBuffer(body)), nil
}

// Get return csv data in array.
func (d *Data) Get() ([][]string, error) {
	if len(d.UnixMapData[d.Date.Unix()]) == 0 {
		data, err := hCache.Get(d.URL(), true)
		if err != nil {
			return nil, fmt.Errorf(errorNetworkFail.Error(), err)
		}
		csvArrayContent := strings.Split(string(data), "\n")
		for i := range csvArrayContent {
			csvArrayContent[i] = strings.TrimSpace(csvArrayContent[i])
		}
		var csvReader *csv.Reader
		if (d.exchange == "tse" && len(csvArrayContent) > 2) || (d.exchange == "otc" && len(csvArrayContent) > 5) {
			if d.exchange == "tse" {
				if d.Name == "" {
					d.Name = strings.Split(csvArrayContent[0], " ")[2]
				}
				csvReader = csv.NewReader(strings.NewReader(strings.Join(csvArrayContent[2:], "\n")))
			} else if d.exchange == "otc" {
				if d.Name == "" {
					d.Name = strings.Split(csvArrayContent[2], ":")[1]
				}
				csvReader = csv.NewReader(strings.NewReader(strings.Join(csvArrayContent[5:len(csvArrayContent)-1], "\n")))
			}
			allData, err := csvReader.ReadAll()
			d.RawData = append(allData, d.RawData...)
			d.UnixMapData[d.Date.Unix()] = allData
			d.clearCache()
			return allData, err
		}
		return nil, errorNotEnoughData
	}
	return d.UnixMapData[d.Date.Unix()], nil
}

func parseFlags() (*csv.Reader, *csv.Reader, error) {
	flag.Parse()
	if *dataFile == "" {
		return nil, nil, fmt.Errorf("probs flag required")
	}

	if *remFile == "" {
		return nil, nil, fmt.Errorf("remaining flag required")
	}

	if *weekNumber < 1 || *weekNumber > 13 {
		return nil, nil, fmt.Errorf("week number must be specified and must be in the range [1,13]")
	}

	csvFile, err := os.Open(*dataFile)
	if err != nil {
		return nil, nil, err
	}
	pReader := csv.NewReader(csvFile)

	csvFile2, err := os.Open(*remFile)
	if err != nil {
		return pReader, nil, err
	}
	rReader := csv.NewReader(csvFile2)

	return pReader, rReader, nil
}

func Example() {
	// testFile is a CSV file with CR line endings.
	testFile := bytes.NewBufferString("a,b,c\r1,2,3\r").Bytes()

	// First try reading the csv file the normal way.
	// The CSV reader doesn't recognize the '\r' line ending.
	r1 := csv.NewReader(bytes.NewReader(testFile))
	lines1, err := r1.ReadAll()
	if err != nil {
		fmt.Println(err)
	}
	fmt.Printf("Without macreader: %#v\n", lines1)

	// Now try reading the csv file using macreader.
	// It should work as expected.
	r2 := csv.NewReader(New(bytes.NewReader(testFile)))
	lines2, err := r2.ReadAll()
	if err != nil {
		fmt.Println(err)
	}
	fmt.Printf("With macreader: %#v\n", lines2)

	// Output: Without macreader: [][]string{[]string{"a", "b", "c\r1", "2", "3"}}
	// With macreader: [][]string{[]string{"a", "b", "c"}, []string{"1", "2", "3"}}

}

func readCsv(ch chan []string) {
	var reader *csv.Reader
	if inputFn == "" {
		reader = csv.NewReader(os.Stdin)
	} else {
		file, err := os.Open(inputFn)
		if err != nil {
			fmt.Println("Error:", err)
			os.Exit(1)
		}
		defer file.Close()
		reader = csv.NewReader(file)
	}
	if !strictLen {
		reader.FieldsPerRecord = -1
	}
	r, _ := utf8.DecodeRuneInString(inputSep)
	reader.Comma = r
	reader.LazyQuotes = lazyQuotes

	for {
		record, err := reader.Read()
		if err == io.EOF {
			close(ch)
			break
		} else if err != nil {
			fmt.Println("Error:", err)
			close(ch)
			break
		}
		ch <- record
	}
}

//Read read csv for handle
func ReadLines(file string, isGbk bool) (lines [][]string, err error) {
	//catch panic
	defer func() {
		if rerr := recover(); rerr != nil {
			err = errors.New(fmt.Sprintf("read csv file: %v, error: %v", file, rerr))
		}
	}()

	//open file
	fi, err := os.Open(file)
	if err != nil {
		return nil, err
	}
	defer fi.Close()
	//get reader
	var reader *csv.Reader
	if !isGbk {
		reader = csv.NewReader(fi)
	} else {
		//transform gbk to utf8
		r := transform.NewReader(fi, simplifiedchinese.GBK.NewDecoder())
		reader = csv.NewReader(r)
	}
	lines, err = reader.ReadAll()
	return
}

func TestWriteCSV(t *testing.T) {
	testutil.VerifyTestType(t, testutil.UnitTestType)

	Convey("With a CSV export output", t, func() {
		fields := []string{"_id", "x", " y", "z.1.a"}
		out := &bytes.Buffer{}

		Convey("Headers should be written correctly", func() {
			csvExporter := NewCSVExportOutput(fields, false, out)
			err := csvExporter.WriteHeader()
			So(err, ShouldBeNil)
			csvExporter.ExportDocument(bson.D{{"_id", "12345"}})
			csvExporter.WriteFooter()
			csvExporter.Flush()
			rec, err := csv.NewReader(strings.NewReader(out.String())).Read()
			So(err, ShouldBeNil)
			So(rec, ShouldResemble, []string{"_id", "x", " y", "z.1.a"})
		})

		Convey("Headers should not be written", func() {
			csvExporter := NewCSVExportOutput(fields, true, out)
			err := csvExporter.WriteHeader()
			So(err, ShouldBeNil)
			csvExporter.ExportDocument(bson.D{{"_id", "12345"}})
			csvExporter.WriteFooter()
			csvExporter.Flush()
			rec, err := csv.NewReader(strings.NewReader(out.String())).Read()
			So(err, ShouldBeNil)
			So(rec, ShouldResemble, []string{"12345", "", "", ""})
		})

		Convey("Exported document with missing fields should print as blank", func() {
			csvExporter := NewCSVExportOutput(fields, true, out)
			csvExporter.ExportDocument(bson.D{{"_id", "12345"}})
			csvExporter.WriteFooter()
			csvExporter.Flush()
			rec, err := csv.NewReader(strings.NewReader(out.String())).Read()
			So(err, ShouldBeNil)
			So(rec, ShouldResemble, []string{"12345", "", "", ""})
		})

		Convey("Exported document with index into nested objects should print correctly", func() {
			csvExporter := NewCSVExportOutput(fields, true, out)
			z := []interface{}{"x", bson.D{{"a", "T"}, {"B", 1}}}
			csvExporter.ExportDocument(bson.D{{Name: "z", Value: z}})
			csvExporter.WriteFooter()
			csvExporter.Flush()
			rec, err := csv.NewReader(strings.NewReader(out.String())).Read()
			So(err, ShouldBeNil)
			So(rec, ShouldResemble, []string{"", "", "", "T"})
		})

		Reset(func() {
			out.Reset()
		})

	})
}

// Get is to get TWSE csv data.
func (l *Lists) Get(category string) ([][]string, error) {
	if TWSECLASS[category] == "" {
		return nil, errorNotSupport
	}

	year, month, day := l.Date.Date()
	data, err := hCache.PostForm(fmt.Sprintf("%s%s", utils.TWSEHOST, utils.TWSELISTCSV),
		url.Values{"download": {"csv"}, "selectType": {category},
			"qdate": {fmt.Sprintf("%d/%02d/%02d", year-1911, month, day)}})

	if err != nil {
		return nil, fmt.Errorf(errorNetworkFail.Error(), err)
	}

	csvArrayContent := strings.Split(string(data), "\n")

	var csvReader *csv.Reader
	switch category {
	case "MS":
		if len(csvArrayContent) > 6 {
			csvReader = csv.NewReader(strings.NewReader(strings.Join(csvArrayContent[4:51], "\n")))
		}
	case "ALLBUT0999", "ALL":
		if len(csvArrayContent) > 155 {
			re := regexp.MustCompile("^=?[\"]{1}[0-9A-Z]{4,}")
			var pickdata []string
			for _, v := range csvArrayContent {
				if re.MatchString(v) {
					if v[0] == 61 {
						pickdata = append(pickdata, v[1:])
					} else {
						pickdata = append(pickdata, v)
					}
				}
			}
			csvReader = csv.NewReader(strings.NewReader(strings.Join(pickdata, "\n")))
		}
	default:
		if len(csvArrayContent) > 9 {
			csvReader = csv.NewReader(strings.NewReader(strings.Join(csvArrayContent[4:len(csvArrayContent)-7], "\n")))
		}
	}
	if csvReader != nil {
		returnData, err := csvReader.ReadAll()
		switch category {
		default:
			if err == nil {
				l.categoryRawData[category] = returnData
				l.formatData(category)
			}
		case "MS":
		}
		return returnData, err
	}
	return nil, errorNotEnoughData
}

func main() {

	ht := makeSimpleHashTable(16, false)
	// ht := makeBackgroundHashTable(16)
	// ht := makeMasterHashTable(16)

	// Insertion
	insert_file, _ := os.Open("tests/insert.csv")
	insert_reader := csv.NewReader(insert_file)
	insert_times := make([]time.Duration, 0)

	for i := 0; ; i++ {
		data, err := insert_reader.Read()
		if err != nil {
			break
		}

		key, _ := strconv.ParseInt(data[1], 0, 0)
		val := data[2]
		start := time.Now()
		ht.insert(int(key), val)
		insert_times = Extend(insert_times, time.Now().Sub(start))
	}

	insert_out, _ := os.Create("data/simple4.csv")
	for _, datum := range insert_times {
		insert_out.Write([]byte(datum.String() + "\n"))
	}

	// Get
	get_file, _ := os.Open("tests/get.csv")
	get_reader := csv.NewReader(get_file)
	get_times := make([]time.Duration, 0)
	get_start := time.Now()

	for i := 0; ; i++ {
		data, err := get_reader.Read()
		if err != nil {
			break
		}

		key, _ := strconv.ParseInt(data[1], 0, 0)
		start := time.Now()
		ht.get(int(key))
		get_times = Extend(get_times, time.Now().Sub(start))
	}
	fmt.Println("Elapsed time", time.Now().Sub(get_start))

	get_out, _ := os.Create("data/simple_get4.csv")
	for _, datum := range get_times {
		get_out.Write([]byte(datum.String() + "\n"))
	}

}

func LoadUsers(root string) OSUsers {
	var users OSUsers

	shadowf, serr := os.Open(path.Join(root, "etc", "shadow"))
	passwdf, perr := os.Open(path.Join(root, "etc", "passwd"))
	defer shadowf.Close()
	defer passwdf.Close()

	var shadowentries [][]string

	if serr != nil || perr != nil {
		return users
	}

	shadowreader := csv.NewReader(shadowf)
	shadowreader.Comma = ':'

	for {
		record, err := shadowreader.Read()
		if err != nil {
			if err == io.EOF {
				break
			}
			log.Fatal(err)
		}

		shadowentries = append(shadowentries, record)
	}

	passwdreader := csv.NewReader(passwdf)
	passwdreader.Comma = ':'

	for {
		record, err := passwdreader.Read()
		if err != nil {
			if err == io.EOF {
				break
			}
			log.Fatal(err)
		}

		for _, shadowentry := range shadowentries {
			if shadowentry[0] == record[0] {
				found_user := new(OSUser)
				found_user.shadowEntry = shadowentry
				found_user.passwdEntry = record
				users = append(users, found_user)
				break
			}
		}
	}

	return users
}

func (r *Reader) setupCsvReader(reader io.Reader, dialect *FileDialect) error {
	if dialect.Encoding == "sjis" {
		r.logger.Info("use ShiftJIS decoder for input.")
		decoder := japanese.ShiftJIS.NewDecoder()
		r.csvReader = csv.NewReader(transform.NewReader(reader, decoder))
	} else {
		r.csvReader = csv.NewReader(reader)
	}
	r.csvReader.Comma = dialect.Comma
	r.csvReader.Comment = dialect.Comment
	r.csvReader.FieldsPerRecord = dialect.FieldsPerRecord
	return nil
}

// Takes two csv files and starts the diffing operation. Goes through revisionX
// comparing each row against every row in revisionY and then repeats the
// process in reverse.
func csvDiff(fileX, fileY io.Reader) (err error) {
	revisionXRows, err := csv.NewReader(fileX).ReadAll()
	revisionYRows, err := csv.NewReader(fileY).ReadAll()

	for _, revisionXRow := range revisionXRows {
		compareRowAgainstRows(revisionXRow, revisionYRows, true)
	}

	for _, revisionYRow := range revisionYRows {
		compareRowAgainstRows(revisionYRow, revisionXRows, false)
	}

	return
}

// Load data from CSV files in order to detect gender. If new files are being used, call this again.
func NewGenderData(femaleFilename string, maleFilename string) {
	femaleNamesFile, err := os.Open(femaleFilename)
	if err != nil {
		// err is printable
		// elements passed are separated by space automatically
		log.Println("Error:", err)
	}
	// automatically call Close() at the end of current method
	defer femaleNamesFile.Close()
	femaleReader := csv.NewReader(femaleNamesFile)

	femaleReader.Comma = ','
	var fName UsCensusName
	for {
		err := unmarshalCensusData(femaleReader, &fName)
		if err == io.EOF {
			break
		}
		if err != nil {
			break
			//panic(err)
		}
		femaleNames = append(femaleNames, fName)
	}

	maleNamesFile, err := os.Open(maleFilename)
	if err != nil {
		// err is printable
		// elements passed are separated by space automatically
		log.Println("Error:", err)
	}
	// automatically call Close() at the end of current method
	defer maleNamesFile.Close()
	maleReader := csv.NewReader(maleNamesFile)

	maleReader.Comma = ','
	var mName UsCensusName
	for {
		err := unmarshalCensusData(maleReader, &mName)
		if err == io.EOF {
			break
		}
		if err != nil {
			//break
			panic(err)
		}
		maleNames = append(maleNames, mName)
	}

}

func Fuzz(data []byte) int {
	score := 0

	r := csv.NewReader(bytes.NewReader(data))
	r.Comment = '#'
	r.LazyQuotes = true
	r.TrimLeadingSpace = true
	rec, err := r.ReadAll()
	if err != nil {
		if rec != nil {
			panic("rec is not nil on error")
		}
	} else {
		score = 1
	}

	r = csv.NewReader(bytes.NewReader(data))
	rec, err = r.ReadAll()
	if err != nil {
		if rec != nil {
			panic("rec is not nil on error")
		}
	} else {
		score = 1
		var rec0 [][]string
		for _, r := range rec {
			if len(r) > 0 {
				rec0 = append(rec0)
			}
		}

		buf := new(bytes.Buffer)
		w := csv.NewWriter(buf)
		w.WriteAll(rec0)
		r := csv.NewReader(buf)
		rec1, err := r.ReadAll()
		if err != nil {
			panic(err)
		}
		if !fuzz.DeepEqual(rec0, rec1) {
			fmt.Printf("rec0: %+v\n", rec0)
			fmt.Printf("rec1: %+v\n", rec1)
			panic("records differ")
		}
	}

	return score
}

func main() {
	m1 := matrix.MakeDenseMatrix([]float64{1, 2, 3, 4, 5, 6}, 3, 2)
	fmt.Println(m1)
	fmt.Println(matrix.Transpose(m1))
	res, _ := http.Get("http://archive.ics.uci.edu/ml/machine-learning-databases/wine-quality/winequality-red.csv")
	defer res.Body.Close()

	reader := csv.NewReader(res.Body)
	reader.Comma = ';'
	//contents, _ := ioutil.ReadAll(res.Body)
	testData := make([][]float64, 0)
	for {
		col, err := reader.Read()
		if err == io.EOF {
			break
		}
		//strings.Split(col, ";")
		x := make([]float64, len(col))
		for i := range col {
			x[i], _ = strconv.ParseFloat(col[i], 64)
		}
		testData = append(testData, x)
	}
	ae := NewAutoEncoder()
	ae.Train(testData[1])

}

func (e *Exporter) scrape(csvRows chan<- []string) {
	defer close(csvRows)

	e.totalScrapes.Inc()

	resp, err := e.client.Get(e.URI)
	if err != nil {
		e.up.Set(0)
		log.Printf("Error while scraping HAProxy: %v", err)
		return
	}
	defer resp.Body.Close()
	e.up.Set(1)

	reader := csv.NewReader(resp.Body)
	reader.TrailingComma = true
	reader.Comment = '#'

	for {
		row, err := reader.Read()
		if err == io.EOF {
			break
		}
		if err != nil {
			log.Printf("Error while reading CSV: %v", err)
			e.csvParseFailures.Inc()
			break
		}
		if len(row) == 0 {
			continue
		}
		csvRows <- row
	}
}

func Parse(filePath string) []Record {
	src, err := os.Open(filePath)
	if err != nil {
		panic(err)
	}
	defer src.Close()

	rows, err := csv.NewReader(src).ReadAll()
	if err != nil {
		panic(err)
	}

	records := make([]Record, 0, len(rows))

	for _, row := range rows {
		date, _ := time.Parse("2006-01-02", row[0])
		open, _ := strconv.ParseFloat(row[1], 64)

		records = append(records, Record{
			Date: date,
			Open: open,
		})
	}

	return records

}

func FromLineRead(lines []int, path string) {
	File, err := os.Open(path)
	if err != nil {
		log.Println("读取csv文件失败:", err.Error())
		return
	}
	defer File.Close()
	r := csv.NewReader(File)
	r.Read()
	var list []string
	var line, index int
	for {
		list, err = r.Read()
		if err != nil {
			if err != io.EOF {
				log.Println("读取文件内容失败,错误信息:", err.Error())
			}
			break
		}
		if lines[index] == line {
			log.Println(list)
			index++
			if index >= len(lines) {
				break
			}
		}
		line++
	}
}

// Prepares the route to be used in matching.
func NewRoute(method, path, action, fixedArgs, routesPath string, line int) (r *Route) {
	// Handle fixed arguments
	argsReader := strings.NewReader(fixedArgs)
	csv := csv.NewReader(argsReader)
	fargs, err := csv.Read()
	if err != nil && err != io.EOF {
		ERROR.Printf("Invalid fixed parameters (%v): for string '%v'", err.Error(), fixedArgs)
	}

	r = &Route{
		Method:      strings.ToUpper(method),
		Path:        path,
		Action:      action,
		FixedParams: fargs,
		TreePath:    treePath(strings.ToUpper(method), path),
		routesPath:  routesPath,
		line:        line,
	}

	// URL pattern
	if !strings.HasPrefix(r.Path, "/") {
		ERROR.Print("Absolute URL required.")
		return
	}

	actionSplit := strings.Split(action, ".")
	if len(actionSplit) == 2 {
		r.ControllerName = actionSplit[0]
		r.MethodName = actionSplit[1]
	}

	return
}