// ToAlbum converts APIAlbum to Album type.
func (apiAlbum *APIAlbum) ToAlbum() *Album {
	album := NewAlbum()
	album.Id = apiAlbum.Id
	keyArr := apiAlbum.Url.FindAllStringSubmatch(`.+/(.+?)\.html`, 1)
	if len(keyArr) > 0 {
		album.Key = keyArr[0][1]
	}
	album.Title = apiAlbum.Title
	album.Artists = apiAlbum.Artists.Split(" ft ").FilterEmpty()
	album.Plays = apiAlbum.Plays
	songids := dna.IntArray{}
	for _, song := range apiAlbum.SongList {
		songids.Push(song.Id)
	}
	album.Songids = songids
	album.Nsongs = dna.Int(len(apiAlbum.SongList))
	album.Description = ""
	album.Coverart = apiAlbum.Coverart
	dateCreatedArr := album.Coverart.FindAllStringSubmatch(`/?(\d{4}/\d{2}/\d{2})`, -1)
	if len(dateCreatedArr) > 0 {
		year := dateCreatedArr[0][1].FindAllStringSubmatch(`(\d{4})/\d{2}/\d{2}`, -1)[0][1].ToInt()
		month := dateCreatedArr[0][1].FindAllStringSubmatch(`\d{4}/(\d{2})/\d{2}`, -1)[0][1].ToInt()
		day := dateCreatedArr[0][1].FindAllStringSubmatch(`\d{4}/\d{2}/(\d{2})`, -1)[0][1].ToInt()
		album.DateCreated = time.Date(int(year), time.Month(month), int(day), 0, 0, 0, 0, time.UTC)

	}
	album.Checktime = time.Now()
	return album
}

func (sf *APISongFreaksAlbum) ToAlbum() (*Album, error) {
	if sferr := sf.HasError(); sferr != nil {
		return nil, sferr
	}
	album := NewAlbum()
	album.Id = sf.Album.Id
	album.AMG = sf.Album.AMG
	album.UrlSlug = sf.Album.UrlSlug
	album.Year = sf.Album.Year
	album.Coverart = sf.Album.Coverart
	album.CoverartLarge = sf.Album.CoverartLarge
	album.Title = sf.Album.Title
	album.Ratings = dna.IntArray{sf.Album.Rating.AverageRating.ToInt(), sf.Album.Rating.UserRating, sf.Album.Rating.TotalRatings}
	album.Artistid = sf.Album.Artist.Id
	album.Artists = sf.Album.Artist.Name.ToStringArray()
	album.Link = sf.Album.Link
	songids := dna.IntArray{}
	for _, track := range sf.Album.Tracks {
		songids.Push(track.Id)
	}
	album.Songids = songids
	album.ReviewAuthor = sf.Album.Review.Author
	album.Review = sf.Album.Review.Value.DecodeHTML()
	album.Checktime = time.Now()
	return album, nil
}

// SelectMissingIds accepts a table name as an input and a list of ids as a source.
// It returns a new list of ids that does not exist in the destination table
//
// 	* tblName : a table name
// 	* srcIds : a source ids
// 	* db : a pointer to connected databased
// 	* Returns a new list of ids which are not from the specified table
//
// The format of sql statement is:
// 	WITH dna (id) AS (VALUES (5),(6),(7),(8),(9))
// 	SELECT id FROM dna WHERE NOT EXISTS
// 	(SELECT 1 from ziartists WHERE id=dna.id)
func SelectMissingIds(tblName dna.String, srcIds *dna.IntArray, db *sqlpg.DB) (*dna.IntArray, error) {

	if srcIds.Length() > 0 {

		val := dna.StringArray(srcIds.Map(func(val dna.Int, idx dna.Int) dna.String {
			return "(" + val.ToString() + ")"
		}).([]dna.String))
		selectStmt := "with dna (id) as (values " + val.Join(",") + ") \n"
		selectStmt += "SELECT id FROM dna WHERE NOT EXISTS\n (SELECT 1 from " + tblName + "  WHERE id=dna.id)"
		ids := &[]dna.Int{}
		err := db.Select(ids, selectStmt)
		switch {
		case err != nil:
			return nil, err
		case err == nil && ids != nil:
			slice := dna.IntArray(*ids)
			return &slice, nil
		case err == nil && ids == nil:
			return &dna.IntArray{}, nil
		default:
			panic("Default case triggered. Case is not expected. Cannot select non existed ids")
		}
	} else {
		return nil, errors.New("Empty input array")
	}

}

func amGetErrIds(inputFile dna.String, mode dna.Int) *dna.IntArray {
	var ret = dna.IntArray{}
	b, err := ioutil.ReadFile(inputFile.String())
	if err != nil {
		panic(err)
	}
	data := dna.String(string(b))
	lines := data.Split("\n")
	for _, line := range lines {
		switch mode {
		case 1:
			idArr := line.FindAllStringSubmatch(`([0-9]+) Post.+no such host`, 1)
			if len(idArr) > 0 {
				ret.Push(idArr[0][1].ToInt())
			}
			idArr = line.FindAllStringSubmatch(`Timeout.+at id :([0-9]+)`, 1)
			if len(idArr) > 0 {
				ret.Push(idArr[0][1].ToInt())
			}
		case 2:
			ret.Push(line.ToInt())
		}
	}
	if mode == 1 {
		err = ioutil.WriteFile(inputFile.String(), []byte{}, 0644)
		if err != nil {
			dna.Log("Cannot write to file1:", err.Error())
		}

	}
	ret = ret.Unique()
	return &ret
}

func (apiAlbum *APIAlbum) FillAlbum(album *Album) {
	var (
		nSongs, nVideos dna.Int = 0, 0
	)
	album.Id = apiAlbum.Id
	album.Key = apiAlbum.Key
	album.Title = apiAlbum.Title
	album.Artists = apiAlbum.Artist.Split(", ")
	album.Topics = apiAlbum.Genre.ToStringArray()
	album.Likes = apiAlbum.Likes
	album.Plays = apiAlbum.Plays
	ids := dna.IntArray{}
	for _, song := range apiAlbum.Listsong {
		if isSongFormat(song.StreamUrl) == true {
			nSongs += 1
		}
		if isVideoFormat(song.StreamUrl) == true {
			nVideos += 1
		}
		ids.Push(song.Id)
	}
	// dna.Log("NSONGS AND VIDEOS:", nSongs, nVideos)
	album.Songids = ids
	album.Nsongs = dna.Int(len(apiAlbum.Listsong))
	album.Description = apiAlbum.Description
	album.Coverart = apiAlbum.Coverart
	album.LinkShare = apiAlbum.Linkshare
	switch {
	case nSongs == album.Nsongs:
		album.Type = "PLAYLIST_SONG"
	case nVideos == album.Nsongs:
		album.Type = "PLAYLIST_VIDEO"
	case nVideos+nSongs == album.Nsongs:
		album.Type = "PLAYLIST_MIXED"
	default:
		album.Type = "PLAYLIST"
	}
	datecreatedArr := album.Coverart.FindAllStringSubmatch(`/([0-9]+)[_500]*\..+$`, -1)
	if len(datecreatedArr) > 0 {
		// Log(int64(datecreatedArr[0][1].ToInt()))
		album.DateCreated = time.Unix(int64(datecreatedArr[0][1].ToInt()/1000), 0)
	} else {
		dateCreatedArr := album.Coverart.FindAllStringSubmatch(`/?(\d{4}/\d{2}/\d{2})`, -1)
		if len(dateCreatedArr) > 0 {
			year := dateCreatedArr[0][1].FindAllStringSubmatch(`(\d{4})/\d{2}/\d{2}`, -1)[0][1].ToInt()
			month := dateCreatedArr[0][1].FindAllStringSubmatch(`\d{4}/(\d{2})/\d{2}`, -1)[0][1].ToInt()
			day := dateCreatedArr[0][1].FindAllStringSubmatch(`\d{4}/\d{2}/(\d{2})`, -1)[0][1].ToInt()
			album.DateCreated = time.Date(int(year), time.Month(month), int(day), 0, 0, 0, 0, time.UTC)

		}
	}
	album.Checktime = time.Now()
}

// DecodeEncodedKey decodes an encoded key to a key
func DecodeEncodedKey(key dna.String) dna.String {
	var y dna.IntArray = key[5:15].Split("").Map(func(v dna.String, i dna.Int) dna.Int {
		for j, val := range c {
			for _, char := range val.Split("") {
				if char == v {
					return dna.Int(j)
				}
			}
		}
		return -1
	}).([]dna.Int)
	return GetKey(y.ToString().ToInt() + 307843200)
}

func convertSongToIntArray(songStr dna.String) dna.IntArray {
	var apiSongs []APISong
	var ret = dna.IntArray{}
	err := json.Unmarshal([]byte(string(songStr)), &apiSongs)
	if err != nil {
		panic("Invalid song string input")
	} else {
		for _, song := range apiSongs {
			ret.Push(song.Id)
		}
	}
	return ret
}

func (sf *APISongFreaksTrack) ToSong() (*Song, error) {
	if sferr := sf.HasError(); sferr != nil {
		return nil, sferr
	}
	song := NewSong()
	song.Id = sf.Track.Id
	song.TrackGroupId = sf.Track.TrackGroupId
	song.AMG = sf.Track.AMG
	song.UrlSlug = sf.Track.UrlSlug
	song.IsInstrumental = sf.Track.IsInstrumental
	song.Viewable = sf.Track.Viewable
	song.Duration = utils.ToSeconds(sf.Track.Duration)
	song.Lyricid = sf.Track.LyricId
	song.HasLrc = sf.Track.HasLrc
	song.TrackNumber = sf.Track.TrackNumber
	song.DiscNumber = sf.Track.DiscNumber
	song.Title = sf.Track.Title
	song.Rating = dna.IntArray{sf.Track.Rating.AverageRating.ToInt(), sf.Track.Rating.UserRating, sf.Track.Rating.TotalRatings}
	song.Albumid = sf.Track.Album.Id

	artistIds := dna.IntArray{}
	artists := dna.StringArray{}
	for _, artist := range sf.Track.Artists {
		artistIds.Push(artist.Id)
		artists.Push(artist.Name)
	}
	song.Artistids = artistIds
	song.Artists = artists

	if sf.Track.Lrc.Lines != nil && len(sf.Track.Lrc.Lines) > 0 {
		lines, err := json.Marshal(sf.Track.Lrc.Lines)
		if err == nil {
			song.Lrc = dna.String(string(lines))

		}
	}

	song.Link = sf.Track.Link
	song.Lyric = sf.Track.Lyrics
	if song.Lyric != "" {
		song.HasLyric = true
	}
	song.Copyright = sf.Track.Copyright
	song.Writer = sf.Track.Writer
	song.SubmittedLyric = sf.Track.SubmittedLyric
	song.Checktime = time.Now()
	return song, nil
}

func GetNewEpisodeKeys(movieid, currentEps dna.Int) (dna.IntArray, error) {
	movie, err := GetMovie(movieid)
	// Reset hdv.EpisodeKeyList
	EpisodeKeyList = dna.IntArray{}
	if err == nil {
		if movie.CurrentEps > currentEps {
			var ret = dna.IntArray{}
			LastestMovieCurrentEps[movieid] = movie.CurrentEps
			for i := currentEps + 1; i <= movie.CurrentEps; i++ {
				ret.Push(ToEpisodeKey(movieid, i))
			}
			return ret, nil
		} else {
			return nil, errors.New(dna.Sprintf("Ep ID: %v has to updated episode", movieid).String())
		}
	} else {
		return nil, err
	}
}

func (apiSong *APISong) FillSong(song *Song) {
	song.Id = apiSong.Id
	song.Title = apiSong.Title
	song.Artists = apiSong.Artist.ToStringArray()
	song.Url = apiSong.Url
	song.Lyric = apiSong.Lyric
	song.Status = apiSong.Status
	song.Gracenote = apiSong.Gracenote
	song.Publishers = apiSong.Publishers
	song.Writers = apiSong.Writers
	song.LineCount = apiSong.LineCount
	lt := dna.IntArray{}
	for _, ts := range apiSong.LineTimestamps {
		lt.Push(ts.Line)
		lt.Push(ts.Timestamp)

	}
	song.LineTimestamps = lt

}

func (apiaward *APIAward) ToAward() *Award {
	award := NewAward()
	award.Id = apiaward.Id
	award.Title = apiaward.Title
	award.Section = ""
	award.Year = apiaward.Year
	award.Chart = apiaward.Chart
	award.Peak = apiaward.Peak
	award.Type = apiaward.Type
	award.Prize = apiaward.Award
	winnerids := dna.IntArray{}
	winners := dna.StringArray{}
	for _, winner := range apiaward.Winners {
		winnerids.Push(winner.Id)
		winners.Push(winner.Name)
	}
	award.Winnerids = winnerids
	award.Winners = winners
	return award
}

func (alca *AlbumCategory) Save(db *sqlpg.DB) error {

	var last error
	var aids = dna.IntArray{}
	albums := &[]Album{}
	for _, album := range *(alca.Albums) {
		aids.Push(album.Id)
		// dna.Log(album)
	}
	query := "SELECT id, topics, genres from nsalbums WHERE id IN (" + aids.Join(",") + ")"
	// dna.Log(query)
	err := db.Select(albums, query)
	if err != nil {
		dna.Log(query, alca, *alca.Albums)
		dna.PanicError(err)
	}

	for _, album := range *(alca.Albums) {
		foundIndex := 0
		for j, anotherAlbum := range *(albums) {
			if album.Id == anotherAlbum.Id {
				foundIndex = j
			}
		}
		if foundIndex < len(*albums) {
			cat := album.Category.Concat((*albums)[foundIndex].Topics).Concat((*albums)[foundIndex].Genres).Unique()
			album.Category = cat.Filter(func(v dna.String, i dna.Int) dna.Bool {
				if v != "" {
					return true
				} else {
					return false
				}
			})

		}
		last = db.Update(album, "id", "category")

	}

	return last
}

func (apisong *APISong) ToSong() *Song {
	song := NewSong()
	song.Id = apisong.Id
	song.Title = apisong.Title.DecodeHTML()
	song.Duration = apisong.Duration

	artistids := dna.IntArray{}
	artists := dna.StringArray{}
	for _, artist := range apisong.Artists {
		artistids.Push(artist.Id)
		artists.Push(artist.Name)
	}
	song.Artistids = artistids
	song.Artists = artists

	composerids := dna.IntArray{}
	composers := dna.StringArray{}
	for _, composer := range apisong.Composers {
		composerids.Push(composer.Id)
		composers.Push(composer.Name)
	}
	song.Composerids = composerids
	song.Composers = composers
	song.Checktime = time.Now()

	return song
}

func (apiSong *APIFullSong) FillSong(song *Song) {
	song.Id = apiSong.Track.Id
	song.Title = apiSong.Track.Title

	artists := dna.StringArray{}
	for _, artist := range apiSong.Track.Artists {
		artists.Push(artist.Name)
	}
	song.Artists = artists
	artistids := dna.IntArray{}
	for _, artist := range apiSong.Track.Artists {
		artistids.Push(artist.Id)
	}

	song.Artistids = artistids
	song.Albumid = apiSong.Track.Album.Id
	song.AlbumTitle = apiSong.Track.Album.Title
	song.Duration = utils.ToSeconds(apiSong.Track.Duration)
	song.Instrumental = apiSong.Track.Instrumental
	song.Viewable = apiSong.Track.Viewable
	song.HasLrc = apiSong.Track.HasLrc
	song.LrcVerified = apiSong.Track.LrcVerified
	song.Lyricid = apiSong.Track.Lyricid
	song.Lyric = apiSong.Track.Lyrics
	song.Copyright = apiSong.Track.Copyright
	song.Writers = apiSong.Track.Writer
	// Mon Jan 2 15:04:05 MST 2006
	if apiSong.Track.LastUpdate != "" {
		lastUpdate, err := time.Parse("2006-01-02 15:04:05", apiSong.Track.LastUpdate.String())
		if err == nil {
			song.DateUpdated = lastUpdate
		} else {
			dna.Log(err.Error(), " Song ID:", song.Id, " GOT:", apiSong.Track.LastUpdate, "\n\n")
		}
	}
}

func getCategory(songs *[]*Song, genre Genre, page dna.Int) <-chan bool {
	channel := make(chan bool, 1)
	go func() {
		link := "http://nhacso.net/bai-hat-theo-the-loai-" + genre.Id.ToString() + "/joke-link-2-" + page.ToString() + ".html"
		// dna.Log(link)
		result, err := http.Get(link)
		if err == nil {
			data := &result.Data

			// transform string {"2":[0,3,5,7,9,11,13,15,29],"10":[1,2,4,6,8,]}
			// to  map[dna.Int]dna.Int{20:2, 28:2, 4:10, 12:10} Ex: map[29] = 2
			temp := data.FindAllStringSubmatch(`getCategory.+'(\{.+\})'`, -1)
			mapping := map[dna.Int]dna.Int{}
			if len(temp) > 0 && temp[0].Length() > 0 {
				vals := temp[0][1].FindAllString(`"[0-9]+":\[[0-9,]+?\]`, -1)
				if vals.Length() > 0 {
					for _, val := range vals {
						target := val.FindAllStringSubmatch(`"(.+)"`, -1)[0][1].ToInt()
						arr := val.FindAllStringSubmatch(`\[(.+)\]`, -1)[0][1].Split(",").ToIntArray()
						for _, itm := range arr {
							mapping[itm] = target
						}
					}
				}
			}
			// Finding cat id for each song. cats var is 2-dimentional array.
			// Each index of it represents the correspondent song, its value is the categories the song belongs to
			catStrings := data.FindAllString(`Thể loại :.+`, -1)
			cats := []dna.IntArray{}
			for _, val := range catStrings {
				tagids := dna.IntArray{}
				tmp := val.FindAllStringSubmatch(`cate_tag_song_([0-9]+)`, -1)
				if len(tmp) > 0 {
					for _, el := range tmp {
						tagids.Push(el[1].ToInt())
					}
				}
				cats = append(cats, tagids)
			}
			// Log(cats)

			// get songids
			temps := data.FindAllStringSubmatch(`play" id="blocksongtag_([0-9]+)`, -1)
			songids := dna.IntArray{}
			if len(temps) > 0 {
				for _, val := range temps {
					songids.Push(val[1].ToInt())
				}
			}

			tmpsongs := &[]*Song{}
			for idx, songid := range songids {
				song := NewSong()
				song.Id = songid
				category := dna.StringArray{}
				for _, val := range cats[idx] {
					if mapping[val] > 0 && mapping[val] < CatTags.Length() {
						if CatTags[mapping[val]] != "" {
							category.Push(CatTags[mapping[val]])
						}
					} else {
						mess := dna.Sprintf("WRONG INDEX AT CATTAGS: %v %v %v - %v", mapping[val], genre, page, link)
						panic(mess.String())
					}

				}
				category.Push(genre.Name)
				song.Category = transformCats(category.Unique()).Unique()
				*tmpsongs = append(*tmpsongs, song)
			}
			*songs = *tmpsongs

		}
		channel <- true

	}()
	return channel
}

// UpdateHDViet gets lastest items from hdviet.com.
//
// The update process goes through 8 steps and 2 parts:
//
// 	PART I: FINDING AND UPDATING NEW EPS OF MOVIES IN DB
// 	Step 1: Initalizing db connection, loading site config and state handler.
// 	Step 2: Finding all movies possibly having new episodes in DB
// 	Step 3: Finding all movies found in Step 2 actually having new episodes available from source website.
// 	Step 4: Checking consitency of movies' new found episodes from the source.
// 	Step 5: Getting new episode data from the source, save them to DB and update current_eps field of movies having new eps.
//
// 	PART II: GETTING NEW MOVIES FROM HDVIET SITE.
// 	Step 6: Find newest movies from the source.
// 	Step 7: Updating newest episodes from newest found movies in Step 6.
// 	Step 8: Recovering failed sql statements.
func UpdateHDViet() {
	var mvTable = dna.String("hdvmovies")
	db, err := sqlpg.Connect(sqlpg.NewSQLConfig(SqlConfigPath))
	dna.PanicError(err)
	siteConf, err := LoadSiteConfig("hdv", SiteConfigPath)
	dna.PanicError(err)
	TIMEOUT_SECS = 100

	// PART 1: UPDATING NEW EPISODES OF MOVIES IN DB
	// STEP 2: Finding all movies possibly having new episodes in DB.
	movieCurrentEps, err := hdv.GetMoviesCurrentEps(db, mvTable)
	dna.PanicError(err)
	newEpisodeKeys := dna.IntArray{}
	for movieid, currentEps := range movieCurrentEps {
		newEpisodeKeys.Push(hdv.ToEpisodeKey(movieid, currentEps))
	}
	newEpisodeKeys.Sort()

	// STEP 3: Checking and getting new episodes if available from source website.
	state := NewStateHandlerWithExtSlice(new(hdv.EpUpdater), &newEpisodeKeys, siteConf, db)
	Update(state)

	// STEP 4:Checking consitency of new found episodes of movies from the source.
	movieIdList := dna.IntArray{}
	for _, epKey := range hdv.LastestEpisodeKeyList {
		mvid, _ := hdv.ToMovieIdAndEpisodeId(epKey)
		movieIdList.Push(mvid)
	}
	movieIdList = movieIdList.Unique()
	dna.Log("\nNumber of movies: ", movieIdList.Length(), "having new episodes:", hdv.LastestEpisodeKeyList.Length())

	if movieIdList.Length() != dna.Int(len(hdv.LastestMovieCurrentEps)) {
		dna.Log("LastestEpisodeKeyList & LastestMovieCurrentEps do not match! GOT:", movieIdList.Length(), len(hdv.LastestMovieCurrentEps))
	}

	// STEP 5: Getting new episode data from the source, save them to DB and update current_eps field of movies having new eps.
	if hdv.LastestEpisodeKeyList.Length() > 0 {
		state = NewStateHandlerWithExtSlice(new(hdv.Episode), &hdv.LastestEpisodeKeyList, siteConf, db)
		Update(state)
		RecoverErrorQueries(SqlErrorLogPath, db)
		// dna.Log(hdv.LastestEpisodeKeyList)

		hdv.SaveLastestMovieCurrentEps(db, mvTable, SQLERROR)
		RecoverErrorQueries(SqlErrorLogPath, db)
	} else {
		dna.Log("No new episodes found. Update operation has been aborted!")
	}

	// PART 2: UPDATING NEW MOVIES FROM hdv site.
	// STEP 6: Find newest movies from sources.
	db.Ping()
	state = NewStateHandler(new(hdv.Movie), siteConf, db)
	Update(state)

	// STEP 7: Updating newest episodes from newest found movies in Step 6.
	hdv.EpisodeKeyList = hdv.EpisodeKeyList.Unique()
	if hdv.EpisodeKeyList.Length() > 0 {
		state = NewStateHandlerWithExtSlice(new(hdv.Episode), &hdv.EpisodeKeyList, siteConf, db)
		Update(state)
	} else {
		dna.Log("No new movies found")
	}

	// STEP 8: Recovering failed sql statements.
	RecoverErrorQueries(SqlErrorLogPath, db)

	CountDown(3*time.Second, QuittingMessage, EndingMessage)
	db.Close()
}