package backend import ( "crypto/md5" "encoding/hex" "errors" "fmt" "io" "io/fs" "log" "music-server/internal/db" "music-server/internal/db/repository" "os" "sort" "strconv" "strings" "sync" "time" "github.com/panjf2000/ants/v2" "github.com/MShekow/directory-checksum/directory_checksum" "github.com/spf13/afero" ) var Syncing = false var foldersSynced float32 var numberOfFoldersToSync float32 var totalTime time.Duration var timeSpent time.Duration var allGames []repository.Game var gamesBeforeSync []repository.Game var gamesAfterSync []repository.Game var gamesAdded []string var gamesReAdded []string var gamesChangedTitle map[string]string var gamesChangedContent []string var gamesRemoved []string var catchedErrors []string var brokenSongs []string type SyncResponse struct { GamesAdded []string `json:"games_added"` GamesReAdded []string `json:"games_re_added"` GamesChangedTitle map[string]string `json:"games_changed_title"` GamesChangedContent []string `json:"games_changed_content"` GamesRemoved []string `json:"games_removed"` CatchedErrors []string `json:"catched_errors"` TotalTime string `json:"total_time"` } type ProgressResponse struct { Progress string `json:"progress"` TimeSpent string `json:"time_spent"` } type GameStatus int const ( NotChanged GameStatus = iota TitleChanged GameChanged NewGame ) var statusName = map[GameStatus]string{ NotChanged: "Not changed", TitleChanged: "Title changed", GameChanged: "Game changed", NewGame: "New game", } func (gs GameStatus) String() string { return statusName[gs] } func ResetDB() { repo.ClearSongs(db.Ctx) repo.ClearGames(db.Ctx) } func SyncProgress() ProgressResponse { progress := int((foldersSynced / numberOfFoldersToSync) * 100) out := time.Time{}.Add(timeSpent) fmt.Printf("\nTotal time: %v\n", timeSpent) fmt.Printf("Total time: %v\n", out.Format("15:04:05.00000")) log.Printf("Progress: %v/%v %v%%\n", int(foldersSynced), int(numberOfFoldersToSync), progress) return ProgressResponse{ Progress: fmt.Sprintf("%v", progress), TimeSpent: out.Format("15:04:05.00000"), } } func SyncResult() SyncResponse { fmt.Printf("\nGames Before: %d\n", len(gamesBeforeSync)) fmt.Printf("Games After: %d\n", len(gamesAfterSync)) fmt.Printf("\nGames added: \n") for _, game := range gamesAdded { fmt.Printf("%s\n", game) } fmt.Printf("\nGames readded: \n") for _, game := range gamesReAdded { fmt.Printf("%s\n", game) } fmt.Printf("\nGames with changed title: \n") for key, value := range gamesChangedTitle { fmt.Printf("The game: %s changed title to: %s\n", key, value) } fmt.Printf("\nGames with changed content: \n") for _, game := range gamesChangedContent { fmt.Printf("%s\n", game) } fmt.Printf("\n\n") var gamesRemovedTemp []string for _, beforeGame := range gamesBeforeSync { var found = false for _, afterGame := range gamesAfterSync { if beforeGame.GameName == afterGame.GameName { found = true fmt.Printf("Game: %s, Found: %v break\n", beforeGame.GameName, found) break } } if !found { fmt.Printf("Game: %s, Found: %v\n", beforeGame.GameName, found) gamesRemovedTemp = append(gamesRemovedTemp, beforeGame.GameName) } } for _, game := range gamesRemovedTemp { var found bool = false for key := range gamesChangedTitle { if game == key { found = true //fmt.Printf("Game: %s, Found: %v break2\n", game, found) break } } if !found { gamesRemoved = append(gamesRemoved, game) } } fmt.Printf("\nGames removed: \n") for _, game := range gamesRemoved { fmt.Printf("%s\n", game) } fmt.Printf("\nErrors catched: \n") for _, error := range catchedErrors { fmt.Printf("%s\n", error) } out := time.Time{}.Add(totalTime) fmt.Printf("\nTotal time: %v\n", totalTime) fmt.Printf("Total time: %v\n", out.Format("15:04:05.00000")) return SyncResponse{ GamesAdded: gamesAdded, GamesReAdded: gamesReAdded, GamesChangedTitle: gamesChangedTitle, GamesChangedContent: gamesChangedContent, GamesRemoved: gamesRemoved, CatchedErrors: catchedErrors, TotalTime: out.Format("15:04:05.00000"), } } func SyncGamesNewFull() { syncGamesNew(true) Reset() } func SyncGamesNewOnlyChanges() { syncGamesNew(false) Reset() } func syncGamesNew(full bool) { Syncing = true musicPath := os.Getenv("MUSIC_PATH") fmt.Printf("dir: %s\n", musicPath) if !strings.HasSuffix(musicPath, "/") { musicPath += "/" } var syncWg sync.WaitGroup initRepo() start := time.Now() foldersToSkip := []string{".sync", "dist", "old", "characters"} fmt.Println(foldersToSkip) var err error gamesAdded = nil gamesReAdded = nil gamesChangedTitle = nil gamesChangedContent = nil gamesRemoved = nil catchedErrors = nil brokenSongs = nil gamesBeforeSync, err = repo.FindAllGames(db.Ctx) handleError("FindAllGames Before", err, "") fmt.Printf("Games Before: %d\n", len(gamesBeforeSync)) allGames, err = repo.GetAllGamesIncludingDeleted(db.Ctx) handleError("GetAllGamesIncludingDeleted", err, "") err = repo.SetGameDeletionDate(db.Ctx) handleError("SetGameDeletionDate", err, "") directories, err := os.ReadDir(musicPath) if err != nil { log.Fatal(err) } pool, _ := ants.NewPool(50, ants.WithPreAlloc(true)) defer pool.Release() numberOfFoldersToSync = float32(len(directories)) syncWg.Add(int(numberOfFoldersToSync)) for _, dir := range directories { pool.Submit(func() { defer syncWg.Done() syncGameNew(dir, foldersToSkip, musicPath, full) }) } syncWg.Wait() checkBrokenSongsNew() gamesAfterSync, err = repo.FindAllGames(db.Ctx) handleError("FindAllGames After", err, "") finished := time.Now() totalTime = finished.Sub(start) out := time.Time{}.Add(totalTime) fmt.Printf("\nTotal time: %v\n", totalTime) fmt.Printf("Total time: %v\n", out.Format("15:04:05.00000")) Syncing = false } func checkBrokenSongsNew() { allSongs, err := repo.FetchAllSongs(db.Ctx) handleError("FetchAllSongs", err, "") var brokenWg sync.WaitGroup poolBroken, _ := ants.NewPool(50, ants.WithPreAlloc(true)) defer poolBroken.Release() brokenWg.Add(len(allSongs)) for _, song := range allSongs { poolBroken.Submit(func() { defer brokenWg.Done() checkBrokenSongNew(song) }) } brokenWg.Wait() err = repo.RemoveBrokenSongs(db.Ctx, brokenSongs) handleError("RemoveBrokenSongs", err, "") } func checkBrokenSongNew(song repository.Song) { //Check if file exists and open openFile, err := os.Open(song.Path) if err != nil { //File not found brokenSongs = append(brokenSongs, song.Path) fmt.Printf("song broken: %v\n", song.Path) } else { err = openFile.Close() if err != nil { log.Println(err) } } } func syncGameNew(file os.DirEntry, foldersToSkip []string, baseDir string, full bool) { if file.IsDir() && !contains(foldersToSkip, file.Name()) { fmt.Printf("Syncing: %s\n", file.Name()) gameDir := baseDir + file.Name() + "/" dirHash := getHashForDir(gameDir) var status GameStatus = NewGame var oldGame repository.Game var id int32 = -1 //fmt.Printf("Games before: %d\n", len(gamesBeforeSync)) for _, currentGame := range allGames { oldGame = currentGame //fmt.Printf("%s | %s\n", oldGame.GameName, oldGame.Hash) if oldGame.GameName == file.Name() && oldGame.Hash == dirHash { status = NotChanged id = oldGame.ID //fmt.Printf("Game not changed\n") break } else if oldGame.GameName == file.Name() && oldGame.Hash != dirHash { status = GameChanged id = oldGame.ID //fmt.Printf("Game changed\n") break } else if oldGame.GameName != file.Name() && oldGame.Hash == dirHash { status = TitleChanged id = oldGame.ID //fmt.Printf("GameName changed\n") break } } if full { status = TitleChanged } entries, err := os.ReadDir(gameDir) if err != nil { log.Println(err) } switch status { case NewGame: if id != -1 { for _, entry := range entries { fileInfo, err := entry.Info() if err != nil { log.Println(err) } id = getIdFromFileNew(fileInfo) if id != -1 { break } } err = repo.InsertGameWithExistingId(db.Ctx, repository.InsertGameWithExistingIdParams{ID: id, GameName: file.Name(), Path: gameDir, Hash: dirHash}) handleError("InsertGameWithExistingId", err, "") if err != nil { fmt.Printf("id = %v\n", id) fileName := gameDir + "/." + strconv.Itoa(int(id)) + ".id" fmt.Printf("fileName = %v\n", fileName) err := os.Remove(fileName) if err != nil { fmt.Printf("%s\n", err) } newDirHash := getHashForDir(gameDir) id = insertGameNew(file.Name(), gameDir, newDirHash) } } else { id = insertGameNew(file.Name(), gameDir, dirHash) } //fmt.Printf("\n\nID: %d | GameName: %s | GameHash: %s | Status: %s\n", id, file.Name(), dirHash, status) gamesAdded = append(gamesAdded, file.Name()) newCheckSongs(entries, gameDir, id) case GameChanged: //fmt.Printf("\n\nID: %d | GameName: %s | GameHash: %s | Status: %s\n", id, file.Name(), dirHash, status) err = repo.UpdateGameHash(db.Ctx, repository.UpdateGameHashParams{Hash: dirHash, ID: id}) handleError("UpdateGameHash", err, "") gamesChangedContent = append(gamesChangedContent, file.Name()) newCheckSongs(entries, gameDir, id) case TitleChanged: //fmt.Printf("\n\nID: %d | GameName: %s | GameHash: %s | Status: %s\n", id, file.Name(), dirHash, status) //println("TitleChanged") err = repo.UpdateGameName(db.Ctx, repository.UpdateGameNameParams{Name: file.Name(), Path: gameDir, ID: id}) handleError("UpdateGameName", err, "") newCheckSongs(entries, gameDir, id) if gamesChangedTitle == nil { gamesChangedTitle = make(map[string]string) } gamesChangedTitle[oldGame.GameName] = file.Name() case NotChanged: //println("NotChanged") var found bool = false for _, beforeGame := range gamesBeforeSync { if dirHash == beforeGame.Hash { found = true //fmt.Printf("Game %s | %s | %s | %s | %v\n", beforeGame.GameName, beforeGame.Hash, dirHash, status, beforeGame.Deleted) } } if !found { newCheckSongs(entries, gameDir, id) gamesReAdded = append(gamesReAdded, file.Name()) } } fmt.Printf("\n\nID: %d | GameName: %s | GameHash: %s | Status: %s\n", id, file.Name(), dirHash, status) err = repo.RemoveDeletionDate(db.Ctx, id) handleError("RemoveDeletionDate", err, "") } foldersSynced++ log.Printf("Progress: %v/%v %v%%\n", int(foldersSynced), int(numberOfFoldersToSync), int((foldersSynced/numberOfFoldersToSync)*100)) } func insertGameNew(name string, path string, hash string) int32 { var duplicateError = errors.New("ERROR: duplicate key value violates unique") id, err := repo.InsertGame(db.Ctx, repository.InsertGameParams{GameName: name, Path: path, Hash: hash}) handleError("InsertGame", err, "") if err != nil { fmt.Printf("Handle id busy\n") if strings.HasPrefix(err.Error(), duplicateError.Error()) { fmt.Printf("Handeling this id\n") _, err = repo.ResetGameIdSeq(db.Ctx) handleError("ResetGameIdSeq", err, "") id = insertGameNew(name, path, hash) } } return id } func newCheckSongs(entries []os.DirEntry, gameDir string, id int32) int32 { //hasher := md5.New() var numberOfSongs int32 numberOfFiles := len(entries) var songWg sync.WaitGroup poolSong, _ := ants.NewPool(numberOfFiles, ants.WithPreAlloc(true)) defer poolSong.Release() songWg.Add(numberOfFiles) for _, entry := range entries { poolSong.Submit(func() { defer songWg.Done() if newCheckSong(entry, gameDir, id) { numberOfSongs++ } }) } songWg.Wait() return numberOfSongs } func newCheckSong(entry os.DirEntry, gameDir string, id int32) bool { fileInfo, err := entry.Info() if err != nil { log.Println(err) } if isSong(fileInfo) { path := gameDir + entry.Name() songHash := getHashForFile(path) //numberOfSongs++ fileName := entry.Name() songName, _ := strings.CutSuffix(fileName, ".mp3") song, err := repo.GetSongWithHash(db.Ctx, songHash) handleError("GetSongWithHash", err, fmt.Sprintf("GameID: %d | Path: %s | SongName: %s | SongHash: %s\n", id, path, entry.Name(), songHash)) if err == nil { if song.SongName == songName && song.Path == path { return false } } fmt.Printf("Song Changed\n") fmt.Printf("Path: %s | SongHash: %s\n", path, songHash) count, err := repo.CheckSongWithHash(db.Ctx, songHash) handleError("CheckSongWithHash", err, fmt.Sprintf("GameID: %d | Path: %s | SongName: %s | SongHash: %s\n", id, path, entry.Name(), songHash)) if err != nil { count2, err := repo.CheckSong(db.Ctx, path) handleError("CheckSong", err, fmt.Sprintf("GameID: %d | Path: %s | SongName: %s | SongHash: %s\n", id, path, entry.Name(), songHash)) if count2 > 0 { err = repo.AddHashToSong(db.Ctx, repository.AddHashToSongParams{Hash: songHash, Path: path}) handleError("AddHashToSong", err, fmt.Sprintf("GameID: %d | Path: %s | SongName: %s | SongHash: %s\n", id, path, entry.Name(), songHash)) count, err = repo.CheckSongWithHash(db.Ctx, songHash) handleError("CheckSongWithHash 2", err, fmt.Sprintf("GameID: %d | Path: %s | SongName: %s | SongHash: %s\n", id, path, entry.Name(), songHash)) } } //count, _ := repo.CheckSong(ctx, path) if count > 0 { err = repo.UpdateSong(db.Ctx, repository.UpdateSongParams{SongName: songName, FileName: &fileName, Path: path, Hash: songHash}) handleError("UpdateSong", err, fmt.Sprintf("GameID: %d | Path: %s | SongName: %s | SongHash: %s\n", id, path, entry.Name(), songHash)) } else { count2, err := repo.CheckSong(db.Ctx, path) handleError("CheckSong", err, fmt.Sprintf("GameID: %d | Path: %s | SongName: %s | SongHash: %s\n", id, path, entry.Name(), songHash)) if count2 > 0 { err = repo.AddHashToSong(db.Ctx, repository.AddHashToSongParams{Hash: songHash, Path: path}) handleError("AddHashToSong", err, fmt.Sprintf("GameID: %d | Path: %s | SongName: %s | SongHash: %s\n", id, path, entry.Name(), songHash)) } else { err = repo.AddSong(db.Ctx, repository.AddSongParams{GameID: id, SongName: songName, Path: path, FileName: &fileName, Hash: songHash}) handleError("AddSong", err, fmt.Sprintf("GameID: %d | Path: %s | SongName: %s | SongHash: %s\n", id, path, entry.Name(), songHash)) } } return true } else if isCoverImage(fileInfo) { //TODO: Later add cover art image here in db } return false } func handleError(funcName string, err error, msg string) { var compareError = errors.New("no rows in result set") if err != nil { if compareError.Error() != err.Error() { fmt.Printf("%s Error: %s\n", funcName, err) if msg != "" { fmt.Printf("%s\n", msg) catchedErrors = append(catchedErrors, fmt.Sprintf("Func: %s\nError message: %s\nDebug message: %s\n", funcName, err, msg)) } else { catchedErrors = append(catchedErrors, fmt.Sprintf("Func: %s\nError message: %s\n", funcName, err)) } } } } func getHashForDir(gameDir string) string { directory, _ := directory_checksum.ScanDirectory(gameDir, afero.NewOsFs()) hash, _ := directory.ComputeDirectoryChecksums() //fmt.Printf("Hash: |%s|\n", hash) return hash } func getHashForFile(path string) string { hasher := md5.New() readFile, err := os.Open(path) if err != nil { panic(err) } defer readFile.Close() hasher.Reset() _, err = io.Copy(hasher, readFile) if err != nil { log.Fatal(err) } return hex.EncodeToString(hasher.Sum(nil)) } func getIdFromFileNew(file os.FileInfo) int32 { name := file.Name() if !file.IsDir() && strings.HasSuffix(name, ".id") { name = strings.Replace(name, ".id", "", 1) name = strings.Replace(name, ".", "", 1) i, _ := strconv.Atoi(name) return int32(i) } return -1 } func isSong(entry fs.FileInfo) bool { return !entry.IsDir() && strings.HasSuffix(entry.Name(), ".mp3") } func isCoverImage(entry fs.FileInfo) bool { return !entry.IsDir() && strings.Contains(entry.Name(), "cover") && (strings.HasSuffix(entry.Name(), ".jpg") || strings.HasSuffix(entry.Name(), ".png")) } func contains(s []string, searchTerm string) bool { i := sort.SearchStrings(s, searchTerm) return i < len(s) && s[i] == searchTerm }