package backend import ( "crypto/md5" "encoding/hex" "errors" "fmt" "io" "io/fs" "log" "music-server/db/repository" "music-server/pkg/db" "music-server/pkg/models" "os" "sort" "strconv" "strings" "sync" "time" "github.com/MShekow/directory-checksum/directory_checksum" "github.com/spf13/afero" ) var allGames []repository.Game var gamesBeforeSync []repository.Game var gamesAfterSync []repository.Game var gamesAdded []string var gamesReAdded []string var gamesChangedTitle map[string]string var gamesChangedContent []string var gamesRemoved []string var catchedErrors []string var brokenSongs []string type Response struct { GamesAdded []string `json:"games_added"` GamesReAdded []string `json:"games_re_added"` GamesChangedTitle map[string]string `json:"games_changed_title"` GamesChangedContent []string `json:"games_changed_content"` GamesRemoved []string `json:"games_removed"` CatchedErrors []string `json:"catched_errors"` } type GameStatus int const ( NotChanged GameStatus = iota TitleChanged GameChanged NewGame ) var statusName = map[GameStatus]string{ NotChanged: "Not changed", TitleChanged: "Title changed", GameChanged: "Game changed", NewGame: "New game", } func (gs GameStatus) String() string { return statusName[gs] } var syncWg sync.WaitGroup var repo *repository.Queries var wg sync.WaitGroup func SyncGames() { start := time.Now() host := os.Getenv("DB_HOST") var dir string if host != "" { dir = "/sorted/" } else { dir = "/Users/sebastian/ResilioSync/Sorterat_test/" } fmt.Printf("dir: %s\n", dir) foldersToSkip := []string{".sync", "dist", "old"} fmt.Println(foldersToSkip) db.SetGameDeletionDate() checkBrokenSongs() files, err := os.ReadDir(dir) if err != nil { log.Fatal(err) } for _, file := range files { syncGame(file, foldersToSkip, dir) } finished := time.Now() totalTime := finished.Sub(start) out := time.Time{}.Add(totalTime) fmt.Printf("\nTotal time: %v\n", totalTime) fmt.Printf("Total time: %v\n", out.Format("15:04:05.00000")) } func SyncGamesQuick() { start := time.Now() host := os.Getenv("DB_HOST") var dir string if host != "" { dir = "/sorted/" } else { dir = "/Users/sebastian/ResilioSync/Sorterat_test/" } fmt.Printf("dir: %s\n", dir) foldersToSkip := []string{".sync", "dist", "old"} fmt.Println(foldersToSkip) db.SetGameDeletionDate() checkBrokenSongs() files, err := os.ReadDir(dir) if err != nil { log.Fatal(err) } for _, file := range files { wg.Add(1) go func() { defer wg.Done() syncGame(file, foldersToSkip, dir) }() } wg.Wait() finished := time.Now() totalTime := finished.Sub(start) out := time.Time{}.Add(totalTime) fmt.Printf("\nTotal time: %v\n", totalTime) fmt.Printf("Total time: %v\n", out.Format("15:04:05.00000")) } func syncGame(file os.DirEntry, foldersToSkip []string, dir string) { if file.IsDir() && !contains(foldersToSkip, file.Name()) { fmt.Println(file.Name()) path := dir + file.Name() + "/" fmt.Println(path) entries, err := os.ReadDir(path) if err != nil { log.Println(err) } id := -1 for _, entry := range entries { fileInfo, err := entry.Info() if err != nil { log.Println(err) } id = getIdFromFile(fileInfo) if id != -1 { break } } if id == -1 { addNewGame(file.Name(), path) } else { checkIfChanged(id, file.Name(), path) checkSongs(path, id) } } } func ResetDB() { db.ClearSongs(-1) db.ClearGames() } func getIdFromFile(file os.FileInfo) int { name := file.Name() if !file.IsDir() && strings.HasSuffix(name, ".id") { name = strings.Replace(name, ".id", "", 1) name = strings.Replace(name, ".", "", 1) i, _ := strconv.Atoi(name) return i } return -1 } func checkIfChanged(id int, name string, path string) { fmt.Printf("Id from file: %v\n", id) nameFromDb := db.GetGameName(id) fmt.Printf("Name from file: %v\n", name) fmt.Printf("Name from DB: %v\n", nameFromDb) if nameFromDb == "" { fmt.Println("Not in db") db.InsertGameWithExistingId(id, name, path) fmt.Println("Added to db") } else if name != nameFromDb { fmt.Println("Diff name") db.UpdateGameName(id, name, path) checkBrokenSongs() } db.RemoveDeletionDate(id) } func addNewGame(name string, path string) { newId := db.GetIdByGameName(name) if newId != -1 { checkBrokenSongs() db.RemoveDeletionDate(newId) } else { newId = db.InsertGame(name, path) } fmt.Printf("newId = %v", newId) fileName := path + "/." + strconv.Itoa(newId) + ".id" fmt.Printf("fileName = %v", fileName) err := os.WriteFile(fileName, nil, 0644) if err != nil { panic(err) } checkSongs(path, newId) } func checkSongs(gameDir string, gameId int) { files, err := os.ReadDir(gameDir) if err != nil { log.Println(err) } for _, file := range files { entry, err := file.Info() if err != nil { log.Println(err) } if isSong(entry) { path := gameDir + entry.Name() fileName := entry.Name() songName, _ := strings.CutSuffix(fileName, ".mp3") if db.CheckSong(path) { db.UpdateSong(songName, fileName, path) } else { db.AddSong(models.SongData{GameId: gameId, SongName: songName, Path: path, FileName: fileName}) } } else if isCoverImage(entry) { //TODO: Later add cover art image here in db } } //TODO: Add number of songs here } func checkBrokenSongs() { allSongs := db.FetchAllSongs() var brokenSongs []models.SongData for _, song := range allSongs { //Check if file exists and open openFile, err := os.Open(song.Path) if err != nil { //File not found brokenSongs = append(brokenSongs, song) fmt.Printf("song broken: %v", song.Path) } else { err = openFile.Close() if err != nil { log.Println(err) } } } db.RemoveBrokenSongs(brokenSongs) } func SyncGamesNewFull() Response { return syncGamesNew(true) } func SyncGamesNewOnlyChanges() Response { return syncGamesNew(false) } func syncGamesNew(full bool) Response { musicPath := os.Getenv("MUSIC_PATH") fmt.Printf("dir: %s\n", musicPath) repo = repository.New(db.Dbpool) start := time.Now() foldersToSkip := []string{".sync", "dist", "old"} fmt.Println(foldersToSkip) var err error gamesAdded = nil gamesReAdded = nil gamesChangedTitle = nil gamesChangedContent = nil gamesRemoved = nil catchedErrors = nil brokenSongs = nil gamesBeforeSync, err = repo.FindAllGames(db.Ctx) handleError("FindAllGames Before", err, "") fmt.Printf("Games Before: %d\n", len(gamesBeforeSync)) allGames, err = repo.GetAllGamesIncludingDeleted(db.Ctx) handleError("GetAllGamesIncludingDeleted", err, "") err = repo.SetGameDeletionDate(db.Ctx) handleError("SetGameDeletionDate", err, "") directories, err := os.ReadDir(musicPath) if err != nil { log.Fatal(err) } syncWg.Add(len(directories)) for _, dir := range directories { go func() { defer syncWg.Done() syncGameNew(dir, foldersToSkip, musicPath, full) }() } syncWg.Wait() checkBrokenSongsNew() gamesAfterSync, err = repo.FindAllGames(db.Ctx) handleError("FindAllGames After", err, "") fmt.Printf("\nGames Before: %d\n", len(gamesBeforeSync)) fmt.Printf("Games After: %d\n", len(gamesAfterSync)) fmt.Printf("\nGames added: \n") for _, game := range gamesAdded { fmt.Printf("%s\n", game) } fmt.Printf("\nGames readded: \n") for _, game := range gamesReAdded { fmt.Printf("%s\n", game) } fmt.Printf("\nGames with changed title: \n") for key, value := range gamesChangedTitle { fmt.Printf("The game: %s changed title to: %s\n", key, value) } fmt.Printf("\nGames with changed content: \n") for _, game := range gamesChangedContent { fmt.Printf("%s\n", game) } fmt.Printf("\n\n") var gamesRemovedTemp []string for _, beforeGame := range gamesBeforeSync { var found bool = false for _, afterGame := range gamesAfterSync { if beforeGame.GameName == afterGame.GameName { found = true //fmt.Printf("Game: %s, Found: %v break\n", beforeGame.GameName, found) break } } if !found { //fmt.Printf("Game: %s, Found: %v\n", beforeGame.GameName, found) gamesRemovedTemp = append(gamesRemovedTemp, beforeGame.GameName) } } for _, game := range gamesRemovedTemp { var found bool = false for key := range gamesChangedTitle { if game == key { found = true //fmt.Printf("Game: %s, Found: %v break2\n", game, found) break } } if !found { gamesRemoved = append(gamesRemoved, game) } } fmt.Printf("\nGames removed: \n") for _, game := range gamesRemoved { fmt.Printf("%s\n", game) } fmt.Printf("\nErrors catched: \n") for _, error := range catchedErrors { fmt.Printf("%s\n", error) } finished := time.Now() totalTime := finished.Sub(start) out := time.Time{}.Add(totalTime) fmt.Printf("\nTotal time: %v\n", totalTime) fmt.Printf("Total time: %v\n", out.Format("15:04:05.00000")) return Response{ GamesAdded: gamesAdded, GamesReAdded: gamesReAdded, GamesChangedTitle: gamesChangedTitle, GamesChangedContent: gamesChangedContent, GamesRemoved: gamesRemoved, CatchedErrors: catchedErrors, } } func checkBrokenSongsNew() { allSongs, err := repo.FetchAllSongs(db.Ctx) handleError("FetchAllSongs", err, "") var brokenWg sync.WaitGroup brokenWg.Add(len(allSongs)) for _, song := range allSongs { go func() { defer brokenWg.Done() checkBrokenSongNew(song) }() } brokenWg.Wait() err = repo.RemoveBrokenSongs(db.Ctx, brokenSongs) handleError("RemoveBrokenSongs", err, "") } func checkBrokenSongNew(song repository.Song) { //Check if file exists and open openFile, err := os.Open(song.Path) if err != nil { //File not found brokenSongs = append(brokenSongs, song.Path) fmt.Printf("song broken: %v\n", song.Path) } else { err = openFile.Close() if err != nil { log.Println(err) } } } func syncGameNew(file os.DirEntry, foldersToSkip []string, baseDir string, full bool) { if file.IsDir() && !contains(foldersToSkip, file.Name()) { gameDir := baseDir + file.Name() + "/" dirHash := getHashForDir(gameDir) var status GameStatus = NewGame var oldGame repository.Game var id int32 = -1 //fmt.Printf("Games before: %d\n", len(gamesBeforeSync)) for _, currentGame := range allGames { oldGame = currentGame //fmt.Printf("%s | %s\n", oldGame.GameName, oldGame.Hash) if oldGame.GameName == file.Name() && oldGame.Hash == dirHash { status = NotChanged id = oldGame.ID //fmt.Printf("Game not changed\n") break } else if oldGame.GameName == file.Name() && oldGame.Hash != dirHash { status = GameChanged id = oldGame.ID //fmt.Printf("Game changed\n") break } else if oldGame.GameName != file.Name() && oldGame.Hash == dirHash { status = TitleChanged id = oldGame.ID //fmt.Printf("GameName changed\n") break } } if full { status = TitleChanged } entries, err := os.ReadDir(gameDir) if err != nil { log.Println(err) } switch status { case NewGame: fmt.Printf("\n\nID: %d | GameName: %s | GameHash: %s | Status: %s\n", id, file.Name(), dirHash, status) if id != -1 { for _, entry := range entries { fileInfo, err := entry.Info() if err != nil { log.Println(err) } id = getIdFromFileNew(fileInfo) if id != -1 { break } } err = repo.InsertGameWithExistingId(db.Ctx, repository.InsertGameWithExistingIdParams{ID: id, GameName: file.Name(), Path: gameDir, Hash: dirHash}) handleError("InsertGameWithExistingId", err, "") if err != nil { fmt.Printf("id = %v\n", id) fileName := gameDir + "/." + strconv.Itoa(int(id)) + ".id" fmt.Printf("fileName = %v\n", fileName) err := os.Remove(fileName) if err != nil { fmt.Printf("%s\n", err) } newDirHash := getHashForDir(gameDir) id = insertGameNew(file.Name(), gameDir, newDirHash) } } else { id = insertGameNew(file.Name(), gameDir, dirHash) } gamesAdded = append(gamesAdded, file.Name()) newCheckSongs(entries, gameDir, id) case GameChanged: fmt.Printf("\n\nID: %d | GameName: %s | GameHash: %s | Status: %s\n", id, file.Name(), dirHash, status) err = repo.UpdateGameHash(db.Ctx, repository.UpdateGameHashParams{Hash: dirHash, ID: id}) handleError("UpdateGameHash", err, "") gamesChangedContent = append(gamesChangedContent, file.Name()) newCheckSongs(entries, gameDir, id) case TitleChanged: fmt.Printf("\n\nID: %d | GameName: %s | GameHash: %s | Status: %s\n", id, file.Name(), dirHash, status) //println("TitleChanged") err = repo.UpdateGameName(db.Ctx, repository.UpdateGameNameParams{Name: file.Name(), Path: gameDir, ID: id}) handleError("UpdateGameName", err, "") newCheckSongs(entries, gameDir, id) if gamesChangedTitle == nil { gamesChangedTitle = make(map[string]string) } gamesChangedTitle[oldGame.GameName] = file.Name() case NotChanged: //println("NotChanged") var found bool = false for _, beforeGame := range gamesBeforeSync { if dirHash == beforeGame.Hash { found = true //fmt.Printf("Game %s | %s | %s | %s | %v\n", beforeGame.GameName, beforeGame.Hash, dirHash, status, beforeGame.Deleted) } } if !found { newCheckSongs(entries, gameDir, id) gamesReAdded = append(gamesReAdded, file.Name()) } } err = repo.RemoveDeletionDate(db.Ctx, id) handleError("RemoveDeletionDate", err, "") } } func insertGameNew(name string, path string, hash string) int32 { var duplicateError = errors.New("ERROR: duplicate key value violates unique") id, err := repo.InsertGame(db.Ctx, repository.InsertGameParams{GameName: name, Path: path, Hash: hash}) handleError("InsertGame", err, "") if err != nil { fmt.Printf("Handle id busy\n") if strings.HasPrefix(err.Error(), duplicateError.Error()) { fmt.Printf("Handeling this id\n") _, err = repo.ResetGameIdSeq(db.Ctx) handleError("ResetGameIdSeq", err, "") id = insertGameNew(name, path, hash) } } return id } func newCheckSongs(entries []os.DirEntry, gameDir string, id int32) int32 { //hasher := md5.New() var numberOfSongs int32 var songWg sync.WaitGroup songWg.Add(len(entries)) for _, entry := range entries { go func() { defer songWg.Done() newCheckSong(entry, gameDir, id) }() } songWg.Wait() return numberOfSongs } func newCheckSong(entry os.DirEntry, gameDir string, id int32) { fileInfo, err := entry.Info() if err != nil { log.Println(err) } if isSong(fileInfo) { path := gameDir + entry.Name() songHash := getHashForFile(path) //numberOfSongs++ fileName := entry.Name() songName, _ := strings.CutSuffix(fileName, ".mp3") song, err := repo.GetSongWithHash(db.Ctx, songHash) handleError("GetSongWithHash", err, fmt.Sprintf("GameID: %d | Path: %s | SongName: %s | SongHash: %s\n", id, path, entry.Name(), songHash)) if err == nil { if song.SongName == songName && song.Path == path { return } } fmt.Printf("Song Changed\n") fmt.Printf("Path: %s | SongHash: %s\n", path, songHash) count, err := repo.CheckSongWithHash(db.Ctx, songHash) handleError("CheckSongWithHash", err, fmt.Sprintf("GameID: %d | Path: %s | SongName: %s | SongHash: %s\n", id, path, entry.Name(), songHash)) if err != nil { count2, err := repo.CheckSong(db.Ctx, path) handleError("CheckSong", err, fmt.Sprintf("GameID: %d | Path: %s | SongName: %s | SongHash: %s\n", id, path, entry.Name(), songHash)) if count2 > 0 { err = repo.AddHashToSong(db.Ctx, repository.AddHashToSongParams{Hash: songHash, Path: path}) handleError("AddHashToSong", err, fmt.Sprintf("GameID: %d | Path: %s | SongName: %s | SongHash: %s\n", id, path, entry.Name(), songHash)) count, err = repo.CheckSongWithHash(db.Ctx, songHash) handleError("CheckSongWithHash 2", err, fmt.Sprintf("GameID: %d | Path: %s | SongName: %s | SongHash: %s\n", id, path, entry.Name(), songHash)) } } //count, _ := repo.CheckSong(ctx, path) if count > 0 { err = repo.UpdateSong(db.Ctx, repository.UpdateSongParams{SongName: songName, FileName: &fileName, Path: path, Hash: songHash}) handleError("UpdateSong", err, fmt.Sprintf("GameID: %d | Path: %s | SongName: %s | SongHash: %s\n", id, path, entry.Name(), songHash)) } else { count2, err := repo.CheckSong(db.Ctx, path) handleError("CheckSong", err, fmt.Sprintf("GameID: %d | Path: %s | SongName: %s | SongHash: %s\n", id, path, entry.Name(), songHash)) if count2 > 0 { err = repo.AddHashToSong(db.Ctx, repository.AddHashToSongParams{Hash: songHash, Path: path}) handleError("AddHashToSong", err, fmt.Sprintf("GameID: %d | Path: %s | SongName: %s | SongHash: %s\n", id, path, entry.Name(), songHash)) } else { err = repo.AddSong(db.Ctx, repository.AddSongParams{GameID: id, SongName: songName, Path: path, FileName: &fileName, Hash: songHash}) handleError("AddSong", err, fmt.Sprintf("GameID: %d | Path: %s | SongName: %s | SongHash: %s\n", id, path, entry.Name(), songHash)) } } } else if isCoverImage(fileInfo) { //TODO: Later add cover art image here in db } } func handleError(funcName string, err error, msg string) { var compareError = errors.New("no rows in result set") if err != nil { if compareError.Error() != err.Error() { fmt.Printf("%s Error: %s\n", funcName, err) if msg != "" { fmt.Printf("%s\n", msg) catchedErrors = append(catchedErrors, fmt.Sprintf("Func: %s\nError message: %s\nDebug message: %s\n", funcName, err, msg)) } else { catchedErrors = append(catchedErrors, fmt.Sprintf("Func: %s\nError message: %s\n", funcName, err)) } } } } func getHashForDir(gameDir string) string { directory, _ := directory_checksum.ScanDirectory(gameDir, afero.NewOsFs()) hash, _ := directory.ComputeDirectoryChecksums() //fmt.Printf("Hash: |%s|\n", hash) return hash } func getHashForFile(path string) string { hasher := md5.New() readFile, err := os.Open(path) if err != nil { panic(err) } defer readFile.Close() hasher.Reset() _, err = io.Copy(hasher, readFile) if err != nil { log.Fatal(err) } return hex.EncodeToString(hasher.Sum(nil)) } func getIdFromFileNew(file os.FileInfo) int32 { name := file.Name() if !file.IsDir() && strings.HasSuffix(name, ".id") { name = strings.Replace(name, ".id", "", 1) name = strings.Replace(name, ".", "", 1) i, _ := strconv.Atoi(name) return int32(i) } return -1 } func isSong(entry fs.FileInfo) bool { return !entry.IsDir() && strings.HasSuffix(entry.Name(), ".mp3") } func isCoverImage(entry fs.FileInfo) bool { return !entry.IsDir() && strings.Contains(entry.Name(), "cover") && (strings.HasSuffix(entry.Name(), ".jpg") || strings.HasSuffix(entry.Name(), ".png")) } func contains(s []string, searchTerm string) bool { i := sort.SearchStrings(s, searchTerm) return i < len(s) && s[i] == searchTerm }