Added a new sync that uses hash

Added a new sync that uses hash and sqlc for the queries. Added db
migration. Started adding a config file.

pkg/server/sync.go

@@ -1,9 +1,14 @@
 package server
 
 import (
+	"crypto/md5"
+	"encoding/hex"
+	"errors"
 	"fmt"
+	"io"
 	"io/fs"
 	"log"
+	"music-server/db/repository"
 	"music-server/pkg/db"
 	"music-server/pkg/models"
 	"os"
@@ -11,17 +16,65 @@ import (
 	"strconv"
 	"strings"
 	"sync"
+	"time"
+
+	"github.com/MShekow/directory-checksum/directory_checksum"
+	"github.com/spf13/afero"
 )
 
+var allGames []repository.Game
+var gamesBeforeSync []repository.Game
+var gamesAfterSync []repository.Game
+var gamesAdded []string
+var gamesReAdded []string
+var gamesChangedTitle map[string]string
+var gamesChangedContent []string
+var gamesRemoved []string
+var catchedErrors []string
+var brokenSongs []string
+
+type Response struct {
+	GamesAdded          []string          `json:"games_added"`
+	GamesReAdded        []string          `json:"games_re_added"`
+	GamesChangedTitle   map[string]string `json:"games_changed_title"`
+	GamesChangedContent []string          `json:"games_changed_content"`
+	GamesRemoved        []string          `json:"games_removed"`
+	CatchedErrors       []string          `json:"catched_errors"`
+}
+
+type GameStatus int
+
+const (
+	NotChanged GameStatus = iota
+	TitleChanged
+	GameChanged
+	NewGame
+)
+
+var statusName = map[GameStatus]string{
+	NotChanged:   "Not changed",
+	TitleChanged: "Title changed",
+	GameChanged:  "Game changed",
+	NewGame:      "New game",
+}
+
+func (gs GameStatus) String() string {
+	return statusName[gs]
+}
+
+var syncWg sync.WaitGroup
+var repo *repository.Queries
+
 var wg sync.WaitGroup
 
 func SyncGames() {
+	start := time.Now()
 	host := os.Getenv("DB_HOST")
 	var dir string
 	if host != "" {
 		dir = "/sorted/"
 	} else {
-		dir = "/Users/sebastian/Resilio Sync/Sorterat_test/"
+		dir = "/Users/sebastian/ResilioSync/Sorterat_test/"
 	}
 	fmt.Printf("dir: %s\n", dir)
 	foldersToSkip := []string{".sync", "dist", "old"}
@@ -37,15 +90,21 @@ func SyncGames() {
 	for _, file := range files {
 		syncGame(file, foldersToSkip, dir)
 	}
+	finished := time.Now()
+	totalTime := finished.Sub(start)
+	out := time.Time{}.Add(totalTime)
+	fmt.Printf("\nTotal time: %v\n", totalTime)
+	fmt.Printf("Total time: %v\n", out.Format("15:04:05.00000"))
 }
 
 func SyncGamesQuick() {
+	start := time.Now()
 	host := os.Getenv("DB_HOST")
 	var dir string
 	if host != "" {
 		dir = "/sorted/"
 	} else {
-		dir = "/Users/sebastian/Resilio Sync/Sorterat_test/"
+		dir = "/Users/sebastian/ResilioSync/Sorterat_test/"
 	}
 	fmt.Printf("dir: %s\n", dir)
 	foldersToSkip := []string{".sync", "dist", "old"}
@@ -66,6 +125,11 @@ func SyncGamesQuick() {
 		}()
 	}
 	wg.Wait()
+	finished := time.Now()
+	totalTime := finished.Sub(start)
+	out := time.Time{}.Add(totalTime)
+	fmt.Printf("\nTotal time: %v\n", totalTime)
+	fmt.Printf("Total time: %v\n", out.Format("15:04:05.00000"))
 }
 
 func syncGame(file os.DirEntry, foldersToSkip []string, dir string) {
@@ -133,7 +197,10 @@ func checkIfChanged(id int, name string, path string) {
 
 func addNewGame(name string, path string) {
 	newId := db.GetIdByGameName(name)
-	if newId == -1 {
+	if newId != -1 {
+		checkBrokenSongs()
+		db.RemoveDeletionDate(newId)
+	} else {
 		newId = db.InsertGame(name, path)
 	}
 
@@ -194,6 +261,401 @@ func checkBrokenSongs() {
 	db.RemoveBrokenSongs(brokenSongs)
 }
 
+func SyncGamesNew() Response {
+	repo = repository.New(db.Dbpool)
+	start := time.Now()
+	fmt.Printf("dir: %s\n", Conf.Path)
+	foldersToSkip := []string{".sync", "dist", "old"}
+	fmt.Println(foldersToSkip)
+
+	var err error
+
+	gamesBeforeSync, err = repo.FindAllGames(db.Ctx)
+	handleError("FindAllGames Before", err, "")
+	fmt.Printf("Games Before: %d\n", len(gamesBeforeSync))
+
+	allGames, err = repo.GetAllGamesIncludingDeleted(db.Ctx)
+	handleError("GetAllGamesIncludingDeleted", err, "")
+	err = repo.SetGameDeletionDate(db.Ctx)
+	handleError("SetGameDeletionDate", err, "")
+
+	directories, err := os.ReadDir(Conf.Path)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	syncWg.Add(len(directories))
+	for _, dir := range directories {
+		go func() {
+			defer syncWg.Done()
+			syncGameNew(dir, foldersToSkip, Conf.Path)
+		}()
+	}
+	syncWg.Wait()
+	checkBrokenSongsNew()
+
+	gamesAfterSync, err = repo.FindAllGames(db.Ctx)
+	handleError("FindAllGames After", err, "")
+
+	fmt.Printf("\nGames Before: %d\n", len(gamesBeforeSync))
+	fmt.Printf("Games After: %d\n", len(gamesAfterSync))
+
+	fmt.Printf("\nGames added: \n")
+	for _, game := range gamesAdded {
+		fmt.Printf("%s\n", game)
+	}
+
+	fmt.Printf("\nGames readded: \n")
+	for _, game := range gamesReAdded {
+		fmt.Printf("%s\n", game)
+	}
+
+	fmt.Printf("\nGames with changed title: \n")
+	for key, value := range gamesChangedTitle {
+		fmt.Printf("The game: %s changed title to: %s\n", key, value)
+	}
+
+	fmt.Printf("\nGames with changed content: \n")
+	for _, game := range gamesChangedContent {
+		fmt.Printf("%s\n", game)
+	}
+
+	fmt.Printf("\n\n")
+	var gamesRemovedTemp []string
+	for _, beforeGame := range gamesBeforeSync {
+		var found bool = false
+		for _, afterGame := range gamesAfterSync {
+			if beforeGame.GameName == afterGame.GameName {
+				found = true
+				fmt.Printf("Game: %s, Found: %v break\n", beforeGame.GameName, found)
+				break
+			}
+		}
+		if !found {
+			fmt.Printf("Game: %s, Found: %v\n", beforeGame.GameName, found)
+			gamesRemovedTemp = append(gamesRemovedTemp, beforeGame.GameName)
+		}
+	}
+
+	for _, game := range gamesRemovedTemp {
+		var found bool = false
+		for key, _ := range gamesChangedTitle {
+			if game == key {
+				found = true
+				fmt.Printf("Game: %s, Found: %v break2\n", game, found)
+				break
+			}
+		}
+		if !found {
+			gamesRemoved = append(gamesRemoved, game)
+		}
+	}
+
+	fmt.Printf("\nGames removed: \n")
+	for _, game := range gamesRemoved {
+		fmt.Printf("%s\n", game)
+	}
+
+	fmt.Printf("\nErrors catched: \n")
+	for _, error := range catchedErrors {
+		fmt.Printf("%s\n", error)
+	}
+
+	finished := time.Now()
+	totalTime := finished.Sub(start)
+	out := time.Time{}.Add(totalTime)
+	fmt.Printf("\nTotal time: %v\n", totalTime)
+	fmt.Printf("Total time: %v\n", out.Format("15:04:05.00000"))
+
+	return Response{
+		GamesAdded:          gamesAdded,
+		GamesReAdded:        gamesReAdded,
+		GamesChangedTitle:   gamesChangedTitle,
+		GamesChangedContent: gamesChangedContent,
+		GamesRemoved:        gamesRemoved,
+		CatchedErrors:       catchedErrors,
+	}
+}
+
+func checkBrokenSongsNew() {
+	allSongs, err := repo.FetchAllSongs(db.Ctx)
+	handleError("FetchAllSongs", err, "")
+	var brokenWg sync.WaitGroup
+	brokenWg.Add(len(allSongs))
+	for _, song := range allSongs {
+		go func() {
+			defer brokenWg.Done()
+			checkBrokenSongNew(song)
+		}()
+	}
+	brokenWg.Wait()
+	err = repo.RemoveBrokenSongs(db.Ctx, brokenSongs)
+	handleError("RemoveBrokenSongs", err, "")
+}
+
+func checkBrokenSongNew(song repository.Song) {
+	//Check if file exists and open
+	openFile, err := os.Open(song.Path)
+	if err != nil {
+		//File not found
+		brokenSongs = append(brokenSongs, song.Path)
+		fmt.Printf("song broken: %v\n", song.Path)
+	} else {
+		err = openFile.Close()
+		if err != nil {
+			log.Println(err)
+		}
+	}
+}
+
+func syncGameNew(file os.DirEntry, foldersToSkip []string, baseDir string) {
+	if file.IsDir() && !contains(foldersToSkip, file.Name()) {
+		gameDir := baseDir + file.Name() + "/"
+
+		dirHash := getHashForDir(gameDir)
+
+		var status GameStatus = NewGame
+		var oldGame repository.Game
+		var id int32 = -1
+
+		//fmt.Printf("Games before: %d\n", len(gamesBeforeSync))
+
+		for _, currentGame := range allGames {
+			oldGame = currentGame
+			//fmt.Printf("%s | %s\n", oldGame.GameName, oldGame.Hash)
+			if oldGame.GameName == file.Name() && oldGame.Hash == dirHash {
+				status = NotChanged
+				id = oldGame.ID
+				//fmt.Printf("Game not changed\n")
+				break
+			} else if oldGame.GameName == file.Name() && oldGame.Hash != dirHash {
+				status = GameChanged
+				id = oldGame.ID
+				//fmt.Printf("Game changed\n")
+				break
+			} else if oldGame.GameName != file.Name() && oldGame.Hash == dirHash {
+				status = TitleChanged
+				id = oldGame.ID
+				//fmt.Printf("GameName changed\n")
+				break
+			}
+		}
+
+		fmt.Printf("\n\nID: %d | GameName: %s | GameHash: %s | Status: %s\n", id, file.Name(), dirHash, status)
+
+		entries, err := os.ReadDir(gameDir)
+		if err != nil {
+			log.Println(err)
+		}
+		switch status {
+		case NewGame:
+			if id != -1 {
+				for _, entry := range entries {
+					fileInfo, err := entry.Info()
+					if err != nil {
+						log.Println(err)
+					}
+					id = getIdFromFileNew(fileInfo)
+					if id != -1 {
+						break
+					}
+				}
+				err = repo.InsertGameWithExistingId(db.Ctx, repository.InsertGameWithExistingIdParams{ID: id, GameName: file.Name(), Path: gameDir, Hash: dirHash})
+				handleError("InsertGameWithExistingId", err, "")
+				if err != nil {
+					fmt.Printf("id = %v\n", id)
+					fileName := gameDir + "/." + strconv.Itoa(int(id)) + ".id"
+					fmt.Printf("fileName = %v\n", fileName)
+
+					err := os.Remove(fileName)
+					if err != nil {
+						fmt.Printf("%s\n", err)
+					}
+
+					newDirHash := getHashForDir(gameDir)
+
+					id = insertGameNew(file.Name(), gameDir, newDirHash)
+				}
+			} else {
+				id = insertGameNew(file.Name(), gameDir, dirHash)
+			}
+			gamesAdded = append(gamesAdded, file.Name())
+			newCheckSongs(entries, gameDir, id)
+		case GameChanged:
+			err = repo.UpdateGameHash(db.Ctx, repository.UpdateGameHashParams{Hash: dirHash, ID: id})
+			handleError("UpdateGameHash", err, "")
+			gamesChangedContent = append(gamesChangedContent, file.Name())
+			newCheckSongs(entries, gameDir, id)
+		case TitleChanged:
+			//println("TitleChanged")
+			err = repo.UpdateGameName(db.Ctx, repository.UpdateGameNameParams{Name: file.Name(), Path: gameDir, ID: id})
+			handleError("UpdateGameName", err, "")
+			newCheckSongs(entries, gameDir, id)
+			if gamesChangedTitle == nil {
+				gamesChangedTitle = make(map[string]string)
+			}
+			gamesChangedTitle[oldGame.GameName] = file.Name()
+		case NotChanged:
+			//println("NotChanged")
+			var found bool = false
+			for _, beforeGame := range gamesBeforeSync {
+				if dirHash == beforeGame.Hash {
+					found = true
+					//fmt.Printf("Game %s | %s | %s | %s | %v\n", beforeGame.GameName, beforeGame.Hash, dirHash, status, beforeGame.Deleted)
+				}
+			}
+			if !found {
+				newCheckSongs(entries, gameDir, id)
+				gamesReAdded = append(gamesReAdded, file.Name())
+
+			}
+		}
+		err = repo.RemoveDeletionDate(db.Ctx, id)
+		handleError("RemoveDeletionDate", err, "")
+	}
+}
+
+func insertGameNew(name string, path string, hash string) int32 {
+	var duplicateError = errors.New("ERROR: duplicate key value violates unique")
+	id, err := repo.InsertGame(db.Ctx, repository.InsertGameParams{GameName: name, Path: path, Hash: hash})
+	handleError("InsertGame", err, "")
+	if err != nil {
+		fmt.Printf("Handle id busy\n")
+		if strings.HasPrefix(err.Error(), duplicateError.Error()) {
+			fmt.Printf("Handeling this id\n")
+			_, err = repo.ResetGameIdSeq(db.Ctx)
+			handleError("ResetGameIdSeq", err, "")
+			id = insertGameNew(name, path, hash)
+		}
+	}
+	return id
+
+}
+
+func newCheckSongs(entries []os.DirEntry, gameDir string, id int32) int32 {
+	//hasher := md5.New()
+	var numberOfSongs int32
+	var songWg sync.WaitGroup
+	songWg.Add(len(entries))
+	for _, entry := range entries {
+		go func() {
+			defer songWg.Done()
+			newCheckSong(entry, gameDir, id)
+		}()
+	}
+	songWg.Wait()
+	return numberOfSongs
+}
+
+func newCheckSong(entry os.DirEntry, gameDir string, id int32) {
+	fileInfo, err := entry.Info()
+	if err != nil {
+		log.Println(err)
+	}
+
+	if isSong(fileInfo) {
+		path := gameDir + entry.Name()
+
+		songHash := getHashForFile(path)
+		//numberOfSongs++
+
+		fileName := entry.Name()
+		songName, _ := strings.CutSuffix(fileName, ".mp3")
+
+		song, err := repo.GetSongWithHash(db.Ctx, songHash)
+		handleError("GetSongWithHash", err, fmt.Sprintf("GameID: %d | Path: %s | SongName: %s | SongHash: %s\n", id, path, entry.Name(), songHash))
+		if err == nil {
+			if song.SongName == songName && song.Path == path {
+				return
+			}
+		}
+		fmt.Printf("Song Changed\n")
+
+		fmt.Printf("Path: %s | SongHash: %s\n", path, songHash)
+
+		count, err := repo.CheckSongWithHash(db.Ctx, songHash)
+		handleError("CheckSongWithHash", err, fmt.Sprintf("GameID: %d | Path: %s | SongName: %s | SongHash: %s\n", id, path, entry.Name(), songHash))
+		if err != nil {
+			count2, err := repo.CheckSong(db.Ctx, path)
+			handleError("CheckSong", err, fmt.Sprintf("GameID: %d | Path: %s | SongName: %s | SongHash: %s\n", id, path, entry.Name(), songHash))
+			if count2 > 0 {
+				err = repo.AddHashToSong(db.Ctx, repository.AddHashToSongParams{Hash: songHash, Path: path})
+				handleError("AddHashToSong", err, fmt.Sprintf("GameID: %d | Path: %s | SongName: %s | SongHash: %s\n", id, path, entry.Name(), songHash))
+				count, err = repo.CheckSongWithHash(db.Ctx, songHash)
+				handleError("CheckSongWithHash 2", err, fmt.Sprintf("GameID: %d | Path: %s | SongName: %s | SongHash: %s\n", id, path, entry.Name(), songHash))
+			}
+		}
+
+		//count, _ := repo.CheckSong(ctx, path)
+		if count > 0 {
+			err = repo.UpdateSong(db.Ctx, repository.UpdateSongParams{SongName: songName, FileName: &fileName, Path: path, Hash: songHash})
+			handleError("UpdateSong", err, fmt.Sprintf("GameID: %d | Path: %s | SongName: %s | SongHash: %s\n", id, path, entry.Name(), songHash))
+		} else {
+			count2, err := repo.CheckSong(db.Ctx, path)
+			handleError("CheckSong", err, fmt.Sprintf("GameID: %d | Path: %s | SongName: %s | SongHash: %s\n", id, path, entry.Name(), songHash))
+			if count2 > 0 {
+				err = repo.AddHashToSong(db.Ctx, repository.AddHashToSongParams{Hash: songHash, Path: path})
+				handleError("AddHashToSong", err, fmt.Sprintf("GameID: %d | Path: %s | SongName: %s | SongHash: %s\n", id, path, entry.Name(), songHash))
+			} else {
+				err = repo.AddSong(db.Ctx, repository.AddSongParams{GameID: id, SongName: songName, Path: path, FileName: &fileName, Hash: songHash})
+				handleError("AddSong", err, fmt.Sprintf("GameID: %d | Path: %s | SongName: %s | SongHash: %s\n", id, path, entry.Name(), songHash))
+
+			}
+		}
+	} else if isCoverImage(fileInfo) {
+		//TODO: Later add cover art image here in db
+	}
+}
+
+func handleError(funcName string, err error, msg string) {
+	var compareError = errors.New("no rows in result set")
+	if err != nil {
+		if compareError.Error() != err.Error() {
+			fmt.Printf("%s Error: %s\n", funcName, err)
+			if msg != "" {
+				fmt.Printf("%s\n", msg)
+				catchedErrors = append(catchedErrors, fmt.Sprintf("Func: %s\nError message: %s\nDebug message: %s\n", funcName, err, msg))
+			} else {
+				catchedErrors = append(catchedErrors, fmt.Sprintf("Func: %s\nError message: %s\n", funcName, err))
+			}
+		}
+	}
+}
+
+func getHashForDir(gameDir string) string {
+	directory, _ := directory_checksum.ScanDirectory(gameDir, afero.NewOsFs())
+	hash, _ := directory.ComputeDirectoryChecksums()
+
+	//fmt.Printf("Hash: |%s|\n", hash)
+	return hash
+}
+
+func getHashForFile(path string) string {
+	hasher := md5.New()
+	readFile, err := os.Open(path)
+	if err != nil {
+		panic(err)
+	}
+	defer readFile.Close()
+	hasher.Reset()
+	_, err = io.Copy(hasher, readFile)
+	if err != nil {
+		log.Fatal(err)
+	}
+	return hex.EncodeToString(hasher.Sum(nil))
+}
+
+func getIdFromFileNew(file os.FileInfo) int32 {
+	name := file.Name()
+	if !file.IsDir() && strings.HasSuffix(name, ".id") {
+		name = strings.Replace(name, ".id", "", 1)
+		name = strings.Replace(name, ".", "", 1)
+		i, _ := strconv.Atoi(name)
+		return int32(i)
+	}
+	return -1
+}
+
 func isSong(entry fs.FileInfo) bool {
 	return !entry.IsDir() && strings.HasSuffix(entry.Name(), ".mp3")
 }