Sebastian
806e88adeb
All checks were successful
Build / build (push) Successful in 2m35s
#2 - Added request to download the newest version of the app #3 - Added request to check progress during sync #4 - Now blocking all request while sync is in progress #5 - Implemented ants for thread pooling #6 - Changed the sync request to now only start the sync
546 lines
16 KiB
Go
546 lines
16 KiB
Go
package backend
|
|
|
|
import (
|
|
"crypto/md5"
|
|
"encoding/hex"
|
|
"errors"
|
|
"fmt"
|
|
"github.com/panjf2000/ants/v2"
|
|
"io"
|
|
"io/fs"
|
|
"log"
|
|
"music-server/internal/db"
|
|
"music-server/internal/db/repository"
|
|
"os"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/MShekow/directory-checksum/directory_checksum"
|
|
"github.com/spf13/afero"
|
|
)
|
|
|
|
var Syncing = false
|
|
var foldersSynced float32
|
|
var numberOfFoldersToSync float32
|
|
var totalTime time.Duration
|
|
|
|
var allGames []repository.Game
|
|
var gamesBeforeSync []repository.Game
|
|
var gamesAfterSync []repository.Game
|
|
var gamesAdded []string
|
|
var gamesReAdded []string
|
|
var gamesChangedTitle map[string]string
|
|
var gamesChangedContent []string
|
|
var gamesRemoved []string
|
|
var catchedErrors []string
|
|
var brokenSongs []string
|
|
|
|
type Response struct {
|
|
GamesAdded []string `json:"games_added"`
|
|
GamesReAdded []string `json:"games_re_added"`
|
|
GamesChangedTitle map[string]string `json:"games_changed_title"`
|
|
GamesChangedContent []string `json:"games_changed_content"`
|
|
GamesRemoved []string `json:"games_removed"`
|
|
CatchedErrors []string `json:"catched_errors"`
|
|
}
|
|
|
|
type GameStatus int
|
|
|
|
const (
|
|
NotChanged GameStatus = iota
|
|
TitleChanged
|
|
GameChanged
|
|
NewGame
|
|
)
|
|
|
|
var statusName = map[GameStatus]string{
|
|
NotChanged: "Not changed",
|
|
TitleChanged: "Title changed",
|
|
GameChanged: "Game changed",
|
|
NewGame: "New game",
|
|
}
|
|
|
|
func (gs GameStatus) String() string {
|
|
return statusName[gs]
|
|
}
|
|
|
|
func ResetDB() {
|
|
repo.ClearSongs(db.Ctx)
|
|
repo.ClearGames(db.Ctx)
|
|
}
|
|
|
|
func SyncProgress() string {
|
|
//log.Printf("Progress: %v%%\n", int((foldersSynced/numberOfFoldersToSync)*10))
|
|
log.Printf("Progress: %v/%v %v%%\n", int(foldersSynced), int(numberOfFoldersToSync), int((foldersSynced/numberOfFoldersToSync)*100))
|
|
return fmt.Sprintf("Progress: %v%%", int((foldersSynced/numberOfFoldersToSync)*100))
|
|
}
|
|
|
|
func SyncResult() Response {
|
|
fmt.Printf("\nGames Before: %d\n", len(gamesBeforeSync))
|
|
fmt.Printf("Games After: %d\n", len(gamesAfterSync))
|
|
|
|
fmt.Printf("\nGames added: \n")
|
|
for _, game := range gamesAdded {
|
|
fmt.Printf("%s\n", game)
|
|
}
|
|
|
|
fmt.Printf("\nGames readded: \n")
|
|
for _, game := range gamesReAdded {
|
|
fmt.Printf("%s\n", game)
|
|
}
|
|
|
|
fmt.Printf("\nGames with changed title: \n")
|
|
for key, value := range gamesChangedTitle {
|
|
fmt.Printf("The game: %s changed title to: %s\n", key, value)
|
|
}
|
|
|
|
fmt.Printf("\nGames with changed content: \n")
|
|
for _, game := range gamesChangedContent {
|
|
fmt.Printf("%s\n", game)
|
|
}
|
|
|
|
fmt.Printf("\n\n")
|
|
var gamesRemovedTemp []string
|
|
for _, beforeGame := range gamesBeforeSync {
|
|
var found = false
|
|
for _, afterGame := range gamesAfterSync {
|
|
if beforeGame.GameName == afterGame.GameName {
|
|
found = true
|
|
fmt.Printf("Game: %s, Found: %v break\n", beforeGame.GameName, found)
|
|
break
|
|
}
|
|
}
|
|
if !found {
|
|
fmt.Printf("Game: %s, Found: %v\n", beforeGame.GameName, found)
|
|
gamesRemovedTemp = append(gamesRemovedTemp, beforeGame.GameName)
|
|
}
|
|
}
|
|
|
|
for _, game := range gamesRemovedTemp {
|
|
var found bool = false
|
|
for key := range gamesChangedTitle {
|
|
if game == key {
|
|
found = true
|
|
//fmt.Printf("Game: %s, Found: %v break2\n", game, found)
|
|
break
|
|
}
|
|
}
|
|
if !found {
|
|
gamesRemoved = append(gamesRemoved, game)
|
|
}
|
|
}
|
|
|
|
fmt.Printf("\nGames removed: \n")
|
|
for _, game := range gamesRemoved {
|
|
fmt.Printf("%s\n", game)
|
|
}
|
|
|
|
fmt.Printf("\nErrors catched: \n")
|
|
for _, error := range catchedErrors {
|
|
fmt.Printf("%s\n", error)
|
|
}
|
|
|
|
out := time.Time{}.Add(totalTime)
|
|
fmt.Printf("\nTotal time: %v\n", totalTime)
|
|
fmt.Printf("Total time: %v\n", out.Format("15:04:05.00000"))
|
|
|
|
return Response{
|
|
GamesAdded: gamesAdded,
|
|
GamesReAdded: gamesReAdded,
|
|
GamesChangedTitle: gamesChangedTitle,
|
|
GamesChangedContent: gamesChangedContent,
|
|
GamesRemoved: gamesRemoved,
|
|
CatchedErrors: catchedErrors,
|
|
}
|
|
}
|
|
|
|
func SyncGamesNewFull() {
|
|
syncGamesNew(true)
|
|
Reset()
|
|
}
|
|
|
|
func SyncGamesNewOnlyChanges() {
|
|
syncGamesNew(false)
|
|
Reset()
|
|
}
|
|
|
|
func syncGamesNew(full bool) {
|
|
Syncing = true
|
|
|
|
musicPath := os.Getenv("MUSIC_PATH")
|
|
fmt.Printf("dir: %s\n", musicPath)
|
|
if !strings.HasSuffix(musicPath, "/") {
|
|
musicPath += "/"
|
|
}
|
|
|
|
var syncWg sync.WaitGroup
|
|
|
|
initRepo()
|
|
start := time.Now()
|
|
foldersToSkip := []string{".sync", "dist", "old", "characters"}
|
|
fmt.Println(foldersToSkip)
|
|
|
|
var err error
|
|
gamesAdded = nil
|
|
gamesReAdded = nil
|
|
gamesChangedTitle = nil
|
|
gamesChangedContent = nil
|
|
gamesRemoved = nil
|
|
catchedErrors = nil
|
|
brokenSongs = nil
|
|
|
|
gamesBeforeSync, err = repo.FindAllGames(db.Ctx)
|
|
handleError("FindAllGames Before", err, "")
|
|
fmt.Printf("Games Before: %d\n", len(gamesBeforeSync))
|
|
|
|
allGames, err = repo.GetAllGamesIncludingDeleted(db.Ctx)
|
|
handleError("GetAllGamesIncludingDeleted", err, "")
|
|
err = repo.SetGameDeletionDate(db.Ctx)
|
|
handleError("SetGameDeletionDate", err, "")
|
|
|
|
directories, err := os.ReadDir(musicPath)
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
|
|
pool, _ := ants.NewPool(50, ants.WithPreAlloc(true))
|
|
defer pool.Release()
|
|
|
|
numberOfFoldersToSync = float32(len(directories))
|
|
syncWg.Add(int(numberOfFoldersToSync))
|
|
for _, dir := range directories {
|
|
pool.Submit(func() {
|
|
defer syncWg.Done()
|
|
syncGameNew(dir, foldersToSkip, musicPath, full)
|
|
})
|
|
}
|
|
syncWg.Wait()
|
|
checkBrokenSongsNew()
|
|
|
|
gamesAfterSync, err = repo.FindAllGames(db.Ctx)
|
|
handleError("FindAllGames After", err, "")
|
|
|
|
finished := time.Now()
|
|
totalTime = finished.Sub(start)
|
|
out := time.Time{}.Add(totalTime)
|
|
fmt.Printf("\nTotal time: %v\n", totalTime)
|
|
fmt.Printf("Total time: %v\n", out.Format("15:04:05.00000"))
|
|
|
|
Syncing = false
|
|
}
|
|
|
|
func checkBrokenSongsNew() {
|
|
allSongs, err := repo.FetchAllSongs(db.Ctx)
|
|
handleError("FetchAllSongs", err, "")
|
|
var brokenWg sync.WaitGroup
|
|
poolBroken, _ := ants.NewPool(50, ants.WithPreAlloc(true))
|
|
defer poolBroken.Release()
|
|
|
|
brokenWg.Add(len(allSongs))
|
|
for _, song := range allSongs {
|
|
poolBroken.Submit(func() {
|
|
defer brokenWg.Done()
|
|
checkBrokenSongNew(song)
|
|
})
|
|
}
|
|
brokenWg.Wait()
|
|
err = repo.RemoveBrokenSongs(db.Ctx, brokenSongs)
|
|
handleError("RemoveBrokenSongs", err, "")
|
|
}
|
|
|
|
func checkBrokenSongNew(song repository.Song) {
|
|
//Check if file exists and open
|
|
openFile, err := os.Open(song.Path)
|
|
if err != nil {
|
|
//File not found
|
|
brokenSongs = append(brokenSongs, song.Path)
|
|
fmt.Printf("song broken: %v\n", song.Path)
|
|
} else {
|
|
err = openFile.Close()
|
|
if err != nil {
|
|
log.Println(err)
|
|
}
|
|
}
|
|
}
|
|
|
|
func syncGameNew(file os.DirEntry, foldersToSkip []string, baseDir string, full bool) {
|
|
if file.IsDir() && !contains(foldersToSkip, file.Name()) {
|
|
fmt.Printf("Syncing: %s\n", file.Name())
|
|
gameDir := baseDir + file.Name() + "/"
|
|
dirHash := getHashForDir(gameDir)
|
|
|
|
var status GameStatus = NewGame
|
|
var oldGame repository.Game
|
|
var id int32 = -1
|
|
|
|
//fmt.Printf("Games before: %d\n", len(gamesBeforeSync))
|
|
|
|
for _, currentGame := range allGames {
|
|
oldGame = currentGame
|
|
//fmt.Printf("%s | %s\n", oldGame.GameName, oldGame.Hash)
|
|
if oldGame.GameName == file.Name() && oldGame.Hash == dirHash {
|
|
status = NotChanged
|
|
id = oldGame.ID
|
|
//fmt.Printf("Game not changed\n")
|
|
break
|
|
} else if oldGame.GameName == file.Name() && oldGame.Hash != dirHash {
|
|
status = GameChanged
|
|
id = oldGame.ID
|
|
//fmt.Printf("Game changed\n")
|
|
break
|
|
} else if oldGame.GameName != file.Name() && oldGame.Hash == dirHash {
|
|
status = TitleChanged
|
|
id = oldGame.ID
|
|
//fmt.Printf("GameName changed\n")
|
|
break
|
|
}
|
|
}
|
|
|
|
if full {
|
|
status = TitleChanged
|
|
}
|
|
entries, err := os.ReadDir(gameDir)
|
|
if err != nil {
|
|
log.Println(err)
|
|
}
|
|
switch status {
|
|
case NewGame:
|
|
if id != -1 {
|
|
for _, entry := range entries {
|
|
fileInfo, err := entry.Info()
|
|
if err != nil {
|
|
log.Println(err)
|
|
}
|
|
id = getIdFromFileNew(fileInfo)
|
|
if id != -1 {
|
|
break
|
|
}
|
|
}
|
|
err = repo.InsertGameWithExistingId(db.Ctx, repository.InsertGameWithExistingIdParams{ID: id, GameName: file.Name(), Path: gameDir, Hash: dirHash})
|
|
handleError("InsertGameWithExistingId", err, "")
|
|
if err != nil {
|
|
fmt.Printf("id = %v\n", id)
|
|
fileName := gameDir + "/." + strconv.Itoa(int(id)) + ".id"
|
|
fmt.Printf("fileName = %v\n", fileName)
|
|
|
|
err := os.Remove(fileName)
|
|
if err != nil {
|
|
fmt.Printf("%s\n", err)
|
|
}
|
|
|
|
newDirHash := getHashForDir(gameDir)
|
|
|
|
id = insertGameNew(file.Name(), gameDir, newDirHash)
|
|
}
|
|
} else {
|
|
id = insertGameNew(file.Name(), gameDir, dirHash)
|
|
}
|
|
//fmt.Printf("\n\nID: %d | GameName: %s | GameHash: %s | Status: %s\n", id, file.Name(), dirHash, status)
|
|
gamesAdded = append(gamesAdded, file.Name())
|
|
newCheckSongs(entries, gameDir, id)
|
|
case GameChanged:
|
|
//fmt.Printf("\n\nID: %d | GameName: %s | GameHash: %s | Status: %s\n", id, file.Name(), dirHash, status)
|
|
err = repo.UpdateGameHash(db.Ctx, repository.UpdateGameHashParams{Hash: dirHash, ID: id})
|
|
handleError("UpdateGameHash", err, "")
|
|
gamesChangedContent = append(gamesChangedContent, file.Name())
|
|
newCheckSongs(entries, gameDir, id)
|
|
case TitleChanged:
|
|
//fmt.Printf("\n\nID: %d | GameName: %s | GameHash: %s | Status: %s\n", id, file.Name(), dirHash, status)
|
|
//println("TitleChanged")
|
|
err = repo.UpdateGameName(db.Ctx, repository.UpdateGameNameParams{Name: file.Name(), Path: gameDir, ID: id})
|
|
handleError("UpdateGameName", err, "")
|
|
newCheckSongs(entries, gameDir, id)
|
|
if gamesChangedTitle == nil {
|
|
gamesChangedTitle = make(map[string]string)
|
|
}
|
|
gamesChangedTitle[oldGame.GameName] = file.Name()
|
|
case NotChanged:
|
|
//println("NotChanged")
|
|
var found bool = false
|
|
for _, beforeGame := range gamesBeforeSync {
|
|
if dirHash == beforeGame.Hash {
|
|
found = true
|
|
//fmt.Printf("Game %s | %s | %s | %s | %v\n", beforeGame.GameName, beforeGame.Hash, dirHash, status, beforeGame.Deleted)
|
|
}
|
|
}
|
|
if !found {
|
|
newCheckSongs(entries, gameDir, id)
|
|
gamesReAdded = append(gamesReAdded, file.Name())
|
|
|
|
}
|
|
}
|
|
fmt.Printf("\n\nID: %d | GameName: %s | GameHash: %s | Status: %s\n", id, file.Name(), dirHash, status)
|
|
err = repo.RemoveDeletionDate(db.Ctx, id)
|
|
handleError("RemoveDeletionDate", err, "")
|
|
}
|
|
foldersSynced++
|
|
log.Printf("Progress: %v/%v %v%%\n", int(foldersSynced), int(numberOfFoldersToSync), int((foldersSynced/numberOfFoldersToSync)*100))
|
|
}
|
|
|
|
func insertGameNew(name string, path string, hash string) int32 {
|
|
var duplicateError = errors.New("ERROR: duplicate key value violates unique")
|
|
id, err := repo.InsertGame(db.Ctx, repository.InsertGameParams{GameName: name, Path: path, Hash: hash})
|
|
handleError("InsertGame", err, "")
|
|
if err != nil {
|
|
fmt.Printf("Handle id busy\n")
|
|
if strings.HasPrefix(err.Error(), duplicateError.Error()) {
|
|
fmt.Printf("Handeling this id\n")
|
|
_, err = repo.ResetGameIdSeq(db.Ctx)
|
|
handleError("ResetGameIdSeq", err, "")
|
|
id = insertGameNew(name, path, hash)
|
|
}
|
|
}
|
|
return id
|
|
|
|
}
|
|
|
|
func newCheckSongs(entries []os.DirEntry, gameDir string, id int32) int32 {
|
|
//hasher := md5.New()
|
|
var numberOfSongs int32
|
|
numberOfFiles := len(entries)
|
|
|
|
var songWg sync.WaitGroup
|
|
poolSong, _ := ants.NewPool(numberOfFiles, ants.WithPreAlloc(true))
|
|
defer poolSong.Release()
|
|
|
|
songWg.Add(numberOfFiles)
|
|
for _, entry := range entries {
|
|
poolSong.Submit(func() {
|
|
defer songWg.Done()
|
|
if newCheckSong(entry, gameDir, id) {
|
|
numberOfSongs++
|
|
}
|
|
})
|
|
}
|
|
songWg.Wait()
|
|
return numberOfSongs
|
|
}
|
|
|
|
func newCheckSong(entry os.DirEntry, gameDir string, id int32) bool {
|
|
fileInfo, err := entry.Info()
|
|
if err != nil {
|
|
log.Println(err)
|
|
}
|
|
|
|
if isSong(fileInfo) {
|
|
path := gameDir + entry.Name()
|
|
|
|
songHash := getHashForFile(path)
|
|
//numberOfSongs++
|
|
|
|
fileName := entry.Name()
|
|
songName, _ := strings.CutSuffix(fileName, ".mp3")
|
|
|
|
song, err := repo.GetSongWithHash(db.Ctx, songHash)
|
|
handleError("GetSongWithHash", err, fmt.Sprintf("GameID: %d | Path: %s | SongName: %s | SongHash: %s\n", id, path, entry.Name(), songHash))
|
|
if err == nil {
|
|
if song.SongName == songName && song.Path == path {
|
|
return false
|
|
}
|
|
}
|
|
fmt.Printf("Song Changed\n")
|
|
|
|
fmt.Printf("Path: %s | SongHash: %s\n", path, songHash)
|
|
|
|
count, err := repo.CheckSongWithHash(db.Ctx, songHash)
|
|
handleError("CheckSongWithHash", err, fmt.Sprintf("GameID: %d | Path: %s | SongName: %s | SongHash: %s\n", id, path, entry.Name(), songHash))
|
|
if err != nil {
|
|
count2, err := repo.CheckSong(db.Ctx, path)
|
|
handleError("CheckSong", err, fmt.Sprintf("GameID: %d | Path: %s | SongName: %s | SongHash: %s\n", id, path, entry.Name(), songHash))
|
|
if count2 > 0 {
|
|
err = repo.AddHashToSong(db.Ctx, repository.AddHashToSongParams{Hash: songHash, Path: path})
|
|
handleError("AddHashToSong", err, fmt.Sprintf("GameID: %d | Path: %s | SongName: %s | SongHash: %s\n", id, path, entry.Name(), songHash))
|
|
count, err = repo.CheckSongWithHash(db.Ctx, songHash)
|
|
handleError("CheckSongWithHash 2", err, fmt.Sprintf("GameID: %d | Path: %s | SongName: %s | SongHash: %s\n", id, path, entry.Name(), songHash))
|
|
}
|
|
}
|
|
|
|
//count, _ := repo.CheckSong(ctx, path)
|
|
if count > 0 {
|
|
err = repo.UpdateSong(db.Ctx, repository.UpdateSongParams{SongName: songName, FileName: &fileName, Path: path, Hash: songHash})
|
|
handleError("UpdateSong", err, fmt.Sprintf("GameID: %d | Path: %s | SongName: %s | SongHash: %s\n", id, path, entry.Name(), songHash))
|
|
} else {
|
|
count2, err := repo.CheckSong(db.Ctx, path)
|
|
handleError("CheckSong", err, fmt.Sprintf("GameID: %d | Path: %s | SongName: %s | SongHash: %s\n", id, path, entry.Name(), songHash))
|
|
if count2 > 0 {
|
|
err = repo.AddHashToSong(db.Ctx, repository.AddHashToSongParams{Hash: songHash, Path: path})
|
|
handleError("AddHashToSong", err, fmt.Sprintf("GameID: %d | Path: %s | SongName: %s | SongHash: %s\n", id, path, entry.Name(), songHash))
|
|
} else {
|
|
err = repo.AddSong(db.Ctx, repository.AddSongParams{GameID: id, SongName: songName, Path: path, FileName: &fileName, Hash: songHash})
|
|
handleError("AddSong", err, fmt.Sprintf("GameID: %d | Path: %s | SongName: %s | SongHash: %s\n", id, path, entry.Name(), songHash))
|
|
|
|
}
|
|
}
|
|
return true
|
|
} else if isCoverImage(fileInfo) {
|
|
//TODO: Later add cover art image here in db
|
|
}
|
|
return false
|
|
}
|
|
|
|
func handleError(funcName string, err error, msg string) {
|
|
var compareError = errors.New("no rows in result set")
|
|
if err != nil {
|
|
if compareError.Error() != err.Error() {
|
|
fmt.Printf("%s Error: %s\n", funcName, err)
|
|
if msg != "" {
|
|
fmt.Printf("%s\n", msg)
|
|
catchedErrors = append(catchedErrors, fmt.Sprintf("Func: %s\nError message: %s\nDebug message: %s\n", funcName, err, msg))
|
|
} else {
|
|
catchedErrors = append(catchedErrors, fmt.Sprintf("Func: %s\nError message: %s\n", funcName, err))
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func getHashForDir(gameDir string) string {
|
|
directory, _ := directory_checksum.ScanDirectory(gameDir, afero.NewOsFs())
|
|
hash, _ := directory.ComputeDirectoryChecksums()
|
|
|
|
//fmt.Printf("Hash: |%s|\n", hash)
|
|
return hash
|
|
}
|
|
|
|
func getHashForFile(path string) string {
|
|
hasher := md5.New()
|
|
readFile, err := os.Open(path)
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
defer readFile.Close()
|
|
hasher.Reset()
|
|
_, err = io.Copy(hasher, readFile)
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
return hex.EncodeToString(hasher.Sum(nil))
|
|
}
|
|
|
|
func getIdFromFileNew(file os.FileInfo) int32 {
|
|
name := file.Name()
|
|
if !file.IsDir() && strings.HasSuffix(name, ".id") {
|
|
name = strings.Replace(name, ".id", "", 1)
|
|
name = strings.Replace(name, ".", "", 1)
|
|
i, _ := strconv.Atoi(name)
|
|
return int32(i)
|
|
}
|
|
return -1
|
|
}
|
|
|
|
func isSong(entry fs.FileInfo) bool {
|
|
return !entry.IsDir() && strings.HasSuffix(entry.Name(), ".mp3")
|
|
}
|
|
|
|
func isCoverImage(entry fs.FileInfo) bool {
|
|
return !entry.IsDir() && strings.Contains(entry.Name(), "cover") &&
|
|
(strings.HasSuffix(entry.Name(), ".jpg") || strings.HasSuffix(entry.Name(), ".png"))
|
|
}
|
|
|
|
func contains(s []string, searchTerm string) bool {
|
|
i := sort.SearchStrings(s, searchTerm)
|
|
return i < len(s) && s[i] == searchTerm
|
|
}
|