From 47cc65a824eabe3777bb3282130aa1a8d577458a Mon Sep 17 00:00:00 2001 From: deranjer Date: Fri, 19 Jun 2020 22:57:42 -0400 Subject: [PATCH] working on adding files to repo and making sure no duplicates --- client/.gvc/gvclog.log | 6 --- client/.gvc/logs/gvclog.log | 26 +++++++++++++ client/client.go | 13 +++++-- client/clientcmd/add.go | 48 +++++++++++++++++------ client/clientcmd/commonlib.go | 3 ++ common/database/db-ops.go | 11 +++--- common/manager/manager.go | 73 ++++++++++++++++------------------- 7 files changed, 113 insertions(+), 67 deletions(-) delete mode 100644 client/.gvc/gvclog.log create mode 100644 client/.gvc/logs/gvclog.log diff --git a/client/.gvc/gvclog.log b/client/.gvc/gvclog.log deleted file mode 100644 index 271e8e0..0000000 --- a/client/.gvc/gvclog.log +++ /dev/null @@ -1,6 +0,0 @@ -{"level":"warn","module":"database","message":"No existing databse found. initialising new database"} -{"level":"info","message":"Creating new Manager..."} -{"level":"info","message":"Creating new Manager..."} -{"level":"warn","module":"database","message":"no file found"} -{"level":"info","message":"The file was [not found], so continuing to create it in the database"} -{"level":"info","message":"added file: client.go at path: client.go with hash: \ufffd\ufffd\u0001{\ufffd[Ȼ[8\ufffdR!\ufffd\ufffdB at time: %!s(func() string=0x68c5b0)"} diff --git a/client/.gvc/logs/gvclog.log b/client/.gvc/logs/gvclog.log new file mode 100644 index 0000000..34df549 --- /dev/null +++ b/client/.gvc/logs/gvclog.log @@ -0,0 +1,26 @@ +{"level":"info","message":"Creating new Manager..."} +{"level":"info","message":"added file: client.go at path: client.go with hash: \ufffd\ufffd\u0001{\ufffd[Ȼ[8\ufffdR!\ufffd\ufffdB at time: %!s(func() string=0x68c530)"} +{"level":"info","message":"Creating new Manager..."} +{"level":"info","message":"added file: client.go at path: client.go with hash: \ufffd\ufffd\u0001{\ufffd[Ȼ[8\ufffdR!\ufffd\ufffdB at time: %!s(func() string=0x68c6a0)"} +{"level":"info","message":"Creating new Manager..."} +{"level":"info","message":"added file: client.go at path: client.go with hash: \ufffd\ufffd\u0001{\ufffd[Ȼ[8\ufffdR!\ufffd\ufffdB at time: %!s(func() string=0x68c6a0)"} +{"level":"info","message":"Creating new Manager..."} +{"level":"info","message":"Creating new Manager..."} +{"level":"info","message":"Creating new Manager..."} +{"level":"info","message":"Creating new Manager..."} +{"level":"info","message":"Creating new Manager..."} +{"level":"info","message":"Creating new Manager..."} +{"level":"info","message":"Creating new Manager..."} +{"level":"info","message":"Creating new Manager..."} +{"level":"info","message":"Creating new Manager..."} +{"level":"info","message":"Creating new Manager..."} +{"level":"info","message":"Creating new Manager..."} +{"level":"info","message":"Creating new Manager..."} +{"level":"info","message":"Creating new Manager..."} +{"level":"info","message":"Creating new Manager..."} +{"level":"info","message":"Creating new Manager..."} +{"level":"info","message":"Creating new Manager..."} +{"level":"info","message":"Creating new Manager..."} +{"level":"info","message":"Creating new Manager..."} +{"level":"info","message":"Creating new Manager..."} +{"level":"info","message":"Creating new Manager..."} diff --git a/client/client.go b/client/client.go index 0b753ee..b3475bd 100644 --- a/client/client.go +++ b/client/client.go @@ -46,10 +46,15 @@ func main() { os.Exit(0) } fmt.Println("Attempting to start logger...") - // Setting up the logger to file output - logFile, err := os.OpenFile(".gvc/gvclog.log", os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666) + // Checking the .gvc structure + dirPaths, err := manager.CheckPaths(rootPath) if err != nil { - log.Fatalf("unable to open log file at: %s, exiting with error: %s", ".gvc/gvclog.log", err) + log.Fatalf("unable to create/verify .gvc folder structure: %s", err) + } + // Setting up the logger to file output + logFile, err := os.OpenFile(".gvc/logs/gvclog.log", os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666) + if err != nil { + log.Fatalf("unable to open log file at: %s, exiting with error: %s", ".gvc/logs/gvclog.log", err) } defer logFile.Close() // Setup non-http logging (using middleware for Echo http logging) @@ -65,7 +70,7 @@ func main() { // clientlog.Fatal().Msgf("Unable to open or create a database in the .gvc folder, fatal") //} informer := make(chan manager.OperatingMessage) - m, err = manager.NewManager(rootPath, version, ".gvc/gvc.db", informer, &clientlog) + m, err = manager.NewManager(rootPath, version, ".gvc/gvc.db", informer, dirPaths, &clientlog) if err != nil { clientlog.Fatal().Msgf("unable to create new manager object... %s", err) } diff --git a/client/clientcmd/add.go b/client/clientcmd/add.go index f935b46..19544d7 100644 --- a/client/clientcmd/add.go +++ b/client/clientcmd/add.go @@ -1,6 +1,7 @@ package clientcmd import ( + "errors" "fmt" "os" "path/filepath" @@ -21,6 +22,7 @@ func AddFiles(input string, inputType string, ignore common.FileTypes, m *manage return err } var trackedFiles []string + var SkipDir = errors.New("ignore folder, either .gvc or on ignores") // default variable for filewalk errors to skip the entire dir switch inputType { case "file": // If the -file flag was used, then make sure it is a file, make sure not on ignore list, then add it to tracked files _, err := os.Stat(input) @@ -39,6 +41,7 @@ func AddFiles(input string, inputType string, ignore common.FileTypes, m *manage // } // } relativePath := input //TODO: Figure out if path is corrrect? + fmt.Println("Relative Path: ", relativePath) err = m.AddFileToRepo(relativePath) if err != nil { return fmt.Errorf("unable to add file to repo: %s", err) @@ -59,13 +62,19 @@ func AddFiles(input string, inputType string, ignore common.FileTypes, m *manage return fmt.Errorf("failure accessing path %s", err) } if info.IsDir() { + if currentFile == ".gvc" { // can't track anything in .gvc + return SkipDir + } err = common.CheckFileTypes(currentFile, "folder", ignore) + if err != nil { + return SkipDir + } } else { err = common.CheckFileTypes(currentFile, "file", ignore) - } - if err != nil { - fmt.Printf("Not adding file %s as it is on the ingores list \n", currentFile) - return nil + if err != nil { + fmt.Printf("Not adding file %s as it is on the ingores list \n", currentFile) + return nil + } } fmt.Println("adding file: ", path) relativePath, err := filepath.Rel(workingDir, path) @@ -92,17 +101,25 @@ func AddFiles(input string, inputType string, ignore common.FileTypes, m *manage } filepath.Walk(workingDir, func(path string, info os.FileInfo, err error) error { currentFile := filepath.Base(path) // Stripping all the pathing so we just get the filename + fmt.Println("path: ", path) + fmt.Printf("Checking file: %s\n", currentFile) if err != nil { return fmt.Errorf("failure accessing path %s", err) } if info.IsDir() { + if currentFile == ".gvc" { // can't track anything in .gvc + return nil + } err = common.CheckFileTypes(currentFile, "folder", ignore) + if err != nil { + return err + } } else { err = common.CheckFileTypes(currentFile, "file", ignore) - } - if err != nil { - fmt.Printf("Not adding file %s as it is on the ingores list \n", currentFile) - return nil + if err != nil { + fmt.Printf("Not adding file %s as it is on the ingores list \n", currentFile) + return nil + } } fileExt := filepath.Ext(path) if fileExt == wildcard { // seeing if the file ext matches the wildcard @@ -111,6 +128,7 @@ func AddFiles(input string, inputType string, ignore common.FileTypes, m *manage if err != nil { return fmt.Errorf("unable to create relative path for file: %s", err) } + fmt.Println("WILDCARD Relative Path: ", relativePath) err = m.AddFileToRepo(relativePath) if err != nil { return fmt.Errorf("unable to add file to repo: %s", err) @@ -126,13 +144,19 @@ func AddFiles(input string, inputType string, ignore common.FileTypes, m *manage return fmt.Errorf("failure accessing path %s", err) } if info.IsDir() { + if currentFile == ".gvc" { // can't track anything in .gvc + return SkipDir + } err = common.CheckFileTypes(currentFile, "folder", ignore) + if err != nil { + return SkipDir + } } else { err = common.CheckFileTypes(currentFile, "file", ignore) - } - if err != nil { - fmt.Printf("Not adding file %s as it is on the ingores list \n", currentFile) - return nil + if err != nil { + fmt.Printf("Not adding file %s as it is on the ingores list \n", currentFile) + return nil + } } relativePath, err := filepath.Rel(workingDir, path) if err != nil { diff --git a/client/clientcmd/commonlib.go b/client/clientcmd/commonlib.go index 9514520..81678e9 100644 --- a/client/clientcmd/commonlib.go +++ b/client/clientcmd/commonlib.go @@ -13,6 +13,9 @@ import ( var ConfigPath string func validateFileType(path string, inputType string) error { + if inputType == "wildcard" { // Can't stat wildcard type + return nil + } fullPath, err := filepath.Abs(path) if err != nil { return fmt.Errorf("cannot stat file, invalid input? %s", err) diff --git a/common/database/db-ops.go b/common/database/db-ops.go index ef52717..e2ccd59 100644 --- a/common/database/db-ops.go +++ b/common/database/db-ops.go @@ -33,19 +33,18 @@ func (db *DB) ConfigureDB(dbPath string) error { // CheckIfFileCurrentlyMonitored checks if the file is already being monitored. This is a read-only check // to see whether the file was correctly initialised // (BUG) The hash causes the same file to be in database multiple times! -func (db *DB) CheckIfFileCurrentlyMonitored(src string, hash [16]byte) (File, error) { +func (db *DB) CheckIfFileCurrentlyMonitored(path string) bool { var file File - //TODO: check this actually works still (don't need hash passed to this anymore) - if err := db.One("Path", src, &file); err != nil { + if err := db.One("Path", path, &file); err != nil { if err.Error() != "not found" { db.Err(err).Msg("Error finding file by path") - return File{}, err + return false } db.Warn().Msg("no file found") - return File{}, err + return false } - return file, nil + return true } // RetrieveWatchedFiles all files that are in the database as "watched files" diff --git a/common/manager/manager.go b/common/manager/manager.go index b3a5b47..703e452 100644 --- a/common/manager/manager.go +++ b/common/manager/manager.go @@ -15,24 +15,17 @@ import ( // NewManager creates a new manager interface that contains all the needed information to make changes to the repo // rootPath is passed by client or server to let the manager know where to look for the .gvc folder and all the components needed -func NewManager(rootDir string, version string, dbPath string, informer chan OperatingMessage, log *zerolog.Logger) (*Manager, error) { +func NewManager(rootDir string, version string, dbPath string, informer chan OperatingMessage, dirPaths *FilePaths, log *zerolog.Logger) (*Manager, error) { log.Info().Msg("Creating new Manager...") - dirPaths, err := checkPaths(rootDir) - if err != nil { - return &Manager{}, err - } + // Create new patcher patcher := engine.Patcher{ - log, - dirPaths.KeyFolder, - dirPaths.DownloadFolder, - dirPaths.SyncFolder, - dirPaths.ThumbFolder, - dirPaths.DiffFolder, - } - if err != nil { - log.Fatal().Msgf("Error creating a patcher %s", err) - return &Manager{}, err + Logger: log, + KeyFolder: dirPaths.KeyFolder, + DownloadFolder: dirPaths.DownloadFolder, + SyncFolder: dirPaths.SyncFolder, + ThumbFolder: dirPaths.ThumbFolder, + DiffFolder: dirPaths.DiffFolder, } gvcDB, err := database.OpenOrCreateDB(dbPath, log) if err != nil { @@ -52,7 +45,8 @@ func NewManager(rootDir string, version string, dbPath string, informer chan Ope return &m, nil } -func checkPaths(rootDir string) (filePaths *FilePaths, err error) { +// CheckPaths just checks the .gvc folder structure +func CheckPaths(rootDir string) (filePaths *FilePaths, err error) { // checking for the .gvc folder (the client (but not the server) already checks for the .gvc folder, but this checks all subdirects to make sure they are there) rootFolder, err := filepath.Abs(rootDir) if err != nil { @@ -110,6 +104,8 @@ func (m *Manager) AddFileToRepo(relFilePath string) error { //see commsManagment.go // f := NewFileManager() //DELAYED: this feature affects only large files and user experience. It can wait. + relFilePath = strings.TrimSpace(relFilePath) //purging any odd spaces TODO: Make sure not needed + fmt.Println("Checking for file: ", relFilePath) var tmpFile database.File filename := filepath.Base(relFilePath) var hash [16]byte @@ -123,30 +119,29 @@ func (m *Manager) AddFileToRepo(relFilePath string) error { if hash, err = engine.UniqueFileHash(relFilePath); err != nil { return err } - if tmpFile, err = m.dB.CheckIfFileCurrentlyMonitored(relFilePath, hash); err != nil { - if strings.Index(err.Error(), "not found") != -1 { - //the file wasn't found, this is an ok error - m.Info().Msgf("The file was [%s], so continuing to create it in the database", err) - } else { - return fmt.Errorf("File was not found in repo, please add file first") - } - tmpFile.CurrentHash = hash - tmpFile.Name = filename - tmpFile.Path = relFilePath - tmpFile.CreatedAt = time.Now() - tmpFile.Unique = base64.URLEncoding.EncodeToString([]byte(filename)) + "_" + base64.URLEncoding.EncodeToString((tmpFile.CurrentHash[:])) + "_" + strconv.FormatInt(tmpFile.CreatedAt.Unix(), 10) + "_" + filename - //tmpFile.BkpLocation = filepath.Join(m.SyncFolder, tmpFile.Unique) - //tmpFile.CurrentBase = tmpFile.BkpLocation - //tmpFile.Ignore = false //we can have files in the database that are ignored. TODO: This was initially added so that 'All Files' would show up as a file (its a hack as it adds a dummy to the database) - //we should now have a unique name for this file - //if needs be, we can find out the real file name from the string - //the hash will give us a reasonable indication of the similarity of the files - //define filename of backup(s) - _, err := m.prepareDatabaseForFile(tmpFile) - if err != nil { - return err - } + alreadyTracked := m.dB.CheckIfFileCurrentlyMonitored(relFilePath) + if alreadyTracked { + return fmt.Errorf("file already found in tracked files, not adding: %s", relFilePath) } + tmpFile = database.File{} + + tmpFile.CurrentHash = hash + tmpFile.Name = filename + tmpFile.Path = relFilePath + tmpFile.CreatedAt = time.Now() + tmpFile.Unique = base64.URLEncoding.EncodeToString([]byte(filename)) + "_" + base64.URLEncoding.EncodeToString((tmpFile.CurrentHash[:])) + "_" + strconv.FormatInt(tmpFile.CreatedAt.Unix(), 10) + "_" + filename + //tmpFile.BkpLocation = filepath.Join(m.SyncFolder, tmpFile.Unique) + //tmpFile.CurrentBase = tmpFile.BkpLocation + //tmpFile.Ignore = false //we can have files in the database that are ignored. TODO: This was initially added so that 'All Files' would show up as a file (its a hack as it adds a dummy to the database) + //we should now have a unique name for this file + //if needs be, we can find out the real file name from the string + //the hash will give us a reasonable indication of the similarity of the files + //define filename of backup(s) + _, err = m.prepareDatabaseForFile(tmpFile) + if err != nil { + return err + } + m.Info().Msgf("added file: %s at path: %s with hash: %s at time: %s", filename, relFilePath, tmpFile.CurrentHash, tmpFile.CreatedAt.String) return nil }