deranjer/gvc
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

working on adding files to repo and making sure no duplicates

Browse Source
This commit is contained in:
deranjer
2020-06-19 22:57:42 -04:00
parent e75991da22
commit 47cc65a824
7 changed files with 113 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
client/.gvc/gvclog.log
View File

@@ -1,6 +0,0 @@
{"level":"warn","module":"database","message":"No existing databse found. initialising new database"}
{"level":"info","message":"Creating new Manager..."}
{"level":"info","message":"Creating new Manager..."}
{"level":"warn","module":"database","message":"no file found"}
{"level":"info","message":"The file was [not found], so continuing to create it in the database"}
{"level":"info","message":"added file: client.go at path: client.go with hash: \ufffd\ufffd\u0001{\ufffd[Ȼ[8\ufffdR!\ufffd\ufffdB at time: %!s(func() string=0x68c5b0)"}

26
client/.gvc/logs/gvclog.log Normal file
View File

@@ -0,0 +1,26 @@
{"level":"info","message":"Creating new Manager..."}
{"level":"info","message":"added file: client.go at path: client.go with hash: \ufffd\ufffd\u0001{\ufffd[Ȼ[8\ufffdR!\ufffd\ufffdB at time: %!s(func() string=0x68c530)"}
{"level":"info","message":"Creating new Manager..."}
{"level":"info","message":"added file: client.go at path: client.go with hash: \ufffd\ufffd\u0001{\ufffd[Ȼ[8\ufffdR!\ufffd\ufffdB at time: %!s(func() string=0x68c6a0)"}
{"level":"info","message":"Creating new Manager..."}
{"level":"info","message":"added file: client.go at path: client.go with hash: \ufffd\ufffd\u0001{\ufffd[Ȼ[8\ufffdR!\ufffd\ufffdB at time: %!s(func() string=0x68c6a0)"}
{"level":"info","message":"Creating new Manager..."}
{"level":"info","message":"Creating new Manager..."}
{"level":"info","message":"Creating new Manager..."}
{"level":"info","message":"Creating new Manager..."}
{"level":"info","message":"Creating new Manager..."}
{"level":"info","message":"Creating new Manager..."}
{"level":"info","message":"Creating new Manager..."}
{"level":"info","message":"Creating new Manager..."}
{"level":"info","message":"Creating new Manager..."}
{"level":"info","message":"Creating new Manager..."}
{"level":"info","message":"Creating new Manager..."}
{"level":"info","message":"Creating new Manager..."}
{"level":"info","message":"Creating new Manager..."}
{"level":"info","message":"Creating new Manager..."}
{"level":"info","message":"Creating new Manager..."}
{"level":"info","message":"Creating new Manager..."}
{"level":"info","message":"Creating new Manager..."}
{"level":"info","message":"Creating new Manager..."}
{"level":"info","message":"Creating new Manager..."}
{"level":"info","message":"Creating new Manager..."}

13
client/client.go
View File

@@ -46,10 +46,15 @@ func main() {
os.Exit(0)
}
fmt.Println("Attempting to start logger...")
// Setting up the logger to file output
logFile, err := os.OpenFile(".gvc/gvclog.log", os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
// Checking the .gvc structure
dirPaths, err := manager.CheckPaths(rootPath)
if err != nil {
log.Fatalf("unable to open log file at: %s, exiting with error: %s", ".gvc/gvclog.log", err)
log.Fatalf("unable to create/verify .gvc folder structure: %s", err)
}
// Setting up the logger to file output
logFile, err := os.OpenFile(".gvc/logs/gvclog.log", os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
if err != nil {
log.Fatalf("unable to open log file at: %s, exiting with error: %s", ".gvc/logs/gvclog.log", err)
}
defer logFile.Close()
// Setup non-http logging (using middleware for Echo http logging)
@@ -65,7 +70,7 @@ func main() {
// clientlog.Fatal().Msgf("Unable to open or create a database in the .gvc folder, fatal")
//}
informer := make(chan manager.OperatingMessage)
m, err = manager.NewManager(rootPath, version, ".gvc/gvc.db", informer, &clientlog)
m, err = manager.NewManager(rootPath, version, ".gvc/gvc.db", informer, dirPaths, &clientlog)
if err != nil {
clientlog.Fatal().Msgf("unable to create new manager object... %s", err)
}

48
client/clientcmd/add.go
View File

@@ -1,6 +1,7 @@
package clientcmd
import (
"errors"
"fmt"
"os"
"path/filepath"
@@ -21,6 +22,7 @@ func AddFiles(input string, inputType string, ignore common.FileTypes, m *manage
return err
}
var trackedFiles []string
var SkipDir = errors.New("ignore folder, either .gvc or on ignores") // default variable for filewalk errors to skip the entire dir
switch inputType {
case "file": // If the -file flag was used, then make sure it is a file, make sure not on ignore list, then add it to tracked files
_, err := os.Stat(input)
@@ -39,6 +41,7 @@ func AddFiles(input string, inputType string, ignore common.FileTypes, m *manage
// }
// }
relativePath := input //TODO: Figure out if path is corrrect?
fmt.Println("Relative Path: ", relativePath)
err = m.AddFileToRepo(relativePath)
if err != nil {
return fmt.Errorf("unable to add file to repo: %s", err)
@@ -59,13 +62,19 @@ func AddFiles(input string, inputType string, ignore common.FileTypes, m *manage
return fmt.Errorf("failure accessing path %s", err)
}
if info.IsDir() {
if currentFile == ".gvc" { // can't track anything in .gvc
return SkipDir
}
err = common.CheckFileTypes(currentFile, "folder", ignore)
if err != nil {
return SkipDir
}
} else {
err = common.CheckFileTypes(currentFile, "file", ignore)
}
if err != nil {
fmt.Printf("Not adding file %s as it is on the ingores list \n", currentFile)
return nil
if err != nil {
fmt.Printf("Not adding file %s as it is on the ingores list \n", currentFile)
return nil
}
}
fmt.Println("adding file: ", path)
relativePath, err := filepath.Rel(workingDir, path)
@@ -92,17 +101,25 @@ func AddFiles(input string, inputType string, ignore common.FileTypes, m *manage
}
filepath.Walk(workingDir, func(path string, info os.FileInfo, err error) error {
currentFile := filepath.Base(path) // Stripping all the pathing so we just get the filename
fmt.Println("path: ", path)
fmt.Printf("Checking file: %s\n", currentFile)
if err != nil {
return fmt.Errorf("failure accessing path %s", err)
}
if info.IsDir() {
if currentFile == ".gvc" { // can't track anything in .gvc
return nil
}
err = common.CheckFileTypes(currentFile, "folder", ignore)
if err != nil {
return err
}
} else {
err = common.CheckFileTypes(currentFile, "file", ignore)
}
if err != nil {
fmt.Printf("Not adding file %s as it is on the ingores list \n", currentFile)
return nil
if err != nil {
fmt.Printf("Not adding file %s as it is on the ingores list \n", currentFile)
return nil
}
}
fileExt := filepath.Ext(path)
if fileExt == wildcard { // seeing if the file ext matches the wildcard
@@ -111,6 +128,7 @@ func AddFiles(input string, inputType string, ignore common.FileTypes, m *manage
if err != nil {
return fmt.Errorf("unable to create relative path for file: %s", err)
}
fmt.Println("WILDCARD Relative Path: ", relativePath)
err = m.AddFileToRepo(relativePath)
if err != nil {
return fmt.Errorf("unable to add file to repo: %s", err)
@@ -126,13 +144,19 @@ func AddFiles(input string, inputType string, ignore common.FileTypes, m *manage
return fmt.Errorf("failure accessing path %s", err)
}
if info.IsDir() {
if currentFile == ".gvc" { // can't track anything in .gvc
return SkipDir
}
err = common.CheckFileTypes(currentFile, "folder", ignore)
if err != nil {
return SkipDir
}
} else {
err = common.CheckFileTypes(currentFile, "file", ignore)
}
if err != nil {
fmt.Printf("Not adding file %s as it is on the ingores list \n", currentFile)
return nil
if err != nil {
fmt.Printf("Not adding file %s as it is on the ingores list \n", currentFile)
return nil
}
}
relativePath, err := filepath.Rel(workingDir, path)
if err != nil {

3
client/clientcmd/commonlib.go
View File

@@ -13,6 +13,9 @@ import (
var ConfigPath string
func validateFileType(path string, inputType string) error {
if inputType == "wildcard" { // Can't stat wildcard type
return nil
}
fullPath, err := filepath.Abs(path)
if err != nil {
return fmt.Errorf("cannot stat file, invalid input? %s", err)

11
common/database/db-ops.go
View File

@@ -33,19 +33,18 @@ func (db *DB) ConfigureDB(dbPath string) error {
// CheckIfFileCurrentlyMonitored checks if the file is already being monitored. This is a read-only check
// to see whether the file was correctly initialised
// (BUG) The hash causes the same file to be in database multiple times!
func (db *DB) CheckIfFileCurrentlyMonitored(src string, hash [16]byte) (File, error) {
func (db *DB) CheckIfFileCurrentlyMonitored(path string) bool {
var file File
//TODO: check this actually works still (don't need hash passed to this anymore)
if err := db.One("Path", src, &file); err != nil {
if err := db.One("Path", path, &file); err != nil {
if err.Error() != "not found" {
db.Err(err).Msg("Error finding file by path")
return File{}, err
return false
}
db.Warn().Msg("no file found")
return File{}, err
return false
}
return file, nil
return true
}
// RetrieveWatchedFiles all files that are in the database as "watched files"

73
common/manager/manager.go
View File

@@ -15,24 +15,17 @@ import (
// NewManager creates a new manager interface that contains all the needed information to make changes to the repo
// rootPath is passed by client or server to let the manager know where to look for the .gvc folder and all the components needed
func NewManager(rootDir string, version string, dbPath string, informer chan OperatingMessage, log *zerolog.Logger) (*Manager, error) {
func NewManager(rootDir string, version string, dbPath string, informer chan OperatingMessage, dirPaths *FilePaths, log *zerolog.Logger) (*Manager, error) {
log.Info().Msg("Creating new Manager...")
dirPaths, err := checkPaths(rootDir)
if err != nil {
return &Manager{}, err
}
// Create new patcher
patcher := engine.Patcher{
log,
dirPaths.KeyFolder,
dirPaths.DownloadFolder,
dirPaths.SyncFolder,
dirPaths.ThumbFolder,
dirPaths.DiffFolder,
}
if err != nil {
log.Fatal().Msgf("Error creating a patcher %s", err)
return &Manager{}, err
Logger: log,
KeyFolder: dirPaths.KeyFolder,
DownloadFolder: dirPaths.DownloadFolder,
SyncFolder: dirPaths.SyncFolder,
ThumbFolder: dirPaths.ThumbFolder,
DiffFolder: dirPaths.DiffFolder,
}
gvcDB, err := database.OpenOrCreateDB(dbPath, log)
if err != nil {
@@ -52,7 +45,8 @@ func NewManager(rootDir string, version string, dbPath string, informer chan Ope
return &m, nil
}
func checkPaths(rootDir string) (filePaths *FilePaths, err error) {
// CheckPaths just checks the .gvc folder structure
func CheckPaths(rootDir string) (filePaths *FilePaths, err error) {
// checking for the .gvc folder (the client (but not the server) already checks for the .gvc folder, but this checks all subdirects to make sure they are there)
rootFolder, err := filepath.Abs(rootDir)
if err != nil {
@@ -110,6 +104,8 @@ func (m *Manager) AddFileToRepo(relFilePath string) error {
//see commsManagment.go
// f := NewFileManager()
//DELAYED: this feature affects only large files and user experience. It can wait.
relFilePath = strings.TrimSpace(relFilePath) //purging any odd spaces TODO: Make sure not needed
fmt.Println("Checking for file: ", relFilePath)
var tmpFile database.File
filename := filepath.Base(relFilePath)
var hash [16]byte
@@ -123,30 +119,29 @@ func (m *Manager) AddFileToRepo(relFilePath string) error {
if hash, err = engine.UniqueFileHash(relFilePath); err != nil {
return err
}
if tmpFile, err = m.dB.CheckIfFileCurrentlyMonitored(relFilePath, hash); err != nil {
if strings.Index(err.Error(), "not found") != -1 {
//the file wasn't found, this is an ok error
m.Info().Msgf("The file was [%s], so continuing to create it in the database", err)
} else {
return fmt.Errorf("File was not found in repo, please add file first")
}
tmpFile.CurrentHash = hash
tmpFile.Name = filename
tmpFile.Path = relFilePath
tmpFile.CreatedAt = time.Now()
tmpFile.Unique = base64.URLEncoding.EncodeToString([]byte(filename)) + "_" + base64.URLEncoding.EncodeToString((tmpFile.CurrentHash[:])) + "_" + strconv.FormatInt(tmpFile.CreatedAt.Unix(), 10) + "_" + filename
//tmpFile.BkpLocation = filepath.Join(m.SyncFolder, tmpFile.Unique)
//tmpFile.CurrentBase = tmpFile.BkpLocation
//tmpFile.Ignore = false //we can have files in the database that are ignored. TODO: This was initially added so that 'All Files' would show up as a file (its a hack as it adds a dummy to the database)
//we should now have a unique name for this file
//if needs be, we can find out the real file name from the string
//the hash will give us a reasonable indication of the similarity of the files
//define filename of backup(s)
_, err := m.prepareDatabaseForFile(tmpFile)
if err != nil {
return err
}
alreadyTracked := m.dB.CheckIfFileCurrentlyMonitored(relFilePath)
if alreadyTracked {
return fmt.Errorf("file already found in tracked files, not adding: %s", relFilePath)
}
tmpFile = database.File{}
tmpFile.CurrentHash = hash
tmpFile.Name = filename
tmpFile.Path = relFilePath
tmpFile.CreatedAt = time.Now()
tmpFile.Unique = base64.URLEncoding.EncodeToString([]byte(filename)) + "_" + base64.URLEncoding.EncodeToString((tmpFile.CurrentHash[:])) + "_" + strconv.FormatInt(tmpFile.CreatedAt.Unix(), 10) + "_" + filename
//tmpFile.BkpLocation = filepath.Join(m.SyncFolder, tmpFile.Unique)
//tmpFile.CurrentBase = tmpFile.BkpLocation
//tmpFile.Ignore = false //we can have files in the database that are ignored. TODO: This was initially added so that 'All Files' would show up as a file (its a hack as it adds a dummy to the database)
//we should now have a unique name for this file
//if needs be, we can find out the real file name from the string
//the hash will give us a reasonable indication of the similarity of the files
//define filename of backup(s)
_, err = m.prepareDatabaseForFile(tmpFile)
if err != nil {
return err
}
m.Info().Msgf("added file: %s at path: %s with hash: %s at time: %s", filename, relFilePath, tmpFile.CurrentHash, tmpFile.CreatedAt.String)
return nil
}