deranjer/gvc
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

cleaning up manager, tying it to add files

Browse Source
This commit is contained in:
deranjer
2020-06-19 17:12:21 -04:00
parent 7610e3f168
commit e75991da22
11 changed files with 140 additions and 148 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
common/database/db-ops.go
View File

@@ -74,7 +74,7 @@ func (db *DB) RetrieveWatchedFiles() ([]File, error) {
// InitialiseFileInDatabase should be called before any file is copied/renamed/diff'd/patched,
// and this should be checked before any operation occurs on a file. Any loss of data is completely as a result
// of losing references
func (db *DB) InitialiseFileInDatabase(file File) (int, error) {
func (db *DB) InitializeFileInDatabase(file File) (int, error) {
if err := db.Save(&file); err != nil {
db.Err(err).Msg("Error initialising file in database")
return file.ID, err
@@ -179,7 +179,7 @@ func (db *DB) RetrieveDiffsByID(ID int) (DiffObject, error) {
// UpdateDescription is a simple function to set the label on a patch
func (db *DB) UpdateDescription(patchID int, description string) error {
fmt.Println("attempting to path with id ", patchID, " description ", description)
if err := db.Update(&DiffObject{ID: patchID, Description: description}); err != nil {
if err := db.Update(&DiffObject{ID: patchID}); err != nil {
db.Err(err).Msg("Error changing diff label")
return err
}

36
common/database/structures.go
View File

@@ -22,10 +22,10 @@ type CommitMeta struct {
// File represents a tracked file
type File struct {
ID int `storm:"id,increment"`
Path string `storm:"index"`
Name string
BkpLocation string //TODO: Needed?
ID int `storm:"id,increment"`
Path string `storm:"index"`
Name string
//BkpLocation string //TODO: Needed?
CurrentBase string
CurrentHash [16]byte `storm:"index,unique"`
CreatedAt time.Time
@@ -44,20 +44,20 @@ type FileIndex struct {
// DiffObject store the information for each diff that is made
type DiffObject struct {
ID int `storm:"id,increment"`
Subject string `storm:"index"`
Object string `storm:"index"`
SubjectHash [16]byte `storm:"index"`
ObjectHash [16]byte `storm:"index"`
Watching string //name of the file being watched
DiffPath string //path of the diff/patch
ID int `storm:"id,increment"`
Target string `storm:"index"`
DiffObject string `storm:"index"`
TargetHash [16]byte `storm:"index"`
DiffObjectHash [16]byte `storm:"index"`
//Watching string //name of the file being watched
DiffPath string //path of the diff/patch //path would be .gvc/hashofcommit/
//Label string //store a comment if the user wants to (user written)
//Screenshot string //path to the screen shot when the diff was made
Fs bool //whether it was written to the directly
Description string //record of forward or backward (just a quick helper)
E error //a record of the error when it was created. Maybe able to optimize out later
Diff *[]byte //the diff itself (incase we want to store in memory) - unused as of now
DiffSize int64 //the size of the diff in bytes
StartTime time.Time //when was the diff created (can take a while to create)
Message string //any message we want to store against the diff while its created
//Fs bool //whether it was written to the directly
//Description string //record of forward or backward (just a quick helper)
E error //a record of the error when it was created. Maybe able to optimize out later
Diff *[]byte //the diff itself (incase we want to store in memory)
DiffSize int64 //the size of the diff in bytes
StartTime time.Time //when was the diff created (can take a while to create)
Message string //any message we want to store against the diff while its created
}

76
common/engine/diff.go
View File

@@ -22,14 +22,14 @@ import (
// TODO: Be able to cancel a diff creation (for instance if the user resaves). Does this work? Should we block
// creating diffs within 5 minutes of creating one? Cancelling is probably better at this point.
// it might be nice to inform the user when diffs build up
func manageFileDiffing(ctx context.Context, subject, object, diffStorageLocation string, fs bool, diffChannel chan database.DiffObject, wg *sync.WaitGroup) error {
func manageFileDiffing(ctx context.Context, target, diffobject, commitHashPath string, diffChannel chan database.DiffObject, wg *sync.WaitGroup) error {
var subjectHash, objectHash [16]byte
var targetHash, diffobjectHash [16]byte
var err error
if subjectHash, err = UniqueFileHash(subject); err != nil {
if targetHash, err = UniqueFileHash(target); err != nil {
return err
}
if objectHash, err = UniqueFileHash(object); err != nil {
if diffobjectHash, err = UniqueFileHash(diffobject); err != nil {
return err
}
@@ -37,30 +37,21 @@ func manageFileDiffing(ctx context.Context, subject, object, diffStorageLocation
wg.Add(1)
go func(messages chan<- database.DiffObject) {
defer wg.Done()
var dO database.DiffObject
//doing this on routine to not lose anytime... does it change anything?
dO.Description = ""
dO.Subject = object
dO.Object = subject
//dO.Description = ""
dO.Target = target
dO.DiffObject = diffobject
dO.StartTime = diffTime
dO.SubjectHash = objectHash //TODO: these being the wrong way round is a legacy thing. Swapping them needs testing, but should be fine
dO.ObjectHash = subjectHash
dO.TargetHash = targetHash //TODO: these being the wrong way round is a legacy thing. Swapping them needs testing, but should be fine
dO.DiffObjectHash = diffobjectHash
fmt.Println("creating diff object now")
if diff, err := binaryDiff(ctx, &dO, diffStorageLocation, fs); err != nil { //binaryDiff
if diff, err := binaryDiff(ctx, &dO, commitHashPath); err != nil { //binaryDiff
fmt.Println("error from binary diff ", err)
dO.E = err
} else {
dO.Diff = &diff
dO.Diff = &diff //Storing it in memory as a complete failure //TODO: Remove this at some point
}
// ssStruct := <-ssChannel
// fmt.Printf("received over ssChannel %+v\r\n", ssStruct)
// if ssStruct.ScreenshotError != nil {
// fmt.Println("screenshot failed, ", ssStruct.ScreenshotError)
// } else {
// fmt.Println("diff reeived screenshot ", ssStruct.Screenshot)
// dO.Screenshot = ssStruct.Screenshot
// }
elapsed := time.Since(diffTime)
dO.Message = "elapsed time:" + elapsed.String()
messages <- dO
@@ -69,7 +60,7 @@ func manageFileDiffing(ctx context.Context, subject, object, diffStorageLocation
}
//run instead of binaryDiff to turn it off
func dryrun(ctx context.Context, dO *database.DiffObject, diffStorageLocation string, fs bool) ([]byte, error) {
func dryrun(ctx context.Context, dO *database.DiffObject, commitHash string) ([]byte, error) {
return []byte{}, nil
}
@@ -80,41 +71,24 @@ func dryrun(ctx context.Context, dO *database.DiffObject, diffStorageLocation st
// 2. Whether to save diffs in both directions
// 3. Creates a diff object that contains any necessary metadata about the diff files
// subject is the file that changed, object is file on record
func binaryDiff(ctx context.Context, dO *database.DiffObject, diffStorageLocation string, fs bool) ([]byte, error) {
func binaryDiff(ctx context.Context, dO *database.DiffObject, commitHashPath string) ([]byte, error) {
var fileName string
_, fileName = filepath.Split(dO.Subject) // dirPath
dO.Watching = fileName
// var sub io.Reader
// if sub, err = os.Open(dO.Subject); err != nil {
// return []byte{}, err
// }
// var obj io.Reader
// if obj, err = os.Open(dO.Object); err != nil {
// return []byte{}, err
// }
_, fileName = filepath.Split(dO.Target) // dirPath
startTime := strconv.FormatInt(dO.StartTime.Unix(), 10)
if fs { //if the wish is to store to the filesystem
dO.DiffPath = filepath.Join(diffStorageLocation, fileName+"_"+startTime+"_"+dO.Description) + "_diff.patch"
if writeDiff, err := os.Create(dO.DiffPath); err != nil {
dO.DiffPath = filepath.Join(commitHashPath, fileName+"_"+startTime) + "_diff.patch"
if writeDiff, err := os.Create(dO.DiffPath); err != nil {
return []byte{}, err
} else if deltaBytes, err := fdeltaDiff(ctx, dO.Target, dO.DiffObject); err != nil {
return []byte{}, err
} else {
bytesWritten, err := writeDiff.Write(deltaBytes)
if err != nil {
return []byte{}, err
} else if deltaBytes, err := fdeltaDiff(ctx, dO.Subject, dO.Object); err != nil {
return []byte{}, err
} else {
if bytesWritten, err := writeDiff.Write(deltaBytes); err != nil {
return []byte{}, err
} else {
dO.DiffSize = int64(bytesWritten)
return []byte{}, nil
}
}
} else { //if we actually want the bytes we have to set fs to false (can do this above.)
if deltaBytes, err := fdeltaDiff(ctx, dO.Subject, dO.Object); err != nil {
return []byte{}, err
} else {
dO.DiffSize = int64(len(deltaBytes))
return deltaBytes, nil
}
dO.DiffSize = int64(bytesWritten)
return []byte{}, nil
}
}
//sub is the original

26
common/engine/filesystem.go
View File

@@ -4,10 +4,8 @@ import (
"bytes"
"compress/gzip"
"encoding/binary"
"errors"
"fmt"
"os"
"path/filepath"
"strings"
logger "github.com/apsdehal/go-logger"
@@ -66,19 +64,19 @@ func ExpandToIntArray(length int64, arry []byte, intArray *[]int64) error {
// VerifySrcFile checks to see that the file is a regular file
// that the OS has meta information about and that can be read by
// the os.
func VerifySrcFile(src string) (string, error) {
_, fileName := filepath.Split(src) //dirPath
sourceFileStat, err := os.Stat(src)
if err != nil {
return fileName, errors.New("error on os.Stat " + err.Error())
}
// the os. Currently done in client or server before handing off to engine since some checks don't involve database/manager
// func VerifySrcFile(src string) (string, error) {
// _, fileName := filepath.Split(src) //dirPath
// sourceFileStat, err := os.Stat(src)
// if err != nil {
// return fileName, errors.New("error on os.Stat " + err.Error())
// }
if !sourceFileStat.Mode().IsRegular() {
return fileName, errors.New("%s is not a regular file" + src)
}
return fileName, nil
}
// if !sourceFileStat.Mode().IsRegular() {
// return fileName, errors.New("%s is not a regular file" + src)
// }
// return fileName, nil
// }
//InitiateDirectory checks all of the directories to make sure they exist
func InitiateDirectory(directory string) {

9
common/engine/patcher.go
View File

@@ -2,8 +2,6 @@ package engine
import (
"fmt"
"github.com/rs/zerolog"
)
// The watcher is responsible for not only seeing when a file changes,
@@ -12,13 +10,6 @@ import (
// * copying any versions and keeping them safe (even if temporary)
// * creating the diff of the file, in both directions if necessary
// * storing the details in the database
func NewPatcher(logger *zerolog.Logger, KeyFolder, DownloadFolder, SyncFolder, ThumbFolder, DiffFolder string) (Patcher, error) {
p := Patcher{
logger,
KeyFolder, DownloadFolder, SyncFolder, ThumbFolder, DiffFolder,
}
return p, nil
}
// PatchFromFile takes the version of the file that was backed up
// and applies the specified patch to it, to get the latest file. This is incase the

2
common/engine/watcher.go
View File

@@ -103,7 +103,7 @@ func (fw *FileWatcher) BeginWatcherRoutine(ctx context.Context, wg *sync.WaitGro
Total: 100,
}
eventContext := context.WithValue(cancelContext, key(event.Path), e)
if err := manageFileDiffing(eventContext, event.Path, syncFilePath, fw.DiffFolder, true, diffChannel, wg); err != nil {
if err := manageFileDiffing(eventContext, event.Path, syncFilePath, fw.DiffFolder, diffChannel, wg); err != nil {
// I don't think this can be reached...
fw.Warn().Msgf("Error managing the diffing process %s", err)
}

76
common/manager/manager.go
View File

@@ -2,6 +2,7 @@ package manager
import (
"encoding/base64"
"fmt"
"path/filepath"
"strconv"
"strings"
@@ -20,7 +21,15 @@ func NewManager(rootDir string, version string, dbPath string, informer chan Ope
if err != nil {
return &Manager{}, err
}
patcher, err := engine.NewPatcher(log, dirPaths)
// Create new patcher
patcher := engine.Patcher{
log,
dirPaths.KeyFolder,
dirPaths.DownloadFolder,
dirPaths.SyncFolder,
dirPaths.ThumbFolder,
dirPaths.DiffFolder,
}
if err != nil {
log.Fatal().Msgf("Error creating a patcher %s", err)
return &Manager{}, err
@@ -86,12 +95,12 @@ func checkPaths(rootDir string) (filePaths *FilePaths, err error) {
// do checks to make sure that it is successfully monitoring it, and that there
// is a historical breadcrumb trail to recreate all the versions that the database
// claims to have a copy of
func (m *Manager) AddFileToRepo(file string, hardCopy bool) (string, error) {
func (m *Manager) AddFileToRepo(relFilePath string) error {
var err error
// the filepath should be absolute, but this should be done dynamically
if file, err = filepath.Abs(file); err != nil {
return "", err
}
// if file, err = filepath.Abs(file); err != nil {
// return "", err
// }
//TODO: what needs to happen is a channel for errors/progress is created
//then pass that channel to a routine, and put all of the following in it
// whenever an error returns, fire the string to the channel,
@@ -101,60 +110,45 @@ func (m *Manager) AddFileToRepo(file string, hardCopy bool) (string, error) {
//see commsManagment.go
// f := NewFileManager()
//DELAYED: this feature affects only large files and user experience. It can wait.
var tmpFile database.File
var filename string //we might aswell only verify the files validity once
filename := filepath.Base(relFilePath)
var hash [16]byte
//check that the file actually exists
if filename, err = engine.VerifySrcFile(file); err != nil {
//there was no source file or it was not recognisable as a file
return "", err
}
//check that the file actually exists (currently done by client/server)
// if filename, err = engine.VerifySrcFile(relFilePath); err != nil {
// //there was no source file or it was not recognisable as a file
// return "", err
// }
//generate a unique file name from the hash and the moment it was created
//a sampled (and therefore) fast, hash of the file for 'uniqueness'
if hash, err = engine.UniqueFileHash(file); err != nil {
return "", err
if hash, err = engine.UniqueFileHash(relFilePath); err != nil {
return err
}
if tmpFile, err = m.dB.CheckIfFileCurrentlyMonitored(file, hash); err != nil {
if tmpFile, err = m.dB.CheckIfFileCurrentlyMonitored(relFilePath, hash); err != nil {
if strings.Index(err.Error(), "not found") != -1 {
//the file wasn't found, this is an ok error
m.Info().Msgf("The file was [%s], so continuing to create it in the database", err)
} else {
m.Error().Msgf("Error checking if file [%s] is monitored so will init file. Error: %s", tmpFile.Path, err)
return fmt.Errorf("File was not found in repo, please add file first")
}
tmpFile.CurrentHash = hash
tmpFile.Name = filename
tmpFile.Path = file
tmpFile.Path = relFilePath
tmpFile.CreatedAt = time.Now()
tmpFile.Unique = base64.URLEncoding.EncodeToString([]byte(filename)) + "_" + base64.URLEncoding.EncodeToString((tmpFile.CurrentHash[:])) + "_" + strconv.FormatInt(tmpFile.CreatedAt.Unix(), 10) + "_" + filename
tmpFile.BkpLocation = filepath.Join(SyncFolder, tmpFile.Unique)
tmpFile.CurrentBase = tmpFile.BkpLocation
//tmpFile.BkpLocation = filepath.Join(m.SyncFolder, tmpFile.Unique)
//tmpFile.CurrentBase = tmpFile.BkpLocation
//tmpFile.Ignore = false //we can have files in the database that are ignored. TODO: This was initially added so that 'All Files' would show up as a file (its a hack as it adds a dummy to the database)
//we should now have a unique name for this file
//if needs be, we can find out the real file name from the string
//the hash will give us a reasononable indication of the similarity of the files
//the hash will give us a reasonable indication of the similarity of the files
//define filename of backup(s)
if _, err := m.prepareDatabaseForFile(tmpFile); err != nil {
return "", err
} else {
if err := m.copyFileToNewLocation(tmpFile.Path, tmpFile.BkpLocation, hardCopy); err != nil {
m.ErrorF("There was an error copying the file to the backup location %s", err)
return "", err
}
m.Informer <- Op_NewFile.Retrieve()
}
} else {
m.Debug().Msgf("file [%s] is already in the database. Assuming sync file in place", tmpFile.Path)
// we should check if the backup file exists, otherwise there is an issue
if _, err := engine.VerifySrcFile(tmpFile.BkpLocation); err != nil {
//if the backup doesn't exist, something has gone quite wrong....
m.Debug().Msgf("The backup file doesn't seem to exist at the expected location, %s", err)
return "", err
_, err := m.prepareDatabaseForFile(tmpFile)
if err != nil {
return err
}
}
return tmpFile.Path, m.watcher.Add(tmpFile.Path)
m.Info().Msgf("added file: %s at path: %s with hash: %s at time: %s", filename, relFilePath, tmpFile.CurrentHash, tmpFile.CreatedAt.String)
return nil
}
// prepareDatabaseForFile is responsible for keeping all references to the version of the file,
@@ -162,8 +156,8 @@ func (m *Manager) AddFileToRepo(file string, hardCopy bool) (string, error) {
//
// TODO: This will need to initialise a diff object in the database, currently created by the diff package,
// however going forward a diff maybe defined by the manager.
func (m *Manager) prepareDatabaseForFile(tmpFile engine.File) (int, error) {
fileID, err := m.dB.InitialiseFileInDatabase(tmpFile)
func (m *Manager) prepareDatabaseForFile(tmpFile database.File) (int, error) {
fileID, err := m.dB.InitializeFileInDatabase(tmpFile)
if err != nil {
m.Error().Msgf("Error checking if file [%s] is monitored. Error %s", tmpFile.Path, err)
return 0, err