deranjer/gvc
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

cleaning up manager, tying it to add files

Browse Source
This commit is contained in:
deranjer
2020-06-19 17:12:21 -04:00
parent 7610e3f168
commit e75991da22
11 changed files with 140 additions and 148 deletions
Download Patch File Download Diff File Expand all files Collapse all files

76
common/engine/diff.go
View File

@@ -22,14 +22,14 @@ import (
// TODO: Be able to cancel a diff creation (for instance if the user resaves). Does this work? Should we block
// creating diffs within 5 minutes of creating one? Cancelling is probably better at this point.
// it might be nice to inform the user when diffs build up
func manageFileDiffing(ctx context.Context, subject, object, diffStorageLocation string, fs bool, diffChannel chan database.DiffObject, wg *sync.WaitGroup) error {
func manageFileDiffing(ctx context.Context, target, diffobject, commitHashPath string, diffChannel chan database.DiffObject, wg *sync.WaitGroup) error {
var subjectHash, objectHash [16]byte
var targetHash, diffobjectHash [16]byte
var err error
if subjectHash, err = UniqueFileHash(subject); err != nil {
if targetHash, err = UniqueFileHash(target); err != nil {
return err
}
if objectHash, err = UniqueFileHash(object); err != nil {
if diffobjectHash, err = UniqueFileHash(diffobject); err != nil {
return err
}
@@ -37,30 +37,21 @@ func manageFileDiffing(ctx context.Context, subject, object, diffStorageLocation
wg.Add(1)
go func(messages chan<- database.DiffObject) {
defer wg.Done()
var dO database.DiffObject
//doing this on routine to not lose anytime... does it change anything?
dO.Description = ""
dO.Subject = object
dO.Object = subject
//dO.Description = ""
dO.Target = target
dO.DiffObject = diffobject
dO.StartTime = diffTime
dO.SubjectHash = objectHash //TODO: these being the wrong way round is a legacy thing. Swapping them needs testing, but should be fine
dO.ObjectHash = subjectHash
dO.TargetHash = targetHash //TODO: these being the wrong way round is a legacy thing. Swapping them needs testing, but should be fine
dO.DiffObjectHash = diffobjectHash
fmt.Println("creating diff object now")
if diff, err := binaryDiff(ctx, &dO, diffStorageLocation, fs); err != nil { //binaryDiff
if diff, err := binaryDiff(ctx, &dO, commitHashPath); err != nil { //binaryDiff
fmt.Println("error from binary diff ", err)
dO.E = err
} else {
dO.Diff = &diff
dO.Diff = &diff //Storing it in memory as a complete failure //TODO: Remove this at some point
}
// ssStruct := <-ssChannel
// fmt.Printf("received over ssChannel %+v\r\n", ssStruct)
// if ssStruct.ScreenshotError != nil {
// fmt.Println("screenshot failed, ", ssStruct.ScreenshotError)
// } else {
// fmt.Println("diff reeived screenshot ", ssStruct.Screenshot)
// dO.Screenshot = ssStruct.Screenshot
// }
elapsed := time.Since(diffTime)
dO.Message = "elapsed time:" + elapsed.String()
messages <- dO
@@ -69,7 +60,7 @@ func manageFileDiffing(ctx context.Context, subject, object, diffStorageLocation
}
//run instead of binaryDiff to turn it off
func dryrun(ctx context.Context, dO *database.DiffObject, diffStorageLocation string, fs bool) ([]byte, error) {
func dryrun(ctx context.Context, dO *database.DiffObject, commitHash string) ([]byte, error) {
return []byte{}, nil
}
@@ -80,41 +71,24 @@ func dryrun(ctx context.Context, dO *database.DiffObject, diffStorageLocation st
// 2. Whether to save diffs in both directions
// 3. Creates a diff object that contains any necessary metadata about the diff files
// subject is the file that changed, object is file on record
func binaryDiff(ctx context.Context, dO *database.DiffObject, diffStorageLocation string, fs bool) ([]byte, error) {
func binaryDiff(ctx context.Context, dO *database.DiffObject, commitHashPath string) ([]byte, error) {
var fileName string
_, fileName = filepath.Split(dO.Subject) // dirPath
dO.Watching = fileName
// var sub io.Reader
// if sub, err = os.Open(dO.Subject); err != nil {
// return []byte{}, err
// }
// var obj io.Reader
// if obj, err = os.Open(dO.Object); err != nil {
// return []byte{}, err
// }
_, fileName = filepath.Split(dO.Target) // dirPath
startTime := strconv.FormatInt(dO.StartTime.Unix(), 10)
if fs { //if the wish is to store to the filesystem
dO.DiffPath = filepath.Join(diffStorageLocation, fileName+"_"+startTime+"_"+dO.Description) + "_diff.patch"
if writeDiff, err := os.Create(dO.DiffPath); err != nil {
dO.DiffPath = filepath.Join(commitHashPath, fileName+"_"+startTime) + "_diff.patch"
if writeDiff, err := os.Create(dO.DiffPath); err != nil {
return []byte{}, err
} else if deltaBytes, err := fdeltaDiff(ctx, dO.Target, dO.DiffObject); err != nil {
return []byte{}, err
} else {
bytesWritten, err := writeDiff.Write(deltaBytes)
if err != nil {
return []byte{}, err
} else if deltaBytes, err := fdeltaDiff(ctx, dO.Subject, dO.Object); err != nil {
return []byte{}, err
} else {
if bytesWritten, err := writeDiff.Write(deltaBytes); err != nil {
return []byte{}, err
} else {
dO.DiffSize = int64(bytesWritten)
return []byte{}, nil
}
}
} else { //if we actually want the bytes we have to set fs to false (can do this above.)
if deltaBytes, err := fdeltaDiff(ctx, dO.Subject, dO.Object); err != nil {
return []byte{}, err
} else {
dO.DiffSize = int64(len(deltaBytes))
return deltaBytes, nil
}
dO.DiffSize = int64(bytesWritten)
return []byte{}, nil
}
}
//sub is the original

26
common/engine/filesystem.go
View File

@@ -4,10 +4,8 @@ import (
"bytes"
"compress/gzip"
"encoding/binary"
"errors"
"fmt"
"os"
"path/filepath"
"strings"
logger "github.com/apsdehal/go-logger"
@@ -66,19 +64,19 @@ func ExpandToIntArray(length int64, arry []byte, intArray *[]int64) error {
// VerifySrcFile checks to see that the file is a regular file
// that the OS has meta information about and that can be read by
// the os.
func VerifySrcFile(src string) (string, error) {
_, fileName := filepath.Split(src) //dirPath
sourceFileStat, err := os.Stat(src)
if err != nil {
return fileName, errors.New("error on os.Stat " + err.Error())
}
// the os. Currently done in client or server before handing off to engine since some checks don't involve database/manager
// func VerifySrcFile(src string) (string, error) {
// _, fileName := filepath.Split(src) //dirPath
// sourceFileStat, err := os.Stat(src)
// if err != nil {
// return fileName, errors.New("error on os.Stat " + err.Error())
// }
if !sourceFileStat.Mode().IsRegular() {
return fileName, errors.New("%s is not a regular file" + src)
}
return fileName, nil
}
// if !sourceFileStat.Mode().IsRegular() {
// return fileName, errors.New("%s is not a regular file" + src)
// }
// return fileName, nil
// }
//InitiateDirectory checks all of the directories to make sure they exist
func InitiateDirectory(directory string) {

9
common/engine/patcher.go
View File

@@ -2,8 +2,6 @@ package engine
import (
"fmt"
"github.com/rs/zerolog"
)
// The watcher is responsible for not only seeing when a file changes,
@@ -12,13 +10,6 @@ import (
// * copying any versions and keeping them safe (even if temporary)
// * creating the diff of the file, in both directions if necessary
// * storing the details in the database
func NewPatcher(logger *zerolog.Logger, KeyFolder, DownloadFolder, SyncFolder, ThumbFolder, DiffFolder string) (Patcher, error) {
p := Patcher{
logger,
KeyFolder, DownloadFolder, SyncFolder, ThumbFolder, DiffFolder,
}
return p, nil
}
// PatchFromFile takes the version of the file that was backed up
// and applies the specified patch to it, to get the latest file. This is incase the

2
common/engine/watcher.go
View File

@@ -103,7 +103,7 @@ func (fw *FileWatcher) BeginWatcherRoutine(ctx context.Context, wg *sync.WaitGro
Total: 100,
}
eventContext := context.WithValue(cancelContext, key(event.Path), e)
if err := manageFileDiffing(eventContext, event.Path, syncFilePath, fw.DiffFolder, true, diffChannel, wg); err != nil {
if err := manageFileDiffing(eventContext, event.Path, syncFilePath, fw.DiffFolder, diffChannel, wg); err != nil {
// I don't think this can be reached...
fw.Warn().Msgf("Error managing the diffing process %s", err)
}