diff --git a/common/database/create-db.go b/common/database/create-db.go new file mode 100644 index 0000000..dc9d029 --- /dev/null +++ b/common/database/create-db.go @@ -0,0 +1,32 @@ +package database + +import ( + "io" + + logger "github.com/apsdehal/go-logger" + "github.com/asdine/storm" +) + +type DB struct { + *storm.DB + *logger.Logger +} + +// NewDB returns a new database object, +// it configures the database for you. +func NewDB(dbPath, format string, logLevel int, logWriter io.WriteCloser) (*DB, error) { + //Note! Do not use logger as you have no idea if logWriter has been configured for output yet + var db DB + log, err := logger.New("db logger", 1, logWriter) + if err != nil { + return &db, err + } + log.SetLogLevel(logger.LogLevel(logLevel)) + log.SetFormat(format) + db.Logger = log + if err := db.ConfigureDB(dbPath); err != nil { + log.ErrorF("Error configuring the database ", err) + return &db, err + } + return &db, nil +} diff --git a/common/database/db-ops.go b/common/database/db-ops.go new file mode 100644 index 0000000..ca6e631 --- /dev/null +++ b/common/database/db-ops.go @@ -0,0 +1,189 @@ +package database + +import ( + "fmt" + + "github.com/asdine/storm" +) + +// ConfigureDB sets up bolt and Storm according to the path of the database +// this is done here so that different databases can be configured in different scenarios +func (db *DB) ConfigureDB(dbPath string) error { + var err error + if db.DB, err = storm.Open(dbPath); err != nil { + return err + } + var file File + if err := db.One("Name", "-- All Files --", &file); err != nil { + if err.Error() != "not found" { + db.ErrorF("Error finding file by path %s", err) + return err + } + db.WarningF("No file found. initialising the database") + file.Name = "-- All Files --" + //file.Ignore = true //this is currently not used however could result in this file being ignored when file watching (etc) starts + if err := db.Save(&file); err != nil { + db.ErrorF("Error storing the diff %s", err) + return err + } + } + return nil +} + +// CheckIfFileCurrentlyMonitored checks if the file is already being monitored. This is a read-only check +// to see whether the file was correctly initialised +// (BUG) The hash causes the same file to be in database multiple times! +func (db *DB) CheckIfFileCurrentlyMonitored(src string, hash [16]byte) (File, error) { + var file File + + //TODO: check this actually works still (don't need hash passed to this anymore) + if err := db.One("Path", src, &file); err != nil { + if err.Error() != "not found" { + db.ErrorF("Error finding file by path %s", err) + return File{}, err + } + db.WarningF("no file found, %s", err) + return File{}, err + } else { + return file, nil + } +} + +// RetrieveWatchedFiles all files that are in the database as "watched files" +// This can be used to trigger the same files to be watched again +func (db *DB) RetrieveWatchedFiles() ([]File, error) { + var files []File + if err := db.All(&files); err != nil { + db.ErrorF("Error retrieving all watched files %s", err) + return []File{}, err + } else { + return files, nil + } +} + +// RetrieveAllDiffs retrieves all files that are in the database as "watched files" +// This can be used to trigger the same files to be watched again +// func (db *DB) RetrieveAllDiffs() ([]DiffObject, error) { +// var diffs []DiffObject +// if err := db.All(&diffs); err != nil { +// db.ErrorF("Error retrieving all diffs %s", err) +// return []DiffObject{}, err +// } else { +// return diffs, nil +// } +// } + +// InitialiseFileInDatabase should be called before any file is copied/renamed/diff'd/patched, +// and this should be checked before any operation occurs on a file. Any loss of data is completely as a result +// of losing references +func (db *DB) InitialiseFileInDatabase(file File) (int, error) { + if err := db.Save(&file); err != nil { + db.ErrorF("Error initialising file in database %s", err) + return file.ID, err + } + return file.ID, nil +} + +// FindFileByPath is a search function looking for file details +func (db *DB) FindFileByPath(filePath string) (File, error) { + var file File + if err := db.One("Path", filePath, &file); err != nil { + db.ErrorF("Error finding file by path %s", err) + return File{}, err + } + return file, nil +} + +// FindFileByID is a search function looking for file details +func (db *DB) FindFileByID(ID int) (File, error) { + var file File + if err := db.One("ID", ID, &file); err != nil { + db.ErrorF("Error finding file by path %s", err) + return File{}, err + } + return file, nil +} + +// UpdateFileData updates the current base file that diffs will compare to +func (db *DB) UpdateFileData(filePath, basePath string, hash [16]byte) error { + if file, err := db.FindFileByPath(filePath); err != nil { + db.ErrorF("Error updating the file base %s", err) + return err + } else { + err := db.Update(&File{ID: file.ID, CurrentBase: basePath, CurrentHash: hash}) + return err + } +} + +// RetrieveDiffsForFileByHash returns the diffs for a file. Diffs can be applied to a specific file (by its hash), +// so when looking for the diffs, the hash is a good place to start in terms of finding the diffs +func (db *DB) RetrieveDiffsForFileByHash(fileHash [16]byte, direction bool) ([]DiffObject, error) { + var diffs []DiffObject + var field string + if direction { + field = "ObjectHash" + } else { + field = "SubjectHash" + } + if err := db.Find(field, fileHash, &diffs); err != nil { + return []DiffObject{}, err + } + return diffs, nil +} + +// RetrieveDiffsForFileByPath returns the diffs for a file base on the file path. Diffs are very specific to a file, +// so this may not be as robust as searching by diff, however we may not have the diff available +func (db *DB) RetrieveDiffsForFileByPath(filePath string) ([]DiffObject, error) { + var objDiffs []DiffObject + var subDiffs []DiffObject + if err := db.Find("Object", filePath, &objDiffs); err != nil && err.Error() != "not found" { + db.ErrorF("Error finding diff by object %s", err) + return []DiffObject{}, err + } + if err := db.Find("Subject", filePath, &subDiffs); err != nil && err.Error() != "not found" { + db.ErrorF("Error finding diff by subject %s", err) + return []DiffObject{}, err + } + return append(objDiffs, subDiffs...), nil +} + +// StoreDiff just places the information about a diff in the database. Currently there is no protection +// to stop the entire diff entering the database (if fs is false), which may be very slow/bulky... +// TODO: decide what to do with diffs in memory +// func (db *DB) StoreDiff(diff DiffObject) error { +// if err := db.Save(&diff); err != nil { +// db.ErrorF("Error storing the diff %s", err) +// return err +// } +// return nil +// } + +// FindDiffByPath is a search function looking for a diff +func (db *DB) FindDiffByPath(patchPath string) (DiffObject, error) { + var diff DiffObject + if err := db.One("DiffPath", patchPath, &diff); err != nil { + db.ErrorF("Error finding diff by path %s", err) + return DiffObject{}, err + } + return diff, nil +} + +//RetrieveDiffsByID returns a diff based on the id it has in the database +func (db *DB) RetrieveDiffsByID(ID int) (DiffObject, error) { + var diff DiffObject + if err := db.One("ID", ID, &diff); err != nil { + db.ErrorF("Error finding diff by ID %s", err) + return DiffObject{}, err + } + return diff, nil +} + +// UpdateDescription is a simple function to set the label on a patch +func (db *DB) UpdateDescription(patchID int, description string) error { + fmt.Println("attempting to path with id ", patchID, " description ", description) + if err := db.Update(&DiffObject{ID: patchID, Description: description}); err != nil { + db.ErrorF("Error changing diff label %s", err) + return err + } + return nil +} diff --git a/common/database/structures.go b/common/database/structures.go new file mode 100644 index 0000000..e5420f3 --- /dev/null +++ b/common/database/structures.go @@ -0,0 +1,63 @@ +package database + +import "time" + +// Commit stores all the necessary information for a commit +type Commit struct { + CommitHash string // The hash of the commit (generated by hashing commit author name, time, the previous commit, and more? TODO: Not sure what else) + TrackedFiles []File // All of the tracked files for this commit + Date string + Version string //User can tag a commit with a version number + +} + +// CommitMeta stores the meta information about the commit +type CommitMeta struct { + Tag string + Flavour string + PersistLogs bool + Production bool + Virtual bool +} + +// File represents a tracked file +type File struct { + ID int `storm:"id,increment"` + Path string `storm:"index"` + Name string + BkpLocation string //TODO: Needed? + CurrentBase string + CurrentHash [16]byte `storm:"index,unique"` + CreatedAt time.Time + Unique string + Version float64 + NoCompress bool // Whether or not to compress this file +} + +type FileIndex struct { + ID int `storm:"id,increment"` + FileID int `storm:"index"` + FileHash [16]byte `storm:"index,unique"` + Index []byte + Length int64 +} + +// DiffObject store the information for each diff that is made +type DiffObject struct { + ID int `storm:"id,increment"` + Subject string `storm:"index"` + Object string `storm:"index"` + SubjectHash [16]byte `storm:"index"` + ObjectHash [16]byte `storm:"index"` + Watching string //name of the file being watched + DiffPath string //path of the diff/patch + //Label string //store a comment if the user wants to (user written) + //Screenshot string //path to the screen shot when the diff was made + Fs bool //whether it was written to the directly + Description string //record of forward or backward (just a quick helper) + E error //a record of the error when it was created. Maybe able to optimize out later + //Diff *[]byte //the diff itself (incase we want to store in memory) - unused as of now + DiffSize int64 //the size of the diff in bytes + StartTime time.Time //when was the diff created (can take a while to create) + Message string //any message we want to store against the diff while its created +} diff --git a/common/engine/compressor.go b/common/engine/compressor.go new file mode 100644 index 0000000..ecddf78 --- /dev/null +++ b/common/engine/compressor.go @@ -0,0 +1,70 @@ +package engine + +import ( + "bytes" + "compress/gzip" + "encoding/gob" + "fmt" + "io" +) + +type SomeStruct struct { + A string + B int64 + C float64 +} + +//1. +func StructToBytes(obj SomeStruct) (bytes.Buffer, error) { + //now gob this + var indexBuffer bytes.Buffer + encoder := gob.NewEncoder(&indexBuffer) + if err := encoder.Encode(obj); err != nil { + return indexBuffer, err + } + return indexBuffer, nil +} + +//1. +func BytesToGob(obj []byte) (bytes.Buffer, error) { + //now gob this + var indexBuffer bytes.Buffer + encoder := gob.NewEncoder(&indexBuffer) + if err := encoder.Encode(obj); err != nil { + return indexBuffer, err + } + return indexBuffer, nil +} + +//2. +func CompressBinary(binaryBuffer *bytes.Buffer) (bytes.Buffer, error) { + //now compress it + var compressionBuffer bytes.Buffer + compressor := gzip.NewWriter(&compressionBuffer) + _, err := compressor.Write(binaryBuffer.Bytes()) + err = compressor.Close() + return compressionBuffer, err +} + +//3. +func DecompressBinary(compressionBuffer bytes.Buffer) (*gzip.Reader, error) { + //now decompress it + dataReader := bytes.NewReader(compressionBuffer.Bytes()) + if reader, err := gzip.NewReader(dataReader); err != nil { + fmt.Println("gzip failed ", err) + return &gzip.Reader{}, err + } else { + err := reader.Close() + return reader, err + } +} + +//4. +func GobToBytes(binaryBytes io.Reader) ([]byte, error) { + decoder := gob.NewDecoder(binaryBytes) + var tmp []byte + if err := decoder.Decode(&tmp); err != nil { + return tmp, err + } + return tmp, nil +} diff --git a/common/engine/compressor_test.go b/common/engine/compressor_test.go new file mode 100644 index 0000000..cefff58 --- /dev/null +++ b/common/engine/compressor_test.go @@ -0,0 +1,51 @@ +package engine + +import ( + "fmt" + "io/ioutil" + "testing" +) + +var testStruct = SomeStruct{ + A: "alex walker", + B: 24, + C: 1.234, +} + +func TestMain(t *testing.T) { + //use assert library to check for similarity or require library to check something exists + t.Run("check that we can 'gob' the data correctly", func(t *testing.T) { + if structBytes, err := StructToBytes(testStruct); err != nil { + t.Error("failed to create the bytes from the struct provided ", err) + t.FailNow() + } else if binaryToGobBuffer, err := BytesToGob(structBytes.Bytes()); err != nil { + t.Error("failed to create the gob from the bytes provided ", err) + t.FailNow() + //issue here with gob reading from an interface + } else if compressedData, err := CompressBinary(&binaryToGobBuffer); err != nil { + t.Error("failed to create the gob from the struct provided ", err) + t.FailNow() + } else if decompressionReader, err := DecompressBinary(compressedData); err != nil { + t.Error("failed to decompress the binary data ", err) + t.FailNow() + } else if res, err := GobToBytes(decompressionReader); err != nil { + t.Error("failed to convert bytes to struct ", err) + t.FailNow() + } else { + fmt.Printf("result %+v\r\n", res) + } + }) +} + +func openFile(path string) ([]byte, error) { + data, err := ioutil.ReadFile(path) + if err != nil { + fmt.Println("File reading error", err) + } + return data, err +} + +func writeFile(path string, data []byte) error { + err := ioutil.WriteFile(path, data, 0644) + return err +} diff --git a/common/engine/fdelta.go b/common/engine/fdelta.go new file mode 100644 index 0000000..6d9e2ec --- /dev/null +++ b/common/engine/fdelta.go @@ -0,0 +1,94 @@ +package engine + +import ( + "bytes" + "fmt" + "io" + "io/ioutil" + + "github.com/amlwwalker/fdelta" +) + +// var originalFile string +// var newFile string +// var patchFile string +// var appliedFile string + +func openReader(file io.Reader) ([]byte, error) { + buffer, err := ioutil.ReadAll(file) + return buffer, err +} +func openFile(path string) ([]byte, error) { + data, err := ioutil.ReadFile(path) + if err != nil { + fmt.Println("File reading error", err) + } + return data, err +} + +func writeFile(path string, data []byte) error { + err := ioutil.WriteFile(path, data, 0644) + return err +} + +func getOriginalBytes(originalFile string) ([]byte, error) { + originalBytes, err := openFile(originalFile) + if err != nil { + return []byte{}, err + } + return originalBytes, nil +} +func createDelta(newFile string, originalBytes []byte) ([]byte, error) { + newBytes, err := openFile(newFile) + if err != nil { + return []byte{}, err + } + delta := fdelta.Create(originalBytes, newBytes) + fmt.Println("size of delta ", len(delta)) + return delta, nil +} +func compressDelta(delta []byte) ([]byte, error) { + if binaryToGobBuffer, err := compressor.BytesToGob(delta); err != nil { + return []byte{}, err + } else if compressedData, err := compressor.CompressBinary(&binaryToGobBuffer); err != nil { + return []byte{}, err + } else { + return compressedData.Bytes(), nil + } +} +func storeDelta(patchFile string, delta []byte) ([]byte, error) { + if compressedData, err := compressDelta(delta); err != nil { + return []byte{}, err + } else if err := writeFile(patchFile, compressedData); err != nil { + return []byte{}, err + } else { + return compressedData, nil + } +} +func retrieveDelta(patchFile string) ([]byte, error) { + compressedData, err := openFile(patchFile) + if err != nil { + return []byte{}, err + } + return compressedData, nil +} + +func decompressDelta(compressedData []byte) ([]byte, error) { + var compressedBuffer bytes.Buffer + compressedBuffer.Write(compressedData) + if decompressionReader, err := compressor.DecompressBinary(compressedBuffer); err != nil { + return []byte{}, err + } else if res, err := compressor.GobToBytes(decompressionReader); err != nil { + return []byte{}, err + } else { + return res, nil + } +} + +func applyPatchToFile(originalbytes, delta []byte) ([]byte, error) { + if patchedBytes, err := fdelta.Apply(originalbytes, delta); err != nil { + return []byte{}, err + } else { + return patchedBytes, nil + } +} diff --git a/common/engine/fdelta_test.go b/common/engine/fdelta_test.go new file mode 100644 index 0000000..aea6a82 --- /dev/null +++ b/common/engine/fdelta_test.go @@ -0,0 +1,33 @@ +package engine + +import ( + "fmt" + "os" + + "github.com/amlwwalker/fdelta" +) + +func main() { + DefineFiles() + originalBytes := GetOriginalBytes() + delta := CreateDelta(originalBytes) + StoreDelta(delta) + retrievedDelta := RetrieveDelta() + // var deltaBytes []byte + fmt.Printf("res : `%s`\n", len(retrievedDelta)) + //test loading the delta from disk + appliedBytes, err := fdelta.Apply(originalBytes, retrievedDelta) + if err != nil { + panic(err) + } + fmt.Println("exporting delta") + err = writeFile(appliedFile, appliedBytes) + if err != nil { + fmt.Println("error reading bytes [3]", err) + os.Exit(1) + } + fmt.Printf("Origin : `%s`\n", originalFile) + fmt.Printf("Target : `%s`\n", len(appliedBytes)) + fmt.Printf("Delta : `%s`\n", len(delta)) + fmt.Printf("Result: `%s`\n", appliedFile) +} diff --git a/common/engine/patcher.go b/common/engine/patcher.go new file mode 100644 index 0000000..52c5a20 --- /dev/null +++ b/common/engine/patcher.go @@ -0,0 +1,52 @@ +package engine + +import ( + logger "github.com/apsdehal/go-logger" +) + +// The watcher is responsible for not only seeing when a file changes, +// but also keeping track of +// * the file hash so that if it changes again any modifications can be handled +// * copying any versions and keeping them safe (even if temporary) +// * creating the diff of the file, in both directions if necessary +// * storing the details in the database +func NewPatcher(logger *logger.Logger, KEYFOLDER, DOWNLOADFOLDER, SYNCFOLDER, THUMBFOLDER, DIFFFOLDER string) (Patcher, error) { + p := Patcher{ + logger, + KEYFOLDER, DOWNLOADFOLDER, SYNCFOLDER, THUMBFOLDER, DIFFFOLDER, + } + return p, nil +} + +// PatchFromFile takes the version of the file that was backed up +// and applies the specified patch to it, to get the latest file. This is incase the +// last save is the file you want to get. +func (p *Patcher) PatchFromFile(filePath, patchPath, restorePath string) error { + if subject, err := openFile(filePath); err != nil { + p.ErrorF("error on subject file: ", err) + } else if patch, err := openFile(patchPath); err != nil { + p.ErrorF("error on patch file: ", err) + } else { + return p.applyPatch(subject, patch, restorePath) + } + return nil +} + +//applyPatch actively applies the patch to the subject. This could eventually +// be upgraded for different patching algorithms +func (p *Patcher) applyPatch(subject, patch []byte, restorePath string) error { + if delta, err := decompressDelta(patch); err != nil { + p.ErrorF("error decompressing delta", err) + } else { + if appliedBytes, err := applyPatchToFile(subject, delta); err != nil { + p.ErrorF("error applying delta to original file", err) + return err + } else if err := writeFile(restorePath, appliedBytes); err != nil { + p.ErrorF("error writing patchedFile", err) + return err + } else { + return nil + } + } + return nil +} diff --git a/common/engine/structures.go b/common/engine/structures.go new file mode 100644 index 0000000..c92428e --- /dev/null +++ b/common/engine/structures.go @@ -0,0 +1,25 @@ +package engine + +import ( + logger "github.com/apsdehal/go-logger" + radovskyb "github.com/radovskyb/watcher" +) + +type Watcher struct { + *radovskyb.Watcher + *logger.Logger + Enabled bool + KEYFOLDER string + DOWNLOADFOLDER string + SYNCFOLDER string + THUMBFOLDER string + DIFFFOLDER string +} +type Patcher struct { + *logger.Logger + KEYFOLDER string + DOWNLOADFOLDER string + SYNCFOLDER string + THUMBFOLDER string + DIFFFOLDER string +} diff --git a/common/engine/watcher.go b/common/engine/watcher.go new file mode 100644 index 0000000..27da0ae --- /dev/null +++ b/common/engine/watcher.go @@ -0,0 +1,132 @@ +package engine + +import ( + "context" + "path/filepath" + "sync" + "time" + + logger "github.com/apsdehal/go-logger" + radovskyb "github.com/radovskyb/watcher" +) + +type key string +type Event struct { + Name string + Progress int + Total int +} + +// The watcher is responsible for not only seeing when a file changes, +// but also keeping track of +// * the file hash so that if it changes again any modifications can be handled +// * copying any versions and keeping them safe (even if temporary) +// * creating the diff of the file, in both directions if necessary +// * storing the details in the database +func NewWatcher(logger *logger.Logger, KEYFOLDER, DOWNLOADFOLDER, SYNCFOLDER, THUMBFOLDER, DIFFFOLDER string) (Watcher, error) { + w := Watcher{ + radovskyb.New(), + logger, + true, //used to temporarily ignore events if necessary + KEYFOLDER, DOWNLOADFOLDER, SYNCFOLDER, THUMBFOLDER, DIFFFOLDER, + } + return w, nil +} + +func (w *Watcher) Ignore() bool { + w.Enabled = false + return w.Enabled +} +func (w *Watcher) Enable() bool { + w.Enabled = true + return w.Enabled +} +func (w *Watcher) IsEnabled() bool { + return w.Enabled +} + +// BeginWatcherRoutine kicks off the watcher. When the watcher noticies a file change, +// certain actions will be taken in case of event and error +// the routine will handle these whenever this occurs. +// If certain functions need to be called then this will +// need to be specified as part of the managers lambda functions +// TODO: Should return an error +func (w *Watcher) BeginWatcherRoutine(ctx context.Context, wg *sync.WaitGroup, diffChannel chan utilities.DiffObject, onFileChanged func(string) (utilities.File, error)) { + + //seems a bit barking, but we can now cancel any diff that is occuring on a file when it fires again + cancelFunctions := make(map[string]func()) + for { + select { + // we have filtered already on the [Op]erations we want to listen for so no need to check here + case event := <-w.Event: + if !w.IsEnabled() { + w.Infof("ignoring event and reenabling the watcher %s\r\n", event) + w.Enable() + continue + } + w.Infof("event fired ", event) + //this is currently slow as it does a db lookup on the path. + //TODO: On load (or whenever a file is added to the watcher, the db information for files being watched, could be cached in memory. This would be much faster) + fileInfo, err := onFileChanged(event.Path) //could return the 'Event' object here + syncFilePath := fileInfo.CurrentBase + uniqueName := fileInfo.Unique + // //begin taking screenshot if we are supposed to + screenshotChannel := make(chan utilities.ScreenshotWrapper) + go func(ssChannel chan utilities.ScreenshotWrapper) { + w.Infof("beginning taking screenshot at ", time.Now()) + var ssStruct utilities.ScreenshotWrapper + if screenshotFileName, err := takeScreenShot(w.THUMBFOLDER, uniqueName); err != nil { + w.WarningF("could not take screenshot", err) + ssStruct.ScreenshotError = err + } else { + ssStruct.Screenshot = filepath.Join(w.THUMBFOLDER, screenshotFileName) + w.Infof("screenshot recorded ", ssStruct.Screenshot, " at ", time.Now()) + } + ssChannel <- ssStruct + }(screenshotChannel) + + // fileID := fileInfo.ID + //we need the hash of the current base, not the hash of the original file + // fileHash := fileInfo.CurrentHash //hash needs to come from + if err != nil { + w.ErrorF("path was not returned to sync path", err) + continue + } + //cancel the event if it indeed is running... + if cancelFunctions[event.Path] != nil { + cancelFunctions[event.Path]() + delete(cancelFunctions, event.Path) + } + + //context for the current event. Calling cancel will cancel the routines + //to kill a context you must have access to the cancel function. + // i could add the cancel function to a map of them + // if you want to kill it you call on the correct cancel function + // that will kill the context. Fine. + // If however you want to kill it from another place.... + // then you need a cancel channel, which inturn calls the equivelent cancel function.... ok + // sounds about best i can do right now... + // kind of bonkers right.... + cancelContext, cancel := context.WithCancel(ctx) + cancelFunctions[event.Path] = cancel + // good idea to not use strings as keys directly as can conflict across namespaces + // this needs to be sorted out -- too many things called an event.... + // TODO: its totally bananas + e := Event{ + Name: event.Path, + Progress: 0, + Total: 100, + } + eventContext := context.WithValue(cancelContext, key(event.Path), e) + if err := manageFileDiffing(eventContext, event.Path, syncFilePath, w.DIFFFOLDER, true, screenshotChannel, diffChannel, wg); err != nil { + // I don't think this can be reached... + w.WarningF("Error managing the diffing process %s", err) + } + case err := <-w.Watcher.Error: + w.ErrorF("%s\r\n", err) + case <-w.Closed: + w.Notice("radovskyb closed") + return + } + } +}