making changes to common library, starting to integrate database functions

common/engine/diff.go

@@ -0,0 +1,136 @@
+package engine
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strconv"
+	"sync"
+	"time"
+
+	"github.com/deranjer/gvc/common/database"
+)
+
+// ManageFileDiffing handles creating the diffs on the background routines and creating the information
+// about each diff that is made.
+//
+// TODO: fs works however it takes a while to write the diffs to disk. It maybe a better idea to keep the diffs
+// in memory (although they could get huge??) and then write them to disk at a later point in time.
+// In any event, this works now.
+//
+// TODO: Be able to cancel a diff creation (for instance if the user resaves). Does this work? Should we block
+// creating diffs within 5 minutes of creating one? Cancelling is probably better at this point.
+// it might be nice to inform the user when diffs build up
+func manageFileDiffing(ctx context.Context, subject, object, diffStorageLocation string, fs bool, diffChannel chan database.DiffObject, wg *sync.WaitGroup) error {
+
+	var subjectHash, objectHash [16]byte
+	var err error
+	if subjectHash, err = UniqueFileHash(subject); err != nil {
+		return err
+	}
+	if objectHash, err = UniqueFileHash(object); err != nil {
+		return err
+	}
+
+	diffTime := time.Now()
+	wg.Add(1)
+	go func(messages chan<- database.DiffObject) {
+		defer wg.Done()
+
+		var dO database.DiffObject
+		//doing this on routine to not lose anytime... does it change anything?
+		dO.Description = ""
+		dO.Subject = object
+		dO.Object = subject
+		dO.StartTime = diffTime
+		dO.SubjectHash = objectHash //TODO: these being the wrong way round is a legacy thing. Swapping them needs testing, but should be fine
+		dO.ObjectHash = subjectHash
+		fmt.Println("creating diff object now")
+		if diff, err := binaryDiff(ctx, &dO, diffStorageLocation, fs); err != nil { //binaryDiff
+			fmt.Println("error from binary diff ", err)
+			dO.E = err
+		} else {
+			dO.Diff = &diff
+		}
+		// ssStruct := <-ssChannel
+		// fmt.Printf("received over ssChannel %+v\r\n", ssStruct)
+		// if ssStruct.ScreenshotError != nil {
+		// 	fmt.Println("screenshot failed, ", ssStruct.ScreenshotError)
+		// } else {
+		// 	fmt.Println("diff reeived screenshot ", ssStruct.Screenshot)
+		// 	dO.Screenshot = ssStruct.Screenshot
+		// }
+		elapsed := time.Since(diffTime)
+		dO.Message = "elapsed time:" + elapsed.String()
+		messages <- dO
+	}(diffChannel)
+	return nil
+}
+
+//run instead of binaryDiff to turn it off
+func dryrun(ctx context.Context, dO *database.DiffObject, diffStorageLocation string, fs bool) ([]byte, error) {
+	return []byte{}, nil
+}
+
+// Diff manages the creation of the diffs but doesn't actually create the diffs itself.
+// Sources are file system sources in this case and an array of diffs (io.Writers) are returned
+// 1. This handles whether to save the diffs directly to the drive, and if so, will save to the
+// specified location. If so, it will return the diffs.
+// 2. Whether to save diffs in both directions
+// 3. Creates a diff object that contains any necessary metadata about the diff files
+// subject is the file that changed, object is file on record
+func binaryDiff(ctx context.Context, dO *database.DiffObject, diffStorageLocation string, fs bool) ([]byte, error) {
+	var fileName string
+	_, fileName = filepath.Split(dO.Subject) // dirPath
+	dO.Watching = fileName
+	// var sub io.Reader
+	// if sub, err = os.Open(dO.Subject); err != nil {
+	// 	return []byte{}, err
+	// }
+	// var obj io.Reader
+	// if obj, err = os.Open(dO.Object); err != nil {
+	// 	return []byte{}, err
+	// }
+	startTime := strconv.FormatInt(dO.StartTime.Unix(), 10)
+	if fs { //if the wish is to store to the filesystem
+		dO.DiffPath = filepath.Join(diffStorageLocation, fileName+"_"+startTime+"_"+dO.Description) + "_diff.patch"
+		if writeDiff, err := os.Create(dO.DiffPath); err != nil {
+			return []byte{}, err
+		} else if deltaBytes, err := fdeltaDiff(ctx, dO.Subject, dO.Object); err != nil {
+			return []byte{}, err
+		} else {
+			if bytesWritten, err := writeDiff.Write(deltaBytes); err != nil {
+				return []byte{}, err
+			} else {
+				dO.DiffSize = int64(bytesWritten)
+				return []byte{}, nil
+			}
+		}
+	} else { //if we actually want the bytes we have to set fs to false (can do this above.)
+		if deltaBytes, err := fdeltaDiff(ctx, dO.Subject, dO.Object); err != nil {
+			return []byte{}, err
+		} else {
+			dO.DiffSize = int64(len(deltaBytes))
+			return deltaBytes, nil
+		}
+	}
+}
+
+//sub is the original
+func fdeltaDiff(ctx context.Context, sub, obj string) ([]byte, error) {
+	//now follow what is found in fdelta to retrieve the bytes and get back a delta
+	//you can use the gob/compression code to save the files according to where in pickle it they are saved
+	//TODO: currently the code is used to compress the bsdiff index, but we dont need that, just need to store the
+	// delta on disk. This is currently already done somewhere, so can possibly add/swap out the delta and compressor code
+	// so that it uses the new code.
+	if originalBytes, err := getOriginalBytes(sub); err != nil {
+		return []byte{}, err
+	} else if deltaBytes, err := createDelta(obj, originalBytes); err != nil {
+		return []byte{}, err
+	} else if compressedDelta, err := compressDelta(deltaBytes); err != nil {
+		return []byte{}, err
+	} else {
+		return compressedDelta, nil
+	}
+}