adding decompress

client/.gvc/logs/gvclog.log

@@ -243,3 +243,62 @@
 {"level":"error","module":"database","error":"already exists","message":"Error initialising file in database"}
 {"level":"error","message":"Error checking if file [test.go] is monitored. Error already exists"}
 {"level":"info","message":"Adding the following files to tracked: [client.go clientcmd\\add.go clientcmd\\branch.go clientcmd\\commit.go clientcmd\\commonlib.go clientcmd\\ignore.go clientcmd\\info.go clientcmd\\init.go clientcmd\\lock.go clientcmd\\refresh.go clientcmd\\remote.go clientcmd\\switch.go clientconfig\\config.go clientconfig\\structures.go test\\test.go.old test\\test2.go]"}
+{"level":"info","message":"Creating new Manager..."}
+{"level":"info","module":"database","message":"Fetching last commit on branch: master"}
+{"level":"error","module":"database","error":"not found","message":"Failed to find last commit on branch: master"}
+{"level":"info","message":"Creating new Manager..."}
+{"level":"info","module":"database","message":"Fetching last commit on branch: master"}
+{"level":"error","module":"database","error":"not found","message":"Failed to find last commit on branch: master"}
+{"level":"info","message":"Creating new Manager..."}
+{"level":"info","module":"database","message":"Fetching last commit on branch: master"}
+{"level":"error","module":"database","error":"not found","message":"Failed to find last commit on branch: master"}
+{"level":"info","message":"unable to fetch last commit on branch, assuming first commit on branch%!(EXTRA *errors.errorString=not found)"}
+{"level":"info","message":"Starting initial commit on branch: test commit"}
+{"level":"info","message":"Creating new Manager..."}
+{"level":"info","module":"database","message":"Fetching last commit on branch: master"}
+{"level":"error","module":"database","error":"not found","message":"Failed to find last commit on branch: master"}
+{"level":"info","message":"unable to fetch last commit on branch, assuming first commit on branch%!(EXTRA *errors.errorString=not found)"}
+{"level":"info","message":"Starting initial commit on branch: test commit"}
+{"level":"info","message":"Creating new Manager..."}
+{"level":"info","module":"database","message":"Fetching last commit on branch: master"}
+{"level":"error","module":"database","error":"not found","message":"Failed to find last commit on branch: master"}
+{"level":"info","message":"unable to fetch last commit on branch, assuming first commit on branch%!(EXTRA *errors.errorString=not found)"}
+{"level":"info","message":"Starting initial commit on branch: master"}
+{"level":"info","message":"Creating new Manager..."}
+{"level":"info","module":"database","message":"Fetching last commit on branch: master"}
+{"level":"error","module":"database","error":"not found","message":"Failed to find last commit on branch: master"}
+{"level":"info","message":"unable to fetch last commit on branch, assuming first commit on branch%!(EXTRA *errors.errorString=not found)"}
+{"level":"info","message":"Starting initial commit on branch: master"}
+{"level":"info","message":"Creating new Manager..."}
+{"level":"info","module":"database","message":"Fetching last commit on branch: master"}
+{"level":"error","module":"database","error":"not found","message":"Failed to find last commit on branch: master"}
+{"level":"info","message":"unable to fetch last commit on branch, assuming first commit on branch%!(EXTRA *errors.errorString=not found)"}
+{"level":"info","message":"Starting initial commit on branch: master"}
+{"level":"info","message":"Total number of files to commit: %!s(int=16)"}
+{"level":"info","module":"database","message":"Writing new commit: %!s(int=1) on branch: master"}
+{"level":"error","module":"database","error":"provided target must be a pointer to struct","message":"failure to write commit to database"}
+{"level":"error","error":"provided target must be a pointer to struct","message":"unable to create initial commit: provided target must be a pointer to struct"}
+{"level":"info","message":"Creating new Manager..."}
+{"level":"info","module":"database","message":"Fetching last commit on branch: master"}
+{"level":"error","module":"database","error":"not found","message":"Failed to find last commit on branch: master"}
+{"level":"info","message":"unable to fetch last commit on branch, assuming first commit on branch%!(EXTRA *errors.errorString=not found)"}
+{"level":"info","message":"Starting initial commit on branch: master"}
+{"level":"info","message":"Total number of files to commit: %!s(int=16)"}
+{"level":"info","module":"database","message":"Writing new commit: %!s(int=1) on branch: master"}
+{"level":"error","module":"database","error":"missing struct tag id or ID field","message":"failure to write commit to database"}
+{"level":"error","error":"missing struct tag id or ID field","message":"unable to create initial commit: missing struct tag id or ID field"}
+{"level":"info","message":"Creating new Manager..."}
+{"level":"info","module":"database","message":"Fetching last commit on branch: master"}
+{"level":"error","module":"database","error":"not found","message":"Failed to find last commit on branch: master"}
+{"level":"info","message":"unable to fetch last commit on branch, assuming first commit on branch%!(EXTRA *errors.errorString=not found)"}
+{"level":"info","message":"Starting initial commit on branch: master"}
+{"level":"info","message":"Total number of files to commit: %!s(int=16)"}
+{"level":"info","module":"database","message":"Writing new commit: %!s(int=1) on branch: master"}
+{"level":"info","message":"Creating new Manager..."}
+{"level":"info","module":"database","message":"Fetching last commit on branch: master"}
+{"level":"info","message":"Creating new Manager..."}
+{"level":"info","module":"database","message":"Fetching last commit on branch: master"}
+{"level":"info","message":"Creating new Manager..."}
+{"level":"info","module":"database","message":"Fetching last commit on branch: master"}
+{"level":"info","message":"Creating new Manager..."}
+{"level":"info","module":"database","message":"Fetching last commit on branch: master"}

common/database/db-ops.go

@@ -88,7 +88,7 @@ func (db *DB) UpdateFileData(filePath, basePath string, hash []byte) error {
 		db.Err(err).Msg("Error updating the file base")
 		return err
 	} else {
-		err := db.Update(&File{ID: file.ID, CurrentBase: basePath, Hash: hash})
+		err := db.Update(&File{ID: file.ID, BaseFilePath: basePath, Hash: hash})
 		return err
 	}
 }
@@ -198,6 +198,18 @@ func (db *DB) FetchLastCommitOnBranch(branch string) (commitResult Commit, err e
 	err = query.OrderBy("Number").Reverse().First(&commit) // Getting the last entry by number
 	if err != nil {
 		db.Err(err).Msgf("Failed to find last commit on branch: %s", branch)
+		return commit, err
 	}
 	return commit, nil
 }
+
+// NewCommit writes a new commit to the specified branch into the database
+func (db *DB) NewCommit(newCommit Commit, branch string) error {
+	db.Info().Msgf("Writing new commit: %s on branch: %s", newCommit.Number, branch)
+	err := db.Save(&newCommit)
+	if err != nil {
+		db.Err(err).Msg("failure to write commit to database")
+		return err
+	}
+	return nil
+}

common/database/structures.go

@@ -24,7 +24,7 @@ type GVCInfo struct {
 
 // Commit stores all the necessary information for a commit
 type Commit struct {
-	CommitHash   []byte `storm:"index,unique"` // The hash of the commit (generated by hashing commit author name, time, the previous commit, and more? TODO: Not sure what else)
+	CommitHash   []byte `storm:"id,index,unique"` // The hash of the commit (generated by hashing commit author name, time, the previous commit, and more? TODO: Not sure what else)
 	TrackedFiles []File // All of the tracked files for this commit
 	Date         string
 	Version      string //User can tag a commit with a version number
@@ -43,19 +43,19 @@ type CommitMeta struct {
 
 // File represents a tracked file
 type File struct {
-	ID   int    `storm:"id,increment"`
+	ID           int    `storm:"id,increment"`
-	Path string `storm:"index"`
+	Path         string `storm:"index"`
-	Name string
+	BaseFilePath string // This stores the path to the tracked version of the file in the object database
-	//BkpLocation string //TODO: Needed?
+	Name         string
-	CurrentBase string
+	//CurrentBase string
-	Hash        []byte `storm:"index,unique"` // with []byte can't use sha256.sum256 since that is [32]byte, so everything done manually.
+	Hash       []byte `storm:"index,unique"` // with []byte can't use sha256.sum256 since that is [32]byte, so everything done manually.
-	CreatedAt   time.Time
+	CreatedAt  time.Time
-	Unique      string
+	Unique     string
-	Version     float64
+	Version    float64
-	NoCompress  bool // Whether or not to compress this file
+	NoCompress bool // Whether or not to compress this file
 }
 
-// CalculateHash creates a hash for the file
+// CalculateHash creates a hash for the file //Requires path
 func (f *File) CalculateHash() error {
 	fileContents, err := ioutil.ReadFile(f.Path)
 	if err != nil {

common/engine/commits.go

@@ -11,7 +11,8 @@ import (
 )
 
 // CreateInitialCommit copies the files over and compresses them if they are not in the NoCompress struct
-func (m *Manager) CreateInitialCommit(fileList []database.File, commitMessage string) error { // ONLY HAPPENS FOR MASTER I THINK, SO NO BRANCH NEEDED
+func (m *Manager) CreateInitialCommit(fileList []database.File, branch string, commitMessage string) error { // ONLY HAPPENS FOR MASTER I THINK, SO NO BRANCH NEEDED
+	m.Info().Msgf("Starting initial commit on branch: %s", branch)
 	//Need to deduplicate so we aren't storing duplicates of files, storing all the files in one folder won't work, will need something like git
 	//For initial commit no changes are made to files, so don't store anything, just save the list so you can send to server
 	var initialCommit database.Commit
@@ -30,8 +31,19 @@ func (m *Manager) CreateInitialCommit(fileList []database.File, commitMessage st
 	if err != nil {
 		return fmt.Errorf("unable to create commit directory in object dir: %s err: %s", folder, err)
 	}
+	m.Info().Msgf("Total number of files to commit: %s", len(fileList))
+	fmt.Println("Total number of files to commit: ", len(fileList))
 	for _, file := range fileList {
-		ConvertFileForStorage(&file, folder)
+		filename, err := ConvertFileForStorage(&file, folder)
+		if err != nil {
+			// TODO: roll back commit on error (folders/files created need to be deleted)
+			return err
+		}
+		file.BaseFilePath = filename
+	}
+	err = m.dB.NewCommit(initialCommit, branch)
+	if err != nil {
+		return err
 	}
 	//var hashList [][]byte
 	return nil

common/engine/compressor.go

@@ -2,10 +2,14 @@ package engine
 
 import (
 	"bytes"
-	"compress/gzip"
+	"io/ioutil"
+	"os"
+
 	"encoding/gob"
 	"fmt"
 	"io"
+
+	gzip "github.com/klauspost/pgzip"
 )
 
 type SomeStruct struct {
@@ -14,6 +18,51 @@ type SomeStruct struct {
 	C float64
 }
 
+// CompressFile uses pgzip to compress a file for storage
+func CompressFile(path string) error {
+	fileBytes, err := ioutil.ReadFile(path)
+	if err != nil {
+		return fmt.Errorf("unable to read file: %s", path)
+	}
+	file, err := os.Create(path)
+	if err != nil {
+		return err
+	}
+	w := gzip.NewWriter(file)
+	w.SetConcurrency(100000, 10)
+	bytesWritten, err := w.Write(fileBytes)
+	if err != nil {
+		return err
+	}
+	fmt.Println("%d Bytes Written", bytesWritten)
+	w.Close()
+	return nil
+}
+
+// RestoreFile takes the compressed file path and the target path and decompresses the file into the target path
+func RestoreFile(compressPath, targetPath string) error {
+	fileBytes, err := os.Open(compressPath)
+	if err != nil {
+		return fmt.Errorf("unable to read file: %s", compressPath)
+	}
+	err = os.Remove(targetPath)
+	if err != nil {
+		return fmt.Errorf("unable to remove target file to replace with decompressed file: %s", targetPath)
+	}
+	file, err := os.Create(targetPath)
+	if err != nil {
+		return err
+	}
+	r, err := gzip.NewReader(fileBytes)
+	bytesWritten, err := r.WriteTo(file)
+	if err != nil {
+		return err
+	}
+	fmt.Println("%d Bytes Written", bytesWritten)
+	r.Close()
+	return nil
+}
+
 //1.
 func StructToBytes(obj SomeStruct) (bytes.Buffer, error) {
 	//now gob this

common/engine/filesystem.go

@@ -5,7 +5,6 @@ import (
 	"compress/gzip"
 	"encoding/binary"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strings"
@@ -68,19 +67,15 @@ func ExpandToIntArray(length int64, arry []byte, intArray *[]int64) error {
 // 	return fileName, nil
 // }
 
-func ConvertFileForStorage(file *database.File, folder string) error {
+// ConvertFileForStorage opens the original file and dumps the bytes to pgzip to compress it for storage
-	fileBytes, err := ioutil.ReadFile(file.Path)
+func ConvertFileForStorage(file *database.File, folder string) (fileName string, err error) {
+	filename := folder + string(filepath.Separator) + file.Name //fileName is under the object folder then the hash, then the filename
+	err = CompressFile(filename)
 	if err != nil {
-		return err
+		return filename, err
 	}
-	fmt.Println("REMOVE: ", fileBytes)
+	fmt.Println("Compressed and written", file.Path)
-	filename := folder + string(filepath.Separator) + file.Name
+	return filename, nil
-	err = ioutil.WriteFile(filename, fileBytes, 0666)
-	if err != nil {
-		return err
-	}
-	return nil
-
 }
 
 func IsDirectory(path string) (bool, error) {

common/engine/manager.go

@@ -149,8 +149,12 @@ func (m *Manager) AddFileToRepo(relFilePath string) error {
 	//DELAYED: this feature affects only large files and user experience. It can wait.
 	relFilePath = strings.TrimSpace(relFilePath) //purging any odd spaces TODO: Make sure not needed
 	var tmpFile database.File
-	filename := filepath.Base(relFilePath)
+	tmpFile.Path = relFilePath
-	var hash []byte
+	filename := filepath.Base(tmpFile.Path)
+	alreadyTracked := m.dB.CheckIfFileCurrentlyMonitored(tmpFile.Path)
+	if alreadyTracked {
+		return fmt.Errorf("file already found in tracked files, not adding: %s", relFilePath)
+	}
 	//check that the file actually exists (currently done by client/server)
 	// if filename, err = VerifySrcFile(relFilePath); err != nil {
 	// 	//there was no source file or it was not recognisable as a file
@@ -158,19 +162,12 @@ func (m *Manager) AddFileToRepo(relFilePath string) error {
 	// }
 	//generate a unique file name from the hash and the moment it was created
 	//a sampled (and therefore) fast, hash of the file for 'uniqueness'
-	if hash, err = UniqueFileHash(relFilePath); err != nil {
+
+	if err = tmpFile.CalculateHash(); err != nil { //creates the hash for the file
 		return err
 	}
-	m.Info().Msgf("Hash generated for file: %s hash: %s", relFilePath, hex.EncodeToString(hash))
+	m.Info().Msgf("Hash generated for file: %s hash: %s", tmpFile.Path, hex.EncodeToString(tmpFile.Hash))
-	alreadyTracked := m.dB.CheckIfFileCurrentlyMonitored(relFilePath)
-	if alreadyTracked {
-		return fmt.Errorf("file already found in tracked files, not adding: %s", relFilePath)
-	}
-	tmpFile = database.File{}
-
-	tmpFile.Hash = hash
 	tmpFile.Name = filename
-	tmpFile.Path = relFilePath
 	tmpFile.CreatedAt = time.Now()
 	tmpFile.Unique = hex.EncodeToString([]byte(filename)) + "_" + hex.EncodeToString((tmpFile.Hash)) + "_" + strconv.FormatInt(tmpFile.CreatedAt.Unix(), 10) + "_" + filename
 	//tmpFile.BkpLocation = filepath.Join(m.SyncFolder, tmpFile.Unique)
@@ -207,6 +204,7 @@ func (m *Manager) prepareDatabaseForFile(tmpFile database.File) (int, error) {
 // BeginCommit starts the commit process
 func (m *Manager) BeginCommit(branch string, commitMessage string) error {
 	fmt.Println("Beginning Commit on Branch: ", branch)
+
 	trackedFiles, err := m.dB.RetrieveTrackedFiles()
 	if err != nil {
 		return err
@@ -214,6 +212,15 @@ func (m *Manager) BeginCommit(branch string, commitMessage string) error {
 	if len(trackedFiles) == 0 {
 		return fmt.Errorf("no files show as tracked in repo, cannot commit, aborting...")
 	}
+	commit, err := m.dB.FetchLastCommitOnBranch(branch) //Check for previous commits
+	if err != nil {
+		m.Info().Msgf("unable to fetch last commit on branch, assuming first commit on branch", err)
+		err := m.CreateInitialCommit(trackedFiles, branch, commitMessage) // Create the initial commit
+		if err != nil {
+			m.Err(err).Msgf("unable to create initial commit: %s", err)
+			return err
+		}
+	}
 	var filesToDiff []database.File // Contains the list of files that have changed
 	for _, trackedFile := range trackedFiles {
 		fmt.Println("Working on file: ", trackedFile.Path)
@@ -226,7 +233,6 @@ func (m *Manager) BeginCommit(branch string, commitMessage string) error {
 			fmt.Printf("unable to stat tracked file: %s error: %s\n", currentFile.Name(), err)
 			continue
 		}
-
 		currentFileHash, err := UniqueFileHash(trackedFile.Path)
 		if err != nil {
 			fmt.Printf("unable to create hash for file: %s error: %s\n", currentFile.Name(), err)
@@ -242,14 +248,10 @@ func (m *Manager) BeginCommit(branch string, commitMessage string) error {
 	//diffChannel := make(chan database.DiffObject)
 	//diffContext := context.Background()
 	//m.WaitGroup.Add(2)
-	commit, err := m.dB.FetchLastCommitOnBranch(branch)
+	fmt.Println("Changed Files: ", filesToDiff)
-	if err != nil {
+	if len(filesToDiff) == 0 {
-		m.Info().Msgf("unable to fetch last commit on branch, assuming first commit on branch", err)
+		m.Info().Msgf("No changed files found to commit on branch: %s", branch)
-		err := m.CreateInitialCommit(filesToDiff, commitMessage)
+		return fmt.Errorf("no changed files, cannot commit on branch: %s", branch)
-		if err != nil {
-			m.Err(err).Msgf("unable to create initial commit: %s", err)
-			return err
-		}
 	}
 	fmt.Println("COMMIT: ", commit.CommitHash)
 	return nil

common/engine/watcher.go

@@ -67,7 +67,7 @@ func (fw *FileWatcher) BeginWatcherRoutine(ctx context.Context, wg *sync.WaitGro
 			//this is currently slow as it does a db lookup on the path.
 			//TODO: On load (or whenever a file is added to the watcher, the db information for files being watched, could be cached in memory. This would be much faster)
 			fileInfo, err := onFileChanged(event.Path) //could return the 'Event' object here
-			syncFilePath := fileInfo.CurrentBase
+			syncFilePath := fileInfo.BaseFilePath
 			//uniqueName := fileInfo.Unique
 
 			// fileID := fileInfo.ID

go.mod

@@ -17,6 +17,8 @@ require (
 	github.com/golang/snappy v0.0.1 // indirect
 	github.com/imdario/mergo v0.3.9
 	github.com/kalafut/imohash v1.0.0
+	github.com/klauspost/compress v1.10.10 // indirect
+	github.com/klauspost/pgzip v1.2.4
 	github.com/labstack/echo v3.3.10+incompatible
 	github.com/labstack/gommon v0.3.0 // indirect
 	github.com/mattn/go-colorable v0.1.6 // indirect

go.sum

@@ -41,6 +41,10 @@ github.com/imdario/mergo v0.3.9 h1:UauaLniWCFHWd+Jp9oCEkTBj8VO/9DKg3PV3VCNMDIg=
 github.com/imdario/mergo v0.3.9/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
 github.com/kalafut/imohash v1.0.0 h1:LgCJ+p/BwM2HKpOxFopkeddpzVCfm15EtXMroXD1SYE=
 github.com/kalafut/imohash v1.0.0/go.mod h1:c3RHT80ZAp5C/aYgQI92ZlrOymqkZnRDprU87kg75HI=
+github.com/klauspost/compress v1.10.10 h1:a/y8CglcM7gLGYmlbP/stPE5sR3hbhFRUjCBfd/0B3I=
+github.com/klauspost/compress v1.10.10/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
+github.com/klauspost/pgzip v1.2.4 h1:TQ7CNpYKovDOmqzRHKxJh0BeaBI7UdQZYc6p7pMQh1A=
+github.com/klauspost/pgzip v1.2.4/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
 github.com/labstack/echo v3.3.10+incompatible h1:pGRcYk231ExFAyoAjAfD85kQzRJCRI8bbnE7CX5OEgg=
 github.com/labstack/echo v3.3.10+incompatible/go.mod h1:0INS7j/VjnFxD4E2wkz67b8cVwCLbBmJyDaka6Cmk1s=
 github.com/labstack/gommon v0.3.0 h1:JEeO0bvc78PKdyHxloTKiF8BD5iGrH8T6MSeGvSgob0=