deranjer/gvc
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

making changes to common library, starting to integrate database functions

Browse Source
This commit is contained in:
deranjer
2020-06-11 17:24:35 -04:00
parent 0ecb0b96ce
commit d335549fd5
15 changed files with 317 additions and 145 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
common/database/db-ops.go
View File

@@ -19,7 +19,7 @@ func (db *DB) ConfigureDB(dbPath string) error {
db.ErrorF("Error finding file by path %s", err)
return err
}
db.WarningF("No file found. initialising the database")
db.WarningF("No existing databse found. initialising new database")
file.Name = "-- All Files --"
//file.Ignore = true //this is currently not used however could result in this file being ignored when file watching (etc) starts
if err := db.Save(&file); err != nil {

14
common/database/structures.go
View File

@@ -53,11 +53,11 @@ type DiffObject struct {
DiffPath string //path of the diff/patch
//Label string //store a comment if the user wants to (user written)
//Screenshot string //path to the screen shot when the diff was made
Fs bool //whether it was written to the directly
Description string //record of forward or backward (just a quick helper)
E error //a record of the error when it was created. Maybe able to optimize out later
//Diff *[]byte //the diff itself (incase we want to store in memory) - unused as of now
DiffSize int64 //the size of the diff in bytes
StartTime time.Time //when was the diff created (can take a while to create)
Message string //any message we want to store against the diff while its created
Fs bool //whether it was written to the directly
Description string //record of forward or backward (just a quick helper)
E error //a record of the error when it was created. Maybe able to optimize out later
Diff *[]byte //the diff itself (incase we want to store in memory) - unused as of now
DiffSize int64 //the size of the diff in bytes
StartTime time.Time //when was the diff created (can take a while to create)
Message string //any message we want to store against the diff while its created
}

4
common/engine/compressor_test.go
View File

@@ -37,7 +37,7 @@ func TestMain(t *testing.T) {
})
}
func openFile(path string) ([]byte, error) {
func testOpenFile(path string) ([]byte, error) {
data, err := ioutil.ReadFile(path)
if err != nil {
fmt.Println("File reading error", err)
@@ -45,7 +45,7 @@ func openFile(path string) ([]byte, error) {
return data, err
}
func writeFile(path string, data []byte) error {
func testWriteFile(path string, data []byte) error {
err := ioutil.WriteFile(path, data, 0644)
return err
}

136
common/engine/diff.go Normal file
View File

@@ -0,0 +1,136 @@
package engine
import (
"context"
"fmt"
"os"
"path/filepath"
"strconv"
"sync"
"time"
"github.com/deranjer/gvc/common/database"
)
// ManageFileDiffing handles creating the diffs on the background routines and creating the information
// about each diff that is made.
//
// TODO: fs works however it takes a while to write the diffs to disk. It maybe a better idea to keep the diffs
// in memory (although they could get huge??) and then write them to disk at a later point in time.
// In any event, this works now.
//
// TODO: Be able to cancel a diff creation (for instance if the user resaves). Does this work? Should we block
// creating diffs within 5 minutes of creating one? Cancelling is probably better at this point.
// it might be nice to inform the user when diffs build up
func manageFileDiffing(ctx context.Context, subject, object, diffStorageLocation string, fs bool, diffChannel chan database.DiffObject, wg *sync.WaitGroup) error {
var subjectHash, objectHash [16]byte
var err error
if subjectHash, err = UniqueFileHash(subject); err != nil {
return err
}
if objectHash, err = UniqueFileHash(object); err != nil {
return err
}
diffTime := time.Now()
wg.Add(1)
go func(messages chan<- database.DiffObject) {
defer wg.Done()
var dO database.DiffObject
//doing this on routine to not lose anytime... does it change anything?
dO.Description = ""
dO.Subject = object
dO.Object = subject
dO.StartTime = diffTime
dO.SubjectHash = objectHash //TODO: these being the wrong way round is a legacy thing. Swapping them needs testing, but should be fine
dO.ObjectHash = subjectHash
fmt.Println("creating diff object now")
if diff, err := binaryDiff(ctx, &dO, diffStorageLocation, fs); err != nil { //binaryDiff
fmt.Println("error from binary diff ", err)
dO.E = err
} else {
dO.Diff = &diff
}
// ssStruct := <-ssChannel
// fmt.Printf("received over ssChannel %+v\r\n", ssStruct)
// if ssStruct.ScreenshotError != nil {
// fmt.Println("screenshot failed, ", ssStruct.ScreenshotError)
// } else {
// fmt.Println("diff reeived screenshot ", ssStruct.Screenshot)
// dO.Screenshot = ssStruct.Screenshot
// }
elapsed := time.Since(diffTime)
dO.Message = "elapsed time:" + elapsed.String()
messages <- dO
}(diffChannel)
return nil
}
//run instead of binaryDiff to turn it off
func dryrun(ctx context.Context, dO *database.DiffObject, diffStorageLocation string, fs bool) ([]byte, error) {
return []byte{}, nil
}
// Diff manages the creation of the diffs but doesn't actually create the diffs itself.
// Sources are file system sources in this case and an array of diffs (io.Writers) are returned
// 1. This handles whether to save the diffs directly to the drive, and if so, will save to the
// specified location. If so, it will return the diffs.
// 2. Whether to save diffs in both directions
// 3. Creates a diff object that contains any necessary metadata about the diff files
// subject is the file that changed, object is file on record
func binaryDiff(ctx context.Context, dO *database.DiffObject, diffStorageLocation string, fs bool) ([]byte, error) {
var fileName string
_, fileName = filepath.Split(dO.Subject) // dirPath
dO.Watching = fileName
// var sub io.Reader
// if sub, err = os.Open(dO.Subject); err != nil {
// return []byte{}, err
// }
// var obj io.Reader
// if obj, err = os.Open(dO.Object); err != nil {
// return []byte{}, err
// }
startTime := strconv.FormatInt(dO.StartTime.Unix(), 10)
if fs { //if the wish is to store to the filesystem
dO.DiffPath = filepath.Join(diffStorageLocation, fileName+"_"+startTime+"_"+dO.Description) + "_diff.patch"
if writeDiff, err := os.Create(dO.DiffPath); err != nil {
return []byte{}, err
} else if deltaBytes, err := fdeltaDiff(ctx, dO.Subject, dO.Object); err != nil {
return []byte{}, err
} else {
if bytesWritten, err := writeDiff.Write(deltaBytes); err != nil {
return []byte{}, err
} else {
dO.DiffSize = int64(bytesWritten)
return []byte{}, nil
}
}
} else { //if we actually want the bytes we have to set fs to false (can do this above.)
if deltaBytes, err := fdeltaDiff(ctx, dO.Subject, dO.Object); err != nil {
return []byte{}, err
} else {
dO.DiffSize = int64(len(deltaBytes))
return deltaBytes, nil
}
}
}
//sub is the original
func fdeltaDiff(ctx context.Context, sub, obj string) ([]byte, error) {
//now follow what is found in fdelta to retrieve the bytes and get back a delta
//you can use the gob/compression code to save the files according to where in pickle it they are saved
//TODO: currently the code is used to compress the bsdiff index, but we dont need that, just need to store the
// delta on disk. This is currently already done somewhere, so can possibly add/swap out the delta and compressor code
// so that it uses the new code.
if originalBytes, err := getOriginalBytes(sub); err != nil {
return []byte{}, err
} else if deltaBytes, err := createDelta(obj, originalBytes); err != nil {
return []byte{}, err
} else if compressedDelta, err := compressDelta(deltaBytes); err != nil {
return []byte{}, err
} else {
return compressedDelta, nil
}
}

8
common/engine/fdelta.go
View File

@@ -48,9 +48,9 @@ func createDelta(newFile string, originalBytes []byte) ([]byte, error) {
return delta, nil
}
func compressDelta(delta []byte) ([]byte, error) {
if binaryToGobBuffer, err := compressor.BytesToGob(delta); err != nil {
if binaryToGobBuffer, err := BytesToGob(delta); err != nil {
return []byte{}, err
} else if compressedData, err := compressor.CompressBinary(&binaryToGobBuffer); err != nil {
} else if compressedData, err := CompressBinary(&binaryToGobBuffer); err != nil {
return []byte{}, err
} else {
return compressedData.Bytes(), nil
@@ -76,9 +76,9 @@ func retrieveDelta(patchFile string) ([]byte, error) {
func decompressDelta(compressedData []byte) ([]byte, error) {
var compressedBuffer bytes.Buffer
compressedBuffer.Write(compressedData)
if decompressionReader, err := compressor.DecompressBinary(compressedBuffer); err != nil {
if decompressionReader, err := DecompressBinary(compressedBuffer); err != nil {
return []byte{}, err
} else if res, err := compressor.GobToBytes(decompressionReader); err != nil {
} else if res, err := GobToBytes(decompressionReader); err != nil {
return []byte{}, err
} else {
return res, nil

55
common/engine/fdelta_test.go
View File

@@ -1,33 +1,26 @@
package engine
import (
"fmt"
"os"
"github.com/amlwwalker/fdelta"
)
func main() {
DefineFiles()
originalBytes := GetOriginalBytes()
delta := CreateDelta(originalBytes)
StoreDelta(delta)
retrievedDelta := RetrieveDelta()
// var deltaBytes []byte
fmt.Printf("res : `%s`\n", len(retrievedDelta))
//test loading the delta from disk
appliedBytes, err := fdelta.Apply(originalBytes, retrievedDelta)
if err != nil {
panic(err)
}
fmt.Println("exporting delta")
err = writeFile(appliedFile, appliedBytes)
if err != nil {
fmt.Println("error reading bytes [3]", err)
os.Exit(1)
}
fmt.Printf("Origin : `%s`\n", originalFile)
fmt.Printf("Target : `%s`\n", len(appliedBytes))
fmt.Printf("Delta : `%s`\n", len(delta))
fmt.Printf("Result: `%s`\n", appliedFile)
}
// func main() {
// DefineFiles()
// originalBytes := GetOriginalBytes()
// delta := CreateDelta(originalBytes)
// StoreDelta(delta)
// retrievedDelta := RetrieveDelta()
// // var deltaBytes []byte
// fmt.Printf("res : `%s`\n", len(retrievedDelta))
// //test loading the delta from disk
// appliedBytes, err := fdelta.Apply(originalBytes, retrievedDelta)
// if err != nil {
// panic(err)
// }
// fmt.Println("exporting delta")
// err = writeFile(appliedFile, appliedBytes)
// if err != nil {
// fmt.Println("error reading bytes [3]", err)
// os.Exit(1)
// }
// fmt.Printf("Origin : `%s`\n", originalFile)
// fmt.Printf("Target : `%s`\n", len(appliedBytes))
// fmt.Printf("Delta : `%s`\n", len(delta))
// fmt.Printf("Result: `%s`\n", appliedFile)
// }

14
common/engine/filehashing.go Normal file
View File

@@ -0,0 +1,14 @@
package engine
import (
"github.com/kalafut/imohash"
)
// UniqueFileHash creats a fast hash of a file. It's not bullet proof (could cause a collision, but in practice unlikely) but its fast
func UniqueFileHash(src string) ([16]byte, error) {
hash, err := imohash.SumFile(src)
if err != nil {
return [16]byte{}, err
}
return hash, nil
}

14
common/engine/patcher.go
View File

@@ -1,6 +1,8 @@
package engine
import (
"fmt"
logger "github.com/apsdehal/go-logger"
)
@@ -23,9 +25,9 @@ func NewPatcher(logger *logger.Logger, KEYFOLDER, DOWNLOADFOLDER, SYNCFOLDER, TH
// last save is the file you want to get.
func (p *Patcher) PatchFromFile(filePath, patchPath, restorePath string) error {
if subject, err := openFile(filePath); err != nil {
p.ErrorF("error on subject file: ", err)
return fmt.Errorf("error on subject file: ", err)
} else if patch, err := openFile(patchPath); err != nil {
p.ErrorF("error on patch file: ", err)
return fmt.Errorf("error on patch file: ", err)
} else {
return p.applyPatch(subject, patch, restorePath)
}
@@ -36,14 +38,12 @@ func (p *Patcher) PatchFromFile(filePath, patchPath, restorePath string) error {
// be upgraded for different patching algorithms
func (p *Patcher) applyPatch(subject, patch []byte, restorePath string) error {
if delta, err := decompressDelta(patch); err != nil {
p.ErrorF("error decompressing delta", err)
return fmt.Errorf("error decompressing delta", err)
} else {
if appliedBytes, err := applyPatchToFile(subject, delta); err != nil {
p.ErrorF("error applying delta to original file", err)
return err
return fmt.Errorf("error applying delta to original file", err)
} else if err := writeFile(restorePath, appliedBytes); err != nil {
p.ErrorF("error writing patchedFile", err)
return err
return fmt.Errorf("error writing patchedFile", err)
} else {
return nil
}

6
common/engine/structures.go
View File

@@ -2,11 +2,11 @@ package engine
import (
logger "github.com/apsdehal/go-logger"
radovskyb "github.com/radovskyb/watcher"
watcher "github.com/radovskyb/watcher"
)
type Watcher struct {
*radovskyb.Watcher
type FileWatcher struct {
*watcher.Watcher
*logger.Logger
Enabled bool
KEYFOLDER string

69
common/engine/watcher.go
View File

@@ -2,12 +2,11 @@ package engine
import (
"context"
"path/filepath"
"sync"
"time"
logger "github.com/apsdehal/go-logger"
radovskyb "github.com/radovskyb/watcher"
"github.com/deranjer/gvc/common/database"
watcher "github.com/radovskyb/watcher"
)
type key string
@@ -23,9 +22,9 @@ type Event struct {
// * copying any versions and keeping them safe (even if temporary)
// * creating the diff of the file, in both directions if necessary
// * storing the details in the database
func NewWatcher(logger *logger.Logger, KEYFOLDER, DOWNLOADFOLDER, SYNCFOLDER, THUMBFOLDER, DIFFFOLDER string) (Watcher, error) {
w := Watcher{
radovskyb.New(),
func NewWatcher(logger *logger.Logger, KEYFOLDER, DOWNLOADFOLDER, SYNCFOLDER, THUMBFOLDER, DIFFFOLDER string) (FileWatcher, error) {
w := FileWatcher{
watcher.New(),
logger,
true, //used to temporarily ignore events if necessary
KEYFOLDER, DOWNLOADFOLDER, SYNCFOLDER, THUMBFOLDER, DIFFFOLDER,
@@ -33,16 +32,16 @@ func NewWatcher(logger *logger.Logger, KEYFOLDER, DOWNLOADFOLDER, SYNCFOLDER, TH
return w, nil
}
func (w *Watcher) Ignore() bool {
w.Enabled = false
return w.Enabled
func (fw *FileWatcher) Ignore() bool {
fw.Enabled = false
return fw.Enabled
}
func (w *Watcher) Enable() bool {
w.Enabled = true
return w.Enabled
func (fw *FileWatcher) Enable() bool {
fw.Enabled = true
return fw.Enabled
}
func (w *Watcher) IsEnabled() bool {
return w.Enabled
func (fw *FileWatcher) IsEnabled() bool {
return fw.Enabled
}
// BeginWatcherRoutine kicks off the watcher. When the watcher noticies a file change,
@@ -51,45 +50,31 @@ func (w *Watcher) IsEnabled() bool {
// If certain functions need to be called then this will
// need to be specified as part of the managers lambda functions
// TODO: Should return an error
func (w *Watcher) BeginWatcherRoutine(ctx context.Context, wg *sync.WaitGroup, diffChannel chan utilities.DiffObject, onFileChanged func(string) (utilities.File, error)) {
func (fw *FileWatcher) BeginWatcherRoutine(ctx context.Context, wg *sync.WaitGroup, diffChannel chan database.DiffObject, onFileChanged func(string) (database.File, error)) {
//seems a bit barking, but we can now cancel any diff that is occuring on a file when it fires again
cancelFunctions := make(map[string]func())
for {
select {
// we have filtered already on the [Op]erations we want to listen for so no need to check here
case event := <-w.Event:
if !w.IsEnabled() {
w.Infof("ignoring event and reenabling the watcher %s\r\n", event)
w.Enable()
case event := <-fw.Watcher.Event:
if !fw.IsEnabled() {
fw.Infof("ignoring event and reenabling the watcher %s\r\n", event)
fw.Enable()
continue
}
w.Infof("event fired ", event)
fw.Infof("event fired ", event)
//this is currently slow as it does a db lookup on the path.
//TODO: On load (or whenever a file is added to the watcher, the db information for files being watched, could be cached in memory. This would be much faster)
fileInfo, err := onFileChanged(event.Path) //could return the 'Event' object here
syncFilePath := fileInfo.CurrentBase
uniqueName := fileInfo.Unique
// //begin taking screenshot if we are supposed to
screenshotChannel := make(chan utilities.ScreenshotWrapper)
go func(ssChannel chan utilities.ScreenshotWrapper) {
w.Infof("beginning taking screenshot at ", time.Now())
var ssStruct utilities.ScreenshotWrapper
if screenshotFileName, err := takeScreenShot(w.THUMBFOLDER, uniqueName); err != nil {
w.WarningF("could not take screenshot", err)
ssStruct.ScreenshotError = err
} else {
ssStruct.Screenshot = filepath.Join(w.THUMBFOLDER, screenshotFileName)
w.Infof("screenshot recorded ", ssStruct.Screenshot, " at ", time.Now())
}
ssChannel <- ssStruct
}(screenshotChannel)
//uniqueName := fileInfo.Unique
// fileID := fileInfo.ID
//we need the hash of the current base, not the hash of the original file
// fileHash := fileInfo.CurrentHash //hash needs to come from
if err != nil {
w.ErrorF("path was not returned to sync path", err)
fw.ErrorF("path was not returned to sync path", err)
continue
}
//cancel the event if it indeed is running...
@@ -118,14 +103,14 @@ func (w *Watcher) BeginWatcherRoutine(ctx context.Context, wg *sync.WaitGroup, d
Total: 100,
}
eventContext := context.WithValue(cancelContext, key(event.Path), e)
if err := manageFileDiffing(eventContext, event.Path, syncFilePath, w.DIFFFOLDER, true, screenshotChannel, diffChannel, wg); err != nil {
if err := manageFileDiffing(eventContext, event.Path, syncFilePath, fw.DIFFFOLDER, true, diffChannel, wg); err != nil {
// I don't think this can be reached...
w.WarningF("Error managing the diffing process %s", err)
fw.WarningF("Error managing the diffing process %s", err)
}
case err := <-w.Watcher.Error:
w.ErrorF("%s\r\n", err)
case <-w.Closed:
w.Notice("radovskyb closed")
case err := <-fw.Watcher.Error:
fw.Errorf("%s\r\n", err)
case <-fw.Watcher.Closed:
fw.Notice("radovskyb closed")
return
}
}

23
go.mod
View File

@@ -3,13 +3,26 @@ module github.com/deranjer/gvc
go 1.14
require (
github.com/DataDog/zstd v1.4.5 // indirect
github.com/Sereal/Sereal v0.0.0-20200611165018-70572ef94023 // indirect
github.com/amlwwalker/fdelta v0.0.0-20200513211915-3b53ff25eff6
github.com/apsdehal/go-logger v0.0.0-20190515212710-b0d6ccfee0e6
github.com/asdine/storm v2.1.2+incompatible
github.com/deranjer/clir v1.0.5
github.com/deranjer/store v0.0.0-20200526205429-464dd59c6031
github.com/firstrow/tcp_server v0.0.0-20190424084220-b7a05ff2879d
github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect
github.com/go-resty/resty/v2 v2.3.0
github.com/imdario/mergo v0.3.9 // indirect
github.com/golang/protobuf v1.4.2 // indirect
github.com/golang/snappy v0.0.1 // indirect
github.com/imdario/mergo v0.3.9
github.com/kalafut/imohash v1.0.0
github.com/labstack/echo v3.3.10+incompatible
github.com/labstack/echo/v4 v4.1.16 // indirect
github.com/smartystreets/goconvey v1.6.4 // indirect
github.com/ziflex/lecho/v2 v2.0.0 // indirect
github.com/labstack/gommon v0.3.0 // indirect
github.com/mattn/go-colorable v0.1.6 // indirect
github.com/radovskyb/watcher v1.0.7
github.com/valyala/fasttemplate v1.1.0 // indirect
github.com/vmihailenco/msgpack v4.0.4+incompatible // indirect
go.etcd.io/bbolt v1.3.4 // indirect
golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d // indirect
google.golang.org/appengine v1.6.6 // indirect
)

83
go.sum
View File

@@ -1,6 +1,16 @@
github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ=
github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo=
github.com/Sereal/Sereal v0.0.0-20200611165018-70572ef94023 h1:94Me5RcKWY1qI9MLDed6dMeyhPeYC1ubY7/SuQwBWlE=
github.com/Sereal/Sereal v0.0.0-20200611165018-70572ef94023/go.mod h1:D0JMgToj/WdxCgd30Kc1UcA9E+WdZoJqeVOuYW7iTBM=
github.com/amlwwalker/fdelta v0.0.0-20200513211915-3b53ff25eff6 h1:GVsprsmVmG4ufOyGxlWogL7WZuz/CvWd+a5Mm26tcOI=
github.com/amlwwalker/fdelta v0.0.0-20200513211915-3b53ff25eff6/go.mod h1:VXKr/zswe9Ayq3FjtEnBhB56TOUSvmaXtlGcCuTM9KA=
github.com/apsdehal/go-logger v0.0.0-20190515212710-b0d6ccfee0e6 h1:qISSdUEX4sjDHfdD/vf65fhuCh3pIhiILDB7ktjJrqU=
github.com/apsdehal/go-logger v0.0.0-20190515212710-b0d6ccfee0e6/go.mod h1:U3/8D6R9+bVpX0ORZjV+3mU9pQ86m7h1lESgJbXNvXA=
github.com/asdine/storm v2.1.2+incompatible h1:dczuIkyqwY2LrtXPz8ixMrU/OFgZp71kbKTHGrXYt/Q=
github.com/asdine/storm v2.1.2+incompatible/go.mod h1:RarYDc9hq1UPLImuiXK3BIWPJLdIygvV3PsInK0FbVQ=
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/deranjer/clir v1.0.5 h1:tEunZj5qJLYNBtzMQ/vH8hEAIv4NptWFmTldsP9U2qY=
github.com/deranjer/clir v1.0.5/go.mod h1:x/FAjr5CHGsBT0yjs+NYxX3qFxx8G15gbeCcN6FFuyU=
@@ -8,23 +18,28 @@ github.com/deranjer/store v0.0.0-20200526205429-464dd59c6031 h1:sPjxPMNILoBbu6uh
github.com/deranjer/store v0.0.0-20200526205429-464dd59c6031/go.mod h1:wPOs9IJ77lRTXyjEOQeegCFjIlm21qOFcv33lXmU7gE=
github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM=
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
github.com/firstrow/tcp_server v0.0.0-20190424084220-b7a05ff2879d h1:3/oQzvZhwA8Jb5ykb0KehJfsdHokCJdC96k7xy6SJcs=
github.com/firstrow/tcp_server v0.0.0-20190424084220-b7a05ff2879d/go.mod h1:hGkv6sO57ZC+XrSTyzdIGXX7+O6S3RJb9G8sPopEF/4=
github.com/go-resty/resty v1.12.0 h1:L1P5qymrXL5H/doXe2pKUr1wxovAI5ilm2LdVLbwThc=
github.com/go-resty/resty/v2 v2.3.0 h1:JOOeAvjSlapTT92p8xiS19Zxev1neGikoHsXJeOq8So=
github.com/go-resty/resty/v2 v2.3.0/go.mod h1:UpN9CgLZNsv4e9XG50UU8xdI0F43UQ4HmxLBDwaroHU=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0=
github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/imdario/mergo v0.3.9 h1:UauaLniWCFHWd+Jp9oCEkTBj8VO/9DKg3PV3VCNMDIg=
github.com/imdario/mergo v0.3.9/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
github.com/labstack/echo v1.4.4 h1:1bEiBNeGSUKxcPDGfZ/7IgdhJJZx8wV/pICJh4W2NJI=
github.com/kalafut/imohash v1.0.0 h1:LgCJ+p/BwM2HKpOxFopkeddpzVCfm15EtXMroXD1SYE=
github.com/kalafut/imohash v1.0.0/go.mod h1:c3RHT80ZAp5C/aYgQI92ZlrOymqkZnRDprU87kg75HI=
github.com/labstack/echo v3.3.10+incompatible h1:pGRcYk231ExFAyoAjAfD85kQzRJCRI8bbnE7CX5OEgg=
github.com/labstack/echo v3.3.10+incompatible/go.mod h1:0INS7j/VjnFxD4E2wkz67b8cVwCLbBmJyDaka6Cmk1s=
github.com/labstack/echo/v4 v4.1.10/go.mod h1:i541M3Fj6f76NZtHSj7TXnyM8n2gaodfvfxNnFqi74g=
github.com/labstack/echo/v4 v4.1.16 h1:8swiwjE5Jkai3RPfZoahp8kjVCRNq+y7Q0hPji2Kz0o=
github.com/labstack/echo/v4 v4.1.16/go.mod h1:awO+5TzAjvL8XpibdsfXxPgHr+orhtXZJZIQCVjogKI=
github.com/labstack/gommon v0.3.0 h1:JEeO0bvc78PKdyHxloTKiF8BD5iGrH8T6MSeGvSgob0=
github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k=
github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
@@ -34,42 +49,37 @@ github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hd
github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
github.com/rs/zerolog v1.15.0 h1:uPRuwkWF4J6fGsJ2R0Gn2jB1EQiav9k3S6CSdygQJXY=
github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
github.com/radovskyb/watcher v1.0.7 h1:AYePLih6dpmS32vlHfhCeli8127LzkIgwJGcwwe8tUE=
github.com/radovskyb/watcher v1.0.7/go.mod h1:78okwvY5wPdzcb1UYnip1pvrZNIVEIh/Cm+ZuvsUYIg=
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ=
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8=
github.com/valyala/fasttemplate v1.1.0 h1:RZqt0yGBsps8NGvLSGW804QQqCUYYLsaOjTVHy1Ocw4=
github.com/valyala/fasttemplate v1.1.0/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8=
github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q=
github.com/ziflex/lecho v1.2.0 h1:/ykfd7V/aTsWUYNFimgbdhUiEMnWzvNaCxtbM/LX5F8=
github.com/ziflex/lecho/v2 v2.0.0 h1:ggrWF5LaGAC+Y+WX71jFK7uYR7cUFbHjIgGqCyrYC5Q=
github.com/ziflex/lecho/v2 v2.0.0/go.mod h1:s7dy9Fynjx6z+/7xE2BsK13vXIS3oQoo4ZaKXYG5xUs=
github.com/vmihailenco/msgpack v4.0.4+incompatible h1:dSLoQfGFAo3F6OoNhwUmLwVgaUXK79GlxNBwueZn0xI=
github.com/vmihailenco/msgpack v4.0.4+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk=
go.etcd.io/bbolt v1.3.4 h1:hi1bXHMVrlQh6WwxAy+qZCV/SYIlqo+Ushwdpa4tAKg=
go.etcd.io/bbolt v1.3.4/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d h1:1ZiEyfaQIg3Qh0EoqpwAakHVhecoE5wlSg5GjnafJGw=
golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b h1:0mm1VjtFUOIlE1SbDlwjYaDxZVDP2S5ou6y0gSgXHu8=
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
golang.org/x/net v0.0.0-20200513185701-a91f0712d120 h1:EZ3cVSzKOlJxAd8e8YAJ7no8nNypTxexh/YE/xW3ZEY=
golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae h1:/WDfKMnPU+m5M4xB+6x4kaepxRw6jWvR5iDRdvjHgy8=
golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
@@ -78,10 +88,21 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/appengine v1.6.6 h1:lMO5rYAqUxkmaj76jAkRUvt5JZgFymx/+Q5Mzfivuhc=
google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM=
google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/tylerb/is.v1 v1.1.2 h1:AB/MANFml2ySf+adwcinvajyHvsYltAOD+rb/8njfSU=
gopkg.in/tylerb/is.v1 v1.1.2/go.mod h1:9yQB2tyIhZ5oph6Kk5Sq7cJMd9c5Jpa1p3hr9kxzPqo=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

1
server/config/serverConfig.toml
View File

@@ -4,6 +4,7 @@ port = 80
ip = ""
rawport = 0
reporootpath = "F:\\repos"
databaselocation = "gvc.db"
[[repo]]
rootpath = ""

12
server/server.go
View File

@@ -5,6 +5,7 @@ import (
"log"
"os"
"github.com/deranjer/gvc/common/database"
"github.com/deranjer/gvc/server/engine"
serverconfig "github.com/deranjer/gvc/server/serverconfig"
"github.com/deranjer/store"
@@ -24,10 +25,11 @@ func main() {
fmt.Printf("Unable to find config file: %s\n", err)
fmt.Println("Since no config found, creating a default config to use...")
conf = serverconfig.GvcServerConfig{
LogFile: "gvclog.log",
Version: "0.1",
Port: 80,
RepoRootPath: "repos", //default repos directory will be cwd\repos
LogFile: "gvclog.log",
Version: "0.1",
Port: 80,
RepoRootPath: "repos", //default repos directory will be cwd\repos
DatabaseLocation: "gvc.db", //database is stored in root dir by default
}
configPath = serverconfig.DefaultConfigPath // set the root path for our config so we can save that back to TOML
err = store.Save(serverconfig.DefaultConfigPath, &conf) // Save our new default config back to TOML so it can be read in
@@ -53,6 +55,8 @@ func main() {
log.Fatalf("unable to open log file at: %s, exiting with error: %s", conf.LogFile, err)
}
defer logFile.Close()
// Check/Setup the database
database.NewDB(conf.DatabaseLocation)
// Setup the web server
e := echo.New()
// Setup the logger to print to the file specified in config

21
server/serverconfig/structures.go
View File

@@ -1,16 +1,21 @@
package config
import "github.com/deranjer/gvc/common"
import (
"github.com/asdine/storm"
"github.com/deranjer/gvc/common"
)
// GvcServerConfig will hold the base server settings
type GvcServerConfig struct {
LogFile string `toml:"logfile"` // Where to store the echo logs
Version string `toml:"version"` // The server version
Port int `toml:"port"` // The port that the server will listed on
BindIP string `toml:"ip"` // What IP to bind the server to. If empty will bind to all interfaces
RawPort int `toml:"rawport"` // The optional TCP port that the server will send raw files over
RepoRootPath string `toml:"reporootpath"` // This will be the root path where (by default) all new repos will be stored at
Repos []RepoConfig `toml:"repo"` // A struct of all the repos and their settings for the serve
LogFile string `toml:"logfile"` // Where to store the echo logs
DatabaseLocation string `toml:"databaselocation"` // Location of the database
Database *storm.DB // DB Handle for passing around
Version string `toml:"version"` // The server version
Port int `toml:"port"` // The port that the server will listed on
BindIP string `toml:"ip"` // What IP to bind the server to. If empty will bind to all interfaces
RawPort int `toml:"rawport"` // The optional TCP port that the server will send raw files over
RepoRootPath string `toml:"reporootpath"` // This will be the root path where (by default) all new repos will be stored at
Repos []RepoConfig `toml:"repo"` // A struct of all the repos and their settings for the serve
}
// RepoConfig will be the struct that holds the config for a single repo