deranjer/gvc
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

adding path checks to manager

Browse Source
This commit is contained in:
deranjer
2020-06-18 23:15:30 -04:00
parent 8c01cdbcf4
commit 7610e3f168
10 changed files with 188 additions and 74 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
common/database/create-db.go
View File

@@ -10,9 +10,8 @@ type DB struct {
*zerolog.Logger
}
// NewDB returns a new database object,
// it configures the database for you.
func NewDB(dbPath string, log *zerolog.Logger) (*DB, error) {
// OpenOrCreate returns a new database object, either from existing database or creates a new one
func OpenOrCreateDB(dbPath string, log *zerolog.Logger) (*DB, error) {
var db DB
databaseLogger := log.With().Str("module", "database").Logger() // Setting up a sub logger for the database module
db.Logger = &databaseLogger

1
common/engine/filesystem.go
View File

@@ -80,6 +80,7 @@ func VerifySrcFile(src string) (string, error) {
return fileName, nil
}
//InitiateDirectory checks all of the directories to make sure they exist
func InitiateDirectory(directory string) {
// For the keys-folder we need to check if the folder exists...
checkDir, err := IsDirectory(directory)

4
common/engine/patcher.go
View File

@@ -12,10 +12,10 @@ import (
// * copying any versions and keeping them safe (even if temporary)
// * creating the diff of the file, in both directions if necessary
// * storing the details in the database
func NewPatcher(logger *zerolog.Logger, KEYFOLDER, DOWNLOADFOLDER, SYNCFOLDER, THUMBFOLDER, DIFFFOLDER string) (Patcher, error) {
func NewPatcher(logger *zerolog.Logger, KeyFolder, DownloadFolder, SyncFolder, ThumbFolder, DiffFolder string) (Patcher, error) {
p := Patcher{
logger,
KEYFOLDER, DOWNLOADFOLDER, SYNCFOLDER, THUMBFOLDER, DIFFFOLDER,
KeyFolder, DownloadFolder, SyncFolder, ThumbFolder, DiffFolder,
}
return p, nil
}

22
common/engine/structures.go
View File

@@ -9,17 +9,19 @@ type FileWatcher struct {
*watcher.Watcher
*zerolog.Logger
Enabled bool
KEYFOLDER string
DOWNLOADFOLDER string
SYNCFOLDER string
THUMBFOLDER string
DIFFFOLDER string
KeyFolder string
DownloadFolder string
SyncFolder string
ThumbFolder string
DiffFolder string
}
//Patcher contains the information needed to patch files
type Patcher struct {
*zerolog.Logger
KEYFOLDER string
DOWNLOADFOLDER string
SYNCFOLDER string
THUMBFOLDER string
DIFFFOLDER string
KeyFolder string
DownloadFolder string
SyncFolder string
ThumbFolder string
DiffFolder string
}

6
common/engine/watcher.go
View File

@@ -22,12 +22,12 @@ type Event struct {
// * copying any versions and keeping them safe (even if temporary)
// * creating the diff of the file, in both directions if necessary
// * storing the details in the database
func NewWatcher(logger *zerolog.Logger, KEYFOLDER, DOWNLOADFOLDER, SYNCFOLDER, THUMBFOLDER, DIFFFOLDER string) (FileWatcher, error) {
func NewWatcher(logger *zerolog.Logger, keyFolder, downloadFolder, syncFolder, thumbFolder, diffFolder string) (FileWatcher, error) {
w := FileWatcher{
watcher.New(),
logger,
true, //used to temporarily ignore events if necessary
KEYFOLDER, DOWNLOADFOLDER, SYNCFOLDER, THUMBFOLDER, DIFFFOLDER,
keyFolder, downloadFolder, syncFolder, thumbFolder, diffFolder,
}
return w, nil
}
@@ -103,7 +103,7 @@ func (fw *FileWatcher) BeginWatcherRoutine(ctx context.Context, wg *sync.WaitGro
Total: 100,
}
eventContext := context.WithValue(cancelContext, key(event.Path), e)
if err := manageFileDiffing(eventContext, event.Path, syncFilePath, fw.DIFFFOLDER, true, diffChannel, wg); err != nil {
if err := manageFileDiffing(eventContext, event.Path, syncFilePath, fw.DiffFolder, true, diffChannel, wg); err != nil {
// I don't think this can be reached...
fw.Warn().Msgf("Error managing the diffing process %s", err)
}

47
common/manager/initialize.go
View File

@@ -1,47 +0,0 @@
package manager
import (
"path/filepath"
"github.com/atrox/homedir"
engine "github.com/deranjer/gvc/common/engine"
)
const (
storageDirectory = "pickleit"
)
var (
PATH, KEYFOLDER, DOWNLOADFOLDER, SYNCFOLDER, DIFFFOLDER, THUMBFOLDER, LOGFOLDER, PLUGINFOLDER string
)
func init() {
homeDirectory, err := homedir.Dir()
if err != nil {
panic(err)
}
PATH = filepath.Join(homeDirectory, storageDirectory)
//where private and public keys are kept
KEYFOLDER = filepath.Join(PATH, "keys")
//where downloaded files start
DOWNLOADFOLDER = filepath.Join(PATH, "downloads")
//where file originals live
SYNCFOLDER = filepath.Join(PATH, "sync")
//where patches and last versions live
DIFFFOLDER = filepath.Join(PATH, "diff")
//where the thumbnails are stored
THUMBFOLDER = filepath.Join(PATH, "thumb")
//where the logs are stored
LOGFOLDER = filepath.Join(PATH, "logs")
//where plugins are stored
PLUGINFOLDER = filepath.Join(PATH, "plugins")
engine.InitiateDirectory(KEYFOLDER)
engine.InitiateDirectory(DOWNLOADFOLDER)
engine.InitiateDirectory(SYNCFOLDER)
engine.InitiateDirectory(DIFFFOLDER)
engine.InitiateDirectory(THUMBFOLDER)
engine.InitiateDirectory(LOGFOLDER)
engine.InitiateDirectory(PLUGINFOLDER)
}

155
common/manager/manager.go
View File

@@ -1,26 +1,173 @@
package manager
import (
"encoding/base64"
"path/filepath"
"strconv"
"strings"
"time"
"github.com/deranjer/gvc/common/database"
engine "github.com/deranjer/gvc/common/engine"
"github.com/rs/zerolog"
)
// NewManager creates a new manager interface that contains all the needed information to make changes to the repo
func NewManager(version string, informer chan OperatingMessage, log zerolog.Logger) (*Manager, error) {
// rootPath is passed by client or server to let the manager know where to look for the .gvc folder and all the components needed
func NewManager(rootDir string, version string, dbPath string, informer chan OperatingMessage, log *zerolog.Logger) (*Manager, error) {
log.Info().Msg("Creating new Manager...")
patcher, err := engine.NewPatcher(&log, KEYFOLDER, DOWNLOADFOLDER, SYNCFOLDER, THUMBFOLDER, DIFFFOLDER)
dirPaths, err := checkPaths(rootDir)
if err != nil {
return &Manager{}, err
}
patcher, err := engine.NewPatcher(log, dirPaths)
if err != nil {
log.Fatal().Msgf("Error creating a patcher %s", err)
return &Manager{}, err
}
gvcDB, err := database.OpenOrCreateDB(dbPath, log)
if err != nil {
log.Fatal().Msgf("unable to create or open db: %s", err)
}
m := Manager{
version,
//settings,
&log,
log,
patcher,
database,
gvcDB,
informer,
dirPaths,
}
return &m, nil
}
func checkPaths(rootDir string) (filePaths *FilePaths, err error) {
// checking for the .gvc folder (the client (but not the server) already checks for the .gvc folder, but this checks all subdirects to make sure they are there)
rootFolder, err := filepath.Abs(rootDir)
if err != nil {
return &FilePaths{}, err
}
path := rootFolder + string(filepath.Separator) + ".gvc"
//path = filepath.Join(rootFolder, filepath.Separator+".gvc")
var fullFilePaths FilePaths
//where private and public keys are kept
fullFilePaths.KeyFolder = filepath.Join(path, "keys")
//where downloaded files start
fullFilePaths.DownloadFolder = filepath.Join(path, "downloads")
//where file originals live
fullFilePaths.SyncFolder = filepath.Join(path, "sync")
//where patches and last versions live
fullFilePaths.DiffFolder = filepath.Join(path, "diff")
//where the thumbnails are stored
fullFilePaths.ThumbFolder = filepath.Join(path, "thumb")
//where the logs are stored
fullFilePaths.LogFolder = filepath.Join(path, "logs")
//where plugins are stored
fullFilePaths.PluginFolder = filepath.Join(path, "plugins")
engine.InitiateDirectory(fullFilePaths.KeyFolder)
engine.InitiateDirectory(fullFilePaths.DownloadFolder)
engine.InitiateDirectory(fullFilePaths.SyncFolder)
engine.InitiateDirectory(fullFilePaths.DiffFolder)
engine.InitiateDirectory(fullFilePaths.ThumbFolder)
engine.InitiateDirectory(fullFilePaths.LogFolder)
engine.InitiateDirectory(fullFilePaths.PluginFolder)
return &fullFilePaths, nil
}
// This adds a file for the watcher to keep an eye on
// however the file will also need to be backedup
// and added to the database.
// This changes all paths to absolute paths rather than relative
// when adding a file to monitor, this should check if the database
// is already expecting to monitor this file. If it is this function should
// do checks to make sure that it is successfully monitoring it, and that there
// is a historical breadcrumb trail to recreate all the versions that the database
// claims to have a copy of
func (m *Manager) AddFileToRepo(file string, hardCopy bool) (string, error) {
var err error
// the filepath should be absolute, but this should be done dynamically
if file, err = filepath.Abs(file); err != nil {
return "", err
}
//TODO: what needs to happen is a channel for errors/progress is created
//then pass that channel to a routine, and put all of the following in it
// whenever an error returns, fire the string to the channel,
// or send progress on the progress channel
//however need to work out best way of returning the final result to the caller
//- the way to do that is send the result on a third channel, for which is just the result
//see commsManagment.go
// f := NewFileManager()
//DELAYED: this feature affects only large files and user experience. It can wait.
var tmpFile database.File
var filename string //we might aswell only verify the files validity once
var hash [16]byte
//check that the file actually exists
if filename, err = engine.VerifySrcFile(file); err != nil {
//there was no source file or it was not recognisable as a file
return "", err
}
//generate a unique file name from the hash and the moment it was created
//a sampled (and therefore) fast, hash of the file for 'uniqueness'
if hash, err = engine.UniqueFileHash(file); err != nil {
return "", err
}
if tmpFile, err = m.dB.CheckIfFileCurrentlyMonitored(file, hash); err != nil {
if strings.Index(err.Error(), "not found") != -1 {
//the file wasn't found, this is an ok error
m.Info().Msgf("The file was [%s], so continuing to create it in the database", err)
} else {
m.Error().Msgf("Error checking if file [%s] is monitored so will init file. Error: %s", tmpFile.Path, err)
}
tmpFile.CurrentHash = hash
tmpFile.Name = filename
tmpFile.Path = file
tmpFile.CreatedAt = time.Now()
tmpFile.Unique = base64.URLEncoding.EncodeToString([]byte(filename)) + "_" + base64.URLEncoding.EncodeToString((tmpFile.CurrentHash[:])) + "_" + strconv.FormatInt(tmpFile.CreatedAt.Unix(), 10) + "_" + filename
tmpFile.BkpLocation = filepath.Join(SyncFolder, tmpFile.Unique)
tmpFile.CurrentBase = tmpFile.BkpLocation
//tmpFile.Ignore = false //we can have files in the database that are ignored. TODO: This was initially added so that 'All Files' would show up as a file (its a hack as it adds a dummy to the database)
//we should now have a unique name for this file
//if needs be, we can find out the real file name from the string
//the hash will give us a reasononable indication of the similarity of the files
//define filename of backup(s)
if _, err := m.prepareDatabaseForFile(tmpFile); err != nil {
return "", err
} else {
if err := m.copyFileToNewLocation(tmpFile.Path, tmpFile.BkpLocation, hardCopy); err != nil {
m.ErrorF("There was an error copying the file to the backup location %s", err)
return "", err
}
m.Informer <- Op_NewFile.Retrieve()
}
} else {
m.Debug().Msgf("file [%s] is already in the database. Assuming sync file in place", tmpFile.Path)
// we should check if the backup file exists, otherwise there is an issue
if _, err := engine.VerifySrcFile(tmpFile.BkpLocation); err != nil {
//if the backup doesn't exist, something has gone quite wrong....
m.Debug().Msgf("The backup file doesn't seem to exist at the expected location, %s", err)
return "", err
}
}
return tmpFile.Path, m.watcher.Add(tmpFile.Path)
}
// prepareDatabaseForFile is responsible for keeping all references to the version of the file,
// the diff and the metadata of the diffs. Before any file is copied and stored, it should be managed by the database
//
// TODO: This will need to initialise a diff object in the database, currently created by the diff package,
// however going forward a diff maybe defined by the manager.
func (m *Manager) prepareDatabaseForFile(tmpFile engine.File) (int, error) {
fileID, err := m.dB.InitialiseFileInDatabase(tmpFile)
if err != nil {
m.Error().Msgf("Error checking if file [%s] is monitored. Error %s", tmpFile.Path, err)
return 0, err
}
return fileID, nil
}

8
common/manager/manager.go.old
View File

@@ -36,12 +36,12 @@ func NewManager(version engine.Version, logLevel int, format string, informer ch
log.SetLogLevel(logger.LogLevel(logLevel))
var wg sync.WaitGroup
watcher, err := engine.NewWatcher(log, KEYFOLDER, DOWNLOADFOLDER, SYNCFOLDER, THUMBFOLDER, DIFFFOLDER)
watcher, err := engine.NewWatcher(log, KeyFolder, DownloadFolder, SyncFolder, ThumbFolder, DiffFolder)
if err != nil {
log.CriticalF("Error creating a watcher %s", err)
return &Manager{}, err
}
patcher, err := engine.NewPatcher(log, KEYFOLDER, DOWNLOADFOLDER, SYNCFOLDER, THUMBFOLDER, DIFFFOLDER)
patcher, err := engine.NewPatcher(log, KeyFolder, DownloadFolder, SyncFolder, ThumbFolder, DiffFolder)
if err != nil {
log.CriticalF("Error creating a patcher %s", err)
return &Manager{}, err
@@ -161,7 +161,7 @@ func (m *Manager) AddFileToMonitor(file string, hardCopy bool) (string, error) {
tmpFile.Path = file
tmpFile.CreatedAt = time.Now()
tmpFile.Unique = base64.URLEncoding.EncodeToString([]byte(filename)) + "_" + base64.URLEncoding.EncodeToString((tmpFile.CurrentHash[:])) + "_" + strconv.FormatInt(tmpFile.CreatedAt.Unix(), 10) + "_" + filename
tmpFile.BkpLocation = filepath.Join(SYNCFOLDER, tmpFile.Unique)
tmpFile.BkpLocation = filepath.Join(SyncFolder, tmpFile.Unique)
tmpFile.CurrentBase = tmpFile.BkpLocation
tmpFile.Ignore = false //we can have files in the database that are ignored. TODO: This was initially added so that 'All Files' would show up as a file (its a hack as it adds a dummy to the database)
//we should now have a unique name for this file
@@ -260,7 +260,7 @@ func (m *Manager) BeginWatching() error {
_, fileName := filepath.Split(diff.Object)
unique := base64.URLEncoding.EncodeToString([]byte(fileName)) + "_" + base64.URLEncoding.EncodeToString((diff.ObjectHash[:])) + "_" + strconv.FormatInt(time.Now().Unix(), 10) + "_" + fileName
restoreFile := filepath.Join(SYNCFOLDER, unique)
restoreFile := filepath.Join(SyncFolder, unique)
//generate a unique file name from the hash and the moment it was created
//a sampled (and therefore) fast, hash of the file for 'uniqueness'
if err := m.BeginForwardPatch(diff.Object, diff.DiffPath, restoreFile); err != nil {

12
common/manager/structures.go
View File

@@ -19,6 +19,7 @@ type Manager struct {
patcher engine.Patcher
dB *database.DB
Informer chan OperatingMessage
*FilePaths
//ProgressCommunicator io.WriteCloser
}
@@ -28,6 +29,17 @@ type CustomPlugin interface {
Description() string
}
//FilePaths holds the full paths to all the relevant folders
type FilePaths struct {
KeyFolder string
DownloadFolder string
SyncFolder string
ThumbFolder string
DiffFolder string
LogFolder string
PluginFolder string
}
// type PluginManager struct {
// engine *qml.QQmlApplicationEngine
// informer chan OperatingMessage

2
server/server.go
View File

@@ -66,7 +66,7 @@ func main() {
serverlog := zerolog.New(logFile)
serverlog.WithLevel(logLevel)
// Check/Setup the database
database.NewDB(conf.DatabaseLocation, &serverlog)
database.OpenOrCreateDB(conf.DatabaseLocation, &serverlog)
// Setup the web server
e := echo.New()
// Setup the logger to print to the file specified in config