adding path checks to manager

common/database/create-db.go

@@ -10,9 +10,8 @@ type DB struct {
 	*zerolog.Logger
 }
 
-// NewDB returns a new database object,
+// OpenOrCreate returns a new database object, either from existing database or creates a new one
-// it configures the database for you.
+func OpenOrCreateDB(dbPath string, log *zerolog.Logger) (*DB, error) {
-func NewDB(dbPath string, log *zerolog.Logger) (*DB, error) {
 	var db DB
 	databaseLogger := log.With().Str("module", "database").Logger() // Setting up a sub logger for the database module
 	db.Logger = &databaseLogger

common/engine/filesystem.go

@@ -80,6 +80,7 @@ func VerifySrcFile(src string) (string, error) {
 	return fileName, nil
 }
 
+//InitiateDirectory checks all of the directories to make sure they exist
 func InitiateDirectory(directory string) {
 	// For the keys-folder we need to check if the folder exists...
 	checkDir, err := IsDirectory(directory)

common/engine/patcher.go

@@ -12,10 +12,10 @@ import (
 // * copying any versions and keeping them safe (even if temporary)
 // * creating the diff of the file, in both directions if necessary
 // * storing the details in the database
-func NewPatcher(logger *zerolog.Logger, KEYFOLDER, DOWNLOADFOLDER, SYNCFOLDER, THUMBFOLDER, DIFFFOLDER string) (Patcher, error) {
+func NewPatcher(logger *zerolog.Logger, KeyFolder, DownloadFolder, SyncFolder, ThumbFolder, DiffFolder string) (Patcher, error) {
 	p := Patcher{
 		logger,
-		KEYFOLDER, DOWNLOADFOLDER, SYNCFOLDER, THUMBFOLDER, DIFFFOLDER,
+		KeyFolder, DownloadFolder, SyncFolder, ThumbFolder, DiffFolder,
 	}
 	return p, nil
 }

common/engine/structures.go

@@ -9,17 +9,19 @@ type FileWatcher struct {
 	*watcher.Watcher
 	*zerolog.Logger
 	Enabled        bool
-	KEYFOLDER      string
+	KeyFolder      string
-	DOWNLOADFOLDER string
+	DownloadFolder string
-	SYNCFOLDER     string
+	SyncFolder     string
-	THUMBFOLDER    string
+	ThumbFolder    string
-	DIFFFOLDER     string
+	DiffFolder     string
 }
+
+//Patcher contains the information needed to patch files
 type Patcher struct {
 	*zerolog.Logger
-	KEYFOLDER      string
+	KeyFolder      string
-	DOWNLOADFOLDER string
+	DownloadFolder string
-	SYNCFOLDER     string
+	SyncFolder     string
-	THUMBFOLDER    string
+	ThumbFolder    string
-	DIFFFOLDER     string
+	DiffFolder     string
 }

common/engine/watcher.go

@@ -22,12 +22,12 @@ type Event struct {
 // * copying any versions and keeping them safe (even if temporary)
 // * creating the diff of the file, in both directions if necessary
 // * storing the details in the database
-func NewWatcher(logger *zerolog.Logger, KEYFOLDER, DOWNLOADFOLDER, SYNCFOLDER, THUMBFOLDER, DIFFFOLDER string) (FileWatcher, error) {
+func NewWatcher(logger *zerolog.Logger, keyFolder, downloadFolder, syncFolder, thumbFolder, diffFolder string) (FileWatcher, error) {
 	w := FileWatcher{
 		watcher.New(),
 		logger,
 		true, //used to temporarily ignore events if necessary
-		KEYFOLDER, DOWNLOADFOLDER, SYNCFOLDER, THUMBFOLDER, DIFFFOLDER,
+		keyFolder, downloadFolder, syncFolder, thumbFolder, diffFolder,
 	}
 	return w, nil
 }
@@ -103,7 +103,7 @@ func (fw *FileWatcher) BeginWatcherRoutine(ctx context.Context, wg *sync.WaitGro
 				Total:    100,
 			}
 			eventContext := context.WithValue(cancelContext, key(event.Path), e)
-			if err := manageFileDiffing(eventContext, event.Path, syncFilePath, fw.DIFFFOLDER, true, diffChannel, wg); err != nil {
+			if err := manageFileDiffing(eventContext, event.Path, syncFilePath, fw.DiffFolder, true, diffChannel, wg); err != nil {
 				// I don't think this can be reached...
 				fw.Warn().Msgf("Error managing the diffing process %s", err)
 			}

common/manager/initialize.go

@@ -1,47 +0,0 @@
-package manager
-
-import (
-	"path/filepath"
-
-	"github.com/atrox/homedir"
-	engine "github.com/deranjer/gvc/common/engine"
-)
-
-const (
-	storageDirectory = "pickleit"
-)
-
-var (
-	PATH, KEYFOLDER, DOWNLOADFOLDER, SYNCFOLDER, DIFFFOLDER, THUMBFOLDER, LOGFOLDER, PLUGINFOLDER string
-)
-
-func init() {
-	homeDirectory, err := homedir.Dir()
-	if err != nil {
-		panic(err)
-	}
-	PATH = filepath.Join(homeDirectory, storageDirectory)
-
-	//where private and public keys are kept
-	KEYFOLDER = filepath.Join(PATH, "keys")
-	//where downloaded files start
-	DOWNLOADFOLDER = filepath.Join(PATH, "downloads")
-	//where file originals live
-	SYNCFOLDER = filepath.Join(PATH, "sync")
-	//where patches and last versions live
-	DIFFFOLDER = filepath.Join(PATH, "diff")
-	//where the thumbnails are stored
-	THUMBFOLDER = filepath.Join(PATH, "thumb")
-	//where the logs are stored
-	LOGFOLDER = filepath.Join(PATH, "logs")
-	//where plugins are stored
-	PLUGINFOLDER = filepath.Join(PATH, "plugins")
-
-	engine.InitiateDirectory(KEYFOLDER)
-	engine.InitiateDirectory(DOWNLOADFOLDER)
-	engine.InitiateDirectory(SYNCFOLDER)
-	engine.InitiateDirectory(DIFFFOLDER)
-	engine.InitiateDirectory(THUMBFOLDER)
-	engine.InitiateDirectory(LOGFOLDER)
-	engine.InitiateDirectory(PLUGINFOLDER)
-}

common/manager/manager.go

@@ -1,26 +1,173 @@
 package manager
 
 import (
+	"encoding/base64"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/deranjer/gvc/common/database"
 	engine "github.com/deranjer/gvc/common/engine"
 	"github.com/rs/zerolog"
 )
 
 // NewManager creates a new manager interface that contains all the needed information to make changes to the repo
-func NewManager(version string, informer chan OperatingMessage, log zerolog.Logger) (*Manager, error) {
+// rootPath is passed by client or server to let the manager know where to look for the .gvc folder and all the components needed
+func NewManager(rootDir string, version string, dbPath string, informer chan OperatingMessage, log *zerolog.Logger) (*Manager, error) {
 	log.Info().Msg("Creating new Manager...")
-	patcher, err := engine.NewPatcher(&log, KEYFOLDER, DOWNLOADFOLDER, SYNCFOLDER, THUMBFOLDER, DIFFFOLDER)
+	dirPaths, err := checkPaths(rootDir)
+	if err != nil {
+		return &Manager{}, err
+	}
+	patcher, err := engine.NewPatcher(log, dirPaths)
 	if err != nil {
 		log.Fatal().Msgf("Error creating a patcher %s", err)
 		return &Manager{}, err
 	}
+	gvcDB, err := database.OpenOrCreateDB(dbPath, log)
+	if err != nil {
+		log.Fatal().Msgf("unable to create or open db: %s", err)
+	}
 
 	m := Manager{
 		version,
+
 		//settings,
-		&log,
+		log,
 		patcher,
-		database,
+		gvcDB,
 		informer,
+		dirPaths,
 	}
 	return &m, nil
 }
+
+func checkPaths(rootDir string) (filePaths *FilePaths, err error) {
+	// checking for the .gvc folder (the client (but not the server) already checks for the .gvc folder, but this checks all subdirects to make sure they are there)
+	rootFolder, err := filepath.Abs(rootDir)
+	if err != nil {
+		return &FilePaths{}, err
+	}
+	path := rootFolder + string(filepath.Separator) + ".gvc"
+	//path = filepath.Join(rootFolder, filepath.Separator+".gvc")
+	var fullFilePaths FilePaths
+	//where private and public keys are kept
+	fullFilePaths.KeyFolder = filepath.Join(path, "keys")
+	//where downloaded files start
+	fullFilePaths.DownloadFolder = filepath.Join(path, "downloads")
+	//where file originals live
+	fullFilePaths.SyncFolder = filepath.Join(path, "sync")
+	//where patches and last versions live
+	fullFilePaths.DiffFolder = filepath.Join(path, "diff")
+	//where the thumbnails are stored
+	fullFilePaths.ThumbFolder = filepath.Join(path, "thumb")
+	//where the logs are stored
+	fullFilePaths.LogFolder = filepath.Join(path, "logs")
+	//where plugins are stored
+	fullFilePaths.PluginFolder = filepath.Join(path, "plugins")
+
+	engine.InitiateDirectory(fullFilePaths.KeyFolder)
+	engine.InitiateDirectory(fullFilePaths.DownloadFolder)
+	engine.InitiateDirectory(fullFilePaths.SyncFolder)
+	engine.InitiateDirectory(fullFilePaths.DiffFolder)
+	engine.InitiateDirectory(fullFilePaths.ThumbFolder)
+	engine.InitiateDirectory(fullFilePaths.LogFolder)
+	engine.InitiateDirectory(fullFilePaths.PluginFolder)
+	return &fullFilePaths, nil
+}
+
+// This adds a file for the watcher to keep an eye on
+// however the file will also need to be backedup
+// and added to the database.
+// This changes all paths to absolute paths rather than relative
+// when adding a file to monitor, this should check if the database
+// is already expecting to monitor this file. If it is this function should
+// do checks to make sure that it is successfully monitoring it, and that there
+// is a historical breadcrumb trail to recreate all the versions that the database
+// claims to have a copy of
+func (m *Manager) AddFileToRepo(file string, hardCopy bool) (string, error) {
+	var err error
+	// the filepath should be absolute, but this should be done dynamically
+	if file, err = filepath.Abs(file); err != nil {
+		return "", err
+	}
+	//TODO: what needs to happen is a channel for errors/progress is created
+	//then pass that channel to a routine, and put all of the following in it
+	// whenever an error returns, fire the string to the channel,
+	// or send progress on the progress channel
+	//however need to work out best way of returning the final result to the caller
+	//- the way to do that is send the result on a third channel, for which is just the result
+	//see commsManagment.go
+	// f := NewFileManager()
+	//DELAYED: this feature affects only large files and user experience. It can wait.
+
+	var tmpFile database.File
+	var filename string //we might aswell only verify the files validity once
+	var hash [16]byte
+	//check that the file actually exists
+	if filename, err = engine.VerifySrcFile(file); err != nil {
+		//there was no source file or it was not recognisable as a file
+		return "", err
+	}
+	//generate a unique file name from the hash and the moment it was created
+	//a sampled (and therefore) fast, hash of the file for 'uniqueness'
+	if hash, err = engine.UniqueFileHash(file); err != nil {
+		return "", err
+	}
+
+	if tmpFile, err = m.dB.CheckIfFileCurrentlyMonitored(file, hash); err != nil {
+		if strings.Index(err.Error(), "not found") != -1 {
+			//the file wasn't found, this is an ok error
+			m.Info().Msgf("The file was [%s], so continuing to create it in the database", err)
+		} else {
+			m.Error().Msgf("Error checking if file [%s] is monitored so will init file. Error: %s", tmpFile.Path, err)
+		}
+		tmpFile.CurrentHash = hash
+		tmpFile.Name = filename
+		tmpFile.Path = file
+		tmpFile.CreatedAt = time.Now()
+		tmpFile.Unique = base64.URLEncoding.EncodeToString([]byte(filename)) + "_" + base64.URLEncoding.EncodeToString((tmpFile.CurrentHash[:])) + "_" + strconv.FormatInt(tmpFile.CreatedAt.Unix(), 10) + "_" + filename
+		tmpFile.BkpLocation = filepath.Join(SyncFolder, tmpFile.Unique)
+		tmpFile.CurrentBase = tmpFile.BkpLocation
+		//tmpFile.Ignore = false //we can have files in the database that are ignored. TODO: This was initially added so that 'All Files' would show up as a file (its a hack as it adds a dummy to the database)
+		//we should now have a unique name for this file
+		//if needs be, we can find out the real file name from the string
+		//the hash will give us a reasononable indication of the similarity of the files
+		//define filename of backup(s)
+		if _, err := m.prepareDatabaseForFile(tmpFile); err != nil {
+			return "", err
+		} else {
+			if err := m.copyFileToNewLocation(tmpFile.Path, tmpFile.BkpLocation, hardCopy); err != nil {
+				m.ErrorF("There was an error copying the file to the backup location %s", err)
+				return "", err
+			}
+			m.Informer <- Op_NewFile.Retrieve()
+		}
+	} else {
+		m.Debug().Msgf("file [%s] is already in the database. Assuming sync file in place", tmpFile.Path)
+		// we should check if the backup file exists, otherwise there is an issue
+		if _, err := engine.VerifySrcFile(tmpFile.BkpLocation); err != nil {
+			//if the backup doesn't exist, something has gone quite wrong....
+			m.Debug().Msgf("The backup file doesn't seem to exist at the expected location, %s", err)
+			return "", err
+		}
+	}
+
+	return tmpFile.Path, m.watcher.Add(tmpFile.Path)
+}
+
+// prepareDatabaseForFile is responsible for keeping all references to the version of the file,
+// the diff and the metadata of the diffs. Before any file is copied and stored, it should be managed by the database
+//
+// TODO: This will need to initialise a diff object in the database, currently created by the diff package,
+// however going forward a diff maybe defined by the manager.
+func (m *Manager) prepareDatabaseForFile(tmpFile engine.File) (int, error) {
+	fileID, err := m.dB.InitialiseFileInDatabase(tmpFile)
+	if err != nil {
+		m.Error().Msgf("Error checking if file [%s] is monitored. Error %s", tmpFile.Path, err)
+		return 0, err
+	}
+	return fileID, nil
+
+}

common/manager/manager.go.old

@@ -36,12 +36,12 @@ func NewManager(version engine.Version, logLevel int, format string, informer ch
 	log.SetLogLevel(logger.LogLevel(logLevel))
 	var wg sync.WaitGroup
 
-	watcher, err := engine.NewWatcher(log, KEYFOLDER, DOWNLOADFOLDER, SYNCFOLDER, THUMBFOLDER, DIFFFOLDER)
+	watcher, err := engine.NewWatcher(log, KeyFolder, DownloadFolder, SyncFolder, ThumbFolder, DiffFolder)
 	if err != nil {
 		log.CriticalF("Error creating a watcher %s", err)
 		return &Manager{}, err
 	}
-	patcher, err := engine.NewPatcher(log, KEYFOLDER, DOWNLOADFOLDER, SYNCFOLDER, THUMBFOLDER, DIFFFOLDER)
+	patcher, err := engine.NewPatcher(log, KeyFolder, DownloadFolder, SyncFolder, ThumbFolder, DiffFolder)
 	if err != nil {
 		log.CriticalF("Error creating a patcher %s", err)
 		return &Manager{}, err
@@ -161,7 +161,7 @@ func (m *Manager) AddFileToMonitor(file string, hardCopy bool) (string, error) {
 		tmpFile.Path = file
 		tmpFile.CreatedAt = time.Now()
 		tmpFile.Unique = base64.URLEncoding.EncodeToString([]byte(filename)) + "_" + base64.URLEncoding.EncodeToString((tmpFile.CurrentHash[:])) + "_" + strconv.FormatInt(tmpFile.CreatedAt.Unix(), 10) + "_" + filename
-		tmpFile.BkpLocation = filepath.Join(SYNCFOLDER, tmpFile.Unique)
+		tmpFile.BkpLocation = filepath.Join(SyncFolder, tmpFile.Unique)
 		tmpFile.CurrentBase = tmpFile.BkpLocation
 		tmpFile.Ignore = false //we can have files in the database that are ignored. TODO: This was initially added so that 'All Files' would show up as a file (its a hack as it adds a dummy to the database)
 		//we should now have a unique name for this file
@@ -260,7 +260,7 @@ func (m *Manager) BeginWatching() error {
 
 						_, fileName := filepath.Split(diff.Object)
 						unique := base64.URLEncoding.EncodeToString([]byte(fileName)) + "_" + base64.URLEncoding.EncodeToString((diff.ObjectHash[:])) + "_" + strconv.FormatInt(time.Now().Unix(), 10) + "_" + fileName
-						restoreFile := filepath.Join(SYNCFOLDER, unique)
+						restoreFile := filepath.Join(SyncFolder, unique)
 						//generate a unique file name from the hash and the moment it was created
 						//a sampled (and therefore) fast, hash of the file for 'uniqueness'
 						if err := m.BeginForwardPatch(diff.Object, diff.DiffPath, restoreFile); err != nil {

common/manager/structures.go

@@ -19,6 +19,7 @@ type Manager struct {
 	patcher  engine.Patcher
 	dB       *database.DB
 	Informer chan OperatingMessage
+	*FilePaths
 	//ProgressCommunicator io.WriteCloser
 }
 
@@ -28,6 +29,17 @@ type CustomPlugin interface {
 	Description() string
 }
 
+//FilePaths holds the full paths to all the relevant folders
+type FilePaths struct {
+	KeyFolder      string
+	DownloadFolder string
+	SyncFolder     string
+	ThumbFolder    string
+	DiffFolder     string
+	LogFolder      string
+	PluginFolder   string
+}
+
 // type PluginManager struct {
 // 	engine   *qml.QQmlApplicationEngine
 // 	informer chan OperatingMessage

server/server.go

@@ -66,7 +66,7 @@ func main() {
 	serverlog := zerolog.New(logFile)
 	serverlog.WithLevel(logLevel)
 	// Check/Setup the database
-	database.NewDB(conf.DatabaseLocation, &serverlog)
+	database.OpenOrCreateDB(conf.DatabaseLocation, &serverlog)
 	// Setup the web server
 	e := echo.New()
 	// Setup the logger to print to the file specified in config