deranjer/gvc
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

reworking database, init and add commands

Browse Source
This commit is contained in:
deranjer
2020-07-02 15:00:30 -04:00
parent 6379c73e38
commit 5af55ed62e
15 changed files with 220 additions and 295 deletions
Download Patch File Download Diff File Expand all files Collapse all files

49
common/database/create-db.go
View File

@@ -1,23 +1,54 @@
package database
import (
"fmt"
"os"
"time"
"github.com/asdine/storm"
"github.com/rs/zerolog"
)
type DB struct {
*storm.DB
*zerolog.Logger
}
// OpenOrCreate returns a new database object, either from existing database or creates a new one
func OpenOrCreateDB(dbPath string, log *zerolog.Logger) (*DB, error) {
// OpenDB returns a new database object, from opening existing db
func OpenDB(dbPath string, log *zerolog.Logger, version string) (*DB, error) {
var db DB
var err error
databaseLogger := log.With().Str("module", "database").Logger() // Setting up a sub logger for the database module
db.Logger = &databaseLogger
if err := db.ConfigureDB(dbPath); err != nil {
db.Err(err).Msg("unable to configure database")
db.DB, err = storm.Open(dbPath)
if err != nil {
db.Err(err).Msg("No existing database found. this does not appear to be a repo, please init repo")
return &db, err
}
defer db.Close()
return &db, nil
}
// CreateDB sets up bolt and Storm according to the path of the database //TODO: Save a backup of the config in DB on creation
func CreateDB(dbPath string, version string, repoName string) error {
db, err := storm.Open(dbPath)
if err != nil {
fmt.Println("error initializing database")
return err
}
defer db.Close()
fmt.Println("Initializing new database..")
initTime := time.Now().String()
var gvcInit GVCInfo
gvcInit.InitTime = initTime
gvcInit.GVCVersion = version
gvcInit.RepoName = repoName
if err := db.Save(&gvcInit); err != nil {
fmt.Println("Unable to init db")
return err
}
return nil
}
// CheckDB checks to see if database exists
func CheckDB(dbPath string) error {
if _, err := os.Stat(dbPath); os.IsNotExist(err) {
return err
}
return nil
}

27
common/database/db-ops.go
View File

@@ -3,34 +3,9 @@ package database
import (
"fmt"
"github.com/asdine/storm"
"github.com/asdine/storm/q"
)
// ConfigureDB sets up bolt and Storm according to the path of the database
// this is done here so that different databases can be configured in different scenarios
func (db *DB) ConfigureDB(dbPath string) error {
var err error
if db.DB, err = storm.Open(dbPath); err != nil {
return err
}
var file File
if err := db.One("Name", "-- All Files --", &file); err != nil {
if err.Error() != "not found" {
db.Err(err).Msg("Error finding file by path")
return err
}
db.Warn().Msg("No existing databse found. initialising new database")
file.Name = "-- All Files --"
//file.Ignore = true //this is currently not used however could result in this file being ignored when file watching (etc) starts
if err := db.Save(&file); err != nil {
db.Err(err).Msg("Error storing the diff")
return err
}
}
return nil
}
// CheckIfFileCurrentlyMonitored checks if the file is already being monitored. This is a read-only check
// to see whether the file was correctly initialised
// (BUG) The hash causes the same file to be in database multiple times!
@@ -55,10 +30,12 @@ func (db *DB) CheckIfFileCurrentlyMonitored(path string) bool {
// This can be used to trigger the same files to be watched again
func (db *DB) RetrieveTrackedFiles() ([]File, error) {
var files []File
fmt.Println("Starting file extraction")
if err := db.All(&files); err != nil {
db.Err(err).Msg("Error retrieving all watched files")
return []File{}, err
}
fmt.Println("Ending file extraction")
return files, nil
}

17
common/database/structures.go
View File

@@ -5,11 +5,26 @@ import (
"fmt"
"io/ioutil"
"time"
"github.com/asdine/storm"
"github.com/rs/zerolog"
)
type DB struct {
*storm.DB
*zerolog.Logger
}
// GVCInfo stores the basic information about when the repo was init'd and some basic information
type GVCInfo struct {
GVCVersion string `storm:"id"`
InitTime string
RepoName string
}
// Commit stores all the necessary information for a commit
type Commit struct {
CommitHash []byte // The hash of the commit (generated by hashing commit author name, time, the previous commit, and more? TODO: Not sure what else)
CommitHash []byte `storm:"index,unique"` // The hash of the commit (generated by hashing commit author name, time, the previous commit, and more? TODO: Not sure what else)
TrackedFiles []File // All of the tracked files for this commit
Date string
Version string //User can tag a commit with a version number

12
common/engine/commits.go
View File

@@ -1,6 +1,10 @@
package engine
import (
"encoding/hex"
"fmt"
"os"
"path/filepath"
"time"
"github.com/deranjer/gvc/common/database"
@@ -21,9 +25,13 @@ func (m *Manager) CreateInitialCommit(fileList []database.File, commitMessage st
initialCommit.Number = 1
initialCommit.TrackedFiles = fileList
initialCommit.Date = currentTime.String()
folder := m.FilePaths.ObjectFolder + string(filepath.Separator) + hex.EncodeToString(hashBytes)
err = os.Mkdir(folder, 0666)
if err != nil {
return fmt.Errorf("unable to create commit directory in object dir: %s err: %s", folder, err)
}
for _, file := range fileList {
go ConvertFileForStorage(&file, folder)
ConvertFileForStorage(&file, folder)
}
//var hashList [][]byte
return nil

6
common/engine/filesystem.go
View File

@@ -7,6 +7,7 @@ import (
"fmt"
"io/ioutil"
"os"
"path/filepath"
"strings"
"github.com/deranjer/gvc/common/database"
@@ -95,6 +96,11 @@ func ConvertFileForStorage(file *database.File, folder string) error {
return err
}
fmt.Println("REMOVE: ", fileBytes)
filename := folder + string(filepath.Separator) + file.Name
err = ioutil.WriteFile(filename, fileBytes, 0666)
if err != nil {
return err
}
return nil
}

85
common/engine/manager.go
View File

@@ -13,7 +13,6 @@ import (
"github.com/deranjer/gvc/common/database"
"github.com/rs/zerolog"
"golang.org/x/net/context"
)
// NewManager creates a new manager interface that contains all the needed information to make changes to the repo
@@ -30,9 +29,10 @@ func NewManager(rootDir string, version string, dbPath string, informer chan Ope
ThumbFolder: dirPaths.ThumbFolder,
DiffFolder: dirPaths.ObjectFolder,
}
gvcDB, err := database.OpenOrCreateDB(dbPath, log)
gvcDB, err := database.OpenDB(dbPath, log, version)
if err != nil {
log.Fatal().Msgf("unable to create or open db: %s", err)
log.Err(err).Msgf("unable to create or open db: %s", err)
return nil, err
}
var wg *sync.WaitGroup
m := Manager{
@@ -49,16 +49,16 @@ func NewManager(rootDir string, version string, dbPath string, informer chan Ope
return &m, nil
}
// CheckPaths just checks the .gvc folder structure
func CheckPaths(rootDir string) (filePaths *FilePaths, err error) {
func generatePaths(rootDir string) (filePaths *FilePaths, err error) {
var fullFilePaths FilePaths
// checking for the .gvc folder (the client (but not the server) already checks for the .gvc folder, but this checks all subdirects to make sure they are there)
rootFolder, err := filepath.Abs(rootDir)
if err != nil {
return &FilePaths{}, err
return &fullFilePaths, err
}
path := rootFolder + string(filepath.Separator) + ".gvc"
//path = filepath.Join(rootFolder, filepath.Separator+".gvc")
var fullFilePaths FilePaths
//where private and public keys are kept
fullFilePaths.KeyFolder = filepath.Join(path, "keys")
//where downloaded files start
@@ -73,7 +73,45 @@ func CheckPaths(rootDir string) (filePaths *FilePaths, err error) {
fullFilePaths.LogFolder = filepath.Join(path, "logs")
//where plugins are stored
fullFilePaths.PluginFolder = filepath.Join(path, "plugins")
return &fullFilePaths, nil
}
// CheckPaths just checks the .gvc folder structure
func CheckPaths(rootDir string) (filePaths *FilePaths, err error) {
fullFilePaths, err := generatePaths(rootDir)
if err != nil {
return nil, fmt.Errorf("unable to generate paths, err: %s", err)
}
if _, err := os.Stat(fullFilePaths.DownloadFolder); os.IsNotExist(err) {
return nil, err
}
if _, err := os.Stat(fullFilePaths.KeyFolder); os.IsNotExist(err) {
return nil, err
}
if _, err := os.Stat(fullFilePaths.LogFolder); os.IsNotExist(err) {
return nil, err
}
if _, err := os.Stat(fullFilePaths.ObjectFolder); os.IsNotExist(err) {
return nil, err
}
if _, err := os.Stat(fullFilePaths.PluginFolder); os.IsNotExist(err) {
return nil, err
}
if _, err := os.Stat(fullFilePaths.SyncFolder); os.IsNotExist(err) {
return nil, err
}
if _, err := os.Stat(fullFilePaths.ThumbFolder); os.IsNotExist(err) {
return nil, err
}
return fullFilePaths, nil
}
// CreatePaths creates the paths needed in the .gvc folder
func CreatePaths(rootDir string) error {
fullFilePaths, err := generatePaths(rootDir)
if err != nil {
return fmt.Errorf("unable to generate file paths.. %s", err)
}
InitiateDirectory(fullFilePaths.KeyFolder)
InitiateDirectory(fullFilePaths.DownloadFolder)
InitiateDirectory(fullFilePaths.SyncFolder)
@@ -81,7 +119,7 @@ func CheckPaths(rootDir string) (filePaths *FilePaths, err error) {
InitiateDirectory(fullFilePaths.ThumbFolder)
InitiateDirectory(fullFilePaths.LogFolder)
InitiateDirectory(fullFilePaths.PluginFolder)
return &fullFilePaths, nil
return nil
}
// This adds a file for the watcher to keep an eye on
@@ -164,18 +202,29 @@ func (m *Manager) prepareDatabaseForFile(tmpFile database.File) (int, error) {
}
// BeginCommit starts the commit process
func (m *Manager) BeginCommit(branch string, commitMessage string) error {
trackedFiles, err := m.FetchTrackedFiles()
fmt.Println("Beginning Commit on Branch: ", branch)
trackedFiles, err := m.dB.RetrieveTrackedFiles()
if err != nil {
return err
}
if len(trackedFiles) == 0 {
return fmt.Errorf("no files show as tracked in repo, cannot commit, aborting...")
}
var filesToDiff []database.File // Contains the list of files that have changed
for _, trackedFile := range trackedFiles {
fmt.Println("Working on file: ", trackedFile.Path)
if trackedFile.Path == "" {
fmt.Println("No filepath found for file: ", trackedFile.Name)
continue
}
currentFile, err := os.Stat(trackedFile.Path)
if err != nil {
fmt.Printf("unable to stat tracked file: %s error: %s\n", currentFile.Name(), err)
continue
}
currentFileHash, err := UniqueFileHash(trackedFile.Path)
if err != nil {
fmt.Printf("unable to create hash for file: %s error: %s\n", currentFile.Name(), err)
@@ -188,18 +237,19 @@ func (m *Manager) BeginCommit(branch string, commitMessage string) error {
}
filesToDiff = append(filesToDiff, trackedFile)
}
diffChannel := make(chan database.DiffObject)
diffContext := context.Background()
m.WaitGroup.Add(2)
//diffChannel := make(chan database.DiffObject)
//diffContext := context.Background()
//m.WaitGroup.Add(2)
commit, err := m.dB.FetchLastCommitOnBranch(branch)
if err != nil {
m.Info().Msgf("unable to fetch last commit on branch, assuming first commit on branch", err)
err := CreateInitialCommit(filesToDiff, commitMessage)
err := m.CreateInitialCommit(filesToDiff, commitMessage)
if err != nil {
m.Err(err).Msgf("unable to create initial commit: %s", err)
return err
}
}
fmt.Println("COMMIT: ", commit.CommitHash)
return nil
}
@@ -207,12 +257,3 @@ func (m *Manager) FetchCommitByNumber(branch string, commitNumber string) error
return nil
}
// FetchTrackedFiles just grabbes all the files currently tracked in the repo
func (m *Manager) FetchTrackedFiles() ([]database.File, error) {
files, err := m.dB.RetrieveTrackedFiles()
if err != nil {
return nil, fmt.Errorf("unable to retrieve tracked files: %s", err)
}
return files, nil
}