deranjer/goTorrent
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

working on pulling settings from file using viper, finished basic RSS feed and cron job

Browse Source
This commit is contained in:
deranjer
2018-01-10 20:07:00 -05:00
parent 08b3a14576
commit f079a5f067
18 changed files with 30486 additions and 28934 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
engine/clientStructs.go
View File

@@ -17,6 +17,19 @@ type Message struct {
//Next are the messages the server sends to the client
//RSSJSONList is a slice of gofeed.Feeds sent to the client
type RSSJSONList struct {
MessageType string
TotalRSSFeeds int
RSSFeeds []RSSFeedsNames //strings of the full rss feed
}
//RSSFeedsNames stores all of the feeds by name and with URL
type RSSFeedsNames struct {
RSSName string
RSSFeedURL string
}
//TorrentList struct contains the torrent list that is sent to the client
type TorrentList struct { //helps create the JSON structure that react expects to recieve
MessageType string `json:"MessageType"`
@@ -74,4 +87,6 @@ type ClientDB struct { //TODO maybe seperate out the internal bits into another
DataBytesRead int64 //Internal used for calculating dl speed
UpdatedAt time.Time //Internal used for calculating speeds of upload and download
TorrentHash metainfo.Hash //Used to create string for TorrentHashString... not sure why I have it... make that a TODO I guess
NumberofFiles int
NumberofPieces int
}

48
engine/cronJobs.go Normal file
View File

@@ -0,0 +1,48 @@
package engine
import (
"fmt"
"github.com/asdine/storm"
Storage "github.com/deranjer/goTorrent/storage"
"github.com/mmcdole/gofeed"
"github.com/robfig/cron"
)
//InitializeCronEngine initializes and starts the cron engine so we can add tasks as needed, returns pointer to the engine
func InitializeCronEngine() *cron.Cron { //TODO add a cron to inspect cron jobs and log the outputs
c := cron.New()
c.Start()
return c
}
//RefreshRSSCron refreshes all of the RSS feeds on an hourly basis
func RefreshRSSCron(c *cron.Cron, db *storm.DB) {
c.AddFunc("@hourly", func() {
RSSFeedStore := Storage.FetchRSSFeeds(db)
singleRSSTorrent := Storage.SingleRSSTorrent{}
newFeedStore := Storage.RSSFeedStore{ID: RSSFeedStore.ID} //creating a new feed store just using old one to parse for new torrents
fp := gofeed.NewParser()
for _, singleFeed := range RSSFeedStore.RSSFeeds {
feed, err := fp.ParseURL(singleFeed.URL)
if err != nil {
fmt.Println("Unable to parse URL", singleFeed.URL, err)
}
for _, RSSTorrent := range feed.Items {
singleRSSTorrent.Link = RSSTorrent.Link
singleRSSTorrent.Title = RSSTorrent.Title
singleRSSTorrent.PubDate = RSSTorrent.Published
singleFeed.Torrents = append(singleFeed.Torrents, singleRSSTorrent)
}
newFeedStore.RSSFeeds = append(newFeedStore.RSSFeeds, singleFeed)
}
Storage.UpdateRSSFeeds(db, newFeedStore) //Calling this to fully update storage will all rss feeds
})
}
//LogCronStatus prints out the status of the cron jobs to the log
func LogCronStatus(c *cron.Cron) {
}

142
engine/engine.go
View File

@@ -2,6 +2,7 @@ package engine //main file for all the calculations and data gathering needed fo
import (
"fmt"
"io/ioutil"
"os"
"strconv"
"strings"
@@ -11,9 +12,58 @@ import (
"github.com/anacrolix/torrent/metainfo"
"github.com/asdine/storm"
Storage "github.com/deranjer/goTorrent/storage"
"github.com/mmcdole/gofeed"
)
func timeOutInfo(clientTorrent *torrent.Torrent, seconds time.Duration) (deleted bool) { //forcing a timeout of the torrent if it doesn't load from program restart
//RefreshSingleRSSFeed refreshing a single RSS feed to send to the client (so no updating database) mainly by updating the torrent list to display any changes
func RefreshSingleRSSFeed(db *storm.DB, RSSFeed Storage.SingleRSSFeed) Storage.SingleRSSFeed { //Todo.. duplicate as cron job... any way to merge these to reduce duplication?
singleRSSFeed := Storage.SingleRSSFeed{URL: RSSFeed.URL, Name: RSSFeed.Name}
singleRSSTorrent := Storage.SingleRSSTorrent{}
fp := gofeed.NewParser()
feed, err := fp.ParseURL(RSSFeed.URL)
if err != nil {
fmt.Println("Unable to parse URL", RSSFeed.URL, err)
}
for _, RSSTorrent := range feed.Items {
singleRSSTorrent.Link = RSSTorrent.Link
singleRSSTorrent.Title = RSSTorrent.Title
singleRSSTorrent.PubDate = RSSTorrent.Published
singleRSSFeed.Torrents = append(singleRSSFeed.Torrents, singleRSSTorrent)
}
return singleRSSFeed
}
//ForceRSSRefresh forces a refresh (in addition to the cron schedule) to add the new RSS feed
func ForceRSSRefresh(db *storm.DB, RSSFeedStore Storage.RSSFeedStore) { //Todo.. duplicate as cron job... any way to merge these to reduce duplication?
singleRSSTorrent := Storage.SingleRSSTorrent{}
newFeedStore := Storage.RSSFeedStore{ID: RSSFeedStore.ID} //creating a new feed store just using old one to parse for new torrents
fp := gofeed.NewParser()
fmt.Println("Length of RSS feeds (should be ONE)", len(RSSFeedStore.RSSFeeds))
for _, singleFeed := range RSSFeedStore.RSSFeeds {
feed, err := fp.ParseURL(singleFeed.URL)
if err != nil {
fmt.Println("Unable to parse URL", singleFeed.URL, err)
}
fmt.Println("SingleFeed is: ", singleFeed)
for _, RSSTorrent := range feed.Items {
singleRSSTorrent.Link = RSSTorrent.Link
singleRSSTorrent.Title = RSSTorrent.Title
singleRSSTorrent.PubDate = RSSTorrent.Published
singleFeed.Torrents = append(singleFeed.Torrents, singleRSSTorrent)
}
newFeedStore.RSSFeeds = append(newFeedStore.RSSFeeds, singleFeed)
}
fmt.Println("ABOUT TO WRITE TO DB", newFeedStore.RSSFeeds)
Storage.UpdateRSSFeeds(db, newFeedStore) //Calling this to fully update storage will all rss feeds
}
//timeOutInfo forcing a timeout of the torrent if it doesn't load from program restart
func timeOutInfo(clientTorrent *torrent.Torrent, seconds time.Duration) (deleted bool) {
fmt.Println("Attempting to pull information for torrent... ", clientTorrent.Name())
timeout := make(chan bool, 1) //creating a timeout channel for our gotinfo
go func() {
time.Sleep(seconds * time.Second)
@@ -21,11 +71,11 @@ func timeOutInfo(clientTorrent *torrent.Torrent, seconds time.Duration) (deleted
}()
select {
case <-clientTorrent.GotInfo(): //attempting to retrieve info for torrent
fmt.Println("Recieved torrent info for...", clientTorrent.Name())
//fmt.Println("Recieved torrent info for...", clientTorrent.Name())
clientTorrent.DownloadAll()
return false
case <-timeout: // getting info for torrent has timed out so purging the torrent
fmt.Println("Dropping Torrent")
fmt.Println("Dropping Torrent from information timeout...", clientTorrent.Name())
clientTorrent.Drop()
return true
}
@@ -33,23 +83,37 @@ func timeOutInfo(clientTorrent *torrent.Torrent, seconds time.Duration) (deleted
}
//StartTorrent creates the storage.db entry and starts A NEW TORRENT and adds to the running torrent array
func StartTorrent(clientTorrent *torrent.Torrent, torrentLocalStorage Storage.TorrentLocal, torrentDbStorage *storm.DB, dataDir string, torrentFile string, torrentFileName string) {
func StartTorrent(clientTorrent *torrent.Torrent, torrentLocalStorage Storage.TorrentLocal, torrentDbStorage *storm.DB, dataDir string, torrentType string, torrentFileName string) {
timeOutInfo(clientTorrent, 45) //seeing if adding the torrrent times out (giving 45 seconds)
var TempHash metainfo.Hash
TempHash = clientTorrent.InfoHash()
fmt.Println(clientTorrent.Info().Source)
torrentLocalStorage.Hash = TempHash.String() // we will store the infohash to add it back later on client restart (if needed)
torrentLocalStorage.InfoBytes = clientTorrent.Metainfo().InfoBytes
torrentLocalStorage.DateAdded = time.Now().Format("Jan _2 2006")
torrentLocalStorage.StoragePath = dataDir //TODO check full path information for torrent storage
torrentLocalStorage.TorrentName = clientTorrent.Name()
torrentLocalStorage.TorrentStatus = "downloading" //by default start all the torrents as downloading.
torrentLocalStorage.TorrentType = torrentFile //either "file" or "magnet" maybe more in the future
if torrentFile == "file" {
torrentLocalStorage.TorrentStatus = "Running" //by default start all the torrents as downloading.
torrentLocalStorage.TorrentType = torrentType //either "file" or "magnet" maybe more in the future
if torrentType == "file" { //if it is a file read the entire file into the database for us to spit out later
torrentLocalStorage.TorrentFileName = torrentFileName
} else {
torrentLocalStorage.TorrentFileName = ""
torrentfile, err := ioutil.ReadFile(torrentFileName)
if err != nil {
fmt.Println("Unable to read the torrent file...")
}
torrentLocalStorage.TorrentFile = torrentfile //storing the entire file in to database
}
torrentFiles := clientTorrent.Files() //storing all of the files in the database along with the priority
var TorrentFilePriorityArray = []Storage.TorrentFilePriority{}
for _, singleFile := range torrentFiles { //creating the database setup for the file array
var torrentFilePriority = Storage.TorrentFilePriority{}
torrentFilePriority.TorrentFilePath = singleFile.DisplayPath()
torrentFilePriority.TorrentFilePriority = "Normal"
TorrentFilePriorityArray = append(TorrentFilePriorityArray, torrentFilePriority)
}
torrentLocalStorage.TorrentFilePriority = TorrentFilePriorityArray
fmt.Printf("%+v\n", torrentLocalStorage)
Storage.AddTorrentLocalStorage(torrentDbStorage, torrentLocalStorage) //writing all of the data to the database
clientTorrent.DownloadAll() //starting the download
@@ -64,12 +128,27 @@ func CreateRunningTorrentArray(tclient *torrent.Client, TorrentLocalArray []*Sto
if element.TorrentType == "file" { //if it is a file pull it from the uploaded torrent folder
//fmt.Println("Filename", element.TorrentFileName)
tempFile, err := ioutil.TempFile("", "TorrentFileTemp")
if err != nil {
fmt.Println("Unable to create a temp file for adding file torrent in", err)
}
defer os.Remove(tempFile.Name())
if _, err := tempFile.Write(element.TorrentFile); err != nil {
fmt.Println("Unable to write to the temp file...", err)
}
if err := tempFile.Close(); err != nil {
fmt.Println("Error closing Temp file", err)
}
singleTorrent, _ = tclient.AddTorrentFromFile(tempFile.Name())
if _, err := os.Stat(element.TorrentFileName); err == nil { //if we CAN find the torrent, add it
//fmt.Println("Adding file name...", element.TorrentFileName)
singleTorrent, _ = tclient.AddTorrentFromFile(element.TorrentFileName)
} else { //if we cant find the torrent delete it
fmt.Println("File Error", err)
Storage.DelTorrentLocalStorage(db, element)
Storage.DelTorrentLocalStorage(db, element.Hash)
continue
}
@@ -77,11 +156,17 @@ func CreateRunningTorrentArray(tclient *torrent.Client, TorrentLocalArray []*Sto
elementMagnet := "magnet:?xt=urn:btih:" + element.Hash //For magnet links just need to prepend the magnet part to the hash to readd
singleTorrent, _ = tclient.AddMagnet(elementMagnet)
}
var TempHash metainfo.Hash
TempHash = singleTorrent.InfoHash()
timeOut := timeOutInfo(singleTorrent, 45)
singleTorrentStorageInfo := Storage.FetchTorrentFromStorage(db, TempHash.String())
singleTorrent.SetInfoBytes(singleTorrentStorageInfo.InfoBytes) //setting the infobytes back into the torrent
/* timeOut := timeOutInfo(singleTorrent, 45) //Shouldn't need this anymore as we pull in the infohash from the database
if timeOut == true { // if we did timeout then drop the torrent from the boltdb database
Storage.DelTorrentLocalStorage(db, element) //purging torrent from the local database
}
Storage.DelTorrentLocalStorage(db, element.Hash) //purging torrent from the local database
continue
} */
fullClientDB := new(ClientDB)
fullStruct := singleTorrent.Stats()
@@ -97,10 +182,7 @@ func CreateRunningTorrentArray(tclient *torrent.Client, TorrentLocalArray []*Sto
}
activePeersString := strconv.Itoa(fullStruct.ActivePeers) //converting to strings
totalPeersString := fmt.Sprintf("%v", fullStruct.TotalPeers)
var TempHash metainfo.Hash
TempHash = singleTorrent.InfoHash()
singleTorrentStorageInfo := Storage.FetchTorrentFromStorage(db, TempHash.String()) //fetching all the info from the database
//fetching all the info from the database
var torrentTypeTemp string
torrentTypeTemp = singleTorrentStorageInfo.TorrentType //either "file" or "magnet" maybe more in the future
if torrentTypeTemp == "file" {
@@ -110,8 +192,8 @@ func CreateRunningTorrentArray(tclient *torrent.Client, TorrentLocalArray []*Sto
}
fullClientDB.StoragePath = singleTorrentStorageInfo.StoragePath //grabbed from database
totalSizeHumanized := HumanizeBytes(float32(singleTorrent.BytesCompleted())) //convert size to GB if needed
downloadedSizeHumanized := HumanizeBytes(float32(singleTorrent.Length()))
downloadedSizeHumanized := HumanizeBytes(float32(singleTorrent.BytesCompleted())) //convert size to GB if needed
totalSizeHumanized := HumanizeBytes(float32(singleTorrent.Length()))
//grabbed from torrent client
fullClientDB.DownloadedSize = downloadedSizeHumanized
@@ -127,6 +209,7 @@ func CreateRunningTorrentArray(tclient *torrent.Client, TorrentLocalArray []*Sto
fullClientDB.TorrentName = element.TorrentName
fullClientDB.DateAdded = element.DateAdded
fullClientDB.BytesCompleted = singleTorrent.BytesCompleted()
fullClientDB.NumberofFiles = len(singleTorrent.Files())
CalculateTorrentETA(singleTorrent, fullClientDB) //calculating the ETA for the torrent
fullClientDB.TotalUploadedBytes = singleTorrentStorageInfo.UploadedBytes
@@ -139,7 +222,11 @@ func CreateRunningTorrentArray(tclient *torrent.Client, TorrentLocalArray []*Sto
tickUpdateStruct.Hash = fullClientDB.TorrentHashString //needed for index
Storage.UpdateStorageTick(db, tickUpdateStruct)
CalculateTorrentStatus(singleTorrent, fullClientDB) //calculate the status of the torrent, ie downloading seeding etc
if singleTorrentStorageInfo.TorrentStatus != "Stopped" { //if the torrent is not stopped, try to discern the status of the torrent
CalculateTorrentStatus(singleTorrent, fullClientDB) //calculate the status of the torrent, ie downloading seeding etc
} else {
fullClientDB.Status = "Stopped"
}
RunningTorrentArray = append(RunningTorrentArray, *fullClientDB)
@@ -156,12 +243,20 @@ func CreateFileListArray(tclient *torrent.Client, selectedHash string) TorrentFi
tempHash := singleTorrent.InfoHash().String()
if tempHash == selectedHash { // if our selection hash equals our torrent hash
torrentFilesRaw := singleTorrent.Files()
fmt.Println(torrentFilesRaw)
for _, singleFile := range torrentFilesRaw {
TorrentFileStruct.TorrentHashString = tempHash
TorrentFileStruct.FileName = singleFile.DisplayPath()
TorrentFileStruct.FilePath = singleFile.Path()
TorrentFileStruct.FilePercent = fmt.Sprintf("%.2f", float32(singleFile.Length())/float32(singleFile.Length())) //TODO figure out downloaded size of file
TorrentFileStruct.FilePriority = "Normal" //TODO, figure out how to store this per file in storage and also tie a priority to a file
PieceState := singleFile.State()
var downloadedBytes int64
for _, piece := range PieceState {
if piece.Complete {
downloadedBytes = downloadedBytes + piece.Bytes //adding up the bytes in the completed pieces
}
}
TorrentFileStruct.FilePercent = fmt.Sprintf("%.2f", float32(downloadedBytes)/float32(singleFile.Length()))
TorrentFileStruct.FilePriority = "Normal" //TODO, figure out how to store this per file in storage and also tie a priority to a file
TorrentFileStruct.FileSize = HumanizeBytes(float32(singleFile.Length()))
TorrentFileListSelected.FileList = append(TorrentFileListSelected.FileList, TorrentFileStruct)
}
@@ -205,7 +300,6 @@ func CreateTorrentDetailJSON(tclient *torrent.Client, selectedHash string, torre
if tempHash == selectedHash {
fmt.Println("CreateTorrentDetail", localTorrentInfo)
return TorrentDetailStruct
break //only looking for one result
}
}
return TorrentDetailStruct

101
engine/settings.go
View File

@@ -1,32 +1,109 @@
package engine //Settings.go contains all of the program settings
package engine
import (
"fmt"
"golang.org/x/time/rate"
"github.com/anacrolix/dht"
"github.com/anacrolix/torrent"
"github.com/spf13/viper"
)
//FullCLientSettings struct is a struct that can be read into anacrolix/torrent to setup a torrent client
type FullClientSettings struct {
Version int
torrent.Config
Version int
TorrentConfig torrent.Config
TFileUploadFolder string
}
//FullClientSettingsNew creates a new torrent client config TODO read from a TOML file
func FullClientSettingsNew() FullClientSettings {
//Config := fullClientSettings //generate a new struct
func defaultConfig() FullClientSettings {
var Config FullClientSettings
Config.Version = 1.0
Config.DataDir = "downloads" //the full OR relative path of the default download directory for torrents
Config.TorrentConfig.DataDir = "downloads" //the full OR relative path of the default download directory for torrents
Config.TFileUploadFolder = "uploadedTorrents"
Config.Seed = true
Config.TorrentConfig.Seed = true
Config.DHTConfig = dht.ServerConfig{
Config.TorrentConfig.DHTConfig = dht.ServerConfig{
StartingNodes: dht.GlobalBootstrapAddrs,
}
return Config
}
func dhtServerSettings(dhtConfig dht.ServerConfig) dht.ServerConfig {
viper.UnmarshalKey("DHTConfig", &dhtConfig)
fmt.Println("dhtconfig", dhtConfig)
return dhtConfig
}
func FullClientSettingsNew() FullClientSettings {
viper.SetConfigName("config")
viper.AddConfigPath("./")
err := viper.ReadInConfig()
if err != nil {
fmt.Println("Error reading in config, using defaults", err)
FullClientSettings := defaultConfig()
return FullClientSettings
}
dataDir := viper.GetString("torrentClientConfig.DownloadDir")
listenAddr := viper.GetString("torrentClientConfig.ListenAddr")
disablePex := viper.GetBool("torrentClientConfig.DisablePEX")
noDHT := viper.GetBool("torrentClientConfig.NoDHT")
noUpload := viper.GetBool("torrentClientConfig.NoUpload")
seed := viper.GetBool("torrentClientConfig.Seed")
peerID := viper.GetString("torrentClientConfig.PeerID")
disableUTP := viper.GetBool("torrentClientConfig.DisableUTP")
disableTCP := viper.GetBool("torrentClientConfig.DisableTCP")
disableIPv6 := viper.GetBool("torrentClientConfig.DisableIPv6")
debug := viper.GetBool("torrentClientConfig.Debug")
dhtServerConfig := dht.ServerConfig{
StartingNodes: dht.GlobalBootstrapAddrs,
}
if viper.IsSet("DHTConfig") {
fmt.Println("Reading in custom DHT config")
dhtServerConfig = dhtServerSettings(dhtServerConfig)
}
uploadRateLimiter := new(rate.Limiter)
viper.UnmarshalKey("UploadRateLimiter", &uploadRateLimiter)
downloadRateLimiter := new(rate.Limiter)
viper.UnmarshalKey("DownloadRateLimiter", &downloadRateLimiter)
rreferNoEncryption := viper.GetBool("EncryptionPolicy.PreferNoEncryption")
fmt.Println("Encryption", rreferNoEncryption)
encryptionPolicy := torrent.EncryptionPolicy{
DisableEncryption: viper.GetBool("EncryptionPolicy.DisableEncryption"),
ForceEncryption: viper.GetBool("EncryptionPolicy.ForceEncryption"),
PreferNoEncryption: viper.GetBool("EncryptionPolicy.PreferNoEncryption"),
}
tConfig := torrent.Config{
DataDir: dataDir,
ListenAddr: listenAddr,
DisablePEX: disablePex,
NoDHT: noDHT,
DHTConfig: dhtServerConfig,
NoUpload: noUpload,
Seed: seed,
//UploadRateLimiter: uploadRateLimiter,
//DownloadRateLimiter: downloadRateLimiter,
PeerID: peerID,
DisableUTP: disableUTP,
DisableTCP: disableTCP,
DisableIPv6: disableIPv6,
Debug: debug,
EncryptionPolicy: encryptionPolicy,
}
Config := FullClientSettings{TorrentConfig: tConfig, TFileUploadFolder: "uploadedTorrents"}
return Config
}