package engine import ( "bytes" "io/ioutil" "os" "encoding/gob" "fmt" "io" gzip "github.com/klauspost/pgzip" ) type SomeStruct struct { A string B int64 C float64 } // CompressFile uses pgzip to compress a file for storage func CompressFile(path string, newPath string) error { fileBytes, err := ioutil.ReadFile(path) if err != nil { return fmt.Errorf("unable to read file: %s", path) } file, err := os.Create(newPath) if err != nil { return err } w := gzip.NewWriter(file) w.SetConcurrency(100000, 10) bytesWritten, err := w.Write(fileBytes) if err != nil { return err } fmt.Println("%d Bytes Written", bytesWritten) w.Close() return nil } // RestoreFile takes the compressed file path and the target path and decompresses the file into the target path func RestoreFile(compressPath, targetPath string) error { fileBytes, err := os.Open(compressPath) if err != nil { return fmt.Errorf("unable to read file: %s", compressPath) } err = os.Remove(targetPath) if err != nil { return fmt.Errorf("unable to remove target file to replace with decompressed file: %s", targetPath) } file, err := os.Create(targetPath) if err != nil { return err } r, err := gzip.NewReader(fileBytes) bytesWritten, err := r.WriteTo(file) if err != nil { return err } fmt.Println("%d Bytes Written", bytesWritten) r.Close() return nil } //1. func StructToBytes(obj SomeStruct) (bytes.Buffer, error) { //now gob this var indexBuffer bytes.Buffer encoder := gob.NewEncoder(&indexBuffer) if err := encoder.Encode(obj); err != nil { return indexBuffer, err } return indexBuffer, nil } //1. func BytesToGob(obj []byte) (bytes.Buffer, error) { //now gob this var indexBuffer bytes.Buffer encoder := gob.NewEncoder(&indexBuffer) if err := encoder.Encode(obj); err != nil { return indexBuffer, err } return indexBuffer, nil } //2. func CompressBinary(binaryBuffer *bytes.Buffer) (bytes.Buffer, error) { //now compress it var compressionBuffer bytes.Buffer compressor := gzip.NewWriter(&compressionBuffer) _, err := compressor.Write(binaryBuffer.Bytes()) err = compressor.Close() return compressionBuffer, err } //3. func DecompressBinary(compressionBuffer bytes.Buffer) (*gzip.Reader, error) { //now decompress it dataReader := bytes.NewReader(compressionBuffer.Bytes()) if reader, err := gzip.NewReader(dataReader); err != nil { fmt.Println("gzip failed ", err) return &gzip.Reader{}, err } else { err := reader.Close() return reader, err } } //4. func GobToBytes(binaryBytes io.Reader) ([]byte, error) { decoder := gob.NewDecoder(binaryBytes) var tmp []byte if err := decoder.Decode(&tmp); err != nil { return tmp, err } return tmp, nil }