From d7dbfe3ed22c37791215e5f0ef18eedb37988946 Mon Sep 17 00:00:00 2001 From: Stefan Benten Date: Sat, 28 Mar 2020 22:34:29 +0100 Subject: [PATCH] Implement File Cleanup and remove Interface Function --- server/handlers.go | 4 +-- server/storage/common.go | 9 +++++-- server/storage/gdrive.go | 3 ++- server/storage/local.go | 56 ++++++++++++++++++++++++++++++++-------- server/storage/s3.go | 5 ---- 5 files changed, 56 insertions(+), 21 deletions(-) diff --git a/server/handlers.go b/server/handlers.go index e90a545..178c374 100644 --- a/server/handlers.go +++ b/server/handlers.go @@ -419,7 +419,7 @@ func (s *Server) putHandler(w http.ResponseWriter, r *http.Request) { if contentLength == 0 { log.Print("Empty content-length") - http.Error(w, errors.New("Could not upload empty file").Error(), 400) + http.Error(w, errors.New("could not upload empty file").Error(), 400) return } @@ -439,7 +439,7 @@ func (s *Server) putHandler(w http.ResponseWriter, r *http.Request) { if err = s.storage.Put(token, filename, reader, metadata); err != nil { log.Printf("Error putting new file: %s", err.Error()) - http.Error(w, errors.New("Could not save file").Error(), 500) + http.Error(w, errors.New("could not save file").Error(), 500) return } diff --git a/server/storage/common.go b/server/storage/common.go index af203cf..5880ca6 100644 --- a/server/storage/common.go +++ b/server/storage/common.go @@ -23,13 +23,18 @@ type Metadata struct { } type Storage interface { + // Get returns the entire file as io.ReadCloser and its metadata Get(token string, filename string) (reader io.ReadCloser, metaData Metadata, err error) + // Head returns the metadata Head(token string, filename string) (metadata Metadata, err error) + // Meta updates the file's metadata Meta(token string, filename string, metadata Metadata) error + // Put stores the content of reader including the metadata Put(token string, filename string, reader io.Reader, metadata Metadata) error + // Delete deletes the file Delete(token string, filename string) error + // IsNotExist checks if the error is an "not file does not exist" error IsNotExist(err error) bool - DeleteExpired() error - + // Type returns the name of the storage it implements Type() string } diff --git a/server/storage/gdrive.go b/server/storage/gdrive.go index cfc615c..1fc6748 100644 --- a/server/storage/gdrive.go +++ b/server/storage/gdrive.go @@ -227,7 +227,8 @@ func (s *GDrive) IsNotExist(err error) bool { return false } -func (s *GDrive) DeleteExpired() error { +func (s *GDrive) deleteExpired() error { + //ToDo: figure out if necessary return nil } diff --git a/server/storage/local.go b/server/storage/local.go index 77f5415..36d022c 100644 --- a/server/storage/local.go +++ b/server/storage/local.go @@ -8,16 +8,34 @@ import ( "log" "os" "path/filepath" + "time" ) type LocalStorage struct { Storage - basedir string - logger *log.Logger + basedir string + logger *log.Logger + cleanupJob chan struct{} } -func NewLocalStorage(basedir string, logger *log.Logger) (*LocalStorage, error) { - return &LocalStorage{basedir: basedir, logger: logger}, nil +func NewLocalStorage(basedir string, cleanupInterval int, logger *log.Logger) (*LocalStorage, error) { + storage := &LocalStorage{basedir: basedir, logger: logger} + + ticker := time.NewTicker(time.Duration(cleanupInterval) * time.Hour * 24) + storage.cleanupJob = make(chan struct{}) + go func() { + for { + select { + case <-ticker.C: + err := storage.deleteExpired() + log.Printf("error cleaning up expired files: %v", err) + case <-storage.cleanupJob: + ticker.Stop() + return + } + } + }() + return storage, nil } func (s *LocalStorage) Type() string { @@ -74,12 +92,7 @@ func (s *LocalStorage) Put(token string, filename string, reader io.Reader, meta } func (s *LocalStorage) Delete(token string, filename string) (err error) { - metadata := filepath.Join(s.basedir, token, fmt.Sprintf("%s.metadata", filename)) - _ = os.Remove(metadata) - - path := filepath.Join(s.basedir, token, filename) - err = os.Remove(path) - return + return os.RemoveAll(filepath.Join(s.basedir, token)) } func (s *LocalStorage) IsNotExist(err error) bool { @@ -90,7 +103,28 @@ func (s *LocalStorage) IsNotExist(err error) bool { return os.IsNotExist(err) } -func (s *LocalStorage) DeleteExpired() error { +func (s *LocalStorage) deleteExpired() error { + // search for all metadata files + metaFiles, err := filepath.Glob(fmt.Sprintf("%s/*/*.metadata", s.basedir)) + if err != nil { + log.Printf("error searching for expired files %v \n", err) + return err + } + var meta Metadata + for _, file := range metaFiles { + f, err := os.Open(file) + if err != nil { + log.Printf("error opening file: %s \n", file) + return err + } + err = json.NewDecoder(f).Decode(&meta) + if err == nil { + if time.Now().After(meta.MaxDate) { + // remove folder and all files in it + _ = os.RemoveAll(filepath.Dir(file)) + } + } + } return nil } diff --git a/server/storage/s3.go b/server/storage/s3.go index 1517fd3..e54060b 100644 --- a/server/storage/s3.go +++ b/server/storage/s3.go @@ -210,11 +210,6 @@ func (s *S3Storage) IsNotExist(err error) bool { return false } -func (s *S3Storage) DeleteExpired() error { - // not necessary, as S3 has expireDate on files to automatically delete the them - return nil -} - func getAwsSession(accessKey, secretKey, region, endpoint string, forcePathStyle bool) *session.Session { return session.Must(session.NewSession(&aws.Config{ Region: aws.String(region),