From 1ce21ef1974f60c5199681b78e0163de05ed8cbb Mon Sep 17 00:00:00 2001 From: Stefan Benten Date: Sun, 29 Mar 2020 12:41:17 +0200 Subject: [PATCH] Create Utils Subpackage and add cleanup flag --- cmd/cmd.go | 9 ++++- server/clamav.go | 4 +- server/handlers.go | 77 +++++++++++++++++++------------------ server/handlers_test.go | 4 +- server/server.go | 3 +- server/server_fuzz.go | 5 ++- server/{ => utils}/codec.go | 2 +- server/{ => utils}/utils.go | 28 +++++++------- server/virustotal.go | 11 +++--- 9 files changed, 77 insertions(+), 66 deletions(-) rename server/{ => utils}/codec.go (99%) rename server/{ => utils}/utils.go (92%) diff --git a/cmd/cmd.go b/cmd/cmd.go index 7df57da..81a07fd 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -177,6 +177,11 @@ var globalFlags = []cli.Flag{ Usage: "path to storage", Value: "", }, + cli.IntFlag{ + Name: "cleanup-interval", + Usage: "interval to clean up expired files from local storage", + Value: 1, + }, cli.StringFlag{ Name: "clamav-host", Usage: "clamav-host", @@ -377,7 +382,9 @@ func New() *Cmd { case "local": if v := c.String("basedir"); v == "" { panic("basedir not set.") - } else if localStorage, err := storage.NewLocalStorage(v, logger); err != nil { + } else if cleanUp := c.Int("cleanup-interval"); cleanUp <= 0 { + panic("cleanup-interval invalid.") + } else if localStorage, err := storage.NewLocalStorage(v, cleanUp, logger); err != nil { panic(err) } else { options = append(options, server.UseStorage(localStorage)) diff --git a/server/clamav.go b/server/clamav.go index 4352fbc..8c24371 100644 --- a/server/clamav.go +++ b/server/clamav.go @@ -35,14 +35,14 @@ import ( "time" clamd "github.com/dutchcoders/go-clamd" - + "github.com/dutchcoders/transfer.sh/server/utils" "github.com/gorilla/mux" ) func (s *Server) scanHandler(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) - filename := sanitize(vars["filename"]) + filename := utils.Sanitize(vars["filename"]) contentLength := r.ContentLength contentType := r.Header.Get("Content-Type") diff --git a/server/handlers.go b/server/handlers.go index 570e9a6..7803a94 100644 --- a/server/handlers.go +++ b/server/handlers.go @@ -33,7 +33,7 @@ import ( "errors" "fmt" "html" - html_template "html/template" + htmlTemplate "html/template" "io" "io/ioutil" "log" @@ -47,14 +47,15 @@ import ( "strconv" "strings" "sync" - text_template "text/template" + textTemplate "text/template" "time" web "github.com/dutchcoders/transfer.sh-web" "github.com/dutchcoders/transfer.sh/server/storage" + "github.com/dutchcoders/transfer.sh/server/utils" "github.com/gorilla/mux" "github.com/microcosm-cc/bluemonday" - blackfriday "github.com/russross/blackfriday/v2" + "github.com/russross/blackfriday/v2" "github.com/skip2/go-qrcode" ) @@ -67,19 +68,19 @@ func stripPrefix(path string) string { return strings.Replace(path, web.Prefix+"/", "", -1) } -func initTextTemplates() *text_template.Template { - templateMap := text_template.FuncMap{"format": formatNumber} +func initTextTemplates() *textTemplate.Template { + templateMap := textTemplate.FuncMap{"format": utils.FormatNumber} // Templates with functions available to them - var templates = text_template.New("").Funcs(templateMap) + var templates = textTemplate.New("").Funcs(templateMap) return templates } -func initHTMLTemplates() *html_template.Template { - templateMap := html_template.FuncMap{"format": formatNumber} +func initHTMLTemplates() *htmlTemplate.Template { + templateMap := htmlTemplate.FuncMap{"format": utils.FormatNumber} // Templates with functions available to them - var templates = html_template.New("").Funcs(templateMap) + var templates = htmlTemplate.New("").Funcs(templateMap) return templates } @@ -111,12 +112,12 @@ func (s *Server) RedirectHandler(h http.Handler) http.HandlerFunc { // we don't want to enforce https } else if r.URL.Path == "/health.html" { // health check url won't redirect - } else if strings.HasSuffix(ipAddrFromRemoteAddr(r.Host), ".onion") { + } else if strings.HasSuffix(utils.IpAddrFromRemoteAddr(r.Host), ".onion") { // .onion addresses cannot get a valid certificate, so don't redirect } else if r.Header.Get("X-Forwarded-Proto") == "https" { } else if r.URL.Scheme == "https" { } else { - u := getURL(r) + u := utils.GetURL(r) u.Scheme = "https" http.Redirect(w, r, u.String(), http.StatusPermanentRedirect) @@ -173,7 +174,7 @@ func (s *Server) previewHandler(w http.ResponseWriter, r *http.Request) { contentType := metadata.ContentType var templatePath string - var content html_template.HTML + var content htmlTemplate.HTML switch { case strings.HasPrefix(contentType, "image/"): @@ -201,9 +202,9 @@ func (s *Server) previewHandler(w http.ResponseWriter, r *http.Request) { if strings.HasPrefix(contentType, "text/x-markdown") || strings.HasPrefix(contentType, "text/markdown") { unsafe := blackfriday.Run(data) output := bluemonday.UGCPolicy().SanitizeBytes(unsafe) - content = html_template.HTML(output) + content = htmlTemplate.HTML(output) } else if strings.HasPrefix(contentType, "text/plain") { - content = html_template.HTML(fmt.Sprintf("
%s
", html.EscapeString(string(data)))) + content = htmlTemplate.HTML(fmt.Sprintf("
%s
", html.EscapeString(string(data)))) } else { templatePath = "download.sandbox.html" } @@ -213,9 +214,9 @@ func (s *Server) previewHandler(w http.ResponseWriter, r *http.Request) { } relativeURL, _ := url.Parse(path.Join(s.proxyPath, token, filename)) - resolvedURL := resolveURL(r, relativeURL) + resolvedURL := utils.ResolveURL(r, relativeURL) relativeURLGet, _ := url.Parse(path.Join(s.proxyPath, "get", token, filename)) - resolvedURLGet := resolveURL(r, relativeURLGet) + resolvedURLGet := utils.ResolveURL(r, relativeURLGet) var png []byte png, err = qrcode.Encode(resolvedURL, qrcode.High, 150) if err != nil { @@ -225,12 +226,12 @@ func (s *Server) previewHandler(w http.ResponseWriter, r *http.Request) { qrCode := base64.StdEncoding.EncodeToString(png) - hostname := getURL(r).Host - webAddress := resolveWebAddress(r, s.proxyPath) + hostname := utils.GetURL(r).Host + webAddress := utils.ResolveWebAddress(r, s.proxyPath) data := struct { ContentType string - Content html_template.HTML + Content htmlTemplate.HTML Filename string Url string UrlGet string @@ -265,8 +266,8 @@ func (s *Server) previewHandler(w http.ResponseWriter, r *http.Request) { func (s *Server) viewHandler(w http.ResponseWriter, r *http.Request) { // vars := mux.Vars(r) - hostname := getURL(r).Host - webAddress := resolveWebAddress(r, s.proxyPath) + hostname := utils.GetURL(r).Host + webAddress := utils.ResolveWebAddress(r, s.proxyPath) data := struct { Hostname string @@ -280,7 +281,7 @@ func (s *Server) viewHandler(w http.ResponseWriter, r *http.Request) { s.userVoiceKey, } - if acceptsHTML(r.Header) { + if utils.AcceptsHTML(r.Header) { if err := htmlTemplates.ExecuteTemplate(w, "index.html", data); err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return @@ -304,13 +305,13 @@ func (s *Server) postHandler(w http.ResponseWriter, r *http.Request) { return } - token := Encode(10000000 + int64(rand.Intn(1000000000))) + token := utils.Encode(10000000 + int64(rand.Intn(1000000000))) w.Header().Set("Content-Type", "text/plain") for _, fheaders := range r.MultipartForm.File { for _, fheader := range fheaders { - filename := sanitize(fheader.Filename) + filename := utils.Sanitize(fheader.Filename) contentType := fheader.Header.Get("Content-Type") if contentType == "" { @@ -344,7 +345,7 @@ func (s *Server) postHandler(w http.ResponseWriter, r *http.Request) { log.Fatal(err) } - defer cleanTmpFile(file) + defer utils.CleanTmpFile(file) n, err = io.Copy(file, io.MultiReader(&b, f)) if err != nil { @@ -371,7 +372,7 @@ func (s *Server) postHandler(w http.ResponseWriter, r *http.Request) { filename = url.PathEscape(filename) relativeURL, _ := url.Parse(path.Join(s.proxyPath, token, filename)) - _, _ = fmt.Fprintln(w, getURL(r).ResolveReference(relativeURL).String()) + _, _ = fmt.Fprintln(w, utils.GetURL(r).ResolveReference(relativeURL).String()) } } } @@ -379,7 +380,7 @@ func (s *Server) postHandler(w http.ResponseWriter, r *http.Request) { func (s *Server) putHandler(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) - filename := sanitize(vars["filename"]) + filename := utils.Sanitize(vars["filename"]) contentLength := r.ContentLength @@ -415,7 +416,7 @@ func (s *Server) putHandler(w http.ResponseWriter, r *http.Request) { return } - defer cleanTmpFile(file) + defer utils.CleanTmpFile(file) n, err = io.Copy(file, io.MultiReader(&b, f)) if err != nil { @@ -444,7 +445,7 @@ func (s *Server) putHandler(w http.ResponseWriter, r *http.Request) { contentType = mime.TypeByExtension(filepath.Ext(vars["filename"])) } - token := Encode(10000000 + int64(rand.Intn(1000000000))) + token := utils.Encode(10000000 + int64(rand.Intn(1000000000))) metadata := s.metadataForRequest(contentType, contentLength, r) @@ -466,9 +467,9 @@ func (s *Server) putHandler(w http.ResponseWriter, r *http.Request) { relativeURL, _ := url.Parse(path.Join(s.proxyPath, token, filename)) deleteURL, _ := url.Parse(path.Join(s.proxyPath, token, filename, metadata.DeletionToken)) - w.Header().Set("X-Url-Delete", resolveURL(r, deleteURL)) + w.Header().Set("X-Url-Delete", utils.ResolveURL(r, deleteURL)) - _, _ = fmt.Fprint(w, resolveURL(r, relativeURL)) + _, _ = fmt.Fprint(w, utils.ResolveURL(r, relativeURL)) } func (s *Server) deleteHandler(w http.ResponseWriter, r *http.Request) { @@ -509,10 +510,10 @@ func (s *Server) zipHandler(w http.ResponseWriter, r *http.Request) { zw := zip.NewWriter(w) for _, key := range strings.Split(files, ",") { - key = resolveKey(key, s.proxyPath) + key = utils.ResolveKey(key, s.proxyPath) token := strings.Split(key, "/")[0] - filename := sanitize(strings.Split(key, "/")[1]) + filename := utils.Sanitize(strings.Split(key, "/")[1]) if _, err := s.checkMetadata(token, filename, true); err != nil { log.Printf("Error metadata: %s", err.Error()) @@ -580,10 +581,10 @@ func (s *Server) tarGzHandler(w http.ResponseWriter, r *http.Request) { defer zw.Close() for _, key := range strings.Split(files, ",") { - key = resolveKey(key, s.proxyPath) + key = utils.ResolveKey(key, s.proxyPath) token := strings.Split(key, "/")[0] - filename := sanitize(strings.Split(key, "/")[1]) + filename := utils.Sanitize(strings.Split(key, "/")[1]) if _, err := s.checkMetadata(token, filename, true); err != nil { log.Printf("Error metadata: %s", err.Error()) @@ -639,7 +640,7 @@ func (s *Server) tarHandler(w http.ResponseWriter, r *http.Request) { defer zw.Close() for _, key := range strings.Split(files, ",") { - key = resolveKey(key, s.proxyPath) + key = utils.ResolveKey(key, s.proxyPath) token := strings.Split(key, "/")[0] filename := strings.Split(key, "/")[1] @@ -776,7 +777,7 @@ func (s *Server) getHandler(w http.ResponseWriter, r *http.Request) { return } - defer cleanTmpFile(file) + defer utils.CleanTmpFile(file) tee := io.TeeReader(reader, file) for { @@ -803,7 +804,7 @@ func (s *Server) metadataForRequest(contentType string, contentLength int64, r * MaxDate: time.Now().Add(s.lifetime), Downloads: 0, MaxDownloads: -1, - DeletionToken: Encode(10000000+int64(rand.Intn(1000000000))) + Encode(10000000+int64(rand.Intn(1000000000))), + DeletionToken: utils.Encode(10000000+int64(rand.Intn(1000000000))) + utils.Encode(10000000+int64(rand.Intn(1000000000))), } if v := r.Header.Get("Max-Downloads"); v == "" { diff --git a/server/handlers_test.go b/server/handlers_test.go index ae0113a..6e59dc2 100644 --- a/server/handlers_test.go +++ b/server/handlers_test.go @@ -26,7 +26,7 @@ func (s *SuiteRedirectWithForceHTTPs) SetUpTest(c *C) { c.Assert(err, IsNil) handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintln(w, "Hello, client") + _, _ = fmt.Fprintln(w, "Hello, client") }) s.handler = srvr.RedirectHandler(handler) @@ -83,7 +83,7 @@ func (s *SuiteRedirectWithoutForceHTTPs) SetUpTest(c *C) { c.Assert(err, IsNil) handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintln(w, "Hello, client") + _, _ = fmt.Fprintln(w, "Hello, client") }) s.handler = srvr.RedirectHandler(handler) diff --git a/server/server.go b/server/server.go index c3bda9d..f342b86 100644 --- a/server/server.go +++ b/server/server.go @@ -47,6 +47,7 @@ import ( "github.com/VojtechVitek/ratelimit/memory" web "github.com/dutchcoders/transfer.sh-web" "github.com/dutchcoders/transfer.sh/server/storage" + "github.com/dutchcoders/transfer.sh/server/utils" assetfs "github.com/elazarl/go-bindata-assetfs" "github.com/gorilla/mux" "golang.org/x/crypto/acme/autocert" @@ -375,7 +376,7 @@ func (s *Server) Run() { // The file will show a preview page when opening the link in browser directly or // from external link. If the referer url path and current path are the same it will be // downloaded. - if !acceptsHTML(r.Header) { + if !utils.AcceptsHTML(r.Header) { return false } diff --git a/server/server_fuzz.go b/server/server_fuzz.go index 9949be7..c31b8a0 100644 --- a/server/server_fuzz.go +++ b/server/server_fuzz.go @@ -4,6 +4,7 @@ package server import ( "bytes" + "github.com/dutchcoders/transfer.sh/server/utils" "io" "math/rand" "reflect" @@ -23,8 +24,8 @@ func FuzzLocalStorage(fuzz []byte) int { panic("unable to create local storage") } - token := Encode(10000000 + int64(rand.Intn(1000000000))) - filename := Encode(10000000+int64(rand.Intn(1000000000))) + ".bin" + token := utils.Encode(10000000 + int64(rand.Intn(1000000000))) + filename := utils.Encode(10000000+int64(rand.Intn(1000000000))) + ".bin" input := bytes.NewReader(fuzz) err = storage.Put(token, filename, input, applicationOctetStream, fuzzLength) diff --git a/server/codec.go b/server/utils/codec.go similarity index 99% rename from server/codec.go rename to server/utils/codec.go index fda3682..9ec8cd3 100644 --- a/server/codec.go +++ b/server/utils/codec.go @@ -22,7 +22,7 @@ IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package server +package utils import ( "math" diff --git a/server/utils.go b/server/utils/utils.go similarity index 92% rename from server/utils.go rename to server/utils/utils.go index 0d2c4dd..4894d6e 100644 --- a/server/utils.go +++ b/server/utils/utils.go @@ -22,7 +22,7 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package server +package utils import ( "fmt" @@ -40,7 +40,7 @@ import ( "github.com/golang/gddo/httputil/header" ) -func cleanTmpFile(f *os.File) { +func CleanTmpFile(f *os.File) { if f != nil { err := f.Close() if err != nil { @@ -54,17 +54,17 @@ func cleanTmpFile(f *os.File) { } } -func sanitize(fileName string) string { +func Sanitize(fileName string) string { return path.Clean(path.Base(fileName)) } -func resolveURL(r *http.Request, u *url.URL) string { +func ResolveURL(r *http.Request, u *url.URL) string { r.URL.Path = "" - return getURL(r).ResolveReference(u).String() + return GetURL(r).ResolveReference(u).String() } -func resolveKey(key, proxyPath string) string { +func ResolveKey(key, proxyPath string) string { if strings.HasPrefix(key, "/") { key = key[1:] } @@ -78,8 +78,8 @@ func resolveKey(key, proxyPath string) string { return key } -func resolveWebAddress(r *http.Request, proxyPath string) string { - rUrl := getURL(r) +func ResolveWebAddress(r *http.Request, proxyPath string) string { + rUrl := GetURL(r) var webAddress string @@ -97,7 +97,7 @@ func resolveWebAddress(r *http.Request, proxyPath string) string { return webAddress } -func getURL(r *http.Request) *url.URL { +func GetURL(r *http.Request) *url.URL { u, _ := url.Parse(r.URL.String()) if r.TLS != nil { @@ -124,7 +124,7 @@ func getURL(r *http.Request) *url.URL { return u } -func formatNumber(format string, s int64) string { +func FormatNumber(format string, s int64) string { return RenderFloat(format, float64(s)) } @@ -279,7 +279,7 @@ func RenderFloat(format string, n float64) string { // Request.RemoteAddress contains port, which we want to remove i.e.: // "[::1]:58292" => "[::1]" -func ipAddrFromRemoteAddr(s string) string { +func IpAddrFromRemoteAddr(s string) string { idx := strings.LastIndex(s, ":") if idx == -1 { return s @@ -287,12 +287,12 @@ func ipAddrFromRemoteAddr(s string) string { return s[:idx] } -func getIPAddress(r *http.Request) string { +func GetIPAddress(r *http.Request) string { hdr := r.Header hdrRealIP := hdr.Get("X-Real-Ip") hdrForwardedFor := hdr.Get("X-Forwarded-For") if hdrRealIP == "" && hdrForwardedFor == "" { - return ipAddrFromRemoteAddr(r.RemoteAddr) + return IpAddrFromRemoteAddr(r.RemoteAddr) } if hdrForwardedFor != "" { // X-Forwarded-For is potentially a list of addresses separated with "," @@ -316,7 +316,7 @@ func encodeRFC2047(s string) string { return strings.Trim(addr.String(), " <>") } -func acceptsHTML(hdr http.Header) bool { +func AcceptsHTML(hdr http.Header) bool { actual := header.ParseAccept(hdr, "Accept") for _, s := range actual { diff --git a/server/virustotal.go b/server/virustotal.go index 3e0f618..d22c98c 100644 --- a/server/virustotal.go +++ b/server/virustotal.go @@ -29,16 +29,15 @@ import ( "io" "net/http" - _ "github.com/PuerkitoBio/ghost/handlers" + "github.com/dutchcoders/go-virustotal" + "github.com/dutchcoders/transfer.sh/server/utils" "github.com/gorilla/mux" - - virustotal "github.com/dutchcoders/go-virustotal" ) func (s *Server) virusTotalHandler(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) - filename := sanitize(vars["filename"]) + filename := utils.Sanitize(vars["filename"]) contentLength := r.ContentLength contentType := r.Header.Get("Content-Type") @@ -48,6 +47,7 @@ func (s *Server) virusTotalHandler(w http.ResponseWriter, r *http.Request) { vt, err := virustotal.NewVirusTotal(s.VirusTotalKey) if err != nil { http.Error(w, err.Error(), 500) + return } var reader io.Reader @@ -57,8 +57,9 @@ func (s *Server) virusTotalHandler(w http.ResponseWriter, r *http.Request) { result, err := vt.Scan(filename, reader) if err != nil { http.Error(w, err.Error(), 500) + return } s.logger.Println(result) - w.Write([]byte(fmt.Sprintf("%v\n", result.Permalink))) + _, _ = w.Write([]byte(fmt.Sprintf("%v\n", result.Permalink))) }