[feature] Gin enable gzip encoding (#405)

* add gin gzip dependency

* add gzip middleware to router

* go mod tidy

README.md

@@ -176,6 +176,7 @@ The following libraries and frameworks are used by GoToSocial, with gratitude
 - [coreos/go-oidc](https://github.com/coreos/go-oidc); OIDC client library. [Apache-2.0 License](https://spdx.org/licenses/Apache-2.0.html).
 - [gin-gonic/gin](https://github.com/gin-gonic/gin); speedy router engine. [MIT License](https://spdx.org/licenses/MIT.html).
   - [gin-contrib/cors](https://github.com/gin-contrib/cors); Gin CORS middleware. [MIT License](https://spdx.org/licenses/MIT.html).
+  - [gin-contrib/gzip](https://github.com/gin-contrib/gzip); Gin gzip middleware. [MIT License](https://spdx.org/licenses/MIT.html).
   - [gin-contrib/sessions](https://github.com/gin-contrib/sessions); Gin sessions middleware. [MIT License](https://spdx.org/licenses/MIT.html)
   - [gin-contrib/static](https://github.com/gin-contrib/static); Gin static page middleware. [MIT License](https://spdx.org/licenses/MIT.html)
 - [go-fed/httpsig](https://github.com/go-fed/httpsig); secure HTTP signature library. [BSD-3-Clause License](https://spdx.org/licenses/BSD-3-Clause.html).

go.mod

@@ -10,6 +10,7 @@ require (
 	github.com/buckket/go-blurhash v1.1.0
 	github.com/coreos/go-oidc/v3 v3.1.0
 	github.com/gin-contrib/cors v1.3.1
+	github.com/gin-contrib/gzip v0.0.5
 	github.com/gin-contrib/sessions v0.0.4
 	github.com/gin-gonic/gin v1.7.7
 	github.com/go-fed/httpsig v1.1.0

go.sum

@@ -196,6 +196,8 @@ github.com/gavv/httpexpect v2.0.0+incompatible/go.mod h1:x+9tiU1YnrOvnB725RkpoLv
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/gin-contrib/cors v1.3.1 h1:doAsuITavI4IOcd0Y19U4B+O0dNWihRyX//nn4sEmgA=
 github.com/gin-contrib/cors v1.3.1/go.mod h1:jjEJ4268OPZUcU7k9Pm653S7lXUGcqMADzFA61xsmDk=
+github.com/gin-contrib/gzip v0.0.5 h1:mhnVU32YnnBh2LPH2iqRqsA/eR7SAqRaD388jL2s/j0=
+github.com/gin-contrib/gzip v0.0.5/go.mod h1:OPIK6HR0Um2vNmBUTlayD7qle4yVVRZT0PyhdUigrKk=
 github.com/gin-contrib/sessions v0.0.4 h1:gq4fNa1Zmp564iHP5G6EBuktilEos8VKhe2sza1KMgo=
 github.com/gin-contrib/sessions v0.0.4/go.mod h1:pQ3sIyviBBGcxgyR8mkeJuXbeV3h3NYmhJADQTq5+Vo=
 github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=

internal/router/gzip.go

@@ -0,0 +1,30 @@
+/*
+   GoToSocial
+   Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package router
+
+import (
+	ginGzip "github.com/gin-contrib/gzip"
+	"github.com/gin-gonic/gin"
+)
+
+func useGzip(engine *gin.Engine) error {
+	gzipMiddleware := ginGzip.Gzip(ginGzip.DefaultCompression)
+	engine.Use(gzipMiddleware)
+	return nil
+}

internal/router/router.go

@@ -137,11 +137,8 @@ func New(ctx context.Context, db db.DB) (Router, error) {
 		return nil, err
 	}
 
-	// set template functions
+	// enable gzip compression on the engine
-	LoadTemplateFunctions(engine)
+	if err := useGzip(engine); err != nil {
-
-	// load templates onto the engine
-	if err := loadTemplates(engine); err != nil {
 		return nil, err
 	}
 
@@ -150,6 +147,14 @@ func New(ctx context.Context, db db.DB) (Router, error) {
 		return nil, err
 	}
 
+	// set template functions
+	LoadTemplateFunctions(engine)
+
+	// load templates onto the engine
+	if err := loadTemplates(engine); err != nil {
+		return nil, err
+	}
+
 	// create the http server here, passing the gin engine as handler
 	bindAddress := viper.GetString(keys.BindAddress)
 	port := viper.GetInt(keys.Port)

vendor/github.com/gin-contrib/gzip/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2017 Gin-Gonic
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

vendor/github.com/gin-contrib/gzip/README.md

@@ -0,0 +1,135 @@
+# GZIP gin's middleware
+
+[![Run Tests](https://github.com/gin-contrib/gzip/actions/workflows/go.yml/badge.svg)](https://github.com/gin-contrib/gzip/actions/workflows/go.yml)
+[![codecov](https://codecov.io/gh/gin-contrib/gzip/branch/master/graph/badge.svg)](https://codecov.io/gh/gin-contrib/gzip)
+[![Go Report Card](https://goreportcard.com/badge/github.com/gin-contrib/gzip)](https://goreportcard.com/report/github.com/gin-contrib/gzip)
+[![GoDoc](https://godoc.org/github.com/gin-contrib/gzip?status.svg)](https://godoc.org/github.com/gin-contrib/gzip)
+[![Join the chat at https://gitter.im/gin-gonic/gin](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/gin-gonic/gin)
+
+Gin middleware to enable `GZIP` support.
+
+## Usage
+
+Download and install it:
+
+```sh
+go get github.com/gin-contrib/gzip
+```
+
+Import it in your code:
+
+```go
+import "github.com/gin-contrib/gzip"
+```
+
+Canonical example:
+
+```go
+package main
+
+import (
+  "fmt"
+  "net/http"
+  "time"
+
+  "github.com/gin-contrib/gzip"
+  "github.com/gin-gonic/gin"
+)
+
+func main() {
+  r := gin.Default()
+  r.Use(gzip.Gzip(gzip.DefaultCompression))
+  r.GET("/ping", func(c *gin.Context) {
+    c.String(http.StatusOK, "pong "+fmt.Sprint(time.Now().Unix()))
+  })
+
+  // Listen and Server in 0.0.0.0:8080
+  if err := r.Run(":8080"); err != nil {
+    log.Fatal(err)
+  }
+}
+```
+
+Customized Excluded Extensions
+
+```go
+package main
+
+import (
+  "fmt"
+  "net/http"
+  "time"
+
+  "github.com/gin-contrib/gzip"
+  "github.com/gin-gonic/gin"
+)
+
+func main() {
+  r := gin.Default()
+  r.Use(gzip.Gzip(gzip.DefaultCompression, gzip.WithExcludedExtensions([]string{".pdf", ".mp4"})))
+  r.GET("/ping", func(c *gin.Context) {
+    c.String(http.StatusOK, "pong "+fmt.Sprint(time.Now().Unix()))
+  })
+
+  // Listen and Server in 0.0.0.0:8080
+  if err := r.Run(":8080"); err != nil {
+    log.Fatal(err)
+  }
+}
+```
+
+Customized Excluded Paths
+
+```go
+package main
+
+import (
+  "fmt"
+  "net/http"
+  "time"
+
+  "github.com/gin-contrib/gzip"
+  "github.com/gin-gonic/gin"
+)
+
+func main() {
+  r := gin.Default()
+  r.Use(gzip.Gzip(gzip.DefaultCompression, gzip.WithExcludedPaths([]string{"/api/"})))
+  r.GET("/ping", func(c *gin.Context) {
+    c.String(http.StatusOK, "pong "+fmt.Sprint(time.Now().Unix()))
+  })
+
+  // Listen and Server in 0.0.0.0:8080
+  if err := r.Run(":8080"); err != nil {
+    log.Fatal(err)
+  }
+}
+```
+
+Customized Excluded Paths
+
+```go
+package main
+
+import (
+  "fmt"
+  "net/http"
+  "time"
+
+  "github.com/gin-contrib/gzip"
+  "github.com/gin-gonic/gin"
+)
+
+func main() {
+  r := gin.Default()
+  r.Use(gzip.Gzip(gzip.DefaultCompression, gzip.WithExcludedPathsRegexs([]string{".*"})))
+  r.GET("/ping", func(c *gin.Context) {
+    c.String(http.StatusOK, "pong "+fmt.Sprint(time.Now().Unix()))
+  })
+
+  // Listen and Server in 0.0.0.0:8080
+  if err := r.Run(":8080"); err != nil {
+    log.Fatal(err)
+  }
+}
+```

vendor/github.com/gin-contrib/gzip/gzip.go

@@ -0,0 +1,39 @@
+package gzip
+
+import (
+	"compress/gzip"
+
+	"github.com/gin-gonic/gin"
+)
+
+const (
+	BestCompression    = gzip.BestCompression
+	BestSpeed          = gzip.BestSpeed
+	DefaultCompression = gzip.DefaultCompression
+	NoCompression      = gzip.NoCompression
+)
+
+func Gzip(level int, options ...Option) gin.HandlerFunc {
+	return newGzipHandler(level, options...).Handle
+}
+
+type gzipWriter struct {
+	gin.ResponseWriter
+	writer *gzip.Writer
+}
+
+func (g *gzipWriter) WriteString(s string) (int, error) {
+	g.Header().Del("Content-Length")
+	return g.writer.Write([]byte(s))
+}
+
+func (g *gzipWriter) Write(data []byte) (int, error) {
+	g.Header().Del("Content-Length")
+	return g.writer.Write(data)
+}
+
+// Fix: https://github.com/mholt/caddy/issues/38
+func (g *gzipWriter) WriteHeader(code int) {
+	g.Header().Del("Content-Length")
+	g.ResponseWriter.WriteHeader(code)
+}

vendor/github.com/gin-contrib/gzip/handler.go

@@ -0,0 +1,84 @@
+package gzip
+
+import (
+	"compress/gzip"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"path/filepath"
+	"strings"
+	"sync"
+
+	"github.com/gin-gonic/gin"
+)
+
+type gzipHandler struct {
+	*Options
+	gzPool sync.Pool
+}
+
+func newGzipHandler(level int, options ...Option) *gzipHandler {
+	handler := &gzipHandler{
+		Options: DefaultOptions,
+		gzPool: sync.Pool{
+			New: func() interface{} {
+				gz, err := gzip.NewWriterLevel(ioutil.Discard, level)
+				if err != nil {
+					panic(err)
+				}
+				return gz
+			},
+		},
+	}
+	for _, setter := range options {
+		setter(handler.Options)
+	}
+	return handler
+}
+
+func (g *gzipHandler) Handle(c *gin.Context) {
+	if fn := g.DecompressFn; fn != nil && c.Request.Header.Get("Content-Encoding") == "gzip" {
+		fn(c)
+	}
+
+	if !g.shouldCompress(c.Request) {
+		return
+	}
+
+	gz := g.gzPool.Get().(*gzip.Writer)
+	defer g.gzPool.Put(gz)
+	defer gz.Reset(ioutil.Discard)
+	gz.Reset(c.Writer)
+
+	c.Header("Content-Encoding", "gzip")
+	c.Header("Vary", "Accept-Encoding")
+	c.Writer = &gzipWriter{c.Writer, gz}
+	defer func() {
+		gz.Close()
+		c.Header("Content-Length", fmt.Sprint(c.Writer.Size()))
+	}()
+	c.Next()
+}
+
+func (g *gzipHandler) shouldCompress(req *http.Request) bool {
+	if !strings.Contains(req.Header.Get("Accept-Encoding"), "gzip") ||
+		strings.Contains(req.Header.Get("Connection"), "Upgrade") ||
+		strings.Contains(req.Header.Get("Accept"), "text/event-stream") {
+
+		return false
+	}
+
+	extension := filepath.Ext(req.URL.Path)
+	if g.ExcludedExtensions.Contains(extension) {
+		return false
+	}
+
+	if g.ExcludedPaths.Contains(req.URL.Path) {
+		return false
+	}
+	if g.ExcludedPathesRegexs.Contains(req.URL.Path) {
+		return false
+	}
+
+	return true
+}

vendor/github.com/gin-contrib/gzip/options.go

@@ -0,0 +1,116 @@
+package gzip
+
+import (
+	"compress/gzip"
+	"net/http"
+	"regexp"
+	"strings"
+
+	"github.com/gin-gonic/gin"
+)
+
+var (
+	DefaultExcludedExtentions = NewExcludedExtensions([]string{
+		".png", ".gif", ".jpeg", ".jpg",
+	})
+	DefaultOptions = &Options{
+		ExcludedExtensions: DefaultExcludedExtentions,
+	}
+)
+
+type Options struct {
+	ExcludedExtensions   ExcludedExtensions
+	ExcludedPaths        ExcludedPaths
+	ExcludedPathesRegexs ExcludedPathesRegexs
+	DecompressFn         func(c *gin.Context)
+}
+
+type Option func(*Options)
+
+func WithExcludedExtensions(args []string) Option {
+	return func(o *Options) {
+		o.ExcludedExtensions = NewExcludedExtensions(args)
+	}
+}
+
+func WithExcludedPaths(args []string) Option {
+	return func(o *Options) {
+		o.ExcludedPaths = NewExcludedPaths(args)
+	}
+}
+
+func WithExcludedPathsRegexs(args []string) Option {
+	return func(o *Options) {
+		o.ExcludedPathesRegexs = NewExcludedPathesRegexs(args)
+	}
+}
+
+func WithDecompressFn(decompressFn func(c *gin.Context)) Option {
+	return func(o *Options) {
+		o.DecompressFn = decompressFn
+	}
+}
+
+// Using map for better lookup performance
+type ExcludedExtensions map[string]bool
+
+func NewExcludedExtensions(extensions []string) ExcludedExtensions {
+	res := make(ExcludedExtensions)
+	for _, e := range extensions {
+		res[e] = true
+	}
+	return res
+}
+
+func (e ExcludedExtensions) Contains(target string) bool {
+	_, ok := e[target]
+	return ok
+}
+
+type ExcludedPaths []string
+
+func NewExcludedPaths(paths []string) ExcludedPaths {
+	return ExcludedPaths(paths)
+}
+
+func (e ExcludedPaths) Contains(requestURI string) bool {
+	for _, path := range e {
+		if strings.HasPrefix(requestURI, path) {
+			return true
+		}
+	}
+	return false
+}
+
+type ExcludedPathesRegexs []*regexp.Regexp
+
+func NewExcludedPathesRegexs(regexs []string) ExcludedPathesRegexs {
+	result := make([]*regexp.Regexp, len(regexs))
+	for i, reg := range regexs {
+		result[i] = regexp.MustCompile(reg)
+	}
+	return result
+}
+
+func (e ExcludedPathesRegexs) Contains(requestURI string) bool {
+	for _, reg := range e {
+		if reg.MatchString(requestURI) {
+			return true
+		}
+	}
+	return false
+}
+
+func DefaultDecompressHandle(c *gin.Context) {
+	if c.Request.Body == nil {
+		return
+	}
+	r, err := gzip.NewReader(c.Request.Body)
+	if err != nil {
+		_ = c.AbortWithError(http.StatusBadRequest, err)
+		return
+	}
+	c.Request.Header.Del("Content-Encoding")
+	c.Request.Header.Del("Content-Length")
+	c.Request.Body = r
+}

vendor/modules.txt

@@ -74,6 +74,9 @@ github.com/fsnotify/fsnotify
 # github.com/gin-contrib/cors v1.3.1
 ## explicit; go 1.13
 github.com/gin-contrib/cors
+# github.com/gin-contrib/gzip v0.0.5
+## explicit; go 1.13
+github.com/gin-contrib/gzip
 # github.com/gin-contrib/sessions v0.0.4
 ## explicit; go 1.13
 github.com/gin-contrib/sessions