fileserver working

PROGRESS.md

@@ -69,7 +69,7 @@
     * [ ] /api/v1/suggestions GET                           (Get suggested accounts to follow)
     * [ ] /api/v1/suggestions/:account_id DELETE            (Delete a suggestion)
   * [ ] Statuses
-    * [ ] /api/v1/statuses POST                             (Create a new status)
+    * [x] /api/v1/statuses POST                             (Create a new status)
     * [ ] /api/v1/statuses/:id GET                          (View an existing status)
     * [ ] /api/v1/statuses/:id DELETE                       (Delete a status)
     * [ ] /api/v1/statuses/:id/context GET                  (View statuses above and below status ID)
@@ -86,7 +86,7 @@
     * [ ] /api/v1/statuses/:id/pin POST                     (Pin a status to profile)
     * [ ] /api/v1/statuses/:id/unpin POST                   (Unpin a status from profile)
   * [ ] Media
-    * [ ] /api/v1/media POST                                (Upload a media attachment)
+    * [x] /api/v1/media POST                                (Upload a media attachment)
     * [ ] /api/v1/media/:id GET                             (Get a media attachment)
     * [ ] /api/v1/media/:id PUT                             (Update an attachment)
   * [ ] Polls
@@ -178,8 +178,8 @@
 * [ ] Storage
   * [x] Internal/statuses/preferences etc
     * [x] Postgres interface
-  * [ ] Media storage
+  * [x] Media storage
-    * [ ] Local storage interface
+    * [x] Local storage interface
     * [ ] S3 storage interface
 * [ ] Cache
   * [ ] In-memory cache

cmd/gotosocial/main.go

@@ -100,7 +100,7 @@ func main() {
 			&cli.StringFlag{
 				Name:    flagNames.DbPassword,
 				Usage:   "Database password",
-				Value: defaults.DbPassword,
+				Value:   defaults.DbPassword,
 				EnvVars: []string{envNames.DbPassword},
 			},
 			&cli.StringFlag{

internal/apimodule/auth/auth_test.go

@@ -22,7 +22,6 @@ import (
 	"context"
 	"fmt"
 	"testing"
-	"time"
 
 	"github.com/google/uuid"
 	"github.com/sirupsen/logrus"
@@ -31,7 +30,6 @@ import (
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/db/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/oauth"
-	"github.com/superseriousbusiness/gotosocial/internal/router"
 	"golang.org/x/crypto/bcrypt"
 )
 
@@ -163,27 +161,6 @@ func (suite *AuthTestSuite) TearDownTest() {
 	suite.db = nil
 }
 
-func (suite *AuthTestSuite) TestAPIInitialize() {
-	log := logrus.New()
-	log.SetLevel(logrus.TraceLevel)
-
-	r, err := router.New(suite.config, log)
-	if err != nil {
-		suite.FailNow(fmt.Sprintf("error mapping routes onto router: %s", err))
-	}
-
-	api := New(suite.oauthServer, suite.db, log)
-	if err := api.Route(r); err != nil {
-		suite.FailNow(fmt.Sprintf("error mapping routes onto router: %s", err))
-	}
-
-	r.Start()
-	time.Sleep(60 * time.Second)
-	if err := r.Stop(context.Background()); err != nil {
-		suite.FailNow(fmt.Sprintf("error stopping router: %s", err))
-	}
-}
-
 func TestAuthTestSuite(t *testing.T) {
 	suite.Run(t, new(AuthTestSuite))
 }

internal/apimodule/fileserver/fileserver.go

@@ -1,7 +1,26 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
 package fileserver
 
 import (
 	"fmt"
+	"net/http"
 
 	"github.com/sirupsen/logrus"
 	"github.com/superseriousbusiness/gotosocial/internal/apimodule"
@@ -12,6 +31,13 @@ import (
 	"github.com/superseriousbusiness/gotosocial/internal/storage"
 )
 
+const (
+	accountIDKey = "account_id"
+	mediaTypeKey = "media_type"
+	mediaSizeKey = "media_size"
+	fileNameKey  = "file_name"
+)
+
 // fileServer implements the RESTAPIModule interface.
 // The goal here is to serve requested media files if the gotosocial server is configured to use local storage.
 type fileServer struct {
@@ -24,33 +50,23 @@ type fileServer struct {
 
 // New returns a new fileServer module
 func New(config *config.Config, db db.DB, storage storage.Storage, log *logrus.Logger) apimodule.ClientAPIModule {
-
-	storageBase := config.StorageConfig.BasePath // TODO: do this properly
-
 	return &fileServer{
 		config:      config,
 		db:          db,
 		storage:     storage,
 		log:         log,
-		storageBase: storageBase,
+		storageBase: config.StorageConfig.ServeBasePath,
 	}
 }
 
 // Route satisfies the RESTAPIModule interface
 func (m *fileServer) Route(s router.Router) error {
-	// s.AttachHandler(http.MethodPost, appsPath, m.appsPOSTHandler)
+	s.AttachHandler(http.MethodGet, fmt.Sprintf("%s/:%s/:%s/:%s/:%s", m.storageBase, accountIDKey, mediaTypeKey, mediaSizeKey, fileNameKey), m.ServeFile)
 	return nil
 }
 
 func (m *fileServer) CreateTables(db db.DB) error {
 	models := []interface{}{
-		&gtsmodel.User{},
-		&gtsmodel.Account{},
-		&gtsmodel.Follow{},
-		&gtsmodel.FollowRequest{},
-		&gtsmodel.Status{},
-		&gtsmodel.Application{},
-		&gtsmodel.EmailDomainBlock{},
 		&gtsmodel.MediaAttachment{},
 	}
 

internal/apimodule/fileserver/servefile.go

@@ -0,0 +1,152 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package fileserver
+
+import (
+	"bytes"
+	"net/http"
+	"strings"
+
+	"github.com/gin-gonic/gin"
+	"github.com/sirupsen/logrus"
+	"github.com/superseriousbusiness/gotosocial/internal/db/gtsmodel"
+	"github.com/superseriousbusiness/gotosocial/internal/media"
+)
+
+// ServeFile is for serving attachments, headers, and avatars to the requester from instance storage.
+//
+// Note: to mitigate scraping attempts, no information should be given out on a bad request except "404 page not found".
+// Don't give away account ids or media ids or anything like that; callers shouldn't be able to infer anything.
+func (m *fileServer) ServeFile(c *gin.Context) {
+	l := m.log.WithFields(logrus.Fields{
+		"func":        "ServeFile",
+		"request_uri": c.Request.RequestURI,
+		"user_agent":  c.Request.UserAgent(),
+		"origin_ip":   c.ClientIP(),
+	})
+	l.Trace("received request")
+
+	// We use request params to check what to pull out of the database/storage so check everything. A request URL should be formatted as follows:
+	// "https://example.org/fileserver/[ACCOUNT_ID]/[MEDIA_TYPE]/[MEDIA_SIZE]/[FILE_NAME]"
+	// "FILE_NAME" consists of two parts, the attachment's database id, a period, and the file extension.
+	accountID := c.Param(accountIDKey)
+	if accountID == "" {
+		l.Debug("missing accountID from request")
+		c.String(http.StatusNotFound, "404 page not found")
+		return
+	}
+
+	mediaType := c.Param(mediaTypeKey)
+	if mediaType == "" {
+		l.Debug("missing mediaType from request")
+		c.String(http.StatusNotFound, "404 page not found")
+		return
+	}
+
+	mediaSize := c.Param(mediaSizeKey)
+	if mediaSize == "" {
+		l.Debug("missing mediaSize from request")
+		c.String(http.StatusNotFound, "404 page not found")
+		return
+	}
+
+	fileName := c.Param(fileNameKey)
+	if fileName == "" {
+		l.Debug("missing fileName from request")
+		c.String(http.StatusNotFound, "404 page not found")
+		return
+	}
+
+	// Only serve media types that are defined in our internal media module
+	switch mediaType {
+	case media.MediaHeader:
+	case media.MediaAvatar:
+	case media.MediaAttachment:
+	default:
+		l.Debugf("mediatype %s not recognized", mediaType)
+		c.String(http.StatusNotFound, "404 page not found")
+		return
+	}
+
+	// This corresponds to original-sized image as it was uploaded, or small, which is the thumbnail
+	switch mediaSize {
+	case media.MediaOriginal:
+	case media.MediaSmall:
+	default:
+		l.Debugf("mediasize %s not recognized", mediaSize)
+		c.String(http.StatusNotFound, "404 page not found")
+		return
+	}
+
+	// derive the media id and the file extension from the last part of the request
+	spl := strings.Split(fileName, ".")
+	if len(spl) != 2 {
+		l.Debugf("filename %s not parseable", fileName)
+		c.String(http.StatusNotFound, "404 page not found")
+		return
+	}
+	wantedMediaID := spl[0]
+	fileExtension := spl[1]
+	if wantedMediaID == "" || fileExtension == "" {
+		l.Debugf("filename %s not parseable", fileName)
+		c.String(http.StatusNotFound, "404 page not found")
+		return
+	}
+
+	// now we know the attachment ID that the caller is asking for we can use it to pull the attachment out of the db
+	attachment := &gtsmodel.MediaAttachment{}
+	if err := m.db.GetByID(wantedMediaID, attachment); err != nil {
+		l.Debugf("attachment with id %s not retrievable: %s", wantedMediaID, err)
+		c.String(http.StatusNotFound, "404 page not found")
+		return
+	}
+
+	// make sure the given account id owns the requested attachment
+	if accountID != attachment.AccountID {
+		l.Debugf("account %s does not own attachment with id %s", accountID, wantedMediaID)
+		c.String(http.StatusNotFound, "404 page not found")
+		return
+	}
+
+	// now we can start preparing the response depending on whether we're serving a thumbnail or a larger attachment
+	var storagePath string
+	var contentType string
+	var contentLength int
+	switch mediaSize {
+	case media.MediaOriginal:
+		storagePath = attachment.File.Path
+		contentType = attachment.File.ContentType
+		contentLength = attachment.File.FileSize
+	case media.MediaSmall:
+		storagePath = attachment.Thumbnail.Path
+		contentType = attachment.Thumbnail.ContentType
+		contentLength = attachment.Thumbnail.FileSize
+	}
+
+	// use the path listed on the attachment we pulled out of the database to retrieve the object from storage
+	attachmentBytes, err := m.storage.RetrieveFileFrom(storagePath)
+	if err != nil {
+		l.Debugf("error retrieving from storage: %s", err)
+		c.String(http.StatusNotFound, "404 page not found")
+		return
+	}
+
+	// finally we can return with all the information we derived above
+	c.DataFromReader(http.StatusOK, int64(contentLength), contentType, bytes.NewReader(attachmentBytes), map[string]string{})
+}

internal/apimodule/fileserver/servefile_test.go

@@ -0,0 +1,156 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package fileserver
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/gin-gonic/gin"
+	"github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/config"
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/db/gtsmodel"
+	"github.com/superseriousbusiness/gotosocial/internal/mastotypes"
+	"github.com/superseriousbusiness/gotosocial/internal/media"
+	"github.com/superseriousbusiness/gotosocial/internal/oauth"
+	"github.com/superseriousbusiness/gotosocial/internal/storage"
+	"github.com/superseriousbusiness/gotosocial/testrig"
+)
+
+type ServeFileTestSuite struct {
+	// standard suite interfaces
+	suite.Suite
+	config         *config.Config
+	db             db.DB
+	log            *logrus.Logger
+	storage        storage.Storage
+	mastoConverter mastotypes.Converter
+	mediaHandler   media.MediaHandler
+	oauthServer    oauth.Server
+
+	// standard suite models
+	testTokens       map[string]*oauth.Token
+	testClients      map[string]*oauth.Client
+	testApplications map[string]*gtsmodel.Application
+	testUsers        map[string]*gtsmodel.User
+	testAccounts     map[string]*gtsmodel.Account
+	testAttachments  map[string]*gtsmodel.MediaAttachment
+
+	// item being tested
+	fileServer *fileServer
+}
+
+/*
+	TEST INFRASTRUCTURE
+*/
+
+func (suite *ServeFileTestSuite) SetupSuite() {
+	// setup standard items
+	suite.config = testrig.NewTestConfig()
+	suite.db = testrig.NewTestDB()
+	suite.log = testrig.NewTestLog()
+	suite.storage = testrig.NewTestStorage()
+	suite.mastoConverter = testrig.NewTestMastoConverter(suite.db)
+	suite.mediaHandler = testrig.NewTestMediaHandler(suite.db, suite.storage)
+	suite.oauthServer = testrig.NewTestOauthServer(suite.db)
+
+	// setup module being tested
+	suite.fileServer = New(suite.config, suite.db, suite.storage, suite.log).(*fileServer)
+}
+
+func (suite *ServeFileTestSuite) TearDownSuite() {
+	if err := suite.db.Stop(context.Background()); err != nil {
+		logrus.Panicf("error closing db connection: %s", err)
+	}
+}
+
+func (suite *ServeFileTestSuite) SetupTest() {
+	testrig.StandardDBSetup(suite.db)
+	testrig.StandardStorageSetup(suite.storage, "../../../testrig/media")
+	suite.testTokens = testrig.NewTestTokens()
+	suite.testClients = testrig.NewTestClients()
+	suite.testApplications = testrig.NewTestApplications()
+	suite.testUsers = testrig.NewTestUsers()
+	suite.testAccounts = testrig.NewTestAccounts()
+	suite.testAttachments = testrig.NewTestAttachments()
+}
+
+func (suite *ServeFileTestSuite) TearDownTest() {
+	testrig.StandardDBTeardown(suite.db)
+	testrig.StandardStorageTeardown(suite.storage)
+}
+
+/*
+	ACTUAL TESTS
+*/
+
+func (suite *ServeFileTestSuite) TestServeOriginalFileSuccessful() {
+	targetAttachment, ok := suite.testAttachments["admin_account_status_1_attachment_1"]
+	assert.True(suite.T(), ok)
+	assert.NotNil(suite.T(), targetAttachment)
+
+	recorder := httptest.NewRecorder()
+	ctx, _ := gin.CreateTestContext(recorder)
+	ctx.Request = httptest.NewRequest(http.MethodGet, targetAttachment.URL, nil)
+
+	// normally the router would populate these params from the path values,
+	// but because we're calling the ServeFile function directly, we need to set them manually.
+	ctx.Params = gin.Params{
+		gin.Param{
+			Key:   accountIDKey,
+			Value: targetAttachment.AccountID,
+		},
+		gin.Param{
+			Key:   mediaTypeKey,
+			Value: media.MediaAttachment,
+		},
+		gin.Param{
+			Key:   mediaSizeKey,
+			Value: media.MediaOriginal,
+		},
+		gin.Param{
+			Key:   fileNameKey,
+			Value: fmt.Sprintf("%s.jpeg", targetAttachment.ID),
+		},
+	}
+
+	// call the function we're testing and check status code
+	suite.fileServer.ServeFile(ctx)
+	suite.EqualValues(http.StatusOK, recorder.Code)
+
+	b, err := ioutil.ReadAll(recorder.Body)
+	assert.NoError(suite.T(), err)
+	assert.NotNil(suite.T(), b)
+
+	fileInStorage, err := suite.storage.RetrieveFileFrom(targetAttachment.File.Path)
+	assert.NoError(suite.T(), err)
+	assert.NotNil(suite.T(), fileInStorage)
+	assert.Equal(suite.T(), b, fileInStorage)
+}
+
+func TestServeFileTestSuite(t *testing.T) {
+	suite.Run(t, new(ServeFileTestSuite))
+}

internal/apimodule/media/mediacreate_test.go

@@ -1,3 +1,21 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
 package media
 
 import (
@@ -167,7 +185,7 @@ func (suite *MediaCreateTestSuite) TestStatusCreatePOSTImageHandlerSuccessful()
 	assert.NotEmpty(suite.T(), attachmentReply.ID)
 	assert.NotEmpty(suite.T(), attachmentReply.URL)
 	assert.NotEmpty(suite.T(), attachmentReply.PreviewURL)
-	assert.Equal(suite.T(), len(storageKeysBeforeRequest) + 2, len(storageKeysAfterRequest)) // 2 images should be added to storage: the original and the thumbnail
+	assert.Equal(suite.T(), len(storageKeysBeforeRequest)+2, len(storageKeysAfterRequest)) // 2 images should be added to storage: the original and the thumbnail
 }
 
 func TestMediaCreateTestSuite(t *testing.T) {

internal/apimodule/status/statuscreate.go

@@ -169,13 +169,27 @@ func (m *statusModule) statusCreatePOSTHandler(c *gin.Context) {
 	}
 	newStatus.Emojis = emojis
 
-	// put the new status in the database
+	/*
+		FROM THIS POINT ONWARDS WE ARE HAPPY WITH THE STATUS -- it is valid and we will try to create it
+	*/
+
+	// put the new status in the database, generating an ID for it in the process
 	if err := m.db.Put(newStatus); err != nil {
 		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
 		return
 	}
 
-	// pass to the distributor to take care of side effects -- federation, mentions, updating metadata, etc, etc
+	// change the status ID of the media attachments to the new status
+	for _, a := range newStatus.Attachments {
+		a.StatusID = newStatus.ID
+		a.UpdatedAt = time.Now()
+		if err := m.db.UpdateByID(a.ID, a); err != nil {
+			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
+			return
+		}
+	}
+
+	// pass to the distributor to take care of side effects asynchronously -- federation, mentions, updating metadata, etc, etc
 	m.distributor.FromClientAPI() <- distributor.FromClientAPI{
 		APObjectType:   gtsmodel.ActivityStreamsNote,
 		APActivityType: gtsmodel.ActivityStreamsCreate,
@@ -430,6 +444,10 @@ func (m *statusModule) parseMediaIDs(form *advancedStatusCreateForm, thisAccount
 		if a.AccountID != thisAccountID {
 			return fmt.Errorf("media with id %s does not belong to account %s", mediaID, thisAccountID)
 		}
+		// check they're not already used in a status
+		if a.StatusID != "" || a.ScheduledStatusID != "" {
+			return fmt.Errorf("media with id %s is already attached to a status", mediaID)
+		}
 		attachments = append(attachments, a)
 	}
 	status.Attachments = attachments

internal/apimodule/status/statuscreate_test.go

@@ -19,7 +19,6 @@
 package status
 
 import (
-	"context"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
@@ -45,21 +44,27 @@ import (
 )
 
 type StatusCreateTestSuite struct {
+	// standard suite interfaces
 	suite.Suite
-	config           *config.Config
+	config         *config.Config
-	mockOauthServer  *oauth.MockServer
+	db             db.DB
-	mockStorage      *storage.MockStorage
+	log            *logrus.Logger
-	mediaHandler     media.MediaHandler
+	storage        storage.Storage
-	mastoConverter   mastotypes.Converter
+	mastoConverter mastotypes.Converter
-	distributor      *distributor.MockDistributor
+	mediaHandler   media.MediaHandler
+	oauthServer    oauth.Server
+	distributor    distributor.Distributor
+
+	// standard suite models
 	testTokens       map[string]*oauth.Token
 	testClients      map[string]*oauth.Client
 	testApplications map[string]*gtsmodel.Application
 	testUsers        map[string]*gtsmodel.User
 	testAccounts     map[string]*gtsmodel.Account
-	log              *logrus.Logger
+	testAttachments  map[string]*gtsmodel.MediaAttachment
-	db               db.DB
+
-	statusModule     *statusModule
+	// module being tested
+	statusModule *statusModule
 }
 
 /*
@@ -68,73 +73,34 @@ type StatusCreateTestSuite struct {
 
 // SetupSuite sets some variables on the suite that we can use as consts (more or less) throughout
 func (suite *StatusCreateTestSuite) SetupSuite() {
-	// some of our subsequent entities need a log so create this here
+	// setup standard items
-	log := logrus.New()
+	suite.config = testrig.NewTestConfig()
-	log.SetLevel(logrus.TraceLevel)
+	suite.db = testrig.NewTestDB()
-	suite.log = log
+	suite.log = testrig.NewTestLog()
+	suite.storage = testrig.NewTestStorage()
+	suite.mastoConverter = testrig.NewTestMastoConverter(suite.db)
+	suite.mediaHandler = testrig.NewTestMediaHandler(suite.db, suite.storage)
+	suite.oauthServer = testrig.NewTestOauthServer(suite.db)
+	suite.distributor = testrig.NewTestDistributor()
 
-	// Direct config to local postgres instance
+	// setup module being tested
-	c := config.Empty()
+	suite.statusModule = New(suite.config, suite.db, suite.oauthServer, suite.mediaHandler, suite.mastoConverter, suite.distributor, suite.log).(*statusModule)
-	c.Protocol = "http"
-	c.Host = "localhost"
-	c.DBConfig = &config.DBConfig{
-		Type:            "postgres",
-		Address:         "localhost",
-		Port:            5432,
-		User:            "postgres",
-		Password:        "postgres",
-		Database:        "postgres",
-		ApplicationName: "gotosocial",
-	}
-	c.MediaConfig = &config.MediaConfig{
-		MaxImageSize: 2 << 20,
-	}
-	c.StorageConfig = &config.StorageConfig{
-		Backend:       "local",
-		BasePath:      "/tmp",
-		ServeProtocol: "http",
-		ServeHost:     "localhost",
-		ServeBasePath: "/fileserver/media",
-	}
-	c.StatusesConfig = &config.StatusesConfig{
-		MaxChars:           500,
-		CWMaxChars:         50,
-		PollMaxOptions:     4,
-		PollOptionMaxChars: 50,
-		MaxMediaFiles:      4,
-	}
-	suite.config = c
-
-	// use an actual database for this, because it's just easier than mocking one out
-	database, err := db.New(context.Background(), c, log)
-	if err != nil {
-		suite.FailNow(err.Error())
-	}
-	suite.db = database
-
-	suite.mockOauthServer = &oauth.MockServer{}
-	suite.mockStorage = &storage.MockStorage{}
-	suite.mediaHandler = media.New(suite.config, suite.db, suite.mockStorage, log)
-	suite.mastoConverter = mastotypes.New(suite.config, suite.db)
-	suite.distributor = &distributor.MockDistributor{}
-	suite.distributor.On("FromClientAPI").Return(make(chan distributor.FromClientAPI, 100))
-
-	suite.statusModule = New(suite.config, suite.db, suite.mockOauthServer, suite.mediaHandler, suite.mastoConverter, suite.distributor, suite.log).(*statusModule)
 }
 
 func (suite *StatusCreateTestSuite) TearDownSuite() {
-	if err := suite.db.Stop(context.Background()); err != nil {
+	testrig.StandardDBTeardown(suite.db)
-		logrus.Panicf("error closing db connection: %s", err)
+	testrig.StandardStorageTeardown(suite.storage)
-	}
 }
 
 func (suite *StatusCreateTestSuite) SetupTest() {
 	testrig.StandardDBSetup(suite.db)
+	testrig.StandardStorageSetup(suite.storage, "../../../testrig/media")
 	suite.testTokens = testrig.NewTestTokens()
 	suite.testClients = testrig.NewTestClients()
 	suite.testApplications = testrig.NewTestApplications()
 	suite.testUsers = testrig.NewTestUsers()
 	suite.testAccounts = testrig.NewTestAccounts()
+	suite.testAttachments = testrig.NewTestAttachments()
 }
 
 // TearDownTest drops tables to make sure there's no data in the db

internal/config/config.go

@@ -63,8 +63,6 @@ func Empty() *Config {
 	}
 }
 
-
-
 // loadFromFile takes a path to a yaml file and attempts to load a Config object from it
 func loadFromFile(path string) (*Config, error) {
 	bytes, err := os.ReadFile(path)

internal/gotosocial/actions.go

@@ -31,6 +31,7 @@ import (
 	"github.com/superseriousbusiness/gotosocial/internal/apimodule/account"
 	"github.com/superseriousbusiness/gotosocial/internal/apimodule/app"
 	"github.com/superseriousbusiness/gotosocial/internal/apimodule/auth"
+	"github.com/superseriousbusiness/gotosocial/internal/apimodule/fileserver"
 	mediaModule "github.com/superseriousbusiness/gotosocial/internal/apimodule/media"
 	"github.com/superseriousbusiness/gotosocial/internal/cache"
 	"github.com/superseriousbusiness/gotosocial/internal/config"
@@ -72,12 +73,14 @@ var Run action.GTSAction = func(ctx context.Context, c *config.Config, log *logr
 	accountModule := account.New(c, dbService, oauthServer, mediaHandler, mastoConverter, log)
 	appsModule := app.New(oauthServer, dbService, mastoConverter, log)
 	mm := mediaModule.New(dbService, mediaHandler, mastoConverter, c, log)
+	fileServerModule := fileserver.New(c, dbService, storageBackend, log)
 
 	apiModules := []apimodule.ClientAPIModule{
 		authModule, // this one has to go first so the other modules use its middleware
 		accountModule,
 		appsModule,
 		mm,
+		fileServerModule,
 	}
 
 	for _, m := range apiModules {

internal/media/media.go

@@ -32,6 +32,14 @@ import (
 	"github.com/superseriousbusiness/gotosocial/internal/storage"
 )
 
+const (
+	MediaSmall      = "small"
+	MediaOriginal   = "original"
+	MediaAttachment = "attachment"
+	MediaHeader     = "header"
+	MediaAvatar     = "avatar"
+)
+
 // MediaHandler provides an interface for parsing, storing, and retrieving media objects like photos, videos, and gifs.
 type MediaHandler interface {
 	// SetHeaderOrAvatarForAccountID takes a new header image for an account, checks it out, removes exif data from it,
@@ -61,14 +69,6 @@ func New(config *config.Config, database db.DB, storage storage.Storage, log *lo
 	}
 }
 
-// HeaderInfo wraps the urls at which a Header and a StaticHeader is available from the server.
-type HeaderInfo struct {
-	// URL to the header
-	Header string
-	// Static version of the above (eg., a path to a still image if the header is a gif)
-	HeaderStatic string
-}
-
 /*
 	INTERFACE FUNCTIONS
 */
@@ -76,7 +76,7 @@ type HeaderInfo struct {
 func (mh *mediaHandler) SetHeaderOrAvatarForAccountID(img []byte, accountID string, headerOrAvi string) (*gtsmodel.MediaAttachment, error) {
 	l := mh.log.WithField("func", "SetHeaderForAccountID")
 
-	if headerOrAvi != "header" && headerOrAvi != "avatar" {
+	if headerOrAvi != MediaHeader && headerOrAvi != MediaAvatar {
 		return nil, errors.New("header or avatar not selected")
 	}
 
@@ -189,13 +189,13 @@ func (mh *mediaHandler) processImage(data []byte, accountID string, contentType
 	smallURL := fmt.Sprintf("%s/%s/attachment/small/%s.%s", URLbase, accountID, newMediaID, extension)
 
 	// we store the original...
-	originalPath := fmt.Sprintf("%s/%s/attachment/original/%s.%s", mh.config.StorageConfig.BasePath, accountID, newMediaID, extension)
+	originalPath := fmt.Sprintf("%s/%s/%s/%s/%s.%s", mh.config.StorageConfig.BasePath, accountID, MediaAttachment, MediaOriginal, newMediaID, extension)
 	if err := mh.storage.StoreFileAt(originalPath, original.image); err != nil {
 		return nil, fmt.Errorf("storage error: %s", err)
 	}
 
 	// and a thumbnail...
-	smallPath := fmt.Sprintf("%s/%s/attachment/small/%s.%s", mh.config.StorageConfig.BasePath, accountID, newMediaID, extension)
+	smallPath := fmt.Sprintf("%s/%s/%s/%s/%s.%s", mh.config.StorageConfig.BasePath, accountID, MediaAttachment, MediaSmall, newMediaID, extension)
 	if err := mh.storage.StoreFileAt(smallPath, small.image); err != nil {
 		return nil, fmt.Errorf("storage error: %s", err)
 	}
@@ -254,9 +254,9 @@ func (mh *mediaHandler) processHeaderOrAvi(imageBytes []byte, contentType string
 	var isAvatar bool
 
 	switch headerOrAvi {
-	case "header":
+	case MediaHeader:
 		isHeader = true
-	case "avatar":
+	case MediaAvatar:
 		isAvatar = true
 	default:
 		return nil, errors.New("header or avatar not selected")
@@ -299,13 +299,13 @@ func (mh *mediaHandler) processHeaderOrAvi(imageBytes []byte, contentType string
 	smallURL := fmt.Sprintf("%s/%s/%s/small/%s.%s", URLbase, accountID, headerOrAvi, newMediaID, extension)
 
 	// we store the original...
-	originalPath := fmt.Sprintf("%s/%s/%s/original/%s.%s", mh.config.StorageConfig.BasePath, accountID, headerOrAvi, newMediaID, extension)
+	originalPath := fmt.Sprintf("%s/%s/%s/%s/%s.%s", mh.config.StorageConfig.BasePath, accountID, headerOrAvi, MediaOriginal, newMediaID, extension)
 	if err := mh.storage.StoreFileAt(originalPath, original.image); err != nil {
 		return nil, fmt.Errorf("storage error: %s", err)
 	}
 
 	// and a thumbnail...
-	smallPath := fmt.Sprintf("%s/%s/%s/small/%s.%s", mh.config.StorageConfig.BasePath, accountID, headerOrAvi, newMediaID, extension)
+	smallPath := fmt.Sprintf("%s/%s/%s/%s/%s.%s", mh.config.StorageConfig.BasePath, accountID, headerOrAvi, MediaSmall, newMediaID, extension)
 	if err := mh.storage.StoreFileAt(smallPath, small.image); err != nil {
 		return nil, fmt.Errorf("storage error: %s", err)
 	}

internal/storage/inmem.go

@@ -15,13 +15,13 @@ import (
 func NewInMem(c *config.Config, log *logrus.Logger) (Storage, error) {
 	return &inMemStorage{
 		stored: make(map[string][]byte),
-		log: log,
+		log:    log,
 	}, nil
 }
 
 type inMemStorage struct {
 	stored map[string][]byte
-	log *logrus.Logger
+	log    *logrus.Logger
 }
 
 func (s *inMemStorage) StoreFileAt(path string, data []byte) error {
@@ -41,7 +41,7 @@ func (s *inMemStorage) RetrieveFileFrom(path string) ([]byte, error) {
 	return d, nil
 }
 
-func (s *inMemStorage)ListKeys() ([]string, error) {
+func (s *inMemStorage) ListKeys() ([]string, error) {
 	keys := []string{}
 	for k := range s.stored {
 		keys = append(keys, k)

internal/storage/local.go

@@ -28,7 +28,7 @@ func (s *localStorage) StoreFileAt(path string, data []byte) error {
 	l := s.log.WithField("func", "StoreFileAt")
 	l.Debugf("storing at path %s", path)
 	components := strings.Split(path, "/")
-	dir := strings.Join(components[0:len(components) - 1], "/")
+	dir := strings.Join(components[0:len(components)-1], "/")
 	if err := os.MkdirAll(dir, 0777); err != nil {
 		return fmt.Errorf("error writing file at %s: %s", path, err)
 	}

internal/storage/storage.go

@@ -25,6 +25,6 @@ package storage
 type Storage interface {
 	StoreFileAt(path string, data []byte) error
 	RetrieveFileFrom(path string) ([]byte, error)
-   ListKeys() ([]string, error)
+	ListKeys() ([]string, error)
-   RemoveFileAt(path string) error
+	RemoveFileAt(path string) error
 }

testrig/distributor.go

@@ -0,0 +1,25 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package testrig
+
+import "github.com/superseriousbusiness/gotosocial/internal/distributor"
+
+func NewTestDistributor() distributor.Distributor {
+	return distributor.New(nil, NewTestLog())
+}

testrig/testmodels.go

@@ -472,8 +472,8 @@ func NewTestAttachments() map[string]*gtsmodel.MediaAttachment {
 			StatusID:  "502ccd6f-0edf-48d7-9016-2dfa4d3714cd",
 			URL:       "http://localhost:8080/fileserver/8020dbb4-1e7b-4d99-a872-4cf94e64210f/attachment/original/b052241b-f30f-4dc6-92fc-2bad0be1f8d8.jpeg",
 			RemoteURL: "",
-			CreatedAt:      time.Now().Add(-71 * time.Hour),
+			CreatedAt: time.Now().Add(-71 * time.Hour),
-			UpdatedAt:      time.Now().Add(-71 * time.Hour),
+			UpdatedAt: time.Now().Add(-71 * time.Hour),
 			Type:      gtsmodel.FileTypeImage,
 			FileMeta: gtsmodel.FileMeta{
 				Original: gtsmodel.Original{
@@ -516,7 +516,7 @@ func NewTestAttachments() map[string]*gtsmodel.MediaAttachment {
 
 // NewTestStored returns a map of filenames, keyed according to which attachment they pertain to.
 func NewTestStored() map[string]string {
-	return map[string]string {
+	return map[string]string{
 		"admin_account_status_1_attachment_1": "welcome-*.jpeg",
 	}
 }

testrig/util.go

@@ -1,3 +1,21 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
 package testrig
 
 import (