From 6e947ff2662d0b9fc475c5b21e9f1cfb0bb308a0 Mon Sep 17 00:00:00 2001 From: Sashanoraa Date: Sun, 15 May 2022 08:52:46 -0400 Subject: [PATCH] [feature] Media cleanup endpoint (#560) Adds an admin endpoint to trigger a remote media cleanup. Fixed #348 Signed-off-by: Sashanoraa --- docs/api/swagger.yaml | 32 +++++ internal/api/client/admin/admin.go | 2 + internal/api/client/admin/mediacleanup.go | 110 +++++++++++++++++ .../api/client/admin/mediacleanup_test.go | 114 ++++++++++++++++++ internal/api/model/admin.go | 9 ++ internal/db/bundb/media_test.go | 2 +- internal/media/pruneremote_test.go | 8 +- testrig/testmodels.go | 49 ++++++++ 8 files changed, 321 insertions(+), 5 deletions(-) create mode 100644 internal/api/client/admin/mediacleanup.go create mode 100644 internal/api/client/admin/mediacleanup_test.go diff --git a/docs/api/swagger.yaml b/docs/api/swagger.yaml index 4c78baff9..7bfb96a35 100644 --- a/docs/api/swagger.yaml +++ b/docs/api/swagger.yaml @@ -2574,6 +2574,38 @@ paths: summary: View domain block with the given ID. tags: - admin + /api/v1/admin/media_cleanup: + post: + consumes: + - application/json + - application/xml + - application/x-www-form-urlencoded + operationId: mediaCleanup + parameters: + - description: |- + Number of days of remote media to keep. Native values will be treated as 0. + If value is not specified, the value of media-remote-cache-days in the server config will be used. + format: int64 + in: query + name: remote_cache_days + type: integer + x-go-name: RemoteCacheDays + produces: + - application/json + responses: + "200": + description: Echos the number of days requested. The cleanup is performed + asynchronously after the request completes. + "400": + description: bad request + "403": + description: forbidden + security: + - OAuth2 Bearer: + - admin + summary: Clean up remote media older than the specified number of days. + tags: + - admin /api/v1/apps: post: consumes: diff --git a/internal/api/client/admin/admin.go b/internal/api/client/admin/admin.go index 7cf2c9bfc..2044c4ab0 100644 --- a/internal/api/client/admin/admin.go +++ b/internal/api/client/admin/admin.go @@ -41,6 +41,7 @@ const ( AccountsPathWithID = AccountsPath + "/:" + IDKey // AccountsActionPath is used for taking action on a single account. AccountsActionPath = AccountsPathWithID + "/action" + MediaCleanupPath = BasePath + "/media_cleanup" // ExportQueryKey is for requesting a public export of some data. ExportQueryKey = "export" @@ -70,5 +71,6 @@ func (m *Module) Route(r router.Router) error { r.AttachHandler(http.MethodGet, DomainBlocksPathWithID, m.DomainBlockGETHandler) r.AttachHandler(http.MethodDelete, DomainBlocksPathWithID, m.DomainBlockDELETEHandler) r.AttachHandler(http.MethodPost, AccountsActionPath, m.AccountActionPOSTHandler) + r.AttachHandler(http.MethodPost, MediaCleanupPath, m.MediaCleanupPOSTHandler) return nil } diff --git a/internal/api/client/admin/mediacleanup.go b/internal/api/client/admin/mediacleanup.go new file mode 100644 index 000000000..0a8852ff3 --- /dev/null +++ b/internal/api/client/admin/mediacleanup.go @@ -0,0 +1,110 @@ +/* + GoToSocial + Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package admin + +import ( + "fmt" + "net/http" + + "github.com/gin-gonic/gin" + "github.com/sirupsen/logrus" + "github.com/spf13/viper" + "github.com/superseriousbusiness/gotosocial/internal/api/model" + "github.com/superseriousbusiness/gotosocial/internal/config" + "github.com/superseriousbusiness/gotosocial/internal/oauth" +) + +// MediaCleanupPOSTHandler swagger:operation POST /api/v1/admin/media_cleanup mediaCleanup +// +// Clean up remote media older than the specified number of days. +// +// --- +// tags: +// - admin +// +// consumes: +// - application/json +// - application/xml +// - application/x-www-form-urlencoded +// +// produces: +// - application/json +// +// security: +// - OAuth2 Bearer: +// - admin +// +// responses: +// '200': +// description: |- +// Echos the number of days requested. The cleanup is performed asynchronously after the request completes. +// '403': +// description: forbidden +// '400': +// description: bad request +func (m *Module) MediaCleanupPOSTHandler(c *gin.Context) { + l := logrus.WithFields(logrus.Fields{ + "func": "MediaCleanupPOSTHandler", + "request_uri": c.Request.RequestURI, + "user_agent": c.Request.UserAgent(), + "origin_ip": c.ClientIP(), + }) + + // make sure we're authed... + authed, err := oauth.Authed(c, true, true, true, true) + if err != nil { + l.Debugf("couldn't auth: %s", err) + c.JSON(http.StatusUnauthorized, gin.H{"error": err.Error()}) + return + } + + // with an admin account + if !authed.User.Admin { + l.Debugf("user %s not an admin", authed.User.ID) + c.JSON(http.StatusForbidden, gin.H{"error": "not an admin"}) + return + } + + // extract the form from the request context + l.Tracef("parsing request form: %+v", c.Request.Form) + form := &model.MediaCleanupRequest{} + if err := c.ShouldBind(form); err != nil { + l.Debugf("error parsing form %+v: %s", c.Request.Form, err) + c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("could not parse form: %s", err)}) + return + } + + var remoteCacheDays int + if form.RemoteCacheDays == nil { + remoteCacheDays = viper.GetInt(config.Keys.MediaRemoteCacheDays) + } else { + remoteCacheDays = *form.RemoteCacheDays + } + if remoteCacheDays < 0 { + remoteCacheDays = 0 + } + + if errWithCode := m.processor.AdminMediaRemotePrune(c.Request.Context(), remoteCacheDays); errWithCode != nil { + l.Debugf("error starting prune of remote media: %s", errWithCode.Error()) + c.JSON(errWithCode.Code(), gin.H{"error": errWithCode.Safe()}) + return + } + + c.JSON(http.StatusOK, remoteCacheDays) +} diff --git a/internal/api/client/admin/mediacleanup_test.go b/internal/api/client/admin/mediacleanup_test.go new file mode 100644 index 000000000..449777541 --- /dev/null +++ b/internal/api/client/admin/mediacleanup_test.go @@ -0,0 +1,114 @@ +/* + GoToSocial + Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package admin_test + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/api/client/admin" +) + +type MediaCleanupTestSuite struct { + AdminStandardTestSuite +} + +func (suite *MediaCleanupTestSuite) TestMediaCleanup() { + testAttachment := suite.testAttachments["remote_account_1_status_1_attachment_2"] + suite.True(testAttachment.Cached) + + // set up the request + recorder := httptest.NewRecorder() + ctx := suite.newContext(recorder, http.MethodPost, []byte("{\"remote_cache_days\": 1}"), admin.EmojiPath, "application/json") + + // call the handler + suite.adminModule.MediaCleanupPOSTHandler(ctx) + + // we should have OK because our request was valid + suite.Equal(http.StatusOK, recorder.Code) + + // Wait for async task to finish + time.Sleep(1 * time.Second) + + // Get media we prunes + prunedAttachment, err := suite.db.GetAttachmentByID(context.Background(), testAttachment.ID) + suite.NoError(err) + + // the media should no longer be cached + suite.False(prunedAttachment.Cached) +} + +func (suite *MediaCleanupTestSuite) TestMediaCleanupNoArg() { + testAttachment := suite.testAttachments["remote_account_1_status_1_attachment_2"] + suite.True(testAttachment.Cached) + println("TIME: ", testAttachment.CreatedAt.String()) + + // set up the request + recorder := httptest.NewRecorder() + ctx := suite.newContext(recorder, http.MethodPost, []byte("{}"), admin.EmojiPath, "application/json") + + // call the handler + suite.adminModule.MediaCleanupPOSTHandler(ctx) + + // we should have OK because our request was valid + suite.Equal(http.StatusOK, recorder.Code) + + // Wait for async task to finish + time.Sleep(1 * time.Second) + + // Get media we prunes + prunedAttachment, err := suite.db.GetAttachmentByID(context.Background(), testAttachment.ID) + suite.NoError(err) + + // the media should no longer be cached + suite.True(prunedAttachment.Cached) +} + +func (suite *MediaCleanupTestSuite) TestMediaCleanupNotOldEnough() { + testAttachment := suite.testAttachments["remote_account_1_status_1_attachment_2"] + suite.True(testAttachment.Cached) + + // set up the request + recorder := httptest.NewRecorder() + ctx := suite.newContext(recorder, http.MethodPost, []byte("{\"remote_cache_days\": 3}"), admin.EmojiPath, "application/json") + + // call the handler + suite.adminModule.MediaCleanupPOSTHandler(ctx) + + // we should have OK because our request was valid + suite.Equal(http.StatusOK, recorder.Code) + + // Wait for async task to finish + time.Sleep(1 * time.Second) + + // Get media we prunes + prunedAttachment, err := suite.db.GetAttachmentByID(context.Background(), testAttachment.ID) + suite.NoError(err) + + // the media should still be cached + suite.True(prunedAttachment.Cached) +} + +func TestMediaCleanupTestSuite(t *testing.T) { + suite.Run(t, &MediaCleanupTestSuite{}) +} diff --git a/internal/api/model/admin.go b/internal/api/model/admin.go index ada415546..023ba42b1 100644 --- a/internal/api/model/admin.go +++ b/internal/api/model/admin.go @@ -91,3 +91,12 @@ type AdminAccountActionRequest struct { // ID of the account to be acted on. TargetAccountID string `form:"-" json:"-" xml:"-"` } + +// MediaCleanupRequest models admin media cleanup parameters +// +// swagger:parameters mediaCleanup +type MediaCleanupRequest struct { + // Number of days of remote media to keep. Native values will be treated as 0. + // If value is not specified, the value of media-remote-cache-days in the server config will be used. + RemoteCacheDays *int `form:"remote_cache_days" json:"remote_cache_days" xml:"remote_cache_days"` +} diff --git a/internal/db/bundb/media_test.go b/internal/db/bundb/media_test.go index 1d0b86275..3138caf3b 100644 --- a/internal/db/bundb/media_test.go +++ b/internal/db/bundb/media_test.go @@ -40,7 +40,7 @@ func (suite *MediaTestSuite) TestGetAttachmentByID() { func (suite *MediaTestSuite) TestGetOlder() { attachments, err := suite.db.GetRemoteOlderThan(context.Background(), time.Now(), 20) suite.NoError(err) - suite.Len(attachments, 1) + suite.Len(attachments, 2) } func TestMediaTestSuite(t *testing.T) { diff --git a/internal/media/pruneremote_test.go b/internal/media/pruneremote_test.go index 9f3b65dd2..c9d040a6f 100644 --- a/internal/media/pruneremote_test.go +++ b/internal/media/pruneremote_test.go @@ -39,7 +39,7 @@ func (suite *PruneRemoteTestSuite) TestPruneRemote() { totalPruned, err := suite.manager.PruneRemote(context.Background(), 1) suite.NoError(err) - suite.Equal(1, totalPruned) + suite.Equal(2, totalPruned) prunedAttachment, err := suite.db.GetAttachmentByID(context.Background(), testAttachment.ID) suite.NoError(err) @@ -51,7 +51,7 @@ func (suite *PruneRemoteTestSuite) TestPruneRemote() { func (suite *PruneRemoteTestSuite) TestPruneRemoteTwice() { totalPruned, err := suite.manager.PruneRemote(context.Background(), 1) suite.NoError(err) - suite.Equal(1, totalPruned) + suite.Equal(2, totalPruned) // final prune should prune nothing, since the first prune already happened totalPrunedAgain, err := suite.manager.PruneRemote(context.Background(), 1) @@ -65,7 +65,7 @@ func (suite *PruneRemoteTestSuite) TestPruneAndRecache() { totalPruned, err := suite.manager.PruneRemote(ctx, 1) suite.NoError(err) - suite.Equal(1, totalPruned) + suite.Equal(2, totalPruned) // media should no longer be stored _, err = suite.storage.Get(testAttachment.File.Path) @@ -118,7 +118,7 @@ func (suite *PruneRemoteTestSuite) TestPruneOneNonExistent() { // Now attempt to prune remote for item with db entry no file totalPruned, err := suite.manager.PruneRemote(ctx, 1) suite.NoError(err) - suite.Equal(1, totalPruned) + suite.Equal(2, totalPruned) } func TestPruneRemoteTestSuite(t *testing.T) { diff --git a/testrig/testmodels.go b/testrig/testmodels.go index a74357feb..08b0e13dc 100644 --- a/testrig/testmodels.go +++ b/testrig/testmodels.go @@ -831,6 +831,55 @@ func NewTestAttachments() map[string]*gtsmodel.MediaAttachment { Header: false, Cached: true, }, + "remote_account_1_status_1_attachment_2": { + ID: "01FVW7RXPQ8YJHTEXYPE7Q8ZY1", + StatusID: "01FVW7JHQFSFK166WWKR8CBA6M", + URL: "http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/attachment/original/01FVW7RXPQ8YJHTEXYPE7Q8ZY0.jpeg", + RemoteURL: "http://fossbros-anonymous.io/attachments/original/13bbc3f8-2b5e-46ea-9531-40b4974d9912.jpeg", + CreatedAt: time.Now().Add(-48 * time.Hour), + UpdatedAt: time.Now().Add(-48 * time.Hour), + Type: gtsmodel.FileTypeImage, + FileMeta: gtsmodel.FileMeta{ + Original: gtsmodel.Original{ + Width: 472, + Height: 291, + Size: 137352, + Aspect: 1.6219931271477663, + }, + Small: gtsmodel.Small{ + Width: 472, + Height: 291, + Size: 137352, + Aspect: 1.6219931271477663, + }, + Focus: gtsmodel.Focus{ + X: 0, + Y: 0, + }, + }, + AccountID: "01F8MH1H7YV1Z7D2C8K2730QBF", + Description: "tweet from thoughts of dog: i drank. all the water. in my bowl. earlier. but just now. i returned. to the same bowl. and it was. full again.. the bowl. is haunted", + ScheduledStatusID: "", + Blurhash: "LARysgM_IU_3~pD%M_Rj_39FIAt6", + Processing: 2, + File: gtsmodel.File{ + Path: "01F8MH1H7YV1Z7D2C8K2730QBF/attachment/original/01FVW7RXPQ8YJHTEXYPE7Q8ZY0.jpeg", + ContentType: "image/jpeg", + FileSize: 19310, + UpdatedAt: time.Now().Add(-48 * time.Hour), + }, + Thumbnail: gtsmodel.Thumbnail{ + Path: "01F8MH1H7YV1Z7D2C8K2730QBF/attachment/small/01FVW7RXPQ8YJHTEXYPE7Q8ZY0.jpeg", + ContentType: "image/jpeg", + FileSize: 20395, + UpdatedAt: time.Now().Add(-48 * time.Hour), + URL: "http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/attachment/small/01FVW7RXPQ8YJHTEXYPE7Q8ZY0.jpeg", + RemoteURL: "http://fossbros-anonymous.io/attachments/small/a499f55b-2d1e-4acd-98d2-1ac2ba6d79b9.jpeg", + }, + Avatar: false, + Header: false, + Cached: true, + }, } }