diff --git a/cmd/gotosocial/action/server/server.go b/cmd/gotosocial/action/server/server.go index 4a8993598..709b3d481 100644 --- a/cmd/gotosocial/action/server/server.go +++ b/cmd/gotosocial/action/server/server.go @@ -239,7 +239,7 @@ var Start action.GTSAction = func(ctx context.Context) error { } // perform initial media prune in case value of MediaRemoteCacheDays changed - if err := processor.AdminMediaRemotePrune(ctx, viper.GetInt(config.Keys.MediaRemoteCacheDays)); err != nil { + if err := processor.AdminMediaPrune(ctx, viper.GetInt(config.Keys.MediaRemoteCacheDays)); err != nil { return fmt.Errorf("error during initial media prune: %s", err) } diff --git a/docs/api/swagger.yaml b/docs/api/swagger.yaml index 7bfb96a35..f61bcbaea 100644 --- a/docs/api/swagger.yaml +++ b/docs/api/swagger.yaml @@ -2580,6 +2580,7 @@ paths: - application/json - application/xml - application/x-www-form-urlencoded + description: Also cleans up unused headers + avatars from the media cache. operationId: mediaCleanup parameters: - description: |- diff --git a/internal/api/client/admin/mediacleanup.go b/internal/api/client/admin/mediacleanup.go index 0a8852ff3..508840b23 100644 --- a/internal/api/client/admin/mediacleanup.go +++ b/internal/api/client/admin/mediacleanup.go @@ -33,6 +33,7 @@ import ( // MediaCleanupPOSTHandler swagger:operation POST /api/v1/admin/media_cleanup mediaCleanup // // Clean up remote media older than the specified number of days. +// Also cleans up unused headers + avatars from the media cache. // // --- // tags: @@ -100,7 +101,7 @@ func (m *Module) MediaCleanupPOSTHandler(c *gin.Context) { remoteCacheDays = 0 } - if errWithCode := m.processor.AdminMediaRemotePrune(c.Request.Context(), remoteCacheDays); errWithCode != nil { + if errWithCode := m.processor.AdminMediaPrune(c.Request.Context(), remoteCacheDays); errWithCode != nil { l.Debugf("error starting prune of remote media: %s", errWithCode.Error()) c.JSON(errWithCode.Code(), gin.H{"error": errWithCode.Safe()}) return diff --git a/internal/db/bundb/media.go b/internal/db/bundb/media.go index 4da80e757..fc3280ddf 100644 --- a/internal/db/bundb/media.go +++ b/internal/db/bundb/media.go @@ -72,3 +72,29 @@ func (m *mediaDB) GetRemoteOlderThan(ctx context.Context, olderThan time.Time, l } return attachments, nil } + +func (m *mediaDB) GetAvatarsAndHeaders(ctx context.Context, maxID string, limit int) ([]*gtsmodel.MediaAttachment, db.Error) { + attachments := []*gtsmodel.MediaAttachment{} + + q := m.newMediaQ(&attachments). + WhereGroup(" AND ", func(innerQ *bun.SelectQuery) *bun.SelectQuery { + return innerQ. + WhereOr("media_attachment.avatar = true"). + WhereOr("media_attachment.header = true") + }). + Order("media_attachment.id DESC") + + if maxID != "" { + q = q.Where("media_attachment.id < ?", maxID) + } + + if limit != 0 { + q = q.Limit(limit) + } + + if err := q.Scan(ctx); err != nil { + return nil, m.conn.ProcessError(err) + } + + return attachments, nil +} diff --git a/internal/db/bundb/media_test.go b/internal/db/bundb/media_test.go index 3138caf3b..f1809b3fb 100644 --- a/internal/db/bundb/media_test.go +++ b/internal/db/bundb/media_test.go @@ -43,6 +43,14 @@ func (suite *MediaTestSuite) TestGetOlder() { suite.Len(attachments, 2) } +func (suite *MediaTestSuite) TestGetAvisAndHeaders() { + ctx := context.Background() + + attachments, err := suite.db.GetAvatarsAndHeaders(ctx, "", 20) + suite.NoError(err) + suite.Len(attachments, 2) +} + func TestMediaTestSuite(t *testing.T) { suite.Run(t, new(MediaTestSuite)) } diff --git a/internal/db/media.go b/internal/db/media.go index c734502a1..636fc61f2 100644 --- a/internal/db/media.go +++ b/internal/db/media.go @@ -35,4 +35,7 @@ type Media interface { // The selected media attachments will be those with both a URL and a RemoteURL filled in. // In other words, media attachments that originated remotely, and that we currently have cached locally. GetRemoteOlderThan(ctx context.Context, olderThan time.Time, limit int) ([]*gtsmodel.MediaAttachment, Error) + // GetAvatarsAndHeaders fetches limit n avatars and headers with an id < maxID. These headers + // and avis may be in use or not; the caller should check this if it's important. + GetAvatarsAndHeaders(ctx context.Context, maxID string, limit int) ([]*gtsmodel.MediaAttachment, Error) } diff --git a/internal/gtsmodel/mediaattachment.go b/internal/gtsmodel/mediaattachment.go index 20cc6d3bf..2cd287eea 100644 --- a/internal/gtsmodel/mediaattachment.go +++ b/internal/gtsmodel/mediaattachment.go @@ -34,7 +34,7 @@ type MediaAttachment struct { Type FileType `validate:"oneof=Image Gif Audio Video Unknown" bun:",nullzero,notnull"` // Type of file (image/gif/audio/video) FileMeta FileMeta `validate:"required" bun:",embed:filemeta_,nullzero,notnull"` // Metadata about the file AccountID string `validate:"required,ulid" bun:"type:CHAR(26),nullzero,notnull"` // To which account does this attachment belong - Account *Account `validate:"-" bun:"rel:has-one"` // Account corresponding to accountID + Account *Account `validate:"-" bun:"rel:belongs-to,join:account_id=id"` // Account corresponding to accountID Description string `validate:"-" bun:""` // Description of the attachment (for screenreaders) ScheduledStatusID string `validate:"omitempty,ulid" bun:"type:CHAR(26),nullzero"` // To which scheduled status does this attachment belong Blurhash string `validate:"required_if=Type Image,required_if=Type Gif,required_if=Type Video" bun:",nullzero"` // What is the generated blurhash of this attachment diff --git a/internal/media/manager.go b/internal/media/manager.go index 5b4a01021..60290e4ff 100644 --- a/internal/media/manager.go +++ b/internal/media/manager.go @@ -32,6 +32,9 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/db" ) +// selectPruneLimit is the amount of media entries to select at a time from the db when pruning +const selectPruneLimit = 20 + // Manager provides an interface for managing media: parsing, storing, and retrieving media objects like photos, videos, and gifs. type Manager interface { // ProcessMedia begins the process of decoding and storing the given data as an attachment. @@ -66,10 +69,19 @@ type Manager interface { ProcessEmoji(ctx context.Context, data DataFunc, postData PostDataCallbackFunc, shortcode string, id string, uri string, ai *AdditionalEmojiInfo) (*ProcessingEmoji, error) // RecacheMedia refetches, reprocesses, and recaches an existing attachment that has been uncached via pruneRemote. RecacheMedia(ctx context.Context, data DataFunc, postData PostDataCallbackFunc, attachmentID string) (*ProcessingMedia, error) - // PruneRemote prunes all remote media cached on this instance that's older than the given amount of days. + + // PruneAllRemote prunes all remote media attachments cached on this instance which are older than the given amount of days. // 'Pruning' in this context means removing the locally stored data of the attachment (both thumbnail and full size), // and setting 'cached' to false on the associated attachment. - PruneRemote(ctx context.Context, olderThanDays int) (int, error) + // + // The returned int is the amount of media that was pruned by this function. + PruneAllRemote(ctx context.Context, olderThanDays int) (int, error) + // PruneAllMeta prunes unused meta media -- currently, this means unused avatars + headers, but can also be extended + // to include things like attachments that were uploaded on this server but left unused, etc. + // + // The returned int is the amount of media that was pruned by this function. + PruneAllMeta(ctx context.Context) (int, error) + // Stop stops the underlying worker pool of the manager. It should be called // when closing GoToSocial in order to cleanly finish any in-progress jobs. // It will block until workers are finished processing. @@ -128,53 +140,8 @@ func NewManager(database db.DB, storage *kv.KVStore) (Manager, error) { return nil, err } - // start remote cache cleanup cronjob if configured - cacheCleanupDays := viper.GetInt(config.Keys.MediaRemoteCacheDays) - if cacheCleanupDays != 0 { - // we need a way of cancelling running jobs if the media manager is told to stop - pruneCtx, pruneCancel := context.WithCancel(context.Background()) - - // create a new cron instance and add a function to it - c := cron.New(cron.WithLogger(&logrusWrapper{})) - - pruneFunc := func() { - begin := time.Now() - pruned, err := m.PruneRemote(pruneCtx, cacheCleanupDays) - if err != nil { - logrus.Errorf("media manager: error pruning remote cache: %s", err) - return - } - logrus.Infof("media manager: pruned %d remote cache entries in %s", pruned, time.Since(begin)) - } - - // run every night - entryID, err := c.AddFunc("@midnight", pruneFunc) - if err != nil { - pruneCancel() - return nil, fmt.Errorf("error starting media manager remote cache cleanup job: %s", err) - } - - // since we're running a cron job, we should define how the manager should stop them - m.stopCronJobs = func() error { - // try to stop any jobs gracefully by waiting til they're finished - cronCtx := c.Stop() - - select { - case <-cronCtx.Done(): - logrus.Infof("media manager: cron finished jobs and stopped gracefully") - case <-time.After(1 * time.Minute): - logrus.Infof("media manager: cron didn't stop after 60 seconds, will force close") - break - } - - // whether the job is finished neatly or we had to wait a minute, cancel the context on the prune job - pruneCancel() - return nil - } - - // now start all the cron stuff we've lined up - c.Start() - logrus.Infof("media manager: next scheduled remote cache cleanup is %q", c.Entry(entryID).Next) + if err := scheduleCleanupJobs(m); err != nil { + return nil, err } return m, nil @@ -213,9 +180,7 @@ func (m *manager) Stop() error { emojiErr := m.emojiWorker.Stop() var cronErr error - if m.stopCronJobs != nil { - // only set if cache prune age > 0 cronErr = m.stopCronJobs() } @@ -224,5 +189,60 @@ func (m *manager) Stop() error { } else if emojiErr != nil { return emojiErr } + return cronErr } + +func scheduleCleanupJobs(m *manager) error { + // create a new cron instance for scheduling cleanup jobs + c := cron.New(cron.WithLogger(&logrusWrapper{})) + pruneCtx, pruneCancel := context.WithCancel(context.Background()) + + if _, err := c.AddFunc("@midnight", func() { + begin := time.Now() + pruned, err := m.PruneAllMeta(pruneCtx) + if err != nil { + logrus.Errorf("media manager: error pruning meta: %s", err) + return + } + logrus.Infof("media manager: pruned %d meta entries in %s", pruned, time.Since(begin)) + }); err != nil { + pruneCancel() + return fmt.Errorf("error starting media manager meta cleanup job: %s", err) + } + + // start remote cache cleanup cronjob if configured + if mediaRemoteCacheDays := viper.GetInt(config.Keys.MediaRemoteCacheDays); mediaRemoteCacheDays > 0 { + if _, err := c.AddFunc("@midnight", func() { + begin := time.Now() + pruned, err := m.PruneAllRemote(pruneCtx, mediaRemoteCacheDays) + if err != nil { + logrus.Errorf("media manager: error pruning remote cache: %s", err) + return + } + logrus.Infof("media manager: pruned %d remote cache entries in %s", pruned, time.Since(begin)) + }); err != nil { + pruneCancel() + return fmt.Errorf("error starting media manager remote cache cleanup job: %s", err) + } + } + + // try to stop any jobs gracefully by waiting til they're finished + m.stopCronJobs = func() error { + cronCtx := c.Stop() + + select { + case <-cronCtx.Done(): + logrus.Infof("media manager: cron finished jobs and stopped gracefully") + case <-time.After(1 * time.Minute): + logrus.Infof("media manager: cron didn't stop after 60 seconds, will force close jobs") + break + } + + pruneCancel() + return nil + } + + c.Start() + return nil +} diff --git a/internal/media/media_test.go b/internal/media/media_test.go index ee0fd8eea..1b5011801 100644 --- a/internal/media/media_test.go +++ b/internal/media/media_test.go @@ -34,6 +34,7 @@ type MediaStandardTestSuite struct { storage *kv.KVStore manager media.Manager testAttachments map[string]*gtsmodel.MediaAttachment + testAccounts map[string]*gtsmodel.Account } func (suite *MediaStandardTestSuite) SetupSuite() { @@ -48,6 +49,7 @@ func (suite *MediaStandardTestSuite) SetupTest() { testrig.StandardStorageSetup(suite.storage, "../../testrig/media") testrig.StandardDBSetup(suite.db, nil) suite.testAttachments = testrig.NewTestAttachments() + suite.testAccounts = testrig.NewTestAccounts() suite.manager = testrig.NewTestMediaManager(suite.db, suite.storage) } diff --git a/internal/media/prunemeta.go b/internal/media/prunemeta.go new file mode 100644 index 000000000..aa838d2a4 --- /dev/null +++ b/internal/media/prunemeta.go @@ -0,0 +1,87 @@ +/* + GoToSocial + Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package media + +import ( + "context" + + "codeberg.org/gruf/go-store/storage" + "github.com/sirupsen/logrus" + "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" +) + +func (m *manager) PruneAllMeta(ctx context.Context) (int, error) { + var totalPruned int + var maxID string + var attachments []*gtsmodel.MediaAttachment + var err error + + // select 20 attachments at a time and prune them + for attachments, err = m.db.GetAvatarsAndHeaders(ctx, maxID, selectPruneLimit); err == nil && len(attachments) != 0; attachments, err = m.db.GetAvatarsAndHeaders(ctx, maxID, selectPruneLimit) { + // use the id of the last attachment in the slice as the next 'maxID' value + l := len(attachments) + logrus.Tracef("PruneAllMeta: got %d attachments with maxID < %s", l, maxID) + maxID = attachments[l-1].ID + + // prune each attachment that meets one of the following criteria: + // - has no owning account in the database + // - is a header but isn't the owning account's current header + // - is an avatar but isn't the owning account's current avatar + for _, attachment := range attachments { + if attachment.Account == nil || + (attachment.Header && attachment.ID != attachment.Account.HeaderMediaAttachmentID) || + (attachment.Avatar && attachment.ID != attachment.Account.AvatarMediaAttachmentID) { + if err := m.pruneOneAvatarOrHeader(ctx, attachment); err != nil { + return totalPruned, err + } + totalPruned++ + } + } + } + + // make sure we don't have a real error when we leave the loop + if err != nil && err != db.ErrNoEntries { + return totalPruned, err + } + + logrus.Infof("PruneAllMeta: finished pruning avatars + headers: pruned %d entries", totalPruned) + return totalPruned, nil +} + +func (m *manager) pruneOneAvatarOrHeader(ctx context.Context, attachment *gtsmodel.MediaAttachment) error { + if attachment.File.Path != "" { + // delete the full size attachment from storage + logrus.Tracef("pruneOneAvatarOrHeader: deleting %s", attachment.File.Path) + if err := m.storage.Delete(attachment.File.Path); err != nil && err != storage.ErrNotFound { + return err + } + } + + if attachment.Thumbnail.Path != "" { + // delete the thumbnail from storage + logrus.Tracef("pruneOneAvatarOrHeader: deleting %s", attachment.Thumbnail.Path) + if err := m.storage.Delete(attachment.Thumbnail.Path); err != nil && err != storage.ErrNotFound { + return err + } + } + + // delete the attachment entry completely + return m.db.DeleteByID(ctx, attachment.ID, >smodel.MediaAttachment{}) +} diff --git a/internal/media/prunemeta_test.go b/internal/media/prunemeta_test.go new file mode 100644 index 000000000..1358208a8 --- /dev/null +++ b/internal/media/prunemeta_test.go @@ -0,0 +1,131 @@ +/* + GoToSocial + Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package media_test + +import ( + "context" + "testing" + + "codeberg.org/gruf/go-store/storage" + "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/db" +) + +type PruneMetaTestSuite struct { + MediaStandardTestSuite +} + +func (suite *PruneMetaTestSuite) TestPruneMeta() { + ctx := context.Background() + + // start by clearing zork's avatar + header + zorkOldAvatar := suite.testAttachments["local_account_1_avatar"] + zorkOldHeader := suite.testAttachments["local_account_1_avatar"] + zork := suite.testAccounts["local_account_1"] + zork.AvatarMediaAttachmentID = "" + zork.HeaderMediaAttachmentID = "" + if err := suite.db.UpdateByPrimaryKey(ctx, zork); err != nil { + panic(err) + } + + totalPruned, err := suite.manager.PruneAllMeta(ctx) + suite.NoError(err) + suite.Equal(2, totalPruned) + + // media should no longer be stored + _, err = suite.storage.Get(zorkOldAvatar.File.Path) + suite.ErrorIs(err, storage.ErrNotFound) + _, err = suite.storage.Get(zorkOldAvatar.Thumbnail.Path) + suite.ErrorIs(err, storage.ErrNotFound) + _, err = suite.storage.Get(zorkOldHeader.File.Path) + suite.ErrorIs(err, storage.ErrNotFound) + _, err = suite.storage.Get(zorkOldHeader.Thumbnail.Path) + suite.ErrorIs(err, storage.ErrNotFound) + + // attachments should no longer be in the db + _, err = suite.db.GetAttachmentByID(ctx, zorkOldAvatar.ID) + suite.ErrorIs(err, db.ErrNoEntries) + _, err = suite.db.GetAttachmentByID(ctx, zorkOldHeader.ID) + suite.ErrorIs(err, db.ErrNoEntries) +} + +func (suite *PruneMetaTestSuite) TestPruneMetaTwice() { + ctx := context.Background() + + // start by clearing zork's avatar + header + zork := suite.testAccounts["local_account_1"] + zork.AvatarMediaAttachmentID = "" + zork.HeaderMediaAttachmentID = "" + if err := suite.db.UpdateByPrimaryKey(ctx, zork); err != nil { + panic(err) + } + + totalPruned, err := suite.manager.PruneAllMeta(ctx) + suite.NoError(err) + suite.Equal(2, totalPruned) + + // final prune should prune nothing, since the first prune already happened + totalPruned, err = suite.manager.PruneAllMeta(ctx) + suite.NoError(err) + suite.Equal(0, totalPruned) +} +func (suite *PruneMetaTestSuite) TestPruneMetaMultipleAccounts() { + ctx := context.Background() + + // start by clearing zork's avatar + header + zorkOldAvatar := suite.testAttachments["local_account_1_avatar"] + zorkOldHeader := suite.testAttachments["local_account_1_avatar"] + zork := suite.testAccounts["local_account_1"] + zork.AvatarMediaAttachmentID = "" + zork.HeaderMediaAttachmentID = "" + if err := suite.db.UpdateByPrimaryKey(ctx, zork); err != nil { + panic(err) + } + + // set zork's unused header as belonging to turtle + turtle := suite.testAccounts["local_account_1"] + zorkOldHeader.AccountID = turtle.ID + if err := suite.db.UpdateByPrimaryKey(ctx, zorkOldHeader); err != nil { + panic(err) + } + + totalPruned, err := suite.manager.PruneAllMeta(ctx) + suite.NoError(err) + suite.Equal(2, totalPruned) + + // media should no longer be stored + _, err = suite.storage.Get(zorkOldAvatar.File.Path) + suite.ErrorIs(err, storage.ErrNotFound) + _, err = suite.storage.Get(zorkOldAvatar.Thumbnail.Path) + suite.ErrorIs(err, storage.ErrNotFound) + _, err = suite.storage.Get(zorkOldHeader.File.Path) + suite.ErrorIs(err, storage.ErrNotFound) + _, err = suite.storage.Get(zorkOldHeader.Thumbnail.Path) + suite.ErrorIs(err, storage.ErrNotFound) + + // attachments should no longer be in the db + _, err = suite.db.GetAttachmentByID(ctx, zorkOldAvatar.ID) + suite.ErrorIs(err, db.ErrNoEntries) + _, err = suite.db.GetAttachmentByID(ctx, zorkOldHeader.ID) + suite.ErrorIs(err, db.ErrNoEntries) +} + +func TestPruneMetaTestSuite(t *testing.T) { + suite.Run(t, &PruneMetaTestSuite{}) +} diff --git a/internal/media/pruneremote.go b/internal/media/pruneremote.go index 372f7bbb9..f7b77d32e 100644 --- a/internal/media/pruneremote.go +++ b/internal/media/pruneremote.go @@ -29,10 +29,7 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" ) -// amount of media attachments to select at a time from the db when pruning -const selectPruneLimit = 20 - -func (m *manager) PruneRemote(ctx context.Context, olderThanDays int) (int, error) { +func (m *manager) PruneAllRemote(ctx context.Context, olderThanDays int) (int, error) { var totalPruned int // convert days into a duration string @@ -40,23 +37,23 @@ func (m *manager) PruneRemote(ctx context.Context, olderThanDays int) (int, erro // parse the duration string into a duration olderThanHours, err := time.ParseDuration(olderThanHoursString) if err != nil { - return totalPruned, fmt.Errorf("PruneRemote: %d", err) + return totalPruned, fmt.Errorf("PruneAllRemote: %d", err) } // 'subtract' that from the time now to give our threshold olderThan := time.Now().Add(-olderThanHours) - logrus.Infof("PruneRemote: pruning media older than %s", olderThan) + logrus.Infof("PruneAllRemote: pruning media older than %s", olderThan) // select 20 attachments at a time and prune them for attachments, err := m.db.GetRemoteOlderThan(ctx, olderThan, selectPruneLimit); err == nil && len(attachments) != 0; attachments, err = m.db.GetRemoteOlderThan(ctx, olderThan, selectPruneLimit) { // use the age of the oldest attachment (the last one in the slice) as the next 'older than' value l := len(attachments) - logrus.Tracef("PruneRemote: got %d attachments older than %s", l, olderThan) + logrus.Tracef("PruneAllRemote: got %d attachments older than %s", l, olderThan) olderThan = attachments[l-1].CreatedAt // prune each attachment for _, attachment := range attachments { - if err := m.PruneOne(ctx, attachment); err != nil { + if err := m.pruneOneRemote(ctx, attachment); err != nil { return totalPruned, err } totalPruned++ @@ -68,14 +65,14 @@ func (m *manager) PruneRemote(ctx context.Context, olderThanDays int) (int, erro return totalPruned, err } - logrus.Infof("PruneRemote: finished pruning remote media: pruned %d entries", totalPruned) + logrus.Infof("PruneAllRemote: finished pruning remote media: pruned %d entries", totalPruned) return totalPruned, nil } -func (m *manager) PruneOne(ctx context.Context, attachment *gtsmodel.MediaAttachment) error { +func (m *manager) pruneOneRemote(ctx context.Context, attachment *gtsmodel.MediaAttachment) error { if attachment.File.Path != "" { // delete the full size attachment from storage - logrus.Tracef("PruneOne: deleting %s", attachment.File.Path) + logrus.Tracef("pruneOneRemote: deleting %s", attachment.File.Path) if err := m.storage.Delete(attachment.File.Path); err != nil && err != storage.ErrNotFound { return err } @@ -84,7 +81,7 @@ func (m *manager) PruneOne(ctx context.Context, attachment *gtsmodel.MediaAttach if attachment.Thumbnail.Path != "" { // delete the thumbnail from storage - logrus.Tracef("PruneOne: deleting %s", attachment.Thumbnail.Path) + logrus.Tracef("pruneOneRemote: deleting %s", attachment.Thumbnail.Path) if err := m.storage.Delete(attachment.Thumbnail.Path); err != nil && err != storage.ErrNotFound { return err } diff --git a/internal/media/pruneremote_test.go b/internal/media/pruneremote_test.go index c9d040a6f..31c5128ff 100644 --- a/internal/media/pruneremote_test.go +++ b/internal/media/pruneremote_test.go @@ -37,7 +37,7 @@ func (suite *PruneRemoteTestSuite) TestPruneRemote() { testAttachment := suite.testAttachments["remote_account_1_status_1_attachment_1"] suite.True(testAttachment.Cached) - totalPruned, err := suite.manager.PruneRemote(context.Background(), 1) + totalPruned, err := suite.manager.PruneAllRemote(context.Background(), 1) suite.NoError(err) suite.Equal(2, totalPruned) @@ -49,12 +49,12 @@ func (suite *PruneRemoteTestSuite) TestPruneRemote() { } func (suite *PruneRemoteTestSuite) TestPruneRemoteTwice() { - totalPruned, err := suite.manager.PruneRemote(context.Background(), 1) + totalPruned, err := suite.manager.PruneAllRemote(context.Background(), 1) suite.NoError(err) suite.Equal(2, totalPruned) // final prune should prune nothing, since the first prune already happened - totalPrunedAgain, err := suite.manager.PruneRemote(context.Background(), 1) + totalPrunedAgain, err := suite.manager.PruneAllRemote(context.Background(), 1) suite.NoError(err) suite.Equal(0, totalPrunedAgain) } @@ -63,7 +63,7 @@ func (suite *PruneRemoteTestSuite) TestPruneAndRecache() { ctx := context.Background() testAttachment := suite.testAttachments["remote_account_1_status_1_attachment_1"] - totalPruned, err := suite.manager.PruneRemote(ctx, 1) + totalPruned, err := suite.manager.PruneAllRemote(ctx, 1) suite.NoError(err) suite.Equal(2, totalPruned) @@ -116,7 +116,7 @@ func (suite *PruneRemoteTestSuite) TestPruneOneNonExistent() { suite.NoError(err) // Now attempt to prune remote for item with db entry no file - totalPruned, err := suite.manager.PruneRemote(ctx, 1) + totalPruned, err := suite.manager.PruneAllRemote(ctx, 1) suite.NoError(err) suite.Equal(2, totalPruned) } diff --git a/internal/processing/admin.go b/internal/processing/admin.go index 10f3ff8ba..cbbea05b1 100644 --- a/internal/processing/admin.go +++ b/internal/processing/admin.go @@ -54,6 +54,6 @@ func (p *processor) AdminDomainBlockDelete(ctx context.Context, authed *oauth.Au return p.adminProcessor.DomainBlockDelete(ctx, authed.Account, id) } -func (p *processor) AdminMediaRemotePrune(ctx context.Context, mediaRemoteCacheDays int) gtserror.WithCode { - return p.adminProcessor.MediaRemotePrune(ctx, mediaRemoteCacheDays) +func (p *processor) AdminMediaPrune(ctx context.Context, mediaRemoteCacheDays int) gtserror.WithCode { + return p.adminProcessor.MediaPrune(ctx, mediaRemoteCacheDays) } diff --git a/internal/processing/admin/admin.go b/internal/processing/admin/admin.go index 6779f59b7..c528f0fb8 100644 --- a/internal/processing/admin/admin.go +++ b/internal/processing/admin/admin.go @@ -41,7 +41,7 @@ type Processor interface { DomainBlockDelete(ctx context.Context, account *gtsmodel.Account, id string) (*apimodel.DomainBlock, gtserror.WithCode) AccountAction(ctx context.Context, account *gtsmodel.Account, form *apimodel.AdminAccountActionRequest) gtserror.WithCode EmojiCreate(ctx context.Context, account *gtsmodel.Account, user *gtsmodel.User, form *apimodel.EmojiCreateRequest) (*apimodel.Emoji, gtserror.WithCode) - MediaRemotePrune(ctx context.Context, mediaRemoteCacheDays int) gtserror.WithCode + MediaPrune(ctx context.Context, mediaRemoteCacheDays int) gtserror.WithCode } type processor struct { diff --git a/internal/processing/admin/mediaremoteprune.go b/internal/processing/admin/mediaprune.go similarity index 60% rename from internal/processing/admin/mediaremoteprune.go rename to internal/processing/admin/mediaprune.go index e4a50cab8..0e6abe028 100644 --- a/internal/processing/admin/mediaremoteprune.go +++ b/internal/processing/admin/mediaprune.go @@ -26,18 +26,27 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/gtserror" ) -func (p *processor) MediaRemotePrune(ctx context.Context, mediaRemoteCacheDays int) gtserror.WithCode { +func (p *processor) MediaPrune(ctx context.Context, mediaRemoteCacheDays int) gtserror.WithCode { if mediaRemoteCacheDays < 0 { - err := fmt.Errorf("invalid value for mediaRemoteCacheDays prune: value was %d, cannot be less than 0", mediaRemoteCacheDays) + err := fmt.Errorf("MediaPrune: invalid value for mediaRemoteCacheDays prune: value was %d, cannot be less than 0", mediaRemoteCacheDays) return gtserror.NewErrorBadRequest(err, err.Error()) } go func() { - pruned, err := p.mediaManager.PruneRemote(ctx, mediaRemoteCacheDays) + pruned, err := p.mediaManager.PruneAllRemote(ctx, mediaRemoteCacheDays) if err != nil { - logrus.Errorf("MediaRemotePrune: error pruning: %s", err) + logrus.Errorf("MediaPrune: error pruning remote cache: %s", err) } else { - logrus.Infof("MediaRemotePrune: pruned %d entries", pruned) + logrus.Infof("MediaPrune: pruned %d remote cache entries", pruned) + } + }() + + go func() { + pruned, err := p.mediaManager.PruneAllMeta(ctx) + if err != nil { + logrus.Errorf("MediaPrune: error pruning meta: %s", err) + } else { + logrus.Infof("MediaPrune: pruned %d meta entries", pruned) } }() diff --git a/internal/processing/processor.go b/internal/processing/processor.go index d30f2f37e..f34cc568f 100644 --- a/internal/processing/processor.go +++ b/internal/processing/processor.go @@ -114,7 +114,7 @@ type Processor interface { // AdminDomainBlockDelete deletes one domain block, specified by ID, returning the deleted domain block. AdminDomainBlockDelete(ctx context.Context, authed *oauth.Auth, id string) (*apimodel.DomainBlock, gtserror.WithCode) // AdminMediaRemotePrune triggers a prune of remote media according to the given number of mediaRemoteCacheDays - AdminMediaRemotePrune(ctx context.Context, mediaRemoteCacheDays int) gtserror.WithCode + AdminMediaPrune(ctx context.Context, mediaRemoteCacheDays int) gtserror.WithCode // AppCreate processes the creation of a new API application AppCreate(ctx context.Context, authed *oauth.Auth, form *apimodel.ApplicationCreateRequest) (*apimodel.Application, error)