From 70d65b683fa963d2a8761182a2ddd2f4f9a86bb4 Mon Sep 17 00:00:00 2001 From: tobi <31960611+tsmethurst@users.noreply.github.com> Date: Thu, 13 Oct 2022 15:16:24 +0200 Subject: [PATCH] [feature] Refetch emojis when they change on remote instances (#905) * select emoji using image_static_url * use updated on AP emojis * allow refetch of updated emojis * cheeky workaround for test * clean up old files for refreshed emoji * check error for originalPostData * shorten GetEmojiByStaticImageURL * delete kirby (sorry nintendo) --- internal/ap/extract.go | 5 + internal/api/s2s/user/inboxpost_test.go | 8 +- internal/cache/emoji.go | 21 ++- internal/db/bundb/emoji.go | 33 ++++ internal/db/bundb/emoji_test.go | 7 + .../20221011125732_refetch_updated_emojis.go | 48 ++++++ internal/db/emoji.go | 5 + .../federation/dereferencing/account_test.go | 10 +- .../federation/dereferencing/dereferencer.go | 2 +- internal/federation/dereferencing/emoji.go | 32 +++- .../federation/dereferencing/emoji_test.go | 2 +- internal/media/manager.go | 8 +- internal/media/manager_test.go | 101 ++++++++++++- internal/media/media_test.go | 2 + internal/media/processingemoji.go | 142 ++++++++++++++---- .../media/test/gts_pixellated-original.png | Bin 0 -> 10296 bytes internal/media/test/gts_pixellated-static.png | Bin 0 -> 1010 bytes internal/media/types.go | 4 +- internal/processing/admin/createemoji.go | 2 +- internal/processing/media/getfile.go | 45 +++--- internal/typeutils/internaltoas.go | 4 + internal/typeutils/internaltoas_test.go | 6 +- 22 files changed, 413 insertions(+), 74 deletions(-) create mode 100644 internal/db/bundb/migrations/20221011125732_refetch_updated_emojis.go create mode 100644 internal/media/test/gts_pixellated-original.png create mode 100644 internal/media/test/gts_pixellated-static.png diff --git a/internal/ap/extract.go b/internal/ap/extract.go index 8bd8aa3f4..a7a46e51c 100644 --- a/internal/ap/extract.go +++ b/internal/ap/extract.go @@ -535,6 +535,11 @@ func ExtractEmoji(i Emojiable) (*gtsmodel.Emoji, error) { emoji.Disabled = new(bool) emoji.VisibleInPicker = new(bool) + updatedProp := i.GetActivityStreamsUpdated() + if updatedProp != nil && updatedProp.IsXMLSchemaDateTime() { + emoji.UpdatedAt = updatedProp.Get() + } + return emoji, nil } diff --git a/internal/api/s2s/user/inboxpost_test.go b/internal/api/s2s/user/inboxpost_test.go index a408b3773..364ad336f 100644 --- a/internal/api/s2s/user/inboxpost_test.go +++ b/internal/api/s2s/user/inboxpost_test.go @@ -239,7 +239,13 @@ func (suite *InboxPostTestSuite) TestPostUnblock() { func (suite *InboxPostTestSuite) TestPostUpdate() { updatedAccount := *suite.testAccounts["remote_account_1"] updatedAccount.DisplayName = "updated display name!" - testEmoji := testrig.NewTestEmojis()["rainbow"] + + // ad an emoji to the account; because we're serializing this remote + // account from our own instance, we need to cheat a bit to get the emoji + // to work properly, just for this test + testEmoji := >smodel.Emoji{} + *testEmoji = *testrig.NewTestEmojis()["yell"] + testEmoji.ImageURL = testEmoji.ImageRemoteURL // <- here's the cheat updatedAccount.Emojis = []*gtsmodel.Emoji{testEmoji} asAccount, err := suite.tc.AccountToAS(context.Background(), &updatedAccount) diff --git a/internal/cache/emoji.go b/internal/cache/emoji.go index eda7583ea..117f5475e 100644 --- a/internal/cache/emoji.go +++ b/internal/cache/emoji.go @@ -37,19 +37,26 @@ func NewEmojiCache() *EmojiCache { RegisterLookups: func(lm *cache.LookupMap[string, string]) { lm.RegisterLookup("uri") lm.RegisterLookup("shortcodedomain") + lm.RegisterLookup("imagestaticurl") }, AddLookups: func(lm *cache.LookupMap[string, string], emoji *gtsmodel.Emoji) { + lm.Set("shortcodedomain", shortcodeDomainKey(emoji.Shortcode, emoji.Domain), emoji.ID) if uri := emoji.URI; uri != "" { - lm.Set("uri", uri, emoji.URI) - lm.Set("shortcodedomain", shortcodeDomainKey(emoji.Shortcode, emoji.Domain), emoji.ID) + lm.Set("uri", uri, emoji.ID) + } + if imageStaticURL := emoji.ImageStaticURL; imageStaticURL != "" { + lm.Set("imagestaticurl", imageStaticURL, emoji.ID) } }, DeleteLookups: func(lm *cache.LookupMap[string, string], emoji *gtsmodel.Emoji) { + lm.Delete("shortcodedomain", shortcodeDomainKey(emoji.Shortcode, emoji.Domain)) if uri := emoji.URI; uri != "" { lm.Delete("uri", uri) - lm.Delete("shortcodedomain", shortcodeDomainKey(emoji.Shortcode, emoji.Domain)) + } + if imageStaticURL := emoji.ImageStaticURL; imageStaticURL != "" { + lm.Delete("imagestaticurl", imageStaticURL) } }, }) @@ -72,6 +79,10 @@ func (c *EmojiCache) GetByShortcodeDomain(shortcode string, domain string) (*gts return c.cache.GetBy("shortcodedomain", shortcodeDomainKey(shortcode, domain)) } +func (c *EmojiCache) GetByImageStaticURL(imageStaticURL string) (*gtsmodel.Emoji, bool) { + return c.cache.GetBy("imagestaticurl", imageStaticURL) +} + // Put places an emoji in the cache, ensuring that the object place is a copy for thread-safety func (c *EmojiCache) Put(emoji *gtsmodel.Emoji) { if emoji == nil || emoji.ID == "" { @@ -80,6 +91,10 @@ func (c *EmojiCache) Put(emoji *gtsmodel.Emoji) { c.cache.Set(emoji.ID, copyEmoji(emoji)) } +func (c *EmojiCache) Invalidate(emojiID string) { + c.cache.Invalidate(emojiID) +} + // copyEmoji performs a surface-level copy of emoji, only keeping attached IDs intact, not the objects. // due to all the data being copied being 99% primitive types or strings (which are immutable and passed by ptr) // this should be a relatively cheap process diff --git a/internal/db/bundb/emoji.go b/internal/db/bundb/emoji.go index 640e354c4..4fb4f0ce6 100644 --- a/internal/db/bundb/emoji.go +++ b/internal/db/bundb/emoji.go @@ -21,6 +21,7 @@ package bundb import ( "context" "strings" + "time" "github.com/superseriousbusiness/gotosocial/internal/cache" "github.com/superseriousbusiness/gotosocial/internal/db" @@ -50,6 +51,23 @@ func (e *emojiDB) PutEmoji(ctx context.Context, emoji *gtsmodel.Emoji) db.Error return nil } +func (e *emojiDB) UpdateEmoji(ctx context.Context, emoji *gtsmodel.Emoji, columns ...string) (*gtsmodel.Emoji, db.Error) { + // Update the emoji's last-updated + emoji.UpdatedAt = time.Now() + + if _, err := e.conn. + NewUpdate(). + Model(emoji). + Where("? = ?", bun.Ident("emoji.id"), emoji.ID). + Column(columns...). + Exec(ctx); err != nil { + return nil, e.conn.ProcessError(err) + } + + e.cache.Invalidate(emoji.ID) + return emoji, nil +} + func (e *emojiDB) GetEmojis(ctx context.Context, domain string, includeDisabled bool, includeEnabled bool, shortcode string, maxShortcodeDomain string, minShortcodeDomain string, limit int) ([]*gtsmodel.Emoji, db.Error) { emojiIDs := []string{} @@ -232,6 +250,21 @@ func (e *emojiDB) GetEmojiByShortcodeDomain(ctx context.Context, shortcode strin ) } +func (e *emojiDB) GetEmojiByStaticURL(ctx context.Context, imageStaticURL string) (*gtsmodel.Emoji, db.Error) { + return e.getEmoji( + ctx, + func() (*gtsmodel.Emoji, bool) { + return e.cache.GetByImageStaticURL(imageStaticURL) + }, + func(emoji *gtsmodel.Emoji) error { + return e. + newEmojiQ(emoji). + Where("? = ?", bun.Ident("emoji.image_static_url"), imageStaticURL). + Scan(ctx) + }, + ) +} + func (e *emojiDB) getEmoji(ctx context.Context, cacheGet func() (*gtsmodel.Emoji, bool), dbQuery func(*gtsmodel.Emoji) error) (*gtsmodel.Emoji, db.Error) { // Attempt to fetch cached emoji emoji, cached := cacheGet() diff --git a/internal/db/bundb/emoji_test.go b/internal/db/bundb/emoji_test.go index 3c61fb620..c6577a721 100644 --- a/internal/db/bundb/emoji_test.go +++ b/internal/db/bundb/emoji_test.go @@ -38,6 +38,13 @@ func (suite *EmojiTestSuite) TestGetUseableEmojis() { suite.Equal("rainbow", emojis[0].Shortcode) } +func (suite *EmojiTestSuite) TestGetEmojiByStaticURL() { + emoji, err := suite.db.GetEmojiByStaticURL(context.Background(), "http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/emoji/static/01F8MH9H8E4VG3KDYJR9EGPXCQ.png") + suite.NoError(err) + suite.NotNil(emoji) + suite.Equal("rainbow", emoji.Shortcode) +} + func (suite *EmojiTestSuite) TestGetAllEmojis() { emojis, err := suite.db.GetEmojis(context.Background(), db.EmojiAllDomains, true, true, "", "", "", 0) diff --git a/internal/db/bundb/migrations/20221011125732_refetch_updated_emojis.go b/internal/db/bundb/migrations/20221011125732_refetch_updated_emojis.go new file mode 100644 index 000000000..28ba41a3a --- /dev/null +++ b/internal/db/bundb/migrations/20221011125732_refetch_updated_emojis.go @@ -0,0 +1,48 @@ +/* + GoToSocial + Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package migrations + +import ( + "context" + + gtsmodel "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/uptrace/bun" +) + +func init() { + up := func(ctx context.Context, db *bun.DB) error { + _, err := db. + NewCreateIndex(). + Model(>smodel.Emoji{}). + Index("emojis_image_static_url_idx"). + Column("image_static_url"). + Exec(ctx) + return err + } + + down := func(ctx context.Context, db *bun.DB) error { + return db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { + return nil + }) + } + + if err := Migrations.Register(up, down); err != nil { + panic(err) + } +} diff --git a/internal/db/emoji.go b/internal/db/emoji.go index 4316a43ef..831629232 100644 --- a/internal/db/emoji.go +++ b/internal/db/emoji.go @@ -32,6 +32,9 @@ const EmojiAllDomains string = "all" type Emoji interface { // PutEmoji puts one emoji in the database. PutEmoji(ctx context.Context, emoji *gtsmodel.Emoji) Error + // UpdateEmoji updates the given columns of one emoji. + // If no columns are specified, every column is updated. + UpdateEmoji(ctx context.Context, emoji *gtsmodel.Emoji, columns ...string) (*gtsmodel.Emoji, Error) // GetUseableEmojis gets all emojis which are useable by accounts on this instance. GetUseableEmojis(ctx context.Context) ([]*gtsmodel.Emoji, Error) // GetEmojis gets emojis based on given parameters. Useful for admin actions. @@ -43,4 +46,6 @@ type Emoji interface { GetEmojiByShortcodeDomain(ctx context.Context, shortcode string, domain string) (*gtsmodel.Emoji, Error) // GetEmojiByURI returns one emoji based on its ActivityPub URI. GetEmojiByURI(ctx context.Context, uri string) (*gtsmodel.Emoji, Error) + // GetEmojiByStaticURL gets an emoji using the URL of the static version of the emoji image. + GetEmojiByStaticURL(ctx context.Context, imageStaticURL string) (*gtsmodel.Emoji, Error) } diff --git a/internal/federation/dereferencing/account_test.go b/internal/federation/dereferencing/account_test.go index aec612ac8..ddd9456e8 100644 --- a/internal/federation/dereferencing/account_test.go +++ b/internal/federation/dereferencing/account_test.go @@ -224,6 +224,7 @@ func (suite *AccountTestSuite) TestDereferenceRemoteAccountWithPartial() { URI: "http://fossbros-anonymous.io/emoji/01GD5HCC2YECT012TK8PAGX4D1", Shortcode: "kip_van_den_bos", UpdatedAt: testrig.TimeMustParse("2022-09-13T12:13:12+02:00"), + ImageUpdatedAt: testrig.TimeMustParse("2022-09-13T12:13:12+02:00"), ImageRemoteURL: "http://fossbros-anonymous.io/emoji/kip.gif", Disabled: testrig.FalseBool(), VisibleInPicker: testrig.FalseBool(), @@ -275,10 +276,12 @@ func (suite *AccountTestSuite) TestDereferenceRemoteAccountWithPartial2() { { URI: knownEmoji.URI, Shortcode: knownEmoji.Shortcode, - UpdatedAt: knownEmoji.CreatedAt, + UpdatedAt: knownEmoji.UpdatedAt, + ImageUpdatedAt: knownEmoji.ImageUpdatedAt, ImageRemoteURL: knownEmoji.ImageRemoteURL, Disabled: knownEmoji.Disabled, VisibleInPicker: knownEmoji.VisibleInPicker, + Domain: knownEmoji.Domain, }, }, } @@ -326,10 +329,12 @@ func (suite *AccountTestSuite) TestDereferenceRemoteAccountWithPartial3() { { URI: knownEmoji.URI, Shortcode: knownEmoji.Shortcode, - UpdatedAt: knownEmoji.CreatedAt, + UpdatedAt: knownEmoji.UpdatedAt, + ImageUpdatedAt: knownEmoji.ImageUpdatedAt, ImageRemoteURL: knownEmoji.ImageRemoteURL, Disabled: knownEmoji.Disabled, VisibleInPicker: knownEmoji.VisibleInPicker, + Domain: knownEmoji.Domain, }, }, } @@ -372,6 +377,7 @@ func (suite *AccountTestSuite) TestDereferenceRemoteAccountWithPartial3() { URI: "http://fossbros-anonymous.io/emoji/01GD5HCC2YECT012TK8PAGX4D1", Shortcode: "kip_van_den_bos", UpdatedAt: testrig.TimeMustParse("2022-09-13T12:13:12+02:00"), + ImageUpdatedAt: testrig.TimeMustParse("2022-09-13T12:13:12+02:00"), ImageRemoteURL: "http://fossbros-anonymous.io/emoji/kip.gif", Disabled: testrig.FalseBool(), VisibleInPicker: testrig.FalseBool(), diff --git a/internal/federation/dereferencing/dereferencer.go b/internal/federation/dereferencing/dereferencer.go index 331df3215..a6cb9b15f 100644 --- a/internal/federation/dereferencing/dereferencer.go +++ b/internal/federation/dereferencing/dereferencer.go @@ -41,7 +41,7 @@ type Dereferencer interface { GetRemoteInstance(ctx context.Context, username string, remoteInstanceURI *url.URL) (*gtsmodel.Instance, error) GetRemoteMedia(ctx context.Context, requestingUsername string, accountID string, remoteURL string, ai *media.AdditionalMediaInfo) (*media.ProcessingMedia, error) - GetRemoteEmoji(ctx context.Context, requestingUsername string, remoteURL string, shortcode string, id string, emojiURI string, ai *media.AdditionalEmojiInfo) (*media.ProcessingEmoji, error) + GetRemoteEmoji(ctx context.Context, requestingUsername string, remoteURL string, shortcode string, id string, emojiURI string, ai *media.AdditionalEmojiInfo, refresh bool) (*media.ProcessingEmoji, error) DereferenceAnnounce(ctx context.Context, announce *gtsmodel.Status, requestingUsername string) error DereferenceThread(ctx context.Context, username string, statusIRI *url.URL, status *gtsmodel.Status, statusable ap.Statusable) diff --git a/internal/federation/dereferencing/emoji.go b/internal/federation/dereferencing/emoji.go index 622b131c9..3cdb1d52d 100644 --- a/internal/federation/dereferencing/emoji.go +++ b/internal/federation/dereferencing/emoji.go @@ -31,7 +31,7 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/media" ) -func (d *deref) GetRemoteEmoji(ctx context.Context, requestingUsername string, remoteURL string, shortcode string, id string, emojiURI string, ai *media.AdditionalEmojiInfo) (*media.ProcessingEmoji, error) { +func (d *deref) GetRemoteEmoji(ctx context.Context, requestingUsername string, remoteURL string, shortcode string, id string, emojiURI string, ai *media.AdditionalEmojiInfo, refresh bool) (*media.ProcessingEmoji, error) { t, err := d.transportController.NewTransportForUsername(ctx, requestingUsername) if err != nil { return nil, fmt.Errorf("GetRemoteEmoji: error creating transport: %s", err) @@ -46,7 +46,7 @@ func (d *deref) GetRemoteEmoji(ctx context.Context, requestingUsername string, r return t.DereferenceMedia(innerCtx, derefURI) } - processingMedia, err := d.mediaManager.ProcessEmoji(ctx, dataFunc, nil, shortcode, id, emojiURI, ai) + processingMedia, err := d.mediaManager.ProcessEmoji(ctx, dataFunc, nil, shortcode, id, emojiURI, ai, refresh) if err != nil { return nil, fmt.Errorf("GetRemoteEmoji: error processing emoji: %s", err) } @@ -69,12 +69,34 @@ func (d *deref) populateEmojis(ctx context.Context, rawEmojis []*gtsmodel.Emoji, var err error // check if we've already got this emoji in the db - if gotEmoji, err = d.db.GetEmojiByURI(ctx, e.URI); err != nil && err != db.ErrNoEntries { + if gotEmoji, err = d.db.GetEmojiByShortcodeDomain(ctx, e.Shortcode, e.Domain); err != nil && err != db.ErrNoEntries { log.Errorf("populateEmojis: error checking database for emoji %s: %s", e.URI, err) continue } - if gotEmoji == nil { + if gotEmoji != nil { + // we had the emoji in our database already; make sure the one we have is up to date + if (e.UpdatedAt.After(gotEmoji.ImageUpdatedAt)) || (e.URI != gotEmoji.URI) || (e.ImageRemoteURL != gotEmoji.ImageRemoteURL) { + emojiID := gotEmoji.ID // use existing ID + processingEmoji, err := d.GetRemoteEmoji(ctx, requestingUsername, e.ImageRemoteURL, e.Shortcode, emojiID, e.URI, &media.AdditionalEmojiInfo{ + Domain: &e.Domain, + ImageRemoteURL: &e.ImageRemoteURL, + ImageStaticRemoteURL: &e.ImageRemoteURL, + Disabled: gotEmoji.Disabled, + VisibleInPicker: gotEmoji.VisibleInPicker, + }, true) + + if err != nil { + log.Errorf("populateEmojis: couldn't refresh remote emoji %s: %s", e.URI, err) + continue + } + + if gotEmoji, err = processingEmoji.LoadEmoji(ctx); err != nil { + log.Errorf("populateEmojis: couldn't load refreshed remote emoji %s: %s", e.URI, err) + continue + } + } + } else { // it's new! go get it! newEmojiID, err := id.NewRandomULID() if err != nil { @@ -88,7 +110,7 @@ func (d *deref) populateEmojis(ctx context.Context, rawEmojis []*gtsmodel.Emoji, ImageStaticRemoteURL: &e.ImageRemoteURL, Disabled: e.Disabled, VisibleInPicker: e.VisibleInPicker, - }) + }, false) if err != nil { log.Errorf("populateEmojis: couldn't get remote emoji %s: %s", e.URI, err) diff --git a/internal/federation/dereferencing/emoji_test.go b/internal/federation/dereferencing/emoji_test.go index b03d839ce..3093a1e7f 100644 --- a/internal/federation/dereferencing/emoji_test.go +++ b/internal/federation/dereferencing/emoji_test.go @@ -51,7 +51,7 @@ func (suite *EmojiTestSuite) TestDereferenceEmojiBlocking() { VisibleInPicker: &emojiVisibleInPicker, } - processingEmoji, err := suite.dereferencer.GetRemoteEmoji(ctx, fetchingAccount.Username, emojiImageRemoteURL, emojiShortcode, emojiID, emojiURI, ai) + processingEmoji, err := suite.dereferencer.GetRemoteEmoji(ctx, fetchingAccount.Username, emojiImageRemoteURL, emojiShortcode, emojiID, emojiURI, ai, false) suite.NoError(err) // make a blocking call to load the emoji from the in-process media diff --git a/internal/media/manager.go b/internal/media/manager.go index 828aa033b..62998156e 100644 --- a/internal/media/manager.go +++ b/internal/media/manager.go @@ -69,7 +69,9 @@ type Manager interface { // uri is the ActivityPub URI/ID of the emoji. // // ai is optional and can be nil. Any additional information about the emoji provided will be put in the database. - ProcessEmoji(ctx context.Context, data DataFunc, postData PostDataCallbackFunc, shortcode string, id string, uri string, ai *AdditionalEmojiInfo) (*ProcessingEmoji, error) + // + // If refresh is true, this indicates that the emoji image has changed and should be updated. + ProcessEmoji(ctx context.Context, data DataFunc, postData PostDataCallbackFunc, shortcode string, id string, uri string, ai *AdditionalEmojiInfo, refresh bool) (*ProcessingEmoji, error) // RecacheMedia refetches, reprocesses, and recaches an existing attachment that has been uncached via pruneRemote. RecacheMedia(ctx context.Context, data DataFunc, postData PostDataCallbackFunc, attachmentID string) (*ProcessingMedia, error) @@ -164,8 +166,8 @@ func (m *manager) ProcessMedia(ctx context.Context, data DataFunc, postData Post return processingMedia, nil } -func (m *manager) ProcessEmoji(ctx context.Context, data DataFunc, postData PostDataCallbackFunc, shortcode string, id string, uri string, ai *AdditionalEmojiInfo) (*ProcessingEmoji, error) { - processingEmoji, err := m.preProcessEmoji(ctx, data, postData, shortcode, id, uri, ai) +func (m *manager) ProcessEmoji(ctx context.Context, data DataFunc, postData PostDataCallbackFunc, shortcode string, id string, uri string, ai *AdditionalEmojiInfo, refresh bool) (*ProcessingEmoji, error) { + processingEmoji, err := m.preProcessEmoji(ctx, data, postData, shortcode, id, uri, ai, refresh) if err != nil { return nil, err } diff --git a/internal/media/manager_test.go b/internal/media/manager_test.go index f9c96259d..e00cdd98d 100644 --- a/internal/media/manager_test.go +++ b/internal/media/manager_test.go @@ -55,7 +55,7 @@ func (suite *ManagerTestSuite) TestEmojiProcessBlocking() { emojiID := "01GDQ9G782X42BAMFASKP64343" emojiURI := "http://localhost:8080/emoji/01GDQ9G782X42BAMFASKP64343" - processingEmoji, err := suite.manager.ProcessEmoji(ctx, data, nil, "rainbow_test", emojiID, emojiURI, nil) + processingEmoji, err := suite.manager.ProcessEmoji(ctx, data, nil, "rainbow_test", emojiID, emojiURI, nil, false) suite.NoError(err) // do a blocking call to fetch the emoji @@ -101,6 +101,99 @@ func (suite *ManagerTestSuite) TestEmojiProcessBlocking() { suite.Equal(processedStaticBytesExpected, processedStaticBytes) } +func (suite *ManagerTestSuite) TestEmojiProcessBlockingRefresh() { + ctx := context.Background() + + // we're going to 'refresh' the remote 'yell' emoji by changing the image url to the pixellated gts logo + originalEmoji := suite.testEmojis["yell"] + + emojiToUpdate := >smodel.Emoji{} + *emojiToUpdate = *originalEmoji + newImageRemoteURL := "http://fossbros-anonymous.io/some/image/path.png" + + oldEmojiImagePath := emojiToUpdate.ImagePath + oldEmojiImageStaticPath := emojiToUpdate.ImageStaticPath + + data := func(_ context.Context) (io.Reader, int64, error) { + b, err := os.ReadFile("./test/gts_pixellated-original.png") + if err != nil { + panic(err) + } + return bytes.NewBuffer(b), int64(len(b)), nil + } + + emojiID := emojiToUpdate.ID + emojiURI := emojiToUpdate.URI + + processingEmoji, err := suite.manager.ProcessEmoji(ctx, data, nil, "yell", emojiID, emojiURI, &media.AdditionalEmojiInfo{ + CreatedAt: &emojiToUpdate.CreatedAt, + Domain: &emojiToUpdate.Domain, + ImageRemoteURL: &newImageRemoteURL, + }, true) + suite.NoError(err) + + // do a blocking call to fetch the emoji + emoji, err := processingEmoji.LoadEmoji(ctx) + suite.NoError(err) + suite.NotNil(emoji) + + // make sure it's got the stuff set on it that we expect + suite.Equal(emojiID, emoji.ID) + + // file meta should be correctly derived from the image + suite.Equal("image/png", emoji.ImageContentType) + suite.Equal("image/png", emoji.ImageStaticContentType) + suite.Equal(10296, emoji.ImageFileSize) + + // now make sure the emoji is in the database + dbEmoji, err := suite.db.GetEmojiByID(ctx, emojiID) + suite.NoError(err) + suite.NotNil(dbEmoji) + + // make sure the processed emoji file is in storage + processedFullBytes, err := suite.storage.Get(ctx, emoji.ImagePath) + suite.NoError(err) + suite.NotEmpty(processedFullBytes) + + // load the processed bytes from our test folder, to compare + processedFullBytesExpected, err := os.ReadFile("./test/gts_pixellated-original.png") + suite.NoError(err) + suite.NotEmpty(processedFullBytesExpected) + + // the bytes in storage should be what we expected + suite.Equal(processedFullBytesExpected, processedFullBytes) + + // now do the same for the thumbnail and make sure it's what we expected + processedStaticBytes, err := suite.storage.Get(ctx, emoji.ImageStaticPath) + suite.NoError(err) + suite.NotEmpty(processedStaticBytes) + + processedStaticBytesExpected, err := os.ReadFile("./test/gts_pixellated-static.png") + suite.NoError(err) + suite.NotEmpty(processedStaticBytesExpected) + + suite.Equal(processedStaticBytesExpected, processedStaticBytes) + + // most fields should be different on the emoji now from what they were before + suite.Equal(originalEmoji.ID, dbEmoji.ID) + suite.NotEqual(originalEmoji.ImageRemoteURL, dbEmoji.ImageRemoteURL) + suite.NotEqual(originalEmoji.ImageURL, dbEmoji.ImageURL) + suite.NotEqual(originalEmoji.ImageStaticURL, dbEmoji.ImageStaticURL) + suite.NotEqual(originalEmoji.ImageFileSize, dbEmoji.ImageFileSize) + suite.NotEqual(originalEmoji.ImageStaticFileSize, dbEmoji.ImageStaticFileSize) + suite.NotEqual(originalEmoji.ImagePath, dbEmoji.ImagePath) + suite.NotEqual(originalEmoji.ImageStaticPath, dbEmoji.ImageStaticPath) + suite.NotEqual(originalEmoji.ImageStaticPath, dbEmoji.ImageStaticPath) + suite.NotEqual(originalEmoji.UpdatedAt, dbEmoji.UpdatedAt) + suite.NotEqual(originalEmoji.ImageUpdatedAt, dbEmoji.ImageUpdatedAt) + + // the old image files should no longer be in storage + _, err = suite.storage.Get(ctx, oldEmojiImagePath) + suite.ErrorIs(err, storage.ErrNotFound) + _, err = suite.storage.Get(ctx, oldEmojiImageStaticPath) + suite.ErrorIs(err, storage.ErrNotFound) +} + func (suite *ManagerTestSuite) TestEmojiProcessBlockingTooLarge() { ctx := context.Background() @@ -116,7 +209,7 @@ func (suite *ManagerTestSuite) TestEmojiProcessBlockingTooLarge() { emojiID := "01GDQ9G782X42BAMFASKP64343" emojiURI := "http://localhost:8080/emoji/01GDQ9G782X42BAMFASKP64343" - processingEmoji, err := suite.manager.ProcessEmoji(ctx, data, nil, "big_panda", emojiID, emojiURI, nil) + processingEmoji, err := suite.manager.ProcessEmoji(ctx, data, nil, "big_panda", emojiID, emojiURI, nil, false) suite.NoError(err) // do a blocking call to fetch the emoji @@ -140,7 +233,7 @@ func (suite *ManagerTestSuite) TestEmojiProcessBlockingTooLargeNoSizeGiven() { emojiID := "01GDQ9G782X42BAMFASKP64343" emojiURI := "http://localhost:8080/emoji/01GDQ9G782X42BAMFASKP64343" - processingEmoji, err := suite.manager.ProcessEmoji(ctx, data, nil, "big_panda", emojiID, emojiURI, nil) + processingEmoji, err := suite.manager.ProcessEmoji(ctx, data, nil, "big_panda", emojiID, emojiURI, nil, false) suite.NoError(err) // do a blocking call to fetch the emoji @@ -165,7 +258,7 @@ func (suite *ManagerTestSuite) TestEmojiProcessBlockingNoFileSizeGiven() { emojiURI := "http://localhost:8080/emoji/01GDQ9G782X42BAMFASKP64343" // process the media with no additional info provided - processingEmoji, err := suite.manager.ProcessEmoji(ctx, data, nil, "rainbow_test", emojiID, emojiURI, nil) + processingEmoji, err := suite.manager.ProcessEmoji(ctx, data, nil, "rainbow_test", emojiID, emojiURI, nil, false) suite.NoError(err) // do a blocking call to fetch the emoji diff --git a/internal/media/media_test.go b/internal/media/media_test.go index fda1963a7..e2c3914a3 100644 --- a/internal/media/media_test.go +++ b/internal/media/media_test.go @@ -35,6 +35,7 @@ type MediaStandardTestSuite struct { manager media.Manager testAttachments map[string]*gtsmodel.MediaAttachment testAccounts map[string]*gtsmodel.Account + testEmojis map[string]*gtsmodel.Emoji } func (suite *MediaStandardTestSuite) SetupSuite() { @@ -50,6 +51,7 @@ func (suite *MediaStandardTestSuite) SetupTest() { testrig.StandardDBSetup(suite.db, nil) suite.testAttachments = testrig.NewTestAttachments() suite.testAccounts = testrig.NewTestAccounts() + suite.testEmojis = testrig.NewTestEmojis() suite.manager = testrig.NewTestMediaManager(suite.db, suite.storage) } diff --git a/internal/media/processingemoji.go b/internal/media/processingemoji.go index 6495b991e..e1c6f2efb 100644 --- a/internal/media/processingemoji.go +++ b/internal/media/processingemoji.go @@ -21,6 +21,7 @@ package media import ( "bytes" "context" + "errors" "fmt" "io" "strings" @@ -28,9 +29,11 @@ import ( "sync/atomic" "time" + gostore "codeberg.org/gruf/go-store/storage" "github.com/superseriousbusiness/gotosocial/internal/config" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/id" "github.com/superseriousbusiness/gotosocial/internal/log" "github.com/superseriousbusiness/gotosocial/internal/storage" "github.com/superseriousbusiness/gotosocial/internal/uris" @@ -71,6 +74,11 @@ type ProcessingEmoji struct { // track whether this emoji has already been put in the databse insertedInDB bool + + // is this a refresh of an existing emoji? + refresh bool + // if it is a refresh, which alternate ID should we use in the storage and URL paths? + newPathID string } // EmojiID returns the ID of the underlying emoji without blocking processing. @@ -94,8 +102,28 @@ func (p *ProcessingEmoji) LoadEmoji(ctx context.Context) (*gtsmodel.Emoji, error // store the result in the database before returning it if !p.insertedInDB { - if err := p.database.PutEmoji(ctx, p.emoji); err != nil { - return nil, err + if p.refresh { + columns := []string{ + "updated_at", + "image_remote_url", + "image_static_remote_url", + "image_url", + "image_static_url", + "image_path", + "image_static_path", + "image_content_type", + "image_file_size", + "image_static_file_size", + "image_updated_at", + "uri", + } + if _, err := p.database.UpdateEmoji(ctx, p.emoji, columns...); err != nil { + return nil, err + } + } else { + if err := p.database.PutEmoji(ctx, p.emoji); err != nil { + return nil, err + } } p.insertedInDB = true } @@ -203,8 +231,14 @@ func (p *ProcessingEmoji) store(ctx context.Context) error { // set some additional fields on the emoji now that // we know more about what the underlying image actually is - p.emoji.ImageURL = uris.GenerateURIForAttachment(p.instanceAccountID, string(TypeEmoji), string(SizeOriginal), p.emoji.ID, extension) - p.emoji.ImagePath = fmt.Sprintf("%s/%s/%s/%s.%s", p.instanceAccountID, TypeEmoji, SizeOriginal, p.emoji.ID, extension) + var pathID string + if p.refresh { + pathID = p.newPathID + } else { + pathID = p.emoji.ID + } + p.emoji.ImageURL = uris.GenerateURIForAttachment(p.instanceAccountID, string(TypeEmoji), string(SizeOriginal), pathID, extension) + p.emoji.ImagePath = fmt.Sprintf("%s/%s/%s/%s.%s", p.instanceAccountID, TypeEmoji, SizeOriginal, pathID, extension) p.emoji.ImageContentType = contentType // concatenate the first bytes with the existing bytes still in the reader (thanks Mara) @@ -251,39 +285,87 @@ func (p *ProcessingEmoji) store(ctx context.Context) error { return nil } -func (m *manager) preProcessEmoji(ctx context.Context, data DataFunc, postData PostDataCallbackFunc, shortcode string, id string, uri string, ai *AdditionalEmojiInfo) (*ProcessingEmoji, error) { +func (m *manager) preProcessEmoji(ctx context.Context, data DataFunc, postData PostDataCallbackFunc, shortcode string, emojiID string, uri string, ai *AdditionalEmojiInfo, refresh bool) (*ProcessingEmoji, error) { instanceAccount, err := m.db.GetInstanceAccount(ctx, "") if err != nil { return nil, fmt.Errorf("preProcessEmoji: error fetching this instance account from the db: %s", err) } - disabled := false - visibleInPicker := true + var newPathID string + var emoji *gtsmodel.Emoji + if refresh { + emoji, err = m.db.GetEmojiByID(ctx, emojiID) + if err != nil { + return nil, fmt.Errorf("preProcessEmoji: error fetching emoji to refresh from the db: %s", err) + } - // populate initial fields on the emoji -- some of these will be overwritten as we proceed - emoji := >smodel.Emoji{ - ID: id, - CreatedAt: time.Now(), - UpdatedAt: time.Now(), - Shortcode: shortcode, - Domain: "", // assume our own domain unless told otherwise - ImageRemoteURL: "", - ImageStaticRemoteURL: "", - ImageURL: "", // we don't know yet - ImageStaticURL: uris.GenerateURIForAttachment(instanceAccount.ID, string(TypeEmoji), string(SizeStatic), id, mimePng), // all static emojis are encoded as png - ImagePath: "", // we don't know yet - ImageStaticPath: fmt.Sprintf("%s/%s/%s/%s.%s", instanceAccount.ID, TypeEmoji, SizeStatic, id, mimePng), // all static emojis are encoded as png - ImageContentType: "", // we don't know yet - ImageStaticContentType: mimeImagePng, // all static emojis are encoded as png - ImageFileSize: 0, - ImageStaticFileSize: 0, - ImageUpdatedAt: time.Now(), - Disabled: &disabled, - URI: uri, - VisibleInPicker: &visibleInPicker, - CategoryID: "", + // if this is a refresh, we will end up with new images + // stored for this emoji, so we can use the postData function + // to perform clean up of the old images from storage + originalPostData := postData + originalImagePath := emoji.ImagePath + originalImageStaticPath := emoji.ImageStaticPath + postData = func(ctx context.Context) error { + // trigger the original postData function if it was provided + if originalPostData != nil { + if err := originalPostData(ctx); err != nil { + return err + } + } + + l := log.WithField("shortcode@domain", emoji.Shortcode+"@"+emoji.Domain) + l.Debug("postData: cleaning up old emoji files for refreshed emoji") + if err := m.storage.Delete(ctx, originalImagePath); err != nil && !errors.Is(err, gostore.ErrNotFound) { + l.Errorf("postData: error cleaning up old emoji image at %s for refreshed emoji: %s", originalImagePath, err) + } + if err := m.storage.Delete(ctx, originalImageStaticPath); err != nil && !errors.Is(err, gostore.ErrNotFound) { + l.Errorf("postData: error cleaning up old emoji static image at %s for refreshed emoji: %s", originalImageStaticPath, err) + } + + return nil + } + + newPathID, err = id.NewRandomULID() + if err != nil { + return nil, fmt.Errorf("preProcessEmoji: error generating alternateID for emoji refresh: %s", err) + } + + // store + serve static image at new path ID + emoji.ImageStaticURL = uris.GenerateURIForAttachment(instanceAccount.ID, string(TypeEmoji), string(SizeStatic), newPathID, mimePng) + emoji.ImageStaticPath = fmt.Sprintf("%s/%s/%s/%s.%s", instanceAccount.ID, TypeEmoji, SizeStatic, newPathID, mimePng) + + // update these fields as we go + emoji.URI = uri + } else { + disabled := false + visibleInPicker := true + + // populate initial fields on the emoji -- some of these will be overwritten as we proceed + emoji = >smodel.Emoji{ + ID: emojiID, + CreatedAt: time.Now(), + Shortcode: shortcode, + Domain: "", // assume our own domain unless told otherwise + ImageRemoteURL: "", + ImageStaticRemoteURL: "", + ImageURL: "", // we don't know yet + ImageStaticURL: uris.GenerateURIForAttachment(instanceAccount.ID, string(TypeEmoji), string(SizeStatic), emojiID, mimePng), // all static emojis are encoded as png + ImagePath: "", // we don't know yet + ImageStaticPath: fmt.Sprintf("%s/%s/%s/%s.%s", instanceAccount.ID, TypeEmoji, SizeStatic, emojiID, mimePng), // all static emojis are encoded as png + ImageContentType: "", // we don't know yet + ImageStaticContentType: mimeImagePng, // all static emojis are encoded as png + ImageFileSize: 0, + ImageStaticFileSize: 0, + Disabled: &disabled, + URI: uri, + VisibleInPicker: &visibleInPicker, + CategoryID: "", + } } + emoji.ImageUpdatedAt = time.Now() + emoji.UpdatedAt = time.Now() + // check if we have additional info to add to the emoji, // and overwrite some of the emoji fields if so if ai != nil { @@ -324,6 +406,8 @@ func (m *manager) preProcessEmoji(ctx context.Context, data DataFunc, postData P staticState: int32(received), database: m.db, storage: m.storage, + refresh: refresh, + newPathID: newPathID, } return processingEmoji, nil diff --git a/internal/media/test/gts_pixellated-original.png b/internal/media/test/gts_pixellated-original.png new file mode 100644 index 0000000000000000000000000000000000000000..564ee76d842b2495e5c271b3162d5823bda61474 GIT binary patch literal 10296 zcmY+qV|XRN(k;A$iEZ1NiJghLV>=Vuwrx&q?%4Lkwr$%J-8t|5zI*Ta(cRC}UA1b} zs_K6gp&%!L2!{s;000nylA=og&W3-h9tP^)4lDVo1^|#fd8%kQD;c&ZptXs2vowKha`JN-p z`hG*&LvA44y|FbcZzUiE4~&rv8$^ZMm-ANGUIXt?7_=T)@l)LpV-#x*ttbA}~e8sCCJx*VP8%kDsO&}(? z;jy1`1?lJAwcC=57NtTNcq->~2-QJ2O)Ku$<3zu~1syB>?kzQvUcZITB&Ng5jDsES zKWW@EK81B10q*w9HnZK)*1Ifn(|beaexkE{y?G>L-E3#R89U+hGli}(we1R z_ctb{rVXizs-~^3Qyq{~_0(;)dML5MK!P}t$-=C7Zqal!>*UYz%7)4AcR2Vo4UgZd z>e@cM4MLHGZfhQ?x^C+q@mADX?sKE1S)OaBJq1tVk8w3$B$ih(89fPe4u|Rs?u*HH z4sg_5z(`!w`&rc60vMP+3|4L3=wAuS6V~?_Vs{8Ev>c|~V$mo=m)<;A>KDEeS9*h! zMSpAdj=nlZa6_*A$Q?sxK5$!lL~SCDk78DoLo$@{LHx0&-FTmx<1K!l-1`ps3#f+t z%@?i&>bS)NzuPi$DT<=fTZ6Mri%aOh_!4LdC@*KG!ke;2iyppb#Z(Xs^*XnhZ&p_i zE4!udIybbYs;Mzrg>m0)$XLL-KKR`J7IyY3VnGu6hTOku66}Zt5rjP~W5Ub8l7Erh zCZjd*=|50$kjz1v?J>x+EW_{8vh3*efaR9SU0tWNOYVAo;{bMl3Fmzb*R|k|?C>Z$ z(iGD!iWgjN;+Iy=0Za6HJ~WaEk(JJioH`sGJ3Cy5o>Xsndu?r;dsiMYbIp%WoK{+W zss>0;tT|7v9riBZ82ag-vp0YF^-}tJ8iRQEL&{>P+h$F4ST51hvKzX_s`vp$v^vR(m(iLZK2po|QR~0X*DPOK zR}I%RLsLPr@Zv;ssYf#-FseM;NM~KlE<@|+y|SNk7Fzxli;Z6jf# z1Rm^3O}MRYp{Qu%;k>ER5>=a1U-d~goupal=riCy*V?Ab^2iEqg3IxRB*JLo804J? z#*L22>`CzKTUKEq$$BoE=}zrPzcL7U8-L|Mr9T^XGjJ4AV9`qOq!L~NJHF8+AXcW-QP1i3T8%)z4tK}eoVG?ojxi4>0ti5g$OcyZ;-)}x~087q}p4FQ1}F@Eh+P0q+PJBVR_2 zaB6DzHQJ_*VaJv|P%wZVjau7JP+!&rX4BuQ*AGX<5BfvGV&ZMg31y*`&9-)K+xxcI zY|PWloQE@w!M>pXgM=@0mX!OfqxNwb+7;4gWLGfK#5M%>;_nDbvz-Ek4JCTW;KAQb z$j^h=IrzF-p}P!GA1n&_$6zuKBvXkU9n}om4RItt(YbeDup{DHLverRSm*c=%BauG z>1O&jG!yyNW@9(v@?Yu&(@mH#e<;9;vo{mNEvEPgL?%e{FkFSU{ryHTsL&u?^ci+S zr%1|ujDLzk#Gx1zr=#JV-s(WAz{I0l4kk6d!&1YZ+S6hhQuPFP6TTad@kt(LwRlKH z$1+Vq%}9Pk`QvB`kLvKqp7H=b5e)WP$z4Q8!sj$F&~cXXh0nDe=vi`&aM%zOGd6Oi*6+@u88eC3!B&m6;S%|H&=;UD<;t*lN#LG6M#6?qU1f#Vf+=a=V`41s& zff3eC?Hui8=DNJp#!aJw?WBzw3@Y?WF@%RB0a#fBSazSZ;c64bf&#W<$0kBv??SSY z3J|QM3bQW;$P5w!qqNt?nJ5FS1pp002qJ>+Mb4And54GWr633ZJbR=gIL`vd%jf+2 z!_Yk|ZysSicXWzjeHPIt{fO-EQxUQOHC`pRm6OTjjvi~=z< zj)Kf-j^?0kQPBKkX$(Qpi;zFh{P=;7_l1LnGl0x43q$E!ntv|fjn4cr0NJs1;!(Cz73GXxtEEX-Z=Ro0}yJE08e{@nX z{kyrgY5ru6J(d#EMz|hII`Go8Hcb3W}Qp|HDtoHj;Yk#_lJT=(m*rHQVp9nPSas}%B zja7nQsR*vzQ^H6$50N*S3IYhndnf6w2z9cgZX~`Vf1~+O=kxs_E?M^gIru`T{P~Ye z0UbAf0%p641$B^Hu)%DT3QBL674!}2{el14o?@(jmM?^F=Lo0}hz*y=dT0z?tWZu$ zi%~n9zzPrZ$ChP-*(yPvOQW16E>PVG*lL63&VUpuPWp>YNaU@cS4N!r0_YQHzr&n^ z!XopjWYQ+dpdI8POj?L*riFSf6sxb8X2hG;#1=;W?n5fSnJI>3LN6BsiWtyU3>5Sp z`lk95b1ex53^D#=5F*0DFl?6@r<^?gVj0mT8EPi4azDd1kEr=4x2vBtWXJD#3Z2@( z!tld!K2^BoUZ&@bOf+0lC#RshyuA3Kn2R|4ze9T1Iv?gATaIC)@fr-bgf;~hT3Ij{ zf11HrpyL~5peYG9$?5q4m-FMDAZeh9>#(p?^lJQuKLlpVwsWNvhHOM$3Ixg5V z4rc_14sAeWUg35l27jl}-`g3ooK=oNYpR0$iL+HU#s(^mcXh$@nbg&Q;1nOTp^w6* z^}#SRuJpaSiPH6&e#le2qb0bnvs$cyM!m$9M%jJWYk3~Mjp;_CncAZ|vpJMLe`Lpc z5{3Ji9rJLs;Lum>kZ&73x$EwkmItaMb88}R4v>+F{b*Mz_M>biH=s3+W^~E=7@>uq ze^bfw@NO3t1~nH#DF7K1=HDqzMz$K1@wdA>MgY8xbDkp>%yqkDB4INplg98!w+=!5 zNYxTD>kui64Vjqg+laJWw|_EH?v+M^B0>^R8@fyK?P5hh9QBwl+4kt`{>a~|GLRCC zh8YtS!x1>W*q}VaY?0{LUm`_Q(BuNik7{O`hb6c(9tZ<{H)T6gFamekl{}5m=~{Ts z-daJQhQ%SE>)-(|j{14ajyyoBR5nsqPoyQ4&s zeM}GmjFu7_2LR6qjB4T6*pc%3HwK~xAA@+JBtqO#XC~4aZySIs_LP#+_Ffx!V10d9 zK)!_QezJ>bFub)b;5hP-fhcAVdLnX4s0%#9!!YJz1(Y#vnqtpf(?kJkBQ<*mwtbg_ z!1&RR;*PH((`MJ!n*SuEFYHJwy1`C>D(DaQjCxs3H)F<3WPG;nW>`zZ(d*nN%Bqpf zrKFnR7Bylo^TX!OgAVVZ+^huD@z~`K?JG_ja*@i$h=+#POu<=?Sw?AvvanXvt>{hR z`X__LkU_&Nsybk;>^=sp|G=FlJZj|G$Yeo^T5KpN3>7$)2dgZ~L>2a@hQ=6B1p~AU zBP+++0R`^xLx$Cv(_mQYQw3&Hbk1SM@mYmis#Kr)i#zwPDw}|Hyf_%~$f;~5#Ygwx zY5M3BicDnP^Q%QwLcWawhEmj}BXN&>ON;t1&V`B}Tcs?*IRKagCvvZb1mZHJr%?%5 z*&}F8u4CgKqaS5M%4G5&%~*-QCS$rSAw9}jZ#yKxKTN@1#kiRlxjf_T= zzxuqG5G>!*!cP?U?G)3_3~V8p*^yJNU80LB0jzF{C~*5%&CdO0y*l884lO6%e$iDE z4DHOlD8C%r3&E3kw2li*b%iMs8OuZbm3z9i=_JL=_u{~C9_q>tm?Pt1;vwu?Gfq~{ z5(Ah?nu}rCB<=Oh+ql_jP@ago3u>)<9L>nXqA^AU$d&rPC6)sJf~&)vca#uxdc=%J z^b$H_v|_LaI1i#=LB)X13V=t-9}pJ#I)+JvYHrycEiJgx^)MqON}6B6K{}Fq#9ND* zHXCFjN)XUN9W-53O(8Ya$UpQDO=6Z#D#tuOUXR|0%x#NFcag5X2T@@r)G%N&^)Qua z(e*w<*Z^hW-%l@r2%0rVjY1w}wL@<`od7mk$H9mD|VC9FDw{d3(a^ z!ul9-et1e`uIc*5`d6j}D0FV<>+38PA8s8Ko-_C+bgvGuhT>FUi!58phsD06V`iCr zO(l7jcE2w?q5q~jj7|K$+0V}OYj?5w@|~J~Cfe13Y-Cp#9RJw#wTCdk!4^Aws59|$ zBcb_?O7iGIpynmFhuH|Jdb0-Nd8A7vt?=?pjx6_l;6)pSdEWfy_r!Vk*x$nH>Kpe_%ZouTZF5=97;BCdLk0|*iuRIP78C1sRLw`Q#+3fV2a-2GJ23hfhA zcYF|(DhrqDmQT83L50x%SQuW@sHfsDPEiK=)W_kj1N$s!QKrz%`Oxkx3n*E3mV)k{Oyk;#=A%Dmmt0$n)jJwB-^VuNWC`G z-L7nrC>Kx23ClR=(}ZiTXN>5KuA6Y4Vi8DKoMVj@dmo!`oz zI?;okVgAfPM5Zs}C5xpSN-hN*f1DO*&ER|EoivFRL+QoboZ4h58RdFzmowII%;V6| zic8xrO!gKnVZxK0%s!C{S99ZwqJs|<2Bn3gHeRJR;sv#fFL!{e+EXi{h7a~>!~l3& zW!VyGe*S2gs0&|{F){>$*1{v%i5*2Qs{x-o{+TyG9=_?;MQTMFS*b@|*e29*0VjGL z2{fk`$i%9otnjwfuy8&BX@~{cKjOK zLpWZ|#U-4u#K3Fo067ShGsg! zBjDvL#g&hff@p~|k|yZm)i9U2fav>{^Q(k943sNS95nA?6ZN-nLG*?USLJ9HC07Esy4p{ZyFd+Mvt@@k!QwXEsIiS#&?G+ zrgI=?nJaC-LbFpAHN)nW5PLgk6k|a_OB>x3`Unh%=3(4h7 zwI31Gij7JhpfRBV_;!E9PeH?eDZogrG)r4JKkm8|#=Q$uc#3vD=?HyO0W-7+d^|WG zswetMF$)$Pu$Hfj1nnBYbew2Ps=y!;c;XT4U6&tReTWhXi?E<2aHY)4G1C&U_l?Dk zxwwq5WvGEcX-L-*y!CvNvJ>0kRvtUzU}bG;v8oyK1hb7n)Y^)L-F@`yT6s4EQTeud zp0%!(rKA~-l=Q9mvBBGznZv&4b_)>Li*#neN!?1p-zeVN!}9}aOvAOlHlOnc72**% zD(_$Dm^tIWeXB6Mz;IiHUZnhCYVP{w6fo8UEX7Or?^laTIym-!@U@3-EKV!_&=4qc z{vJA(qKv>G7Elm9-v2}L&ARJs5t3sfNTGSX+>5j-iFG4H%pHDS;?( z=Dx~lX*Q)PHEuGgst{}rhQFOQ-4J_BaZFp*r*l6T>Jtl$RS?+8&}QW}lfC_i{lODt zL$u~ESEXv;WvjK7%*w}oILa_stOB27=;aPzzi%$p9{m!l>Rk^YZD~5tsrT_jT-!~= z?U$1nH=7{Z*FX#c1XkdB;OFS;w%eR|p|^-C_f*NWc_%t_^BF@eU1u~55C#G|Tg=aa z=@>T8J8Gqh%j1Q=@+}|9M4TF6eMU4#zRc~SH{A;1W7WFaxLJQe(kIN?|0r~actJr# zS#L_arjdk!)Z>0O5}kEl<6^(#tuAdF*EgU#LO9Q&k2R8|P^L9Cmu0-}KQQAILSYwd zs{zx?`m3KPuhhzUAaSDds*Nwt#I?pCX1KS^0_D@EhWK4#sNx*x89Bu9Go_KEq>Fpa zZYkAOIvbq`9z-L95^$#*tf$(uSbD~Qc_P`>E%kt6ttCYh2yjcI;F@IJ2N;BQ@C?R$ zUw|XcfB+hjuu9-PmRKvfd3Hz>ugJdfvfFCWjpYU*#8i@qg@le#c}qp~5d38z@*WGj z+gN+{W_k)U0JxF+NVQ(SJ|RE}C@tdGq7*&t_{g1PPN9I6KVG+J@<>sTiJ@M-U@!2* z<4-n~N^%BIRT4{wGR@s*E)Lywuoe4gd!db4!DT^U&OEBC5}=u<{SeG7MBbuX;MT}P@Di$s4r6`7M7`h2M zoL8zY1pyulzM}aj&I?WpC6EgDG7RjFhYJFZ@*Q(2C8GkL(@c!Jh7+u&bBUxwD$t7H zLLTU*Sf9c-Nm7TW5!?f*aR{_-<+W@yL-BCY7GO2+0+V1S9vt``QP>%vEFWl;H7lij zo*qaoGZM{HyS?-~CXkdx6+U^_cVn-zu{jpCitZ=&HlOvHQTn9vX%(RDtiq%v=dN*g zl_tpIn_6A0NE1V<_w$!|Sn1z(9n5NC8+>b^MHd@Nm&Ng~OPA*8$H#cnb1i968^|9n zPEk{*NIq|^ka!TkiTA3ZA_D0tiG47QWG?mXWFX|qp`l+*Y{>pVt8u|)0!B1!7y!|o zp*V5W$431xh{q&fGn_MFlYe|vgAE+q91d+E-h8*8t8=}ohs&hfV*@-omJTYyQ$0qa zg)@bD?KohEfu& zQ3Rj58mT8JEOW;^qZ_mPu~uQwf-pYc~K7JPBWOG4N`Z8*H|p)0UCuZx2^earySCW2=YQFyG|TpLr< zCG$*aHLrrqCudNmKD`a(77>d-^mx9oS+s5t8aGR0tLlufkk$n3TCnHTezz8!e+m&W z6&^{iqY7F=8bg{!Hs1H9r0mVfR?C@<%7z8wg1kv1XKVe5suNeWx8a^(!j)qQdd;C$ zS`h6~g%%MK=5q;U`Pg@uZohAd6woeJRPXb<@M0Ci-xjnHy&LuPAAu;&j%Y9jN7>$S zs5<11PvjByDg5QzWbj;*LYw51;QYcakWur5iwrKu+qrMz=C6Go-oS8ahU<6b#iIA? zhQTT>@y(kSL=wf+sn_yjWl+7shrb|J72)Q0scTFEnTh%C?_^A^n_1IyG6-1rl6-F-#+%e&LXqosz>HAn7<0>69<19z{7v{kZC#kyn$~2DY=n_H!TmxEd#(T=tlX9+DdCkM?H#8JADUUNlFib4{h6=DFH6M<$l74bq z$RU<)i>zRvgsM`^Gz>n}8Dg!mQ%h|ds4z~(mYmYb#Nknm-=_{c*tSPVP*PK^&27<_YrHfl>*K4yfQlb}p2^7M zzLxdO5EaN&lP}jKF=yol=i61Vx4MXwwDk!Q!InSEM*IxeKQ<_Y?P}u4);gYJ6m3X5 z`G$y=hHoWd!70TE1V3|htBX;9uo-=i>W#q;Ua9usDzwOy8eLiGeKZLcws zc;^+Ax?UstU^QWup~_<=Mh1JgaK&q^PdS^2s!! zHiv%~7U4$AC@w{9B*X|H#mKRYhRSZo5{cydJM*@*UD0`ctJc>ev+#*pp-LOtx2t{r zGTBG6LI0bQ-j?ZQ`kq~6TxAXR8=Hcm6^b&nN+SEK?LJ^7ljso2;ZY5IuPSG?pR;VR zy!d7juLaz&2-a#K^#!M|^f-g4#H;zL@k1r=>fK|`#JHcLN3dTI3JH}Ul-G=;ZKPic z$xzFHdspJrv-8zlV7p;dUkM5PQBk%lrnN+zVoX*S`hnnZ44_pq16qoi#E%7!QYIvx z+T7M9S6I{C86?X*?sNosdcx=zdBUOg&$Q%S7M+CD2U2KLTrbU&3TdTQg*i%QrHuYI zlq9G}a6+0mOu1V_6`w=0b5G5paFz})VPIPnv^*2~;plw&%IaO<8X{ClWkjqn`65{B zj)Fn@?I9EnaZMqCz534!O@4K|#AD6dSVn}an_pC7ge&?ZbL3Ew9XPp)rx`6H3D)szTFwr(cwWaf~7(@;z|cyxar3EsuNsGYtyCXB%<&w7K`ye%{`nHBPAA(P=#ZepO!|SBT7F zvwhQ-r_=R)UyhEa8=C?O-$80O;>M@~G`J=6Fp3Nq7Y7D9P{`#qxv&)<&-QO<^nzx( z_=o#H9&J=BnFqXv$pd#!)l%O>q1b~!tx^c^ke8)0J~F!C7psJ$5%fb8%7_HH zio1h_;%8~u*t}0dZ+B5ouhmRVE3OMQwQIR9SIXM2qudOORzvVs7P_iaJxjGHYg=m8 zOGZw2{tcGDAI-xq29meB9)q8y_CFTZjtxIRs`nUHyqZ>0XD?>y{E29G6qh)P9H?}H zC(l_yuG;Iex9WUb%;2hn$@<8$w`-5~`pA5JwXlg%5{Ql4Cs<#T3~u*_785Fso1It~ zCo(TuJ;uRUSQ{4f?(huqqXk8-j8{m?CB;(C8O_Xpehr`D4t-M}Q-zjG{?X1^fM>Gh z04?_;PShm#{3YRHElUlL%oQXb89sxDblN>Xxh*1<9eloko_DC}Qw~zh@+T2eVSVhgp|WhH<*2pkyW>8d05`x1_x@r zuA-f;Vcr6Or&>M?)8RO8!&FDB!^`p@OxxVZcItPZ=`;1E`VU zM`&)x7Pic3W#_Q>b96VchhvL+J?gQIbWbWcFF>0)7;u_il28Oam8}WewvhZGeyF^q z5V_7=;8m1uXH=an(lBzZa!)de^1Ogr@}V&?C!{S(kJ;CPz~}Gij3DX#7l5{rD-|5L z^$Y-j1zU)SC;&x7{{IP>e{(NcJ_&r1{elF8dfLTGS_Bs)d-3@^nkc`??SF8+)M>cbJ1i~ssiHs%2ohvxVdxeOd{KsAF%uCy?{9VuZ}RqT zSF$W#;6kfRvapmD#gEs(^{EG=h(oeUN=_^si!vil-?|_Z>s{~t9Dm!$B`?5`fqH@5 z(=5Tqx`lEuoS);qMwigkR0^Mt7)EF(xg}pVN!{j3C3w1@03}RLpi4xAC}EH5PKxax=Qw-&NSU+>E?nZ=o?V`wuG3xp1h&{5=fs0IK>r4lpbVuYL;(eWO~Ct~PrJ?k zj4IhnYB~V`4CMb=VD7~NuKzBfoq@7q&_{3xh`e}%rS_Zv00b*gR7k~r?Q&DrcAW{u zU$Jf^=HlF$ejDWPOl6^%CpbO#yIoEpO=#FCHtX3{Y-T`!7{*k1=8`KG_E#S#BbXr+ zvq&r_V;@`7($>eD#?cHT+GTV1Tc|H%9ZOIeDr-3I0Kz|;`#)3MX20x^vmuO3pqPWw zgeLyyA@plW&SQy#%k1;c?r&61OEZg6=6@jnw`Zt%4U^Hk`}H7)p0NCxV;c?moi9R$ zk#>6BB;fyGh$N@63V6zG7{z)!-p^G|teE=$Lnkr)7dDh7$5Ft^F1B;>s9Pib{|gDl zOm50m7M5k^`{Z$8J;i&x_RRP6`91UgzdU*P+0A_a5_tR){^IkI>)a|Cn-(z7G;fGcJiK%0F5D zC6J#;dYrzz_0ie)=2e*D!SGSCxzzNqkN@~r^S@Ip$};;^I^}V2a6jf{hU5i1CDd>X zn&ozUx{uQ;+1WLiR^Qb5C!u zzW5$(c^sZUX8gTxvFUhwJbN~E*4w5%{)Iu$`v1@~2*>)I=<$Q~MbejloQA!a%=dCS zNX7mj5d1H*{KPCTR!^sa18!%PhfYVk*sPBaiFLc@T)m}j*#E8vspAyuv-j=&l-cw7 z@pXsZ*ZscN)mw)(KQziZ!haYJME|Rug2dRzgu&o6Z>~>l*LB}pLef>&%fq?dRJ0A$ z|8$M`zkEOxhg;F-?JZt<*Hd*-+a0Jm^J>^x|Cii9v|X7s2ZDX#fre>iYyn2X z6z{Ezn5zE|JUDp|0V_QX_$_R6MH59Z~kieY0}#Bjp@-<|85rl;J28G8bUzD z``#PhuZ{nC^ZAK0rdI9WEE}$%TfF*>as0G*&t>=QFTe7$BIfJY>2vg^Z@qtL`L$pF zpJQ4)W7GMXo4Rk5{qKeE(>;^O-&}m*X7QDamY?1{moL4?`%m_uopC85lECggX7$~e zUuy2+yjs59PgdLicgGT8uiqH!NB#r`WbM=G)2FXoU%xlb{_n4A@6SdU?K%I45lgt% z{V@K2&-jOBgZH+7mW|%UFP2&s-Fxwzq3!@?S}}ZnuV<;{qbHVMR$Q!o3XH5b^Y?GN z{qpnk*!Yf3M5A6ZZVbVk}|X qX=(gs&Y3CTTmy-a{frC@|No~kT(OJr&g}!{Wd=`IKbLh*2~7aVfcAs{ literal 0 HcmV?d00001 diff --git a/internal/media/types.go b/internal/media/types.go index d71080658..3238916b8 100644 --- a/internal/media/types.go +++ b/internal/media/types.go @@ -98,8 +98,8 @@ type AdditionalMediaInfo struct { FocusY *float32 } -// AdditionalMediaInfo represents additional information -// that should be added to an emoji when processing it. +// AdditionalEmojiInfo represents additional information +// that should be taken into account when processing an emoji. type AdditionalEmojiInfo struct { // Time that this emoji was created; defaults to time.Now(). CreatedAt *time.Time diff --git a/internal/processing/admin/createemoji.go b/internal/processing/admin/createemoji.go index 50399279c..93ae17496 100644 --- a/internal/processing/admin/createemoji.go +++ b/internal/processing/admin/createemoji.go @@ -57,7 +57,7 @@ func (p *processor) EmojiCreate(ctx context.Context, account *gtsmodel.Account, return f, form.Image.Size, err } - processingEmoji, err := p.mediaManager.ProcessEmoji(ctx, data, nil, form.Shortcode, emojiID, emojiURI, nil) + processingEmoji, err := p.mediaManager.ProcessEmoji(ctx, data, nil, form.Shortcode, emojiID, emojiURI, nil, false) if err != nil { return nil, gtserror.NewErrorInternalError(fmt.Errorf("error processing emoji: %s", err), "error processing emoji") } diff --git a/internal/processing/media/getfile.go b/internal/processing/media/getfile.go index 104bca770..693b8685b 100644 --- a/internal/processing/media/getfile.go +++ b/internal/processing/media/getfile.go @@ -30,9 +30,10 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/gtserror" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/media" + "github.com/superseriousbusiness/gotosocial/internal/uris" ) -func (p *processor) GetFile(ctx context.Context, account *gtsmodel.Account, form *apimodel.GetContentRequestForm) (*apimodel.Content, gtserror.WithCode) { +func (p *processor) GetFile(ctx context.Context, requestingAccount *gtsmodel.Account, form *apimodel.GetContentRequestForm) (*apimodel.Content, gtserror.WithCode) { // parse the form fields mediaSize, err := media.ParseMediaSize(form.MediaSize) if err != nil { @@ -49,25 +50,25 @@ func (p *processor) GetFile(ctx context.Context, account *gtsmodel.Account, form return nil, gtserror.NewErrorNotFound(fmt.Errorf("file name %s not parseable", form.FileName)) } wantedMediaID := spl[0] - expectedAccountID := form.AccountID + owningAccountID := form.AccountID // get the account that owns the media and make sure it's not suspended - acct, err := p.db.GetAccountByID(ctx, expectedAccountID) + owningAccount, err := p.db.GetAccountByID(ctx, owningAccountID) if err != nil { - return nil, gtserror.NewErrorNotFound(fmt.Errorf("account with id %s could not be selected from the db: %s", expectedAccountID, err)) + return nil, gtserror.NewErrorNotFound(fmt.Errorf("account with id %s could not be selected from the db: %s", owningAccountID, err)) } - if !acct.SuspendedAt.IsZero() { - return nil, gtserror.NewErrorNotFound(fmt.Errorf("account with id %s is suspended", expectedAccountID)) + if !owningAccount.SuspendedAt.IsZero() { + return nil, gtserror.NewErrorNotFound(fmt.Errorf("account with id %s is suspended", owningAccountID)) } // make sure the requesting account and the media account don't block each other - if account != nil { - blocked, err := p.db.IsBlocked(ctx, account.ID, expectedAccountID, true) + if requestingAccount != nil { + blocked, err := p.db.IsBlocked(ctx, requestingAccount.ID, owningAccountID, true) if err != nil { - return nil, gtserror.NewErrorNotFound(fmt.Errorf("block status could not be established between accounts %s and %s: %s", expectedAccountID, account.ID, err)) + return nil, gtserror.NewErrorNotFound(fmt.Errorf("block status could not be established between accounts %s and %s: %s", owningAccountID, requestingAccount.ID, err)) } if blocked { - return nil, gtserror.NewErrorNotFound(fmt.Errorf("block exists between accounts %s and %s", expectedAccountID, account.ID)) + return nil, gtserror.NewErrorNotFound(fmt.Errorf("block exists between accounts %s and %s", owningAccountID, requestingAccount.ID)) } } @@ -75,15 +76,15 @@ func (p *processor) GetFile(ctx context.Context, account *gtsmodel.Account, form // so we need to take different steps depending on the media type being requested switch mediaType { case media.TypeEmoji: - return p.getEmojiContent(ctx, wantedMediaID, mediaSize) + return p.getEmojiContent(ctx, wantedMediaID, owningAccountID, mediaSize) case media.TypeAttachment, media.TypeHeader, media.TypeAvatar: - return p.getAttachmentContent(ctx, account, wantedMediaID, expectedAccountID, mediaSize) + return p.getAttachmentContent(ctx, requestingAccount, wantedMediaID, owningAccountID, mediaSize) default: return nil, gtserror.NewErrorNotFound(fmt.Errorf("media type %s not recognized", mediaType)) } } -func (p *processor) getAttachmentContent(ctx context.Context, requestingAccount *gtsmodel.Account, wantedMediaID string, expectedAccountID string, mediaSize media.Size) (*apimodel.Content, gtserror.WithCode) { +func (p *processor) getAttachmentContent(ctx context.Context, requestingAccount *gtsmodel.Account, wantedMediaID string, owningAccountID string, mediaSize media.Size) (*apimodel.Content, gtserror.WithCode) { attachmentContent := &apimodel.Content{} var storagePath string @@ -93,8 +94,8 @@ func (p *processor) getAttachmentContent(ctx context.Context, requestingAccount return nil, gtserror.NewErrorNotFound(fmt.Errorf("attachment %s could not be taken from the db: %s", wantedMediaID, err)) } - if a.AccountID != expectedAccountID { - return nil, gtserror.NewErrorNotFound(fmt.Errorf("attachment %s is not owned by %s", wantedMediaID, expectedAccountID)) + if a.AccountID != owningAccountID { + return nil, gtserror.NewErrorNotFound(fmt.Errorf("attachment %s is not owned by %s", wantedMediaID, owningAccountID)) } // get file information from the attachment depending on the requested media size @@ -227,17 +228,23 @@ func (p *processor) getAttachmentContent(ctx context.Context, requestingAccount return attachmentContent, nil } -func (p *processor) getEmojiContent(ctx context.Context, wantedEmojiID string, emojiSize media.Size) (*apimodel.Content, gtserror.WithCode) { +func (p *processor) getEmojiContent(ctx context.Context, fileName string, owningAccountID string, emojiSize media.Size) (*apimodel.Content, gtserror.WithCode) { emojiContent := &apimodel.Content{} var storagePath string - e, err := p.db.GetEmojiByID(ctx, wantedEmojiID) + // reconstruct the static emoji image url -- reason + // for using the static URL rather than full size url + // is that static emojis are always encoded as png, + // so this is more reliable than using full size url + imageStaticURL := uris.GenerateURIForAttachment(owningAccountID, string(media.TypeEmoji), string(media.SizeStatic), fileName, "png") + + e, err := p.db.GetEmojiByStaticURL(ctx, imageStaticURL) if err != nil { - return nil, gtserror.NewErrorNotFound(fmt.Errorf("emoji %s could not be taken from the db: %s", wantedEmojiID, err)) + return nil, gtserror.NewErrorNotFound(fmt.Errorf("emoji %s could not be taken from the db: %s", fileName, err)) } if *e.Disabled { - return nil, gtserror.NewErrorNotFound(fmt.Errorf("emoji %s has been disabled", wantedEmojiID)) + return nil, gtserror.NewErrorNotFound(fmt.Errorf("emoji %s has been disabled", fileName)) } switch emojiSize { diff --git a/internal/typeutils/internaltoas.go b/internal/typeutils/internaltoas.go index 6194dba82..c84dd09f4 100644 --- a/internal/typeutils/internaltoas.go +++ b/internal/typeutils/internaltoas.go @@ -760,6 +760,10 @@ func (c *converter) EmojiToAS(ctx context.Context, e *gtsmodel.Emoji) (vocab.Too iconProperty.AppendActivityStreamsImage(iconImage) emoji.SetActivityStreamsIcon(iconProperty) + updatedProp := streams.NewActivityStreamsUpdatedProperty() + updatedProp.Set(e.ImageUpdatedAt) + emoji.SetActivityStreamsUpdated(updatedProp) + return emoji, nil } diff --git a/internal/typeutils/internaltoas_test.go b/internal/typeutils/internaltoas_test.go index f2845be02..b63fa7aea 100644 --- a/internal/typeutils/internaltoas_test.go +++ b/internal/typeutils/internaltoas_test.go @@ -72,7 +72,7 @@ func (suite *InternalToASTestSuite) TestAccountToASWithEmoji() { // this is necessary because the order of multiple 'context' entries is not determinate trimmed := strings.Split(string(bytes), "\"discoverable\"")[1] - suite.Equal(`:true,"featured":"http://localhost:8080/users/the_mighty_zork/collections/featured","followers":"http://localhost:8080/users/the_mighty_zork/followers","following":"http://localhost:8080/users/the_mighty_zork/following","icon":{"mediaType":"image/jpeg","type":"Image","url":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/original/01F8MH58A357CV5K7R7TJMSH6S.jpeg"},"id":"http://localhost:8080/users/the_mighty_zork","image":{"mediaType":"image/jpeg","type":"Image","url":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/header/original/01PFPMWK2FF0D9WMHEJHR07C3Q.jpeg"},"inbox":"http://localhost:8080/users/the_mighty_zork/inbox","manuallyApprovesFollowers":false,"name":"original zork (he/they)","outbox":"http://localhost:8080/users/the_mighty_zork/outbox","preferredUsername":"the_mighty_zork","publicKey":{"id":"http://localhost:8080/users/the_mighty_zork/main-key","owner":"http://localhost:8080/users/the_mighty_zork","publicKeyPem":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAwXTcOAvM1Jiw5Ffpk0qn\nr0cwbNvFe/5zQ+Tp7tumK/ZnT37o7X0FUEXrxNi+dkhmeJ0gsaiN+JQGNUewvpSk\nPIAXKvi908aSfCGjs7bGlJCJCuDuL5d6m7hZnP9rt9fJc70GElPpG0jc9fXwlz7T\nlsPb2ecatmG05Y4jPwdC+oN4MNCv9yQzEvCVMzl76EJaM602kIHC1CISn0rDFmYd\n9rSN7XPlNJw1F6PbpJ/BWQ+pXHKw3OEwNTETAUNYiVGnZU+B7a7bZC9f6/aPbJuV\nt8Qmg+UnDvW1Y8gmfHnxaWG2f5TDBvCHmcYtucIZPLQD4trAozC4ryqlmCWQNKbt\n0wIDAQAB\n-----END PUBLIC KEY-----\n"},"summary":"\u003cp\u003ehey yo this is my profile!\u003c/p\u003e","tag":{"icon":{"mediaType":"image/png","type":"Image","url":"http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/emoji/original/01F8MH9H8E4VG3KDYJR9EGPXCQ.png"},"id":"http://localhost:8080/emoji/01F8MH9H8E4VG3KDYJR9EGPXCQ","name":":rainbow:","type":"Emoji"},"type":"Person","url":"http://localhost:8080/@the_mighty_zork"}`, trimmed) + suite.Equal(`:true,"featured":"http://localhost:8080/users/the_mighty_zork/collections/featured","followers":"http://localhost:8080/users/the_mighty_zork/followers","following":"http://localhost:8080/users/the_mighty_zork/following","icon":{"mediaType":"image/jpeg","type":"Image","url":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/original/01F8MH58A357CV5K7R7TJMSH6S.jpeg"},"id":"http://localhost:8080/users/the_mighty_zork","image":{"mediaType":"image/jpeg","type":"Image","url":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/header/original/01PFPMWK2FF0D9WMHEJHR07C3Q.jpeg"},"inbox":"http://localhost:8080/users/the_mighty_zork/inbox","manuallyApprovesFollowers":false,"name":"original zork (he/they)","outbox":"http://localhost:8080/users/the_mighty_zork/outbox","preferredUsername":"the_mighty_zork","publicKey":{"id":"http://localhost:8080/users/the_mighty_zork/main-key","owner":"http://localhost:8080/users/the_mighty_zork","publicKeyPem":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAwXTcOAvM1Jiw5Ffpk0qn\nr0cwbNvFe/5zQ+Tp7tumK/ZnT37o7X0FUEXrxNi+dkhmeJ0gsaiN+JQGNUewvpSk\nPIAXKvi908aSfCGjs7bGlJCJCuDuL5d6m7hZnP9rt9fJc70GElPpG0jc9fXwlz7T\nlsPb2ecatmG05Y4jPwdC+oN4MNCv9yQzEvCVMzl76EJaM602kIHC1CISn0rDFmYd\n9rSN7XPlNJw1F6PbpJ/BWQ+pXHKw3OEwNTETAUNYiVGnZU+B7a7bZC9f6/aPbJuV\nt8Qmg+UnDvW1Y8gmfHnxaWG2f5TDBvCHmcYtucIZPLQD4trAozC4ryqlmCWQNKbt\n0wIDAQAB\n-----END PUBLIC KEY-----\n"},"summary":"\u003cp\u003ehey yo this is my profile!\u003c/p\u003e","tag":{"icon":{"mediaType":"image/png","type":"Image","url":"http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/emoji/original/01F8MH9H8E4VG3KDYJR9EGPXCQ.png"},"id":"http://localhost:8080/emoji/01F8MH9H8E4VG3KDYJR9EGPXCQ","name":":rainbow:","type":"Emoji","updated":"2021-09-20T12:40:37+02:00"},"type":"Person","url":"http://localhost:8080/@the_mighty_zork"}`, trimmed) } func (suite *InternalToASTestSuite) TestAccountToASWithSharedInbox() { @@ -157,7 +157,7 @@ func (suite *InternalToASTestSuite) TestStatusWithTagsToASWithIDs() { // http://joinmastodon.org/ns, https://www.w3.org/ns/activitystreams -- // will appear, so trim them out of the string for consistency trimmed := strings.SplitAfter(string(bytes), `"attachment":`)[1] - suite.Equal(`{"blurhash":"LNJRdVM{00Rj%Mayt7j[4nWBofRj","mediaType":"image/jpeg","name":"Black and white image of some 50's style text saying: Welcome On Board","type":"Document","url":"http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/original/01F8MH6NEM8D7527KZAECTCR76.jpeg"},"attributedTo":"http://localhost:8080/users/admin","cc":"http://localhost:8080/users/admin/followers","content":"hello world! #welcome ! first post on the instance :rainbow: !","id":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R","published":"2021-10-20T11:36:45Z","replies":{"first":{"id":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R/replies?page=true","next":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R/replies?only_other_accounts=false\u0026page=true","partOf":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R/replies","type":"CollectionPage"},"id":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R/replies","type":"Collection"},"sensitive":false,"summary":"","tag":{"icon":{"mediaType":"image/png","type":"Image","url":"http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/emoji/original/01F8MH9H8E4VG3KDYJR9EGPXCQ.png"},"id":"http://localhost:8080/emoji/01F8MH9H8E4VG3KDYJR9EGPXCQ","name":":rainbow:","type":"Emoji"},"to":"https://www.w3.org/ns/activitystreams#Public","type":"Note","url":"http://localhost:8080/@admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R"}`, trimmed) + suite.Equal(`{"blurhash":"LNJRdVM{00Rj%Mayt7j[4nWBofRj","mediaType":"image/jpeg","name":"Black and white image of some 50's style text saying: Welcome On Board","type":"Document","url":"http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/original/01F8MH6NEM8D7527KZAECTCR76.jpeg"},"attributedTo":"http://localhost:8080/users/admin","cc":"http://localhost:8080/users/admin/followers","content":"hello world! #welcome ! first post on the instance :rainbow: !","id":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R","published":"2021-10-20T11:36:45Z","replies":{"first":{"id":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R/replies?page=true","next":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R/replies?only_other_accounts=false\u0026page=true","partOf":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R/replies","type":"CollectionPage"},"id":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R/replies","type":"Collection"},"sensitive":false,"summary":"","tag":{"icon":{"mediaType":"image/png","type":"Image","url":"http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/emoji/original/01F8MH9H8E4VG3KDYJR9EGPXCQ.png"},"id":"http://localhost:8080/emoji/01F8MH9H8E4VG3KDYJR9EGPXCQ","name":":rainbow:","type":"Emoji","updated":"2021-09-20T10:40:37Z"},"to":"https://www.w3.org/ns/activitystreams#Public","type":"Note","url":"http://localhost:8080/@admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R"}`, trimmed) } func (suite *InternalToASTestSuite) TestStatusWithTagsToASFromDB() { @@ -179,7 +179,7 @@ func (suite *InternalToASTestSuite) TestStatusWithTagsToASFromDB() { // http://joinmastodon.org/ns, https://www.w3.org/ns/activitystreams -- // will appear, so trim them out of the string for consistency trimmed := strings.SplitAfter(string(bytes), `"attachment":`)[1] - suite.Equal(`{"blurhash":"LNJRdVM{00Rj%Mayt7j[4nWBofRj","mediaType":"image/jpeg","name":"Black and white image of some 50's style text saying: Welcome On Board","type":"Document","url":"http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/original/01F8MH6NEM8D7527KZAECTCR76.jpeg"},"attributedTo":"http://localhost:8080/users/admin","cc":"http://localhost:8080/users/admin/followers","content":"hello world! #welcome ! first post on the instance :rainbow: !","id":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R","published":"2021-10-20T11:36:45Z","replies":{"first":{"id":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R/replies?page=true","next":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R/replies?only_other_accounts=false\u0026page=true","partOf":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R/replies","type":"CollectionPage"},"id":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R/replies","type":"Collection"},"sensitive":false,"summary":"","tag":{"icon":{"mediaType":"image/png","type":"Image","url":"http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/emoji/original/01F8MH9H8E4VG3KDYJR9EGPXCQ.png"},"id":"http://localhost:8080/emoji/01F8MH9H8E4VG3KDYJR9EGPXCQ","name":":rainbow:","type":"Emoji"},"to":"https://www.w3.org/ns/activitystreams#Public","type":"Note","url":"http://localhost:8080/@admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R"}`, trimmed) + suite.Equal(`{"blurhash":"LNJRdVM{00Rj%Mayt7j[4nWBofRj","mediaType":"image/jpeg","name":"Black and white image of some 50's style text saying: Welcome On Board","type":"Document","url":"http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/original/01F8MH6NEM8D7527KZAECTCR76.jpeg"},"attributedTo":"http://localhost:8080/users/admin","cc":"http://localhost:8080/users/admin/followers","content":"hello world! #welcome ! first post on the instance :rainbow: !","id":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R","published":"2021-10-20T11:36:45Z","replies":{"first":{"id":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R/replies?page=true","next":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R/replies?only_other_accounts=false\u0026page=true","partOf":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R/replies","type":"CollectionPage"},"id":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R/replies","type":"Collection"},"sensitive":false,"summary":"","tag":{"icon":{"mediaType":"image/png","type":"Image","url":"http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/emoji/original/01F8MH9H8E4VG3KDYJR9EGPXCQ.png"},"id":"http://localhost:8080/emoji/01F8MH9H8E4VG3KDYJR9EGPXCQ","name":":rainbow:","type":"Emoji","updated":"2021-09-20T10:40:37Z"},"to":"https://www.w3.org/ns/activitystreams#Public","type":"Note","url":"http://localhost:8080/@admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R"}`, trimmed) } func (suite *InternalToASTestSuite) TestStatusToASWithMentions() {