From 23fc70f4e68730b7eec91d58dac54ec00099ed8d Mon Sep 17 00:00:00 2001 From: kim <89579420+NyaaaWhatsUpDoc@users.noreply.github.com> Date: Thu, 5 Dec 2024 13:35:07 +0000 Subject: [PATCH] [feature] add support for receiving federated status edits (#3597) * add support for extracting Updated field from Statusable implementers * add support for status edits in the database, and update status dereferencer to handle them * remove unused AdditionalInfo{}.CreatedAt * remove unused AdditionalEmojiInfo{}.CreatedAt * update new mention creation to use status.UpdatedAt * remove mention.UpdatedAt, fixes related to NewULIDFromTime() change * add migration to remove Mention{}.UpdatedAt field * add migration to add the StatusEdit{} table * start adding tests, add delete function for status edits * add more of status edit migrations, fill in more of the necessary edit delete functionality * remove unused function * allow generating gotosocial compatible ulid via CLI with `go run ./cmd/gen-ulid` * add StatusEdit{} test models * fix new statusedits sql * use model instead of table name * actually remove the Mention.UpdatedAt field... * fix tests now new models are added, add more status edit DB tests * fix panic wording * add test for deleting status edits * don't automatically set `updated_at` field on updated statuses * flesh out more of the dereferencer status edit tests, ensure updated at field set on outgoing AS statuses * remove media_attachments.updated_at column * fix up more tests, further complete the dereferencer status edit tests * update more status serialization tests not expecting 'updated' AS property * gah!! json serialization tests!! * undo some gtscontext wrapping changes * more serialization test fixing :smiling_face_with_tear: * more test fixing, ensure the edit.status_id field is actually set :facepalm: * fix status edit test * grrr linter * add edited_at field to apimodel status * remove the choice of paging on the timeline public filtered test (otherwise it needs updating every time you add statuses ...) * ensure that status.updated_at always fits chronologically * fix more serialization tests ... * add more code comments * fix envparsing * update swagger file * properly handle media description changes during status edits * slight formatting tweak * code comment --- cmd/gen-ulid/main.go | 22 + docs/api/swagger.yaml | 10 + internal/ap/interfaces.go | 9 +- internal/ap/interfaces_test.go | 93 +++ internal/ap/properties.go | 19 + .../api/activitypub/users/outboxget_test.go | 16 +- .../api/client/accounts/accountverify_test.go | 2 +- .../api/client/admin/accountsgetv2_test.go | 12 +- internal/api/client/admin/reportsget_test.go | 47 +- internal/api/client/exports/exports_test.go | 2 +- .../api/client/instance/instancepatch_test.go | 12 +- internal/api/client/mutes/mutesget_test.go | 2 +- internal/api/client/reports/reportget_test.go | 4 +- .../api/client/reports/reportsget_test.go | 16 +- internal/api/client/search/searchget_test.go | 6 +- .../api/client/statuses/statusboost_test.go | 6 + .../api/client/statuses/statuscreate_test.go | 12 + .../api/client/statuses/statusfave_test.go | 2 + .../api/client/statuses/statushistory_test.go | 4 +- .../api/client/statuses/statusmute_test.go | 10 +- internal/api/model/content.go | 3 - internal/api/model/status.go | 4 + internal/cache/cache.go | 1 + internal/cache/db.go | 35 + internal/cache/invalidate.go | 5 + internal/cache/size.go | 19 +- internal/config/config.go | 1 + internal/config/defaults.go | 1 + internal/config/helpers.gen.go | 25 + internal/db/bundb/account_test.go | 10 +- internal/db/bundb/basic_test.go | 2 +- internal/db/bundb/bundb.go | 5 + internal/db/bundb/bundb_test.go | 2 + internal/db/bundb/instance_test.go | 4 +- internal/db/bundb/interaction_test.go | 18 +- internal/db/bundb/media.go | 6 - ...9134448_interaction_requests_client_api.go | 11 +- ...0241113151042_remove_mention_updated_at.go | 57 ++ .../20241113152126_add_status_edits.go | 67 ++ .../20241113152126_add_status_edits/status.go | 97 +++ .../statusedit.go | 48 ++ .../20241121121623_enum_strings_to_ints.go | 94 --- ...4608_remove_media_attachment_updated_at.go | 57 ++ internal/db/bundb/migrations/util.go | 198 ++++++ internal/db/bundb/status.go | 48 +- internal/db/bundb/statusedit.go | 198 ++++++ internal/db/bundb/statusedit_test.go | 168 +++++ internal/db/bundb/timeline.go | 28 +- internal/db/bundb/timeline_test.go | 21 +- internal/db/db.go | 1 + internal/db/statusedit.go | 43 ++ internal/federation/dereferencing/announce.go | 7 +- internal/federation/dereferencing/media.go | 1 + internal/federation/dereferencing/status.go | 607 +++++++++++++----- .../dereferencing/status_permitted.go | 10 +- .../federation/dereferencing/status_test.go | 220 +++++++ internal/federation/dereferencing/util.go | 6 +- .../federation/federatingdb/announce_test.go | 2 +- internal/gtsmodel/mediaattachment.go | 1 - internal/gtsmodel/mention.go | 1 - internal/gtsmodel/status.go | 60 +- internal/gtsmodel/statusedit.go | 62 ++ internal/id/ulid.go | 20 +- internal/media/manager.go | 7 - internal/media/manager_test.go | 1 - internal/media/types.go | 9 - internal/processing/account/rss_test.go | 16 +- internal/processing/media/getfile.go | 4 +- internal/processing/media/unattach_test.go | 3 - internal/processing/status/get.go | 2 +- .../processing/stream/notification_test.go | 4 +- .../processing/stream/statusupdate_test.go | 5 +- internal/processing/timeline/public_test.go | 4 +- internal/processing/workers/util.go | 51 +- internal/timeline/get_test.go | 6 +- internal/timeline/prune_test.go | 8 +- internal/typeutils/astointernal.go | 22 +- internal/typeutils/internal.go | 24 +- internal/typeutils/internaltoas.go | 7 +- internal/typeutils/internaltoas_test.go | 4 + internal/typeutils/internaltofrontend.go | 16 +- internal/typeutils/internaltofrontend_test.go | 100 +-- internal/typeutils/wrap_test.go | 1 + test/envparsing.sh | 1 + testrig/db.go | 91 +-- testrig/testmodels.go | 242 ++++++- 86 files changed, 2557 insertions(+), 651 deletions(-) create mode 100644 cmd/gen-ulid/main.go create mode 100644 internal/ap/interfaces_test.go create mode 100644 internal/db/bundb/migrations/20241113151042_remove_mention_updated_at.go create mode 100644 internal/db/bundb/migrations/20241113152126_add_status_edits.go create mode 100644 internal/db/bundb/migrations/20241113152126_add_status_edits/status.go create mode 100644 internal/db/bundb/migrations/20241113152126_add_status_edits/statusedit.go create mode 100644 internal/db/bundb/migrations/20241203124608_remove_media_attachment_updated_at.go create mode 100644 internal/db/bundb/statusedit.go create mode 100644 internal/db/bundb/statusedit_test.go create mode 100644 internal/db/statusedit.go create mode 100644 internal/gtsmodel/statusedit.go diff --git a/cmd/gen-ulid/main.go b/cmd/gen-ulid/main.go new file mode 100644 index 000000000..f96df4415 --- /dev/null +++ b/cmd/gen-ulid/main.go @@ -0,0 +1,22 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package main + +import "github.com/superseriousbusiness/gotosocial/internal/id" + +func main() { println(id.NewULID()) } diff --git a/docs/api/swagger.yaml b/docs/api/swagger.yaml index a3a79e2fb..c2e5c5fc3 100644 --- a/docs/api/swagger.yaml +++ b/docs/api/swagger.yaml @@ -2692,6 +2692,11 @@ definitions: example: "2021-07-30T09:20:25+00:00" type: string x-go-name: CreatedAt + edited_at: + description: Timestamp of when the status was last edited (ISO 8601 Datetime). + example: "2021-07-30T09:20:25+00:00" + type: string + x-go-name: EditedAt emojis: description: Custom emoji to be used when rendering status content. items: @@ -2889,6 +2894,11 @@ definitions: example: "2021-07-30T09:20:25+00:00" type: string x-go-name: CreatedAt + edited_at: + description: Timestamp of when the status was last edited (ISO 8601 Datetime). + example: "2021-07-30T09:20:25+00:00" + type: string + x-go-name: EditedAt emojis: description: Custom emoji to be used when rendering status content. items: diff --git a/internal/ap/interfaces.go b/internal/ap/interfaces.go index a721fa997..1f08fde37 100644 --- a/internal/ap/interfaces.go +++ b/internal/ap/interfaces.go @@ -25,8 +25,11 @@ import ( // IsActivityable returns whether AS vocab type name is acceptable as Activityable. func IsActivityable(typeName string) bool { - return isActivity(typeName) || - isIntransitiveActivity(typeName) + return isActivity(typeName) + // See interfaces_test.go comment + // about intransitive activities: + // + // || isIntransitiveActivity(typeName) } // ToActivityable safely tries to cast vocab.Type as Activityable, also checking for expected AS type names. @@ -184,6 +187,7 @@ type Accountable interface { WithEndpoints WithTag WithPublished + WithUpdated } // Statusable represents the minimum activitypub interface for representing a 'status'. @@ -196,6 +200,7 @@ type Statusable interface { WithName WithInReplyTo WithPublished + WithUpdated WithURL WithAttributedTo WithTo diff --git a/internal/ap/interfaces_test.go b/internal/ap/interfaces_test.go new file mode 100644 index 000000000..d3248cb1d --- /dev/null +++ b/internal/ap/interfaces_test.go @@ -0,0 +1,93 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package ap_test + +import ( + "github.com/superseriousbusiness/activity/streams/vocab" + "github.com/superseriousbusiness/gotosocial/internal/ap" +) + +var ( + // NOTE: the below aren't actually tests that are run, + // we just move them into an _test.go file to declutter + // the main interfaces.go file, which is already long. + + // Compile-time checks for Activityable interface methods. + _ ap.Activityable = (vocab.ActivityStreamsAccept)(nil) + _ ap.Activityable = (vocab.ActivityStreamsTentativeAccept)(nil) + _ ap.Activityable = (vocab.ActivityStreamsAdd)(nil) + _ ap.Activityable = (vocab.ActivityStreamsCreate)(nil) + _ ap.Activityable = (vocab.ActivityStreamsDelete)(nil) + _ ap.Activityable = (vocab.ActivityStreamsFollow)(nil) + _ ap.Activityable = (vocab.ActivityStreamsIgnore)(nil) + _ ap.Activityable = (vocab.ActivityStreamsJoin)(nil) + _ ap.Activityable = (vocab.ActivityStreamsLeave)(nil) + _ ap.Activityable = (vocab.ActivityStreamsLike)(nil) + _ ap.Activityable = (vocab.ActivityStreamsOffer)(nil) + _ ap.Activityable = (vocab.ActivityStreamsInvite)(nil) + _ ap.Activityable = (vocab.ActivityStreamsReject)(nil) + _ ap.Activityable = (vocab.ActivityStreamsTentativeReject)(nil) + _ ap.Activityable = (vocab.ActivityStreamsRemove)(nil) + _ ap.Activityable = (vocab.ActivityStreamsUndo)(nil) + _ ap.Activityable = (vocab.ActivityStreamsUpdate)(nil) + _ ap.Activityable = (vocab.ActivityStreamsView)(nil) + _ ap.Activityable = (vocab.ActivityStreamsListen)(nil) + _ ap.Activityable = (vocab.ActivityStreamsRead)(nil) + _ ap.Activityable = (vocab.ActivityStreamsMove)(nil) + _ ap.Activityable = (vocab.ActivityStreamsAnnounce)(nil) + _ ap.Activityable = (vocab.ActivityStreamsBlock)(nil) + _ ap.Activityable = (vocab.ActivityStreamsFlag)(nil) + _ ap.Activityable = (vocab.ActivityStreamsDislike)(nil) + + // the below intransitive activities don't fit the interface definition because they're + // missing an attached object (as the activity itself contains the details), but we don't + // actually end up using them so it's simpler to just comment them out and not have to do + // a WithObject{} interface check on every single incoming activity: + // + // _ Activityable = (vocab.ActivityStreamsArrive)(nil) + // _ Activityable = (vocab.ActivityStreamsTravel)(nil) + // _ Activityable = (vocab.ActivityStreamsQuestion)(nil) + + // Compile-time checks for Accountable interface methods. + _ ap.Accountable = (vocab.ActivityStreamsPerson)(nil) + _ ap.Accountable = (vocab.ActivityStreamsApplication)(nil) + _ ap.Accountable = (vocab.ActivityStreamsOrganization)(nil) + _ ap.Accountable = (vocab.ActivityStreamsService)(nil) + _ ap.Accountable = (vocab.ActivityStreamsGroup)(nil) + + // Compile-time checks for Statusable interface methods. + _ ap.Statusable = (vocab.ActivityStreamsArticle)(nil) + _ ap.Statusable = (vocab.ActivityStreamsDocument)(nil) + _ ap.Statusable = (vocab.ActivityStreamsImage)(nil) + _ ap.Statusable = (vocab.ActivityStreamsVideo)(nil) + _ ap.Statusable = (vocab.ActivityStreamsNote)(nil) + _ ap.Statusable = (vocab.ActivityStreamsPage)(nil) + _ ap.Statusable = (vocab.ActivityStreamsEvent)(nil) + _ ap.Statusable = (vocab.ActivityStreamsPlace)(nil) + _ ap.Statusable = (vocab.ActivityStreamsProfile)(nil) + _ ap.Statusable = (vocab.ActivityStreamsQuestion)(nil) + + // Compile-time checks for Pollable interface methods. + _ ap.Pollable = (vocab.ActivityStreamsQuestion)(nil) + + // Compile-time checks for PollOptionable interface methods. + _ ap.PollOptionable = (vocab.ActivityStreamsNote)(nil) + + // Compile-time checks for Acceptable interface methods. + _ ap.Acceptable = (vocab.ActivityStreamsAccept)(nil) +) diff --git a/internal/ap/properties.go b/internal/ap/properties.go index 38e58ebc0..0a2564168 100644 --- a/internal/ap/properties.go +++ b/internal/ap/properties.go @@ -408,6 +408,25 @@ func SetPublished(with WithPublished, published time.Time) { publishProp.Set(published) } +// GetUpdated returns the time contained in the Updated property of 'with'. +func GetUpdated(with WithUpdated) time.Time { + updateProp := with.GetActivityStreamsUpdated() + if updateProp == nil || !updateProp.IsXMLSchemaDateTime() { + return time.Time{} + } + return updateProp.Get() +} + +// SetUpdated sets the given time on the Updated property of 'with'. +func SetUpdated(with WithUpdated, updated time.Time) { + updateProp := with.GetActivityStreamsUpdated() + if updateProp == nil { + updateProp = streams.NewActivityStreamsUpdatedProperty() + with.SetActivityStreamsUpdated(updateProp) + } + updateProp.Set(updated) +} + // GetEndTime returns the time contained in the EndTime property of 'with'. func GetEndTime(with WithEndTime) time.Time { endTimeProp := with.GetActivityStreamsEndTime() diff --git a/internal/api/activitypub/users/outboxget_test.go b/internal/api/activitypub/users/outboxget_test.go index cba1ef31d..2de3b0456 100644 --- a/internal/api/activitypub/users/outboxget_test.go +++ b/internal/api/activitypub/users/outboxget_test.go @@ -82,7 +82,7 @@ func (suite *OutboxGetTestSuite) TestGetOutbox() { "@context": "https://www.w3.org/ns/activitystreams", "first": "http://localhost:8080/users/the_mighty_zork/outbox?limit=40", "id": "http://localhost:8080/users/the_mighty_zork/outbox", - "totalItems": 8, + "totalItems": 9, "type": "OrderedCollection" }`, dst.String()) @@ -142,6 +142,14 @@ func (suite *OutboxGetTestSuite) TestGetOutboxFirstPage() { "id": "http://localhost:8080/users/the_mighty_zork/outbox?limit=40", "next": "http://localhost:8080/users/the_mighty_zork/outbox?limit=40\u0026max_id=01F8MHAMCHF6Y650WCRSCP4WMY", "orderedItems": [ + { + "actor": "http://localhost:8080/users/the_mighty_zork", + "cc": "http://localhost:8080/users/the_mighty_zork/followers", + "id": "http://localhost:8080/users/the_mighty_zork/statuses/01JDPZC707CKDN8N4QVWM4Z1NR/activity#Create", + "object": "http://localhost:8080/users/the_mighty_zork/statuses/01JDPZC707CKDN8N4QVWM4Z1NR", + "to": "https://www.w3.org/ns/activitystreams#Public", + "type": "Create" + }, { "actor": "http://localhost:8080/users/the_mighty_zork", "cc": "http://localhost:8080/users/the_mighty_zork/followers", @@ -160,8 +168,8 @@ func (suite *OutboxGetTestSuite) TestGetOutboxFirstPage() { } ], "partOf": "http://localhost:8080/users/the_mighty_zork/outbox", - "prev": "http://localhost:8080/users/the_mighty_zork/outbox?limit=40\u0026min_id=01HH9KYNQPA416TNJ53NSATP40", - "totalItems": 8, + "prev": "http://localhost:8080/users/the_mighty_zork/outbox?limit=40\u0026min_id=01JDPZC707CKDN8N4QVWM4Z1NR", + "totalItems": 9, "type": "OrderedCollectionPage" }`, dst.String()) @@ -224,7 +232,7 @@ func (suite *OutboxGetTestSuite) TestGetOutboxNextPage() { "id": "http://localhost:8080/users/the_mighty_zork/outbox?limit=40&max_id=01F8MHAMCHF6Y650WCRSCP4WMY", "orderedItems": [], "partOf": "http://localhost:8080/users/the_mighty_zork/outbox", - "totalItems": 8, + "totalItems": 9, "type": "OrderedCollectionPage" }`, dst.String()) diff --git a/internal/api/client/accounts/accountverify_test.go b/internal/api/client/accounts/accountverify_test.go index 3f67cdefb..df5c21389 100644 --- a/internal/api/client/accounts/accountverify_test.go +++ b/internal/api/client/accounts/accountverify_test.go @@ -97,7 +97,7 @@ func (suite *AccountVerifyTestSuite) TestAccountVerifyGet() { suite.Equal("http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/header/small/01PFPMWK2FF0D9WMHEJHR07C3Q.webp", apimodelAccount.HeaderStatic) suite.Equal(2, apimodelAccount.FollowersCount) suite.Equal(2, apimodelAccount.FollowingCount) - suite.Equal(8, apimodelAccount.StatusesCount) + suite.Equal(9, apimodelAccount.StatusesCount) suite.EqualValues(apimodel.VisibilityPublic, apimodelAccount.Source.Privacy) suite.Equal(testAccount.Settings.Language, apimodelAccount.Source.Language) suite.Equal(testAccount.NoteRaw, apimodelAccount.Source.Note) diff --git a/internal/api/client/admin/accountsgetv2_test.go b/internal/api/client/admin/accountsgetv2_test.go index 77ca135eb..489a245d0 100644 --- a/internal/api/client/admin/accountsgetv2_test.go +++ b/internal/api/client/admin/accountsgetv2_test.go @@ -99,8 +99,8 @@ func (suite *AccountsGetTestSuite) TestAccountsGetFromTop() { "header_description": "Flat gray background (default header).", "followers_count": 1, "following_count": 1, - "statuses_count": 8, - "last_status_at": "2021-07-28", + "statuses_count": 9, + "last_status_at": "2024-11-01", "emojis": [], "fields": [ { @@ -262,8 +262,8 @@ func (suite *AccountsGetTestSuite) TestAccountsGetFromTop() { "header_media_id": "01PFPMWK2FF0D9WMHEJHR07C3Q", "followers_count": 2, "following_count": 2, - "statuses_count": 8, - "last_status_at": "2024-01-10", + "statuses_count": 9, + "last_status_at": "2024-11-01", "emojis": [], "fields": [], "enable_rss": true @@ -403,8 +403,8 @@ func (suite *AccountsGetTestSuite) TestAccountsGetFromTop() { "header_description": "Flat gray background (default header).", "followers_count": 0, "following_count": 0, - "statuses_count": 3, - "last_status_at": "2021-09-11", + "statuses_count": 4, + "last_status_at": "2024-11-01", "emojis": [], "fields": [] } diff --git a/internal/api/client/admin/reportsget_test.go b/internal/api/client/admin/reportsget_test.go index 12a307836..255e32c3b 100644 --- a/internal/api/client/admin/reportsget_test.go +++ b/internal/api/client/admin/reportsget_test.go @@ -186,8 +186,8 @@ func (suite *ReportsGetTestSuite) TestReportsGetAll() { "header_description": "Flat gray background (default header).", "followers_count": 0, "following_count": 0, - "statuses_count": 3, - "last_status_at": "2021-09-11", + "statuses_count": 4, + "last_status_at": "2024-11-01", "emojis": [], "fields": [] } @@ -232,8 +232,8 @@ func (suite *ReportsGetTestSuite) TestReportsGetAll() { "header_description": "Flat gray background (default header).", "followers_count": 1, "following_count": 1, - "statuses_count": 8, - "last_status_at": "2021-07-28", + "statuses_count": 9, + "last_status_at": "2024-11-01", "emojis": [], "fields": [ { @@ -414,8 +414,8 @@ func (suite *ReportsGetTestSuite) TestReportsGetAll() { "header_description": "Flat gray background (default header).", "followers_count": 1, "following_count": 1, - "statuses_count": 8, - "last_status_at": "2021-07-28", + "statuses_count": 9, + "last_status_at": "2024-11-01", "emojis": [], "fields": [ { @@ -473,8 +473,8 @@ func (suite *ReportsGetTestSuite) TestReportsGetAll() { "header_description": "Flat gray background (default header).", "followers_count": 0, "following_count": 0, - "statuses_count": 3, - "last_status_at": "2021-09-11", + "statuses_count": 4, + "last_status_at": "2024-11-01", "emojis": [], "fields": [] } @@ -485,6 +485,7 @@ func (suite *ReportsGetTestSuite) TestReportsGetAll() { { "id": "01FVW7JHQFSFK166WWKR8CBA6M", "created_at": "2021-09-20T10:40:37.000Z", + "edited_at": null, "in_reply_to_id": null, "in_reply_to_account_id": null, "sensitive": false, @@ -521,8 +522,8 @@ func (suite *ReportsGetTestSuite) TestReportsGetAll() { "header_description": "Flat gray background (default header).", "followers_count": 0, "following_count": 0, - "statuses_count": 3, - "last_status_at": "2021-09-11", + "statuses_count": 4, + "last_status_at": "2024-11-01", "emojis": [], "fields": [] }, @@ -667,8 +668,8 @@ func (suite *ReportsGetTestSuite) TestReportsGetCreatedByAccount() { "header_description": "Flat gray background (default header).", "followers_count": 1, "following_count": 1, - "statuses_count": 8, - "last_status_at": "2021-07-28", + "statuses_count": 9, + "last_status_at": "2024-11-01", "emojis": [], "fields": [ { @@ -726,8 +727,8 @@ func (suite *ReportsGetTestSuite) TestReportsGetCreatedByAccount() { "header_description": "Flat gray background (default header).", "followers_count": 0, "following_count": 0, - "statuses_count": 3, - "last_status_at": "2021-09-11", + "statuses_count": 4, + "last_status_at": "2024-11-01", "emojis": [], "fields": [] } @@ -738,6 +739,7 @@ func (suite *ReportsGetTestSuite) TestReportsGetCreatedByAccount() { { "id": "01FVW7JHQFSFK166WWKR8CBA6M", "created_at": "2021-09-20T10:40:37.000Z", + "edited_at": null, "in_reply_to_id": null, "in_reply_to_account_id": null, "sensitive": false, @@ -774,8 +776,8 @@ func (suite *ReportsGetTestSuite) TestReportsGetCreatedByAccount() { "header_description": "Flat gray background (default header).", "followers_count": 0, "following_count": 0, - "statuses_count": 3, - "last_status_at": "2021-09-11", + "statuses_count": 4, + "last_status_at": "2024-11-01", "emojis": [], "fields": [] }, @@ -920,8 +922,8 @@ func (suite *ReportsGetTestSuite) TestReportsGetTargetAccount() { "header_description": "Flat gray background (default header).", "followers_count": 1, "following_count": 1, - "statuses_count": 8, - "last_status_at": "2021-07-28", + "statuses_count": 9, + "last_status_at": "2024-11-01", "emojis": [], "fields": [ { @@ -979,8 +981,8 @@ func (suite *ReportsGetTestSuite) TestReportsGetTargetAccount() { "header_description": "Flat gray background (default header).", "followers_count": 0, "following_count": 0, - "statuses_count": 3, - "last_status_at": "2021-09-11", + "statuses_count": 4, + "last_status_at": "2024-11-01", "emojis": [], "fields": [] } @@ -991,6 +993,7 @@ func (suite *ReportsGetTestSuite) TestReportsGetTargetAccount() { { "id": "01FVW7JHQFSFK166WWKR8CBA6M", "created_at": "2021-09-20T10:40:37.000Z", + "edited_at": null, "in_reply_to_id": null, "in_reply_to_account_id": null, "sensitive": false, @@ -1027,8 +1030,8 @@ func (suite *ReportsGetTestSuite) TestReportsGetTargetAccount() { "header_description": "Flat gray background (default header).", "followers_count": 0, "following_count": 0, - "statuses_count": 3, - "last_status_at": "2021-09-11", + "statuses_count": 4, + "last_status_at": "2024-11-01", "emojis": [], "fields": [] }, diff --git a/internal/api/client/exports/exports_test.go b/internal/api/client/exports/exports_test.go index 62997af5d..13f7bea05 100644 --- a/internal/api/client/exports/exports_test.go +++ b/internal/api/client/exports/exports_test.go @@ -229,7 +229,7 @@ Cool Ass Posters From This Instance,admin@localhost:8080 "media_storage": "", "followers_count": 2, "following_count": 2, - "statuses_count": 8, + "statuses_count": 9, "lists_count": 1, "blocks_count": 0, "mutes_count": 0 diff --git a/internal/api/client/instance/instancepatch_test.go b/internal/api/client/instance/instancepatch_test.go index efcb3762f..f126ee6ae 100644 --- a/internal/api/client/instance/instancepatch_test.go +++ b/internal/api/client/instance/instancepatch_test.go @@ -155,7 +155,7 @@ func (suite *InstancePatchTestSuite) TestInstancePatch1() { }, "stats": { "domain_count": 2, - "status_count": 19, + "status_count": 21, "user_count": 4 }, "thumbnail": "http://localhost:8080/assets/logo.webp", @@ -296,7 +296,7 @@ func (suite *InstancePatchTestSuite) TestInstancePatch2() { }, "stats": { "domain_count": 2, - "status_count": 19, + "status_count": 21, "user_count": 4 }, "thumbnail": "http://localhost:8080/assets/logo.webp", @@ -437,7 +437,7 @@ func (suite *InstancePatchTestSuite) TestInstancePatch3() { }, "stats": { "domain_count": 2, - "status_count": 19, + "status_count": 21, "user_count": 4 }, "thumbnail": "http://localhost:8080/assets/logo.webp", @@ -629,7 +629,7 @@ func (suite *InstancePatchTestSuite) TestInstancePatch6() { }, "stats": { "domain_count": 2, - "status_count": 19, + "status_count": 21, "user_count": 4 }, "thumbnail": "http://localhost:8080/assets/logo.webp", @@ -792,7 +792,7 @@ func (suite *InstancePatchTestSuite) TestInstancePatch8() { }, "stats": { "domain_count": 2, - "status_count": 19, + "status_count": 21, "user_count": 4 }, "thumbnail": "http://localhost:8080/fileserver/01AY6P665V14JJR0AFVRT7311Y/attachment/original/`+instanceAccount.AvatarMediaAttachment.ID+`.gif",`+` @@ -974,7 +974,7 @@ func (suite *InstancePatchTestSuite) TestInstancePatch9() { }, "stats": { "domain_count": 2, - "status_count": 19, + "status_count": 21, "user_count": 4 }, "thumbnail": "http://localhost:8080/assets/logo.webp", diff --git a/internal/api/client/mutes/mutesget_test.go b/internal/api/client/mutes/mutesget_test.go index fa52c9aa9..13d826398 100644 --- a/internal/api/client/mutes/mutesget_test.go +++ b/internal/api/client/mutes/mutesget_test.go @@ -148,7 +148,7 @@ func (suite *MutesTestSuite) TestIndefinitelyMutedAccountSerializesMuteExpiratio // Fetch all muted accounts for the logged-in account. // The expected body contains `"mute_expires_at":null`. - _, err = suite.getMutedAccounts(http.StatusOK, `[{"id":"01F8MH5ZK5VRH73AKHQM6Y9VNX","username":"foss_satan","acct":"foss_satan@fossbros-anonymous.io","display_name":"big gerald","locked":false,"discoverable":true,"bot":false,"created_at":"2021-09-26T10:52:36.000Z","note":"i post about like, i dunno, stuff, or whatever!!!!","url":"http://fossbros-anonymous.io/@foss_satan","avatar":"","avatar_static":"","header":"http://localhost:8080/assets/default_header.webp","header_static":"http://localhost:8080/assets/default_header.webp","header_description":"Flat gray background (default header).","followers_count":0,"following_count":0,"statuses_count":3,"last_status_at":"2021-09-11","emojis":[],"fields":[],"mute_expires_at":null}]`) + _, err = suite.getMutedAccounts(http.StatusOK, `[{"id":"01F8MH5ZK5VRH73AKHQM6Y9VNX","username":"foss_satan","acct":"foss_satan@fossbros-anonymous.io","display_name":"big gerald","locked":false,"discoverable":true,"bot":false,"created_at":"2021-09-26T10:52:36.000Z","note":"i post about like, i dunno, stuff, or whatever!!!!","url":"http://fossbros-anonymous.io/@foss_satan","avatar":"","avatar_static":"","header":"http://localhost:8080/assets/default_header.webp","header_static":"http://localhost:8080/assets/default_header.webp","header_description":"Flat gray background (default header).","followers_count":0,"following_count":0,"statuses_count":4,"last_status_at":"2024-11-01","emojis":[],"fields":[],"mute_expires_at":null}]`) if err != nil { suite.FailNow(err.Error()) } diff --git a/internal/api/client/reports/reportget_test.go b/internal/api/client/reports/reportget_test.go index 8c9dfa1e5..afbcb2e28 100644 --- a/internal/api/client/reports/reportget_test.go +++ b/internal/api/client/reports/reportget_test.go @@ -130,8 +130,8 @@ func (suite *ReportGetTestSuite) TestGetReport1() { "header_description": "Flat gray background (default header).", "followers_count": 0, "following_count": 0, - "statuses_count": 3, - "last_status_at": "2021-09-11", + "statuses_count": 4, + "last_status_at": "2024-11-01", "emojis": [], "fields": [] } diff --git a/internal/api/client/reports/reportsget_test.go b/internal/api/client/reports/reportsget_test.go index 0eb66e778..b5988e331 100644 --- a/internal/api/client/reports/reportsget_test.go +++ b/internal/api/client/reports/reportsget_test.go @@ -156,8 +156,8 @@ func (suite *ReportsGetTestSuite) TestGetReports() { "header_description": "Flat gray background (default header).", "followers_count": 0, "following_count": 0, - "statuses_count": 3, - "last_status_at": "2021-09-11", + "statuses_count": 4, + "last_status_at": "2024-11-01", "emojis": [], "fields": [] } @@ -247,8 +247,8 @@ func (suite *ReportsGetTestSuite) TestGetReports4() { "header_description": "Flat gray background (default header).", "followers_count": 0, "following_count": 0, - "statuses_count": 3, - "last_status_at": "2021-09-11", + "statuses_count": 4, + "last_status_at": "2024-11-01", "emojis": [], "fields": [] } @@ -322,8 +322,8 @@ func (suite *ReportsGetTestSuite) TestGetReports6() { "header_description": "Flat gray background (default header).", "followers_count": 0, "following_count": 0, - "statuses_count": 3, - "last_status_at": "2021-09-11", + "statuses_count": 4, + "last_status_at": "2024-11-01", "emojis": [], "fields": [] } @@ -381,8 +381,8 @@ func (suite *ReportsGetTestSuite) TestGetReports7() { "header_description": "Flat gray background (default header).", "followers_count": 0, "following_count": 0, - "statuses_count": 3, - "last_status_at": "2021-09-11", + "statuses_count": 4, + "last_status_at": "2024-11-01", "emojis": [], "fields": [] } diff --git a/internal/api/client/search/searchget_test.go b/internal/api/client/search/searchget_test.go index ab4f46689..2c4efd19c 100644 --- a/internal/api/client/search/searchget_test.go +++ b/internal/api/client/search/searchget_test.go @@ -916,7 +916,7 @@ func (suite *SearchGetTestSuite) TestSearchAAny() { } suite.Len(searchResult.Accounts, 5) - suite.Len(searchResult.Statuses, 7) + suite.Len(searchResult.Statuses, 8) suite.Len(searchResult.Hashtags, 0) } @@ -959,7 +959,7 @@ func (suite *SearchGetTestSuite) TestSearchAAnyFollowingOnly() { } suite.Len(searchResult.Accounts, 2) - suite.Len(searchResult.Statuses, 7) + suite.Len(searchResult.Statuses, 8) suite.Len(searchResult.Hashtags, 0) } @@ -1002,7 +1002,7 @@ func (suite *SearchGetTestSuite) TestSearchAStatuses() { } suite.Len(searchResult.Accounts, 0) - suite.Len(searchResult.Statuses, 7) + suite.Len(searchResult.Statuses, 8) suite.Len(searchResult.Hashtags, 0) } diff --git a/internal/api/client/statuses/statusboost_test.go b/internal/api/client/statuses/statusboost_test.go index 1f92d8b3f..51b7d7652 100644 --- a/internal/api/client/statuses/statusboost_test.go +++ b/internal/api/client/statuses/statusboost_test.go @@ -100,6 +100,7 @@ func (suite *StatusBoostTestSuite) TestPostBoost() { "card": null, "content": "", "created_at": "right the hell just now babyee", + "edited_at": null, "emojis": [], "favourited": true, "favourites_count": 0, @@ -145,6 +146,7 @@ func (suite *StatusBoostTestSuite) TestPostBoost() { "card": null, "content": "hello world! #welcome ! first post on the instance :rainbow: !", "created_at": "right the hell just now babyee", + "edited_at": null, "emojis": [ { "category": "reactions", @@ -280,6 +282,7 @@ func (suite *StatusBoostTestSuite) TestPostBoostOwnFollowersOnly() { "card": null, "content": "", "created_at": "right the hell just now babyee", + "edited_at": null, "emojis": [], "favourited": false, "favourites_count": 0, @@ -329,6 +332,7 @@ func (suite *StatusBoostTestSuite) TestPostBoostOwnFollowersOnly() { "card": null, "content": "hi!", "created_at": "right the hell just now babyee", + "edited_at": null, "emojis": [], "favourited": false, "favourites_count": 0, @@ -494,6 +498,7 @@ func (suite *StatusBoostTestSuite) TestPostBoostImplicitAccept() { "card": null, "content": "", "created_at": "right the hell just now babyee", + "edited_at": null, "emojis": [], "favourited": false, "favourites_count": 0, @@ -539,6 +544,7 @@ func (suite *StatusBoostTestSuite) TestPostBoostImplicitAccept() { "card": null, "content": "

Hi @1happyturtle, can I reply?

", "created_at": "right the hell just now babyee", + "edited_at": null, "emojis": [], "favourited": false, "favourites_count": 0, diff --git a/internal/api/client/statuses/statuscreate_test.go b/internal/api/client/statuses/statuscreate_test.go index 5f5386dd5..227e7d83e 100644 --- a/internal/api/client/statuses/statuscreate_test.go +++ b/internal/api/client/statuses/statuscreate_test.go @@ -102,6 +102,7 @@ func (suite *StatusCreateTestSuite) TestPostNewStatus() { "card": null, "content": "

this is a brand new status! #helloworld

", "created_at": "right the hell just now babyee", + "edited_at": null, "emojis": [], "favourited": false, "favourites_count": 0, @@ -187,6 +188,7 @@ func (suite *StatusCreateTestSuite) TestPostNewStatusIntPolicy() { "card": null, "content": "

this is a brand new status! #helloworld

", "created_at": "right the hell just now babyee", + "edited_at": null, "emojis": [], "favourited": false, "favourites_count": 0, @@ -282,6 +284,7 @@ func (suite *StatusCreateTestSuite) TestPostNewStatusIntPolicyJSON() { "card": null, "content": "

this is a brand new status! #helloworld

", "created_at": "right the hell just now babyee", + "edited_at": null, "emojis": [], "favourited": false, "favourites_count": 0, @@ -407,6 +410,7 @@ func (suite *StatusCreateTestSuite) TestPostNewStatusMarkdown() { "card": null, "content": "

Title

Smaller title

This is a post written in markdown

", "created_at": "right the hell just now babyee", + "edited_at": null, "emojis": [], "favourited": false, "favourites_count": 0, @@ -490,6 +494,7 @@ func (suite *StatusCreateTestSuite) TestMentionUnknownAccount() { "card": null, "content": "

hello @brand_new_person

", "created_at": "right the hell just now babyee", + "edited_at": null, "emojis": [], "favourited": false, "favourites_count": 0, @@ -567,6 +572,7 @@ func (suite *StatusCreateTestSuite) TestPostStatusWithLinksAndTags() { "card": null, "content": "

#test alright, should be able to post #links with fragments in them now, let's see........

https://docs.gotosocial.org/en/latest/user_guide/posts/#links

#gotosocial

(tobi remember to pull the docker image challenge)

", "created_at": "right the hell just now babyee", + "edited_at": null, "emojis": [], "favourited": false, "favourites_count": 0, @@ -650,6 +656,7 @@ func (suite *StatusCreateTestSuite) TestPostNewStatusWithEmoji() { "card": null, "content": "

here is a rainbow emoji a few times! :rainbow: :rainbow: :rainbow:
here's an emoji that isn't in the db: :test_emoji:

", "created_at": "right the hell just now babyee", + "edited_at": null, "emojis": [ { "category": "reactions", @@ -747,6 +754,7 @@ func (suite *StatusCreateTestSuite) TestReplyToLocalStatus() { "card": null, "content": "

hello @1happyturtle this reply should work!

", "created_at": "right the hell just now babyee", + "edited_at": null, "emojis": [], "favourited": false, "favourites_count": 0, @@ -829,6 +837,7 @@ func (suite *StatusCreateTestSuite) TestAttachNewMediaSuccess() { "card": null, "content": "

here's an image attachment

", "created_at": "right the hell just now babyee", + "edited_at": null, "emojis": [], "favourited": false, "favourites_count": 0, @@ -933,6 +942,7 @@ func (suite *StatusCreateTestSuite) TestPostNewStatusWithNoncanonicalLanguageTag "card": null, "content": "

English? what's English? i speak American

", "created_at": "right the hell just now babyee", + "edited_at": null, "emojis": [], "favourited": false, "favourites_count": 0, @@ -1007,6 +1017,7 @@ func (suite *StatusCreateTestSuite) TestPostNewStatusWithPollForm() { "card": null, "content": "

this is a status with a poll!

", "created_at": "right the hell just now babyee", + "edited_at": null, "emojis": [], "favourited": false, "favourites_count": 0, @@ -1103,6 +1114,7 @@ func (suite *StatusCreateTestSuite) TestPostNewStatusWithPollJSON() { "card": null, "content": "

this is a status with a poll!

", "created_at": "right the hell just now babyee", + "edited_at": null, "emojis": [], "favourited": false, "favourites_count": 0, diff --git a/internal/api/client/statuses/statusfave_test.go b/internal/api/client/statuses/statusfave_test.go index bd81c0cf9..8851b4d58 100644 --- a/internal/api/client/statuses/statusfave_test.go +++ b/internal/api/client/statuses/statusfave_test.go @@ -105,6 +105,7 @@ func (suite *StatusFaveTestSuite) TestPostFave() { "card": null, "content": "🐕🐕🐕🐕🐕", "created_at": "right the hell just now babyee", + "edited_at": null, "emojis": [], "favourited": true, "favourites_count": 1, @@ -228,6 +229,7 @@ func (suite *StatusFaveTestSuite) TestPostFaveImplicitAccept() { "card": null, "content": "

Hi @1happyturtle, can I reply?

", "created_at": "right the hell just now babyee", + "edited_at": null, "emojis": [], "favourited": true, "favourites_count": 1, diff --git a/internal/api/client/statuses/statushistory_test.go b/internal/api/client/statuses/statushistory_test.go index aea666dbb..3878f54e4 100644 --- a/internal/api/client/statuses/statushistory_test.go +++ b/internal/api/client/statuses/statushistory_test.go @@ -116,8 +116,8 @@ func (suite *StatusHistoryTestSuite) TestGetHistory() { "header_media_id": "01PFPMWK2FF0D9WMHEJHR07C3Q", "followers_count": 2, "following_count": 2, - "statuses_count": 8, - "last_status_at": "2024-01-10", + "statuses_count": 9, + "last_status_at": "2024-11-01", "emojis": [], "fields": [], "enable_rss": true diff --git a/internal/api/client/statuses/statusmute_test.go b/internal/api/client/statuses/statusmute_test.go index 384761fc6..66bd4a420 100644 --- a/internal/api/client/statuses/statusmute_test.go +++ b/internal/api/client/statuses/statusmute_test.go @@ -91,6 +91,7 @@ func (suite *StatusMuteTestSuite) TestMuteUnmuteStatus() { suite.Equal(`{ "id": "01F8MHAMCHF6Y650WCRSCP4WMY", "created_at": "2021-10-20T10:40:37.000Z", + "edited_at": null, "in_reply_to_id": null, "in_reply_to_account_id": null, "sensitive": true, @@ -134,8 +135,8 @@ func (suite *StatusMuteTestSuite) TestMuteUnmuteStatus() { "header_media_id": "01PFPMWK2FF0D9WMHEJHR07C3Q", "followers_count": 2, "following_count": 2, - "statuses_count": 8, - "last_status_at": "2024-01-10", + "statuses_count": 9, + "last_status_at": "2024-11-01", "emojis": [], "fields": [], "enable_rss": true @@ -178,6 +179,7 @@ func (suite *StatusMuteTestSuite) TestMuteUnmuteStatus() { suite.Equal(`{ "id": "01F8MHAMCHF6Y650WCRSCP4WMY", "created_at": "2021-10-20T10:40:37.000Z", + "edited_at": null, "in_reply_to_id": null, "in_reply_to_account_id": null, "sensitive": true, @@ -221,8 +223,8 @@ func (suite *StatusMuteTestSuite) TestMuteUnmuteStatus() { "header_media_id": "01PFPMWK2FF0D9WMHEJHR07C3Q", "followers_count": 2, "following_count": 2, - "statuses_count": 8, - "last_status_at": "2024-01-10", + "statuses_count": 9, + "last_status_at": "2024-11-01", "emojis": [], "fields": [], "enable_rss": true diff --git a/internal/api/model/content.go b/internal/api/model/content.go index 7da389ed1..5af81b11b 100644 --- a/internal/api/model/content.go +++ b/internal/api/model/content.go @@ -19,7 +19,6 @@ package model import ( "io" - "time" "github.com/superseriousbusiness/gotosocial/internal/storage" ) @@ -30,8 +29,6 @@ type Content struct { ContentType string // ContentLength in bytes ContentLength int64 - // Time when the content was last updated. - ContentUpdated time.Time // Actual content Content io.ReadCloser // Resource URL to forward to if the file can be fetched from the storage directly (e.g signed S3 URL) diff --git a/internal/api/model/status.go b/internal/api/model/status.go index c29ab3e82..724134b77 100644 --- a/internal/api/model/status.go +++ b/internal/api/model/status.go @@ -29,6 +29,10 @@ type Status struct { // The date when this status was created (ISO 8601 Datetime). // example: 2021-07-30T09:20:25+00:00 CreatedAt string `json:"created_at"` + // Timestamp of when the status was last edited (ISO 8601 Datetime). + // example: 2021-07-30T09:20:25+00:00 + // nullable: true + EditedAt *string `json:"edited_at"` // ID of the status being replied to. // example: 01FBVD42CQ3ZEEVMW180SBX03B // nullable: true diff --git a/internal/cache/cache.go b/internal/cache/cache.go index a4f9f2044..1a66fcd6b 100644 --- a/internal/cache/cache.go +++ b/internal/cache/cache.go @@ -105,6 +105,7 @@ func (c *Caches) Init() { c.initStatus() c.initStatusBookmark() c.initStatusBookmarkIDs() + c.initStatusEdit() c.initStatusFave() c.initStatusFaveIDs() c.initTag() diff --git a/internal/cache/db.go b/internal/cache/db.go index aac11236a..dc47bc31c 100644 --- a/internal/cache/db.go +++ b/internal/cache/db.go @@ -226,6 +226,9 @@ type DBCaches struct { // StatusBookmarkIDs provides access to the status bookmark IDs list database cache. StatusBookmarkIDs SliceCache[string] + // StatusEdit provides access to the gtsmodel StatusEdit database cache. + StatusEdit StructCache[*gtsmodel.StatusEdit] + // StatusFave provides access to the gtsmodel StatusFave database cache. StatusFave StructCache[*gtsmodel.StatusFave] @@ -1385,6 +1388,38 @@ func (c *Caches) initStatusBookmarkIDs() { c.DB.StatusBookmarkIDs.Init(0, cap) } +func (c *Caches) initStatusEdit() { + // Calculate maximum cache size. + cap := calculateResultCacheMax( + sizeofStatusEdit(), // model in-mem size. + config.GetCacheStatusEditMemRatio(), + ) + + log.Infof(nil, "cache size = %d", cap) + + copyF := func(s1 *gtsmodel.StatusEdit) *gtsmodel.StatusEdit { + s2 := new(gtsmodel.StatusEdit) + *s2 = *s1 + + // Don't include ptr fields that + // will be populated separately. + s2.Attachments = nil + + return s2 + } + + c.DB.StatusEdit.Init(structr.CacheConfig[*gtsmodel.StatusEdit]{ + Indices: []structr.IndexConfig{ + {Fields: "ID"}, + {Fields: "StatusID", Multiple: true}, + }, + MaxSize: cap, + IgnoreErr: ignoreErrors, + Copy: copyF, + Invalidate: c.OnInvalidateStatusEdit, + }) +} + func (c *Caches) initStatusFave() { // Calculate maximum cache size. cap := calculateResultCacheMax( diff --git a/internal/cache/invalidate.go b/internal/cache/invalidate.go index 9b42e88f6..42d7b7399 100644 --- a/internal/cache/invalidate.go +++ b/internal/cache/invalidate.go @@ -273,6 +273,11 @@ func (c *Caches) OnInvalidateStatusBookmark(bookmark *gtsmodel.StatusBookmark) { c.DB.StatusBookmarkIDs.Invalidate(bookmark.StatusID) } +func (c *Caches) OnInvalidateStatusEdit(edit *gtsmodel.StatusEdit) { + // Invalidate cache of related status model. + c.DB.Status.Invalidate("ID", edit.StatusID) +} + func (c *Caches) OnInvalidateStatusFave(fave *gtsmodel.StatusFave) { // Invalidate status fave ID list for this status. c.DB.StatusFaveIDs.Invalidate(fave.StatusID) diff --git a/internal/cache/size.go b/internal/cache/size.go index 26f4096ed..988755099 100644 --- a/internal/cache/size.go +++ b/internal/cache/size.go @@ -505,7 +505,6 @@ func sizeofMedia() uintptr { URL: exampleURI, RemoteURL: exampleURI, CreatedAt: exampleTime, - UpdatedAt: exampleTime, Type: gtsmodel.FileTypeImage, AccountID: exampleID, Description: exampleText, @@ -532,7 +531,6 @@ func sizeofMention() uintptr { ID: exampleURI, StatusID: exampleURI, CreatedAt: exampleTime, - UpdatedAt: exampleTime, OriginAccountID: exampleURI, OriginAccountURI: exampleURI, TargetAccountID: exampleID, @@ -674,6 +672,23 @@ func sizeofStatusBookmark() uintptr { })) } +func sizeofStatusEdit() uintptr { + return uintptr(size.Of(>smodel.StatusEdit{ + ID: exampleID, + Content: exampleText, + ContentWarning: exampleUsername, // similar length + Text: exampleText, + Language: "en", + Sensitive: func() *bool { ok := false; return &ok }(), + AttachmentIDs: []string{exampleID, exampleID, exampleID}, + Attachments: nil, + PollOptions: []string{exampleTextSmall, exampleTextSmall, exampleTextSmall, exampleTextSmall}, + PollVotes: []int{69, 420, 1337, 1969}, + StatusID: exampleID, + CreatedAt: exampleTime, + })) +} + func sizeofStatusFave() uintptr { return uintptr(size.Of(>smodel.StatusFave{ ID: exampleID, diff --git a/internal/config/config.go b/internal/config/config.go index 2e3ad8ec1..413743409 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -238,6 +238,7 @@ type CacheConfiguration struct { StatusMemRatio float64 `name:"status-mem-ratio"` StatusBookmarkMemRatio float64 `name:"status-bookmark-mem-ratio"` StatusBookmarkIDsMemRatio float64 `name:"status-bookmark-ids-mem-ratio"` + StatusEditMemRatio float64 `name:"status-edit-mem-ratio"` StatusFaveMemRatio float64 `name:"status-fave-mem-ratio"` StatusFaveIDsMemRatio float64 `name:"status-fave-ids-mem-ratio"` TagMemRatio float64 `name:"tag-mem-ratio"` diff --git a/internal/config/defaults.go b/internal/config/defaults.go index 9b45002d0..f77c5c456 100644 --- a/internal/config/defaults.go +++ b/internal/config/defaults.go @@ -199,6 +199,7 @@ var Defaults = Configuration{ StatusMemRatio: 5, StatusBookmarkMemRatio: 0.5, StatusBookmarkIDsMemRatio: 2, + StatusEditMemRatio: 2, StatusFaveMemRatio: 2, StatusFaveIDsMemRatio: 3, TagMemRatio: 2, diff --git a/internal/config/helpers.gen.go b/internal/config/helpers.gen.go index a35622f8e..543292ebe 100644 --- a/internal/config/helpers.gen.go +++ b/internal/config/helpers.gen.go @@ -3912,6 +3912,31 @@ func GetCacheStatusBookmarkIDsMemRatio() float64 { return global.GetCacheStatusB // SetCacheStatusBookmarkIDsMemRatio safely sets the value for global configuration 'Cache.StatusBookmarkIDsMemRatio' field func SetCacheStatusBookmarkIDsMemRatio(v float64) { global.SetCacheStatusBookmarkIDsMemRatio(v) } +// GetCacheStatusEditMemRatio safely fetches the Configuration value for state's 'Cache.StatusEditMemRatio' field +func (st *ConfigState) GetCacheStatusEditMemRatio() (v float64) { + st.mutex.RLock() + v = st.config.Cache.StatusEditMemRatio + st.mutex.RUnlock() + return +} + +// SetCacheStatusEditMemRatio safely sets the Configuration value for state's 'Cache.StatusEditMemRatio' field +func (st *ConfigState) SetCacheStatusEditMemRatio(v float64) { + st.mutex.Lock() + defer st.mutex.Unlock() + st.config.Cache.StatusEditMemRatio = v + st.reloadToViper() +} + +// CacheStatusEditMemRatioFlag returns the flag name for the 'Cache.StatusEditMemRatio' field +func CacheStatusEditMemRatioFlag() string { return "cache-status-edit-mem-ratio" } + +// GetCacheStatusEditMemRatio safely fetches the value for global configuration 'Cache.StatusEditMemRatio' field +func GetCacheStatusEditMemRatio() float64 { return global.GetCacheStatusEditMemRatio() } + +// SetCacheStatusEditMemRatio safely sets the value for global configuration 'Cache.StatusEditMemRatio' field +func SetCacheStatusEditMemRatio(v float64) { global.SetCacheStatusEditMemRatio(v) } + // GetCacheStatusFaveMemRatio safely fetches the Configuration value for state's 'Cache.StatusFaveMemRatio' field func (st *ConfigState) GetCacheStatusFaveMemRatio() (v float64) { st.mutex.RLock() diff --git a/internal/db/bundb/account_test.go b/internal/db/bundb/account_test.go index 7dcc0f9e7..879250408 100644 --- a/internal/db/bundb/account_test.go +++ b/internal/db/bundb/account_test.go @@ -46,7 +46,7 @@ type AccountTestSuite struct { func (suite *AccountTestSuite) TestGetAccountStatuses() { statuses, err := suite.db.GetAccountStatuses(context.Background(), suite.testAccounts["local_account_1"].ID, 20, false, false, "", "", false, false) suite.NoError(err) - suite.Len(statuses, 8) + suite.Len(statuses, 9) } func (suite *AccountTestSuite) TestGetAccountStatusesPageDown() { @@ -69,7 +69,7 @@ func (suite *AccountTestSuite) TestGetAccountStatusesPageDown() { if err != nil { suite.FailNow(err.Error()) } - suite.Len(statuses, 2) + suite.Len(statuses, 3) // try to get the last page (should be empty) statuses, err = suite.db.GetAccountStatuses(context.Background(), suite.testAccounts["local_account_1"].ID, 3, false, false, statuses[len(statuses)-1].ID, "", false, false) @@ -80,13 +80,13 @@ func (suite *AccountTestSuite) TestGetAccountStatusesPageDown() { func (suite *AccountTestSuite) TestGetAccountStatusesExcludeRepliesAndReblogs() { statuses, err := suite.db.GetAccountStatuses(context.Background(), suite.testAccounts["local_account_1"].ID, 20, true, true, "", "", false, false) suite.NoError(err) - suite.Len(statuses, 7) + suite.Len(statuses, 8) } func (suite *AccountTestSuite) TestGetAccountStatusesExcludeRepliesAndReblogsPublicOnly() { statuses, err := suite.db.GetAccountStatuses(context.Background(), suite.testAccounts["local_account_1"].ID, 20, true, true, "", "", false, true) suite.NoError(err) - suite.Len(statuses, 3) + suite.Len(statuses, 4) } // populateTestStatus adds mandatory fields to a partially populated status. @@ -173,7 +173,7 @@ func (suite *AccountTestSuite) TestGetAccountStatusesExcludeRepliesExcludesSelfR testAccount := suite.testAccounts["local_account_1"] statuses, err := suite.db.GetAccountStatuses(context.Background(), testAccount.ID, 20, true, true, "", "", false, false) suite.NoError(err) - suite.Len(statuses, 8) + suite.Len(statuses, 9) for _, status := range statuses { if status.InReplyToID != "" && status.InReplyToAccountID != testAccount.ID { suite.FailNowf("", "Status with ID %s is a non-self reply and should have been excluded", status.ID) diff --git a/internal/db/bundb/basic_test.go b/internal/db/bundb/basic_test.go index 56159dc25..e20aab765 100644 --- a/internal/db/bundb/basic_test.go +++ b/internal/db/bundb/basic_test.go @@ -114,7 +114,7 @@ func (suite *BasicTestSuite) TestGetAllStatuses() { s := []*gtsmodel.Status{} err := suite.db.GetAll(context.Background(), &s) suite.NoError(err) - suite.Len(s, 25) + suite.Len(s, 28) } func (suite *BasicTestSuite) TestGetAllNotNull() { diff --git a/internal/db/bundb/bundb.go b/internal/db/bundb/bundb.go index 70132fe58..cf612fd2e 100644 --- a/internal/db/bundb/bundb.go +++ b/internal/db/bundb/bundb.go @@ -81,6 +81,7 @@ type DBService struct { db.SinBinStatus db.Status db.StatusBookmark + db.StatusEdit db.StatusFave db.Tag db.Thread @@ -272,6 +273,10 @@ func NewBunDBService(ctx context.Context, state *state.State) (db.DB, error) { db: db, state: state, }, + StatusEdit: &statusEditDB{ + db: db, + state: state, + }, StatusFave: &statusFaveDB{ db: db, state: state, diff --git a/internal/db/bundb/bundb_test.go b/internal/db/bundb/bundb_test.go index e976199e4..2fcf61aed 100644 --- a/internal/db/bundb/bundb_test.go +++ b/internal/db/bundb/bundb_test.go @@ -57,6 +57,7 @@ type BunDBStandardTestSuite struct { testPolls map[string]*gtsmodel.Poll testPollVotes map[string]*gtsmodel.PollVote testInteractionRequests map[string]*gtsmodel.InteractionRequest + testStatusEdits map[string]*gtsmodel.StatusEdit } func (suite *BunDBStandardTestSuite) SetupSuite() { @@ -83,6 +84,7 @@ func (suite *BunDBStandardTestSuite) SetupSuite() { suite.testPolls = testrig.NewTestPolls() suite.testPollVotes = testrig.NewTestPollVotes() suite.testInteractionRequests = testrig.NewTestInteractionRequests() + suite.testStatusEdits = testrig.NewTestStatusEdits() } func (suite *BunDBStandardTestSuite) SetupTest() { diff --git a/internal/db/bundb/instance_test.go b/internal/db/bundb/instance_test.go index 4b8ec9962..1364bacc2 100644 --- a/internal/db/bundb/instance_test.go +++ b/internal/db/bundb/instance_test.go @@ -47,13 +47,13 @@ func (suite *InstanceTestSuite) TestCountInstanceUsersRemote() { func (suite *InstanceTestSuite) TestCountInstanceStatuses() { count, err := suite.db.CountInstanceStatuses(context.Background(), config.GetHost()) suite.NoError(err) - suite.Equal(19, count) + suite.Equal(21, count) } func (suite *InstanceTestSuite) TestCountInstanceStatusesRemote() { count, err := suite.db.CountInstanceStatuses(context.Background(), "fossbros-anonymous.io") suite.NoError(err) - suite.Equal(3, count) + suite.Equal(4, count) } func (suite *InstanceTestSuite) TestCountInstanceDomains() { diff --git a/internal/db/bundb/interaction_test.go b/internal/db/bundb/interaction_test.go index 37684f18c..1eb8154c1 100644 --- a/internal/db/bundb/interaction_test.go +++ b/internal/db/bundb/interaction_test.go @@ -59,11 +59,7 @@ func (suite *InteractionTestSuite) markInteractionsPending( // Put an interaction request // in the DB for this reply. - req, err := typeutils.StatusToInteractionRequest(ctx, reply) - if err != nil { - suite.FailNow(err.Error()) - } - + req := typeutils.StatusToInteractionRequest(reply) if err := suite.state.DB.PutInteractionRequest(ctx, req); err != nil { suite.FailNow(err.Error()) } @@ -90,11 +86,7 @@ func (suite *InteractionTestSuite) markInteractionsPending( // Put an interaction request // in the DB for this boost. - req, err := typeutils.StatusToInteractionRequest(ctx, boost) - if err != nil { - suite.FailNow(err.Error()) - } - + req := typeutils.StatusToInteractionRequest(boost) if err := suite.state.DB.PutInteractionRequest(ctx, req); err != nil { suite.FailNow(err.Error()) } @@ -121,11 +113,7 @@ func (suite *InteractionTestSuite) markInteractionsPending( // Put an interaction request // in the DB for this fave. - req, err := typeutils.StatusFaveToInteractionRequest(ctx, fave) - if err != nil { - suite.FailNow(err.Error()) - } - + req := typeutils.StatusFaveToInteractionRequest(fave) if err := suite.state.DB.PutInteractionRequest(ctx, req); err != nil { suite.FailNow(err.Error()) } diff --git a/internal/db/bundb/media.go b/internal/db/bundb/media.go index 453ad856a..09c8188f0 100644 --- a/internal/db/bundb/media.go +++ b/internal/db/bundb/media.go @@ -104,12 +104,6 @@ func (m *mediaDB) PutAttachment(ctx context.Context, media *gtsmodel.MediaAttach } func (m *mediaDB) UpdateAttachment(ctx context.Context, media *gtsmodel.MediaAttachment, columns ...string) error { - media.UpdatedAt = time.Now() - if len(columns) > 0 { - // If we're updating by column, ensure "updated_at" is included. - columns = append(columns, "updated_at") - } - return m.state.Caches.DB.Media.Store(media, func() error { _, err := m.db.NewUpdate(). Model(media). diff --git a/internal/db/bundb/migrations/20240809134448_interaction_requests_client_api.go b/internal/db/bundb/migrations/20240809134448_interaction_requests_client_api.go index 82c2b4016..a3fb8675e 100644 --- a/internal/db/bundb/migrations/20240809134448_interaction_requests_client_api.go +++ b/internal/db/bundb/migrations/20240809134448_interaction_requests_client_api.go @@ -93,11 +93,7 @@ func init() { // For each currently pending status, check whether it's a reply or // a boost, and insert a corresponding interaction request into the db. for _, pendingStatus := range pendingStatuses { - req, err := typeutils.StatusToInteractionRequest(ctx, pendingStatus) - if err != nil { - return err - } - + req := typeutils.StatusToInteractionRequest(pendingStatus) if _, err := tx. NewInsert(). Model(req). @@ -125,10 +121,7 @@ func init() { } for _, pendingFave := range pendingFaves { - req, err := typeutils.StatusFaveToInteractionRequest(ctx, pendingFave) - if err != nil { - return err - } + req := typeutils.StatusFaveToInteractionRequest(pendingFave) if _, err := tx. NewInsert(). diff --git a/internal/db/bundb/migrations/20241113151042_remove_mention_updated_at.go b/internal/db/bundb/migrations/20241113151042_remove_mention_updated_at.go new file mode 100644 index 000000000..bd72dc109 --- /dev/null +++ b/internal/db/bundb/migrations/20241113151042_remove_mention_updated_at.go @@ -0,0 +1,57 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package migrations + +import ( + "context" + + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/uptrace/bun" +) + +func init() { + up := func(ctx context.Context, db *bun.DB) error { + return db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { + + // Check for 'updated_at' column on mentions table, else return. + exists, err := doesColumnExist(ctx, tx, "mentions", "updated_at") + if err != nil { + return err + } else if !exists { + return nil + } + + // Remove 'updated_at' column. + _, err = tx.NewDropColumn(). + Model((*gtsmodel.Mention)(nil)). + Column("updated_at"). + Exec(ctx) + return err + }) + } + + down := func(ctx context.Context, db *bun.DB) error { + return db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { + return nil + }) + } + + if err := Migrations.Register(up, down); err != nil { + panic(err) + } +} diff --git a/internal/db/bundb/migrations/20241113152126_add_status_edits.go b/internal/db/bundb/migrations/20241113152126_add_status_edits.go new file mode 100644 index 000000000..aa0b0d4b9 --- /dev/null +++ b/internal/db/bundb/migrations/20241113152126_add_status_edits.go @@ -0,0 +1,67 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package migrations + +import ( + "context" + "reflect" + + gtsmodel "github.com/superseriousbusiness/gotosocial/internal/db/bundb/migrations/20241113152126_add_status_edits" + + "github.com/uptrace/bun" +) + +func init() { + up := func(ctx context.Context, db *bun.DB) error { + return db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { + statusType := reflect.TypeOf((*gtsmodel.Status)(nil)) + + // Generate new Status.EditIDs column definition from bun. + colDef, err := getBunColumnDef(tx, statusType, "EditIDs") + if err != nil { + return err + } + + // Add EditIDs column to Status table. + _, err = tx.NewAddColumn(). + Model((*gtsmodel.Status)(nil)). + ColumnExpr(colDef). + Exec(ctx) + if err != nil { + return err + } + + // Create the main StatusEdits table. + _, err = tx.NewCreateTable(). + IfNotExists(). + Model((*gtsmodel.StatusEdit)(nil)). + Exec(ctx) + return err + }) + } + + down := func(ctx context.Context, db *bun.DB) error { + return db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { + return nil + }) + } + + if err := Migrations.Register(up, down); err != nil { + panic(err) + } +} diff --git a/internal/db/bundb/migrations/20241113152126_add_status_edits/status.go b/internal/db/bundb/migrations/20241113152126_add_status_edits/status.go new file mode 100644 index 000000000..1b7d93f70 --- /dev/null +++ b/internal/db/bundb/migrations/20241113152126_add_status_edits/status.go @@ -0,0 +1,97 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package gtsmodel + +import ( + "time" + + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" +) + +// Status represents a user-created 'post' or 'status' in the database, either remote or local +type Status struct { + ID string `bun:"type:CHAR(26),pk,nullzero,notnull,unique"` // id of this item in the database + CreatedAt time.Time `bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // when was item created + UpdatedAt time.Time `bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // when was item last updated + FetchedAt time.Time `bun:"type:timestamptz,nullzero"` // when was item (remote) last fetched. + PinnedAt time.Time `bun:"type:timestamptz,nullzero"` // Status was pinned by owning account at this time. + URI string `bun:",unique,nullzero,notnull"` // activitypub URI of this status + URL string `bun:",nullzero"` // web url for viewing this status + Content string `bun:""` // content of this status; likely html-formatted but not guaranteed + AttachmentIDs []string `bun:"attachments,array"` // Database IDs of any media attachments associated with this status + Attachments []*gtsmodel.MediaAttachment `bun:"attached_media,rel:has-many"` // Attachments corresponding to attachmentIDs + TagIDs []string `bun:"tags,array"` // Database IDs of any tags used in this status + Tags []*gtsmodel.Tag `bun:"attached_tags,m2m:status_to_tags"` // Tags corresponding to tagIDs. https://bun.uptrace.dev/guide/relations.html#many-to-many-relation + MentionIDs []string `bun:"mentions,array"` // Database IDs of any mentions in this status + Mentions []*gtsmodel.Mention `bun:"attached_mentions,rel:has-many"` // Mentions corresponding to mentionIDs + EmojiIDs []string `bun:"emojis,array"` // Database IDs of any emojis used in this status + Emojis []*gtsmodel.Emoji `bun:"attached_emojis,m2m:status_to_emojis"` // Emojis corresponding to emojiIDs. https://bun.uptrace.dev/guide/relations.html#many-to-many-relation + Local *bool `bun:",nullzero,notnull,default:false"` // is this status from a local account? + AccountID string `bun:"type:CHAR(26),nullzero,notnull"` // which account posted this status? + Account *gtsmodel.Account `bun:"rel:belongs-to"` // account corresponding to accountID + AccountURI string `bun:",nullzero,notnull"` // activitypub uri of the owner of this status + InReplyToID string `bun:"type:CHAR(26),nullzero"` // id of the status this status replies to + InReplyToURI string `bun:",nullzero"` // activitypub uri of the status this status is a reply to + InReplyToAccountID string `bun:"type:CHAR(26),nullzero"` // id of the account that this status replies to + InReplyTo *Status `bun:"-"` // status corresponding to inReplyToID + InReplyToAccount *gtsmodel.Account `bun:"rel:belongs-to"` // account corresponding to inReplyToAccountID + BoostOfID string `bun:"type:CHAR(26),nullzero"` // id of the status this status is a boost of + BoostOfURI string `bun:"-"` // URI of the status this status is a boost of; field not inserted in the db, just for dereferencing purposes. + BoostOfAccountID string `bun:"type:CHAR(26),nullzero"` // id of the account that owns the boosted status + BoostOf *Status `bun:"-"` // status that corresponds to boostOfID + BoostOfAccount *gtsmodel.Account `bun:"rel:belongs-to"` // account that corresponds to boostOfAccountID + ThreadID string `bun:"type:CHAR(26),nullzero"` // id of the thread to which this status belongs; only set for remote statuses if a local account is involved at some point in the thread, otherwise null + EditIDs []string `bun:"edits,array"` // + Edits []*StatusEdit `bun:"-"` // + PollID string `bun:"type:CHAR(26),nullzero"` // + Poll *gtsmodel.Poll `bun:"-"` // + ContentWarning string `bun:",nullzero"` // cw string for this status + Visibility Visibility `bun:",nullzero,notnull"` // visibility entry for this status + Sensitive *bool `bun:",nullzero,notnull,default:false"` // mark the status as sensitive? + Language string `bun:",nullzero"` // what language is this status written in? + CreatedWithApplicationID string `bun:"type:CHAR(26),nullzero"` // Which application was used to create this status? + CreatedWithApplication *gtsmodel.Application `bun:"rel:belongs-to"` // application corresponding to createdWithApplicationID + ActivityStreamsType string `bun:",nullzero,notnull"` // What is the activitystreams type of this status? See: https://www.w3.org/TR/activitystreams-vocabulary/#object-types. Will probably almost always be Note but who knows!. + Text string `bun:""` // Original text of the status without formatting + Federated *bool `bun:",notnull"` // This status will be federated beyond the local timeline(s) + InteractionPolicy *gtsmodel.InteractionPolicy `bun:""` // InteractionPolicy for this status. If null then the default InteractionPolicy should be assumed for this status's Visibility. Always null for boost wrappers. + PendingApproval *bool `bun:",nullzero,notnull,default:false"` // If true then status is a reply or boost wrapper that must be Approved by the reply-ee or boost-ee before being fully distributed. + PreApproved bool `bun:"-"` // If true, then status is a reply to or boost wrapper of a status on our instance, has permission to do the interaction, and an Accept should be sent out for it immediately. Field not stored in the DB. + ApprovedByURI string `bun:",nullzero"` // URI of an Accept Activity that approves the Announce or Create Activity that this status was/will be attached to. +} + +// Visibility represents the visibility granularity of a status. +type Visibility string + +const ( + // VisibilityNone means nobody can see this. + // It's only used for web status visibility. + VisibilityNone Visibility = "none" + // VisibilityPublic means this status will be visible to everyone on all timelines. + VisibilityPublic Visibility = "public" + // VisibilityUnlocked means this status will be visible to everyone, but will only show on home timeline to followers, and in lists. + VisibilityUnlocked Visibility = "unlocked" + // VisibilityFollowersOnly means this status is viewable to followers only. + VisibilityFollowersOnly Visibility = "followers_only" + // VisibilityMutualsOnly means this status is visible to mutual followers only. + VisibilityMutualsOnly Visibility = "mutuals_only" + // VisibilityDirect means this status is visible only to mentioned recipients. + VisibilityDirect Visibility = "direct" + // VisibilityDefault is used when no other setting can be found. + VisibilityDefault Visibility = VisibilityUnlocked +) diff --git a/internal/db/bundb/migrations/20241113152126_add_status_edits/statusedit.go b/internal/db/bundb/migrations/20241113152126_add_status_edits/statusedit.go new file mode 100644 index 000000000..b27c3b343 --- /dev/null +++ b/internal/db/bundb/migrations/20241113152126_add_status_edits/statusedit.go @@ -0,0 +1,48 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package gtsmodel + +import ( + "time" +) + +// StatusEdit represents a **historical** view of a Status +// after a received edit. The Status itself will always +// contain the latest up-to-date information. +// +// Note that stored status edits may not exactly match that +// of the origin server, they are a best-effort by receiver +// to store version history. There is no AP history endpoint. +type StatusEdit struct { + ID string `bun:"type:CHAR(26),pk,nullzero,notnull,unique"` // ID of this item in the database. + Content string `bun:""` // Content of status at time of edit; likely html-formatted but not guaranteed. + ContentWarning string `bun:",nullzero"` // Content warning of status at time of edit. + Text string `bun:""` // Original status text, without formatting, at time of edit. + Language string `bun:",nullzero"` // Status language at time of edit. + Sensitive *bool `bun:",nullzero,notnull,default:false"` // Status sensitive flag at time of edit. + AttachmentIDs []string `bun:"attachments,array"` // Database IDs of media attachments associated with status at time of edit. + AttachmentDescriptions []string `bun:",array"` // Previous media descriptions of media attachments associated with status at time of edit. + PollOptions []string `bun:",array"` // Poll options of status at time of edit, only set if status contains a poll. + PollVotes []int `bun:",array"` // Poll vote count at time of status edit, only set if poll votes were reset. + StatusID string `bun:"type:CHAR(26),nullzero,notnull"` // The originating status ID this is a historical edit of. + CreatedAt time.Time `bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // The creation time of this version of the status content (according to receiving server). + + // We don't bother having a *gtsmodel.Status model here + // as the StatusEdit is always just attached to a Status, + // so it doesn't need a self-reference back to it. +} diff --git a/internal/db/bundb/migrations/20241121121623_enum_strings_to_ints.go b/internal/db/bundb/migrations/20241121121623_enum_strings_to_ints.go index 10ae95c17..7621ddc6c 100644 --- a/internal/db/bundb/migrations/20241121121623_enum_strings_to_ints.go +++ b/internal/db/bundb/migrations/20241121121623_enum_strings_to_ints.go @@ -19,12 +19,9 @@ package migrations import ( "context" - "errors" old_gtsmodel "github.com/superseriousbusiness/gotosocial/internal/db/bundb/migrations/20241121121623_enum_strings_to_ints" - "github.com/superseriousbusiness/gotosocial/internal/gtserror" new_gtsmodel "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" - "github.com/superseriousbusiness/gotosocial/internal/log" "github.com/superseriousbusiness/gotosocial/internal/util" "github.com/uptrace/bun" @@ -128,97 +125,6 @@ func init() { } } -// convertEnums performs a transaction that converts -// a table's column of our old-style enums (strings) to -// more performant and space-saving integer types. -func convertEnums[OldType ~string, NewType ~int16]( - ctx context.Context, - tx bun.Tx, - table string, - column string, - mapping map[OldType]NewType, - defaultValue *NewType, -) error { - if len(mapping) == 0 { - return errors.New("empty mapping") - } - - // Generate new column name. - newColumn := column + "_new" - - log.Infof(ctx, "converting %s.%s enums; "+ - "this may take a while, please don't interrupt!", - table, column, - ) - - // Ensure a default value. - if defaultValue == nil { - var zero NewType - defaultValue = &zero - } - - // Add new column to database. - if _, err := tx.NewAddColumn(). - Table(table). - ColumnExpr("? SMALLINT NOT NULL DEFAULT ?", - bun.Ident(newColumn), - *defaultValue). - Exec(ctx); err != nil { - return gtserror.Newf("error adding new column: %w", err) - } - - // Get a count of all in table. - total, err := tx.NewSelect(). - Table(table). - Count(ctx) - if err != nil { - return gtserror.Newf("error selecting total count: %w", err) - } - - var updated int - for old, new := range mapping { - - // Update old to new values. - res, err := tx.NewUpdate(). - Table(table). - Where("? = ?", bun.Ident(column), old). - Set("? = ?", bun.Ident(newColumn), new). - Exec(ctx) - if err != nil { - return gtserror.Newf("error updating old column values: %w", err) - } - - // Count number items updated. - n, _ := res.RowsAffected() - updated += int(n) - } - - // Check total updated. - if total != updated { - log.Warnf(ctx, "total=%d does not match updated=%d", total, updated) - } - - // Drop the old column from table. - if _, err := tx.NewDropColumn(). - Table(table). - ColumnExpr("?", bun.Ident(column)). - Exec(ctx); err != nil { - return gtserror.Newf("error dropping old column: %w", err) - } - - // Rename new to old name. - if _, err := tx.NewRaw( - "ALTER TABLE ? RENAME COLUMN ? TO ?", - bun.Ident(table), - bun.Ident(newColumn), - bun.Ident(column), - ).Exec(ctx); err != nil { - return gtserror.Newf("error renaming new column: %w", err) - } - - return nil -} - // visibilityEnumMapping maps old Visibility enum values to their newer integer type. func visibilityEnumMapping[T ~string]() map[T]new_gtsmodel.Visibility { return map[T]new_gtsmodel.Visibility{ diff --git a/internal/db/bundb/migrations/20241203124608_remove_media_attachment_updated_at.go b/internal/db/bundb/migrations/20241203124608_remove_media_attachment_updated_at.go new file mode 100644 index 000000000..344168b38 --- /dev/null +++ b/internal/db/bundb/migrations/20241203124608_remove_media_attachment_updated_at.go @@ -0,0 +1,57 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package migrations + +import ( + "context" + + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/uptrace/bun" +) + +func init() { + up := func(ctx context.Context, db *bun.DB) error { + return db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { + + // Check for 'updated_at' column on media attachments table, else return. + exists, err := doesColumnExist(ctx, tx, "media_attachments", "updated_at") + if err != nil { + return err + } else if !exists { + return nil + } + + // Remove 'updated_at' column. + _, err = tx.NewDropColumn(). + Model((*gtsmodel.MediaAttachment)(nil)). + Column("updated_at"). + Exec(ctx) + return err + }) + } + + down := func(ctx context.Context, db *bun.DB) error { + return db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { + return nil + }) + } + + if err := Migrations.Register(up, down); err != nil { + panic(err) + } +} diff --git a/internal/db/bundb/migrations/util.go b/internal/db/bundb/migrations/util.go index 47de09e23..edf7c1d05 100644 --- a/internal/db/bundb/migrations/util.go +++ b/internal/db/bundb/migrations/util.go @@ -19,11 +19,209 @@ package migrations import ( "context" + "errors" + "fmt" + "reflect" + "strconv" + "strings" + "codeberg.org/gruf/go-byteutil" + "github.com/superseriousbusiness/gotosocial/internal/gtserror" + "github.com/superseriousbusiness/gotosocial/internal/log" "github.com/uptrace/bun" "github.com/uptrace/bun/dialect" + "github.com/uptrace/bun/dialect/feature" + "github.com/uptrace/bun/dialect/sqltype" + "github.com/uptrace/bun/schema" ) +// convertEnums performs a transaction that converts +// a table's column of our old-style enums (strings) to +// more performant and space-saving integer types. +func convertEnums[OldType ~string, NewType ~int16]( + ctx context.Context, + tx bun.Tx, + table string, + column string, + mapping map[OldType]NewType, + defaultValue *NewType, +) error { + if len(mapping) == 0 { + return errors.New("empty mapping") + } + + // Generate new column name. + newColumn := column + "_new" + + log.Infof(ctx, "converting %s.%s enums; "+ + "this may take a while, please don't interrupt!", + table, column, + ) + + // Ensure a default value. + if defaultValue == nil { + var zero NewType + defaultValue = &zero + } + + // Add new column to database. + if _, err := tx.NewAddColumn(). + Table(table). + ColumnExpr("? SMALLINT NOT NULL DEFAULT ?", + bun.Ident(newColumn), + *defaultValue). + Exec(ctx); err != nil { + return gtserror.Newf("error adding new column: %w", err) + } + + // Get a count of all in table. + total, err := tx.NewSelect(). + Table(table). + Count(ctx) + if err != nil { + return gtserror.Newf("error selecting total count: %w", err) + } + + var updated int + for old, new := range mapping { + + // Update old to new values. + res, err := tx.NewUpdate(). + Table(table). + Where("? = ?", bun.Ident(column), old). + Set("? = ?", bun.Ident(newColumn), new). + Exec(ctx) + if err != nil { + return gtserror.Newf("error updating old column values: %w", err) + } + + // Count number items updated. + n, _ := res.RowsAffected() + updated += int(n) + } + + // Check total updated. + if total != updated { + log.Warnf(ctx, "total=%d does not match updated=%d", total, updated) + } + + // Drop the old column from table. + if _, err := tx.NewDropColumn(). + Table(table). + ColumnExpr("?", bun.Ident(column)). + Exec(ctx); err != nil { + return gtserror.Newf("error dropping old column: %w", err) + } + + // Rename new to old name. + if _, err := tx.NewRaw( + "ALTER TABLE ? RENAME COLUMN ? TO ?", + bun.Ident(table), + bun.Ident(newColumn), + bun.Ident(column), + ).Exec(ctx); err != nil { + return gtserror.Newf("error renaming new column: %w", err) + } + + return nil +} + +// getBunColumnDef generates a column definition string for the SQL table represented by +// Go type, with the SQL column represented by the given Go field name. This ensures when +// adding a new column for table by migration that it will end up as bun would create it. +// +// NOTE: this function must stay in sync with (*bun.CreateTableQuery{}).AppendQuery(), +// specifically where it loops over table fields appending each column definition. +func getBunColumnDef(db bun.IDB, rtype reflect.Type, fieldName string) (string, error) { + d := db.Dialect() + f := d.Features() + + // Get bun schema definitions for Go type and its field. + field, table, err := getModelField(db, rtype, fieldName) + if err != nil { + return "", err + } + + // Start with reasonable buf. + buf := make([]byte, 0, 64) + + // Start with the SQL column name. + buf = append(buf, field.SQLName...) + buf = append(buf, " "...) + + // Append the SQL + // type information. + switch { + + // Most of the time these two will match, but for the cases where DiscoveredSQLType is dialect-specific, + // e.g. pgdialect would change sqltype.SmallInt to pgTypeSmallSerial for columns that have `bun:",autoincrement"` + case !strings.EqualFold(field.CreateTableSQLType, field.DiscoveredSQLType): + buf = append(buf, field.CreateTableSQLType...) + + // For all common SQL types except VARCHAR, both UserDefinedSQLType and DiscoveredSQLType specify the correct type, + // and we needn't modify it. For VARCHAR columns, we will stop to check if a valid length has been set in .Varchar(int). + case !strings.EqualFold(field.CreateTableSQLType, sqltype.VarChar): + buf = append(buf, field.CreateTableSQLType...) + + // All else falls back + // to a default varchar. + default: + if d.Name() == dialect.Oracle { + buf = append(buf, "VARCHAR2"...) + } else { + buf = append(buf, sqltype.VarChar...) + } + buf = append(buf, "("...) + buf = strconv.AppendInt(buf, int64(d.DefaultVarcharLen()), 10) + buf = append(buf, ")"...) + } + + // Append not null definition if field requires. + if field.NotNull && d.Name() != dialect.Oracle { + buf = append(buf, " NOT NULL"...) + } + + // Append autoincrement definition if field requires. + if field.Identity && f.Has(feature.GeneratedIdentity) || + (field.AutoIncrement && (f.Has(feature.AutoIncrement) || f.Has(feature.Identity))) { + buf = d.AppendSequence(buf, table, field) + } + + // Append any default value. + if field.SQLDefault != "" { + buf = append(buf, " DEFAULT "...) + buf = append(buf, field.SQLDefault...) + } + + return byteutil.B2S(buf), nil +} + +// getModelField returns the uptrace/bun schema details for given Go type and field name. +func getModelField(db bun.IDB, rtype reflect.Type, fieldName string) (*schema.Field, *schema.Table, error) { + + // Get the associated table for Go type. + table := db.Dialect().Tables().Get(rtype) + if table == nil { + return nil, nil, fmt.Errorf("no table found for type: %s", rtype) + } + + var field *schema.Field + + // Look for field matching Go name. + for i := range table.Fields { + if table.Fields[i].GoName == fieldName { + field = table.Fields[i] + break + } + } + + if field == nil { + return nil, nil, fmt.Errorf("no bun field found on %s with name: %s", rtype, fieldName) + } + + return field, table, nil +} + // doesColumnExist safely checks whether given column exists on table, handling both SQLite and PostgreSQL appropriately. func doesColumnExist(ctx context.Context, tx bun.Tx, table, col string) (bool, error) { var n int diff --git a/internal/db/bundb/status.go b/internal/db/bundb/status.go index 45e9864a3..fa31f3459 100644 --- a/internal/db/bundb/status.go +++ b/internal/db/bundb/status.go @@ -21,7 +21,6 @@ import ( "context" "errors" "slices" - "time" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/gtscontext" @@ -181,7 +180,7 @@ func (s *statusDB) getStatus(ctx context.Context, lookup string, dbQuery func(*g func (s *statusDB) PopulateStatus(ctx context.Context, status *gtsmodel.Status) error { var ( err error - errs = gtserror.NewMultiError(9) + errs gtserror.MultiError ) if status.Account == nil { @@ -257,7 +256,7 @@ func (s *statusDB) PopulateStatus(ctx context.Context, status *gtsmodel.Status) if !status.AttachmentsPopulated() { // Status attachments are out-of-date with IDs, repopulate. status.Attachments, err = s.state.DB.GetAttachmentsByIDs( - ctx, // these are already barebones + gtscontext.SetBarebones(ctx), status.AttachmentIDs, ) if err != nil { @@ -268,7 +267,7 @@ func (s *statusDB) PopulateStatus(ctx context.Context, status *gtsmodel.Status) if !status.TagsPopulated() { // Status tags are out-of-date with IDs, repopulate. status.Tags, err = s.state.DB.GetTags( - ctx, + gtscontext.SetBarebones(ctx), status.TagIDs, ) if err != nil { @@ -279,7 +278,7 @@ func (s *statusDB) PopulateStatus(ctx context.Context, status *gtsmodel.Status) if !status.MentionsPopulated() { // Status mentions are out-of-date with IDs, repopulate. status.Mentions, err = s.state.DB.GetMentions( - ctx, // leave fully populated for now + ctx, // TODO: manually populate mentions for places expecting these populated status.MentionIDs, ) if err != nil { @@ -290,7 +289,7 @@ func (s *statusDB) PopulateStatus(ctx context.Context, status *gtsmodel.Status) if !status.EmojisPopulated() { // Status emojis are out-of-date with IDs, repopulate. status.Emojis, err = s.state.DB.GetEmojisByIDs( - ctx, // these are already barebones + gtscontext.SetBarebones(ctx), status.EmojiIDs, ) if err != nil { @@ -298,10 +297,21 @@ func (s *statusDB) PopulateStatus(ctx context.Context, status *gtsmodel.Status) } } + if !status.EditsPopulated() { + // Status edits are out-of-date with IDs, repopulate. + status.Edits, err = s.state.DB.GetStatusEditsByIDs( + gtscontext.SetBarebones(ctx), + status.EditIDs, + ) + if err != nil { + errs.Appendf("error populating status edits: %w", err) + } + } + if status.CreatedWithApplicationID != "" && status.CreatedWithApplication == nil { // Populate the status' expected CreatedWithApplication (not always set). status.CreatedWithApplication, err = s.state.DB.GetApplicationByID( - ctx, // these are already barebones + gtscontext.SetBarebones(ctx), status.CreatedWithApplicationID, ) if err != nil { @@ -350,14 +360,14 @@ func (s *statusDB) PutStatus(ctx context.Context, status *gtsmodel.Status) error } } - // change the status ID of the media attachments to the new status + // change the status ID of the media + // attachments to the current status for _, a := range status.Attachments { a.StatusID = status.ID - a.UpdatedAt = time.Now() if _, err := tx. NewUpdate(). Model(a). - Column("status_id", "updated_at"). + Column("status_id"). Where("? = ?", bun.Ident("media_attachment.id"), a.ID). Exec(ctx); err != nil { if !errors.Is(err, db.ErrAlreadyExists) { @@ -384,19 +394,15 @@ func (s *statusDB) PutStatus(ctx context.Context, status *gtsmodel.Status) error } // Finally, insert the status - _, err := tx.NewInsert().Model(status).Exec(ctx) + _, err := tx.NewInsert(). + Model(status). + Exec(ctx) return err }) }) } func (s *statusDB) UpdateStatus(ctx context.Context, status *gtsmodel.Status, columns ...string) error { - status.UpdatedAt = time.Now() - if len(columns) > 0 { - // If we're updating by column, ensure "updated_at" is included. - columns = append(columns, "updated_at") - } - return s.state.Caches.DB.Status.Store(status, func() error { // It is safe to run this database transaction within cache.Store // as the cache does not attempt a mutex lock until AFTER hook. @@ -434,13 +440,14 @@ func (s *statusDB) UpdateStatus(ctx context.Context, status *gtsmodel.Status, co } } - // change the status ID of the media attachments to the new status + // change the status ID of the media + // attachments to the current status. for _, a := range status.Attachments { a.StatusID = status.ID - a.UpdatedAt = time.Now() if _, err := tx. NewUpdate(). Model(a). + Column("status_id"). Where("? = ?", bun.Ident("media_attachment.id"), a.ID). Exec(ctx); err != nil { if !errors.Is(err, db.ErrAlreadyExists) { @@ -467,8 +474,7 @@ func (s *statusDB) UpdateStatus(ctx context.Context, status *gtsmodel.Status, co } // Finally, update the status - _, err := tx. - NewUpdate(). + _, err := tx.NewUpdate(). Model(status). Column(columns...). Where("? = ?", bun.Ident("status.id"), status.ID). diff --git a/internal/db/bundb/statusedit.go b/internal/db/bundb/statusedit.go new file mode 100644 index 000000000..c932968fd --- /dev/null +++ b/internal/db/bundb/statusedit.go @@ -0,0 +1,198 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package bundb + +import ( + "context" + "errors" + "slices" + + "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/gtscontext" + "github.com/superseriousbusiness/gotosocial/internal/gtserror" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/log" + "github.com/superseriousbusiness/gotosocial/internal/state" + "github.com/superseriousbusiness/gotosocial/internal/util/xslices" + "github.com/uptrace/bun" +) + +type statusEditDB struct { + db *bun.DB + state *state.State +} + +func (s *statusEditDB) GetStatusEditByID(ctx context.Context, id string) (*gtsmodel.StatusEdit, error) { + // Fetch edit from database cache with loader callback. + edit, err := s.state.Caches.DB.StatusEdit.LoadOne("ID", + func() (*gtsmodel.StatusEdit, error) { + var edit gtsmodel.StatusEdit + + // Not cached, load edit + // from database by its ID. + if err := s.db.NewSelect(). + Model(&edit). + Where("? = ?", bun.Ident("id"), id). + Scan(ctx); err != nil { + return nil, err + } + + return &edit, nil + }, id, + ) + if err != nil { + return nil, err + } + + if gtscontext.Barebones(ctx) { + // no need to fully populate. + return edit, nil + } + + // Further populate the edit fields where applicable. + if err := s.PopulateStatusEdit(ctx, edit); err != nil { + return nil, err + } + + return edit, nil +} + +func (s *statusEditDB) GetStatusEditsByIDs(ctx context.Context, ids []string) ([]*gtsmodel.StatusEdit, error) { + // Load status edits for IDs via cache loader callbacks. + edits, err := s.state.Caches.DB.StatusEdit.LoadIDs("ID", + ids, + func(uncached []string) ([]*gtsmodel.StatusEdit, error) { + // Preallocate expected length of uncached edits. + edits := make([]*gtsmodel.StatusEdit, 0, len(uncached)) + + // Perform database query scanning + // the remaining (uncached) edit IDs. + if err := s.db.NewSelect(). + Model(&edits). + Where("? IN (?)", bun.Ident("id"), bun.In(uncached)). + Scan(ctx); err != nil { + return nil, err + } + + return edits, nil + }, + ) + if err != nil { + return nil, err + } + + // Reorder the edits by their + // IDs to ensure in correct order. + getID := func(e *gtsmodel.StatusEdit) string { return e.ID } + xslices.OrderBy(edits, ids, getID) + + if gtscontext.Barebones(ctx) { + // no need to fully populate. + return edits, nil + } + + // Populate all loaded edits, removing those we fail to + // populate (removes needing so many nil checks everywhere). + edits = slices.DeleteFunc(edits, func(edit *gtsmodel.StatusEdit) bool { + if err := s.PopulateStatusEdit(ctx, edit); err != nil { + log.Errorf(ctx, "error populating edit %s: %v", edit.ID, err) + return true + } + return false + }) + + return edits, nil +} + +func (s *statusEditDB) PopulateStatusEdit(ctx context.Context, edit *gtsmodel.StatusEdit) error { + var err error + var errs gtserror.MultiError + + // For sub-models we only want + // barebones versions of them. + ctx = gtscontext.SetBarebones(ctx) + + if !edit.AttachmentsPopulated() { + // Fetch all attachments for status edit's IDs. + edit.Attachments, err = s.state.DB.GetAttachmentsByIDs( + ctx, + edit.AttachmentIDs, + ) + if err != nil { + errs.Appendf("error populating edit attachments: %w", err) + } + } + + return errs.Combine() +} + +func (s *statusEditDB) PutStatusEdit(ctx context.Context, edit *gtsmodel.StatusEdit) error { + return s.state.Caches.DB.StatusEdit.Store(edit, func() error { + _, err := s.db.NewInsert().Model(edit).Exec(ctx) + return err + }) +} + +func (s *statusEditDB) DeleteStatusEdits(ctx context.Context, ids []string) error { + // Gather necessary fields from + // deleted for cache invalidation. + deleted := make([]*gtsmodel.StatusEdit, 0, len(ids)) + + // Delete all edits with IDs pertaining + // to given slice, returning status IDs. + if _, err := s.db.NewDelete(). + Model(&deleted). + Where("? IN (?)", bun.Ident("id"), bun.In(ids)). + Returning("?", bun.Ident("status_id")). + Exec(ctx); err != nil && + !errors.Is(err, db.ErrNoEntries) { + return err + } + + // Check for no deletes. + if len(deleted) == 0 { + return nil + } + + // Invalidate all the cached status edits with IDs. + s.state.Caches.DB.StatusEdit.InvalidateIDs("ID", ids) + + // With each invalidate hook mark status ID of + // edit we just called for. We only want to call + // invalidate hooks of edits from unique statuses. + invalidated := make(map[string]struct{}, 1) + + // Invalidate the first delete manually, this + // opt negates need for initial hashmap lookup. + s.state.Caches.OnInvalidateStatusEdit(deleted[0]) + invalidated[deleted[0].StatusID] = struct{}{} + + for _, edit := range deleted { + // Check not already called for status. + _, ok := invalidated[edit.StatusID] + if ok { + continue + } + + // Manually call status edit invalidate hook. + s.state.Caches.OnInvalidateStatusEdit(edit) + invalidated[edit.StatusID] = struct{}{} + } + + return nil +} diff --git a/internal/db/bundb/statusedit_test.go b/internal/db/bundb/statusedit_test.go new file mode 100644 index 000000000..b6a15e825 --- /dev/null +++ b/internal/db/bundb/statusedit_test.go @@ -0,0 +1,168 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package bundb_test + +import ( + "context" + "errors" + "reflect" + "slices" + "testing" + "time" + + "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" +) + +type StatusEditTestSuite struct { + BunDBStandardTestSuite +} + +func (suite *StatusEditTestSuite) TestGetStatusEditBy() { + t := suite.T() + + // Create a new context for this test. + ctx, cncl := context.WithCancel(context.Background()) + defer cncl() + + // Sentinel error to mark avoiding a test case. + sentinelErr := errors.New("sentinel") + + for _, edit := range suite.testStatusEdits { + for lookup, dbfunc := range map[string]func() (*gtsmodel.StatusEdit, error){ + "id": func() (*gtsmodel.StatusEdit, error) { + return suite.db.GetStatusEditByID(ctx, edit.ID) + }, + } { + // Clear database caches. + suite.state.Caches.Init() + + t.Logf("checking database lookup %q", lookup) + + // Perform database function. + checkEdit, err := dbfunc() + if err != nil { + if err == sentinelErr { + continue + } + + t.Errorf("error encountered for database lookup %q: %v", lookup, err) + continue + } + + // Check received account data. + if !areEditsEqual(edit, checkEdit) { + t.Errorf("edit does not contain expected data: %+v", checkEdit) + continue + } + } + } +} + +func (suite *StatusEditTestSuite) TestGetStatusEditsByIDs() { + t := suite.T() + + // Create a new context for this test. + ctx, cncl := context.WithCancel(context.Background()) + defer cncl() + + // editsByStatus returns all test edits by the given status with ID. + editsByStatus := func(status *gtsmodel.Status) []*gtsmodel.StatusEdit { + var edits []*gtsmodel.StatusEdit + for _, edit := range suite.testStatusEdits { + if edit.StatusID == status.ID { + edits = append(edits, edit) + } + } + return edits + } + + for _, status := range suite.testStatuses { + // Get test status edit models + // that should be found for status. + check := editsByStatus(status) + + // Fetch edits for the slice of IDs attached to status from database. + edits, err := suite.state.DB.GetStatusEditsByIDs(ctx, status.EditIDs) + suite.NoError(err) + + // Ensure both slices + // sorted the same. + sortEdits(check) + sortEdits(edits) + + // Check whether slices of status edits match. + if !slices.EqualFunc(check, edits, areEditsEqual) { + t.Error("status edit slices do not match") + } + } +} + +func (suite *StatusEditTestSuite) TestDeleteStatusEdits() { + // Create a new context for this test. + ctx, cncl := context.WithCancel(context.Background()) + defer cncl() + + for _, status := range suite.testStatuses { + // Delete all edits for status with given IDs from database. + err := suite.state.DB.DeleteStatusEdits(ctx, status.EditIDs) + suite.NoError(err) + + // Now attempt to fetch these edits from database, should be empty. + edits, err := suite.state.DB.GetStatusEditsByIDs(ctx, status.EditIDs) + suite.NoError(err) + suite.Empty(edits) + } +} + +func TestStatusEditTestSuite(t *testing.T) { + suite.Run(t, new(StatusEditTestSuite)) +} + +func areEditsEqual(e1, e2 *gtsmodel.StatusEdit) bool { + // Clone the 1st status edit. + e1Copy := new(gtsmodel.StatusEdit) + *e1Copy = *e1 + e1 = e1Copy + + // Clone the 2nd status edit. + e2Copy := new(gtsmodel.StatusEdit) + *e2Copy = *e2 + e2 = e2Copy + + // Clear populated sub-models. + e1.Attachments = nil + e2.Attachments = nil + + // Clear database-set fields. + e1.CreatedAt = time.Time{} + e2.CreatedAt = time.Time{} + + return reflect.DeepEqual(*e1, *e2) +} + +func sortEdits(edits []*gtsmodel.StatusEdit) { + slices.SortFunc(edits, func(a, b *gtsmodel.StatusEdit) int { + if a.CreatedAt.Before(b.CreatedAt) { + return +1 + } else if b.CreatedAt.Before(a.CreatedAt) { + return -1 + } + return 0 + }) +} diff --git a/internal/db/bundb/timeline.go b/internal/db/bundb/timeline.go index bcb7953d4..fcea0178a 100644 --- a/internal/db/bundb/timeline.go +++ b/internal/db/bundb/timeline.go @@ -123,13 +123,8 @@ func (t *timelineDB) GetHomeTimeline(ctx context.Context, accountID string, maxI if maxID == "" || maxID >= id.Highest { const future = 24 * time.Hour - var err error - // don't return statuses more than 24hr in the future - maxID, err = id.NewULIDFromTime(time.Now().Add(future)) - if err != nil { - return nil, err - } + maxID = id.NewULIDFromTime(time.Now().Add(future)) } // return only statuses LOWER (ie., older) than maxID @@ -223,13 +218,8 @@ func (t *timelineDB) GetPublicTimeline(ctx context.Context, maxID string, sinceI if maxID == "" || maxID >= id.Highest { const future = 24 * time.Hour - var err error - // don't return statuses more than 24hr in the future - maxID, err = id.NewULIDFromTime(time.Now().Add(future)) - if err != nil { - return nil, err - } + maxID = id.NewULIDFromTime(time.Now().Add(future)) } // return only statuses LOWER (ie., older) than maxID @@ -409,13 +399,8 @@ func (t *timelineDB) GetListTimeline( if maxID == "" || maxID >= id.Highest { const future = 24 * time.Hour - var err error - // don't return statuses more than 24hr in the future - maxID, err = id.NewULIDFromTime(time.Now().Add(future)) - if err != nil { - return nil, err - } + maxID = id.NewULIDFromTime(time.Now().Add(future)) } // return only statuses LOWER (ie., older) than maxID @@ -508,13 +493,8 @@ func (t *timelineDB) GetTagTimeline( if maxID == "" || maxID >= id.Highest { const future = 24 * time.Hour - var err error - // don't return statuses more than 24hr in the future - maxID, err = id.NewULIDFromTime(time.Now().Add(future)) - if err != nil { - return nil, err - } + maxID = id.NewULIDFromTime(time.Now().Add(future)) } // return only statuses LOWER (ie., older) than maxID diff --git a/internal/db/bundb/timeline_test.go b/internal/db/bundb/timeline_test.go index 00df2b3a6..75a335512 100644 --- a/internal/db/bundb/timeline_test.go +++ b/internal/db/bundb/timeline_test.go @@ -37,10 +37,7 @@ type TimelineTestSuite struct { func getFutureStatus() *gtsmodel.Status { theDistantFuture := time.Now().Add(876600 * time.Hour) - id, err := id.NewULIDFromTime(theDistantFuture) - if err != nil { - panic(err) - } + id := id.NewULIDFromTime(theDistantFuture) return >smodel.Status{ ID: id, @@ -182,7 +179,7 @@ func (suite *TimelineTestSuite) TestGetHomeTimelineIgnoreExclusive() { if err != nil { suite.FailNow(err.Error()) } - suite.checkStatuses(s, id.Highest, id.Lowest, 8) + suite.checkStatuses(s, id.Highest, id.Lowest, 9) // Remove admin account from the exclusive list. listEntry := suite.testListEntries["local_account_1_list_1_entry_2"] @@ -196,7 +193,7 @@ func (suite *TimelineTestSuite) TestGetHomeTimelineIgnoreExclusive() { if err != nil { suite.FailNow(err.Error()) } - suite.checkStatuses(s, id.Highest, id.Lowest, 12) + suite.checkStatuses(s, id.Highest, id.Lowest, 13) } func (suite *TimelineTestSuite) TestGetHomeTimelineNoFollowing() { @@ -228,7 +225,7 @@ func (suite *TimelineTestSuite) TestGetHomeTimelineNoFollowing() { suite.FailNow(err.Error()) } - suite.checkStatuses(s, id.Highest, id.Lowest, 8) + suite.checkStatuses(s, id.Highest, id.Lowest, 9) } func (suite *TimelineTestSuite) TestGetHomeTimelineWithFutureStatus() { @@ -281,8 +278,8 @@ func (suite *TimelineTestSuite) TestGetHomeTimelineFromHighest() { } suite.checkStatuses(s, id.Highest, id.Lowest, 5) - suite.Equal("01J2M1HPFSS54S60Y0KYV23KJE", s[0].ID) - suite.Equal("01G36SF3V6Y6V5BF9P4R7PQG7G", s[len(s)-1].ID) + suite.Equal("01JDPZEZ77X1NX0TY9M10BK1HM", s[0].ID) + suite.Equal("01HEN2RZ8BG29Y5Z9VJC73HZW7", s[len(s)-1].ID) } func (suite *TimelineTestSuite) TestGetListTimelineNoParams() { @@ -296,7 +293,7 @@ func (suite *TimelineTestSuite) TestGetListTimelineNoParams() { suite.FailNow(err.Error()) } - suite.checkStatuses(s, id.Highest, id.Lowest, 12) + suite.checkStatuses(s, id.Highest, id.Lowest, 13) } func (suite *TimelineTestSuite) TestGetListTimelineMaxID() { @@ -311,8 +308,8 @@ func (suite *TimelineTestSuite) TestGetListTimelineMaxID() { } suite.checkStatuses(s, id.Highest, id.Lowest, 5) - suite.Equal("01HEN2PRXT0TF4YDRA64FZZRN7", s[0].ID) - suite.Equal("01FF25D5Q0DH7CHD57CTRS6WK0", s[len(s)-1].ID) + suite.Equal("01JDPZEZ77X1NX0TY9M10BK1HM", s[0].ID) + suite.Equal("01FN3VJGFH10KR7S2PB0GFJZYG", s[len(s)-1].ID) } func (suite *TimelineTestSuite) TestGetListTimelineMinID() { diff --git a/internal/db/db.go b/internal/db/db.go index c42985912..11dd2e507 100644 --- a/internal/db/db.go +++ b/internal/db/db.go @@ -51,6 +51,7 @@ type DB interface { SinBinStatus Status StatusBookmark + StatusEdit StatusFave Tag Thread diff --git a/internal/db/statusedit.go b/internal/db/statusedit.go new file mode 100644 index 000000000..32e770fb9 --- /dev/null +++ b/internal/db/statusedit.go @@ -0,0 +1,43 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package db + +import ( + "context" + + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" +) + +type StatusEdit interface { + + // GetStatusEditByID fetches the StatusEdit with given ID from the database. + GetStatusEditByID(ctx context.Context, id string) (*gtsmodel.StatusEdit, error) + + // GetStatusEditsByIDs fetches all StatusEdits with given IDs from database, + // this is optimized and faster than multiple calls to GetStatusEditByID. + GetStatusEditsByIDs(ctx context.Context, ids []string) ([]*gtsmodel.StatusEdit, error) + + // PopulateStatusEdit ensures the given StatusEdit's sub-models are populated. + PopulateStatusEdit(ctx context.Context, edit *gtsmodel.StatusEdit) error + + // PutStatusEdit inserts the given new StatusEdit into the database. + PutStatusEdit(ctx context.Context, edit *gtsmodel.StatusEdit) error + + // DeleteStatusEdits deletes the StatusEdits with given IDs from the database. + DeleteStatusEdits(ctx context.Context, ids []string) error +} diff --git a/internal/federation/dereferencing/announce.go b/internal/federation/dereferencing/announce.go index a3eaf199d..eb949f159 100644 --- a/internal/federation/dereferencing/announce.go +++ b/internal/federation/dereferencing/announce.go @@ -87,7 +87,7 @@ func (d *Dereferencer) EnrichAnnounce( boost.Federated = target.Federated // Ensure this Announce is permitted by the Announcee. - permit, err := d.isPermittedStatus(ctx, requestUser, nil, boost) + permit, err := d.isPermittedStatus(ctx, requestUser, nil, boost, true) if err != nil { return nil, gtserror.Newf("error checking permitted status %s: %w", boost.URI, err) } @@ -99,10 +99,7 @@ func (d *Dereferencer) EnrichAnnounce( } // Generate an ID for the boost wrapper status. - boost.ID, err = id.NewULIDFromTime(boost.CreatedAt) - if err != nil { - return nil, gtserror.Newf("error generating id: %w", err) - } + boost.ID = id.NewULIDFromTime(boost.CreatedAt) // Store the boost wrapper status in database. switch err = d.state.DB.PutStatus(ctx, boost); { diff --git a/internal/federation/dereferencing/media.go b/internal/federation/dereferencing/media.go index 3bed4b198..d22eeb237 100644 --- a/internal/federation/dereferencing/media.go +++ b/internal/federation/dereferencing/media.go @@ -128,6 +128,7 @@ func (d *Dereferencer) RefreshMedia( // Check emoji is up-to-date // with provided extra info. switch { + case force: case info.Blurhash != nil && *info.Blurhash != attach.Blurhash: attach.Blurhash = *info.Blurhash diff --git a/internal/federation/dereferencing/status.go b/internal/federation/dereferencing/status.go index c90730826..d19669891 100644 --- a/internal/federation/dereferencing/status.go +++ b/internal/federation/dereferencing/status.go @@ -302,6 +302,7 @@ func (d *Dereferencer) enrichStatusSafely( uri, status, statusable, + isNew, ) // Check for a returned HTTP code via error. @@ -374,6 +375,7 @@ func (d *Dereferencer) enrichStatus( uri *url.URL, status *gtsmodel.Status, statusable ap.Statusable, + isNew bool, ) ( *gtsmodel.Status, ap.Statusable, @@ -476,8 +478,7 @@ func (d *Dereferencer) enrichStatus( // Ensure the final parsed status URI or URL matches // the input URI we fetched (or received) it as. - matches, err := util.URIMatches( - uri, + matches, err := util.URIMatches(uri, append( ap.GetURL(statusable), // status URL(s) ap.GetJSONLDId(statusable), // status URI @@ -497,19 +498,10 @@ func (d *Dereferencer) enrichStatus( ) } - var isNew bool - - // Based on the original provided - // status model, determine whether - // this is a new insert / update. - if isNew = (status.ID == ""); isNew { + if isNew { // Generate new status ID from the provided creation date. - latestStatus.ID, err = id.NewULIDFromTime(latestStatus.CreatedAt) - if err != nil { - log.Errorf(ctx, "invalid created at date (falling back to 'now'): %v", err) - latestStatus.ID = id.NewULID() // just use "now" - } + latestStatus.ID = id.NewULIDFromTime(latestStatus.CreatedAt) } else { // Reuse existing status ID. @@ -519,7 +511,6 @@ func (d *Dereferencer) enrichStatus( // Set latest fetch time and carry- // over some values from "old" status. latestStatus.FetchedAt = time.Now() - latestStatus.UpdatedAt = status.UpdatedAt latestStatus.Local = status.Local latestStatus.PinnedAt = status.PinnedAt @@ -538,8 +529,9 @@ func (d *Dereferencer) enrichStatus( } // Check if this is a permitted status we should accept. - // Function also sets "PendingApproval" bool as necessary. - permit, err := d.isPermittedStatus(ctx, requestUser, status, latestStatus) + // Function also sets "PendingApproval" bool as necessary, + // and handles removal of existing statuses no longer permitted. + permit, err := d.isPermittedStatus(ctx, requestUser, status, latestStatus, isNew) if err != nil { return nil, nil, gtserror.Newf("error checking permissibility for status %s: %w", uri, err) } @@ -550,59 +542,113 @@ func (d *Dereferencer) enrichStatus( return nil, nil, gtserror.SetNotPermitted(err) } - // Ensure the status' mentions are populated, and pass in existing to check for changes. - if err := d.fetchStatusMentions(ctx, requestUser, status, latestStatus); err != nil { + // Insert / update any attached status poll. + pollChanged, err := d.handleStatusPoll(ctx, + status, + latestStatus, + ) + if err != nil { + return nil, nil, gtserror.Newf("error handling poll for status %s: %w", uri, err) + } + + // Populate mentions associated with status, passing + // in existing status to reuse old where possible. + // (especially important here to reduce need to dereference). + mentionsChanged, err := d.fetchStatusMentions(ctx, + requestUser, + status, + latestStatus, + ) + if err != nil { return nil, nil, gtserror.Newf("error populating mentions for status %s: %w", uri, err) } - // Ensure the status' poll remains consistent, else reset the poll. - if err := d.fetchStatusPoll(ctx, status, latestStatus); err != nil { - return nil, nil, gtserror.Newf("error populating poll for status %s: %w", uri, err) + // Ensure status in a thread is connected. + threadChanged, err := d.threadStatus(ctx, + status, + latestStatus, + ) + if err != nil { + return nil, nil, gtserror.Newf("error handling threading for status %s: %w", uri, err) } - // Now that we know who this status replies to (handled by ASStatusToStatus) - // and who it mentions, we can add a ThreadID to it if necessary. - if err := d.threadStatus(ctx, latestStatus); err != nil { - return nil, nil, gtserror.Newf("error checking / creating threadID for status %s: %w", uri, err) - } - - // Ensure the status' tags are populated, (changes are expected / okay). - if err := d.fetchStatusTags(ctx, status, latestStatus); err != nil { + // Populate tags associated with status, passing + // in existing status to reuse old where possible. + tagsChanged, err := d.fetchStatusTags(ctx, + status, + latestStatus, + ) + if err != nil { return nil, nil, gtserror.Newf("error populating tags for status %s: %w", uri, err) } - // Ensure the status' media attachments are populated, passing in existing to check for changes. - if err := d.fetchStatusAttachments(ctx, requestUser, status, latestStatus); err != nil { + // Populate media attachments associated with status, + // passing in existing status to reuse old where possible + // (especially important here to reduce need to dereference). + mediaChanged, err := d.fetchStatusAttachments(ctx, + requestUser, + status, + latestStatus, + ) + if err != nil { return nil, nil, gtserror.Newf("error populating attachments for status %s: %w", uri, err) } - // Ensure the status' emoji attachments are populated, passing in existing to check for changes. - if err := d.fetchStatusEmojis(ctx, status, latestStatus); err != nil { + // Populate emoji associated with status, passing + // in existing status to reuse old where possible + // (especially important here to reduce need to dereference). + emojiChanged, err := d.fetchStatusEmojis(ctx, + status, + latestStatus, + ) + if err != nil { return nil, nil, gtserror.Newf("error populating emojis for status %s: %w", uri, err) } if isNew { - // This is new, put the status in the database. - err := d.state.DB.PutStatus(ctx, latestStatus) - if err != nil { - return nil, nil, gtserror.Newf("error putting in database: %w", err) + // Simplest case, insert this new status into the database. + if err := d.state.DB.PutStatus(ctx, latestStatus); err != nil { + return nil, nil, gtserror.Newf("error inserting new status %s: %w", uri, err) } } else { - // This is an existing status, update the model in the database. - if err := d.state.DB.UpdateStatus(ctx, latestStatus); err != nil { - return nil, nil, gtserror.Newf("error updating database: %w", err) + // Check for and handle any edits to status, inserting + // historical edit if necessary. Also determines status + // columns that need updating in below query. + cols, err := d.handleStatusEdit(ctx, + status, + latestStatus, + pollChanged, + mentionsChanged, + threadChanged, + tagsChanged, + mediaChanged, + emojiChanged, + ) + if err != nil { + return nil, nil, gtserror.Newf("error handling edit for status %s: %w", uri, err) + } + + // With returned changed columns, now update the existing status entry. + if err := d.state.DB.UpdateStatus(ctx, latestStatus, cols...); err != nil { + return nil, nil, gtserror.Newf("error updating existing status %s: %w", uri, err) } } return latestStatus, statusable, nil } +// fetchStatusMentions populates the mentions on 'status', creating +// new where needed, or using unchanged mentions from 'existing' status. func (d *Dereferencer) fetchStatusMentions( ctx context.Context, requestUser string, existing *gtsmodel.Status, status *gtsmodel.Status, -) error { +) ( + changed bool, + err error, +) { + // Allocate new slice to take the yet-to-be created mention IDs. status.MentionIDs = make([]string, len(status.Mentions)) @@ -610,7 +656,6 @@ func (d *Dereferencer) fetchStatusMentions( var ( mention = status.Mentions[i] alreadyExists bool - err error ) // Search existing status for a mention already stored, @@ -633,19 +678,16 @@ func (d *Dereferencer) fetchStatusMentions( continue } + // Mark status as + // having changed. + changed = true + // This mention didn't exist yet. - // Generate new ID according to status creation. - // TODO: update this to use "edited_at" when we add - // support for edited status revision history. - mention.ID, err = id.NewULIDFromTime(status.CreatedAt) - if err != nil { - log.Errorf(ctx, "invalid created at date (falling back to 'now'): %v", err) - mention.ID = id.NewULID() // just use "now" - } + // Generate new ID according to latest update. + mention.ID = id.NewULIDFromTime(status.UpdatedAt) // Set known further mention details. - mention.CreatedAt = status.CreatedAt - mention.UpdatedAt = status.UpdatedAt + mention.CreatedAt = status.UpdatedAt mention.OriginAccount = status.Account mention.OriginAccountID = status.AccountID mention.OriginAccountURI = status.AccountURI @@ -657,7 +699,7 @@ func (d *Dereferencer) fetchStatusMentions( // Place the new mention into the database. if err := d.state.DB.PutMention(ctx, mention); err != nil { - return gtserror.Newf("error putting mention in database: %w", err) + return changed, gtserror.Newf("error putting mention in database: %w", err) } // Set the *new* mention and ID. @@ -678,17 +720,42 @@ func (d *Dereferencer) fetchStatusMentions( i++ } - return nil + return changed, nil } -func (d *Dereferencer) threadStatus(ctx context.Context, status *gtsmodel.Status) error { - if status.InReplyTo != nil { - if parentThreadID := status.InReplyTo.ThreadID; parentThreadID != "" { - // Simplest case: parent status - // is threaded, so inherit threadID. - status.ThreadID = parentThreadID - return nil +// threadStatus ensures that given status is threaded correctly +// where necessary. that is it will inherit a thread ID from the +// existing copy if it is threaded correctly, else it will inherit +// a thread ID from a parent with existing thread, else it will +// generate a new thread ID if status mentions a local account. +func (d *Dereferencer) threadStatus( + ctx context.Context, + existing *gtsmodel.Status, + status *gtsmodel.Status, +) ( + changed bool, + err error, +) { + + // Check for existing status + // that is already threaded. + if existing.ThreadID != "" { + + // Existing is threaded correctly. + if existing.InReplyTo == nil || + existing.InReplyTo.ThreadID == existing.ThreadID { + status.ThreadID = existing.ThreadID + return false, nil } + + // TODO: delete incorrect thread + } + + // Check for existing parent to inherit threading from. + if inReplyTo := status.InReplyTo; inReplyTo != nil && + inReplyTo.ThreadID != "" { + status.ThreadID = inReplyTo.ThreadID + return true, nil } // Parent wasn't threaded. If this @@ -711,7 +778,7 @@ func (d *Dereferencer) threadStatus(ctx context.Context, status *gtsmodel.Status // Status doesn't mention a // local account, so we don't // need to thread it. - return nil + return false, nil } // Status mentions a local account. @@ -719,24 +786,30 @@ func (d *Dereferencer) threadStatus(ctx context.Context, status *gtsmodel.Status // it to the status. threadID := id.NewULID() - if err := d.state.DB.PutThread( - ctx, - >smodel.Thread{ - ID: threadID, - }, + // Insert new thread model into db. + if err := d.state.DB.PutThread(ctx, + >smodel.Thread{ID: threadID}, ); err != nil { - return gtserror.Newf("error inserting new thread in db: %w", err) + return false, gtserror.Newf("error inserting new thread in db: %w", err) } + // Set thread on latest status. status.ThreadID = threadID - return nil + return true, nil } +// fetchStatusTags populates the tags on 'status', fetching existing +// from the database and creating new where needed. 'existing' is used +// to fetch tags that have not changed since previous stored status. func (d *Dereferencer) fetchStatusTags( ctx context.Context, existing *gtsmodel.Status, status *gtsmodel.Status, -) error { +) ( + changed bool, + err error, +) { + // Allocate new slice to take the yet-to-be determined tag IDs. status.TagIDs = make([]string, len(status.Tags)) @@ -751,10 +824,14 @@ func (d *Dereferencer) fetchStatusTags( continue } + // Mark status as + // having changed. + changed = true + // Look for existing tag with name in the database. existing, err := d.state.DB.GetTagByName(ctx, tag.Name) if err != nil && !errors.Is(err, db.ErrNoEntries) { - return gtserror.Newf("db error getting tag %s: %w", tag.Name, err) + return changed, gtserror.Newf("db error getting tag %s: %w", tag.Name, err) } else if existing != nil { status.Tags[i] = existing status.TagIDs[i] = existing.ID @@ -788,106 +865,21 @@ func (d *Dereferencer) fetchStatusTags( i++ } - return nil -} - -func (d *Dereferencer) fetchStatusPoll( - ctx context.Context, - existing *gtsmodel.Status, - status *gtsmodel.Status, -) error { - var ( - // insertStatusPoll generates ID and inserts the poll attached to status into the database. - insertStatusPoll = func(ctx context.Context, status *gtsmodel.Status) error { - var err error - - // Generate new ID for poll from the status CreatedAt. - // TODO: update this to use "edited_at" when we add - // support for edited status revision history. - status.Poll.ID, err = id.NewULIDFromTime(status.CreatedAt) - if err != nil { - log.Errorf(ctx, "invalid created at date (falling back to 'now'): %v", err) - status.Poll.ID = id.NewULID() // just use "now" - } - - // Update the status<->poll links. - status.PollID = status.Poll.ID - status.Poll.StatusID = status.ID - status.Poll.Status = status - - // Insert this latest poll into the database. - err = d.state.DB.PutPoll(ctx, status.Poll) - if err != nil { - return gtserror.Newf("error putting in database: %w", err) - } - - return nil - } - - // deleteStatusPoll deletes the poll with ID, and all attached votes, from the database. - deleteStatusPoll = func(ctx context.Context, pollID string) error { - if err := d.state.DB.DeletePollByID(ctx, pollID); err != nil { - return gtserror.Newf("error deleting existing poll from database: %w", err) - } - return nil - } - ) - - switch { - case existing.Poll == nil && status.Poll == nil: - // no poll before or after, nothing to do. - return nil - - case existing.Poll == nil && status.Poll != nil: - // no previous poll, insert new poll! - return insertStatusPoll(ctx, status) - - case status.Poll == nil: - // existing poll has been deleted, remove this. - return deleteStatusPoll(ctx, existing.PollID) - - case pollChanged(existing.Poll, status.Poll): - // poll has changed since original, delete and reinsert new. - if err := deleteStatusPoll(ctx, existing.PollID); err != nil { - return err - } - return insertStatusPoll(ctx, status) - - case pollUpdated(existing.Poll, status.Poll): - // Since we last saw it, the poll has updated! - // Whether that be stats, or close time. - poll := existing.Poll - poll.Closing = pollJustClosed(existing.Poll, status.Poll) - poll.ClosedAt = status.Poll.ClosedAt - poll.Voters = status.Poll.Voters - poll.Votes = status.Poll.Votes - - // Update poll model in the database (specifically only the possible changed columns). - if err := d.state.DB.UpdatePoll(ctx, poll, "closed_at", "voters", "votes"); err != nil { - return gtserror.Newf("error updating poll: %w", err) - } - - // Update poll on status. - status.PollID = poll.ID - status.Poll = poll - return nil - - default: - // latest and existing - // polls are up to date. - poll := existing.Poll - status.PollID = poll.ID - status.Poll = poll - return nil - } + return changed, nil } +// fetchStatusAttachments populates the attachments on 'status', creating new database +// entries where needed and dereferencing it, or using unchanged from 'existing' status. func (d *Dereferencer) fetchStatusAttachments( ctx context.Context, requestUser string, existing *gtsmodel.Status, status *gtsmodel.Status, -) error { +) ( + changed bool, + err error, +) { + // Allocate new slice to take the yet-to-be fetched attachment IDs. status.AttachmentIDs = make([]string, len(status.Attachments)) @@ -897,9 +889,26 @@ func (d *Dereferencer) fetchStatusAttachments( // Look for existing media attachment with remote URL first. existing, ok := existing.GetAttachmentByRemoteURL(placeholder.RemoteURL) if ok && existing.ID != "" { + var info media.AdditionalMediaInfo - // Ensure the existing media attachment is up-to-date and cached. - existing, err := d.updateAttachment(ctx, requestUser, existing, placeholder) + // Look for any difference in stored media description. + diff := (existing.Description != placeholder.Description) + if diff { + info.Description = &placeholder.Description + } + + // If description changed, + // we mark media as changed. + changed = changed || diff + + // Store any attachment updates and + // ensure media is locally cached. + existing, err := d.RefreshMedia(ctx, + requestUser, + existing, + info, + diff, + ) if err != nil { log.Errorf(ctx, "error updating existing attachment: %v", err) @@ -915,9 +924,12 @@ func (d *Dereferencer) fetchStatusAttachments( continue } + // Mark status as + // having changed. + changed = true + // Load this new media attachment. - attachment, err := d.GetMedia( - ctx, + attachment, err := d.GetMedia(ctx, requestUser, status.AccountID, placeholder.RemoteURL, @@ -955,28 +967,34 @@ func (d *Dereferencer) fetchStatusAttachments( i++ } - return nil + return changed, nil } +// fetchStatusEmojis populates the emojis on 'status', creating new database entries +// where needed and dereferencing it, or using unchanged from 'existing' status. func (d *Dereferencer) fetchStatusEmojis( ctx context.Context, existing *gtsmodel.Status, status *gtsmodel.Status, -) error { +) ( + changed bool, + err error, +) { + // Fetch the updated emojis for our status. emojis, changed, err := d.fetchEmojis(ctx, existing.Emojis, status.Emojis, ) if err != nil { - return gtserror.Newf("error fetching emojis: %w", err) + return changed, gtserror.Newf("error fetching emojis: %w", err) } if !changed { // Use existing status emoji objects. status.EmojiIDs = existing.EmojiIDs status.Emojis = existing.Emojis - return nil + return false, nil } // Set latest emojis. @@ -988,9 +1006,254 @@ func (d *Dereferencer) fetchStatusEmojis( status.EmojiIDs[i] = emoji.ID } + return true, nil +} + +// handleStatusPoll handles both inserting of new status poll or the +// update of an existing poll. this handles the case of simple vote +// count updates (without being classified as a change of the poll +// itself), as well as full poll changes that delete existing instance. +func (d *Dereferencer) handleStatusPoll( + ctx context.Context, + existing *gtsmodel.Status, + status *gtsmodel.Status, +) ( + changed bool, + err error, +) { + switch { + case existing.Poll == nil && status.Poll == nil: + // no poll before or after, nothing to do. + return false, nil + + case existing.Poll == nil && status.Poll != nil: + // no previous poll, insert new status poll! + return true, d.insertStatusPoll(ctx, status) + + case status.Poll == nil: + // existing status poll has been deleted, remove this from the database. + if err = d.state.DB.DeletePollByID(ctx, existing.Poll.ID); err != nil { + err = gtserror.Newf("error deleting poll from database: %w", err) + } + return true, err + + case pollChanged(existing.Poll, status.Poll): + // existing status poll has been changed, remove this from the database. + if err = d.state.DB.DeletePollByID(ctx, existing.Poll.ID); err != nil { + return true, gtserror.Newf("error deleting poll from database: %w", err) + } + + // insert latest poll version into database. + return true, d.insertStatusPoll(ctx, status) + + case pollStateUpdated(existing.Poll, status.Poll): + // Since we last saw it, the poll has updated! + // Whether that be stats, or close time. + poll := existing.Poll + poll.Closing = pollJustClosed(existing.Poll, status.Poll) + poll.ClosedAt = status.Poll.ClosedAt + poll.Voters = status.Poll.Voters + poll.Votes = status.Poll.Votes + + // Update poll model in the database (specifically only the possible changed columns). + if err = d.state.DB.UpdatePoll(ctx, poll, "closed_at", "voters", "votes"); err != nil { + return false, gtserror.Newf("error updating poll: %w", err) + } + + // Update poll on status. + status.PollID = poll.ID + status.Poll = poll + return false, nil + + default: + // latest and existing + // polls are up to date. + poll := existing.Poll + status.PollID = poll.ID + status.Poll = poll + return false, nil + } +} + +// insertStatusPoll inserts an assumed new poll attached to status into the database, this +// also handles generating new ID for the poll and setting necessary fields on the status. +func (d *Dereferencer) insertStatusPoll(ctx context.Context, status *gtsmodel.Status) error { + var err error + + // Generate new ID for poll from latest updated time. + status.Poll.ID = id.NewULIDFromTime(status.UpdatedAt) + + // Update the status<->poll links. + status.PollID = status.Poll.ID + status.Poll.StatusID = status.ID + status.Poll.Status = status + + // Insert this latest poll into the database. + err = d.state.DB.PutPoll(ctx, status.Poll) + if err != nil { + return gtserror.Newf("error putting poll in database: %w", err) + } + return nil } +// handleStatusEdit compiles a list of changed status table columns between +// existing and latest status model, and where necessary inserts a historic +// edit of the status into the database to store its previous state. the +// returned slice is a list of columns requiring updating in the database. +func (d *Dereferencer) handleStatusEdit( + ctx context.Context, + existing *gtsmodel.Status, + status *gtsmodel.Status, + pollChanged bool, + mentionsChanged bool, + threadChanged bool, + tagsChanged bool, + mediaChanged bool, + emojiChanged bool, +) ( + cols []string, + err error, +) { + var edited bool + + // Preallocate max slice length. + cols = make([]string, 0, 13) + + // Always update `fetched_at`. + cols = append(cols, "fetched_at") + + // Check for edited status content. + if existing.Content != status.Content { + cols = append(cols, "content") + edited = true + } + + // Check for edited status content warning. + if existing.ContentWarning != status.ContentWarning { + cols = append(cols, "content_warning") + edited = true + } + + // Check for edited status sensitive flag. + if *existing.Sensitive != *status.Sensitive { + cols = append(cols, "sensitive") + edited = true + } + + // Check for edited status language tag. + if existing.Language != status.Language { + cols = append(cols, "language") + edited = true + } + + if pollChanged { + // Attached poll was changed. + cols = append(cols, "poll_id") + edited = true + } + + if mentionsChanged { + cols = append(cols, "mentions") // i.e. MentionIDs + + // Mentions changed doesn't necessarily + // indicate an edit, it may just not have + // been previously populated properly. + } + + if threadChanged { + cols = append(cols, "thread_id") + + // Thread changed doesn't necessarily + // indicate an edit, it may just now + // actually be included in a thread. + } + + if tagsChanged { + cols = append(cols, "tags") // i.e. TagIDs + + // Tags changed doesn't necessarily + // indicate an edit, it may just not have + // been previously populated properly. + } + + if mediaChanged { + // Attached media was changed. + cols = append(cols, "attachments") // i.e. AttachmentIDs + edited = true + } + + if emojiChanged { + // Attached emojis changed. + cols = append(cols, "emojis") // i.e. EmojiIDs + + // Emojis changed doesn't necessarily + // indicate an edit, it may just not have + // been previously populated properly. + } + + if edited { + // We prefer to use provided 'upated_at', but ensure + // it fits chronologically with creation / last update. + if !status.UpdatedAt.After(status.CreatedAt) || + !status.UpdatedAt.After(existing.UpdatedAt) { + + // Else fallback to now as update time. + status.UpdatedAt = status.FetchedAt + } + + // Status has been editted since last + // we saw it, take snapshot of existing. + var edit gtsmodel.StatusEdit + edit.ID = id.NewULIDFromTime(status.UpdatedAt) + edit.Content = existing.Content + edit.ContentWarning = existing.ContentWarning + edit.Text = existing.Text + edit.Language = existing.Language + edit.Sensitive = existing.Sensitive + edit.StatusID = status.ID + + // Copy existing attachments and descriptions. + edit.AttachmentIDs = existing.AttachmentIDs + edit.Attachments = existing.Attachments + if l := len(existing.Attachments); l > 0 { + edit.AttachmentDescriptions = make([]string, l) + for i, attach := range existing.Attachments { + edit.AttachmentDescriptions[i] = attach.Description + } + } + + // Edit creation is last update time. + edit.CreatedAt = existing.UpdatedAt + + if existing.Poll != nil { + // Poll only set if existing contained them. + edit.PollOptions = existing.Poll.Options + + if !*existing.Poll.HideCounts || pollChanged { + // If the counts are allowed to be + // shown, or poll has changed, then + // include poll vote counts in edit. + edit.PollVotes = existing.Poll.Votes + } + } + + // Insert this new edit of existing status into database. + if err := d.state.DB.PutStatusEdit(ctx, &edit); err != nil { + return nil, gtserror.Newf("error putting edit in database: %w", err) + } + + // Add edit to list of edits on the status. + status.EditIDs = append(status.EditIDs, edit.ID) + status.Edits = append(status.Edits, &edit) + + // Add updated_at and edits to list of cols. + cols = append(cols, "updated_at", "edits") + } + + return cols, nil +} + // getPopulatedMention tries to populate the given // mention with the correct TargetAccount and (if not // yet set) TargetAccountURI, returning the populated diff --git a/internal/federation/dereferencing/status_permitted.go b/internal/federation/dereferencing/status_permitted.go index 9ad425c2f..5d05c5de4 100644 --- a/internal/federation/dereferencing/status_permitted.go +++ b/internal/federation/dereferencing/status_permitted.go @@ -62,6 +62,7 @@ func (d *Dereferencer) isPermittedStatus( requestUser string, existing *gtsmodel.Status, status *gtsmodel.Status, + isNew bool, ) ( permitted bool, // is permitted? err error, @@ -98,7 +99,7 @@ func (d *Dereferencer) isPermittedStatus( permitted = true } - if !permitted && existing != nil { + if !permitted && !isNew { log.Infof(ctx, "deleting unpermitted: %s", existing.URI) // Delete existing status from database as it's no longer permitted. @@ -110,11 +111,13 @@ func (d *Dereferencer) isPermittedStatus( return } +// isPermittedReply ... func (d *Dereferencer) isPermittedReply( ctx context.Context, requestUser string, reply *gtsmodel.Status, ) (bool, error) { + var ( replyURI = reply.URI // Definitely set. inReplyToURI = reply.InReplyToURI // Definitely set. @@ -149,8 +152,7 @@ func (d *Dereferencer) isPermittedReply( // If this status's parent was rejected, // implicitly this reply should be too; // there's nothing more to check here. - return false, d.unpermittedByParent( - ctx, + return false, d.unpermittedByParent(ctx, reply, thisReq, parentReq, @@ -164,6 +166,7 @@ func (d *Dereferencer) isPermittedReply( // be approved, then we should just reject it // again, as nothing's changed since last time. if thisRejected && acceptIRI == "" { + // Nothing changed, // still rejected. return false, nil @@ -174,6 +177,7 @@ func (d *Dereferencer) isPermittedReply( // to be approved. Continue permission checks. if inReplyTo == nil { + // If we didn't have the replied-to status // in our database (yet), we can't check // right now if this reply is permitted. diff --git a/internal/federation/dereferencing/status_test.go b/internal/federation/dereferencing/status_test.go index 3b2c2bff2..4b3bd6d67 100644 --- a/internal/federation/dereferencing/status_test.go +++ b/internal/federation/dereferencing/status_test.go @@ -21,14 +21,21 @@ import ( "context" "fmt" "testing" + "time" "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/activity/streams" "github.com/superseriousbusiness/gotosocial/internal/ap" "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/util" "github.com/superseriousbusiness/gotosocial/testrig" ) +// instantFreshness is the shortest possible freshness window. +var instantFreshness = util.Ptr(dereferencing.FreshnessWindow(0)) + type StatusTestSuite struct { DereferencerStandardTestSuite } @@ -229,6 +236,219 @@ func (suite *StatusTestSuite) TestDereferenceStatusWithNonMatchingURI() { suite.Nil(fetchedStatus) } +func (suite *StatusTestSuite) TestDereferencerRefreshStatusUpdated() { + // Create a new context for this test. + ctx, cncl := context.WithCancel(context.Background()) + defer cncl() + + // The local account we will be fetching statuses as. + fetchingAccount := suite.testAccounts["local_account_1"] + + // The test status in question that we will be dereferencing from "remote". + testURIStr := "https://unknown-instance.com/users/brand_new_person/statuses/01FE4NTHKWW7THT67EF10EB839" + testURI := testrig.URLMustParse(testURIStr) + testStatusable := suite.client.TestRemoteStatuses[testURIStr] + + // Fetch the remote status first to load it into instance. + testStatus, statusable, err := suite.dereferencer.GetStatusByURI(ctx, + fetchingAccount.Username, + testURI, + ) + suite.NotNil(statusable) + suite.NoError(err) + + // Run through multiple possible edits. + for _, testCase := range []struct { + editedContent string + editedContentWarning string + editedLanguage string + editedSensitive bool + editedAttachmentIDs []string + editedPollOptions []string + editedPollVotes []int + editedAt time.Time + }{ + { + editedContent: "updated status content!", + editedContentWarning: "CW: edited status content", + editedLanguage: testStatus.Language, // no change + editedSensitive: *testStatus.Sensitive, // no change + editedAttachmentIDs: testStatus.AttachmentIDs, // no change + editedPollOptions: getPollOptions(testStatus), // no change + editedPollVotes: getPollVotes(testStatus), // no change + editedAt: time.Now(), + }, + } { + // Take a snapshot of current + // state of the test status. + testStatus = copyStatus(testStatus) + + // Edit the "remote" statusable obj. + suite.editStatusable(testStatusable, + testCase.editedContent, + testCase.editedContentWarning, + testCase.editedLanguage, + testCase.editedSensitive, + testCase.editedAttachmentIDs, + testCase.editedPollOptions, + testCase.editedPollVotes, + testCase.editedAt, + ) + + // Refresh with a given statusable to updated to edited copy. + latest, statusable, err := suite.dereferencer.RefreshStatus(ctx, + fetchingAccount.Username, + testStatus, + nil, // NOTE: can provide testStatusable here to test as being received (not deref'd) + instantFreshness, + ) + suite.NotNil(statusable) + suite.NoError(err) + + // verify updated status details. + suite.verifyEditedStatusUpdate( + + // the original status + // before any changes. + testStatus, + + // latest status + // being tested. + latest, + + // expected current state. + >smodel.StatusEdit{ + Content: testCase.editedContent, + ContentWarning: testCase.editedContentWarning, + Language: testCase.editedLanguage, + Sensitive: &testCase.editedSensitive, + AttachmentIDs: testCase.editedAttachmentIDs, + PollOptions: testCase.editedPollOptions, + PollVotes: testCase.editedPollVotes, + // createdAt never changes + }, + + // expected historic edit. + >smodel.StatusEdit{ + Content: testStatus.Content, + ContentWarning: testStatus.ContentWarning, + Language: testStatus.Language, + Sensitive: testStatus.Sensitive, + AttachmentIDs: testStatus.AttachmentIDs, + PollOptions: getPollOptions(testStatus), + PollVotes: getPollVotes(testStatus), + CreatedAt: testStatus.UpdatedAt, + }, + ) + } +} + +// editStatusable updates the given statusable attributes. +// note that this acts on the original object, no copying. +func (suite *StatusTestSuite) editStatusable( + statusable ap.Statusable, + content string, + contentWarning string, + language string, + sensitive bool, + attachmentIDs []string, // TODO: this will require some thinking as to how ... + pollOptions []string, // TODO: this will require changing statusable type to question + pollVotes []int, // TODO: this will require changing statusable type to question + editedAt time.Time, +) { + // simply reset all mentions / emojis / tags + statusable.SetActivityStreamsTag(nil) + + // Update the statusable content property + language (if set). + contentProp := streams.NewActivityStreamsContentProperty() + statusable.SetActivityStreamsContent(contentProp) + contentProp.AppendXMLSchemaString(content) + if language != "" { + contentProp.AppendRDFLangString(map[string]string{ + language: content, + }) + } + + // Update the statusable content-warning property. + summaryProp := streams.NewActivityStreamsSummaryProperty() + statusable.SetActivityStreamsSummary(summaryProp) + summaryProp.AppendXMLSchemaString(contentWarning) + + // Update the statusable sensitive property. + sensitiveProp := streams.NewActivityStreamsSensitiveProperty() + statusable.SetActivityStreamsSensitive(sensitiveProp) + sensitiveProp.AppendXMLSchemaBoolean(sensitive) + + // Update the statusable updated property. + ap.SetUpdated(statusable, editedAt) +} + +// verifyEditedStatusUpdate verifies that a given status has +// the expected number of historic edits, the 'current' status +// attributes (encapsulated as an edit for minimized no. args), +// and the last given 'historic' status edit attributes. +func (suite *StatusTestSuite) verifyEditedStatusUpdate( + testStatus *gtsmodel.Status, // the original model + status *gtsmodel.Status, // the status to check + current *gtsmodel.StatusEdit, // expected current state + historic *gtsmodel.StatusEdit, // historic edit we expect to have +) { + // don't use this func + // name in error msgs. + suite.T().Helper() + + // Check we have expected number of edits. + previousEdits := len(testStatus.Edits) + suite.Len(status.Edits, previousEdits+1) + suite.Len(status.EditIDs, previousEdits+1) + + // Check current state of status. + suite.Equal(current.Content, status.Content) + suite.Equal(current.ContentWarning, status.ContentWarning) + suite.Equal(current.Language, status.Language) + suite.Equal(*current.Sensitive, *status.Sensitive) + suite.Equal(current.AttachmentIDs, status.AttachmentIDs) + suite.Equal(current.PollOptions, getPollOptions(status)) + suite.Equal(current.PollVotes, getPollVotes(status)) + + // Check the latest historic edit matches expected. + latestEdit := status.Edits[len(status.Edits)-1] + suite.Equal(historic.Content, latestEdit.Content) + suite.Equal(historic.ContentWarning, latestEdit.ContentWarning) + suite.Equal(historic.Language, latestEdit.Language) + suite.Equal(*historic.Sensitive, *latestEdit.Sensitive) + suite.Equal(historic.AttachmentIDs, latestEdit.AttachmentIDs) + suite.Equal(historic.PollOptions, latestEdit.PollOptions) + suite.Equal(historic.PollVotes, latestEdit.PollVotes) + suite.Equal(historic.CreatedAt, latestEdit.CreatedAt) + + // The status creation date should never change. + suite.Equal(testStatus.CreatedAt, status.CreatedAt) +} + func TestStatusTestSuite(t *testing.T) { suite.Run(t, new(StatusTestSuite)) } + +// copyStatus returns a copy of the given status model (not including sub-structs). +func copyStatus(status *gtsmodel.Status) *gtsmodel.Status { + copy := new(gtsmodel.Status) + *copy = *status + return copy +} + +// getPollOptions extracts poll option strings from status (if poll is set). +func getPollOptions(status *gtsmodel.Status) []string { + if status.Poll != nil { + return status.Poll.Options + } + return nil +} + +// getPollVotes extracts poll vote counts from status (if poll is set). +func getPollVotes(status *gtsmodel.Status) []int { + if status.Poll != nil { + return status.Poll.Votes + } + return nil +} diff --git a/internal/federation/dereferencing/util.go b/internal/federation/dereferencing/util.go index 297e90adc..208117660 100644 --- a/internal/federation/dereferencing/util.go +++ b/internal/federation/dereferencing/util.go @@ -52,15 +52,15 @@ func emojiChanged(existing, latest *gtsmodel.Emoji) bool { // pollChanged returns whether a poll has changed in way that // indicates that this should be an entirely new poll. i.e. if -// the available options have changed, or the expiry has increased. +// the available options have changed, or the expiry has changed. func pollChanged(existing, latest *gtsmodel.Poll) bool { return !slices.Equal(existing.Options, latest.Options) || !existing.ExpiresAt.Equal(latest.ExpiresAt) } -// pollUpdated returns whether a poll has updated, i.e. if the +// pollStateUpdated returns whether a poll has updated, i.e. if // vote counts have changed, or if it has expired / been closed. -func pollUpdated(existing, latest *gtsmodel.Poll) bool { +func pollStateUpdated(existing, latest *gtsmodel.Poll) bool { return *existing.Voters != *latest.Voters || !slices.Equal(existing.Votes, latest.Votes) || !existing.ClosedAt.Equal(latest.ClosedAt) diff --git a/internal/federation/federatingdb/announce_test.go b/internal/federation/federatingdb/announce_test.go index 264279253..5bb2fc877 100644 --- a/internal/federation/federatingdb/announce_test.go +++ b/internal/federation/federatingdb/announce_test.go @@ -79,7 +79,7 @@ func (suite *AnnounceTestSuite) TestAnnounceTwice() { // Insert the boost-of status into the // DB cache to emulate processor handling - boost.ID, _ = id.NewULIDFromTime(boost.CreatedAt) + boost.ID = id.NewULIDFromTime(boost.CreatedAt) suite.state.Caches.DB.Status.Put(boost) // only the URI will be set for the boosted status diff --git a/internal/gtsmodel/mediaattachment.go b/internal/gtsmodel/mediaattachment.go index f4bfb5929..5cf6f60a6 100644 --- a/internal/gtsmodel/mediaattachment.go +++ b/internal/gtsmodel/mediaattachment.go @@ -26,7 +26,6 @@ import ( type MediaAttachment struct { ID string `bun:"type:CHAR(26),pk,nullzero,notnull,unique"` // id of this item in the database CreatedAt time.Time `bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // when was item created - UpdatedAt time.Time `bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // when was item last updated StatusID string `bun:"type:CHAR(26),nullzero"` // ID of the status to which this is attached URL string `bun:",nullzero"` // Where can the attachment be retrieved on *this* server RemoteURL string `bun:",nullzero"` // Where can the attachment be retrieved on a remote server (empty for local media) diff --git a/internal/gtsmodel/mention.go b/internal/gtsmodel/mention.go index 24e83f904..180193f0f 100644 --- a/internal/gtsmodel/mention.go +++ b/internal/gtsmodel/mention.go @@ -26,7 +26,6 @@ import ( type Mention struct { ID string `bun:"type:CHAR(26),pk,nullzero,notnull,unique"` // id of this item in the database CreatedAt time.Time `bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // when was item created - UpdatedAt time.Time `bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // when was item last updated StatusID string `bun:"type:CHAR(26),nullzero,notnull"` // ID of the status this mention originates from Status *Status `bun:"rel:belongs-to"` // status referred to by statusID OriginAccountID string `bun:"type:CHAR(26),nullzero,notnull"` // ID of the mention creator account diff --git a/internal/gtsmodel/status.go b/internal/gtsmodel/status.go index f8bd068ab..4c65d8a88 100644 --- a/internal/gtsmodel/status.go +++ b/internal/gtsmodel/status.go @@ -20,6 +20,8 @@ package gtsmodel import ( "slices" "time" + + "github.com/superseriousbusiness/gotosocial/internal/util/xslices" ) // Status represents a user-created 'post' or 'status' in the database, either remote or local @@ -55,6 +57,8 @@ type Status struct { BoostOf *Status `bun:"-"` // status that corresponds to boostOfID BoostOfAccount *Account `bun:"rel:belongs-to"` // account that corresponds to boostOfAccountID ThreadID string `bun:"type:CHAR(26),nullzero"` // id of the thread to which this status belongs; only set for remote statuses if a local account is involved at some point in the thread, otherwise null + EditIDs []string `bun:"edits,array"` // + Edits []*StatusEdit `bun:"-"` // PollID string `bun:"type:CHAR(26),nullzero"` // Poll *Poll `bun:"-"` // ContentWarning string `bun:",nullzero"` // cw string for this status @@ -92,7 +96,8 @@ func (s *Status) GetBoostOfAccountID() string { return s.BoostOfAccountID } -// AttachmentsPopulated returns whether media attachments are populated according to current AttachmentIDs. +// AttachmentsPopulated returns whether media attachments +// are populated according to current AttachmentIDs. func (s *Status) AttachmentsPopulated() bool { if len(s.AttachmentIDs) != len(s.Attachments) { // this is the quickest indicator. @@ -106,7 +111,8 @@ func (s *Status) AttachmentsPopulated() bool { return true } -// TagsPopulated returns whether tags are populated according to current TagIDs. +// TagsPopulated returns whether tags are +// populated according to current TagIDs. func (s *Status) TagsPopulated() bool { if len(s.TagIDs) != len(s.Tags) { // this is the quickest indicator. @@ -120,7 +126,8 @@ func (s *Status) TagsPopulated() bool { return true } -// MentionsPopulated returns whether mentions are populated according to current MentionIDs. +// MentionsPopulated returns whether mentions are +// populated according to current MentionIDs. func (s *Status) MentionsPopulated() bool { if len(s.MentionIDs) != len(s.Mentions) { // this is the quickest indicator. @@ -134,7 +141,8 @@ func (s *Status) MentionsPopulated() bool { return true } -// EmojisPopulated returns whether emojis are populated according to current EmojiIDs. +// EmojisPopulated returns whether emojis are +// populated according to current EmojiIDs. func (s *Status) EmojisPopulated() bool { if len(s.EmojiIDs) != len(s.Emojis) { // this is the quickest indicator. @@ -148,6 +156,21 @@ func (s *Status) EmojisPopulated() bool { return true } +// EditsPopulated returns whether edits are +// populated according to current EditIDs. +func (s *Status) EditsPopulated() bool { + if len(s.EditIDs) != len(s.Edits) { + // this is quickest indicator. + return false + } + for i, id := range s.EditIDs { + if s.Edits[i].ID != id { + return false + } + } + return true +} + // EmojissUpToDate returns whether status emoji attachments of receiving status are up-to-date // according to emoji attachments of the passed status, by comparing their emoji URIs. We don't // use IDs as this is used to determine whether there are new emojis to fetch. @@ -247,6 +270,35 @@ func (s *Status) IsLocalOnly() bool { return s.Federated == nil || !*s.Federated } +// AllAttachmentIDs gathers ALL media attachment IDs from both the +// receiving Status{}, and any historical Status{}.Edits. Note that +// this function will panic if Status{}.Edits is not populated. +func (s *Status) AllAttachmentIDs() []string { + var total int + + if len(s.EditIDs) != len(s.Edits) { + panic("status edits not populated") + } + + // Get count of attachment IDs. + total += len(s.Attachments) + for _, edit := range s.Edits { + total += len(edit.AttachmentIDs) + } + + // Start gathering of all IDs with *current* attachment IDs. + attachmentIDs := make([]string, len(s.AttachmentIDs), total) + copy(attachmentIDs, s.AttachmentIDs) + + // Append IDs of historical edits. + for _, edit := range s.Edits { + attachmentIDs = append(attachmentIDs, edit.AttachmentIDs...) + } + + // Deduplicate these IDs in case of shared media. + return xslices.Deduplicate(attachmentIDs) +} + // StatusToTag is an intermediate struct to facilitate the many2many relationship between a status and one or more tags. type StatusToTag struct { StatusID string `bun:"type:CHAR(26),unique:statustag,nullzero,notnull"` diff --git a/internal/gtsmodel/statusedit.go b/internal/gtsmodel/statusedit.go new file mode 100644 index 000000000..199d47736 --- /dev/null +++ b/internal/gtsmodel/statusedit.go @@ -0,0 +1,62 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package gtsmodel + +import "time" + +// StatusEdit represents a **historical** view of a Status +// after a received edit. The Status itself will always +// contain the latest up-to-date information. +// +// Note that stored status edits may not exactly match that +// of the origin server, they are a best-effort by receiver +// to store version history. There is no AP history endpoint. +type StatusEdit struct { + ID string `bun:"type:CHAR(26),pk,nullzero,notnull,unique"` // ID of this item in the database. + Content string `bun:""` // Content of status at time of edit; likely html-formatted but not guaranteed. + ContentWarning string `bun:",nullzero"` // Content warning of status at time of edit. + Text string `bun:""` // Original status text, without formatting, at time of edit. + Language string `bun:",nullzero"` // Status language at time of edit. + Sensitive *bool `bun:",nullzero,notnull,default:false"` // Status sensitive flag at time of edit. + AttachmentIDs []string `bun:"attachments,array"` // Database IDs of media attachments associated with status at time of edit. + AttachmentDescriptions []string `bun:",array"` // Previous media descriptions of media attachments associated with status at time of edit. + Attachments []*MediaAttachment `bun:"-"` // Media attachments relating to .AttachmentIDs field (not always populated). + PollOptions []string `bun:",array"` // Poll options of status at time of edit, only set if status contains a poll. + PollVotes []int `bun:",array"` // Poll vote count at time of status edit, only set if poll votes were reset. + StatusID string `bun:"type:CHAR(26),nullzero,notnull"` // The originating status ID this is a historical edit of. + CreatedAt time.Time `bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // The creation time of this version of the status content (according to receiving server). + + // We don't bother having a *gtsmodel.Status model here + // as the StatusEdit is always just attached to a Status, + // so it doesn't need a self-reference back to it. +} + +// AttachmentsPopulated returns whether media attachments +// are populated according to current AttachmentIDs. +func (e *StatusEdit) AttachmentsPopulated() bool { + if len(e.AttachmentIDs) != len(e.Attachments) { + // this is the quickest indicator. + return false + } + for i, id := range e.AttachmentIDs { + if e.Attachments[i].ID != id { + return false + } + } + return true +} diff --git a/internal/id/ulid.go b/internal/id/ulid.go index 8de4cc4cc..8c0b1e94c 100644 --- a/internal/id/ulid.go +++ b/internal/id/ulid.go @@ -22,7 +22,9 @@ import ( "math/big" "time" + "codeberg.org/gruf/go-kv" "github.com/oklog/ulid" + "github.com/superseriousbusiness/gotosocial/internal/log" ) const ( @@ -45,13 +47,19 @@ func NewULID() string { return ulid.String() } -// NewULIDFromTime returns a new ULID string using the given time, or an error if something goes wrong. -func NewULIDFromTime(t time.Time) (string, error) { - newUlid, err := ulid.New(ulid.Timestamp(t), rand.Reader) - if err != nil { - return "", err +// NewULIDFromTime returns a new ULID string using +// given time, or from current time on any error. +func NewULIDFromTime(t time.Time) string { + ts := ulid.Timestamp(t) + if ts > ulid.MaxTime() { + log.WarnKVs(nil, kv.Fields{ + {K: "caller", V: log.Caller(2)}, + {K: "value", V: t}, + {K: "msg", V: "invalid ulid time"}, + }...) + ts = ulid.Now() } - return newUlid.String(), nil + return ulid.MustNew(ts, rand.Reader).String() } // NewRandomULID returns a new ULID string using a random time in an ~80 year range around the current datetime, or an error if something goes wrong. diff --git a/internal/media/manager.go b/internal/media/manager.go index 2807848bd..6aa13c17b 100644 --- a/internal/media/manager.go +++ b/internal/media/manager.go @@ -118,15 +118,11 @@ func (m *Manager) CreateMedia( Header: util.Ptr(false), Cached: util.Ptr(false), CreatedAt: now, - UpdatedAt: now, } // Check if we were provided additional info // to add to the attachment, and overwrite // some of the attachment fields if so. - if info.CreatedAt != nil { - attachment.CreatedAt = *info.CreatedAt - } if info.StatusID != nil { attachment.StatusID = *info.StatusID } @@ -372,9 +368,6 @@ func (m *Manager) createOrUpdateEmoji( if info.URI != nil { emoji.URI = *info.URI } - if info.CreatedAt != nil { - emoji.CreatedAt = *info.CreatedAt - } if info.Domain != nil { emoji.Domain = *info.Domain } diff --git a/internal/media/manager_test.go b/internal/media/manager_test.go index e175369f5..5b6882100 100644 --- a/internal/media/manager_test.go +++ b/internal/media/manager_test.go @@ -109,7 +109,6 @@ func (suite *ManagerTestSuite) TestEmojiProcessRefresh() { emojiToUpdate, data, media.AdditionalEmojiInfo{ - CreatedAt: &emojiToUpdate.CreatedAt, Domain: &emojiToUpdate.Domain, ImageRemoteURL: &newImageRemoteURL, }, diff --git a/internal/media/types.go b/internal/media/types.go index 9631a15bd..827752941 100644 --- a/internal/media/types.go +++ b/internal/media/types.go @@ -20,7 +20,6 @@ package media import ( "context" "io" - "time" ) type Size string @@ -44,10 +43,6 @@ const ( // should be added to attachment when processing a piece of media. type AdditionalMediaInfo struct { - // Time that this media was - // created; defaults to time.Now(). - CreatedAt *time.Time - // ID of the status to which this // media is attached; defaults to "". StatusID *string @@ -93,10 +88,6 @@ type AdditionalEmojiInfo struct { // this remote emoji. URI *string - // Time that this emoji was - // created; defaults to time.Now(). - CreatedAt *time.Time - // Domain the emoji originated from. Blank // for this instance's domain. Defaults to "". Domain *string diff --git a/internal/processing/account/rss_test.go b/internal/processing/account/rss_test.go index e4706d3b7..5606151c2 100644 --- a/internal/processing/account/rss_test.go +++ b/internal/processing/account/rss_test.go @@ -70,7 +70,7 @@ func (suite *GetRSSTestSuite) TestGetAccountRSSAdmin() { func (suite *GetRSSTestSuite) TestGetAccountRSSZork() { getFeed, lastModified, err := suite.accountProcessor.GetRSSFeedForUsername(context.Background(), "the_mighty_zork") suite.NoError(err) - suite.EqualValues(1704878640, lastModified.Unix()) + suite.EqualValues(1730451600, lastModified.Unix()) feed, err := getFeed() suite.NoError(err) @@ -79,13 +79,23 @@ func (suite *GetRSSTestSuite) TestGetAccountRSSZork() { Posts from @the_mighty_zork@localhost:8080 http://localhost:8080/@the_mighty_zork Posts from @the_mighty_zork@localhost:8080 - Wed, 10 Jan 2024 09:24:00 +0000 - Wed, 10 Jan 2024 09:24:00 +0000 + Fri, 01 Nov 2024 09:00:00 +0000 + Fri, 01 Nov 2024 09:00:00 +0000 http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/small/01F8MH58A357CV5K7R7TJMSH6S.webp Avatar for @the_mighty_zork@localhost:8080 http://localhost:8080/@the_mighty_zork + + edited status + http://localhost:8080/@the_mighty_zork/statuses/01JDPZC707CKDN8N4QVWM4Z1NR + @the_mighty_zork@localhost:8080 made a new post: "this is the latest revision of the status, with a content-warning" + this is the latest revision of the status, with a content-warning

]]> + @the_mighty_zork@localhost:8080 + http://localhost:8080/@the_mighty_zork/statuses/01JDPZC707CKDN8N4QVWM4Z1NR + Fri, 01 Nov 2024 09:00:00 +0000 + http://localhost:8080/@the_mighty_zork/feed.rss + HTML in post http://localhost:8080/@the_mighty_zork/statuses/01HH9KYNQPA416TNJ53NSATP40 diff --git a/internal/processing/media/getfile.go b/internal/processing/media/getfile.go index 6962601f2..11d8f7eb5 100644 --- a/internal/processing/media/getfile.go +++ b/internal/processing/media/getfile.go @@ -177,9 +177,7 @@ func (p *Processor) getAttachmentContent( } // Start preparing API content model. - apiContent := &apimodel.Content{ - ContentUpdated: attach.UpdatedAt, - } + apiContent := &apimodel.Content{} // Retrieve appropriate // size file from storage. diff --git a/internal/processing/media/unattach_test.go b/internal/processing/media/unattach_test.go index 051caa4d3..02d2c7077 100644 --- a/internal/processing/media/unattach_test.go +++ b/internal/processing/media/unattach_test.go @@ -20,7 +20,6 @@ package media_test import ( "context" "testing" - "time" "github.com/stretchr/testify/suite" ) @@ -42,8 +41,6 @@ func (suite *UnattachTestSuite) TestUnattachMedia() { dbAttachment, errWithCode := suite.db.GetAttachmentByID(ctx, a.ID) suite.NoError(errWithCode) - - suite.WithinDuration(dbAttachment.UpdatedAt, time.Now(), 1*time.Minute) suite.Empty(dbAttachment.StatusID) } diff --git a/internal/processing/status/get.go b/internal/processing/status/get.go index 75a687db2..470b93a8f 100644 --- a/internal/processing/status/get.go +++ b/internal/processing/status/get.go @@ -67,7 +67,6 @@ func (p *Processor) Get(ctx context.Context, requestingAccount *gtsmodel.Account if errWithCode != nil { return nil, errWithCode } - return p.c.GetAPIStatus(ctx, requestingAccount, targetStatus) } @@ -106,5 +105,6 @@ func (p *Processor) SourceGet(ctx context.Context, requestingAccount *gtsmodel.A err = gtserror.Newf("error converting status: %w", err) return nil, gtserror.NewErrorInternalError(err) } + return statusSource, nil } diff --git a/internal/processing/stream/notification_test.go b/internal/processing/stream/notification_test.go index 169e4f5ce..5c89e1f40 100644 --- a/internal/processing/stream/notification_test.go +++ b/internal/processing/stream/notification_test.go @@ -79,8 +79,8 @@ func (suite *NotificationTestSuite) TestStreamNotification() { "header_description": "Flat gray background (default header).", "followers_count": 0, "following_count": 0, - "statuses_count": 3, - "last_status_at": "2021-09-11", + "statuses_count": 4, + "last_status_at": "2024-11-01", "emojis": [], "fields": [] } diff --git a/internal/processing/stream/statusupdate_test.go b/internal/processing/stream/statusupdate_test.go index b61a9c623..6bf5e436c 100644 --- a/internal/processing/stream/statusupdate_test.go +++ b/internal/processing/stream/statusupdate_test.go @@ -54,6 +54,7 @@ func (suite *StatusUpdateTestSuite) TestStreamNotification() { suite.Equal(`{ "id": "01FVW7JHQFSFK166WWKR8CBA6M", "created_at": "2021-09-20T10:40:37.000Z", + "edited_at": null, "in_reply_to_id": null, "in_reply_to_account_id": null, "sensitive": false, @@ -90,8 +91,8 @@ func (suite *StatusUpdateTestSuite) TestStreamNotification() { "header_description": "Flat gray background (default header).", "followers_count": 0, "following_count": 0, - "statuses_count": 3, - "last_status_at": "2021-09-11", + "statuses_count": 4, + "last_status_at": "2024-11-01", "emojis": [], "fields": [] }, diff --git a/internal/processing/timeline/public_test.go b/internal/processing/timeline/public_test.go index 6b01c9849..ab8e33429 100644 --- a/internal/processing/timeline/public_test.go +++ b/internal/processing/timeline/public_test.go @@ -102,8 +102,8 @@ func (suite *PublicTestSuite) TestPublicTimelineGetHideFiltered() { requester = suite.testAccounts["local_account_1"] maxID = "" sinceID = "" - minID = "01F8MHAAY43M6RJ473VQFCVH36" // 1 before filteredStatus - limit = 10 + minID = "" + limit = 100 local = false filteredStatus = suite.testStatuses["admin_account_status_2"] filteredStatusFound = false diff --git a/internal/processing/workers/util.go b/internal/processing/workers/util.go index 62ea6c95c..b358dc951 100644 --- a/internal/processing/workers/util.go +++ b/internal/processing/workers/util.go @@ -75,6 +75,21 @@ func (u *utils) wipeStatus( } } + // Before handling media, ensure + // historic edits are populated. + if !status.EditsPopulated() { + var err error + + // Fetch all historical edits of status from database. + status.Edits, err = u.state.DB.GetStatusEditsByIDs( + gtscontext.SetBarebones(ctx), + status.EditIDs, + ) + if err != nil { + errs.Appendf("error getting status edits from database: %w", err) + } + } + // Either delete all attachments for this status, // or simply detach + clean them separately later. // @@ -83,20 +98,27 @@ func (u *utils) wipeStatus( // status immediately (in case of delete + redraft). if deleteAttachments { // todo:u.state.DB.DeleteAttachmentsForStatus - for _, id := range status.AttachmentIDs { + for _, id := range status.AllAttachmentIDs() { if err := u.media.Delete(ctx, id); err != nil { errs.Appendf("error deleting media: %w", err) } } } else { // todo:u.state.DB.UnattachAttachmentsForStatus - for _, id := range status.AttachmentIDs { + for _, id := range status.AllAttachmentIDs() { if _, err := u.media.Unattach(ctx, status.Account, id); err != nil { errs.Appendf("error unattaching media: %w", err) } } } + // Delete all historical edits of status. + if ids := status.EditIDs; len(ids) > 0 { + if err := u.state.DB.DeleteStatusEdits(ctx, ids); err != nil { + errs.Appendf("error deleting status edits: %w", err) + } + } + // Delete all mentions generated by this status. // todo:u.state.DB.DeleteMentionsForStatus for _, id := range status.MentionIDs { @@ -120,19 +142,20 @@ func (u *utils) wipeStatus( errs.Appendf("error deleting status faves: %w", err) } - if pollID := status.PollID; pollID != "" { + if id := status.PollID; id != "" { // Delete this poll by ID from the database. - if err := u.state.DB.DeletePollByID(ctx, pollID); err != nil { + if err := u.state.DB.DeletePollByID(ctx, id); err != nil { errs.Appendf("error deleting status poll: %w", err) } // Cancel any scheduled expiry task for poll. - _ = u.state.Workers.Scheduler.Cancel(pollID) + _ = u.state.Workers.Scheduler.Cancel(id) } // Get all boost of this status so that we can // delete those boosts + remove them from timelines. boosts, err := u.state.DB.GetStatusBoosts( + // We MUST set a barebones context here, // as depending on where it came from the // original BoostOf may already be gone. @@ -537,11 +560,7 @@ func (u *utils) requestFave( } // Create + store new interaction request. - req, err = typeutils.StatusFaveToInteractionRequest(ctx, fave) - if err != nil { - return gtserror.Newf("error creating interaction request: %w", err) - } - + req = typeutils.StatusFaveToInteractionRequest(fave) if err := u.state.DB.PutInteractionRequest(ctx, req); err != nil { return gtserror.Newf("db error storing interaction request: %w", err) } @@ -584,11 +603,7 @@ func (u *utils) requestReply( } // Create + store interaction request. - req, err = typeutils.StatusToInteractionRequest(ctx, reply) - if err != nil { - return gtserror.Newf("error creating interaction request: %w", err) - } - + req = typeutils.StatusToInteractionRequest(reply) if err := u.state.DB.PutInteractionRequest(ctx, req); err != nil { return gtserror.Newf("db error storing interaction request: %w", err) } @@ -631,11 +646,7 @@ func (u *utils) requestAnnounce( } // Create + store interaction request. - req, err = typeutils.StatusToInteractionRequest(ctx, boost) - if err != nil { - return gtserror.Newf("error creating interaction request: %w", err) - } - + req = typeutils.StatusToInteractionRequest(boost) if err := u.state.DB.PutInteractionRequest(ctx, req); err != nil { return gtserror.Newf("db error storing interaction request: %w", err) } diff --git a/internal/timeline/get_test.go b/internal/timeline/get_test.go index 6b01ca812..91a456560 100644 --- a/internal/timeline/get_test.go +++ b/internal/timeline/get_test.go @@ -228,7 +228,7 @@ func (suite *GetTestSuite) TestGetNewTimelineMoreThanPossible() { if err != nil { suite.FailNow(err.Error()) } - suite.checkStatuses(statuses, id.Highest, id.Lowest, 20) + suite.checkStatuses(statuses, id.Highest, id.Lowest, 22) } func (suite *GetTestSuite) TestGetNewTimelineMoreThanPossiblePageUp() { @@ -255,7 +255,7 @@ func (suite *GetTestSuite) TestGetNewTimelineMoreThanPossiblePageUp() { if err != nil { suite.FailNow(err.Error()) } - suite.checkStatuses(statuses, id.Highest, id.Lowest, 20) + suite.checkStatuses(statuses, id.Highest, id.Lowest, 22) } func (suite *GetTestSuite) TestGetNewTimelineNoFollowing() { @@ -284,7 +284,7 @@ func (suite *GetTestSuite) TestGetNewTimelineNoFollowing() { if err != nil { suite.FailNow(err.Error()) } - suite.checkStatuses(statuses, id.Highest, id.Lowest, 8) + suite.checkStatuses(statuses, id.Highest, id.Lowest, 9) for _, s := range statuses { if s.GetAccountID() != testAccount.ID { diff --git a/internal/timeline/prune_test.go b/internal/timeline/prune_test.go index e78db64e8..4b909540c 100644 --- a/internal/timeline/prune_test.go +++ b/internal/timeline/prune_test.go @@ -40,7 +40,7 @@ func (suite *PruneTestSuite) TestPrune() { pruned, err := suite.state.Timelines.Home.Prune(ctx, testAccountID, desiredPreparedItemsLength, desiredIndexedItemsLength) suite.NoError(err) - suite.Equal(20, pruned) + suite.Equal(23, pruned) suite.Equal(5, suite.state.Timelines.Home.GetIndexedLength(ctx, testAccountID)) } @@ -56,7 +56,7 @@ func (suite *PruneTestSuite) TestPruneTwice() { pruned, err := suite.state.Timelines.Home.Prune(ctx, testAccountID, desiredPreparedItemsLength, desiredIndexedItemsLength) suite.NoError(err) - suite.Equal(20, pruned) + suite.Equal(23, pruned) suite.Equal(5, suite.state.Timelines.Home.GetIndexedLength(ctx, testAccountID)) // Prune same again, nothing should be pruned this time. @@ -78,7 +78,7 @@ func (suite *PruneTestSuite) TestPruneTo0() { pruned, err := suite.state.Timelines.Home.Prune(ctx, testAccountID, desiredPreparedItemsLength, desiredIndexedItemsLength) suite.NoError(err) - suite.Equal(25, pruned) + suite.Equal(28, pruned) suite.Equal(0, suite.state.Timelines.Home.GetIndexedLength(ctx, testAccountID)) } @@ -95,7 +95,7 @@ func (suite *PruneTestSuite) TestPruneToInfinityAndBeyond() { pruned, err := suite.state.Timelines.Home.Prune(ctx, testAccountID, desiredPreparedItemsLength, desiredIndexedItemsLength) suite.NoError(err) suite.Equal(0, pruned) - suite.Equal(25, suite.state.Timelines.Home.GetIndexedLength(ctx, testAccountID)) + suite.Equal(28, suite.state.Timelines.Home.GetIndexedLength(ctx, testAccountID)) } func TestPruneTestSuite(t *testing.T) { diff --git a/internal/typeutils/astointernal.go b/internal/typeutils/astointernal.go index cf0c0719a..1a7098673 100644 --- a/internal/typeutils/astointernal.go +++ b/internal/typeutils/astointernal.go @@ -111,6 +111,13 @@ func (c *Converter) ASRepresentationToAccount( acct.UpdatedAt = pub } + // Extract updated time if possible, i.e. last edited. + if upd := ap.GetUpdated(accountable); !upd.IsZero() { + acct.UpdatedAt = upd + } else { + acct.UpdatedAt = acct.CreatedAt + } + // Extract a preferred name (display name), fallback to username. if displayName := ap.ExtractName(accountable); displayName != "" { acct.DisplayName = displayName @@ -348,18 +355,25 @@ func (c *Converter) ASStatusToStatus(ctx context.Context, statusable ap.Statusab // zero-time will fall back to db defaults. if pub := ap.GetPublished(statusable); !pub.IsZero() { status.CreatedAt = pub - status.UpdatedAt = pub } else { log.Warnf(ctx, "unusable published property on %s", uri) } + // status.Updated + // + // Extract updated time for status, defaults to Published. + if upd := ap.GetUpdated(statusable); !upd.IsZero() { + status.UpdatedAt = upd + } else { + status.UpdatedAt = status.CreatedAt + } + // status.AccountURI // status.AccountID // status.Account // - // Account that created the status. Assume we have - // this in the db by the time this function is called, - // error if we don't. + // Account that created the status. Assume we have this + // in the db by the time this function is called, else error. status.Account, err = c.getASAttributedToAccount(ctx, status.URI, statusable, diff --git a/internal/typeutils/internal.go b/internal/typeutils/internal.go index ccde6a38f..573495e0a 100644 --- a/internal/typeutils/internal.go +++ b/internal/typeutils/internal.go @@ -104,14 +104,8 @@ func (c *Converter) StatusToBoost( return boost, nil } -func StatusToInteractionRequest( - ctx context.Context, - status *gtsmodel.Status, -) (*gtsmodel.InteractionRequest, error) { - reqID, err := id.NewULIDFromTime(status.CreatedAt) - if err != nil { - return nil, gtserror.Newf("error generating ID: %w", err) - } +func StatusToInteractionRequest(status *gtsmodel.Status) *gtsmodel.InteractionRequest { + reqID := id.NewULIDFromTime(status.CreatedAt) var ( targetID string @@ -154,17 +148,11 @@ func StatusToInteractionRequest( InteractionType: interactionType, Reply: reply, Announce: announce, - }, nil + } } -func StatusFaveToInteractionRequest( - ctx context.Context, - fave *gtsmodel.StatusFave, -) (*gtsmodel.InteractionRequest, error) { - reqID, err := id.NewULIDFromTime(fave.CreatedAt) - if err != nil { - return nil, gtserror.Newf("error generating ID: %w", err) - } +func StatusFaveToInteractionRequest(fave *gtsmodel.StatusFave) *gtsmodel.InteractionRequest { + reqID := id.NewULIDFromTime(fave.CreatedAt) return >smodel.InteractionRequest{ ID: reqID, @@ -178,7 +166,7 @@ func StatusFaveToInteractionRequest( InteractionURI: fave.URI, InteractionType: gtsmodel.InteractionLike, Like: fave, - }, nil + } } func (c *Converter) StatusToSinBinStatus( diff --git a/internal/typeutils/internaltoas.go b/internal/typeutils/internaltoas.go index d46ce64e2..7d0c483dd 100644 --- a/internal/typeutils/internaltoas.go +++ b/internal/typeutils/internaltoas.go @@ -484,10 +484,9 @@ func (c *Converter) StatusToAS(ctx context.Context, s *gtsmodel.Status) (ap.Stat status.SetActivityStreamsInReplyTo(inReplyToProp) } - // published - publishedProp := streams.NewActivityStreamsPublishedProperty() - publishedProp.Set(s.CreatedAt) - status.SetActivityStreamsPublished(publishedProp) + // Set created / updated at properties. + ap.SetPublished(status, s.CreatedAt) + ap.SetUpdated(status, s.UpdatedAt) // url if s.URL != "" { diff --git a/internal/typeutils/internaltoas_test.go b/internal/typeutils/internaltoas_test.go index c847cfc93..9870c760a 100644 --- a/internal/typeutils/internaltoas_test.go +++ b/internal/typeutils/internaltoas_test.go @@ -499,6 +499,7 @@ func (suite *InternalToASTestSuite) TestStatusToAS() { "tag": [], "to": "https://www.w3.org/ns/activitystreams#Public", "type": "Note", + "updated": "2021-10-20T12:40:37+02:00", "url": "http://localhost:8080/@the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY" }`, string(bytes)) } @@ -598,6 +599,7 @@ func (suite *InternalToASTestSuite) TestStatusWithTagsToASWithIDs() { ], "to": "https://www.w3.org/ns/activitystreams#Public", "type": "Note", + "updated": "2021-10-20T11:36:45Z", "url": "http://localhost:8080/@admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R" }`, string(bytes)) } @@ -698,6 +700,7 @@ func (suite *InternalToASTestSuite) TestStatusWithTagsToASFromDB() { ], "to": "https://www.w3.org/ns/activitystreams#Public", "type": "Note", + "updated": "2021-10-20T11:36:45Z", "url": "http://localhost:8080/@admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R" }`, string(bytes)) } @@ -778,6 +781,7 @@ func (suite *InternalToASTestSuite) TestStatusToASWithMentions() { }, "to": "https://www.w3.org/ns/activitystreams#Public", "type": "Note", + "updated": "2021-11-20T13:32:16Z", "url": "http://localhost:8080/@admin/statuses/01FF25D5Q0DH7CHD57CTRS6WK0" }`, string(bytes)) } diff --git a/internal/typeutils/internaltofrontend.go b/internal/typeutils/internaltofrontend.go index fda59610b..e0276a53b 100644 --- a/internal/typeutils/internaltofrontend.go +++ b/internal/typeutils/internaltofrontend.go @@ -1399,17 +1399,13 @@ func (c *Converter) baseStatusToFrontend( } // Nullable fields. - if s.InReplyToID != "" { - apiStatus.InReplyToID = util.Ptr(s.InReplyToID) - } - - if s.InReplyToAccountID != "" { - apiStatus.InReplyToAccountID = util.Ptr(s.InReplyToAccountID) - } - - if s.Language != "" { - apiStatus.Language = util.Ptr(s.Language) + if !s.UpdatedAt.Equal(s.CreatedAt) { + timestamp := util.FormatISO8601(s.UpdatedAt) + apiStatus.EditedAt = util.Ptr(timestamp) } + apiStatus.InReplyToID = util.PtrIf(s.InReplyToID) + apiStatus.InReplyToAccountID = util.PtrIf(s.InReplyToAccountID) + apiStatus.Language = util.PtrIf(s.Language) if app := s.CreatedWithApplication; app != nil { apiStatus.Application, err = c.AppToAPIAppPublic(ctx, app) diff --git a/internal/typeutils/internaltofrontend_test.go b/internal/typeutils/internaltofrontend_test.go index db37d3abd..0ec9ea05f 100644 --- a/internal/typeutils/internaltofrontend_test.go +++ b/internal/typeutils/internaltofrontend_test.go @@ -67,8 +67,8 @@ func (suite *InternalToFrontendTestSuite) TestAccountToFrontend() { "header_media_id": "01PFPMWK2FF0D9WMHEJHR07C3Q", "followers_count": 2, "following_count": 2, - "statuses_count": 8, - "last_status_at": "2024-01-10", + "statuses_count": 9, + "last_status_at": "2024-11-01", "emojis": [], "fields": [], "enable_rss": true @@ -119,8 +119,8 @@ func (suite *InternalToFrontendTestSuite) TestAccountToFrontendAliasedAndMoved() "header_media_id": "01PFPMWK2FF0D9WMHEJHR07C3Q", "followers_count": 2, "following_count": 2, - "statuses_count": 8, - "last_status_at": "2024-01-10", + "statuses_count": 9, + "last_status_at": "2024-11-01", "emojis": [], "fields": [], "source": { @@ -162,8 +162,8 @@ func (suite *InternalToFrontendTestSuite) TestAccountToFrontendAliasedAndMoved() "header_description": "Flat gray background (default header).", "followers_count": 1, "following_count": 1, - "statuses_count": 8, - "last_status_at": "2021-07-28", + "statuses_count": 9, + "last_status_at": "2024-11-01", "emojis": [], "fields": [ { @@ -217,8 +217,8 @@ func (suite *InternalToFrontendTestSuite) TestAccountToFrontendWithEmojiStruct() "header_media_id": "01PFPMWK2FF0D9WMHEJHR07C3Q", "followers_count": 2, "following_count": 2, - "statuses_count": 8, - "last_status_at": "2024-01-10", + "statuses_count": 9, + "last_status_at": "2024-11-01", "emojis": [ { "shortcode": "rainbow", @@ -266,8 +266,8 @@ func (suite *InternalToFrontendTestSuite) TestAccountToFrontendWithEmojiIDs() { "header_media_id": "01PFPMWK2FF0D9WMHEJHR07C3Q", "followers_count": 2, "following_count": 2, - "statuses_count": 8, - "last_status_at": "2024-01-10", + "statuses_count": 9, + "last_status_at": "2024-11-01", "emojis": [ { "shortcode": "rainbow", @@ -311,8 +311,8 @@ func (suite *InternalToFrontendTestSuite) TestAccountToFrontendSensitive() { "header_media_id": "01PFPMWK2FF0D9WMHEJHR07C3Q", "followers_count": 2, "following_count": 2, - "statuses_count": 8, - "last_status_at": "2024-01-10", + "statuses_count": 9, + "last_status_at": "2024-11-01", "emojis": [], "fields": [], "source": { @@ -463,6 +463,7 @@ func (suite *InternalToFrontendTestSuite) TestStatusToFrontend() { suite.Equal(`{ "id": "01F8MH75CBF9JFX4ZAD54N0W0R", "created_at": "2021-10-20T11:36:45.000Z", + "edited_at": null, "in_reply_to_id": null, "in_reply_to_account_id": null, "sensitive": false, @@ -641,6 +642,7 @@ func (suite *InternalToFrontendTestSuite) TestWarnFilteredStatusToFrontend() { suite.Equal(`{ "id": "01F8MH75CBF9JFX4ZAD54N0W0R", "created_at": "2021-10-20T11:36:45.000Z", + "edited_at": null, "in_reply_to_id": null, "in_reply_to_account_id": null, "sensitive": false, @@ -807,6 +809,7 @@ func (suite *InternalToFrontendTestSuite) TestWarnFilteredBoostToFrontend() { suite.Equal(`{ "id": "01G36SF3V6Y6V5BF9P4R7PQG7G", "created_at": "2021-10-20T10:41:37.000Z", + "edited_at": null, "in_reply_to_id": null, "in_reply_to_account_id": null, "sensitive": false, @@ -827,6 +830,7 @@ func (suite *InternalToFrontendTestSuite) TestWarnFilteredBoostToFrontend() { "reblog": { "id": "01F8MH75CBF9JFX4ZAD54N0W0R", "created_at": "2021-10-20T11:36:45.000Z", + "edited_at": null, "in_reply_to_id": null, "in_reply_to_account_id": null, "sensitive": false, @@ -870,8 +874,8 @@ func (suite *InternalToFrontendTestSuite) TestWarnFilteredBoostToFrontend() { "header_media_id": "01PFPMWK2FF0D9WMHEJHR07C3Q", "followers_count": 2, "following_count": 2, - "statuses_count": 8, - "last_status_at": "2024-01-10", + "statuses_count": 9, + "last_status_at": "2024-11-01", "emojis": [], "fields": [], "enable_rss": true @@ -1218,6 +1222,7 @@ func (suite *InternalToFrontendTestSuite) TestStatusToFrontendUnknownAttachments suite.Equal(`{ "id": "01HE7XJ1CG84TBKH5V9XKBVGF5", "created_at": "2023-11-02T10:44:25.000Z", + "edited_at": null, "in_reply_to_id": "01F8MH75CBF9JFX4ZAD54N0W0R", "in_reply_to_account_id": "01F8MH17FWEB39HZJ76B6VXSKF", "sensitive": true, @@ -1350,6 +1355,7 @@ func (suite *InternalToFrontendTestSuite) TestStatusToWebStatus() { suite.Equal(`{ "id": "01HE7XJ1CG84TBKH5V9XKBVGF5", "created_at": "2023-11-02T10:44:25.000Z", + "edited_at": null, "in_reply_to_id": "01F8MH75CBF9JFX4ZAD54N0W0R", "in_reply_to_account_id": "01F8MH17FWEB39HZJ76B6VXSKF", "sensitive": true, @@ -1511,6 +1517,7 @@ func (suite *InternalToFrontendTestSuite) TestStatusToFrontendUnknownLanguage() suite.Equal(`{ "id": "01F8MH75CBF9JFX4ZAD54N0W0R", "created_at": "2021-10-20T11:36:45.000Z", + "edited_at": null, "in_reply_to_id": null, "in_reply_to_account_id": null, "sensitive": false, @@ -1654,6 +1661,7 @@ func (suite *InternalToFrontendTestSuite) TestStatusToFrontendPartialInteraction suite.Equal(`{ "id": "01F8MHBBN8120SYH7D5S050MGK", "created_at": "2021-10-20T10:40:37.000Z", + "edited_at": null, "in_reply_to_id": null, "in_reply_to_account_id": null, "sensitive": false, @@ -1697,8 +1705,8 @@ func (suite *InternalToFrontendTestSuite) TestStatusToFrontendPartialInteraction "header_media_id": "01PFPMWK2FF0D9WMHEJHR07C3Q", "followers_count": 2, "following_count": 2, - "statuses_count": 8, - "last_status_at": "2024-01-10", + "statuses_count": 9, + "last_status_at": "2024-11-01", "emojis": [], "fields": [], "enable_rss": true @@ -1764,6 +1772,7 @@ func (suite *InternalToFrontendTestSuite) TestStatusToAPIStatusPendingApproval() suite.Equal(`{ "id": "01J5QVB9VC76NPPRQ207GG4DRZ", "created_at": "2024-02-20T10:41:37.000Z", + "edited_at": null, "in_reply_to_id": "01F8MHC8VWDRBQR0N1BATDDEM5", "in_reply_to_account_id": "01F8MH5NBDF2MV7CTC4Q5128HF", "sensitive": false, @@ -1993,7 +2002,7 @@ func (suite *InternalToFrontendTestSuite) TestInstanceV1ToFrontend() { }, "stats": { "domain_count": 2, - "status_count": 19, + "status_count": 21, "user_count": 4 }, "thumbnail": "http://localhost:8080/assets/logo.webp", @@ -2277,8 +2286,8 @@ func (suite *InternalToFrontendTestSuite) TestReportToFrontend1() { "header_description": "Flat gray background (default header).", "followers_count": 0, "following_count": 0, - "statuses_count": 3, - "last_status_at": "2021-09-11", + "statuses_count": 4, + "last_status_at": "2024-11-01", "emojis": [], "fields": [] } @@ -2321,8 +2330,8 @@ func (suite *InternalToFrontendTestSuite) TestReportToFrontend2() { "header_description": "Flat gray background (default header).", "followers_count": 1, "following_count": 1, - "statuses_count": 8, - "last_status_at": "2021-07-28", + "statuses_count": 9, + "last_status_at": "2024-11-01", "emojis": [], "fields": [ { @@ -2398,8 +2407,8 @@ func (suite *InternalToFrontendTestSuite) TestAdminReportToFrontend1() { "header_description": "Flat gray background (default header).", "followers_count": 0, "following_count": 0, - "statuses_count": 3, - "last_status_at": "2021-09-11", + "statuses_count": 4, + "last_status_at": "2024-11-01", "emojis": [], "fields": [] } @@ -2444,8 +2453,8 @@ func (suite *InternalToFrontendTestSuite) TestAdminReportToFrontend1() { "header_description": "Flat gray background (default header).", "followers_count": 1, "following_count": 1, - "statuses_count": 8, - "last_status_at": "2021-07-28", + "statuses_count": 9, + "last_status_at": "2024-11-01", "emojis": [], "fields": [ { @@ -2636,8 +2645,8 @@ func (suite *InternalToFrontendTestSuite) TestAdminReportToFrontend2() { "header_description": "Flat gray background (default header).", "followers_count": 1, "following_count": 1, - "statuses_count": 8, - "last_status_at": "2021-07-28", + "statuses_count": 9, + "last_status_at": "2024-11-01", "emojis": [], "fields": [ { @@ -2695,8 +2704,8 @@ func (suite *InternalToFrontendTestSuite) TestAdminReportToFrontend2() { "header_description": "Flat gray background (default header).", "followers_count": 0, "following_count": 0, - "statuses_count": 3, - "last_status_at": "2021-09-11", + "statuses_count": 4, + "last_status_at": "2024-11-01", "emojis": [], "fields": [] } @@ -2707,6 +2716,7 @@ func (suite *InternalToFrontendTestSuite) TestAdminReportToFrontend2() { { "id": "01FVW7JHQFSFK166WWKR8CBA6M", "created_at": "2021-09-20T10:40:37.000Z", + "edited_at": null, "in_reply_to_id": null, "in_reply_to_account_id": null, "sensitive": false, @@ -2743,8 +2753,8 @@ func (suite *InternalToFrontendTestSuite) TestAdminReportToFrontend2() { "header_description": "Flat gray background (default header).", "followers_count": 0, "following_count": 0, - "statuses_count": 3, - "last_status_at": "2021-09-11", + "statuses_count": 4, + "last_status_at": "2024-11-01", "emojis": [], "fields": [] }, @@ -2902,8 +2912,8 @@ func (suite *InternalToFrontendTestSuite) TestAdminReportToFrontendSuspendedLoca "header_description": "Flat gray background (default header).", "followers_count": 0, "following_count": 0, - "statuses_count": 3, - "last_status_at": "2021-09-11", + "statuses_count": 4, + "last_status_at": "2024-11-01", "emojis": [], "fields": [] } @@ -3214,6 +3224,7 @@ func (suite *InternalToFrontendTestSuite) TestIntReqToAPI() { "status": { "id": "01F8MHC8VWDRBQR0N1BATDDEM5", "created_at": "2021-10-20T10:40:37.000Z", + "edited_at": null, "in_reply_to_id": null, "in_reply_to_account_id": null, "sensitive": true, @@ -3254,8 +3265,8 @@ func (suite *InternalToFrontendTestSuite) TestIntReqToAPI() { "header_description": "Flat gray background (default header).", "followers_count": 1, "following_count": 1, - "statuses_count": 8, - "last_status_at": "2021-07-28", + "statuses_count": 9, + "last_status_at": "2024-11-01", "emojis": [], "fields": [ { @@ -3307,6 +3318,7 @@ func (suite *InternalToFrontendTestSuite) TestIntReqToAPI() { "reply": { "id": "01J5QVB9VC76NPPRQ207GG4DRZ", "created_at": "2024-02-20T10:41:37.000Z", + "edited_at": null, "in_reply_to_id": "01F8MHC8VWDRBQR0N1BATDDEM5", "in_reply_to_account_id": "01F8MH5NBDF2MV7CTC4Q5128HF", "sensitive": false, @@ -3464,8 +3476,8 @@ func (suite *InternalToFrontendTestSuite) TestConversationToAPISelfConvo() { "header_media_id": "01PFPMWK2FF0D9WMHEJHR07C3Q", "followers_count": 2, "following_count": 2, - "statuses_count": 8, - "last_status_at": "2024-01-10", + "statuses_count": 9, + "last_status_at": "2024-11-01", "emojis": [], "fields": [], "enable_rss": true @@ -3474,6 +3486,7 @@ func (suite *InternalToFrontendTestSuite) TestConversationToAPISelfConvo() { "last_status": { "id": "01F8MHAMCHF6Y650WCRSCP4WMY", "created_at": "2021-10-20T10:40:37.000Z", + "edited_at": null, "in_reply_to_id": null, "in_reply_to_account_id": null, "sensitive": true, @@ -3517,8 +3530,8 @@ func (suite *InternalToFrontendTestSuite) TestConversationToAPISelfConvo() { "header_media_id": "01PFPMWK2FF0D9WMHEJHR07C3Q", "followers_count": 2, "following_count": 2, - "statuses_count": 8, - "last_status_at": "2024-01-10", + "statuses_count": 9, + "last_status_at": "2024-11-01", "emojis": [], "fields": [], "enable_rss": true @@ -3619,8 +3632,8 @@ func (suite *InternalToFrontendTestSuite) TestConversationToAPI() { "header_description": "Flat gray background (default header).", "followers_count": 1, "following_count": 1, - "statuses_count": 8, - "last_status_at": "2021-07-28", + "statuses_count": 9, + "last_status_at": "2024-11-01", "emojis": [], "fields": [ { @@ -3640,6 +3653,7 @@ func (suite *InternalToFrontendTestSuite) TestConversationToAPI() { "last_status": { "id": "01F8MHAMCHF6Y650WCRSCP4WMY", "created_at": "2021-10-20T10:40:37.000Z", + "edited_at": null, "in_reply_to_id": null, "in_reply_to_account_id": null, "sensitive": true, @@ -3683,8 +3697,8 @@ func (suite *InternalToFrontendTestSuite) TestConversationToAPI() { "header_media_id": "01PFPMWK2FF0D9WMHEJHR07C3Q", "followers_count": 2, "following_count": 2, - "statuses_count": 8, - "last_status_at": "2024-01-10", + "statuses_count": 9, + "last_status_at": "2024-11-01", "emojis": [], "fields": [], "enable_rss": true diff --git a/internal/typeutils/wrap_test.go b/internal/typeutils/wrap_test.go index 1085c8c66..c2c9c9464 100644 --- a/internal/typeutils/wrap_test.go +++ b/internal/typeutils/wrap_test.go @@ -131,6 +131,7 @@ func (suite *WrapTestSuite) TestWrapNoteInCreate() { "tag": [], "to": "https://www.w3.org/ns/activitystreams#Public", "type": "Note", + "updated": "2021-10-20T12:40:37+02:00", "url": "http://localhost:8080/@the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY" }, "published": "2021-10-20T12:40:37+02:00", diff --git a/test/envparsing.sh b/test/envparsing.sh index 927c5f98b..372c62327 100755 --- a/test/envparsing.sh +++ b/test/envparsing.sh @@ -64,6 +64,7 @@ EXPECT=$(cat << "EOF" "sin-bin-status-mem-ratio": 0.5, "status-bookmark-ids-mem-ratio": 2, "status-bookmark-mem-ratio": 0.5, + "status-edit-mem-ratio": 2, "status-fave-ids-mem-ratio": 3, "status-fave-mem-ratio": 2, "status-mem-ratio": 5, diff --git a/testrig/db.go b/testrig/db.go index 92d963e89..e53e9c9f0 100644 --- a/testrig/db.go +++ b/testrig/db.go @@ -52,6 +52,7 @@ var testModels = []interface{}{ >smodel.Status{}, >smodel.StatusToEmoji{}, >smodel.StatusToTag{}, + >smodel.StatusEdit{}, >smodel.StatusFave{}, >smodel.StatusBookmark{}, >smodel.Tag{}, @@ -101,7 +102,7 @@ func CreateTestTables(db db.DB) { ctx := context.Background() for _, m := range testModels { if err := db.CreateTable(ctx, m); err != nil { - log.Panicf(nil, "error creating table for %+v: %s", m, err) + log.Panicf(ctx, "error creating table for %+v: %s", m, err) } } } @@ -125,243 +126,249 @@ func StandardDBSetup(db db.DB, accounts map[string]*gtsmodel.Account) { for _, v := range NewTestTokens() { if err := db.Put(ctx, v); err != nil { - log.Panic(nil, err) + log.Panic(ctx, err) } } for _, v := range NewTestClients() { if err := db.Put(ctx, v); err != nil { - log.Panic(nil, err) + log.Panic(ctx, err) } } for _, v := range NewTestApplications() { if err := db.Put(ctx, v); err != nil { - log.Panic(nil, err) + log.Panic(ctx, err) } } for _, v := range NewTestBlocks() { if err := db.Put(ctx, v); err != nil { - log.Panic(nil, err) + log.Panic(ctx, err) } } for _, v := range NewTestReports() { if err := db.Put(ctx, v); err != nil { - log.Panic(nil, err) + log.Panic(ctx, err) } } for _, v := range NewTestRules() { if err := db.Put(ctx, v); err != nil { - log.Panic(nil, err) + log.Panic(ctx, err) } } for _, v := range NewTestDomainBlocks() { if err := db.Put(ctx, v); err != nil { - log.Panic(nil, err) + log.Panic(ctx, err) } } for _, v := range NewTestInstances() { if err := db.Put(ctx, v); err != nil { - log.Panic(nil, err) + log.Panic(ctx, err) } } for _, v := range NewTestUsers() { if err := db.Put(ctx, v); err != nil { - log.Panic(nil, err) + log.Panic(ctx, err) } } if accounts == nil { for _, v := range NewTestAccounts() { if err := db.Put(ctx, v); err != nil { - log.Panic(nil, err) + log.Panic(ctx, err) } } } else { for _, v := range accounts { if err := db.Put(ctx, v); err != nil { - log.Panic(nil, err) + log.Panic(ctx, err) } } } for _, v := range NewTestAccountSettings() { if err := db.Put(ctx, v); err != nil { - log.Panic(nil, err) + log.Panic(ctx, err) } } for _, v := range NewTestAttachments() { if err := db.Put(ctx, v); err != nil { - log.Panic(nil, err) + log.Panic(ctx, err) } } for _, v := range NewTestStatuses() { if err := db.Put(ctx, v); err != nil { - log.Panic(nil, err) + log.Panic(ctx, err) } } for _, v := range NewTestEmojis() { if err := db.Put(ctx, v); err != nil { - log.Panic(nil, err) + log.Panic(ctx, err) } } for _, v := range NewTestEmojiCategories() { if err := db.Put(ctx, v); err != nil { - log.Panic(nil, err) + log.Panic(ctx, err) } } for _, v := range NewTestStatusToEmojis() { if err := db.Put(ctx, v); err != nil { - log.Panic(nil, err) + log.Panic(ctx, err) } } for _, v := range NewTestTags() { if err := db.Put(ctx, v); err != nil { - log.Panic(nil, err) + log.Panic(ctx, err) } } for _, v := range NewTestStatusToTags() { if err := db.Put(ctx, v); err != nil { - log.Panic(nil, err) + log.Panic(ctx, err) } } for _, v := range NewTestMentions() { if err := db.Put(ctx, v); err != nil { - log.Panic(nil, err) + log.Panic(ctx, err) } } for _, v := range NewTestFaves() { if err := db.Put(ctx, v); err != nil { - log.Panic(nil, err) + log.Panic(ctx, err) } } for _, v := range NewTestFollows() { if err := db.Put(ctx, v); err != nil { - log.Panic(nil, err) + log.Panic(ctx, err) } } for _, v := range NewTestLists() { if err := db.Put(ctx, v); err != nil { - log.Panic(nil, err) + log.Panic(ctx, err) } } for _, v := range NewTestListEntries() { if err := db.Put(ctx, v); err != nil { - log.Panic(nil, err) + log.Panic(ctx, err) } } for _, v := range NewTestNotifications() { if err := db.Put(ctx, v); err != nil { - log.Panic(nil, err) + log.Panic(ctx, err) } } for _, v := range NewTestTombstones() { if err := db.Put(ctx, v); err != nil { - log.Panic(nil, err) + log.Panic(ctx, err) } } for _, v := range NewTestBookmarks() { if err := db.Put(ctx, v); err != nil { - log.Panic(nil, err) + log.Panic(ctx, err) } } for _, v := range NewTestAccountNotes() { if err := db.Put(ctx, v); err != nil { - log.Panic(nil, err) + log.Panic(ctx, err) } } for _, v := range NewTestMarkers() { if err := db.Put(ctx, v); err != nil { - log.Panic(nil, err) + log.Panic(ctx, err) } } for _, v := range NewTestThreads() { if err := db.Put(ctx, v); err != nil { - log.Panic(nil, err) + log.Panic(ctx, err) } } for _, v := range NewTestThreadToStatus() { if err := db.Put(ctx, v); err != nil { - log.Panic(nil, err) + log.Panic(ctx, err) } } for _, v := range NewTestPolls() { if err := db.Put(ctx, v); err != nil { - log.Panic(nil, err) + log.Panic(ctx, err) } } for _, v := range NewTestPollVotes() { if err := db.Put(ctx, v); err != nil { - log.Panic(nil, err) + log.Panic(ctx, err) } } for _, v := range NewTestFilters() { if err := db.Put(ctx, v); err != nil { - log.Panic(nil, err) + log.Panic(ctx, err) } } for _, v := range NewTestFilterKeywords() { if err := db.Put(ctx, v); err != nil { - log.Panic(nil, err) + log.Panic(ctx, err) } } for _, v := range NewTestFilterStatuses() { if err := db.Put(ctx, v); err != nil { - log.Panic(nil, err) + log.Panic(ctx, err) } } for _, v := range NewTestUserMutes() { if err := db.Put(ctx, v); err != nil { - log.Panic(nil, err) + log.Panic(ctx, err) } } for _, v := range NewTestInteractionRequests() { if err := db.Put(ctx, v); err != nil { - log.Panic(nil, err) + log.Panic(ctx, err) + } + } + + for _, v := range NewTestStatusEdits() { + if err := db.Put(ctx, v); err != nil { + log.Panic(ctx, err) } } if err := db.CreateInstanceAccount(ctx); err != nil { - log.Panic(nil, err) + log.Panic(ctx, err) } if err := db.CreateInstanceInstance(ctx); err != nil { - log.Panic(nil, err) + log.Panic(ctx, err) } - log.Debug(nil, "testing db setup complete") + log.Debug(ctx, "testing db setup complete") } // StandardDBTeardown drops all the standard testing tables/models from the database to ensure it's clean for the next test. diff --git a/testrig/testmodels.go b/testrig/testmodels.go index ae69b9e81..2b83c2102 100644 --- a/testrig/testmodels.go +++ b/testrig/testmodels.go @@ -718,7 +718,6 @@ func NewTestAttachments() map[string]*gtsmodel.MediaAttachment { URL: "http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/original/01F8MH6NEM8D7527KZAECTCR76.jpg", RemoteURL: "", CreatedAt: TimeMustParse("2022-06-04T13:12:00Z"), - UpdatedAt: TimeMustParse("2022-06-04T13:12:00Z"), Type: gtsmodel.FileTypeImage, FileMeta: gtsmodel.FileMeta{ Original: gtsmodel.Original{ @@ -761,7 +760,6 @@ func NewTestAttachments() map[string]*gtsmodel.MediaAttachment { URL: "http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/attachment/original/01F8MH7TDVANYKWVE8VVKFPJTJ.gif", RemoteURL: "", CreatedAt: TimeMustParse("2022-06-09T13:12:00Z"), - UpdatedAt: TimeMustParse("2022-06-09T13:12:00Z"), Type: gtsmodel.FileTypeImage, FileMeta: gtsmodel.FileMeta{ Original: gtsmodel.Original{ @@ -808,7 +806,6 @@ func NewTestAttachments() map[string]*gtsmodel.MediaAttachment { URL: "http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/attachment/original/01CDR64G398ADCHXK08WWTHEZ5.mp4", RemoteURL: "", CreatedAt: TimeMustParse("2022-06-09T13:12:00Z"), - UpdatedAt: TimeMustParse("2022-06-09T13:12:00Z"), Type: gtsmodel.FileTypeVideo, FileMeta: gtsmodel.FileMeta{ Original: gtsmodel.Original{ @@ -858,7 +855,6 @@ func NewTestAttachments() map[string]*gtsmodel.MediaAttachment { URL: "http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/attachment/original/01F8MH8RMYQ6MSNY3JM2XT1CQ5.jpg", RemoteURL: "", CreatedAt: TimeMustParse("2022-06-09T13:12:00Z"), - UpdatedAt: TimeMustParse("2022-06-09T13:12:00Z"), Type: gtsmodel.FileTypeImage, FileMeta: gtsmodel.FileMeta{ Original: gtsmodel.Original{ @@ -905,7 +901,6 @@ func NewTestAttachments() map[string]*gtsmodel.MediaAttachment { URL: "http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/original/01F8MH58A357CV5K7R7TJMSH6S.jpg", RemoteURL: "", CreatedAt: TimeMustParse("2022-06-09T13:12:00Z"), - UpdatedAt: TimeMustParse("2022-06-09T13:12:00Z"), Type: gtsmodel.FileTypeImage, FileMeta: gtsmodel.FileMeta{ Original: gtsmodel.Original{ @@ -952,7 +947,6 @@ func NewTestAttachments() map[string]*gtsmodel.MediaAttachment { URL: "http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/header/original/01PFPMWK2FF0D9WMHEJHR07C3Q.jpg", RemoteURL: "", CreatedAt: TimeMustParse("2022-06-09T13:12:00Z"), - UpdatedAt: TimeMustParse("2022-06-09T13:12:00Z"), Type: gtsmodel.FileTypeImage, FileMeta: gtsmodel.FileMeta{ Original: gtsmodel.Original{ @@ -999,7 +993,6 @@ func NewTestAttachments() map[string]*gtsmodel.MediaAttachment { URL: "http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/attachment/original/01J2M20K6K9XQC4WSB961YJHV6.mp3", RemoteURL: "", CreatedAt: TimeMustParse("2024-01-10T11:24:00+02:00"), - UpdatedAt: TimeMustParse("2024-01-10T11:24:00+02:00"), Type: gtsmodel.FileTypeAudio, FileMeta: gtsmodel.FileMeta{ Original: gtsmodel.Original{ @@ -1043,13 +1036,30 @@ func NewTestAttachments() map[string]*gtsmodel.MediaAttachment { Header: util.Ptr(false), Cached: util.Ptr(true), }, + "local_account_2_status_9_attachment_1": { + ID: "01JDQ164HM08SGJ7ZEK9003Z4B", + StatusID: "01JDPZEZ77X1NX0TY9M10BK1HM", + URL: "http://localhost:8080/fileserver/01FHMQX3GAABWSM0S2VZEC2SWC/attachment/original/01HE88YG74PVAB81PX2XA9F3FG.mp3", + RemoteURL: "http://example.org/fileserver/01HE7Y659ZWZ02JM4AWYJZ176Q/attachment/original/01HE892Y8ZS68TQCNPX7J888P3.mp3", + CreatedAt: TimeMustParse("2024-11-01T10:01:00+02:00"), + Type: gtsmodel.FileTypeUnknown, + FileMeta: gtsmodel.FileMeta{}, + AccountID: "01F8MH5NBDF2MV7CTC4Q5128HF", + Description: "Jolly salsa song, public domain.", + Blurhash: "", + Processing: gtsmodel.ProcessingStatusProcessed, + File: gtsmodel.File{}, + Thumbnail: gtsmodel.Thumbnail{RemoteURL: ""}, + Avatar: util.Ptr(false), + Header: util.Ptr(false), + Cached: util.Ptr(false), + }, "remote_account_1_status_1_attachment_1": { ID: "01FVW7RXPQ8YJHTEXYPE7Q8ZY0", StatusID: "01FVW7JHQFSFK166WWKR8CBA6M", URL: "http://localhost:8080/fileserver/01F8MH5ZK5VRH73AKHQM6Y9VNX/attachment/original/01FVW7RXPQ8YJHTEXYPE7Q8ZY0.jpg", RemoteURL: "http://fossbros-anonymous.io/attachments/original/13bbc3f8-2b5e-46ea-9531-40b4974d9912.jpg", CreatedAt: TimeMustParse("2021-09-20T12:40:37+02:00"), - UpdatedAt: TimeMustParse("2021-09-20T12:40:37+02:00"), Type: gtsmodel.FileTypeImage, FileMeta: gtsmodel.FileMeta{ Original: gtsmodel.Original{ @@ -1095,7 +1105,6 @@ func NewTestAttachments() map[string]*gtsmodel.MediaAttachment { URL: "http://localhost:8080/fileserver/062G5WYKY35KKD12EMSM3F8PJ8/header/original/01PFPMWK2FF0D9WMHEJHR07C3R.jpg", RemoteURL: "http://fossbros-anonymous.io/attachments/small/a499f55b-2d1e-4acd-98d2-1ac2ba6d79b9.jpg", CreatedAt: TimeMustParse("2022-06-09T13:12:00Z"), - UpdatedAt: TimeMustParse("2022-06-09T13:12:00Z"), Type: gtsmodel.FileTypeImage, FileMeta: gtsmodel.FileMeta{ Original: gtsmodel.Original{ @@ -1141,7 +1150,6 @@ func NewTestAttachments() map[string]*gtsmodel.MediaAttachment { URL: "http://localhost:8080/fileserver/01FHMQX3GAABWSM0S2VZEC2SWC/attachment/original/01HE7Y3C432WRSNS10EZM86SA5.jpg", RemoteURL: "http://example.org/fileserver/01HE7Y659ZWZ02JM4AWYJZ176Q/attachment/original/01HE7Y6G0EMCKST3Q0914WW0MS.jpg", CreatedAt: TimeMustParse("2023-11-02T12:44:25+02:00"), - UpdatedAt: TimeMustParse("2023-11-02T12:44:25+02:00"), Type: gtsmodel.FileTypeImage, FileMeta: gtsmodel.FileMeta{ Original: gtsmodel.Original{ @@ -1186,7 +1194,6 @@ func NewTestAttachments() map[string]*gtsmodel.MediaAttachment { URL: "http://localhost:8080/fileserver/01FHMQX3GAABWSM0S2VZEC2SWC/attachment/original/01HE7ZFX9GKA5ZZVD4FACABSS9.svg", RemoteURL: "http://example.org/fileserver/01HE7Y659ZWZ02JM4AWYJZ176Q/attachment/original/01HE7ZGJYTSYMXF927GF9353KR.svg", CreatedAt: TimeMustParse("2023-11-02T12:44:25+02:00"), - UpdatedAt: TimeMustParse("2023-11-02T12:44:25+02:00"), Type: gtsmodel.FileTypeUnknown, FileMeta: gtsmodel.FileMeta{}, AccountID: "01FHMQX3GAABWSM0S2VZEC2SWC", @@ -1205,7 +1212,6 @@ func NewTestAttachments() map[string]*gtsmodel.MediaAttachment { URL: "http://localhost:8080/fileserver/01FHMQX3GAABWSM0S2VZEC2SWC/attachment/original/01HE88YG74PVAB81PX2XA9F3FG.mp3", RemoteURL: "http://example.org/fileserver/01HE7Y659ZWZ02JM4AWYJZ176Q/attachment/original/01HE892Y8ZS68TQCNPX7J888P3.mp3", CreatedAt: TimeMustParse("2023-11-02T12:44:25+02:00"), - UpdatedAt: TimeMustParse("2023-11-02T12:44:25+02:00"), Type: gtsmodel.FileTypeUnknown, FileMeta: gtsmodel.FileMeta{}, AccountID: "01FHMQX3GAABWSM0S2VZEC2SWC", @@ -1739,6 +1745,32 @@ func NewTestStatuses() map[string]*gtsmodel.Status { Federated: util.Ptr(true), ActivityStreamsType: ap.ObjectNote, }, + "local_account_1_status_9": { + ID: "01JDPZC707CKDN8N4QVWM4Z1NR", + URI: "http://localhost:8080/users/the_mighty_zork/statuses/01JDPZC707CKDN8N4QVWM4Z1NR", + URL: "http://localhost:8080/@the_mighty_zork/statuses/01JDPZC707CKDN8N4QVWM4Z1NR", + Content: "

this is the latest revision of the status, with a content-warning

", + Text: "this is the latest revision of the status, with a content-warning", + ContentWarning: "edited status", + AttachmentIDs: nil, + CreatedAt: TimeMustParse("2024-11-01T11:00:00+02:00"), + UpdatedAt: TimeMustParse("2024-11-01T11:02:00+02:00"), + Local: util.Ptr(true), + AccountURI: "http://localhost:8080/users/the_mighty_zork", + AccountID: "01F8MH1H7YV1Z7D2C8K2730QBF", + InReplyToID: "", + InReplyToAccountID: "", + InReplyToURI: "", + BoostOfID: "", + ThreadID: "", + EditIDs: []string{"01JDPZCZ2Y9KSGZW0R7ZG8T8Y2", "01JDPZDADMD1T9HKF94RECF7PP"}, + Visibility: gtsmodel.VisibilityPublic, + Sensitive: util.Ptr(false), + Language: "en", + CreatedWithApplicationID: "01F8MGY43H3N2C8EWPR2FPYEXG", + Federated: util.Ptr(true), + ActivityStreamsType: ap.ObjectNote, + }, "local_account_2_status_1": { ID: "01F8MHBQCBTDKN6X5VHGMMN4MA", URI: "http://localhost:8080/users/1happyturtle/statuses/01F8MHBQCBTDKN6X5VHGMMN4MA", @@ -1967,6 +1999,32 @@ func NewTestStatuses() map[string]*gtsmodel.Status { PollID: "01HEN2QB5NR4NCEHGYC3HN84K6", PendingApproval: util.Ptr(false), }, + "local_account_2_status_9": { + ID: "01JDPZEZ77X1NX0TY9M10BK1HM", + URI: "http://localhost:8080/users/1happyturtle/statuses/01JDPZEZ77X1NX0TY9M10BK1HM", + URL: "http://localhost:8080/@1happyturtle/statuses/01JDPZEZ77X1NX0TY9M10BK1HM", + Content: "

now edited to bring back the previous edit's media!

", + Text: "now edited to bring back the previous edit's media!", + ContentWarning: "edit with media attachments", + AttachmentIDs: []string{"01JDQ164HM08SGJ7ZEK9003Z4B"}, + CreatedAt: TimeMustParse("2024-11-01T10:00:00+02:00"), + UpdatedAt: TimeMustParse("2024-11-01T10:03:00+02:00"), + Local: util.Ptr(true), + AccountURI: "http://localhost:8080/users/the_mighty_zork", + AccountID: "01F8MH5NBDF2MV7CTC4Q5128HF", + InReplyToID: "", + InReplyToAccountID: "", + InReplyToURI: "", + BoostOfID: "", + ThreadID: "", + EditIDs: []string{"01JDPZPBXAX0M02YSEPB21KX4R", "01JDPZPJHKP7E3M0YQXEXPS1YT", "01JDPZPY3F85Y7B78ETRXEMWD9"}, + Visibility: gtsmodel.VisibilityPublic, + Sensitive: util.Ptr(false), + Language: "en", + CreatedWithApplicationID: "01F8MGYG9E893WRHW0TAEXR8GJ", + Federated: util.Ptr(true), + ActivityStreamsType: ap.ObjectNote, + }, "remote_account_1_status_1": { ID: "01FVW7JHQFSFK166WWKR8CBA6M", URI: "http://fossbros-anonymous.io/users/foss_satan/statuses/01FVW7JHQFSFK166WWKR8CBA6M", @@ -2042,6 +2100,33 @@ func NewTestStatuses() map[string]*gtsmodel.Status { PollID: "01HEWV1GW2D49R919NPEDXPTZ5", PendingApproval: util.Ptr(false), }, + "remote_account_1_status_4": { + ID: "01JDQ07JZTX9CMDJP67CNA71YD", + URI: "http://fossbros-anonymous.io/users/foss_satan/statuses/______", + URL: "http://fossbros-anonymous.io/@foss_satan/statuses/______", + Content: "

this is the latest status edit without poll change

", + Text: "this is the latest status edit without poll change", + ContentWarning: "", + AttachmentIDs: nil, + CreatedAt: TimeMustParse("2024-11-01T09:00:00+02:00"), + UpdatedAt: TimeMustParse("2024-11-01T09:02:00+02:00"), + Local: util.Ptr(false), + AccountURI: "http://fossbros-anonymous.io/users/foss_satan", + AccountID: "01F8MH5ZK5VRH73AKHQM6Y9VNX", + InReplyToID: "", + InReplyToAccountID: "", + InReplyToURI: "", + BoostOfID: "", + ThreadID: "", + EditIDs: []string{"01JDQ07ZZ4FGP13YN8TF63P5A6", "01JDQ08AYQC0G6413VAHA51CV9"}, + PollID: "01JDQ0EZ5HM9T4WXRQ5WSVD40J", + Visibility: gtsmodel.VisibilityPublic, + Sensitive: util.Ptr(false), + Language: "en", + CreatedWithApplicationID: "01F8MGYG9E893WRHW0TAEXR8GJ", + Federated: util.Ptr(true), + ActivityStreamsType: ap.ObjectNote, + }, "remote_account_2_status_1": { ID: "01HE7XJ1CG84TBKH5V9XKBVGF5", URI: "http://example.org/users/Some_User/statuses/01HE7XJ1CG84TBKH5V9XKBVGF5", @@ -2125,6 +2210,19 @@ func NewTestPolls() map[string]*gtsmodel.Poll { ClosedAt: time.Time{}, Closing: false, }, + "remote_account_1_status_4_poll": { + ID: "01JDQ0EZ5HM9T4WXRQ5WSVD40J", + Multiple: util.Ptr(false), + HideCounts: util.Ptr(false), + Options: []string{"yes", "no", "maybe", "i don't know", "can you repeat the question"}, + Votes: []int{0, 0, 0, 0, 2}, + Voters: util.Ptr(2), + StatusID: "01JDQ07JZTX9CMDJP67CNA71YD", + // empty expiry AND closed date, i.e. no end + ExpiresAt: time.Time{}, + ClosedAt: time.Time{}, + Closing: false, + }, } } @@ -2184,6 +2282,24 @@ func NewTestPollVotes() map[string]*gtsmodel.PollVote { Poll: nil, CreatedAt: TimeMustParse("2021-09-11T11:47:37+02:00"), }, + "remote_account_1_status_4_poll_vote_local_account_1": { + ID: "01JDQ0SX9QVVFHS7P8M1PA3SVG", + Choices: []int{4}, + AccountID: "01F8MH1H7YV1Z7D2C8K2730QBF", + Account: nil, + PollID: "01JDQ0EZ5HM9T4WXRQ5WSVD40J", + Poll: nil, + CreatedAt: TimeMustParse("2024-11-01T09:01:30+02:00"), + }, + "remote_account_1_status_4_poll_vote_local_account_2": { + ID: "01JDQ0T3EEDN7SAVBQMQP4PR12", + Choices: []int{4}, + AccountID: "01F8MH5NBDF2MV7CTC4Q5128HF", + Account: nil, + PollID: "01JDQ0EZ5HM9T4WXRQ5WSVD40J", + Poll: nil, + CreatedAt: TimeMustParse("2024-11-01T09:02:30+02:00"), + }, } } @@ -2341,7 +2457,6 @@ func NewTestMentions() map[string]*gtsmodel.Mention { ID: "01FCTA2Y6FGHXQA4ZE6N5NMNEX", StatusID: "01FCTA44PW9H1TB328S9AQXKDS", CreatedAt: TimeMustParse("2022-05-14T13:21:09+02:00"), - UpdatedAt: TimeMustParse("2022-05-14T13:21:09+02:00"), OriginAccountID: "01F8MH1H7YV1Z7D2C8K2730QBF", OriginAccountURI: "http://localhost:8080/users/the_mighty_zork", TargetAccountID: "01F8MH5ZK5VRH73AKHQM6Y9VNX", @@ -2353,7 +2468,6 @@ func NewTestMentions() map[string]*gtsmodel.Mention { ID: "01FDF2HM2NF6FSRZCDEDV451CN", StatusID: "01FCQSQ667XHJ9AV9T27SJJSX5", CreatedAt: TimeMustParse("2022-05-14T13:21:09+02:00"), - UpdatedAt: TimeMustParse("2022-05-14T13:21:09+02:00"), OriginAccountID: "01F8MH5NBDF2MV7CTC4Q5128HF", OriginAccountURI: "http://localhost:8080/users/1happyturtle", TargetAccountID: "01F8MH1H7YV1Z7D2C8K2730QBF", @@ -2365,7 +2479,6 @@ func NewTestMentions() map[string]*gtsmodel.Mention { ID: "01FN3VKDEF4CN2W9TKX339BEHB", StatusID: "01FN3VJGFH10KR7S2PB0GFJZYG", CreatedAt: TimeMustParse("2022-05-14T13:21:09+02:00"), - UpdatedAt: TimeMustParse("2022-05-14T13:21:09+02:00"), OriginAccountID: "01F8MH5NBDF2MV7CTC4Q5128HF", OriginAccountURI: "http://localhost:8080/users/1happyturtle", TargetAccountID: "01F8MH1H7YV1Z7D2C8K2730QBF", @@ -2377,7 +2490,6 @@ func NewTestMentions() map[string]*gtsmodel.Mention { ID: "01FF26A6BGEKCZFWNEHXB2ZZ6M", StatusID: "01FF25D5Q0DH7CHD57CTRS6WK0", CreatedAt: TimeMustParse("2022-05-14T13:21:09+02:00"), - UpdatedAt: TimeMustParse("2022-05-14T13:21:09+02:00"), OriginAccountID: "01F8MH17FWEB39HZJ76B6VXSKF", OriginAccountURI: "http://localhost:8080/users/admin", TargetAccountID: "01F8MH1H7YV1Z7D2C8K2730QBF", @@ -2389,7 +2501,6 @@ func NewTestMentions() map[string]*gtsmodel.Mention { ID: "01J5QVP69ANF1K4WHES6GA4WXP", StatusID: "01J5QVB9VC76NPPRQ207GG4DRZ", CreatedAt: TimeMustParse("2024-02-20T12:41:37+02:00"), - UpdatedAt: TimeMustParse("2024-02-20T12:41:37+02:00"), OriginAccountID: "01F8MH17FWEB39HZJ76B6VXSKF", OriginAccountURI: "http://localhost:8080/users/admin", TargetAccountID: "01F8MH5NBDF2MV7CTC4Q5128HF", @@ -2401,7 +2512,6 @@ func NewTestMentions() map[string]*gtsmodel.Mention { ID: "01HE7XQNMKTVC8MNPCE1JGK4J3", StatusID: "01HE7XJ1CG84TBKH5V9XKBVGF5", CreatedAt: TimeMustParse("2023-11-02T12:44:25+02:00"), - UpdatedAt: TimeMustParse("2023-11-02T12:44:25+02:00"), OriginAccountID: "01FHMQX3GAABWSM0S2VZEC2SWC", OriginAccountURI: "http://example.org/users/Some_User", TargetAccountID: "01F8MH17FWEB39HZJ76B6VXSKF", @@ -3490,6 +3600,102 @@ func NewTestInteractionRequests() map[string]*gtsmodel.InteractionRequest { } } +func NewTestStatusEdits() map[string]*gtsmodel.StatusEdit { + return map[string]*gtsmodel.StatusEdit{ + "local_account_1_status_9_edit_1": { + ID: "01JDPZCZ2Y9KSGZW0R7ZG8T8Y2", + Content: "

this is the original status

", + ContentWarning: "", + Text: "this is the original status", + Language: "en", + Sensitive: util.Ptr(false), + AttachmentIDs: nil, + PollOptions: nil, + PollVotes: nil, + StatusID: "01JDPZC707CKDN8N4QVWM4Z1NR", + CreatedAt: TimeMustParse("2024-11-01T11:00:00+02:00"), + }, + "local_account_1_status_9_edit_2": { + ID: "01JDPZDADMD1T9HKF94RECF7PP", + Content: "

this is the first status edit! now with content-warning

", + ContentWarning: "edited status", + Text: "this is the first status edit! now with content-warning", + Language: "en", + Sensitive: util.Ptr(false), + AttachmentIDs: nil, + PollOptions: nil, + PollVotes: nil, + StatusID: "01JDPZC707CKDN8N4QVWM4Z1NR", + CreatedAt: TimeMustParse("2024-11-01T11:01:00+02:00"), + }, + "local_account_2_status_9_edit_1": { + ID: "01JDPZPBXAX0M02YSEPB21KX4R", + Content: "

this is the original status

", + ContentWarning: "", + Text: "this is the original status", + Language: "en", + Sensitive: util.Ptr(false), + AttachmentIDs: nil, + PollOptions: nil, + PollVotes: nil, + StatusID: "01JDPZEZ77X1NX0TY9M10BK1HM", + CreatedAt: TimeMustParse("2024-11-01T10:00:00+02:00"), + }, + "local_account_2_status_9_edit_2": { + ID: "01JDPZPJHKP7E3M0YQXEXPS1YT", + Content: "

now edited to have some media!

", + ContentWarning: "edit with media attachments", + Text: "now edited to have some media!", + Language: "en", + Sensitive: util.Ptr(true), + AttachmentIDs: []string{"01JDQ164HM08SGJ7ZEK9003Z4B"}, + PollOptions: nil, + PollVotes: nil, + StatusID: "01JDPZEZ77X1NX0TY9M10BK1HM", + CreatedAt: TimeMustParse("2024-11-01T10:01:00+02:00"), + }, + "local_account_2_status_9_edit_3": { + ID: "01JDPZPY3F85Y7B78ETRXEMWD9", + Content: "

now edited to remove the media

", + ContentWarning: "edit missing previous media attachments", + Text: "now edited to remove the media", + Language: "en", + Sensitive: util.Ptr(false), + AttachmentIDs: nil, + PollOptions: nil, + PollVotes: nil, + StatusID: "01JDPZEZ77X1NX0TY9M10BK1HM", + CreatedAt: TimeMustParse("2024-11-01T10:02:00+02:00"), + }, + "remote_account_1_status_4_edit_1": { + ID: "01JDQ07ZZ4FGP13YN8TF63P5A6", + Content: "

this is the original status, with a poll!

", + ContentWarning: "", + Text: "this is the original status, with a poll!", + Language: "en", + Sensitive: util.Ptr(false), + AttachmentIDs: nil, + PollOptions: []string{"yes", "no", "spiderman"}, + PollVotes: []int{42, 42, 69}, + StatusID: "01JDQ07JZTX9CMDJP67CNA71YD", + CreatedAt: TimeMustParse("2024-11-01T09:00:00+02:00"), + }, + "remote_account_1_status_4_edit_2": { + ID: "01JDQ08AYQC0G6413VAHA51CV9", + Content: "

this is the first status edit! now with a different poll!

", + ContentWarning: "edited status", + Text: "this is the first status edit! now with a different poll!", + Language: "en", + Sensitive: util.Ptr(false), + AttachmentIDs: nil, + PollOptions: []string{"yes", "no", "maybe", "i don't know", "can you repeat the question"}, + PollVotes: []int{0, 0, 0, 0, 1}, + StatusID: "01JDQ07JZTX9CMDJP67CNA71YD", + CreatedAt: TimeMustParse("2024-11-01T09:01:00+02:00"), + }, + } +} + // GetSignatureForActivity prepares a mock HTTP request as if it were going to deliver activity to destination signed for privkey and pubKeyID, signs the request and returns the header values. func GetSignatureForActivity(activity pub.Activity, pubKeyID string, privkey *rsa.PrivateKey, destination *url.URL) (signatureHeader string, digestHeader string, dateHeader string) { // convert the activity into json bytes