diff --git a/internal/ap/activitystreams_test.go b/internal/ap/activitystreams_test.go index edade9718..050825a1a 100644 --- a/internal/ap/activitystreams_test.go +++ b/internal/ap/activitystreams_test.go @@ -25,6 +25,7 @@ import ( "github.com/superseriousbusiness/activity/streams/vocab" "github.com/superseriousbusiness/gotosocial/internal/ap" "github.com/superseriousbusiness/gotosocial/internal/paging" + "github.com/superseriousbusiness/gotosocial/internal/util" ) func TestASCollection(t *testing.T) { @@ -51,7 +52,7 @@ func TestASCollection(t *testing.T) { ID: parseURI(idURI), First: new(paging.Page), Query: url.Values{"limit": []string{"40"}}, - Total: total, + Total: util.Ptr(total), }) // Serialize collection. @@ -82,7 +83,7 @@ func TestASCollectionTotalOnly(t *testing.T) { // Create new collection using builder function. c := ap.NewASCollection(ap.CollectionParams{ ID: parseURI(idURI), - Total: total, + Total: util.Ptr(total), }) // Serialize collection. @@ -128,7 +129,7 @@ func TestASCollectionPage(t *testing.T) { p := ap.NewASCollectionPage(ap.CollectionPageParams{ CollectionParams: ap.CollectionParams{ ID: parseURI(idURI), - Total: total, + Total: util.Ptr(total), }, Current: currPg, @@ -166,7 +167,7 @@ func TestASOrderedCollection(t *testing.T) { ID: parseURI(idURI), First: new(paging.Page), Query: url.Values{"limit": []string{"40"}}, - Total: total, + Total: util.Ptr(total), }) // Serialize collection. @@ -193,7 +194,31 @@ func TestASOrderedCollectionTotalOnly(t *testing.T) { // Create new collection using builder function. c := ap.NewASOrderedCollection(ap.CollectionParams{ ID: parseURI(idURI), - Total: total, + Total: util.Ptr(total), + }) + + // Serialize collection. + s := toJSON(c) + + // Ensure outputs are equal. + assert.Equal(t, expect, s) +} + +func TestASOrderedCollectionNoTotal(t *testing.T) { + const ( + idURI = "https://zorg.flabormagorg.xyz/users/itsa_me_mario" + ) + + // Create JSON string of expected output. + expect := toJSON(map[string]any{ + "@context": "https://www.w3.org/ns/activitystreams", + "type": "OrderedCollection", + "id": idURI, + }) + + // Create new collection using builder function. + c := ap.NewASOrderedCollection(ap.CollectionParams{ + ID: parseURI(idURI), }) // Serialize collection. @@ -239,7 +264,7 @@ func TestASOrderedCollectionPage(t *testing.T) { p := ap.NewASOrderedCollectionPage(ap.CollectionPageParams{ CollectionParams: ap.CollectionParams{ ID: parseURI(idURI), - Total: total, + Total: util.Ptr(total), }, Current: currPg, diff --git a/internal/ap/collections.go b/internal/ap/collections.go index 62c81fd57..43b7541e4 100644 --- a/internal/ap/collections.go +++ b/internal/ap/collections.go @@ -321,7 +321,8 @@ type CollectionParams struct { Query url.Values // Total no. items. - Total int + // Omitted if nil. + Total *int } type CollectionPageParams struct { @@ -367,6 +368,7 @@ type CollectionPageBuilder interface { // vocab.ActivityStreamsOrderedItemsProperty type ItemsPropertyBuilder interface { AppendIRI(*url.URL) + AppendActivityStreamsCreate(vocab.ActivityStreamsCreate) // NOTE: add more of the items-property-like interface // functions here as you require them for building pages. @@ -409,9 +411,11 @@ func buildCollection[C CollectionBuilder](collection C, params CollectionParams) collection.SetJSONLDId(idProp) // Add the collection totalItems count property. - totalItems := streams.NewActivityStreamsTotalItemsProperty() - totalItems.Set(params.Total) - collection.SetActivityStreamsTotalItems(totalItems) + if params.Total != nil { + totalItems := streams.NewActivityStreamsTotalItemsProperty() + totalItems.Set(*params.Total) + collection.SetActivityStreamsTotalItems(totalItems) + } // No First page means we're done. if params.First == nil { @@ -497,9 +501,11 @@ func buildCollectionPage[C CollectionPageBuilder, I ItemsPropertyBuilder](collec } // Add the collection totalItems count property. - totalItems := streams.NewActivityStreamsTotalItemsProperty() - totalItems.Set(params.Total) - collectionPage.SetActivityStreamsTotalItems(totalItems) + if params.Total != nil { + totalItems := streams.NewActivityStreamsTotalItemsProperty() + totalItems.Set(*params.Total) + collectionPage.SetActivityStreamsTotalItems(totalItems) + } if params.Append == nil { // nil check outside the for loop. diff --git a/internal/api/activitypub/users/outboxget.go b/internal/api/activitypub/users/outboxget.go index 7dcc354ac..43379ad8f 100644 --- a/internal/api/activitypub/users/outboxget.go +++ b/internal/api/activitypub/users/outboxget.go @@ -19,14 +19,13 @@ package users import ( "errors" - "fmt" "net/http" - "strconv" "strings" "github.com/gin-gonic/gin" apiutil "github.com/superseriousbusiness/gotosocial/internal/api/util" "github.com/superseriousbusiness/gotosocial/internal/gtserror" + "github.com/superseriousbusiness/gotosocial/internal/paging" ) // OutboxGETHandler swagger:operation GET /users/{username}/outbox s2sOutboxGet @@ -105,30 +104,17 @@ func (m *Module) OutboxGETHandler(c *gin.Context) { return } - var page bool - if pageString := c.Query(PageKey); pageString != "" { - i, err := strconv.ParseBool(pageString) - if err != nil { - err := fmt.Errorf("error parsing %s: %s", PageKey, err) - apiutil.ErrorHandler(c, gtserror.NewErrorBadRequest(err, err.Error()), m.processor.InstanceGetV1) - return - } - page = i + page, errWithCode := paging.ParseIDPage(c, + 1, // min limit + 80, // max limit + 0, // default = disabled + ) + if errWithCode != nil { + apiutil.ErrorHandler(c, errWithCode, m.processor.InstanceGetV1) + return } - minID := "" - minIDString := c.Query(MinIDKey) - if minIDString != "" { - minID = minIDString - } - - maxID := "" - maxIDString := c.Query(MaxIDKey) - if maxIDString != "" { - maxID = maxIDString - } - - resp, errWithCode := m.processor.Fedi().OutboxGet(c.Request.Context(), requestedUsername, page, maxID, minID) + resp, errWithCode := m.processor.Fedi().OutboxGet(c.Request.Context(), requestedUsername, page) if errWithCode != nil { apiutil.ErrorHandler(c, errWithCode, m.processor.InstanceGetV1) return diff --git a/internal/api/activitypub/users/outboxget_test.go b/internal/api/activitypub/users/outboxget_test.go index 55e9f2f78..521af0ff0 100644 --- a/internal/api/activitypub/users/outboxget_test.go +++ b/internal/api/activitypub/users/outboxget_test.go @@ -80,8 +80,9 @@ func (suite *OutboxGetTestSuite) TestGetOutbox() { suite.NoError(err) suite.Equal(`{ "@context": "https://www.w3.org/ns/activitystreams", - "first": "http://localhost:8080/users/the_mighty_zork/outbox?page=true", + "first": "http://localhost:8080/users/the_mighty_zork/outbox?limit=40", "id": "http://localhost:8080/users/the_mighty_zork/outbox", + "totalItems": 7, "type": "OrderedCollection" }`, dst.String()) @@ -105,7 +106,7 @@ func (suite *OutboxGetTestSuite) TestGetOutboxFirstPage() { // setup request recorder := httptest.NewRecorder() ctx, _ := testrig.CreateGinTestContext(recorder, nil) - ctx.Request = httptest.NewRequest(http.MethodGet, targetAccount.OutboxURI+"?page=true", nil) // the endpoint we're hitting + ctx.Request = httptest.NewRequest(http.MethodGet, targetAccount.OutboxURI+"?limit=40", nil) // the endpoint we're hitting ctx.Request.Header.Set("accept", "application/activity+json") ctx.Request.Header.Set("Signature", signedRequest.SignatureHeader) ctx.Request.Header.Set("Date", signedRequest.DateHeader) @@ -138,8 +139,8 @@ func (suite *OutboxGetTestSuite) TestGetOutboxFirstPage() { suite.NoError(err) suite.Equal(`{ "@context": "https://www.w3.org/ns/activitystreams", - "id": "http://localhost:8080/users/the_mighty_zork/outbox?page=true", - "next": "http://localhost:8080/users/the_mighty_zork/outbox?page=true\u0026max_id=01F8MHAMCHF6Y650WCRSCP4WMY", + "id": "http://localhost:8080/users/the_mighty_zork/outbox?limit=40", + "next": "http://localhost:8080/users/the_mighty_zork/outbox?limit=40\u0026max_id=01F8MHAMCHF6Y650WCRSCP4WMY", "orderedItems": [ { "actor": "http://localhost:8080/users/the_mighty_zork", @@ -159,7 +160,8 @@ func (suite *OutboxGetTestSuite) TestGetOutboxFirstPage() { } ], "partOf": "http://localhost:8080/users/the_mighty_zork/outbox", - "prev": "http://localhost:8080/users/the_mighty_zork/outbox?page=true\u0026min_id=01HH9KYNQPA416TNJ53NSATP40", + "prev": "http://localhost:8080/users/the_mighty_zork/outbox?limit=40\u0026min_id=01HH9KYNQPA416TNJ53NSATP40", + "totalItems": 7, "type": "OrderedCollectionPage" }`, dst.String()) @@ -183,7 +185,7 @@ func (suite *OutboxGetTestSuite) TestGetOutboxNextPage() { // setup request recorder := httptest.NewRecorder() ctx, _ := testrig.CreateGinTestContext(recorder, nil) - ctx.Request = httptest.NewRequest(http.MethodGet, targetAccount.OutboxURI+"?page=true&max_id=01F8MHAMCHF6Y650WCRSCP4WMY", nil) // the endpoint we're hitting + ctx.Request = httptest.NewRequest(http.MethodGet, targetAccount.OutboxURI+"?limit=40&max_id=01F8MHAMCHF6Y650WCRSCP4WMY", nil) // the endpoint we're hitting ctx.Request.Header.Set("accept", "application/activity+json") ctx.Request.Header.Set("Signature", signedRequest.SignatureHeader) ctx.Request.Header.Set("Date", signedRequest.DateHeader) @@ -219,9 +221,10 @@ func (suite *OutboxGetTestSuite) TestGetOutboxNextPage() { suite.NoError(err) suite.Equal(`{ "@context": "https://www.w3.org/ns/activitystreams", - "id": "http://localhost:8080/users/the_mighty_zork/outbox?page=true&maxID=01F8MHAMCHF6Y650WCRSCP4WMY", + "id": "http://localhost:8080/users/the_mighty_zork/outbox?limit=40&max_id=01F8MHAMCHF6Y650WCRSCP4WMY", "orderedItems": [], "partOf": "http://localhost:8080/users/the_mighty_zork/outbox", + "totalItems": 7, "type": "OrderedCollectionPage" }`, dst.String()) diff --git a/internal/db/account.go b/internal/db/account.go index 4f02a4d29..45a4ccc09 100644 --- a/internal/db/account.go +++ b/internal/db/account.go @@ -140,10 +140,23 @@ type Account interface { // Update local account settings. UpdateAccountSettings(ctx context.Context, settings *gtsmodel.AccountSettings, columns ...string) error - // PopulateAccountStats gets (or creates and gets) account stats for - // the given account, and attaches them to the account model. + // PopulateAccountStats either creates account stats for the given + // account by performing COUNT(*) database queries, or retrieves + // existing stats from the database, and attaches stats to account. + // + // If account is local and stats were last regenerated > 48 hours ago, + // stats will always be regenerated using COUNT(*) queries, to prevent drift. PopulateAccountStats(ctx context.Context, account *gtsmodel.Account) error + // StubAccountStats creates zeroed account stats for the given account, + // skipping COUNT(*) queries, upserts them in the DB, and attaches them + // to the account model. + // + // Useful following fresh dereference of a remote account, or fresh + // creation of a local account, when you know all COUNT(*) queries + // would return 0 anyway. + StubAccountStats(ctx context.Context, account *gtsmodel.Account) error + // RegenerateAccountStats creates, upserts, and returns stats // for the given account, and attaches them to the account model. // diff --git a/internal/db/bundb/account.go b/internal/db/bundb/account.go index eb5385c70..43978243a 100644 --- a/internal/db/bundb/account.go +++ b/internal/db/bundb/account.go @@ -1217,6 +1217,35 @@ func (a *accountDB) PopulateAccountStats(ctx context.Context, account *gtsmodel. return nil } +func (a *accountDB) StubAccountStats(ctx context.Context, account *gtsmodel.Account) error { + stats := >smodel.AccountStats{ + AccountID: account.ID, + RegeneratedAt: time.Now(), + FollowersCount: util.Ptr(0), + FollowingCount: util.Ptr(0), + FollowRequestsCount: util.Ptr(0), + StatusesCount: util.Ptr(0), + StatusesPinnedCount: util.Ptr(0), + } + + // Upsert this stats in case a race + // meant someone else inserted it first. + if err := a.state.Caches.GTS.AccountStats.Store(stats, func() error { + if _, err := NewUpsert(a.db). + Model(stats). + Constraint("account_id"). + Exec(ctx); err != nil { + return err + } + return nil + }); err != nil { + return err + } + + account.Stats = stats + return nil +} + func (a *accountDB) RegenerateAccountStats(ctx context.Context, account *gtsmodel.Account) error { // Initialize a new stats struct. stats := >smodel.AccountStats{ diff --git a/internal/db/bundb/admin.go b/internal/db/bundb/admin.go index e9191b7c7..ff398fca5 100644 --- a/internal/db/bundb/admin.go +++ b/internal/db/bundb/admin.go @@ -120,16 +120,6 @@ func (a *adminDB) NewSignup(ctx context.Context, newSignup gtsmodel.NewSignup) ( return nil, err } - settings := >smodel.AccountSettings{ - AccountID: accountID, - Privacy: gtsmodel.VisibilityDefault, - } - - // Insert the settings! - if err := a.state.DB.PutAccountSettings(ctx, settings); err != nil { - return nil, err - } - account = >smodel.Account{ ID: accountID, Username: newSignup.Username, @@ -145,13 +135,26 @@ func (a *adminDB) NewSignup(ctx context.Context, newSignup gtsmodel.NewSignup) ( PrivateKey: privKey, PublicKey: &privKey.PublicKey, PublicKeyURI: uris.PublicKeyURI, - Settings: settings, } // Insert the new account! if err := a.state.DB.PutAccount(ctx, account); err != nil { return nil, err } + + // Insert basic settings for new account. + account.Settings = >smodel.AccountSettings{ + AccountID: accountID, + Privacy: gtsmodel.VisibilityDefault, + } + if err := a.state.DB.PutAccountSettings(ctx, account.Settings); err != nil { + return nil, err + } + + // Stub empty stats for new account. + if err := a.state.DB.StubAccountStats(ctx, account); err != nil { + return nil, err + } } // Created or already had an account. diff --git a/internal/federation/dereferencing/account.go b/internal/federation/dereferencing/account.go index 33a71ceb9..069fca1bc 100644 --- a/internal/federation/dereferencing/account.go +++ b/internal/federation/dereferencing/account.go @@ -102,11 +102,15 @@ func (d *Dereferencer) GetAccountByURI(ctx context.Context, requestUser string, } if accountable != nil { - // This account was updated, enqueue re-dereference featured posts. + // This account was updated, enqueue re-dereference featured posts + stats. d.state.Workers.Dereference.Queue.Push(func(ctx context.Context) { if err := d.dereferenceAccountFeatured(ctx, requestUser, account); err != nil { log.Errorf(ctx, "error fetching account featured collection: %v", err) } + + if err := d.dereferenceAccountStats(ctx, requestUser, account); err != nil { + log.Errorf(ctx, "error fetching account stats: %v", err) + } }) } @@ -150,11 +154,22 @@ func (d *Dereferencer) getAccountByURI(ctx context.Context, requestUser string, } // Create and pass-through a new bare-bones model for dereferencing. - return d.enrichAccountSafely(ctx, requestUser, uri, >smodel.Account{ + account, accountable, err := d.enrichAccountSafely(ctx, requestUser, uri, >smodel.Account{ ID: id.NewULID(), Domain: uri.Host, URI: uriStr, }, nil) + if err != nil { + return nil, nil, err + } + + // We have a new account. Ensure basic account stats populated; + // real stats will be fetched from remote asynchronously. + if err := d.state.DB.StubAccountStats(ctx, account); err != nil { + return nil, nil, gtserror.Newf("error stubbing account stats: %w", err) + } + + return account, accountable, nil } if accountFresh(account, nil) { @@ -199,11 +214,15 @@ func (d *Dereferencer) GetAccountByUsernameDomain(ctx context.Context, requestUs } if accountable != nil { - // This account was updated, enqueue re-dereference featured posts. + // This account was updated, enqueue re-dereference featured posts + stats. d.state.Workers.Dereference.Queue.Push(func(ctx context.Context) { if err := d.dereferenceAccountFeatured(ctx, requestUser, account); err != nil { log.Errorf(ctx, "error fetching account featured collection: %v", err) } + + if err := d.dereferenceAccountStats(ctx, requestUser, account); err != nil { + log.Errorf(ctx, "error fetching account stats: %v", err) + } }) } @@ -251,6 +270,12 @@ func (d *Dereferencer) getAccountByUsernameDomain( return nil, nil, err } + // We have a new account. Ensure basic account stats populated; + // real stats will be fetched from remote asynchronously. + if err := d.state.DB.StubAccountStats(ctx, account); err != nil { + return nil, nil, gtserror.Newf("error stubbing account stats: %w", err) + } + return account, accountable, nil } @@ -320,11 +345,15 @@ func (d *Dereferencer) RefreshAccount( } if accountable != nil { - // This account was updated, enqueue re-dereference featured posts. + // This account was updated, enqueue re-dereference featured posts + stats. d.state.Workers.Dereference.Queue.Push(func(ctx context.Context) { if err := d.dereferenceAccountFeatured(ctx, requestUser, latest); err != nil { log.Errorf(ctx, "error fetching account featured collection: %v", err) } + + if err := d.dereferenceAccountStats(ctx, requestUser, latest); err != nil { + log.Errorf(ctx, "error fetching account stats: %v", err) + } }) } @@ -369,10 +398,14 @@ func (d *Dereferencer) RefreshAccountAsync( } if accountable != nil { - // This account was updated, enqueue re-dereference featured posts. + // This account was updated, enqueue re-dereference featured posts + stats. if err := d.dereferenceAccountFeatured(ctx, requestUser, latest); err != nil { log.Errorf(ctx, "error fetching account featured collection: %v", err) } + + if err := d.dereferenceAccountStats(ctx, requestUser, latest); err != nil { + log.Errorf(ctx, "error fetching account stats: %v", err) + } } }) } @@ -697,12 +730,12 @@ func (d *Dereferencer) enrichAccount( latestAcc.ID = account.ID latestAcc.FetchedAt = time.Now() - // Ensure the account's avatar media is populated, passing in existing to check for chages. + // Ensure the account's avatar media is populated, passing in existing to check for changes. if err := d.fetchRemoteAccountAvatar(ctx, tsport, account, latestAcc); err != nil { log.Errorf(ctx, "error fetching remote avatar for account %s: %v", uri, err) } - // Ensure the account's avatar media is populated, passing in existing to check for chages. + // Ensure the account's avatar media is populated, passing in existing to check for changes. if err := d.fetchRemoteAccountHeader(ctx, tsport, account, latestAcc); err != nil { log.Errorf(ctx, "error fetching remote header for account %s: %v", uri, err) } @@ -712,11 +745,6 @@ func (d *Dereferencer) enrichAccount( log.Errorf(ctx, "error fetching remote emojis for account %s: %v", uri, err) } - // Fetch followers/following count for this account. - if err := d.fetchRemoteAccountStats(ctx, latestAcc, requestUser); err != nil { - log.Errorf(ctx, "error fetching remote stats for account %s: %v", uri, err) - } - if account.IsNew() { // Prefer published/created time from // apubAcc, fall back to FetchedAt value. @@ -1007,7 +1035,7 @@ func (d *Dereferencer) fetchRemoteAccountEmojis(ctx context.Context, targetAccou return changed, nil } -func (d *Dereferencer) fetchRemoteAccountStats(ctx context.Context, account *gtsmodel.Account, requestUser string) error { +func (d *Dereferencer) dereferenceAccountStats(ctx context.Context, requestUser string, account *gtsmodel.Account) error { // Ensure we have a stats model for this account. if account.Stats == nil { if err := d.state.DB.PopulateAccountStats(ctx, account); err != nil { diff --git a/internal/processing/fedi/collections.go b/internal/processing/fedi/collections.go index 7a6c99adb..a1fc5e1f4 100644 --- a/internal/processing/fedi/collections.go +++ b/internal/processing/fedi/collections.go @@ -29,6 +29,8 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/gtserror" "github.com/superseriousbusiness/gotosocial/internal/log" "github.com/superseriousbusiness/gotosocial/internal/paging" + "github.com/superseriousbusiness/gotosocial/internal/typeutils" + "github.com/superseriousbusiness/gotosocial/internal/util" ) // InboxPost handles POST requests to a user's inbox for new activitypub messages. @@ -45,91 +47,32 @@ func (p *Processor) InboxPost(ctx context.Context, w http.ResponseWriter, r *htt return p.federator.FederatingActor().PostInbox(ctx, w, r) } -// OutboxGet returns the activitypub representation of a local user's outbox. -// This contains links to PUBLIC posts made by this user. +// OutboxGet returns the serialized ActivityPub +// collection of a local account's outbox, which +// contains links to PUBLIC posts by this account. func (p *Processor) OutboxGet( ctx context.Context, requestedUser string, - page bool, - maxID string, - minID string, + page *paging.Page, ) (interface{}, gtserror.WithCode) { - // Authenticate the incoming request, getting related user accounts. - _, receiver, errWithCode := p.authenticate(ctx, requestedUser) - if errWithCode != nil { - return nil, errWithCode - } - - var data map[string]interface{} - // There are two scenarios: - // 1. we're asked for the whole collection and not a page -- we can just return the collection, with no items, but a link to 'first' page. - // 2. we're asked for a specific page; this can be either the first page or any other page - - if !page { - /* - scenario 1: return the collection with no items - we want something that looks like this: - { - "@context": "https://www.w3.org/ns/activitystreams", - "id": "https://example.org/users/whatever/outbox", - "type": "OrderedCollection", - "first": "https://example.org/users/whatever/outbox?page=true", - "last": "https://example.org/users/whatever/outbox?min_id=0&page=true" - } - */ - collection, err := p.converter.OutboxToASCollection(ctx, receiver.OutboxURI) - if err != nil { - return nil, gtserror.NewErrorInternalError(err) - } - - data, err = ap.Serialize(collection) - if err != nil { - return nil, gtserror.NewErrorInternalError(err) - } - - return data, nil - } - - // scenario 2 -- get the requested page - // limit pages to 30 entries per page - publicStatuses, err := p.state.DB.GetAccountStatuses(ctx, receiver.ID, 30, true, true, maxID, minID, false, true) - if err != nil && !errors.Is(err, db.ErrNoEntries) { - return nil, gtserror.NewErrorInternalError(err) - } - - outboxPage, err := p.converter.StatusesToASOutboxPage(ctx, receiver.OutboxURI, maxID, minID, publicStatuses) - if err != nil { - return nil, gtserror.NewErrorInternalError(err) - } - - data, err = ap.Serialize(outboxPage) - if err != nil { - return nil, gtserror.NewErrorInternalError(err) - } - - return data, nil -} - -// FollowersGet handles the getting of a fedi/activitypub representation of a user/account's followers, performing appropriate -// authentication before returning a JSON serializable interface to the caller. -func (p *Processor) FollowersGet(ctx context.Context, requestedUser string, page *paging.Page) (interface{}, gtserror.WithCode) { - // Authenticate the incoming request, getting related user accounts. - _, receiver, errWithCode := p.authenticate(ctx, requestedUser) + // Authenticate incoming request, getting related accounts. + auth, errWithCode := p.authenticate(ctx, requestedUser) if errWithCode != nil { return nil, errWithCode } + receivingAcct := auth.receivingAcct // Parse the collection ID object from account's followers URI. - collectionID, err := url.Parse(receiver.FollowersURI) + collectionID, err := url.Parse(receivingAcct.OutboxURI) if err != nil { - err := gtserror.Newf("error parsing account followers uri %s: %w", receiver.FollowersURI, err) + err := gtserror.Newf("error parsing account outbox uri %s: %w", receivingAcct.OutboxURI, err) return nil, gtserror.NewErrorInternalError(err) } // Ensure we have stats for this account. - if receiver.Stats == nil { - if err := p.state.DB.PopulateAccountStats(ctx, receiver); err != nil { - err := gtserror.Newf("error getting stats for account %s: %w", receiver.ID, err) + if receivingAcct.Stats == nil { + if err := p.state.DB.PopulateAccountStats(ctx, receivingAcct); err != nil { + err := gtserror.Newf("error getting stats for account %s: %w", receivingAcct.ID, err) return nil, gtserror.NewErrorInternalError(err) } } @@ -139,29 +82,160 @@ func (p *Processor) FollowersGet(ctx context.Context, requestedUser string, page // Start the AS collection params. var params ap.CollectionParams params.ID = collectionID - params.Total = *receiver.Stats.FollowersCount switch { - case receiver.IsInstance() || - *receiver.Settings.HideCollections: - // Instance account (can't follow/be followed), - // or an account that hides followers/following. - // Respect this by just returning totalItems. + case *receivingAcct.Settings.HideCollections || + receivingAcct.IsInstance(): + // If account that hides collections, or instance + // account (ie., can't post / have relationships), + // just return barest stub of collection. obj = ap.NewASOrderedCollection(params) - case page == nil: - // i.e. paging disabled, return collection - // that links to first page (i.e. path below). + case page == nil || auth.handshakingURI != nil: + // If paging disabled, or we're currently handshaking + // the requester, just return collection that links + // to first page (i.e. path below), with no items. + params.Total = util.Ptr(*receivingAcct.Stats.StatusesCount) params.First = new(paging.Page) params.Query = make(url.Values, 1) params.Query.Set("limit", "40") // enables paging obj = ap.NewASOrderedCollection(params) default: - // i.e. paging enabled - // Get the request page of full follower objects with attached accounts. - followers, err := p.state.DB.GetAccountFollowers(ctx, receiver.ID, page) + // Paging enabled. + // Get page of full public statuses. + statuses, err := p.state.DB.GetAccountStatuses( + ctx, + receivingAcct.ID, + page.GetLimit(), // limit + true, // excludeReplies + true, // excludeReblogs + page.GetMax(), // maxID + page.GetMin(), // minID + false, // mediaOnly + true, // publicOnly + ) + if err != nil && !errors.Is(err, db.ErrNoEntries) { + err := gtserror.Newf("error getting statuses: %w", err) + return nil, gtserror.NewErrorInternalError(err) + } + + // page ID values. + var lo, hi string + + if len(statuses) > 0 { + // Get the lowest and highest + // ID values, used for paging. + lo = statuses[len(statuses)-1].ID + hi = statuses[0].ID + } + + // Start building AS collection page params. + params.Total = util.Ptr(*receivingAcct.Stats.StatusesCount) + var pageParams ap.CollectionPageParams + pageParams.CollectionParams = params + + // Current page details. + pageParams.Current = page + pageParams.Count = len(statuses) + + // Set linked next/prev parameters. + pageParams.Next = page.Next(lo, hi) + pageParams.Prev = page.Prev(lo, hi) + + // Set the collection item property builder function. + pageParams.Append = func(i int, itemsProp ap.ItemsPropertyBuilder) { + // Get status at index. + status := statuses[i] + + // Derive statusable from status. + statusable, err := p.converter.StatusToAS(ctx, status) + if err != nil { + log.Errorf(ctx, "error converting %s to statusable: %v", status.URI, err) + return + } + + // Derive create from statusable, using the IRI only. + create := typeutils.WrapStatusableInCreate(statusable, true) + + // Add to item property. + itemsProp.AppendActivityStreamsCreate(create) + } + + // Build AS collection page object from params. + obj = ap.NewASOrderedCollectionPage(pageParams) + } + + // Serialize the prepared object. + data, err := ap.Serialize(obj) + if err != nil { + err := gtserror.Newf("error serializing: %w", err) + return nil, gtserror.NewErrorInternalError(err) + } + + return data, nil +} + +// FollowersGet returns the serialized ActivityPub +// collection of a local account's followers collection, +// which contains links to accounts following this account. +func (p *Processor) FollowersGet( + ctx context.Context, + requestedUser string, + page *paging.Page, +) (interface{}, gtserror.WithCode) { + // Authenticate incoming request, getting related accounts. + auth, errWithCode := p.authenticate(ctx, requestedUser) + if errWithCode != nil { + return nil, errWithCode + } + receivingAcct := auth.receivingAcct + + // Parse the collection ID object from account's followers URI. + collectionID, err := url.Parse(receivingAcct.FollowersURI) + if err != nil { + err := gtserror.Newf("error parsing account followers uri %s: %w", receivingAcct.FollowersURI, err) + return nil, gtserror.NewErrorInternalError(err) + } + + // Ensure we have stats for this account. + if receivingAcct.Stats == nil { + if err := p.state.DB.PopulateAccountStats(ctx, receivingAcct); err != nil { + err := gtserror.Newf("error getting stats for account %s: %w", receivingAcct.ID, err) + return nil, gtserror.NewErrorInternalError(err) + } + } + + var obj vocab.Type + + // Start the AS collection params. + var params ap.CollectionParams + params.ID = collectionID + + switch { + + case receivingAcct.IsInstance() || + *receivingAcct.Settings.HideCollections: + // If account that hides collections, or instance + // account (ie., can't post / have relationships), + // just return barest stub of collection. + obj = ap.NewASOrderedCollection(params) + + case page == nil || auth.handshakingURI != nil: + // If paging disabled, or we're currently handshaking + // the requester, just return collection that links + // to first page (i.e. path below), with no items. + params.Total = util.Ptr(*receivingAcct.Stats.FollowersCount) + params.First = new(paging.Page) + params.Query = make(url.Values, 1) + params.Query.Set("limit", "40") // enables paging + obj = ap.NewASOrderedCollection(params) + + default: + // Paging enabled. + // Get page of full follower objects with attached accounts. + followers, err := p.state.DB.GetAccountFollowers(ctx, receivingAcct.ID, page) if err != nil { err := gtserror.Newf("error getting followers: %w", err) return nil, gtserror.NewErrorInternalError(err) @@ -178,6 +252,7 @@ func (p *Processor) FollowersGet(ctx context.Context, requestedUser string, page } // Start building AS collection page params. + params.Total = util.Ptr(*receivingAcct.Stats.FollowersCount) var pageParams ap.CollectionPageParams pageParams.CollectionParams = params @@ -210,7 +285,7 @@ func (p *Processor) FollowersGet(ctx context.Context, requestedUser string, page obj = ap.NewASOrderedCollectionPage(pageParams) } - // Serialized the prepared object. + // Serialize the prepared object. data, err := ap.Serialize(obj) if err != nil { err := gtserror.Newf("error serializing: %w", err) @@ -220,26 +295,28 @@ func (p *Processor) FollowersGet(ctx context.Context, requestedUser string, page return data, nil } -// FollowingGet handles the getting of a fedi/activitypub representation of a user/account's following, performing appropriate -// authentication before returning a JSON serializable interface to the caller. +// FollowingGet returns the serialized ActivityPub +// collection of a local account's following collection, +// which contains links to accounts followed by this account. func (p *Processor) FollowingGet(ctx context.Context, requestedUser string, page *paging.Page) (interface{}, gtserror.WithCode) { - // Authenticate the incoming request, getting related user accounts. - _, receiver, errWithCode := p.authenticate(ctx, requestedUser) + // Authenticate incoming request, getting related accounts. + auth, errWithCode := p.authenticate(ctx, requestedUser) if errWithCode != nil { return nil, errWithCode } + receivingAcct := auth.receivingAcct // Parse collection ID from account's following URI. - collectionID, err := url.Parse(receiver.FollowingURI) + collectionID, err := url.Parse(receivingAcct.FollowingURI) if err != nil { - err := gtserror.Newf("error parsing account following uri %s: %w", receiver.FollowingURI, err) + err := gtserror.Newf("error parsing account following uri %s: %w", receivingAcct.FollowingURI, err) return nil, gtserror.NewErrorInternalError(err) } // Ensure we have stats for this account. - if receiver.Stats == nil { - if err := p.state.DB.PopulateAccountStats(ctx, receiver); err != nil { - err := gtserror.Newf("error getting stats for account %s: %w", receiver.ID, err) + if receivingAcct.Stats == nil { + if err := p.state.DB.PopulateAccountStats(ctx, receivingAcct); err != nil { + err := gtserror.Newf("error getting stats for account %s: %w", receivingAcct.ID, err) return nil, gtserror.NewErrorInternalError(err) } } @@ -249,28 +326,29 @@ func (p *Processor) FollowingGet(ctx context.Context, requestedUser string, page // Start AS collection params. var params ap.CollectionParams params.ID = collectionID - params.Total = *receiver.Stats.FollowingCount switch { - case receiver.IsInstance() || - *receiver.Settings.HideCollections: - // Instance account (can't follow/be followed), - // or an account that hides followers/following. - // Respect this by just returning totalItems. + case receivingAcct.IsInstance() || + *receivingAcct.Settings.HideCollections: + // If account that hides collections, or instance + // account (ie., can't post / have relationships), + // just return barest stub of collection. obj = ap.NewASOrderedCollection(params) - case page == nil: - // i.e. paging disabled, return collection - // that links to first page (i.e. path below). + case page == nil || auth.handshakingURI != nil: + // If paging disabled, or we're currently handshaking + // the requester, just return collection that links + // to first page (i.e. path below), with no items. + params.Total = util.Ptr(*receivingAcct.Stats.FollowingCount) params.First = new(paging.Page) params.Query = make(url.Values, 1) params.Query.Set("limit", "40") // enables paging obj = ap.NewASOrderedCollection(params) default: - // i.e. paging enabled - // Get the request page of full follower objects with attached accounts. - follows, err := p.state.DB.GetAccountFollows(ctx, receiver.ID, page) + // Paging enabled. + // Get page of full follower objects with attached accounts. + follows, err := p.state.DB.GetAccountFollows(ctx, receivingAcct.ID, page) if err != nil { err := gtserror.Newf("error getting follows: %w", err) return nil, gtserror.NewErrorInternalError(err) @@ -287,6 +365,7 @@ func (p *Processor) FollowingGet(ctx context.Context, requestedUser string, page } // Start AS collection page params. + params.Total = util.Ptr(*receivingAcct.Stats.FollowingCount) var pageParams ap.CollectionPageParams pageParams.CollectionParams = params @@ -319,7 +398,7 @@ func (p *Processor) FollowingGet(ctx context.Context, requestedUser string, page obj = ap.NewASOrderedCollectionPage(pageParams) } - // Serialized the prepared object. + // Serialize the prepared object. data, err := ap.Serialize(obj) if err != nil { err := gtserror.Newf("error serializing: %w", err) @@ -332,20 +411,21 @@ func (p *Processor) FollowingGet(ctx context.Context, requestedUser string, page // FeaturedCollectionGet returns an ordered collection of the requested username's Pinned posts. // The returned collection have an `items` property which contains an ordered list of status URIs. func (p *Processor) FeaturedCollectionGet(ctx context.Context, requestedUser string) (interface{}, gtserror.WithCode) { - // Authenticate the incoming request, getting related user accounts. - _, receiver, errWithCode := p.authenticate(ctx, requestedUser) + // Authenticate incoming request, getting related accounts. + auth, errWithCode := p.authenticate(ctx, requestedUser) if errWithCode != nil { return nil, errWithCode } + receivingAcct := auth.receivingAcct - statuses, err := p.state.DB.GetAccountPinnedStatuses(ctx, receiver.ID) + statuses, err := p.state.DB.GetAccountPinnedStatuses(ctx, receivingAcct.ID) if err != nil { if !errors.Is(err, db.ErrNoEntries) { return nil, gtserror.NewErrorInternalError(err) } } - collection, err := p.converter.StatusesToASFeaturedCollection(ctx, receiver.FeaturedCollectionURI, statuses) + collection, err := p.converter.StatusesToASFeaturedCollection(ctx, receivingAcct.FeaturedCollectionURI, statuses) if err != nil { return nil, gtserror.NewErrorInternalError(err) } diff --git a/internal/processing/fedi/common.go b/internal/processing/fedi/common.go index a02779e73..1a4d38bc1 100644 --- a/internal/processing/fedi/common.go +++ b/internal/processing/fedi/common.go @@ -20,55 +20,64 @@ package fedi import ( "context" "errors" + "net/url" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/gtserror" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" ) -func (p *Processor) authenticate(ctx context.Context, requestedUser string) ( - *gtsmodel.Account, // requester: i.e. user making the request - *gtsmodel.Account, // receiver: i.e. the receiving inbox user - gtserror.WithCode, -) { +type commonAuth struct { + handshakingURI *url.URL // Set to requestingAcct's URI if we're currently handshaking them. + requestingAcct *gtsmodel.Account // Remote account making request to this instance. + receivingAcct *gtsmodel.Account // Local account receiving the request. +} + +func (p *Processor) authenticate(ctx context.Context, requestedUser string) (*commonAuth, gtserror.WithCode) { // First get the requested (receiving) LOCAL account with username from database. receiver, err := p.state.DB.GetAccountByUsernameDomain(ctx, requestedUser, "") if err != nil { if !errors.Is(err, db.ErrNoEntries) { // Real db error. err = gtserror.Newf("db error getting account %s: %w", requestedUser, err) - return nil, nil, gtserror.NewErrorInternalError(err) + return nil, gtserror.NewErrorInternalError(err) } // Account just not found in the db. - return nil, nil, gtserror.NewErrorNotFound(err) + return nil, gtserror.NewErrorNotFound(err) } // Ensure request signed, and use signature URI to // get requesting account, dereferencing if necessary. pubKeyAuth, errWithCode := p.federator.AuthenticateFederatedRequest(ctx, requestedUser) if errWithCode != nil { - return nil, nil, errWithCode + return nil, errWithCode } if pubKeyAuth.Handshaking { - // This should happen very rarely, we are in the middle of handshaking. - err := gtserror.Newf("network race handshaking %s", pubKeyAuth.OwnerURI) - return nil, nil, gtserror.NewErrorInternalError(err) + // We're still handshaking so we + // don't know the requester yet. + return &commonAuth{ + handshakingURI: pubKeyAuth.OwnerURI, + receivingAcct: receiver, + }, nil } // Get requester from auth. requester := pubKeyAuth.Owner - // Check that block does not exist between receiver and requester. + // Ensure block does not exist between receiver and requester. blocked, err := p.state.DB.IsEitherBlocked(ctx, receiver.ID, requester.ID) if err != nil { err := gtserror.Newf("error checking block: %w", err) - return nil, nil, gtserror.NewErrorInternalError(err) + return nil, gtserror.NewErrorInternalError(err) } else if blocked { const text = "block exists between accounts" - return nil, nil, gtserror.NewErrorForbidden(errors.New(text)) + return nil, gtserror.NewErrorForbidden(errors.New(text)) } - return requester, receiver, nil + return &commonAuth{ + requestingAcct: requester, + receivingAcct: receiver, + }, nil } diff --git a/internal/processing/fedi/status.go b/internal/processing/fedi/status.go index c6d611eee..7c4d4beec 100644 --- a/internal/processing/fedi/status.go +++ b/internal/processing/fedi/status.go @@ -30,23 +30,35 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/log" "github.com/superseriousbusiness/gotosocial/internal/paging" + "github.com/superseriousbusiness/gotosocial/internal/util" ) // StatusGet handles the getting of a fedi/activitypub representation of a local status. // It performs appropriate authentication before returning a JSON serializable interface. func (p *Processor) StatusGet(ctx context.Context, requestedUser string, statusID string) (interface{}, gtserror.WithCode) { - // Authenticate the incoming request, getting related user accounts. - requester, receiver, errWithCode := p.authenticate(ctx, requestedUser) + // Authenticate incoming request, getting related accounts. + auth, errWithCode := p.authenticate(ctx, requestedUser) if errWithCode != nil { return nil, errWithCode } + if auth.handshakingURI != nil { + // We're currently handshaking, which means + // we don't know this account yet. This should + // be a very rare race condition. + err := gtserror.Newf("network race handshaking %s", auth.handshakingURI) + return nil, gtserror.NewErrorInternalError(err) + } + + receivingAcct := auth.receivingAcct + requestingAcct := auth.requestingAcct + status, err := p.state.DB.GetStatusByID(ctx, statusID) if err != nil { return nil, gtserror.NewErrorNotFound(err) } - if status.AccountID != receiver.ID { + if status.AccountID != receivingAcct.ID { const text = "status does not belong to receiving account" return nil, gtserror.NewErrorNotFound(errors.New(text)) } @@ -56,7 +68,7 @@ func (p *Processor) StatusGet(ctx context.Context, requestedUser string, statusI return nil, gtserror.NewErrorNotFound(errors.New(text)) } - visible, err := p.filter.StatusVisible(ctx, requester, status) + visible, err := p.filter.StatusVisible(ctx, requestingAcct, status) if err != nil { return nil, gtserror.NewErrorInternalError(err) } @@ -90,15 +102,26 @@ func (p *Processor) StatusRepliesGet( page *paging.Page, onlyOtherAccounts bool, ) (interface{}, gtserror.WithCode) { - // Authenticate the incoming request, getting related user accounts. - requester, receiver, errWithCode := p.authenticate(ctx, requestedUser) + // Authenticate incoming request, getting related accounts. + auth, errWithCode := p.authenticate(ctx, requestedUser) if errWithCode != nil { return nil, errWithCode } + if auth.handshakingURI != nil { + // We're currently handshaking, which means + // we don't know this account yet. This should + // be a very rare race condition. + err := gtserror.Newf("network race handshaking %s", auth.handshakingURI) + return nil, gtserror.NewErrorInternalError(err) + } + + receivingAcct := auth.receivingAcct + requestingAcct := auth.requestingAcct + // Get target status and ensure visible to requester. status, errWithCode := p.c.GetVisibleTargetStatus(ctx, - requester, + requestingAcct, statusID, nil, // default freshness ) @@ -107,7 +130,7 @@ func (p *Processor) StatusRepliesGet( } // Ensure status is by receiving account. - if status.AccountID != receiver.ID { + if status.AccountID != receivingAcct.ID { const text = "status does not belong to receiving account" return nil, gtserror.NewErrorNotFound(errors.New(text)) } @@ -140,7 +163,7 @@ func (p *Processor) StatusRepliesGet( } // Reslice replies dropping all those invisible to requester. - replies, err = p.filter.StatusesVisible(ctx, requester, replies) + replies, err = p.filter.StatusesVisible(ctx, requestingAcct, replies) if err != nil { err := gtserror.Newf("error filtering status replies: %w", err) return nil, gtserror.NewErrorInternalError(err) @@ -151,7 +174,7 @@ func (p *Processor) StatusRepliesGet( // Start AS collection params. var params ap.CollectionParams params.ID = collectionID - params.Total = len(replies) + params.Total = util.Ptr(len(replies)) if page == nil { // i.e. paging disabled, return collection diff --git a/internal/typeutils/internaltoas.go b/internal/typeutils/internaltoas.go index a795541d0..2fb782029 100644 --- a/internal/typeutils/internaltoas.go +++ b/internal/typeutils/internaltoas.go @@ -1560,39 +1560,6 @@ func (c *Converter) StatusesToASOutboxPage(ctx context.Context, outboxID string, return page, nil } -// OutboxToASCollection returns an ordered collection with appropriate id, next, and last fields. -// The returned collection won't have any actual entries; just links to where entries can be obtained. -// we want something that looks like this: -// -// { -// "@context": "https://www.w3.org/ns/activitystreams", -// "id": "https://example.org/users/whatever/outbox", -// "type": "OrderedCollection", -// "first": "https://example.org/users/whatever/outbox?page=true" -// } -func (c *Converter) OutboxToASCollection(ctx context.Context, outboxID string) (vocab.ActivityStreamsOrderedCollection, error) { - collection := streams.NewActivityStreamsOrderedCollection() - - collectionIDProp := streams.NewJSONLDIdProperty() - outboxIDURI, err := url.Parse(outboxID) - if err != nil { - return nil, fmt.Errorf("error parsing url %s", outboxID) - } - collectionIDProp.SetIRI(outboxIDURI) - collection.SetJSONLDId(collectionIDProp) - - collectionFirstProp := streams.NewActivityStreamsFirstProperty() - collectionFirstPropID := fmt.Sprintf("%s?page=true", outboxID) - collectionFirstPropIDURI, err := url.Parse(collectionFirstPropID) - if err != nil { - return nil, fmt.Errorf("error parsing url %s", collectionFirstPropID) - } - collectionFirstProp.SetIRI(collectionFirstPropIDURI) - collection.SetActivityStreamsFirst(collectionFirstProp) - - return collection, nil -} - // StatusesToASFeaturedCollection converts a slice of statuses into an ordered collection // of URIs, suitable for serializing and serving via the activitypub API. func (c *Converter) StatusesToASFeaturedCollection(ctx context.Context, featuredCollectionID string, statuses []*gtsmodel.Status) (vocab.ActivityStreamsOrderedCollection, error) { diff --git a/internal/typeutils/internaltoas_test.go b/internal/typeutils/internaltoas_test.go index 740938220..26e86c516 100644 --- a/internal/typeutils/internaltoas_test.go +++ b/internal/typeutils/internaltoas_test.go @@ -366,27 +366,6 @@ func (suite *InternalToASTestSuite) TestAccountToASWithSharedInbox() { }`, trimmed) } -func (suite *InternalToASTestSuite) TestOutboxToASCollection() { - testAccount := suite.testAccounts["admin_account"] - ctx := context.Background() - - collection, err := suite.typeconverter.OutboxToASCollection(ctx, testAccount.OutboxURI) - suite.NoError(err) - - ser, err := ap.Serialize(collection) - suite.NoError(err) - - bytes, err := json.MarshalIndent(ser, "", " ") - suite.NoError(err) - - suite.Equal(`{ - "@context": "https://www.w3.org/ns/activitystreams", - "first": "http://localhost:8080/users/admin/outbox?page=true", - "id": "http://localhost:8080/users/admin/outbox", - "type": "OrderedCollection" -}`, string(bytes)) -} - func (suite *InternalToASTestSuite) TestStatusToAS() { testStatus := suite.testStatuses["local_account_1_status_1"] ctx := context.Background() diff --git a/testrig/testmodels.go b/testrig/testmodels.go index e1d99fed8..e3d31b7d2 100644 --- a/testrig/testmodels.go +++ b/testrig/testmodels.go @@ -3227,7 +3227,7 @@ func NewTestDereferenceRequests(accounts map[string]*gtsmodel.Account) map[strin DateHeader: date, } - target = URLMustParse(accounts["local_account_1"].OutboxURI + "?page=true") + target = URLMustParse(accounts["local_account_1"].OutboxURI + "?limit=40") sig, digest, date = GetSignatureForDereference(accounts["remote_account_1"].PublicKeyURI, accounts["remote_account_1"].PrivateKey, target) fossSatanDereferenceZorkOutboxFirst := ActivityWithSignature{ SignatureHeader: sig, @@ -3235,7 +3235,7 @@ func NewTestDereferenceRequests(accounts map[string]*gtsmodel.Account) map[strin DateHeader: date, } - target = URLMustParse(accounts["local_account_1"].OutboxURI + "?page=true&max_id=01F8MHAMCHF6Y650WCRSCP4WMY") + target = URLMustParse(accounts["local_account_1"].OutboxURI + "?limit=40&max_id=01F8MHAMCHF6Y650WCRSCP4WMY") sig, digest, date = GetSignatureForDereference(accounts["remote_account_1"].PublicKeyURI, accounts["remote_account_1"].PrivateKey, target) fossSatanDereferenceZorkOutboxNext := ActivityWithSignature{ SignatureHeader: sig,