mirrors
/
gotosocial
mirror of
1
Fork 0
Code Issues Releases Activity

[bugfix] Deref stats async, serve stub collections if handshaking (#2990) 

* [bugfix] Deref stats async, allow peek if handshaking

* don't return totalItems when handshaking or hiding collections

* use GetLimit()

* use StubAccountStats
This commit is contained in:
tobi 2024-06-11 11:54:59 +02:00 committed by GitHub
parent fd6637df4a
commit 611f9de39b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
14 changed files with 412 additions and 261 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
internal/ap/activitystreams_test.go
View File

@ -25,6 +25,7 @@ import (
"github.com/superseriousbusiness/activity/streams/vocab"
"github.com/superseriousbusiness/gotosocial/internal/ap"
"github.com/superseriousbusiness/gotosocial/internal/paging"
"github.com/superseriousbusiness/gotosocial/internal/util"
)
func TestASCollection(t *testing.T) {
@ -51,7 +52,7 @@ func TestASCollection(t *testing.T) {
ID: parseURI(idURI),
First: new(paging.Page),
Query: url.Values{"limit": []string{"40"}},
Total: total,
Total: util.Ptr(total),
})
// Serialize collection.
@ -82,7 +83,7 @@ func TestASCollectionTotalOnly(t *testing.T) {
// Create new collection using builder function.
c := ap.NewASCollection(ap.CollectionParams{
ID: parseURI(idURI),
Total: total,
Total: util.Ptr(total),
})
// Serialize collection.
@ -128,7 +129,7 @@ func TestASCollectionPage(t *testing.T) {
p := ap.NewASCollectionPage(ap.CollectionPageParams{
CollectionParams: ap.CollectionParams{
ID: parseURI(idURI),
Total: total,
Total: util.Ptr(total),
},
Current: currPg,
@ -166,7 +167,7 @@ func TestASOrderedCollection(t *testing.T) {
ID: parseURI(idURI),
First: new(paging.Page),
Query: url.Values{"limit": []string{"40"}},
Total: total,
Total: util.Ptr(total),
})
// Serialize collection.
@ -193,7 +194,31 @@ func TestASOrderedCollectionTotalOnly(t *testing.T) {
// Create new collection using builder function.
c := ap.NewASOrderedCollection(ap.CollectionParams{
ID: parseURI(idURI),
Total: total,
Total: util.Ptr(total),
})
// Serialize collection.
s := toJSON(c)
// Ensure outputs are equal.
assert.Equal(t, expect, s)
}
func TestASOrderedCollectionNoTotal(t *testing.T) {
const (
idURI = "https://zorg.flabormagorg.xyz/users/itsa_me_mario"
)
// Create JSON string of expected output.
expect := toJSON(map[string]any{
"@context": "https://www.w3.org/ns/activitystreams",
"type": "OrderedCollection",
"id": idURI,
})
// Create new collection using builder function.
c := ap.NewASOrderedCollection(ap.CollectionParams{
ID: parseURI(idURI),
})
// Serialize collection.
@ -239,7 +264,7 @@ func TestASOrderedCollectionPage(t *testing.T) {
p := ap.NewASOrderedCollectionPage(ap.CollectionPageParams{
CollectionParams: ap.CollectionParams{
ID: parseURI(idURI),
Total: total,
Total: util.Ptr(total),
},
Current: currPg,

20
internal/ap/collections.go
View File

@ -321,7 +321,8 @@ type CollectionParams struct {
Query url.Values
// Total no. items.
Total int
// Omitted if nil.
Total *int
}
type CollectionPageParams struct {
@ -367,6 +368,7 @@ type CollectionPageBuilder interface {
// vocab.ActivityStreamsOrderedItemsProperty
type ItemsPropertyBuilder interface {
AppendIRI(*url.URL)
AppendActivityStreamsCreate(vocab.ActivityStreamsCreate)
// NOTE: add more of the items-property-like interface
// functions here as you require them for building pages.
@ -409,9 +411,11 @@ func buildCollection[C CollectionBuilder](collection C, params CollectionParams)
collection.SetJSONLDId(idProp)
// Add the collection totalItems count property.
totalItems := streams.NewActivityStreamsTotalItemsProperty()
totalItems.Set(params.Total)
collection.SetActivityStreamsTotalItems(totalItems)
if params.Total != nil {
totalItems := streams.NewActivityStreamsTotalItemsProperty()
totalItems.Set(*params.Total)
collection.SetActivityStreamsTotalItems(totalItems)
}
// No First page means we're done.
if params.First == nil {
@ -497,9 +501,11 @@ func buildCollectionPage[C CollectionPageBuilder, I ItemsPropertyBuilder](collec
}
// Add the collection totalItems count property.
totalItems := streams.NewActivityStreamsTotalItemsProperty()
totalItems.Set(params.Total)
collectionPage.SetActivityStreamsTotalItems(totalItems)
if params.Total != nil {
totalItems := streams.NewActivityStreamsTotalItemsProperty()
totalItems.Set(*params.Total)
collectionPage.SetActivityStreamsTotalItems(totalItems)
}
if params.Append == nil {
// nil check outside the for loop.

34
internal/api/activitypub/users/outboxget.go
View File

@ -19,14 +19,13 @@ package users
import (
"errors"
"fmt"
"net/http"
"strconv"
"strings"
"github.com/gin-gonic/gin"
apiutil "github.com/superseriousbusiness/gotosocial/internal/api/util"
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
"github.com/superseriousbusiness/gotosocial/internal/paging"
)
// OutboxGETHandler swagger:operation GET /users/{username}/outbox s2sOutboxGet
@ -105,30 +104,17 @@ func (m *Module) OutboxGETHandler(c *gin.Context) {
return
}
var page bool
if pageString := c.Query(PageKey); pageString != "" {
i, err := strconv.ParseBool(pageString)
if err != nil {
err := fmt.Errorf("error parsing %s: %s", PageKey, err)
apiutil.ErrorHandler(c, gtserror.NewErrorBadRequest(err, err.Error()), m.processor.InstanceGetV1)
return
}
page = i
page, errWithCode := paging.ParseIDPage(c,
1, // min limit
80, // max limit
0, // default = disabled
)
if errWithCode != nil {
apiutil.ErrorHandler(c, errWithCode, m.processor.InstanceGetV1)
return
}
minID := ""
minIDString := c.Query(MinIDKey)
if minIDString != "" {
minID = minIDString
}
maxID := ""
maxIDString := c.Query(MaxIDKey)
if maxIDString != "" {
maxID = maxIDString
}
resp, errWithCode := m.processor.Fedi().OutboxGet(c.Request.Context(), requestedUsername, page, maxID, minID)
resp, errWithCode := m.processor.Fedi().OutboxGet(c.Request.Context(), requestedUsername, page)
if errWithCode != nil {
apiutil.ErrorHandler(c, errWithCode, m.processor.InstanceGetV1)
return

17
internal/api/activitypub/users/outboxget_test.go
View File

@ -80,8 +80,9 @@ func (suite *OutboxGetTestSuite) TestGetOutbox() {
suite.NoError(err)
suite.Equal(`{
"@context": "https://www.w3.org/ns/activitystreams",
"first": "http://localhost:8080/users/the_mighty_zork/outbox?page=true",
"first": "http://localhost:8080/users/the_mighty_zork/outbox?limit=40",
"id": "http://localhost:8080/users/the_mighty_zork/outbox",
"totalItems": 7,
"type": "OrderedCollection"
}`, dst.String())
@ -105,7 +106,7 @@ func (suite *OutboxGetTestSuite) TestGetOutboxFirstPage() {
// setup request
recorder := httptest.NewRecorder()
ctx, _ := testrig.CreateGinTestContext(recorder, nil)
ctx.Request = httptest.NewRequest(http.MethodGet, targetAccount.OutboxURI+"?page=true", nil) // the endpoint we're hitting
ctx.Request = httptest.NewRequest(http.MethodGet, targetAccount.OutboxURI+"?limit=40", nil) // the endpoint we're hitting
ctx.Request.Header.Set("accept", "application/activity+json")
ctx.Request.Header.Set("Signature", signedRequest.SignatureHeader)
ctx.Request.Header.Set("Date", signedRequest.DateHeader)
@ -138,8 +139,8 @@ func (suite *OutboxGetTestSuite) TestGetOutboxFirstPage() {
suite.NoError(err)
suite.Equal(`{
"@context": "https://www.w3.org/ns/activitystreams",
"id": "http://localhost:8080/users/the_mighty_zork/outbox?page=true",
"next": "http://localhost:8080/users/the_mighty_zork/outbox?page=true\u0026max_id=01F8MHAMCHF6Y650WCRSCP4WMY",
"id": "http://localhost:8080/users/the_mighty_zork/outbox?limit=40",
"next": "http://localhost:8080/users/the_mighty_zork/outbox?limit=40\u0026max_id=01F8MHAMCHF6Y650WCRSCP4WMY",
"orderedItems": [
{
"actor": "http://localhost:8080/users/the_mighty_zork",
@ -159,7 +160,8 @@ func (suite *OutboxGetTestSuite) TestGetOutboxFirstPage() {
}
],
"partOf": "http://localhost:8080/users/the_mighty_zork/outbox",
"prev": "http://localhost:8080/users/the_mighty_zork/outbox?page=true\u0026min_id=01HH9KYNQPA416TNJ53NSATP40",
"prev": "http://localhost:8080/users/the_mighty_zork/outbox?limit=40\u0026min_id=01HH9KYNQPA416TNJ53NSATP40",
"totalItems": 7,
"type": "OrderedCollectionPage"
}`, dst.String())
@ -183,7 +185,7 @@ func (suite *OutboxGetTestSuite) TestGetOutboxNextPage() {
// setup request
recorder := httptest.NewRecorder()
ctx, _ := testrig.CreateGinTestContext(recorder, nil)
ctx.Request = httptest.NewRequest(http.MethodGet, targetAccount.OutboxURI+"?page=true&max_id=01F8MHAMCHF6Y650WCRSCP4WMY", nil) // the endpoint we're hitting
ctx.Request = httptest.NewRequest(http.MethodGet, targetAccount.OutboxURI+"?limit=40&max_id=01F8MHAMCHF6Y650WCRSCP4WMY", nil) // the endpoint we're hitting
ctx.Request.Header.Set("accept", "application/activity+json")
ctx.Request.Header.Set("Signature", signedRequest.SignatureHeader)
ctx.Request.Header.Set("Date", signedRequest.DateHeader)
@ -219,9 +221,10 @@ func (suite *OutboxGetTestSuite) TestGetOutboxNextPage() {
suite.NoError(err)
suite.Equal(`{
"@context": "https://www.w3.org/ns/activitystreams",
"id": "http://localhost:8080/users/the_mighty_zork/outbox?page=true&maxID=01F8MHAMCHF6Y650WCRSCP4WMY",
"id": "http://localhost:8080/users/the_mighty_zork/outbox?limit=40&max_id=01F8MHAMCHF6Y650WCRSCP4WMY",
"orderedItems": [],
"partOf": "http://localhost:8080/users/the_mighty_zork/outbox",
"totalItems": 7,
"type": "OrderedCollectionPage"
}`, dst.String())

17
internal/db/account.go
View File

@ -140,10 +140,23 @@ type Account interface {
// Update local account settings.
UpdateAccountSettings(ctx context.Context, settings *gtsmodel.AccountSettings, columns ...string) error
// PopulateAccountStats gets (or creates and gets) account stats for
// the given account, and attaches them to the account model.
// PopulateAccountStats either creates account stats for the given
// account by performing COUNT(*) database queries, or retrieves
// existing stats from the database, and attaches stats to account.
//
// If account is local and stats were last regenerated > 48 hours ago,
// stats will always be regenerated using COUNT(*) queries, to prevent drift.
PopulateAccountStats(ctx context.Context, account *gtsmodel.Account) error
// StubAccountStats creates zeroed account stats for the given account,
// skipping COUNT(*) queries, upserts them in the DB, and attaches them
// to the account model.
//
// Useful following fresh dereference of a remote account, or fresh
// creation of a local account, when you know all COUNT(*) queries
// would return 0 anyway.
StubAccountStats(ctx context.Context, account *gtsmodel.Account) error
// RegenerateAccountStats creates, upserts, and returns stats
// for the given account, and attaches them to the account model.
//

29
internal/db/bundb/account.go
View File

@ -1217,6 +1217,35 @@ func (a *accountDB) PopulateAccountStats(ctx context.Context, account *gtsmodel.
return nil
}
func (a *accountDB) StubAccountStats(ctx context.Context, account *gtsmodel.Account) error {
stats := &gtsmodel.AccountStats{
AccountID: account.ID,
RegeneratedAt: time.Now(),
FollowersCount: util.Ptr(0),
FollowingCount: util.Ptr(0),
FollowRequestsCount: util.Ptr(0),
StatusesCount: util.Ptr(0),
StatusesPinnedCount: util.Ptr(0),
}
// Upsert this stats in case a race
// meant someone else inserted it first.
if err := a.state.Caches.GTS.AccountStats.Store(stats, func() error {
if _, err := NewUpsert(a.db).
Model(stats).
Constraint("account_id").
Exec(ctx); err != nil {
return err
}
return nil
}); err != nil {
return err
}
account.Stats = stats
return nil
}
func (a *accountDB) RegenerateAccountStats(ctx context.Context, account *gtsmodel.Account) error {
// Initialize a new stats struct.
stats := &gtsmodel.AccountStats{

25
internal/db/bundb/admin.go
View File

@ -120,16 +120,6 @@ func (a *adminDB) NewSignup(ctx context.Context, newSignup gtsmodel.NewSignup) (
return nil, err
}
settings := &gtsmodel.AccountSettings{
AccountID: accountID,
Privacy: gtsmodel.VisibilityDefault,
}
// Insert the settings!
if err := a.state.DB.PutAccountSettings(ctx, settings); err != nil {
return nil, err
}
account = &gtsmodel.Account{
ID: accountID,
Username: newSignup.Username,
@ -145,13 +135,26 @@ func (a *adminDB) NewSignup(ctx context.Context, newSignup gtsmodel.NewSignup) (
PrivateKey: privKey,
PublicKey: &privKey.PublicKey,
PublicKeyURI: uris.PublicKeyURI,
Settings: settings,
}
// Insert the new account!
if err := a.state.DB.PutAccount(ctx, account); err != nil {
return nil, err
}
// Insert basic settings for new account.
account.Settings = &gtsmodel.AccountSettings{
AccountID: accountID,
Privacy: gtsmodel.VisibilityDefault,
}
if err := a.state.DB.PutAccountSettings(ctx, account.Settings); err != nil {
return nil, err
}
// Stub empty stats for new account.
if err := a.state.DB.StubAccountStats(ctx, account); err != nil {
return nil, err
}
}
// Created or already had an account.

54
internal/federation/dereferencing/account.go
View File

@ -102,11 +102,15 @@ func (d *Dereferencer) GetAccountByURI(ctx context.Context, requestUser string,
}
if accountable != nil {
// This account was updated, enqueue re-dereference featured posts.
// This account was updated, enqueue re-dereference featured posts + stats.
d.state.Workers.Dereference.Queue.Push(func(ctx context.Context) {
if err := d.dereferenceAccountFeatured(ctx, requestUser, account); err != nil {
log.Errorf(ctx, "error fetching account featured collection: %v", err)
}
if err := d.dereferenceAccountStats(ctx, requestUser, account); err != nil {
log.Errorf(ctx, "error fetching account stats: %v", err)
}
})
}
@ -150,11 +154,22 @@ func (d *Dereferencer) getAccountByURI(ctx context.Context, requestUser string,
}
// Create and pass-through a new bare-bones model for dereferencing.
return d.enrichAccountSafely(ctx, requestUser, uri, &gtsmodel.Account{
account, accountable, err := d.enrichAccountSafely(ctx, requestUser, uri, &gtsmodel.Account{
ID: id.NewULID(),
Domain: uri.Host,
URI: uriStr,
}, nil)
if err != nil {
return nil, nil, err
}
// We have a new account. Ensure basic account stats populated;
// real stats will be fetched from remote asynchronously.
if err := d.state.DB.StubAccountStats(ctx, account); err != nil {
return nil, nil, gtserror.Newf("error stubbing account stats: %w", err)
}
return account, accountable, nil
}
if accountFresh(account, nil) {
@ -199,11 +214,15 @@ func (d *Dereferencer) GetAccountByUsernameDomain(ctx context.Context, requestUs
}
if accountable != nil {
// This account was updated, enqueue re-dereference featured posts.
// This account was updated, enqueue re-dereference featured posts + stats.
d.state.Workers.Dereference.Queue.Push(func(ctx context.Context) {
if err := d.dereferenceAccountFeatured(ctx, requestUser, account); err != nil {
log.Errorf(ctx, "error fetching account featured collection: %v", err)
}
if err := d.dereferenceAccountStats(ctx, requestUser, account); err != nil {
log.Errorf(ctx, "error fetching account stats: %v", err)
}
})
}
@ -251,6 +270,12 @@ func (d *Dereferencer) getAccountByUsernameDomain(
return nil, nil, err
}
// We have a new account. Ensure basic account stats populated;
// real stats will be fetched from remote asynchronously.
if err := d.state.DB.StubAccountStats(ctx, account); err != nil {
return nil, nil, gtserror.Newf("error stubbing account stats: %w", err)
}
return account, accountable, nil
}
@ -320,11 +345,15 @@ func (d *Dereferencer) RefreshAccount(
}
if accountable != nil {
// This account was updated, enqueue re-dereference featured posts.
// This account was updated, enqueue re-dereference featured posts + stats.
d.state.Workers.Dereference.Queue.Push(func(ctx context.Context) {
if err := d.dereferenceAccountFeatured(ctx, requestUser, latest); err != nil {
log.Errorf(ctx, "error fetching account featured collection: %v", err)
}
if err := d.dereferenceAccountStats(ctx, requestUser, latest); err != nil {
log.Errorf(ctx, "error fetching account stats: %v", err)
}
})
}
@ -369,10 +398,14 @@ func (d *Dereferencer) RefreshAccountAsync(
}
if accountable != nil {
// This account was updated, enqueue re-dereference featured posts.
// This account was updated, enqueue re-dereference featured posts + stats.
if err := d.dereferenceAccountFeatured(ctx, requestUser, latest); err != nil {
log.Errorf(ctx, "error fetching account featured collection: %v", err)
}
if err := d.dereferenceAccountStats(ctx, requestUser, latest); err != nil {
log.Errorf(ctx, "error fetching account stats: %v", err)
}
}
})
}
@ -697,12 +730,12 @@ func (d *Dereferencer) enrichAccount(
latestAcc.ID = account.ID
latestAcc.FetchedAt = time.Now()
// Ensure the account's avatar media is populated, passing in existing to check for chages.
// Ensure the account's avatar media is populated, passing in existing to check for changes.
if err := d.fetchRemoteAccountAvatar(ctx, tsport, account, latestAcc); err != nil {
log.Errorf(ctx, "error fetching remote avatar for account %s: %v", uri, err)
}
// Ensure the account's avatar media is populated, passing in existing to check for chages.
// Ensure the account's avatar media is populated, passing in existing to check for changes.
if err := d.fetchRemoteAccountHeader(ctx, tsport, account, latestAcc); err != nil {
log.Errorf(ctx, "error fetching remote header for account %s: %v", uri, err)
}
@ -712,11 +745,6 @@ func (d *Dereferencer) enrichAccount(
log.Errorf(ctx, "error fetching remote emojis for account %s: %v", uri, err)
}
// Fetch followers/following count for this account.
if err := d.fetchRemoteAccountStats(ctx, latestAcc, requestUser); err != nil {
log.Errorf(ctx, "error fetching remote stats for account %s: %v", uri, err)
}
if account.IsNew() {
// Prefer published/created time from
// apubAcc, fall back to FetchedAt value.
@ -1007,7 +1035,7 @@ func (d *Dereferencer) fetchRemoteAccountEmojis(ctx context.Context, targetAccou
return changed, nil
}
func (d *Dereferencer) fetchRemoteAccountStats(ctx context.Context, account *gtsmodel.Account, requestUser string) error {
func (d *Dereferencer) dereferenceAccountStats(ctx context.Context, requestUser string, account *gtsmodel.Account) error {
// Ensure we have a stats model for this account.
if account.Stats == nil {
if err := d.state.DB.PopulateAccountStats(ctx, account); err != nil {

300
internal/processing/fedi/collections.go
View File

@ -29,6 +29,8 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
"github.com/superseriousbusiness/gotosocial/internal/log"
"github.com/superseriousbusiness/gotosocial/internal/paging"
"github.com/superseriousbusiness/gotosocial/internal/typeutils"
"github.com/superseriousbusiness/gotosocial/internal/util"
)
// InboxPost handles POST requests to a user's inbox for new activitypub messages.
@ -45,91 +47,32 @@ func (p *Processor) InboxPost(ctx context.Context, w http.ResponseWriter, r *htt
return p.federator.FederatingActor().PostInbox(ctx, w, r)
}
// OutboxGet returns the activitypub representation of a local user's outbox.
// This contains links to PUBLIC posts made by this user.
// OutboxGet returns the serialized ActivityPub
// collection of a local account's outbox, which
// contains links to PUBLIC posts by this account.
func (p *Processor) OutboxGet(
ctx context.Context,
requestedUser string,
page bool,
maxID string,
minID string,
page *paging.Page,
) (interface{}, gtserror.WithCode) {
// Authenticate the incoming request, getting related user accounts.
_, receiver, errWithCode := p.authenticate(ctx, requestedUser)
if errWithCode != nil {
return nil, errWithCode
}
var data map[string]interface{}
// There are two scenarios:
// 1. we're asked for the whole collection and not a page -- we can just return the collection, with no items, but a link to 'first' page.
// 2. we're asked for a specific page; this can be either the first page or any other page
if !page {
/*
scenario 1: return the collection with no items
we want something that looks like this:
{
"@context": "https://www.w3.org/ns/activitystreams",
"id": "https://example.org/users/whatever/outbox",
"type": "OrderedCollection",
"first": "https://example.org/users/whatever/outbox?page=true",
"last": "https://example.org/users/whatever/outbox?min_id=0&page=true"
}
*/
collection, err := p.converter.OutboxToASCollection(ctx, receiver.OutboxURI)
if err != nil {
return nil, gtserror.NewErrorInternalError(err)
}
data, err = ap.Serialize(collection)
if err != nil {
return nil, gtserror.NewErrorInternalError(err)
}
return data, nil
}
// scenario 2 -- get the requested page
// limit pages to 30 entries per page
publicStatuses, err := p.state.DB.GetAccountStatuses(ctx, receiver.ID, 30, true, true, maxID, minID, false, true)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
return nil, gtserror.NewErrorInternalError(err)
}
outboxPage, err := p.converter.StatusesToASOutboxPage(ctx, receiver.OutboxURI, maxID, minID, publicStatuses)
if err != nil {
return nil, gtserror.NewErrorInternalError(err)
}
data, err = ap.Serialize(outboxPage)
if err != nil {
return nil, gtserror.NewErrorInternalError(err)
}
return data, nil
}
// FollowersGet handles the getting of a fedi/activitypub representation of a user/account's followers, performing appropriate
// authentication before returning a JSON serializable interface to the caller.
func (p *Processor) FollowersGet(ctx context.Context, requestedUser string, page *paging.Page) (interface{}, gtserror.WithCode) {
// Authenticate the incoming request, getting related user accounts.
_, receiver, errWithCode := p.authenticate(ctx, requestedUser)
// Authenticate incoming request, getting related accounts.
auth, errWithCode := p.authenticate(ctx, requestedUser)
if errWithCode != nil {
return nil, errWithCode
}
receivingAcct := auth.receivingAcct
// Parse the collection ID object from account's followers URI.
collectionID, err := url.Parse(receiver.FollowersURI)
collectionID, err := url.Parse(receivingAcct.OutboxURI)
if err != nil {
err := gtserror.Newf("error parsing account followers uri %s: %w", receiver.FollowersURI, err)
err := gtserror.Newf("error parsing account outbox uri %s: %w", receivingAcct.OutboxURI, err)
return nil, gtserror.NewErrorInternalError(err)
}
// Ensure we have stats for this account.
if receiver.Stats == nil {
if err := p.state.DB.PopulateAccountStats(ctx, receiver); err != nil {
err := gtserror.Newf("error getting stats for account %s: %w", receiver.ID, err)
if receivingAcct.Stats == nil {
if err := p.state.DB.PopulateAccountStats(ctx, receivingAcct); err != nil {
err := gtserror.Newf("error getting stats for account %s: %w", receivingAcct.ID, err)
return nil, gtserror.NewErrorInternalError(err)
}
}
@ -139,29 +82,160 @@ func (p *Processor) FollowersGet(ctx context.Context, requestedUser string, page
// Start the AS collection params.
var params ap.CollectionParams
params.ID = collectionID
params.Total = *receiver.Stats.FollowersCount
switch {
case receiver.IsInstance() ||
*receiver.Settings.HideCollections:
// Instance account (can't follow/be followed),
// or an account that hides followers/following.
// Respect this by just returning totalItems.
case *receivingAcct.Settings.HideCollections ||
receivingAcct.IsInstance():
// If account that hides collections, or instance
// account (ie., can't post / have relationships),
// just return barest stub of collection.
obj = ap.NewASOrderedCollection(params)
case page == nil:
// i.e. paging disabled, return collection
// that links to first page (i.e. path below).
case page == nil || auth.handshakingURI != nil:
// If paging disabled, or we're currently handshaking
// the requester, just return collection that links
// to first page (i.e. path below), with no items.
params.Total = util.Ptr(*receivingAcct.Stats.StatusesCount)
params.First = new(paging.Page)
params.Query = make(url.Values, 1)
params.Query.Set("limit", "40") // enables paging
obj = ap.NewASOrderedCollection(params)
default:
// i.e. paging enabled
// Get the request page of full follower objects with attached accounts.
followers, err := p.state.DB.GetAccountFollowers(ctx, receiver.ID, page)
// Paging enabled.
// Get page of full public statuses.
statuses, err := p.state.DB.GetAccountStatuses(
ctx,
receivingAcct.ID,
page.GetLimit(), // limit
true, // excludeReplies
true, // excludeReblogs
page.GetMax(), // maxID
page.GetMin(), // minID
false, // mediaOnly
true, // publicOnly
)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
err := gtserror.Newf("error getting statuses: %w", err)
return nil, gtserror.NewErrorInternalError(err)
}
// page ID values.
var lo, hi string
if len(statuses) > 0 {
// Get the lowest and highest
// ID values, used for paging.
lo = statuses[len(statuses)-1].ID
hi = statuses[0].ID
}
// Start building AS collection page params.
params.Total = util.Ptr(*receivingAcct.Stats.StatusesCount)
var pageParams ap.CollectionPageParams
pageParams.CollectionParams = params
// Current page details.
pageParams.Current = page
pageParams.Count = len(statuses)
// Set linked next/prev parameters.
pageParams.Next = page.Next(lo, hi)
pageParams.Prev = page.Prev(lo, hi)
// Set the collection item property builder function.
pageParams.Append = func(i int, itemsProp ap.ItemsPropertyBuilder) {
// Get status at index.
status := statuses[i]
// Derive statusable from status.
statusable, err := p.converter.StatusToAS(ctx, status)
if err != nil {
log.Errorf(ctx, "error converting %s to statusable: %v", status.URI, err)
return
}
// Derive create from statusable, using the IRI only.
create := typeutils.WrapStatusableInCreate(statusable, true)
// Add to item property.
itemsProp.AppendActivityStreamsCreate(create)
}
// Build AS collection page object from params.
obj = ap.NewASOrderedCollectionPage(pageParams)
}
// Serialize the prepared object.
data, err := ap.Serialize(obj)
if err != nil {
err := gtserror.Newf("error serializing: %w", err)
return nil, gtserror.NewErrorInternalError(err)
}
return data, nil
}
// FollowersGet returns the serialized ActivityPub
// collection of a local account's followers collection,
// which contains links to accounts following this account.
func (p *Processor) FollowersGet(
ctx context.Context,
requestedUser string,
page *paging.Page,
) (interface{}, gtserror.WithCode) {
// Authenticate incoming request, getting related accounts.
auth, errWithCode := p.authenticate(ctx, requestedUser)
if errWithCode != nil {
return nil, errWithCode
}
receivingAcct := auth.receivingAcct
// Parse the collection ID object from account's followers URI.
collectionID, err := url.Parse(receivingAcct.FollowersURI)
if err != nil {
err := gtserror.Newf("error parsing account followers uri %s: %w", receivingAcct.FollowersURI, err)
return nil, gtserror.NewErrorInternalError(err)
}
// Ensure we have stats for this account.
if receivingAcct.Stats == nil {
if err := p.state.DB.PopulateAccountStats(ctx, receivingAcct); err != nil {
err := gtserror.Newf("error getting stats for account %s: %w", receivingAcct.ID, err)
return nil, gtserror.NewErrorInternalError(err)
}
}
var obj vocab.Type
// Start the AS collection params.
var params ap.CollectionParams
params.ID = collectionID
switch {
case receivingAcct.IsInstance() ||
*receivingAcct.Settings.HideCollections:
// If account that hides collections, or instance
// account (ie., can't post / have relationships),
// just return barest stub of collection.
obj = ap.NewASOrderedCollection(params)
case page == nil || auth.handshakingURI != nil:
// If paging disabled, or we're currently handshaking
// the requester, just return collection that links
// to first page (i.e. path below), with no items.
params.Total = util.Ptr(*receivingAcct.Stats.FollowersCount)
params.First = new(paging.Page)
params.Query = make(url.Values, 1)
params.Query.Set("limit", "40") // enables paging
obj = ap.NewASOrderedCollection(params)
default:
// Paging enabled.
// Get page of full follower objects with attached accounts.
followers, err := p.state.DB.GetAccountFollowers(ctx, receivingAcct.ID, page)
if err != nil {
err := gtserror.Newf("error getting followers: %w", err)
return nil, gtserror.NewErrorInternalError(err)
@ -178,6 +252,7 @@ func (p *Processor) FollowersGet(ctx context.Context, requestedUser string, page
}
// Start building AS collection page params.
params.Total = util.Ptr(*receivingAcct.Stats.FollowersCount)
var pageParams ap.CollectionPageParams
pageParams.CollectionParams = params
@ -210,7 +285,7 @@ func (p *Processor) FollowersGet(ctx context.Context, requestedUser string, page
obj = ap.NewASOrderedCollectionPage(pageParams)
}
// Serialized the prepared object.
// Serialize the prepared object.
data, err := ap.Serialize(obj)
if err != nil {
err := gtserror.Newf("error serializing: %w", err)
@ -220,26 +295,28 @@ func (p *Processor) FollowersGet(ctx context.Context, requestedUser string, page
return data, nil
}
// FollowingGet handles the getting of a fedi/activitypub representation of a user/account's following, performing appropriate
// authentication before returning a JSON serializable interface to the caller.
// FollowingGet returns the serialized ActivityPub
// collection of a local account's following collection,
// which contains links to accounts followed by this account.
func (p *Processor) FollowingGet(ctx context.Context, requestedUser string, page *paging.Page) (interface{}, gtserror.WithCode) {
// Authenticate the incoming request, getting related user accounts.
_, receiver, errWithCode := p.authenticate(ctx, requestedUser)
// Authenticate incoming request, getting related accounts.
auth, errWithCode := p.authenticate(ctx, requestedUser)
if errWithCode != nil {
return nil, errWithCode
}
receivingAcct := auth.receivingAcct
// Parse collection ID from account's following URI.
collectionID, err := url.Parse(receiver.FollowingURI)
collectionID, err := url.Parse(receivingAcct.FollowingURI)
if err != nil {
err := gtserror.Newf("error parsing account following uri %s: %w", receiver.FollowingURI, err)
err := gtserror.Newf("error parsing account following uri %s: %w", receivingAcct.FollowingURI, err)
return nil, gtserror.NewErrorInternalError(err)
}
// Ensure we have stats for this account.
if receiver.Stats == nil {
if err := p.state.DB.PopulateAccountStats(ctx, receiver); err != nil {
err := gtserror.Newf("error getting stats for account %s: %w", receiver.ID, err)
if receivingAcct.Stats == nil {
if err := p.state.DB.PopulateAccountStats(ctx, receivingAcct); err != nil {
err := gtserror.Newf("error getting stats for account %s: %w", receivingAcct.ID, err)
return nil, gtserror.NewErrorInternalError(err)
}
}
@ -249,28 +326,29 @@ func (p *Processor) FollowingGet(ctx context.Context, requestedUser string, page
// Start AS collection params.
var params ap.CollectionParams
params.ID = collectionID
params.Total = *receiver.Stats.FollowingCount
switch {
case receiver.IsInstance() ||
*receiver.Settings.HideCollections:
// Instance account (can't follow/be followed),
// or an account that hides followers/following.
// Respect this by just returning totalItems.
case receivingAcct.IsInstance() ||
*receivingAcct.Settings.HideCollections:
// If account that hides collections, or instance
// account (ie., can't post / have relationships),
// just return barest stub of collection.
obj = ap.NewASOrderedCollection(params)
case page == nil:
// i.e. paging disabled, return collection
// that links to first page (i.e. path below).
case page == nil || auth.handshakingURI != nil:
// If paging disabled, or we're currently handshaking
// the requester, just return collection that links
// to first page (i.e. path below), with no items.
params.Total = util.Ptr(*receivingAcct.Stats.FollowingCount)
params.First = new(paging.Page)
params.Query = make(url.Values, 1)
params.Query.Set("limit", "40") // enables paging
obj = ap.NewASOrderedCollection(params)
default:
// i.e. paging enabled
// Get the request page of full follower objects with attached accounts.
follows, err := p.state.DB.GetAccountFollows(ctx, receiver.ID, page)
// Paging enabled.
// Get page of full follower objects with attached accounts.
follows, err := p.state.DB.GetAccountFollows(ctx, receivingAcct.ID, page)
if err != nil {
err := gtserror.Newf("error getting follows: %w", err)
return nil, gtserror.NewErrorInternalError(err)
@ -287,6 +365,7 @@ func (p *Processor) FollowingGet(ctx context.Context, requestedUser string, page
}
// Start AS collection page params.
params.Total = util.Ptr(*receivingAcct.Stats.FollowingCount)
var pageParams ap.CollectionPageParams
pageParams.CollectionParams = params
@ -319,7 +398,7 @@ func (p *Processor) FollowingGet(ctx context.Context, requestedUser string, page
obj = ap.NewASOrderedCollectionPage(pageParams)
}
// Serialized the prepared object.
// Serialize the prepared object.
data, err := ap.Serialize(obj)
if err != nil {
err := gtserror.Newf("error serializing: %w", err)
@ -332,20 +411,21 @@ func (p *Processor) FollowingGet(ctx context.Context, requestedUser string, page
// FeaturedCollectionGet returns an ordered collection of the requested username's Pinned posts.
// The returned collection have an `items` property which contains an ordered list of status URIs.
func (p *Processor) FeaturedCollectionGet(ctx context.Context, requestedUser string) (interface{}, gtserror.WithCode) {
// Authenticate the incoming request, getting related user accounts.
_, receiver, errWithCode := p.authenticate(ctx, requestedUser)
// Authenticate incoming request, getting related accounts.
auth, errWithCode := p.authenticate(ctx, requestedUser)
if errWithCode != nil {
return nil, errWithCode
}
receivingAcct := auth.receivingAcct
statuses, err := p.state.DB.GetAccountPinnedStatuses(ctx, receiver.ID)
statuses, err := p.state.DB.GetAccountPinnedStatuses(ctx, receivingAcct.ID)
if err != nil {
if !errors.Is(err, db.ErrNoEntries) {
return nil, gtserror.NewErrorInternalError(err)
}
}
collection, err := p.converter.StatusesToASFeaturedCollection(ctx, receiver.FeaturedCollectionURI, statuses)
collection, err := p.converter.StatusesToASFeaturedCollection(ctx, receivingAcct.FeaturedCollectionURI, statuses)
if err != nil {
return nil, gtserror.NewErrorInternalError(err)
}

39
internal/processing/fedi/common.go
View File

@ -20,55 +20,64 @@ package fedi
import (
"context"
"errors"
"net/url"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
)
func (p *Processor) authenticate(ctx context.Context, requestedUser string) (
*gtsmodel.Account, // requester: i.e. user making the request
*gtsmodel.Account, // receiver: i.e. the receiving inbox user
gtserror.WithCode,
) {
type commonAuth struct {
handshakingURI *url.URL // Set to requestingAcct's URI if we're currently handshaking them.
requestingAcct *gtsmodel.Account // Remote account making request to this instance.
receivingAcct *gtsmodel.Account // Local account receiving the request.
}
func (p *Processor) authenticate(ctx context.Context, requestedUser string) (*commonAuth, gtserror.WithCode) {
// First get the requested (receiving) LOCAL account with username from database.
receiver, err := p.state.DB.GetAccountByUsernameDomain(ctx, requestedUser, "")
if err != nil {
if !errors.Is(err, db.ErrNoEntries) {
// Real db error.
err = gtserror.Newf("db error getting account %s: %w", requestedUser, err)
return nil, nil, gtserror.NewErrorInternalError(err)
return nil, gtserror.NewErrorInternalError(err)
}
// Account just not found in the db.
return nil, nil, gtserror.NewErrorNotFound(err)
return nil, gtserror.NewErrorNotFound(err)
}
// Ensure request signed, and use signature URI to
// get requesting account, dereferencing if necessary.
pubKeyAuth, errWithCode := p.federator.AuthenticateFederatedRequest(ctx, requestedUser)
if errWithCode != nil {
return nil, nil, errWithCode
return nil, errWithCode
}
if pubKeyAuth.Handshaking {
// This should happen very rarely, we are in the middle of handshaking.
err := gtserror.Newf("network race handshaking %s", pubKeyAuth.OwnerURI)
return nil, nil, gtserror.NewErrorInternalError(err)
// We're still handshaking so we
// don't know the requester yet.
return &commonAuth{
handshakingURI: pubKeyAuth.OwnerURI,
receivingAcct: receiver,
}, nil
}
// Get requester from auth.
requester := pubKeyAuth.Owner
// Check that block does not exist between receiver and requester.
// Ensure block does not exist between receiver and requester.
blocked, err := p.state.DB.IsEitherBlocked(ctx, receiver.ID, requester.ID)
if err != nil {
err := gtserror.Newf("error checking block: %w", err)
return nil, nil, gtserror.NewErrorInternalError(err)
return nil, gtserror.NewErrorInternalError(err)
} else if blocked {
const text = "block exists between accounts"
return nil, nil, gtserror.NewErrorForbidden(errors.New(text))
return nil, gtserror.NewErrorForbidden(errors.New(text))
}
return requester, receiver, nil
return &commonAuth{
requestingAcct: requester,
receivingAcct: receiver,
}, nil
}

43
internal/processing/fedi/status.go
View File

@ -30,23 +30,35 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/log"
"github.com/superseriousbusiness/gotosocial/internal/paging"
"github.com/superseriousbusiness/gotosocial/internal/util"
)
// StatusGet handles the getting of a fedi/activitypub representation of a local status.
// It performs appropriate authentication before returning a JSON serializable interface.
func (p *Processor) StatusGet(ctx context.Context, requestedUser string, statusID string) (interface{}, gtserror.WithCode) {
// Authenticate the incoming request, getting related user accounts.
requester, receiver, errWithCode := p.authenticate(ctx, requestedUser)
// Authenticate incoming request, getting related accounts.
auth, errWithCode := p.authenticate(ctx, requestedUser)
if errWithCode != nil {
return nil, errWithCode
}
if auth.handshakingURI != nil {
// We're currently handshaking, which means
// we don't know this account yet. This should
// be a very rare race condition.
err := gtserror.Newf("network race handshaking %s", auth.handshakingURI)
return nil, gtserror.NewErrorInternalError(err)
}
receivingAcct := auth.receivingAcct
requestingAcct := auth.requestingAcct
status, err := p.state.DB.GetStatusByID(ctx, statusID)
if err != nil {
return nil, gtserror.NewErrorNotFound(err)
}
if status.AccountID != receiver.ID {
if status.AccountID != receivingAcct.ID {
const text = "status does not belong to receiving account"
return nil, gtserror.NewErrorNotFound(errors.New(text))
}
@ -56,7 +68,7 @@ func (p *Processor) StatusGet(ctx context.Context, requestedUser string, statusI
return nil, gtserror.NewErrorNotFound(errors.New(text))
}
visible, err := p.filter.StatusVisible(ctx, requester, status)
visible, err := p.filter.StatusVisible(ctx, requestingAcct, status)
if err != nil {
return nil, gtserror.NewErrorInternalError(err)
}
@ -90,15 +102,26 @@ func (p *Processor) StatusRepliesGet(
page *paging.Page,
onlyOtherAccounts bool,
) (interface{}, gtserror.WithCode) {
// Authenticate the incoming request, getting related user accounts.
requester, receiver, errWithCode := p.authenticate(ctx, requestedUser)
// Authenticate incoming request, getting related accounts.
auth, errWithCode := p.authenticate(ctx, requestedUser)
if errWithCode != nil {
return nil, errWithCode
}
if auth.handshakingURI != nil {
// We're currently handshaking, which means
// we don't know this account yet. This should
// be a very rare race condition.
err := gtserror.Newf("network race handshaking %s", auth.handshakingURI)
return nil, gtserror.NewErrorInternalError(err)
}
receivingAcct := auth.receivingAcct
requestingAcct := auth.requestingAcct
// Get target status and ensure visible to requester.
status, errWithCode := p.c.GetVisibleTargetStatus(ctx,
requester,
requestingAcct,
statusID,
nil, // default freshness
)
@ -107,7 +130,7 @@ func (p *Processor) StatusRepliesGet(
}
// Ensure status is by receiving account.
if status.AccountID != receiver.ID {
if status.AccountID != receivingAcct.ID {
const text = "status does not belong to receiving account"
return nil, gtserror.NewErrorNotFound(errors.New(text))
}
@ -140,7 +163,7 @@ func (p *Processor) StatusRepliesGet(
}
// Reslice replies dropping all those invisible to requester.
replies, err = p.filter.StatusesVisible(ctx, requester, replies)
replies, err = p.filter.StatusesVisible(ctx, requestingAcct, replies)
if err != nil {
err := gtserror.Newf("error filtering status replies: %w", err)
return nil, gtserror.NewErrorInternalError(err)
@ -151,7 +174,7 @@ func (p *Processor) StatusRepliesGet(
// Start AS collection params.
var params ap.CollectionParams
params.ID = collectionID
params.Total = len(replies)
params.Total = util.Ptr(len(replies))
if page == nil {
// i.e. paging disabled, return collection

33
internal/typeutils/internaltoas.go
View File

@ -1560,39 +1560,6 @@ func (c *Converter) StatusesToASOutboxPage(ctx context.Context, outboxID string,
return page, nil
}
// OutboxToASCollection returns an ordered collection with appropriate id, next, and last fields.
// The returned collection won't have any actual entries; just links to where entries can be obtained.
// we want something that looks like this:
//
// {
// "@context": "https://www.w3.org/ns/activitystreams",
// "id": "https://example.org/users/whatever/outbox",
// "type": "OrderedCollection",
// "first": "https://example.org/users/whatever/outbox?page=true"
// }
func (c *Converter) OutboxToASCollection(ctx context.Context, outboxID string) (vocab.ActivityStreamsOrderedCollection, error) {
collection := streams.NewActivityStreamsOrderedCollection()
collectionIDProp := streams.NewJSONLDIdProperty()
outboxIDURI, err := url.Parse(outboxID)
if err != nil {
return nil, fmt.Errorf("error parsing url %s", outboxID)
}
collectionIDProp.SetIRI(outboxIDURI)
collection.SetJSONLDId(collectionIDProp)
collectionFirstProp := streams.NewActivityStreamsFirstProperty()
collectionFirstPropID := fmt.Sprintf("%s?page=true", outboxID)
collectionFirstPropIDURI, err := url.Parse(collectionFirstPropID)
if err != nil {
return nil, fmt.Errorf("error parsing url %s", collectionFirstPropID)
}
collectionFirstProp.SetIRI(collectionFirstPropIDURI)
collection.SetActivityStreamsFirst(collectionFirstProp)
return collection, nil
}
// StatusesToASFeaturedCollection converts a slice of statuses into an ordered collection
// of URIs, suitable for serializing and serving via the activitypub API.
func (c *Converter) StatusesToASFeaturedCollection(ctx context.Context, featuredCollectionID string, statuses []*gtsmodel.Status) (vocab.ActivityStreamsOrderedCollection, error) {

21
internal/typeutils/internaltoas_test.go
View File

@ -366,27 +366,6 @@ func (suite *InternalToASTestSuite) TestAccountToASWithSharedInbox() {
}`, trimmed)
}
func (suite *InternalToASTestSuite) TestOutboxToASCollection() {
testAccount := suite.testAccounts["admin_account"]
ctx := context.Background()
collection, err := suite.typeconverter.OutboxToASCollection(ctx, testAccount.OutboxURI)
suite.NoError(err)
ser, err := ap.Serialize(collection)
suite.NoError(err)
bytes, err := json.MarshalIndent(ser, "", " ")
suite.NoError(err)
suite.Equal(`{
"@context": "https://www.w3.org/ns/activitystreams",
"first": "http://localhost:8080/users/admin/outbox?page=true",
"id": "http://localhost:8080/users/admin/outbox",
"type": "OrderedCollection"
}`, string(bytes))
}
func (suite *InternalToASTestSuite) TestStatusToAS() {
testStatus := suite.testStatuses["local_account_1_status_1"]
ctx := context.Background()

4
testrig/testmodels.go
View File

@ -3227,7 +3227,7 @@ func NewTestDereferenceRequests(accounts map[string]*gtsmodel.Account) map[strin
DateHeader: date,
}
target = URLMustParse(accounts["local_account_1"].OutboxURI + "?page=true")
target = URLMustParse(accounts["local_account_1"].OutboxURI + "?limit=40")
sig, digest, date = GetSignatureForDereference(accounts["remote_account_1"].PublicKeyURI, accounts["remote_account_1"].PrivateKey, target)
fossSatanDereferenceZorkOutboxFirst := ActivityWithSignature{
SignatureHeader: sig,
@ -3235,7 +3235,7 @@ func NewTestDereferenceRequests(accounts map[string]*gtsmodel.Account) map[strin
DateHeader: date,
}
target = URLMustParse(accounts["local_account_1"].OutboxURI + "?page=true&max_id=01F8MHAMCHF6Y650WCRSCP4WMY")
target = URLMustParse(accounts["local_account_1"].OutboxURI + "?limit=40&max_id=01F8MHAMCHF6Y650WCRSCP4WMY")
sig, digest, date = GetSignatureForDereference(accounts["remote_account_1"].PublicKeyURI, accounts["remote_account_1"].PrivateKey, target)
fossSatanDereferenceZorkOutboxNext := ActivityWithSignature{
SignatureHeader: sig,