mirrors
/
gotosocial
mirror of
1
Fork 0
Code Issues Releases Activity

[performance] add account block DB cache and remove block query joins (#1085) 

* add account block DB cache and remove reliance on relational joins

* actually include cache key arguments...

* add a PutBlock() method which also updates the block cache, update tests accordingly

* use `PutBlock` instead of `Put(ctx, block)`

* add + use functions for deleting + invalidating blocks

Signed-off-by: kim <grufwub@gmail.com>
Co-authored-by: tsmethurst <tobi.smethurst@protonmail.com>
This commit is contained in:
kim 2022-11-20 16:33:49 +00:00 committed by GitHub
parent 9be16852f2
commit 5d55e8d920
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 319 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
internal/api/s2s/user/inboxpost_test.go
View File

@ -153,7 +153,7 @@ func (suite *InboxPostTestSuite) TestPostUnblock() {
TargetAccountID: blockedAccount.ID,
}
err = suite.db.Put(context.Background(), dbBlock)
err = suite.db.PutBlock(context.Background(), dbBlock)
suite.NoError(err)
asBlock, err := suite.tc.BlockToAS(context.Background(), dbBlock)

7
internal/db/bundb/bundb.go
View File

@ -166,6 +166,7 @@ func NewBunDBService(ctx context.Context) (db.DB, error) {
notif := &notificationDB{conn: conn}
status := &statusDB{conn: conn}
emoji := &emojiDB{conn: conn}
relationship := &relationshipDB{conn: conn}
timeline := &timelineDB{conn: conn}
tombstone := &tombstoneDB{conn: conn}
user := &userDB{conn: conn}
@ -174,6 +175,7 @@ func NewBunDBService(ctx context.Context) (db.DB, error) {
account.emojis = emoji
account.status = status
admin.users = user
relationship.accounts = account
status.accounts = account
status.emojis = emoji
status.mentions = mention
@ -185,6 +187,7 @@ func NewBunDBService(ctx context.Context) (db.DB, error) {
emoji.init()
mention.init()
notif.init()
relationship.init()
status.init()
tombstone.init()
user.init()
@ -209,9 +212,7 @@ func NewBunDBService(ctx context.Context) (db.DB, error) {
},
Mention: mention,
Notification: notif,
Relationship: &relationshipDB{
conn: conn,
},
Relationship: relationship,
Session: &sessionDB{
conn: conn,
},

202
internal/db/bundb/relationship.go
View File

@ -21,8 +21,11 @@ package bundb
import (
"context"
"database/sql"
"errors"
"fmt"
"time"
"codeberg.org/gruf/go-cache/v3/result"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/uptrace/bun"
@ -30,14 +33,25 @@ import (
type relationshipDB struct {
conn *DBConn
accounts *accountDB
blockCache *result.Cache[*gtsmodel.Block]
}
func (r *relationshipDB) newBlockQ(block *gtsmodel.Block) *bun.SelectQuery {
return r.conn.
NewSelect().
Model(block).
Relation("Account").
Relation("TargetAccount")
func (r *relationshipDB) init() {
// Initialize block result cache
r.blockCache = result.NewSized([]result.Lookup{
{Name: "ID"},
{Name: "AccountID.TargetAccountID"},
{Name: "URI"},
}, func(b1 *gtsmodel.Block) *gtsmodel.Block {
b2 := new(gtsmodel.Block)
*b2 = *b1
return b2
}, 1000)
// Set cache TTL and start sweep routine
r.blockCache.SetTTL(time.Minute*5, false)
r.blockCache.Start(time.Second * 10)
}
func (r *relationshipDB) newFollowQ(follow interface{}) *bun.SelectQuery {
@ -49,43 +63,143 @@ func (r *relationshipDB) newFollowQ(follow interface{}) *bun.SelectQuery {
}
func (r *relationshipDB) IsBlocked(ctx context.Context, account1 string, account2 string, eitherDirection bool) (bool, db.Error) {
q := r.conn.
NewSelect().
TableExpr("? AS ?", bun.Ident("blocks"), bun.Ident("block")).
Column("block.id")
if eitherDirection {
q = q.
WhereGroup(" OR ", func(inner *bun.SelectQuery) *bun.SelectQuery {
return inner.
Where("? = ?", bun.Ident("block.account_id"), account1).
Where("? = ?", bun.Ident("block.target_account_id"), account2)
}).
WhereGroup(" OR ", func(inner *bun.SelectQuery) *bun.SelectQuery {
return inner.
Where("? = ?", bun.Ident("block.account_id"), account2).
Where("? = ?", bun.Ident("block.target_account_id"), account1)
})
} else {
q = q.
Where("? = ?", bun.Ident("block.account_id"), account1).
Where("? = ?", bun.Ident("block.target_account_id"), account2)
// Look for a block in direction of account1->account2
block1, err := r.getBlock(ctx, account1, account2)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
return false, err
}
return r.conn.Exists(ctx, q)
if block1 != nil {
// account1 blocks account2
return true, nil
} else if !eitherDirection {
// Don't check for mutli-directional
return false, nil
}
// Look for a block in direction of account2->account1
block2, err := r.getBlock(ctx, account2, account1)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
return false, err
}
return (block2 != nil), nil
}
func (r *relationshipDB) GetBlock(ctx context.Context, account1 string, account2 string) (*gtsmodel.Block, db.Error) {
block := &gtsmodel.Block{}
// Fetch block from database
block, err := r.getBlock(ctx, account1, account2)
if err != nil {
return nil, err
}
q := r.newBlockQ(block).
// Set the block originating account
block.Account, err = r.accounts.GetAccountByID(ctx, block.AccountID)
if err != nil {
return nil, err
}
// Set the block target account
block.TargetAccount, err = r.accounts.GetAccountByID(ctx, block.TargetAccountID)
if err != nil {
return nil, err
}
return block, nil
}
func (r *relationshipDB) getBlock(ctx context.Context, account1 string, account2 string) (*gtsmodel.Block, db.Error) {
return r.blockCache.Load("AccountID.TargetAccountID", func() (*gtsmodel.Block, error) {
var block gtsmodel.Block
q := r.conn.NewSelect().Model(&block).
Where("? = ?", bun.Ident("block.account_id"), account1).
Where("? = ?", bun.Ident("block.target_account_id"), account2)
if err := q.Scan(ctx); err != nil {
return nil, r.conn.ProcessError(err)
}
return block, nil
return &block, nil
}, account1, account2)
}
func (r *relationshipDB) PutBlock(ctx context.Context, block *gtsmodel.Block) db.Error {
return r.blockCache.Store(block, func() error {
_, err := r.conn.NewInsert().Model(block).Exec(ctx)
return r.conn.ProcessError(err)
})
}
func (r *relationshipDB) DeleteBlockByID(ctx context.Context, id string) db.Error {
if _, err := r.conn.
NewDelete().
TableExpr("? AS ?", bun.Ident("blocks"), bun.Ident("block")).
Where("? = ?", bun.Ident("block.id"), id).
Exec(ctx); err != nil {
return r.conn.ProcessError(err)
}
// Drop any old value from cache by this ID
r.blockCache.Invalidate("ID", id)
return nil
}
func (r *relationshipDB) DeleteBlockByURI(ctx context.Context, uri string) db.Error {
if _, err := r.conn.
NewDelete().
TableExpr("? AS ?", bun.Ident("blocks"), bun.Ident("block")).
Where("? = ?", bun.Ident("block.uri"), uri).
Exec(ctx); err != nil {
return r.conn.ProcessError(err)
}
// Drop any old value from cache by this URI
r.blockCache.Invalidate("URI", uri)
return nil
}
func (r *relationshipDB) DeleteBlocksByOriginAccountID(ctx context.Context, originAccountID string) db.Error {
blockIDs := []string{}
q := r.conn.
NewSelect().
TableExpr("? AS ?", bun.Ident("blocks"), bun.Ident("block")).
Column("block.id").
Where("? = ?", bun.Ident("block.account_id"), originAccountID)
if err := q.Scan(ctx, &blockIDs); err != nil {
return r.conn.ProcessError(err)
}
for _, blockID := range blockIDs {
if err := r.DeleteBlockByID(ctx, blockID); err != nil {
return err
}
}
return nil
}
func (r *relationshipDB) DeleteBlocksByTargetAccountID(ctx context.Context, targetAccountID string) db.Error {
blockIDs := []string{}
q := r.conn.
NewSelect().
TableExpr("? AS ?", bun.Ident("blocks"), bun.Ident("block")).
Column("block.id").
Where("? = ?", bun.Ident("block.target_account_id"), targetAccountID)
if err := q.Scan(ctx, &blockIDs); err != nil {
return r.conn.ProcessError(err)
}
for _, blockID := range blockIDs {
if err := r.DeleteBlockByID(ctx, blockID); err != nil {
return err
}
}
return nil
}
func (r *relationshipDB) GetRelationship(ctx context.Context, requestingAccount string, targetAccount string) (*gtsmodel.Relationship, db.Error) {
@ -144,30 +258,18 @@ func (r *relationshipDB) GetRelationship(ctx context.Context, requestingAccount
rel.Requested = requested
// check if the requesting account is blocking the target account
blockingQ := r.conn.
NewSelect().
TableExpr("? AS ?", bun.Ident("blocks"), bun.Ident("block")).
Column("block.id").
Where("? = ?", bun.Ident("block.account_id"), requestingAccount).
Where("? = ?", bun.Ident("block.target_account_id"), targetAccount)
blocking, err := r.conn.Exists(ctx, blockingQ)
if err != nil {
blockA2T, err := r.getBlock(ctx, requestingAccount, targetAccount)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
return nil, fmt.Errorf("GetRelationship: error checking blocking: %s", err)
}
rel.Blocking = blocking
rel.Blocking = (blockA2T != nil)
// check if the requesting account is blocked by the target account
blockedByQ := r.conn.
NewSelect().
TableExpr("? AS ?", bun.Ident("blocks"), bun.Ident("block")).
Column("block.id").
Where("? = ?", bun.Ident("block.account_id"), targetAccount).
Where("? = ?", bun.Ident("block.target_account_id"), requestingAccount)
blockedBy, err := r.conn.Exists(ctx, blockedByQ)
if err != nil {
blockT2A, err := r.getBlock(ctx, targetAccount, requestingAccount)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
return nil, fmt.Errorf("GetRelationship: error checking blockedBy: %s", err)
}
rel.BlockedBy = blockedBy
rel.BlockedBy = (blockT2A != nil)
return rel, nil
}

128
internal/db/bundb/relationship_test.go
View File

@ -47,7 +47,7 @@ func (suite *RelationshipTestSuite) TestIsBlocked() {
suite.False(blocked)
// have account1 block account2
if err := suite.db.Put(ctx, &gtsmodel.Block{
if err := suite.db.PutBlock(ctx, &gtsmodel.Block{
ID: "01G202BCSXXJZ70BHB5KCAHH8C",
URI: "http://localhost:8080/some_block_uri_1",
AccountID: account1,
@ -81,7 +81,7 @@ func (suite *RelationshipTestSuite) TestGetBlock() {
account1 := suite.testAccounts["local_account_1"].ID
account2 := suite.testAccounts["local_account_2"].ID
if err := suite.db.Put(ctx, &gtsmodel.Block{
if err := suite.db.PutBlock(ctx, &gtsmodel.Block{
ID: "01G202BCSXXJZ70BHB5KCAHH8C",
URI: "http://localhost:8080/some_block_uri_1",
AccountID: account1,
@ -96,6 +96,130 @@ func (suite *RelationshipTestSuite) TestGetBlock() {
suite.Equal("01G202BCSXXJZ70BHB5KCAHH8C", block.ID)
}
func (suite *RelationshipTestSuite) TestDeleteBlockByID() {
ctx := context.Background()
// put a block in first
account1 := suite.testAccounts["local_account_1"].ID
account2 := suite.testAccounts["local_account_2"].ID
if err := suite.db.PutBlock(ctx, &gtsmodel.Block{
ID: "01G202BCSXXJZ70BHB5KCAHH8C",
URI: "http://localhost:8080/some_block_uri_1",
AccountID: account1,
TargetAccountID: account2,
}); err != nil {
suite.FailNow(err.Error())
}
// make sure the block is in the db
block, err := suite.db.GetBlock(ctx, account1, account2)
suite.NoError(err)
suite.NotNil(block)
suite.Equal("01G202BCSXXJZ70BHB5KCAHH8C", block.ID)
// delete the block by ID
err = suite.db.DeleteBlockByID(ctx, "01G202BCSXXJZ70BHB5KCAHH8C")
suite.NoError(err)
// block should be gone
block, err = suite.db.GetBlock(ctx, account1, account2)
suite.ErrorIs(err, db.ErrNoEntries)
suite.Nil(block)
}
func (suite *RelationshipTestSuite) TestDeleteBlockByURI() {
ctx := context.Background()
// put a block in first
account1 := suite.testAccounts["local_account_1"].ID
account2 := suite.testAccounts["local_account_2"].ID
if err := suite.db.PutBlock(ctx, &gtsmodel.Block{
ID: "01G202BCSXXJZ70BHB5KCAHH8C",
URI: "http://localhost:8080/some_block_uri_1",
AccountID: account1,
TargetAccountID: account2,
}); err != nil {
suite.FailNow(err.Error())
}
// make sure the block is in the db
block, err := suite.db.GetBlock(ctx, account1, account2)
suite.NoError(err)
suite.NotNil(block)
suite.Equal("01G202BCSXXJZ70BHB5KCAHH8C", block.ID)
// delete the block by uri
err = suite.db.DeleteBlockByURI(ctx, "http://localhost:8080/some_block_uri_1")
suite.NoError(err)
// block should be gone
block, err = suite.db.GetBlock(ctx, account1, account2)
suite.ErrorIs(err, db.ErrNoEntries)
suite.Nil(block)
}
func (suite *RelationshipTestSuite) TestDeleteBlocksByOriginAccountID() {
ctx := context.Background()
// put a block in first
account1 := suite.testAccounts["local_account_1"].ID
account2 := suite.testAccounts["local_account_2"].ID
if err := suite.db.PutBlock(ctx, &gtsmodel.Block{
ID: "01G202BCSXXJZ70BHB5KCAHH8C",
URI: "http://localhost:8080/some_block_uri_1",
AccountID: account1,
TargetAccountID: account2,
}); err != nil {
suite.FailNow(err.Error())
}
// make sure the block is in the db
block, err := suite.db.GetBlock(ctx, account1, account2)
suite.NoError(err)
suite.NotNil(block)
suite.Equal("01G202BCSXXJZ70BHB5KCAHH8C", block.ID)
// delete the block by originAccountID
err = suite.db.DeleteBlocksByOriginAccountID(ctx, account1)
suite.NoError(err)
// block should be gone
block, err = suite.db.GetBlock(ctx, account1, account2)
suite.ErrorIs(err, db.ErrNoEntries)
suite.Nil(block)
}
func (suite *RelationshipTestSuite) TestDeleteBlocksByTargetAccountID() {
ctx := context.Background()
// put a block in first
account1 := suite.testAccounts["local_account_1"].ID
account2 := suite.testAccounts["local_account_2"].ID
if err := suite.db.PutBlock(ctx, &gtsmodel.Block{
ID: "01G202BCSXXJZ70BHB5KCAHH8C",
URI: "http://localhost:8080/some_block_uri_1",
AccountID: account1,
TargetAccountID: account2,
}); err != nil {
suite.FailNow(err.Error())
}
// make sure the block is in the db
block, err := suite.db.GetBlock(ctx, account1, account2)
suite.NoError(err)
suite.NotNil(block)
suite.Equal("01G202BCSXXJZ70BHB5KCAHH8C", block.ID)
// delete the block by targetAccountID
err = suite.db.DeleteBlocksByTargetAccountID(ctx, account2)
suite.NoError(err)
// block should be gone
block, err = suite.db.GetBlock(ctx, account1, account2)
suite.ErrorIs(err, db.ErrNoEntries)
suite.Nil(block)
}
func (suite *RelationshipTestSuite) TestGetRelationship() {
requestingAccount := suite.testAccounts["local_account_1"]
targetAccount := suite.testAccounts["admin_account"]

15
internal/db/relationship.go
View File

@ -36,6 +36,21 @@ type Relationship interface {
// not if you're just checking for the existence of a block.
GetBlock(ctx context.Context, account1 string, account2 string) (*gtsmodel.Block, Error)
// PutBlock attempts to place the given account block in the database.
PutBlock(ctx context.Context, block *gtsmodel.Block) Error
// DeleteBlockByID removes block with given ID from the database.
DeleteBlockByID(ctx context.Context, id string) Error
// DeleteBlockByURI removes block with given AP URI from the database.
DeleteBlockByURI(ctx context.Context, uri string) Error
// DeleteBlocksByOriginAccountID removes any blocks with accountID equal to originAccountID.
DeleteBlocksByOriginAccountID(ctx context.Context, originAccountID string) Error
// DeleteBlocksByTargetAccountID removes any blocks with given targetAccountID.
DeleteBlocksByTargetAccountID(ctx context.Context, targetAccountID string) Error
// GetRelationship retrieves the relationship of the targetAccount to the requestingAccount.
GetRelationship(ctx context.Context, requestingAccount string, targetAccount string) (*gtsmodel.Relationship, Error)

2
internal/federation/federatingdb/create.go
View File

@ -103,7 +103,7 @@ func (f *federatingDB) activityBlock(ctx context.Context, asType vocab.Type, rec
}
block.ID = newID
if err := f.db.Put(ctx, block); err != nil {
if err := f.db.PutBlock(ctx, block); err != nil {
return fmt.Errorf("activityBlock: database error inserting block: %s", err)
}

2
internal/federation/federatingdb/undo.go
View File

@ -114,7 +114,7 @@ func (f *federatingDB) Undo(ctx context.Context, undo vocab.ActivityStreamsUndo)
return errors.New("UNDO: block object account and inbox account were not the same")
}
// delete any existing BLOCK
if err := f.db.DeleteWhere(ctx, []db.Where{{Key: "uri", Value: gtsBlock.URI}}, &gtsmodel.Block{}); err != nil {
if err := f.db.DeleteBlockByURI(ctx, gtsBlock.URI); err != nil {
return fmt.Errorf("UNDO: db error removing block: %s", err)
}
l.Debug("block undone")

4
internal/federation/federatingprotocol_test.go
View File

@ -312,7 +312,7 @@ func (suite *FederatingProtocolTestSuite) TestBlocked2() {
ctxWithOtherInvolvedIRIs := context.WithValue(ctxWithRequestingAccount, ap.ContextOtherInvolvedIRIs, otherInvolvedIRIs)
// insert a block from inboxAccount targeting sendingAccount
if err := suite.db.Put(context.Background(), &gtsmodel.Block{
if err := suite.db.PutBlock(context.Background(), &gtsmodel.Block{
ID: "01G3KBEMJD4VQ2D615MPV7KTRD",
URI: "whatever",
AccountID: inboxAccount.ID,
@ -350,7 +350,7 @@ func (suite *FederatingProtocolTestSuite) TestBlocked3() {
ctxWithOtherInvolvedIRIs := context.WithValue(ctxWithRequestingAccount, ap.ContextOtherInvolvedIRIs, otherInvolvedIRIs)
// insert a block from inboxAccount targeting CCed account
if err := suite.db.Put(context.Background(), &gtsmodel.Block{
if err := suite.db.PutBlock(context.Background(), &gtsmodel.Block{
ID: "01G3KBEMJD4VQ2D615MPV7KTRD",
URI: "whatever",
AccountID: inboxAccount.ID,

2
internal/processing/account/createblock.go
View File

@ -65,7 +65,7 @@ func (p *processor) BlockCreate(ctx context.Context, requestingAccount *gtsmodel
block.URI = uris.GenerateURIForBlock(requestingAccount.Username, newBlockID)
// whack it in the database
if err := p.db.Put(ctx, block); err != nil {
if err := p.db.PutBlock(ctx, block); err != nil {
return nil, gtserror.NewErrorInternalError(fmt.Errorf("BlockCreate: error creating block in db: %s", err))
}

4
internal/processing/account/delete.go
View File

@ -99,12 +99,12 @@ func (p *processor) Delete(ctx context.Context, account *gtsmodel.Account, origi
// 2. Delete account's blocks
l.Trace("deleting account blocks")
// first delete any blocks that this account created
if err := p.db.DeleteWhere(ctx, []db.Where{{Key: "account_id", Value: account.ID}}, &[]*gtsmodel.Block{}); err != nil {
if err := p.db.DeleteBlocksByOriginAccountID(ctx, account.ID); err != nil {
l.Errorf("error deleting blocks created by account: %s", err)
}
// now delete any blocks that target this account
if err := p.db.DeleteWhere(ctx, []db.Where{{Key: "target_account_id", Value: account.ID}}, &[]*gtsmodel.Block{}); err != nil {
if err := p.db.DeleteBlocksByTargetAccountID(ctx, account.ID); err != nil {
l.Errorf("error deleting blocks targeting account: %s", err)
}

21
internal/processing/account/removeblock.go
View File

@ -20,6 +20,7 @@ package account
import (
"context"
"errors"
"fmt"
"github.com/superseriousbusiness/gotosocial/internal/ap"
@ -37,23 +38,17 @@ func (p *processor) BlockRemove(ctx context.Context, requestingAccount *gtsmodel
return nil, gtserror.NewErrorNotFound(fmt.Errorf("BlockCreate: error getting account %s from the db: %s", targetAccountID, err))
}
// check if a block exists, and remove it if it does (storing the URI for later)
var blockChanged bool
block := &gtsmodel.Block{}
if err := p.db.GetWhere(ctx, []db.Where{
{Key: "account_id", Value: requestingAccount.ID},
{Key: "target_account_id", Value: targetAccountID},
}, block); err == nil {
// check if a block exists, and remove it if it does
block, err := p.db.GetBlock(ctx, requestingAccount.ID, targetAccountID)
if err == nil {
// we got a block, remove it
block.Account = requestingAccount
block.TargetAccount = targetAccount
if err := p.db.DeleteByID(ctx, block.ID, &gtsmodel.Block{}); err != nil {
if err := p.db.DeleteBlockByID(ctx, block.ID); err != nil {
return nil, gtserror.NewErrorInternalError(fmt.Errorf("BlockRemove: error removing block from db: %s", err))
}
blockChanged = true
}
// block status changed so send the UNDO activity to the channel for async processing
if blockChanged {
// send the UNDO activity to the client worker for async processing
p.clientWorker.Queue(messages.FromClientAPI{
APObjectType: ap.ActivityBlock,
APActivityType: ap.ActivityUndo,
@ -61,6 +56,8 @@ func (p *processor) BlockRemove(ctx context.Context, requestingAccount *gtsmodel
OriginAccount: requestingAccount,
TargetAccount: targetAccount,
})
} else if !errors.Is(err, db.ErrNoEntries) {
return nil, gtserror.NewErrorInternalError(fmt.Errorf("BlockRemove: error getting possible block from db: %s", err))
}
// return whatever relationship results from all this