mirrors
/
gotosocial
mirror of
1
Fork 0
Code Issues Releases Activity

[performance] Add dereference shortcuts to avoid making http calls to self (#430) 

* update transport (controller) to allow shortcuts

* go fmt

* expose underlying sig transport to allow test sigs
This commit is contained in:
tobi 2022-03-15 15:01:19 +01:00 committed by GitHub
parent 4b4c935e02
commit e63b653199
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 114 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/gotosocial/action/server/server.go
View File

@ -117,7 +117,7 @@ var Start action.GTSAction = func(ctx context.Context) error {
return fmt.Errorf("error creating media manager: %s", err)
}
oauthServer := oauth.New(ctx, dbService)
transportController := transport.NewController(dbService, &federation.Clock{}, http.DefaultClient)
transportController := transport.NewController(dbService, federatingDB, &federation.Clock{}, http.DefaultClient)
federator := federation.NewFederator(dbService, federatingDB, transportController, typeConverter, mediaManager)
// decide whether to create a noop email sender (won't send emails) or a real one

80
internal/transport/controller.go
View File

@ -21,14 +21,18 @@ package transport
import (
"context"
"crypto"
"encoding/json"
"fmt"
"net/url"
"sync"
"github.com/go-fed/httpsig"
"github.com/spf13/viper"
"github.com/superseriousbusiness/activity/pub"
"github.com/superseriousbusiness/activity/streams"
"github.com/superseriousbusiness/gotosocial/internal/config"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/federation/federatingdb"
)
// Controller generates transports for use in making federation requests to other servers.
@ -42,19 +46,65 @@ type controller struct {
clock pub.Clock
client pub.HttpClient
appAgent string
// dereferenceFollowersShortcut is a shortcut to dereference followers of an
// account on this instance, without making any external api/http calls.
//
// It is passed to new transports, and should only be invoked when the iri.Host == this host.
dereferenceFollowersShortcut func(ctx context.Context, iri *url.URL) ([]byte, error)
// dereferenceUserShortcut is a shortcut to dereference followers an account on
// this instance, without making any external api/http calls.
//
// It is passed to new transports, and should only be invoked when the iri.Host == this host.
dereferenceUserShortcut func(ctx context.Context, iri *url.URL) ([]byte, error)
}
func dereferenceFollowersShortcut(federatingDB federatingdb.DB) func(context.Context, *url.URL) ([]byte, error) {
return func(ctx context.Context, iri *url.URL) ([]byte, error) {
followers, err := federatingDB.Followers(ctx, iri)
if err != nil {
return nil, err
}
i, err := streams.Serialize(followers)
if err != nil {
return nil, err
}
return json.Marshal(i)
}
}
func dereferenceUserShortcut(federatingDB federatingdb.DB) func(context.Context, *url.URL) ([]byte, error) {
return func(ctx context.Context, iri *url.URL) ([]byte, error) {
user, err := federatingDB.Get(ctx, iri)
if err != nil {
return nil, err
}
i, err := streams.Serialize(user)
if err != nil {
return nil, err
}
return json.Marshal(i)
}
}
// NewController returns an implementation of the Controller interface for creating new transports
func NewController(db db.DB, clock pub.Clock, client pub.HttpClient) Controller {
func NewController(db db.DB, federatingDB federatingdb.DB, clock pub.Clock, client pub.HttpClient) Controller {
applicationName := viper.GetString(config.Keys.ApplicationName)
host := viper.GetString(config.Keys.Host)
appAgent := fmt.Sprintf("%s %s", applicationName, host)
return &controller{
db: db,
clock: clock,
client: client,
appAgent: appAgent,
db: db,
clock: clock,
client: client,
appAgent: appAgent,
dereferenceFollowersShortcut: dereferenceFollowersShortcut(federatingDB),
dereferenceUserShortcut: dereferenceUserShortcut(federatingDB),
}
}
@ -78,15 +128,17 @@ func (c *controller) NewTransport(pubKeyID string, privkey crypto.PrivateKey) (T
sigTransport := pub.NewHttpSigTransport(c.client, c.appAgent, c.clock, getSigner, postSigner, pubKeyID, privkey)
return &transport{
client: c.client,
appAgent: c.appAgent,
gofedAgent: "(go-fed/activity v1.0.0)",
clock: c.clock,
pubKeyID: pubKeyID,
privkey: privkey,
sigTransport: sigTransport,
getSigner: getSigner,
getSignerMu: &sync.Mutex{},
client: c.client,
appAgent: c.appAgent,
gofedAgent: "(go-fed/activity v1.0.0)",
clock: c.clock,
pubKeyID: pubKeyID,
privkey: privkey,
sigTransport: sigTransport,
getSigner: getSigner,
getSignerMu: &sync.Mutex{},
dereferenceFollowersShortcut: c.dereferenceFollowersShortcut,
dereferenceUserShortcut: c.dereferenceUserShortcut,
}, nil
}

7
internal/transport/deliver.go
View File

@ -23,6 +23,8 @@ import (
"net/url"
"github.com/sirupsen/logrus"
"github.com/spf13/viper"
"github.com/superseriousbusiness/gotosocial/internal/config"
)
func (t *transport) BatchDeliver(ctx context.Context, b []byte, recipients []*url.URL) error {
@ -30,6 +32,11 @@ func (t *transport) BatchDeliver(ctx context.Context, b []byte, recipients []*ur
}
func (t *transport) Deliver(ctx context.Context, b []byte, to *url.URL) error {
// if the 'to' host is our own, just skip this delivery since we by definition already have the message!
if to.Host == viper.GetString(config.Keys.Host) {
return nil
}
l := logrus.WithField("func", "Deliver")
l.Debugf("performing POST to %s", to.String())
return t.sigTransport.Deliver(ctx, b, to)

19
internal/transport/dereference.go
View File

@ -23,10 +23,29 @@ import (
"net/url"
"github.com/sirupsen/logrus"
"github.com/spf13/viper"
"github.com/superseriousbusiness/gotosocial/internal/config"
"github.com/superseriousbusiness/gotosocial/internal/uris"
)
func (t *transport) Dereference(ctx context.Context, iri *url.URL) ([]byte, error) {
l := logrus.WithField("func", "Dereference")
// if the request is to us, we can shortcut for certain URIs rather than going through
// the normal request flow, thereby saving time and energy
if iri.Host == viper.GetString(config.Keys.Host) {
if uris.IsFollowersPath(iri) {
// the request is for followers of one of our accounts, which we can shortcut
return t.dereferenceFollowersShortcut(ctx, iri)
}
if uris.IsUserPath(iri) {
// the request is for one of our accounts, which we can shortcut
return t.dereferenceUserShortcut(ctx, iri)
}
}
// the request is either for a remote host or for us but we don't have a shortcut, so continue as normal
l.Debugf("performing GET to %s", iri.String())
return t.sigTransport.Dereference(ctx, iri)
}

18
internal/transport/transport.go
View File

@ -30,8 +30,11 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
)
// Transport wraps the pub.Transport interface with some additional
// functionality for fetching remote media.
// Transport wraps the pub.Transport interface with some additional functionality for fetching remote media.
//
// Since the transport has the concept of 'shortcuts' for fetching data locally rather than remotely, it is
// not *always* the case that calling a Transport function does an http call, but it usually will for remote
// hosts or resources for which a shortcut isn't provided by the transport controller (also in this package).
type Transport interface {
pub.Transport
// DereferenceMedia fetches the given media attachment IRI, returning the reader and filesize.
@ -40,6 +43,8 @@ type Transport interface {
DereferenceInstance(ctx context.Context, iri *url.URL) (*gtsmodel.Instance, error)
// Finger performs a webfinger request with the given username and domain, and returns the bytes from the response body.
Finger(ctx context.Context, targetUsername string, targetDomains string) ([]byte, error)
// SigTransport returns the underlying http signature transport wrapped by the GoToSocial transport.
SigTransport() pub.Transport
}
// transport implements the Transport interface
@ -53,4 +58,13 @@ type transport struct {
sigTransport *pub.HttpSigTransport
getSigner httpsig.Signer
getSignerMu *sync.Mutex
// shortcuts for dereferencing things that exist on our instance without making an http call to ourself
dereferenceFollowersShortcut func(ctx context.Context, iri *url.URL) ([]byte, error)
dereferenceUserShortcut func(ctx context.Context, iri *url.URL) ([]byte, error)
}
func (t *transport) SigTransport() pub.Transport {
return t.sigTransport
}

8
testrig/testmodels.go
View File

@ -1749,8 +1749,8 @@ func GetSignatureForActivity(activity pub.Activity, pubKeyID string, privkey cry
panic(err)
}
// trigger the delivery function, which will trigger the 'do' function of the recorder above
if err := tp.Deliver(context.Background(), bytes, destination); err != nil {
// trigger the delivery function for the underlying signature transport, which will trigger the 'do' function of the recorder above
if err := tp.SigTransport().Deliver(context.Background(), bytes, destination); err != nil {
panic(err)
}
@ -1781,8 +1781,8 @@ func GetSignatureForDereference(pubKeyID string, privkey crypto.PrivateKey, dest
panic(err)
}
// trigger the delivery function, which will trigger the 'do' function of the recorder above
if _, err := tp.Dereference(context.Background(), destination); err != nil {
// trigger the dereference function for the underlying signature transport, which will trigger the 'do' function of the recorder above
if _, err := tp.SigTransport().Dereference(context.Background(), destination); err != nil {
panic(err)
}

2
testrig/transportcontroller.go
View File

@ -39,7 +39,7 @@ import (
// PER TEST rather than per suite, so that the do function can be set on a test by test (or even more granular)
// basis.
func NewTestTransportController(client pub.HttpClient, db db.DB) transport.Controller {
return transport.NewController(db, &federation.Clock{}, client)
return transport.NewController(db, NewTestFederatingDB(db), &federation.Clock{}, client)
}
// NewMockHTTPClient returns a client that conforms to the pub.HttpClient interface,