[chore] Webfinger rework (#627)

* move finger to dereferencer

* totally break GetRemoteAccount

* start reworking finger func a bit

* start reworking getRemoteAccount a bit

* move mention parts to namestring

* rework webfingerget

* use util function to extract webfinger parts

* use accountDomain

* rework finger again, final form

* just a real nasty commit, the worst

* remove refresh from account

* use new ASRepToAccount signature

* fix incorrect debug call

* fix for new getRemoteAccount

* rework GetRemoteAccount

* start updating tests to remove repetition

* break a lot of tests
Move shared test logic into the testrig,
rather than having it scattered all over
the place. This allows us to just mock
the transport controller once, and have
all tests use it (unless they need not to
for some other reason).

* fix up tests to use main mock httpclient

* webfinger only if necessary

* cheeky linting with the lads

* update mentionName regex
recognize instance accounts

* don't finger instance accounts

* test webfinger part extraction

* increase default worker count to 4 per cpu

* don't repeat regex parsing

* final search for discovered accountDomain

* be more permissive in namestring lookup

* add more extraction tests

* simplify GetParseMentionFunc

* skip long search if local account

* fix broken test

cmd/gotosocial/action/testrig/testrig.go

@@ -85,7 +85,7 @@ var Start action.GTSAction = func(ctx context.Context) error {
 			StatusCode: 200,
 			Body:       r,
 		}, nil
-	}), dbService, fedWorker)
+	}, ""), dbService, fedWorker)
 	mediaManager := testrig.NewTestMediaManager(dbService, storageBackend)
 	federator := testrig.NewTestFederator(dbService, transportController, storageBackend, mediaManager, fedWorker)
 

internal/ap/extract.go

@@ -574,7 +574,7 @@ func ExtractMention(i Mentionable) (*gtsmodel.Mention, error) {
 	}
 
 	// just make sure the mention string is valid so we can handle it properly later on...
-	_, _, err = util.ExtractMentionParts(mentionString)
+	_, _, err = util.ExtractNamestringParts(mentionString)
 	if err != nil {
 		return nil, err
 	}

internal/api/client/account/account_test.go

@@ -67,7 +67,7 @@ func (suite *AccountStandardTestSuite) SetupTest() {
 	suite.db = testrig.NewTestDB()
 	suite.storage = testrig.NewTestStorage()
 	suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
-	suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker)
+	suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker)
 	suite.sentEmails = make(map[string]string)
 	suite.emailSender = testrig.NewEmailSender("../../../../web/template/", suite.sentEmails)
 	suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager, clientWorker, fedWorker)

internal/api/client/admin/admin_test.go

@@ -85,7 +85,7 @@ func (suite *AdminStandardTestSuite) SetupTest() {
 	suite.db = testrig.NewTestDB()
 	suite.storage = testrig.NewTestStorage()
 	suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
-	suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker)
+	suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker)
 	suite.sentEmails = make(map[string]string)
 	suite.emailSender = testrig.NewEmailSender("../../../../web/template/", suite.sentEmails)
 	suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager, clientWorker, fedWorker)

internal/api/client/auth/auth_test.go

@@ -90,7 +90,7 @@ func (suite *AuthStandardTestSuite) SetupTest() {
 	suite.db = testrig.NewTestDB()
 	suite.storage = testrig.NewTestStorage()
 	suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
-	suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker)
+	suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker)
 	suite.emailSender = testrig.NewEmailSender("../../../../web/template/", nil)
 	suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager, clientWorker, fedWorker)
 

internal/api/client/fileserver/servefile_test.go

@@ -82,7 +82,7 @@ func (suite *ServeFileTestSuite) SetupSuite() {
 
 	suite.db = testrig.NewTestDB()
 	suite.storage = testrig.NewTestStorage()
-	suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker), suite.storage, testrig.NewTestMediaManager(suite.db, suite.storage), fedWorker)
+	suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker)
 	suite.emailSender = testrig.NewEmailSender("../../../../web/template/", nil)
 
 	suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, testrig.NewTestMediaManager(suite.db, suite.storage), clientWorker, fedWorker)

internal/api/client/followrequest/followrequest_test.go

@@ -82,7 +82,7 @@ func (suite *FollowRequestStandardTestSuite) SetupTest() {
 	suite.db = testrig.NewTestDB()
 	suite.storage = testrig.NewTestStorage()
 	suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
-	suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker)
+	suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker)
 	suite.emailSender = testrig.NewEmailSender("../../../../web/template/", nil)
 	suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager, clientWorker, fedWorker)
 	suite.followRequestModule = followrequest.New(suite.processor).(*followrequest.Module)

internal/api/client/instance/instance_test.go

@@ -85,7 +85,7 @@ func (suite *InstanceStandardTestSuite) SetupTest() {
 	suite.db = testrig.NewTestDB()
 	suite.storage = testrig.NewTestStorage()
 	suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
-	suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker)
+	suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker)
 	suite.sentEmails = make(map[string]string)
 	suite.emailSender = testrig.NewEmailSender("../../../../web/template/", suite.sentEmails)
 	suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager, clientWorker, fedWorker)

internal/api/client/media/mediacreate_test.go

@@ -91,7 +91,7 @@ func (suite *MediaCreateTestSuite) SetupSuite() {
 	suite.tc = testrig.NewTestTypeConverter(suite.db)
 	suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
 	suite.oauthServer = testrig.NewTestOauthServer(suite.db)
-	suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker)
+	suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker)
 	suite.emailSender = testrig.NewEmailSender("../../../../web/template/", nil)
 	suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager, clientWorker, fedWorker)
 

internal/api/client/media/mediaupdate_test.go

@@ -89,7 +89,7 @@ func (suite *MediaUpdateTestSuite) SetupSuite() {
 	suite.tc = testrig.NewTestTypeConverter(suite.db)
 	suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
 	suite.oauthServer = testrig.NewTestOauthServer(suite.db)
-	suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker)
+	suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker)
 	suite.emailSender = testrig.NewEmailSender("../../../../web/template/", nil)
 	suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager, clientWorker, fedWorker)
 

internal/api/client/status/status_test.go

@@ -19,18 +19,8 @@
 package status_test
 
 import (
-	"bytes"
-	"context"
-	"encoding/json"
-	"fmt"
-	"io"
-	"io/ioutil"
-	"net/http"
-
 	"codeberg.org/gruf/go-store/kv"
 	"github.com/stretchr/testify/suite"
-	"github.com/superseriousbusiness/activity/pub"
-	"github.com/superseriousbusiness/activity/streams"
 	"github.com/superseriousbusiness/gotosocial/internal/api/client/status"
 	"github.com/superseriousbusiness/gotosocial/internal/concurrency"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
@@ -94,7 +84,7 @@ func (suite *StatusStandardTestSuite) SetupTest() {
 	clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1)
 
 	suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
-	suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(suite.testHttpClient(), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker)
+	suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker)
 	suite.emailSender = testrig.NewEmailSender("../../../../web/template/", nil)
 	suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager, clientWorker, fedWorker)
 	suite.statusModule = status.New(suite.processor).(*status.Module)
@@ -104,88 +94,3 @@ func (suite *StatusStandardTestSuite) TearDownTest() {
 	testrig.StandardDBTeardown(suite.db)
 	testrig.StandardStorageTeardown(suite.storage)
 }
-
-func (suite *StatusStandardTestSuite) testHttpClient() pub.HttpClient {
-	remoteAccount := suite.testAccounts["remote_account_1"]
-	remoteAccountNamestring := fmt.Sprintf("acct:%s@%s", remoteAccount.Username, remoteAccount.Domain)
-	remoteAccountWebfingerURI := fmt.Sprintf("https://%s/.well-known/webfinger?resource=%s", remoteAccount.Domain, remoteAccountNamestring)
-
-	fmt.Println(remoteAccountWebfingerURI)
-
-	httpClient := testrig.NewMockHTTPClient(func(req *http.Request) (*http.Response, error) {
-		// respond correctly to a webfinger lookup
-		if req.URL.String() == remoteAccountWebfingerURI {
-			responseJson := fmt.Sprintf(`
-			{
-				"subject": "%s",
-				"aliases": [
-				  "%s",
-				  "%s"
-				],
-				"links": [
-				  {
-					"rel": "http://webfinger.net/rel/profile-page",
-					"type": "text/html",
-					"href": "%s"
-				  },
-				  {
-					"rel": "self",
-					"type": "application/activity+json",
-					"href": "%s"
-				  }
-				]
-			}`, remoteAccountNamestring, remoteAccount.URI, remoteAccount.URL, remoteAccount.URL, remoteAccount.URI)
-			responseType := "application/json"
-
-			reader := bytes.NewReader([]byte(responseJson))
-			readCloser := io.NopCloser(reader)
-			response := &http.Response{
-				StatusCode:    200,
-				Body:          readCloser,
-				ContentLength: int64(len(responseJson)),
-				Header: http.Header{
-					"content-type": {responseType},
-				},
-			}
-			return response, nil
-		}
-
-		// respond correctly to an account dereference
-		if req.URL.String() == remoteAccount.URI {
-			satanAS, err := suite.tc.AccountToAS(context.Background(), remoteAccount)
-			if err != nil {
-				panic(err)
-			}
-
-			satanI, err := streams.Serialize(satanAS)
-			if err != nil {
-				panic(err)
-			}
-			satanJson, err := json.Marshal(satanI)
-			if err != nil {
-				panic(err)
-			}
-			responseType := "application/activity+json"
-
-			reader := bytes.NewReader(satanJson)
-			readCloser := io.NopCloser(reader)
-			response := &http.Response{
-				StatusCode:    200,
-				Body:          readCloser,
-				ContentLength: int64(len(satanJson)),
-				Header: http.Header{
-					"content-type": {responseType},
-				},
-			}
-			return response, nil
-		}
-
-		r := ioutil.NopCloser(bytes.NewReader([]byte{}))
-		return &http.Response{
-			StatusCode: 200,
-			Body:       r,
-		}, nil
-	})
-
-	return httpClient
-}

internal/api/client/status/statuscreate_test.go

@@ -128,7 +128,7 @@ func (suite *StatusCreateTestSuite) TestMentionUnknownAccount() {
 	ctx.Request = httptest.NewRequest(http.MethodPost, fmt.Sprintf("http://localhost:8080/%s", status.BasePath), nil) // the endpoint we're hitting
 	ctx.Request.Header.Set("accept", "application/json")
 	ctx.Request.Form = url.Values{
-		"status":     {"hello @foss_satan@fossbros-anonymous.io"},
+		"status":     {"hello @brand_new_person@unknown-instance.com"},
 		"visibility": {string(model.VisibilityPublic)},
 	}
 	suite.statusModule.StatusCreatePOSTHandler(ctx)
@@ -145,7 +145,7 @@ func (suite *StatusCreateTestSuite) TestMentionUnknownAccount() {
 	suite.NoError(err)
 
 	// if the status is properly formatted, that means the account has been put in the db
-	suite.Equal("<p>hello <span class=\"h-card\"><a href=\"http://fossbros-anonymous.io/@foss_satan\" class=\"u-url mention\" rel=\"nofollow noreferrer noopener\" target=\"_blank\">@<span>foss_satan</span></a></span></p>", statusReply.Content)
+	suite.Equal(`<p>hello <span class="h-card"><a href="https://unknown-instance.com/@brand_new_person" class="u-url mention" rel="nofollow noreferrer noopener" target="_blank">@<span>brand_new_person</span></a></span></p>`, statusReply.Content)
 	suite.Equal(model.VisibilityPublic, statusReply.Visibility)
 }
 

internal/api/client/user/user_test.go

@@ -69,7 +69,7 @@ func (suite *UserStandardTestSuite) SetupTest() {
 	suite.storage = testrig.NewTestStorage()
 	suite.tc = testrig.NewTestTypeConverter(suite.db)
 	suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
-	suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker)
+	suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker)
 	suite.sentEmails = make(map[string]string)
 	suite.emailSender = testrig.NewEmailSender("../../../../web/template/", suite.sentEmails)
 	suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager, clientWorker, fedWorker)

internal/api/s2s/user/inboxpost_test.go

@@ -88,7 +88,7 @@ func (suite *InboxPostTestSuite) TestPostBlock() {
 	clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1)
 	fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1)
 
-	tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker)
+	tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker)
 	federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager, fedWorker)
 	emailSender := testrig.NewEmailSender("../../../../web/template/", nil)
 	processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager, clientWorker, fedWorker)
@@ -191,7 +191,7 @@ func (suite *InboxPostTestSuite) TestPostUnblock() {
 	clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1)
 	fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1)
 
-	tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker)
+	tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker)
 	federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager, fedWorker)
 	emailSender := testrig.NewEmailSender("../../../../web/template/", nil)
 	processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager, clientWorker, fedWorker)
@@ -284,7 +284,7 @@ func (suite *InboxPostTestSuite) TestPostUpdate() {
 	clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1)
 	fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1)
 
-	tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker)
+	tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker)
 	federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager, fedWorker)
 	emailSender := testrig.NewEmailSender("../../../../web/template/", nil)
 	processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager, clientWorker, fedWorker)
@@ -406,7 +406,7 @@ func (suite *InboxPostTestSuite) TestPostDelete() {
 	clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1)
 	fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1)
 
-	tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker)
+	tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker)
 	federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager, fedWorker)
 	emailSender := testrig.NewEmailSender("../../../../web/template/", nil)
 	processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager, clientWorker, fedWorker)

internal/api/s2s/user/outboxget_test.go

@@ -49,7 +49,7 @@ func (suite *OutboxGetTestSuite) TestGetOutbox() {
 	clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1)
 	fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1)
 
-	tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker)
+	tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker)
 	federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager, fedWorker)
 	emailSender := testrig.NewEmailSender("../../../../web/template/", nil)
 	processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager, clientWorker, fedWorker)
@@ -107,7 +107,7 @@ func (suite *OutboxGetTestSuite) TestGetOutboxFirstPage() {
 	clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1)
 	fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1)
 
-	tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker)
+	tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker)
 	federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager, fedWorker)
 	emailSender := testrig.NewEmailSender("../../../../web/template/", nil)
 	processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager, clientWorker, fedWorker)
@@ -165,7 +165,7 @@ func (suite *OutboxGetTestSuite) TestGetOutboxNextPage() {
 	clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1)
 	fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1)
 
-	tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker)
+	tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker)
 	federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager, fedWorker)
 	emailSender := testrig.NewEmailSender("../../../../web/template/", nil)
 	processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager, clientWorker, fedWorker)

internal/api/s2s/user/repliesget_test.go

@@ -52,7 +52,7 @@ func (suite *RepliesGetTestSuite) TestGetReplies() {
 	clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1)
 	fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1)
 
-	tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker)
+	tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker)
 	federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager, fedWorker)
 	emailSender := testrig.NewEmailSender("../../../../web/template/", nil)
 	processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager, clientWorker, fedWorker)
@@ -116,7 +116,7 @@ func (suite *RepliesGetTestSuite) TestGetRepliesNext() {
 	clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1)
 	fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1)
 
-	tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker)
+	tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker)
 	federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager, fedWorker)
 	emailSender := testrig.NewEmailSender("../../../../web/template/", nil)
 	processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager, clientWorker, fedWorker)
@@ -183,7 +183,7 @@ func (suite *RepliesGetTestSuite) TestGetRepliesLast() {
 	clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1)
 	fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1)
 
-	tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker)
+	tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker)
 	federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager, fedWorker)
 	emailSender := testrig.NewEmailSender("../../../../web/template/", nil)
 	processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager, clientWorker, fedWorker)

internal/api/s2s/user/statusget_test.go

@@ -51,7 +51,7 @@ func (suite *StatusGetTestSuite) TestGetStatus() {
 	clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1)
 	fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1)
 
-	tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker)
+	tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker)
 	federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager, fedWorker)
 	emailSender := testrig.NewEmailSender("../../../../web/template/", nil)
 	processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager, clientWorker, fedWorker)
@@ -119,7 +119,7 @@ func (suite *StatusGetTestSuite) TestGetStatusLowercase() {
 	clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1)
 	fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1)
 
-	tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker)
+	tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker)
 	federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager, fedWorker)
 	emailSender := testrig.NewEmailSender("../../../../web/template/", nil)
 	processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager, clientWorker, fedWorker)

internal/api/s2s/user/user_test.go

@@ -85,7 +85,7 @@ func (suite *UserStandardTestSuite) SetupTest() {
 	suite.tc = testrig.NewTestTypeConverter(suite.db)
 	suite.storage = testrig.NewTestStorage()
 	suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
-	suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker)
+	suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker)
 	suite.emailSender = testrig.NewEmailSender("../../../../web/template/", nil)
 	suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager, clientWorker, fedWorker)
 	suite.userModule = user.New(suite.processor).(*user.Module)

internal/api/s2s/user/userget_test.go

@@ -52,7 +52,7 @@ func (suite *UserGetTestSuite) TestGetUser() {
 	clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1)
 	fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1)
 
-	tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker)
+	tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker)
 	federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager, fedWorker)
 	emailSender := testrig.NewEmailSender("../../../../web/template/", nil)
 	processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager, clientWorker, fedWorker)
@@ -102,7 +102,7 @@ func (suite *UserGetTestSuite) TestGetUser() {
 
 	// convert person to account
 	// since this account is already known, we should get a pretty full model of it from the conversion
-	a, err := suite.tc.ASRepresentationToAccount(context.Background(), person, false)
+	a, err := suite.tc.ASRepresentationToAccount(context.Background(), person, "", false)
 	suite.NoError(err)
 	suite.EqualValues(targetAccount.Username, a.Username)
 }
@@ -133,7 +133,7 @@ func (suite *UserGetTestSuite) TestGetUserPublicKeyDeleted() {
 	clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1)
 	fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1)
 
-	tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker)
+	tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker)
 	federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager, fedWorker)
 	emailSender := testrig.NewEmailSender("../../../../web/template/", nil)
 	processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager, clientWorker, fedWorker)
@@ -182,7 +182,7 @@ func (suite *UserGetTestSuite) TestGetUserPublicKeyDeleted() {
 	suite.True(ok)
 
 	// convert person to account
-	a, err := suite.tc.ASRepresentationToAccount(context.Background(), person, false)
+	a, err := suite.tc.ASRepresentationToAccount(context.Background(), person, "", false)
 	suite.NoError(err)
 	suite.EqualValues(targetAccount.Username, a.Username)
 }

internal/api/s2s/webfinger/webfinger_test.go

@@ -88,7 +88,7 @@ func (suite *WebfingerStandardTestSuite) SetupTest() {
 	suite.tc = testrig.NewTestTypeConverter(suite.db)
 	suite.storage = testrig.NewTestStorage()
 	suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
-	suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker)
+	suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker)
 	suite.emailSender = testrig.NewEmailSender("../../../../web/template/", nil)
 	suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager, clientWorker, fedWorker)
 	suite.webfingerModule = webfinger.New(suite.processor).(*webfinger.Module)

internal/api/s2s/webfinger/webfingerget.go

@@ -22,13 +22,13 @@ import (
 	"context"
 	"fmt"
 	"net/http"
-	"strings"
 
 	"github.com/gin-gonic/gin"
 	"github.com/sirupsen/logrus"
 	"github.com/superseriousbusiness/gotosocial/internal/ap"
 	"github.com/superseriousbusiness/gotosocial/internal/api"
 	"github.com/superseriousbusiness/gotosocial/internal/config"
+	"github.com/superseriousbusiness/gotosocial/internal/util"
 )
 
 // WebfingerGETRequest swagger:operation GET /.well-known/webfinger webfingerGet
@@ -67,35 +67,19 @@ func (m *Module) WebfingerGETRequest(c *gin.Context) {
 		return
 	}
 
-	// remove the acct: prefix if it's present
+	requestedUsername, requestedHost, err := util.ExtractWebfingerParts(resourceQuery)
-	trimAcct := strings.TrimPrefix(resourceQuery, "acct:")
+	if err != nil {
-	// remove the first @ in @whatever@example.org if it's present
+		l.Debugf("bad webfinger request with resource query %s: %s", resourceQuery, err)
-	namestring := strings.TrimPrefix(trimAcct, "@")
+		c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("bad webfinger request with resource query %s", resourceQuery)})
-
-	// at this point we should have a string like some_user@example.org
-	l.Debugf("got finger request for '%s'", namestring)
-
-	usernameAndAccountDomain := strings.Split(namestring, "@")
-	if len(usernameAndAccountDomain) != 2 {
-		l.Debugf("aborting request because username and domain could not be parsed from %s", namestring)
-		c.JSON(http.StatusBadRequest, gin.H{"error": "bad request"})
-		return
-	}
-
-	username := strings.ToLower(usernameAndAccountDomain[0])
-	requestedAccountDomain := strings.ToLower(usernameAndAccountDomain[1])
-	if username == "" || requestedAccountDomain == "" {
-		l.Debug("aborting request because username or domain was empty")
-		c.JSON(http.StatusBadRequest, gin.H{"error": "bad request"})
 		return
 	}
 
 	accountDomain := config.GetAccountDomain()
 	host := config.GetHost()
 
-	if requestedAccountDomain != accountDomain && requestedAccountDomain != host {
+	if requestedHost != host && requestedHost != accountDomain {
-		l.Debugf("aborting request because accountDomain %s does not belong to this instance", requestedAccountDomain)
+		l.Debugf("aborting request because requestedHost %s does not belong to this instance", requestedHost)
-		c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("accountDomain %s does not belong to this instance", requestedAccountDomain)})
+		c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("requested host %s does not belong to this instance", requestedHost)})
 		return
 	}
 
@@ -106,7 +90,7 @@ func (m *Module) WebfingerGETRequest(c *gin.Context) {
 		ctx = context.WithValue(ctx, ap.ContextRequestingPublicKeyVerifier, verifier)
 	}
 
-	resp, errWithCode := m.processor.GetWebfingerAccount(ctx, username)
+	resp, errWithCode := m.processor.GetWebfingerAccount(ctx, requestedUsername)
 	if errWithCode != nil {
 		api.ErrorHandler(c, errWithCode, m.processor.InstanceGet)
 		return

internal/concurrency/workers.go

@@ -45,7 +45,7 @@ func NewWorkerPool[MsgType any](workers int, queueRatio int) *WorkerPool[MsgType
 
 	if workers < 1 {
 		// ensure sensible workers
-		workers = runtime.GOMAXPROCS(0) * 2
+		workers = runtime.GOMAXPROCS(0) * 4
 	}
 	if queueRatio < 1 {
 		// ensure sensible ratio

internal/federation/dereference.go

@@ -23,11 +23,12 @@ import (
 	"net/url"
 
 	"github.com/superseriousbusiness/gotosocial/internal/ap"
+	"github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 )
 
-func (f *federator) GetRemoteAccount(ctx context.Context, username string, remoteAccountID *url.URL, blocking bool, refresh bool) (*gtsmodel.Account, error) {
+func (f *federator) GetRemoteAccount(ctx context.Context, params dereferencing.GetRemoteAccountParams) (*gtsmodel.Account, error) {
-	return f.dereferencer.GetRemoteAccount(ctx, username, remoteAccountID, blocking, refresh)
+	return f.dereferencer.GetRemoteAccount(ctx, params)
 }
 
 func (f *federator) GetRemoteStatus(ctx context.Context, username string, remoteStatusID *url.URL, refetch, includeParent bool) (*gtsmodel.Status, ap.Statusable, error) {

internal/federation/dereferencing/account.go

@@ -33,12 +33,15 @@ import (
 	"github.com/superseriousbusiness/activity/streams"
 	"github.com/superseriousbusiness/activity/streams/vocab"
 	"github.com/superseriousbusiness/gotosocial/internal/ap"
+	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/id"
 	"github.com/superseriousbusiness/gotosocial/internal/media"
 	"github.com/superseriousbusiness/gotosocial/internal/transport"
 )
 
+var webfingerInterval = -48 * time.Hour // 2 days in the past
+
 func instanceAccount(account *gtsmodel.Account) bool {
 	return strings.EqualFold(account.Username, account.Domain) ||
 		account.FollowersURI == "" ||
@@ -46,97 +49,238 @@ func instanceAccount(account *gtsmodel.Account) bool {
 		(account.Username == "internal.fetch" && strings.Contains(account.Note, "internal service actor"))
 }
 
-// GetRemoteAccount completely dereferences a remote account, converts it to a GtS model account,
+// GetRemoteAccountParams wraps parameters for a remote account lookup.
-// puts it in the database, and returns it to a caller.
+type GetRemoteAccountParams struct {
-//
+	// The username of the user doing the lookup request (optional).
-// Refresh indicates whether--if the account exists in our db already--it should be refreshed by calling
+	// If not set, then the GtS instance account will be used to do the lookup.
-// the remote instance again. Blocking indicates whether the function should block until processing of
+	RequestingUsername string
-// the fetched account is complete.
+	// The ActivityPub URI of the remote account (optional).
-//
+	// If not set (nil), the ActivityPub URI of the remote account will be discovered
-// SIDE EFFECTS: remote account will be stored in the database, or updated if it already exists (and refresh is true).
+	// via webfinger, so you must set RemoteAccountUsername and RemoteAccountHost
-func (d *deref) GetRemoteAccount(ctx context.Context, username string, remoteAccountID *url.URL, blocking bool, refresh bool) (*gtsmodel.Account, error) {
+	// if this parameter is not set.
-	new := true
+	RemoteAccountID *url.URL
+	// The username of the remote account (optional).
+	// If RemoteAccountID is not set, then this value must be set.
+	RemoteAccountUsername string
+	// The host of the remote account (optional).
+	// If RemoteAccountID is not set, then this value must be set.
+	RemoteAccountHost string
+	// Whether to do a blocking call to the remote instance. If true,
+	// then the account's media and other fields will be fully dereferenced before it is returned.
+	// If false, then the account's media and other fields will be dereferenced in the background,
+	// so only a minimal account representation will be returned by GetRemoteAccount.
+	Blocking bool
+	// Whether to skip making calls to remote instances. This is useful when you want to
+	// quickly fetch a remote account from the database or fail, and don't want to cause
+	// http requests to go flying around.
+	SkipResolve bool
+}
+
+// GetRemoteAccount completely dereferences a remote account, converts it to a GtS model account,
+// puts or updates it in the database (if necessary), and returns it to a caller.
+func (d *deref) GetRemoteAccount(ctx context.Context, params GetRemoteAccountParams) (remoteAccount *gtsmodel.Account, err error) {
+
+	/*
+		In this function we want to retrieve a gtsmodel representation of a remote account, with its proper
+		accountDomain set, while making as few calls to remote instances as possible to save time and bandwidth.
+
+		There are a few different paths through this function, and the path taken depends on how much
+		initial information we are provided with via parameters, how much information we already have stored,
+		and what we're allowed to do according to the parameters we've been passed.
+
+		Scenario 1: We're not allowed to resolve remotely, but we've got either the account URI or the
+		            account username + host, so we can check in our database and return if possible.
+
+		Scenario 2: We are allowed to resolve remotely, and we have an account URI but no username or host.
+		            In this case, we can use the URI to resolve the remote account and find the username,
+					and then we can webfinger the account to discover the accountDomain if necessary.
+
+		Scenario 3: We are allowed to resolve remotely, and we have the username and host but no URI.
+		            In this case, we can webfinger the account to discover the URI, and then dereference
+					from that.
+	*/
+
+	// first check if we can retrieve the account locally just with what we've been given
+	switch {
+	case params.RemoteAccountID != nil:
+		// try with uri
+		if a, dbErr := d.db.GetAccountByURI(ctx, params.RemoteAccountID.String()); dbErr == nil {
+			remoteAccount = a
+		} else if dbErr != db.ErrNoEntries {
+			err = fmt.Errorf("GetRemoteAccount: database error looking for account %s: %s", params.RemoteAccountID, err)
+		}
+	case params.RemoteAccountUsername != "" && params.RemoteAccountHost != "":
+		// try with username/host
+		a := &gtsmodel.Account{}
+		where := []db.Where{{Key: "username", Value: params.RemoteAccountUsername}, {Key: "domain", Value: params.RemoteAccountHost}}
+		if dbErr := d.db.GetWhere(ctx, where, a); dbErr == nil {
+			remoteAccount = a
+		} else if dbErr != db.ErrNoEntries {
+			err = fmt.Errorf("GetRemoteAccount: database error looking for account with username %s and host %s: %s", params.RemoteAccountUsername, params.RemoteAccountHost, err)
+		}
+	default:
+		err = errors.New("GetRemoteAccount: no identifying parameters were set so we cannot get account")
+	}
+
+	if err != nil {
+		return
+	}
+
+	if params.SkipResolve {
+		// if we can't resolve, return already since there's nothing more we can do
+		if remoteAccount == nil {
+			err = errors.New("GetRemoteAccount: error retrieving account with skipResolve set true")
+		}
+		return
+	}
+
+	var accountable ap.Accountable
+	if params.RemoteAccountUsername == "" || params.RemoteAccountHost == "" {
+		// try to populate the missing params
+		// the first one is easy ...
+		params.RemoteAccountHost = params.RemoteAccountID.Host
+		// ... but we still need the username so we can do a finger for the accountDomain
+
+		// check if we had the account stored already and got it earlier
+		if remoteAccount != nil {
+			params.RemoteAccountUsername = remoteAccount.Username
+		} else {
+			// if we didn't already have it, we have dereference it from remote and just...
+			accountable, err = d.dereferenceAccountable(ctx, params.RequestingUsername, params.RemoteAccountID)
+			if err != nil {
+				err = fmt.Errorf("GetRemoteAccount: error dereferencing accountable: %s", err)
+				return
+			}
+
+			// ... take the username (for now)
+			params.RemoteAccountUsername, err = ap.ExtractPreferredUsername(accountable)
+			if err != nil {
+				err = fmt.Errorf("GetRemoteAccount: error extracting accountable username: %s", err)
+				return
+			}
+		}
+	}
+
+	// if we reach this point, params.RemoteAccountHost and params.RemoteAccountUsername must be set
+	// params.RemoteAccountID may or may not be set, but we have enough information to fetch it if we need it
+
+	// we finger to fetch the account domain but just in case we're not fingering, make a best guess
+	// already about what the account domain might be; this var will be overwritten later if necessary
+	var accountDomain string
+	switch {
+	case remoteAccount != nil:
+		accountDomain = remoteAccount.Domain
+	case params.RemoteAccountID != nil:
+		accountDomain = params.RemoteAccountID.Host
+	default:
+		accountDomain = params.RemoteAccountHost
+	}
+
+	// to save on remote calls: only webfinger if we don't have a remoteAccount yet, or if we haven't
+	// fingered the remote account for at least 2 days; don't finger instance accounts
+	var fingered time.Time
+	if remoteAccount == nil || (remoteAccount.LastWebfingeredAt.Before(time.Now().Add(webfingerInterval)) && !instanceAccount(remoteAccount)) {
+		accountDomain, params.RemoteAccountID, err = d.fingerRemoteAccount(ctx, params.RequestingUsername, params.RemoteAccountUsername, params.RemoteAccountHost)
+		if err != nil {
+			err = fmt.Errorf("GetRemoteAccount: error while fingering: %s", err)
+			return
+		}
+		fingered = time.Now()
+	}
+
+	if !fingered.IsZero() && remoteAccount == nil {
+		// if we just fingered and now have a discovered account domain but still no account,
+		// we should do a final lookup in the database with the discovered username + accountDomain
+		// to make absolutely sure we don't already have this account
+		a := &gtsmodel.Account{}
+		where := []db.Where{{Key: "username", Value: params.RemoteAccountUsername}, {Key: "domain", Value: accountDomain}}
+		if dbErr := d.db.GetWhere(ctx, where, a); dbErr == nil {
+			remoteAccount = a
+		} else if dbErr != db.ErrNoEntries {
+			err = fmt.Errorf("GetRemoteAccount: database error looking for account with username %s and host %s: %s", params.RemoteAccountUsername, params.RemoteAccountHost, err)
+			return
+		}
+	}
+
+	// we may also have some extra information already, like the account we had in the db, or the
+	// accountable representation that we dereferenced from remote
+	if remoteAccount == nil {
+		// we still don't have the account, so deference it if we didn't earlier
+		if accountable == nil {
+			accountable, err = d.dereferenceAccountable(ctx, params.RequestingUsername, params.RemoteAccountID)
+			if err != nil {
+				err = fmt.Errorf("GetRemoteAccount: error dereferencing accountable: %s", err)
+				return
+			}
+		}
+
+		// then convert
+		remoteAccount, err = d.typeConverter.ASRepresentationToAccount(ctx, accountable, accountDomain, false)
+		if err != nil {
+			err = fmt.Errorf("GetRemoteAccount: error converting accountable to account: %s", err)
+			return
+		}
+
+		// this is a new account so we need to generate a new ID for it
+		var ulid string
+		ulid, err = id.NewRandomULID()
+		if err != nil {
+			err = fmt.Errorf("GetRemoteAccount: error generating new id for account: %s", err)
+			return
+		}
+		remoteAccount.ID = ulid
+
+		_, err = d.populateAccountFields(ctx, remoteAccount, params.RequestingUsername, params.Blocking)
+		if err != nil {
+			err = fmt.Errorf("GetRemoteAccount: error populating further account fields: %s", err)
+			return
+		}
+
+		remoteAccount.LastWebfingeredAt = fingered
+		remoteAccount.UpdatedAt = time.Now()
+
+		err = d.db.Put(ctx, remoteAccount)
+		if err != nil {
+			err = fmt.Errorf("GetRemoteAccount: error putting new account: %s", err)
+			return
+		}
+
+		return // the new account
+	}
+
+	// we had the account already, but now we know the account domain, so update it if it's different
+	if !strings.EqualFold(remoteAccount.Domain, accountDomain) {
+		remoteAccount.Domain = accountDomain
+		remoteAccount, err = d.db.UpdateAccount(ctx, remoteAccount)
+		if err != nil {
+			err = fmt.Errorf("GetRemoteAccount: error updating account: %s", err)
+			return
+		}
+	}
 
-	// check if we already have the account in our db, and just return it unless we'd doing a refresh
-	remoteAccount, err := d.db.GetAccountByURI(ctx, remoteAccountID.String())
-	if err == nil {
-		new = false
-		if !refresh {
 	// make sure the account fields are populated before returning:
-			// even if we're not doing a refresh, the caller might want to block
+	// the caller might want to block until everything is loaded
-			// until everything is loaded
+	var fieldsChanged bool
-			changed, err := d.populateAccountFields(ctx, remoteAccount, username, refresh, blocking)
+	fieldsChanged, err = d.populateAccountFields(ctx, remoteAccount, params.RequestingUsername, params.Blocking)
 	if err != nil {
 		return nil, fmt.Errorf("GetRemoteAccount: error populating remoteAccount fields: %s", err)
 	}
 
-			if changed {
+	var fingeredChanged bool
-				updatedAccount, err := d.db.UpdateAccount(ctx, remoteAccount)
+	if !fingered.IsZero() {
+		fingeredChanged = true
+		remoteAccount.LastWebfingeredAt = fingered
+	}
+
+	if fieldsChanged || fingeredChanged {
+		remoteAccount.UpdatedAt = time.Now()
+		remoteAccount, err = d.db.UpdateAccount(ctx, remoteAccount)
 		if err != nil {
 			return nil, fmt.Errorf("GetRemoteAccount: error updating remoteAccount: %s", err)
 		}
-				return updatedAccount, err
 	}
 
-			return remoteAccount, nil
+	return // the account we already had + possibly updated
-		}
-	}
-
-	if new {
-		// we haven't seen this account before: dereference it from remote
-		accountable, err := d.dereferenceAccountable(ctx, username, remoteAccountID)
-		if err != nil {
-			return nil, fmt.Errorf("GetRemoteAccount: error dereferencing accountable: %s", err)
-		}
-
-		newAccount, err := d.typeConverter.ASRepresentationToAccount(ctx, accountable, refresh)
-		if err != nil {
-			return nil, fmt.Errorf("GetRemoteAccount: error converting accountable to account: %s", err)
-		}
-
-		ulid, err := id.NewRandomULID()
-		if err != nil {
-			return nil, fmt.Errorf("GetRemoteAccount: error generating new id for account: %s", err)
-		}
-		newAccount.ID = ulid
-
-		if _, err := d.populateAccountFields(ctx, newAccount, username, refresh, blocking); err != nil {
-			return nil, fmt.Errorf("GetRemoteAccount: error populating further account fields: %s", err)
-		}
-
-		if err := d.db.Put(ctx, newAccount); err != nil {
-			return nil, fmt.Errorf("GetRemoteAccount: error putting new account: %s", err)
-		}
-
-		return newAccount, nil
-	}
-
-	// we have seen this account before, but we have to refresh it
-	refreshedAccountable, err := d.dereferenceAccountable(ctx, username, remoteAccountID)
-	if err != nil {
-		return nil, fmt.Errorf("GetRemoteAccount: error dereferencing refreshedAccountable: %s", err)
-	}
-
-	refreshedAccount, err := d.typeConverter.ASRepresentationToAccount(ctx, refreshedAccountable, refresh)
-	if err != nil {
-		return nil, fmt.Errorf("GetRemoteAccount: error converting refreshedAccountable to refreshedAccount: %s", err)
-	}
-	refreshedAccount.ID = remoteAccount.ID
-
-	changed, err := d.populateAccountFields(ctx, refreshedAccount, username, refresh, blocking)
-	if err != nil {
-		return nil, fmt.Errorf("GetRemoteAccount: error populating further refreshedAccount fields: %s", err)
-	}
-
-	if changed {
-		updatedAccount, err := d.db.UpdateAccount(ctx, refreshedAccount)
-		if err != nil {
-			return nil, fmt.Errorf("GetRemoteAccount: error updating refreshedAccount: %s", err)
-		}
-		return updatedAccount, nil
-	}
-
-	return refreshedAccount, nil
 }
 
 // dereferenceAccountable calls remoteAccountID with a GET request, and tries to parse whatever
@@ -209,7 +353,7 @@ func (d *deref) dereferenceAccountable(ctx context.Context, username string, rem
 
 // populateAccountFields populates any fields on the given account that weren't populated by the initial
 // dereferencing. This includes things like header and avatar etc.
-func (d *deref) populateAccountFields(ctx context.Context, account *gtsmodel.Account, requestingUsername string, blocking bool, refresh bool) (bool, error) {
+func (d *deref) populateAccountFields(ctx context.Context, account *gtsmodel.Account, requestingUsername string, blocking bool) (bool, error) {
 	// if we're dealing with an instance account, just bail, we don't need to do anything
 	if instanceAccount(account) {
 		return false, nil
@@ -230,7 +374,7 @@ func (d *deref) populateAccountFields(ctx context.Context, account *gtsmodel.Acc
 	}
 
 	// fetch the header and avatar
-	changed, err := d.fetchRemoteAccountMedia(ctx, account, t, refresh, blocking)
+	changed, err := d.fetchRemoteAccountMedia(ctx, account, t, blocking)
 	if err != nil {
 		return false, fmt.Errorf("populateAccountFields: error fetching header/avi for account: %s", err)
 	}
@@ -250,7 +394,7 @@ func (d *deref) populateAccountFields(ctx context.Context, account *gtsmodel.Acc
 //
 // If blocking is true, then the calls to the media manager made by this function will be blocking:
 // in other words, the function won't return until the header and the avatar have been fully processed.
-func (d *deref) fetchRemoteAccountMedia(ctx context.Context, targetAccount *gtsmodel.Account, t transport.Transport, blocking bool, refresh bool) (bool, error) {
+func (d *deref) fetchRemoteAccountMedia(ctx context.Context, targetAccount *gtsmodel.Account, t transport.Transport, blocking bool) (bool, error) {
 	changed := false
 
 	accountURI, err := url.Parse(targetAccount.URI)
@@ -262,7 +406,7 @@ func (d *deref) fetchRemoteAccountMedia(ctx context.Context, targetAccount *gtsm
 		return changed, fmt.Errorf("fetchRemoteAccountMedia: domain %s is blocked", accountURI.Host)
 	}
 
-	if targetAccount.AvatarRemoteURL != "" && (targetAccount.AvatarMediaAttachmentID == "" || refresh) {
+	if targetAccount.AvatarRemoteURL != "" && (targetAccount.AvatarMediaAttachmentID == "") {
 		var processingMedia *media.ProcessingMedia
 
 		d.dereferencingAvatarsLock.Lock() // LOCK HERE
@@ -320,7 +464,7 @@ func (d *deref) fetchRemoteAccountMedia(ctx context.Context, targetAccount *gtsm
 		changed = true
 	}
 
-	if targetAccount.HeaderRemoteURL != "" && (targetAccount.HeaderMediaAttachmentID == "" || refresh) {
+	if targetAccount.HeaderRemoteURL != "" && (targetAccount.HeaderMediaAttachmentID == "") {
 		var processingMedia *media.ProcessingMedia
 
 		d.dereferencingHeadersLock.Lock() // LOCK HERE

internal/federation/dereferencing/account_test.go

@@ -24,6 +24,7 @@ import (
 
 	"github.com/stretchr/testify/suite"
 	"github.com/superseriousbusiness/gotosocial/internal/ap"
+	"github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing"
 	"github.com/superseriousbusiness/gotosocial/testrig"
 )
 
@@ -35,7 +36,10 @@ func (suite *AccountTestSuite) TestDereferenceGroup() {
 	fetchingAccount := suite.testAccounts["local_account_1"]
 
 	groupURL := testrig.URLMustParse("https://unknown-instance.com/groups/some_group")
-	group, err := suite.dereferencer.GetRemoteAccount(context.Background(), fetchingAccount.Username, groupURL, false, false)
+	group, err := suite.dereferencer.GetRemoteAccount(context.Background(), dereferencing.GetRemoteAccountParams{
+		RequestingUsername: fetchingAccount.Username,
+		RemoteAccountID:    groupURL,
+	})
 	suite.NoError(err)
 	suite.NotNil(group)
 	suite.NotNil(group)
@@ -55,7 +59,10 @@ func (suite *AccountTestSuite) TestDereferenceService() {
 	fetchingAccount := suite.testAccounts["local_account_1"]
 
 	serviceURL := testrig.URLMustParse("https://owncast.example.org/federation/user/rgh")
-	service, err := suite.dereferencer.GetRemoteAccount(context.Background(), fetchingAccount.Username, serviceURL, false, false)
+	service, err := suite.dereferencer.GetRemoteAccount(context.Background(), dereferencing.GetRemoteAccountParams{
+		RequestingUsername: fetchingAccount.Username,
+		RemoteAccountID:    serviceURL,
+	})
 	suite.NoError(err)
 	suite.NotNil(service)
 	suite.NotNil(service)
@@ -69,6 +76,7 @@ func (suite *AccountTestSuite) TestDereferenceService() {
 	suite.NoError(err)
 	suite.Equal(service.ID, dbService.ID)
 	suite.Equal(ap.ActorService, dbService.ActorType)
+	suite.Equal("example.org", dbService.Domain)
 }
 
 func TestAccountTestSuite(t *testing.T) {

internal/federation/dereferencing/dereferencer.go

@@ -33,7 +33,7 @@ import (
 
 // Dereferencer wraps logic and functionality for doing dereferencing of remote accounts, statuses, etc, from federated instances.
 type Dereferencer interface {
-	GetRemoteAccount(ctx context.Context, username string, remoteAccountID *url.URL, blocking bool, refresh bool) (*gtsmodel.Account, error)
+	GetRemoteAccount(ctx context.Context, params GetRemoteAccountParams) (*gtsmodel.Account, error)
 
 	GetRemoteStatus(ctx context.Context, username string, remoteStatusID *url.URL, refetch, includeParent bool) (*gtsmodel.Status, ap.Statusable, error)
 	EnrichRemoteStatus(ctx context.Context, username string, status *gtsmodel.Status, includeParent bool) (*gtsmodel.Status, error)

internal/federation/dereferencing/dereferencer_test.go

@@ -19,22 +19,14 @@
 package dereferencing_test
 
 import (
-	"bytes"
-	"encoding/json"
-	"io"
-	"net/http"
-
 	"codeberg.org/gruf/go-store/kv"
-	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/suite"
-	"github.com/superseriousbusiness/activity/streams"
 	"github.com/superseriousbusiness/activity/streams/vocab"
 	"github.com/superseriousbusiness/gotosocial/internal/concurrency"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/messages"
-	"github.com/superseriousbusiness/gotosocial/internal/transport"
 	"github.com/superseriousbusiness/gotosocial/testrig"
 )
 
@@ -66,106 +58,10 @@ func (suite *DereferencerStandardTestSuite) SetupTest() {
 
 	suite.db = testrig.NewTestDB()
 	suite.storage = testrig.NewTestStorage()
-	suite.dereferencer = dereferencing.NewDereferencer(suite.db, testrig.NewTestTypeConverter(suite.db), suite.mockTransportController(), testrig.NewTestMediaManager(suite.db, suite.storage))
+	suite.dereferencer = dereferencing.NewDereferencer(suite.db, testrig.NewTestTypeConverter(suite.db), testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../testrig/media"), suite.db, concurrency.NewWorkerPool[messages.FromFederator](-1, -1)), testrig.NewTestMediaManager(suite.db, suite.storage))
 	testrig.StandardDBSetup(suite.db, nil)
 }
 
 func (suite *DereferencerStandardTestSuite) TearDownTest() {
 	testrig.StandardDBTeardown(suite.db)
 }
-
-// mockTransportController returns basically a miniature muxer, which returns a different
-// value based on the request URL. It can be used to return remote statuses, profiles, etc,
-// as though they were actually being dereferenced. If the URL doesn't correspond to any person
-// or note or attachment that we have stored, then just a 200 code will be returned, with an empty body.
-func (suite *DereferencerStandardTestSuite) mockTransportController() transport.Controller {
-	do := func(req *http.Request) (*http.Response, error) {
-		logrus.Debugf("received request for %s", req.URL)
-
-		responseBytes := []byte{}
-		responseType := ""
-		responseLength := 0
-
-		if note, ok := suite.testRemoteStatuses[req.URL.String()]; ok {
-			// the request is for a note that we have stored
-			noteI, err := streams.Serialize(note)
-			if err != nil {
-				panic(err)
-			}
-			noteJson, err := json.Marshal(noteI)
-			if err != nil {
-				panic(err)
-			}
-			responseBytes = noteJson
-			responseType = "application/activity+json"
-		}
-
-		if person, ok := suite.testRemotePeople[req.URL.String()]; ok {
-			// the request is for a person that we have stored
-			personI, err := streams.Serialize(person)
-			if err != nil {
-				panic(err)
-			}
-			personJson, err := json.Marshal(personI)
-			if err != nil {
-				panic(err)
-			}
-			responseBytes = personJson
-			responseType = "application/activity+json"
-		}
-
-		if group, ok := suite.testRemoteGroups[req.URL.String()]; ok {
-			// the request is for a person that we have stored
-			groupI, err := streams.Serialize(group)
-			if err != nil {
-				panic(err)
-			}
-			groupJson, err := json.Marshal(groupI)
-			if err != nil {
-				panic(err)
-			}
-			responseBytes = groupJson
-			responseType = "application/activity+json"
-		}
-
-		if service, ok := suite.testRemoteServices[req.URL.String()]; ok {
-			serviceI, err := streams.Serialize(service)
-			if err != nil {
-				panic(err)
-			}
-			serviceJson, err := json.Marshal(serviceI)
-			if err != nil {
-				panic(err)
-			}
-			responseBytes = serviceJson
-			responseType = "application/activity+json"
-		}
-
-		if attachment, ok := suite.testRemoteAttachments[req.URL.String()]; ok {
-			responseBytes = attachment.Data
-			responseType = attachment.ContentType
-		}
-
-		if len(responseBytes) != 0 {
-			// we found something, so print what we're going to return
-			logrus.Debugf("returning response %s", string(responseBytes))
-		}
-		responseLength = len(responseBytes)
-
-		reader := bytes.NewReader(responseBytes)
-		readCloser := io.NopCloser(reader)
-		response := &http.Response{
-			StatusCode:    200,
-			Body:          readCloser,
-			ContentLength: int64(responseLength),
-			Header: http.Header{
-				"content-type": {responseType},
-			},
-		}
-
-		return response, nil
-	}
-	fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1)
-	mockClient := testrig.NewMockHTTPClient(do)
-	return testrig.NewTestTransportController(mockClient, suite.db, fedWorker)
-}

internal/federation/dereferencing/finger.go

@@ -0,0 +1,80 @@
+/*
+   GoToSocial
+   Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package dereferencing
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/url"
+	"strings"
+
+	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
+	"github.com/superseriousbusiness/gotosocial/internal/util"
+)
+
+func (d *deref) fingerRemoteAccount(ctx context.Context, username string, targetUsername string, targetHost string) (accountDomain string, accountURI *url.URL, err error) {
+	t, err := d.transportController.NewTransportForUsername(ctx, username)
+	if err != nil {
+		err = fmt.Errorf("fingerRemoteAccount: error getting transport for %s: %s", username, err)
+		return
+	}
+
+	b, err := t.Finger(ctx, targetUsername, targetHost)
+	if err != nil {
+		err = fmt.Errorf("fingerRemoteAccount: error fingering @%s@%s: %s", targetUsername, targetHost, err)
+		return
+	}
+
+	resp := &apimodel.WellKnownResponse{}
+	if err = json.Unmarshal(b, resp); err != nil {
+		err = fmt.Errorf("fingerRemoteAccount: could not unmarshal server response as WebfingerAccountResponse while dereferencing @%s@%s: %s", targetUsername, targetHost, err)
+		return
+	}
+
+	if len(resp.Links) == 0 {
+		err = fmt.Errorf("fingerRemoteAccount: no links found in webfinger response %s", string(b))
+		return
+	}
+
+	if resp.Subject == "" {
+		err = fmt.Errorf("fingerRemoteAccount: no subject found in webfinger response %s", string(b))
+		return
+	}
+
+	_, accountDomain, err = util.ExtractWebfingerParts(resp.Subject)
+	if err != nil {
+		err = fmt.Errorf("fingerRemoteAccount: error extracting webfinger subject parts: %s", err)
+	}
+
+	// look through the links for the first one that matches what we need
+	for _, l := range resp.Links {
+		if l.Rel == "self" && (strings.EqualFold(l.Type, "application/activity+json") || strings.EqualFold(l.Type, "application/ld+json; profile=\"https://www.w3.org/ns/activitystreams\"")) {
+			if uri, thiserr := url.Parse(l.Href); thiserr == nil && (uri.Scheme == "http" || uri.Scheme == "https") {
+				// found it!
+				accountURI = uri
+				return
+			}
+		}
+	}
+
+	err = errors.New("fingerRemoteAccount: no match found in webfinger response")
+	return
+}

internal/federation/dereferencing/status.go

@@ -85,7 +85,10 @@ func (d *deref) GetRemoteStatus(ctx context.Context, username string, remoteStat
 		return nil, nil, fmt.Errorf("GetRemoteStatus: error extracting attributedTo: %s", err)
 	}
 
-	_, err = d.GetRemoteAccount(ctx, username, accountURI, true, false)
+	_, err = d.GetRemoteAccount(ctx, GetRemoteAccountParams{
+		RequestingUsername: username,
+		RemoteAccountID:    accountURI,
+	})
 	if err != nil {
 		return nil, nil, fmt.Errorf("GetRemoteStatus: couldn't get status author: %s", err)
 	}
@@ -316,7 +319,10 @@ func (d *deref) populateStatusMentions(ctx context.Context, status *gtsmodel.Sta
 		if targetAccount == nil {
 			// we didn't find the account in our database already
 			// check if we can get the account remotely (dereference it)
-			if a, err := d.GetRemoteAccount(ctx, requestingUsername, targetAccountURI, false, false); err != nil {
+			if a, err := d.GetRemoteAccount(ctx, GetRemoteAccountParams{
+				RequestingUsername: requestingUsername,
+				RemoteAccountID:    targetAccountURI,
+			}); err != nil {
 				errs = append(errs, err.Error())
 			} else {
 				logrus.Debugf("populateStatusMentions: got target account %s with id %s through GetRemoteAccount", targetAccountURI, a.ID)

internal/federation/federatingactor_test.go

@@ -19,10 +19,7 @@
 package federation_test
 
 import (
-	"bytes"
 	"context"
-	"io/ioutil"
-	"net/http"
 	"net/url"
 	"testing"
 	"time"
@@ -60,15 +57,9 @@ func (suite *FederatingActorTestSuite) TestSendNoRemoteFollowers() {
 	fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1)
 
 	// setup transport controller with a no-op client so we don't make external calls
-	sentMessages := []*url.URL{}
+	httpClient := testrig.NewMockHTTPClient(nil, "../../testrig/media")
-	tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(func(req *http.Request) (*http.Response, error) {
+	tc := testrig.NewTestTransportController(httpClient, suite.db, fedWorker)
-		sentMessages = append(sentMessages, req.URL)
+
-		r := ioutil.NopCloser(bytes.NewReader([]byte{}))
-		return &http.Response{
-			StatusCode: 200,
-			Body:       r,
-		}, nil
-	}), suite.db, fedWorker)
 	// setup module being tested
 	federator := federation.NewFederator(suite.db, testrig.NewTestFederatingDB(suite.db, fedWorker), tc, suite.tc, testrig.NewTestMediaManager(suite.db, suite.storage))
 
@@ -77,7 +68,7 @@ func (suite *FederatingActorTestSuite) TestSendNoRemoteFollowers() {
 	suite.NotNil(activity)
 
 	// because zork has no remote followers, sent messages should be empty (no messages sent to own instance)
-	suite.Empty(sentMessages)
+	suite.Empty(httpClient.SentMessages)
 }
 
 func (suite *FederatingActorTestSuite) TestSendRemoteFollower() {
@@ -87,8 +78,8 @@ func (suite *FederatingActorTestSuite) TestSendRemoteFollower() {
 
 	err := suite.db.Put(ctx, &gtsmodel.Follow{
 		ID:              "01G1TRWV4AYCDBX5HRWT2EVBCV",
-		CreatedAt:       time.Now(),
+		CreatedAt:       testrig.TimeMustParse("2022-06-02T12:22:21+02:00"),
-		UpdatedAt:       time.Now(),
+		UpdatedAt:       testrig.TimeMustParse("2022-06-02T12:22:21+02:00"),
 		AccountID:       testRemoteAccount.ID,
 		TargetAccountID: testAccount.ID,
 		ShowReblogs:     true,
@@ -100,7 +91,7 @@ func (suite *FederatingActorTestSuite) TestSendRemoteFollower() {
 	testNote := testrig.NewAPNote(
 		testrig.URLMustParse("http://localhost:8080/users/the_mighty_zork/statuses/01G1TR6BADACCZWQMNF9X21TV5"),
 		testrig.URLMustParse("http://localhost:8080/@the_mighty_zork/statuses/01G1TR6BADACCZWQMNF9X21TV5"),
-		time.Now(),
+		testrig.TimeMustParse("2022-06-02T12:22:21+02:00"),
 		"boobies",
 		"",
 		testrig.URLMustParse(testAccount.URI),
@@ -110,20 +101,12 @@ func (suite *FederatingActorTestSuite) TestSendRemoteFollower() {
 		nil,
 		nil,
 	)
-	testActivity := testrig.WrapAPNoteInCreate(testrig.URLMustParse("http://localhost:8080/whatever_some_create"), testrig.URLMustParse(testAccount.URI), time.Now(), testNote)
+	testActivity := testrig.WrapAPNoteInCreate(testrig.URLMustParse("http://localhost:8080/whatever_some_create"), testrig.URLMustParse(testAccount.URI), testrig.TimeMustParse("2022-06-02T12:22:21+02:00"), testNote)
 
 	fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1)
 
-	// setup transport controller with a no-op client so we don't make external calls
+	httpClient := testrig.NewMockHTTPClient(nil, "../../testrig/media")
-	sentMessages := []*url.URL{}
+	tc := testrig.NewTestTransportController(httpClient, suite.db, fedWorker)
-	tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(func(req *http.Request) (*http.Response, error) {
-		sentMessages = append(sentMessages, req.URL)
-		r := ioutil.NopCloser(bytes.NewReader([]byte{}))
-		return &http.Response{
-			StatusCode: 200,
-			Body:       r,
-		}, nil
-	}), suite.db, fedWorker)
 	// setup module being tested
 	federator := federation.NewFederator(suite.db, testrig.NewTestFederatingDB(suite.db, fedWorker), tc, suite.tc, testrig.NewTestMediaManager(suite.db, suite.storage))
 
@@ -132,8 +115,10 @@ func (suite *FederatingActorTestSuite) TestSendRemoteFollower() {
 	suite.NotNil(activity)
 
 	// because we added 1 remote follower for zork, there should be a url in sentMessage
-	suite.Len(sentMessages, 1)
+	suite.Len(httpClient.SentMessages, 1)
-	suite.Equal(testRemoteAccount.InboxURI, sentMessages[0].String())
+	msg, ok := httpClient.SentMessages[testRemoteAccount.InboxURI]
+	suite.True(ok)
+	suite.Equal(`{"@context":"https://www.w3.org/ns/activitystreams","actor":"http://localhost:8080/users/the_mighty_zork","id":"http://localhost:8080/whatever_some_create","object":{"attributedTo":"http://localhost:8080/users/the_mighty_zork","content":"boobies","id":"http://localhost:8080/users/the_mighty_zork/statuses/01G1TR6BADACCZWQMNF9X21TV5","published":"2022-06-02T12:22:21+02:00","tag":[],"to":"http://localhost:8080/users/the_mighty_zork/followers","type":"Note","url":"http://localhost:8080/@the_mighty_zork/statuses/01G1TR6BADACCZWQMNF9X21TV5"},"published":"2022-06-02T12:22:21+02:00","to":"http://localhost:8080/users/the_mighty_zork/followers","type":"Create"}`, string(msg))
 }
 
 func TestFederatingActorTestSuite(t *testing.T) {

internal/federation/federatingdb/update.go

@@ -119,7 +119,7 @@ func (f *federatingDB) Update(ctx context.Context, asType vocab.Type) error {
 			accountable = i
 		}
 
-		updatedAcct, err := f.typeConverter.ASRepresentationToAccount(ctx, accountable, true)
+		updatedAcct, err := f.typeConverter.ASRepresentationToAccount(ctx, accountable, "", true)
 		if err != nil {
 			return fmt.Errorf("UPDATE: error converting to account: %s", err)
 		}

internal/federation/federatingprotocol.go

@@ -31,6 +31,7 @@ import (
 	"github.com/superseriousbusiness/activity/streams/vocab"
 	"github.com/superseriousbusiness/gotosocial/internal/ap"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/uris"
 	"github.com/superseriousbusiness/gotosocial/internal/util"
@@ -197,7 +198,10 @@ func (f *federator) AuthenticatePostInbox(ctx context.Context, w http.ResponseWr
 		}
 	}
 
-	requestingAccount, err := f.GetRemoteAccount(ctx, username, publicKeyOwnerURI, false, false)
+	requestingAccount, err := f.GetRemoteAccount(ctx, dereferencing.GetRemoteAccountParams{
+		RequestingUsername: username,
+		RemoteAccountID:    publicKeyOwnerURI,
+	})
 	if err != nil {
 		return nil, false, fmt.Errorf("couldn't get requesting account %s: %s", publicKeyOwnerURI, err)
 	}

internal/federation/federatingprotocol_test.go

@@ -45,10 +45,8 @@ func (suite *FederatingProtocolTestSuite) TestPostInboxRequestBodyHook1() {
 
 	fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1)
 
-	// setup transport controller with a no-op client so we don't make external calls
+	httpClient := testrig.NewMockHTTPClient(nil, "../../testrig/media")
-	tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(func(req *http.Request) (*http.Response, error) {
+	tc := testrig.NewTestTransportController(httpClient, suite.db, fedWorker)
-		return nil, nil
-	}), suite.db, fedWorker)
 	// setup module being tested
 	federator := federation.NewFederator(suite.db, testrig.NewTestFederatingDB(suite.db, fedWorker), tc, suite.tc, testrig.NewTestMediaManager(suite.db, suite.storage))
 
@@ -78,10 +76,9 @@ func (suite *FederatingProtocolTestSuite) TestPostInboxRequestBodyHook2() {
 
 	fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1)
 
-	// setup transport controller with a no-op client so we don't make external calls
+	httpClient := testrig.NewMockHTTPClient(nil, "../../testrig/media")
-	tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(func(req *http.Request) (*http.Response, error) {
+	tc := testrig.NewTestTransportController(httpClient, suite.db, fedWorker)
-		return nil, nil
+
-	}), suite.db, fedWorker)
 	// setup module being tested
 	federator := federation.NewFederator(suite.db, testrig.NewTestFederatingDB(suite.db, fedWorker), tc, suite.tc, testrig.NewTestMediaManager(suite.db, suite.storage))
 
@@ -112,10 +109,9 @@ func (suite *FederatingProtocolTestSuite) TestPostInboxRequestBodyHook3() {
 
 	fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1)
 
-	// setup transport controller with a no-op client so we don't make external calls
+	httpClient := testrig.NewMockHTTPClient(nil, "../../testrig/media")
-	tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(func(req *http.Request) (*http.Response, error) {
+	tc := testrig.NewTestTransportController(httpClient, suite.db, fedWorker)
-		return nil, nil
+
-	}), suite.db, fedWorker)
 	// setup module being tested
 	federator := federation.NewFederator(suite.db, testrig.NewTestFederatingDB(suite.db, fedWorker), tc, suite.tc, testrig.NewTestMediaManager(suite.db, suite.storage))
 
@@ -148,7 +144,9 @@ func (suite *FederatingProtocolTestSuite) TestAuthenticatePostInbox() {
 
 	fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1)
 
-	tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker)
+	httpClient := testrig.NewMockHTTPClient(nil, "../../testrig/media")
+	tc := testrig.NewTestTransportController(httpClient, suite.db, fedWorker)
+
 	// now setup module being tested, with the mock transport controller
 	federator := federation.NewFederator(suite.db, testrig.NewTestFederatingDB(suite.db, fedWorker), tc, suite.tc, testrig.NewTestMediaManager(suite.db, suite.storage))
 
@@ -186,7 +184,8 @@ func (suite *FederatingProtocolTestSuite) TestAuthenticatePostInbox() {
 
 func (suite *FederatingProtocolTestSuite) TestBlocked1() {
 	fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1)
-	tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker)
+	httpClient := testrig.NewMockHTTPClient(nil, "../../testrig/media")
+	tc := testrig.NewTestTransportController(httpClient, suite.db, fedWorker)
 	federator := federation.NewFederator(suite.db, testrig.NewTestFederatingDB(suite.db, fedWorker), tc, suite.tc, testrig.NewTestMediaManager(suite.db, suite.storage))
 
 	sendingAccount := suite.testAccounts["remote_account_1"]
@@ -208,7 +207,8 @@ func (suite *FederatingProtocolTestSuite) TestBlocked1() {
 
 func (suite *FederatingProtocolTestSuite) TestBlocked2() {
 	fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1)
-	tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker)
+	httpClient := testrig.NewMockHTTPClient(nil, "../../testrig/media")
+	tc := testrig.NewTestTransportController(httpClient, suite.db, fedWorker)
 	federator := federation.NewFederator(suite.db, testrig.NewTestFederatingDB(suite.db, fedWorker), tc, suite.tc, testrig.NewTestMediaManager(suite.db, suite.storage))
 
 	sendingAccount := suite.testAccounts["remote_account_1"]
@@ -241,7 +241,8 @@ func (suite *FederatingProtocolTestSuite) TestBlocked2() {
 
 func (suite *FederatingProtocolTestSuite) TestBlocked3() {
 	fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1)
-	tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker)
+	httpClient := testrig.NewMockHTTPClient(nil, "../../testrig/media")
+	tc := testrig.NewTestTransportController(httpClient, suite.db, fedWorker)
 	federator := federation.NewFederator(suite.db, testrig.NewTestFederatingDB(suite.db, fedWorker), tc, suite.tc, testrig.NewTestMediaManager(suite.db, suite.storage))
 
 	sendingAccount := suite.testAccounts["remote_account_1"]
@@ -277,7 +278,8 @@ func (suite *FederatingProtocolTestSuite) TestBlocked3() {
 
 func (suite *FederatingProtocolTestSuite) TestBlocked4() {
 	fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1)
-	tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker)
+	httpClient := testrig.NewMockHTTPClient(nil, "../../testrig/media")
+	tc := testrig.NewTestTransportController(httpClient, suite.db, fedWorker)
 	federator := federation.NewFederator(suite.db, testrig.NewTestFederatingDB(suite.db, fedWorker), tc, suite.tc, testrig.NewTestMediaManager(suite.db, suite.storage))
 
 	sendingAccount := suite.testAccounts["remote_account_1"]

internal/federation/federator.go

@@ -53,14 +53,10 @@ type Federator interface {
 	// If something goes wrong during authentication, nil, false, and an error will be returned.
 	AuthenticateFederatedRequest(ctx context.Context, username string) (*url.URL, gtserror.WithCode)
 
-	// FingerRemoteAccount performs a webfinger lookup for a remote account, using the .well-known path. It will return the ActivityPub URI for that
-	// account, or an error if it doesn't exist or can't be retrieved.
-	FingerRemoteAccount(ctx context.Context, requestingUsername string, targetUsername string, targetDomain string) (*url.URL, error)
-
 	DereferenceRemoteThread(ctx context.Context, username string, statusURI *url.URL) error
 	DereferenceAnnounce(ctx context.Context, announce *gtsmodel.Status, requestingUsername string) error
 
-	GetRemoteAccount(ctx context.Context, username string, remoteAccountID *url.URL, blocking bool, refresh bool) (*gtsmodel.Account, error)
+	GetRemoteAccount(ctx context.Context, params dereferencing.GetRemoteAccountParams) (*gtsmodel.Account, error)
 
 	GetRemoteStatus(ctx context.Context, username string, remoteStatusID *url.URL, refetch, includeParent bool) (*gtsmodel.Status, ap.Statusable, error)
 	EnrichRemoteStatus(ctx context.Context, username string, status *gtsmodel.Status, includeParent bool) (*gtsmodel.Status, error)

internal/federation/finger.go

@@ -1,72 +0,0 @@
-/*
-   GoToSocial
-   Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
-
-   This program is free software: you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published by
-   the Free Software Foundation, either version 3 of the License, or
-   (at your option) any later version.
-
-   This program is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-   GNU Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-package federation
-
-import (
-	"context"
-	"encoding/json"
-	"errors"
-	"fmt"
-	"net/url"
-	"strings"
-
-	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
-)
-
-func (f *federator) FingerRemoteAccount(ctx context.Context, requestingUsername string, targetUsername string, targetDomain string) (*url.URL, error) {
-	if blocked, err := f.db.IsDomainBlocked(ctx, targetDomain); blocked || err != nil {
-		return nil, fmt.Errorf("FingerRemoteAccount: domain %s is blocked", targetDomain)
-	}
-
-	t, err := f.transportController.NewTransportForUsername(ctx, requestingUsername)
-	if err != nil {
-		return nil, fmt.Errorf("FingerRemoteAccount: error getting transport for username %s while dereferencing @%s@%s: %s", requestingUsername, targetUsername, targetDomain, err)
-	}
-
-	b, err := t.Finger(ctx, targetUsername, targetDomain)
-	if err != nil {
-		return nil, fmt.Errorf("FingerRemoteAccount: error doing request on behalf of username %s while dereferencing @%s@%s: %s", requestingUsername, targetUsername, targetDomain, err)
-	}
-
-	resp := &apimodel.WellKnownResponse{}
-	if err := json.Unmarshal(b, resp); err != nil {
-		return nil, fmt.Errorf("FingerRemoteAccount: could not unmarshal server response as WebfingerAccountResponse on behalf of username %s while dereferencing @%s@%s: %s", requestingUsername, targetUsername, targetDomain, err)
-	}
-
-	if len(resp.Links) == 0 {
-		return nil, fmt.Errorf("FingerRemoteAccount: no links found in webfinger response %s", string(b))
-	}
-
-	// look through the links for the first one that matches "application/activity+json", this is what we need
-	for _, l := range resp.Links {
-		if strings.EqualFold(l.Type, "application/activity+json") {
-			if l.Href == "" || l.Rel != "self" {
-				continue
-			}
-			accountURI, err := url.Parse(l.Href)
-			if err != nil {
-				return nil, fmt.Errorf("FingerRemoteAccount: couldn't parse url %s: %s", l.Href, err)
-			}
-			// found it!
-			return accountURI, nil
-		}
-	}
-
-	return nil, errors.New("FingerRemoteAccount: no match found in webfinger response")
-}

internal/processing/account/account_test.go

@@ -23,7 +23,6 @@ import (
 
 	"codeberg.org/gruf/go-store/kv"
 	"github.com/stretchr/testify/suite"
-	"github.com/superseriousbusiness/activity/pub"
 	"github.com/superseriousbusiness/gotosocial/internal/concurrency"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/email"
@@ -48,7 +47,6 @@ type AccountStandardTestSuite struct {
 	mediaManager        media.Manager
 	oauthServer         oauth.Server
 	fromClientAPIChan   chan messages.FromClientAPI
-	httpClient          pub.HttpClient
 	transportController transport.Controller
 	federator           federation.Federator
 	emailSender         email.Sender
@@ -97,8 +95,7 @@ func (suite *AccountStandardTestSuite) SetupTest() {
 	suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
 	suite.oauthServer = testrig.NewTestOauthServer(suite.db)
 	suite.fromClientAPIChan = make(chan messages.FromClientAPI, 100)
-	suite.httpClient = testrig.NewMockHTTPClient(nil)
+	suite.transportController = testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../testrig/media"), suite.db, fedWorker)
-	suite.transportController = testrig.NewTestTransportController(suite.httpClient, suite.db, fedWorker)
 	suite.federator = testrig.NewTestFederator(suite.db, suite.transportController, suite.storage, suite.mediaManager, fedWorker)
 	suite.sentEmails = make(map[string]string)
 	suite.emailSender = testrig.NewEmailSender("../../../web/template/", suite.sentEmails)

internal/processing/account/get.go

@@ -26,6 +26,7 @@ import (
 
 	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing"
 	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 )
@@ -80,7 +81,13 @@ func (p *processor) getAccountFor(ctx context.Context, requestingAccount *gtsmod
 			return nil, gtserror.NewErrorInternalError(fmt.Errorf("error parsing url %s: %s", targetAccount.URI, err))
 		}
 
-		a, err := p.federator.GetRemoteAccount(ctx, requestingAccount.Username, targetAccountURI, true, false)
+		a, err := p.federator.GetRemoteAccount(ctx, dereferencing.GetRemoteAccountParams{
+			RequestingUsername:    requestingAccount.Username,
+			RemoteAccountID:       targetAccountURI,
+			RemoteAccountHost:     targetAccount.Domain,
+			RemoteAccountUsername: targetAccount.Username,
+			Blocking:              true,
+		})
 		if err == nil {
 			targetAccount = a
 		}

internal/processing/account_test.go

@@ -60,7 +60,7 @@ func (suite *AccountTestSuite) TestAccountDeleteLocal() {
 	time.Sleep(1 * time.Second) // wait a sec for the delete to process
 
 	// the delete should be federated outwards to the following account's inbox
-	sent, ok := suite.sentHTTPRequests[followingAccount.InboxURI]
+	sent, ok := suite.httpClient.SentMessages[followingAccount.InboxURI]
 	suite.True(ok)
 	delete := &struct {
 		Actor  string `json:"actor"`

internal/processing/federation/getfollowers.go

@@ -24,6 +24,7 @@ import (
 	"net/url"
 
 	"github.com/superseriousbusiness/activity/streams"
+	"github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing"
 	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
 )
 
@@ -40,7 +41,10 @@ func (p *processor) GetFollowers(ctx context.Context, requestedUsername string,
 		return nil, errWithCode
 	}
 
-	requestingAccount, err := p.federator.GetRemoteAccount(ctx, requestedUsername, requestingAccountURI, false, false)
+	requestingAccount, err := p.federator.GetRemoteAccount(ctx, dereferencing.GetRemoteAccountParams{
+		RequestingUsername: requestedUsername,
+		RemoteAccountID:    requestingAccountURI,
+	})
 	if err != nil {
 		return nil, gtserror.NewErrorUnauthorized(err)
 	}

internal/processing/federation/getfollowing.go

@@ -24,6 +24,7 @@ import (
 	"net/url"
 
 	"github.com/superseriousbusiness/activity/streams"
+	"github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing"
 	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
 )
 
@@ -40,7 +41,10 @@ func (p *processor) GetFollowing(ctx context.Context, requestedUsername string,
 		return nil, errWithCode
 	}
 
-	requestingAccount, err := p.federator.GetRemoteAccount(ctx, requestedUsername, requestingAccountURI, false, false)
+	requestingAccount, err := p.federator.GetRemoteAccount(ctx, dereferencing.GetRemoteAccountParams{
+		RequestingUsername: requestedUsername,
+		RemoteAccountID:    requestingAccountURI,
+	})
 	if err != nil {
 		return nil, gtserror.NewErrorUnauthorized(err)
 	}

internal/processing/federation/getoutbox.go

@@ -25,6 +25,7 @@ import (
 
 	"github.com/superseriousbusiness/activity/streams"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing"
 	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
 )
 
@@ -41,7 +42,10 @@ func (p *processor) GetOutbox(ctx context.Context, requestedUsername string, pag
 		return nil, errWithCode
 	}
 
-	requestingAccount, err := p.federator.GetRemoteAccount(ctx, requestedUsername, requestingAccountURI, false, false)
+	requestingAccount, err := p.federator.GetRemoteAccount(ctx, dereferencing.GetRemoteAccountParams{
+		RequestingUsername: requestedUsername,
+		RemoteAccountID:    requestingAccountURI,
+	})
 	if err != nil {
 		return nil, gtserror.NewErrorUnauthorized(err)
 	}

internal/processing/federation/getstatus.go

@@ -24,6 +24,7 @@ import (
 	"net/url"
 
 	"github.com/superseriousbusiness/activity/streams"
+	"github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing"
 	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
 )
 
@@ -40,7 +41,10 @@ func (p *processor) GetStatus(ctx context.Context, requestedUsername string, req
 		return nil, errWithCode
 	}
 
-	requestingAccount, err := p.federator.GetRemoteAccount(ctx, requestedUsername, requestingAccountURI, false, false)
+	requestingAccount, err := p.federator.GetRemoteAccount(ctx, dereferencing.GetRemoteAccountParams{
+		RequestingUsername: requestedUsername,
+		RemoteAccountID:    requestingAccountURI,
+	})
 	if err != nil {
 		return nil, gtserror.NewErrorUnauthorized(err)
 	}

internal/processing/federation/getstatusreplies.go

@@ -25,6 +25,7 @@ import (
 
 	"github.com/superseriousbusiness/activity/streams"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing"
 	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 )
@@ -42,7 +43,10 @@ func (p *processor) GetStatusReplies(ctx context.Context, requestedUsername stri
 		return nil, errWithCode
 	}
 
-	requestingAccount, err := p.federator.GetRemoteAccount(ctx, requestedUsername, requestingAccountURI, false, false)
+	requestingAccount, err := p.federator.GetRemoteAccount(ctx, dereferencing.GetRemoteAccountParams{
+		RequestingUsername: requestedUsername,
+		RemoteAccountID:    requestingAccountURI,
+	})
 	if err != nil {
 		return nil, gtserror.NewErrorUnauthorized(err)
 	}

internal/processing/federation/getuser.go

@@ -25,6 +25,7 @@ import (
 
 	"github.com/superseriousbusiness/activity/streams"
 	"github.com/superseriousbusiness/activity/streams/vocab"
+	"github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing"
 	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
 	"github.com/superseriousbusiness/gotosocial/internal/uris"
 )
@@ -52,7 +53,10 @@ func (p *processor) GetUser(ctx context.Context, requestedUsername string, reque
 
 		// if we're not already handshaking/dereferencing a remote account, dereference it now
 		if !p.federator.Handshaking(ctx, requestedUsername, requestingAccountURI) {
-			requestingAccount, err := p.federator.GetRemoteAccount(ctx, requestedUsername, requestingAccountURI, false, false)
+			requestingAccount, err := p.federator.GetRemoteAccount(ctx, dereferencing.GetRemoteAccountParams{
+				RequestingUsername: requestedUsername,
+				RemoteAccountID:    requestingAccountURI,
+			})
 			if err != nil {
 				return nil, gtserror.NewErrorUnauthorized(err)
 			}

internal/processing/followrequest_test.go

@@ -57,7 +57,7 @@ func (suite *FollowRequestTestSuite) TestFollowRequestAccept() {
 	time.Sleep(1 * time.Second)
 
 	// accept should be sent to some_user
-	sent, ok := suite.sentHTTPRequests[requestingAccount.InboxURI]
+	sent, ok := suite.httpClient.SentMessages[requestingAccount.InboxURI]
 	suite.True(ok)
 
 	accept := &struct {
@@ -109,7 +109,7 @@ func (suite *FollowRequestTestSuite) TestFollowRequestReject() {
 	time.Sleep(1 * time.Second)
 
 	// reject should be sent to some_user
-	sent, ok := suite.sentHTTPRequests[requestingAccount.InboxURI]
+	sent, ok := suite.httpClient.SentMessages[requestingAccount.InboxURI]
 	suite.True(ok)
 
 	reject := &struct {

internal/processing/fromfederator.go

@@ -26,6 +26,7 @@ import (
 
 	"github.com/sirupsen/logrus"
 	"github.com/superseriousbusiness/gotosocial/internal/ap"
+	"github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/id"
 	"github.com/superseriousbusiness/gotosocial/internal/messages"
@@ -130,7 +131,11 @@ func (p *processor) processCreateStatusFromFederator(ctx context.Context, federa
 			return err
 		}
 
-		a, err := p.federator.GetRemoteAccount(ctx, federatorMsg.ReceivingAccount.Username, remoteAccountID, true, false)
+		a, err := p.federator.GetRemoteAccount(ctx, dereferencing.GetRemoteAccountParams{
+			RequestingUsername: federatorMsg.ReceivingAccount.Username,
+			RemoteAccountID:    remoteAccountID,
+			Blocking:           true,
+		})
 		if err != nil {
 			return err
 		}
@@ -172,7 +177,11 @@ func (p *processor) processCreateFaveFromFederator(ctx context.Context, federato
 			return err
 		}
 
-		a, err := p.federator.GetRemoteAccount(ctx, federatorMsg.ReceivingAccount.Username, remoteAccountID, true, false)
+		a, err := p.federator.GetRemoteAccount(ctx, dereferencing.GetRemoteAccountParams{
+			RequestingUsername: federatorMsg.ReceivingAccount.Username,
+			RemoteAccountID:    remoteAccountID,
+			Blocking:           true,
+		})
 		if err != nil {
 			return err
 		}
@@ -210,7 +219,11 @@ func (p *processor) processCreateFollowRequestFromFederator(ctx context.Context,
 			return err
 		}
 
-		a, err := p.federator.GetRemoteAccount(ctx, federatorMsg.ReceivingAccount.Username, remoteAccountID, true, false)
+		a, err := p.federator.GetRemoteAccount(ctx, dereferencing.GetRemoteAccountParams{
+			RequestingUsername: federatorMsg.ReceivingAccount.Username,
+			RemoteAccountID:    remoteAccountID,
+			Blocking:           true,
+		})
 		if err != nil {
 			return err
 		}
@@ -267,7 +280,11 @@ func (p *processor) processCreateAnnounceFromFederator(ctx context.Context, fede
 			return err
 		}
 
-		a, err := p.federator.GetRemoteAccount(ctx, federatorMsg.ReceivingAccount.Username, remoteAccountID, true, false)
+		a, err := p.federator.GetRemoteAccount(ctx, dereferencing.GetRemoteAccountParams{
+			RequestingUsername: federatorMsg.ReceivingAccount.Username,
+			RemoteAccountID:    remoteAccountID,
+			Blocking:           true,
+		})
 		if err != nil {
 			return err
 		}
@@ -332,7 +349,11 @@ func (p *processor) processUpdateAccountFromFederator(ctx context.Context, feder
 		return err
 	}
 
-	if _, err := p.federator.GetRemoteAccount(ctx, federatorMsg.ReceivingAccount.Username, incomingAccountURL, false, true); err != nil {
+	if _, err := p.federator.GetRemoteAccount(ctx, dereferencing.GetRemoteAccountParams{
+		RequestingUsername: federatorMsg.ReceivingAccount.Username,
+		RemoteAccountID:    incomingAccountURL,
+		Blocking:           true,
+	}); err != nil {
 		return fmt.Errorf("error enriching updated account from federator: %s", err)
 	}
 

internal/processing/fromfederator_test.go

@@ -423,7 +423,7 @@ func (suite *FromFederatorTestSuite) TestProcessFollowRequestLocked() {
 	suite.Equal(originAccount.ID, notif.Account.ID)
 
 	// no messages should have been sent out, since we didn't need to federate an accept
-	suite.Empty(suite.sentHTTPRequests)
+	suite.Empty(suite.httpClient.SentMessages)
 }
 
 func (suite *FromFederatorTestSuite) TestProcessFollowRequestUnlocked() {
@@ -474,8 +474,8 @@ func (suite *FromFederatorTestSuite) TestProcessFollowRequestUnlocked() {
 	suite.Equal(originAccount.ID, notif.Account.ID)
 
 	// an accept message should be sent to satan's inbox
-	suite.Len(suite.sentHTTPRequests, 1)
+	suite.Len(suite.httpClient.SentMessages, 1)
-	acceptBytes := suite.sentHTTPRequests[originAccount.InboxURI]
+	acceptBytes := suite.httpClient.SentMessages[originAccount.InboxURI]
 	accept := &struct {
 		Actor  string `json:"actor"`
 		ID     string `json:"id"`

internal/processing/media/media_test.go

@@ -19,12 +19,7 @@
 package media_test
 
 import (
-	"bytes"
-	"io"
-	"net/http"
-
 	"codeberg.org/gruf/go-store/kv"
-	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/suite"
 	"github.com/superseriousbusiness/gotosocial/internal/concurrency"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
@@ -79,7 +74,7 @@ func (suite *MediaStandardTestSuite) SetupTest() {
 	suite.tc = testrig.NewTestTypeConverter(suite.db)
 	suite.storage = testrig.NewTestStorage()
 	suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
-	suite.transportController = suite.mockTransportController()
+	suite.transportController = testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../testrig/media"), suite.db, concurrency.NewWorkerPool[messages.FromFederator](-1, -1))
 	suite.mediaProcessor = mediaprocessing.New(suite.db, suite.tc, suite.mediaManager, suite.transportController, suite.storage)
 	testrig.StandardDBSetup(suite.db, nil)
 	testrig.StandardStorageSetup(suite.storage, "../../../testrig/media")
@@ -89,40 +84,3 @@ func (suite *MediaStandardTestSuite) TearDownTest() {
 	testrig.StandardDBTeardown(suite.db)
 	testrig.StandardStorageTeardown(suite.storage)
 }
-
-func (suite *MediaStandardTestSuite) mockTransportController() transport.Controller {
-	do := func(req *http.Request) (*http.Response, error) {
-		logrus.Debugf("received request for %s", req.URL)
-
-		responseBytes := []byte{}
-		responseType := ""
-		responseLength := 0
-
-		if attachment, ok := suite.testRemoteAttachments[req.URL.String()]; ok {
-			responseBytes = attachment.Data
-			responseType = attachment.ContentType
-		}
-
-		if len(responseBytes) != 0 {
-			// we found something, so print what we're going to return
-			logrus.Debugf("returning response %s", string(responseBytes))
-		}
-		responseLength = len(responseBytes)
-
-		reader := bytes.NewReader(responseBytes)
-		readCloser := io.NopCloser(reader)
-		response := &http.Response{
-			StatusCode:    200,
-			Body:          readCloser,
-			ContentLength: int64(responseLength),
-			Header: http.Header{
-				"content-type": {responseType},
-			},
-		}
-
-		return response, nil
-	}
-	fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1)
-	mockClient := testrig.NewMockHTTPClient(do)
-	return testrig.NewTestTransportController(mockClient, suite.db, fedWorker)
-}

internal/processing/processor_test.go

@@ -19,16 +19,8 @@
 package processing_test
 
 import (
-	"bytes"
-	"context"
-	"encoding/json"
-	"io"
-	"io/ioutil"
-	"net/http"
-
 	"codeberg.org/gruf/go-store/kv"
 	"github.com/stretchr/testify/suite"
-	"github.com/superseriousbusiness/activity/streams"
 	"github.com/superseriousbusiness/gotosocial/internal/concurrency"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/email"
@@ -50,6 +42,7 @@ type ProcessingStandardTestSuite struct {
 	storage             *kv.KVStore
 	mediaManager        media.Manager
 	typeconverter       typeutils.TypeConverter
+	httpClient          *testrig.MockHTTPClient
 	transportController transport.Controller
 	federator           federation.Federator
 	oauthServer         oauth.Server
@@ -69,8 +62,6 @@ type ProcessingStandardTestSuite struct {
 	testBlocks       map[string]*gtsmodel.Block
 	testActivities   map[string]testrig.ActivityWithSignature
 
-	sentHTTPRequests map[string][]byte
-
 	processor processing.Processor
 }
 
@@ -102,123 +93,12 @@ func (suite *ProcessingStandardTestSuite) SetupTest() {
 	suite.testActivities = testrig.NewTestActivities(suite.testAccounts)
 	suite.storage = testrig.NewTestStorage()
 	suite.typeconverter = testrig.NewTestTypeConverter(suite.db)
-
+	suite.httpClient = testrig.NewMockHTTPClient(nil, "../../testrig/media")
-	// make an http client that stores POST requests it receives into a map,
-	// and also responds to correctly to dereference requests
-	suite.sentHTTPRequests = make(map[string][]byte)
-	httpClient := testrig.NewMockHTTPClient(func(req *http.Request) (*http.Response, error) {
-		if req.Method == http.MethodPost && req.Body != nil {
-			requestBytes, err := ioutil.ReadAll(req.Body)
-			if err != nil {
-				panic(err)
-			}
-			if err := req.Body.Close(); err != nil {
-				panic(err)
-			}
-			suite.sentHTTPRequests[req.URL.String()] = requestBytes
-		}
-
-		if req.URL.String() == suite.testAccounts["remote_account_1"].URI {
-			// the request is for remote account 1
-			satan := suite.testAccounts["remote_account_1"]
-
-			satanAS, err := suite.typeconverter.AccountToAS(context.Background(), satan)
-			if err != nil {
-				panic(err)
-			}
-
-			satanI, err := streams.Serialize(satanAS)
-			if err != nil {
-				panic(err)
-			}
-			satanJson, err := json.Marshal(satanI)
-			if err != nil {
-				panic(err)
-			}
-			responseType := "application/activity+json"
-
-			reader := bytes.NewReader(satanJson)
-			readCloser := io.NopCloser(reader)
-			response := &http.Response{
-				StatusCode:    200,
-				Body:          readCloser,
-				ContentLength: int64(len(satanJson)),
-				Header: http.Header{
-					"content-type": {responseType},
-				},
-			}
-			return response, nil
-		}
-
-		if req.URL.String() == suite.testAccounts["remote_account_2"].URI {
-			// the request is for remote account 2
-			someAccount := suite.testAccounts["remote_account_2"]
-
-			someAccountAS, err := suite.typeconverter.AccountToAS(context.Background(), someAccount)
-			if err != nil {
-				panic(err)
-			}
-
-			someAccountI, err := streams.Serialize(someAccountAS)
-			if err != nil {
-				panic(err)
-			}
-			someAccountJson, err := json.Marshal(someAccountI)
-			if err != nil {
-				panic(err)
-			}
-			responseType := "application/activity+json"
-
-			reader := bytes.NewReader(someAccountJson)
-			readCloser := io.NopCloser(reader)
-			response := &http.Response{
-				StatusCode:    200,
-				Body:          readCloser,
-				ContentLength: int64(len(someAccountJson)),
-				Header: http.Header{
-					"content-type": {responseType},
-				},
-			}
-			return response, nil
-		}
-
-		if req.URL.String() == "http://example.org/users/some_user/statuses/afaba698-5740-4e32-a702-af61aa543bc1" {
-			// the request is for the forwarded message
-			message := suite.testActivities["forwarded_message"].Activity.GetActivityStreamsObject().At(0).GetActivityStreamsNote()
-			messageI, err := streams.Serialize(message)
-			if err != nil {
-				panic(err)
-			}
-			messageJson, err := json.Marshal(messageI)
-			if err != nil {
-				panic(err)
-			}
-			responseType := "application/activity+json"
-
-			reader := bytes.NewReader(messageJson)
-			readCloser := io.NopCloser(reader)
-			response := &http.Response{
-				StatusCode:    200,
-				Body:          readCloser,
-				ContentLength: int64(len(messageJson)),
-				Header: http.Header{
-					"content-type": {responseType},
-				},
-			}
-			return response, nil
-		}
-
-		r := ioutil.NopCloser(bytes.NewReader([]byte{}))
-		return &http.Response{
-			StatusCode: 200,
-			Body:       r,
-		}, nil
-	})
 
 	clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1)
 	fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1)
 
-	suite.transportController = testrig.NewTestTransportController(httpClient, suite.db, fedWorker)
+	suite.transportController = testrig.NewTestTransportController(suite.httpClient, suite.db, fedWorker)
 	suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
 	suite.federator = testrig.NewTestFederator(suite.db, suite.transportController, suite.storage, suite.mediaManager, fedWorker)
 	suite.oauthServer = testrig.NewTestOauthServer(suite.db)

internal/processing/search.go

@@ -20,6 +20,7 @@ package processing
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"net/url"
 	"strings"
@@ -28,53 +29,70 @@ import (
 	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
 	"github.com/superseriousbusiness/gotosocial/internal/config"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing"
 	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/oauth"
 	"github.com/superseriousbusiness/gotosocial/internal/util"
 )
 
-func (p *processor) SearchGet(ctx context.Context, authed *oauth.Auth, searchQuery *apimodel.SearchQuery) (*apimodel.SearchResult, gtserror.WithCode) {
+func (p *processor) SearchGet(ctx context.Context, authed *oauth.Auth, search *apimodel.SearchQuery) (*apimodel.SearchResult, gtserror.WithCode) {
 	l := logrus.WithFields(logrus.Fields{
 		"func":  "SearchGet",
-		"query": searchQuery.Query,
+		"query": search.Query,
 	})
 
-	results := &apimodel.SearchResult{
+	// tidy up the query and make sure it wasn't just spaces
+	query := strings.TrimSpace(search.Query)
+	if query == "" {
+		err := errors.New("search query was empty string after trimming space")
+		return nil, gtserror.NewErrorBadRequest(err, err.Error())
+	}
+
+	searchResult := &apimodel.SearchResult{
 		Accounts: []apimodel.Account{},
 		Statuses: []apimodel.Status{},
 		Hashtags: []apimodel.Tag{},
 	}
 	foundAccounts := []*gtsmodel.Account{}
 	foundStatuses := []*gtsmodel.Status{}
-	// foundHashtags := []*gtsmodel.Tag{}
-
-	// convert the query to lowercase and trim leading/trailing spaces
-	query := strings.ToLower(strings.TrimSpace(searchQuery.Query))
 
 	var foundOne bool
-	// check if the query is something like @whatever_username@example.org -- this means it's a remote account
+
-	if _, domain, err := util.ExtractMentionParts(searchQuery.Query); err == nil && domain != "" {
+	/*
-		l.Debug("search term is a mention, looking it up...")
+		SEARCH BY MENTION
-		foundAccount, err := p.searchAccountByMention(ctx, authed, searchQuery.Query, searchQuery.Resolve)
+		check if the query is something like @whatever_username@example.org -- this means it's a remote account
-		if err == nil && foundAccount != nil {
+	*/
+	maybeNamestring := query
+	if maybeNamestring[0] != '@' {
+		maybeNamestring = "@" + maybeNamestring
+	}
+
+	if username, domain, err := util.ExtractNamestringParts(maybeNamestring); err == nil {
+		l.Debugf("search term %s is a mention, looking it up...", maybeNamestring)
+		if foundAccount, err := p.searchAccountByMention(ctx, authed, username, domain, search.Resolve); err == nil && foundAccount != nil {
 			foundAccounts = append(foundAccounts, foundAccount)
 			foundOne = true
 			l.Debug("got an account by searching by mention")
+		} else if err != nil {
+			l.Debugf("error looking up account %s: %s", maybeNamestring, err)
 		}
 	}
 
-	// check if the query is a URI with a recognizable scheme and just do a lookup for that, straight up
+	/*
+		SEARCH BY URI
+		check if the query is a URI with a recognizable scheme and dereference it
+	*/
 	if !foundOne {
 		if uri, err := url.Parse(query); err == nil && (uri.Scheme == "https" || uri.Scheme == "http") {
 			// 1. check if it's a status
-			if foundStatus, err := p.searchStatusByURI(ctx, authed, uri, searchQuery.Resolve); err == nil && foundStatus != nil {
+			if foundStatus, err := p.searchStatusByURI(ctx, authed, uri, search.Resolve); err == nil && foundStatus != nil {
 				foundStatuses = append(foundStatuses, foundStatus)
 				l.Debug("got a status by searching by URI")
 			}
 
 			// 2. check if it's an account
-			if foundAccount, err := p.searchAccountByURI(ctx, authed, uri, searchQuery.Resolve); err == nil && foundAccount != nil {
+			if foundAccount, err := p.searchAccountByURI(ctx, authed, uri, search.Resolve); err == nil && foundAccount != nil {
 				foundAccounts = append(foundAccounts, foundAccount)
 				l.Debug("got an account by searching by URI")
 			}
@@ -90,7 +108,7 @@ func (p *processor) SearchGet(ctx context.Context, authed *oauth.Auth, searchQue
 		if blocked, err := p.db.IsBlocked(ctx, authed.Account.ID, foundAccount.ID, true); err == nil && !blocked {
 			// all good, convert it and add it to the results
 			if apiAcct, err := p.tc.AccountToAPIAccountPublic(ctx, foundAccount); err == nil && apiAcct != nil {
-				results.Accounts = append(results.Accounts, *apiAcct)
+				searchResult.Accounts = append(searchResult.Accounts, *apiAcct)
 			}
 		}
 	}
@@ -105,10 +123,10 @@ func (p *processor) SearchGet(ctx context.Context, authed *oauth.Auth, searchQue
 			continue
 		}
 
-		results.Statuses = append(results.Statuses, *apiStatus)
+		searchResult.Statuses = append(searchResult.Statuses, *apiStatus)
 	}
 
-	return results, nil
+	return searchResult, nil
 }
 
 func (p *processor) searchStatusByURI(ctx context.Context, authed *oauth.Auth, uri *url.URL, resolve bool) (*gtsmodel.Status, error) {
@@ -147,7 +165,10 @@ func (p *processor) searchAccountByURI(ctx context.Context, authed *oauth.Auth,
 
 	if resolve {
 		// we don't have it locally so try and dereference it
-		account, err := p.federator.GetRemoteAccount(ctx, authed.Account.Username, uri, true, true)
+		account, err := p.federator.GetRemoteAccount(ctx, dereferencing.GetRemoteAccountParams{
+			RequestingUsername: authed.Account.Username,
+			RemoteAccountID:    uri,
+		})
 		if err != nil {
 			return nil, fmt.Errorf("searchAccountByURI: error dereferencing account with uri %s: %s", uri.String(), err)
 		}
@@ -156,18 +177,14 @@ func (p *processor) searchAccountByURI(ctx context.Context, authed *oauth.Auth,
 	return nil, nil
 }
 
-func (p *processor) searchAccountByMention(ctx context.Context, authed *oauth.Auth, mention string, resolve bool) (*gtsmodel.Account, error) {
+func (p *processor) searchAccountByMention(ctx context.Context, authed *oauth.Auth, username string, domain string, resolve bool) (*gtsmodel.Account, error) {
-	// query is for a remote account
+	maybeAcct := &gtsmodel.Account{}
-	username, domain, err := util.ExtractMentionParts(mention)
+	var err error
-	if err != nil {
-		return nil, fmt.Errorf("searchAccountByMention: error extracting mention parts: %s", err)
-	}
 
 	// if it's a local account we can skip a whole bunch of stuff
-	maybeAcct := &gtsmodel.Account{}
+	if domain == config.GetHost() || domain == config.GetAccountDomain() || domain == "" {
-	if domain == config.GetHost() {
 		maybeAcct, err = p.db.GetLocalAccountByUsername(ctx, username)
-		if err != nil {
+		if err != nil && err != db.ErrNoEntries {
 			return nil, fmt.Errorf("searchAccountByMention: error getting local account by username: %s", err)
 		}
 		return maybeAcct, nil
@@ -191,22 +208,15 @@ func (p *processor) searchAccountByMention(ctx context.Context, authed *oauth.Au
 
 	// we got a db.ErrNoEntries, so we just don't have the account locally stored -- check if we can dereference it
 	if resolve {
-		// we're allowed to resolve it so let's try
+		maybeAcct, err = p.federator.GetRemoteAccount(ctx, dereferencing.GetRemoteAccountParams{
-		// first we need to webfinger the remote account to convert the username and domain into the activitypub URI for the account
+			RequestingUsername:    authed.Account.Username,
-		acctURI, err := p.federator.FingerRemoteAccount(ctx, authed.Account.Username, username, domain)
+			RemoteAccountUsername: username,
+			RemoteAccountHost:     domain,
+		})
 		if err != nil {
-			// something went wrong doing the webfinger lookup so we can't process the request
+			return nil, fmt.Errorf("searchAccountByMention: error getting remote account: %s", err)
-			return nil, fmt.Errorf("error fingering remote account with username %s and domain %s: %s", username, domain, err)
-		}
-
-		if acctURI.Scheme == "https" || acctURI.Scheme == "http" {
-			acct, err := p.federator.GetRemoteAccount(ctx, authed.Account.Username, acctURI, true, true)
-			if err != nil {
-				logrus.Debugf("could not get remote account by mention %s with uri %s: %s", mention, acctURI, err)
-				return nil, err
-			}
-			return acct, nil
 		}
+		return maybeAcct, nil
 	}
 
 	return nil, nil

internal/processing/status/status_test.go

@@ -80,7 +80,7 @@ func (suite *StatusStandardTestSuite) SetupTest() {
 	suite.db = testrig.NewTestDB()
 	suite.typeConverter = testrig.NewTestTypeConverter(suite.db)
 	suite.clientWorker = concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1)
-	suite.tc = testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker)
+	suite.tc = testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../testrig/media"), suite.db, fedWorker)
 	suite.storage = testrig.NewTestStorage()
 	suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
 	suite.federator = testrig.NewTestFederator(suite.db, suite.tc, suite.storage, suite.mediaManager, fedWorker)

internal/processing/util.go

@@ -21,12 +21,14 @@ package processing
 import (
 	"context"
 	"fmt"
-	"strings"
 
+	"github.com/superseriousbusiness/gotosocial/internal/config"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
+	"github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/id"
+	"github.com/superseriousbusiness/gotosocial/internal/util"
 )
 
 func GetParseMentionFunc(dbConn db.DB, federator federation.Federator) gtsmodel.ParseMentionFunc {
@@ -37,98 +39,35 @@ func GetParseMentionFunc(dbConn db.DB, federator federation.Federator) gtsmodel.
 			return nil, fmt.Errorf("couldn't get mention origin account with id %s", originAccountID)
 		}
 
-		// A mentioned account looks like "@test@example.org" or just "@test" for a local account
+		username, domain, err := util.ExtractNamestringParts(targetAccount)
-		// -- we can guarantee this from the regex that targetAccounts should have been derived from.
+		if err != nil {
-		// But we still need to do a bit of fiddling to get what we need here -- the username and domain (if given).
+			return nil, fmt.Errorf("couldn't extract namestring parts from %s: %s", targetAccount, err)
-
-		// 1.  trim off the first @
-		trimmed := strings.TrimPrefix(targetAccount, "@")
-
-		// 2. split the username and domain
-		split := strings.Split(trimmed, "@")
-
-		// 3. if it's length 1 it's a local account, length 2 means remote, anything else means something is wrong
-
-		var local bool
-		switch len(split) {
-		case 1:
-			local = true
-		case 2:
-			local = false
-		default:
-			return nil, fmt.Errorf("mentioned account format '%s' was not valid", targetAccount)
-		}
-
-		var username, domain string
-		username = split[0]
-		if !local {
-			domain = split[1]
-		}
-
-		// 4. check we now have a proper username and domain
-		if username == "" || (!local && domain == "") {
-			return nil, fmt.Errorf("username or domain for '%s' was nil", targetAccount)
 		}
 
 		var mentionedAccount *gtsmodel.Account
-
+		if domain == "" || domain == config.GetHost() || domain == config.GetAccountDomain() {
-		if local {
 			localAccount, err := dbConn.GetLocalAccountByUsername(ctx, username)
 			if err != nil {
 				return nil, err
 			}
 			mentionedAccount = localAccount
 		} else {
-			remoteAccount := &gtsmodel.Account{}
+			var requestingUsername string
-
-			where := []db.Where{
-				{
-					Key:             "username",
-					Value:           username,
-					CaseInsensitive: true,
-				},
-				{
-					Key:             "domain",
-					Value:           domain,
-					CaseInsensitive: true,
-				},
-			}
-
-			err := dbConn.GetWhere(ctx, where, remoteAccount)
-			if err == nil {
-				// the account was already in the database
-				mentionedAccount = remoteAccount
-			} else {
-				// we couldn't get it from the database
-				if err != db.ErrNoEntries {
-					// a serious error has happened so bail
-					return nil, fmt.Errorf("error getting account with username '%s' and domain '%s': %s", username, domain, err)
-				}
-
-				// We just don't have the account, so try webfingering it.
-				//
-				// If the mention originates from our instance we should use the username of the origin account to do the dereferencing,
-				// otherwise we should just use our instance account (that is, provide an empty string), since obviously we can't use
-				// a remote account to do remote dereferencing!
-				var fingeringUsername string
 			if originAccount.Domain == "" {
-					fingeringUsername = originAccount.Username
+				requestingUsername = originAccount.Username
 			}
-
+			remoteAccount, err := federator.GetRemoteAccount(ctx, dereferencing.GetRemoteAccountParams{
-				acctURI, err := federator.FingerRemoteAccount(ctx, fingeringUsername, username, domain)
+				RequestingUsername:    requestingUsername,
+				RemoteAccountUsername: username,
+				RemoteAccountHost:     domain,
+			})
 			if err != nil {
-					// something went wrong doing the webfinger lookup so we can't process the request
+				return nil, fmt.Errorf("error dereferencing account: %s", err)
-					return nil, fmt.Errorf("error fingering remote account with username %s and domain %s: %s", username, domain, err)
-				}
-
-				resolvedAccount, err := federator.GetRemoteAccount(ctx, fingeringUsername, acctURI, true, true)
-				if err != nil {
-					return nil, fmt.Errorf("error dereferencing account with uri %s: %s", acctURI.String(), err)
 			}
 
 			// we were able to resolve it!
-				mentionedAccount = resolvedAccount
+			mentionedAccount = remoteAccount
-			}
+
 		}
 
 		mentionID, err := id.NewRandomULID()

internal/regexes/regexes.go

@@ -61,7 +61,7 @@ var (
 		return rgx
 	}()
 
-	mentionName = `^@(\w+)(?:@([a-zA-Z0-9_\-\.:]+))?$`
+	mentionName = `^@([\w\-\.]+)(?:@([\w\-\.:]+))?$`
 	// MentionName captures the username and domain part from a mention string
 	// such as @whatever_user@example.org, returning whatever_user and example.org (without the @ symbols)
 	MentionName = regexp.MustCompile(mentionName)

internal/transport/derefinstance.go

@@ -206,7 +206,7 @@ func dereferenceByNodeInfo(c context.Context, t *transport, iri *url.URL) (*gtsm
 			// see if there's a 'name' in the map
 			if name, present := v["name"]; present {
 				// name could be just a username, or could be a mention string eg @whatever@aaaa.com
-				username, _, err := util.ExtractMentionParts(name)
+				username, _, err := util.ExtractNamestringParts(name)
 				if err == nil {
 					// it was a mention string
 					contactAccountUsername = username

internal/typeutils/astointernal.go

@@ -30,7 +30,7 @@ import (
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 )
 
-func (c *converter) ASRepresentationToAccount(ctx context.Context, accountable ap.Accountable, update bool) (*gtsmodel.Account, error) {
+func (c *converter) ASRepresentationToAccount(ctx context.Context, accountable ap.Accountable, accountDomain string, update bool) (*gtsmodel.Account, error) {
 	// first check if we actually already know this account
 	uriProp := accountable.GetJSONLDId()
 	if uriProp == nil || !uriProp.IsIRI() {
@@ -63,7 +63,11 @@ func (c *converter) ASRepresentationToAccount(ctx context.Context, accountable a
 	acct.Username = username
 
 	// Domain
+	if accountDomain != "" {
+		acct.Domain = accountDomain
+	} else {
 		acct.Domain = uri.Host
+	}
 
 	// avatar aka icon
 	// if this one isn't extractable in a format we recognise we'll just skip it

internal/typeutils/astointernal_test.go

@@ -38,7 +38,7 @@ type ASToInternalTestSuite struct {
 func (suite *ASToInternalTestSuite) TestParsePerson() {
 	testPerson := suite.testPeople["https://unknown-instance.com/users/brand_new_person"]
 
-	acct, err := suite.typeconverter.ASRepresentationToAccount(context.Background(), testPerson, false)
+	acct, err := suite.typeconverter.ASRepresentationToAccount(context.Background(), testPerson, "", false)
 	suite.NoError(err)
 
 	suite.Equal("https://unknown-instance.com/users/brand_new_person", acct.URI)
@@ -106,7 +106,7 @@ func (suite *ASToInternalTestSuite) TestParseGargron() {
 	rep, ok := t.(ap.Accountable)
 	suite.True(ok)
 
-	acct, err := suite.typeconverter.ASRepresentationToAccount(context.Background(), rep, false)
+	acct, err := suite.typeconverter.ASRepresentationToAccount(context.Background(), rep, "", false)
 	suite.NoError(err)
 
 	fmt.Printf("%+v", acct)
@@ -168,7 +168,7 @@ func (suite *ASToInternalTestSuite) TestParseOwncastService() {
 	rep, ok := t.(ap.Accountable)
 	suite.True(ok)
 
-	acct, err := suite.typeconverter.ASRepresentationToAccount(context.Background(), rep, false)
+	acct, err := suite.typeconverter.ASRepresentationToAccount(context.Background(), rep, "", false)
 	suite.NoError(err)
 
 	suite.Equal("rgh", acct.Username)

internal/typeutils/converter.go

@@ -99,7 +99,10 @@ type TypeConverter interface {
 	// If update is false, and the account is already known in the database, then the existing account entry will be returned.
 	// If update is true, then even if the account is already known, all fields in the accountable will be parsed and a new *gtsmodel.Account
 	// will be generated. This is useful when one needs to force refresh of an account, eg., during an Update of a Profile.
-	ASRepresentationToAccount(ctx context.Context, accountable ap.Accountable, update bool) (*gtsmodel.Account, error)
+	//
+	// If accountDomain is set (not an empty string) then this value will be used as the account's Domain. If not set,
+	// then the Host of the accountable's AP ID will be used instead.
+	ASRepresentationToAccount(ctx context.Context, accountable ap.Accountable, accountDomain string, update bool) (*gtsmodel.Account, error)
 	// ASStatus converts a remote activitystreams 'status' representation into a gts model status.
 	ASStatusToStatus(ctx context.Context, statusable ap.Statusable) (*gtsmodel.Status, error)
 	// ASFollowToFollowRequest converts a remote activitystreams `follow` representation into gts model follow request.

internal/util/namestring.go

@@ -0,0 +1,59 @@
+/*
+   GoToSocial
+   Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package util
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/superseriousbusiness/gotosocial/internal/regexes"
+)
+
+// ExtractNamestringParts extracts the username test_user and
+// the domain example.org from a string like @test_user@example.org.
+//
+// If nothing is matched, it will return an error.
+func ExtractNamestringParts(mention string) (username, host string, err error) {
+	matches := regexes.MentionName.FindStringSubmatch(mention)
+	switch len(matches) {
+	case 2:
+		return matches[1], "", nil
+	case 3:
+		return matches[1], matches[2], nil
+	default:
+		return "", "", fmt.Errorf("couldn't match mention %s", mention)
+	}
+}
+
+// ExtractWebfingerParts returns username test_user and
+// domain example.org from a string like acct:test_user@example.org,
+// or acct:@test_user@example.org.
+//
+// If nothing is extracted, it will return an error.
+func ExtractWebfingerParts(webfinger string) (username, host string, err error) {
+	// remove the acct: prefix if it's present
+	webfinger = strings.TrimPrefix(webfinger, "acct:")
+
+	// prepend an @ if necessary
+	if webfinger[0] != '@' {
+		webfinger = "@" + webfinger
+	}
+
+	return ExtractNamestringParts(webfinger)
+}

internal/util/namestring_test.go

@@ -0,0 +1,118 @@
+/*
+   GoToSocial
+   Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package util_test
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/util"
+)
+
+type NamestringSuite struct {
+	suite.Suite
+}
+
+func (suite *NamestringSuite) TestExtractWebfingerParts1() {
+	webfinger := "acct:stonerkitty.monster@stonerkitty.monster"
+	username, host, err := util.ExtractWebfingerParts(webfinger)
+	suite.NoError(err)
+
+	suite.Equal("stonerkitty.monster", username)
+	suite.Equal("stonerkitty.monster", host)
+}
+
+func (suite *NamestringSuite) TestExtractWebfingerParts2() {
+	webfinger := "@stonerkitty.monster@stonerkitty.monster"
+	username, host, err := util.ExtractWebfingerParts(webfinger)
+	suite.NoError(err)
+
+	suite.Equal("stonerkitty.monster", username)
+	suite.Equal("stonerkitty.monster", host)
+}
+
+func (suite *NamestringSuite) TestExtractWebfingerParts3() {
+	webfinger := "acct:someone@somewhere"
+	username, host, err := util.ExtractWebfingerParts(webfinger)
+	suite.NoError(err)
+
+	suite.Equal("someone", username)
+	suite.Equal("somewhere", host)
+}
+
+func (suite *NamestringSuite) TestExtractWebfingerParts4() {
+	webfinger := "@stoner-kitty.monster@stonerkitty.monster"
+	username, host, err := util.ExtractWebfingerParts(webfinger)
+	suite.NoError(err)
+
+	suite.Equal("stoner-kitty.monster", username)
+	suite.Equal("stonerkitty.monster", host)
+}
+
+func (suite *NamestringSuite) TestExtractWebfingerParts5() {
+	webfinger := "@stonerkitty.monster"
+	username, host, err := util.ExtractWebfingerParts(webfinger)
+	suite.NoError(err)
+
+	suite.Equal("stonerkitty.monster", username)
+	suite.Empty(host)
+}
+
+func (suite *NamestringSuite) TestExtractWebfingerParts6() {
+	webfinger := "@@stonerkitty.monster"
+	_, _, err := util.ExtractWebfingerParts(webfinger)
+	suite.EqualError(err, "couldn't match mention @@stonerkitty.monster")
+}
+
+func (suite *NamestringSuite) TestExtractNamestringParts1() {
+	namestring := "@stonerkitty.monster@stonerkitty.monster"
+	username, host, err := util.ExtractNamestringParts(namestring)
+	suite.NoError(err)
+
+	suite.Equal("stonerkitty.monster", username)
+	suite.Equal("stonerkitty.monster", host)
+}
+
+func (suite *NamestringSuite) TestExtractNamestringParts2() {
+	namestring := "@stonerkitty.monster"
+	username, host, err := util.ExtractNamestringParts(namestring)
+	suite.NoError(err)
+
+	suite.Equal("stonerkitty.monster", username)
+	suite.Empty(host)
+}
+
+func (suite *NamestringSuite) TestExtractNamestringParts3() {
+	namestring := "@someone@somewhere"
+	username, host, err := util.ExtractWebfingerParts(namestring)
+	suite.NoError(err)
+
+	suite.Equal("someone", username)
+	suite.Equal("somewhere", host)
+}
+
+func (suite *NamestringSuite) TestExtractNamestringParts4() {
+	namestring := ""
+	_, _, err := util.ExtractNamestringParts(namestring)
+	suite.EqualError(err, "couldn't match mention ")
+}
+
+func TestNamestringSuite(t *testing.T) {
+	suite.Run(t, &NamestringSuite{})
+}

internal/util/statustools.go

@@ -19,7 +19,6 @@
 package util
 
 import (
-	"fmt"
 	"strings"
 
 	"github.com/superseriousbusiness/gotosocial/internal/regexes"
@@ -59,19 +58,3 @@ func DeriveEmojisFromText(text string) []string {
 	}
 	return UniqueStrings(emojis)
 }
-
-// ExtractMentionParts extracts the username test_user and the domain example.org
-// from a mention string like @test_user@example.org.
-//
-// If nothing is matched, it will return an error.
-func ExtractMentionParts(mention string) (username, domain string, err error) {
-	matches := regexes.MentionName.FindStringSubmatch(mention)
-	switch len(matches) {
-	case 2:
-		return matches[1], "", nil
-	case 3:
-		return matches[1], matches[2], nil
-	default:
-		return "", "", fmt.Errorf("couldn't match mention %s", mention)
-	}
-}

testrig/testmodels.go

@@ -1909,6 +1909,25 @@ func NewTestFediAttachments(relativePath string) map[string]RemoteAttachmentFile
 
 func NewTestFediStatuses() map[string]vocab.ActivityStreamsNote {
 	return map[string]vocab.ActivityStreamsNote{
+		"http://example.org/users/some_user/statuses/afaba698-5740-4e32-a702-af61aa543bc1": NewAPNote(
+			URLMustParse("http://example.org/users/some_user/statuses/afaba698-5740-4e32-a702-af61aa543bc1"),
+			URLMustParse("http://example.org/@some_user/afaba698-5740-4e32-a702-af61aa543bc1"),
+			time.Now(),
+			"this is a public status, please forward it!",
+			"",
+			URLMustParse("http://example.org/users/some_user"),
+			[]*url.URL{URLMustParse(pub.PublicActivityPubIRI)},
+			nil,
+			false,
+			[]vocab.ActivityStreamsMention{},
+			[]vocab.ActivityStreamsImage{
+				newAPImage(
+					URLMustParse("http://example.org/users/some_user/statuses/afaba698-5740-4e32-a702-af61aa543bc1/attachment1.jpeg"),
+					"image/jpeg",
+					"trent reznor looking handsome as balls",
+					"LEDara58O=t5EMSOENEN9]}?aK%0"),
+			},
+		),
 		"https://unknown-instance.com/users/brand_new_person/statuses/01FE4NTHKWW7THT67EF10EB839": NewAPNote(
 			URLMustParse("https://unknown-instance.com/users/brand_new_person/statuses/01FE4NTHKWW7THT67EF10EB839"),
 			URLMustParse("https://unknown-instance.com/users/@brand_new_person/01FE4NTHKWW7THT67EF10EB839"),

testrig/transportcontroller.go

@@ -20,10 +20,16 @@ package testrig
 
 import (
 	"bytes"
-	"io/ioutil"
+	"encoding/json"
+	"io"
 	"net/http"
+	"strings"
 
+	"github.com/sirupsen/logrus"
 	"github.com/superseriousbusiness/activity/pub"
+	"github.com/superseriousbusiness/activity/streams"
+	"github.com/superseriousbusiness/activity/streams/vocab"
+	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
 	"github.com/superseriousbusiness/gotosocial/internal/concurrency"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
@@ -31,6 +37,9 @@ import (
 	"github.com/superseriousbusiness/gotosocial/internal/transport"
 )
 
+const applicationJSON = "application/json"
+const applicationActivityJSON = "application/activity+json"
+
 // NewTestTransportController returns a test transport controller with the given http client.
 //
 // Obviously for testing purposes you should not be making actual http calls to other servers.
@@ -44,33 +53,232 @@ func NewTestTransportController(client pub.HttpClient, db db.DB, fedWorker *conc
 	return transport.NewController(db, NewTestFederatingDB(db, fedWorker), &federation.Clock{}, client)
 }
 
-// NewMockHTTPClient returns a client that conforms to the pub.HttpClient interface,
+type MockHTTPClient struct {
-// but will always just execute the given `do` function, allowing responses to be mocked.
+	do func(req *http.Request) (*http.Response, error)
+
+	testRemoteStatuses    map[string]vocab.ActivityStreamsNote
+	testRemotePeople      map[string]vocab.ActivityStreamsPerson
+	testRemoteGroups      map[string]vocab.ActivityStreamsGroup
+	testRemoteServices    map[string]vocab.ActivityStreamsService
+	testRemoteAttachments map[string]RemoteAttachmentFile
+
+	SentMessages map[string][]byte
+}
+
+// NewMockHTTPClient returns a client that conforms to the pub.HttpClient interface.
 //
-// If 'do' is nil, then a no-op function will be used instead, that just returns status 200.
+// If do is nil, then a standard response set will be mocked out, which includes models stored in the
+// testrig, and webfinger responses as well.
+//
+// If do is not nil, then the given do function will always be used, which allows callers
+// to customize how the client is mocked.
 //
 // Note that you should never ever make ACTUAL http calls with this thing.
-func NewMockHTTPClient(do func(req *http.Request) (*http.Response, error)) pub.HttpClient {
+func NewMockHTTPClient(do func(req *http.Request) (*http.Response, error), relativeMediaPath string) *MockHTTPClient {
-	if do == nil {
+	mockHTTPClient := &MockHTTPClient{}
-		return &mockHTTPClient{
+
-			do: func(req *http.Request) (*http.Response, error) {
+	if do != nil {
-				r := ioutil.NopCloser(bytes.NewReader([]byte{}))
+		mockHTTPClient.do = do
+		return mockHTTPClient
+	}
+
+	mockHTTPClient.testRemoteStatuses = NewTestFediStatuses()
+	mockHTTPClient.testRemotePeople = NewTestFediPeople()
+	mockHTTPClient.testRemoteGroups = NewTestFediGroups()
+	mockHTTPClient.testRemoteServices = NewTestFediServices()
+	mockHTTPClient.testRemoteAttachments = NewTestFediAttachments(relativeMediaPath)
+
+	mockHTTPClient.SentMessages = make(map[string][]byte)
+
+	mockHTTPClient.do = func(req *http.Request) (*http.Response, error) {
+		responseCode := http.StatusNotFound
+		responseBytes := []byte(`{"error":"404 not found"}`)
+		responseContentType := applicationJSON
+		responseContentLength := len(responseBytes)
+
+		if req.Method == http.MethodPost {
+			b, err := io.ReadAll(req.Body)
+			if err != nil {
+				panic(err)
+			}
+			mockHTTPClient.SentMessages[req.URL.String()] = b
+
+			responseCode = http.StatusOK
+			responseBytes = []byte(`{"ok":"accepted"}`)
+			responseContentType = applicationJSON
+			responseContentLength = len(responseBytes)
+		} else if strings.Contains(req.URL.String(), ".well-known/webfinger") {
+			responseCode, responseBytes, responseContentType, responseContentLength = WebfingerResponse(req)
+		} else if note, ok := mockHTTPClient.testRemoteStatuses[req.URL.String()]; ok {
+			// the request is for a note that we have stored
+			noteI, err := streams.Serialize(note)
+			if err != nil {
+				panic(err)
+			}
+			noteJSON, err := json.Marshal(noteI)
+			if err != nil {
+				panic(err)
+			}
+			responseCode = http.StatusOK
+			responseBytes = noteJSON
+			responseContentType = applicationActivityJSON
+			responseContentLength = len(noteJSON)
+		} else if person, ok := mockHTTPClient.testRemotePeople[req.URL.String()]; ok {
+			// the request is for a person that we have stored
+			personI, err := streams.Serialize(person)
+			if err != nil {
+				panic(err)
+			}
+			personJSON, err := json.Marshal(personI)
+			if err != nil {
+				panic(err)
+			}
+			responseCode = http.StatusOK
+			responseBytes = personJSON
+			responseContentType = applicationActivityJSON
+			responseContentLength = len(personJSON)
+		} else if group, ok := mockHTTPClient.testRemoteGroups[req.URL.String()]; ok {
+			// the request is for a person that we have stored
+			groupI, err := streams.Serialize(group)
+			if err != nil {
+				panic(err)
+			}
+			groupJSON, err := json.Marshal(groupI)
+			if err != nil {
+				panic(err)
+			}
+			responseCode = http.StatusOK
+			responseBytes = groupJSON
+			responseContentType = applicationActivityJSON
+			responseContentLength = len(groupJSON)
+		} else if service, ok := mockHTTPClient.testRemoteServices[req.URL.String()]; ok {
+			serviceI, err := streams.Serialize(service)
+			if err != nil {
+				panic(err)
+			}
+			serviceJSON, err := json.Marshal(serviceI)
+			if err != nil {
+				panic(err)
+			}
+			responseCode = http.StatusOK
+			responseBytes = serviceJSON
+			responseContentType = applicationActivityJSON
+			responseContentLength = len(serviceJSON)
+		} else if attachment, ok := mockHTTPClient.testRemoteAttachments[req.URL.String()]; ok {
+			responseCode = http.StatusOK
+			responseBytes = attachment.Data
+			responseContentType = attachment.ContentType
+			responseContentLength = len(attachment.Data)
+		}
+
+		logrus.Debugf("returning response %s", string(responseBytes))
+		reader := bytes.NewReader(responseBytes)
+		readCloser := io.NopCloser(reader)
 		return &http.Response{
-					StatusCode: 200,
+			StatusCode:    responseCode,
-					Body:       r,
+			Body:          readCloser,
+			ContentLength: int64(responseContentLength),
+			Header: http.Header{
+				"content-type": {responseContentType},
+			},
 		}, nil
+	}
+
+	return mockHTTPClient
+}
+
+func (m *MockHTTPClient) Do(req *http.Request) (*http.Response, error) {
+	return m.do(req)
+}
+
+func WebfingerResponse(req *http.Request) (responseCode int, responseBytes []byte, responseContentType string, responseContentLength int) {
+	var wfr *apimodel.WellKnownResponse
+
+	switch req.URL.String() {
+	case "https://unknown-instance.com/.well-known/webfinger?resource=acct:some_group@unknown-instance.com":
+		wfr = &apimodel.WellKnownResponse{
+			Subject: "acct:some_group@unknown-instance.com",
+			Links: []apimodel.Link{
+				{
+					Rel:  "self",
+					Type: applicationActivityJSON,
+					Href: "https://unknown-instance.com/groups/some_group",
+				},
+			},
+		}
+	case "https://owncast.example.org/.well-known/webfinger?resource=acct:rgh@owncast.example.org":
+		wfr = &apimodel.WellKnownResponse{
+			Subject: "acct:rgh@example.org",
+			Links: []apimodel.Link{
+				{
+					Rel:  "self",
+					Type: applicationActivityJSON,
+					Href: "https://owncast.example.org/federation/user/rgh",
+				},
+			},
+		}
+	case "https://unknown-instance.com/.well-known/webfinger?resource=acct:brand_new_person@unknown-instance.com":
+		wfr = &apimodel.WellKnownResponse{
+			Subject: "acct:brand_new_person@unknown-instance.com",
+			Links: []apimodel.Link{
+				{
+					Rel:  "self",
+					Type: applicationActivityJSON,
+					Href: "https://unknown-instance.com/users/brand_new_person",
+				},
+			},
+		}
+	case "https://turnip.farm/.well-known/webfinger?resource=acct:turniplover6969@turnip.farm":
+		wfr = &apimodel.WellKnownResponse{
+			Subject: "acct:turniplover6969@turnip.farm",
+			Links: []apimodel.Link{
+				{
+					Rel:  "self",
+					Type: applicationActivityJSON,
+					Href: "https://turnip.farm/users/turniplover6969",
+				},
+			},
+		}
+	case "https://fossbros-anonymous.io/.well-known/webfinger?resource=acct:foss_satan@fossbros-anonymous.io":
+		wfr = &apimodel.WellKnownResponse{
+			Subject: "acct:foss_satan@fossbros-anonymous.io",
+			Links: []apimodel.Link{
+				{
+					Rel:  "self",
+					Type: applicationActivityJSON,
+					Href: "https://fossbros-anonymous.io/users/foss_satan",
+				},
+			},
+		}
+	case "https://example.org/.well-known/webfinger?resource=acct:some_user@example.org":
+		wfr = &apimodel.WellKnownResponse{
+			Subject: "acct:some_user@example.org",
+			Links: []apimodel.Link{
+				{
+					Rel:  "self",
+					Type: applicationActivityJSON,
+					Href: "https://example.org/users/some_user",
+				},
 			},
 		}
 	}
-	return &mockHTTPClient{
+
-		do: do,
+	if wfr == nil {
+		logrus.Debugf("webfinger response not available for %s", req.URL)
+		responseCode = http.StatusNotFound
+		responseBytes = []byte(`{"error":"not found"}`)
+		responseContentType = applicationJSON
+		responseContentLength = len(responseBytes)
+		return
 	}
-}
 
-type mockHTTPClient struct {
+	wfrJSON, err := json.Marshal(wfr)
-	do func(req *http.Request) (*http.Response, error)
+	if err != nil {
-}
+		panic(err)
-
+	}
-func (m *mockHTTPClient) Do(req *http.Request) (*http.Response, error) {
+	responseCode = http.StatusOK
-	return m.do(req)
+	responseBytes = wfrJSON
+	responseContentType = applicationJSON
+	responseContentLength = len(wfrJSON)
+	return
 }