diff --git a/docs/api/swagger.yaml b/docs/api/swagger.yaml index e72f19f02..faf9f181e 100644 --- a/docs/api/swagger.yaml +++ b/docs/api/swagger.yaml @@ -1562,6 +1562,64 @@ definitions: type: string x-go-name: Visibility x-go-package: github.com/superseriousbusiness/gotosocial/internal/api/model + swaggerStatusRepliesCollection: + properties: + '@context': + description: ActivityStreams context. + example: https://www.w3.org/ns/activitystreams + type: string + x-go-name: Context + first: + $ref: '#/definitions/swaggerStatusRepliesCollectionPage' + id: + description: ActivityStreams ID. + example: https://example.org/users/some_user/statuses/106717595988259568/replies + type: string + x-go-name: ID + type: + description: ActivityStreams type. + example: Collection + type: string + x-go-name: Type + title: SwaggerStatusRepliesCollection represents a response to GET /users/{username}/statuses/{status}/replies. + type: object + x-go-name: SwaggerStatusRepliesCollection + x-go-package: github.com/superseriousbusiness/gotosocial/internal/api/s2s/user + swaggerStatusRepliesCollectionPage: + properties: + id: + description: ActivityStreams ID. + example: https://example.org/users/some_user/statuses/106717595988259568/replies?page=true + type: string + x-go-name: ID + items: + description: Items on this page. + example: + - https://example.org/users/some_other_user/statuses/086417595981111564 + - https://another.example.com/users/another_user/statuses/01FCN8XDV3YG7B4R42QA6YQZ9R + items: + type: string + type: array + x-go-name: Items + next: + description: Link to the next page. + example: https://example.org/users/some_user/statuses/106717595988259568/replies?only_other_accounts=true&page=true + type: string + x-go-name: Next + partOf: + description: Collection this page belongs to. + example: https://example.org/users/some_user/statuses/106717595988259568/replies + type: string + x-go-name: PartOf + type: + description: ActivityStreams type. + example: CollectionPage + type: string + x-go-name: Type + title: SwaggerStatusRepliesCollectionPage represents one page of a collection. + type: object + x-go-name: SwaggerStatusRepliesCollectionPage + x-go-package: github.com/superseriousbusiness/gotosocial/internal/api/s2s/user tag: properties: name: @@ -1621,7 +1679,7 @@ info: name: AGPL3 url: https://www.gnu.org/licenses/agpl-3.0.en.html title: GoToSocial - version: 0.1.0-SNAPSHOT + version: 0.1.0-SNAPSHOT-dereference_remote_replies paths: /api/v1/accounts: post: @@ -2395,11 +2453,10 @@ paths: - blocks /api/v1/instance: get: - description: "This is mostly provided for Mastodon application compatibility, - since many apps that work with Mastodon use `/api/v1/instance` to inform their - connection parameters. \n\nHowever, it can also be used by other instances - for gathering instance information and representing instances in some UI or - other." + description: |- + This is mostly provided for Mastodon application compatibility, since many apps that work with Mastodon use `/api/v1/instance` to inform their connection parameters. + + However, it can also be used by other instances for gathering instance information and representing instances in some UI or other. operationId: instanceGet produces: - application/json @@ -3306,6 +3363,56 @@ paths: summary: See public statuses/posts that your instance is aware of. tags: - timelines + /users/{username}/statuses/{status}/replies: + get: + description: |- + Note that the response will be a Collection with a page as `first`, as shown below, if `page` is `false`. + + If `page` is `true`, then the response will be a single `CollectionPage` without the wrapping `Collection`. + + HTTP signature is required on the request. + operationId: s2sRepliesGet + parameters: + - description: Username of the account. + in: path + name: username + required: true + type: string + - description: ID of the status. + in: path + name: status + required: true + type: string + - default: false + description: Return response as a CollectionPage. + in: query + name: page + type: boolean + - default: false + description: Return replies only from accounts other than the status owner. + in: query + name: only_other_accounts + type: boolean + - description: Minimum ID of the next status, used for paging. + in: query + name: min_id + type: string + produces: + - application/activity+json + responses: + "200": + description: "" + schema: + $ref: '#/definitions/swaggerStatusRepliesCollection' + "401": + description: unauthorized + "403": + description: forbidden + "404": + description: not found + summary: Get the replies collection for a status. + tags: + - s2s/federation schemes: - https - http diff --git a/docs/assets/diagrams/conversation_thread.drawio b/docs/assets/diagrams/conversation_thread.drawio new file mode 100644 index 000000000..99d365dcb --- /dev/null +++ b/docs/assets/diagrams/conversation_thread.drawio @@ -0,0 +1 @@ +7VrBcpswEP0ajs0AMjY+xkncXtqmTaZNTxnFKEYTQIwQsZ2vrwQSBok0rmObmGTG40ELEtq3T29Xsi1wFi8/U5iGX0mAIsu1g6UFzi3XHfku/xaGlTI4pWFOcVCaaoYr/ISk0ZbWHAcoazzICIkYTpvGGUkSNGMNG6SULJqP3ZOo+dYUzpFhuJrByLT+xgELS6vv2Wv7F4TnoXqzY8s7MVQPS0MWwoAsaiZwYYEzSggrr+LlGYoEdgqXst/0mbvVxChK2CYdTp/8m+9T+/pXPP0Bw2sCycPtp0E5yiOMcumwNbAzEiOSIG6/DnEmhhBfUDyH6Er4hmCCk/tcDJ+SjJ1ID9lKwUZJngRIvNmxwGQRYoauUjgTdxecJ9wWsjiStzNGyUMFLwdmImeFKEPLZ911KhA5+RCfMysmJzsAhbsknuv5ZXuxDqOjnglrIRxKG5TMmVdDr8HlFxLf/8B6aGINpo3P61BU/LJ5I4BZWHXcAZq+BqYCqTMwRyaYOnooCU6FAvBWIuisoVKDDi0xu6ld/xEonniydb6UoBaNlWok3I2beqPWSzTX3YqW6ldOEwWG7Gih4K6QnM7Qy4RikM4Re2mRm6Gtxc5rCZ2yURRBhh+b022Lp3zDJcHckYo5LmhSZwA0SpRuyl51/dIH0jgIfG2gEgdjoIJeldvbM843GHdJ0SMmeSZVsG0F8xXHmlyDEZ4n/HrGw40oN4h1iXnCOZU3YhwEovuEogw/wbtiKMGcVHhW+OpNLO9cjJUzkpUpc0frvMq7aqGPzYU+biGLu691Pj5i0fSGby0Fqei+a9kcbCib4y5lczDSZNPZUjY9rQ4COrn2LJuOY1DumxDF/mimHqo2zRwdUjMdt0ei6fhdl5oO6BGcYNA5nC17zlfkoC3zyTa5q4McVC7lzpKQpyUhnRQbJyH7hWy27yTkfdQ9m3MOvKnCZ7Crwmd8YM6Z5z0/URqtelHy6OdBrplV2jiyv5LHPA/qLdjA6xpsc1M+IbyY51opq3rXvlsVUhAThm5NrT22EAAtf1WHWvUqanjIGKiBj7Io1c/mnTY4D1qUuuYu9XjhBKPO4WzZgb67emvT4/nar4FdnM9rJ75b1/iuttM8dI3vtmzTPzj3T73rjHP6dnBrzukb1ENzzjzL6E3Zqf9wB1p+8z1o2emam/jegl2p8u7B5s31n07KhbD+5w64+As= \ No newline at end of file diff --git a/docs/assets/diagrams/conversation_thread.png b/docs/assets/diagrams/conversation_thread.png new file mode 100644 index 000000000..b4eca90f7 Binary files /dev/null and b/docs/assets/diagrams/conversation_thread.png differ diff --git a/docs/federation/behaviors/conversation_threads.md b/docs/federation/behaviors/conversation_threads.md new file mode 100644 index 000000000..4e6f6b0ee --- /dev/null +++ b/docs/federation/behaviors/conversation_threads.md @@ -0,0 +1,45 @@ +# Conversation Threads + +Due to the nature of decentralization and federation, it is practically impossible for any one server on the fediverse to be aware of every post in a given conversation thread. + +With that said, it is possible to do 'best effort' dereferencing of threads, whereby remote replies are fetched from one server onto another, to try to more fully flesh out a conversation. + +GoToSocial does this by iterating up and down the thread of a conversation, pulling in remote statuses where possible. + +## Example + +Let's say we have two accounts: `local_account` on `our.server`, and `remote_1` on `remote.1`. + +In this scenario, `local_account` follows `remote_1`, so posts from `remote_1` show up in the home timeline of `local_account`. + +Now, `remote_1` boosts/reblogs a post from a third account, `remote_2`, residing on server `remote.2`. + +`local_account` does not follow `remote_2`, and neither does anybody else on `our.server`, which means that `our.server` has not seen this post by `remote_2` before. + +![A diagram of the conversation thread, showing the post from remote_2, and possible ancestor and descendant posts](../../assets/diagrams/conversation_thread.png) + +What GoToSocial will do now, is 'dereference' the post by `remote_2` to check if it is part of a thread and, if so, whether any other parts of the thread can be obtained. + +GtS begins by checking the `inReplyTo` property of the post, which is set when a post is a reply to another post. [See here](https://www.w3.org/TR/activitystreams-vocabulary/#dfn-inreplyto). If `inReplyTo` is set, GoToSocial derefences the replied-to post. If *this* post also has an `inReplyTo` set, then GoToSocial dereferences that too, and so on. + +Once all of these **ancestors** of a status have been retrieved, GtS will begin working down through the **descendants** of posts. + +It does this by checking the `replies` property of a derefenced post, and working through replies, and replies of replies. [See here](https://www.w3.org/TR/activitystreams-vocabulary/#dfn-replies). + +This process of thread dereferencing will likely involve making multiple HTTP calls to different servers, especially if the thread is long and complicated. + +The end result of this dereferencing is that, assuming the reblogged post by `remote_2` was part of a thread, then `local_account` should now be able to see posts in the thread when they open the status on their home timeline. In other words, they will see replies from accounts on other servers (who they may not have come across yet), in addition to any previous and next posts in the thread as posted by `remote_2`. + +This gives `local_account` a more complete view on the conversation, as opposed to just seeing the reblogged post in isolation and out of context. It also gives `local_account` the opportunity to discover new accounts to follow, based on replies to `remote_2`. + +## Privacy and Security + +During the dereferencing process, GoToSocial signs outgoing requests using the key of the actor who received the activity that necessitated dereferencing. To use the above example, this means that all dereferencing requests would be signed by `local_account`. This gives remote servers the ability to refuse these dereferencing requests, assuming that `local_account` is blocked by one or more participants in the conversation. + +From GoToSocial's side, domain blocks will be respected during the dereferencing process, to avoid making calls to servers that `our.server` has blocked. + +Individual account blocks will also be respected, meaning that `our.server` won't try to dereference posts from accounts blocked by `local_account`. + +Finally, GoToSocial expects that remote servers will only list replies that are marked as public (either `to` or `cc`). GtS may *try* to dereference followers-only posts, but it will assume that remote servers will check whether or not `local_account` is allowed to view them, and refuse accordingly. + +Of course, when `local_account` opens up the conversation thread in whatever application they are using, GoToSocial will apply the usual post visibility filtering to ensure that they do not see any posts that they shouldn't have access to. diff --git a/docs/federation/glossary.md b/docs/federation/glossary.md new file mode 100644 index 000000000..3362d666a --- /dev/null +++ b/docs/federation/glossary.md @@ -0,0 +1,27 @@ +# Glossary + +Some commonly-used terms in discussions of federation, and their meanings. + +### `ActivityPub` + +A decentralized social networking protocol based on the ActivityStreams data format. See [here](https://www.w3.org/TR/activitypub/). + +GoToSocial uses the ActivityPub protocol to communicate between GtS servers, and with other federated servers like Mastodon, Pixelfed, etc. + +### `ActivityStreams` + +A model/data format for representing potential and completed activities using JSON. See [here](https://www.w3.org/TR/activitystreams-core/). + +GoToSocial uses the ActivityStreams data model to 'speak' ActivityPub with other servers. + +### `Actor` + +An actor is an ActivityStreams object that is capable of performing some Activity like following, liking, creating a post, reblogging, etc. See [here](https://www.w3.org/TR/activitypub/#actors). + +In GoToSocial, each account/user is an actor. + +### `Dereference` + +To 'dereference' a post or a profile means to make an HTTP call to the server that hosts that post or profile, in order to obtain its ActivityStreams representation. + +GoToSocial 'dereferences' posts and profiles on remote servers, in order to convert them to models that GoToSocial can understand and work with. diff --git a/docs/federation/principles.md b/docs/federation/principles.md new file mode 100644 index 000000000..643ab9e1b --- /dev/null +++ b/docs/federation/principles.md @@ -0,0 +1,9 @@ +# Principles + +TODO -- describe the principles GtS uses for federating. + +Eg: + +* Why federate? +* Why ActivityPub? +* Broad overview of how GtS fits into the fediverse. diff --git a/docs/federation/security.md b/docs/federation/security.md new file mode 100644 index 000000000..d7c2b6ae7 --- /dev/null +++ b/docs/federation/security.md @@ -0,0 +1,7 @@ +# Security + +TODO: describe the security model we use for federation. + +* http signatures +* behavior for refusing requests +* how data is protected diff --git a/internal/typeutils/asextractionutil.go b/internal/ap/extract.go similarity index 74% rename from internal/typeutils/asextractionutil.go rename to internal/ap/extract.go index b3e6eb2c4..baffd4bf2 100644 --- a/internal/typeutils/asextractionutil.go +++ b/internal/ap/extract.go @@ -16,7 +16,10 @@ along with this program. If not, see . */ -package typeutils +// Package ap contains models and utilities for working with activitypub/activitystreams representations. +// +// It is built on top of go-fed/activity. +package ap import ( "crypto/rsa" @@ -33,7 +36,8 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/util" ) -func extractPreferredUsername(i withPreferredUsername) (string, error) { +// ExtractPreferredUsername returns a string representation of an interface's preferredUsername property. +func ExtractPreferredUsername(i WithPreferredUsername) (string, error) { u := i.GetActivityStreamsPreferredUsername() if u == nil || !u.IsXMLSchemaString() { return "", errors.New("preferredUsername was not a string") @@ -44,7 +48,8 @@ func extractPreferredUsername(i withPreferredUsername) (string, error) { return u.GetXMLSchemaString(), nil } -func extractName(i withName) (string, error) { +// ExtractName returns a string representation of an interface's name property. +func ExtractName(i WithName) (string, error) { nameProp := i.GetActivityStreamsName() if nameProp == nil { return "", errors.New("activityStreamsName not found") @@ -60,22 +65,42 @@ func extractName(i withName) (string, error) { return "", errors.New("activityStreamsName not found") } -func extractInReplyToURI(i withInReplyTo) (*url.URL, error) { +// ExtractInReplyToURI extracts the inReplyToURI property (if present) from an interface. +func ExtractInReplyToURI(i WithInReplyTo) *url.URL { inReplyToProp := i.GetActivityStreamsInReplyTo() if inReplyToProp == nil { - return nil, errors.New("in reply to prop was nil") + // the property just wasn't set + return nil } for iter := inReplyToProp.Begin(); iter != inReplyToProp.End(); iter = iter.Next() { if iter.IsIRI() { if iter.GetIRI() != nil { - return iter.GetIRI(), nil + return iter.GetIRI() } } } - return nil, errors.New("couldn't find iri for in reply to") + // couldn't find a URI + return nil } -func extractTos(i withTo) ([]*url.URL, error) { +// ExtractURLItems extracts a slice of URLs from a property that has withItems. +func ExtractURLItems(i WithItems) []*url.URL { + urls := []*url.URL{} + items := i.GetActivityStreamsItems() + if items == nil || items.Len() == 0 { + return urls + } + + for iter := items.Begin(); iter != items.End(); iter = iter.Next() { + if iter.IsIRI() { + urls = append(urls, iter.GetIRI()) + } + } + return urls +} + +// ExtractTos returns a list of URIs that the activity addresses as To. +func ExtractTos(i WithTo) ([]*url.URL, error) { to := []*url.URL{} toProp := i.GetActivityStreamsTo() if toProp == nil { @@ -91,7 +116,8 @@ func extractTos(i withTo) ([]*url.URL, error) { return to, nil } -func extractCCs(i withCC) ([]*url.URL, error) { +// ExtractCCs returns a list of URIs that the activity addresses as CC. +func ExtractCCs(i WithCC) ([]*url.URL, error) { cc := []*url.URL{} ccProp := i.GetActivityStreamsCc() if ccProp == nil { @@ -107,7 +133,8 @@ func extractCCs(i withCC) ([]*url.URL, error) { return cc, nil } -func extractAttributedTo(i withAttributedTo) (*url.URL, error) { +// ExtractAttributedTo returns the URL of the actor that the withAttributedTo is attributed to. +func ExtractAttributedTo(i WithAttributedTo) (*url.URL, error) { attributedToProp := i.GetActivityStreamsAttributedTo() if attributedToProp == nil { return nil, errors.New("attributedToProp was nil") @@ -122,7 +149,8 @@ func extractAttributedTo(i withAttributedTo) (*url.URL, error) { return nil, errors.New("couldn't find iri for attributed to") } -func extractPublished(i withPublished) (time.Time, error) { +// ExtractPublished extracts the publication time of an activity. +func ExtractPublished(i WithPublished) (time.Time, error) { publishedProp := i.GetActivityStreamsPublished() if publishedProp == nil { return time.Time{}, errors.New("published prop was nil") @@ -139,13 +167,13 @@ func extractPublished(i withPublished) (time.Time, error) { return t, nil } -// extractIconURL extracts a URL to a supported image file from something like: +// ExtractIconURL extracts a URL to a supported image file from something like: // "icon": { // "mediaType": "image/jpeg", // "type": "Image", // "url": "http://example.org/path/to/some/file.jpeg" // }, -func extractIconURL(i withIcon) (*url.URL, error) { +func ExtractIconURL(i WithIcon) (*url.URL, error) { iconProp := i.GetActivityStreamsIcon() if iconProp == nil { return nil, errors.New("icon property was nil") @@ -166,7 +194,7 @@ func extractIconURL(i withIcon) (*url.URL, error) { } // 2. has a URL so we can grab it - url, err := extractURL(imageValue) + url, err := ExtractURL(imageValue) if err == nil && url != nil { return url, nil } @@ -175,13 +203,13 @@ func extractIconURL(i withIcon) (*url.URL, error) { return nil, errors.New("could not extract valid image from icon") } -// extractImageURL extracts a URL to a supported image file from something like: +// ExtractImageURL extracts a URL to a supported image file from something like: // "image": { // "mediaType": "image/jpeg", // "type": "Image", // "url": "http://example.org/path/to/some/file.jpeg" // }, -func extractImageURL(i withImage) (*url.URL, error) { +func ExtractImageURL(i WithImage) (*url.URL, error) { imageProp := i.GetActivityStreamsImage() if imageProp == nil { return nil, errors.New("icon property was nil") @@ -202,7 +230,7 @@ func extractImageURL(i withImage) (*url.URL, error) { } // 2. has a URL so we can grab it - url, err := extractURL(imageValue) + url, err := ExtractURL(imageValue) if err == nil && url != nil { return url, nil } @@ -211,7 +239,8 @@ func extractImageURL(i withImage) (*url.URL, error) { return nil, errors.New("could not extract valid image from image property") } -func extractSummary(i withSummary) (string, error) { +// ExtractSummary extracts the summary/content warning of an interface. +func ExtractSummary(i WithSummary) (string, error) { summaryProp := i.GetActivityStreamsSummary() if summaryProp == nil { return "", errors.New("summary property was nil") @@ -226,14 +255,16 @@ func extractSummary(i withSummary) (string, error) { return "", errors.New("could not extract summary") } -func extractDiscoverable(i withDiscoverable) (bool, error) { +// ExtractDiscoverable extracts the Discoverable boolean of an interface. +func ExtractDiscoverable(i WithDiscoverable) (bool, error) { if i.GetTootDiscoverable() == nil { return false, errors.New("discoverable was nil") } return i.GetTootDiscoverable().Get(), nil } -func extractURL(i withURL) (*url.URL, error) { +// ExtractURL extracts the URL property of an interface. +func ExtractURL(i WithURL) (*url.URL, error) { urlProp := i.GetActivityStreamsUrl() if urlProp == nil { return nil, errors.New("url property was nil") @@ -248,7 +279,9 @@ func extractURL(i withURL) (*url.URL, error) { return nil, errors.New("could not extract url") } -func extractPublicKeyForOwner(i withPublicKey, forOwner *url.URL) (*rsa.PublicKey, *url.URL, error) { +// ExtractPublicKeyForOwner extracts the public key from an interface, as long as it belongs to the specified owner. +// It will return the public key itself, the id/URL of the public key, or an error if something goes wrong. +func ExtractPublicKeyForOwner(i WithPublicKey, forOwner *url.URL) (*rsa.PublicKey, *url.URL, error) { publicKeyProp := i.GetW3IDSecurityV1PublicKey() if publicKeyProp == nil { return nil, nil, errors.New("public key property was nil") @@ -298,7 +331,8 @@ func extractPublicKeyForOwner(i withPublicKey, forOwner *url.URL) (*rsa.PublicKe return nil, nil, errors.New("couldn't find public key") } -func extractContent(i withContent) (string, error) { +// ExtractContent returns a string representation of the interface's Content property. +func ExtractContent(i WithContent) (string, error) { contentProperty := i.GetActivityStreamsContent() if contentProperty == nil { return "", errors.New("content property was nil") @@ -311,7 +345,8 @@ func extractContent(i withContent) (string, error) { return "", errors.New("no content found") } -func extractAttachments(i withAttachment) ([]*gtsmodel.MediaAttachment, error) { +// ExtractAttachments returns a slice of attachments on the interface. +func ExtractAttachments(i WithAttachment) ([]*gtsmodel.MediaAttachment, error) { attachments := []*gtsmodel.MediaAttachment{} attachmentProp := i.GetActivityStreamsAttachment() if attachmentProp == nil { @@ -326,7 +361,7 @@ func extractAttachments(i withAttachment) ([]*gtsmodel.MediaAttachment, error) { if !ok { continue } - attachment, err := extractAttachment(attachmentable) + attachment, err := ExtractAttachment(attachmentable) if err != nil { continue } @@ -335,12 +370,13 @@ func extractAttachments(i withAttachment) ([]*gtsmodel.MediaAttachment, error) { return attachments, nil } -func extractAttachment(i Attachmentable) (*gtsmodel.MediaAttachment, error) { +// ExtractAttachment returns a gts model of an attachment from an attachmentable interface. +func ExtractAttachment(i Attachmentable) (*gtsmodel.MediaAttachment, error) { attachment := >smodel.MediaAttachment{ File: gtsmodel.File{}, } - attachmentURL, err := extractURL(i) + attachmentURL, err := ExtractURL(i) if err != nil { return nil, err } @@ -356,7 +392,7 @@ func extractAttachment(i Attachmentable) (*gtsmodel.MediaAttachment, error) { attachment.File.ContentType = mediaType.Get() attachment.Type = gtsmodel.FileTypeImage - name, err := extractName(i) + name, err := ExtractName(i) if err == nil { attachment.Description = name } @@ -376,7 +412,8 @@ func extractAttachment(i Attachmentable) (*gtsmodel.MediaAttachment, error) { // return i.GetTootBlurhashProperty().Get(), nil // } -func extractHashtags(i withTag) ([]*gtsmodel.Tag, error) { +// ExtractHashtags returns a slice of tags on the interface. +func ExtractHashtags(i WithTag) ([]*gtsmodel.Tag, error) { tags := []*gtsmodel.Tag{} tagsProp := i.GetActivityStreamsTag() if tagsProp == nil { @@ -397,7 +434,7 @@ func extractHashtags(i withTag) ([]*gtsmodel.Tag, error) { continue } - tag, err := extractHashtag(hashtaggable) + tag, err := ExtractHashtag(hashtaggable) if err != nil { continue } @@ -407,7 +444,8 @@ func extractHashtags(i withTag) ([]*gtsmodel.Tag, error) { return tags, nil } -func extractHashtag(i Hashtaggable) (*gtsmodel.Tag, error) { +// ExtractHashtag returns a gtsmodel tag from a hashtaggable. +func ExtractHashtag(i Hashtaggable) (*gtsmodel.Tag, error) { tag := >smodel.Tag{} hrefProp := i.GetActivityStreamsHref() @@ -416,7 +454,7 @@ func extractHashtag(i Hashtaggable) (*gtsmodel.Tag, error) { } tag.URL = hrefProp.GetIRI().String() - name, err := extractName(i) + name, err := ExtractName(i) if err != nil { return nil, err } @@ -425,7 +463,8 @@ func extractHashtag(i Hashtaggable) (*gtsmodel.Tag, error) { return tag, nil } -func extractEmojis(i withTag) ([]*gtsmodel.Emoji, error) { +// ExtractEmojis returns a slice of emojis on the interface. +func ExtractEmojis(i WithTag) ([]*gtsmodel.Emoji, error) { emojis := []*gtsmodel.Emoji{} tagsProp := i.GetActivityStreamsTag() if tagsProp == nil { @@ -446,7 +485,7 @@ func extractEmojis(i withTag) ([]*gtsmodel.Emoji, error) { continue } - emoji, err := extractEmoji(emojiable) + emoji, err := ExtractEmoji(emojiable) if err != nil { continue } @@ -456,7 +495,8 @@ func extractEmojis(i withTag) ([]*gtsmodel.Emoji, error) { return emojis, nil } -func extractEmoji(i Emojiable) (*gtsmodel.Emoji, error) { +// ExtractEmoji ... +func ExtractEmoji(i Emojiable) (*gtsmodel.Emoji, error) { emoji := >smodel.Emoji{} idProp := i.GetJSONLDId() @@ -467,7 +507,7 @@ func extractEmoji(i Emojiable) (*gtsmodel.Emoji, error) { emoji.URI = uri.String() emoji.Domain = uri.Host - name, err := extractName(i) + name, err := ExtractName(i) if err != nil { return nil, err } @@ -476,7 +516,7 @@ func extractEmoji(i Emojiable) (*gtsmodel.Emoji, error) { if i.GetActivityStreamsIcon() == nil { return nil, errors.New("no icon for emoji") } - imageURL, err := extractIconURL(i) + imageURL, err := ExtractIconURL(i) if err != nil { return nil, errors.New("no url for emoji image") } @@ -485,7 +525,8 @@ func extractEmoji(i Emojiable) (*gtsmodel.Emoji, error) { return emoji, nil } -func extractMentions(i withTag) ([]*gtsmodel.Mention, error) { +// ExtractMentions extracts a slice of gtsmodel Mentions from a WithTag interface. +func ExtractMentions(i WithTag) ([]*gtsmodel.Mention, error) { mentions := []*gtsmodel.Mention{} tagsProp := i.GetActivityStreamsTag() if tagsProp == nil { @@ -506,7 +547,7 @@ func extractMentions(i withTag) ([]*gtsmodel.Mention, error) { continue } - mention, err := extractMention(mentionable) + mention, err := ExtractMention(mentionable) if err != nil { continue } @@ -516,10 +557,11 @@ func extractMentions(i withTag) ([]*gtsmodel.Mention, error) { return mentions, nil } -func extractMention(i Mentionable) (*gtsmodel.Mention, error) { +// ExtractMention extracts a gts model mention from a Mentionable. +func ExtractMention(i Mentionable) (*gtsmodel.Mention, error) { mention := >smodel.Mention{} - mentionString, err := extractName(i) + mentionString, err := ExtractName(i) if err != nil { return nil, err } @@ -543,7 +585,8 @@ func extractMention(i Mentionable) (*gtsmodel.Mention, error) { return mention, nil } -func extractActor(i withActor) (*url.URL, error) { +// ExtractActor extracts the actor ID/IRI from an interface WithActor. +func ExtractActor(i WithActor) (*url.URL, error) { actorProp := i.GetActivityStreamsActor() if actorProp == nil { return nil, errors.New("actor property was nil") @@ -556,7 +599,8 @@ func extractActor(i withActor) (*url.URL, error) { return nil, errors.New("no iri found for actor prop") } -func extractObject(i withObject) (*url.URL, error) { +// ExtractObject extracts a URL object from a WithObject interface. +func ExtractObject(i WithObject) (*url.URL, error) { objectProp := i.GetActivityStreamsObject() if objectProp == nil { return nil, errors.New("object property was nil") diff --git a/internal/ap/interfaces.go b/internal/ap/interfaces.go new file mode 100644 index 000000000..43dd149d5 --- /dev/null +++ b/internal/ap/interfaces.go @@ -0,0 +1,321 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package ap + +import "github.com/go-fed/activity/streams/vocab" + +// Accountable represents the minimum activitypub interface for representing an 'account'. +// This interface is fulfilled by: Person, Application, Organization, Service, and Group +type Accountable interface { + WithJSONLDId + WithTypeName + + WithPreferredUsername + WithIcon + WithName + WithImage + WithSummary + WithDiscoverable + WithURL + WithPublicKey + WithInbox + WithOutbox + WithFollowing + WithFollowers + WithFeatured +} + +// Statusable represents the minimum activitypub interface for representing a 'status'. +// This interface is fulfilled by: Article, Document, Image, Video, Note, Page, Event, Place, Mention, Profile +type Statusable interface { + WithJSONLDId + WithTypeName + + WithSummary + WithInReplyTo + WithPublished + WithURL + WithAttributedTo + WithTo + WithCC + WithSensitive + WithConversation + WithContent + WithAttachment + WithTag + WithReplies +} + +// Attachmentable represents the minimum activitypub interface for representing a 'mediaAttachment'. +// This interface is fulfilled by: Audio, Document, Image, Video +type Attachmentable interface { + WithTypeName + WithMediaType + WithURL + WithName +} + +// Hashtaggable represents the minimum activitypub interface for representing a 'hashtag' tag. +type Hashtaggable interface { + WithTypeName + WithHref + WithName +} + +// Emojiable represents the minimum interface for an 'emoji' tag. +type Emojiable interface { + WithJSONLDId + WithTypeName + WithName + WithUpdated + WithIcon +} + +// Mentionable represents the minimum interface for a 'mention' tag. +type Mentionable interface { + WithName + WithHref +} + +// Followable represents the minimum interface for an activitystreams 'follow' activity. +type Followable interface { + WithJSONLDId + WithTypeName + + WithActor + WithObject +} + +// Likeable represents the minimum interface for an activitystreams 'like' activity. +type Likeable interface { + WithJSONLDId + WithTypeName + + WithActor + WithObject +} + +// Blockable represents the minimum interface for an activitystreams 'block' activity. +type Blockable interface { + WithJSONLDId + WithTypeName + + WithActor + WithObject +} + +// Announceable represents the minimum interface for an activitystreams 'announce' activity. +type Announceable interface { + WithJSONLDId + WithTypeName + + WithActor + WithObject + WithPublished + WithTo + WithCC +} + +// CollectionPageable represents the minimum interface for an activitystreams 'CollectionPage' object. +type CollectionPageable interface { + WithJSONLDId + WithTypeName + + WithNext + WithPartOf + WithItems +} + +// WithJSONLDId represents an activity with JSONLDIdProperty +type WithJSONLDId interface { + GetJSONLDId() vocab.JSONLDIdProperty +} + +// WithTypeName represents an activity with a type name +type WithTypeName interface { + GetTypeName() string +} + +// WithPreferredUsername represents an activity with ActivityStreamsPreferredUsernameProperty +type WithPreferredUsername interface { + GetActivityStreamsPreferredUsername() vocab.ActivityStreamsPreferredUsernameProperty +} + +// WithIcon represents an activity with ActivityStreamsIconProperty +type WithIcon interface { + GetActivityStreamsIcon() vocab.ActivityStreamsIconProperty +} + +// WithName represents an activity with ActivityStreamsNameProperty +type WithName interface { + GetActivityStreamsName() vocab.ActivityStreamsNameProperty +} + +// WithImage represents an activity with ActivityStreamsImageProperty +type WithImage interface { + GetActivityStreamsImage() vocab.ActivityStreamsImageProperty +} + +// WithSummary represents an activity with ActivityStreamsSummaryProperty +type WithSummary interface { + GetActivityStreamsSummary() vocab.ActivityStreamsSummaryProperty +} + +// WithDiscoverable represents an activity with TootDiscoverableProperty +type WithDiscoverable interface { + GetTootDiscoverable() vocab.TootDiscoverableProperty +} + +// WithURL represents an activity with ActivityStreamsUrlProperty +type WithURL interface { + GetActivityStreamsUrl() vocab.ActivityStreamsUrlProperty +} + +// WithPublicKey represents an activity with W3IDSecurityV1PublicKeyProperty +type WithPublicKey interface { + GetW3IDSecurityV1PublicKey() vocab.W3IDSecurityV1PublicKeyProperty +} + +// WithInbox represents an activity with ActivityStreamsInboxProperty +type WithInbox interface { + GetActivityStreamsInbox() vocab.ActivityStreamsInboxProperty +} + +// WithOutbox represents an activity with ActivityStreamsOutboxProperty +type WithOutbox interface { + GetActivityStreamsOutbox() vocab.ActivityStreamsOutboxProperty +} + +// WithFollowing represents an activity with ActivityStreamsFollowingProperty +type WithFollowing interface { + GetActivityStreamsFollowing() vocab.ActivityStreamsFollowingProperty +} + +// WithFollowers represents an activity with ActivityStreamsFollowersProperty +type WithFollowers interface { + GetActivityStreamsFollowers() vocab.ActivityStreamsFollowersProperty +} + +// WithFeatured represents an activity with TootFeaturedProperty +type WithFeatured interface { + GetTootFeatured() vocab.TootFeaturedProperty +} + +// WithAttributedTo represents an activity with ActivityStreamsAttributedToProperty +type WithAttributedTo interface { + GetActivityStreamsAttributedTo() vocab.ActivityStreamsAttributedToProperty +} + +// WithAttachment represents an activity with ActivityStreamsAttachmentProperty +type WithAttachment interface { + GetActivityStreamsAttachment() vocab.ActivityStreamsAttachmentProperty +} + +// WithTo represents an activity with ActivityStreamsToProperty +type WithTo interface { + GetActivityStreamsTo() vocab.ActivityStreamsToProperty +} + +// WithInReplyTo represents an activity with ActivityStreamsInReplyToProperty +type WithInReplyTo interface { + GetActivityStreamsInReplyTo() vocab.ActivityStreamsInReplyToProperty +} + +// WithCC represents an activity with ActivityStreamsCcProperty +type WithCC interface { + GetActivityStreamsCc() vocab.ActivityStreamsCcProperty +} + +// WithSensitive ... +type WithSensitive interface { + // TODO +} + +// WithConversation ... +type WithConversation interface { + // TODO +} + +// WithContent represents an activity with ActivityStreamsContentProperty +type WithContent interface { + GetActivityStreamsContent() vocab.ActivityStreamsContentProperty +} + +// WithPublished represents an activity with ActivityStreamsPublishedProperty +type WithPublished interface { + GetActivityStreamsPublished() vocab.ActivityStreamsPublishedProperty +} + +// WithTag represents an activity with ActivityStreamsTagProperty +type WithTag interface { + GetActivityStreamsTag() vocab.ActivityStreamsTagProperty +} + +// WithReplies represents an activity with ActivityStreamsRepliesProperty +type WithReplies interface { + GetActivityStreamsReplies() vocab.ActivityStreamsRepliesProperty +} + +// WithMediaType represents an activity with ActivityStreamsMediaTypeProperty +type WithMediaType interface { + GetActivityStreamsMediaType() vocab.ActivityStreamsMediaTypeProperty +} + +// type withBlurhash interface { +// GetTootBlurhashProperty() vocab.TootBlurhashProperty +// } + +// type withFocalPoint interface { +// // TODO +// } + +// WithHref represents an activity with ActivityStreamsHrefProperty +type WithHref interface { + GetActivityStreamsHref() vocab.ActivityStreamsHrefProperty +} + +// WithUpdated represents an activity with ActivityStreamsUpdatedProperty +type WithUpdated interface { + GetActivityStreamsUpdated() vocab.ActivityStreamsUpdatedProperty +} + +// WithActor represents an activity with ActivityStreamsActorProperty +type WithActor interface { + GetActivityStreamsActor() vocab.ActivityStreamsActorProperty +} + +// WithObject represents an activity with ActivityStreamsObjectProperty +type WithObject interface { + GetActivityStreamsObject() vocab.ActivityStreamsObjectProperty +} + +// WithNext represents an activity with ActivityStreamsNextProperty +type WithNext interface { + GetActivityStreamsNext() vocab.ActivityStreamsNextProperty +} + +// WithPartOf represents an activity with ActivityStreamsPartOfProperty +type WithPartOf interface { + GetActivityStreamsPartOf() vocab.ActivityStreamsPartOfProperty +} + +// WithItems represents an activity with ActivityStreamsItemsProperty +type WithItems interface { + GetActivityStreamsItems() vocab.ActivityStreamsItemsProperty +} diff --git a/internal/api/client/account/accountupdate_test.go b/internal/api/client/account/accountupdate_test.go index 341b865ff..349429625 100644 --- a/internal/api/client/account/accountupdate_test.go +++ b/internal/api/client/account/accountupdate_test.go @@ -53,10 +53,10 @@ func (suite *AccountUpdateTestSuite) SetupTest() { suite.db = testrig.NewTestDB() suite.storage = testrig.NewTestStorage() suite.log = testrig.NewTestLog() - suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil)), suite.storage) + suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage) suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator) suite.accountModule = account.New(suite.config, suite.processor, suite.log).(*account.Module) - testrig.StandardDBSetup(suite.db) + testrig.StandardDBSetup(suite.db, nil) testrig.StandardStorageSetup(suite.storage, "../../../../testrig/media") } @@ -80,6 +80,8 @@ func (suite *AccountUpdateTestSuite) TestAccountUpdateCredentialsPATCHHandler() ctx, _ := gin.CreateTestContext(recorder) ctx.Set(oauth.SessionAuthorizedAccount, suite.testAccounts["local_account_1"]) ctx.Set(oauth.SessionAuthorizedToken, oauth.TokenToOauthToken(suite.testTokens["local_account_1"])) + ctx.Set(oauth.SessionAuthorizedApplication, suite.testApplications["application_1"]) + ctx.Set(oauth.SessionAuthorizedUser, suite.testUsers["local_account_1"]) ctx.Request = httptest.NewRequest(http.MethodPatch, fmt.Sprintf("http://localhost:8080/%s", account.UpdateCredentialsPath), bytes.NewReader(requestBody.Bytes())) // the endpoint we're hitting ctx.Request.Header.Set("Content-Type", w.FormDataContentType()) suite.accountModule.AccountUpdateCredentialsPATCHHandler(ctx) diff --git a/internal/api/client/fileserver/servefile_test.go b/internal/api/client/fileserver/servefile_test.go index cb503facb..4eec3bae5 100644 --- a/internal/api/client/fileserver/servefile_test.go +++ b/internal/api/client/fileserver/servefile_test.go @@ -78,7 +78,7 @@ func (suite *ServeFileTestSuite) SetupSuite() { suite.db = testrig.NewTestDB() suite.log = testrig.NewTestLog() suite.storage = testrig.NewTestStorage() - suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil)), suite.storage) + suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage) suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator) suite.tc = testrig.NewTestTypeConverter(suite.db) suite.mediaHandler = testrig.NewTestMediaHandler(suite.db, suite.storage) @@ -95,7 +95,7 @@ func (suite *ServeFileTestSuite) TearDownSuite() { } func (suite *ServeFileTestSuite) SetupTest() { - testrig.StandardDBSetup(suite.db) + testrig.StandardDBSetup(suite.db, nil) testrig.StandardStorageSetup(suite.storage, "../../../../testrig/media") suite.testTokens = testrig.NewTestTokens() suite.testClients = testrig.NewTestClients() diff --git a/internal/api/client/media/mediacreate_test.go b/internal/api/client/media/mediacreate_test.go index 89a77a729..a61a36324 100644 --- a/internal/api/client/media/mediacreate_test.go +++ b/internal/api/client/media/mediacreate_test.go @@ -84,7 +84,7 @@ func (suite *MediaCreateTestSuite) SetupSuite() { suite.tc = testrig.NewTestTypeConverter(suite.db) suite.mediaHandler = testrig.NewTestMediaHandler(suite.db, suite.storage) suite.oauthServer = testrig.NewTestOauthServer(suite.db) - suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil)), suite.storage) + suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage) suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator) // setup module being tested @@ -98,7 +98,7 @@ func (suite *MediaCreateTestSuite) TearDownSuite() { } func (suite *MediaCreateTestSuite) SetupTest() { - testrig.StandardDBSetup(suite.db) + testrig.StandardDBSetup(suite.db, nil) testrig.StandardStorageSetup(suite.storage, "../../../../testrig/media") suite.testTokens = testrig.NewTestTokens() suite.testClients = testrig.NewTestClients() diff --git a/internal/api/client/status/statusboost_test.go b/internal/api/client/status/statusboost_test.go index 9400aeddc..fbe267fac 100644 --- a/internal/api/client/status/statusboost_test.go +++ b/internal/api/client/status/statusboost_test.go @@ -52,10 +52,10 @@ func (suite *StatusBoostTestSuite) SetupTest() { suite.db = testrig.NewTestDB() suite.storage = testrig.NewTestStorage() suite.log = testrig.NewTestLog() - suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil)), suite.storage) + suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage) suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator) suite.statusModule = status.New(suite.config, suite.processor, suite.log).(*status.Module) - testrig.StandardDBSetup(suite.db) + testrig.StandardDBSetup(suite.db, nil) testrig.StandardStorageSetup(suite.storage, "../../../../testrig/media") } diff --git a/internal/api/client/status/statuscreate_test.go b/internal/api/client/status/statuscreate_test.go index dd4a4386b..603432724 100644 --- a/internal/api/client/status/statuscreate_test.go +++ b/internal/api/client/status/statuscreate_test.go @@ -58,10 +58,10 @@ func (suite *StatusCreateTestSuite) SetupTest() { suite.storage = testrig.NewTestStorage() suite.log = testrig.NewTestLog() suite.tc = testrig.NewTestTypeConverter(suite.db) - suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil)), suite.storage) + suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage) suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator) suite.statusModule = status.New(suite.config, suite.processor, suite.log).(*status.Module) - testrig.StandardDBSetup(suite.db) + testrig.StandardDBSetup(suite.db, nil) testrig.StandardStorageSetup(suite.storage, "../../../../testrig/media") } diff --git a/internal/api/client/status/statusfave_test.go b/internal/api/client/status/statusfave_test.go index b1cafc2fb..0f44b5e90 100644 --- a/internal/api/client/status/statusfave_test.go +++ b/internal/api/client/status/statusfave_test.go @@ -55,10 +55,10 @@ func (suite *StatusFaveTestSuite) SetupTest() { suite.db = testrig.NewTestDB() suite.storage = testrig.NewTestStorage() suite.log = testrig.NewTestLog() - suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil)), suite.storage) + suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage) suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator) suite.statusModule = status.New(suite.config, suite.processor, suite.log).(*status.Module) - testrig.StandardDBSetup(suite.db) + testrig.StandardDBSetup(suite.db, nil) testrig.StandardStorageSetup(suite.storage, "../../../../testrig/media") } diff --git a/internal/api/client/status/statusfavedby_test.go b/internal/api/client/status/statusfavedby_test.go index b6e1591e0..22a549b30 100644 --- a/internal/api/client/status/statusfavedby_test.go +++ b/internal/api/client/status/statusfavedby_test.go @@ -55,10 +55,10 @@ func (suite *StatusFavedByTestSuite) SetupTest() { suite.db = testrig.NewTestDB() suite.storage = testrig.NewTestStorage() suite.log = testrig.NewTestLog() - suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil)), suite.storage) + suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage) suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator) suite.statusModule = status.New(suite.config, suite.processor, suite.log).(*status.Module) - testrig.StandardDBSetup(suite.db) + testrig.StandardDBSetup(suite.db, nil) testrig.StandardStorageSetup(suite.storage, "../../../../testrig/media") } diff --git a/internal/api/client/status/statusget_test.go b/internal/api/client/status/statusget_test.go index 1bbf48a91..1c700aaa5 100644 --- a/internal/api/client/status/statusget_test.go +++ b/internal/api/client/status/statusget_test.go @@ -45,10 +45,10 @@ func (suite *StatusGetTestSuite) SetupTest() { suite.db = testrig.NewTestDB() suite.storage = testrig.NewTestStorage() suite.log = testrig.NewTestLog() - suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil)), suite.storage) + suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage) suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator) suite.statusModule = status.New(suite.config, suite.processor, suite.log).(*status.Module) - testrig.StandardDBSetup(suite.db) + testrig.StandardDBSetup(suite.db, nil) testrig.StandardStorageSetup(suite.storage, "../../../../testrig/media") } diff --git a/internal/api/client/status/statusunfave_test.go b/internal/api/client/status/statusunfave_test.go index 36144c5ce..a5f267f4c 100644 --- a/internal/api/client/status/statusunfave_test.go +++ b/internal/api/client/status/statusunfave_test.go @@ -55,10 +55,10 @@ func (suite *StatusUnfaveTestSuite) SetupTest() { suite.db = testrig.NewTestDB() suite.storage = testrig.NewTestStorage() suite.log = testrig.NewTestLog() - suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil)), suite.storage) + suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage) suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator) suite.statusModule = status.New(suite.config, suite.processor, suite.log).(*status.Module) - testrig.StandardDBSetup(suite.db) + testrig.StandardDBSetup(suite.db, nil) testrig.StandardStorageSetup(suite.storage, "../../../../testrig/media") } diff --git a/internal/api/s2s/user/repliesget.go b/internal/api/s2s/user/repliesget.go new file mode 100644 index 000000000..951cc428c --- /dev/null +++ b/internal/api/s2s/user/repliesget.go @@ -0,0 +1,186 @@ +package user + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "strconv" + + "github.com/gin-gonic/gin" + "github.com/sirupsen/logrus" + "github.com/superseriousbusiness/gotosocial/internal/util" +) + +// StatusRepliesGETHandler swagger:operation GET /users/{username}/statuses/{status}/replies s2sRepliesGet +// +// Get the replies collection for a status. +// +// Note that the response will be a Collection with a page as `first`, as shown below, if `page` is `false`. +// +// If `page` is `true`, then the response will be a single `CollectionPage` without the wrapping `Collection`. +// +// HTTP signature is required on the request. +// +// --- +// tags: +// - s2s/federation +// +// produces: +// - application/activity+json +// +// parameters: +// - name: username +// type: string +// description: Username of the account. +// in: path +// required: true +// - name: status +// type: string +// description: ID of the status. +// in: path +// required: true +// - name: page +// type: boolean +// description: Return response as a CollectionPage. +// in: query +// default: false +// - name: only_other_accounts +// type: boolean +// description: Return replies only from accounts other than the status owner. +// in: query +// default: false +// - name: min_id +// type: string +// description: Minimum ID of the next status, used for paging. +// in: query +// +// responses: +// '200': +// in: body +// schema: +// "$ref": "#/definitions/swaggerStatusRepliesCollection" +// '400': +// description: bad request +// '401': +// description: unauthorized +// '403': +// description: forbidden +// '404': +// description: not found +func (m *Module) StatusRepliesGETHandler(c *gin.Context) { + l := m.log.WithFields(logrus.Fields{ + "func": "StatusRepliesGETHandler", + "url": c.Request.RequestURI, + }) + + requestedUsername := c.Param(UsernameKey) + if requestedUsername == "" { + c.JSON(http.StatusBadRequest, gin.H{"error": "no username specified in request"}) + return + } + + requestedStatusID := c.Param(StatusIDKey) + if requestedStatusID == "" { + c.JSON(http.StatusBadRequest, gin.H{"error": "no status id specified in request"}) + return + } + + page := false + pageString := c.Query(PageKey) + if pageString != "" { + i, err := strconv.ParseBool(pageString) + if err != nil { + l.Debugf("error parsing page string: %s", err) + c.JSON(http.StatusBadRequest, gin.H{"error": "couldn't parse page query param"}) + return + } + page = i + } + + onlyOtherAccounts := false + onlyOtherAccountsString := c.Query(OnlyOtherAccountsKey) + if onlyOtherAccountsString != "" { + i, err := strconv.ParseBool(onlyOtherAccountsString) + if err != nil { + l.Debugf("error parsing only_other_accounts string: %s", err) + c.JSON(http.StatusBadRequest, gin.H{"error": "couldn't parse only_other_accounts query param"}) + return + } + onlyOtherAccounts = i + } + + minID := "" + minIDString := c.Query(MinIDKey) + if minIDString != "" { + minID = minIDString + } + + // make sure this actually an AP request + format := c.NegotiateFormat(ActivityPubAcceptHeaders...) + if format == "" { + c.JSON(http.StatusNotAcceptable, gin.H{"error": "could not negotiate format with given Accept header(s)"}) + return + } + l.Tracef("negotiated format: %s", format) + + // transfer the signature verifier from the gin context to the request context + ctx := c.Request.Context() + verifier, signed := c.Get(string(util.APRequestingPublicKeyVerifier)) + if signed { + ctx = context.WithValue(ctx, util.APRequestingPublicKeyVerifier, verifier) + } + + replies, err := m.processor.GetFediStatusReplies(ctx, requestedUsername, requestedStatusID, page, onlyOtherAccounts, minID, c.Request.URL) + if err != nil { + l.Info(err.Error()) + c.JSON(err.Code(), gin.H{"error": err.Safe()}) + return + } + + b, mErr := json.Marshal(replies) + if mErr != nil { + err := fmt.Errorf("could not marshal json: %s", mErr) + l.Error(err) + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + + c.Data(http.StatusOK, format, b) +} + +// SwaggerStatusRepliesCollection represents a response to GET /users/{username}/statuses/{status}/replies. +// swagger:model swaggerStatusRepliesCollection +type SwaggerStatusRepliesCollection struct { + // ActivityStreams context. + // example: https://www.w3.org/ns/activitystreams + Context string `json:"@context"` + // ActivityStreams ID. + // example: https://example.org/users/some_user/statuses/106717595988259568/replies + ID string `json:"id"` + // ActivityStreams type. + // example: Collection + Type string `json:"type"` + // ActivityStreams first property. + First SwaggerStatusRepliesCollectionPage `json:"first"` +} + +// SwaggerStatusRepliesCollectionPage represents one page of a collection. +// swagger:model swaggerStatusRepliesCollectionPage +type SwaggerStatusRepliesCollectionPage struct { + // ActivityStreams ID. + // example: https://example.org/users/some_user/statuses/106717595988259568/replies?page=true + ID string `json:"id"` + // ActivityStreams type. + // example: CollectionPage + Type string `json:"type"` + // Link to the next page. + // example: https://example.org/users/some_user/statuses/106717595988259568/replies?only_other_accounts=true&page=true + Next string `json:"next"` + // Collection this page belongs to. + // example: https://example.org/users/some_user/statuses/106717595988259568/replies + PartOf string `json:"partOf"` + // Items on this page. + // example: ["https://example.org/users/some_other_user/statuses/086417595981111564", "https://another.example.com/users/another_user/statuses/01FCN8XDV3YG7B4R42QA6YQZ9R"] + Items []string `json:"items"` +} diff --git a/internal/api/s2s/user/repliesget_test.go b/internal/api/s2s/user/repliesget_test.go new file mode 100644 index 000000000..75edbc882 --- /dev/null +++ b/internal/api/s2s/user/repliesget_test.go @@ -0,0 +1,241 @@ +package user_test + +import ( + "context" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "net/http/httptest" + "testing" + + "github.com/gin-gonic/gin" + "github.com/go-fed/activity/streams" + "github.com/go-fed/activity/streams/vocab" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/api/s2s/user" + "github.com/superseriousbusiness/gotosocial/internal/api/security" + "github.com/superseriousbusiness/gotosocial/testrig" +) + +type RepliesGetTestSuite struct { + UserStandardTestSuite +} + +func (suite *RepliesGetTestSuite) SetupSuite() { + suite.testTokens = testrig.NewTestTokens() + suite.testClients = testrig.NewTestClients() + suite.testApplications = testrig.NewTestApplications() + suite.testUsers = testrig.NewTestUsers() + suite.testAccounts = testrig.NewTestAccounts() + suite.testAttachments = testrig.NewTestAttachments() + suite.testStatuses = testrig.NewTestStatuses() +} + +func (suite *RepliesGetTestSuite) SetupTest() { + suite.config = testrig.NewTestConfig() + suite.db = testrig.NewTestDB() + suite.tc = testrig.NewTestTypeConverter(suite.db) + suite.storage = testrig.NewTestStorage() + suite.log = testrig.NewTestLog() + suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage) + suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator) + suite.userModule = user.New(suite.config, suite.processor, suite.log).(*user.Module) + suite.securityModule = security.New(suite.config, suite.db, suite.log).(*security.Module) + testrig.StandardDBSetup(suite.db, suite.testAccounts) + testrig.StandardStorageSetup(suite.storage, "../../../../testrig/media") +} + +func (suite *RepliesGetTestSuite) TearDownTest() { + testrig.StandardDBTeardown(suite.db) + testrig.StandardStorageTeardown(suite.storage) +} + +func (suite *RepliesGetTestSuite) TestGetReplies() { + // the dereference we're gonna use + derefRequests := testrig.NewTestDereferenceRequests(suite.testAccounts) + signedRequest := derefRequests["foss_satan_dereference_local_account_1_status_1_replies"] + targetAccount := suite.testAccounts["local_account_1"] + targetStatus := suite.testStatuses["local_account_1_status_1"] + + tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db) + federator := testrig.NewTestFederator(suite.db, tc, suite.storage) + processor := testrig.NewTestProcessor(suite.db, suite.storage, federator) + userModule := user.New(suite.config, processor, suite.log).(*user.Module) + + // setup request + recorder := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(recorder) + ctx.Request = httptest.NewRequest(http.MethodGet, targetStatus.URI+"/replies", nil) // the endpoint we're hitting + ctx.Request.Header.Set("Signature", signedRequest.SignatureHeader) + ctx.Request.Header.Set("Date", signedRequest.DateHeader) + + // we need to pass the context through signature check first to set appropriate values on it + suite.securityModule.SignatureCheck(ctx) + + // normally the router would populate these params from the path values, + // but because we're calling the function directly, we need to set them manually. + ctx.Params = gin.Params{ + gin.Param{ + Key: user.UsernameKey, + Value: targetAccount.Username, + }, + gin.Param{ + Key: user.StatusIDKey, + Value: targetStatus.ID, + }, + } + + // trigger the function being tested + userModule.StatusRepliesGETHandler(ctx) + + // check response + suite.EqualValues(http.StatusOK, recorder.Code) + + result := recorder.Result() + defer result.Body.Close() + b, err := ioutil.ReadAll(result.Body) + assert.NoError(suite.T(), err) + assert.Equal(suite.T(), `{"@context":"https://www.w3.org/ns/activitystreams","first":{"id":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies?page=true","next":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies?only_other_accounts=false\u0026page=true","partOf":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies","type":"CollectionPage"},"id":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies","type":"Collection"}`, string(b)) + + // should be a Collection + m := make(map[string]interface{}) + err = json.Unmarshal(b, &m) + assert.NoError(suite.T(), err) + + t, err := streams.ToType(context.Background(), m) + assert.NoError(suite.T(), err) + + _, ok := t.(vocab.ActivityStreamsCollection) + assert.True(suite.T(), ok) +} + +func (suite *RepliesGetTestSuite) TestGetRepliesNext() { + // the dereference we're gonna use + derefRequests := testrig.NewTestDereferenceRequests(suite.testAccounts) + signedRequest := derefRequests["foss_satan_dereference_local_account_1_status_1_replies_next"] + targetAccount := suite.testAccounts["local_account_1"] + targetStatus := suite.testStatuses["local_account_1_status_1"] + + tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db) + federator := testrig.NewTestFederator(suite.db, tc, suite.storage) + processor := testrig.NewTestProcessor(suite.db, suite.storage, federator) + userModule := user.New(suite.config, processor, suite.log).(*user.Module) + + // setup request + recorder := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(recorder) + ctx.Request = httptest.NewRequest(http.MethodGet, targetStatus.URI+"/replies?only_other_accounts=false&page=true", nil) // the endpoint we're hitting + ctx.Request.Header.Set("Signature", signedRequest.SignatureHeader) + ctx.Request.Header.Set("Date", signedRequest.DateHeader) + + // we need to pass the context through signature check first to set appropriate values on it + suite.securityModule.SignatureCheck(ctx) + + // normally the router would populate these params from the path values, + // but because we're calling the function directly, we need to set them manually. + ctx.Params = gin.Params{ + gin.Param{ + Key: user.UsernameKey, + Value: targetAccount.Username, + }, + gin.Param{ + Key: user.StatusIDKey, + Value: targetStatus.ID, + }, + } + + // trigger the function being tested + userModule.StatusRepliesGETHandler(ctx) + + // check response + suite.EqualValues(http.StatusOK, recorder.Code) + + result := recorder.Result() + defer result.Body.Close() + b, err := ioutil.ReadAll(result.Body) + assert.NoError(suite.T(), err) + + assert.Equal(suite.T(), `{"@context":"https://www.w3.org/ns/activitystreams","id":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies?page=true\u0026only_other_accounts=false","items":"http://localhost:8080/users/1happyturtle/statuses/01FCQSQ667XHJ9AV9T27SJJSX5","next":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies?only_other_accounts=false\u0026page=true\u0026min_id=01FCQSQ667XHJ9AV9T27SJJSX5","partOf":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies","type":"CollectionPage"}`, string(b)) + + // should be a Collection + m := make(map[string]interface{}) + err = json.Unmarshal(b, &m) + assert.NoError(suite.T(), err) + + t, err := streams.ToType(context.Background(), m) + assert.NoError(suite.T(), err) + + page, ok := t.(vocab.ActivityStreamsCollectionPage) + assert.True(suite.T(), ok) + + assert.Equal(suite.T(), page.GetActivityStreamsItems().Len(), 1) +} + +func (suite *RepliesGetTestSuite) TestGetRepliesLast() { + // the dereference we're gonna use + derefRequests := testrig.NewTestDereferenceRequests(suite.testAccounts) + signedRequest := derefRequests["foss_satan_dereference_local_account_1_status_1_replies_last"] + targetAccount := suite.testAccounts["local_account_1"] + targetStatus := suite.testStatuses["local_account_1_status_1"] + + tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db) + federator := testrig.NewTestFederator(suite.db, tc, suite.storage) + processor := testrig.NewTestProcessor(suite.db, suite.storage, federator) + userModule := user.New(suite.config, processor, suite.log).(*user.Module) + + // setup request + recorder := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(recorder) + ctx.Request = httptest.NewRequest(http.MethodGet, targetStatus.URI+"/replies?only_other_accounts=false&page=true&min_id=01FCQSQ667XHJ9AV9T27SJJSX5", nil) // the endpoint we're hitting + ctx.Request.Header.Set("Signature", signedRequest.SignatureHeader) + ctx.Request.Header.Set("Date", signedRequest.DateHeader) + + // we need to pass the context through signature check first to set appropriate values on it + suite.securityModule.SignatureCheck(ctx) + + // normally the router would populate these params from the path values, + // but because we're calling the function directly, we need to set them manually. + ctx.Params = gin.Params{ + gin.Param{ + Key: user.UsernameKey, + Value: targetAccount.Username, + }, + gin.Param{ + Key: user.StatusIDKey, + Value: targetStatus.ID, + }, + } + + // trigger the function being tested + userModule.StatusRepliesGETHandler(ctx) + + // check response + suite.EqualValues(http.StatusOK, recorder.Code) + + result := recorder.Result() + defer result.Body.Close() + b, err := ioutil.ReadAll(result.Body) + assert.NoError(suite.T(), err) + + fmt.Println(string(b)) + assert.Equal(suite.T(), `{"@context":"https://www.w3.org/ns/activitystreams","id":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies?page=true\u0026only_other_accounts=false\u0026min_id=01FCQSQ667XHJ9AV9T27SJJSX5","items":[],"next":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies?only_other_accounts=false\u0026page=true","partOf":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies","type":"CollectionPage"}`, string(b)) + + // should be a Collection + m := make(map[string]interface{}) + err = json.Unmarshal(b, &m) + assert.NoError(suite.T(), err) + + t, err := streams.ToType(context.Background(), m) + assert.NoError(suite.T(), err) + + page, ok := t.(vocab.ActivityStreamsCollectionPage) + assert.True(suite.T(), ok) + + assert.Equal(suite.T(), page.GetActivityStreamsItems().Len(), 0) +} + +func TestRepliesGetTestSuite(t *testing.T) { + suite.Run(t, new(RepliesGetTestSuite)) +} diff --git a/internal/api/s2s/user/user.go b/internal/api/s2s/user/user.go index 0cb8e1e90..b5ff9a699 100644 --- a/internal/api/s2s/user/user.go +++ b/internal/api/s2s/user/user.go @@ -34,6 +34,13 @@ const ( UsernameKey = "username" // StatusIDKey is for status IDs StatusIDKey = "status" + // OnlyOtherAccountsKey is for filtering status responses. + OnlyOtherAccountsKey = "only_other_accounts" + // MinIDKey is for filtering status responses. + MinIDKey = "min_id" + // PageKey is for filtering status responses. + PageKey = "page" + // UsersBasePath is the base path for serving information about Users eg https://example.org/users UsersBasePath = "/" + util.UsersPath // UsersBasePathWithUsername is just the users base path with the Username key in it. @@ -50,6 +57,8 @@ const ( UsersFollowingPath = UsersBasePathWithUsername + "/" + util.FollowingPath // UsersStatusPath is for serving GET requests to a particular status by a user, with the given username key and status ID UsersStatusPath = UsersBasePathWithUsername + "/" + util.StatusesPath + "/:" + StatusIDKey + // UsersStatusRepliesPath is for serving the replies collection of a status. + UsersStatusRepliesPath = UsersStatusPath + "/replies" ) // ActivityPubAcceptHeaders represents the Accept headers mentioned here: @@ -83,5 +92,6 @@ func (m *Module) Route(s router.Router) error { s.AttachHandler(http.MethodGet, UsersFollowingPath, m.FollowingGETHandler) s.AttachHandler(http.MethodGet, UsersStatusPath, m.StatusGETHandler) s.AttachHandler(http.MethodGet, UsersPublicKeyPath, m.PublicKeyGETHandler) + s.AttachHandler(http.MethodGet, UsersStatusRepliesPath, m.StatusRepliesGETHandler) return nil } diff --git a/internal/api/s2s/user/user_test.go b/internal/api/s2s/user/user_test.go index 91d1ea32d..71d4395eb 100644 --- a/internal/api/s2s/user/user_test.go +++ b/internal/api/s2s/user/user_test.go @@ -4,6 +4,7 @@ import ( "github.com/sirupsen/logrus" "github.com/stretchr/testify/suite" "github.com/superseriousbusiness/gotosocial/internal/api/s2s/user" + "github.com/superseriousbusiness/gotosocial/internal/api/security" "github.com/superseriousbusiness/gotosocial/internal/blob" "github.com/superseriousbusiness/gotosocial/internal/config" "github.com/superseriousbusiness/gotosocial/internal/db" @@ -18,13 +19,14 @@ import ( type UserStandardTestSuite struct { // standard suite interfaces suite.Suite - config *config.Config - db db.DB - log *logrus.Logger - tc typeutils.TypeConverter - federator federation.Federator - processor processing.Processor - storage blob.Storage + config *config.Config + db db.DB + log *logrus.Logger + tc typeutils.TypeConverter + federator federation.Federator + processor processing.Processor + storage blob.Storage + securityModule *security.Module // standard suite models testTokens map[string]*oauth.Token diff --git a/internal/api/s2s/user/userget_test.go b/internal/api/s2s/user/userget_test.go index d20148802..ab0015c57 100644 --- a/internal/api/s2s/user/userget_test.go +++ b/internal/api/s2s/user/userget_test.go @@ -1,16 +1,11 @@ package user_test import ( - "bytes" "context" - "crypto/x509" "encoding/json" - "encoding/pem" - "fmt" "io/ioutil" "net/http" "net/http/httptest" - "strings" "testing" "github.com/gin-gonic/gin" @@ -19,6 +14,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" "github.com/superseriousbusiness/gotosocial/internal/api/s2s/user" + "github.com/superseriousbusiness/gotosocial/internal/api/security" "github.com/superseriousbusiness/gotosocial/testrig" ) @@ -42,10 +38,11 @@ func (suite *UserGetTestSuite) SetupTest() { suite.tc = testrig.NewTestTypeConverter(suite.db) suite.storage = testrig.NewTestStorage() suite.log = testrig.NewTestLog() - suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil)), suite.storage) + suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage) suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator) suite.userModule = user.New(suite.config, suite.processor, suite.log).(*user.Module) - testrig.StandardDBSetup(suite.db) + suite.securityModule = security.New(suite.config, suite.db, suite.log).(*security.Module) + testrig.StandardDBSetup(suite.db, suite.testAccounts) testrig.StandardStorageSetup(suite.storage, "../../../../testrig/media") } @@ -56,48 +53,11 @@ func (suite *UserGetTestSuite) TearDownTest() { func (suite *UserGetTestSuite) TestGetUser() { // the dereference we're gonna use - signedRequest := testrig.NewTestDereferenceRequests(suite.testAccounts)["foss_satan_dereference_zork"] - - requestingAccount := suite.testAccounts["remote_account_1"] + derefRequests := testrig.NewTestDereferenceRequests(suite.testAccounts) + signedRequest := derefRequests["foss_satan_dereference_zork"] targetAccount := suite.testAccounts["local_account_1"] - encodedPublicKey, err := x509.MarshalPKIXPublicKey(requestingAccount.PublicKey) - assert.NoError(suite.T(), err) - publicKeyBytes := pem.EncodeToMemory(&pem.Block{ - Type: "PUBLIC KEY", - Bytes: encodedPublicKey, - }) - publicKeyString := strings.ReplaceAll(string(publicKeyBytes), "\n", "\\n") - - // for this test we need the client to return the public key of the requester on the 'remote' instance - responseBodyString := fmt.Sprintf(` - { - "@context": [ - "https://www.w3.org/ns/activitystreams", - "https://w3id.org/security/v1" - ], - - "id": "%s", - "type": "Person", - "preferredUsername": "%s", - "inbox": "%s", - - "publicKey": { - "id": "%s", - "owner": "%s", - "publicKeyPem": "%s" - } - }`, requestingAccount.URI, requestingAccount.Username, requestingAccount.InboxURI, requestingAccount.PublicKeyURI, requestingAccount.URI, publicKeyString) - - // create a transport controller whose client will just return the response body string we specified above - tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(func(req *http.Request) (*http.Response, error) { - r := ioutil.NopCloser(bytes.NewReader([]byte(responseBodyString))) - return &http.Response{ - StatusCode: 200, - Body: r, - }, nil - })) - // get this transport controller embedded right in the user module we're testing + tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db) federator := testrig.NewTestFederator(suite.db, tc, suite.storage) processor := testrig.NewTestProcessor(suite.db, suite.storage, federator) userModule := user.New(suite.config, processor, suite.log).(*user.Module) @@ -105,7 +65,12 @@ func (suite *UserGetTestSuite) TestGetUser() { // setup request recorder := httptest.NewRecorder() ctx, _ := gin.CreateTestContext(recorder) - ctx.Request = httptest.NewRequest(http.MethodGet, fmt.Sprintf("http://localhost:8080%s", strings.Replace(user.UsersBasePathWithUsername, ":username", targetAccount.Username, 1)), nil) // the endpoint we're hitting + ctx.Request = httptest.NewRequest(http.MethodGet, targetAccount.URI, nil) // the endpoint we're hitting + ctx.Request.Header.Set("Signature", signedRequest.SignatureHeader) + ctx.Request.Header.Set("Date", signedRequest.DateHeader) + + // we need to pass the context through signature check first to set appropriate values on it + suite.securityModule.SignatureCheck(ctx) // normally the router would populate these params from the path values, // but because we're calling the function directly, we need to set them manually. @@ -116,11 +81,6 @@ func (suite *UserGetTestSuite) TestGetUser() { }, } - // we need these headers for the request to be validated - ctx.Request.Header.Set("Signature", signedRequest.SignatureHeader) - ctx.Request.Header.Set("Date", signedRequest.DateHeader) - ctx.Request.Header.Set("Digest", signedRequest.DigestHeader) - // trigger the function being tested userModule.UsersGETHandler(ctx) diff --git a/internal/cliactions/server/server.go b/internal/cliactions/server/server.go index 3c4f97dea..2314e2608 100644 --- a/internal/cliactions/server/server.go +++ b/internal/cliactions/server/server.go @@ -115,7 +115,7 @@ var Start cliactions.GTSAction = func(ctx context.Context, c *config.Config, log // build backend handlers mediaHandler := media.New(c, dbService, storageBackend, log) oauthServer := oauth.New(dbService, log) - transportController := transport.NewController(c, &federation.Clock{}, http.DefaultClient, log) + transportController := transport.NewController(c, dbService, &federation.Clock{}, http.DefaultClient, log) federator := federation.NewFederator(dbService, federatingDB, transportController, c, log, typeConverter, mediaHandler) processor := processing.NewProcessor(c, typeConverter, federator, oauthServer, mediaHandler, storageBackend, timelineManager, dbService, log) if err := processor.Start(); err != nil { diff --git a/internal/cliactions/testrig/testrig.go b/internal/cliactions/testrig/testrig.go index e2b97fe61..a7032825c 100644 --- a/internal/cliactions/testrig/testrig.go +++ b/internal/cliactions/testrig/testrig.go @@ -46,7 +46,7 @@ import ( var Start cliactions.GTSAction = func(ctx context.Context, _ *config.Config, log *logrus.Logger) error { c := testrig.NewTestConfig() dbService := testrig.NewTestDB() - testrig.StandardDBSetup(dbService) + testrig.StandardDBSetup(dbService, nil) router := testrig.NewTestRouter(dbService) storageBackend := testrig.NewTestStorage() testrig.StandardStorageSetup(storageBackend, "./testrig/media") @@ -59,7 +59,7 @@ var Start cliactions.GTSAction = func(ctx context.Context, _ *config.Config, log StatusCode: 200, Body: r, }, nil - })) + }), dbService) federator := testrig.NewTestFederator(dbService, transportController, storageBackend) processor := testrig.NewTestProcessor(dbService, storageBackend, federator) diff --git a/internal/db/db.go b/internal/db/db.go index c764cc716..d0b23fbc6 100644 --- a/internal/db/db.go +++ b/internal/db/db.go @@ -218,10 +218,14 @@ type DB interface { GetFaveCountForStatus(status *gtsmodel.Status) (int, error) // StatusParents get the parent statuses of a given status. - StatusParents(status *gtsmodel.Status) ([]*gtsmodel.Status, error) + // + // If onlyDirect is true, only the immediate parent will be returned. + StatusParents(status *gtsmodel.Status, onlyDirect bool) ([]*gtsmodel.Status, error) // StatusChildren gets the child statuses of a given status. - StatusChildren(status *gtsmodel.Status) ([]*gtsmodel.Status, error) + // + // If onlyDirect is true, only the immediate children will be returned. + StatusChildren(status *gtsmodel.Status, onlyDirect bool, minID string) ([]*gtsmodel.Status, error) // StatusFavedBy checks if a given status has been faved by a given account ID StatusFavedBy(status *gtsmodel.Status, accountID string) (bool, error) diff --git a/internal/db/pg/statuscontext.go b/internal/db/pg/statuscontext.go index 732485ab5..2ff1a20bb 100644 --- a/internal/db/pg/statuscontext.go +++ b/internal/db/pg/statuscontext.go @@ -25,14 +25,14 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" ) -func (ps *postgresService) StatusParents(status *gtsmodel.Status) ([]*gtsmodel.Status, error) { +func (ps *postgresService) StatusParents(status *gtsmodel.Status, onlyDirect bool) ([]*gtsmodel.Status, error) { parents := []*gtsmodel.Status{} - ps.statusParent(status, &parents) + ps.statusParent(status, &parents, onlyDirect) return parents, nil } -func (ps *postgresService) statusParent(status *gtsmodel.Status, foundStatuses *[]*gtsmodel.Status) { +func (ps *postgresService) statusParent(status *gtsmodel.Status, foundStatuses *[]*gtsmodel.Status, onlyDirect bool) { if status.InReplyToID == "" { return } @@ -42,13 +42,16 @@ func (ps *postgresService) statusParent(status *gtsmodel.Status, foundStatuses * *foundStatuses = append(*foundStatuses, parentStatus) } - ps.statusParent(parentStatus, foundStatuses) + if onlyDirect { + return + } + ps.statusParent(parentStatus, foundStatuses, false) } -func (ps *postgresService) StatusChildren(status *gtsmodel.Status) ([]*gtsmodel.Status, error) { +func (ps *postgresService) StatusChildren(status *gtsmodel.Status, onlyDirect bool, minID string) ([]*gtsmodel.Status, error) { foundStatuses := &list.List{} foundStatuses.PushFront(status) - ps.statusChildren(status, foundStatuses) + ps.statusChildren(status, foundStatuses, onlyDirect, minID) children := []*gtsmodel.Status{} for e := foundStatuses.Front(); e != nil; e = e.Next() { @@ -66,11 +69,15 @@ func (ps *postgresService) StatusChildren(status *gtsmodel.Status) ([]*gtsmodel. return children, nil } -func (ps *postgresService) statusChildren(status *gtsmodel.Status, foundStatuses *list.List) { +func (ps *postgresService) statusChildren(status *gtsmodel.Status, foundStatuses *list.List, onlyDirect bool, minID string) { immediateChildren := []*gtsmodel.Status{} - err := ps.conn.Model(&immediateChildren).Where("in_reply_to_id = ?", status.ID).Select() - if err != nil { + q := ps.conn.Model(&immediateChildren).Where("in_reply_to_id = ?", status.ID) + if minID != "" { + q = q.Where("status.id > ?", minID) + } + + if err := q.Select(); err != nil { return } @@ -88,6 +95,10 @@ func (ps *postgresService) statusChildren(status *gtsmodel.Status, foundStatuses } } - ps.statusChildren(child, foundStatuses) + // only do one loop if we only want direct children + if onlyDirect { + return + } + ps.statusChildren(child, foundStatuses, false, minID) } } diff --git a/internal/federation/authenticate.go b/internal/federation/authenticate.go index 0cb8db6dc..699691ca6 100644 --- a/internal/federation/authenticate.go +++ b/internal/federation/authenticate.go @@ -147,6 +147,7 @@ func (f *federator) AuthenticateFederatedRequest(ctx context.Context, requestedU if strings.EqualFold(requestingHost, f.config.Host) { // LOCAL ACCOUNT REQUEST // the request is coming from INSIDE THE HOUSE so skip the remote dereferencing + l.Tracef("proceeding without dereference for local public key %s", requestingPublicKeyID) if err := f.db.GetWhere([]db.Where{{Key: "public_key_uri", Value: requestingPublicKeyID.String()}}, requestingLocalAccount); err != nil { return nil, false, fmt.Errorf("couldn't get local account with public key uri %s from the database: %s", requestingPublicKeyID.String(), err) } @@ -158,6 +159,7 @@ func (f *federator) AuthenticateFederatedRequest(ctx context.Context, requestedU } else if err := f.db.GetWhere([]db.Where{{Key: "public_key_uri", Value: requestingPublicKeyID.String()}}, requestingRemoteAccount); err == nil { // REMOTE ACCOUNT REQUEST WITH KEY CACHED LOCALLY // this is a remote account and we already have the public key for it so use that + l.Tracef("proceeding without dereference for cached public key %s", requestingPublicKeyID) publicKey = requestingRemoteAccount.PublicKey pkOwnerURI, err = url.Parse(requestingRemoteAccount.URI) if err != nil { @@ -167,7 +169,8 @@ func (f *federator) AuthenticateFederatedRequest(ctx context.Context, requestedU // REMOTE ACCOUNT REQUEST WITHOUT KEY CACHED LOCALLY // the request is remote and we don't have the public key yet, // so we need to authenticate the request properly by dereferencing the remote key - transport, err := f.GetTransportForUser(requestedUsername) + l.Tracef("proceeding with dereference for uncached public key %s", requestingPublicKeyID) + transport, err := f.transportController.NewTransportForUsername(requestedUsername) if err != nil { return nil, false, fmt.Errorf("transport err: %s", err) } @@ -209,15 +212,28 @@ func (f *federator) AuthenticateFederatedRequest(ctx context.Context, requestedU } pkOwnerURI = pkOwnerProp.GetIRI() } + + // after all that, public key should be defined if publicKey == nil { return nil, false, errors.New("returned public key was empty") } // do the actual authentication here! - algo := httpsig.RSA_SHA256 // TODO: make this more robust - if err := verifier.Verify(publicKey, algo); err != nil { - return nil, false, nil + algos := []httpsig.Algorithm{ + httpsig.RSA_SHA512, + httpsig.RSA_SHA256, + httpsig.ED25519, } - return pkOwnerURI, true, nil + for _, algo := range algos { + l.Tracef("trying algo: %s", algo) + if err := verifier.Verify(publicKey, algo); err == nil { + l.Tracef("authentication for %s PASSED with algorithm %s", pkOwnerURI, algo) + return pkOwnerURI, true, nil + } + l.Tracef("authentication for %s NOT PASSED with algorithm %s: %s", pkOwnerURI, algo, err) + } + + l.Infof("authentication not passed for %s", pkOwnerURI) + return nil, false, nil } diff --git a/internal/federation/dereference.go b/internal/federation/dereference.go index b87462acd..8975d6c0c 100644 --- a/internal/federation/dereference.go +++ b/internal/federation/dereference.go @@ -1,526 +1,32 @@ package federation import ( - "context" - "encoding/json" - "errors" - "fmt" "net/url" - "github.com/go-fed/activity/streams" - "github.com/go-fed/activity/streams/vocab" - "github.com/sirupsen/logrus" - "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/ap" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" - "github.com/superseriousbusiness/gotosocial/internal/id" - "github.com/superseriousbusiness/gotosocial/internal/transport" - "github.com/superseriousbusiness/gotosocial/internal/typeutils" ) -func (f *federator) DereferenceRemoteAccount(username string, remoteAccountID *url.URL) (typeutils.Accountable, error) { - f.startHandshake(username, remoteAccountID) - defer f.stopHandshake(username, remoteAccountID) - - if blocked, err := f.blockedDomain(remoteAccountID.Host); blocked || err != nil { - return nil, fmt.Errorf("DereferenceRemoteAccount: domain %s is blocked", remoteAccountID.Host) - } - - transport, err := f.GetTransportForUser(username) - if err != nil { - return nil, fmt.Errorf("transport err: %s", err) - } - - b, err := transport.Dereference(context.Background(), remoteAccountID) - if err != nil { - return nil, fmt.Errorf("error deferencing %s: %s", remoteAccountID.String(), err) - } - - m := make(map[string]interface{}) - if err := json.Unmarshal(b, &m); err != nil { - return nil, fmt.Errorf("error unmarshalling bytes into json: %s", err) - } - - t, err := streams.ToType(context.Background(), m) - if err != nil { - return nil, fmt.Errorf("error resolving json into ap vocab type: %s", err) - } - - switch t.GetTypeName() { - case string(gtsmodel.ActivityStreamsPerson): - p, ok := t.(vocab.ActivityStreamsPerson) - if !ok { - return nil, errors.New("error resolving type as activitystreams person") - } - return p, nil - case string(gtsmodel.ActivityStreamsApplication): - p, ok := t.(vocab.ActivityStreamsApplication) - if !ok { - return nil, errors.New("error resolving type as activitystreams application") - } - return p, nil - case string(gtsmodel.ActivityStreamsService): - p, ok := t.(vocab.ActivityStreamsService) - if !ok { - return nil, errors.New("error resolving type as activitystreams service") - } - return p, nil - } - - return nil, fmt.Errorf("type name %s not supported", t.GetTypeName()) +func (f *federator) GetRemoteAccount(username string, remoteAccountID *url.URL, refresh bool) (*gtsmodel.Account, bool, error) { + return f.dereferencer.GetRemoteAccount(username, remoteAccountID, refresh) } -func (f *federator) DereferenceRemoteStatus(username string, remoteStatusID *url.URL) (typeutils.Statusable, error) { - if blocked, err := f.blockedDomain(remoteStatusID.Host); blocked || err != nil { - return nil, fmt.Errorf("DereferenceRemoteStatus: domain %s is blocked", remoteStatusID.Host) - } - - transport, err := f.GetTransportForUser(username) - if err != nil { - return nil, fmt.Errorf("transport err: %s", err) - } - - b, err := transport.Dereference(context.Background(), remoteStatusID) - if err != nil { - return nil, fmt.Errorf("error deferencing %s: %s", remoteStatusID.String(), err) - } - - m := make(map[string]interface{}) - if err := json.Unmarshal(b, &m); err != nil { - return nil, fmt.Errorf("error unmarshalling bytes into json: %s", err) - } - - t, err := streams.ToType(context.Background(), m) - if err != nil { - return nil, fmt.Errorf("error resolving json into ap vocab type: %s", err) - } - - // Article, Document, Image, Video, Note, Page, Event, Place, Mention, Profile - switch t.GetTypeName() { - case gtsmodel.ActivityStreamsArticle: - p, ok := t.(vocab.ActivityStreamsArticle) - if !ok { - return nil, errors.New("error resolving type as ActivityStreamsArticle") - } - return p, nil - case gtsmodel.ActivityStreamsDocument: - p, ok := t.(vocab.ActivityStreamsDocument) - if !ok { - return nil, errors.New("error resolving type as ActivityStreamsDocument") - } - return p, nil - case gtsmodel.ActivityStreamsImage: - p, ok := t.(vocab.ActivityStreamsImage) - if !ok { - return nil, errors.New("error resolving type as ActivityStreamsImage") - } - return p, nil - case gtsmodel.ActivityStreamsVideo: - p, ok := t.(vocab.ActivityStreamsVideo) - if !ok { - return nil, errors.New("error resolving type as ActivityStreamsVideo") - } - return p, nil - case gtsmodel.ActivityStreamsNote: - p, ok := t.(vocab.ActivityStreamsNote) - if !ok { - return nil, errors.New("error resolving type as ActivityStreamsNote") - } - return p, nil - case gtsmodel.ActivityStreamsPage: - p, ok := t.(vocab.ActivityStreamsPage) - if !ok { - return nil, errors.New("error resolving type as ActivityStreamsPage") - } - return p, nil - case gtsmodel.ActivityStreamsEvent: - p, ok := t.(vocab.ActivityStreamsEvent) - if !ok { - return nil, errors.New("error resolving type as ActivityStreamsEvent") - } - return p, nil - case gtsmodel.ActivityStreamsPlace: - p, ok := t.(vocab.ActivityStreamsPlace) - if !ok { - return nil, errors.New("error resolving type as ActivityStreamsPlace") - } - return p, nil - case gtsmodel.ActivityStreamsProfile: - p, ok := t.(vocab.ActivityStreamsProfile) - if !ok { - return nil, errors.New("error resolving type as ActivityStreamsProfile") - } - return p, nil - } - - return nil, fmt.Errorf("type name %s not supported", t.GetTypeName()) +func (f *federator) GetRemoteStatus(username string, remoteStatusID *url.URL, refresh bool) (*gtsmodel.Status, ap.Statusable, bool, error) { + return f.dereferencer.GetRemoteStatus(username, remoteStatusID, refresh) } -func (f *federator) DereferenceRemoteInstance(username string, remoteInstanceURI *url.URL) (*gtsmodel.Instance, error) { - if blocked, err := f.blockedDomain(remoteInstanceURI.Host); blocked || err != nil { - return nil, fmt.Errorf("DereferenceRemoteInstance: domain %s is blocked", remoteInstanceURI.Host) - } - - transport, err := f.GetTransportForUser(username) - if err != nil { - return nil, fmt.Errorf("transport err: %s", err) - } - - return transport.DereferenceInstance(context.Background(), remoteInstanceURI) +func (f *federator) EnrichRemoteStatus(username string, status *gtsmodel.Status) (*gtsmodel.Status, error) { + return f.dereferencer.EnrichRemoteStatus(username, status) } -// dereferenceStatusFields fetches all the information we temporarily pinned to an incoming -// federated status, back in the federating db's Create function. -// -// When a status comes in from the federation API, there are certain fields that -// haven't been dereferenced yet, because we needed to provide a snappy synchronous -// response to the caller. By the time it reaches this function though, it's being -// processed asynchronously, so we have all the time in the world to fetch the various -// bits and bobs that are attached to the status, and properly flesh it out, before we -// send the status to any timelines and notify people. -// -// Things to dereference and fetch here: -// -// 1. Media attachments. -// 2. Hashtags. -// 3. Emojis. -// 4. Mentions. -// 5. Posting account. -// 6. Replied-to-status. -// -// SIDE EFFECTS: -// This function will deference all of the above, insert them in the database as necessary, -// and attach them to the status. The status itself will not be added to the database yet, -// that's up the caller to do. -func (f *federator) DereferenceStatusFields(status *gtsmodel.Status, requestingUsername string) error { - l := f.log.WithFields(logrus.Fields{ - "func": "dereferenceStatusFields", - "status": fmt.Sprintf("%+v", status), - }) - l.Debug("entering function") - - statusURI, err := url.Parse(status.URI) - if err != nil { - return fmt.Errorf("DereferenceStatusFields: couldn't parse status URI %s: %s", status.URI, err) - } - if blocked, err := f.blockedDomain(statusURI.Host); blocked || err != nil { - return fmt.Errorf("DereferenceStatusFields: domain %s is blocked", statusURI.Host) - } - - t, err := f.GetTransportForUser(requestingUsername) - if err != nil { - return fmt.Errorf("error creating transport: %s", err) - } - - // the status should have an ID by now, but just in case it doesn't let's generate one here - // because we'll need it further down - if status.ID == "" { - newID, err := id.NewULIDFromTime(status.CreatedAt) - if err != nil { - return err - } - status.ID = newID - } - - // 1. Media attachments. - // - // At this point we should know: - // * the media type of the file we're looking for (a.File.ContentType) - // * the blurhash (a.Blurhash) - // * the file type (a.Type) - // * the remote URL (a.RemoteURL) - // This should be enough to pass along to the media processor. - attachmentIDs := []string{} - for _, a := range status.GTSMediaAttachments { - l.Debugf("dereferencing attachment: %+v", a) - - // it might have been processed elsewhere so check first if it's already in the database or not - maybeAttachment := >smodel.MediaAttachment{} - err := f.db.GetWhere([]db.Where{{Key: "remote_url", Value: a.RemoteURL}}, maybeAttachment) - if err == nil { - // we already have it in the db, dereferenced, no need to do it again - l.Debugf("attachment already exists with id %s", maybeAttachment.ID) - attachmentIDs = append(attachmentIDs, maybeAttachment.ID) - continue - } - if _, ok := err.(db.ErrNoEntries); !ok { - // we have a real error - return fmt.Errorf("error checking db for existence of attachment with remote url %s: %s", a.RemoteURL, err) - } - // it just doesn't exist yet so carry on - l.Debug("attachment doesn't exist yet, calling ProcessRemoteAttachment", a) - deferencedAttachment, err := f.mediaHandler.ProcessRemoteAttachment(t, a, status.AccountID) - if err != nil { - l.Errorf("error dereferencing status attachment: %s", err) - continue - } - l.Debugf("dereferenced attachment: %+v", deferencedAttachment) - deferencedAttachment.StatusID = status.ID - deferencedAttachment.Description = a.Description - if err := f.db.Put(deferencedAttachment); err != nil { - return fmt.Errorf("error inserting dereferenced attachment with remote url %s: %s", a.RemoteURL, err) - } - attachmentIDs = append(attachmentIDs, deferencedAttachment.ID) - } - status.Attachments = attachmentIDs - - // 2. Hashtags - - // 3. Emojis - - // 4. Mentions - // At this point, mentions should have the namestring and mentionedAccountURI set on them. - // - // We should dereference any accounts mentioned here which we don't have in our db yet, by their URI. - mentions := []string{} - for _, m := range status.GTSMentions { - if m.ID == "" { - mID, err := id.NewRandomULID() - if err != nil { - return err - } - m.ID = mID - } - - uri, err := url.Parse(m.MentionedAccountURI) - if err != nil { - l.Debugf("error parsing mentioned account uri %s: %s", m.MentionedAccountURI, err) - continue - } - - m.StatusID = status.ID - m.OriginAccountID = status.GTSAuthorAccount.ID - m.OriginAccountURI = status.GTSAuthorAccount.URI - - targetAccount := >smodel.Account{} - if err := f.db.GetWhere([]db.Where{{Key: "uri", Value: uri.String()}}, targetAccount); err != nil { - // proper error - if _, ok := err.(db.ErrNoEntries); !ok { - return fmt.Errorf("db error checking for account with uri %s", uri.String()) - } - - // we just don't have it yet, so we should go get it.... - accountable, err := f.DereferenceRemoteAccount(requestingUsername, uri) - if err != nil { - // we can't dereference it so just skip it - l.Debugf("error dereferencing remote account with uri %s: %s", uri.String(), err) - continue - } - - targetAccount, err = f.typeConverter.ASRepresentationToAccount(accountable, false) - if err != nil { - l.Debugf("error converting remote account with uri %s into gts model: %s", uri.String(), err) - continue - } - - targetAccountID, err := id.NewRandomULID() - if err != nil { - return err - } - targetAccount.ID = targetAccountID - - if err := f.db.Put(targetAccount); err != nil { - return fmt.Errorf("db error inserting account with uri %s", uri.String()) - } - } - - // by this point, we know the targetAccount exists in our database with an ID :) - m.TargetAccountID = targetAccount.ID - if err := f.db.Put(m); err != nil { - return fmt.Errorf("error creating mention: %s", err) - } - mentions = append(mentions, m.ID) - } - status.Mentions = mentions - - return nil +func (f *federator) DereferenceRemoteThread(username string, statusIRI *url.URL) error { + return f.dereferencer.DereferenceThread(username, statusIRI) } -func (f *federator) DereferenceAccountFields(account *gtsmodel.Account, requestingUsername string, refresh bool) error { - l := f.log.WithFields(logrus.Fields{ - "func": "dereferenceAccountFields", - "requestingUsername": requestingUsername, - }) - - accountURI, err := url.Parse(account.URI) - if err != nil { - return fmt.Errorf("DereferenceAccountFields: couldn't parse account URI %s: %s", account.URI, err) - } - if blocked, err := f.blockedDomain(accountURI.Host); blocked || err != nil { - return fmt.Errorf("DereferenceAccountFields: domain %s is blocked", accountURI.Host) - } - - t, err := f.GetTransportForUser(requestingUsername) - if err != nil { - return fmt.Errorf("error getting transport for user: %s", err) - } - - // fetch the header and avatar - if err := f.fetchHeaderAndAviForAccount(account, t, refresh); err != nil { - // if this doesn't work, just skip it -- we can do it later - l.Debugf("error fetching header/avi for account: %s", err) - } - - if err := f.db.UpdateByID(account.ID, account); err != nil { - return fmt.Errorf("error updating account in database: %s", err) - } - - return nil +func (f *federator) GetRemoteInstance(username string, remoteInstanceURI *url.URL) (*gtsmodel.Instance, error) { + return f.dereferencer.GetRemoteInstance(username, remoteInstanceURI) } func (f *federator) DereferenceAnnounce(announce *gtsmodel.Status, requestingUsername string) error { - if announce.GTSBoostedStatus == nil || announce.GTSBoostedStatus.URI == "" { - // we can't do anything unfortunately - return errors.New("DereferenceAnnounce: no URI to dereference") - } - - boostedStatusURI, err := url.Parse(announce.GTSBoostedStatus.URI) - if err != nil { - return fmt.Errorf("DereferenceAnnounce: couldn't parse boosted status URI %s: %s", announce.GTSBoostedStatus.URI, err) - } - if blocked, err := f.blockedDomain(boostedStatusURI.Host); blocked || err != nil { - return fmt.Errorf("DereferenceAnnounce: domain %s is blocked", boostedStatusURI.Host) - } - - // check if we already have the boosted status in the database - boostedStatus := >smodel.Status{} - err = f.db.GetWhere([]db.Where{{Key: "uri", Value: announce.GTSBoostedStatus.URI}}, boostedStatus) - if err == nil { - // nice, we already have it so we don't actually need to dereference it from remote - announce.Content = boostedStatus.Content - announce.ContentWarning = boostedStatus.ContentWarning - announce.ActivityStreamsType = boostedStatus.ActivityStreamsType - announce.Sensitive = boostedStatus.Sensitive - announce.Language = boostedStatus.Language - announce.Text = boostedStatus.Text - announce.BoostOfID = boostedStatus.ID - announce.BoostOfAccountID = boostedStatus.AccountID - announce.Visibility = boostedStatus.Visibility - announce.VisibilityAdvanced = boostedStatus.VisibilityAdvanced - announce.GTSBoostedStatus = boostedStatus - return nil - } - - // we don't have it so we need to dereference it - statusable, err := f.DereferenceRemoteStatus(requestingUsername, boostedStatusURI) - if err != nil { - return fmt.Errorf("dereferenceAnnounce: error dereferencing remote status with id %s: %s", announce.GTSBoostedStatus.URI, err) - } - - // make sure we have the author account in the db - attributedToProp := statusable.GetActivityStreamsAttributedTo() - for iter := attributedToProp.Begin(); iter != attributedToProp.End(); iter = iter.Next() { - accountURI := iter.GetIRI() - if accountURI == nil { - continue - } - - if err := f.db.GetWhere([]db.Where{{Key: "uri", Value: accountURI.String()}}, >smodel.Account{}); err == nil { - // we already have it, fine - continue - } - - // we don't have the boosted status author account yet so dereference it - accountable, err := f.DereferenceRemoteAccount(requestingUsername, accountURI) - if err != nil { - return fmt.Errorf("dereferenceAnnounce: error dereferencing remote account with id %s: %s", accountURI.String(), err) - } - account, err := f.typeConverter.ASRepresentationToAccount(accountable, false) - if err != nil { - return fmt.Errorf("dereferenceAnnounce: error converting dereferenced account with id %s into account : %s", accountURI.String(), err) - } - - accountID, err := id.NewRandomULID() - if err != nil { - return err - } - account.ID = accountID - - if err := f.db.Put(account); err != nil { - return fmt.Errorf("dereferenceAnnounce: error putting dereferenced account with id %s into database : %s", accountURI.String(), err) - } - - if err := f.DereferenceAccountFields(account, requestingUsername, false); err != nil { - return fmt.Errorf("dereferenceAnnounce: error dereferencing fields on account with id %s : %s", accountURI.String(), err) - } - } - - // now convert the statusable into something we can understand - boostedStatus, err = f.typeConverter.ASStatusToStatus(statusable) - if err != nil { - return fmt.Errorf("dereferenceAnnounce: error converting dereferenced statusable with id %s into status : %s", announce.GTSBoostedStatus.URI, err) - } - - boostedStatusID, err := id.NewULIDFromTime(boostedStatus.CreatedAt) - if err != nil { - return nil - } - boostedStatus.ID = boostedStatusID - - if err := f.db.Put(boostedStatus); err != nil { - return fmt.Errorf("dereferenceAnnounce: error putting dereferenced status with id %s into the db: %s", announce.GTSBoostedStatus.URI, err) - } - - // now dereference additional fields straight away (we're already async here so we have time) - if err := f.DereferenceStatusFields(boostedStatus, requestingUsername); err != nil { - return fmt.Errorf("dereferenceAnnounce: error dereferencing status fields for status with id %s: %s", announce.GTSBoostedStatus.URI, err) - } - - // update with the newly dereferenced fields - if err := f.db.UpdateByID(boostedStatus.ID, boostedStatus); err != nil { - return fmt.Errorf("dereferenceAnnounce: error updating dereferenced status in the db: %s", err) - } - - // we have everything we need! - announce.Content = boostedStatus.Content - announce.ContentWarning = boostedStatus.ContentWarning - announce.ActivityStreamsType = boostedStatus.ActivityStreamsType - announce.Sensitive = boostedStatus.Sensitive - announce.Language = boostedStatus.Language - announce.Text = boostedStatus.Text - announce.BoostOfID = boostedStatus.ID - announce.BoostOfAccountID = boostedStatus.AccountID - announce.Visibility = boostedStatus.Visibility - announce.VisibilityAdvanced = boostedStatus.VisibilityAdvanced - announce.GTSBoostedStatus = boostedStatus - return nil -} - -// fetchHeaderAndAviForAccount fetches the header and avatar for a remote account, using a transport -// on behalf of requestingUsername. -// -// targetAccount's AvatarMediaAttachmentID and HeaderMediaAttachmentID will be updated as necessary. -// -// SIDE EFFECTS: remote header and avatar will be stored in local storage, and the database will be updated -// to reflect the creation of these new attachments. -func (f *federator) fetchHeaderAndAviForAccount(targetAccount *gtsmodel.Account, t transport.Transport, refresh bool) error { - accountURI, err := url.Parse(targetAccount.URI) - if err != nil { - return fmt.Errorf("fetchHeaderAndAviForAccount: couldn't parse account URI %s: %s", targetAccount.URI, err) - } - if blocked, err := f.blockedDomain(accountURI.Host); blocked || err != nil { - return fmt.Errorf("fetchHeaderAndAviForAccount: domain %s is blocked", accountURI.Host) - } - - if targetAccount.AvatarRemoteURL != "" && (targetAccount.AvatarMediaAttachmentID == "" || refresh) { - a, err := f.mediaHandler.ProcessRemoteHeaderOrAvatar(t, >smodel.MediaAttachment{ - RemoteURL: targetAccount.AvatarRemoteURL, - Avatar: true, - }, targetAccount.ID) - if err != nil { - return fmt.Errorf("error processing avatar for user: %s", err) - } - targetAccount.AvatarMediaAttachmentID = a.ID - } - - if targetAccount.HeaderRemoteURL != "" && (targetAccount.HeaderMediaAttachmentID == "" || refresh) { - a, err := f.mediaHandler.ProcessRemoteHeaderOrAvatar(t, >smodel.MediaAttachment{ - RemoteURL: targetAccount.HeaderRemoteURL, - Header: true, - }, targetAccount.ID) - if err != nil { - return fmt.Errorf("error processing header for user: %s", err) - } - targetAccount.HeaderMediaAttachmentID = a.ID - } - return nil + return f.dereferencer.DereferenceAnnounce(announce, requestingUsername) } diff --git a/internal/federation/dereferencing/account.go b/internal/federation/dereferencing/account.go new file mode 100644 index 000000000..c403ec66f --- /dev/null +++ b/internal/federation/dereferencing/account.go @@ -0,0 +1,243 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package dereferencing + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/url" + + "github.com/go-fed/activity/streams" + "github.com/go-fed/activity/streams/vocab" + "github.com/sirupsen/logrus" + "github.com/superseriousbusiness/gotosocial/internal/ap" + "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/id" + "github.com/superseriousbusiness/gotosocial/internal/transport" +) + +// EnrichRemoteAccount takes an account that's already been inserted into the database in a minimal form, +// and populates it with additional fields, media, etc. +// +// EnrichRemoteAccount is mostly useful for calling after an account has been initially created by +// the federatingDB's Create function, or during the federated authorization flow. +func (d *deref) EnrichRemoteAccount(username string, account *gtsmodel.Account) (*gtsmodel.Account, error) { + if err := d.populateAccountFields(account, username, false); err != nil { + return nil, err + } + + if err := d.db.UpdateByID(account.ID, account); err != nil { + return nil, fmt.Errorf("EnrichRemoteAccount: error updating account: %s", err) + } + + return account, nil +} + +// GetRemoteAccount completely dereferences a remote account, converts it to a GtS model account, +// puts it in the database, and returns it to a caller. The boolean indicates whether the account is new +// to us or not. If we haven't seen the account before, bool will be true. If we have seen the account before, +// it will be false. +// +// Refresh indicates whether--if the account exists in our db already--it should be refreshed by calling +// the remote instance again. +// +// SIDE EFFECTS: remote account will be stored in the database, or updated if it already exists (and refresh is true). +func (d *deref) GetRemoteAccount(username string, remoteAccountID *url.URL, refresh bool) (*gtsmodel.Account, bool, error) { + new := true + + // check if we already have the account in our db + maybeAccount := >smodel.Account{} + if err := d.db.GetWhere([]db.Where{{Key: "uri", Value: remoteAccountID.String()}}, maybeAccount); err == nil { + // we've seen this account before so it's not new + new = false + + // if we're not being asked to refresh, we can just return the maybeAccount as-is and avoid doing any external calls + if !refresh { + return maybeAccount, new, nil + } + } + + accountable, err := d.dereferenceAccountable(username, remoteAccountID) + if err != nil { + return nil, new, fmt.Errorf("FullyDereferenceAccount: error dereferencing accountable: %s", err) + } + + gtsAccount, err := d.typeConverter.ASRepresentationToAccount(accountable, false) + if err != nil { + return nil, new, fmt.Errorf("FullyDereferenceAccount: error converting accountable to account: %s", err) + } + + if new { + // generate a new id since we haven't seen this account before, and do a put + ulid, err := id.NewRandomULID() + if err != nil { + return nil, new, fmt.Errorf("FullyDereferenceAccount: error generating new id for account: %s", err) + } + gtsAccount.ID = ulid + + if err := d.populateAccountFields(gtsAccount, username, refresh); err != nil { + return nil, new, fmt.Errorf("FullyDereferenceAccount: error populating further account fields: %s", err) + } + + if err := d.db.Put(gtsAccount); err != nil { + return nil, new, fmt.Errorf("FullyDereferenceAccount: error putting new account: %s", err) + } + } else { + // take the id we already have and do an update + gtsAccount.ID = maybeAccount.ID + + if err := d.populateAccountFields(gtsAccount, username, refresh); err != nil { + return nil, new, fmt.Errorf("FullyDereferenceAccount: error populating further account fields: %s", err) + } + + if err := d.db.UpdateByID(gtsAccount.ID, gtsAccount); err != nil { + return nil, new, fmt.Errorf("FullyDereferenceAccount: error updating existing account: %s", err) + } + } + + return gtsAccount, new, nil +} + +// dereferenceAccountable calls remoteAccountID with a GET request, and tries to parse whatever +// it finds as something that an account model can be constructed out of. +// +// Will work for Person, Application, or Service models. +func (d *deref) dereferenceAccountable(username string, remoteAccountID *url.URL) (ap.Accountable, error) { + d.startHandshake(username, remoteAccountID) + defer d.stopHandshake(username, remoteAccountID) + + if blocked, err := d.blockedDomain(remoteAccountID.Host); blocked || err != nil { + return nil, fmt.Errorf("DereferenceAccountable: domain %s is blocked", remoteAccountID.Host) + } + + transport, err := d.transportController.NewTransportForUsername(username) + if err != nil { + return nil, fmt.Errorf("DereferenceAccountable: transport err: %s", err) + } + + b, err := transport.Dereference(context.Background(), remoteAccountID) + if err != nil { + return nil, fmt.Errorf("DereferenceAccountable: error deferencing %s: %s", remoteAccountID.String(), err) + } + + m := make(map[string]interface{}) + if err := json.Unmarshal(b, &m); err != nil { + return nil, fmt.Errorf("DereferenceAccountable: error unmarshalling bytes into json: %s", err) + } + + t, err := streams.ToType(context.Background(), m) + if err != nil { + return nil, fmt.Errorf("DereferenceAccountable: error resolving json into ap vocab type: %s", err) + } + + switch t.GetTypeName() { + case string(gtsmodel.ActivityStreamsPerson): + p, ok := t.(vocab.ActivityStreamsPerson) + if !ok { + return nil, errors.New("DereferenceAccountable: error resolving type as activitystreams person") + } + return p, nil + case string(gtsmodel.ActivityStreamsApplication): + p, ok := t.(vocab.ActivityStreamsApplication) + if !ok { + return nil, errors.New("DereferenceAccountable: error resolving type as activitystreams application") + } + return p, nil + case string(gtsmodel.ActivityStreamsService): + p, ok := t.(vocab.ActivityStreamsService) + if !ok { + return nil, errors.New("DereferenceAccountable: error resolving type as activitystreams service") + } + return p, nil + } + + return nil, fmt.Errorf("DereferenceAccountable: type name %s not supported", t.GetTypeName()) +} + +// populateAccountFields populates any fields on the given account that weren't populated by the initial +// dereferencing. This includes things like header and avatar etc. +func (d *deref) populateAccountFields(account *gtsmodel.Account, requestingUsername string, refresh bool) error { + l := d.log.WithFields(logrus.Fields{ + "func": "PopulateAccountFields", + "requestingUsername": requestingUsername, + }) + + accountURI, err := url.Parse(account.URI) + if err != nil { + return fmt.Errorf("PopulateAccountFields: couldn't parse account URI %s: %s", account.URI, err) + } + if blocked, err := d.blockedDomain(accountURI.Host); blocked || err != nil { + return fmt.Errorf("PopulateAccountFields: domain %s is blocked", accountURI.Host) + } + + t, err := d.transportController.NewTransportForUsername(requestingUsername) + if err != nil { + return fmt.Errorf("PopulateAccountFields: error getting transport for user: %s", err) + } + + // fetch the header and avatar + if err := d.fetchHeaderAndAviForAccount(account, t, refresh); err != nil { + // if this doesn't work, just skip it -- we can do it later + l.Debugf("error fetching header/avi for account: %s", err) + } + + return nil +} + +// fetchHeaderAndAviForAccount fetches the header and avatar for a remote account, using a transport +// on behalf of requestingUsername. +// +// targetAccount's AvatarMediaAttachmentID and HeaderMediaAttachmentID will be updated as necessary. +// +// SIDE EFFECTS: remote header and avatar will be stored in local storage. +func (d *deref) fetchHeaderAndAviForAccount(targetAccount *gtsmodel.Account, t transport.Transport, refresh bool) error { + accountURI, err := url.Parse(targetAccount.URI) + if err != nil { + return fmt.Errorf("fetchHeaderAndAviForAccount: couldn't parse account URI %s: %s", targetAccount.URI, err) + } + if blocked, err := d.blockedDomain(accountURI.Host); blocked || err != nil { + return fmt.Errorf("fetchHeaderAndAviForAccount: domain %s is blocked", accountURI.Host) + } + + if targetAccount.AvatarRemoteURL != "" && (targetAccount.AvatarMediaAttachmentID == "" || refresh) { + a, err := d.mediaHandler.ProcessRemoteHeaderOrAvatar(t, >smodel.MediaAttachment{ + RemoteURL: targetAccount.AvatarRemoteURL, + Avatar: true, + }, targetAccount.ID) + if err != nil { + return fmt.Errorf("error processing avatar for user: %s", err) + } + targetAccount.AvatarMediaAttachmentID = a.ID + } + + if targetAccount.HeaderRemoteURL != "" && (targetAccount.HeaderMediaAttachmentID == "" || refresh) { + a, err := d.mediaHandler.ProcessRemoteHeaderOrAvatar(t, >smodel.MediaAttachment{ + RemoteURL: targetAccount.HeaderRemoteURL, + Header: true, + }, targetAccount.ID) + if err != nil { + return fmt.Errorf("error processing header for user: %s", err) + } + targetAccount.HeaderMediaAttachmentID = a.ID + } + return nil +} diff --git a/internal/federation/dereferencing/announce.go b/internal/federation/dereferencing/announce.go new file mode 100644 index 000000000..2522a4034 --- /dev/null +++ b/internal/federation/dereferencing/announce.go @@ -0,0 +1,65 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package dereferencing + +import ( + "errors" + "fmt" + "net/url" + + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" +) + +func (d *deref) DereferenceAnnounce(announce *gtsmodel.Status, requestingUsername string) error { + if announce.GTSBoostedStatus == nil || announce.GTSBoostedStatus.URI == "" { + // we can't do anything unfortunately + return errors.New("DereferenceAnnounce: no URI to dereference") + } + + boostedStatusURI, err := url.Parse(announce.GTSBoostedStatus.URI) + if err != nil { + return fmt.Errorf("DereferenceAnnounce: couldn't parse boosted status URI %s: %s", announce.GTSBoostedStatus.URI, err) + } + if blocked, err := d.blockedDomain(boostedStatusURI.Host); blocked || err != nil { + return fmt.Errorf("DereferenceAnnounce: domain %s is blocked", boostedStatusURI.Host) + } + + // dereference statuses in the thread of the boosted status + if err := d.DereferenceThread(requestingUsername, boostedStatusURI); err != nil { + return fmt.Errorf("DereferenceAnnounce: error dereferencing thread of boosted status: %s", err) + } + + boostedStatus, _, _, err := d.GetRemoteStatus(requestingUsername, boostedStatusURI, false) + if err != nil { + return fmt.Errorf("DereferenceAnnounce: error dereferencing remote status with id %s: %s", announce.GTSBoostedStatus.URI, err) + } + + announce.Content = boostedStatus.Content + announce.ContentWarning = boostedStatus.ContentWarning + announce.ActivityStreamsType = boostedStatus.ActivityStreamsType + announce.Sensitive = boostedStatus.Sensitive + announce.Language = boostedStatus.Language + announce.Text = boostedStatus.Text + announce.BoostOfID = boostedStatus.ID + announce.BoostOfAccountID = boostedStatus.AccountID + announce.Visibility = boostedStatus.Visibility + announce.VisibilityAdvanced = boostedStatus.VisibilityAdvanced + announce.GTSBoostedStatus = boostedStatus + return nil +} diff --git a/internal/federation/dereferencing/blocked.go b/internal/federation/dereferencing/blocked.go new file mode 100644 index 000000000..a66afbb60 --- /dev/null +++ b/internal/federation/dereferencing/blocked.go @@ -0,0 +1,41 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package dereferencing + +import ( + "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" +) + +func (d *deref) blockedDomain(host string) (bool, error) { + b := >smodel.DomainBlock{} + err := d.db.GetWhere([]db.Where{{Key: "domain", Value: host, CaseInsensitive: true}}, b) + if err == nil { + // block exists + return true, nil + } + + if _, ok := err.(db.ErrNoEntries); ok { + // there are no entries so there's no block + return false, nil + } + + // there's an actual error + return false, err +} diff --git a/internal/federation/dereferencing/collectionpage.go b/internal/federation/dereferencing/collectionpage.go new file mode 100644 index 000000000..5feadc1ad --- /dev/null +++ b/internal/federation/dereferencing/collectionpage.go @@ -0,0 +1,70 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package dereferencing + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/url" + + "github.com/go-fed/activity/streams" + "github.com/go-fed/activity/streams/vocab" + "github.com/superseriousbusiness/gotosocial/internal/ap" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" +) + +// DereferenceCollectionPage returns the activitystreams CollectionPage at the specified IRI, or an error if something goes wrong. +func (d *deref) DereferenceCollectionPage(username string, pageIRI *url.URL) (ap.CollectionPageable, error) { + if blocked, err := d.blockedDomain(pageIRI.Host); blocked || err != nil { + return nil, fmt.Errorf("DereferenceCollectionPage: domain %s is blocked", pageIRI.Host) + } + + transport, err := d.transportController.NewTransportForUsername(username) + if err != nil { + return nil, fmt.Errorf("DereferenceCollectionPage: error creating transport: %s", err) + } + + b, err := transport.Dereference(context.Background(), pageIRI) + if err != nil { + return nil, fmt.Errorf("DereferenceCollectionPage: error deferencing %s: %s", pageIRI.String(), err) + } + + m := make(map[string]interface{}) + if err := json.Unmarshal(b, &m); err != nil { + return nil, fmt.Errorf("DereferenceCollectionPage: error unmarshalling bytes into json: %s", err) + } + + t, err := streams.ToType(context.Background(), m) + if err != nil { + return nil, fmt.Errorf("DereferenceCollectionPage: error resolving json into ap vocab type: %s", err) + } + + if t.GetTypeName() != gtsmodel.ActivityStreamsCollectionPage { + return nil, fmt.Errorf("DereferenceCollectionPage: type name %s not supported", t.GetTypeName()) + } + + p, ok := t.(vocab.ActivityStreamsCollectionPage) + if !ok { + return nil, errors.New("DereferenceCollectionPage: error resolving type as activitystreams collection page") + } + + return p, nil +} diff --git a/internal/federation/dereferencing/dereferencer.go b/internal/federation/dereferencing/dereferencer.go new file mode 100644 index 000000000..03b90569a --- /dev/null +++ b/internal/federation/dereferencing/dereferencer.go @@ -0,0 +1,73 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package dereferencing + +import ( + "net/url" + "sync" + + "github.com/sirupsen/logrus" + "github.com/superseriousbusiness/gotosocial/internal/ap" + "github.com/superseriousbusiness/gotosocial/internal/config" + "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/media" + "github.com/superseriousbusiness/gotosocial/internal/transport" + "github.com/superseriousbusiness/gotosocial/internal/typeutils" +) + +// Dereferencer wraps logic and functionality for doing dereferencing of remote accounts, statuses, etc, from federated instances. +type Dereferencer interface { + GetRemoteAccount(username string, remoteAccountID *url.URL, refresh bool) (*gtsmodel.Account, bool, error) + EnrichRemoteAccount(username string, account *gtsmodel.Account) (*gtsmodel.Account, error) + + GetRemoteStatus(username string, remoteStatusID *url.URL, refresh bool) (*gtsmodel.Status, ap.Statusable, bool, error) + EnrichRemoteStatus(username string, status *gtsmodel.Status) (*gtsmodel.Status, error) + + GetRemoteInstance(username string, remoteInstanceURI *url.URL) (*gtsmodel.Instance, error) + + DereferenceAnnounce(announce *gtsmodel.Status, requestingUsername string) error + DereferenceThread(username string, statusIRI *url.URL) error + + Handshaking(username string, remoteAccountID *url.URL) bool +} + +type deref struct { + log *logrus.Logger + db db.DB + typeConverter typeutils.TypeConverter + transportController transport.Controller + mediaHandler media.Handler + config *config.Config + handshakes map[string][]*url.URL + handshakeSync *sync.Mutex // mutex to lock/unlock when checking or updating the handshakes map +} + +// NewDereferencer returns a Dereferencer initialized with the given parameters. +func NewDereferencer(config *config.Config, db db.DB, typeConverter typeutils.TypeConverter, transportController transport.Controller, mediaHandler media.Handler, log *logrus.Logger) Dereferencer { + return &deref{ + log: log, + db: db, + typeConverter: typeConverter, + transportController: transportController, + mediaHandler: mediaHandler, + config: config, + handshakeSync: &sync.Mutex{}, + } +} diff --git a/internal/federation/dereferencing/handshake.go b/internal/federation/dereferencing/handshake.go new file mode 100644 index 000000000..cda8eafd0 --- /dev/null +++ b/internal/federation/dereferencing/handshake.go @@ -0,0 +1,98 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package dereferencing + +import "net/url" + +func (d *deref) Handshaking(username string, remoteAccountID *url.URL) bool { + d.handshakeSync.Lock() + defer d.handshakeSync.Unlock() + + if d.handshakes == nil { + // handshakes isn't even initialized yet so we can't be handshaking with anyone + return false + } + + remoteIDs, ok := d.handshakes[username] + if !ok { + // user isn't handshaking with anyone, bail + return false + } + + for _, id := range remoteIDs { + if id.String() == remoteAccountID.String() { + // we are currently handshaking with the remote account, yep + return true + } + } + + // didn't find it which means we're not handshaking + return false +} + +func (d *deref) startHandshake(username string, remoteAccountID *url.URL) { + d.handshakeSync.Lock() + defer d.handshakeSync.Unlock() + + // lazily initialize handshakes + if d.handshakes == nil { + d.handshakes = make(map[string][]*url.URL) + } + + remoteIDs, ok := d.handshakes[username] + if !ok { + // there was nothing in there yet, so just add this entry and return + d.handshakes[username] = []*url.URL{remoteAccountID} + return + } + + // add the remote ID to the slice + remoteIDs = append(remoteIDs, remoteAccountID) + d.handshakes[username] = remoteIDs +} + +func (d *deref) stopHandshake(username string, remoteAccountID *url.URL) { + d.handshakeSync.Lock() + defer d.handshakeSync.Unlock() + + if d.handshakes == nil { + return + } + + remoteIDs, ok := d.handshakes[username] + if !ok { + // there was nothing in there yet anyway so just bail + return + } + + newRemoteIDs := []*url.URL{} + for _, id := range remoteIDs { + if id.String() != remoteAccountID.String() { + newRemoteIDs = append(newRemoteIDs, id) + } + } + + if len(newRemoteIDs) == 0 { + // there are no handshakes so just remove this user entry from the map and save a few bytes + delete(d.handshakes, username) + } else { + // there are still other handshakes ongoing + d.handshakes[username] = newRemoteIDs + } +} diff --git a/internal/federation/dereferencing/instance.go b/internal/federation/dereferencing/instance.go new file mode 100644 index 000000000..80f626662 --- /dev/null +++ b/internal/federation/dereferencing/instance.go @@ -0,0 +1,40 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package dereferencing + +import ( + "context" + "fmt" + "net/url" + + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" +) + +func (d *deref) GetRemoteInstance(username string, remoteInstanceURI *url.URL) (*gtsmodel.Instance, error) { + if blocked, err := d.blockedDomain(remoteInstanceURI.Host); blocked || err != nil { + return nil, fmt.Errorf("GetRemoteInstance: domain %s is blocked", remoteInstanceURI.Host) + } + + transport, err := d.transportController.NewTransportForUsername(username) + if err != nil { + return nil, fmt.Errorf("transport err: %s", err) + } + + return transport.DereferenceInstance(context.Background(), remoteInstanceURI) +} diff --git a/internal/federation/dereferencing/status.go b/internal/federation/dereferencing/status.go new file mode 100644 index 000000000..b05f6e72c --- /dev/null +++ b/internal/federation/dereferencing/status.go @@ -0,0 +1,369 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package dereferencing + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/url" + + "github.com/go-fed/activity/streams" + "github.com/go-fed/activity/streams/vocab" + "github.com/sirupsen/logrus" + "github.com/superseriousbusiness/gotosocial/internal/ap" + "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/id" +) + +// EnrichRemoteStatus takes a status that's already been inserted into the database in a minimal form, +// and populates it with additional fields, media, etc. +// +// EnrichRemoteStatus is mostly useful for calling after a status has been initially created by +// the federatingDB's Create function, but additional dereferencing is needed on it. +func (d *deref) EnrichRemoteStatus(username string, status *gtsmodel.Status) (*gtsmodel.Status, error) { + if err := d.populateStatusFields(status, username); err != nil { + return nil, err + } + + if err := d.db.UpdateByID(status.ID, status); err != nil { + return nil, fmt.Errorf("EnrichRemoteStatus: error updating status: %s", err) + } + + return status, nil +} + +// GetRemoteStatus completely dereferences a remote status, converts it to a GtS model status, +// puts it in the database, and returns it to a caller. The boolean indicates whether the status is new +// to us or not. If we haven't seen the status before, bool will be true. If we have seen the status before, +// it will be false. +// +// If refresh is true, then even if we have the status in our database already, it will be dereferenced from its +// remote representation, as will its owner. +// +// If a dereference was performed, then the function also returns the ap.Statusable representation for further processing. +// +// SIDE EFFECTS: remote status will be stored in the database, and the remote status owner will also be stored. +func (d *deref) GetRemoteStatus(username string, remoteStatusID *url.URL, refresh bool) (*gtsmodel.Status, ap.Statusable, bool, error) { + new := true + + // check if we already have the status in our db + maybeStatus := >smodel.Status{} + if err := d.db.GetWhere([]db.Where{{Key: "uri", Value: remoteStatusID.String()}}, maybeStatus); err == nil { + // we've seen this status before so it's not new + new = false + + // if we're not being asked to refresh, we can just return the maybeStatus as-is and avoid doing any external calls + if !refresh { + return maybeStatus, nil, new, nil + } + } + + statusable, err := d.dereferenceStatusable(username, remoteStatusID) + if err != nil { + return nil, statusable, new, fmt.Errorf("GetRemoteStatus: error dereferencing statusable: %s", err) + } + + accountURI, err := ap.ExtractAttributedTo(statusable) + if err != nil { + return nil, statusable, new, fmt.Errorf("GetRemoteStatus: error extracting attributedTo: %s", err) + } + + // do this so we know we have the remote account of the status in the db + _, _, err = d.GetRemoteAccount(username, accountURI, false) + if err != nil { + return nil, statusable, new, fmt.Errorf("GetRemoteStatus: couldn't derive status author: %s", err) + } + + gtsStatus, err := d.typeConverter.ASStatusToStatus(statusable) + if err != nil { + return nil, statusable, new, fmt.Errorf("GetRemoteStatus: error converting statusable to status: %s", err) + } + + if new { + ulid, err := id.NewULIDFromTime(gtsStatus.CreatedAt) + if err != nil { + return nil, statusable, new, fmt.Errorf("GetRemoteStatus: error generating new id for status: %s", err) + } + gtsStatus.ID = ulid + + if err := d.populateStatusFields(gtsStatus, username); err != nil { + return nil, statusable, new, fmt.Errorf("GetRemoteStatus: error populating status fields: %s", err) + } + + if err := d.db.Put(gtsStatus); err != nil { + return nil, statusable, new, fmt.Errorf("GetRemoteStatus: error putting new status: %s", err) + } + } else { + gtsStatus.ID = maybeStatus.ID + + if err := d.populateStatusFields(gtsStatus, username); err != nil { + return nil, statusable, new, fmt.Errorf("GetRemoteStatus: error populating status fields: %s", err) + } + + if err := d.db.UpdateByID(gtsStatus.ID, gtsStatus); err != nil { + return nil, statusable, new, fmt.Errorf("GetRemoteStatus: error updating status: %s", err) + } + } + + return gtsStatus, statusable, new, nil +} + +func (d *deref) dereferenceStatusable(username string, remoteStatusID *url.URL) (ap.Statusable, error) { + if blocked, err := d.blockedDomain(remoteStatusID.Host); blocked || err != nil { + return nil, fmt.Errorf("DereferenceStatusable: domain %s is blocked", remoteStatusID.Host) + } + + transport, err := d.transportController.NewTransportForUsername(username) + if err != nil { + return nil, fmt.Errorf("DereferenceStatusable: transport err: %s", err) + } + + b, err := transport.Dereference(context.Background(), remoteStatusID) + if err != nil { + return nil, fmt.Errorf("DereferenceStatusable: error deferencing %s: %s", remoteStatusID.String(), err) + } + + m := make(map[string]interface{}) + if err := json.Unmarshal(b, &m); err != nil { + return nil, fmt.Errorf("DereferenceStatusable: error unmarshalling bytes into json: %s", err) + } + + t, err := streams.ToType(context.Background(), m) + if err != nil { + return nil, fmt.Errorf("DereferenceStatusable: error resolving json into ap vocab type: %s", err) + } + + // Article, Document, Image, Video, Note, Page, Event, Place, Mention, Profile + switch t.GetTypeName() { + case gtsmodel.ActivityStreamsArticle: + p, ok := t.(vocab.ActivityStreamsArticle) + if !ok { + return nil, errors.New("DereferenceStatusable: error resolving type as ActivityStreamsArticle") + } + return p, nil + case gtsmodel.ActivityStreamsDocument: + p, ok := t.(vocab.ActivityStreamsDocument) + if !ok { + return nil, errors.New("DereferenceStatusable: error resolving type as ActivityStreamsDocument") + } + return p, nil + case gtsmodel.ActivityStreamsImage: + p, ok := t.(vocab.ActivityStreamsImage) + if !ok { + return nil, errors.New("DereferenceStatusable: error resolving type as ActivityStreamsImage") + } + return p, nil + case gtsmodel.ActivityStreamsVideo: + p, ok := t.(vocab.ActivityStreamsVideo) + if !ok { + return nil, errors.New("DereferenceStatusable: error resolving type as ActivityStreamsVideo") + } + return p, nil + case gtsmodel.ActivityStreamsNote: + p, ok := t.(vocab.ActivityStreamsNote) + if !ok { + return nil, errors.New("DereferenceStatusable: error resolving type as ActivityStreamsNote") + } + return p, nil + case gtsmodel.ActivityStreamsPage: + p, ok := t.(vocab.ActivityStreamsPage) + if !ok { + return nil, errors.New("DereferenceStatusable: error resolving type as ActivityStreamsPage") + } + return p, nil + case gtsmodel.ActivityStreamsEvent: + p, ok := t.(vocab.ActivityStreamsEvent) + if !ok { + return nil, errors.New("DereferenceStatusable: error resolving type as ActivityStreamsEvent") + } + return p, nil + case gtsmodel.ActivityStreamsPlace: + p, ok := t.(vocab.ActivityStreamsPlace) + if !ok { + return nil, errors.New("DereferenceStatusable: error resolving type as ActivityStreamsPlace") + } + return p, nil + case gtsmodel.ActivityStreamsProfile: + p, ok := t.(vocab.ActivityStreamsProfile) + if !ok { + return nil, errors.New("DereferenceStatusable: error resolving type as ActivityStreamsProfile") + } + return p, nil + } + + return nil, fmt.Errorf("DereferenceStatusable: type name %s not supported", t.GetTypeName()) +} + +// populateStatusFields fetches all the information we temporarily pinned to an incoming +// federated status, back in the federating db's Create function. +// +// When a status comes in from the federation API, there are certain fields that +// haven't been dereferenced yet, because we needed to provide a snappy synchronous +// response to the caller. By the time it reaches this function though, it's being +// processed asynchronously, so we have all the time in the world to fetch the various +// bits and bobs that are attached to the status, and properly flesh it out, before we +// send the status to any timelines and notify people. +// +// Things to dereference and fetch here: +// +// 1. Media attachments. +// 2. Hashtags. +// 3. Emojis. +// 4. Mentions. +// 5. Posting account. +// 6. Replied-to-status. +// +// SIDE EFFECTS: +// This function will deference all of the above, insert them in the database as necessary, +// and attach them to the status. The status itself will not be added to the database yet, +// that's up the caller to do. +func (d *deref) populateStatusFields(status *gtsmodel.Status, requestingUsername string) error { + l := d.log.WithFields(logrus.Fields{ + "func": "dereferenceStatusFields", + "status": fmt.Sprintf("%+v", status), + }) + l.Debug("entering function") + + // make sure we have a status URI and that the domain in question isn't blocked + statusURI, err := url.Parse(status.URI) + if err != nil { + return fmt.Errorf("DereferenceStatusFields: couldn't parse status URI %s: %s", status.URI, err) + } + if blocked, err := d.blockedDomain(statusURI.Host); blocked || err != nil { + return fmt.Errorf("DereferenceStatusFields: domain %s is blocked", statusURI.Host) + } + + // we can continue -- create a new transport here because we'll probably need it + t, err := d.transportController.NewTransportForUsername(requestingUsername) + if err != nil { + return fmt.Errorf("error creating transport: %s", err) + } + + // in case the status doesn't have an id yet (ie., it hasn't entered the database yet), then create one + if status.ID == "" { + newID, err := id.NewULIDFromTime(status.CreatedAt) + if err != nil { + return err + } + status.ID = newID + } + + // 1. Media attachments. + // + // At this point we should know: + // * the media type of the file we're looking for (a.File.ContentType) + // * the blurhash (a.Blurhash) + // * the file type (a.Type) + // * the remote URL (a.RemoteURL) + // This should be enough to pass along to the media processor. + attachmentIDs := []string{} + for _, a := range status.GTSMediaAttachments { + l.Tracef("dereferencing attachment: %+v", a) + + // it might have been processed elsewhere so check first if it's already in the database or not + maybeAttachment := >smodel.MediaAttachment{} + err := d.db.GetWhere([]db.Where{{Key: "remote_url", Value: a.RemoteURL}}, maybeAttachment) + if err == nil { + // we already have it in the db, dereferenced, no need to do it again + l.Tracef("attachment already exists with id %s", maybeAttachment.ID) + attachmentIDs = append(attachmentIDs, maybeAttachment.ID) + continue + } + if _, ok := err.(db.ErrNoEntries); !ok { + // we have a real error + return fmt.Errorf("error checking db for existence of attachment with remote url %s: %s", a.RemoteURL, err) + } + // it just doesn't exist yet so carry on + l.Debug("attachment doesn't exist yet, calling ProcessRemoteAttachment", a) + deferencedAttachment, err := d.mediaHandler.ProcessRemoteAttachment(t, a, status.AccountID) + if err != nil { + l.Errorf("error dereferencing status attachment: %s", err) + continue + } + l.Debugf("dereferenced attachment: %+v", deferencedAttachment) + deferencedAttachment.StatusID = status.ID + deferencedAttachment.Description = a.Description + if err := d.db.Put(deferencedAttachment); err != nil { + return fmt.Errorf("error inserting dereferenced attachment with remote url %s: %s", a.RemoteURL, err) + } + attachmentIDs = append(attachmentIDs, deferencedAttachment.ID) + } + status.Attachments = attachmentIDs + + // 2. Hashtags + + // 3. Emojis + + // 4. Mentions + // At this point, mentions should have the namestring and mentionedAccountURI set on them. + // + // We should dereference any accounts mentioned here which we don't have in our db yet, by their URI. + mentions := []string{} + for _, m := range status.GTSMentions { + + if m.ID != "" { + continue + // we've already populated this mention, since it has an ID + } + + mID, err := id.NewRandomULID() + if err != nil { + return err + } + m.ID = mID + + uri, err := url.Parse(m.MentionedAccountURI) + if err != nil { + l.Debugf("error parsing mentioned account uri %s: %s", m.MentionedAccountURI, err) + continue + } + + m.StatusID = status.ID + m.OriginAccountID = status.GTSAuthorAccount.ID + m.OriginAccountURI = status.GTSAuthorAccount.URI + + targetAccount, _, err := d.GetRemoteAccount(requestingUsername, uri, false) + if err != nil { + continue + } + + // by this point, we know the targetAccount exists in our database with an ID :) + m.TargetAccountID = targetAccount.ID + if err := d.db.Put(m); err != nil { + return fmt.Errorf("error creating mention: %s", err) + } + mentions = append(mentions, m.ID) + } + status.Mentions = mentions + + // status has replyToURI but we don't have an ID yet for the status it replies to + if status.InReplyToURI != "" && status.InReplyToID == "" { + replyToStatus := >smodel.Status{} + if err := d.db.GetWhere([]db.Where{{Key: "uri", Value: status.InReplyToURI}}, replyToStatus); err == nil { + // we have the status + status.InReplyToID = replyToStatus.ID + status.InReplyToAccountID = replyToStatus.AccountID + } + } + + return nil +} diff --git a/internal/federation/dereferencing/thread.go b/internal/federation/dereferencing/thread.go new file mode 100644 index 000000000..2a407f923 --- /dev/null +++ b/internal/federation/dereferencing/thread.go @@ -0,0 +1,250 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package dereferencing + +import ( + "fmt" + "net/url" + + "github.com/sirupsen/logrus" + "github.com/superseriousbusiness/gotosocial/internal/ap" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/util" +) + +// DereferenceThread takes a statusable (something that has withReplies and withInReplyTo), +// and dereferences statusables in the conversation. +// +// This process involves working up and down the chain of replies, and parsing through the collections of IDs +// presented by remote instances as part of their replies collections, and will likely involve making several calls to +// multiple different hosts. +func (d *deref) DereferenceThread(username string, statusIRI *url.URL) error { + l := d.log.WithFields(logrus.Fields{ + "func": "DereferenceThread", + "username": username, + "statusIRI": statusIRI.String(), + }) + l.Debug("entering DereferenceThread") + + // if it's our status we already have everything stashed so we can bail early + if statusIRI.Host == d.config.Host { + l.Debug("iri belongs to us, bailing") + return nil + } + + // first make sure we have this status in our db + _, statusable, _, err := d.GetRemoteStatus(username, statusIRI, true) + if err != nil { + return fmt.Errorf("DereferenceThread: error getting status with id %s: %s", statusIRI.String(), err) + } + + // first iterate up through ancestors, dereferencing if necessary as we go + if err := d.iterateAncestors(username, *statusIRI); err != nil { + return fmt.Errorf("error iterating ancestors of status %s: %s", statusIRI.String(), err) + } + + // now iterate down through descendants, again dereferencing as we go + if err := d.iterateDescendants(username, *statusIRI, statusable); err != nil { + return fmt.Errorf("error iterating descendants of status %s: %s", statusIRI.String(), err) + } + + return nil +} + +// iterateAncestors has the goal of reaching the oldest ancestor of a given status, and stashing all statuses along the way. +func (d *deref) iterateAncestors(username string, statusIRI url.URL) error { + l := d.log.WithFields(logrus.Fields{ + "func": "iterateAncestors", + "username": username, + "statusIRI": statusIRI.String(), + }) + l.Debug("entering iterateAncestors") + + // if it's our status we don't need to dereference anything so we can immediately move up the chain + if statusIRI.Host == d.config.Host { + l.Debug("iri belongs to us, moving up to next ancestor") + + // since this is our status, we know we can extract the id from the status path + _, id, err := util.ParseStatusesPath(&statusIRI) + if err != nil { + return err + } + + status := >smodel.Status{} + if err := d.db.GetByID(id, status); err != nil { + return err + } + + if status.InReplyToURI == "" { + // status doesn't reply to anything + return nil + } + nextIRI, err := url.Parse(status.URI) + if err != nil { + return err + } + return d.iterateAncestors(username, *nextIRI) + } + + // If we reach here, we're looking at a remote status -- make sure we have it in our db by calling GetRemoteStatus + // We call it with refresh to true because we want the statusable representation to parse inReplyTo from. + status, statusable, _, err := d.GetRemoteStatus(username, &statusIRI, true) + if err != nil { + l.Debugf("error getting remote status: %s", err) + return nil + } + + inReplyTo := ap.ExtractInReplyToURI(statusable) + if inReplyTo == nil || inReplyTo.String() == "" { + // status doesn't reply to anything + return nil + } + + // get the ancestor status into our database if we don't have it yet + if _, _, _, err := d.GetRemoteStatus(username, inReplyTo, false); err != nil { + l.Debugf("error getting remote status: %s", err) + return nil + } + + // now enrich the current status, since we should have the ancestor in the db + if _, err := d.EnrichRemoteStatus(username, status); err != nil { + l.Debugf("error enriching remote status: %s", err) + return nil + } + + // now move up to the next ancestor + return d.iterateAncestors(username, *inReplyTo) +} + +func (d *deref) iterateDescendants(username string, statusIRI url.URL, statusable ap.Statusable) error { + l := d.log.WithFields(logrus.Fields{ + "func": "iterateDescendants", + "username": username, + "statusIRI": statusIRI.String(), + }) + l.Debug("entering iterateDescendants") + + // if it's our status we already have descendants stashed so we can bail early + if statusIRI.Host == d.config.Host { + l.Debug("iri belongs to us, bailing") + return nil + } + + replies := statusable.GetActivityStreamsReplies() + if replies == nil || !replies.IsActivityStreamsCollection() { + l.Debug("no replies, bailing") + return nil + } + + repliesCollection := replies.GetActivityStreamsCollection() + if repliesCollection == nil { + l.Debug("replies collection is nil, bailing") + return nil + } + + first := repliesCollection.GetActivityStreamsFirst() + if first == nil { + l.Debug("replies collection has no first, bailing") + return nil + } + + firstPage := first.GetActivityStreamsCollectionPage() + if firstPage == nil { + l.Debug("first has no collection page, bailing") + return nil + } + + firstPageNext := firstPage.GetActivityStreamsNext() + if firstPageNext == nil || !firstPageNext.IsIRI() { + l.Debug("next is not an iri, bailing") + return nil + } + + var foundReplies int + currentPageIRI := firstPageNext.GetIRI() + +pageLoop: + for { + l.Debugf("dereferencing page %s", currentPageIRI) + nextPage, err := d.DereferenceCollectionPage(username, currentPageIRI) + if err != nil { + return nil + } + + // next items could be either a list of URLs or a list of statuses + + nextItems := nextPage.GetActivityStreamsItems() + if nextItems.Len() == 0 { + // no items on this page, which means we're done + break pageLoop + } + + // have a look through items and see what we can find + for iter := nextItems.Begin(); iter != nextItems.End(); iter = iter.Next() { + // We're looking for a url to feed to GetRemoteStatus. + // Items can be either an IRI, or a Note. + // If a note, we grab the ID from it and call it, rather than parsing the note. + + var itemURI *url.URL + if iter.IsIRI() { + // iri, easy + itemURI = iter.GetIRI() + } else if iter.IsActivityStreamsNote() { + // note, get the id from it to use as iri + n := iter.GetActivityStreamsNote() + id := n.GetJSONLDId() + if id != nil && id.IsIRI() { + itemURI = id.GetIRI() + } + } else { + // if it's not an iri or a note, we don't know how to process it + continue + } + + if itemURI.Host == d.config.Host { + // skip if the reply is from us -- we already have it then + continue + } + + // we can confidently say now that we found something + foundReplies = foundReplies + 1 + + // get the remote statusable and put it in the db + _, statusable, new, err := d.GetRemoteStatus(username, itemURI, false) + if new && err == nil && statusable != nil { + // now iterate descendants of *that* status + if err := d.iterateDescendants(username, *itemURI, statusable); err != nil { + continue + } + } + } + + next := nextPage.GetActivityStreamsNext() + if next != nil && next.IsIRI() { + l.Debug("setting next page") + currentPageIRI = next.GetIRI() + } else { + l.Debug("no next page, bailing") + break pageLoop + } + } + + l.Debugf("foundReplies %d", foundReplies) + return nil +} diff --git a/internal/federation/federatingdb/update.go b/internal/federation/federatingdb/update.go index e4a4920c8..3f4e3e413 100644 --- a/internal/federation/federatingdb/update.go +++ b/internal/federation/federatingdb/update.go @@ -9,8 +9,8 @@ import ( "github.com/go-fed/activity/streams" "github.com/go-fed/activity/streams/vocab" "github.com/sirupsen/logrus" + "github.com/superseriousbusiness/gotosocial/internal/ap" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" - "github.com/superseriousbusiness/gotosocial/internal/typeutils" "github.com/superseriousbusiness/gotosocial/internal/util" ) @@ -78,7 +78,7 @@ func (f *federatingDB) Update(ctx context.Context, asType vocab.Type) error { typeName == gtsmodel.ActivityStreamsPerson || typeName == gtsmodel.ActivityStreamsService { // it's an UPDATE to some kind of account - var accountable typeutils.Accountable + var accountable ap.Accountable switch asType.GetTypeName() { case gtsmodel.ActivityStreamsApplication: diff --git a/internal/federation/federatingprotocol.go b/internal/federation/federatingprotocol.go index 1acdb6cb1..9e21b43bf 100644 --- a/internal/federation/federatingprotocol.go +++ b/internal/federation/federatingprotocol.go @@ -31,7 +31,6 @@ import ( "github.com/sirupsen/logrus" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" - "github.com/superseriousbusiness/gotosocial/internal/id" "github.com/superseriousbusiness/gotosocial/internal/util" ) @@ -139,7 +138,7 @@ func (f *federator) AuthenticatePostInbox(ctx context.Context, w http.ResponseWr } // we don't have an entry for this instance yet so dereference it - i, err = f.DereferenceRemoteInstance(username, &url.URL{ + i, err = f.GetRemoteInstance(username, &url.URL{ Scheme: publicKeyOwnerURI.Scheme, Host: publicKeyOwnerURI.Host, }) @@ -153,51 +152,9 @@ func (f *federator) AuthenticatePostInbox(ctx context.Context, w http.ResponseWr } } - requestingAccount := >smodel.Account{} - if err := f.db.GetWhere([]db.Where{{Key: "uri", Value: publicKeyOwnerURI.String()}}, requestingAccount); err != nil { - // there's been a proper error so return it - if _, ok := err.(db.ErrNoEntries); !ok { - return ctx, false, fmt.Errorf("error getting requesting account with public key id %s: %s", publicKeyOwnerURI.String(), err) - } - - // we don't know this account (yet) so let's dereference it right now - person, err := f.DereferenceRemoteAccount(requestedAccount.Username, publicKeyOwnerURI) - if err != nil { - return ctx, false, fmt.Errorf("error dereferencing account with public key id %s: %s", publicKeyOwnerURI.String(), err) - } - - a, err := f.typeConverter.ASRepresentationToAccount(person, false) - if err != nil { - return ctx, false, fmt.Errorf("error converting person with public key id %s to account: %s", publicKeyOwnerURI.String(), err) - } - - aID, err := id.NewRandomULID() - if err != nil { - return ctx, false, err - } - a.ID = aID - - if err := f.db.Put(a); err != nil { - l.Errorf("error inserting dereferenced remote account: %s", err) - } - - requestingAccount = a - - // send the newly dereferenced account into the processor channel for further async processing - fromFederatorChanI := ctx.Value(util.APFromFederatorChanKey) - if fromFederatorChanI == nil { - l.Error("from federator channel wasn't set on context") - } - fromFederatorChan, ok := fromFederatorChanI.(chan gtsmodel.FromFederator) - if !ok { - l.Error("from federator channel was set on context but couldn't be parsed") - } - - fromFederatorChan <- gtsmodel.FromFederator{ - APObjectType: gtsmodel.ActivityStreamsProfile, - APActivityType: gtsmodel.ActivityStreamsCreate, - GTSModel: requestingAccount, - } + requestingAccount, _, err := f.GetRemoteAccount(username, publicKeyOwnerURI, false) + if err != nil { + return nil, false, fmt.Errorf("couldn't get remote account: %s", err) } withRequester := context.WithValue(ctx, util.APRequestingAccount, requestingAccount) diff --git a/internal/federation/federator.go b/internal/federation/federator.go index a5ffb3de8..ea9e61831 100644 --- a/internal/federation/federator.go +++ b/internal/federation/federator.go @@ -21,12 +21,13 @@ package federation import ( "context" "net/url" - "sync" "github.com/go-fed/activity/pub" "github.com/sirupsen/logrus" + "github.com/superseriousbusiness/gotosocial/internal/ap" "github.com/superseriousbusiness/gotosocial/internal/config" "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing" "github.com/superseriousbusiness/gotosocial/internal/federation/federatingdb" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/media" @@ -40,6 +41,7 @@ type Federator interface { FederatingActor() pub.FederatingActor // FederatingDB returns the underlying FederatingDB interface. FederatingDB() federatingdb.DB + // AuthenticateFederatedRequest can be used to check the authenticity of incoming http-signed requests for federating resources. // The given username will be used to create a transport for making outgoing requests. See the implementation for more detailed comments. // @@ -49,29 +51,21 @@ type Federator interface { // // If something goes wrong during authentication, nil, false, and an error will be returned. AuthenticateFederatedRequest(ctx context.Context, username string) (*url.URL, bool, error) + // FingerRemoteAccount performs a webfinger lookup for a remote account, using the .well-known path. It will return the ActivityPub URI for that // account, or an error if it doesn't exist or can't be retrieved. FingerRemoteAccount(requestingUsername string, targetUsername string, targetDomain string) (*url.URL, error) - // DereferenceRemoteAccount can be used to get the representation of a remote account, based on the account ID (which is a URI). - // The given username will be used to create a transport for making outgoing requests. See the implementation for more detailed comments. - DereferenceRemoteAccount(username string, remoteAccountID *url.URL) (typeutils.Accountable, error) - // DereferenceRemoteStatus can be used to get the representation of a remote status, based on its ID (which is a URI). - // The given username will be used to create a transport for making outgoing requests. See the implementation for more detailed comments. - DereferenceRemoteStatus(username string, remoteStatusID *url.URL) (typeutils.Statusable, error) - // DereferenceRemoteInstance takes the URL of a remote instance, and a username (optional) to spin up a transport with. It then - // does its damnedest to get some kind of information back about the instance, trying /api/v1/instance, then /.well-known/nodeinfo - DereferenceRemoteInstance(username string, remoteInstanceURI *url.URL) (*gtsmodel.Instance, error) - // DereferenceStatusFields does further dereferencing on a status. - DereferenceStatusFields(status *gtsmodel.Status, requestingUsername string) error - // DereferenceAccountFields does further dereferencing on an account. - DereferenceAccountFields(account *gtsmodel.Account, requestingUsername string, refresh bool) error - // DereferenceAnnounce does further dereferencing on an announce. + + DereferenceRemoteThread(username string, statusURI *url.URL) error DereferenceAnnounce(announce *gtsmodel.Status, requestingUsername string) error - // GetTransportForUser returns a new transport initialized with the key credentials belonging to the given username. - // This can be used for making signed http requests. - // - // If username is an empty string, our instance user's credentials will be used instead. - GetTransportForUser(username string) (transport.Transport, error) + + GetRemoteAccount(username string, remoteAccountID *url.URL, refresh bool) (*gtsmodel.Account, bool, error) + + GetRemoteStatus(username string, remoteStatusID *url.URL, refresh bool) (*gtsmodel.Status, ap.Statusable, bool, error) + EnrichRemoteStatus(username string, status *gtsmodel.Status) (*gtsmodel.Status, error) + + GetRemoteInstance(username string, remoteInstanceURI *url.URL) (*gtsmodel.Instance, error) + // Handshaking returns true if the given username is currently in the process of dereferencing the remoteAccountID. Handshaking(username string, remoteAccountID *url.URL) bool pub.CommonBehavior @@ -85,16 +79,17 @@ type federator struct { clock pub.Clock typeConverter typeutils.TypeConverter transportController transport.Controller + dereferencer dereferencing.Dereferencer mediaHandler media.Handler actor pub.FederatingActor log *logrus.Logger - handshakes map[string][]*url.URL - handshakeSync *sync.Mutex // mutex to lock/unlock when checking or updating the handshakes map } // NewFederator returns a new federator func NewFederator(db db.DB, federatingDB federatingdb.DB, transportController transport.Controller, config *config.Config, log *logrus.Logger, typeConverter typeutils.TypeConverter, mediaHandler media.Handler) Federator { + dereferencer := dereferencing.NewDereferencer(config, db, typeConverter, transportController, mediaHandler, log) + clock := &Clock{} f := &federator{ config: config, @@ -103,9 +98,9 @@ func NewFederator(db db.DB, federatingDB federatingdb.DB, transportController tr clock: &Clock{}, typeConverter: typeConverter, transportController: transportController, + dereferencer: dereferencer, mediaHandler: mediaHandler, log: log, - handshakeSync: &sync.Mutex{}, } actor := newFederatingActor(f, f, federatingDB, clock) f.actor = actor diff --git a/internal/federation/federator_test.go b/internal/federation/federator_test.go index 4ba0796cd..d74070487 100644 --- a/internal/federation/federator_test.go +++ b/internal/federation/federator_test.go @@ -69,7 +69,7 @@ func (suite *ProtocolTestSuite) SetupSuite() { } func (suite *ProtocolTestSuite) SetupTest() { - testrig.StandardDBSetup(suite.db) + testrig.StandardDBSetup(suite.db, suite.accounts) } @@ -87,7 +87,7 @@ func (suite *ProtocolTestSuite) TestPostInboxRequestBodyHook() { // setup transport controller with a no-op client so we don't make external calls tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(func(req *http.Request) (*http.Response, error) { return nil, nil - })) + }), suite.db) // setup module being tested federator := federation.NewFederator(suite.db, testrig.NewTestFederatingDB(suite.db), tc, suite.config, suite.log, suite.typeConverter, testrig.NewTestMediaHandler(suite.db, suite.storage)) @@ -152,7 +152,7 @@ func (suite *ProtocolTestSuite) TestAuthenticatePostInbox() { StatusCode: 200, Body: r, }, nil - })) + }), suite.db) // now setup module being tested, with the mock transport controller federator := federation.NewFederator(suite.db, testrig.NewTestFederatingDB(suite.db), tc, suite.config, suite.log, suite.typeConverter, testrig.NewTestMediaHandler(suite.db, suite.storage)) diff --git a/internal/federation/finger.go b/internal/federation/finger.go index 6c6e9f6dc..0ffc60e5a 100644 --- a/internal/federation/finger.go +++ b/internal/federation/finger.go @@ -34,7 +34,7 @@ func (f *federator) FingerRemoteAccount(requestingUsername string, targetUsernam return nil, fmt.Errorf("FingerRemoteAccount: domain %s is blocked", targetDomain) } - t, err := f.GetTransportForUser(requestingUsername) + t, err := f.transportController.NewTransportForUsername(requestingUsername) if err != nil { return nil, fmt.Errorf("FingerRemoteAccount: error getting transport for username %s while dereferencing @%s@%s: %s", requestingUsername, targetUsername, targetDomain, err) } diff --git a/internal/federation/handshake.go b/internal/federation/handshake.go index 511e3e174..47c8a6c84 100644 --- a/internal/federation/handshake.go +++ b/internal/federation/handshake.go @@ -3,78 +3,5 @@ package federation import "net/url" func (f *federator) Handshaking(username string, remoteAccountID *url.URL) bool { - f.handshakeSync.Lock() - defer f.handshakeSync.Unlock() - - if f.handshakes == nil { - // handshakes isn't even initialized yet so we can't be handshaking with anyone - return false - } - - remoteIDs, ok := f.handshakes[username] - if !ok { - // user isn't handshaking with anyone, bail - return false - } - - for _, id := range remoteIDs { - if id.String() == remoteAccountID.String() { - // we are currently handshaking with the remote account, yep - return true - } - } - - // didn't find it which means we're not handshaking - return false -} - -func (f *federator) startHandshake(username string, remoteAccountID *url.URL) { - f.handshakeSync.Lock() - defer f.handshakeSync.Unlock() - - // lazily initialize handshakes - if f.handshakes == nil { - f.handshakes = make(map[string][]*url.URL) - } - - remoteIDs, ok := f.handshakes[username] - if !ok { - // there was nothing in there yet, so just add this entry and return - f.handshakes[username] = []*url.URL{remoteAccountID} - return - } - - // add the remote ID to the slice - remoteIDs = append(remoteIDs, remoteAccountID) - f.handshakes[username] = remoteIDs -} - -func (f *federator) stopHandshake(username string, remoteAccountID *url.URL) { - f.handshakeSync.Lock() - defer f.handshakeSync.Unlock() - - if f.handshakes == nil { - return - } - - remoteIDs, ok := f.handshakes[username] - if !ok { - // there was nothing in there yet anyway so just bail - return - } - - newRemoteIDs := []*url.URL{} - for _, id := range remoteIDs { - if id.String() != remoteAccountID.String() { - newRemoteIDs = append(newRemoteIDs, id) - } - } - - if len(newRemoteIDs) == 0 { - // there are no handshakes so just remove this user entry from the map and save a few bytes - delete(f.handshakes, username) - } else { - // there are still other handshakes ongoing - f.handshakes[username] = newRemoteIDs - } + return f.dereferencer.Handshaking(username, remoteAccountID) } diff --git a/internal/federation/transport.go b/internal/federation/transport.go index a92f66d25..ed28749a1 100644 --- a/internal/federation/transport.go +++ b/internal/federation/transport.go @@ -6,8 +6,6 @@ import ( "net/url" "github.com/go-fed/activity/pub" - "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" - "github.com/superseriousbusiness/gotosocial/internal/transport" "github.com/superseriousbusiness/gotosocial/internal/util" ) @@ -35,7 +33,6 @@ import ( // returned Transport so that any private credentials are able to be // garbage collected. func (f *federator) NewTransport(ctx context.Context, actorBoxIRI *url.URL, gofedAgent string) (pub.Transport, error) { - var username string var err error @@ -53,32 +50,5 @@ func (f *federator) NewTransport(ctx context.Context, actorBoxIRI *url.URL, gofe return nil, fmt.Errorf("id %s was neither an inbox path nor an outbox path", actorBoxIRI.String()) } - account := >smodel.Account{} - if err := f.db.GetLocalAccountByUsername(username, account); err != nil { - return nil, fmt.Errorf("error getting account with username %s from the db: %s", username, err) - } - - return f.transportController.NewTransport(account.PublicKeyURI, account.PrivateKey) -} - -func (f *federator) GetTransportForUser(username string) (transport.Transport, error) { - // We need an account to use to create a transport for dereferecing something. - // If a username has been given, we can fetch the account with that username and use it. - // Otherwise, we can take the instance account and use those credentials to make the request. - ourAccount := >smodel.Account{} - var u string - if username == "" { - u = f.config.Host - } else { - u = username - } - if err := f.db.GetLocalAccountByUsername(u, ourAccount); err != nil { - return nil, fmt.Errorf("error getting account %s from db: %s", username, err) - } - - transport, err := f.transportController.NewTransport(ourAccount.PublicKeyURI, ourAccount.PrivateKey) - if err != nil { - return nil, fmt.Errorf("error creating transport for user %s: %s", username, err) - } - return transport, nil + return f.transportController.NewTransportForUsername(username) } diff --git a/internal/gtsmodel/activitystreams.go b/internal/gtsmodel/activitystreams.go index 77c935c5f..5cd92015c 100644 --- a/internal/gtsmodel/activitystreams.go +++ b/internal/gtsmodel/activitystreams.go @@ -43,6 +43,10 @@ const ( ActivityStreamsTombstone = "Tombstone" // ActivityStreamsVideo https://www.w3.org/TR/activitystreams-vocabulary/#dfn-video ActivityStreamsVideo = "Video" + //ActivityStreamsCollection https://www.w3.org/TR/activitystreams-vocabulary/#dfn-collection + ActivityStreamsCollection = "Collection" + // ActivityStreamsCollectionPage https://www.w3.org/TR/activitystreams-vocabulary/#dfn-collectionpage + ActivityStreamsCollectionPage = "CollectionPage" ) const ( diff --git a/internal/gtsmodel/status.go b/internal/gtsmodel/status.go index 524b9c3ef..106298bcd 100644 --- a/internal/gtsmodel/status.go +++ b/internal/gtsmodel/status.go @@ -18,7 +18,9 @@ package gtsmodel -import "time" +import ( + "time" +) // Status represents a user-created 'post' or 'status' in the database, either remote or local type Status struct { diff --git a/internal/processing/account/get.go b/internal/processing/account/get.go index b937ace5b..d2994e246 100644 --- a/internal/processing/account/get.go +++ b/internal/processing/account/get.go @@ -36,15 +36,6 @@ func (p *processor) Get(requestingAccount *gtsmodel.Account, targetAccountID str return nil, fmt.Errorf("db error: %s", err) } - // lazily dereference things on the account if it hasn't been done yet - var requestingUsername string - if requestingAccount != nil { - requestingUsername = requestingAccount.Username - } - if err := p.federator.DereferenceAccountFields(targetAccount, requestingUsername, false); err != nil { - p.log.WithField("func", "AccountGet").Debugf("dereferencing account: %s", err) - } - var blocked bool var err error if requestingAccount != nil { diff --git a/internal/processing/account/getfollowers.go b/internal/processing/account/getfollowers.go index bfc463d3f..0806a82c0 100644 --- a/internal/processing/account/getfollowers.go +++ b/internal/processing/account/getfollowers.go @@ -63,12 +63,6 @@ func (p *processor) FollowersGet(requestingAccount *gtsmodel.Account, targetAcco return nil, gtserror.NewErrorInternalError(err) } - // derefence account fields in case we haven't done it already - if err := p.federator.DereferenceAccountFields(a, requestingAccount.Username, false); err != nil { - // don't bail if we can't fetch them, we'll try another time - p.log.WithField("func", "AccountFollowersGet").Debugf("error dereferencing account fields: %s", err) - } - account, err := p.tc.AccountToMastoPublic(a) if err != nil { return nil, gtserror.NewErrorInternalError(err) diff --git a/internal/processing/account/getfollowing.go b/internal/processing/account/getfollowing.go index bb6a905f4..75e89dacb 100644 --- a/internal/processing/account/getfollowing.go +++ b/internal/processing/account/getfollowing.go @@ -63,12 +63,6 @@ func (p *processor) FollowingGet(requestingAccount *gtsmodel.Account, targetAcco return nil, gtserror.NewErrorInternalError(err) } - // derefence account fields in case we haven't done it already - if err := p.federator.DereferenceAccountFields(a, requestingAccount.Username, false); err != nil { - // don't bail if we can't fetch them, we'll try another time - p.log.WithField("func", "AccountFollowingGet").Debugf("error dereferencing account fields: %s", err) - } - account, err := p.tc.AccountToMastoPublic(a) if err != nil { return nil, gtserror.NewErrorInternalError(err) diff --git a/internal/processing/federation.go b/internal/processing/federation.go index 966dab08d..765fdf862 100644 --- a/internal/processing/federation.go +++ b/internal/processing/federation.go @@ -31,65 +31,9 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/gtserror" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" - "github.com/superseriousbusiness/gotosocial/internal/id" "github.com/superseriousbusiness/gotosocial/internal/util" ) -// dereferenceFediRequest authenticates the HTTP signature of an incoming federation request, using the given -// username to perform the validation. It will *also* dereference the originator of the request and return it as a gtsmodel account -// for further processing. NOTE that this function will have the side effect of putting the dereferenced account into the database, -// and passing it into the processor through a channel for further asynchronous processing. -func (p *processor) dereferenceFediRequest(username string, requestingAccountURI *url.URL) (*gtsmodel.Account, error) { - // OK now we can do the dereferencing part - // we might already have an entry for this account so check that first - requestingAccount := >smodel.Account{} - - err := p.db.GetWhere([]db.Where{{Key: "uri", Value: requestingAccountURI.String()}}, requestingAccount) - if err == nil { - // we do have it yay, return it - return requestingAccount, nil - } - - if _, ok := err.(db.ErrNoEntries); !ok { - // something has actually gone wrong so bail - return nil, fmt.Errorf("database error getting account with uri %s: %s", requestingAccountURI.String(), err) - } - - // we just don't have an entry for this account yet - // what we do now should depend on our chosen federation method - // for now though, we'll just dereference it - // TODO: slow-fed - requestingPerson, err := p.federator.DereferenceRemoteAccount(username, requestingAccountURI) - if err != nil { - return nil, fmt.Errorf("couldn't dereference %s: %s", requestingAccountURI.String(), err) - } - - // convert it to our internal account representation - requestingAccount, err = p.tc.ASRepresentationToAccount(requestingPerson, false) - if err != nil { - return nil, fmt.Errorf("couldn't convert dereferenced uri %s to gtsmodel account: %s", requestingAccountURI.String(), err) - } - - requestingAccountID, err := id.NewRandomULID() - if err != nil { - return nil, err - } - requestingAccount.ID = requestingAccountID - - if err := p.db.Put(requestingAccount); err != nil { - return nil, fmt.Errorf("database error inserting account with uri %s: %s", requestingAccountURI.String(), err) - } - - // put it in our channel to queue it for async processing - p.fromFederator <- gtsmodel.FromFederator{ - APObjectType: gtsmodel.ActivityStreamsProfile, - APActivityType: gtsmodel.ActivityStreamsCreate, - GTSModel: requestingAccount, - } - - return requestingAccount, nil -} - func (p *processor) GetFediUser(ctx context.Context, requestedUsername string, requestURL *url.URL) (interface{}, gtserror.WithCode) { // get the account the request is referring to requestedAccount := >smodel.Account{} @@ -112,9 +56,9 @@ func (p *processor) GetFediUser(ctx context.Context, requestedUsername string, r return nil, gtserror.NewErrorNotAuthorized(errors.New("not authorized"), "not authorized") } - // if we're already handshaking/dereferencing a remote account, we can skip the dereferencing part + // if we're not already handshaking/dereferencing a remote account, dereference it now if !p.federator.Handshaking(requestedUsername, requestingAccountURI) { - requestingAccount, err := p.dereferenceFediRequest(requestedUsername, requestingAccountURI) + requestingAccount, _, err := p.federator.GetRemoteAccount(requestedUsername, requestingAccountURI, false) if err != nil { return nil, gtserror.NewErrorNotAuthorized(err) } @@ -158,7 +102,7 @@ func (p *processor) GetFediFollowers(ctx context.Context, requestedUsername stri return nil, gtserror.NewErrorNotAuthorized(errors.New("not authorized"), "not authorized") } - requestingAccount, err := p.dereferenceFediRequest(requestedUsername, requestingAccountURI) + requestingAccount, _, err := p.federator.GetRemoteAccount(requestedUsername, requestingAccountURI, false) if err != nil { return nil, gtserror.NewErrorNotAuthorized(err) } @@ -203,7 +147,7 @@ func (p *processor) GetFediFollowing(ctx context.Context, requestedUsername stri return nil, gtserror.NewErrorNotAuthorized(errors.New("not authorized"), "not authorized") } - requestingAccount, err := p.dereferenceFediRequest(requestedUsername, requestingAccountURI) + requestingAccount, _, err := p.federator.GetRemoteAccount(requestedUsername, requestingAccountURI, false) if err != nil { return nil, gtserror.NewErrorNotAuthorized(err) } @@ -248,7 +192,7 @@ func (p *processor) GetFediStatus(ctx context.Context, requestedUsername string, return nil, gtserror.NewErrorNotAuthorized(errors.New("not authorized"), "not authorized") } - requestingAccount, err := p.dereferenceFediRequest(requestedUsername, requestingAccountURI) + requestingAccount, _, err := p.federator.GetRemoteAccount(requestedUsername, requestingAccountURI, false) if err != nil { return nil, gtserror.NewErrorNotAuthorized(err) } @@ -295,6 +239,139 @@ func (p *processor) GetFediStatus(ctx context.Context, requestedUsername string, return data, nil } +func (p *processor) GetFediStatusReplies(ctx context.Context, requestedUsername string, requestedStatusID string, page bool, onlyOtherAccounts bool, minID string, requestURL *url.URL) (interface{}, gtserror.WithCode) { + // get the account the request is referring to + requestedAccount := >smodel.Account{} + if err := p.db.GetLocalAccountByUsername(requestedUsername, requestedAccount); err != nil { + return nil, gtserror.NewErrorNotFound(fmt.Errorf("database error getting account with username %s: %s", requestedUsername, err)) + } + + // authenticate the request + requestingAccountURI, authenticated, err := p.federator.AuthenticateFederatedRequest(ctx, requestedUsername) + if err != nil || !authenticated { + return nil, gtserror.NewErrorNotAuthorized(errors.New("not authorized"), "not authorized") + } + + requestingAccount, _, err := p.federator.GetRemoteAccount(requestedUsername, requestingAccountURI, false) + if err != nil { + return nil, gtserror.NewErrorNotAuthorized(err) + } + + // authorize the request: + // 1. check if a block exists between the requester and the requestee + blocked, err := p.db.Blocked(requestedAccount.ID, requestingAccount.ID) + if err != nil { + return nil, gtserror.NewErrorInternalError(err) + } + + if blocked { + return nil, gtserror.NewErrorNotAuthorized(fmt.Errorf("block exists between accounts %s and %s", requestedAccount.ID, requestingAccount.ID)) + } + + // get the status out of the database here + s := >smodel.Status{} + if err := p.db.GetWhere([]db.Where{ + {Key: "id", Value: requestedStatusID}, + {Key: "account_id", Value: requestedAccount.ID}, + }, s); err != nil { + return nil, gtserror.NewErrorNotFound(fmt.Errorf("database error getting status with id %s and account id %s: %s", requestedStatusID, requestedAccount.ID, err)) + } + + visible, err := p.filter.StatusVisible(s, requestingAccount) + if err != nil { + return nil, gtserror.NewErrorInternalError(err) + } + if !visible { + return nil, gtserror.NewErrorNotFound(fmt.Errorf("status with id %s not visible to user with id %s", s.ID, requestingAccount.ID)) + } + + var data map[string]interface{} + + // now there are three scenarios: + // 1. we're asked for the whole collection and not a page -- we can just return the collection, with no items, but a link to 'first' page. + // 2. we're asked for a page but only_other_accounts has not been set in the query -- so we should just return the first page of the collection, with no items. + // 3. we're asked for a page, and only_other_accounts has been set, and min_id has optionally been set -- so we need to return some actual items! + + if !page { + // scenario 1 + + // get the collection + collection, err := p.tc.StatusToASRepliesCollection(s, onlyOtherAccounts) + if err != nil { + return nil, gtserror.NewErrorInternalError(err) + } + + data, err = streams.Serialize(collection) + if err != nil { + return nil, gtserror.NewErrorInternalError(err) + } + } else if page && requestURL.Query().Get("only_other_accounts") == "" { + // scenario 2 + + // get the collection + collection, err := p.tc.StatusToASRepliesCollection(s, onlyOtherAccounts) + if err != nil { + return nil, gtserror.NewErrorInternalError(err) + } + // but only return the first page + data, err = streams.Serialize(collection.GetActivityStreamsFirst().GetActivityStreamsCollectionPage()) + if err != nil { + return nil, gtserror.NewErrorInternalError(err) + } + } else { + // scenario 3 + // get immediate children + replies, err := p.db.StatusChildren(s, true, minID) + if err != nil { + return nil, gtserror.NewErrorInternalError(err) + } + + // filter children and extract URIs + replyURIs := map[string]*url.URL{} + for _, r := range replies { + // only show public or unlocked statuses as replies + if r.Visibility != gtsmodel.VisibilityPublic && r.Visibility != gtsmodel.VisibilityUnlocked { + continue + } + + // respect onlyOtherAccounts parameter + if onlyOtherAccounts && r.AccountID == requestedAccount.ID { + continue + } + + // only show replies that the status owner can see + visibleToStatusOwner, err := p.filter.StatusVisible(r, requestedAccount) + if err != nil || !visibleToStatusOwner { + continue + } + + // only show replies that the requester can see + visibleToRequester, err := p.filter.StatusVisible(r, requestingAccount) + if err != nil || !visibleToRequester { + continue + } + + rURI, err := url.Parse(r.URI) + if err != nil { + continue + } + + replyURIs[r.ID] = rURI + } + + repliesPage, err := p.tc.StatusURIsToASRepliesPage(s, onlyOtherAccounts, minID, replyURIs) + if err != nil { + return nil, gtserror.NewErrorInternalError(err) + } + data, err = streams.Serialize(repliesPage) + if err != nil { + return nil, gtserror.NewErrorInternalError(err) + } + } + + return data, nil +} + func (p *processor) GetWebfingerAccount(ctx context.Context, requestedUsername string, requestURL *url.URL) (*apimodel.WellKnownResponse, gtserror.WithCode) { // get the account the request is referring to requestedAccount := >smodel.Account{} diff --git a/internal/processing/fromfederator.go b/internal/processing/fromfederator.go index 94a4e5af8..949a734c7 100644 --- a/internal/processing/fromfederator.go +++ b/internal/processing/fromfederator.go @@ -21,6 +21,7 @@ package processing import ( "errors" "fmt" + "net/url" "github.com/sirupsen/logrus" "github.com/superseriousbusiness/gotosocial/internal/db" @@ -47,36 +48,21 @@ func (p *processor) processFromFederator(federatorMsg gtsmodel.FromFederator) er return errors.New("note was not parseable as *gtsmodel.Status") } - l.Trace("will now derefence incoming status") - if err := p.federator.DereferenceStatusFields(incomingStatus, federatorMsg.ReceivingAccount.Username); err != nil { - return fmt.Errorf("error dereferencing status from federator: %s", err) - } - if err := p.db.UpdateByID(incomingStatus.ID, incomingStatus); err != nil { - return fmt.Errorf("error updating dereferenced status in the db: %s", err) - } - - if err := p.timelineStatus(incomingStatus); err != nil { + status, err := p.federator.EnrichRemoteStatus(federatorMsg.ReceivingAccount.Username, incomingStatus) + if err != nil { return err } - if err := p.notifyStatus(incomingStatus); err != nil { + if err := p.timelineStatus(status); err != nil { return err } + if err := p.notifyStatus(status); err != nil { + return err + } case gtsmodel.ActivityStreamsProfile: // CREATE AN ACCOUNT - incomingAccount, ok := federatorMsg.GTSModel.(*gtsmodel.Account) - if !ok { - return errors.New("profile was not parseable as *gtsmodel.Account") - } - - l.Trace("will now derefence incoming account") - if err := p.federator.DereferenceAccountFields(incomingAccount, "", false); err != nil { - return fmt.Errorf("error dereferencing account from federator: %s", err) - } - if err := p.db.UpdateByID(incomingAccount.ID, incomingAccount); err != nil { - return fmt.Errorf("error updating dereferenced account in the db: %s", err) - } + // nothing to do here case gtsmodel.ActivityStreamsLike: // CREATE A FAVE incomingFave, ok := federatorMsg.GTSModel.(*gtsmodel.StatusFave) @@ -154,12 +140,13 @@ func (p *processor) processFromFederator(federatorMsg gtsmodel.FromFederator) er return errors.New("profile was not parseable as *gtsmodel.Account") } - l.Trace("will now derefence incoming account") - if err := p.federator.DereferenceAccountFields(incomingAccount, federatorMsg.ReceivingAccount.Username, true); err != nil { - return fmt.Errorf("error dereferencing account from federator: %s", err) + incomingAccountURI, err := url.Parse(incomingAccount.URI) + if err != nil { + return err } - if err := p.db.UpdateByID(incomingAccount.ID, incomingAccount); err != nil { - return fmt.Errorf("error updating dereferenced account in the db: %s", err) + + if _, _, err := p.federator.GetRemoteAccount(federatorMsg.ReceivingAccount.Username, incomingAccountURI, true); err != nil { + return fmt.Errorf("error dereferencing account from federator: %s", err) } } case gtsmodel.ActivityStreamsDelete: diff --git a/internal/processing/processor.go b/internal/processing/processor.go index a09a370e9..16f9ac2a3 100644 --- a/internal/processing/processor.go +++ b/internal/processing/processor.go @@ -191,6 +191,10 @@ type Processor interface { // authentication before returning a JSON serializable interface to the caller. GetFediStatus(ctx context.Context, requestedUsername string, requestedStatusID string, requestURL *url.URL) (interface{}, gtserror.WithCode) + // GetFediStatus handles the getting of a fedi/activitypub representation of replies to a status, performing appropriate + // authentication before returning a JSON serializable interface to the caller. + GetFediStatusReplies(ctx context.Context, requestedUsername string, requestedStatusID string, page bool, onlyOtherAccounts bool, minID string, requestURL *url.URL) (interface{}, gtserror.WithCode) + // GetWebfingerAccount handles the GET for a webfinger resource. Most commonly, it will be used for returning account lookups. GetWebfingerAccount(ctx context.Context, requestedUsername string, requestURL *url.URL) (*apimodel.WellKnownResponse, gtserror.WithCode) diff --git a/internal/processing/search.go b/internal/processing/search.go index 727ad13bd..737ad8f71 100644 --- a/internal/processing/search.go +++ b/internal/processing/search.go @@ -19,7 +19,6 @@ package processing import ( - "errors" "fmt" "net/url" "strings" @@ -29,7 +28,6 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/gtserror" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" - "github.com/superseriousbusiness/gotosocial/internal/id" "github.com/superseriousbusiness/gotosocial/internal/oauth" "github.com/superseriousbusiness/gotosocial/internal/util" ) @@ -122,6 +120,11 @@ func (p *processor) SearchGet(authed *oauth.Auth, searchQuery *apimodel.SearchQu } func (p *processor) searchStatusByURI(authed *oauth.Auth, uri *url.URL, resolve bool) (*gtsmodel.Status, error) { + l := p.log.WithFields(logrus.Fields{ + "func": "searchStatusByURI", + "uri": uri.String(), + "resolve": resolve, + }) maybeStatus := >smodel.Status{} if err := p.db.GetWhere([]db.Where{{Key: "uri", Value: uri.String(), CaseInsensitive: true}}, maybeStatus); err == nil { @@ -134,57 +137,12 @@ func (p *processor) searchStatusByURI(authed *oauth.Auth, uri *url.URL, resolve // we don't have it locally so dereference it if we're allowed to if resolve { - statusable, err := p.federator.DereferenceRemoteStatus(authed.Account.Username, uri) + status, _, _, err := p.federator.GetRemoteStatus(authed.Account.Username, uri, true) if err == nil { - // it IS a status! - - // extract the status owner's IRI from the statusable - var statusOwnerURI *url.URL - statusAttributedTo := statusable.GetActivityStreamsAttributedTo() - for i := statusAttributedTo.Begin(); i != statusAttributedTo.End(); i = i.Next() { - if i.IsIRI() { - statusOwnerURI = i.GetIRI() - break - } + if err := p.federator.DereferenceRemoteThread(authed.Account.Username, uri); err != nil { + // try to deref the thread while we're here + l.Debugf("searchStatusByURI: error dereferencing remote thread: %s", err) } - if statusOwnerURI == nil { - return nil, errors.New("couldn't extract ownerAccountURI from statusable") - } - - // make sure the status owner exists in the db by searching for it - _, err := p.searchAccountByURI(authed, statusOwnerURI, resolve) - if err != nil { - return nil, err - } - - // we have the status owner, we have the dereferenced status, so now we should finish dereferencing the status properly - - // first turn it into a gtsmodel.Status - status, err := p.tc.ASStatusToStatus(statusable) - if err != nil { - return nil, gtserror.NewErrorInternalError(err) - } - - statusID, err := id.NewULIDFromTime(status.CreatedAt) - if err != nil { - return nil, err - } - status.ID = statusID - - if err := p.db.Put(status); err != nil { - return nil, fmt.Errorf("error putting status in the db: %s", err) - } - - // properly dereference everything in the status (media attachments etc) - if err := p.federator.DereferenceStatusFields(status, authed.Account.Username); err != nil { - return nil, fmt.Errorf("error dereferencing status fields: %s", err) - } - - // update with the nicely dereferenced status - if err := p.db.UpdateByID(status.ID, status); err != nil { - return nil, fmt.Errorf("error updating status in the db: %s", err) - } - return status, nil } } @@ -202,31 +160,10 @@ func (p *processor) searchAccountByURI(authed *oauth.Auth, uri *url.URL, resolve } if resolve { // we don't have it locally so try and dereference it - accountable, err := p.federator.DereferenceRemoteAccount(authed.Account.Username, uri) + account, _, err := p.federator.GetRemoteAccount(authed.Account.Username, uri, true) if err != nil { return nil, fmt.Errorf("searchAccountByURI: error dereferencing account with uri %s: %s", uri.String(), err) } - - // it IS an account! - account, err := p.tc.ASRepresentationToAccount(accountable, false) - if err != nil { - return nil, fmt.Errorf("searchAccountByURI: error dereferencing account with uri %s: %s", uri.String(), err) - } - - accountID, err := id.NewRandomULID() - if err != nil { - return nil, err - } - account.ID = accountID - - if err := p.db.Put(account); err != nil { - return nil, fmt.Errorf("searchAccountByURI: error inserting account with uri %s: %s", uri.String(), err) - } - - if err := p.federator.DereferenceAccountFields(account, authed.Account.Username, false); err != nil { - return nil, fmt.Errorf("searchAccountByURI: error further dereferencing account with uri %s: %s", uri.String(), err) - } - return account, nil } return nil, nil @@ -275,35 +212,12 @@ func (p *processor) searchAccountByMention(authed *oauth.Auth, mention string, r return nil, fmt.Errorf("searchAccountByMention: error fingering remote account with username %s and domain %s: %s", username, domain, err) } - // dereference the account based on the URI we retrieved from the webfinger lookup - accountable, err := p.federator.DereferenceRemoteAccount(authed.Account.Username, acctURI) + // we don't have it locally so try and dereference it + account, _, err := p.federator.GetRemoteAccount(authed.Account.Username, acctURI, true) if err != nil { - // something went wrong doing the dereferencing so we can't process the request - return nil, fmt.Errorf("searchAccountByMention: error dereferencing remote account with uri %s: %s", acctURI.String(), err) - } - - // convert the dereferenced account to the gts model of that account - foundAccount, err := p.tc.ASRepresentationToAccount(accountable, false) - if err != nil { - // something went wrong doing the conversion to a gtsmodel.Account so we can't process the request - return nil, fmt.Errorf("searchAccountByMention: error converting account with uri %s: %s", acctURI.String(), err) - } - - foundAccountID, err := id.NewULID() - if err != nil { - return nil, err - } - foundAccount.ID = foundAccountID - - // put this new account in our database - if err := p.db.Put(foundAccount); err != nil { - return nil, fmt.Errorf("searchAccountByMention: error inserting account with uri %s: %s", acctURI.String(), err) - } - - // properly dereference all the fields on the account immediately - if err := p.federator.DereferenceAccountFields(foundAccount, authed.Account.Username, true); err != nil { - return nil, fmt.Errorf("searchAccountByMention: error dereferencing fields on account with uri %s: %s", acctURI.String(), err) + return nil, fmt.Errorf("searchAccountByMention: error dereferencing account with uri %s: %s", acctURI.String(), err) } + return account, nil } return nil, nil diff --git a/internal/processing/status/context.go b/internal/processing/status/context.go index 72b9b5623..32c528296 100644 --- a/internal/processing/status/context.go +++ b/internal/processing/status/context.go @@ -33,7 +33,7 @@ func (p *processor) Context(account *gtsmodel.Account, targetStatusID string) (* return nil, gtserror.NewErrorForbidden(fmt.Errorf("account with id %s does not have permission to view status %s", account.ID, targetStatusID)) } - parents, err := p.db.StatusParents(targetStatus) + parents, err := p.db.StatusParents(targetStatus, false) if err != nil { return nil, gtserror.NewErrorInternalError(err) } @@ -51,7 +51,7 @@ func (p *processor) Context(account *gtsmodel.Account, targetStatusID string) (* return context.Ancestors[i].ID < context.Ancestors[j].ID }) - children, err := p.db.StatusChildren(targetStatus) + children, err := p.db.StatusChildren(targetStatus, false, "") if err != nil { return nil, gtserror.NewErrorInternalError(err) } diff --git a/internal/text/link_test.go b/internal/text/link_test.go index 15e27f870..83c42f045 100644 --- a/internal/text/link_test.go +++ b/internal/text/link_test.go @@ -86,7 +86,7 @@ func (suite *LinkTestSuite) SetupTest() { suite.log = testrig.NewTestLog() suite.formatter = text.NewFormatter(suite.config, suite.db, suite.log) - testrig.StandardDBSetup(suite.db) + testrig.StandardDBSetup(suite.db, nil) } func (suite *LinkTestSuite) TearDownTest() { diff --git a/internal/text/plain_test.go b/internal/text/plain_test.go index 1e0d1471a..183ccc478 100644 --- a/internal/text/plain_test.go +++ b/internal/text/plain_test.go @@ -57,7 +57,7 @@ func (suite *PlainTestSuite) SetupTest() { suite.log = testrig.NewTestLog() suite.formatter = text.NewFormatter(suite.config, suite.db, suite.log) - testrig.StandardDBSetup(suite.db) + testrig.StandardDBSetup(suite.db, nil) } func (suite *PlainTestSuite) TearDownTest() { diff --git a/internal/transport/controller.go b/internal/transport/controller.go index c01af0900..07d20cdcf 100644 --- a/internal/transport/controller.go +++ b/internal/transport/controller.go @@ -27,15 +27,19 @@ import ( "github.com/go-fed/httpsig" "github.com/sirupsen/logrus" "github.com/superseriousbusiness/gotosocial/internal/config" + "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" ) // Controller generates transports for use in making federation requests to other servers. type Controller interface { NewTransport(pubKeyID string, privkey crypto.PrivateKey) (Transport, error) + NewTransportForUsername(username string) (Transport, error) } type controller struct { config *config.Config + db db.DB clock pub.Clock client pub.HttpClient appAgent string @@ -43,9 +47,10 @@ type controller struct { } // NewController returns an implementation of the Controller interface for creating new transports -func NewController(config *config.Config, clock pub.Clock, client pub.HttpClient, log *logrus.Logger) Controller { +func NewController(config *config.Config, db db.DB, clock pub.Clock, client pub.HttpClient, log *logrus.Logger) Controller { return &controller{ config: config, + db: db, clock: clock, client: client, appAgent: fmt.Sprintf("%s %s", config.ApplicationName, config.Host), @@ -55,10 +60,10 @@ func NewController(config *config.Config, clock pub.Clock, client pub.HttpClient // NewTransport returns a new http signature transport with the given public key id (a URL), and the given private key. func (c *controller) NewTransport(pubKeyID string, privkey crypto.PrivateKey) (Transport, error) { - prefs := []httpsig.Algorithm{httpsig.RSA_SHA256, httpsig.RSA_SHA512} + prefs := []httpsig.Algorithm{httpsig.RSA_SHA512} digestAlgo := httpsig.DigestSha256 - getHeaders := []string{"(request-target)", "host", "date"} - postHeaders := []string{"(request-target)", "host", "date", "digest"} + getHeaders := []string{httpsig.RequestTarget, "host", "date"} + postHeaders := []string{httpsig.RequestTarget, "host", "date", "digest"} getSigner, _, err := httpsig.NewSigner(prefs, digestAlgo, getHeaders, httpsig.Signature, 120) if err != nil { @@ -85,3 +90,25 @@ func (c *controller) NewTransport(pubKeyID string, privkey crypto.PrivateKey) (T log: c.log, }, nil } + +func (c *controller) NewTransportForUsername(username string) (Transport, error) { + // We need an account to use to create a transport for dereferecing something. + // If a username has been given, we can fetch the account with that username and use it. + // Otherwise, we can take the instance account and use those credentials to make the request. + ourAccount := >smodel.Account{} + var u string + if username == "" { + u = c.config.Host + } else { + u = username + } + if err := c.db.GetLocalAccountByUsername(u, ourAccount); err != nil { + return nil, fmt.Errorf("error getting account %s from db: %s", username, err) + } + + transport, err := c.NewTransport(ourAccount.PublicKeyURI, ourAccount.PrivateKey) + if err != nil { + return nil, fmt.Errorf("error creating transport for user %s: %s", username, err) + } + return transport, nil +} diff --git a/internal/typeutils/asinterfaces.go b/internal/typeutils/asinterfaces.go deleted file mode 100644 index d0b1cf617..000000000 --- a/internal/typeutils/asinterfaces.go +++ /dev/null @@ -1,265 +0,0 @@ -/* - GoToSocial - Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . -*/ - -package typeutils - -import "github.com/go-fed/activity/streams/vocab" - -// Accountable represents the minimum activitypub interface for representing an 'account'. -// This interface is fulfilled by: Person, Application, Organization, Service, and Group -type Accountable interface { - withJSONLDId - withTypeName - - withPreferredUsername - withIcon - withName - withImage - withSummary - withDiscoverable - withURL - withPublicKey - withInbox - withOutbox - withFollowing - withFollowers - withFeatured -} - -// Statusable represents the minimum activitypub interface for representing a 'status'. -// This interface is fulfilled by: Article, Document, Image, Video, Note, Page, Event, Place, Mention, Profile -type Statusable interface { - withJSONLDId - withTypeName - - withSummary - withInReplyTo - withPublished - withURL - withAttributedTo - withTo - withCC - withSensitive - withConversation - withContent - withAttachment - withTag - withReplies -} - -// Attachmentable represents the minimum activitypub interface for representing a 'mediaAttachment'. -// This interface is fulfilled by: Audio, Document, Image, Video -type Attachmentable interface { - withTypeName - withMediaType - withURL - withName -} - -// Hashtaggable represents the minimum activitypub interface for representing a 'hashtag' tag. -type Hashtaggable interface { - withTypeName - withHref - withName -} - -// Emojiable represents the minimum interface for an 'emoji' tag. -type Emojiable interface { - withJSONLDId - withTypeName - withName - withUpdated - withIcon -} - -// Mentionable represents the minimum interface for a 'mention' tag. -type Mentionable interface { - withName - withHref -} - -// Followable represents the minimum interface for an activitystreams 'follow' activity. -type Followable interface { - withJSONLDId - withTypeName - - withActor - withObject -} - -// Likeable represents the minimum interface for an activitystreams 'like' activity. -type Likeable interface { - withJSONLDId - withTypeName - - withActor - withObject -} - -// Blockable represents the minimum interface for an activitystreams 'block' activity. -type Blockable interface { - withJSONLDId - withTypeName - - withActor - withObject -} - -// Announceable represents the minimum interface for an activitystreams 'announce' activity. -type Announceable interface { - withJSONLDId - withTypeName - - withActor - withObject - withPublished - withTo - withCC -} - -type withJSONLDId interface { - GetJSONLDId() vocab.JSONLDIdProperty -} - -type withTypeName interface { - GetTypeName() string -} - -type withPreferredUsername interface { - GetActivityStreamsPreferredUsername() vocab.ActivityStreamsPreferredUsernameProperty -} - -type withIcon interface { - GetActivityStreamsIcon() vocab.ActivityStreamsIconProperty -} - -type withName interface { - GetActivityStreamsName() vocab.ActivityStreamsNameProperty -} - -type withImage interface { - GetActivityStreamsImage() vocab.ActivityStreamsImageProperty -} - -type withSummary interface { - GetActivityStreamsSummary() vocab.ActivityStreamsSummaryProperty -} - -type withDiscoverable interface { - GetTootDiscoverable() vocab.TootDiscoverableProperty -} - -type withURL interface { - GetActivityStreamsUrl() vocab.ActivityStreamsUrlProperty -} - -type withPublicKey interface { - GetW3IDSecurityV1PublicKey() vocab.W3IDSecurityV1PublicKeyProperty -} - -type withInbox interface { - GetActivityStreamsInbox() vocab.ActivityStreamsInboxProperty -} - -type withOutbox interface { - GetActivityStreamsOutbox() vocab.ActivityStreamsOutboxProperty -} - -type withFollowing interface { - GetActivityStreamsFollowing() vocab.ActivityStreamsFollowingProperty -} - -type withFollowers interface { - GetActivityStreamsFollowers() vocab.ActivityStreamsFollowersProperty -} - -type withFeatured interface { - GetTootFeatured() vocab.TootFeaturedProperty -} - -type withAttributedTo interface { - GetActivityStreamsAttributedTo() vocab.ActivityStreamsAttributedToProperty -} - -type withAttachment interface { - GetActivityStreamsAttachment() vocab.ActivityStreamsAttachmentProperty -} - -type withTo interface { - GetActivityStreamsTo() vocab.ActivityStreamsToProperty -} - -type withInReplyTo interface { - GetActivityStreamsInReplyTo() vocab.ActivityStreamsInReplyToProperty -} - -type withCC interface { - GetActivityStreamsCc() vocab.ActivityStreamsCcProperty -} - -type withSensitive interface { - // TODO -} - -type withConversation interface { - // TODO -} - -type withContent interface { - GetActivityStreamsContent() vocab.ActivityStreamsContentProperty -} - -type withPublished interface { - GetActivityStreamsPublished() vocab.ActivityStreamsPublishedProperty -} - -type withTag interface { - GetActivityStreamsTag() vocab.ActivityStreamsTagProperty -} - -type withReplies interface { - GetActivityStreamsReplies() vocab.ActivityStreamsRepliesProperty -} - -type withMediaType interface { - GetActivityStreamsMediaType() vocab.ActivityStreamsMediaTypeProperty -} - -// type withBlurhash interface { -// GetTootBlurhashProperty() vocab.TootBlurhashProperty -// } - -// type withFocalPoint interface { -// // TODO -// } - -type withHref interface { - GetActivityStreamsHref() vocab.ActivityStreamsHrefProperty -} - -type withUpdated interface { - GetActivityStreamsUpdated() vocab.ActivityStreamsUpdatedProperty -} - -type withActor interface { - GetActivityStreamsActor() vocab.ActivityStreamsActorProperty -} - -type withObject interface { - GetActivityStreamsObject() vocab.ActivityStreamsObjectProperty -} diff --git a/internal/typeutils/astointernal.go b/internal/typeutils/astointernal.go index 394de6e82..f754d282a 100644 --- a/internal/typeutils/astointernal.go +++ b/internal/typeutils/astointernal.go @@ -24,11 +24,12 @@ import ( "net/url" "strings" + "github.com/superseriousbusiness/gotosocial/internal/ap" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" ) -func (c *converter) ASRepresentationToAccount(accountable Accountable, update bool) (*gtsmodel.Account, error) { +func (c *converter) ASRepresentationToAccount(accountable ap.Accountable, update bool) (*gtsmodel.Account, error) { // first check if we actually already know this account uriProp := accountable.GetJSONLDId() if uriProp == nil || !uriProp.IsIRI() { @@ -55,7 +56,7 @@ func (c *converter) ASRepresentationToAccount(accountable Accountable, update bo // Username aka preferredUsername // We need this one so bail if it's not set. - username, err := extractPreferredUsername(accountable) + username, err := ap.ExtractPreferredUsername(accountable) if err != nil { return nil, fmt.Errorf("couldn't extract username: %s", err) } @@ -66,27 +67,27 @@ func (c *converter) ASRepresentationToAccount(accountable Accountable, update bo // avatar aka icon // if this one isn't extractable in a format we recognise we'll just skip it - if avatarURL, err := extractIconURL(accountable); err == nil { + if avatarURL, err := ap.ExtractIconURL(accountable); err == nil { acct.AvatarRemoteURL = avatarURL.String() } // header aka image // if this one isn't extractable in a format we recognise we'll just skip it - if headerURL, err := extractImageURL(accountable); err == nil { + if headerURL, err := ap.ExtractImageURL(accountable); err == nil { acct.HeaderRemoteURL = headerURL.String() } // display name aka name // we default to the username, but take the more nuanced name property if it exists acct.DisplayName = username - if displayName, err := extractName(accountable); err == nil { + if displayName, err := ap.ExtractName(accountable); err == nil { acct.DisplayName = displayName } // TODO: fields aka attachment array // note aka summary - note, err := extractSummary(accountable) + note, err := ap.ExtractSummary(accountable) if err == nil && note != "" { acct.Note = note } @@ -110,13 +111,13 @@ func (c *converter) ASRepresentationToAccount(accountable Accountable, update bo // discoverable // default to false -- take custom value if it's set though acct.Discoverable = false - discoverable, err := extractDiscoverable(accountable) + discoverable, err := ap.ExtractDiscoverable(accountable) if err == nil { acct.Discoverable = discoverable } // url property - url, err := extractURL(accountable) + url, err := ap.ExtractURL(accountable) if err == nil { // take the URL if we can find it acct.URL = url.String() @@ -155,7 +156,7 @@ func (c *converter) ASRepresentationToAccount(accountable Accountable, update bo // TODO: alsoKnownAs // publicKey - pkey, pkeyURL, err := extractPublicKeyForOwner(accountable, uri) + pkey, pkeyURL, err := ap.ExtractPublicKeyForOwner(accountable, uri) if err != nil { return nil, fmt.Errorf("couldn't get public key for person %s: %s", uri.String(), err) } @@ -165,7 +166,7 @@ func (c *converter) ASRepresentationToAccount(accountable Accountable, update bo return acct, nil } -func (c *converter) ASStatusToStatus(statusable Statusable) (*gtsmodel.Status, error) { +func (c *converter) ASStatusToStatus(statusable ap.Statusable) (*gtsmodel.Status, error) { status := >smodel.Status{} // uri at which this status is reachable @@ -176,49 +177,49 @@ func (c *converter) ASStatusToStatus(statusable Statusable) (*gtsmodel.Status, e status.URI = uriProp.GetIRI().String() // web url for viewing this status - if statusURL, err := extractURL(statusable); err == nil { + if statusURL, err := ap.ExtractURL(statusable); err == nil { status.URL = statusURL.String() } // the html-formatted content of this status - if content, err := extractContent(statusable); err == nil { + if content, err := ap.ExtractContent(statusable); err == nil { status.Content = content } // attachments to dereference and fetch later on (we don't do that here) - if attachments, err := extractAttachments(statusable); err == nil { + if attachments, err := ap.ExtractAttachments(statusable); err == nil { status.GTSMediaAttachments = attachments } // hashtags to dereference later on - if hashtags, err := extractHashtags(statusable); err == nil { + if hashtags, err := ap.ExtractHashtags(statusable); err == nil { status.GTSTags = hashtags } // emojis to dereference and fetch later on - if emojis, err := extractEmojis(statusable); err == nil { + if emojis, err := ap.ExtractEmojis(statusable); err == nil { status.GTSEmojis = emojis } // mentions to dereference later on - if mentions, err := extractMentions(statusable); err == nil { + if mentions, err := ap.ExtractMentions(statusable); err == nil { status.GTSMentions = mentions } // cw string for this status - if cw, err := extractSummary(statusable); err == nil { + if cw, err := ap.ExtractSummary(statusable); err == nil { status.ContentWarning = cw } // when was this status created? - published, err := extractPublished(statusable) + published, err := ap.ExtractPublished(statusable) if err == nil { status.CreatedAt = published } // which account posted this status? // if we don't know the account yet we can dereference it later - attributedTo, err := extractAttributedTo(statusable) + attributedTo, err := ap.ExtractAttributedTo(statusable) if err != nil { return nil, errors.New("attributedTo was empty") } @@ -233,8 +234,8 @@ func (c *converter) ASStatusToStatus(statusable Statusable) (*gtsmodel.Status, e status.GTSAuthorAccount = statusOwner // check if there's a post that this is a reply to - inReplyToURI, err := extractInReplyToURI(statusable) - if err == nil { + inReplyToURI := ap.ExtractInReplyToURI(statusable) + if inReplyToURI != nil { // something is set so we can at least set this field on the // status and dereference using this later if we need to status.InReplyToURI = inReplyToURI.String() @@ -259,12 +260,12 @@ func (c *converter) ASStatusToStatus(statusable Statusable) (*gtsmodel.Status, e // visibility entry for this status var visibility gtsmodel.Visibility - to, err := extractTos(statusable) + to, err := ap.ExtractTos(statusable) if err != nil { return nil, fmt.Errorf("error extracting TO values: %s", err) } - cc, err := extractCCs(statusable) + cc, err := ap.ExtractCCs(statusable) if err != nil { return nil, fmt.Errorf("error extracting CC values: %s", err) } @@ -315,7 +316,7 @@ func (c *converter) ASStatusToStatus(statusable Statusable) (*gtsmodel.Status, e return status, nil } -func (c *converter) ASFollowToFollowRequest(followable Followable) (*gtsmodel.FollowRequest, error) { +func (c *converter) ASFollowToFollowRequest(followable ap.Followable) (*gtsmodel.FollowRequest, error) { idProp := followable.GetJSONLDId() if idProp == nil || !idProp.IsIRI() { @@ -323,7 +324,7 @@ func (c *converter) ASFollowToFollowRequest(followable Followable) (*gtsmodel.Fo } uri := idProp.GetIRI().String() - origin, err := extractActor(followable) + origin, err := ap.ExtractActor(followable) if err != nil { return nil, errors.New("error extracting actor property from follow") } @@ -332,7 +333,7 @@ func (c *converter) ASFollowToFollowRequest(followable Followable) (*gtsmodel.Fo return nil, fmt.Errorf("error extracting account with uri %s from the database: %s", origin.String(), err) } - target, err := extractObject(followable) + target, err := ap.ExtractObject(followable) if err != nil { return nil, errors.New("error extracting object property from follow") } @@ -350,14 +351,14 @@ func (c *converter) ASFollowToFollowRequest(followable Followable) (*gtsmodel.Fo return followRequest, nil } -func (c *converter) ASFollowToFollow(followable Followable) (*gtsmodel.Follow, error) { +func (c *converter) ASFollowToFollow(followable ap.Followable) (*gtsmodel.Follow, error) { idProp := followable.GetJSONLDId() if idProp == nil || !idProp.IsIRI() { return nil, errors.New("no id property set on follow, or was not an iri") } uri := idProp.GetIRI().String() - origin, err := extractActor(followable) + origin, err := ap.ExtractActor(followable) if err != nil { return nil, errors.New("error extracting actor property from follow") } @@ -366,7 +367,7 @@ func (c *converter) ASFollowToFollow(followable Followable) (*gtsmodel.Follow, e return nil, fmt.Errorf("error extracting account with uri %s from the database: %s", origin.String(), err) } - target, err := extractObject(followable) + target, err := ap.ExtractObject(followable) if err != nil { return nil, errors.New("error extracting object property from follow") } @@ -384,14 +385,14 @@ func (c *converter) ASFollowToFollow(followable Followable) (*gtsmodel.Follow, e return follow, nil } -func (c *converter) ASLikeToFave(likeable Likeable) (*gtsmodel.StatusFave, error) { +func (c *converter) ASLikeToFave(likeable ap.Likeable) (*gtsmodel.StatusFave, error) { idProp := likeable.GetJSONLDId() if idProp == nil || !idProp.IsIRI() { return nil, errors.New("no id property set on like, or was not an iri") } uri := idProp.GetIRI().String() - origin, err := extractActor(likeable) + origin, err := ap.ExtractActor(likeable) if err != nil { return nil, errors.New("error extracting actor property from like") } @@ -400,7 +401,7 @@ func (c *converter) ASLikeToFave(likeable Likeable) (*gtsmodel.StatusFave, error return nil, fmt.Errorf("error extracting account with uri %s from the database: %s", origin.String(), err) } - target, err := extractObject(likeable) + target, err := ap.ExtractObject(likeable) if err != nil { return nil, errors.New("error extracting object property from like") } @@ -426,14 +427,14 @@ func (c *converter) ASLikeToFave(likeable Likeable) (*gtsmodel.StatusFave, error }, nil } -func (c *converter) ASBlockToBlock(blockable Blockable) (*gtsmodel.Block, error) { +func (c *converter) ASBlockToBlock(blockable ap.Blockable) (*gtsmodel.Block, error) { idProp := blockable.GetJSONLDId() if idProp == nil || !idProp.IsIRI() { return nil, errors.New("ASBlockToBlock: no id property set on block, or was not an iri") } uri := idProp.GetIRI().String() - origin, err := extractActor(blockable) + origin, err := ap.ExtractActor(blockable) if err != nil { return nil, errors.New("ASBlockToBlock: error extracting actor property from block") } @@ -442,7 +443,7 @@ func (c *converter) ASBlockToBlock(blockable Blockable) (*gtsmodel.Block, error) return nil, fmt.Errorf("ASBlockToBlock: error extracting account with uri %s from the database: %s", origin.String(), err) } - target, err := extractObject(blockable) + target, err := ap.ExtractObject(blockable) if err != nil { return nil, errors.New("ASBlockToBlock: error extracting object property from block") } @@ -461,7 +462,7 @@ func (c *converter) ASBlockToBlock(blockable Blockable) (*gtsmodel.Block, error) }, nil } -func (c *converter) ASAnnounceToStatus(announceable Announceable) (*gtsmodel.Status, bool, error) { +func (c *converter) ASAnnounceToStatus(announceable ap.Announceable) (*gtsmodel.Status, bool, error) { status := >smodel.Status{} isNew := true @@ -480,7 +481,7 @@ func (c *converter) ASAnnounceToStatus(announceable Announceable) (*gtsmodel.Sta status.URI = uri // get the URI of the announced/boosted status - boostedStatusURI, err := extractObject(announceable) + boostedStatusURI, err := ap.ExtractObject(announceable) if err != nil { return nil, isNew, fmt.Errorf("ASAnnounceToStatus: error getting object from announce: %s", err) } @@ -491,7 +492,7 @@ func (c *converter) ASAnnounceToStatus(announceable Announceable) (*gtsmodel.Sta } // get the published time for the announce - published, err := extractPublished(announceable) + published, err := ap.ExtractPublished(announceable) if err != nil { return nil, isNew, fmt.Errorf("ASAnnounceToStatus: error extracting published time: %s", err) } @@ -499,7 +500,7 @@ func (c *converter) ASAnnounceToStatus(announceable Announceable) (*gtsmodel.Sta status.UpdatedAt = published // get the actor's IRI (ie., the person who boosted the status) - actor, err := extractActor(announceable) + actor, err := ap.ExtractActor(announceable) if err != nil { return nil, isNew, fmt.Errorf("ASAnnounceToStatus: error extracting actor: %s", err) } @@ -522,12 +523,12 @@ func (c *converter) ASAnnounceToStatus(announceable Announceable) (*gtsmodel.Sta // parse the visibility from the To and CC entries var visibility gtsmodel.Visibility - to, err := extractTos(announceable) + to, err := ap.ExtractTos(announceable) if err != nil { return nil, isNew, fmt.Errorf("error extracting TO values: %s", err) } - cc, err := extractCCs(announceable) + cc, err := ap.ExtractCCs(announceable) if err != nil { return nil, isNew, fmt.Errorf("error extracting CC values: %s", err) } diff --git a/internal/typeutils/astointernal_test.go b/internal/typeutils/astointernal_test.go index 9d6ce4e0a..2e33271c5 100644 --- a/internal/typeutils/astointernal_test.go +++ b/internal/typeutils/astointernal_test.go @@ -28,6 +28,7 @@ import ( "github.com/go-fed/activity/streams/vocab" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/ap" "github.com/superseriousbusiness/gotosocial/internal/typeutils" "github.com/superseriousbusiness/gotosocial/testrig" ) @@ -342,7 +343,7 @@ func (suite *ASToInternalTestSuite) SetupSuite() { } func (suite *ASToInternalTestSuite) SetupTest() { - testrig.StandardDBSetup(suite.db) + testrig.StandardDBSetup(suite.db, nil) } func (suite *ASToInternalTestSuite) TestParsePerson() { @@ -364,7 +365,7 @@ func (suite *ASToInternalTestSuite) TestParseGargron() { t, err := streams.ToType(context.Background(), m) assert.NoError(suite.T(), err) - rep, ok := t.(typeutils.Accountable) + rep, ok := t.(ap.Accountable) assert.True(suite.T(), ok) acct, err := suite.typeconverter.ASRepresentationToAccount(rep, false) @@ -391,7 +392,7 @@ func (suite *ASToInternalTestSuite) TestParseStatus() { first := obj.Begin() assert.NotNil(suite.T(), first) - rep, ok := first.GetType().(typeutils.Statusable) + rep, ok := first.GetType().(ap.Statusable) assert.True(suite.T(), ok) status, err := suite.typeconverter.ASStatusToStatus(rep) @@ -418,7 +419,7 @@ func (suite *ASToInternalTestSuite) TestParseStatusWithMention() { first := obj.Begin() assert.NotNil(suite.T(), first) - rep, ok := first.GetType().(typeutils.Statusable) + rep, ok := first.GetType().(ap.Statusable) assert.True(suite.T(), ok) status, err := suite.typeconverter.ASStatusToStatus(rep) diff --git a/internal/typeutils/converter.go b/internal/typeutils/converter.go index 57c2a1f6d..10d9a0f18 100644 --- a/internal/typeutils/converter.go +++ b/internal/typeutils/converter.go @@ -19,7 +19,10 @@ package typeutils import ( + "net/url" + "github.com/go-fed/activity/streams/vocab" + "github.com/superseriousbusiness/gotosocial/internal/ap" "github.com/superseriousbusiness/gotosocial/internal/api/model" "github.com/superseriousbusiness/gotosocial/internal/config" "github.com/superseriousbusiness/gotosocial/internal/db" @@ -99,17 +102,17 @@ type TypeConverter interface { // If update is false, and the account is already known in the database, then the existing account entry will be returned. // If update is true, then even if the account is already known, all fields in the accountable will be parsed and a new *gtsmodel.Account // will be generated. This is useful when one needs to force refresh of an account, eg., during an Update of a Profile. - ASRepresentationToAccount(accountable Accountable, update bool) (*gtsmodel.Account, error) + ASRepresentationToAccount(accountable ap.Accountable, update bool) (*gtsmodel.Account, error) // ASStatus converts a remote activitystreams 'status' representation into a gts model status. - ASStatusToStatus(statusable Statusable) (*gtsmodel.Status, error) + ASStatusToStatus(statusable ap.Statusable) (*gtsmodel.Status, error) // ASFollowToFollowRequest converts a remote activitystreams `follow` representation into gts model follow request. - ASFollowToFollowRequest(followable Followable) (*gtsmodel.FollowRequest, error) + ASFollowToFollowRequest(followable ap.Followable) (*gtsmodel.FollowRequest, error) // ASFollowToFollowRequest converts a remote activitystreams `follow` representation into gts model follow. - ASFollowToFollow(followable Followable) (*gtsmodel.Follow, error) + ASFollowToFollow(followable ap.Followable) (*gtsmodel.Follow, error) // ASLikeToFave converts a remote activitystreams 'like' representation into a gts model status fave. - ASLikeToFave(likeable Likeable) (*gtsmodel.StatusFave, error) + ASLikeToFave(likeable ap.Likeable) (*gtsmodel.StatusFave, error) // ASBlockToBlock converts a remote activity streams 'block' representation into a gts model block. - ASBlockToBlock(blockable Blockable) (*gtsmodel.Block, error) + ASBlockToBlock(blockable ap.Blockable) (*gtsmodel.Block, error) // ASAnnounceToStatus converts an activitystreams 'announce' into a status. // // The returned bool indicates whether this status is new (true) or not new (false). @@ -122,7 +125,7 @@ type TypeConverter interface { // This is useful when multiple users on an instance might receive the same boost, and we only want to process the boost once. // // NOTE -- this is different from one status being boosted multiple times! In this case, new boosts should indeed be created. - ASAnnounceToStatus(announceable Announceable) (status *gtsmodel.Status, new bool, err error) + ASAnnounceToStatus(announceable ap.Announceable) (status *gtsmodel.Status, new bool, err error) /* INTERNAL (gts) MODEL TO ACTIVITYSTREAMS MODEL @@ -150,7 +153,10 @@ type TypeConverter interface { BoostToAS(boostWrapperStatus *gtsmodel.Status, boostingAccount *gtsmodel.Account, boostedAccount *gtsmodel.Account) (vocab.ActivityStreamsAnnounce, error) // BlockToAS converts a gts model block into an activityStreams BLOCK, suitable for federation. BlockToAS(block *gtsmodel.Block) (vocab.ActivityStreamsBlock, error) - + // StatusToASRepliesCollection converts a gts model status into an activityStreams REPLIES collection. + StatusToASRepliesCollection(status *gtsmodel.Status, onlyOtherAccounts bool) (vocab.ActivityStreamsCollection, error) + // StatusURIsToASRepliesPage returns a collection page with appropriate next/part of pagination. + StatusURIsToASRepliesPage(status *gtsmodel.Status, onlyOtherAccounts bool, minID string, replies map[string]*url.URL) (vocab.ActivityStreamsCollectionPage, error) /* INTERNAL (gts) MODEL TO INTERNAL MODEL */ diff --git a/internal/typeutils/converter_test.go b/internal/typeutils/converter_test.go index b2272f50c..c104ab06c 100644 --- a/internal/typeutils/converter_test.go +++ b/internal/typeutils/converter_test.go @@ -21,6 +21,7 @@ package typeutils_test import ( "github.com/sirupsen/logrus" "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/ap" "github.com/superseriousbusiness/gotosocial/internal/config" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" @@ -34,7 +35,7 @@ type ConverterStandardTestSuite struct { db db.DB log *logrus.Logger accounts map[string]*gtsmodel.Account - people map[string]typeutils.Accountable + people map[string]ap.Accountable typeconverter typeutils.TypeConverter } diff --git a/internal/typeutils/internaltoas.go b/internal/typeutils/internaltoas.go index b24b07e13..333f131d4 100644 --- a/internal/typeutils/internaltoas.go +++ b/internal/typeutils/internaltoas.go @@ -27,6 +27,7 @@ import ( "github.com/go-fed/activity/streams" "github.com/go-fed/activity/streams/vocab" "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/gtserror" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" ) @@ -505,7 +506,14 @@ func (c *converter) StatusToAS(s *gtsmodel.Status) (vocab.ActivityStreamsNote, e status.SetActivityStreamsAttachment(attachmentProp) // replies - // TODO + repliesCollection, err := c.StatusToASRepliesCollection(s, false) + if err != nil { + return nil, fmt.Errorf("error creating repliesCollection: %s", err) + } + + repliesProp := streams.NewActivityStreamsRepliesProperty() + repliesProp.SetActivityStreamsCollection(repliesCollection) + status.SetActivityStreamsReplies(repliesProp) return status, nil } @@ -850,3 +858,138 @@ func (c *converter) BlockToAS(b *gtsmodel.Block) (vocab.ActivityStreamsBlock, er return block, nil } + +/* + the goal is to end up with something like this: + + { + "@context": "https://www.w3.org/ns/activitystreams", + "id": "https://example.org/users/whatever/statuses/01FCNEXAGAKPEX1J7VJRPJP490/replies", + "type": "Collection", + "first": { + "id": "https://example.org/users/whatever/statuses/01FCNEXAGAKPEX1J7VJRPJP490/replies?page=true", + "type": "CollectionPage", + "next": "https://example.org/users/whatever/statuses/01FCNEXAGAKPEX1J7VJRPJP490/replies?only_other_accounts=true&page=true", + "partOf": "https://example.org/users/whatever/statuses/01FCNEXAGAKPEX1J7VJRPJP490/replies", + "items": [] + } + } +*/ +func (c *converter) StatusToASRepliesCollection(status *gtsmodel.Status, onlyOtherAccounts bool) (vocab.ActivityStreamsCollection, error) { + collectionID := fmt.Sprintf("%s/replies", status.URI) + collectionIDURI, err := url.Parse(collectionID) + if err != nil { + return nil, err + } + + collection := streams.NewActivityStreamsCollection() + + // collection.id + collectionIDProp := streams.NewJSONLDIdProperty() + collectionIDProp.SetIRI(collectionIDURI) + collection.SetJSONLDId(collectionIDProp) + + // first + first := streams.NewActivityStreamsFirstProperty() + firstPage := streams.NewActivityStreamsCollectionPage() + + // first.id + firstPageIDProp := streams.NewJSONLDIdProperty() + firstPageID, err := url.Parse(fmt.Sprintf("%s?page=true", collectionID)) + if err != nil { + return nil, gtserror.NewErrorInternalError(err) + } + firstPageIDProp.SetIRI(firstPageID) + firstPage.SetJSONLDId(firstPageIDProp) + + // first.next + nextProp := streams.NewActivityStreamsNextProperty() + nextPropID, err := url.Parse(fmt.Sprintf("%s?only_other_accounts=%t&page=true", collectionID, onlyOtherAccounts)) + if err != nil { + return nil, gtserror.NewErrorInternalError(err) + } + nextProp.SetIRI(nextPropID) + firstPage.SetActivityStreamsNext(nextProp) + + // first.partOf + partOfProp := streams.NewActivityStreamsPartOfProperty() + partOfProp.SetIRI(collectionIDURI) + firstPage.SetActivityStreamsPartOf(partOfProp) + + first.SetActivityStreamsCollectionPage(firstPage) + + // collection.first + collection.SetActivityStreamsFirst(first) + + return collection, nil +} + +/* + the goal is to end up with something like this: + { + "@context": "https://www.w3.org/ns/activitystreams", + "id": "https://example.org/users/whatever/statuses/01FCNEXAGAKPEX1J7VJRPJP490/replies?only_other_accounts=true&page=true", + "type": "CollectionPage", + "next": "https://example.org/users/whatever/statuses/01FCNEXAGAKPEX1J7VJRPJP490/replies?min_id=106720870266901180&only_other_accounts=true&page=true", + "partOf": "https://example.org/users/whatever/statuses/01FCNEXAGAKPEX1J7VJRPJP490/replies", + "items": [ + "https://example.com/users/someone/statuses/106720752853216226", + "https://somewhere.online/users/eeeeeeeeeep/statuses/106720870163727231" + ] + } +*/ +func (c *converter) StatusURIsToASRepliesPage(status *gtsmodel.Status, onlyOtherAccounts bool, minID string, replies map[string]*url.URL) (vocab.ActivityStreamsCollectionPage, error) { + collectionID := fmt.Sprintf("%s/replies", status.URI) + + page := streams.NewActivityStreamsCollectionPage() + + // .id + pageIDProp := streams.NewJSONLDIdProperty() + pageIDString := fmt.Sprintf("%s?page=true&only_other_accounts=%t", collectionID, onlyOtherAccounts) + if minID != "" { + pageIDString = fmt.Sprintf("%s&min_id=%s", pageIDString, minID) + } + + pageID, err := url.Parse(pageIDString) + if err != nil { + return nil, gtserror.NewErrorInternalError(err) + } + pageIDProp.SetIRI(pageID) + page.SetJSONLDId(pageIDProp) + + // .partOf + collectionIDURI, err := url.Parse(collectionID) + if err != nil { + return nil, err + } + partOfProp := streams.NewActivityStreamsPartOfProperty() + partOfProp.SetIRI(collectionIDURI) + page.SetActivityStreamsPartOf(partOfProp) + + // .items + items := streams.NewActivityStreamsItemsProperty() + var highestID string + for k, v := range replies { + items.AppendIRI(v) + if k > highestID { + highestID = k + } + } + page.SetActivityStreamsItems(items) + + // .next + nextProp := streams.NewActivityStreamsNextProperty() + nextPropIDString := fmt.Sprintf("%s?only_other_accounts=%t&page=true", collectionID, onlyOtherAccounts) + if highestID != "" { + nextPropIDString = fmt.Sprintf("%s&min_id=%s", nextPropIDString, highestID) + } + + nextPropID, err := url.Parse(nextPropIDString) + if err != nil { + return nil, gtserror.NewErrorInternalError(err) + } + nextProp.SetIRI(nextPropID) + page.SetActivityStreamsNext(nextProp) + + return page, nil +} diff --git a/internal/typeutils/internaltoas_test.go b/internal/typeutils/internaltoas_test.go index 8eb827e35..caa56ce0d 100644 --- a/internal/typeutils/internaltoas_test.go +++ b/internal/typeutils/internaltoas_test.go @@ -47,7 +47,7 @@ func (suite *InternalToASTestSuite) SetupSuite() { } func (suite *InternalToASTestSuite) SetupTest() { - testrig.StandardDBSetup(suite.db) + testrig.StandardDBSetup(suite.db, nil) } // TearDownTest drops tables to make sure there's no data in the db diff --git a/testrig/db.go b/testrig/db.go index 01cf93934..fe38c3164 100644 --- a/testrig/db.go +++ b/testrig/db.go @@ -65,7 +65,14 @@ func NewTestDB() db.DB { } // StandardDBSetup populates a given db with all the necessary tables/models for perfoming tests. -func StandardDBSetup(db db.DB) { +// +// The accounts parameter is provided in case the db should be populated with a certain set of accounts. +// If accounts is nil, then the standard test accounts will be used. +// +// When testing http signatures, you should pass into this function the same accounts map that you generated +// signatures with, otherwise this function will randomly generate new keys for accounts and signature +// verification will fail. +func StandardDBSetup(db db.DB, accounts map[string]*gtsmodel.Account) { for _, m := range testModels { if err := db.CreateTable(m); err != nil { panic(err) @@ -96,9 +103,17 @@ func StandardDBSetup(db db.DB) { } } - for _, v := range NewTestAccounts() { - if err := db.Put(v); err != nil { - panic(err) + if accounts == nil { + for _, v := range NewTestAccounts() { + if err := db.Put(v); err != nil { + panic(err) + } + } + } else { + for _, v := range accounts { + if err := db.Put(v); err != nil { + panic(err) + } } } diff --git a/testrig/testmodels.go b/testrig/testmodels.go index 1934170d2..da5cbe7af 100644 --- a/testrig/testmodels.go +++ b/testrig/testmodels.go @@ -36,9 +36,9 @@ import ( "github.com/go-fed/activity/pub" "github.com/go-fed/activity/streams" "github.com/go-fed/activity/streams/vocab" + "github.com/superseriousbusiness/gotosocial/internal/ap" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/oauth" - "github.com/superseriousbusiness/gotosocial/internal/typeutils" ) // NewTestTokens returns a map of tokens keyed according to which account the token belongs to. @@ -443,9 +443,9 @@ func NewTestAccounts() map[string]*gtsmodel.Account { FeaturedCollectionURI: "http://fossbros-anonymous.io/users/foss_satan/collections/featured", ActorType: gtsmodel.ActivityStreamsPerson, AlsoKnownAs: "", - PrivateKey: nil, + PrivateKey: &rsa.PrivateKey{}, PublicKey: &rsa.PublicKey{}, - PublicKeyURI: "http://fossbros-anonymous.io/users/foss_satan#main-key", + PublicKeyURI: "http://fossbros-anonymous.io/users/foss_satan/main-key", SensitizedAt: time.Time{}, SilencedAt: time.Time{}, SuspendedAt: time.Time{}, @@ -1033,6 +1033,32 @@ func NewTestStatuses() map[string]*gtsmodel.Status { }, ActivityStreamsType: gtsmodel.ActivityStreamsNote, }, + "local_account_2_status_5": { + ID: "01FCQSQ667XHJ9AV9T27SJJSX5", + URI: "http://localhost:8080/users/1happyturtle/statuses/01FCQSQ667XHJ9AV9T27SJJSX5", + URL: "http://localhost:8080/@1happyturtle/statuses/01FCQSQ667XHJ9AV9T27SJJSX5", + Content: "🐢 hi zork! 🐢", + CreatedAt: time.Now().Add(-1 * time.Minute), + UpdatedAt: time.Now().Add(-1 * time.Minute), + Local: true, + AccountID: "01F8MH5NBDF2MV7CTC4Q5128HF", + InReplyToID: "01F8MHAMCHF6Y650WCRSCP4WMY", + InReplyToAccountID: "01F8MH1H7YV1Z7D2C8K2730QBF", + InReplyToURI: "http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY", + BoostOfID: "", + ContentWarning: "", + Visibility: gtsmodel.VisibilityPublic, + Sensitive: false, + Language: "en", + CreatedWithApplicationID: "01F8MGYG9E893WRHW0TAEXR8GJ", + VisibilityAdvanced: >smodel.VisibilityAdvanced{ + Federated: true, + Boostable: true, + Replyable: true, + Likeable: true, + }, + ActivityStreamsType: gtsmodel.ActivityStreamsNote, + }, } } @@ -1155,14 +1181,14 @@ func NewTestActivities(accounts map[string]*gtsmodel.Account) map[string]Activit } // NewTestFediPeople returns a bunch of activity pub Person representations for testing converters and so on. -func NewTestFediPeople() map[string]typeutils.Accountable { +func NewTestFediPeople() map[string]ap.Accountable { newPerson1Priv, err := rsa.GenerateKey(rand.Reader, 2048) if err != nil { panic(err) } newPerson1Pub := &newPerson1Priv.PublicKey - return map[string]typeutils.Accountable{ + return map[string]ap.Accountable{ "new_person_1": newPerson( URLMustParse("https://unknown-instance.com/users/brand_new_person"), URLMustParse("https://unknown-instance.com/users/brand_new_person/following"), @@ -1187,13 +1213,47 @@ func NewTestFediPeople() map[string]typeutils.Accountable { // NewTestDereferenceRequests returns a map of incoming dereference requests, with their signatures. func NewTestDereferenceRequests(accounts map[string]*gtsmodel.Account) map[string]ActivityWithSignature { - sig, digest, date := getSignatureForDereference(accounts["remote_account_1"].PublicKeyURI, accounts["remote_account_1"].PrivateKey, URLMustParse(accounts["local_account_1"].URI)) + var sig, digest, date string + var target *url.URL + statuses := NewTestStatuses() + + target = URLMustParse(accounts["local_account_1"].URI) + sig, digest, date = getSignatureForDereference(accounts["remote_account_1"].PublicKeyURI, accounts["remote_account_1"].PrivateKey, target) + fossSatanDereferenceZork := ActivityWithSignature{ + SignatureHeader: sig, + DigestHeader: digest, + DateHeader: date, + } + + target = URLMustParse(statuses["local_account_1_status_1"].URI + "/replies") + sig, digest, date = getSignatureForDereference(accounts["remote_account_1"].PublicKeyURI, accounts["remote_account_1"].PrivateKey, target) + fossSatanDereferenceLocalAccount1Status1Replies := ActivityWithSignature{ + SignatureHeader: sig, + DigestHeader: digest, + DateHeader: date, + } + + target = URLMustParse(statuses["local_account_1_status_1"].URI + "/replies?only_other_accounts=false&page=true") + sig, digest, date = getSignatureForDereference(accounts["remote_account_1"].PublicKeyURI, accounts["remote_account_1"].PrivateKey, target) + fossSatanDereferenceLocalAccount1Status1RepliesNext := ActivityWithSignature{ + SignatureHeader: sig, + DigestHeader: digest, + DateHeader: date, + } + + target = URLMustParse(statuses["local_account_1_status_1"].URI + "/replies?only_other_accounts=false&page=true&min_id=01FCQSQ667XHJ9AV9T27SJJSX5") + sig, digest, date = getSignatureForDereference(accounts["remote_account_1"].PublicKeyURI, accounts["remote_account_1"].PrivateKey, target) + fossSatanDereferenceLocalAccount1Status1RepliesLast := ActivityWithSignature{ + SignatureHeader: sig, + DigestHeader: digest, + DateHeader: date, + } + return map[string]ActivityWithSignature{ - "foss_satan_dereference_zork": { - SignatureHeader: sig, - DigestHeader: digest, - DateHeader: date, - }, + "foss_satan_dereference_zork": fossSatanDereferenceZork, + "foss_satan_dereference_local_account_1_status_1_replies": fossSatanDereferenceLocalAccount1Status1Replies, + "foss_satan_dereference_local_account_1_status_1_replies_next": fossSatanDereferenceLocalAccount1Status1RepliesNext, + "foss_satan_dereference_local_account_1_status_1_replies_last": fossSatanDereferenceLocalAccount1Status1RepliesLast, } } @@ -1215,7 +1275,7 @@ func getSignatureForActivity(activity pub.Activity, pubKeyID string, privkey cry } // use the client to create a new transport - c := NewTestTransportController(client) + c := NewTestTransportController(client, NewTestDB()) tp, err := c.NewTransport(pubKeyID, privkey) if err != nil { panic(err) @@ -1247,7 +1307,6 @@ func getSignatureForDereference(pubKeyID string, privkey crypto.PrivateKey, dest client := &mockHTTPClient{ do: func(req *http.Request) (*http.Response, error) { signatureHeader = req.Header.Get("Signature") - digestHeader = req.Header.Get("Digest") dateHeader = req.Header.Get("Date") r := ioutil.NopCloser(bytes.NewReader([]byte{})) // we only need this so the 'close' func doesn't nil out return &http.Response{ @@ -1258,7 +1317,7 @@ func getSignatureForDereference(pubKeyID string, privkey crypto.PrivateKey, dest } // use the client to create a new transport - c := NewTestTransportController(client) + c := NewTestTransportController(client, NewTestDB()) tp, err := c.NewTransport(pubKeyID, privkey) if err != nil { panic(err) @@ -1290,7 +1349,7 @@ func newPerson( avatarURL *url.URL, avatarContentType string, headerURL *url.URL, - headerContentType string) typeutils.Accountable { + headerContentType string) ap.Accountable { person := streams.NewActivityStreamsPerson() // id should be the activitypub URI of this user diff --git a/testrig/transportcontroller.go b/testrig/transportcontroller.go index f2b5b93f7..ec591a9b1 100644 --- a/testrig/transportcontroller.go +++ b/testrig/transportcontroller.go @@ -24,6 +24,7 @@ import ( "net/http" "github.com/go-fed/activity/pub" + "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/federation" "github.com/superseriousbusiness/gotosocial/internal/transport" ) @@ -37,8 +38,8 @@ import ( // Unlike the other test interfaces provided in this package, you'll probably want to call this function // PER TEST rather than per suite, so that the do function can be set on a test by test (or even more granular) // basis. -func NewTestTransportController(client pub.HttpClient) transport.Controller { - return transport.NewController(NewTestConfig(), &federation.Clock{}, client, NewTestLog()) +func NewTestTransportController(client pub.HttpClient, db db.DB) transport.Controller { + return transport.NewController(NewTestConfig(), db, &federation.Clock{}, client, NewTestLog()) } // NewMockHTTPClient returns a client that conforms to the pub.HttpClient interface,