From 6b4f6dc7555e4a4a632ee1654596b8ed4d09853e Mon Sep 17 00:00:00 2001 From: tobi <31960611+tsmethurst@users.noreply.github.com> Date: Wed, 26 Apr 2023 17:17:22 +0200 Subject: [PATCH] [bugfix] Fix remaining mangled URI escaping issues in statuses + accounts (#1712) * start fiddling with normalize + extract functions * normalize attachment name (image description) * NormalizeAccountableSummary * normalize summary + name --- internal/ap/extract.go | 17 +- internal/ap/interfaces.go | 14 ++ internal/ap/normalize.go | 146 +++++++++-- internal/ap/normalize_test.go | 346 +++++++++++++++++++++++++-- internal/ap/resolve.go | 12 +- internal/processing/fromfederator.go | 12 +- 6 files changed, 498 insertions(+), 49 deletions(-) diff --git a/internal/ap/extract.go b/internal/ap/extract.go index c0a6ab5b5..2742d27ac 100644 --- a/internal/ap/extract.go +++ b/internal/ap/extract.go @@ -56,10 +56,13 @@ func ExtractName(i WithName) string { return "" } - // take the first name string we can find + // Take the first useful value for the name string we can find. for iter := nameProp.Begin(); iter != nameProp.End(); iter = iter.Next() { - if iter.IsXMLSchemaString() && iter.GetXMLSchemaString() != "" { + switch { + case iter.IsXMLSchemaString(): return iter.GetXMLSchemaString() + case iter.IsIRI(): + return iter.GetIRI().String() } } @@ -253,10 +256,10 @@ func ExtractSummary(i WithSummary) string { for iter := summaryProp.Begin(); iter != summaryProp.End(); iter = iter.Next() { switch { - case iter.IsIRI(): - return iter.GetIRI().String() case iter.IsXMLSchemaString(): return iter.GetXMLSchemaString() + case iter.IsIRI(): + return iter.GetIRI().String() } } @@ -354,10 +357,10 @@ func ExtractContent(i WithContent) string { } for iter := contentProperty.Begin(); iter != contentProperty.End(); iter = iter.Next() { - if iter.IsXMLSchemaString() { + switch { + case iter.IsXMLSchemaString(): return iter.GetXMLSchemaString() - } - if iter.IsIRI() && iter.GetIRI() != nil { + case iter.IsIRI(): return iter.GetIRI().String() } } diff --git a/internal/ap/interfaces.go b/internal/ap/interfaces.go index 33b2eb9ca..674791229 100644 --- a/internal/ap/interfaces.go +++ b/internal/ap/interfaces.go @@ -30,6 +30,7 @@ type Accountable interface { WithName WithImage WithSummary + WithSetSummary WithDiscoverable WithURL WithPublicKey @@ -50,7 +51,9 @@ type Statusable interface { WithTypeName WithSummary + WithSetSummary WithName + WithSetName WithInReplyTo WithPublished WithURL @@ -73,6 +76,7 @@ type Attachmentable interface { WithMediaType WithURL WithName + WithSetName WithBlurhash } @@ -193,6 +197,11 @@ type WithName interface { GetActivityStreamsName() vocab.ActivityStreamsNameProperty } +// WithSetName represents an activity with a settable ActivityStreamsNameProperty +type WithSetName interface { + SetActivityStreamsName(vocab.ActivityStreamsNameProperty) +} + // WithImage represents an activity with ActivityStreamsImageProperty type WithImage interface { GetActivityStreamsImage() vocab.ActivityStreamsImageProperty @@ -203,6 +212,11 @@ type WithSummary interface { GetActivityStreamsSummary() vocab.ActivityStreamsSummaryProperty } +// WithSetSummary represents an activity that can have summary set on it. +type WithSetSummary interface { + SetActivityStreamsSummary(vocab.ActivityStreamsSummaryProperty) +} + // WithDiscoverable represents an activity with TootDiscoverableProperty type WithDiscoverable interface { GetTootDiscoverable() vocab.TootDiscoverableProperty diff --git a/internal/ap/normalize.go b/internal/ap/normalize.go index 2425b35a0..52e297080 100644 --- a/internal/ap/normalize.go +++ b/internal/ap/normalize.go @@ -27,7 +27,7 @@ import ( // The rawActivity map should the freshly deserialized json representation of the Activity. // // This function is a noop if the type passed in is anything except a Create with a Statusable as its Object. -func NormalizeActivityObject(activity pub.Activity, rawActivity map[string]interface{}) { +func NormalizeActivityObject(activity pub.Activity, rawJSON map[string]interface{}) { if activity.GetTypeName() != ActivityCreate { // Only interested in Create right now. return @@ -72,45 +72,157 @@ func NormalizeActivityObject(activity pub.Activity, rawActivity map[string]inter return } - object, ok := rawActivity["object"] + rawObject, ok := rawJSON["object"] if !ok { // No object in raw map. return } - rawStatusable, ok := object.(map[string]interface{}) + rawStatusableJSON, ok := rawObject.(map[string]interface{}) if !ok { // Object wasn't a json object. return } - // Pass in the statusable and its raw JSON representation. - NormalizeStatusableContent(statusable, rawStatusable) + // Normalize everything we can on the statusable. + NormalizeContent(statusable, rawStatusableJSON) + NormalizeAttachments(statusable, rawStatusableJSON) + NormalizeSummary(statusable, rawStatusableJSON) + NormalizeName(statusable, rawStatusableJSON) } -// NormalizeStatusableContent replaces the Content of the given statusable -// with the raw 'content' value from the given json object map. +// NormalizeContent replaces the Content of the given item +// with the raw 'content' value from the raw json object map. // -// noop if there was no content in the json object map or the content was -// not a plain string. -func NormalizeStatusableContent(statusable Statusable, rawStatusable map[string]interface{}) { - content, ok := rawStatusable["content"] +// noop if there was no content in the json object map or the +// content was not a plain string. +func NormalizeContent(item WithSetContent, rawJSON map[string]interface{}) { + rawContent, ok := rawJSON["content"] if !ok { - // No content in rawStatusable. + // No content in rawJSON. // TODO: In future we might also // look for "contentMap" property. return } - rawContent, ok := content.(string) + content, ok := rawContent.(string) if !ok { // Not interested in content arrays. return } - // Set normalized content property from the raw string; this - // will replace any existing content property on the statusable. + // Set normalized content property from the raw string; + // this replaces any existing content property on the item. contentProp := streams.NewActivityStreamsContentProperty() - contentProp.AppendXMLSchemaString(rawContent) - statusable.SetActivityStreamsContent(contentProp) + contentProp.AppendXMLSchemaString(content) + item.SetActivityStreamsContent(contentProp) +} + +// NormalizeAttachments normalizes all attachments (if any) of the given +// itm, replacing the 'name' (aka content warning) field of each attachment +// with the raw 'name' value from the raw json object map. +// +// noop if there are no attachments; noop if attachment is not a format +// we can understand. +func NormalizeAttachments(item WithAttachment, rawJSON map[string]interface{}) { + rawAttachments, ok := rawJSON["attachment"] + if !ok { + // No attachments in rawJSON. + return + } + + // Convert to slice if not already, + // so we can iterate through it. + var attachments []interface{} + if attachments, ok = rawAttachments.([]interface{}); !ok { + attachments = []interface{}{rawAttachments} + } + + attachmentProperty := item.GetActivityStreamsAttachment() + if attachmentProperty == nil { + // Nothing to do here. + return + } + + if l := attachmentProperty.Len(); l == 0 || l != len(attachments) { + // Mismatch between item and + // JSON, can't normalize. + return + } + + // Keep an index of where we are in the iter; + // we need this so we can modify the correct + // attachment, in case of multiples. + i := -1 + + for iter := attachmentProperty.Begin(); iter != attachmentProperty.End(); iter = iter.Next() { + i++ + + t := iter.GetType() + if t == nil { + continue + } + + attachmentable, ok := t.(Attachmentable) + if !ok { + continue + } + + rawAttachment, ok := attachments[i].(map[string]interface{}) + if !ok { + continue + } + + NormalizeName(attachmentable, rawAttachment) + } +} + +// NormalizeSummary replaces the Summary of the given item +// with the raw 'summary' value from the raw json object map. +// +// noop if there was no summary in the json object map or the +// summary was not a plain string. +func NormalizeSummary(item WithSetSummary, rawJSON map[string]interface{}) { + rawSummary, ok := rawJSON["summary"] + if !ok { + // No summary in rawJSON. + return + } + + summary, ok := rawSummary.(string) + if !ok { + // Not interested in non-string summary. + return + } + + // Set normalized summary property from the raw string; this + // will replace any existing summary property on the item. + summaryProp := streams.NewActivityStreamsSummaryProperty() + summaryProp.AppendXMLSchemaString(summary) + item.SetActivityStreamsSummary(summaryProp) +} + +// NormalizeName replaces the Name of the given item +// with the raw 'name' value from the raw json object map. +// +// noop if there was no name in the json object map or the +// name was not a plain string. +func NormalizeName(item WithSetName, rawJSON map[string]interface{}) { + rawName, ok := rawJSON["name"] + if !ok { + // No name in rawJSON. + return + } + + name, ok := rawName.(string) + if !ok { + // Not interested in non-string name. + return + } + + // Set normalized name property from the raw string; this + // will replace any existing name property on the item. + nameProp := streams.NewActivityStreamsNameProperty() + nameProp.AppendXMLSchemaString(name) + item.SetActivityStreamsName(nameProp) } diff --git a/internal/ap/normalize_test.go b/internal/ap/normalize_test.go index d2a74a19e..c265b02f5 100644 --- a/internal/ap/normalize_test.go +++ b/internal/ap/normalize_test.go @@ -33,8 +33,37 @@ type NormalizeTestSuite struct { suite.Suite } -func (suite *NormalizeTestSuite) GetStatusable() (vocab.ActivityStreamsNote, map[string]interface{}) { - rawJson := `{ +func (suite *NormalizeTestSuite) jsonToType(rawJson string) (vocab.Type, map[string]interface{}) { + var raw map[string]interface{} + err := json.Unmarshal([]byte(rawJson), &raw) + if err != nil { + panic(err) + } + + t, err := streams.ToType(context.Background(), raw) + if err != nil { + panic(err) + } + + return t, raw +} + +func (suite *NormalizeTestSuite) typeToJson(t vocab.Type) string { + m, err := streams.Serialize(t) + if err != nil { + suite.FailNow(err.Error()) + } + + b, err := json.MarshalIndent(m, "", " ") + if err != nil { + suite.FailNow(err.Error()) + } + + return string(b) +} + +func (suite *NormalizeTestSuite) getStatusable() (vocab.ActivityStreamsNote, map[string]interface{}) { + t, raw := suite.jsonToType(`{ "@context": [ "https://www.w3.org/ns/activitystreams", "https://example.org/schemas/litepub-0.1.jsonld", @@ -74,24 +103,117 @@ func (suite *NormalizeTestSuite) GetStatusable() (vocab.ActivityStreamsNote, map "https://www.w3.org/ns/activitystreams#Public" ], "type": "Note" - }` + }`) - var rawNote map[string]interface{} - err := json.Unmarshal([]byte(rawJson), &rawNote) - if err != nil { - panic(err) - } + return t.(vocab.ActivityStreamsNote), raw +} - t, err := streams.ToType(context.Background(), rawNote) - if err != nil { - panic(err) - } +func (suite *NormalizeTestSuite) getStatusableWithOneAttachment() (vocab.ActivityStreamsNote, map[string]interface{}) { + t, raw := suite.jsonToType(`{ + "@context": "https://www.w3.org/ns/activitystreams", + "id": "https://example.org/users/hourlycatbot/statuses/01GYW48H311PZ78C5G856MGJJJ", + "type": "Note", + "url": "https://example.org/@hourlycatbot/01GYW48H311PZ78C5G856MGJJJ", + "attributedTo": "https://example.org/users/hourlycatbot", + "to": "https://www.w3.org/ns/activitystreams#Public", + "attachment": [ + { + "type": "Document", + "mediaType": "image/jpeg", + "url": "https://files.example.org/media_attachments/files/110/258/459/579/509/026/original/b65392ebe0fb04ef.jpeg", + "name": "DESCRIPTION: here's <> picture of a #cat, it's cute! here's some special characters: \"\" \\ weeee''''" + } + ] + }`) - return t.(vocab.ActivityStreamsNote), rawNote + return t.(vocab.ActivityStreamsNote), raw +} + +func (suite *NormalizeTestSuite) getStatusableWithOneAttachmentEmbedded() (vocab.ActivityStreamsNote, map[string]interface{}) { + t, raw := suite.jsonToType(`{ + "@context": "https://www.w3.org/ns/activitystreams", + "id": "https://example.org/users/hourlycatbot/statuses/01GYW48H311PZ78C5G856MGJJJ", + "type": "Note", + "url": "https://example.org/@hourlycatbot/01GYW48H311PZ78C5G856MGJJJ", + "attributedTo": "https://example.org/users/hourlycatbot", + "to": "https://www.w3.org/ns/activitystreams#Public", + "attachment": { + "type": "Document", + "mediaType": "image/jpeg", + "url": "https://files.example.org/media_attachments/files/110/258/459/579/509/026/original/b65392ebe0fb04ef.jpeg", + "name": "DESCRIPTION: here's <> picture of a #cat, it's cute! here's some special characters: \"\" \\ weeee''''" + } + }`) + + return t.(vocab.ActivityStreamsNote), raw +} + +func (suite *NormalizeTestSuite) getStatusableWithMultipleAttachments() (vocab.ActivityStreamsNote, map[string]interface{}) { + t, raw := suite.jsonToType(`{ + "@context": "https://www.w3.org/ns/activitystreams", + "id": "https://example.org/users/hourlycatbot/statuses/01GYW48H311PZ78C5G856MGJJJ", + "type": "Note", + "url": "https://example.org/@hourlycatbot/01GYW48H311PZ78C5G856MGJJJ", + "attributedTo": "https://example.org/users/hourlycatbot", + "to": "https://www.w3.org/ns/activitystreams#Public", + "attachment": [ + { + "type": "Document", + "mediaType": "image/jpeg", + "url": "https://files.example.org/media_attachments/files/110/258/459/579/509/026/original/b65392ebe0fb04ef.jpeg", + "name": "DESCRIPTION: here's <> picture of a #cat, it's cute! here's some special characters: \"\" \\ weeee''''" + }, + { + "type": "Document", + "mediaType": "image/jpeg", + "url": "https://files.example.org/media_attachments/files/110/258/459/579/509/026/original/b65392ebe0fb04ef.jpeg", + "name": "hello: here's another #picture #of #a #cat, hope you like it!!!!!!!" + }, + { + "type": "Document", + "mediaType": "image/jpeg", + "url": "https://files.example.org/media_attachments/files/110/258/459/579/509/026/original/b65392ebe0fb04ef.jpeg" + }, + { + "type": "Document", + "mediaType": "image/jpeg", + "url": "https://files.example.org/media_attachments/files/110/258/459/579/509/026/original/b65392ebe0fb04ef.jpeg", + "name": "danger: #cute but will claw you :(" + } + ] + }`) + + return t.(vocab.ActivityStreamsNote), raw +} + +func (suite *NormalizeTestSuite) getStatusableWithWeirdSummaryAndName() (vocab.ActivityStreamsNote, map[string]interface{}) { + t, raw := suite.jsonToType(`{ + "@context": "https://www.w3.org/ns/activitystreams", + "id": "https://example.org/users/hourlycatbot/statuses/01GYW48H311PZ78C5G856MGJJJ", + "type": "Note", + "url": "https://example.org/@hourlycatbot/01GYW48H311PZ78C5G856MGJJJ", + "attributedTo": "https://example.org/users/hourlycatbot", + "to": "https://www.w3.org/ns/activitystreams#Public", + "summary": "warning: #WEIRD #SUMMARY ;;;;a;;a;asv khop8273987(*^&^)", + "name": "WARNING: #WEIRD #nameEE ;;;;a;;a;asv khop8273987(*^&^)" + }`) + + return t.(vocab.ActivityStreamsNote), raw +} + +func (suite *NormalizeTestSuite) getAccountable() (vocab.ActivityStreamsPerson, map[string]interface{}) { + t, raw := suite.jsonToType(`{ + "@context": "https://www.w3.org/ns/activitystreams", + "id": "https://example.org/users/someone", + "summary": "about: I'm a #Barbie #girl in a #Barbie #world\nLife in plastic, it's fantastic\nYou can brush my hair, undress me everywhere\nImagination, life is your creation\nI'm a blonde bimbo girl\nIn a fantasy world\nDress me up, make it tight\nI'm your dolly\nYou're my doll, rock and roll\nFeel the glamour in pink\nKiss me here, touch me there\nHanky panky", + "type": "Person" + }`) + + return t.(vocab.ActivityStreamsPerson), raw } func (suite *NormalizeTestSuite) TestNormalizeActivityObject() { - note, rawNote := suite.GetStatusable() + note, rawNote := suite.getStatusable() suite.Equal(`update: As of this morning there are now more than 7 million Mastodon users, most from the #TwitterMigration%3C/a%3E.%3Cbr%3E%3Cbr%3EIn%20fact,%20100,000%20new%20accounts%20have%20been%20created%20since%20last%20night.%3Cbr%3E%3Cbr%3ESince%20last%20night&%2339;s%20spike%208,000-12,000%20new%20accounts%20are%20being%20created%20every%20hour.%3Cbr%3E%3Cbr%3EYesterday,%20I%20estimated%20that%20Mastodon%20would%20have%208%20million%20users%20by%20the%20end%20of%20the%20week.%20That%20might%20happen%20a%20lot%20sooner%20if%20this%20trend%20continues.`, ap.ExtractContent(note)) create := testrig.WrapAPNoteInCreate( @@ -105,6 +227,202 @@ func (suite *NormalizeTestSuite) TestNormalizeActivityObject() { suite.Equal(`UPDATE: As of this morning there are now more than 7 million Mastodon users, most from the #TwitterMigration.

In fact, 100,000 new accounts have been created since last night.

Since last night's spike 8,000-12,000 new accounts are being created every hour.

Yesterday, I estimated that Mastodon would have 8 million users by the end of the week. That might happen a lot sooner if this trend continues.`, ap.ExtractContent(note)) } +func (suite *NormalizeTestSuite) TestNormalizeStatusableAttachmentsOneAttachment() { + note, rawNote := suite.getStatusableWithOneAttachment() + + // Without normalization, the 'name' field of + // the attachment(s) should be all jacked up. + suite.Equal(`{ + "@context": "https://www.w3.org/ns/activitystreams", + "attachment": { + "mediaType": "image/jpeg", + "name": "description: here's \u003c\u003ca\u003e\u003e picture of a #cat,%20it%27s%20cute!%20here%27s%20some%20special%20characters:%20%22%22%20%5C%20weeee%27%27%27%27", + "type": "Document", + "url": "https://files.example.org/media_attachments/files/110/258/459/579/509/026/original/b65392ebe0fb04ef.jpeg" + }, + "attributedTo": "https://example.org/users/hourlycatbot", + "id": "https://example.org/users/hourlycatbot/statuses/01GYW48H311PZ78C5G856MGJJJ", + "to": "https://www.w3.org/ns/activitystreams#Public", + "type": "Note", + "url": "https://example.org/@hourlycatbot/01GYW48H311PZ78C5G856MGJJJ" +}`, suite.typeToJson(note)) + + // Normalize it! + ap.NormalizeAttachments(note, rawNote) + + // After normalization, the 'name' field of the + // attachment should no longer be all jacked up. + suite.Equal(`{ + "@context": "https://www.w3.org/ns/activitystreams", + "attachment": { + "mediaType": "image/jpeg", + "name": "DESCRIPTION: here's \u003c\u003ca\u003e\u003e picture of a #cat, it's cute! here's some special characters: \"\" \\ weeee''''", + "type": "Document", + "url": "https://files.example.org/media_attachments/files/110/258/459/579/509/026/original/b65392ebe0fb04ef.jpeg" + }, + "attributedTo": "https://example.org/users/hourlycatbot", + "id": "https://example.org/users/hourlycatbot/statuses/01GYW48H311PZ78C5G856MGJJJ", + "to": "https://www.w3.org/ns/activitystreams#Public", + "type": "Note", + "url": "https://example.org/@hourlycatbot/01GYW48H311PZ78C5G856MGJJJ" +}`, suite.typeToJson(note)) +} + +func (suite *NormalizeTestSuite) TestNormalizeStatusableAttachmentsOneAttachmentEmbedded() { + note, rawNote := suite.getStatusableWithOneAttachmentEmbedded() + + // Without normalization, the 'name' field of + // the attachment(s) should be all jacked up. + suite.Equal(`{ + "@context": "https://www.w3.org/ns/activitystreams", + "attachment": { + "mediaType": "image/jpeg", + "name": "description: here's \u003c\u003ca\u003e\u003e picture of a #cat,%20it%27s%20cute!%20here%27s%20some%20special%20characters:%20%22%22%20%5C%20weeee%27%27%27%27", + "type": "Document", + "url": "https://files.example.org/media_attachments/files/110/258/459/579/509/026/original/b65392ebe0fb04ef.jpeg" + }, + "attributedTo": "https://example.org/users/hourlycatbot", + "id": "https://example.org/users/hourlycatbot/statuses/01GYW48H311PZ78C5G856MGJJJ", + "to": "https://www.w3.org/ns/activitystreams#Public", + "type": "Note", + "url": "https://example.org/@hourlycatbot/01GYW48H311PZ78C5G856MGJJJ" +}`, suite.typeToJson(note)) + + // Normalize it! + ap.NormalizeAttachments(note, rawNote) + + // After normalization, the 'name' field of the + // attachment should no longer be all jacked up. + suite.Equal(`{ + "@context": "https://www.w3.org/ns/activitystreams", + "attachment": { + "mediaType": "image/jpeg", + "name": "DESCRIPTION: here's \u003c\u003ca\u003e\u003e picture of a #cat, it's cute! here's some special characters: \"\" \\ weeee''''", + "type": "Document", + "url": "https://files.example.org/media_attachments/files/110/258/459/579/509/026/original/b65392ebe0fb04ef.jpeg" + }, + "attributedTo": "https://example.org/users/hourlycatbot", + "id": "https://example.org/users/hourlycatbot/statuses/01GYW48H311PZ78C5G856MGJJJ", + "to": "https://www.w3.org/ns/activitystreams#Public", + "type": "Note", + "url": "https://example.org/@hourlycatbot/01GYW48H311PZ78C5G856MGJJJ" +}`, suite.typeToJson(note)) +} + +func (suite *NormalizeTestSuite) TestNormalizeStatusableAttachmentsMultipleAttachments() { + note, rawNote := suite.getStatusableWithMultipleAttachments() + + // Without normalization, the 'name' field of + // the attachment(s) should be all jacked up. + suite.Equal(`{ + "@context": "https://www.w3.org/ns/activitystreams", + "attachment": [ + { + "mediaType": "image/jpeg", + "name": "description: here's \u003c\u003ca\u003e\u003e picture of a #cat,%20it%27s%20cute!%20here%27s%20some%20special%20characters:%20%22%22%20%5C%20weeee%27%27%27%27", + "type": "Document", + "url": "https://files.example.org/media_attachments/files/110/258/459/579/509/026/original/b65392ebe0fb04ef.jpeg" + }, + { + "mediaType": "image/jpeg", + "name": "hello: here's another #picture%20%23of%20%23a%20%23cat,%20hope%20you%20like%20it!!!!!!!", + "type": "Document", + "url": "https://files.example.org/media_attachments/files/110/258/459/579/509/026/original/b65392ebe0fb04ef.jpeg" + }, + { + "mediaType": "image/jpeg", + "type": "Document", + "url": "https://files.example.org/media_attachments/files/110/258/459/579/509/026/original/b65392ebe0fb04ef.jpeg" + }, + { + "mediaType": "image/jpeg", + "name": "danger: #cute%20but%20will%20claw%20you%20:(", + "type": "Document", + "url": "https://files.example.org/media_attachments/files/110/258/459/579/509/026/original/b65392ebe0fb04ef.jpeg" + } + ], + "attributedTo": "https://example.org/users/hourlycatbot", + "id": "https://example.org/users/hourlycatbot/statuses/01GYW48H311PZ78C5G856MGJJJ", + "to": "https://www.w3.org/ns/activitystreams#Public", + "type": "Note", + "url": "https://example.org/@hourlycatbot/01GYW48H311PZ78C5G856MGJJJ" +}`, suite.typeToJson(note)) + + // Normalize it! + ap.NormalizeAttachments(note, rawNote) + + // After normalization, the 'name' field of the + // attachment should no longer be all jacked up. + suite.Equal(`{ + "@context": "https://www.w3.org/ns/activitystreams", + "attachment": [ + { + "mediaType": "image/jpeg", + "name": "DESCRIPTION: here's \u003c\u003ca\u003e\u003e picture of a #cat, it's cute! here's some special characters: \"\" \\ weeee''''", + "type": "Document", + "url": "https://files.example.org/media_attachments/files/110/258/459/579/509/026/original/b65392ebe0fb04ef.jpeg" + }, + { + "mediaType": "image/jpeg", + "name": "hello: here's another #picture #of #a #cat, hope you like it!!!!!!!", + "type": "Document", + "url": "https://files.example.org/media_attachments/files/110/258/459/579/509/026/original/b65392ebe0fb04ef.jpeg" + }, + { + "mediaType": "image/jpeg", + "type": "Document", + "url": "https://files.example.org/media_attachments/files/110/258/459/579/509/026/original/b65392ebe0fb04ef.jpeg" + }, + { + "mediaType": "image/jpeg", + "name": "danger: #cute but will claw you :(", + "type": "Document", + "url": "https://files.example.org/media_attachments/files/110/258/459/579/509/026/original/b65392ebe0fb04ef.jpeg" + } + ], + "attributedTo": "https://example.org/users/hourlycatbot", + "id": "https://example.org/users/hourlycatbot/statuses/01GYW48H311PZ78C5G856MGJJJ", + "to": "https://www.w3.org/ns/activitystreams#Public", + "type": "Note", + "url": "https://example.org/@hourlycatbot/01GYW48H311PZ78C5G856MGJJJ" +}`, suite.typeToJson(note)) +} + +func (suite *NormalizeTestSuite) TestNormalizeAccountableSummary() { + accountable, rawAccount := suite.getAccountable() + suite.Equal(`about: I'm a #Barbie%20%23girl%20in%20a%20%23Barbie%20%23world%0ALife%20in%20plastic,%20it%27s%20fantastic%0AYou%20can%20brush%20my%20hair,%20undress%20me%20everywhere%0AImagination,%20life%20is%20your%20creation%0AI%27m%20a%20blonde%20bimbo%20girl%0AIn%20a%20fantasy%20world%0ADress%20me%20up,%20make%20it%20tight%0AI%27m%20your%20dolly%0AYou%27re%20my%20doll,%20rock%20and%20roll%0AFeel%20the%20glamour%20in%20pink%0AKiss%20me%20here,%20touch%20me%20there%0AHanky%20panky`, ap.ExtractSummary(accountable)) + + ap.NormalizeSummary(accountable, rawAccount) + suite.Equal(`about: I'm a #Barbie #girl in a #Barbie #world +Life in plastic, it's fantastic +You can brush my hair, undress me everywhere +Imagination, life is your creation +I'm a blonde bimbo girl +In a fantasy world +Dress me up, make it tight +I'm your dolly +You're my doll, rock and roll +Feel the glamour in pink +Kiss me here, touch me there +Hanky panky`, ap.ExtractSummary(accountable)) +} + +func (suite *NormalizeTestSuite) TestNormalizeStatusableSummary() { + statusable, rawAccount := suite.getStatusableWithWeirdSummaryAndName() + suite.Equal(`warning: #WEIRD%20%23SUMMARY%20;;;;a;;a;asv%20%20%20%20khop8273987(*%5E&%5E)`, ap.ExtractSummary(statusable)) + + ap.NormalizeSummary(statusable, rawAccount) + suite.Equal(`warning: #WEIRD #SUMMARY ;;;;a;;a;asv khop8273987(*^&^)`, ap.ExtractSummary(statusable)) +} + +func (suite *NormalizeTestSuite) TestNormalizeStatusableName() { + statusable, rawAccount := suite.getStatusableWithWeirdSummaryAndName() + suite.Equal(`warning: #WEIRD%20%23nameEE%20;;;;a;;a;asv%20%20%20%20khop8273987(*%5E&%5E)`, ap.ExtractName(statusable)) + + ap.NormalizeName(statusable, rawAccount) + suite.Equal(`WARNING: #WEIRD #nameEE ;;;;a;;a;asv khop8273987(*^&^)`, ap.ExtractName(statusable)) +} + func TestNormalizeTestSuite(t *testing.T) { suite.Run(t, new(NormalizeTestSuite)) } diff --git a/internal/ap/resolve.go b/internal/ap/resolve.go index c5c9efd65..8d116751c 100644 --- a/internal/ap/resolve.go +++ b/internal/ap/resolve.go @@ -27,8 +27,7 @@ import ( ) // ResolveStatusable tries to resolve the given bytes into an ActivityPub Statusable representation. -// It will then perform normalization on the Statusable by calling NormalizeStatusable, so that -// callers don't need to bother doing extra steps. +// It will then perform normalization on the Statusable. // // Works for: Article, Document, Image, Video, Note, Page, Event, Place, Profile func ResolveStatusable(ctx context.Context, b []byte) (Statusable, error) { @@ -73,11 +72,16 @@ func ResolveStatusable(ctx context.Context, b []byte) (Statusable, error) { return nil, newErrWrongType(err) } - NormalizeStatusableContent(statusable, rawStatusable) + NormalizeContent(statusable, rawStatusable) + NormalizeAttachments(statusable, rawStatusable) + NormalizeSummary(statusable, rawStatusable) + NormalizeName(statusable, rawStatusable) + return statusable, nil } // ResolveStatusable tries to resolve the given bytes into an ActivityPub Accountable representation. +// It will then perform normalization on the Accountable. // // Works for: Application, Group, Organization, Person, Service func ResolveAccountable(ctx context.Context, b []byte) (Accountable, error) { @@ -114,5 +118,7 @@ func ResolveAccountable(ctx context.Context, b []byte) (Accountable, error) { return nil, newErrWrongType(err) } + NormalizeSummary(accountable, rawAccountable) + return accountable, nil } diff --git a/internal/processing/fromfederator.go b/internal/processing/fromfederator.go index 55e85a526..929ef824f 100644 --- a/internal/processing/fromfederator.go +++ b/internal/processing/fromfederator.go @@ -373,15 +373,11 @@ func (p *Processor) processUpdateAccountFromFederator(ctx context.Context, feder return errors.New("profile was not parseable as *gtsmodel.Account") } - incomingAccountURL, err := url.Parse(incomingAccount.URI) - if err != nil { - return err - } - - // further database updates occur inside getremoteaccount - if _, err := p.federator.GetAccountByURI(ctx, + // Call UpdateAccount with force to reflect that + // we want to fetch new bio, avatar, header, etc. + if _, err := p.federator.UpdateAccount(ctx, federatorMsg.ReceivingAccount.Username, - incomingAccountURL, + incomingAccount, true, ); err != nil { return fmt.Errorf("error enriching updated account from federator: %s", err)