mirrors
/
gotosocial
mirror of
1
Fork 0
Code Issues Releases Activity

[bugfix] Fix emphasis being added to emoji shortcodes with markdown parsing (#856) 

* fix underscored emoji shortcodes being emphasized

* remove footnote parsing from md
This commit is contained in:
tobi 2022-09-27 14:27:53 +02:00 committed by GitHub
parent 429bb770e2
commit 00d38855d4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 83 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
internal/processing/status/create_test.go
View File

@ -24,6 +24,8 @@ import (
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/api/model"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
)
type StatusCreateTestSuite struct {
@ -98,6 +100,45 @@ func (suite *StatusCreateTestSuite) TestProcessContentWarningWithHTMLEscapedQuot
suite.Equal("\"test\"", apiStatus.SpoilerText)
}
func (suite *StatusCreateTestSuite) TestProcessStatusMarkdownWithUnderscoreEmoji() {
ctx := context.Background()
// update the shortcode of the rainbow emoji to surround it in underscores
if err := suite.db.UpdateWhere(ctx, []db.Where{{Key: "shortcode", Value: "rainbow"}}, "shortcode", "_rainbow_", &gtsmodel.Emoji{}); err != nil {
suite.FailNow(err.Error())
}
creatingAccount := suite.testAccounts["local_account_1"]
creatingApplication := suite.testApplications["application_1"]
statusCreateForm := &model.AdvancedStatusCreateForm{
StatusCreateRequest: model.StatusCreateRequest{
Status: "poopoo peepee :_rainbow_:",
MediaIDs: []string{},
Poll: nil,
InReplyToID: "",
Sensitive: false,
Visibility: model.VisibilityPublic,
ScheduledAt: "",
Language: "en",
Format: model.StatusFormatMarkdown,
},
AdvancedVisibilityFlagsForm: model.AdvancedVisibilityFlagsForm{
Federated: nil,
Boostable: nil,
Replyable: nil,
Likeable: nil,
},
}
apiStatus, err := suite.status.Create(ctx, creatingAccount, creatingApplication, statusCreateForm)
suite.NoError(err)
suite.NotNil(apiStatus)
suite.Equal("<p>poopoo peepee :_rainbow_:</p>", apiStatus.Content)
suite.NotEmpty(apiStatus.Emojis)
}
func TestStatusCreateTestSuite(t *testing.T) {
suite.Run(t, new(StatusCreateTestSuite))
}

2
internal/processing/status/util.go
View File

@ -302,7 +302,7 @@ func (p *processor) ProcessContent(ctx context.Context, form *apimodel.AdvancedS
case apimodel.StatusFormatPlain:
formatted = p.formatter.FromPlain(ctx, form.Status, status.Mentions, status.Tags)
case apimodel.StatusFormatMarkdown:
formatted = p.formatter.FromMarkdown(ctx, form.Status, status.Mentions, status.Tags)
formatted = p.formatter.FromMarkdown(ctx, form.Status, status.Mentions, status.Tags, status.Emojis)
default:
return fmt.Errorf("format %s not recognised as a valid status format", form.Format)
}

2
internal/text/formatter.go
View File

@ -30,7 +30,7 @@ type Formatter interface {
// FromPlain parses an HTML text from a plaintext.
FromPlain(ctx context.Context, plain string, mentions []*gtsmodel.Mention, tags []*gtsmodel.Tag) string
// FromMarkdown parses an HTML text from a markdown-formatted text.
FromMarkdown(ctx context.Context, md string, mentions []*gtsmodel.Mention, tags []*gtsmodel.Tag) string
FromMarkdown(ctx context.Context, md string, mentions []*gtsmodel.Mention, tags []*gtsmodel.Tag, emojis []*gtsmodel.Emoji) string
// ReplaceTags takes a piece of text and a slice of tags, and returns the same text with the tags nicely formatted as hrefs.
ReplaceTags(ctx context.Context, in string, tags []*gtsmodel.Tag) string

35
internal/text/markdown.go
View File

@ -22,6 +22,7 @@ import (
"bytes"
"context"
"io"
"strings"
"github.com/russross/blackfriday/v2"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
@ -31,7 +32,7 @@ import (
)
var (
bfExtensions = blackfriday.CommonExtensions | blackfriday.HardLineBreak | blackfriday.Footnotes
bfExtensions = blackfriday.NoIntraEmphasis | blackfriday.FencedCode | blackfriday.Autolink | blackfriday.Strikethrough | blackfriday.SpaceHeadings | blackfriday.HardLineBreak
m *minify.M
)
@ -54,8 +55,7 @@ func (r *renderer) RenderNode(w io.Writer, node *blackfriday.Node, entering bool
html = r.f.ReplaceMentions(r.ctx, html, r.mentions)
// we don't have much recourse if this fails
_, err := io.WriteString(w, html)
if err != nil {
if _, err := io.WriteString(w, html); err != nil {
log.Errorf("error outputting markdown text: %s", err)
}
return status
@ -63,7 +63,7 @@ func (r *renderer) RenderNode(w io.Writer, node *blackfriday.Node, entering bool
return r.HTMLRenderer.RenderNode(w, node, entering)
}
func (f *formatter) FromMarkdown(ctx context.Context, md string, mentions []*gtsmodel.Mention, tags []*gtsmodel.Tag) string {
func (f *formatter) FromMarkdown(ctx context.Context, markdownText string, mentions []*gtsmodel.Mention, tags []*gtsmodel.Tag, emojis []*gtsmodel.Emoji) string {
renderer := &renderer{
f: f,
@ -75,11 +75,28 @@ func (f *formatter) FromMarkdown(ctx context.Context, md string, mentions []*gts
}),
}
// parse markdown, use custom renderer to add hashtag/mention links
contentBytes := blackfriday.Run([]byte(md), blackfriday.WithExtensions(bfExtensions), blackfriday.WithRenderer(renderer))
// Temporarily replace all found emoji shortcodes in the markdown text with
// their ID so that they're not parsed as anything by the markdown parser -
// this fixes cases where emojis with some underscores in them are parsed as
// words with emphasis, eg `:_some_emoji:` becomes `:<em>some</em>emoji:`
//
// Since the IDs of the emojis are just uppercase letters + numbers they should
// be safe to pass through the markdown parser without unexpected effects.
for _, e := range emojis {
markdownText = strings.ReplaceAll(markdownText, ":"+e.Shortcode+":", ":"+e.ID+":")
}
// clean anything dangerous out of it
content := SanitizeHTML(string(contentBytes))
// parse markdown text into html, using custom renderer to add hashtag/mention links
htmlContentBytes := blackfriday.Run([]byte(markdownText), blackfriday.WithExtensions(bfExtensions), blackfriday.WithRenderer(renderer))
htmlContent := string(htmlContentBytes)
// Replace emoji IDs in the parsed html content with their shortcodes again
for _, e := range emojis {
htmlContent = strings.ReplaceAll(htmlContent, ":"+e.ID+":", ":"+e.Shortcode+":")
}
// clean anything dangerous out of the html
htmlContent = SanitizeHTML(htmlContent)
if m == nil {
m = minify.New()
@ -89,7 +106,7 @@ func (f *formatter) FromMarkdown(ctx context.Context, md string, mentions []*gts
})
}
minified, err := m.String("text/html", content)
minified, err := m.String("text/html", htmlContent)
if err != nil {
log.Errorf("error minifying markdown text: %s", err)
}

28
internal/text/markdown_test.go
View File

@ -62,7 +62,7 @@ const (
mdCodeBlockWithNewlines = "some code coming up\n\n```\n\n\n\n```\nthat was some code"
mdCodeBlockWithNewlinesExpected = "<p>some code coming up</p><pre><code>\n\n\n</code></pre><p>that was some code</p>"
mdWithFootnote = "fox mulder,fbi.[^1]\n\n[^1]: federated bureau of investigation"
mdWithFootnoteExpected = "<p>fox mulder,fbi.<sup id=\"fnref:1\"><a href=\"#fn:1\" rel=\"nofollow noreferrer\">1</a></sup></p><div><hr><ol><li id=\"fn:1\">federated bureau of investigation<br></li></ol></div>"
mdWithFootnoteExpected = "<p>fox mulder,fbi.[^1]</p><p>[^1]: federated bureau of investigation</p>"
mdWithBlockQuote = "get ready, there's a block quote coming:\n\n>line1\n>line2\n>\n>line3\n\n"
mdWithBlockQuoteExpected = "<p>get ready, there’s a block quote coming:</p><blockquote><p>line1<br>line2</p><p>line3</p></blockquote>"
mdHashtagAndCodeBlock = "#Hashtag\n\n```\n#Hashtag\n```"
@ -76,22 +76,22 @@ type MarkdownTestSuite struct {
}
func (suite *MarkdownTestSuite) TestParseSimple() {
s := suite.formatter.FromMarkdown(context.Background(), simpleMarkdown, nil, nil)
s := suite.formatter.FromMarkdown(context.Background(), simpleMarkdown, nil, nil, nil)
suite.Equal(simpleMarkdownExpected, s)
}
func (suite *MarkdownTestSuite) TestParseWithCodeBlock() {
s := suite.formatter.FromMarkdown(context.Background(), withCodeBlock, nil, nil)
s := suite.formatter.FromMarkdown(context.Background(), withCodeBlock, nil, nil, nil)
suite.Equal(withCodeBlockExpected, s)
}
func (suite *MarkdownTestSuite) TestParseWithInlineCode() {
s := suite.formatter.FromMarkdown(context.Background(), withInlineCode, nil, nil)
s := suite.formatter.FromMarkdown(context.Background(), withInlineCode, nil, nil, nil)
suite.Equal(withInlineCodeExpected, s)
}
func (suite *MarkdownTestSuite) TestParseWithInlineCode2() {
s := suite.formatter.FromMarkdown(context.Background(), withInlineCode2, nil, nil)
s := suite.formatter.FromMarkdown(context.Background(), withInlineCode2, nil, nil, nil)
suite.Equal(withInlineCode2Expected, s)
}
@ -100,17 +100,17 @@ func (suite *MarkdownTestSuite) TestParseWithHashtag() {
suite.testTags["Hashtag"],
}
s := suite.formatter.FromMarkdown(context.Background(), withHashtag, nil, foundTags)
s := suite.formatter.FromMarkdown(context.Background(), withHashtag, nil, foundTags, nil)
suite.Equal(withHashtagExpected, s)
}
func (suite *MarkdownTestSuite) TestParseWithHTML() {
s := suite.formatter.FromMarkdown(context.Background(), mdWithHTML, nil, nil)
s := suite.formatter.FromMarkdown(context.Background(), mdWithHTML, nil, nil, nil)
suite.Equal(mdWithHTMLExpected, s)
}
func (suite *MarkdownTestSuite) TestParseWithCheekyHTML() {
s := suite.formatter.FromMarkdown(context.Background(), mdWithCheekyHTML, nil, nil)
s := suite.formatter.FromMarkdown(context.Background(), mdWithCheekyHTML, nil, nil, nil)
suite.Equal(mdWithCheekyHTMLExpected, s)
}
@ -118,36 +118,36 @@ func (suite *MarkdownTestSuite) TestParseWithHashtagInitial() {
s := suite.formatter.FromMarkdown(context.Background(), mdWithHashtagInitial, nil, []*gtsmodel.Tag{
suite.testTags["Hashtag"],
suite.testTags["welcome"],
})
}, nil)
suite.Equal(mdWithHashtagInitialExpected, s)
}
func (suite *MarkdownTestSuite) TestParseCodeBlockWithNewlines() {
s := suite.formatter.FromMarkdown(context.Background(), mdCodeBlockWithNewlines, nil, nil)
s := suite.formatter.FromMarkdown(context.Background(), mdCodeBlockWithNewlines, nil, nil, nil)
suite.Equal(mdCodeBlockWithNewlinesExpected, s)
}
func (suite *MarkdownTestSuite) TestParseWithFootnote() {
s := suite.formatter.FromMarkdown(context.Background(), mdWithFootnote, nil, nil)
s := suite.formatter.FromMarkdown(context.Background(), mdWithFootnote, nil, nil, nil)
suite.Equal(mdWithFootnoteExpected, s)
}
func (suite *MarkdownTestSuite) TestParseWithBlockquote() {
s := suite.formatter.FromMarkdown(context.Background(), mdWithBlockQuote, nil, nil)
s := suite.formatter.FromMarkdown(context.Background(), mdWithBlockQuote, nil, nil, nil)
suite.Equal(mdWithBlockQuoteExpected, s)
}
func (suite *MarkdownTestSuite) TestParseHashtagWithCodeBlock() {
s := suite.formatter.FromMarkdown(context.Background(), mdHashtagAndCodeBlock, nil, []*gtsmodel.Tag{
suite.testTags["Hashtag"],
})
}, nil)
suite.Equal(mdHashtagAndCodeBlockExpected, s)
}
func (suite *MarkdownTestSuite) TestParseMentionWithCodeBlock() {
s := suite.formatter.FromMarkdown(context.Background(), mdMentionAndCodeBlock, []*gtsmodel.Mention{
suite.testMentions["local_user_2_mention_zork"],
}, nil)
}, nil, nil)
suite.Equal(mdMentionAndCodeBlockExpected, s)
}