[bugfix] Fix unicode-unaware word boundary check in hashtags (#1049)
* [bugfix] Fix unicode-unaware word boundary check in hashtag regex Go `\b` does not care for Unicode, and without lookahead, the workarounds got very ugly. So I replaced the regex with a parser. The parser runs in O(n) time and performance should not be affected. * [bugfix] Add back hashtag max length and add tests for it
This commit is contained in:
ugla
GitHub
parent
fece7fa706
commit
52109776f6
|
|
@ -47,7 +47,6 @@ const (
|
|||
const (
|
||||
maximumUsernameLength = 64
|
||||
maximumEmojiShortcodeLength = 30
|
||||
maximumHashtagLength = 30
|
||||
)
|
||||
|
||||
var (
|
||||
|
|
@ -66,17 +65,11 @@ var (
|
|||
// such as @whatever_user@example.org, returning whatever_user and example.org (without the @ symbols)
|
||||
MentionName = regexp.MustCompile(mentionName)
|
||||
|
||||
// mention regex can be played around with here: https://regex101.com/r/G1oGR0/1
|
||||
// mention regex can be played around with here: https://regex101.com/r/P0vpYG/1
|
||||
mentionFinder = `(?:^|\s)(@\w+(?:@[a-zA-Z0-9_\-\.]+)?)`
|
||||
// MentionFinder extracts mentions from a piece of text.
|
||||
MentionFinder = regexp.MustCompile(mentionFinder)
|
||||
|
||||
// hashtag regex can be played with here: https://regex101.com/r/bpyGlj/1
|
||||
hashtagFinder = fmt.Sprintf(`(?:^|\s)(?:#*)(#[\p{L}\p{N}]{1,%d})(?:#|\b)`, maximumHashtagLength)
|
||||
// HashtagFinder finds possible hashtags in a string.
|
||||
// It returns just the string part of the hashtag, not the # symbol.
|
||||
HashtagFinder = regexp.MustCompile(hashtagFinder)
|
||||
|
||||
emojiShortcode = fmt.Sprintf(`\w{2,%d}`, maximumEmojiShortcodeLength)
|
||||
// EmojiShortcode validates an emoji name.
|
||||
EmojiShortcode = regexp.MustCompile(fmt.Sprintf("^%s$", emojiShortcode))
|
||||
|
|
|
|||
|
|
@ -27,36 +27,46 @@ import (
|
|||
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/log"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/regexes"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/util"
|
||||
)
|
||||
|
||||
func (f *formatter) ReplaceTags(ctx context.Context, in string, tags []*gtsmodel.Tag) string {
|
||||
return regexes.ReplaceAllStringFunc(regexes.HashtagFinder, in, func(match string, buf *bytes.Buffer) string {
|
||||
// we have a match
|
||||
matchTrimmed := strings.TrimSpace(match)
|
||||
tagAsEntered := matchTrimmed[1:]
|
||||
spans := util.FindHashtagSpansInText(in)
|
||||
|
||||
if len(spans) == 0 {
|
||||
return in
|
||||
}
|
||||
|
||||
var b strings.Builder
|
||||
i := 0
|
||||
|
||||
spans:
|
||||
for _, t := range spans {
|
||||
b.WriteString(in[i:t.First])
|
||||
i = t.Second
|
||||
tagAsEntered := in[t.First+1 : t.Second]
|
||||
|
||||
// check through the tags to find what we're matching
|
||||
for _, tag := range tags {
|
||||
if strings.EqualFold(tagAsEntered, tag.Name) {
|
||||
// Add any dropped space from match
|
||||
if unicode.IsSpace(rune(match[0])) {
|
||||
buf.WriteByte(match[0])
|
||||
}
|
||||
|
||||
// replace the #tag with the formatted tag content
|
||||
// `<a href="tag.URL" class="mention hashtag" rel="tag">#<span>tagAsEntered</span></a>
|
||||
buf.WriteString(`<a href="`)
|
||||
buf.WriteString(tag.URL)
|
||||
buf.WriteString(`" class="mention hashtag" rel="tag">#<span>`)
|
||||
buf.WriteString(tagAsEntered)
|
||||
buf.WriteString(`</span></a>`)
|
||||
return buf.String()
|
||||
b.WriteString(`<a href="`)
|
||||
b.WriteString(tag.URL)
|
||||
b.WriteString(`" class="mention hashtag" rel="tag">#<span>`)
|
||||
b.WriteString(tagAsEntered)
|
||||
b.WriteString(`</span></a>`)
|
||||
continue spans
|
||||
}
|
||||
}
|
||||
|
||||
// the match wasn't in the list of tags for whatever reason, so just return the match as we found it so nothing changes
|
||||
return match
|
||||
})
|
||||
b.WriteString(in[t.First:t.Second])
|
||||
}
|
||||
|
||||
// Get the last bits.
|
||||
i = spans[len(spans)-1].Second
|
||||
b.WriteString(in[i:])
|
||||
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func (f *formatter) ReplaceMentions(ctx context.Context, in string, mentions []*gtsmodel.Mention) string {
|
||||
|
|
|
|||
|
|
@ -19,11 +19,16 @@
|
|||
package util
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"unicode"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/superseriousbusiness/gotosocial/internal/regexes"
|
||||
)
|
||||
|
||||
const (
|
||||
maximumHashtagLength = 30
|
||||
)
|
||||
|
||||
// DeriveMentionNamesFromText takes a plaintext (ie., not html-formatted) text,
|
||||
// and applies a regex to it to return a deduplicated list of account names
|
||||
// mentioned in that text, in the format "@user@example.org" or "@username" for
|
||||
|
|
@ -36,16 +41,71 @@ func DeriveMentionNamesFromText(text string) []string {
|
|||
return UniqueStrings(mentionedAccounts)
|
||||
}
|
||||
|
||||
// DeriveHashtagsFromText takes a plaintext (ie., not html-formatted) text,
|
||||
// and applies a regex to it to return a deduplicated list of hashtags
|
||||
// used in that text, without the leading #. The case of the returned
|
||||
// tags will be lowered, for consistency.
|
||||
type Pair[A, B any] struct {
|
||||
First A
|
||||
Second B
|
||||
}
|
||||
|
||||
// Byte index in original string
|
||||
// `First` includes `#`.
|
||||
type Span = Pair[int, int]
|
||||
|
||||
// Takes a plaintext (ie., not HTML-formatted) text,
|
||||
// and returns a slice of unique hashtags.
|
||||
func DeriveHashtagsFromText(text string) []string {
|
||||
tagsMap := make(map[string]bool)
|
||||
tags := []string{}
|
||||
for _, m := range regexes.HashtagFinder.FindAllStringSubmatch(text, -1) {
|
||||
tags = append(tags, strings.TrimPrefix(m[1], "#"))
|
||||
|
||||
for _, v := range FindHashtagSpansInText(text) {
|
||||
t := text[v.First+1 : v.Second]
|
||||
if _, value := tagsMap[t]; !value {
|
||||
tagsMap[t] = true
|
||||
tags = append(tags, t)
|
||||
}
|
||||
}
|
||||
|
||||
return tags
|
||||
}
|
||||
|
||||
// Takes a plaintext (ie., not HTML-formatted) text,
|
||||
// and returns a list of pairs of indices into the original string, where
|
||||
// hashtags are located.
|
||||
func FindHashtagSpansInText(text string) []Span {
|
||||
tags := []Span{}
|
||||
start := 0
|
||||
// Keep one rune of lookbehind.
|
||||
prev := ' '
|
||||
inTag := false
|
||||
|
||||
for i, r := range text {
|
||||
if r == '#' && isHashtagBoundary(prev) {
|
||||
// Start of hashtag.
|
||||
inTag = true
|
||||
start = i
|
||||
} else if inTag && !isPermittedInHashtag(r) && !isHashtagBoundary(r) {
|
||||
// Inside the hashtag, but it was a phoney, gottem.
|
||||
inTag = false
|
||||
} else if inTag && isHashtagBoundary(r) {
|
||||
// End of hashtag.
|
||||
inTag = false
|
||||
appendTag(&tags, text, start, i)
|
||||
} else if irl := i + utf8.RuneLen(r); inTag && irl == len(text) {
|
||||
// End of text.
|
||||
appendTag(&tags, text, start, irl)
|
||||
}
|
||||
|
||||
prev = r
|
||||
}
|
||||
|
||||
return tags
|
||||
}
|
||||
|
||||
func appendTag(tags *[]Span, text string, start int, end int) {
|
||||
l := end - start - 1
|
||||
// This check could be moved out into the parsing loop if necessary!
|
||||
if 0 < l && l <= maximumHashtagLength {
|
||||
*tags = append(*tags, Span{First: start, Second: end})
|
||||
}
|
||||
return UniqueStrings(tags)
|
||||
}
|
||||
|
||||
// DeriveEmojisFromText takes a plaintext (ie., not html-formatted) text,
|
||||
|
|
@ -58,3 +118,17 @@ func DeriveEmojisFromText(text string) []string {
|
|||
}
|
||||
return UniqueStrings(emojis)
|
||||
}
|
||||
|
||||
func isPermittedInHashtag(r rune) bool {
|
||||
return unicode.IsLetter(r) || unicode.IsNumber(r)
|
||||
}
|
||||
|
||||
// Decides where to break before or after a hashtag.
|
||||
func isHashtagBoundary(r rune) bool {
|
||||
return r == '#' || // `###lol` should work
|
||||
unicode.IsSpace(r) || // All kinds of Unicode whitespace.
|
||||
unicode.IsControl(r) || // All kinds of control characters, like tab.
|
||||
// Most kinds of punctuation except "Pc" ("Punctuation, connecting", like `_`).
|
||||
// But `someurl/#fragment` should not match, neither should HTML entities like `#`.
|
||||
('/' != r && '&' != r && !unicode.Is(unicode.Categories["Pc"], r) && unicode.IsPunct(r))
|
||||
}
|
||||
|
|
|
|||
|
|
@ -77,26 +77,50 @@ func (suite *StatusTestSuite) TestDeriveHashtagsOK() {
|
|||
|
||||
# testing this one shouldn't work
|
||||
|
||||
#thisshouldwork
|
||||
#thisshouldwork #dupe #dupe!! #dupe
|
||||
|
||||
here's a link with a fragment: https://example.org/whatever#ahhh
|
||||
here's another link with a fragment: https://example.org/whatever/#ahhh
|
||||
|
||||
#ThisShouldAlsoWork #not_this_though
|
||||
(#ThisShouldAlsoWork) #not_this_though
|
||||
|
||||
#111111 thisalsoshouldn'twork#### ##
|
||||
|
||||
#alimentación, #saúde
|
||||
#alimentación, #saúde, #lävistää, #ö, #네
|
||||
#ThisOneIsThirtyOneCharactersLon... ...ng
|
||||
#ThisOneIsThirteyCharactersLong
|
||||
`
|
||||
|
||||
tags := util.DeriveHashtagsFromText(statusText)
|
||||
assert.Len(suite.T(), tags, 7)
|
||||
assert.Len(suite.T(), tags, 12)
|
||||
assert.Equal(suite.T(), "testing123", tags[0])
|
||||
assert.Equal(suite.T(), "also", tags[1])
|
||||
assert.Equal(suite.T(), "thisshouldwork", tags[2])
|
||||
assert.Equal(suite.T(), "ThisShouldAlsoWork", tags[3])
|
||||
assert.Equal(suite.T(), "111111", tags[4])
|
||||
assert.Equal(suite.T(), "alimentación", tags[5])
|
||||
assert.Equal(suite.T(), "saúde", tags[6])
|
||||
assert.Equal(suite.T(), "dupe", tags[3])
|
||||
assert.Equal(suite.T(), "ThisShouldAlsoWork", tags[4])
|
||||
assert.Equal(suite.T(), "111111", tags[5])
|
||||
assert.Equal(suite.T(), "alimentación", tags[6])
|
||||
assert.Equal(suite.T(), "saúde", tags[7])
|
||||
assert.Equal(suite.T(), "lävistää", tags[8])
|
||||
assert.Equal(suite.T(), "ö", tags[9])
|
||||
assert.Equal(suite.T(), "네", tags[10])
|
||||
assert.Equal(suite.T(), "ThisOneIsThirteyCharactersLong", tags[11])
|
||||
|
||||
statusText = `#올빼미 hej`
|
||||
tags = util.DeriveHashtagsFromText(statusText)
|
||||
assert.Equal(suite.T(), "올빼미", tags[0])
|
||||
}
|
||||
|
||||
func (suite *StatusTestSuite) TestHashtagSpansOK() {
|
||||
statusText := `#0 #3 #8aa`
|
||||
|
||||
spans := util.FindHashtagSpansInText(statusText)
|
||||
assert.Equal(suite.T(), 0, spans[0].First)
|
||||
assert.Equal(suite.T(), 2, spans[0].Second)
|
||||
assert.Equal(suite.T(), 3, spans[1].First)
|
||||
assert.Equal(suite.T(), 5, spans[1].Second)
|
||||
assert.Equal(suite.T(), 8, spans[2].First)
|
||||
assert.Equal(suite.T(), 12, spans[2].Second)
|
||||
}
|
||||
|
||||
func (suite *StatusTestSuite) TestDeriveEmojiOK() {
|
||||
|
|
@ -127,7 +151,7 @@ Here's some normal text with an :emoji: at the end
|
|||
func (suite *StatusTestSuite) TestDeriveMultiple() {
|
||||
statusText := `Another test @foss_satan@fossbros-anonymous.io
|
||||
|
||||
#Hashtag
|
||||
#HashTag
|
||||
|
||||
Text`
|
||||
|
||||
|
|
@ -139,7 +163,7 @@ func (suite *StatusTestSuite) TestDeriveMultiple() {
|
|||
assert.Equal(suite.T(), "@foss_satan@fossbros-anonymous.io", ms[0])
|
||||
|
||||
assert.Len(suite.T(), hs, 1)
|
||||
assert.Equal(suite.T(), "Hashtag", hs[0])
|
||||
assert.Contains(suite.T(), hs, "HashTag")
|
||||
|
||||
assert.Len(suite.T(), es, 0)
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue