From ce190d867ca126001a1c0417b00810fc03c0b3ba Mon Sep 17 00:00:00 2001 From: Tobi Smethurst <31960611+tsmethurst@users.noreply.github.com> Date: Mon, 16 Aug 2021 19:17:56 +0200 Subject: [PATCH] Text/status parsing fixes (#141) * aaaaaa * vendor minify * update + test markdown parsing --- CONTRIBUTING.md | 4 +- README.md | 1 + dockerbuild.sh | 2 +- dockerpush.sh | 2 +- go.mod | 1 + go.sum | 10 + .../api/client/status/statuscreate_test.go | 6 +- internal/processing/status/util_test.go | 6 +- internal/text/common.go | 33 +- internal/text/markdown.go | 11 +- internal/text/markdown_test.go | 116 ++ internal/text/minify.go | 39 + internal/text/plain.go | 7 + internal/text/plain_test.go | 8 +- internal/text/sanitize.go | 17 +- internal/text/sanitize_test.go | 75 + .../tdewolff/minify/v2/.gitattributes | 2 + .../github.com/tdewolff/minify/v2/.gitignore | 14 + .../tdewolff/minify/v2/.golangci.yml | 16 + .../github.com/tdewolff/minify/v2/Dockerfile | 14 + vendor/github.com/tdewolff/minify/v2/LICENSE | 22 + vendor/github.com/tdewolff/minify/v2/Makefile | 56 + .../github.com/tdewolff/minify/v2/README.md | 693 +++++++++ .../github.com/tdewolff/minify/v2/common.go | 515 +++++++ vendor/github.com/tdewolff/minify/v2/go.mod | 14 + vendor/github.com/tdewolff/minify/v2/go.sum | 17 + .../tdewolff/minify/v2/html/buffer.go | 137 ++ .../tdewolff/minify/v2/html/hash.go | 543 +++++++ .../tdewolff/minify/v2/html/html.go | 511 +++++++ .../tdewolff/minify/v2/html/table.go | 1346 +++++++++++++++++ .../github.com/tdewolff/minify/v2/minify.go | 345 +++++ .../tdewolff/parse/v2/.gitattributes | 1 + .../github.com/tdewolff/parse/v2/.gitignore | 5 + .../tdewolff/parse/v2/.golangci.yml | 16 + .../github.com/tdewolff/parse/v2/LICENSE.md | 22 + vendor/github.com/tdewolff/parse/v2/README.md | 64 + .../tdewolff/parse/v2/buffer/buffer.go | 12 + .../tdewolff/parse/v2/buffer/lexer.go | 164 ++ .../tdewolff/parse/v2/buffer/reader.go | 44 + .../tdewolff/parse/v2/buffer/streamlexer.go | 223 +++ .../tdewolff/parse/v2/buffer/writer.go | 41 + vendor/github.com/tdewolff/parse/v2/common.go | 237 +++ vendor/github.com/tdewolff/parse/v2/error.go | 47 + vendor/github.com/tdewolff/parse/v2/go.mod | 5 + vendor/github.com/tdewolff/parse/v2/go.sum | 2 + .../tdewolff/parse/v2/html/README.md | 98 ++ .../github.com/tdewolff/parse/v2/html/hash.go | 81 + .../github.com/tdewolff/parse/v2/html/lex.go | 493 ++++++ .../github.com/tdewolff/parse/v2/html/util.go | 103 ++ vendor/github.com/tdewolff/parse/v2/input.go | 173 +++ .../github.com/tdewolff/parse/v2/position.go | 95 ++ .../tdewolff/parse/v2/strconv/float.go | 257 ++++ .../tdewolff/parse/v2/strconv/int.go | 88 ++ .../tdewolff/parse/v2/strconv/price.go | 83 + vendor/github.com/tdewolff/parse/v2/util.go | 489 ++++++ vendor/modules.txt | 9 + 56 files changed, 7390 insertions(+), 45 deletions(-) create mode 100644 internal/text/markdown_test.go create mode 100644 internal/text/minify.go create mode 100644 internal/text/sanitize_test.go create mode 100644 vendor/github.com/tdewolff/minify/v2/.gitattributes create mode 100644 vendor/github.com/tdewolff/minify/v2/.gitignore create mode 100644 vendor/github.com/tdewolff/minify/v2/.golangci.yml create mode 100644 vendor/github.com/tdewolff/minify/v2/Dockerfile create mode 100644 vendor/github.com/tdewolff/minify/v2/LICENSE create mode 100644 vendor/github.com/tdewolff/minify/v2/Makefile create mode 100644 vendor/github.com/tdewolff/minify/v2/README.md create mode 100644 vendor/github.com/tdewolff/minify/v2/common.go create mode 100644 vendor/github.com/tdewolff/minify/v2/go.mod create mode 100644 vendor/github.com/tdewolff/minify/v2/go.sum create mode 100644 vendor/github.com/tdewolff/minify/v2/html/buffer.go create mode 100644 vendor/github.com/tdewolff/minify/v2/html/hash.go create mode 100644 vendor/github.com/tdewolff/minify/v2/html/html.go create mode 100644 vendor/github.com/tdewolff/minify/v2/html/table.go create mode 100644 vendor/github.com/tdewolff/minify/v2/minify.go create mode 100644 vendor/github.com/tdewolff/parse/v2/.gitattributes create mode 100644 vendor/github.com/tdewolff/parse/v2/.gitignore create mode 100644 vendor/github.com/tdewolff/parse/v2/.golangci.yml create mode 100644 vendor/github.com/tdewolff/parse/v2/LICENSE.md create mode 100644 vendor/github.com/tdewolff/parse/v2/README.md create mode 100644 vendor/github.com/tdewolff/parse/v2/buffer/buffer.go create mode 100644 vendor/github.com/tdewolff/parse/v2/buffer/lexer.go create mode 100644 vendor/github.com/tdewolff/parse/v2/buffer/reader.go create mode 100644 vendor/github.com/tdewolff/parse/v2/buffer/streamlexer.go create mode 100644 vendor/github.com/tdewolff/parse/v2/buffer/writer.go create mode 100644 vendor/github.com/tdewolff/parse/v2/common.go create mode 100644 vendor/github.com/tdewolff/parse/v2/error.go create mode 100644 vendor/github.com/tdewolff/parse/v2/go.mod create mode 100644 vendor/github.com/tdewolff/parse/v2/go.sum create mode 100644 vendor/github.com/tdewolff/parse/v2/html/README.md create mode 100644 vendor/github.com/tdewolff/parse/v2/html/hash.go create mode 100644 vendor/github.com/tdewolff/parse/v2/html/lex.go create mode 100644 vendor/github.com/tdewolff/parse/v2/html/util.go create mode 100644 vendor/github.com/tdewolff/parse/v2/input.go create mode 100644 vendor/github.com/tdewolff/parse/v2/position.go create mode 100644 vendor/github.com/tdewolff/parse/v2/strconv/float.go create mode 100644 vendor/github.com/tdewolff/parse/v2/strconv/int.go create mode 100644 vendor/github.com/tdewolff/parse/v2/strconv/price.go create mode 100644 vendor/github.com/tdewolff/parse/v2/util.go diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 97220f221..8ccd7e7dc 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -139,7 +139,7 @@ We use [golangci-lint](https://golangci-lint.run/) for linting. To run this loca Then, you can run the linter with: ```bash -golangci-lint run +golangci-lint run --tests=false ``` Note that this linter also runs as a job on the Github repo, so if you make a PR that doesn't pass the linter, it will be rejected. As such, it's good practice to run the linter locally before pushing or opening a PR. @@ -155,7 +155,7 @@ go get -u github.com/golang/lint/golint To run the linter, use: ```bash -golint ./... +golint ./internal/... ``` Then make sure to run `go fmt ./...` to update whitespace and other opinionated formatting. diff --git a/README.md b/README.md index f336b440a..5a64dfa4f 100644 --- a/README.md +++ b/README.md @@ -143,6 +143,7 @@ The following libraries and frameworks are used by GoToSocial, with gratitude * [superseriousbusiness/exifremove](https://github.com/superseriousbusiness/exifremove) forked from [scottleedavis/go-exif-remove](https://github.com/scottleedavis/go-exif-remove); EXIF data removal. [MIT License](https://spdx.org/licenses/MIT.html). * [superseriousbusiness/oauth2](https://github.com/superseriousbusiness/oauth2) forked from [go-oauth2/oauth2](https://github.com/go-oauth2/oauth2); oauth server framework and token handling. [MIT License](https://spdx.org/licenses/MIT.html). * [go-swagger/go-swagger](https://github.com/go-swagger/go-swagger); Swagger OpenAPI spec generation. [Apache-2.0 License](https://spdx.org/licenses/Apache-2.0.html). +* [tdewolff/minify](https://github.com/tdewolff/minify); HTML minification. [MIT License](https://spdx.org/licenses/MIT.html). * [urfave/cli](https://github.com/urfave/cli); command-line interface framework. [MIT License](https://spdx.org/licenses/MIT.html). * [wagslane/go-password-validator](https://github.com/wagslane/go-password-validator); password strength validation. [MIT License](https://spdx.org/licenses/MIT.html). diff --git a/dockerbuild.sh b/dockerbuild.sh index 87893c65c..e2628ce55 100755 --- a/dockerbuild.sh +++ b/dockerbuild.sh @@ -1,3 +1,3 @@ #!/bin/bash -docker build -t "superseriousbusiness/gotosocial:$(cat version)" . +docker build -t "superseriousbusiness/gotosocial:$(git rev-parse --abbrev-ref HEAD)" . diff --git a/dockerpush.sh b/dockerpush.sh index 8377f8e4a..188b3cae0 100755 --- a/dockerpush.sh +++ b/dockerpush.sh @@ -1,3 +1,3 @@ #!/bin/bash -docker push "superseriousbusiness/gotosocial:$(cat version)" +docker push "superseriousbusiness/gotosocial:$(git rev-parse --abbrev-ref HEAD)" diff --git a/go.mod b/go.mod index 48febd4a6..10597a06b 100644 --- a/go.mod +++ b/go.mod @@ -44,6 +44,7 @@ require ( github.com/stretchr/testify v1.7.0 github.com/superseriousbusiness/exifremove v0.0.0-20210330092427-6acd27eac203 github.com/superseriousbusiness/oauth2/v4 v4.3.0-SSB + github.com/tdewolff/minify/v2 v2.9.21 github.com/tidwall/buntdb v1.2.4 // indirect github.com/urfave/cli/v2 v2.3.0 github.com/vmihailenco/msgpack/v5 v5.3.4 // indirect diff --git a/go.sum b/go.sum index e3599fa9d..4d6968ada 100644 --- a/go.sum +++ b/go.sum @@ -46,6 +46,7 @@ github.com/bradleypeabody/gorilla-sessions-memcache v0.0.0-20181103040241-659414 github.com/buckket/go-blurhash v1.1.0 h1:X5M6r0LIvwdvKiUtiNcRL2YlmOfMzYobI3VCKCZc9Do= github.com/buckket/go-blurhash v1.1.0/go.mod h1:aT2iqo5W9vu9GpyoLErKfTHwgODsZp3bQfXjXJUxNb8= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927/go.mod h1:h/aW8ynjgkuj+NQRlZcDbAbM1ORAbXjXX77sX7T289U= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -92,6 +93,7 @@ github.com/dsoprea/go-utility v0.0.0-20200711062821-fab8125e9bdf/go.mod h1:95+K3 github.com/dsoprea/go-utility v0.0.0-20200717064901-2fccff4aa15e h1:ojqYA1mU6LuRm8XzrVOvyfb000y59cbUcu6Wt8sFSAs= github.com/dsoprea/go-utility v0.0.0-20200717064901-2fccff4aa15e/go.mod h1:KVK+/Hul09ujXAGq+42UBgCTnXkiJZRnLYdURGjQUwo= github.com/dsoprea/go-utility/v2 v2.0.0-20200717064901-2fccff4aa15e/go.mod h1:uAzdkPTub5Y9yQwXe8W4m2XuP0tK4a9Q/dantD0+uaU= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -266,6 +268,7 @@ github.com/leodido/go-urn v1.1.0/go.mod h1:+cyI34gQWZcE1eQU7NVgKkkzdXDQHr1dBMtdA github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= +github.com/matryer/try v0.0.0-20161228173917-9ac251b645a2/go.mod h1:0KeJpeMD6o+O4hW7qJOT7vyQPKrWmj26uf5wMc/IiIs= github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= @@ -323,6 +326,7 @@ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykE github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -335,6 +339,12 @@ github.com/superseriousbusiness/exifremove v0.0.0-20210330092427-6acd27eac203 h1 github.com/superseriousbusiness/exifremove v0.0.0-20210330092427-6acd27eac203/go.mod h1:0Xw5cYMOYpgaWs+OOSx41ugycl2qvKTi9tlMMcZhFyY= github.com/superseriousbusiness/oauth2/v4 v4.3.0-SSB h1:dzMVC+oPTxFL5cv29egBrftlqIWPXQ6/VzkuoySwgm4= github.com/superseriousbusiness/oauth2/v4 v4.3.0-SSB/go.mod h1:8p0a/BEN9hhsGzE3tPaFFlIZgxAaLyLN5KY0bPg9ZBc= +github.com/tdewolff/minify/v2 v2.9.21 h1:nO4s1PEMy7aRjlIlbr3Jgr+bJby8QYuifa2Vs2f9lh4= +github.com/tdewolff/minify/v2 v2.9.21/go.mod h1:PoDBts2L7sCwUT28vTAlozGeD6qxjrrihtin4bR/RMM= +github.com/tdewolff/parse/v2 v2.5.19 h1:Kjaj3KQOx/4elIxlBSglus4E2oMfdROphvbq2b+OBZ0= +github.com/tdewolff/parse/v2 v2.5.19/go.mod h1:WzaJpRSbwq++EIQHYIRTpbYKNA3gn9it1Ik++q4zyho= +github.com/tdewolff/test v1.0.6 h1:76mzYJQ83Op284kMT+63iCNCI7NEERsIN8dLM+RiKr4= +github.com/tdewolff/test v1.0.6/go.mod h1:6DAvZliBAAnD7rhVgwaM7DE5/d9NMOAJ09SqYqeK4QE= github.com/tidwall/btree v0.0.0-20191029221954-400434d76274/go.mod h1:huei1BkDWJ3/sLXmO+bsCNELL+Bp2Kks9OLyQFkzvA8= github.com/tidwall/btree v0.5.0 h1:IBfCtOj4uOMQcodv3wzYVo0zPqSJObm71mE039/dlXY= github.com/tidwall/btree v0.5.0/go.mod h1:TzIRzen6yHbibdSfK6t8QimqbUnoxUSrZfeW7Uob0q4= diff --git a/internal/api/client/status/statuscreate_test.go b/internal/api/client/status/statuscreate_test.go index c175a54ec..33912397e 100644 --- a/internal/api/client/status/statuscreate_test.go +++ b/internal/api/client/status/statuscreate_test.go @@ -165,7 +165,7 @@ func (suite *StatusCreateTestSuite) TestPostAnotherNewStatus() { err = json.Unmarshal(b, statusReply) assert.NoError(suite.T(), err) - assert.Equal(suite.T(), "\u003cp\u003e\u003ca href=\"http://localhost:8080/tags/test\" class=\"mention hashtag\" rel=\"tag nofollow noreferrer noopener\" target=\"_blank\"\u003e#\u003cspan\u003etest\u003c/span\u003e\u003c/a\u003e alright, should be able to post \u003ca href=\"http://localhost:8080/tags/links\" class=\"mention hashtag\" rel=\"tag nofollow noreferrer noopener\" target=\"_blank\"\u003e#\u003cspan\u003elinks\u003c/span\u003e\u003c/a\u003e with fragments in them now, let\u0026#39;s see........\u003cbr/\u003e\u003cbr/\u003e\u003ca href=\"https://docs.gotosocial.org/en/latest/user_guide/posts/#links\" rel=\"noopener nofollow noreferrer\" target=\"_blank\"\u003edocs.gotosocial.org/en/latest/user_guide/posts/#links\u003c/a\u003e\u003cbr/\u003e\u003cbr/\u003e\u003ca href=\"http://localhost:8080/tags/gotosocial\" class=\"mention hashtag\" rel=\"tag nofollow noreferrer noopener\" target=\"_blank\"\u003e#\u003cspan\u003egotosocial\u003c/span\u003e\u003c/a\u003e\u003cbr/\u003e\u003cbr/\u003e(tobi remember to pull the docker image challenge)\u003c/p\u003e", statusReply.Content) + assert.Equal(suite.T(), "
#test alright, should be able to post #links with fragments in them now, let's see........
docs.gotosocial.org/en/latest/user_guide/posts/#links
#gotosocial
(tobi remember to pull the docker image challenge)
here is a rainbow emoji a few times! :rainbow: :rainbow: :rainbow:
here's an emoji that isn't in the db: :test_emoji:
here is a rainbow emoji a few times! :rainbow: :rainbow: :rainbow:
here's an emoji that isn't in the db: :test_emoji:
here's an image attachment
", statusResponse.Content) + assert.Equal(suite.T(), "here's an image attachment
", statusResponse.Content) assert.False(suite.T(), statusResponse.Sensitive) assert.Equal(suite.T(), model.VisibilityPublic, statusResponse.Visibility) diff --git a/internal/processing/status/util_test.go b/internal/processing/status/util_test.go index 9a4bd6515..4bf508848 100644 --- a/internal/processing/status/util_test.go +++ b/internal/processing/status/util_test.go @@ -17,8 +17,8 @@ const statusText1 = `Another test @foss_satan@fossbros-anonymous.io #Hashtag Text` -const statusText1ExpectedFull = `Another test @foss_satan
#Hashtag
Text
Another test @foss_satan
#Hashtag
Text
Another test @foss_satan
#Hashtag
Text
Another test @foss_satan
#Hashtag
Text
Another test @foss_satan
#Hashtag
#hashTAG
Another test @foss_satan
#Hashtag
#hashTAG
%s
`, s) - // 3. remove any cheeky newlines - s = strings.ReplaceAll(s, "\n", "") - // 4. remove any whitespace added as a result of the formatting - s = strings.TrimSpace(s) - return s + + // 1. sanitize html to remove potentially dangerous elements + s := SanitizeHTML(in) + + // 2. the sanitize step tends to escape characters inside codeblocks, which is behavior we don't want, so unescape everything again + s = html.UnescapeString(s) + + // 3. minify html to remove any trailing newlines, spaces, unnecessary elements, etc etc + mini, err := minifyHTML(s) + if err != nil { + // if the minify failed, just return what we have + return s + } + // return minified version of the html + return mini } func (f *formatter) ReplaceTags(in string, tags []*gtsmodel.Tag) string { diff --git a/internal/text/markdown.go b/internal/text/markdown.go index f9d12209a..5a7603615 100644 --- a/internal/text/markdown.go +++ b/internal/text/markdown.go @@ -23,21 +23,14 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" ) -var bfExtensions = blackfriday.NoIntraEmphasis | - blackfriday.FencedCode | - blackfriday.Autolink | - blackfriday.Strikethrough | - blackfriday.SpaceHeadings | - blackfriday.BackslashLineBreak - func (f *formatter) FromMarkdown(md string, mentions []*gtsmodel.Mention, tags []*gtsmodel.Tag) string { content := preformat(md) // do the markdown parsing *first* - content = string(blackfriday.Run([]byte(content), blackfriday.WithExtensions(bfExtensions))) + contentBytes := blackfriday.Run([]byte(content)) // format tags nicely - content = f.ReplaceTags(content, tags) + content = f.ReplaceTags(string(contentBytes), tags) // format mentions nicely content = f.ReplaceMentions(content, mentions) diff --git a/internal/text/markdown_test.go b/internal/text/markdown_test.go new file mode 100644 index 000000000..432e9a4ec --- /dev/null +++ b/internal/text/markdown_test.go @@ -0,0 +1,116 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, seeHere’s a simple text in markdown.
Here’s a link.
" + + withCodeBlockExpected = "Below is some JSON.
{\n \"key\": \"value\",\n \"another_key\": [\n \"value1\",\n \"value2\"\n ]\n}\n
that was some JSON :)
" + + withHashtag = "# Title\n\nhere's a simple status that uses hashtag #Hashtag!" + withHashtagExpected = "here’s a simple status that uses hashtag #Hashtag!
" +) + +var ( + withCodeBlock = `# Title + +Below is some JSON. + +` + "```" + `json +{ + "key": "value", + "another_key": [ + "value1", + "value2" + ] +} +` + "```" + ` + +that was some JSON :) +` +) + +type MarkdownTestSuite struct { + TextStandardTestSuite +} + +func (suite *MarkdownTestSuite) SetupSuite() { + suite.testTokens = testrig.NewTestTokens() + suite.testClients = testrig.NewTestClients() + suite.testApplications = testrig.NewTestApplications() + suite.testUsers = testrig.NewTestUsers() + suite.testAccounts = testrig.NewTestAccounts() + suite.testAttachments = testrig.NewTestAttachments() + suite.testStatuses = testrig.NewTestStatuses() + suite.testTags = testrig.NewTestTags() + suite.testMentions = testrig.NewTestMentions() +} + +func (suite *MarkdownTestSuite) SetupTest() { + suite.config = testrig.NewTestConfig() + suite.db = testrig.NewTestDB() + suite.log = testrig.NewTestLog() + suite.formatter = text.NewFormatter(suite.config, suite.db, suite.log) + + testrig.StandardDBSetup(suite.db, suite.testAccounts) +} + +func (suite *MarkdownTestSuite) TearDownTest() { + testrig.StandardDBTeardown(suite.db) +} + +func (suite *MarkdownTestSuite) TestParseSimple() { + s := suite.formatter.FromMarkdown(simpleMarkdown, nil, nil) + suite.Equal(simpleMarkdownExpected, s) +} + +func (suite *MarkdownTestSuite) TestParseWithCodeBlock() { + fmt.Println(withCodeBlock) + s := suite.formatter.FromMarkdown(withCodeBlock, nil, nil) + suite.Equal(withCodeBlockExpected, s) +} + +func (suite *MarkdownTestSuite) TestParseWithHashtag() { + foundTags := []*gtsmodel.Tag{ + suite.testTags["Hashtag"], + } + + s := suite.formatter.FromMarkdown(withHashtag, nil, foundTags) + suite.Equal(withHashtagExpected, s) +} + +func TestMarkdownTestSuite(t *testing.T) { + suite.Run(t, new(MarkdownTestSuite)) +} diff --git a/internal/text/minify.go b/internal/text/minify.go new file mode 100644 index 000000000..c6d7b9bc1 --- /dev/null +++ b/internal/text/minify.go @@ -0,0 +1,39 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see%s
`, content) + return postformat(content) } diff --git a/internal/text/plain_test.go b/internal/text/plain_test.go index 2f9eb3a29..33c95234c 100644 --- a/internal/text/plain_test.go +++ b/internal/text/plain_test.go @@ -33,15 +33,15 @@ const ( simple = "this is a plain and simple status" simpleExpected = "this is a plain and simple status
" - withTag = "this is a simple status that uses hashtag #welcome!" - withTagExpected = "this is a simple status that uses hashtag #welcome!
" + withTag = "here's a simple status that uses hashtag #welcome!" + withTagExpected = "here's a simple status that uses hashtag #welcome!
" moreComplex = `Another test @foss_satan@fossbros-anonymous.io #Hashtag Text` - moreComplexExpected = `Another test @foss_satan
#Hashtag
Text
Another test @foss_satan
#Hashtag
Text
Another test @foss_satan
#Hashtag
Text
gotta test some fucking ''''''''' marks
` + sanitizedOutgoing = `gotta test some fucking ''''''''' marks
` +) + +type SanitizeTestSuite struct { + suite.Suite +} + +func (suite *SanitizeTestSuite) TestRemoveHTML() { + s := text.RemoveHTML(removeHTML) + suite.Equal(removedHTML, s) +} + +func (suite *SanitizeTestSuite) TestSanitizeOutgoing() { + s := text.SanitizeHTML(sanitizeOutgoing) + suite.Equal(sanitizedOutgoing, s) +} + +func (suite *SanitizeTestSuite) TestSanitizeHTML() { + s := text.SanitizeHTML(sanitizeHTML) + suite.Equal(sanitizedHTML, s) +} + +func (suite *SanitizeTestSuite) TestSanitizeWithEscapedLiteral() { + s := text.RemoveHTML(withEscapedLiteral) + suite.Equal(withEscapedLiteralExpected, s) +} + +func (suite *SanitizeTestSuite) TestSanitizeWithEscaped() { + s := text.RemoveHTML(withEscaped) + suite.Equal(withEscapedExpected, s) +} + +func TestSanitizeTestSuite(t *testing.T) { + suite.Run(t, new(SanitizeTestSuite)) +} diff --git a/vendor/github.com/tdewolff/minify/v2/.gitattributes b/vendor/github.com/tdewolff/minify/v2/.gitattributes new file mode 100644 index 000000000..16a3a8b06 --- /dev/null +++ b/vendor/github.com/tdewolff/minify/v2/.gitattributes @@ -0,0 +1,2 @@ +benchmarks/sample_* linguist-generated +tests/*/corpus/* linguist-generated diff --git a/vendor/github.com/tdewolff/minify/v2/.gitignore b/vendor/github.com/tdewolff/minify/v2/.gitignore new file mode 100644 index 000000000..8653de91d --- /dev/null +++ b/vendor/github.com/tdewolff/minify/v2/.gitignore @@ -0,0 +1,14 @@ +dist/ +benchmarks/* +!benchmarks/*.go +!benchmarks/sample_* +tests/*/fuzz-fuzz.zip +tests/*/crashers +tests/*/suppressions +tests/*/corpus/* +!tests/*/corpus/*.* +parse/tests/*/fuzz-fuzz.zip +parse/tests/*/crashers +parse/tests/*/suppressions +parse/tests/*/corpus/* +!parse/tests/*/corpus/*.* diff --git a/vendor/github.com/tdewolff/minify/v2/.golangci.yml b/vendor/github.com/tdewolff/minify/v2/.golangci.yml new file mode 100644 index 000000000..7009f9201 --- /dev/null +++ b/vendor/github.com/tdewolff/minify/v2/.golangci.yml @@ -0,0 +1,16 @@ +linters: + enable: + - depguard + - dogsled + - gofmt + - goimports + - golint + - gosec + - govet + - megacheck + - misspell + - nakedret + - prealloc + - unconvert + - unparam + - wastedassign diff --git a/vendor/github.com/tdewolff/minify/v2/Dockerfile b/vendor/github.com/tdewolff/minify/v2/Dockerfile new file mode 100644 index 000000000..6cc2de9cc --- /dev/null +++ b/vendor/github.com/tdewolff/minify/v2/Dockerfile @@ -0,0 +1,14 @@ +# Use this image to build the executable +FROM golang:1.16-alpine AS compiler + +WORKDIR $GOPATH/src/minify +COPY . . + +RUN apk add --update --update-cache --no-cache git ca-certificates && \ + GO111MODULES=on CGO_ENABLED=0 go build -ldflags "-s -w" -trimpath -o /bin/minify ./cmd/minify + + +# Final image containing the executable from the previous step +FROM alpine:3 + +COPY --from=compiler /bin/minify /bin/minify diff --git a/vendor/github.com/tdewolff/minify/v2/LICENSE b/vendor/github.com/tdewolff/minify/v2/LICENSE new file mode 100644 index 000000000..41677de41 --- /dev/null +++ b/vendor/github.com/tdewolff/minify/v2/LICENSE @@ -0,0 +1,22 @@ +Copyright (c) 2015 Taco de Wolff + + Permission is hereby granted, free of charge, to any person + obtaining a copy of this software and associated documentation + files (the "Software"), to deal in the Software without + restriction, including without limitation the rights to use, + copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following + conditions: + + The above copyright notice and this permission notice shall be + included in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/vendor/github.com/tdewolff/minify/v2/Makefile b/vendor/github.com/tdewolff/minify/v2/Makefile new file mode 100644 index 000000000..22e448a3c --- /dev/null +++ b/vendor/github.com/tdewolff/minify/v2/Makefile @@ -0,0 +1,56 @@ +NAME=minify +CMD=./cmd/minify +TARGETS=linux_amd64 darwin_amd64 freebsd_amd64 netbsd_amd64 openbsd_amd64 windows_amd64 +VERSION=`git describe --tags` +FLAGS=-ldflags "-s -w -X 'main.Version=${VERSION}'" -trimpath +ENVS=GO111MODULES=on CGO_ENABLED=0 + +all: install + +install: + echo "Installing ${VERSION}" + ${ENVS} go install ${FLAGS} ./cmd/minify + . cmd/minify/bash_completion + +release: + TAG=$(shell git describe --tags --exact-match 2> /dev/null); + if [ "${.SHELLSTATUS}" -eq 0 ]; then \ + echo "Releasing ${VERSION}"; \ + else \ + echo "WARNING: commit is not tagged with a version"; \ + echo ""; \ + fi + rm -rf dist + mkdir -p dist + for t in ${TARGETS}; do \ + echo Building $$t...; \ + mkdir dist/$$t; \ + os=$$(echo $$t | cut -f1 -d_); \ + arch=$$(echo $$t | cut -f2 -d_); \ + ${ENVS} GOOS=$$os GOARCH=$$arch go build ${FLAGS} -o dist/$$t/${NAME} ${CMD}; \ + \ + cp LICENSE dist/$$t/.; \ + cp cmd/minify/README.md dist/$$t/.; \ + if [ "$$os" == "windows" ]; then \ + mv dist/$$t/${NAME} dist/$$t/${NAME}.exe; \ + zip -jq dist/${NAME}_$$t.zip dist/$$t/*; \ + cd dist; \ + sha256sum ${NAME}_$$t.zip >> checksums.txt; \ + cd ..; \ + else \ + cp cmd/minify/bash_completion dist/$$t/.; \ + cd dist/$$t; \ + tar -cf - * | gzip -9 > ../${NAME}_$$t.tar.gz; \ + cd ..; \ + sha256sum ${NAME}_$$t.tar.gz >> checksums.txt; \ + cd ..; \ + fi; \ + rm -rf dist/$$t; \ + done + +clean: + echo "Cleaning dist/" + rm -rf dist + +.PHONY: install release clean +.SILENT: install release clean diff --git a/vendor/github.com/tdewolff/minify/v2/README.md b/vendor/github.com/tdewolff/minify/v2/README.md new file mode 100644 index 000000000..0dfcb06c2 --- /dev/null +++ b/vendor/github.com/tdewolff/minify/v2/README.md @@ -0,0 +1,693 @@ +# Minify [![API reference](https://img.shields.io/badge/godoc-reference-5272B4)](https://pkg.go.dev/github.com/tdewolff/minify/v2?tab=doc) [![Go Report Card](https://goreportcard.com/badge/github.com/tdewolff/minify)](https://goreportcard.com/report/github.com/tdewolff/minify) [![codecov](https://codecov.io/gh/tdewolff/minify/branch/master/graph/badge.svg?token=Cr7r2EKPj2)](https://codecov.io/gh/tdewolff/minify) [![Donate](https://img.shields.io/badge/patreon-donate-DFB317)](https://www.patreon.com/tdewolff) + +**[Online demo](https://go.tacodewolff.nl/minify) if you need to minify files *now*.** + +**[Command line tool](https://github.com/tdewolff/minify/tree/master/cmd/minify) that minifies concurrently and watches file changes.** + +**[Releases](https://github.com/tdewolff/minify/releases) of CLI for various platforms.** See [CLI](https://github.com/tdewolff/minify/tree/master/cmd/minify) for more installation instructions. + +**[Parse](https://github.com/tdewolff/minify/tree/master/parse) subpackage on which minify depends.** + +--- + +*Did you know that the shortest valid piece of HTML5 is `