mirrors
/
gotosocial
mirror of
1
Fork 0
Code Issues Releases Activity

[feature] Custom emoji updates (serve emoji via s2s api, tune db models) (#805) 

* migrate emojis

* add get emoji to s2s (federation) API

* add new emoji db + cache functions

* add shortcodeDomain lookup for emojis

* check existing emojis w/cache, not w/constraints

* go fmt

* add putEmoji func

* use new db emoji funcs instead of where

* remove emojistringstotags func

* add unique constraint back in

* fix up broken migration

* update index
This commit is contained in:
tobi 2022-09-06 12:42:55 +02:00 committed by GitHub
parent ee01e030d4
commit a872ddebe6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
21 changed files with 773 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
internal/api/client/admin/emojicreate_test.go
View File

@ -29,8 +29,6 @@ import (
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/api/client/admin"
apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/testrig"
)
@ -79,8 +77,7 @@ func (suite *EmojiCreateTestSuite) TestEmojiCreate() {
suite.True(apiEmoji.VisibleInPicker)
// emoji should be in the db
dbEmoji := &gtsmodel.Emoji{}
err = suite.db.GetWhere(context.Background(), []db.Where{{Key: "shortcode", Value: "new_emoji"}}, dbEmoji)
dbEmoji, err := suite.db.GetEmojiByShortcodeDomain(context.Background(), apiEmoji.Shortcode, "")
suite.NoError(err)
// check fields on the emoji

53
internal/api/s2s/emoji/emoji.go Normal file
View File

@ -0,0 +1,53 @@
/*
GoToSocial
Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package emoji
import (
"net/http"
"github.com/superseriousbusiness/gotosocial/internal/api"
"github.com/superseriousbusiness/gotosocial/internal/processing"
"github.com/superseriousbusiness/gotosocial/internal/router"
"github.com/superseriousbusiness/gotosocial/internal/uris"
)
const (
// EmojiIDKey is for emoji IDs
EmojiIDKey = "id"
// EmojiBasePath is the base path for serving information about Emojis eg https://example.org/emoji
EmojiWithIDPath = "/" + uris.EmojiPath + "/:" + EmojiIDKey
)
// Module implements the FederationModule interface
type Module struct {
processor processing.Processor
}
// New returns a emoji module
func New(processor processing.Processor) api.FederationModule {
return &Module{
processor: processor,
}
}
// Route satisfies the RESTAPIModule interface
func (m *Module) Route(s router.Router) error {
s.AttachHandler(http.MethodGet, EmojiWithIDPath, m.EmojiGetHandler)
return nil
}

74
internal/api/s2s/emoji/emojiget.go Normal file
View File

@ -0,0 +1,74 @@
/*
GoToSocial
Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package emoji
import (
"context"
"encoding/json"
"errors"
"net/http"
"strings"
"github.com/gin-gonic/gin"
"github.com/superseriousbusiness/gotosocial/internal/ap"
"github.com/superseriousbusiness/gotosocial/internal/api"
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
)
// EmojiGetHandler
func (m *Module) EmojiGetHandler(c *gin.Context) {
// usernames on our instance are always lowercase
requestedEmojiID := strings.ToUpper(c.Param(EmojiIDKey))
if requestedEmojiID == "" {
err := errors.New("no emoji id specified in request")
api.ErrorHandler(c, gtserror.NewErrorBadRequest(err, err.Error()), m.processor.InstanceGet)
return
}
format, err := api.NegotiateAccept(c, api.ActivityPubAcceptHeaders...)
if err != nil {
api.ErrorHandler(c, gtserror.NewErrorNotAcceptable(err, err.Error()), m.processor.InstanceGet)
return
}
ctx := c.Request.Context()
verifier, signed := c.Get(string(ap.ContextRequestingPublicKeyVerifier))
if signed {
ctx = context.WithValue(ctx, ap.ContextRequestingPublicKeyVerifier, verifier)
}
signature, signed := c.Get(string(ap.ContextRequestingPublicKeySignature))
if signed {
ctx = context.WithValue(ctx, ap.ContextRequestingPublicKeySignature, signature)
}
resp, errWithCode := m.processor.GetFediEmoji(ctx, requestedEmojiID, c.Request.URL)
if errWithCode != nil {
api.ErrorHandler(c, errWithCode, m.processor.InstanceGet)
return
}
b, err := json.Marshal(resp)
if err != nil {
api.ErrorHandler(c, gtserror.NewErrorInternalError(err), m.processor.InstanceGet)
return
}
c.Data(http.StatusOK, format, b)
}

136
internal/api/s2s/emoji/emojiget_test.go Normal file
View File

@ -0,0 +1,136 @@
/*
GoToSocial
Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package emoji_test
import (
"io/ioutil"
"net/http"
"net/http/httptest"
"testing"
"github.com/gin-gonic/gin"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/api/s2s/emoji"
"github.com/superseriousbusiness/gotosocial/internal/api/security"
"github.com/superseriousbusiness/gotosocial/internal/concurrency"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/email"
"github.com/superseriousbusiness/gotosocial/internal/federation"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/media"
"github.com/superseriousbusiness/gotosocial/internal/messages"
"github.com/superseriousbusiness/gotosocial/internal/oauth"
"github.com/superseriousbusiness/gotosocial/internal/processing"
"github.com/superseriousbusiness/gotosocial/internal/storage"
"github.com/superseriousbusiness/gotosocial/internal/typeutils"
"github.com/superseriousbusiness/gotosocial/testrig"
)
type EmojiGetTestSuite struct {
suite.Suite
db db.DB
tc typeutils.TypeConverter
mediaManager media.Manager
federator federation.Federator
emailSender email.Sender
processor processing.Processor
storage storage.Driver
oauthServer oauth.Server
securityModule *security.Module
testEmojis map[string]*gtsmodel.Emoji
testAccounts map[string]*gtsmodel.Account
emojiModule *emoji.Module
}
func (suite *EmojiGetTestSuite) SetupSuite() {
suite.testAccounts = testrig.NewTestAccounts()
suite.testEmojis = testrig.NewTestEmojis()
}
func (suite *EmojiGetTestSuite) SetupTest() {
testrig.InitTestConfig()
testrig.InitTestLog()
clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1)
fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1)
suite.db = testrig.NewTestDB()
suite.tc = testrig.NewTestTypeConverter(suite.db)
suite.storage = testrig.NewInMemoryStorage()
suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker)
suite.emailSender = testrig.NewEmailSender("../../../../web/template/", nil)
suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager, clientWorker, fedWorker)
suite.emojiModule = emoji.New(suite.processor).(*emoji.Module)
suite.oauthServer = testrig.NewTestOauthServer(suite.db)
suite.securityModule = security.New(suite.db, suite.oauthServer).(*security.Module)
testrig.StandardDBSetup(suite.db, suite.testAccounts)
testrig.StandardStorageSetup(suite.storage, "../../../../testrig/media")
}
func (suite *EmojiGetTestSuite) TearDownTest() {
testrig.StandardDBTeardown(suite.db)
testrig.StandardStorageTeardown(suite.storage)
}
func (suite *EmojiGetTestSuite) TestGetEmoji() {
// the dereference we're gonna use
derefRequests := testrig.NewTestDereferenceRequests(suite.testAccounts)
signedRequest := derefRequests["foss_satan_dereference_emoji"]
targetEmoji := suite.testEmojis["rainbow"]
// setup request
recorder := httptest.NewRecorder()
ctx, _ := testrig.CreateGinTestContext(recorder, nil)
ctx.Request = httptest.NewRequest(http.MethodGet, targetEmoji.URI, nil) // the endpoint we're hitting
ctx.Request.Header.Set("accept", "application/activity+json")
ctx.Request.Header.Set("Signature", signedRequest.SignatureHeader)
ctx.Request.Header.Set("Date", signedRequest.DateHeader)
// we need to pass the context through signature check first to set appropriate values on it
suite.securityModule.SignatureCheck(ctx)
// normally the router would populate these params from the path values,
// but because we're calling the function directly, we need to set them manually.
ctx.Params = gin.Params{
gin.Param{
Key: emoji.EmojiIDKey,
Value: targetEmoji.ID,
},
}
// trigger the function being tested
suite.emojiModule.EmojiGetHandler(ctx)
// check response
suite.EqualValues(http.StatusOK, recorder.Code)
result := recorder.Result()
defer result.Body.Close()
b, err := ioutil.ReadAll(result.Body)
suite.NoError(err)
suite.Contains(string(b), `"icon":{"mediaType":"image/png","type":"Image","url":"http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/emoji/original/01F8MH9H8E4VG3KDYJR9EGPXCQ.png"},"id":"http://localhost:8080/emoji/01F8MH9H8E4VG3KDYJR9EGPXCQ","name":":rainbow:","type":"Emoji"`)
}
func TestEmojiGetTestSuite(t *testing.T) {
suite.Run(t, new(EmojiGetTestSuite))
}

116
internal/cache/emoji.go vendored Normal file
View File

@ -0,0 +1,116 @@
/*
GoToSocial
Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package cache
import (
"time"
"codeberg.org/gruf/go-cache/v2"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
)
// EmojiCache is a cache wrapper to provide ID and URI lookups for gtsmodel.Emoji
type EmojiCache struct {
cache cache.LookupCache[string, string, *gtsmodel.Emoji]
}
// NewEmojiCache returns a new instantiated EmojiCache object
func NewEmojiCache() *EmojiCache {
c := &EmojiCache{}
c.cache = cache.NewLookup(cache.LookupCfg[string, string, *gtsmodel.Emoji]{
RegisterLookups: func(lm *cache.LookupMap[string, string]) {
lm.RegisterLookup("uri")
lm.RegisterLookup("shortcodedomain")
},
AddLookups: func(lm *cache.LookupMap[string, string], emoji *gtsmodel.Emoji) {
if uri := emoji.URI; uri != "" {
lm.Set("uri", uri, emoji.URI)
lm.Set("shortcodedomain", shortcodeDomainKey(emoji.Shortcode, emoji.Domain), emoji.ID)
}
},
DeleteLookups: func(lm *cache.LookupMap[string, string], emoji *gtsmodel.Emoji) {
if uri := emoji.URI; uri != "" {
lm.Delete("uri", uri)
lm.Delete("shortcodedomain", shortcodeDomainKey(emoji.Shortcode, emoji.Domain))
}
},
})
c.cache.SetTTL(time.Minute*5, false)
c.cache.Start(time.Second * 10)
return c
}
// GetByID attempts to fetch an emoji from the cache by its ID, you will receive a copy for thread-safety
func (c *EmojiCache) GetByID(id string) (*gtsmodel.Emoji, bool) {
return c.cache.Get(id)
}
// GetByURI attempts to fetch an emoji from the cache by its URI, you will receive a copy for thread-safety
func (c *EmojiCache) GetByURI(uri string) (*gtsmodel.Emoji, bool) {
return c.cache.GetBy("uri", uri)
}
func (c *EmojiCache) GetByShortcodeDomain(shortcode string, domain string) (*gtsmodel.Emoji, bool) {
return c.cache.GetBy("shortcodedomain", shortcodeDomainKey(shortcode, domain))
}
// Put places an emoji in the cache, ensuring that the object place is a copy for thread-safety
func (c *EmojiCache) Put(emoji *gtsmodel.Emoji) {
if emoji == nil || emoji.ID == "" {
panic("invalid emoji")
}
c.cache.Set(emoji.ID, copyEmoji(emoji))
}
// copyEmoji performs a surface-level copy of emoji, only keeping attached IDs intact, not the objects.
// due to all the data being copied being 99% primitive types or strings (which are immutable and passed by ptr)
// this should be a relatively cheap process
func copyEmoji(emoji *gtsmodel.Emoji) *gtsmodel.Emoji {
return &gtsmodel.Emoji{
ID: emoji.ID,
CreatedAt: emoji.CreatedAt,
UpdatedAt: emoji.UpdatedAt,
Shortcode: emoji.Shortcode,
Domain: emoji.Domain,
ImageRemoteURL: emoji.ImageRemoteURL,
ImageStaticRemoteURL: emoji.ImageStaticRemoteURL,
ImageURL: emoji.ImageURL,
ImageStaticURL: emoji.ImageStaticURL,
ImagePath: emoji.ImagePath,
ImageStaticPath: emoji.ImageStaticPath,
ImageContentType: emoji.ImageContentType,
ImageStaticContentType: emoji.ImageStaticContentType,
ImageFileSize: emoji.ImageFileSize,
ImageStaticFileSize: emoji.ImageStaticFileSize,
ImageUpdatedAt: emoji.ImageUpdatedAt,
Disabled: copyBoolPtr(emoji.Disabled),
URI: emoji.URI,
VisibleInPicker: copyBoolPtr(emoji.VisibleInPicker),
CategoryID: emoji.CategoryID,
}
}
func shortcodeDomainKey(shortcode string, domain string) string {
if domain != "" {
return shortcode + "@" + domain
}
return shortcode
}

24
internal/db/bundb/bundb.go
View File

@ -154,6 +154,7 @@ func NewBunDBService(ctx context.Context) (db.DB, error) {
// Create DB structs that require ptrs to each other
accounts := &accountDB{conn: conn, cache: cache.NewAccountCache()}
status := &statusDB{conn: conn, cache: cache.NewStatusCache()}
emoji := &emojiDB{conn: conn, cache: cache.NewEmojiCache()}
timeline := &timelineDB{conn: conn}
// Setup DB cross-referencing
@ -188,9 +189,7 @@ func NewBunDBService(ctx context.Context) (db.DB, error) {
conn: conn,
cache: blockCache,
},
Emoji: &emojiDB{
conn: conn,
},
Emoji: emoji,
Instance: &instanceDB{
conn: conn,
},
@ -440,22 +439,3 @@ func (ps *bunDBService) TagStringsToTags(ctx context.Context, tags []string, ori
}
return newTags, nil
}
func (ps *bunDBService) EmojiStringsToEmojis(ctx context.Context, emojis []string) ([]*gtsmodel.Emoji, error) {
newEmojis := []*gtsmodel.Emoji{}
for _, e := range emojis {
emoji := &gtsmodel.Emoji{}
err := ps.conn.NewSelect().Model(emoji).Where("shortcode = ?", e).Where("visible_in_picker = true").Where("disabled = false").Scan(ctx)
if err != nil {
if err == sql.ErrNoRows {
// no result found for this username/domain so just don't include it as an emoji and carry on about our business
log.Debugf("no emoji found with shortcode %s, skipping it", e)
continue
}
// a serious error has happened so bail
return nil, fmt.Errorf("error getting emoji with shortcode %s: %s", e, err)
}
newEmojis = append(newEmojis, emoji)
}
return newEmojis, nil
}

117
internal/db/bundb/emoji.go
View File

@ -20,27 +20,136 @@ package bundb
import (
"context"
"strings"
"github.com/superseriousbusiness/gotosocial/internal/cache"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/log"
"github.com/uptrace/bun"
)
type emojiDB struct {
conn *DBConn
cache *cache.EmojiCache
}
func (e emojiDB) GetCustomEmojis(ctx context.Context) ([]*gtsmodel.Emoji, db.Error) {
emojis := []*gtsmodel.Emoji{}
func (e *emojiDB) newEmojiQ(emoji *gtsmodel.Emoji) *bun.SelectQuery {
return e.conn.
NewSelect().
Model(emoji)
}
func (e *emojiDB) PutEmoji(ctx context.Context, emoji *gtsmodel.Emoji) db.Error {
if _, err := e.conn.NewInsert().Model(emoji).Exec(ctx); err != nil {
return e.conn.ProcessError(err)
}
e.cache.Put(emoji)
return nil
}
func (e *emojiDB) GetCustomEmojis(ctx context.Context) ([]*gtsmodel.Emoji, db.Error) {
emojiIDs := []string{}
q := e.conn.
NewSelect().
Model(&emojis).
Table("emojis").
Column("id").
Where("visible_in_picker = true").
Where("disabled = false").
Where("domain IS NULL").
Order("shortcode ASC")
if err := q.Scan(ctx); err != nil {
if err := q.Scan(ctx, &emojiIDs); err != nil {
return nil, e.conn.ProcessError(err)
}
return e.emojisFromIDs(ctx, emojiIDs)
}
func (e *emojiDB) GetEmojiByID(ctx context.Context, id string) (*gtsmodel.Emoji, db.Error) {
return e.getEmoji(
ctx,
func() (*gtsmodel.Emoji, bool) {
return e.cache.GetByID(id)
},
func(emoji *gtsmodel.Emoji) error {
return e.newEmojiQ(emoji).Where("emoji.id = ?", id).Scan(ctx)
},
)
}
func (e *emojiDB) GetEmojiByURI(ctx context.Context, uri string) (*gtsmodel.Emoji, db.Error) {
return e.getEmoji(
ctx,
func() (*gtsmodel.Emoji, bool) {
return e.cache.GetByURI(uri)
},
func(emoji *gtsmodel.Emoji) error {
return e.newEmojiQ(emoji).Where("emoji.uri = ?", uri).Scan(ctx)
},
)
}
func (e *emojiDB) GetEmojiByShortcodeDomain(ctx context.Context, shortcode string, domain string) (*gtsmodel.Emoji, db.Error) {
return e.getEmoji(
ctx,
func() (*gtsmodel.Emoji, bool) {
return e.cache.GetByShortcodeDomain(shortcode, domain)
},
func(emoji *gtsmodel.Emoji) error {
q := e.newEmojiQ(emoji)
if domain != "" {
q = q.Where("emoji.shortcode = ?", shortcode)
q = q.Where("emoji.domain = ?", domain)
} else {
q = q.Where("emoji.shortcode = ?", strings.ToLower(shortcode))
q = q.Where("emoji.domain IS NULL")
}
return q.Scan(ctx)
},
)
}
func (e *emojiDB) getEmoji(ctx context.Context, cacheGet func() (*gtsmodel.Emoji, bool), dbQuery func(*gtsmodel.Emoji) error) (*gtsmodel.Emoji, db.Error) {
// Attempt to fetch cached emoji
emoji, cached := cacheGet()
if !cached {
emoji = &gtsmodel.Emoji{}
// Not cached! Perform database query
err := dbQuery(emoji)
if err != nil {
return nil, e.conn.ProcessError(err)
}
// Place in the cache
e.cache.Put(emoji)
}
return emoji, nil
}
func (e *emojiDB) emojisFromIDs(ctx context.Context, emojiIDs []string) ([]*gtsmodel.Emoji, db.Error) {
// Catch case of no emojis early
if len(emojiIDs) == 0 {
return nil, db.ErrNoEntries
}
emojis := make([]*gtsmodel.Emoji, 0, len(emojiIDs))
for _, id := range emojiIDs {
emoji, err := e.GetEmojiByID(ctx, id)
if err != nil {
log.Errorf("emojisFromIDs: error getting emoji %q: %v", id, err)
}
emojis = append(emojis, emoji)
}
return emojis, nil
}

111
internal/db/bundb/migrations/20220905150505_custom_emoji_updates.go Normal file
View File

@ -0,0 +1,111 @@
/*
GoToSocial
Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package migrations
import (
"context"
"database/sql"
"github.com/superseriousbusiness/gotosocial/internal/db/bundb/migrations/20220905150505_custom_emoji_updates"
"github.com/uptrace/bun"
)
func init() {
up := func(ctx context.Context, db *bun.DB) error {
return db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error {
// create the new emojis table
if _, err := tx.
NewCreateTable().
Model(&gtsmodel.Emoji{}).
ModelTableExpr("new_emojis").
Exec(ctx); err != nil {
return err
}
// move all old emojis to the new table
currentEmojis := []*gtsmodel.Emoji{}
if err := tx.
NewSelect().
Model(&currentEmojis).
Scan(ctx); err != nil && err != sql.ErrNoRows {
return err
}
for _, currentEmoji := range currentEmojis {
if _, err := tx.
NewInsert().
Model(currentEmoji).
ModelTableExpr("new_emojis").
Exec(ctx); err != nil {
return err
}
}
// we have all the data we need from the old table, so we can safely drop it now
if _, err := tx.NewDropTable().Model(&gtsmodel.Emoji{}).Exec(ctx); err != nil {
return err
}
// rename the new table to the same name as the old table was
if _, err := tx.ExecContext(ctx, "ALTER TABLE new_emojis RENAME TO emojis;"); err != nil {
return err
}
// add indexes to the new table
if _, err := tx.
NewCreateIndex().
Model(&gtsmodel.Emoji{}).
Index("emojis_id_idx").
Column("id").
Exec(ctx); err != nil {
return err
}
if _, err := tx.
NewCreateIndex().
Model(&gtsmodel.Emoji{}).
Index("emojis_uri_idx").
Column("uri").
Exec(ctx); err != nil {
return err
}
if _, err := tx.
NewCreateIndex().
Model(&gtsmodel.Emoji{}).
Index("emojis_available_custom_idx").
Column("visible_in_picker", "disabled", "shortcode").
Exec(ctx); err != nil {
return err
}
return nil
})
}
down := func(ctx context.Context, db *bun.DB) error {
return db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error {
return nil
})
}
if err := Migrations.Register(up, down); err != nil {
panic(err)
}
}

45
internal/db/bundb/migrations/20220905150505_custom_emoji_updates/emoji.go Normal file
View File

@ -0,0 +1,45 @@
/*
GoToSocial
Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gtsmodel
import "time"
// Emoji represents a custom emoji that's been uploaded through the admin UI, and is useable by instance denizens.
type Emoji struct {
ID string `validate:"required,ulid" bun:"type:CHAR(26),pk,nullzero,notnull,unique"` // id of this item in the database
CreatedAt time.Time `validate:"-" bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // when was item created
UpdatedAt time.Time `validate:"-" bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // when was item last updated
Shortcode string `validate:"required" bun:",nullzero,notnull,unique:shortcodedomain"` // String shortcode for this emoji -- the part that's between colons. This should be lowercase a-z_ eg., 'blob_hug' 'purple_heart' Must be unique with domain.
Domain string `validate:"omitempty,fqdn" bun:",nullzero,unique:shortcodedomain"` // Origin domain of this emoji, eg 'example.org', 'queer.party'. empty string for local emojis.
ImageRemoteURL string `validate:"required_without=ImageURL,omitempty,url" bun:",nullzero"` // Where can this emoji be retrieved remotely? Null for local emojis.
ImageStaticRemoteURL string `validate:"required_without=ImageStaticURL,omitempty,url" bun:",nullzero"` // Where can a static / non-animated version of this emoji be retrieved remotely? Null for local emojis.
ImageURL string `validate:"required_without=ImageRemoteURL,required_without=Domain,omitempty,url" bun:",nullzero"` // Where can this emoji be retrieved from the local server? Null for remote emojis.
ImageStaticURL string `validate:"required_without=ImageStaticRemoteURL,required_without=Domain,omitempty,url" bun:",nullzero"` // Where can a static version of this emoji be retrieved from the local server? Null for remote emojis.
ImagePath string `validate:"required,file" bun:",nullzero,notnull"` // Path of the emoji image in the server storage system.
ImageStaticPath string `validate:"required,file" bun:",nullzero,notnull"` // Path of a static version of the emoji image in the server storage system
ImageContentType string `validate:"required" bun:",nullzero,notnull"` // MIME content type of the emoji image
ImageStaticContentType string `validate:"required" bun:",nullzero,notnull"` // MIME content type of the static version of the emoji image.
ImageFileSize int `validate:"required,min=1" bun:",nullzero,notnull"` // Size of the emoji image file in bytes, for serving purposes.
ImageStaticFileSize int `validate:"required,min=1" bun:",nullzero,notnull"` // Size of the static version of the emoji image file in bytes, for serving purposes.
ImageUpdatedAt time.Time `validate:"-" bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // When was the emoji image last updated?
Disabled *bool `validate:"-" bun:",nullzero,notnull,default:false"` // Has a moderation action disabled this emoji from being shown?
URI string `validate:"url" bun:",nullzero,notnull,unique"` // ActivityPub uri of this emoji. Something like 'https://example.org/emojis/1234'
VisibleInPicker *bool `validate:"-" bun:",nullzero,notnull,default:true"` // Is this emoji visible in the admin emoji picker?
CategoryID string `validate:"omitempty,ulid" bun:"type:CHAR(26),nullzero"` // In which emoji category is this emoji visible?
}

8
internal/db/db.go
View File

@ -57,12 +57,4 @@ type DB interface {
// Note: this func doesn't/shouldn't do any manipulation of the tags in the DB, it's just for checking
// if they exist in the db already, and conveniently returning them, or creating new tag structs.
TagStringsToTags(ctx context.Context, tags []string, originAccountID string) ([]*gtsmodel.Tag, error)
// EmojiStringsToEmojis takes a slice of deduplicated, lowercase emojis in the form ":emojiname:", which have been
// used in a status. It takes the id of the account that wrote the status, and the id of the status itself, and then
// returns a slice of *model.Emoji corresponding to the given emojis.
//
// Note: this func doesn't/shouldn't do any manipulation of the emoji in the DB, it's just for checking
// if they exist in the db and conveniently returning them if they do.
EmojiStringsToEmojis(ctx context.Context, emojis []string) ([]*gtsmodel.Emoji, error)
}

7
internal/db/emoji.go
View File

@ -26,6 +26,13 @@ import (
// Emoji contains functions for getting emoji in the database.
type Emoji interface {
// PutEmoji puts one emoji in the database.
PutEmoji(ctx context.Context, emoji *gtsmodel.Emoji) Error
// GetCustomEmojis gets all custom emoji for the instance
GetCustomEmojis(ctx context.Context) ([]*gtsmodel.Emoji, Error)
// GetEmojiByID gets a specific emoji by its database ID.
GetEmojiByID(ctx context.Context, id string) (*gtsmodel.Emoji, Error)
// GetEmojiByShortcodeDomain gets an emoji based on its shortcode and domain.
// For local emoji, domain should be an empty string.
GetEmojiByShortcodeDomain(ctx context.Context, shortcode string, domain string) (*gtsmodel.Emoji, Error)
}

2
internal/gtsmodel/emoji.go
View File

@ -26,7 +26,7 @@ type Emoji struct {
CreatedAt time.Time `validate:"-" bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // when was item created
UpdatedAt time.Time `validate:"-" bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // when was item last updated
Shortcode string `validate:"required" bun:",nullzero,notnull,unique:shortcodedomain"` // String shortcode for this emoji -- the part that's between colons. This should be lowercase a-z_ eg., 'blob_hug' 'purple_heart' Must be unique with domain.
Domain string `validate:"omitempty,fqdn" bun:",notnull,default:'',unique:shortcodedomain"` // Origin domain of this emoji, eg 'example.org', 'queer.party'. empty string for local emojis.
Domain string `validate:"omitempty,fqdn" bun:",nullzero,unique:shortcodedomain"` // Origin domain of this emoji, eg 'example.org', 'queer.party'. empty string for local emojis.
ImageRemoteURL string `validate:"required_without=ImageURL,omitempty,url" bun:",nullzero"` // Where can this emoji be retrieved remotely? Null for local emojis.
ImageStaticRemoteURL string `validate:"required_without=ImageStaticURL,omitempty,url" bun:",nullzero"` // Where can a static / non-animated version of this emoji be retrieved remotely? Null for local emojis.
ImageURL string `validate:"required_without=ImageRemoteURL,required_without=Domain,omitempty,url" bun:",nullzero"` // Where can this emoji be retrieved from the local server? Null for remote emojis.

2
internal/media/processingemoji.go
View File

@ -93,7 +93,7 @@ func (p *ProcessingEmoji) LoadEmoji(ctx context.Context) (*gtsmodel.Emoji, error
// store the result in the database before returning it
if !p.insertedInDB {
if err := p.database.Put(ctx, p.emoji); err != nil {
if err := p.database.PutEmoji(ctx, p.emoji); err != nil {
return nil, err
}
p.insertedInDB = true

20
internal/processing/admin/emoji.go
View File

@ -20,7 +20,6 @@ package admin
import (
"context"
"errors"
"fmt"
"io"
@ -37,9 +36,13 @@ func (p *processor) EmojiCreate(ctx context.Context, account *gtsmodel.Account,
return nil, gtserror.NewErrorUnauthorized(fmt.Errorf("user %s not an admin", user.ID), "user is not an admin")
}
data := func(innerCtx context.Context) (io.Reader, int, error) {
f, err := form.Image.Open()
return f, int(form.Image.Size), err
maybeExisting, err := p.db.GetEmojiByShortcodeDomain(ctx, form.Shortcode, "")
if maybeExisting != nil {
return nil, gtserror.NewErrorConflict(fmt.Errorf("emoji with shortcode %s already exists", form.Shortcode), fmt.Sprintf("emoji with shortcode %s already exists", form.Shortcode))
}
if err != nil && err != db.ErrNoEntries {
return nil, gtserror.NewErrorInternalError(fmt.Errorf("error checking existence of emoji with shortcode %s: %s", form.Shortcode, err))
}
emojiID, err := id.NewRandomULID()
@ -49,6 +52,11 @@ func (p *processor) EmojiCreate(ctx context.Context, account *gtsmodel.Account,
emojiURI := uris.GenerateURIForEmoji(emojiID)
data := func(innerCtx context.Context) (io.Reader, int, error) {
f, err := form.Image.Open()
return f, int(form.Image.Size), err
}
processingEmoji, err := p.mediaManager.ProcessEmoji(ctx, data, nil, form.Shortcode, emojiID, emojiURI, nil)
if err != nil {
return nil, gtserror.NewErrorInternalError(fmt.Errorf("error processing emoji: %s", err), "error processing emoji")
@ -56,10 +64,6 @@ func (p *processor) EmojiCreate(ctx context.Context, account *gtsmodel.Account,
emoji, err := processingEmoji.LoadEmoji(ctx)
if err != nil {
var alreadyExistsError *db.ErrAlreadyExists
if errors.As(err, &alreadyExistsError) {
return nil, gtserror.NewErrorConflict(fmt.Errorf("emoji with shortcode %s already exists", form.Shortcode), fmt.Sprintf("emoji with shortcode %s already exists", form.Shortcode))
}
return nil, gtserror.NewErrorInternalError(fmt.Errorf("error loading emoji: %s", err), "error loading emoji")
}

4
internal/processing/federation.go
View File

@ -51,6 +51,10 @@ func (p *processor) GetFediOutbox(ctx context.Context, requestedUsername string,
return p.federationProcessor.GetOutbox(ctx, requestedUsername, page, maxID, minID, requestURL)
}
func (p *processor) GetFediEmoji(ctx context.Context, requestedEmojiID string, requestURL *url.URL) (interface{}, gtserror.WithCode) {
return p.federationProcessor.GetEmoji(ctx, requestedEmojiID, requestURL)
}
func (p *processor) GetWebfingerAccount(ctx context.Context, requestedUsername string) (*apimodel.WellKnownResponse, gtserror.WithCode) {
return p.federationProcessor.GetWebfingerAccount(ctx, requestedUsername)
}

3
internal/processing/federation/federation.go
View File

@ -56,6 +56,9 @@ type Processor interface {
// GetWebfingerAccount handles the GET for a webfinger resource. Most commonly, it will be used for returning account lookups.
GetWebfingerAccount(ctx context.Context, requestedUsername string) (*apimodel.WellKnownResponse, gtserror.WithCode)
// GetFediEmoji handles the GET for a federated emoji originating from this instance.
GetEmoji(ctx context.Context, requestedEmojiID string, requestURL *url.URL) (interface{}, gtserror.WithCode)
// GetNodeInfoRel returns a well known response giving the path to node info.
GetNodeInfoRel(ctx context.Context, request *http.Request) (*apimodel.WellKnownResponse, gtserror.WithCode)

59
internal/processing/federation/getemoji.go Normal file
View File

@ -0,0 +1,59 @@
/*
GoToSocial
Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package federation
import (
"context"
"fmt"
"net/url"
"github.com/superseriousbusiness/activity/streams"
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
)
func (p *processor) GetEmoji(ctx context.Context, requestedEmojiID string, requestURL *url.URL) (interface{}, gtserror.WithCode) {
if _, errWithCode := p.federator.AuthenticateFederatedRequest(ctx, ""); errWithCode != nil {
return nil, errWithCode
}
requestedEmoji, err := p.db.GetEmojiByID(ctx, requestedEmojiID)
if err != nil {
return nil, gtserror.NewErrorNotFound(fmt.Errorf("database error getting emoji with id %s: %s", requestedEmojiID, err))
}
if requestedEmoji.Domain != "" {
return nil, gtserror.NewErrorNotFound(fmt.Errorf("emoji with id %s doesn't belong to this instance (domain %s)", requestedEmojiID, requestedEmoji.Domain))
}
if *requestedEmoji.Disabled {
return nil, gtserror.NewErrorNotFound(fmt.Errorf("emoji with id %s has been disabled", requestedEmojiID))
}
apEmoji, err := p.tc.EmojiToAS(ctx, requestedEmoji)
if err != nil {
return nil, gtserror.NewErrorInternalError(fmt.Errorf("error converting gtsmodel emoji with id %s to ap emoji: %s", requestedEmojiID, err))
}
data, err := streams.Serialize(apEmoji)
if err != nil {
return nil, gtserror.NewErrorInternalError(err)
}
return data, nil
}

4
internal/processing/media/getfile.go
View File

@ -231,8 +231,8 @@ func (p *processor) getEmojiContent(ctx context.Context, wantedEmojiID string, e
emojiContent := &apimodel.Content{}
var storagePath string
e := &gtsmodel.Emoji{}
if err := p.db.GetByID(ctx, wantedEmojiID, e); err != nil {
e, err := p.db.GetEmojiByID(ctx, wantedEmojiID)
if err != nil {
return nil, gtserror.NewErrorNotFound(fmt.Errorf("emoji %s could not be taken from the db: %s", wantedEmojiID, err))
}

2
internal/processing/processor.go
View File

@ -228,6 +228,8 @@ type Processor interface {
GetFediStatusReplies(ctx context.Context, requestedUsername string, requestedStatusID string, page bool, onlyOtherAccounts bool, minID string, requestURL *url.URL) (interface{}, gtserror.WithCode)
// GetFediOutbox returns the public outbox of the requested user, with the given parameters.
GetFediOutbox(ctx context.Context, requestedUsername string, page bool, maxID string, minID string, requestURL *url.URL) (interface{}, gtserror.WithCode)
// GetFediEmoji returns the AP representation of an emoji on this instance.
GetFediEmoji(ctx context.Context, requestedEmojiID string, requestURL *url.URL) (interface{}, gtserror.WithCode)
// GetWebfingerAccount handles the GET for a webfinger resource. Most commonly, it will be used for returning account lookups.
GetWebfingerAccount(ctx context.Context, requestedUsername string) (*apimodel.WellKnownResponse, gtserror.WithCode)
// GetNodeInfoRel returns a well known response giving the path to node info.

27
internal/processing/status/util.go
View File

@ -249,18 +249,27 @@ func (p *processor) ProcessTags(ctx context.Context, form *apimodel.AdvancedStat
}
func (p *processor) ProcessEmojis(ctx context.Context, form *apimodel.AdvancedStatusCreateForm, accountID string, status *gtsmodel.Status) error {
gtsEmojis, err := p.db.EmojiStringsToEmojis(ctx, util.DeriveEmojisFromText(form.Status))
// for each emoji shortcode in the text, check if it's an enabled
// emoji on this instance, and if so, add it to the status
emojiShortcodes := util.DeriveEmojisFromText(form.Status)
status.Emojis = make([]*gtsmodel.Emoji, 0, len(emojiShortcodes))
status.EmojiIDs = make([]string, 0, len(emojiShortcodes))
for _, shortcode := range emojiShortcodes {
emoji, err := p.db.GetEmojiByShortcodeDomain(ctx, shortcode, "")
if err != nil {
return fmt.Errorf("error generating emojis from status: %s", err)
if err != db.ErrNoEntries {
log.Errorf("error getting local emoji with shortcode %s: %s", shortcode, err)
}
emojis := make([]string, 0, len(gtsEmojis))
for _, e := range gtsEmojis {
emojis = append(emojis, e.ID)
continue
}
// add full populated gts emojis to the status for passing them around conveniently
status.Emojis = gtsEmojis
// add just the ids of the used emojis to the status for putting in the db
status.EmojiIDs = emojis
if *emoji.VisibleInPicker && !*emoji.Disabled {
status.Emojis = append(status.Emojis, emoji)
status.EmojiIDs = append(status.EmojiIDs, emoji.ID)
}
}
return nil
}

10
testrig/testmodels.go
View File

@ -2056,6 +2056,7 @@ func NewTestDereferenceRequests(accounts map[string]*gtsmodel.Account) map[strin
var sig, digest, date string
var target *url.URL
statuses := NewTestStatuses()
emojis := NewTestEmojis()
target = URLMustParse(accounts["local_account_1"].URI)
sig, digest, date = GetSignatureForDereference(accounts["remote_account_1"].PublicKeyURI, accounts["remote_account_1"].PrivateKey, target)
@ -2137,6 +2138,14 @@ func NewTestDereferenceRequests(accounts map[string]*gtsmodel.Account) map[strin
DateHeader: date,
}
target = URLMustParse(emojis["rainbow"].URI)
sig, digest, date = GetSignatureForDereference(accounts["remote_account_1"].PublicKeyURI, accounts["remote_account_1"].PrivateKey, target)
fossSatanDereferenceEmoji := ActivityWithSignature{
SignatureHeader: sig,
DigestHeader: digest,
DateHeader: date,
}
return map[string]ActivityWithSignature{
"foss_satan_dereference_zork": fossSatanDereferenceZork,
"foss_satan_dereference_zork_public_key": fossSatanDereferenceZorkPublicKey,
@ -2148,6 +2157,7 @@ func NewTestDereferenceRequests(accounts map[string]*gtsmodel.Account) map[strin
"foss_satan_dereference_zork_outbox": fossSatanDereferenceZorkOutbox,
"foss_satan_dereference_zork_outbox_first": fossSatanDereferenceZorkOutboxFirst,
"foss_satan_dereference_zork_outbox_next": fossSatanDereferenceZorkOutboxNext,
"foss_satan_dereference_emoji": fossSatanDereferenceEmoji,
}
}