[performance] cache media attachments (#1525)

* replace concurrency worker pools with base models in State.Workers, update code and tests accordingly

* add media attachment caching, slightly tweak default cache config

* further tweak default cache config values

* replace other media attachment db calls to go through cache

* update envparsing test

* fix delete media attachment sql

* fix media sql query

* invalidate cached media entries during status create / update

* fix envparsing test

* fix typo in panic log message...

* add 'updated_at' column during UpdateAttachment

* remove unused func

---------

Signed-off-by: kim <grufwub@gmail.com>

example/config.yaml

@@ -239,13 +239,13 @@ cache:
     # ttl        = cached object lifetime
     # sweep-freq = frequency to look for stale cache objects
 
-    account-max-size: 100
+    account-max-size: 500
     account-ttl: "5m"
-    account-sweep-freq: "10s"
+    account-sweep-freq: "30s"
 
     block-max-size: 100
     block-ttl: "5m"
-    block-sweep-freq: "10s"
+    block-sweep-freq: "30s"
 
     domain-block-max-size: 1000
     domain-block-ttl: "24h"
@@ -253,35 +253,39 @@ cache:
 
     emoji-max-size: 500
     emoji-ttl: "5m"
-    emoji-sweep-freq: "10s"
+    emoji-sweep-freq: "30s"
 
     emoji-category-max-size: 100
     emoji-category-ttl: "5m"
-    emoji-category-sweep-freq: "10s"
+    emoji-category-sweep-freq: "30s"
+
+    media-max-size: 500
+    media-ttl: "5m"
+    media-sweep-freq: "30s"
 
     mention-max-size: 500
     mention-ttl: "5m"
-    mention-sweep-freq: "10s"
+    mention-sweep-freq: "30s"
 
     notification-max-size: 500
     notification-ttl: "5m"
-    notification-sweep-freq: "10s"
+    notification-sweep-freq: "30s"
 
     report-max-size: 100
     report-ttl: "5m"
-    report-sweep-freq: "10s"
+    report-sweep-freq: "30s"
 
     status-max-size: 500
     status-ttl: "5m"
-    status-sweep-freq: "10s"
+    status-sweep-freq: "30s"
 
     tombstone-max-size: 100
     tombstone-ttl: "5m"
-    tombstone-sweep-freq: "10s"
+    tombstone-sweep-freq: "30s"
 
     user-max-size: 100
     user-ttl: "5m"
-    user-sweep-freq: "10s"
+    user-sweep-freq: "30s"
 
 ######################
 ##### WEB CONFIG #####

internal/cache/gts.go

@@ -54,6 +54,9 @@ type GTSCaches interface {
 	// Mention provides access to the gtsmodel Mention database cache.
 	Mention() *result.Cache[*gtsmodel.Mention]
 
+	// Media provides access to the gtsmodel Media database cache.
+	Media() *result.Cache[*gtsmodel.MediaAttachment]
+
 	// Notification provides access to the gtsmodel Notification database cache.
 	Notification() *result.Cache[*gtsmodel.Notification]
 
@@ -81,6 +84,7 @@ type gtsCaches struct {
 	domainBlock   *domain.BlockCache
 	emoji         *result.Cache[*gtsmodel.Emoji]
 	emojiCategory *result.Cache[*gtsmodel.EmojiCategory]
+	media         *result.Cache[*gtsmodel.MediaAttachment]
 	mention       *result.Cache[*gtsmodel.Mention]
 	notification  *result.Cache[*gtsmodel.Notification]
 	report        *result.Cache[*gtsmodel.Report]
@@ -95,6 +99,7 @@ func (c *gtsCaches) Init() {
 	c.initDomainBlock()
 	c.initEmoji()
 	c.initEmojiCategory()
+	c.initMedia()
 	c.initMention()
 	c.initNotification()
 	c.initReport()
@@ -119,6 +124,9 @@ func (c *gtsCaches) Start() {
 	tryUntil("starting gtsmodel.EmojiCategory cache", 5, func() bool {
 		return c.emojiCategory.Start(config.GetCacheGTSEmojiCategorySweepFreq())
 	})
+	tryUntil("starting gtsmodel.MediaAttachment cache", 5, func() bool {
+		return c.media.Start(config.GetCacheGTSMediaSweepFreq())
+	})
 	tryUntil("starting gtsmodel.Mention cache", 5, func() bool {
 		return c.mention.Start(config.GetCacheGTSMentionSweepFreq())
 	})
@@ -145,6 +153,7 @@ func (c *gtsCaches) Stop() {
 	tryUntil("stopping gtsmodel.DomainBlock cache", 5, c.domainBlock.Stop)
 	tryUntil("stopping gtsmodel.Emoji cache", 5, c.emoji.Stop)
 	tryUntil("stopping gtsmodel.EmojiCategory cache", 5, c.emojiCategory.Stop)
+	tryUntil("stopping gtsmodel.MediaAttachment cache", 5, c.media.Stop)
 	tryUntil("stopping gtsmodel.Mention cache", 5, c.mention.Stop)
 	tryUntil("stopping gtsmodel.Notification cache", 5, c.notification.Stop)
 	tryUntil("stopping gtsmodel.Report cache", 5, c.report.Stop)
@@ -173,6 +182,10 @@ func (c *gtsCaches) EmojiCategory() *result.Cache[*gtsmodel.EmojiCategory] {
 	return c.emojiCategory
 }
 
+func (c *gtsCaches) Media() *result.Cache[*gtsmodel.MediaAttachment] {
+	return c.media
+}
+
 func (c *gtsCaches) Mention() *result.Cache[*gtsmodel.Mention] {
 	return c.mention
 }
@@ -258,6 +271,17 @@ func (c *gtsCaches) initEmojiCategory() {
 	c.emojiCategory.SetTTL(config.GetCacheGTSEmojiCategoryTTL(), true)
 }
 
+func (c *gtsCaches) initMedia() {
+	c.media = result.New([]result.Lookup{
+		{Name: "ID"},
+	}, func(m1 *gtsmodel.MediaAttachment) *gtsmodel.MediaAttachment {
+		m2 := new(gtsmodel.MediaAttachment)
+		*m2 = *m1
+		return m2
+	}, config.GetCacheGTSMediaMaxSize())
+	c.media.SetTTL(config.GetCacheGTSMediaTTL(), true)
+}
+
 func (c *gtsCaches) initMention() {
 	c.mention = result.New([]result.Lookup{
 		{Name: "ID"},

internal/config/config.go

@@ -177,6 +177,10 @@ type GTSCacheConfiguration struct {
 	EmojiCategoryTTL       time.Duration `name:"emoji-category-ttl"`
 	EmojiCategorySweepFreq time.Duration `name:"emoji-category-sweep-freq"`
 
+	MediaMaxSize   int           `name:"media-max-size"`
+	MediaTTL       time.Duration `name:"media-ttl"`
+	MediaSweepFreq time.Duration `name:"media-sweep-freq"`
+
 	MentionMaxSize   int           `name:"mention-max-size"`
 	MentionTTL       time.Duration `name:"mention-ttl"`
 	MentionSweepFreq time.Duration `name:"mention-sweep-freq"`

internal/config/defaults.go

@@ -116,13 +116,13 @@ var Defaults = Configuration{
 
 	Cache: CacheConfiguration{
 		GTS: GTSCacheConfiguration{
-			AccountMaxSize:   100,
+			AccountMaxSize:   500,
 			AccountTTL:       time.Minute * 5,
-			AccountSweepFreq: time.Second * 10,
+			AccountSweepFreq: time.Second * 30,
 
 			BlockMaxSize:   100,
 			BlockTTL:       time.Minute * 5,
-			BlockSweepFreq: time.Second * 10,
+			BlockSweepFreq: time.Second * 30,
 
 			DomainBlockMaxSize:   1000,
 			DomainBlockTTL:       time.Hour * 24,
@@ -130,35 +130,39 @@ var Defaults = Configuration{
 
 			EmojiMaxSize:   500,
 			EmojiTTL:       time.Minute * 5,
-			EmojiSweepFreq: time.Second * 10,
+			EmojiSweepFreq: time.Second * 30,
 
 			EmojiCategoryMaxSize:   100,
 			EmojiCategoryTTL:       time.Minute * 5,
-			EmojiCategorySweepFreq: time.Second * 10,
+			EmojiCategorySweepFreq: time.Second * 30,
+
+			MediaMaxSize:   500,
+			MediaTTL:       time.Minute * 5,
+			MediaSweepFreq: time.Second * 30,
 
 			MentionMaxSize:   500,
 			MentionTTL:       time.Minute * 5,
-			MentionSweepFreq: time.Second * 10,
+			MentionSweepFreq: time.Second * 30,
 
 			NotificationMaxSize:   500,
 			NotificationTTL:       time.Minute * 5,
-			NotificationSweepFreq: time.Second * 10,
+			NotificationSweepFreq: time.Second * 30,
 
 			ReportMaxSize:   100,
 			ReportTTL:       time.Minute * 5,
-			ReportSweepFreq: time.Second * 10,
+			ReportSweepFreq: time.Second * 30,
 
 			StatusMaxSize:   500,
 			StatusTTL:       time.Minute * 5,
-			StatusSweepFreq: time.Second * 10,
+			StatusSweepFreq: time.Second * 30,
 
 			TombstoneMaxSize:   100,
 			TombstoneTTL:       time.Minute * 5,
-			TombstoneSweepFreq: time.Second * 10,
+			TombstoneSweepFreq: time.Second * 30,
 
 			UserMaxSize:   100,
 			UserTTL:       time.Minute * 5,
-			UserSweepFreq: time.Second * 10,
+			UserSweepFreq: time.Second * 30,
 		},
 	},
 

internal/config/helpers.gen.go

@@ -2426,6 +2426,81 @@ func GetCacheGTSEmojiCategorySweepFreq() time.Duration {
 // SetCacheGTSEmojiCategorySweepFreq safely sets the value for global configuration 'Cache.GTS.EmojiCategorySweepFreq' field
 func SetCacheGTSEmojiCategorySweepFreq(v time.Duration) { global.SetCacheGTSEmojiCategorySweepFreq(v) }
 
+// GetCacheGTSMediaMaxSize safely fetches the Configuration value for state's 'Cache.GTS.MediaMaxSize' field
+func (st *ConfigState) GetCacheGTSMediaMaxSize() (v int) {
+	st.mutex.Lock()
+	v = st.config.Cache.GTS.MediaMaxSize
+	st.mutex.Unlock()
+	return
+}
+
+// SetCacheGTSMediaMaxSize safely sets the Configuration value for state's 'Cache.GTS.MediaMaxSize' field
+func (st *ConfigState) SetCacheGTSMediaMaxSize(v int) {
+	st.mutex.Lock()
+	defer st.mutex.Unlock()
+	st.config.Cache.GTS.MediaMaxSize = v
+	st.reloadToViper()
+}
+
+// CacheGTSMediaMaxSizeFlag returns the flag name for the 'Cache.GTS.MediaMaxSize' field
+func CacheGTSMediaMaxSizeFlag() string { return "cache-gts-media-max-size" }
+
+// GetCacheGTSMediaMaxSize safely fetches the value for global configuration 'Cache.GTS.MediaMaxSize' field
+func GetCacheGTSMediaMaxSize() int { return global.GetCacheGTSMediaMaxSize() }
+
+// SetCacheGTSMediaMaxSize safely sets the value for global configuration 'Cache.GTS.MediaMaxSize' field
+func SetCacheGTSMediaMaxSize(v int) { global.SetCacheGTSMediaMaxSize(v) }
+
+// GetCacheGTSMediaTTL safely fetches the Configuration value for state's 'Cache.GTS.MediaTTL' field
+func (st *ConfigState) GetCacheGTSMediaTTL() (v time.Duration) {
+	st.mutex.Lock()
+	v = st.config.Cache.GTS.MediaTTL
+	st.mutex.Unlock()
+	return
+}
+
+// SetCacheGTSMediaTTL safely sets the Configuration value for state's 'Cache.GTS.MediaTTL' field
+func (st *ConfigState) SetCacheGTSMediaTTL(v time.Duration) {
+	st.mutex.Lock()
+	defer st.mutex.Unlock()
+	st.config.Cache.GTS.MediaTTL = v
+	st.reloadToViper()
+}
+
+// CacheGTSMediaTTLFlag returns the flag name for the 'Cache.GTS.MediaTTL' field
+func CacheGTSMediaTTLFlag() string { return "cache-gts-media-ttl" }
+
+// GetCacheGTSMediaTTL safely fetches the value for global configuration 'Cache.GTS.MediaTTL' field
+func GetCacheGTSMediaTTL() time.Duration { return global.GetCacheGTSMediaTTL() }
+
+// SetCacheGTSMediaTTL safely sets the value for global configuration 'Cache.GTS.MediaTTL' field
+func SetCacheGTSMediaTTL(v time.Duration) { global.SetCacheGTSMediaTTL(v) }
+
+// GetCacheGTSMediaSweepFreq safely fetches the Configuration value for state's 'Cache.GTS.MediaSweepFreq' field
+func (st *ConfigState) GetCacheGTSMediaSweepFreq() (v time.Duration) {
+	st.mutex.Lock()
+	v = st.config.Cache.GTS.MediaSweepFreq
+	st.mutex.Unlock()
+	return
+}
+
+// SetCacheGTSMediaSweepFreq safely sets the Configuration value for state's 'Cache.GTS.MediaSweepFreq' field
+func (st *ConfigState) SetCacheGTSMediaSweepFreq(v time.Duration) {
+	st.mutex.Lock()
+	defer st.mutex.Unlock()
+	st.config.Cache.GTS.MediaSweepFreq = v
+	st.reloadToViper()
+}
+
+// CacheGTSMediaSweepFreqFlag returns the flag name for the 'Cache.GTS.MediaSweepFreq' field
+func CacheGTSMediaSweepFreqFlag() string { return "cache-gts-media-sweep-freq" }
+
+// GetCacheGTSMediaSweepFreq safely fetches the value for global configuration 'Cache.GTS.MediaSweepFreq' field
+func GetCacheGTSMediaSweepFreq() time.Duration { return global.GetCacheGTSMediaSweepFreq() }
+
+// SetCacheGTSMediaSweepFreq safely sets the value for global configuration 'Cache.GTS.MediaSweepFreq' field
+func SetCacheGTSMediaSweepFreq(v time.Duration) { global.SetCacheGTSMediaSweepFreq(v) }
+
 // GetCacheGTSMentionMaxSize safely fetches the Configuration value for state's 'Cache.GTS.MentionMaxSize' field
 func (st *ConfigState) GetCacheGTSMentionMaxSize() (v int) {
 	st.mutex.Lock()

internal/db/bundb/media.go

@@ -34,39 +34,69 @@ type mediaDB struct {
 	state *state.State
 }
 
-func (m *mediaDB) newMediaQ(i *gtsmodel.MediaAttachment) *bun.SelectQuery {
-	return m.conn.
-		NewSelect().
-		Model(i)
-}
-
 func (m *mediaDB) GetAttachmentByID(ctx context.Context, id string) (*gtsmodel.MediaAttachment, db.Error) {
 	return m.getAttachment(
 		ctx,
 		"ID",
 		func(attachment *gtsmodel.MediaAttachment) error {
-			return m.newMediaQ(attachment).Where("? = ?", bun.Ident("media_attachment.id"), id).Scan(ctx)
+			return m.conn.NewSelect().
+				Model(attachment).
+				Where("? = ?", bun.Ident("media_attachment.id"), id).
+				Scan(ctx)
 		},
 		id,
 	)
 }
 
-func (m *mediaDB) getAttachments(ctx context.Context, ids []string) ([]*gtsmodel.MediaAttachment, db.Error) {
+func (m *mediaDB) getAttachment(ctx context.Context, lookup string, dbQuery func(*gtsmodel.MediaAttachment) error, keyParts ...any) (*gtsmodel.MediaAttachment, db.Error) {
-	attachments := make([]*gtsmodel.MediaAttachment, 0, len(ids))
+	return m.state.Caches.GTS.Media().Load(lookup, func() (*gtsmodel.MediaAttachment, error) {
+		var attachment gtsmodel.MediaAttachment
 
-	for _, id := range ids {
+		// Not cached! Perform database query
-		// Attempt fetch from DB
+		if err := dbQuery(&attachment); err != nil {
-		attachment, err := m.GetAttachmentByID(ctx, id)
+			return nil, m.conn.ProcessError(err)
-		if err != nil {
-			log.Errorf(ctx, "error getting attachment %q: %v", id, err)
-			continue
 		}
 
-		// Append attachment
+		return &attachment, nil
-		attachments = append(attachments, attachment)
+	}, keyParts...)
 }
 
-	return attachments, nil
+func (m *mediaDB) PutAttachment(ctx context.Context, media *gtsmodel.MediaAttachment) error {
+	return m.state.Caches.GTS.Media().Store(media, func() error {
+		_, err := m.conn.NewInsert().Model(media).Exec(ctx)
+		return m.conn.ProcessError(err)
+	})
+}
+
+func (m *mediaDB) UpdateAttachment(ctx context.Context, media *gtsmodel.MediaAttachment, columns ...string) error {
+	media.UpdatedAt = time.Now()
+	if len(columns) > 0 {
+		// If we're updating by column, ensure "updated_at" is included.
+		columns = append(columns, "updated_at")
+	}
+
+	return m.state.Caches.GTS.Media().Store(media, func() error {
+		_, err := m.conn.NewUpdate().
+			Model(media).
+			Where("? = ?", bun.Ident("media_attachment.id"), media.ID).
+			Column(columns...).
+			Exec(ctx)
+		return m.conn.ProcessError(err)
+	})
+}
+
+func (m *mediaDB) DeleteAttachment(ctx context.Context, id string) error {
+	// Attempt to delete from database.
+	if _, err := m.conn.NewDelete().
+		TableExpr("? AS ?", bun.Ident("media_attachments"), bun.Ident("media_attachment")).
+		Where("? = ?", bun.Ident("media_attachment.id"), id).
+		Exec(ctx); err != nil {
+		return m.conn.ProcessError(err)
+	}
+
+	// Invalidate this media item from the cache.
+	m.state.Caches.GTS.Media().Invalidate("ID", id)
+	return nil
 }
 
 func (m *mediaDB) GetRemoteOlderThan(ctx context.Context, olderThan time.Time, limit int) ([]*gtsmodel.MediaAttachment, db.Error) {
@@ -183,14 +213,20 @@ func (m *mediaDB) CountLocalUnattachedOlderThan(ctx context.Context, olderThan t
 	return count, nil
 }
 
-func (m *mediaDB) getAttachment(ctx context.Context, lookup string, dbQuery func(*gtsmodel.MediaAttachment) error, keyParts ...any) (*gtsmodel.MediaAttachment, db.Error) {
+func (m *mediaDB) getAttachments(ctx context.Context, ids []string) ([]*gtsmodel.MediaAttachment, db.Error) {
-	// Fetch attachment from database
+	attachments := make([]*gtsmodel.MediaAttachment, 0, len(ids))
-	// todo: cache this lookup
-	attachment := new(gtsmodel.MediaAttachment)
 
-	if err := dbQuery(attachment); err != nil {
+	for _, id := range ids {
-		return nil, m.conn.ProcessError(err)
+		// Attempt fetch from DB
+		attachment, err := m.GetAttachmentByID(ctx, id)
+		if err != nil {
+			log.Errorf(ctx, "error getting attachment %q: %v", id, err)
+			continue
 		}
 
-	return attachment, nil
+		// Append attachment
+		attachments = append(attachments, attachment)
+	}
+
+	return attachments, nil
 }

internal/db/bundb/status.go

@@ -188,7 +188,7 @@ func (s *statusDB) getStatus(ctx context.Context, lookup string, dbQuery func(*g
 }
 
 func (s *statusDB) PutStatus(ctx context.Context, status *gtsmodel.Status) db.Error {
-	return s.state.Caches.GTS.Status().Store(status, func() error {
+	err := s.state.Caches.GTS.Status().Store(status, func() error {
 		// It is safe to run this database transaction within cache.Store
 		// as the cache does not attempt a mutex lock until AFTER hook.
 		//
@@ -248,6 +248,17 @@ func (s *statusDB) PutStatus(ctx context.Context, status *gtsmodel.Status) db.Er
 			return err
 		})
 	})
+	if err != nil {
+		// already processed
+		return err
+	}
+
+	for _, id := range status.AttachmentIDs {
+		// Clear updated media attachment IDs from cache
+		s.state.Caches.GTS.Media().Invalidate("ID", id)
+	}
+
+	return nil
 }
 
 func (s *statusDB) UpdateStatus(ctx context.Context, status *gtsmodel.Status, columns ...string) db.Error {
@@ -317,11 +328,18 @@ func (s *statusDB) UpdateStatus(ctx context.Context, status *gtsmodel.Status, co
 			Exec(ctx)
 		return err
 	}); err != nil {
+		// already processed
 		return err
 	}
 
-	// Drop any old value from cache by this ID
+	for _, id := range status.AttachmentIDs {
+		// Clear updated media attachment IDs from cache
+		s.state.Caches.GTS.Media().Invalidate("ID", id)
+	}
+
+	// Drop any old status value from cache by this ID
 	s.state.Caches.GTS.Status().Invalidate("ID", status.ID)
+
 	return nil
 }
 

internal/db/media.go

@@ -27,9 +27,18 @@ import (
 
 // Media contains functions related to creating/getting/removing media attachments.
 type Media interface {
-	// GetAttachmentByID gets a single attachment by its ID
+	// GetAttachmentByID gets a single attachment by its ID.
 	GetAttachmentByID(ctx context.Context, id string) (*gtsmodel.MediaAttachment, Error)
 
+	// PutAttachment inserts the given attachment into the database.
+	PutAttachment(ctx context.Context, media *gtsmodel.MediaAttachment) error
+
+	// UpdateAttachment will update the given attachment in the database.
+	UpdateAttachment(ctx context.Context, media *gtsmodel.MediaAttachment, columns ...string) error
+
+	// DeleteAttachment deletes the attachment with given ID from the database.
+	DeleteAttachment(ctx context.Context, id string) error
+
 	// GetRemoteOlderThan gets limit n remote media attachments (including avatars and headers) older than the given
 	// olderThan time. These will be returned in order of attachment.created_at descending (newest to oldest in other words).
 	//

internal/media/processingmedia.go

@@ -123,13 +123,13 @@ func (p *ProcessingMedia) load(ctx context.Context) (*gtsmodel.MediaAttachment,
 		}
 
 		if p.recache {
-			// Existing attachment we're recaching, so only need to update.
+			// Existing attachment we're recaching, so only update.
-			err = p.mgr.state.DB.UpdateByID(ctx, p.media, p.media.ID)
+			err = p.mgr.state.DB.UpdateAttachment(ctx, p.media)
 			return err
 		}
 
-		// New attachment, first time caching.
+		// First time caching this attachment, insert it.
-		err = p.mgr.state.DB.Put(ctx, p.media)
+		err = p.mgr.state.DB.PutAttachment(ctx, p.media)
 		return err
 	})
 

internal/media/prune.go

@@ -320,7 +320,7 @@ func (m *manager) deleteAttachment(ctx context.Context, attachment *gtsmodel.Med
 	}
 
 	// Delete attachment completely.
-	return m.state.DB.DeleteByID(ctx, attachment.ID, attachment)
+	return m.state.DB.DeleteAttachment(ctx, attachment.ID)
 }
 
 func (m *manager) uncacheAttachment(ctx context.Context, attachment *gtsmodel.MediaAttachment) error {
@@ -332,7 +332,7 @@ func (m *manager) uncacheAttachment(ctx context.Context, attachment *gtsmodel.Me
 	attachment.UpdatedAt = time.Now()
 	cached := false
 	attachment.Cached = &cached
-	return m.state.DB.UpdateByID(ctx, attachment, attachment.ID, "updated_at", "cached")
+	return m.state.DB.UpdateAttachment(ctx, attachment, "updated_at", "cached")
 }
 
 func (m *manager) removeFiles(ctx context.Context, keys ...string) (int, error) {

internal/processing/instance.go

@@ -233,7 +233,7 @@ func (p *Processor) InstancePatch(ctx context.Context, form *apimodel.InstanceSe
 	} else if form.AvatarDescription != nil && ia.AvatarMediaAttachment != nil {
 		// process just the description for the existing avatar
 		ia.AvatarMediaAttachment.Description = *form.AvatarDescription
-		if err := p.state.DB.UpdateByID(ctx, ia.AvatarMediaAttachment, ia.AvatarMediaAttachmentID, "description"); err != nil {
+		if err := p.state.DB.UpdateAttachment(ctx, ia.AvatarMediaAttachment, "description"); err != nil {
 			return nil, gtserror.NewErrorInternalError(fmt.Errorf("db error updating instance avatar description: %s", err))
 		}
 	}

internal/processing/media/delete.go

@@ -40,7 +40,7 @@ func (p *Processor) Delete(ctx context.Context, mediaAttachmentID string) gtserr
 	}
 
 	// delete the attachment
-	if err := p.state.DB.DeleteByID(ctx, mediaAttachmentID, attachment); err != nil && !errors.Is(err, db.ErrNoEntries) {
+	if err := p.state.DB.DeleteAttachment(ctx, mediaAttachmentID); err != nil && !errors.Is(err, db.ErrNoEntries) {
 		errs = append(errs, fmt.Sprintf("remove attachment: %s", err))
 	}
 

internal/processing/media/unattach.go

@@ -49,7 +49,7 @@ func (p *Processor) Unattach(ctx context.Context, account *gtsmodel.Account, med
 	attachment.UpdatedAt = time.Now()
 	attachment.StatusID = ""
 
-	if err := p.state.DB.UpdateByID(ctx, attachment, attachment.ID, updatingColumns...); err != nil {
+	if err := p.state.DB.UpdateAttachment(ctx, attachment, updatingColumns...); err != nil {
 		return nil, gtserror.NewErrorNotFound(fmt.Errorf("db error updating attachment: %s", err))
 	}
 

test/envparsing.sh

@@ -2,7 +2,7 @@
 
 set -eu
 
-EXPECT='{"account-domain":"peepee","accounts-allow-custom-css":true,"accounts-approval-required":false,"accounts-reason-required":false,"accounts-registration-open":true,"advanced-cookies-samesite":"strict","advanced-rate-limit-requests":6969,"advanced-throttling-multiplier":-1,"advanced-throttling-retry-after":10000000000,"application-name":"gts","bind-address":"127.0.0.1","cache":{"gts":{"account-max-size":99,"account-sweep-freq":1000000000,"account-ttl":10800000000000,"block-max-size":100,"block-sweep-freq":10000000000,"block-ttl":300000000000,"domain-block-max-size":1000,"domain-block-sweep-freq":60000000000,"domain-block-ttl":86400000000000,"emoji-category-max-size":100,"emoji-category-sweep-freq":10000000000,"emoji-category-ttl":300000000000,"emoji-max-size":500,"emoji-sweep-freq":10000000000,"emoji-ttl":300000000000,"mention-max-size":500,"mention-sweep-freq":10000000000,"mention-ttl":300000000000,"notification-max-size":500,"notification-sweep-freq":10000000000,"notification-ttl":300000000000,"report-max-size":100,"report-sweep-freq":10000000000,"report-ttl":300000000000,"status-max-size":500,"status-sweep-freq":10000000000,"status-ttl":300000000000,"tombstone-max-size":100,"tombstone-sweep-freq":10000000000,"tombstone-ttl":300000000000,"user-max-size":100,"user-sweep-freq":10000000000,"user-ttl":300000000000}},"config-path":"internal/config/testdata/test.yaml","db-address":":memory:","db-database":"gotosocial_prod","db-max-open-conns-multiplier":3,"db-password":"hunter2","db-port":6969,"db-sqlite-busy-timeout":1000000000,"db-sqlite-cache-size":0,"db-sqlite-journal-mode":"DELETE","db-sqlite-synchronous":"FULL","db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"sqlite","db-user":"sex-haver","dry-run":true,"email":"","host":"example.com","instance-deliver-to-shared-inboxes":false,"instance-expose-peers":true,"instance-expose-public-timeline":true,"instance-expose-suspended":true,"instance-expose-suspended-web":true,"landing-page-user":"admin","letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":true,"letsencrypt-port":80,"log-db-queries":true,"log-level":"info","media-description-max-chars":5000,"media-description-min-chars":69,"media-emoji-local-max-size":420,"media-emoji-remote-max-size":420,"media-image-max-size":420,"media-remote-cache-days":30,"media-video-max-size":420,"oidc-admin-groups":["steamy"],"oidc-client-id":"1234","oidc-client-secret":"shhhh its a secret","oidc-enabled":true,"oidc-idp-name":"sex-haver","oidc-issuer":"whoknows","oidc-link-existing":true,"oidc-scopes":["read","write"],"oidc-skip-verification":true,"password":"","path":"","port":6969,"protocol":"http","request-id-header":"X-Trace-Id","smtp-from":"queen.rip.in.piss@terfisland.org","smtp-host":"example.com","smtp-password":"hunter2","smtp-port":4269,"smtp-username":"sex-haver","software-version":"","statuses-cw-max-chars":420,"statuses-max-chars":69,"statuses-media-max-files":1,"statuses-poll-max-options":1,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/root/store","storage-s3-access-key":"minio","storage-s3-bucket":"gts","storage-s3-endpoint":"localhost:9000","storage-s3-proxy":true,"storage-s3-secret-key":"miniostorage","storage-s3-use-ssl":false,"syslog-address":"127.0.0.1:6969","syslog-enabled":true,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32","docker.host.local"],"username":"","web-asset-base-dir":"/root","web-template-base-dir":"/root"}'
+EXPECT='{"account-domain":"peepee","accounts-allow-custom-css":true,"accounts-approval-required":false,"accounts-reason-required":false,"accounts-registration-open":true,"advanced-cookies-samesite":"strict","advanced-rate-limit-requests":6969,"advanced-throttling-multiplier":-1,"advanced-throttling-retry-after":10000000000,"application-name":"gts","bind-address":"127.0.0.1","cache":{"gts":{"account-max-size":99,"account-sweep-freq":1000000000,"account-ttl":10800000000000,"block-max-size":100,"block-sweep-freq":30000000000,"block-ttl":300000000000,"domain-block-max-size":1000,"domain-block-sweep-freq":60000000000,"domain-block-ttl":86400000000000,"emoji-category-max-size":100,"emoji-category-sweep-freq":30000000000,"emoji-category-ttl":300000000000,"emoji-max-size":500,"emoji-sweep-freq":30000000000,"emoji-ttl":300000000000,"media-max-size":500,"media-sweep-freq":30000000000,"media-ttl":300000000000,"mention-max-size":500,"mention-sweep-freq":30000000000,"mention-ttl":300000000000,"notification-max-size":500,"notification-sweep-freq":30000000000,"notification-ttl":300000000000,"report-max-size":100,"report-sweep-freq":30000000000,"report-ttl":300000000000,"status-max-size":500,"status-sweep-freq":30000000000,"status-ttl":300000000000,"tombstone-max-size":100,"tombstone-sweep-freq":30000000000,"tombstone-ttl":300000000000,"user-max-size":100,"user-sweep-freq":30000000000,"user-ttl":300000000000}},"config-path":"internal/config/testdata/test.yaml","db-address":":memory:","db-database":"gotosocial_prod","db-max-open-conns-multiplier":3,"db-password":"hunter2","db-port":6969,"db-sqlite-busy-timeout":1000000000,"db-sqlite-cache-size":0,"db-sqlite-journal-mode":"DELETE","db-sqlite-synchronous":"FULL","db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"sqlite","db-user":"sex-haver","dry-run":true,"email":"","host":"example.com","instance-deliver-to-shared-inboxes":false,"instance-expose-peers":true,"instance-expose-public-timeline":true,"instance-expose-suspended":true,"instance-expose-suspended-web":true,"landing-page-user":"admin","letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":true,"letsencrypt-port":80,"log-db-queries":true,"log-level":"info","media-description-max-chars":5000,"media-description-min-chars":69,"media-emoji-local-max-size":420,"media-emoji-remote-max-size":420,"media-image-max-size":420,"media-remote-cache-days":30,"media-video-max-size":420,"oidc-admin-groups":["steamy"],"oidc-client-id":"1234","oidc-client-secret":"shhhh its a secret","oidc-enabled":true,"oidc-idp-name":"sex-haver","oidc-issuer":"whoknows","oidc-link-existing":true,"oidc-scopes":["read","write"],"oidc-skip-verification":true,"password":"","path":"","port":6969,"protocol":"http","request-id-header":"X-Trace-Id","smtp-from":"queen.rip.in.piss@terfisland.org","smtp-host":"example.com","smtp-password":"hunter2","smtp-port":4269,"smtp-username":"sex-haver","software-version":"","statuses-cw-max-chars":420,"statuses-max-chars":69,"statuses-media-max-files":1,"statuses-poll-max-options":1,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/root/store","storage-s3-access-key":"minio","storage-s3-bucket":"gts","storage-s3-endpoint":"localhost:9000","storage-s3-proxy":true,"storage-s3-secret-key":"miniostorage","storage-s3-use-ssl":false,"syslog-address":"127.0.0.1:6969","syslog-enabled":true,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32","docker.host.local"],"username":"","web-asset-base-dir":"/root","web-template-base-dir":"/root"}'
 
 # Set all the environment variables to 
 # ensure that these are parsed without panic

testrig/config.go

@@ -34,7 +34,7 @@ func InitTestConfig() {
 }
 
 var testDefaults = config.Configuration{
-	LogLevel:        "trace",
+	LogLevel:        "info",
 	LogDbQueries:    true,
 	ApplicationName: "gotosocial",
 	LandingPageUser: "",