From 7b48437f172e11daa77c842b18813d5654143cb0 Mon Sep 17 00:00:00 2001 From: rdelaage Date: Wed, 23 Aug 2023 18:01:16 +0200 Subject: [PATCH] [feature] list commands for both attachment and emojis (#2121) * [feature] list commands for both attachment and emojis * use fewer commands, provide `local-only` and `remote-only` as filters * envparsing --------- Co-authored-by: Romain de Laage Co-authored-by: tsmethurst --- cmd/gotosocial/action/admin/media/list.go | 170 ++++++++++++++++++---- cmd/gotosocial/admin.go | 22 +-- docs/admin/cli.md | 68 ++++++++- internal/config/config.go | 12 +- internal/config/flags.go | 11 ++ internal/config/helpers.gen.go | 50 +++++++ test/envparsing.sh | 2 + 7 files changed, 283 insertions(+), 52 deletions(-) diff --git a/cmd/gotosocial/action/admin/media/list.go b/cmd/gotosocial/action/admin/media/list.go index 5b6108b11..0a2e60ede 100644 --- a/cmd/gotosocial/action/admin/media/list.go +++ b/cmd/gotosocial/action/admin/media/list.go @@ -20,6 +20,7 @@ package media import ( "bufio" "context" + "errors" "fmt" "os" "path" @@ -34,14 +35,17 @@ import ( ) type list struct { - dbService db.DB - state *state.State - maxID string - limit int - out *bufio.Writer + dbService db.DB + state *state.State + maxID string + limit int + localOnly bool + remoteOnly bool + out *bufio.Writer } -func (l *list) GetAllMediaPaths(ctx context.Context, filter func(*gtsmodel.MediaAttachment) string) ([]string, error) { +// Get a list of attachment using a custom filter +func (l *list) GetAllAttachmentPaths(ctx context.Context, filter func(*gtsmodel.MediaAttachment) string) ([]string, error) { res := make([]string, 0, 100) for { attachments, err := l.dbService.GetAttachments(ctx, l.maxID, l.limit) @@ -72,8 +76,52 @@ func (l *list) GetAllMediaPaths(ctx context.Context, filter func(*gtsmodel.Media return res, nil } +// Get a list of emojis using a custom filter +func (l *list) GetAllEmojisPaths(ctx context.Context, filter func(*gtsmodel.Emoji) string) ([]string, error) { + res := make([]string, 0, 100) + for { + attachments, err := l.dbService.GetEmojis(ctx, l.maxID, l.limit) + if err != nil { + return nil, fmt.Errorf("failed to retrieve media metadata from database: %w", err) + } + + for _, a := range attachments { + v := filter(a) + if v != "" { + res = append(res, v) + } + } + + // If we got less results than our limit, we've reached the + // last page to retrieve and we can break the loop. If the + // last batch happens to contain exactly the same amount of + // items as the limit we'll end up doing one extra query. + if len(attachments) < l.limit { + break + } + + // Grab the last ID from the batch and set it as the maxID + // that'll be used in the next iteration so we don't get items + // we've already seen. + l.maxID = attachments[len(attachments)-1].ID + } + return res, nil +} + func setupList(ctx context.Context) (*list, error) { - var state state.State + var ( + localOnly = config.GetAdminMediaListLocalOnly() + remoteOnly = config.GetAdminMediaListRemoteOnly() + state state.State + ) + + // Validate flags. + if localOnly && remoteOnly { + return nil, errors.New( + "local-only and remote-only flags cannot be true at the same time; " + + "choose one or the other, or set neither to list all media", + ) + } state.Caches.Init() state.Caches.Start() @@ -87,11 +135,13 @@ func setupList(ctx context.Context) (*list, error) { state.DB = dbService return &list{ - dbService: dbService, - state: &state, - limit: 200, - maxID: "", - out: bufio.NewWriter(os.Stdout), + dbService: dbService, + state: &state, + limit: 200, + maxID: "", + localOnly: localOnly, + remoteOnly: remoteOnly, + out: bufio.NewWriter(os.Stdout), }, nil } @@ -103,7 +153,8 @@ func (l *list) shutdown() error { return err } -var ListLocal action.GTSAction = func(ctx context.Context) error { +// ListAttachments lists local, remote, or all attachment paths. +var ListAttachments action.GTSAction = func(ctx context.Context) error { list, err := setupList(ctx) if err != nil { return err @@ -116,26 +167,53 @@ var ListLocal action.GTSAction = func(ctx context.Context) error { } }() - mediaPath := config.GetStorageLocalBasePath() - media, err := list.GetAllMediaPaths( - ctx, - func(m *gtsmodel.MediaAttachment) string { - if m.RemoteURL == "" { - return path.Join(mediaPath, m.File.Path) + var ( + mediaPath = config.GetStorageLocalBasePath() + filter func(*gtsmodel.MediaAttachment) string + ) + + switch { + case list.localOnly: + filter = func(m *gtsmodel.MediaAttachment) string { + if m.RemoteURL != "" { + // Remote, not + // interested. + return "" } - return "" - }) + + return path.Join(mediaPath, m.File.Path) + } + + case list.remoteOnly: + filter = func(m *gtsmodel.MediaAttachment) string { + if m.RemoteURL == "" { + // Local, not + // interested. + return "" + } + + return path.Join(mediaPath, m.File.Path) + } + + default: + filter = func(m *gtsmodel.MediaAttachment) string { + return path.Join(mediaPath, m.File.Path) + } + } + + attachments, err := list.GetAllAttachmentPaths(ctx, filter) if err != nil { return err } - for _, m := range media { - _, _ = list.out.WriteString(m + "\n") + for _, a := range attachments { + _, _ = list.out.WriteString(a + "\n") } return nil } -var ListRemote action.GTSAction = func(ctx context.Context) error { +// ListEmojis lists local, remote, or all emoji filepaths. +var ListEmojis action.GTSAction = func(ctx context.Context) error { list, err := setupList(ctx) if err != nil { return err @@ -148,17 +226,47 @@ var ListRemote action.GTSAction = func(ctx context.Context) error { } }() - media, err := list.GetAllMediaPaths( - ctx, - func(m *gtsmodel.MediaAttachment) string { - return m.RemoteURL - }) + var ( + mediaPath = config.GetStorageLocalBasePath() + filter func(*gtsmodel.Emoji) string + ) + + switch { + case list.localOnly: + filter = func(e *gtsmodel.Emoji) string { + if e.ImageRemoteURL != "" { + // Remote, not + // interested. + return "" + } + + return path.Join(mediaPath, e.ImagePath) + } + + case list.remoteOnly: + filter = func(e *gtsmodel.Emoji) string { + if e.ImageRemoteURL == "" { + // Local, not + // interested. + return "" + } + + return path.Join(mediaPath, e.ImagePath) + } + + default: + filter = func(e *gtsmodel.Emoji) string { + return path.Join(mediaPath, e.ImagePath) + } + } + + emojis, err := list.GetAllEmojisPaths(ctx, filter) if err != nil { return err } - for _, m := range media { - _, _ = list.out.WriteString(m + "\n") + for _, e := range emojis { + _, _ = list.out.WriteString(e + "\n") } return nil } diff --git a/cmd/gotosocial/admin.go b/cmd/gotosocial/admin.go index a8aab810e..3b2ea69e3 100644 --- a/cmd/gotosocial/admin.go +++ b/cmd/gotosocial/admin.go @@ -178,29 +178,31 @@ func adminCommands() *cobra.Command { ADMIN MEDIA LIST COMMANDS */ - adminMediaListLocalCmd := &cobra.Command{ - Use: "list-local", - Short: "admin command to list media on local storage", + adminMediaListAttachmentsCmd := &cobra.Command{ + Use: "list-attachments", + Short: "list local, remote, or all attachments", PreRunE: func(cmd *cobra.Command, args []string) error { return preRun(preRunArgs{cmd: cmd}) }, RunE: func(cmd *cobra.Command, args []string) error { - return run(cmd.Context(), media.ListLocal) + return run(cmd.Context(), media.ListAttachments) }, } + config.AddAdminMediaList(adminMediaListAttachmentsCmd) + adminMediaCmd.AddCommand(adminMediaListAttachmentsCmd) - adminMediaListRemoteCmd := &cobra.Command{ - Use: "list-remote", - Short: "admin command to list remote media cached on this instance", + adminMediaListEmojisLocalCmd := &cobra.Command{ + Use: "list-emojis", + Short: "list local, remote, or all emojis", PreRunE: func(cmd *cobra.Command, args []string) error { return preRun(preRunArgs{cmd: cmd}) }, RunE: func(cmd *cobra.Command, args []string) error { - return run(cmd.Context(), media.ListRemote) + return run(cmd.Context(), media.ListEmojis) }, } - - adminMediaCmd.AddCommand(adminMediaListLocalCmd, adminMediaListRemoteCmd) + config.AddAdminMediaList(adminMediaListEmojisLocalCmd) + adminMediaCmd.AddCommand(adminMediaListEmojisLocalCmd) /* ADMIN MEDIA PRUNE COMMANDS diff --git a/docs/admin/cli.md b/docs/admin/cli.md index 653c8aa3e..405093531 100644 --- a/docs/admin/cli.md +++ b/docs/admin/cli.md @@ -255,17 +255,73 @@ Example: gotosocial admin import --path example.json --config-path config.yaml ``` -### gotosocial admin media list-local +### gotosocial admin media list-attachments -This command can be used to list local media. Local media is media that belongs to posts by users with an account on the instance. +Can be used to list the storage paths of local, remote, or all media attachments on your instance (including headers and avatars). -The output will be a list of files. The list can be used to drive your backups. +`local-only` and `remote-only` can be used as filters; they cannot both be set at once. -### gotosocial admin media list-remote +If neither `local-only` or `remote-only` are set, all media attachments on your instance will be listed. -This is the corollary to list-local, but instead lists media from remote instances. Remote media belongs to other instances, but was attached to a post we received over federation and have potentially cached locally. +You may want to run this with `GTS_LOG_LEVEL` set to `warn` or `error`, otherwise it will log a lot of info messages you probably don't need. -The output will be a list of URLs to retrieve the original content from. GoToSocial automatically retrieves remote media when it needs it, so you should never need to do so yourself. +`gotosocial admin media list-attachments --help`: + +```text +list local, remote, or all attachments + +Usage: + gotosocial admin media list-attachments [flags] + +Flags: + -h, --help help for list-attachments + --local-only list only local attachments/emojis; if specified then remote-only cannot also be true + --remote-only list only remote attachments/emojis; if specified then local-only cannot also be true +``` + +Example output: + +```text +/gotosocial/062G5WYKY35KKD12EMSM3F8PJ8/attachment/original/01PFPMWK2FF0D9WMHEJHR07C3R.jpg +/gotosocial/01F8MH1H7YV1Z7D2C8K2730QBF/attachment/original/01PFPMWK2FF0D9WMHEJHR07C3Q.jpg +/gotosocial/01F8MH5ZK5VRH73AKHQM6Y9VNX/attachment/original/01FVW7RXPQ8YJHTEXYPE7Q8ZY0.jpg +/gotosocial/01F8MH1H7YV1Z7D2C8K2730QBF/attachment/original/01F8MH8RMYQ6MSNY3JM2XT1CQ5.jpg +/gotosocial/01F8MH1H7YV1Z7D2C8K2730QBF/attachment/original/01F8MH7TDVANYKWVE8VVKFPJTJ.gif +/gotosocial/01F8MH17FWEB39HZJ76B6VXSKF/attachment/original/01F8MH6NEM8D7527KZAECTCR76.jpg +/gotosocial/01F8MH1H7YV1Z7D2C8K2730QBF/attachment/original/01F8MH58A357CV5K7R7TJMSH6S.jpg +/gotosocial/01F8MH1H7YV1Z7D2C8K2730QBF/attachment/original/01CDR64G398ADCHXK08WWTHEZ5.gif +``` + +### gotosocial admin media list-emojis + +Can be used to list the storage paths of local, remote, or all emojis on your instance. + +`local-only` and `remote-only` can be used as filters; they cannot both be set at once. + +If neither `local-only` or `remote-only` are set, all emojis on your instance will be listed. + +You may want to run this with `GTS_LOG_LEVEL` set to `warn` or `error`, otherwise it will log a lot of info messages you probably don't need. + +`gotosocial admin media list-emojis --help`: + +```text +list local, remote, or all emojis + +Usage: + gotosocial admin media list-emojis [flags] + +Flags: + -h, --help help for list-emojis + --local-only list only local attachments/emojis; if specified then remote-only cannot also be true + --remote-only list only remote attachments/emojis; if specified then local-only cannot also be true +``` + +Example output: + +```text +/gotosocial/01AY6P665V14JJR0AFVRT7311Y/emoji/original/01GD5KP5CQEE1R3X43Y1EHS2CW.png +/gotosocial/01AY6P665V14JJR0AFVRT7311Y/emoji/original/01F8MH9H8E4VG3KDYJR9EGPXCQ.png +``` ### gotosocial admin media prune orphaned diff --git a/internal/config/config.go b/internal/config/config.go index 612947c5e..5da226237 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -161,11 +161,13 @@ type Configuration struct { Cache CacheConfiguration `name:"cache"` // TODO: move these elsewhere, these are more ephemeral vs long-running flags like above - AdminAccountUsername string `name:"username" usage:"the username to create/delete/etc"` - AdminAccountEmail string `name:"email" usage:"the email address of this account"` - AdminAccountPassword string `name:"password" usage:"the password to set for this account"` - AdminTransPath string `name:"path" usage:"the path of the file to import from/export to"` - AdminMediaPruneDryRun bool `name:"dry-run" usage:"perform a dry run and only log number of items eligible for pruning"` + AdminAccountUsername string `name:"username" usage:"the username to create/delete/etc"` + AdminAccountEmail string `name:"email" usage:"the email address of this account"` + AdminAccountPassword string `name:"password" usage:"the password to set for this account"` + AdminTransPath string `name:"path" usage:"the path of the file to import from/export to"` + AdminMediaPruneDryRun bool `name:"dry-run" usage:"perform a dry run and only log number of items eligible for pruning"` + AdminMediaListLocalOnly bool `name:"local-only" usage:"list only local attachments/emojis; if specified then remote-only cannot also be true"` + AdminMediaListRemoteOnly bool `name:"remote-only" usage:"list only remote attachments/emojis; if specified then local-only cannot also be true"` RequestIDHeader string `name:"request-id-header" usage:"Header to extract the Request ID from. Eg.,'X-Request-Id'."` } diff --git a/internal/config/flags.go b/internal/config/flags.go index ad07ec2ef..74ceedc00 100644 --- a/internal/config/flags.go +++ b/internal/config/flags.go @@ -203,6 +203,17 @@ func AddAdminTrans(cmd *cobra.Command) { } } +// AddAdminMediaList attaches flags pertaining to media list commands. +func AddAdminMediaList(cmd *cobra.Command) { + localOnly := AdminMediaListLocalOnlyFlag() + localOnlyUsage := fieldtag("AdminMediaListLocalOnly", "usage") + cmd.Flags().Bool(localOnly, false, localOnlyUsage) + + remoteOnly := AdminMediaListRemoteOnlyFlag() + remoteOnlyUsage := fieldtag("AdminMediaListRemoteOnly", "usage") + cmd.Flags().Bool(remoteOnly, false, remoteOnlyUsage) +} + // AddAdminMediaPrune attaches flags pertaining to media storage prune commands. func AddAdminMediaPrune(cmd *cobra.Command) { name := AdminMediaPruneDryRunFlag() diff --git a/internal/config/helpers.gen.go b/internal/config/helpers.gen.go index 801d4f667..a515da200 100644 --- a/internal/config/helpers.gen.go +++ b/internal/config/helpers.gen.go @@ -3374,6 +3374,56 @@ func GetAdminMediaPruneDryRun() bool { return global.GetAdminMediaPruneDryRun() // SetAdminMediaPruneDryRun safely sets the value for global configuration 'AdminMediaPruneDryRun' field func SetAdminMediaPruneDryRun(v bool) { global.SetAdminMediaPruneDryRun(v) } +// GetAdminMediaListLocalOnly safely fetches the Configuration value for state's 'AdminMediaListLocalOnly' field +func (st *ConfigState) GetAdminMediaListLocalOnly() (v bool) { + st.mutex.RLock() + v = st.config.AdminMediaListLocalOnly + st.mutex.RUnlock() + return +} + +// SetAdminMediaListLocalOnly safely sets the Configuration value for state's 'AdminMediaListLocalOnly' field +func (st *ConfigState) SetAdminMediaListLocalOnly(v bool) { + st.mutex.Lock() + defer st.mutex.Unlock() + st.config.AdminMediaListLocalOnly = v + st.reloadToViper() +} + +// AdminMediaListLocalOnlyFlag returns the flag name for the 'AdminMediaListLocalOnly' field +func AdminMediaListLocalOnlyFlag() string { return "local-only" } + +// GetAdminMediaListLocalOnly safely fetches the value for global configuration 'AdminMediaListLocalOnly' field +func GetAdminMediaListLocalOnly() bool { return global.GetAdminMediaListLocalOnly() } + +// SetAdminMediaListLocalOnly safely sets the value for global configuration 'AdminMediaListLocalOnly' field +func SetAdminMediaListLocalOnly(v bool) { global.SetAdminMediaListLocalOnly(v) } + +// GetAdminMediaListRemoteOnly safely fetches the Configuration value for state's 'AdminMediaListRemoteOnly' field +func (st *ConfigState) GetAdminMediaListRemoteOnly() (v bool) { + st.mutex.RLock() + v = st.config.AdminMediaListRemoteOnly + st.mutex.RUnlock() + return +} + +// SetAdminMediaListRemoteOnly safely sets the Configuration value for state's 'AdminMediaListRemoteOnly' field +func (st *ConfigState) SetAdminMediaListRemoteOnly(v bool) { + st.mutex.Lock() + defer st.mutex.Unlock() + st.config.AdminMediaListRemoteOnly = v + st.reloadToViper() +} + +// AdminMediaListRemoteOnlyFlag returns the flag name for the 'AdminMediaListRemoteOnly' field +func AdminMediaListRemoteOnlyFlag() string { return "remote-only" } + +// GetAdminMediaListRemoteOnly safely fetches the value for global configuration 'AdminMediaListRemoteOnly' field +func GetAdminMediaListRemoteOnly() bool { return global.GetAdminMediaListRemoteOnly() } + +// SetAdminMediaListRemoteOnly safely sets the value for global configuration 'AdminMediaListRemoteOnly' field +func SetAdminMediaListRemoteOnly(v bool) { global.SetAdminMediaListRemoteOnly(v) } + // GetRequestIDHeader safely fetches the Configuration value for state's 'RequestIDHeader' field func (st *ConfigState) GetRequestIDHeader() (v string) { st.mutex.RLock() diff --git a/test/envparsing.sh b/test/envparsing.sh index bd4e69309..63f1bc5e3 100755 --- a/test/envparsing.sh +++ b/test/envparsing.sh @@ -87,6 +87,7 @@ EXPECT=$(cat << "EOF" "letsencrypt-email-address": "", "letsencrypt-enabled": true, "letsencrypt-port": 80, + "local-only": false, "log-client-ip": false, "log-db-queries": true, "log-level": "info", @@ -116,6 +117,7 @@ EXPECT=$(cat << "EOF" "path": "", "port": 6969, "protocol": "http", + "remote-only": false, "request-id-header": "X-Trace-Id", "smtp-disclose-recipients": true, "smtp-from": "queen.rip.in.piss@terfisland.org",