From c2ff8f392b6320d45d8667d4e093e8eb8ddf59c1 Mon Sep 17 00:00:00 2001 From: tsmethurst Date: Sat, 8 Jan 2022 13:45:42 +0100 Subject: [PATCH] further refinements --- internal/media/image.go | 70 ----------------- internal/media/manager.go | 56 ++++++++++++++ internal/media/media.go | 114 ++++++++++++++++++---------- internal/processing/media/create.go | 26 ++----- 4 files changed, 133 insertions(+), 133 deletions(-) diff --git a/internal/media/image.go b/internal/media/image.go index 157ae0f4a..acc62a28b 100644 --- a/internal/media/image.go +++ b/internal/media/image.go @@ -20,22 +20,16 @@ package media import ( "bytes" - "context" "errors" "fmt" "image" "image/gif" "image/jpeg" "image/png" - "strings" - "time" "github.com/buckket/go-blurhash" "github.com/nfnt/resize" "github.com/superseriousbusiness/exifremove/pkg/exifremove" - "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" - "github.com/superseriousbusiness/gotosocial/internal/id" - "github.com/superseriousbusiness/gotosocial/internal/uris" ) const ( @@ -53,70 +47,6 @@ type ImageMeta struct { blurhash string } -func (m *manager) preProcessImage(ctx context.Context, data []byte, contentType string, accountID string) (*Media, error) { - if !supportedImage(contentType) { - return nil, fmt.Errorf("image type %s not supported", contentType) - } - - if len(data) == 0 { - return nil, errors.New("image was of size 0") - } - - id, err := id.NewRandomULID() - if err != nil { - return nil, err - } - - extension := strings.Split(contentType, "/")[1] - - attachment := >smodel.MediaAttachment{ - ID: id, - UpdatedAt: time.Now(), - URL: uris.GenerateURIForAttachment(accountID, string(TypeAttachment), string(SizeOriginal), id, extension), - Type: gtsmodel.FileTypeImage, - AccountID: accountID, - Processing: 0, - File: gtsmodel.File{ - Path: fmt.Sprintf("%s/%s/%s/%s.%s", accountID, TypeAttachment, SizeOriginal, id, extension), - ContentType: contentType, - UpdatedAt: time.Now(), - }, - Thumbnail: gtsmodel.Thumbnail{ - URL: uris.GenerateURIForAttachment(accountID, string(TypeAttachment), string(SizeSmall), id, mimeJpeg), // all thumbnails are encoded as jpeg, - Path: fmt.Sprintf("%s/%s/%s/%s.%s", accountID, TypeAttachment, SizeSmall, id, mimeJpeg), // all thumbnails are encoded as jpeg, - ContentType: mimeJpeg, - UpdatedAt: time.Now(), - }, - Avatar: false, - Header: false, - } - - media := &Media{ - attachment: attachment, - } - - return media, nil - - var clean []byte - var original *ImageMeta - var small *ImageMeta - - - - if err != nil { - return nil, err - } - - small, err = deriveThumbnail(clean, contentType) - if err != nil { - return nil, fmt.Errorf("error deriving thumbnail: %s", err) - } - - // now put it in storage, take a new id for the name of the file so we don't store any unnecessary info about it - - return attachment, nil -} - func decodeGif(b []byte) (*ImageMeta, error) { gif, err := gif.DecodeAll(bytes.NewReader(b)) if err != nil { diff --git a/internal/media/manager.go b/internal/media/manager.go index 074ebdb58..8032ab42d 100644 --- a/internal/media/manager.go +++ b/internal/media/manager.go @@ -24,11 +24,15 @@ import ( "fmt" "runtime" "strings" + "time" "codeberg.org/gruf/go-runners" "codeberg.org/gruf/go-store/kv" "github.com/sirupsen/logrus" "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/id" + "github.com/superseriousbusiness/gotosocial/internal/uris" ) // Manager provides an interface for managing media: parsing, storing, and retrieving media objects like photos, videos, and gifs. @@ -92,6 +96,7 @@ func (m *manager) ProcessMedia(ctx context.Context, data []byte, accountID strin // if the inner context is done that means the worker pool is closing, so we should just return return default: + // start preloading the media for the caller's convenience media.PreLoad(innerCtx) } }) @@ -101,3 +106,54 @@ func (m *manager) ProcessMedia(ctx context.Context, data []byte, accountID strin return nil, fmt.Errorf("content type %s not (yet) supported", contentType) } } + +// preProcessImage initializes processing +func (m *manager) preProcessImage(ctx context.Context, data []byte, contentType string, accountID string) (*Media, error) { + if !supportedImage(contentType) { + return nil, fmt.Errorf("image type %s not supported", contentType) + } + + if len(data) == 0 { + return nil, errors.New("image was of size 0") + } + + id, err := id.NewRandomULID() + if err != nil { + return nil, err + } + + extension := strings.Split(contentType, "/")[1] + + attachment := >smodel.MediaAttachment{ + ID: id, + UpdatedAt: time.Now(), + URL: uris.GenerateURIForAttachment(accountID, string(TypeAttachment), string(SizeOriginal), id, extension), + Type: gtsmodel.FileTypeImage, + AccountID: accountID, + Processing: 0, + File: gtsmodel.File{ + Path: fmt.Sprintf("%s/%s/%s/%s.%s", accountID, TypeAttachment, SizeOriginal, id, extension), + ContentType: contentType, + UpdatedAt: time.Now(), + }, + Thumbnail: gtsmodel.Thumbnail{ + URL: uris.GenerateURIForAttachment(accountID, string(TypeAttachment), string(SizeSmall), id, mimeJpeg), // all thumbnails are encoded as jpeg, + Path: fmt.Sprintf("%s/%s/%s/%s.%s", accountID, TypeAttachment, SizeSmall, id, mimeJpeg), // all thumbnails are encoded as jpeg, + ContentType: mimeJpeg, + UpdatedAt: time.Now(), + }, + Avatar: false, + Header: false, + } + + media := &Media{ + attachment: attachment, + rawData: data, + thumbstate: received, + fullSizeState: received, + database: m.db, + storage: m.storage, + } + + return media, nil +} diff --git a/internal/media/media.go b/internal/media/media.go index aa11787b2..022de063e 100644 --- a/internal/media/media.go +++ b/internal/media/media.go @@ -33,15 +33,15 @@ type Media struct { below fields represent the processing state of the media thumbnail */ - thumbing processState - thumb *ImageMeta + thumbstate processState + thumb *ImageMeta /* below fields represent the processing state of the full-sized media */ - processing processState - processed *ImageMeta + fullSizeState processState + fullSize *ImageMeta /* below pointers to database and storage are maintained so that @@ -58,20 +58,20 @@ func (m *Media) Thumb(ctx context.Context) (*ImageMeta, error) { m.mu.Lock() defer m.mu.Unlock() - switch m.thumbing { + switch m.thumbstate { case received: // we haven't processed a thumbnail for this media yet so do it now thumb, err := deriveThumbnail(m.rawData, m.attachment.File.ContentType) if err != nil { m.err = fmt.Errorf("error deriving thumbnail: %s", err) - m.thumbing = errored + m.thumbstate = errored return nil, m.err } // put the thumbnail in storage if err := m.storage.Put(m.attachment.Thumbnail.Path, thumb.image); err != nil { m.err = fmt.Errorf("error storing thumbnail: %s", err) - m.thumbing = errored + m.thumbstate = errored return nil, m.err } @@ -89,12 +89,12 @@ func (m *Media) Thumb(ctx context.Context) (*ImageMeta, error) { if err := m.database.Put(ctx, m.attachment); err != nil { if err != db.ErrAlreadyExists { m.err = fmt.Errorf("error putting attachment: %s", err) - m.thumbing = errored + m.thumbstate = errored return nil, m.err } if err := m.database.UpdateByPrimaryKey(ctx, m.attachment); err != nil { m.err = fmt.Errorf("error updating attachment: %s", err) - m.thumbing = errored + m.thumbstate = errored return nil, m.err } } @@ -103,7 +103,7 @@ func (m *Media) Thumb(ctx context.Context) (*ImageMeta, error) { m.thumb = thumb // we're done processing the thumbnail! - m.thumbing = complete + m.thumbstate = complete fallthrough case complete: return m.thumb, nil @@ -111,46 +111,76 @@ func (m *Media) Thumb(ctx context.Context) (*ImageMeta, error) { return nil, m.err } - return nil, fmt.Errorf("thumbnail processing status %d unknown", m.thumbing) + return nil, fmt.Errorf("thumbnail processing status %d unknown", m.thumbstate) } -func (m *Media) Full(ctx context.Context) (*ImageMeta, error) { - var clean []byte - var err error - var original *ImageMeta +func (m *Media) FullSize(ctx context.Context) (*ImageMeta, error) { + m.mu.Lock() + defer m.mu.Unlock() - ct := m.attachment.File.ContentType -aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa - switch ct { - case mimeImageJpeg, mimeImagePng: - // first 'clean' image by purging exif data from it - var exifErr error - if clean, exifErr = purgeExif(m.rawData); exifErr != nil { - return nil, fmt.Errorf("error cleaning exif data: %s", exifErr) + switch m.fullSizeState { + case received: + var clean []byte + var err error + var decoded *ImageMeta + + ct := m.attachment.File.ContentType + switch ct { + case mimeImageJpeg, mimeImagePng: + // first 'clean' image by purging exif data from it + var exifErr error + if clean, exifErr = purgeExif(m.rawData); exifErr != nil { + err = exifErr + break + } + decoded, err = decodeImage(clean, ct) + case mimeImageGif: + // gifs are already clean - no exif data to remove + clean = m.rawData + decoded, err = decodeGif(clean) + default: + err = fmt.Errorf("content type %s not a processible image type", ct) } - original, err = decodeImage(clean, ct) - case mimeImageGif: - // gifs are already clean - no exif data to remove - clean = m.rawData - original, err = decodeGif(clean) - default: - err = fmt.Errorf("content type %s not a processible image type", ct) + + if err != nil { + m.err = err + m.fullSizeState = errored + return nil, err + } + + // set the fullsize of this media + m.fullSize = decoded + + // we're done processing the full-size image + m.fullSizeState = complete + fallthrough + case complete: + return m.fullSize, nil + case errored: + return nil, m.err } - if err != nil { + return nil, fmt.Errorf("full size processing status %d unknown", m.fullSizeState) +} + +// PreLoad begins the process of deriving the thumbnail and encoding the full-size image. +// It does this in a non-blocking way, so you can call it and then come back later and check +// if it's finished. +func (m *Media) PreLoad(ctx context.Context) { + go m.Thumb(ctx) + go m.FullSize(ctx) +} + +// Load is the blocking equivalent of pre-load. It makes sure the thumbnail and full-size image +// have been processed, then it returns the full-size image. +func (m *Media) Load(ctx context.Context) (*gtsmodel.MediaAttachment, error) { + if _, err := m.Thumb(ctx); err != nil { return nil, err } - return original, nil -} + if _, err := m.FullSize(ctx); err != nil { + return nil, err + } -func (m *Media) PreLoad(ctx context.Context) { - go m.Thumb(ctx) - m.mu.Lock() - defer m.mu.Unlock() -} - -func (m *Media) Load() { - m.mu.Lock() - defer m.mu.Unlock() + return m.attachment, nil } diff --git a/internal/processing/media/create.go b/internal/processing/media/create.go index 68a011683..357278e64 100644 --- a/internal/processing/media/create.go +++ b/internal/processing/media/create.go @@ -24,11 +24,9 @@ import ( "errors" "fmt" "io" - "time" apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" - "github.com/superseriousbusiness/gotosocial/internal/text" ) func (p *processor) Create(ctx context.Context, account *gtsmodel.Account, form *apimodel.AttachmentRequest) (*apimodel.Attachment, error) { @@ -46,29 +44,15 @@ func (p *processor) Create(ctx context.Context, account *gtsmodel.Account, form return nil, errors.New("could not read provided attachment: size 0 bytes") } - // now parse the focus parameter - focusx, focusy, err := parseFocus(form.Focus) + // process the media and load it immediately + media, err := p.mediaManager.ProcessMedia(ctx, buf.Bytes(), account.ID) if err != nil { - return nil, fmt.Errorf("couldn't parse attachment focus: %s", err) + return nil, err } - minAttachment := >smodel.MediaAttachment{ - CreatedAt: time.Now(), - UpdatedAt: time.Now(), - AccountID: account.ID, - Description: text.SanitizeCaption(form.Description), - FileMeta: gtsmodel.FileMeta{ - Focus: gtsmodel.Focus{ - X: focusx, - Y: focusy, - }, - }, - } - - // allow the mediaManager to work its magic of processing the attachment bytes, and putting them in whatever storage backend we're using - attachment, err := p.mediaManager.ProcessAttachment(ctx, buf.Bytes(), minAttachment) + attachment, err := media.Load(ctx) if err != nil { - return nil, fmt.Errorf("error reading attachment: %s", err) + return nil, err } // prepare the frontend representation now -- if there are any errors here at least we can bail without