[feature] For video attachments, store + return fps, bitrate, duration (#1282)

* start messing about with different mp4 metadata extraction

* heyyooo it works

* add test cow

* move useful multierror to gtserror package

* error out if video doesn't seem to be a real mp4

* test parsing mkv in disguise as mp4

* tidy up error handling

* remove extraneous line

* update framerate formatting

* use float32 for aspect

* fixy mctesterson

internal/db/bundb/migrations/20221220134514_mp4_jiggery_pokery.go

@@ -0,0 +1,59 @@
+/*
+   GoToSocial
+   Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package migrations
+
+import (
+	"context"
+	"strings"
+
+	"github.com/uptrace/bun"
+)
+
+func init() {
+	up := func(ctx context.Context, db *bun.DB) error {
+		return db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error {
+			_, err := tx.ExecContext(ctx, "ALTER TABLE ? ADD COLUMN ? REAL", bun.Ident("media_attachments"), bun.Ident("original_duration"))
+			if err != nil && !(strings.Contains(err.Error(), "already exists") || strings.Contains(err.Error(), "duplicate column name") || strings.Contains(err.Error(), "SQLSTATE 42701")) {
+				return err
+			}
+
+			_, err = tx.ExecContext(ctx, "ALTER TABLE ? ADD COLUMN ? REAL", bun.Ident("media_attachments"), bun.Ident("original_framerate"))
+			if err != nil && !(strings.Contains(err.Error(), "already exists") || strings.Contains(err.Error(), "duplicate column name") || strings.Contains(err.Error(), "SQLSTATE 42701")) {
+				return err
+			}
+
+			_, err = tx.ExecContext(ctx, "ALTER TABLE ? ADD COLUMN ? INTEGER", bun.Ident("media_attachments"), bun.Ident("original_bitrate"))
+			if err != nil && !(strings.Contains(err.Error(), "already exists") || strings.Contains(err.Error(), "duplicate column name") || strings.Contains(err.Error(), "SQLSTATE 42701")) {
+				return err
+			}
+
+			return nil
+		})
+	}
+
+	down := func(ctx context.Context, db *bun.DB) error {
+		return db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error {
+			return nil
+		})
+	}
+
+	if err := Migrations.Register(up, down); err != nil {
+		panic(err)
+	}
+}

internal/federation/dereferencing/media_test.go

@@ -66,7 +66,7 @@ func (suite *AttachmentTestSuite) TestDereferenceAttachmentBlocking() {
 	suite.NotEmpty(attachment.ID)
 	suite.NotEmpty(attachment.CreatedAt)
 	suite.NotEmpty(attachment.UpdatedAt)
-	suite.Equal(1.336546184738956, attachment.FileMeta.Original.Aspect)
+	suite.EqualValues(1.3365462, attachment.FileMeta.Original.Aspect)
 	suite.Equal(2071680, attachment.FileMeta.Original.Size)
 	suite.Equal(1245, attachment.FileMeta.Original.Height)
 	suite.Equal(1664, attachment.FileMeta.Original.Width)
@@ -92,7 +92,7 @@ func (suite *AttachmentTestSuite) TestDereferenceAttachmentBlocking() {
 	suite.NotEmpty(dbAttachment.ID)
 	suite.NotEmpty(dbAttachment.CreatedAt)
 	suite.NotEmpty(dbAttachment.UpdatedAt)
-	suite.Equal(1.336546184738956, dbAttachment.FileMeta.Original.Aspect)
+	suite.EqualValues(1.3365462, dbAttachment.FileMeta.Original.Aspect)
 	suite.Equal(2071680, dbAttachment.FileMeta.Original.Size)
 	suite.Equal(1245, dbAttachment.FileMeta.Original.Height)
 	suite.Equal(1664, dbAttachment.FileMeta.Original.Width)
@@ -147,7 +147,7 @@ func (suite *AttachmentTestSuite) TestDereferenceAttachmentAsync() {
 	suite.NotEmpty(attachment.ID)
 	suite.NotEmpty(attachment.CreatedAt)
 	suite.NotEmpty(attachment.UpdatedAt)
-	suite.Equal(1.336546184738956, attachment.FileMeta.Original.Aspect)
+	suite.EqualValues(1.3365462, attachment.FileMeta.Original.Aspect)
 	suite.Equal(2071680, attachment.FileMeta.Original.Size)
 	suite.Equal(1245, attachment.FileMeta.Original.Height)
 	suite.Equal(1664, attachment.FileMeta.Original.Width)

internal/gtserror/multi.go

@@ -0,0 +1,45 @@
+/*
+   GoToSocial
+   Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package gtserror
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+)
+
+// MultiError allows encapsulating multiple errors under a singular instance,
+// which is useful when you only want to log on errors, not return early / bubble up.
+type MultiError []string
+
+func (e *MultiError) Append(err error) {
+	*e = append(*e, err.Error())
+}
+
+func (e *MultiError) Appendf(format string, args ...any) {
+	*e = append(*e, fmt.Sprintf(format, args...))
+}
+
+// Combine converts this multiError to a singular error instance, returning nil if empty.
+func (e MultiError) Combine() error {
+	if len(e) == 0 {
+		return nil
+	}
+	return errors.New(`"` + strings.Join(e, `","`) + `"`)
+}

internal/gtsmodel/mediaattachment.go

@@ -99,15 +99,18 @@ type Small struct {
 	Width  int     `validate:"required_with=Height Size Aspect"`  // width in pixels
 	Height int     `validate:"required_with=Width Size Aspect"`   // height in pixels
 	Size   int     `validate:"required_with=Width Height Aspect"` // size in pixels (width * height)
-	Aspect float64 `validate:"required_with=Widhth Height Size"`  // aspect ratio (width / height)
+	Aspect float32 `validate:"required_with=Width Height Size"`   // aspect ratio (width / height)
 }
 
 // Original can be used for original metadata for any media type
 type Original struct {
-	Width  int     `validate:"required_with=Height Size Aspect"`  // width in pixels
+	Width     int      `validate:"required_with=Height Size Aspect"`  // width in pixels
-	Height int     `validate:"required_with=Width Size Aspect"`   // height in pixels
+	Height    int      `validate:"required_with=Width Size Aspect"`   // height in pixels
-	Size   int     `validate:"required_with=Width Height Aspect"` // size in pixels (width * height)
+	Size      int      `validate:"required_with=Width Height Aspect"` // size in pixels (width * height)
-	Aspect float64 `validate:"required_with=Widhth Height Size"`  // aspect ratio (width / height)
+	Aspect    float32  `validate:"required_with=Width Height Size"`   // aspect ratio (width / height)
+	Duration  *float32 `validate:"-"`                                 // video-specific: duration of the video in seconds
+	Framerate *float32 `validate:"-"`                                 // video-specific: fps
+	Bitrate   *uint64  `validate:"-"`                                 // video-specific: bitrate
 }
 
 // Focus describes the 'center' of the image for display purposes.

internal/media/image.go

@@ -48,7 +48,7 @@ func decodeGif(r io.Reader) (*mediaMeta, error) {
 	width := gif.Config.Width
 	height := gif.Config.Height
 	size := width * height
-	aspect := float64(width) / float64(height)
+	aspect := float32(width) / float32(height)
 
 	return &mediaMeta{
 		width:  width,
@@ -85,7 +85,7 @@ func decodeImage(r io.Reader, contentType string) (*mediaMeta, error) {
 	width := i.Bounds().Size().X
 	height := i.Bounds().Size().Y
 	size := width * height
-	aspect := float64(width) / float64(height)
+	aspect := float32(width) / float32(height)
 
 	return &mediaMeta{
 		width:  width,
@@ -167,7 +167,7 @@ func deriveThumbnailFromImage(r io.Reader, contentType string, createBlurhash bo
 	thumbX := thumb.Bounds().Size().X
 	thumbY := thumb.Bounds().Size().Y
 	size := thumbX * thumbY
-	aspect := float64(thumbX) / float64(thumbY)
+	aspect := float32(thumbX) / float32(thumbY)
 
 	im := &mediaMeta{
 		width:  thumbX,

internal/media/manager_test.go

@@ -407,9 +407,13 @@ func (suite *ManagerTestSuite) TestSlothVineProcessBlocking() {
 	suite.Equal(accountID, attachment.AccountID)
 
 	// file meta should be correctly derived from the video
-	suite.EqualValues(gtsmodel.Original{
+	suite.Equal(338, attachment.FileMeta.Original.Width)
-		Width: 338, Height: 240, Size: 81120, Aspect: 1.4083333333333334,
+	suite.Equal(240, attachment.FileMeta.Original.Height)
-	}, attachment.FileMeta.Original)
+	suite.Equal(81120, attachment.FileMeta.Original.Size)
+	suite.EqualValues(1.4083333, attachment.FileMeta.Original.Aspect)
+	suite.EqualValues(6.5862, *attachment.FileMeta.Original.Duration)
+	suite.EqualValues(29.000029, *attachment.FileMeta.Original.Framerate)
+	suite.EqualValues(0x3b3e1, *attachment.FileMeta.Original.Bitrate)
 	suite.EqualValues(gtsmodel.Small{
 		Width: 338, Height: 240, Size: 81120, Aspect: 1.4083333333333334,
 	}, attachment.FileMeta.Small)
@@ -448,6 +452,108 @@ func (suite *ManagerTestSuite) TestSlothVineProcessBlocking() {
 	suite.Equal(processedThumbnailBytesExpected, processedThumbnailBytes)
 }
 
+func (suite *ManagerTestSuite) TestLongerMp4ProcessBlocking() {
+	ctx := context.Background()
+
+	data := func(_ context.Context) (io.ReadCloser, int64, error) {
+		// load bytes from a test video
+		b, err := os.ReadFile("./test/longer-mp4-original.mp4")
+		if err != nil {
+			panic(err)
+		}
+		return io.NopCloser(bytes.NewBuffer(b)), int64(len(b)), nil
+	}
+
+	accountID := "01FS1X72SK9ZPW0J1QQ68BD264"
+
+	// process the media with no additional info provided
+	processingMedia, err := suite.manager.ProcessMedia(ctx, data, nil, accountID, nil)
+	suite.NoError(err)
+	// fetch the attachment id from the processing media
+	attachmentID := processingMedia.AttachmentID()
+
+	// do a blocking call to fetch the attachment
+	attachment, err := processingMedia.LoadAttachment(ctx)
+	suite.NoError(err)
+	suite.NotNil(attachment)
+
+	// make sure it's got the stuff set on it that we expect
+	// the attachment ID and accountID we expect
+	suite.Equal(attachmentID, attachment.ID)
+	suite.Equal(accountID, attachment.AccountID)
+
+	// file meta should be correctly derived from the video
+	suite.Equal(600, attachment.FileMeta.Original.Width)
+	suite.Equal(330, attachment.FileMeta.Original.Height)
+	suite.Equal(198000, attachment.FileMeta.Original.Size)
+	suite.EqualValues(1.8181819, attachment.FileMeta.Original.Aspect)
+	suite.EqualValues(16.6, *attachment.FileMeta.Original.Duration)
+	suite.EqualValues(10, *attachment.FileMeta.Original.Framerate)
+	suite.EqualValues(0xc8fb, *attachment.FileMeta.Original.Bitrate)
+	suite.EqualValues(gtsmodel.Small{
+		Width: 600, Height: 330, Size: 198000, Aspect: 1.8181819,
+	}, attachment.FileMeta.Small)
+	suite.Equal("video/mp4", attachment.File.ContentType)
+	suite.Equal("image/jpeg", attachment.Thumbnail.ContentType)
+	suite.Equal(109549, attachment.File.FileSize)
+	suite.Equal("", attachment.Blurhash)
+
+	// now make sure the attachment is in the database
+	dbAttachment, err := suite.db.GetAttachmentByID(ctx, attachmentID)
+	suite.NoError(err)
+	suite.NotNil(dbAttachment)
+
+	// make sure the processed file is in storage
+	processedFullBytes, err := suite.storage.Get(ctx, attachment.File.Path)
+	suite.NoError(err)
+	suite.NotEmpty(processedFullBytes)
+
+	// load the processed bytes from our test folder, to compare
+	processedFullBytesExpected, err := os.ReadFile("./test/longer-mp4-processed.mp4")
+	suite.NoError(err)
+	suite.NotEmpty(processedFullBytesExpected)
+
+	// the bytes in storage should be what we expected
+	suite.Equal(processedFullBytesExpected, processedFullBytes)
+
+	// now do the same for the thumbnail and make sure it's what we expected
+	processedThumbnailBytes, err := suite.storage.Get(ctx, attachment.Thumbnail.Path)
+	suite.NoError(err)
+	suite.NotEmpty(processedThumbnailBytes)
+
+	processedThumbnailBytesExpected, err := os.ReadFile("./test/longer-mp4-thumbnail.jpg")
+	suite.NoError(err)
+	suite.NotEmpty(processedThumbnailBytesExpected)
+
+	suite.Equal(processedThumbnailBytesExpected, processedThumbnailBytes)
+}
+
+func (suite *ManagerTestSuite) TestNotAnMp4ProcessBlocking() {
+	// try to load an 'mp4' that's actually an mkv in disguise
+
+	ctx := context.Background()
+
+	data := func(_ context.Context) (io.ReadCloser, int64, error) {
+		// load bytes from a test video
+		b, err := os.ReadFile("./test/not-an.mp4")
+		if err != nil {
+			panic(err)
+		}
+		return io.NopCloser(bytes.NewBuffer(b)), int64(len(b)), nil
+	}
+
+	accountID := "01FS1X72SK9ZPW0J1QQ68BD264"
+
+	// pre processing should go fine but...
+	processingMedia, err := suite.manager.ProcessMedia(ctx, data, nil, accountID, nil)
+	suite.NoError(err)
+
+	// we should get an error while loading
+	attachment, err := processingMedia.LoadAttachment(ctx)
+	suite.EqualError(err, "\"video width could not be discovered\",\"video height could not be discovered\",\"video duration could not be discovered\",\"video framerate could not be discovered\",\"video bitrate could not be discovered\"")
+	suite.Nil(attachment)
+}
+
 func (suite *ManagerTestSuite) TestSimpleJpegProcessBlockingNoContentLengthGiven() {
 	ctx := context.Background()
 

internal/media/processingmedia.go

@@ -249,16 +249,32 @@ func (p *ProcessingMedia) loadFullSize(ctx context.Context) error {
 		}
 
 		// set appropriate fields on the attachment based on the image we derived
+
+		// generic fields
+		p.attachment.File.UpdatedAt = time.Now()
 		p.attachment.FileMeta.Original = gtsmodel.Original{
 			Width:  decoded.width,
 			Height: decoded.height,
 			Size:   decoded.size,
 			Aspect: decoded.aspect,
 		}
-		p.attachment.File.UpdatedAt = time.Now()
+
-		p.attachment.Processing = gtsmodel.ProcessingStatusProcessed
+		// nullable fields
+		if decoded.duration != 0 {
+			i := decoded.duration
+			p.attachment.FileMeta.Original.Duration = &i
+		}
+		if decoded.framerate != 0 {
+			i := decoded.framerate
+			p.attachment.FileMeta.Original.Framerate = &i
+		}
+		if decoded.bitrate != 0 {
+			i := decoded.bitrate
+			p.attachment.FileMeta.Original.Bitrate = &i
+		}
 
 		// we're done processing the full-size image
+		p.attachment.Processing = gtsmodel.ProcessingStatusProcessed
 		atomic.StoreInt32(&p.fullSizeState, int32(complete))
 		log.Tracef("finished processing full size image for attachment %s", p.attachment.URL)
 		fallthrough

internal/media/types.go

@@ -137,7 +137,12 @@ type mediaMeta struct {
 	width    int
 	height   int
 	size     int
-	aspect   float64
+	aspect   float32
 	blurhash string
 	small    []byte
+
+	// video-specific properties
+	duration  float32
+	framerate float32
+	bitrate   uint64
 }

internal/media/video.go

@@ -20,7 +20,6 @@ package media
 
 import (
 	"bytes"
-	"errors"
 	"fmt"
 	"image"
 	"image/color"
@@ -30,6 +29,7 @@ import (
 	"os"
 
 	"github.com/abema/go-mp4"
+	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
 	"github.com/superseriousbusiness/gotosocial/internal/log"
 )
 
@@ -61,62 +61,82 @@ func decodeVideo(r io.Reader, contentType string) (*mediaMeta, error) {
 		return nil, fmt.Errorf("could not copy video reader into temporary file %s: %w", tempFileName, err)
 	}
 
-	// define some vars we need to pull the width/height out of the video
 	var (
-		height      int
+		width     int
-		width       int
+		height    int
-		readHandler = getReadHandler(&height, &width)
+		duration  float32
+		framerate float32
+		bitrate   uint64
 	)
 
-	// do the actual decoding here, providing the temporary file we created as readseeker
+	// probe the video file to extract useful metadata from it; for methodology, see:
-	if _, err := mp4.ReadBoxStructure(tempFile, readHandler); err != nil {
+	// https://github.com/abema/go-mp4/blob/7d8e5a7c5e644e0394261b0cf72fef79ce246d31/mp4tool/probe/probe.go#L85-L154
-		return nil, fmt.Errorf("parsing video data: %w", err)
+	info, err := mp4.Probe(tempFile)
+	if err != nil {
+		return nil, fmt.Errorf("could not probe temporary video file %s: %w", tempFileName, err)
+	}
+
+	for _, tr := range info.Tracks {
+		if tr.AVC == nil {
+			continue
+		}
+
+		if w := int(tr.AVC.Width); w > width {
+			width = w
+		}
+
+		if h := int(tr.AVC.Height); h > height {
+			height = h
+		}
+
+		if br := tr.Samples.GetBitrate(tr.Timescale); br > bitrate {
+			bitrate = br
+		} else if br := info.Segments.GetBitrate(tr.TrackID, tr.Timescale); br > bitrate {
+			bitrate = br
+		}
+
+		if d := float32(tr.Duration) / float32(tr.Timescale); d > duration {
+			duration = d
+			framerate = float32(len(tr.Samples)) / duration
+		}
+	}
+
+	var errs gtserror.MultiError
+	if width == 0 {
+		errs = append(errs, "video width could not be discovered")
+	}
+
+	if height == 0 {
+		errs = append(errs, "video height could not be discovered")
+	}
+
+	if duration == 0 {
+		errs = append(errs, "video duration could not be discovered")
+	}
+
+	if framerate == 0 {
+		errs = append(errs, "video framerate could not be discovered")
+	}
+
+	if bitrate == 0 {
+		errs = append(errs, "video bitrate could not be discovered")
+	}
+
+	if errs != nil {
+		return nil, errs.Combine()
 	}
 
-	// width + height should now be updated by the readHandler
 	return &mediaMeta{
-		width:  width,
+		width:     width,
-		height: height,
+		height:    height,
-		size:   height * width,
+		duration:  duration,
-		aspect: float64(width) / float64(height),
+		framerate: framerate,
+		bitrate:   bitrate,
+		size:      height * width,
+		aspect:    float32(width) / float32(height),
 	}, nil
 }
 
-// getReadHandler returns a handler function that updates the underling
-// values of the given height and width int pointers to the hightest and
-// widest points of the video.
-func getReadHandler(height *int, width *int) func(h *mp4.ReadHandle) (interface{}, error) {
-	return func(rh *mp4.ReadHandle) (interface{}, error) {
-		if rh.BoxInfo.Type == mp4.BoxTypeTkhd() {
-			box, _, err := rh.ReadPayload()
-			if err != nil {
-				return nil, fmt.Errorf("could not read mp4 payload: %w", err)
-			}
-
-			tkhd, ok := box.(*mp4.Tkhd)
-			if !ok {
-				return nil, errors.New("box was not of type *mp4.Tkhd")
-			}
-
-			// if height + width of this box are greater than what
-			// we have stored, then update our stored values
-			if h := int(tkhd.GetHeight()); h > *height {
-				*height = h
-			}
-
-			if w := int(tkhd.GetWidth()); w > *width {
-				*width = w
-			}
-		}
-
-		if rh.BoxInfo.IsSupportedType() {
-			return rh.Expand()
-		}
-
-		return nil, nil
-	}
-}
-
 func deriveThumbnailFromVideo(height int, width int) (*mediaMeta, error) {
 	// create a rectangle with the same dimensions as the video
 	img := image.NewRGBA(image.Rect(0, 0, width, height))
@@ -134,7 +154,7 @@ func deriveThumbnailFromVideo(height int, width int) (*mediaMeta, error) {
 		width:  width,
 		height: height,
 		size:   width * height,
-		aspect: float64(width) / float64(height),
+		aspect: float32(width) / float32(height),
 		small:  out.Bytes(),
 	}, nil
 }

internal/typeutils/converter_test.go

@@ -469,11 +469,12 @@ const (
 
 type TypeUtilsTestSuite struct {
 	suite.Suite
-	db           db.DB
+	db              db.DB
-	testAccounts map[string]*gtsmodel.Account
+	testAccounts    map[string]*gtsmodel.Account
-	testStatuses map[string]*gtsmodel.Status
+	testStatuses    map[string]*gtsmodel.Status
-	testPeople   map[string]vocab.ActivityStreamsPerson
+	testAttachments map[string]*gtsmodel.MediaAttachment
-	testEmojis   map[string]*gtsmodel.Emoji
+	testPeople      map[string]vocab.ActivityStreamsPerson
+	testEmojis      map[string]*gtsmodel.Emoji
 
 	typeconverter typeutils.TypeConverter
 }
@@ -485,6 +486,7 @@ func (suite *TypeUtilsTestSuite) SetupSuite() {
 	suite.db = testrig.NewTestDB()
 	suite.testAccounts = testrig.NewTestAccounts()
 	suite.testStatuses = testrig.NewTestStatuses()
+	suite.testAttachments = testrig.NewTestAttachments()
 	suite.testPeople = testrig.NewTestFediPeople()
 	suite.testEmojis = testrig.NewTestEmojis()
 	suite.typeconverter = typeutils.NewConverter(suite.db)

internal/typeutils/internaltofrontend.go

@@ -22,11 +22,14 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"math"
+	"strconv"
 	"strings"
 
 	"github.com/superseriousbusiness/gotosocial/internal/api/model"
 	"github.com/superseriousbusiness/gotosocial/internal/config"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/log"
 	"github.com/superseriousbusiness/gotosocial/internal/media"
@@ -299,26 +302,38 @@ func (c *converter) AttachmentToAPIAttachment(ctx context.Context, a *gtsmodel.M
 	}
 
 	// nullable fields
-	if a.URL != "" {
+	if i := a.URL; i != "" {
-		i := a.URL
 		apiAttachment.URL = &i
 	}
 
-	if a.RemoteURL != "" {
+	if i := a.RemoteURL; i != "" {
-		i := a.RemoteURL
 		apiAttachment.RemoteURL = &i
 	}
 
-	if a.Thumbnail.RemoteURL != "" {
+	if i := a.Thumbnail.RemoteURL; i != "" {
-		i := a.Thumbnail.RemoteURL
 		apiAttachment.PreviewRemoteURL = &i
 	}
 
-	if a.Description != "" {
+	if i := a.Description; i != "" {
-		i := a.Description
 		apiAttachment.Description = &i
 	}
 
+	if i := a.FileMeta.Original.Duration; i != nil {
+		apiAttachment.Meta.Original.Duration = *i
+	}
+
+	if i := a.FileMeta.Original.Framerate; i != nil {
+		// the masto api expects this as a string in
+		// the format `integer/1`, so 30fps is `30/1`
+		round := math.Round(float64(*i))
+		fr := strconv.FormatInt(int64(round), 10)
+		apiAttachment.Meta.Original.FrameRate = fr + "/1"
+	}
+
+	if i := a.FileMeta.Original.Bitrate; i != nil {
+		apiAttachment.Meta.Original.Bitrate = int(*i)
+	}
+
 	return apiAttachment, nil
 }
 
@@ -789,7 +804,7 @@ func (c *converter) DomainBlockToAPIDomainBlock(ctx context.Context, b *gtsmodel
 
 // convertAttachmentsToAPIAttachments will convert a slice of GTS model attachments to frontend API model attachments, falling back to IDs if no GTS models supplied.
 func (c *converter) convertAttachmentsToAPIAttachments(ctx context.Context, attachments []*gtsmodel.MediaAttachment, attachmentIDs []string) ([]model.Attachment, error) {
-	var errs multiError
+	var errs gtserror.MultiError
 
 	if len(attachments) == 0 {
 		// GTS model attachments were not populated
@@ -826,7 +841,7 @@ func (c *converter) convertAttachmentsToAPIAttachments(ctx context.Context, atta
 
 // convertEmojisToAPIEmojis will convert a slice of GTS model emojis to frontend API model emojis, falling back to IDs if no GTS models supplied.
 func (c *converter) convertEmojisToAPIEmojis(ctx context.Context, emojis []*gtsmodel.Emoji, emojiIDs []string) ([]model.Emoji, error) {
-	var errs multiError
+	var errs gtserror.MultiError
 
 	if len(emojis) == 0 {
 		// GTS model attachments were not populated
@@ -863,7 +878,7 @@ func (c *converter) convertEmojisToAPIEmojis(ctx context.Context, emojis []*gtsm
 
 // convertMentionsToAPIMentions will convert a slice of GTS model mentions to frontend API model mentions, falling back to IDs if no GTS models supplied.
 func (c *converter) convertMentionsToAPIMentions(ctx context.Context, mentions []*gtsmodel.Mention, mentionIDs []string) ([]model.Mention, error) {
-	var errs multiError
+	var errs gtserror.MultiError
 
 	if len(mentions) == 0 {
 		var err error
@@ -895,7 +910,7 @@ func (c *converter) convertMentionsToAPIMentions(ctx context.Context, mentions [
 
 // convertTagsToAPITags will convert a slice of GTS model tags to frontend API model tags, falling back to IDs if no GTS models supplied.
 func (c *converter) convertTagsToAPITags(ctx context.Context, tags []*gtsmodel.Tag, tagIDs []string) ([]model.Tag, error) {
-	var errs multiError
+	var errs gtserror.MultiError
 
 	if len(tags) == 0 {
 		// GTS model tags were not populated
@@ -929,24 +944,3 @@ func (c *converter) convertTagsToAPITags(ctx context.Context, tags []*gtsmodel.T
 
 	return apiTags, errs.Combine()
 }
-
-// multiError allows encapsulating multiple errors under a singular instance,
-// which is useful when you only want to log on errors, not return early / bubble up.
-// TODO: if this is useful elsewhere, move into a separate gts subpackage.
-type multiError []string
-
-func (e *multiError) Append(err error) {
-	*e = append(*e, err.Error())
-}
-
-func (e *multiError) Appendf(format string, args ...any) {
-	*e = append(*e, fmt.Sprintf(format, args...))
-}
-
-// Combine converts this multiError to a singular error instance, returning nil if empty.
-func (e multiError) Combine() error {
-	if len(e) == 0 {
-		return nil
-	}
-	return errors.New(`"` + strings.Join(e, `","`) + `"`)
-}

internal/typeutils/internaltofrontend_test.go

@@ -110,6 +110,17 @@ func (suite *InternalToFrontendTestSuite) TestStatusToFrontendUnknownLanguage()
 	suite.Equal(`{"id":"01F8MH75CBF9JFX4ZAD54N0W0R","created_at":"2021-10-20T11:36:45.000Z","in_reply_to_id":null,"in_reply_to_account_id":null,"sensitive":false,"spoiler_text":"","visibility":"public","language":null,"uri":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R","url":"http://localhost:8080/@admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R","replies_count":0,"reblogs_count":0,"favourites_count":1,"favourited":true,"reblogged":false,"muted":false,"bookmarked":true,"pinned":false,"content":"hello world! #welcome ! first post on the instance :rainbow: !","reblog":null,"application":{"name":"superseriousbusiness","website":"https://superserious.business"},"account":{"id":"01F8MH17FWEB39HZJ76B6VXSKF","username":"admin","acct":"admin","display_name":"","locked":false,"bot":false,"created_at":"2022-05-17T13:10:59.000Z","note":"","url":"http://localhost:8080/@admin","avatar":"","avatar_static":"","header":"http://localhost:8080/assets/default_header.png","header_static":"http://localhost:8080/assets/default_header.png","followers_count":1,"following_count":1,"statuses_count":4,"last_status_at":"2021-10-20T10:41:37.000Z","emojis":[],"fields":[],"enable_rss":true,"role":"admin"},"media_attachments":[{"id":"01F8MH6NEM8D7527KZAECTCR76","type":"image","url":"http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/original/01F8MH6NEM8D7527KZAECTCR76.jpeg","text_url":"http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/original/01F8MH6NEM8D7527KZAECTCR76.jpeg","preview_url":"http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/small/01F8MH6NEM8D7527KZAECTCR76.jpeg","remote_url":null,"preview_remote_url":null,"meta":{"original":{"width":1200,"height":630,"size":"1200x630","aspect":1.9047619},"small":{"width":256,"height":134,"size":"256x134","aspect":1.9104477},"focus":{"x":0,"y":0}},"description":"Black and white image of some 50's style text saying: Welcome On Board","blurhash":"LNJRdVM{00Rj%Mayt7j[4nWBofRj"}],"mentions":[],"tags":[{"name":"welcome","url":"http://localhost:8080/tags/welcome"}],"emojis":[{"shortcode":"rainbow","url":"http://localhost:8080/fileserver/01AY6P665V14JJR0AFVRT7311Y/emoji/original/01F8MH9H8E4VG3KDYJR9EGPXCQ.png","static_url":"http://localhost:8080/fileserver/01AY6P665V14JJR0AFVRT7311Y/emoji/static/01F8MH9H8E4VG3KDYJR9EGPXCQ.png","visible_in_picker":true,"category":"reactions"}],"card":null,"poll":null,"text":"hello world! #welcome ! first post on the instance :rainbow: !"}`, string(b))
 }
 
+func (suite *InternalToFrontendTestSuite) TestVideoAttachmentToFrontend() {
+	testAttachment := suite.testAttachments["local_account_1_status_4_attachment_2"]
+	apiAttachment, err := suite.typeconverter.AttachmentToAPIAttachment(context.Background(), testAttachment)
+	suite.NoError(err)
+
+	b, err := json.Marshal(apiAttachment)
+	suite.NoError(err)
+
+	suite.Equal(`{"id":"01CDR64G398ADCHXK08WWTHEZ5","type":"video","url":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/attachment/original/01CDR64G398ADCHXK08WWTHEZ5.mp4","text_url":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/attachment/original/01CDR64G398ADCHXK08WWTHEZ5.mp4","preview_url":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/attachment/small/01CDR64G398ADCHXK08WWTHEZ5.jpeg","remote_url":null,"preview_remote_url":null,"meta":{"original":{"width":720,"height":404,"frame_rate":"30/1","duration":15.033334,"bitrate":1206522,"size":"720x404","aspect":1.7821782},"small":{"width":720,"height":404,"size":"720x404","aspect":1.7821782},"focus":{"x":0,"y":0}},"description":"A cow adorably licking another cow!"}`, string(b))
+}
+
 func (suite *InternalToFrontendTestSuite) TestInstanceToFrontend() {
 	testInstance := &gtsmodel.Instance{
 		CreatedAt:        testrig.TimeMustParse("2021-10-20T11:36:45Z"),

testrig/testmodels.go

@@ -60,6 +60,14 @@ func StringPtr(in string) *string {
 	return &in
 }
 
+func Float32Ptr(in float32) *float32 {
+	return &in
+}
+
+func Uint64Ptr(in uint64) *uint64 {
+	return &in
+}
+
 // NewTestTokens returns a map of tokens keyed according to which account the token belongs to.
 func NewTestTokens() map[string]*gtsmodel.Token {
 	tokens := map[string]*gtsmodel.Token{
@@ -772,6 +780,58 @@ func NewTestAttachments() map[string]*gtsmodel.MediaAttachment {
 			Header: FalseBool(),
 			Cached: TrueBool(),
 		},
+		"local_account_1_status_4_attachment_2": {
+			ID:        "01CDR64G398ADCHXK08WWTHEZ5",
+			StatusID:  "01F8MH82FYRXD2RC6108DAJ5HB",
+			URL:       "http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/attachment/original/01CDR64G398ADCHXK08WWTHEZ5.mp4",
+			RemoteURL: "",
+			CreatedAt: TimeMustParse("2022-06-09T13:12:00Z"),
+			UpdatedAt: TimeMustParse("2022-06-09T13:12:00Z"),
+			Type:      gtsmodel.FileTypeVideo,
+			FileMeta: gtsmodel.FileMeta{
+				Original: gtsmodel.Original{
+					Width:     720,
+					Height:    404,
+					Size:      290880,
+					Aspect:    1.78217821782178,
+					Duration:  Float32Ptr(15.033334),
+					Framerate: Float32Ptr(30.0),
+					Bitrate:   Uint64Ptr(1206522),
+				},
+				Small: gtsmodel.Small{
+					Width:  720,
+					Height: 404,
+					Size:   290880,
+					Aspect: 1.78217821782178,
+				},
+				Focus: gtsmodel.Focus{
+					X: 0,
+					Y: 0,
+				},
+			},
+			AccountID:         "01F8MH1H7YV1Z7D2C8K2730QBF",
+			Description:       "A cow adorably licking another cow!",
+			ScheduledStatusID: "",
+			Blurhash:          "",
+			Processing:        2,
+			File: gtsmodel.File{
+				Path:        "01F8MH1H7YV1Z7D2C8K2730QBF/attachment/original/01CDR64G398ADCHXK08WWTHEZ5.gif",
+				ContentType: "video/mp4",
+				FileSize:    2273532,
+				UpdatedAt:   TimeMustParse("2022-06-09T13:12:00Z"),
+			},
+			Thumbnail: gtsmodel.Thumbnail{
+				Path:        "01F8MH1H7YV1Z7D2C8K2730QBF/attachment/small/01CDR64G398ADCHXK08WWTHEZ5.jpeg",
+				ContentType: "image/jpeg",
+				FileSize:    5272,
+				UpdatedAt:   TimeMustParse("2022-06-09T13:12:00Z"),
+				URL:         "http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/attachment/small/01CDR64G398ADCHXK08WWTHEZ5.jpeg",
+				RemoteURL:   "",
+			},
+			Avatar: FalseBool(),
+			Header: FalseBool(),
+			Cached: TrueBool(),
+		},
 		"local_account_1_unattached_1": {
 			ID:        "01F8MH8RMYQ6MSNY3JM2XT1CQ5",
 			StatusID:  "", // this attachment isn't connected to a status YET
@@ -1209,6 +1269,10 @@ func newTestStoredAttachments() map[string]filenames {
 			Original: "trent-original.gif",
 			Small:    "trent-small.jpeg",
 		},
+		"local_account_1_status_4_attachment_2": {
+			Original: "cowlick-original.mp4",
+			Small:    "cowlick-small.jpeg",
+		},
 		"local_account_1_unattached_1": {
 			Original: "ohyou-original.jpeg",
 			Small:    "ohyou-small.jpeg",
@@ -1434,9 +1498,9 @@ func NewTestStatuses() map[string]*gtsmodel.Status {
 			ID:                       "01F8MH82FYRXD2RC6108DAJ5HB",
 			URI:                      "http://localhost:8080/users/the_mighty_zork/statuses/01F8MH82FYRXD2RC6108DAJ5HB",
 			URL:                      "http://localhost:8080/@the_mighty_zork/statuses/01F8MH82FYRXD2RC6108DAJ5HB",
-			Content:                  "here's a little gif of trent",
+			Content:                  "here's a little gif of trent.... and also a cow",
-			Text:                     "here's a little gif of trent",
+			Text:                     "here's a little gif of trent.... and also a cow",
-			AttachmentIDs:            []string{"01F8MH7TDVANYKWVE8VVKFPJTJ"},
+			AttachmentIDs:            []string{"01F8MH7TDVANYKWVE8VVKFPJTJ", "01CDR64G398ADCHXK08WWTHEZ5"},
 			CreatedAt:                TimeMustParse("2021-10-20T12:40:37+02:00"),
 			UpdatedAt:                TimeMustParse("2021-10-20T12:40:37+02:00"),
 			Local:                    TrueBool(),
@@ -1444,7 +1508,7 @@ func NewTestStatuses() map[string]*gtsmodel.Status {
 			AccountID:                "01F8MH1H7YV1Z7D2C8K2730QBF",
 			InReplyToID:              "",
 			BoostOfID:                "",
-			ContentWarning:           "eye contact, trent reznor gif",
+			ContentWarning:           "eye contact, trent reznor gif, cow",
 			Visibility:               gtsmodel.VisibilityMutualsOnly,
 			Sensitive:                FalseBool(),
 			Language:                 "en",