From 368c97f0f85f243796a0407960dc5a2ccad24bab Mon Sep 17 00:00:00 2001
From: kim <89579420+NyaaaWhatsUpDoc@users.noreply.github.com>
Date: Sun, 28 Jul 2024 19:10:41 +0000
Subject: [PATCH] [bugfix] take into account rotation when generating thumbnail
 (#3147)

* take into account rotation when generating thumbnail, simplify ffprobe output to show only fields we need

* only show rotation side data

* remove unnecessary comment

* fix code comments

* remove debug logging
---
 internal/media/ffmpeg.go          | 86 +++++++++++++++++++++++++++----
 internal/media/manager_test.go    |  4 +-
 internal/media/processingmedia.go |  7 +--
 internal/media/util.go            | 25 ++++++---
 4 files changed, 102 insertions(+), 20 deletions(-)

diff --git a/internal/media/ffmpeg.go b/internal/media/ffmpeg.go
index ad76e5198..1cfaf891c 100644
--- a/internal/media/ffmpeg.go
+++ b/internal/media/ffmpeg.go
@@ -52,6 +52,8 @@ func ffmpegClearMetadata(ctx context.Context, filepath string) error {
 
 	// Clear metadata with ffmpeg.
 	if err := ffmpeg(ctx, dirpath,
+
+		// Only log errors.
 		"-loglevel", "error",
 
 		// Input file path.
@@ -101,6 +103,8 @@ func ffmpegGenerateThumb(ctx context.Context, filepath string, width, height int
 
 	// Generate thumb with ffmpeg.
 	if err := ffmpeg(ctx, dirpath,
+
+		// Only log errors.
 		"-loglevel", "error",
 
 		// Input file.
@@ -158,6 +162,8 @@ func ffmpegGenerateStatic(ctx context.Context, filepath string) (string, error)
 
 	// Generate static with ffmpeg.
 	if err := ffmpeg(ctx, dirpath,
+
+		// Only log errors.
 		"-loglevel", "error",
 
 		// Input file.
@@ -216,12 +222,29 @@ func ffprobe(ctx context.Context, filepath string) (*result, error) {
 		Stdout: &stdout,
 
 		Args: []string{
-			"-i", filepath,
+			// Don't show any excess logging
+			// information, all goes in JSON.
 			"-loglevel", "quiet",
+
+			// Print in compact JSON format.
 			"-print_format", "json=compact=1",
-			"-show_streams",
-			"-show_format",
+
+			// Show error in our
+			// chosen format type.
 			"-show_error",
+
+			// Show specifically container format, total duration and bitrate.
+			"-show_entries", "format=format_name,duration,bit_rate" + ":" +
+
+				// Show specifically stream codec names, types, frame rate, duration and dimens.
+				"stream=codec_name,codec_type,r_frame_rate,duration_ts,width,height" + ":" +
+
+				// Show any rotation
+				// side data stored.
+				"side_data=rotation",
+
+			// Input file.
+			"-i", filepath,
 		},
 
 		Config: func(modcfg wazero.ModuleConfig) wazero.ModuleConfig {
@@ -257,8 +280,9 @@ type result struct {
 	format   string
 	audio    []audioStream
 	video    []videoStream
-	bitrate  uint64
 	duration float64
+	bitrate  uint64
+	rotation int
 }
 
 type stream struct {
@@ -456,15 +480,61 @@ func (res *ffprobeResult) Process() (*result, error) {
 		}
 	}
 
+	// Check extra packet / frame information
+	// for provided orientation (not always set).
+	for _, pf := range res.PacketsAndFrames {
+		for _, d := range pf.SideDataList {
+
+			// Ensure frame side
+			// data IS rotation data.
+			if d.Rotation == 0 {
+				continue
+			}
+
+			// Ensure rotation not
+			// already been specified.
+			if r.rotation != 0 {
+				return nil, errors.New("multiple sets of rotation data")
+			}
+
+			// Drop any decimal
+			// rotation value.
+			rot := int(d.Rotation)
+
+			// Round rotation to multiple of 90.
+			// More granularity is not needed.
+			if q := rot % 90; q > 45 {
+				rot += (90 - q)
+			} else {
+				rot -= q
+			}
+
+			// Drop any value above 360
+			// or below -360, these are
+			// just repeat full turns.
+			r.rotation = (rot % 360)
+		}
+	}
+
 	return &r, nil
 }
 
 // ffprobeResult contains parsed JSON data from
 // result of calling `ffprobe` on a media file.
 type ffprobeResult struct {
-	Streams []ffprobeStream `json:"streams"`
-	Format  *ffprobeFormat  `json:"format"`
-	Error   *ffprobeError   `json:"error"`
+	PacketsAndFrames []ffprobePacketOrFrame `json:"packets_and_frames"`
+	Streams          []ffprobeStream        `json:"streams"`
+	Format           *ffprobeFormat         `json:"format"`
+	Error            *ffprobeError          `json:"error"`
+}
+
+type ffprobePacketOrFrame struct {
+	Type         string            `json:"type"`
+	SideDataList []ffprobeSideData `json:"side_data_list"`
+}
+
+type ffprobeSideData struct {
+	Rotation float64 `json:"rotation"`
 }
 
 type ffprobeStream struct {
@@ -474,14 +544,12 @@ type ffprobeStream struct {
 	DurationTS uint   `json:"duration_ts"`
 	Width      int    `json:"width"`
 	Height     int    `json:"height"`
-	// + unused fields.
 }
 
 type ffprobeFormat struct {
 	FormatName string `json:"format_name"`
 	Duration   string `json:"duration"`
 	BitRate    string `json:"bit_rate"`
-	// + unused fields
 }
 
 type ffprobeError struct {
diff --git a/internal/media/manager_test.go b/internal/media/manager_test.go
index 26b103908..68de74dd6 100644
--- a/internal/media/manager_test.go
+++ b/internal/media/manager_test.go
@@ -483,7 +483,7 @@ func (suite *ManagerTestSuite) TestLongerMp4Process() {
 	suite.EqualValues(float32(10), *attachment.FileMeta.Original.Framerate)
 	suite.EqualValues(0xce3a, *attachment.FileMeta.Original.Bitrate)
 	suite.EqualValues(gtsmodel.Small{
-		Width: 512, Height: 281, Size: 143872, Aspect: 1.822064,
+		Width: 512, Height: 281, Size: 143872, Aspect: 1.8181819,
 	}, attachment.FileMeta.Small)
 	suite.Equal("video/mp4", attachment.File.ContentType)
 	suite.Equal("image/webp", attachment.Thumbnail.ContentType)
@@ -543,7 +543,7 @@ func (suite *ManagerTestSuite) TestBirdnestMp4Process() {
 	suite.EqualValues(float32(30), *attachment.FileMeta.Original.Framerate)
 	suite.EqualValues(0x11844c, *attachment.FileMeta.Original.Bitrate)
 	suite.EqualValues(gtsmodel.Small{
-		Width: 287, Height: 512, Size: 146944, Aspect: 0.5605469,
+		Width: 287, Height: 512, Size: 146944, Aspect: 0.5611111,
 	}, attachment.FileMeta.Small)
 	suite.Equal("video/mp4", attachment.File.ContentType)
 	suite.Equal("image/webp", attachment.Thumbnail.ContentType)
diff --git a/internal/media/processingmedia.go b/internal/media/processingmedia.go
index 32c0531bc..b68d8d680 100644
--- a/internal/media/processingmedia.go
+++ b/internal/media/processingmedia.go
@@ -176,10 +176,11 @@ func (p *ProcessingMedia) store(ctx context.Context) error {
 	// This will always be used regardless of type,
 	// as even audio files may contain embedded album art.
 	width, height, framerate := result.ImageMeta()
+	aspect := util.Div(float32(width), float32(height))
 	p.media.FileMeta.Original.Width = width
 	p.media.FileMeta.Original.Height = height
 	p.media.FileMeta.Original.Size = (width * height)
-	p.media.FileMeta.Original.Aspect = util.Div(float32(width), float32(height))
+	p.media.FileMeta.Original.Aspect = aspect
 	p.media.FileMeta.Original.Framerate = util.PtrIf(framerate)
 	p.media.FileMeta.Original.Duration = util.PtrIf(float32(result.duration))
 	p.media.FileMeta.Original.Bitrate = util.PtrIf(result.bitrate)
@@ -218,11 +219,11 @@ func (p *ProcessingMedia) store(ctx context.Context) error {
 
 	if width > 0 && height > 0 {
 		// Determine thumbnail dimensions to use.
-		thumbWidth, thumbHeight := thumbSize(width, height)
+		thumbWidth, thumbHeight := thumbSize(width, height, aspect, result.rotation)
 		p.media.FileMeta.Small.Width = thumbWidth
 		p.media.FileMeta.Small.Height = thumbHeight
 		p.media.FileMeta.Small.Size = (thumbWidth * thumbHeight)
-		p.media.FileMeta.Small.Aspect = float32(thumbWidth) / float32(thumbHeight)
+		p.media.FileMeta.Small.Aspect = aspect
 
 		// Generate a thumbnail image from input image path.
 		thumbpath, err = ffmpegGenerateThumb(ctx, temppath,
diff --git a/internal/media/util.go b/internal/media/util.go
index b643cd9c8..dd445844d 100644
--- a/internal/media/util.go
+++ b/internal/media/util.go
@@ -37,12 +37,23 @@ import (
 
 // thumbSize returns the dimensions to use for an input
 // image of given width / height, for its outgoing thumbnail.
-// This maintains the original image aspect ratio.
-func thumbSize(width, height int) (int, int) {
+// This attempts to maintains the original image aspect ratio.
+func thumbSize(width, height int, aspect float32, rotation int) (int, int) {
 	const (
 		maxThumbWidth  = 512
 		maxThumbHeight = 512
 	)
+
+	// If image is rotated by
+	// any odd multiples of 90,
+	// flip width / height to
+	// get the correct scale.
+	switch rotation {
+	case -90, 90, -270, 270:
+		width, height = height, width
+		aspect = 1 / aspect
+	}
+
 	switch {
 	// Simplest case, within bounds!
 	case width < maxThumbWidth &&
@@ -51,13 +62,15 @@ func thumbSize(width, height int) (int, int) {
 
 	// Width is larger side.
 	case width > height:
-		p := float32(width) / float32(maxThumbWidth)
-		return maxThumbWidth, int(float32(height) / p)
+		// i.e. height = newWidth * (height / width)
+		height = int(float32(maxThumbWidth) / aspect)
+		return maxThumbWidth, height
 
 	// Height is larger side.
 	case height > width:
-		p := float32(height) / float32(maxThumbHeight)
-		return int(float32(width) / p), maxThumbHeight
+		// i.e. width = newHeight * (width / height)
+		width = int(float32(maxThumbHeight) * aspect)
+		return width, maxThumbHeight
 
 	// Square.
 	default: