mirrors
/
gotosocial
mirror of
1
Fork 0
Code Issues Releases Activity

[feature] more filetype support! (#3107) 

* add more supported file types to our media processor that ffmpeg supports, update supported mime type lists

* add code comments to the supported mime types slice

* don't check for zero value string, just parse

* remove some unneeded consts which make the code a bit harder to read

* fix test expected instance media mime types, use compact ffprobe json, simple media processing by type

* final tweaks to media processing code

* don't use safe divide where we don't need to
This commit is contained in:
kim 2024-07-15 14:24:53 +00:00 committed by GitHub
parent 9efb11d848
commit de45c0be60
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
12 changed files with 495 additions and 351 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
cmd/process-emoji/main.go
View File

@ -29,6 +29,7 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/db/bundb"
"github.com/superseriousbusiness/gotosocial/internal/log"
"github.com/superseriousbusiness/gotosocial/internal/media"
"github.com/superseriousbusiness/gotosocial/internal/media/ffmpeg"
"github.com/superseriousbusiness/gotosocial/internal/state"
"github.com/superseriousbusiness/gotosocial/internal/storage"
"github.com/superseriousbusiness/gotosocial/internal/util"
@ -43,6 +44,14 @@ func main() {
log.Panic(ctx, "Usage: go run ./cmd/process-emoji <input-file> <output-static>")
}
if err := ffmpeg.InitFfprobe(ctx, 1); err != nil {
log.Panic(ctx, err)
}
if err := ffmpeg.InitFfmpeg(ctx, 1); err != nil {
log.Panic(ctx, err)
}
var st storage.Driver
st.Storage = memory.Open(10, true)

12
cmd/process-media/main.go
View File

@ -29,6 +29,7 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/db/bundb"
"github.com/superseriousbusiness/gotosocial/internal/log"
"github.com/superseriousbusiness/gotosocial/internal/media"
"github.com/superseriousbusiness/gotosocial/internal/media/ffmpeg"
"github.com/superseriousbusiness/gotosocial/internal/state"
"github.com/superseriousbusiness/gotosocial/internal/storage"
)
@ -42,6 +43,14 @@ func main() {
log.Panic(ctx, "Usage: go run ./cmd/process-media <input-file> <output-processed> <output-thumbnail>")
}
if err := ffmpeg.InitFfprobe(ctx, 1); err != nil {
log.Panic(ctx, err)
}
if err := ffmpeg.InitFfmpeg(ctx, 1); err != nil {
log.Panic(ctx, err)
}
var st storage.Driver
st.Storage = memory.Open(10, true)
@ -105,6 +114,9 @@ func main() {
func copyFile(ctx context.Context, st *storage.Driver, key string, path string) {
rc, err := st.GetStream(ctx, key)
if err != nil {
if storage.IsNotFound(err) {
return
}
log.Panic(ctx, err)
}
defer rc.Close()

102
internal/api/client/instance/instancepatch_test.go
View File

@ -105,9 +105,22 @@ func (suite *InstancePatchTestSuite) TestInstancePatch1() {
"supported_mime_types": [
"image/jpeg",
"image/gif",
"image/png",
"image/webp",
"video/mp4"
"audio/mp2",
"audio/mp3",
"video/x-msvideo",
"image/png",
"image/apng",
"audio/ogg",
"video/ogg",
"audio/x-m4a",
"video/mp4",
"video/quicktime",
"audio/x-ms-wma",
"video/x-ms-wmv",
"video/webm",
"audio/x-matroska",
"video/x-matroska"
],
"image_size_limit": 41943040,
"image_matrix_limit": 16777216,
@ -226,9 +239,22 @@ func (suite *InstancePatchTestSuite) TestInstancePatch2() {
"supported_mime_types": [
"image/jpeg",
"image/gif",
"image/png",
"image/webp",
"video/mp4"
"audio/mp2",
"audio/mp3",
"video/x-msvideo",
"image/png",
"image/apng",
"audio/ogg",
"video/ogg",
"audio/x-m4a",
"video/mp4",
"video/quicktime",
"audio/x-ms-wma",
"video/x-ms-wmv",
"video/webm",
"audio/x-matroska",
"video/x-matroska"
],
"image_size_limit": 41943040,
"image_matrix_limit": 16777216,
@ -347,9 +373,22 @@ func (suite *InstancePatchTestSuite) TestInstancePatch3() {
"supported_mime_types": [
"image/jpeg",
"image/gif",
"image/png",
"image/webp",
"video/mp4"
"audio/mp2",
"audio/mp3",
"video/x-msvideo",
"image/png",
"image/apng",
"audio/ogg",
"video/ogg",
"audio/x-m4a",
"video/mp4",
"video/quicktime",
"audio/x-ms-wma",
"video/x-ms-wmv",
"video/webm",
"audio/x-matroska",
"video/x-matroska"
],
"image_size_limit": 41943040,
"image_matrix_limit": 16777216,
@ -519,9 +558,22 @@ func (suite *InstancePatchTestSuite) TestInstancePatch6() {
"supported_mime_types": [
"image/jpeg",
"image/gif",
"image/png",
"image/webp",
"video/mp4"
"audio/mp2",
"audio/mp3",
"video/x-msvideo",
"image/png",
"image/apng",
"audio/ogg",
"video/ogg",
"audio/x-m4a",
"video/mp4",
"video/quicktime",
"audio/x-ms-wma",
"video/x-ms-wmv",
"video/webm",
"audio/x-matroska",
"video/x-matroska"
],
"image_size_limit": 41943040,
"image_matrix_limit": 16777216,
@ -662,9 +714,22 @@ func (suite *InstancePatchTestSuite) TestInstancePatch8() {
"supported_mime_types": [
"image/jpeg",
"image/gif",
"image/png",
"image/webp",
"video/mp4"
"audio/mp2",
"audio/mp3",
"video/x-msvideo",
"image/png",
"image/apng",
"audio/ogg",
"video/ogg",
"audio/x-m4a",
"video/mp4",
"video/quicktime",
"audio/x-ms-wma",
"video/x-ms-wmv",
"video/webm",
"audio/x-matroska",
"video/x-matroska"
],
"image_size_limit": 41943040,
"image_matrix_limit": 16777216,
@ -820,9 +885,22 @@ func (suite *InstancePatchTestSuite) TestInstancePatch9() {
"supported_mime_types": [
"image/jpeg",
"image/gif",
"image/png",
"image/webp",
"video/mp4"
"audio/mp2",
"audio/mp3",
"video/x-msvideo",
"image/png",
"image/apng",
"audio/ogg",
"video/ogg",
"audio/x-m4a",
"video/mp4",
"video/quicktime",
"audio/x-ms-wma",
"video/x-ms-wmv",
"video/webm",
"audio/x-matroska",
"video/x-matroska"
],
"image_size_limit": 41943040,
"image_matrix_limit": 16777216,

375
internal/media/ffmpeg.go
View File

@ -18,7 +18,6 @@
package media
import (
"cmp"
"context"
"encoding/json"
"errors"
@ -135,7 +134,7 @@ func ffmpeg(ctx context.Context, dirpath string, args ...string) error {
}
// ffprobe calls `ffprobe` (WASM) on filepath, returning parsed JSON output.
func ffprobe(ctx context.Context, filepath string) (*ffprobeResult, error) {
func ffprobe(ctx context.Context, filepath string) (*result, error) {
var stdout byteutil.Buffer
// Get directory from filepath.
@ -148,7 +147,7 @@ func ffprobe(ctx context.Context, filepath string) (*ffprobeResult, error) {
Args: []string{
"-i", filepath,
"-loglevel", "quiet",
"-print_format", "json",
"-print_format", "json=compact=1",
"-show_streams",
"-show_format",
"-show_error",
@ -172,7 +171,219 @@ func ffprobe(ctx context.Context, filepath string) (*ffprobeResult, error) {
return nil, gtserror.Newf("error unmarshaling json: %w", err)
}
return &result, nil
// Convert raw result data.
res, err := result.Process()
if err != nil {
return nil, err
}
return res, nil
}
// result contains parsed ffprobe result
// data in a more useful data format.
type result struct {
format string
audio []audioStream
video []videoStream
bitrate uint64
duration float64
}
type stream struct {
codec string
}
type audioStream struct {
stream
}
type videoStream struct {
stream
width int
height int
framerate float32
}
// GetFileType determines file type and extension to use for media data. This
// function helps to abstract away the horrible complexities that are possible
// media container (i.e. the file) types and and possible sub-types within that.
//
// Note the checks for (len(res.video) > 0) may catch some audio files with embedded
// album art as video, but i blame that on the hellscape that is media filetypes.
//
// TODO: we can update this code to also return a mimetype and avoid later parsing!
func (res *result) GetFileType() (gtsmodel.FileType, string) {
switch res.format {
case "mpeg":
return gtsmodel.FileTypeVideo, "mpeg"
case "mjpeg":
return gtsmodel.FileTypeVideo, "mjpeg"
case "mov,mp4,m4a,3gp,3g2,mj2":
switch {
case len(res.video) > 0:
return gtsmodel.FileTypeVideo, "mp4"
case len(res.audio) > 0 &&
res.audio[0].codec == "aac":
// m4a only supports [aac] audio.
return gtsmodel.FileTypeAudio, "m4a"
}
case "apng":
return gtsmodel.FileTypeImage, "apng"
case "png_pipe":
return gtsmodel.FileTypeImage, "png"
case "image2", "image2pipe", "jpeg_pipe":
return gtsmodel.FileTypeImage, "jpeg"
case "webp", "webp_pipe":
return gtsmodel.FileTypeImage, "webp"
case "gif":
return gtsmodel.FileTypeImage, "gif"
case "mp3":
if len(res.audio) > 0 {
switch res.audio[0].codec {
case "mp2":
return gtsmodel.FileTypeAudio, "mp2"
case "mp3":
return gtsmodel.FileTypeAudio, "mp3"
}
}
case "asf":
switch {
case len(res.video) > 0:
return gtsmodel.FileTypeVideo, "wmv"
case len(res.audio) > 0:
return gtsmodel.FileTypeAudio, "wma"
}
case "ogg":
switch {
case len(res.video) > 0:
return gtsmodel.FileTypeVideo, "ogv"
case len(res.audio) > 0:
return gtsmodel.FileTypeAudio, "ogg"
}
case "matroska,webm":
switch {
case len(res.video) > 0:
switch res.video[0].codec {
case "vp8", "vp9", "av1":
default:
return gtsmodel.FileTypeVideo, "mkv"
}
if len(res.audio) > 0 {
switch res.audio[0].codec {
case "vorbis", "opus", "libopus":
// webm only supports [VP8/VP9/AV1]+[vorbis/opus]
return gtsmodel.FileTypeVideo, "webm"
}
}
case len(res.audio) > 0:
return gtsmodel.FileTypeAudio, "mka"
}
case "avi":
return gtsmodel.FileTypeVideo, "avi"
}
return gtsmodel.FileTypeUnknown, res.format
}
// ImageMeta extracts image metadata contained within ffprobe'd media result streams.
func (res *result) ImageMeta() (width int, height int, framerate float32) {
for _, stream := range res.video {
if stream.width > width {
width = stream.width
}
if stream.height > height {
height = stream.height
}
if fr := float32(stream.framerate); fr > 0 {
if framerate == 0 || fr < framerate {
framerate = fr
}
}
}
return
}
// Process converts raw ffprobe result data into our more usable result{} type.
func (res *ffprobeResult) Process() (*result, error) {
if res.Error != nil {
return nil, res.Error
}
if res.Format == nil {
return nil, errors.New("missing format data")
}
var r result
var err error
// Copy over container format.
r.format = res.Format.FormatName
// Parsed media bitrate (if it was set).
if str := res.Format.BitRate; str != "" {
r.bitrate, err = strconv.ParseUint(str, 10, 64)
if err != nil {
return nil, gtserror.Newf("invalid bitrate %s: %w", str, err)
}
}
// Parse media duration (if it was set).
if str := res.Format.Duration; str != "" {
r.duration, err = strconv.ParseFloat(str, 32)
if err != nil {
return nil, gtserror.Newf("invalid duration %s: %w", str, err)
}
}
// Preallocate streams to max possible lengths.
r.audio = make([]audioStream, 0, len(res.Streams))
r.video = make([]videoStream, 0, len(res.Streams))
// Convert streams to separate types.
for _, s := range res.Streams {
switch s.CodecType {
case "audio":
// Append audio stream data to result.
r.audio = append(r.audio, audioStream{
stream: stream{codec: s.CodecName},
})
case "video":
var framerate float32
// Parse stream framerate, bearing in
// mind that some static container formats
// (e.g. jpeg) still return a framerate, so
// we also check for a non-1 timebase (dts).
if str := s.RFrameRate; str != "" &&
s.DurationTS > 1 {
var num, den uint32
den = 1
// Check for inequality (numerator / denominator).
if p := strings.SplitN(str, "/", 2); len(p) == 2 {
n, _ := strconv.ParseUint(p[0], 10, 32)
d, _ := strconv.ParseUint(p[1], 10, 32)
num, den = uint32(n), uint32(d)
} else {
n, _ := strconv.ParseUint(p[0], 10, 32)
num = uint32(n)
}
// Set final divised framerate.
framerate = float32(num / den)
}
// Append video stream data to result.
r.video = append(r.video, videoStream{
stream: stream{codec: s.CodecName},
width: s.Width,
height: s.Height,
framerate: framerate,
})
}
}
return &r, nil
}
// ffprobeResult contains parsed JSON data from
@ -183,175 +394,33 @@ type ffprobeResult struct {
Error *ffprobeError `json:"error"`
}
// ImageMeta extracts image metadata contained within ffprobe'd media result streams.
func (res *ffprobeResult) ImageMeta() (width int, height int, err error) {
for _, stream := range res.Streams {
if stream.Width > width {
width = stream.Width
}
if stream.Height > height {
height = stream.Height
}
}
if width == 0 || height == 0 {
err = errors.New("invalid image stream(s)")
}
return
}
// EmbeddedImageMeta extracts embedded image metadata contained within ffprobe'd media result
// streams, should be used for pulling album image (can be animated image) from audio files.
func (res *ffprobeResult) EmbeddedImageMeta() (width int, height int, framerate float32, err error) {
for _, stream := range res.Streams {
if stream.Width > width {
width = stream.Width
}
if stream.Height > height {
height = stream.Height
}
if fr := stream.GetFrameRate(); fr > 0 {
if framerate == 0 || fr < framerate {
framerate = fr
}
}
}
// Need width + height but
// no framerate is fine.
if width == 0 || height == 0 {
err = errors.New("invalid image stream(s)")
}
return
}
// VideoMeta extracts video metadata contained within ffprobe'd media result streams.
func (res *ffprobeResult) VideoMeta() (width, height int, framerate float32, err error) {
for _, stream := range res.Streams {
if stream.Width > width {
width = stream.Width
}
if stream.Height > height {
height = stream.Height
}
if fr := stream.GetFrameRate(); fr > 0 {
if framerate == 0 || fr < framerate {
framerate = fr
}
}
}
if width == 0 || height == 0 || framerate == 0 {
err = errors.New("invalid video stream(s)")
}
return
}
type ffprobeStream struct {
CodecName string `json:"codec_name"`
AvgFrameRate string `json:"avg_frame_rate"`
CodecType string `json:"codec_type"`
RFrameRate string `json:"r_frame_rate"`
DurationTS uint `json:"duration_ts"`
Width int `json:"width"`
Height int `json:"height"`
// + unused fields.
}
// GetFrameRate calculates float32 framerate value from stream json string.
func (str *ffprobeStream) GetFrameRate() float32 {
numDen := func(strFR string) (float32, float32) {
var (
// numerator
num float32
// denominator
den float32
)
// Check for a provided inequality, i.e. numerator / denominator.
if p := strings.SplitN(strFR, "/", 2); len(p) == 2 {
n, _ := strconv.ParseFloat(p[0], 32)
d, _ := strconv.ParseFloat(p[1], 32)
num, den = float32(n), float32(d)
} else {
n, _ := strconv.ParseFloat(p[0], 32)
num = float32(n)
}
return num, den
}
var num, den float32
if str.AvgFrameRate != "" {
// Check if we have avg_frame_rate.
num, den = numDen(str.AvgFrameRate)
}
if num == 0 && str.RFrameRate != "" {
// Check if we have r_frame_rate.
num, den = numDen(str.RFrameRate)
}
if num != 0 {
// Found it.
// Avoid divide by zero.
return num / cmp.Or(den, 1)
}
return 0
}
type ffprobeFormat struct {
Filename string `json:"filename"`
FormatName string `json:"format_name"`
Duration string `json:"duration"`
BitRate string `json:"bit_rate"`
// + unused fields
}
// GetFileType determines file type and extension to use for media data.
func (fmt *ffprobeFormat) GetFileType() (gtsmodel.FileType, string) {
switch fmt.FormatName {
case "mov,mp4,m4a,3gp,3g2,mj2":
return gtsmodel.FileTypeVideo, "mp4"
case "apng":
return gtsmodel.FileTypeImage, "apng"
case "png_pipe":
return gtsmodel.FileTypeImage, "png"
case "image2", "jpeg_pipe":
return gtsmodel.FileTypeImage, "jpeg"
case "webp_pipe":
return gtsmodel.FileTypeImage, "webp"
case "gif":
return gtsmodel.FileTypeImage, "gif"
case "mp3":
return gtsmodel.FileTypeAudio, "mp3"
case "ogg":
return gtsmodel.FileTypeAudio, "ogg"
default:
return gtsmodel.FileTypeUnknown, fmt.FormatName
}
}
// GetDuration calculates float32 framerate value from format json string.
func (fmt *ffprobeFormat) GetDuration() float32 {
if fmt.Duration != "" {
dur, _ := strconv.ParseFloat(fmt.Duration, 32)
return float32(dur)
}
return 0
}
// GetBitRate calculates uint64 bitrate value from format json string.
func (fmt *ffprobeFormat) GetBitRate() uint64 {
if fmt.BitRate != "" {
r, _ := strconv.ParseUint(fmt.BitRate, 10, 64)
return r
}
return 0
}
type ffprobeError struct {
Code int `json:"code"`
String string `json:"string"`
}
func isUnsupportedTypeErr(err error) bool {
ffprobeErr, ok := err.(*ffprobeError)
return ok && ffprobeErr.Code == -1094995529
}
func (err *ffprobeError) Error() string {
return err.String + " (" + strconv.Itoa(err.Code) + ")"
}

65
internal/media/manager.go
View File

@ -34,17 +34,46 @@ import (
)
var SupportedMIMETypes = []string{
mimeImageJpeg,
mimeImageGif,
mimeImagePng,
mimeImageWebp,
mimeVideoMp4,
"image/jpeg", // .jpeg
"image/gif", // .gif
"image/webp", // .webp
"audio/mp2", // .mp2
"audio/mp3", // .mp3
"video/x-msvideo", // .avi
// png types
"image/png", // .png
"image/apng", // .apng
// ogg types
"audio/ogg", // .ogg
"video/ogg", // .ogv
// mpeg4 types
"audio/x-m4a", // .m4a
"video/mp4", // .mp4
"video/quicktime", // .mov
// asf types
"audio/x-ms-wma", // .wma
"video/x-ms-wmv", // .wmv
// matroska types
"video/webm", // .webm
"audio/x-matroska", // .mka
"video/x-matroska", // .mkv
}
var SupportedEmojiMIMETypes = []string{
mimeImageGif,
mimeImagePng,
mimeImageWebp,
"image/jpeg", // .jpeg
"image/gif", // .gif
"image/webp", // .webp
// png types
"image/png", // .png
"image/apng", // .apng
}
type Manager struct {
@ -102,8 +131,8 @@ func (m *Manager) CreateMedia(
id,
// Always encode attachment
// thumbnails as jpg.
"jpg",
// thumbnails as jpeg.
"jpeg",
)
// Calculate attachment thumbnail URL.
@ -114,8 +143,8 @@ func (m *Manager) CreateMedia(
id,
// Always encode attachment
// thumbnails as jpg.
"jpg",
// thumbnails as jpeg.
"jpeg",
)
// Populate initial fields on the new media,
@ -134,7 +163,7 @@ func (m *Manager) CreateMedia(
Path: path,
},
Thumbnail: gtsmodel.Thumbnail{
ContentType: mimeImageJpeg, // thumbs always jpg.
ContentType: "image/jpeg",
Path: thumbPath,
URL: thumbURL,
},
@ -244,7 +273,7 @@ func (m *Manager) CreateEmoji(
// All static emojis
// are encoded as png.
mimePng,
"png",
)
// Generate static image path for attachment.
@ -256,7 +285,7 @@ func (m *Manager) CreateEmoji(
// All static emojis
// are encoded as png.
mimePng,
"png",
)
// Populate initial fields on the new emoji,
@ -268,7 +297,7 @@ func (m *Manager) CreateEmoji(
Domain: domain,
ImageStaticURL: staticURL,
ImageStaticPath: staticPath,
ImageStaticContentType: mimeImagePng,
ImageStaticContentType: "image/png",
Disabled: util.Ptr(false),
VisibleInPicker: util.Ptr(true),
CreatedAt: now,
@ -368,7 +397,7 @@ func (m *Manager) RefreshEmoji(
// All static emojis
// are encoded as png.
mimePng,
"png",
)
// Generate new static image storage path for emoji.
@ -380,7 +409,7 @@ func (m *Manager) RefreshEmoji(
// All static emojis
// are encoded as png.
mimePng,
"png",
)
// Finally, create new emoji in database.

2
internal/media/manager_test.go
View File

@ -421,7 +421,7 @@ func (suite *ManagerTestSuite) TestSlothVineProcess() {
suite.Equal(81120, attachment.FileMeta.Original.Size)
suite.EqualValues(float32(1.4083333), attachment.FileMeta.Original.Aspect)
suite.EqualValues(float32(6.641), *attachment.FileMeta.Original.Duration)
suite.EqualValues(float32(29.00003), *attachment.FileMeta.Original.Framerate)
suite.EqualValues(float32(29), *attachment.FileMeta.Original.Framerate)
suite.EqualValues(0x5be18, *attachment.FileMeta.Original.Bitrate)
suite.EqualValues(gtsmodel.Small{
Width: 338, Height: 240, Size: 81120, Aspect: 1.4083333333333334,

20
internal/media/processingemoji.go
View File

@ -160,27 +160,17 @@ func (p *ProcessingEmoji) store(ctx context.Context) error {
// Pass input file through ffprobe to
// parse further metadata information.
result, err := ffprobe(ctx, temppath)
if err != nil {
return gtserror.Newf("error ffprobing data: %w", err)
}
switch {
// No errors parsing data.
case result.Error == nil:
// Data type unhandleable by ffprobe.
case result.Error.Code == -1094995529:
if err != nil && !isUnsupportedTypeErr(err) {
return gtserror.Newf("ffprobe error: %w", err)
} else if result == nil {
log.Warn(ctx, "unsupported data type")
return nil
default:
return gtserror.Newf("ffprobe error: %w", err)
}
var ext string
// Set media type from ffprobe format data.
fileType, ext := result.Format.GetFileType()
// Get type from ffprobe format data.
fileType, ext := result.GetFileType()
if fileType != gtsmodel.FileTypeImage {
return gtserror.Newf("unsupported emoji filetype: %s (%s)", fileType, ext)
}

155
internal/media/processingmedia.go
View File

@ -180,36 +180,33 @@ func (p *ProcessingMedia) store(ctx context.Context) error {
// Pass input file through ffprobe to
// parse further metadata information.
result, err := ffprobe(ctx, temppath)
if err != nil {
return gtserror.Newf("error ffprobing data: %w", err)
}
switch {
// No errors parsing data.
case result.Error == nil:
// Data type unhandleable by ffprobe.
case result.Error.Code == -1094995529:
if err != nil && !isUnsupportedTypeErr(err) {
return gtserror.Newf("ffprobe error: %w", err)
} else if result == nil {
log.Warn(ctx, "unsupported data type")
return nil
default:
return gtserror.Newf("ffprobe error: %w", err)
}
var ext string
// Set the media type from ffprobe format data.
p.media.Type, ext = result.Format.GetFileType()
if p.media.Type == gtsmodel.FileTypeUnknown {
// Return early (deleting file)
// for unhandled file types.
return nil
}
// Extract any video stream metadata from media.
// This will always be used regardless of type,
// as even audio files may contain embedded album art.
width, height, framerate := result.ImageMeta()
p.media.FileMeta.Original.Width = width
p.media.FileMeta.Original.Height = height
p.media.FileMeta.Original.Size = (width * height)
p.media.FileMeta.Original.Aspect = util.Div(float32(width), float32(height))
p.media.FileMeta.Original.Framerate = util.PtrIf(framerate)
p.media.FileMeta.Original.Duration = util.PtrIf(float32(result.duration))
p.media.FileMeta.Original.Bitrate = util.PtrIf(result.bitrate)
// Set media type from ffprobe format data.
p.media.Type, ext = result.GetFileType()
switch p.media.Type {
case gtsmodel.FileTypeImage:
case gtsmodel.FileTypeImage,
gtsmodel.FileTypeVideo:
// Pass file through ffmpeg clearing
// any excess metadata (e.g. EXIF).
if err := ffmpegClearMetadata(ctx,
@ -218,96 +215,16 @@ func (p *ProcessingMedia) store(ctx context.Context) error {
return gtserror.Newf("error cleaning metadata: %w", err)
}
// Extract image metadata from streams.
width, height, err := result.ImageMeta()
if err != nil {
return err
}
p.media.FileMeta.Original.Width = width
p.media.FileMeta.Original.Height = height
p.media.FileMeta.Original.Size = (width * height)
p.media.FileMeta.Original.Aspect = float32(width) / float32(height)
// Determine thumbnail dimensions to use.
thumbWidth, thumbHeight := thumbSize(width, height)
p.media.FileMeta.Small.Width = thumbWidth
p.media.FileMeta.Small.Height = thumbHeight
p.media.FileMeta.Small.Size = (thumbWidth * thumbHeight)
p.media.FileMeta.Small.Aspect = float32(thumbWidth) / float32(thumbHeight)
// Generate a thumbnail image from input image path.
thumbpath, err = ffmpegGenerateThumb(ctx, temppath,
thumbWidth,
thumbHeight,
)
if err != nil {
return gtserror.Newf("error generating image thumb: %w", err)
}
case gtsmodel.FileTypeVideo:
// Pass file through ffmpeg clearing
// any excess metadata (e.g. EXIF).
if err := ffmpegClearMetadata(ctx,
temppath, ext,
); err != nil {
return gtserror.Newf("error cleaning metadata: %w", err)
}
// Extract video metadata we can from streams.
width, height, framerate, err := result.VideoMeta()
if err != nil {
return err
}
p.media.FileMeta.Original.Width = width
p.media.FileMeta.Original.Height = height
p.media.FileMeta.Original.Size = (width * height)
p.media.FileMeta.Original.Aspect = float32(width) / float32(height)
p.media.FileMeta.Original.Framerate = &framerate
// Extract total duration from format.
duration := result.Format.GetDuration()
p.media.FileMeta.Original.Duration = &duration
// Extract total bitrate from format.
bitrate := result.Format.GetBitRate()
p.media.FileMeta.Original.Bitrate = &bitrate
// Determine thumbnail dimensions to use.
thumbWidth, thumbHeight := thumbSize(width, height)
p.media.FileMeta.Small.Width = thumbWidth
p.media.FileMeta.Small.Height = thumbHeight
p.media.FileMeta.Small.Size = (thumbWidth * thumbHeight)
p.media.FileMeta.Small.Aspect = float32(thumbWidth) / float32(thumbHeight)
// Extract a thumbnail frame from input video path.
thumbpath, err = ffmpegGenerateThumb(ctx, temppath,
thumbWidth,
thumbHeight,
)
if err != nil {
return gtserror.Newf("error extracting video frame: %w", err)
}
case gtsmodel.FileTypeAudio:
// Extract total duration from format.
duration := result.Format.GetDuration()
p.media.FileMeta.Original.Duration = &duration
// NOTE: we do not clean audio file
// metadata, in order to keep tags.
// Extract total bitrate from format.
bitrate := result.Format.GetBitRate()
p.media.FileMeta.Original.Bitrate = &bitrate
// Extract image metadata from streams (if any),
// this will only exist for embedded album art.
width, height, framerate, _ := result.EmbeddedImageMeta()
if width > 0 && height > 0 {
// Unlikely to need these but masto API includes them.
p.media.FileMeta.Original.Width = width
p.media.FileMeta.Original.Height = height
if framerate != 0 {
p.media.FileMeta.Original.Framerate = &framerate
default:
log.Warn(ctx, "unsupported data type: %s", result.format)
return nil
}
if width > 0 && height > 0 {
// Determine thumbnail dimensions to use.
thumbWidth, thumbHeight := thumbSize(width, height)
p.media.FileMeta.Small.Width = thumbWidth
@ -323,11 +240,14 @@ func (p *ProcessingMedia) store(ctx context.Context) error {
if err != nil {
return gtserror.Newf("error generating image thumb: %w", err)
}
}
default:
log.Warnf(ctx, "unsupported type: %s (%s)", p.media.Type, result.Format.FormatName)
return nil
if p.media.Blurhash == "" {
// Generate blurhash (if not already) from thumbnail.
p.media.Blurhash, err = generateBlurhash(thumbpath)
if err != nil {
return gtserror.Newf("error generating thumb blurhash: %w", err)
}
}
}
// Calculate final media attachment file path.
@ -352,17 +272,6 @@ func (p *ProcessingMedia) store(ctx context.Context) error {
p.media.File.FileSize = int(filesz)
if thumbpath != "" {
// Note that neither thumbnail storage
// nor a blurhash are needed for audio.
if p.media.Blurhash == "" {
// Generate blurhash (if not already) from thumbnail.
p.media.Blurhash, err = generateBlurhash(thumbpath)
if err != nil {
return gtserror.Newf("error generating thumb blurhash: %w", err)
}
}
// Copy thumbnail file into storage at path.
thumbsz, err := p.mgr.state.Storage.PutFile(ctx,
p.media.Thumbnail.Path,

21
internal/media/types.go
View File

@ -23,27 +23,6 @@ import (
"time"
)
// mime consts
const (
mimeImage = "image"
mimeVideo = "video"
mimeJpeg = "jpeg"
mimeImageJpeg = mimeImage + "/" + mimeJpeg
mimeGif = "gif"
mimeImageGif = mimeImage + "/" + mimeGif
mimePng = "png"
mimeImagePng = mimeImage + "/" + mimePng
mimeWebp = "webp"
mimeImageWebp = mimeImage + "/" + mimeWebp
mimeMp4 = "mp4"
mimeVideoMp4 = mimeVideo + "/" + mimeMp4
)
type Size string
const (

34
internal/typeutils/internaltofrontend_test.go
View File

@ -1225,9 +1225,22 @@ func (suite *InternalToFrontendTestSuite) TestInstanceV1ToFrontend() {
"supported_mime_types": [
"image/jpeg",
"image/gif",
"image/png",
"image/webp",
"video/mp4"
"audio/mp2",
"audio/mp3",
"video/x-msvideo",
"image/png",
"image/apng",
"audio/ogg",
"video/ogg",
"audio/x-m4a",
"video/mp4",
"video/quicktime",
"audio/x-ms-wma",
"video/x-ms-wmv",
"video/webm",
"audio/x-matroska",
"video/x-matroska"
],
"image_size_limit": 41943040,
"image_matrix_limit": 16777216,
@ -1350,9 +1363,22 @@ func (suite *InternalToFrontendTestSuite) TestInstanceV2ToFrontend() {
"supported_mime_types": [
"image/jpeg",
"image/gif",
"image/png",
"image/webp",
"video/mp4"
"audio/mp2",
"audio/mp3",
"video/x-msvideo",
"image/png",
"image/apng",
"audio/ogg",
"video/ogg",
"audio/x-m4a",
"video/mp4",
"video/quicktime",
"audio/x-ms-wma",
"video/x-ms-wmv",
"video/webm",
"audio/x-matroska",
"video/x-matroska"
],
"image_size_limit": 41943040,
"image_matrix_limit": 16777216,

34
internal/util/math.go Normal file
View File

@ -0,0 +1,34 @@
// GoToSocial
// Copyright (C) GoToSocial Authors admin@gotosocial.org
// SPDX-License-Identifier: AGPL-3.0-or-later
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package util
type Number interface {
~int | ~int8 | ~int16 | ~int32 | ~int64 |
~uint | ~uint8 | ~uint16 | ~uint32 | ~uint64 |
~uintptr | ~float32 | ~float64
}
// Div performs a safe division of
// n1 and n2, checking for zero n2. In the
// case of zero n2, zero is returned.
func Div[N Number](n1, n2 N) N {
if n2 == 0 {
return 0
}
return n1 / n2
}

9
internal/util/ptr.go
View File

@ -34,6 +34,15 @@ func Ptr[T any](t T) *T {
return &t
}
// PtrIf returns ptr value only if 't' non-zero.
func PtrIf[T comparable](t T) *T {
var z T
if t == z {
return nil
}
return &t
}
// PtrValueOr returns either value of ptr, or default.
func PtrValueOr[T any](t *T, _default T) T {
if t != nil {