[feature] more filetype support! (#3107)
* add more supported file types to our media processor that ffmpeg supports, update supported mime type lists * add code comments to the supported mime types slice * don't check for zero value string, just parse * remove some unneeded consts which make the code a bit harder to read * fix test expected instance media mime types, use compact ffprobe json, simple media processing by type * final tweaks to media processing code * don't use safe divide where we don't need to
This commit is contained in:
kim
GitHub
parent
9efb11d848
commit
de45c0be60
|
|
@ -29,6 +29,7 @@ import (
|
|||
"github.com/superseriousbusiness/gotosocial/internal/db/bundb"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/log"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/media"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/media/ffmpeg"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/state"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/storage"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/util"
|
||||
|
|
@ -43,6 +44,14 @@ func main() {
|
|||
log.Panic(ctx, "Usage: go run ./cmd/process-emoji <input-file> <output-static>")
|
||||
}
|
||||
|
||||
if err := ffmpeg.InitFfprobe(ctx, 1); err != nil {
|
||||
log.Panic(ctx, err)
|
||||
}
|
||||
|
||||
if err := ffmpeg.InitFfmpeg(ctx, 1); err != nil {
|
||||
log.Panic(ctx, err)
|
||||
}
|
||||
|
||||
var st storage.Driver
|
||||
st.Storage = memory.Open(10, true)
|
||||
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@ import (
|
|||
"github.com/superseriousbusiness/gotosocial/internal/db/bundb"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/log"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/media"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/media/ffmpeg"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/state"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/storage"
|
||||
)
|
||||
|
|
@ -42,6 +43,14 @@ func main() {
|
|||
log.Panic(ctx, "Usage: go run ./cmd/process-media <input-file> <output-processed> <output-thumbnail>")
|
||||
}
|
||||
|
||||
if err := ffmpeg.InitFfprobe(ctx, 1); err != nil {
|
||||
log.Panic(ctx, err)
|
||||
}
|
||||
|
||||
if err := ffmpeg.InitFfmpeg(ctx, 1); err != nil {
|
||||
log.Panic(ctx, err)
|
||||
}
|
||||
|
||||
var st storage.Driver
|
||||
st.Storage = memory.Open(10, true)
|
||||
|
||||
|
|
@ -105,6 +114,9 @@ func main() {
|
|||
func copyFile(ctx context.Context, st *storage.Driver, key string, path string) {
|
||||
rc, err := st.GetStream(ctx, key)
|
||||
if err != nil {
|
||||
if storage.IsNotFound(err) {
|
||||
return
|
||||
}
|
||||
log.Panic(ctx, err)
|
||||
}
|
||||
defer rc.Close()
|
||||
|
|
|
|||
|
|
@ -105,9 +105,22 @@ func (suite *InstancePatchTestSuite) TestInstancePatch1() {
|
|||
"supported_mime_types": [
|
||||
"image/jpeg",
|
||||
"image/gif",
|
||||
"image/png",
|
||||
"image/webp",
|
||||
"video/mp4"
|
||||
"audio/mp2",
|
||||
"audio/mp3",
|
||||
"video/x-msvideo",
|
||||
"image/png",
|
||||
"image/apng",
|
||||
"audio/ogg",
|
||||
"video/ogg",
|
||||
"audio/x-m4a",
|
||||
"video/mp4",
|
||||
"video/quicktime",
|
||||
"audio/x-ms-wma",
|
||||
"video/x-ms-wmv",
|
||||
"video/webm",
|
||||
"audio/x-matroska",
|
||||
"video/x-matroska"
|
||||
],
|
||||
"image_size_limit": 41943040,
|
||||
"image_matrix_limit": 16777216,
|
||||
|
|
@ -226,9 +239,22 @@ func (suite *InstancePatchTestSuite) TestInstancePatch2() {
|
|||
"supported_mime_types": [
|
||||
"image/jpeg",
|
||||
"image/gif",
|
||||
"image/png",
|
||||
"image/webp",
|
||||
"video/mp4"
|
||||
"audio/mp2",
|
||||
"audio/mp3",
|
||||
"video/x-msvideo",
|
||||
"image/png",
|
||||
"image/apng",
|
||||
"audio/ogg",
|
||||
"video/ogg",
|
||||
"audio/x-m4a",
|
||||
"video/mp4",
|
||||
"video/quicktime",
|
||||
"audio/x-ms-wma",
|
||||
"video/x-ms-wmv",
|
||||
"video/webm",
|
||||
"audio/x-matroska",
|
||||
"video/x-matroska"
|
||||
],
|
||||
"image_size_limit": 41943040,
|
||||
"image_matrix_limit": 16777216,
|
||||
|
|
@ -347,9 +373,22 @@ func (suite *InstancePatchTestSuite) TestInstancePatch3() {
|
|||
"supported_mime_types": [
|
||||
"image/jpeg",
|
||||
"image/gif",
|
||||
"image/png",
|
||||
"image/webp",
|
||||
"video/mp4"
|
||||
"audio/mp2",
|
||||
"audio/mp3",
|
||||
"video/x-msvideo",
|
||||
"image/png",
|
||||
"image/apng",
|
||||
"audio/ogg",
|
||||
"video/ogg",
|
||||
"audio/x-m4a",
|
||||
"video/mp4",
|
||||
"video/quicktime",
|
||||
"audio/x-ms-wma",
|
||||
"video/x-ms-wmv",
|
||||
"video/webm",
|
||||
"audio/x-matroska",
|
||||
"video/x-matroska"
|
||||
],
|
||||
"image_size_limit": 41943040,
|
||||
"image_matrix_limit": 16777216,
|
||||
|
|
@ -519,9 +558,22 @@ func (suite *InstancePatchTestSuite) TestInstancePatch6() {
|
|||
"supported_mime_types": [
|
||||
"image/jpeg",
|
||||
"image/gif",
|
||||
"image/png",
|
||||
"image/webp",
|
||||
"video/mp4"
|
||||
"audio/mp2",
|
||||
"audio/mp3",
|
||||
"video/x-msvideo",
|
||||
"image/png",
|
||||
"image/apng",
|
||||
"audio/ogg",
|
||||
"video/ogg",
|
||||
"audio/x-m4a",
|
||||
"video/mp4",
|
||||
"video/quicktime",
|
||||
"audio/x-ms-wma",
|
||||
"video/x-ms-wmv",
|
||||
"video/webm",
|
||||
"audio/x-matroska",
|
||||
"video/x-matroska"
|
||||
],
|
||||
"image_size_limit": 41943040,
|
||||
"image_matrix_limit": 16777216,
|
||||
|
|
@ -662,9 +714,22 @@ func (suite *InstancePatchTestSuite) TestInstancePatch8() {
|
|||
"supported_mime_types": [
|
||||
"image/jpeg",
|
||||
"image/gif",
|
||||
"image/png",
|
||||
"image/webp",
|
||||
"video/mp4"
|
||||
"audio/mp2",
|
||||
"audio/mp3",
|
||||
"video/x-msvideo",
|
||||
"image/png",
|
||||
"image/apng",
|
||||
"audio/ogg",
|
||||
"video/ogg",
|
||||
"audio/x-m4a",
|
||||
"video/mp4",
|
||||
"video/quicktime",
|
||||
"audio/x-ms-wma",
|
||||
"video/x-ms-wmv",
|
||||
"video/webm",
|
||||
"audio/x-matroska",
|
||||
"video/x-matroska"
|
||||
],
|
||||
"image_size_limit": 41943040,
|
||||
"image_matrix_limit": 16777216,
|
||||
|
|
@ -820,9 +885,22 @@ func (suite *InstancePatchTestSuite) TestInstancePatch9() {
|
|||
"supported_mime_types": [
|
||||
"image/jpeg",
|
||||
"image/gif",
|
||||
"image/png",
|
||||
"image/webp",
|
||||
"video/mp4"
|
||||
"audio/mp2",
|
||||
"audio/mp3",
|
||||
"video/x-msvideo",
|
||||
"image/png",
|
||||
"image/apng",
|
||||
"audio/ogg",
|
||||
"video/ogg",
|
||||
"audio/x-m4a",
|
||||
"video/mp4",
|
||||
"video/quicktime",
|
||||
"audio/x-ms-wma",
|
||||
"video/x-ms-wmv",
|
||||
"video/webm",
|
||||
"audio/x-matroska",
|
||||
"video/x-matroska"
|
||||
],
|
||||
"image_size_limit": 41943040,
|
||||
"image_matrix_limit": 16777216,
|
||||
|
|
|
|||
|
|
@ -18,7 +18,6 @@
|
|||
package media
|
||||
|
||||
import (
|
||||
"cmp"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
|
|
@ -135,7 +134,7 @@ func ffmpeg(ctx context.Context, dirpath string, args ...string) error {
|
|||
}
|
||||
|
||||
// ffprobe calls `ffprobe` (WASM) on filepath, returning parsed JSON output.
|
||||
func ffprobe(ctx context.Context, filepath string) (*ffprobeResult, error) {
|
||||
func ffprobe(ctx context.Context, filepath string) (*result, error) {
|
||||
var stdout byteutil.Buffer
|
||||
|
||||
// Get directory from filepath.
|
||||
|
|
@ -148,7 +147,7 @@ func ffprobe(ctx context.Context, filepath string) (*ffprobeResult, error) {
|
|||
Args: []string{
|
||||
"-i", filepath,
|
||||
"-loglevel", "quiet",
|
||||
"-print_format", "json",
|
||||
"-print_format", "json=compact=1",
|
||||
"-show_streams",
|
||||
"-show_format",
|
||||
"-show_error",
|
||||
|
|
@ -172,7 +171,219 @@ func ffprobe(ctx context.Context, filepath string) (*ffprobeResult, error) {
|
|||
return nil, gtserror.Newf("error unmarshaling json: %w", err)
|
||||
}
|
||||
|
||||
return &result, nil
|
||||
// Convert raw result data.
|
||||
res, err := result.Process()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// result contains parsed ffprobe result
|
||||
// data in a more useful data format.
|
||||
type result struct {
|
||||
format string
|
||||
audio []audioStream
|
||||
video []videoStream
|
||||
bitrate uint64
|
||||
duration float64
|
||||
}
|
||||
|
||||
type stream struct {
|
||||
codec string
|
||||
}
|
||||
|
||||
type audioStream struct {
|
||||
stream
|
||||
}
|
||||
|
||||
type videoStream struct {
|
||||
stream
|
||||
width int
|
||||
height int
|
||||
framerate float32
|
||||
}
|
||||
|
||||
// GetFileType determines file type and extension to use for media data. This
|
||||
// function helps to abstract away the horrible complexities that are possible
|
||||
// media container (i.e. the file) types and and possible sub-types within that.
|
||||
//
|
||||
// Note the checks for (len(res.video) > 0) may catch some audio files with embedded
|
||||
// album art as video, but i blame that on the hellscape that is media filetypes.
|
||||
//
|
||||
// TODO: we can update this code to also return a mimetype and avoid later parsing!
|
||||
func (res *result) GetFileType() (gtsmodel.FileType, string) {
|
||||
switch res.format {
|
||||
case "mpeg":
|
||||
return gtsmodel.FileTypeVideo, "mpeg"
|
||||
case "mjpeg":
|
||||
return gtsmodel.FileTypeVideo, "mjpeg"
|
||||
case "mov,mp4,m4a,3gp,3g2,mj2":
|
||||
switch {
|
||||
case len(res.video) > 0:
|
||||
return gtsmodel.FileTypeVideo, "mp4"
|
||||
case len(res.audio) > 0 &&
|
||||
res.audio[0].codec == "aac":
|
||||
// m4a only supports [aac] audio.
|
||||
return gtsmodel.FileTypeAudio, "m4a"
|
||||
}
|
||||
case "apng":
|
||||
return gtsmodel.FileTypeImage, "apng"
|
||||
case "png_pipe":
|
||||
return gtsmodel.FileTypeImage, "png"
|
||||
case "image2", "image2pipe", "jpeg_pipe":
|
||||
return gtsmodel.FileTypeImage, "jpeg"
|
||||
case "webp", "webp_pipe":
|
||||
return gtsmodel.FileTypeImage, "webp"
|
||||
case "gif":
|
||||
return gtsmodel.FileTypeImage, "gif"
|
||||
case "mp3":
|
||||
if len(res.audio) > 0 {
|
||||
switch res.audio[0].codec {
|
||||
case "mp2":
|
||||
return gtsmodel.FileTypeAudio, "mp2"
|
||||
case "mp3":
|
||||
return gtsmodel.FileTypeAudio, "mp3"
|
||||
}
|
||||
}
|
||||
case "asf":
|
||||
switch {
|
||||
case len(res.video) > 0:
|
||||
return gtsmodel.FileTypeVideo, "wmv"
|
||||
case len(res.audio) > 0:
|
||||
return gtsmodel.FileTypeAudio, "wma"
|
||||
}
|
||||
case "ogg":
|
||||
switch {
|
||||
case len(res.video) > 0:
|
||||
return gtsmodel.FileTypeVideo, "ogv"
|
||||
case len(res.audio) > 0:
|
||||
return gtsmodel.FileTypeAudio, "ogg"
|
||||
}
|
||||
case "matroska,webm":
|
||||
switch {
|
||||
case len(res.video) > 0:
|
||||
switch res.video[0].codec {
|
||||
case "vp8", "vp9", "av1":
|
||||
default:
|
||||
return gtsmodel.FileTypeVideo, "mkv"
|
||||
}
|
||||
if len(res.audio) > 0 {
|
||||
switch res.audio[0].codec {
|
||||
case "vorbis", "opus", "libopus":
|
||||
// webm only supports [VP8/VP9/AV1]+[vorbis/opus]
|
||||
return gtsmodel.FileTypeVideo, "webm"
|
||||
}
|
||||
}
|
||||
case len(res.audio) > 0:
|
||||
return gtsmodel.FileTypeAudio, "mka"
|
||||
}
|
||||
case "avi":
|
||||
return gtsmodel.FileTypeVideo, "avi"
|
||||
}
|
||||
return gtsmodel.FileTypeUnknown, res.format
|
||||
}
|
||||
|
||||
// ImageMeta extracts image metadata contained within ffprobe'd media result streams.
|
||||
func (res *result) ImageMeta() (width int, height int, framerate float32) {
|
||||
for _, stream := range res.video {
|
||||
if stream.width > width {
|
||||
width = stream.width
|
||||
}
|
||||
if stream.height > height {
|
||||
height = stream.height
|
||||
}
|
||||
if fr := float32(stream.framerate); fr > 0 {
|
||||
if framerate == 0 || fr < framerate {
|
||||
framerate = fr
|
||||
}
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// Process converts raw ffprobe result data into our more usable result{} type.
|
||||
func (res *ffprobeResult) Process() (*result, error) {
|
||||
if res.Error != nil {
|
||||
return nil, res.Error
|
||||
}
|
||||
|
||||
if res.Format == nil {
|
||||
return nil, errors.New("missing format data")
|
||||
}
|
||||
|
||||
var r result
|
||||
var err error
|
||||
|
||||
// Copy over container format.
|
||||
r.format = res.Format.FormatName
|
||||
|
||||
// Parsed media bitrate (if it was set).
|
||||
if str := res.Format.BitRate; str != "" {
|
||||
r.bitrate, err = strconv.ParseUint(str, 10, 64)
|
||||
if err != nil {
|
||||
return nil, gtserror.Newf("invalid bitrate %s: %w", str, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Parse media duration (if it was set).
|
||||
if str := res.Format.Duration; str != "" {
|
||||
r.duration, err = strconv.ParseFloat(str, 32)
|
||||
if err != nil {
|
||||
return nil, gtserror.Newf("invalid duration %s: %w", str, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Preallocate streams to max possible lengths.
|
||||
r.audio = make([]audioStream, 0, len(res.Streams))
|
||||
r.video = make([]videoStream, 0, len(res.Streams))
|
||||
|
||||
// Convert streams to separate types.
|
||||
for _, s := range res.Streams {
|
||||
switch s.CodecType {
|
||||
case "audio":
|
||||
// Append audio stream data to result.
|
||||
r.audio = append(r.audio, audioStream{
|
||||
stream: stream{codec: s.CodecName},
|
||||
})
|
||||
case "video":
|
||||
var framerate float32
|
||||
|
||||
// Parse stream framerate, bearing in
|
||||
// mind that some static container formats
|
||||
// (e.g. jpeg) still return a framerate, so
|
||||
// we also check for a non-1 timebase (dts).
|
||||
if str := s.RFrameRate; str != "" &&
|
||||
s.DurationTS > 1 {
|
||||
var num, den uint32
|
||||
den = 1
|
||||
|
||||
// Check for inequality (numerator / denominator).
|
||||
if p := strings.SplitN(str, "/", 2); len(p) == 2 {
|
||||
n, _ := strconv.ParseUint(p[0], 10, 32)
|
||||
d, _ := strconv.ParseUint(p[1], 10, 32)
|
||||
num, den = uint32(n), uint32(d)
|
||||
} else {
|
||||
n, _ := strconv.ParseUint(p[0], 10, 32)
|
||||
num = uint32(n)
|
||||
}
|
||||
|
||||
// Set final divised framerate.
|
||||
framerate = float32(num / den)
|
||||
}
|
||||
|
||||
// Append video stream data to result.
|
||||
r.video = append(r.video, videoStream{
|
||||
stream: stream{codec: s.CodecName},
|
||||
width: s.Width,
|
||||
height: s.Height,
|
||||
framerate: framerate,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return &r, nil
|
||||
}
|
||||
|
||||
// ffprobeResult contains parsed JSON data from
|
||||
|
|
@ -183,175 +394,33 @@ type ffprobeResult struct {
|
|||
Error *ffprobeError `json:"error"`
|
||||
}
|
||||
|
||||
// ImageMeta extracts image metadata contained within ffprobe'd media result streams.
|
||||
func (res *ffprobeResult) ImageMeta() (width int, height int, err error) {
|
||||
for _, stream := range res.Streams {
|
||||
if stream.Width > width {
|
||||
width = stream.Width
|
||||
}
|
||||
if stream.Height > height {
|
||||
height = stream.Height
|
||||
}
|
||||
}
|
||||
if width == 0 || height == 0 {
|
||||
err = errors.New("invalid image stream(s)")
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// EmbeddedImageMeta extracts embedded image metadata contained within ffprobe'd media result
|
||||
// streams, should be used for pulling album image (can be animated image) from audio files.
|
||||
func (res *ffprobeResult) EmbeddedImageMeta() (width int, height int, framerate float32, err error) {
|
||||
for _, stream := range res.Streams {
|
||||
if stream.Width > width {
|
||||
width = stream.Width
|
||||
}
|
||||
if stream.Height > height {
|
||||
height = stream.Height
|
||||
}
|
||||
if fr := stream.GetFrameRate(); fr > 0 {
|
||||
if framerate == 0 || fr < framerate {
|
||||
framerate = fr
|
||||
}
|
||||
}
|
||||
}
|
||||
// Need width + height but
|
||||
// no framerate is fine.
|
||||
if width == 0 || height == 0 {
|
||||
err = errors.New("invalid image stream(s)")
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// VideoMeta extracts video metadata contained within ffprobe'd media result streams.
|
||||
func (res *ffprobeResult) VideoMeta() (width, height int, framerate float32, err error) {
|
||||
for _, stream := range res.Streams {
|
||||
if stream.Width > width {
|
||||
width = stream.Width
|
||||
}
|
||||
if stream.Height > height {
|
||||
height = stream.Height
|
||||
}
|
||||
if fr := stream.GetFrameRate(); fr > 0 {
|
||||
if framerate == 0 || fr < framerate {
|
||||
framerate = fr
|
||||
}
|
||||
}
|
||||
}
|
||||
if width == 0 || height == 0 || framerate == 0 {
|
||||
err = errors.New("invalid video stream(s)")
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
type ffprobeStream struct {
|
||||
CodecName string `json:"codec_name"`
|
||||
AvgFrameRate string `json:"avg_frame_rate"`
|
||||
RFrameRate string `json:"r_frame_rate"`
|
||||
Width int `json:"width"`
|
||||
Height int `json:"height"`
|
||||
CodecName string `json:"codec_name"`
|
||||
CodecType string `json:"codec_type"`
|
||||
RFrameRate string `json:"r_frame_rate"`
|
||||
DurationTS uint `json:"duration_ts"`
|
||||
Width int `json:"width"`
|
||||
Height int `json:"height"`
|
||||
// + unused fields.
|
||||
}
|
||||
|
||||
// GetFrameRate calculates float32 framerate value from stream json string.
|
||||
func (str *ffprobeStream) GetFrameRate() float32 {
|
||||
numDen := func(strFR string) (float32, float32) {
|
||||
var (
|
||||
// numerator
|
||||
num float32
|
||||
|
||||
// denominator
|
||||
den float32
|
||||
)
|
||||
|
||||
// Check for a provided inequality, i.e. numerator / denominator.
|
||||
if p := strings.SplitN(strFR, "/", 2); len(p) == 2 {
|
||||
n, _ := strconv.ParseFloat(p[0], 32)
|
||||
d, _ := strconv.ParseFloat(p[1], 32)
|
||||
num, den = float32(n), float32(d)
|
||||
} else {
|
||||
n, _ := strconv.ParseFloat(p[0], 32)
|
||||
num = float32(n)
|
||||
}
|
||||
|
||||
return num, den
|
||||
}
|
||||
|
||||
var num, den float32
|
||||
if str.AvgFrameRate != "" {
|
||||
// Check if we have avg_frame_rate.
|
||||
num, den = numDen(str.AvgFrameRate)
|
||||
}
|
||||
|
||||
if num == 0 && str.RFrameRate != "" {
|
||||
// Check if we have r_frame_rate.
|
||||
num, den = numDen(str.RFrameRate)
|
||||
}
|
||||
|
||||
if num != 0 {
|
||||
// Found it.
|
||||
// Avoid divide by zero.
|
||||
return num / cmp.Or(den, 1)
|
||||
}
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
type ffprobeFormat struct {
|
||||
Filename string `json:"filename"`
|
||||
FormatName string `json:"format_name"`
|
||||
Duration string `json:"duration"`
|
||||
BitRate string `json:"bit_rate"`
|
||||
// + unused fields
|
||||
}
|
||||
|
||||
// GetFileType determines file type and extension to use for media data.
|
||||
func (fmt *ffprobeFormat) GetFileType() (gtsmodel.FileType, string) {
|
||||
switch fmt.FormatName {
|
||||
case "mov,mp4,m4a,3gp,3g2,mj2":
|
||||
return gtsmodel.FileTypeVideo, "mp4"
|
||||
case "apng":
|
||||
return gtsmodel.FileTypeImage, "apng"
|
||||
case "png_pipe":
|
||||
return gtsmodel.FileTypeImage, "png"
|
||||
case "image2", "jpeg_pipe":
|
||||
return gtsmodel.FileTypeImage, "jpeg"
|
||||
case "webp_pipe":
|
||||
return gtsmodel.FileTypeImage, "webp"
|
||||
case "gif":
|
||||
return gtsmodel.FileTypeImage, "gif"
|
||||
case "mp3":
|
||||
return gtsmodel.FileTypeAudio, "mp3"
|
||||
case "ogg":
|
||||
return gtsmodel.FileTypeAudio, "ogg"
|
||||
default:
|
||||
return gtsmodel.FileTypeUnknown, fmt.FormatName
|
||||
}
|
||||
}
|
||||
|
||||
// GetDuration calculates float32 framerate value from format json string.
|
||||
func (fmt *ffprobeFormat) GetDuration() float32 {
|
||||
if fmt.Duration != "" {
|
||||
dur, _ := strconv.ParseFloat(fmt.Duration, 32)
|
||||
return float32(dur)
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// GetBitRate calculates uint64 bitrate value from format json string.
|
||||
func (fmt *ffprobeFormat) GetBitRate() uint64 {
|
||||
if fmt.BitRate != "" {
|
||||
r, _ := strconv.ParseUint(fmt.BitRate, 10, 64)
|
||||
return r
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
type ffprobeError struct {
|
||||
Code int `json:"code"`
|
||||
String string `json:"string"`
|
||||
}
|
||||
|
||||
func isUnsupportedTypeErr(err error) bool {
|
||||
ffprobeErr, ok := err.(*ffprobeError)
|
||||
return ok && ffprobeErr.Code == -1094995529
|
||||
}
|
||||
|
||||
func (err *ffprobeError) Error() string {
|
||||
return err.String + " (" + strconv.Itoa(err.Code) + ")"
|
||||
}
|
||||
|
|
|
|||
|
|
@ -34,17 +34,46 @@ import (
|
|||
)
|
||||
|
||||
var SupportedMIMETypes = []string{
|
||||
mimeImageJpeg,
|
||||
mimeImageGif,
|
||||
mimeImagePng,
|
||||
mimeImageWebp,
|
||||
mimeVideoMp4,
|
||||
"image/jpeg", // .jpeg
|
||||
"image/gif", // .gif
|
||||
"image/webp", // .webp
|
||||
|
||||
"audio/mp2", // .mp2
|
||||
"audio/mp3", // .mp3
|
||||
|
||||
"video/x-msvideo", // .avi
|
||||
|
||||
// png types
|
||||
"image/png", // .png
|
||||
"image/apng", // .apng
|
||||
|
||||
// ogg types
|
||||
"audio/ogg", // .ogg
|
||||
"video/ogg", // .ogv
|
||||
|
||||
// mpeg4 types
|
||||
"audio/x-m4a", // .m4a
|
||||
"video/mp4", // .mp4
|
||||
"video/quicktime", // .mov
|
||||
|
||||
// asf types
|
||||
"audio/x-ms-wma", // .wma
|
||||
"video/x-ms-wmv", // .wmv
|
||||
|
||||
// matroska types
|
||||
"video/webm", // .webm
|
||||
"audio/x-matroska", // .mka
|
||||
"video/x-matroska", // .mkv
|
||||
}
|
||||
|
||||
var SupportedEmojiMIMETypes = []string{
|
||||
mimeImageGif,
|
||||
mimeImagePng,
|
||||
mimeImageWebp,
|
||||
"image/jpeg", // .jpeg
|
||||
"image/gif", // .gif
|
||||
"image/webp", // .webp
|
||||
|
||||
// png types
|
||||
"image/png", // .png
|
||||
"image/apng", // .apng
|
||||
}
|
||||
|
||||
type Manager struct {
|
||||
|
|
@ -102,8 +131,8 @@ func (m *Manager) CreateMedia(
|
|||
id,
|
||||
|
||||
// Always encode attachment
|
||||
// thumbnails as jpg.
|
||||
"jpg",
|
||||
// thumbnails as jpeg.
|
||||
"jpeg",
|
||||
)
|
||||
|
||||
// Calculate attachment thumbnail URL.
|
||||
|
|
@ -114,8 +143,8 @@ func (m *Manager) CreateMedia(
|
|||
id,
|
||||
|
||||
// Always encode attachment
|
||||
// thumbnails as jpg.
|
||||
"jpg",
|
||||
// thumbnails as jpeg.
|
||||
"jpeg",
|
||||
)
|
||||
|
||||
// Populate initial fields on the new media,
|
||||
|
|
@ -134,7 +163,7 @@ func (m *Manager) CreateMedia(
|
|||
Path: path,
|
||||
},
|
||||
Thumbnail: gtsmodel.Thumbnail{
|
||||
ContentType: mimeImageJpeg, // thumbs always jpg.
|
||||
ContentType: "image/jpeg",
|
||||
Path: thumbPath,
|
||||
URL: thumbURL,
|
||||
},
|
||||
|
|
@ -244,7 +273,7 @@ func (m *Manager) CreateEmoji(
|
|||
|
||||
// All static emojis
|
||||
// are encoded as png.
|
||||
mimePng,
|
||||
"png",
|
||||
)
|
||||
|
||||
// Generate static image path for attachment.
|
||||
|
|
@ -256,7 +285,7 @@ func (m *Manager) CreateEmoji(
|
|||
|
||||
// All static emojis
|
||||
// are encoded as png.
|
||||
mimePng,
|
||||
"png",
|
||||
)
|
||||
|
||||
// Populate initial fields on the new emoji,
|
||||
|
|
@ -268,7 +297,7 @@ func (m *Manager) CreateEmoji(
|
|||
Domain: domain,
|
||||
ImageStaticURL: staticURL,
|
||||
ImageStaticPath: staticPath,
|
||||
ImageStaticContentType: mimeImagePng,
|
||||
ImageStaticContentType: "image/png",
|
||||
Disabled: util.Ptr(false),
|
||||
VisibleInPicker: util.Ptr(true),
|
||||
CreatedAt: now,
|
||||
|
|
@ -368,7 +397,7 @@ func (m *Manager) RefreshEmoji(
|
|||
|
||||
// All static emojis
|
||||
// are encoded as png.
|
||||
mimePng,
|
||||
"png",
|
||||
)
|
||||
|
||||
// Generate new static image storage path for emoji.
|
||||
|
|
@ -380,7 +409,7 @@ func (m *Manager) RefreshEmoji(
|
|||
|
||||
// All static emojis
|
||||
// are encoded as png.
|
||||
mimePng,
|
||||
"png",
|
||||
)
|
||||
|
||||
// Finally, create new emoji in database.
|
||||
|
|
|
|||
|
|
@ -421,7 +421,7 @@ func (suite *ManagerTestSuite) TestSlothVineProcess() {
|
|||
suite.Equal(81120, attachment.FileMeta.Original.Size)
|
||||
suite.EqualValues(float32(1.4083333), attachment.FileMeta.Original.Aspect)
|
||||
suite.EqualValues(float32(6.641), *attachment.FileMeta.Original.Duration)
|
||||
suite.EqualValues(float32(29.00003), *attachment.FileMeta.Original.Framerate)
|
||||
suite.EqualValues(float32(29), *attachment.FileMeta.Original.Framerate)
|
||||
suite.EqualValues(0x5be18, *attachment.FileMeta.Original.Bitrate)
|
||||
suite.EqualValues(gtsmodel.Small{
|
||||
Width: 338, Height: 240, Size: 81120, Aspect: 1.4083333333333334,
|
||||
|
|
|
|||
|
|
@ -160,27 +160,17 @@ func (p *ProcessingEmoji) store(ctx context.Context) error {
|
|||
// Pass input file through ffprobe to
|
||||
// parse further metadata information.
|
||||
result, err := ffprobe(ctx, temppath)
|
||||
if err != nil {
|
||||
return gtserror.Newf("error ffprobing data: %w", err)
|
||||
}
|
||||
|
||||
switch {
|
||||
// No errors parsing data.
|
||||
case result.Error == nil:
|
||||
|
||||
// Data type unhandleable by ffprobe.
|
||||
case result.Error.Code == -1094995529:
|
||||
if err != nil && !isUnsupportedTypeErr(err) {
|
||||
return gtserror.Newf("ffprobe error: %w", err)
|
||||
} else if result == nil {
|
||||
log.Warn(ctx, "unsupported data type")
|
||||
return nil
|
||||
|
||||
default:
|
||||
return gtserror.Newf("ffprobe error: %w", err)
|
||||
}
|
||||
|
||||
var ext string
|
||||
|
||||
// Set media type from ffprobe format data.
|
||||
fileType, ext := result.Format.GetFileType()
|
||||
// Get type from ffprobe format data.
|
||||
fileType, ext := result.GetFileType()
|
||||
if fileType != gtsmodel.FileTypeImage {
|
||||
return gtserror.Newf("unsupported emoji filetype: %s (%s)", fileType, ext)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -180,36 +180,33 @@ func (p *ProcessingMedia) store(ctx context.Context) error {
|
|||
// Pass input file through ffprobe to
|
||||
// parse further metadata information.
|
||||
result, err := ffprobe(ctx, temppath)
|
||||
if err != nil {
|
||||
return gtserror.Newf("error ffprobing data: %w", err)
|
||||
}
|
||||
|
||||
switch {
|
||||
// No errors parsing data.
|
||||
case result.Error == nil:
|
||||
|
||||
// Data type unhandleable by ffprobe.
|
||||
case result.Error.Code == -1094995529:
|
||||
if err != nil && !isUnsupportedTypeErr(err) {
|
||||
return gtserror.Newf("ffprobe error: %w", err)
|
||||
} else if result == nil {
|
||||
log.Warn(ctx, "unsupported data type")
|
||||
return nil
|
||||
|
||||
default:
|
||||
return gtserror.Newf("ffprobe error: %w", err)
|
||||
}
|
||||
|
||||
var ext string
|
||||
|
||||
// Set the media type from ffprobe format data.
|
||||
p.media.Type, ext = result.Format.GetFileType()
|
||||
if p.media.Type == gtsmodel.FileTypeUnknown {
|
||||
|
||||
// Return early (deleting file)
|
||||
// for unhandled file types.
|
||||
return nil
|
||||
}
|
||||
// Extract any video stream metadata from media.
|
||||
// This will always be used regardless of type,
|
||||
// as even audio files may contain embedded album art.
|
||||
width, height, framerate := result.ImageMeta()
|
||||
p.media.FileMeta.Original.Width = width
|
||||
p.media.FileMeta.Original.Height = height
|
||||
p.media.FileMeta.Original.Size = (width * height)
|
||||
p.media.FileMeta.Original.Aspect = util.Div(float32(width), float32(height))
|
||||
p.media.FileMeta.Original.Framerate = util.PtrIf(framerate)
|
||||
p.media.FileMeta.Original.Duration = util.PtrIf(float32(result.duration))
|
||||
p.media.FileMeta.Original.Bitrate = util.PtrIf(result.bitrate)
|
||||
|
||||
// Set media type from ffprobe format data.
|
||||
p.media.Type, ext = result.GetFileType()
|
||||
switch p.media.Type {
|
||||
case gtsmodel.FileTypeImage:
|
||||
|
||||
case gtsmodel.FileTypeImage,
|
||||
gtsmodel.FileTypeVideo:
|
||||
// Pass file through ffmpeg clearing
|
||||
// any excess metadata (e.g. EXIF).
|
||||
if err := ffmpegClearMetadata(ctx,
|
||||
|
|
@ -218,16 +215,16 @@ func (p *ProcessingMedia) store(ctx context.Context) error {
|
|||
return gtserror.Newf("error cleaning metadata: %w", err)
|
||||
}
|
||||
|
||||
// Extract image metadata from streams.
|
||||
width, height, err := result.ImageMeta()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
p.media.FileMeta.Original.Width = width
|
||||
p.media.FileMeta.Original.Height = height
|
||||
p.media.FileMeta.Original.Size = (width * height)
|
||||
p.media.FileMeta.Original.Aspect = float32(width) / float32(height)
|
||||
case gtsmodel.FileTypeAudio:
|
||||
// NOTE: we do not clean audio file
|
||||
// metadata, in order to keep tags.
|
||||
|
||||
default:
|
||||
log.Warn(ctx, "unsupported data type: %s", result.format)
|
||||
return nil
|
||||
}
|
||||
|
||||
if width > 0 && height > 0 {
|
||||
// Determine thumbnail dimensions to use.
|
||||
thumbWidth, thumbHeight := thumbSize(width, height)
|
||||
p.media.FileMeta.Small.Width = thumbWidth
|
||||
|
|
@ -244,90 +241,13 @@ func (p *ProcessingMedia) store(ctx context.Context) error {
|
|||
return gtserror.Newf("error generating image thumb: %w", err)
|
||||
}
|
||||
|
||||
case gtsmodel.FileTypeVideo:
|
||||
// Pass file through ffmpeg clearing
|
||||
// any excess metadata (e.g. EXIF).
|
||||
if err := ffmpegClearMetadata(ctx,
|
||||
temppath, ext,
|
||||
); err != nil {
|
||||
return gtserror.Newf("error cleaning metadata: %w", err)
|
||||
}
|
||||
|
||||
// Extract video metadata we can from streams.
|
||||
width, height, framerate, err := result.VideoMeta()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
p.media.FileMeta.Original.Width = width
|
||||
p.media.FileMeta.Original.Height = height
|
||||
p.media.FileMeta.Original.Size = (width * height)
|
||||
p.media.FileMeta.Original.Aspect = float32(width) / float32(height)
|
||||
p.media.FileMeta.Original.Framerate = &framerate
|
||||
|
||||
// Extract total duration from format.
|
||||
duration := result.Format.GetDuration()
|
||||
p.media.FileMeta.Original.Duration = &duration
|
||||
|
||||
// Extract total bitrate from format.
|
||||
bitrate := result.Format.GetBitRate()
|
||||
p.media.FileMeta.Original.Bitrate = &bitrate
|
||||
|
||||
// Determine thumbnail dimensions to use.
|
||||
thumbWidth, thumbHeight := thumbSize(width, height)
|
||||
p.media.FileMeta.Small.Width = thumbWidth
|
||||
p.media.FileMeta.Small.Height = thumbHeight
|
||||
p.media.FileMeta.Small.Size = (thumbWidth * thumbHeight)
|
||||
p.media.FileMeta.Small.Aspect = float32(thumbWidth) / float32(thumbHeight)
|
||||
|
||||
// Extract a thumbnail frame from input video path.
|
||||
thumbpath, err = ffmpegGenerateThumb(ctx, temppath,
|
||||
thumbWidth,
|
||||
thumbHeight,
|
||||
)
|
||||
if err != nil {
|
||||
return gtserror.Newf("error extracting video frame: %w", err)
|
||||
}
|
||||
|
||||
case gtsmodel.FileTypeAudio:
|
||||
// Extract total duration from format.
|
||||
duration := result.Format.GetDuration()
|
||||
p.media.FileMeta.Original.Duration = &duration
|
||||
|
||||
// Extract total bitrate from format.
|
||||
bitrate := result.Format.GetBitRate()
|
||||
p.media.FileMeta.Original.Bitrate = &bitrate
|
||||
|
||||
// Extract image metadata from streams (if any),
|
||||
// this will only exist for embedded album art.
|
||||
width, height, framerate, _ := result.EmbeddedImageMeta()
|
||||
if width > 0 && height > 0 {
|
||||
// Unlikely to need these but masto API includes them.
|
||||
p.media.FileMeta.Original.Width = width
|
||||
p.media.FileMeta.Original.Height = height
|
||||
if framerate != 0 {
|
||||
p.media.FileMeta.Original.Framerate = &framerate
|
||||
}
|
||||
|
||||
// Determine thumbnail dimensions to use.
|
||||
thumbWidth, thumbHeight := thumbSize(width, height)
|
||||
p.media.FileMeta.Small.Width = thumbWidth
|
||||
p.media.FileMeta.Small.Height = thumbHeight
|
||||
p.media.FileMeta.Small.Size = (thumbWidth * thumbHeight)
|
||||
p.media.FileMeta.Small.Aspect = float32(thumbWidth) / float32(thumbHeight)
|
||||
|
||||
// Generate a thumbnail image from input image path.
|
||||
thumbpath, err = ffmpegGenerateThumb(ctx, temppath,
|
||||
thumbWidth,
|
||||
thumbHeight,
|
||||
)
|
||||
if p.media.Blurhash == "" {
|
||||
// Generate blurhash (if not already) from thumbnail.
|
||||
p.media.Blurhash, err = generateBlurhash(thumbpath)
|
||||
if err != nil {
|
||||
return gtserror.Newf("error generating image thumb: %w", err)
|
||||
return gtserror.Newf("error generating thumb blurhash: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
default:
|
||||
log.Warnf(ctx, "unsupported type: %s (%s)", p.media.Type, result.Format.FormatName)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Calculate final media attachment file path.
|
||||
|
|
@ -352,17 +272,6 @@ func (p *ProcessingMedia) store(ctx context.Context) error {
|
|||
p.media.File.FileSize = int(filesz)
|
||||
|
||||
if thumbpath != "" {
|
||||
// Note that neither thumbnail storage
|
||||
// nor a blurhash are needed for audio.
|
||||
|
||||
if p.media.Blurhash == "" {
|
||||
// Generate blurhash (if not already) from thumbnail.
|
||||
p.media.Blurhash, err = generateBlurhash(thumbpath)
|
||||
if err != nil {
|
||||
return gtserror.Newf("error generating thumb blurhash: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Copy thumbnail file into storage at path.
|
||||
thumbsz, err := p.mgr.state.Storage.PutFile(ctx,
|
||||
p.media.Thumbnail.Path,
|
||||
|
|
|
|||
|
|
@ -23,27 +23,6 @@ import (
|
|||
"time"
|
||||
)
|
||||
|
||||
// mime consts
|
||||
const (
|
||||
mimeImage = "image"
|
||||
mimeVideo = "video"
|
||||
|
||||
mimeJpeg = "jpeg"
|
||||
mimeImageJpeg = mimeImage + "/" + mimeJpeg
|
||||
|
||||
mimeGif = "gif"
|
||||
mimeImageGif = mimeImage + "/" + mimeGif
|
||||
|
||||
mimePng = "png"
|
||||
mimeImagePng = mimeImage + "/" + mimePng
|
||||
|
||||
mimeWebp = "webp"
|
||||
mimeImageWebp = mimeImage + "/" + mimeWebp
|
||||
|
||||
mimeMp4 = "mp4"
|
||||
mimeVideoMp4 = mimeVideo + "/" + mimeMp4
|
||||
)
|
||||
|
||||
type Size string
|
||||
|
||||
const (
|
||||
|
|
|
|||
|
|
@ -1225,9 +1225,22 @@ func (suite *InternalToFrontendTestSuite) TestInstanceV1ToFrontend() {
|
|||
"supported_mime_types": [
|
||||
"image/jpeg",
|
||||
"image/gif",
|
||||
"image/png",
|
||||
"image/webp",
|
||||
"video/mp4"
|
||||
"audio/mp2",
|
||||
"audio/mp3",
|
||||
"video/x-msvideo",
|
||||
"image/png",
|
||||
"image/apng",
|
||||
"audio/ogg",
|
||||
"video/ogg",
|
||||
"audio/x-m4a",
|
||||
"video/mp4",
|
||||
"video/quicktime",
|
||||
"audio/x-ms-wma",
|
||||
"video/x-ms-wmv",
|
||||
"video/webm",
|
||||
"audio/x-matroska",
|
||||
"video/x-matroska"
|
||||
],
|
||||
"image_size_limit": 41943040,
|
||||
"image_matrix_limit": 16777216,
|
||||
|
|
@ -1350,9 +1363,22 @@ func (suite *InternalToFrontendTestSuite) TestInstanceV2ToFrontend() {
|
|||
"supported_mime_types": [
|
||||
"image/jpeg",
|
||||
"image/gif",
|
||||
"image/png",
|
||||
"image/webp",
|
||||
"video/mp4"
|
||||
"audio/mp2",
|
||||
"audio/mp3",
|
||||
"video/x-msvideo",
|
||||
"image/png",
|
||||
"image/apng",
|
||||
"audio/ogg",
|
||||
"video/ogg",
|
||||
"audio/x-m4a",
|
||||
"video/mp4",
|
||||
"video/quicktime",
|
||||
"audio/x-ms-wma",
|
||||
"video/x-ms-wmv",
|
||||
"video/webm",
|
||||
"audio/x-matroska",
|
||||
"video/x-matroska"
|
||||
],
|
||||
"image_size_limit": 41943040,
|
||||
"image_matrix_limit": 16777216,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,34 @@
|
|||
// GoToSocial
|
||||
// Copyright (C) GoToSocial Authors admin@gotosocial.org
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package util
|
||||
|
||||
type Number interface {
|
||||
~int | ~int8 | ~int16 | ~int32 | ~int64 |
|
||||
~uint | ~uint8 | ~uint16 | ~uint32 | ~uint64 |
|
||||
~uintptr | ~float32 | ~float64
|
||||
}
|
||||
|
||||
// Div performs a safe division of
|
||||
// n1 and n2, checking for zero n2. In the
|
||||
// case of zero n2, zero is returned.
|
||||
func Div[N Number](n1, n2 N) N {
|
||||
if n2 == 0 {
|
||||
return 0
|
||||
}
|
||||
return n1 / n2
|
||||
}
|
||||
|
|
@ -34,6 +34,15 @@ func Ptr[T any](t T) *T {
|
|||
return &t
|
||||
}
|
||||
|
||||
// PtrIf returns ptr value only if 't' non-zero.
|
||||
func PtrIf[T comparable](t T) *T {
|
||||
var z T
|
||||
if t == z {
|
||||
return nil
|
||||
}
|
||||
return &t
|
||||
}
|
||||
|
||||
// PtrValueOr returns either value of ptr, or default.
|
||||
func PtrValueOr[T any](t *T, _default T) T {
|
||||
if t != nil {
|
||||
|
|
|
|||
Loading…
Reference in New Issue