pico-pixel-player

pico-pixel-player/server/src/ffmpeg.ts
ffmpeg.ts
⎇
Raw
1import { AudioCodec, MediaContainer, type Metadata, VideoCodec, VideoEncodingSetting } from "music-server-shared/types";
2import { createRFC6381String, ladderMaxWidth, type VideoStreamInfo } from "music-server-shared/utils";
3
4const KEYFRAME_INTERVAL_SECONDS = 5;
5
6//+frag_keyframe fragments the mp4 at video keyframes, of which an audio-only stream has none. without
7//an explicit duration the entire track ends up as a single moof/mdat pair
8const AUDIO_FRAGMENT_MICROSECONDS = 2_000_000;
9
10const AUDIO_ENCODERS: Record<AudioCodec, string> = {
11 [AudioCodec.aac]: "aac",
12 [AudioCodec.mp3]: "libmp3lame",
13 [AudioCodec.opus]: "libopus",
14 [AudioCodec.vorbis]: "libvorbis",
15 [AudioCodec.flac]: "flac",
16};
17
18//x264 and x265 share the same preset names
19const X26X_PRESETS: Record<VideoEncodingSetting, string> = {
20 [VideoEncodingSetting.veryfast]: "veryfast",
21 [VideoEncodingSetting.fast]: "fast",
22 [VideoEncodingSetting.balanced]: "medium",
23 [VideoEncodingSetting.quality]: "slow",
24};
25const AV1_PRESETS: Record<VideoEncodingSetting, string> = {
26 [VideoEncodingSetting.veryfast]: "8",
27 [VideoEncodingSetting.fast]: "7",
28 [VideoEncodingSetting.balanced]: "6",
29 [VideoEncodingSetting.quality]: "5",
30};
31//libvpx wants a deadline/cpu-used pair instead of a preset name
32const VP9_PRESETS: Record<VideoEncodingSetting, string[]> = {
33 [VideoEncodingSetting.veryfast]: ["-deadline", "realtime", "-cpu-used", "5"],
34 [VideoEncodingSetting.fast]: ["-deadline", "realtime", "-cpu-used", "4"],
35 [VideoEncodingSetting.balanced]: ["-deadline", "good", "-cpu-used", "2"],
36 [VideoEncodingSetting.quality]: ["-deadline", "good", "-cpu-used", "0"],
37};
38
39function lookup<T>(table: Record<string, T>, key: string, what: string): T {
40 const value = table[key];
41 if (value === undefined) throw new Error(`Unsupported ${what}: ${key}`);
42 return value;
43}
44
45//pick the audio track matching the earliest entry of the caller's preference list. with no match at
46//all, prefer whichever track the file itself flags as default, and only then fall back to the first
47function selectAudioTrack(languages: string, tracks: NonNullable<Metadata["audioTracks"]>): number {
48 const preferences = languages ? languages.toLocaleLowerCase().split(",") : [];
49 let best = -1;
50 let bestPreference = Number.POSITIVE_INFINITY;
51 for (let i = 0; i < tracks.length; i++) {
52 const language = tracks[i].language?.toLocaleLowerCase();
53 const preference = language ? preferences.indexOf(language) : -1;
54 if (preference >= 0 && preference < bestPreference) {
55 best = i;
56 bestPreference = preference;
57 if (preference === 0) break;
58 }
59 }
60 if (best >= 0) return best;
61 const defaultTrack = tracks.findIndex((track) => track.default);
62 return defaultTrack >= 0 ? defaultTrack : 0;
63}
64
65function videoArgsFor(
66 requestedVideoCodec: VideoCodec,
67 requestedVideoEncodingSetting: VideoEncodingSetting,
68 videoBitrate: number,
69): string[] {
70 const args: string[] = [];
71 //TODO: support hardware encoding
72 switch (requestedVideoCodec) {
73 case VideoCodec.av1:
74 args.push("-c:v", "libsvtav1", "-preset", lookup(AV1_PRESETS, requestedVideoEncodingSetting, "encoding setting"));
75 break;
76 case VideoCodec.h264:
77 args.push("-c:v", "libx264", "-profile:v", "high");
78 args.push("-preset", lookup(X26X_PRESETS, requestedVideoEncodingSetting, "encoding setting"));
79 break;
80 case VideoCodec.h265:
81 args.push("-c:v", "libx265", "-profile:v", "main");
82 args.push("-preset", lookup(X26X_PRESETS, requestedVideoEncodingSetting, "encoding setting"));
83 args.push("-x265-params", "log-level=error");
84 break;
85 case VideoCodec.vp9:
86 args.push("-c:v", "libvpx-vp9", ...lookup(VP9_PRESETS, requestedVideoEncodingSetting, "encoding setting"));
87 args.push("-row-mt", "1", "-tile-columns", "2");
88 break;
89 default:
90 throw new Error(`Unsupported video codec: ${requestedVideoCodec}`);
91 }
92
93 const maxWidth = ladderMaxWidth(videoBitrate);
94 args.push(
95 "-b:v",
96 videoBitrate.toString(), //TODO: better bitrate selection
97 //make sure pixel format is browser compatible
98 "-pix_fmt",
99 "yuv420p",
100 //automatically deinterlace video to double fps if video is interlaced
101 //scale video down based on bitrate
102 "-vf",
103 `yadif=deint=1:mode=1${maxWidth ? `, scale='if(gt(iw,${maxWidth}),${maxWidth},iw)':-2` : ""}`,
104 "-force_key_frames",
105 `expr:gte(t,n_forced*${KEYFRAME_INTERVAL_SECONDS})`,
106 "-map",
107 "0:v:0",
108 );
109 return args;
110}
111
112export async function convertWithFFmpeg(
113 filePath: string,
114 //undefined means no -b:a at all, which is the only setting that means anything for a lossless codec
115 audioBitrate: number | undefined,
116 videoBitrate: number,
117 container: MediaContainer,
118 requestedAudioCodec: AudioCodec,
119 requestedVideoCodec: VideoCodec,
120 requestedVideoEncodingSetting: VideoEncodingSetting,
121 seekTo: number | undefined,
122 languages: string,
123 metadata: Metadata,
124) {
125 const videoStreamInfo: VideoStreamInfo | undefined =
126 metadata.videoWidth && metadata.videoHeight && metadata.frameRate
127 ? {
128 sourceWidth: metadata.videoWidth,
129 sourceHeight: metadata.videoHeight,
130 frameRate: metadata.frameRate,
131 interlaced: metadata.interlaced === true,
132 bitrate: videoBitrate,
133 }
134 : undefined;
135 const mimeType = createRFC6381String(container, requestedVideoCodec, requestedAudioCodec, videoStreamInfo);
136 const audioEncoder = lookup(AUDIO_ENCODERS, requestedAudioCodec, "audio codec");
137 const hasVideo = requestedVideoCodec !== VideoCodec.none;
138
139 const containerArgs: string[] = [];
140 if (container === MediaContainer.mp4) {
141 //note: no +faststart - it relocates the moov atom, which needs a seekable output, and this
142 //muxes to a pipe
143 containerArgs.push("-avoid_negative_ts", "make_zero", "-movflags", "+frag_keyframe+empty_moov+default_base_moof");
144 if (!hasVideo) containerArgs.push("-frag_duration", AUDIO_FRAGMENT_MICROSECONDS.toString());
145 }
146
147 const selectedAudioTrack = selectAudioTrack(languages, metadata.audioTracks || []);
148
149 const args = [
150 "ffmpeg",
151 "-nostats",
152 "-loglevel",
153 "error",
154 //put seek first to do a fast keyframe seek
155 ...(seekTo === undefined ? [] : ["-ss", seekTo.toString()]),
156 "-i",
157 filePath,
158 ...containerArgs,
159 ...(hasVideo ? videoArgsFor(requestedVideoCodec, requestedVideoEncodingSetting, videoBitrate) : ["-vn"]),
160 "-map",
161 `0:a:${selectedAudioTrack}?`,
162 "-c:a",
163 audioEncoder,
164 ...(audioBitrate === undefined ? [] : ["-b:a", audioBitrate.toString()]),
165 "-ac",
166 "2",
167 "-f",
168 container,
169 "-",
170 ];
171 console.log(args.join(" "));
172 const cmd = Bun.spawn(args, { stdin: "ignore", stdout: "pipe", stderr: "pipe" });
173 const stderrText = new Response(cmd.stderr).text().catch(() => "");
174 void cmd.exited
175 .then(async (exitCode) => {
176 if (exitCode !== 0 && exitCode !== null) {
177 console.error("ffmpeg failed with status code", exitCode);
178 console.error(await stderrText);
179 }
180 })
181 .catch(() => {});
182 return { cmd, mimeType, stderrText };
183}
184