pico-pixel-player

pico-pixel-player/server/src/ffmpeg.ts
ffmpeg.ts
⎇
Raw
1import { AudioCodec, MediaContainer, type Metadata, VideoCodec, VideoEncodingSetting } from "music-server-shared/types";
2import { createRFC6381String, ladderMaxWidth, videoStreamInfo } from "music-server-shared/utils";
3
4const KEYFRAME_INTERVAL_SECONDS = 5;
5
6//+frag_keyframe fragments the mp4 at video keyframes, of which an audio-only stream has none. without
7//an explicit duration the entire track ends up as a single moof/mdat pair
8const AUDIO_FRAGMENT_MICROSECONDS = 2_000_000;
9
10const AUDIO_ENCODERS: Record<AudioCodec, string> = {
11 [AudioCodec.aac]: "aac",
12 [AudioCodec.mp3]: "libmp3lame",
13 [AudioCodec.opus]: "libopus",
14 [AudioCodec.vorbis]: "libvorbis",
15 [AudioCodec.flac]: "flac",
16};
17
18//x264 and x265 share the same preset names
19const X26X_PRESETS: Record<VideoEncodingSetting, string> = {
20 [VideoEncodingSetting.veryfast]: "veryfast",
21 [VideoEncodingSetting.fast]: "fast",
22 [VideoEncodingSetting.balanced]: "medium",
23 [VideoEncodingSetting.quality]: "slow",
24};
25const AV1_PRESETS: Record<VideoEncodingSetting, string> = {
26 [VideoEncodingSetting.veryfast]: "8",
27 [VideoEncodingSetting.fast]: "7",
28 [VideoEncodingSetting.balanced]: "6",
29 [VideoEncodingSetting.quality]: "5",
30};
31//libvpx wants a deadline/cpu-used pair instead of a preset name
32const VP9_PRESETS: Record<VideoEncodingSetting, string[]> = {
33 [VideoEncodingSetting.veryfast]: ["-deadline", "realtime", "-cpu-used", "5"],
34 [VideoEncodingSetting.fast]: ["-deadline", "realtime", "-cpu-used", "4"],
35 [VideoEncodingSetting.balanced]: ["-deadline", "good", "-cpu-used", "2"],
36 [VideoEncodingSetting.quality]: ["-deadline", "good", "-cpu-used", "0"],
37};
38
39function lookup<T>(table: Record<string, T>, key: string, what: string): T {
40 const value = table[key];
41 if (value === undefined) throw new Error(`Unsupported ${what}: ${key}`);
42 return value;
43}
44
45//Choose the first matching preferred language. Audio always needs a track, so an unmatched
46//preference falls back to the source default and then the first audio stream.
47function selectAudioTrack(audioLanguagePreference: string, tracks: NonNullable<Metadata["audioTracks"]>): number {
48 const preferences = audioLanguagePreference
49 .split(",")
50 .map((language) => language.trim().toLowerCase())
51 .filter(Boolean);
52 let best = -1;
53 let bestPreference = Number.POSITIVE_INFINITY;
54 for (let index = 0; index < tracks.length; index++) {
55 const language = tracks[index].language?.trim().toLowerCase();
56 const preference = language ? preferences.indexOf(language) : -1;
57 if (preference >= 0 && preference < bestPreference) {
58 best = index;
59 bestPreference = preference;
60 if (preference === 0) break;
61 }
62 }
63 if (best >= 0) return best;
64 const defaultTrack = tracks.findIndex((track) => track.default);
65 return defaultTrack >= 0 ? defaultTrack : 0;
66}
67
68function videoArgsFor(
69 requestedVideoCodec: VideoCodec,
70 requestedVideoEncodingSetting: VideoEncodingSetting,
71 videoBitrate: number,
72): string[] {
73 const args: string[] = [];
74 //TODO: support hardware encoding
75 switch (requestedVideoCodec) {
76 case VideoCodec.av1:
77 args.push("-c:v", "libsvtav1", "-preset", lookup(AV1_PRESETS, requestedVideoEncodingSetting, "encoding setting"));
78 break;
79 case VideoCodec.h264:
80 args.push("-c:v", "libx264", "-profile:v", "high");
81 args.push("-preset", lookup(X26X_PRESETS, requestedVideoEncodingSetting, "encoding setting"));
82 break;
83 case VideoCodec.h265:
84 args.push("-c:v", "libx265", "-profile:v", "main");
85 args.push("-preset", lookup(X26X_PRESETS, requestedVideoEncodingSetting, "encoding setting"));
86 args.push("-x265-params", "log-level=error");
87 break;
88 case VideoCodec.vp9:
89 args.push("-c:v", "libvpx-vp9", ...lookup(VP9_PRESETS, requestedVideoEncodingSetting, "encoding setting"));
90 args.push("-row-mt", "1", "-tile-columns", "2");
91 break;
92 default:
93 throw new Error(`Unsupported video codec: ${requestedVideoCodec}`);
94 }
95
96 const maxWidth = ladderMaxWidth(videoBitrate);
97 args.push(
98 "-b:v",
99 videoBitrate.toString(), //TODO: better bitrate selection
100 //make sure pixel format is browser compatible
101 "-pix_fmt",
102 "yuv420p",
103 //automatically deinterlace video to double fps if video is interlaced
104 //scale video down based on bitrate
105 "-vf",
106 `yadif=deint=1:mode=1${maxWidth ? `, scale='if(gt(iw,${maxWidth}),${maxWidth},iw)':-2` : ""}`,
107 "-force_key_frames",
108 `expr:gte(t,n_forced*${KEYFRAME_INTERVAL_SECONDS})`,
109 "-map",
110 "0:v:0",
111 );
112 return args;
113}
114
115export async function convertWithFFmpeg(
116 filePath: string,
117 //undefined means no -b:a at all, which is the only setting that means anything for a lossless codec
118 audioBitrate: number | undefined,
119 videoBitrate: number,
120 container: MediaContainer,
121 requestedAudioCodec: AudioCodec,
122 requestedVideoCodec: VideoCodec,
123 requestedVideoEncodingSetting: VideoEncodingSetting,
124 seekTo: number | undefined,
125 audioLanguagePreference: string,
126 metadata: Metadata,
127) {
128 const video = videoStreamInfo(metadata, videoBitrate);
129 const mimeType = createRFC6381String(container, requestedVideoCodec, requestedAudioCodec, video);
130 const audioEncoder = lookup(AUDIO_ENCODERS, requestedAudioCodec, "audio codec");
131 const hasVideo = requestedVideoCodec !== VideoCodec.none;
132
133 const containerArgs: string[] = [];
134 if (container === MediaContainer.mp4) {
135 //note: no +faststart - it relocates the moov atom, which needs a seekable output, and this
136 //muxes to a pipe
137 containerArgs.push("-avoid_negative_ts", "make_zero", "-movflags", "+frag_keyframe+empty_moov+default_base_moof");
138 if (!hasVideo) containerArgs.push("-frag_duration", AUDIO_FRAGMENT_MICROSECONDS.toString());
139 }
140
141 const selectedAudioTrack = selectAudioTrack(audioLanguagePreference, metadata.audioTracks || []);
142 const args = [
143 "ffmpeg",
144 "-nostats",
145 "-loglevel",
146 "error",
147 //put seek first to do a fast keyframe seek
148 ...(seekTo === undefined ? [] : ["-ss", seekTo.toString()]),
149 "-i",
150 filePath,
151 ...containerArgs,
152 ...(hasVideo ? videoArgsFor(requestedVideoCodec, requestedVideoEncodingSetting, videoBitrate) : ["-vn"]),
153 "-map",
154 `0:a:${selectedAudioTrack}?`,
155 "-c:a",
156 audioEncoder,
157 ...(audioBitrate === undefined ? [] : ["-b:a", audioBitrate.toString()]),
158 "-ac",
159 "2",
160 "-f",
161 container,
162 "-",
163 ];
164 console.log(args.join(" "));
165 const cmd = Bun.spawn(args, { stdin: "ignore", stdout: "pipe", stderr: "pipe" });
166 const stderrText = new Response(cmd.stderr).text().catch(() => "");
167 return { cmd, mimeType, stderrText };
168}
169
170export async function convertSubtitleWithFFmpeg(filePath: string, streamIndex: number) {
171 const args = [
172 "ffmpeg",
173 "-nostats",
174 "-loglevel",
175 "error",
176 "-i",
177 filePath,
178 "-map",
179 `0:s:${streamIndex}`,
180 "-c:s",
181 "webvtt",
182 "-f",
183 "webvtt",
184 "-",
185 ];
186 console.log(args.join(" "));
187 const cmd = Bun.spawn(args, { stdin: "ignore", stdout: "pipe", stderr: "pipe" });
188 const stderrText = new Response(cmd.stderr).text().catch(() => "");
189 return { cmd, mimeType: "text/vtt; charset=utf-8", stderrText };
190}
191