pico-pixel-player

pico-pixel-player/server/src/ffmpeg.ts
ffmpeg.ts
⎇
Raw
1import { AudioCodec, MediaContainer, type Metadata, VideoCodec, VideoEncodingSetting } from "music-server-shared/types";
2import { createRFC6381String, ladderMaxWidth, videoStreamInfo } from "music-server-shared/utils";
3
4const KEYFRAME_INTERVAL_SECONDS = 5;
5
6//+frag_keyframe fragments the mp4 at video keyframes, of which an audio-only stream has none. without
7//an explicit duration the entire track ends up as a single moof/mdat pair
8const AUDIO_FRAGMENT_MICROSECONDS = 2_000_000;
9
10const AUDIO_ENCODERS: Record<AudioCodec, string> = {
11 [AudioCodec.aac]: "aac",
12 [AudioCodec.mp3]: "libmp3lame",
13 [AudioCodec.opus]: "libopus",
14 [AudioCodec.vorbis]: "libvorbis",
15 [AudioCodec.flac]: "flac",
16};
17
18//x264 and x265 share the same preset names
19const X26X_PRESETS: Record<VideoEncodingSetting, string> = {
20 [VideoEncodingSetting.veryfast]: "veryfast",
21 [VideoEncodingSetting.fast]: "fast",
22 [VideoEncodingSetting.balanced]: "medium",
23 [VideoEncodingSetting.quality]: "slow",
24};
25const AV1_PRESETS: Record<VideoEncodingSetting, string> = {
26 [VideoEncodingSetting.veryfast]: "8",
27 [VideoEncodingSetting.fast]: "7",
28 [VideoEncodingSetting.balanced]: "6",
29 [VideoEncodingSetting.quality]: "5",
30};
31//libvpx wants a deadline/cpu-used pair instead of a preset name
32const VP9_PRESETS: Record<VideoEncodingSetting, string[]> = {
33 [VideoEncodingSetting.veryfast]: ["-deadline", "realtime", "-cpu-used", "5"],
34 [VideoEncodingSetting.fast]: ["-deadline", "realtime", "-cpu-used", "4"],
35 [VideoEncodingSetting.balanced]: ["-deadline", "good", "-cpu-used", "2"],
36 [VideoEncodingSetting.quality]: ["-deadline", "good", "-cpu-used", "0"],
37};
38
39//Choose the first matching preferred language. Audio always needs a track, so an unmatched
40//preference falls back to the source default and then the first audio stream.
41function selectAudioTrack(audioLanguagePreference: string, tracks: NonNullable<Metadata["audioTracks"]>): number {
42 const preferences = audioLanguagePreference
43 .split(",")
44 .map((language) => language.trim().toLowerCase())
45 .filter(Boolean);
46 let best = -1;
47 let bestPreference = Number.POSITIVE_INFINITY;
48 for (let index = 0; index < tracks.length; index++) {
49 const language = tracks[index].language?.trim().toLowerCase();
50 const preference = language ? preferences.indexOf(language) : -1;
51 if (preference >= 0 && preference < bestPreference) {
52 best = index;
53 bestPreference = preference;
54 if (preference === 0) break;
55 }
56 }
57 if (best >= 0) return best;
58 const defaultTrack = tracks.findIndex((track) => track.default);
59 return defaultTrack >= 0 ? defaultTrack : 0;
60}
61
62function videoArgsFor(
63 requestedVideoCodec: VideoCodec,
64 requestedVideoEncodingSetting: VideoEncodingSetting,
65 videoBitrate: number,
66): string[] {
67 const args: string[] = [];
68 //TODO: support hardware encoding
69 switch (requestedVideoCodec) {
70 case VideoCodec.av1:
71 args.push("-c:v", "libsvtav1", "-preset", AV1_PRESETS[requestedVideoEncodingSetting]);
72 break;
73 case VideoCodec.h264:
74 args.push("-c:v", "libx264", "-profile:v", "high");
75 args.push("-preset", X26X_PRESETS[requestedVideoEncodingSetting]);
76 break;
77 case VideoCodec.h265:
78 args.push("-c:v", "libx265", "-profile:v", "main");
79 args.push("-preset", X26X_PRESETS[requestedVideoEncodingSetting]);
80 args.push("-x265-params", "log-level=error");
81 break;
82 case VideoCodec.vp9:
83 args.push("-c:v", "libvpx-vp9", ...VP9_PRESETS[requestedVideoEncodingSetting]);
84 args.push("-row-mt", "1", "-tile-columns", "2");
85 break;
86 default:
87 throw new Error(`Unsupported video codec: ${requestedVideoCodec}`);
88 }
89
90 const maxWidth = ladderMaxWidth(videoBitrate);
91 args.push(
92 "-b:v",
93 videoBitrate.toString(), //TODO: better bitrate selection
94 //make sure pixel format is browser compatible
95 "-pix_fmt",
96 "yuv420p",
97 //automatically deinterlace video to double fps if video is interlaced
98 //scale video down based on bitrate
99 "-vf",
100 `yadif=deint=1:mode=1${maxWidth ? `, scale='if(gt(iw,${maxWidth}),${maxWidth},iw)':-2` : ""}`,
101 "-force_key_frames",
102 `expr:gte(t,n_forced*${KEYFRAME_INTERVAL_SECONDS})`,
103 "-map",
104 "0:v:0",
105 );
106 return args;
107}
108
109export async function convertWithFFmpeg(
110 filePath: string,
111 //undefined means no -b:a at all, which is the only setting that means anything for a lossless codec
112 audioBitrate: number | undefined,
113 videoBitrate: number,
114 container: MediaContainer,
115 requestedAudioCodec: AudioCodec,
116 requestedVideoCodec: VideoCodec,
117 requestedVideoEncodingSetting: VideoEncodingSetting,
118 seekTo: number | undefined,
119 audioLanguagePreference: string,
120 metadata: Metadata,
121) {
122 const video = videoStreamInfo(metadata, videoBitrate);
123 const mimeType = createRFC6381String(container, requestedVideoCodec, requestedAudioCodec, video);
124 const audioEncoder = AUDIO_ENCODERS[requestedAudioCodec];
125 const hasVideo = requestedVideoCodec !== VideoCodec.none;
126
127 const containerArgs: string[] = [];
128 if (container === MediaContainer.mp4) {
129 //note: no +faststart - it relocates the moov atom, which needs a seekable output, and this
130 //muxes to a pipe
131 containerArgs.push("-avoid_negative_ts", "make_zero", "-movflags", "+frag_keyframe+empty_moov+default_base_moof");
132 if (!hasVideo) containerArgs.push("-frag_duration", AUDIO_FRAGMENT_MICROSECONDS.toString());
133 }
134
135 const selectedAudioTrack = selectAudioTrack(audioLanguagePreference, metadata.audioTracks || []);
136 const args = [
137 "ffmpeg",
138 "-nostats",
139 "-loglevel",
140 "error",
141 //put seek first to do a fast keyframe seek
142 ...(seekTo === undefined ? [] : ["-ss", seekTo.toString()]),
143 "-i",
144 filePath,
145 ...containerArgs,
146 ...(hasVideo ? videoArgsFor(requestedVideoCodec, requestedVideoEncodingSetting, videoBitrate) : ["-vn"]),
147 "-map",
148 `0:a:${selectedAudioTrack}?`,
149 "-c:a",
150 audioEncoder,
151 ...(audioBitrate === undefined ? [] : ["-b:a", audioBitrate.toString()]),
152 "-ac",
153 "2",
154 "-f",
155 container,
156 "-",
157 ];
158 console.log(args.join(" "));
159 const cmd = Bun.spawn(args, { stdin: "ignore", stdout: "pipe", stderr: "pipe" });
160 const stderrText = new Response(cmd.stderr).text().catch(() => "");
161 return { cmd, mimeType, stderrText };
162}
163
164export async function convertSubtitleWithFFmpeg(filePath: string, streamIndex: number) {
165 const args = [
166 "ffmpeg",
167 "-nostats",
168 "-loglevel",
169 "error",
170 "-i",
171 filePath,
172 "-map",
173 `0:s:${streamIndex}`,
174 "-c:s",
175 "webvtt",
176 "-f",
177 "webvtt",
178 "-",
179 ];
180 console.log(args.join(" "));
181 const cmd = Bun.spawn(args, { stdin: "ignore", stdout: "pipe", stderr: "pipe" });
182 const stderrText = new Response(cmd.stderr).text().catch(() => "");
183 return { cmd, mimeType: "text/vtt; charset=utf-8", stderrText };
184}
185