pico-pixel-player

pico-pixel-player/webclient/src/capabilities.ts
capabilities.ts
⎇
Raw
1import { AudioCodec, MediaContainer, VideoCodec } from "music-server-shared/types";
2import {
3 audioCodecToken,
4 containerMimeType,
5 containerNamesCodecs,
6 createRFC6381String,
7 ladderMaxWidth,
8 type VideoStreamInfo,
9 videoCodecToken,
10} from "music-server-shared/utils";
11import { StreamingMode, VIDEO_BITRATES } from "./types";
12
13export interface Support {
14 supported: boolean;
15 reason?: string;
16}
17
18//what a transcoding profile would produce, which is all it takes to decide whether a browser can play it.
19//deliberately the shape of AppOptions' two settings objects, so either can be passed straight in and a
20//single field varied with a spread - no argument threading and no positional mix-ups at the call sites
21export interface Combination {
22 container: MediaContainer;
23 streamingMode: StreamingMode;
24 audioCodec: AudioCodec;
25 videoCodec?: VideoCodec;
26 videoBitrate?: number;
27}
28
29export const hasMediaSource = "MediaSource" in window;
30export const DEFAULT_STREAMING_MODE = hasMediaSource ? StreamingMode.mse : StreamingMode.progressive;
31
32//the W3C MSE Byte Stream Format Registry is a closed list: ISO BMFF, WebM, MPEG-2 TS and MPEG Audio. a
33//container missing from it cannot be fed to a SourceBuffer by any conforming browser, no matter how well
34//that same browser plays it progressively - so this is a fact about the spec, not something to probe for
35const MSE_BYTE_STREAM_FORMATS: MediaContainer[] = [MediaContainer.mp4, MediaContainer.webm, MediaContainer.mp3];
36
37//audio codecs that are at least plausible in each container - ffmpeg refuses the rest outright, so there
38//is no browser verdict to ask for
39const CONTAINER_AUDIO_CODECS: Record<MediaContainer, AudioCodec[]> = {
40 [MediaContainer.mp4]: [AudioCodec.mp3, AudioCodec.opus, AudioCodec.aac, AudioCodec.flac],
41 [MediaContainer.webm]: [AudioCodec.opus, AudioCodec.vorbis],
42 [MediaContainer.ogg]: [AudioCodec.opus, AudioCodec.vorbis, AudioCodec.flac],
43 [MediaContainer.mp3]: [AudioCodec.mp3],
44};
45
46const CONTAINER_VIDEO_CODECS: Record<MediaContainer, VideoCodec[]> = {
47 [MediaContainer.mp4]: [VideoCodec.av1, VideoCodec.h264, VideoCodec.h265, VideoCodec.vp9],
48 [MediaContainer.webm]: [VideoCodec.av1, VideoCodec.vp9],
49 [MediaContainer.ogg]: [],
50 [MediaContainer.mp3]: [],
51};
52
53export function containerAudioCodecs(container: MediaContainer): AudioCodec[] {
54 return CONTAINER_AUDIO_CODECS[container];
55}
56
57//"none" is always an option: every container can carry audio alone
58export function containerVideoCodecs(container: MediaContainer): VideoCodec[] {
59 return [...CONTAINER_VIDEO_CODECS[container], VideoCodec.none];
60}
61
62//actual values don't seem to matter, besides for a possible "smoothnes" verdict
63//though any valid codec string should result in the same playability result for the same codec
64const NOMINAL_FRAMERATE = 30;
65const NOMINAL_UNSCALED_WIDTH = 3840;
66
67function nominalVideoStream(videoBitrate: number): VideoStreamInfo {
68 const width = ladderMaxWidth(videoBitrate) ?? NOMINAL_UNSCALED_WIDTH;
69 return {
70 sourceWidth: width,
71 sourceHeight: Math.round(width / (16 / 9) / 2) * 2,
72 frameRate: NOMINAL_FRAMERATE,
73 interlaced: false,
74 bitrate: videoBitrate,
75 };
76}
77
78let probeElement: HTMLVideoElement | undefined;
79
80function progressivelyPlayable(mimeType: string): boolean {
81 probeElement ??= document.createElement("video");
82 return probeElement.canPlayType(mimeType) !== "";
83}
84
85//MediaSource.isTypeSupported is the cheap gate but a lenient one: it parses the type and checks the
86//codecs in isolation, so it says yes to combinations the decoder then refuses. decodingInfo asks the
87//actual decoder pipeline, per track, with the resolution and framerate attached
88interface DecodingVerdict {
89 supported: boolean;
90 smooth: boolean;
91}
92
93//the final verdict for every combination initCapabilities enumerates - the only thing checkSupport reads
94const supportCache = new Map<string, Support>();
95const pending = new Map<string, Promise<DecodingVerdict>>();
96
97//buffered and progressive have the same support: both are checked with canPlayType and probed as decoding
98//type "file", so the key collapses them and initCapabilities only has to enumerate one of them. the
99//bitrate is keyed only when there is a video track, so an audio-only verdict is not stored once per
100//resolution ladder rung for an answer that cannot depend on it
101function cacheKey(combination: Combination): string {
102 const { container, streamingMode, audioCodec, videoBitrate } = combination;
103 const videoCodec = combination.videoCodec ?? VideoCodec.none;
104 const mode = streamingMode === StreamingMode.mse ? "mse" : "file";
105 return `${mode}|${container}|${audioCodec}|${videoCodec}|${videoCodec === VideoCodec.none ? "" : videoBitrate}`;
106}
107
108function startDecodingProbe(key: string, configuration: MediaDecodingConfiguration): Promise<DecodingVerdict> {
109 const probe = navigator.mediaCapabilities
110 .decodingInfo(configuration)
111 .then((info) => ({ supported: info.supported, smooth: info.smooth }))
112 .catch(() => {
113 //a configuration this browser cannot even parse tells us nothing beyond what the sync checks
114 //already established, so record the permissive answer rather than disabling a working option
115 return { supported: true, smooth: true };
116 })
117 .finally(() => {
118 pending.delete(key);
119 });
120 pending.set(key, probe);
121 return probe;
122}
123
124function decodingSupport(verdict: DecodingVerdict, mimeType: string): Support {
125 if (!verdict.supported) return { supported: false, reason: `the browser's decoders reject ${mimeType}` };
126 if (!verdict.smooth)
127 return { supported: true, reason: "supported, but the browser does not expect to decode this smoothly" };
128 return { supported: true };
129}
130
131function probeSupport(combination: Combination): void {
132 const { container, streamingMode, audioCodec } = combination;
133 const videoCodec = combination.videoCodec ?? VideoCodec.none;
134 const hasVideo = videoCodec !== VideoCodec.none;
135 const key = cacheKey(combination);
136 if (streamingMode === StreamingMode.mse) {
137 if (!hasMediaSource) {
138 supportCache.set(key, { supported: false, reason: "this browser has no MediaSource support" });
139 return;
140 }
141 if (!MSE_BYTE_STREAM_FORMATS.includes(container)) {
142 supportCache.set(key, {
143 supported: false,
144 reason: `${container} has no registered MSE byte stream format, so no browser can stream it - pick progressive or buffered`,
145 });
146 return;
147 }
148 //measured against -live 1, -dash 1, -default_mode infer, explicit dispositions, forced cluster
149 //limits and both track orders: the append succeeds, updateend fires, and buffered stays empty
150 if (container === MediaContainer.webm && hasVideo) {
151 supportCache.set(key, { supported: false, reason: "effectively unsupported by browsers in mse" });
152 return;
153 }
154 }
155
156 const video = hasVideo ? nominalVideoStream(combination.videoBitrate ?? 0) : undefined;
157 const mimeType = createRFC6381String(container, videoCodec, audioCodec, video);
158
159 if (streamingMode === StreamingMode.mse) {
160 if (!MediaSource.isTypeSupported(mimeType)) {
161 supportCache.set(key, { supported: false, reason: `the browser rejects ${mimeType} in a SourceBuffer` });
162 return;
163 }
164 } else if (!progressivelyPlayable(mimeType)) {
165 supportCache.set(key, { supported: false, reason: `the browser cannot play ${mimeType}` });
166 return;
167 }
168
169 //decodingInfo needs one content type per track, which is why the tokens are built here rather than
170 //reusing the combined string above. a container that names no codec has nothing to ask about
171 if (!containerNamesCodecs(container)) {
172 supportCache.set(key, { supported: true });
173 return;
174 }
175 const configuration: MediaDecodingConfiguration = {
176 type: streamingMode === StreamingMode.mse ? "media-source" : "file",
177 audio: { contentType: `${containerMimeType(container, false)}; codecs="${audioCodecToken(audioCodec)}"` },
178 ...(video
179 ? {
180 video: {
181 contentType: `${containerMimeType(container, true)}; codecs="${videoCodecToken(videoCodec, video)}"`,
182 width: video.sourceWidth,
183 height: video.sourceHeight,
184 bitrate: video.bitrate,
185 framerate: video.frameRate,
186 },
187 }
188 : {}),
189 };
190
191 const decodingKey = `${configuration.type}|${configuration.audio?.contentType}|${configuration.video?.contentType}|${video?.bitrate}`;
192 const probe = pending.get(decodingKey) ?? startDecodingProbe(decodingKey, configuration);
193 void probe.then((verdict) => supportCache.set(key, decodingSupport(verdict, mimeType)));
194}
195
196export function checkSupport(combination: Combination): Support {
197 const support = supportCache.get(cacheKey(combination));
198 if (support === undefined) {
199 console.error("support queried for a combination that was never probed", combination);
200 return { supported: false, reason: "this combination was never probed - a bug, see the console" };
201 }
202 return support;
203}
204
205//a container is offerable when at least one codec combination inside it works in the current mode
206function checkContainer(combination: Combination, videoCodecs: VideoCodec[]): Support {
207 let firstFailure: Support | undefined;
208 for (const videoCodec of videoCodecs) {
209 for (const audioCodec of containerAudioCodecs(combination.container)) {
210 const support = checkSupport({ ...combination, videoCodec, audioCodec });
211 if (support.supported) return { supported: true };
212 firstFailure ??= support;
213 }
214 }
215 return firstFailure ?? { supported: false, reason: `${combination.container} offers no usable codecs` };
216}
217
218//an audio-only profile must not be told mp4 works because mp4 video works, only because mp4 audio does -
219//and an absent videoCodec is exactly what makes a profile audio-only
220export function checkContainerSupport(combination: Combination): Support {
221 return checkContainer(
222 combination,
223 combination.videoCodec === undefined ? [VideoCodec.none] : containerVideoCodecs(combination.container),
224 );
225}
226
227function usableAudioCodecs(combination: Combination): AudioCodec[] {
228 return containerAudioCodecs(combination.container).filter(
229 (audioCodec) => checkSupport({ ...combination, audioCodec }).supported,
230 );
231}
232
233function usableVideoCodecs(combination: Combination): VideoCodec[] {
234 return containerVideoCodecs(combination.container).filter(
235 (videoCodec) => checkContainer(combination, [videoCodec]).supported,
236 );
237}
238
239function usableContainers(combination: Combination): MediaContainer[] {
240 return Object.values(MediaContainer).filter(
241 (container) => checkContainerSupport({ ...combination, container }).supported,
242 );
243}
244
245//used to decide whether the video section is worth showing at all - it is not when nothing but "none" would
246//be selectable
247export function selectableVideoCodecs(combination: Combination): VideoCodec[] {
248 return usableVideoCodecs(combination).filter((videoCodec) => videoCodec !== VideoCodec.none);
249}
250
251function firstUsable<T>(current: T, usable: T[]): T | undefined {
252 return usable.includes(current) ? current : usable[0];
253}
254
255function knownMember<T>(values: T[], value: T, fallback: T): T {
256 return values.includes(value) ? value : fallback;
257}
258
259//make sure settings are playable (e.g. might come from an older browser version with different codec support)
260export function playableSettings<T extends Combination>(settings: T): T {
261 const known = {
262 ...settings,
263 streamingMode: knownMember(Object.values(StreamingMode), settings.streamingMode, DEFAULT_STREAMING_MODE),
264 container: knownMember(Object.values(MediaContainer), settings.container, MediaContainer.mp4),
265 audioCodec: knownMember(Object.values(AudioCodec), settings.audioCodec, AudioCodec.opus),
266 ...(settings.videoCodec === undefined
267 ? {}
268 : { videoCodec: knownMember(Object.values(VideoCodec), settings.videoCodec, VideoCodec.none) }),
269 ...(settings.videoBitrate === undefined
270 ? {}
271 : { videoBitrate: knownMember(VIDEO_BITRATES, settings.videoBitrate, 8_000_000) }),
272 };
273 if (checkSupport(known).supported) return known;
274
275 //outermost field first: the container decides which codecs are on offer, and the video codec decides
276 //which audio codecs are left over
277 const container = firstUsable(known.container, usableContainers(known));
278 if (container === undefined) {
279 if (known.streamingMode === DEFAULT_STREAMING_MODE) return known;
280 return playableSettings({ ...known, streamingMode: DEFAULT_STREAMING_MODE });
281 }
282 const withContainer = { ...known, container };
283
284 const videoCodec =
285 known.videoCodec === undefined ? undefined : firstUsable(known.videoCodec, usableVideoCodecs(withContainer));
286 const withVideoCodec = videoCodec === undefined ? withContainer : { ...withContainer, videoCodec };
287
288 const audioCodec = firstUsable(withVideoCodec.audioCodec, usableAudioCodecs(withVideoCodec));
289 //the container was only offered because some codec pair inside it works, so this cannot come up empty
290 if (audioCodec === undefined) return known;
291 return { ...withVideoCodec, audioCodec };
292}
293
294export async function initCapabilities(): Promise<void> {
295 //buffered and progressive have the same support ("file"), so only one of them is probed
296 for (const streamingMode of [StreamingMode.mse, StreamingMode.progressive]) {
297 for (const container of Object.values(MediaContainer)) {
298 for (const audioCodec of containerAudioCodecs(container)) {
299 for (const videoCodec of containerVideoCodecs(container)) {
300 //the bitrate picks the resolution ladder rung and with it the codec level, so each one the
301 //menu offers is its own probe. an audio-only combination has none to vary - and "none" is
302 //exactly what an absent videoCodec resolves to in probeSupport
303 const videoBitrates = videoCodec === VideoCodec.none ? [undefined] : VIDEO_BITRATES;
304 for (const videoBitrate of videoBitrates) {
305 probeSupport({ container, streamingMode, audioCodec, videoCodec, videoBitrate });
306 }
307 }
308 }
309 }
310 }
311 await Promise.all(pending.values());
312}
313