import { AudioCodec, MediaContainer, VideoCodec } from "music-server-shared/types"; import { audioCodecToken, containerMimeType, containerNamesCodecs, createRFC6381String, ladderMaxWidth, type VideoStreamInfo, videoCodecToken, } from "music-server-shared/utils"; import { StreamingMode, VIDEO_BITRATES } from "./types"; export interface Support { supported: boolean; reason?: string; } //what a transcoding profile would produce, which is all it takes to decide whether a browser can play it. //deliberately the shape of AppOptions' two settings objects, so either can be passed straight in and a //single field varied with a spread - no argument threading and no positional mix-ups at the call sites export interface Combination { container: MediaContainer; streamingMode: StreamingMode; audioCodec: AudioCodec; videoCodec?: VideoCodec; videoBitrate?: number; } export const hasMediaSource = "MediaSource" in window; export const DEFAULT_STREAMING_MODE = hasMediaSource ? StreamingMode.mse : StreamingMode.progressive; //the W3C MSE Byte Stream Format Registry is a closed list: ISO BMFF, WebM, MPEG-2 TS and MPEG Audio. a //container missing from it cannot be fed to a SourceBuffer by any conforming browser, no matter how well //that same browser plays it progressively - so this is a fact about the spec, not something to probe for const MSE_BYTE_STREAM_FORMATS: MediaContainer[] = [MediaContainer.mp4, MediaContainer.webm, MediaContainer.mp3]; //audio codecs that are at least plausible in each container - ffmpeg refuses the rest outright, so there //is no browser verdict to ask for const CONTAINER_AUDIO_CODECS: Record = { [MediaContainer.mp4]: [AudioCodec.mp3, AudioCodec.opus, AudioCodec.aac, AudioCodec.flac], [MediaContainer.webm]: [AudioCodec.opus, AudioCodec.vorbis], [MediaContainer.ogg]: [AudioCodec.opus, AudioCodec.vorbis, AudioCodec.flac], [MediaContainer.mp3]: [AudioCodec.mp3], }; const CONTAINER_VIDEO_CODECS: Record = { [MediaContainer.mp4]: [VideoCodec.av1, VideoCodec.h264, VideoCodec.h265, VideoCodec.vp9], [MediaContainer.webm]: [VideoCodec.av1, VideoCodec.vp9], [MediaContainer.ogg]: [], [MediaContainer.mp3]: [], }; export function containerAudioCodecs(container: MediaContainer): AudioCodec[] { return CONTAINER_AUDIO_CODECS[container]; } //"none" is always an option: every container can carry audio alone export function containerVideoCodecs(container: MediaContainer): VideoCodec[] { return [...CONTAINER_VIDEO_CODECS[container], VideoCodec.none]; } //actual values don't seem to matter, besides for a possible "smoothnes" verdict //though any valid codec string should result in the same playability result for the same codec const NOMINAL_FRAMERATE = 30; const NOMINAL_UNSCALED_WIDTH = 3840; function nominalVideoStream(videoBitrate: number): VideoStreamInfo { const width = ladderMaxWidth(videoBitrate) ?? NOMINAL_UNSCALED_WIDTH; return { sourceWidth: width, sourceHeight: Math.round(width / (16 / 9) / 2) * 2, frameRate: NOMINAL_FRAMERATE, interlaced: false, bitrate: videoBitrate, }; } let probeElement: HTMLVideoElement | undefined; function progressivelyPlayable(mimeType: string): boolean { probeElement ??= document.createElement("video"); return probeElement.canPlayType(mimeType) !== ""; } //MediaSource.isTypeSupported is the cheap gate but a lenient one: it parses the type and checks the //codecs in isolation, so it says yes to combinations the decoder then refuses. decodingInfo asks the //actual decoder pipeline, per track, with the resolution and framerate attached interface DecodingVerdict { supported: boolean; smooth: boolean; } //the final verdict for every combination initCapabilities enumerates - the only thing checkSupport reads const supportCache = new Map(); const pending = new Map>(); //buffered and progressive have the same support: both are checked with canPlayType and probed as decoding //type "file", so the key collapses them and initCapabilities only has to enumerate one of them. the //bitrate is keyed only when there is a video track, so an audio-only verdict is not stored once per //resolution ladder rung for an answer that cannot depend on it function cacheKey(combination: Combination): string { const { container, streamingMode, audioCodec, videoBitrate } = combination; const videoCodec = combination.videoCodec ?? VideoCodec.none; const mode = streamingMode === StreamingMode.mse ? "mse" : "file"; return `${mode}|${container}|${audioCodec}|${videoCodec}|${videoCodec === VideoCodec.none ? "" : videoBitrate}`; } function startDecodingProbe(key: string, configuration: MediaDecodingConfiguration): Promise { const probe = navigator.mediaCapabilities .decodingInfo(configuration) .then((info) => ({ supported: info.supported, smooth: info.smooth })) .catch(() => { //a configuration this browser cannot even parse tells us nothing beyond what the sync checks //already established, so record the permissive answer rather than disabling a working option return { supported: true, smooth: true }; }) .finally(() => { pending.delete(key); }); pending.set(key, probe); return probe; } function decodingSupport(verdict: DecodingVerdict, mimeType: string): Support { if (!verdict.supported) return { supported: false, reason: `the browser's decoders reject ${mimeType}` }; if (!verdict.smooth) return { supported: true, reason: "supported, but the browser does not expect to decode this smoothly" }; return { supported: true }; } function probeSupport(combination: Combination): void { const { container, streamingMode, audioCodec } = combination; const videoCodec = combination.videoCodec ?? VideoCodec.none; const hasVideo = videoCodec !== VideoCodec.none; const key = cacheKey(combination); if (streamingMode === StreamingMode.mse) { if (!hasMediaSource) { supportCache.set(key, { supported: false, reason: "this browser has no MediaSource support" }); return; } if (!MSE_BYTE_STREAM_FORMATS.includes(container)) { supportCache.set(key, { supported: false, reason: `${container} has no registered MSE byte stream format, so no browser can stream it - pick progressive or buffered`, }); return; } //measured against -live 1, -dash 1, -default_mode infer, explicit dispositions, forced cluster //limits and both track orders: the append succeeds, updateend fires, and buffered stays empty if (container === MediaContainer.webm && hasVideo) { supportCache.set(key, { supported: false, reason: "effectively unsupported by browsers in mse" }); return; } } const video = hasVideo ? nominalVideoStream(combination.videoBitrate ?? 0) : undefined; const mimeType = createRFC6381String(container, videoCodec, audioCodec, video); if (streamingMode === StreamingMode.mse) { if (!MediaSource.isTypeSupported(mimeType)) { supportCache.set(key, { supported: false, reason: `the browser rejects ${mimeType} in a SourceBuffer` }); return; } } else if (!progressivelyPlayable(mimeType)) { supportCache.set(key, { supported: false, reason: `the browser cannot play ${mimeType}` }); return; } //decodingInfo needs one content type per track, which is why the tokens are built here rather than //reusing the combined string above. a container that names no codec has nothing to ask about if (!containerNamesCodecs(container)) { supportCache.set(key, { supported: true }); return; } const configuration: MediaDecodingConfiguration = { type: streamingMode === StreamingMode.mse ? "media-source" : "file", audio: { contentType: `${containerMimeType(container, false)}; codecs="${audioCodecToken(audioCodec)}"` }, ...(video ? { video: { contentType: `${containerMimeType(container, true)}; codecs="${videoCodecToken(videoCodec, video)}"`, width: video.sourceWidth, height: video.sourceHeight, bitrate: video.bitrate, framerate: video.frameRate, }, } : {}), }; const decodingKey = `${configuration.type}|${configuration.audio?.contentType}|${configuration.video?.contentType}|${video?.bitrate}`; const probe = pending.get(decodingKey) ?? startDecodingProbe(decodingKey, configuration); void probe.then((verdict) => supportCache.set(key, decodingSupport(verdict, mimeType))); } export function checkSupport(combination: Combination): Support { const support = supportCache.get(cacheKey(combination)); if (support === undefined) { console.error("support queried for a combination that was never probed", combination); return { supported: false, reason: "this combination was never probed - a bug, see the console" }; } return support; } //a container is offerable when at least one codec combination inside it works in the current mode function checkContainer(combination: Combination, videoCodecs: VideoCodec[]): Support { let firstFailure: Support | undefined; for (const videoCodec of videoCodecs) { for (const audioCodec of containerAudioCodecs(combination.container)) { const support = checkSupport({ ...combination, videoCodec, audioCodec }); if (support.supported) return { supported: true }; firstFailure ??= support; } } return firstFailure ?? { supported: false, reason: `${combination.container} offers no usable codecs` }; } //an audio-only profile must not be told mp4 works because mp4 video works, only because mp4 audio does - //and an absent videoCodec is exactly what makes a profile audio-only export function checkContainerSupport(combination: Combination): Support { return checkContainer( combination, combination.videoCodec === undefined ? [VideoCodec.none] : containerVideoCodecs(combination.container), ); } function usableAudioCodecs(combination: Combination): AudioCodec[] { return containerAudioCodecs(combination.container).filter( (audioCodec) => checkSupport({ ...combination, audioCodec }).supported, ); } function usableVideoCodecs(combination: Combination): VideoCodec[] { return containerVideoCodecs(combination.container).filter( (videoCodec) => checkContainer(combination, [videoCodec]).supported, ); } function usableContainers(combination: Combination): MediaContainer[] { return Object.values(MediaContainer).filter( (container) => checkContainerSupport({ ...combination, container }).supported, ); } //used to decide whether the video section is worth showing at all - it is not when nothing but "none" would //be selectable export function selectableVideoCodecs(combination: Combination): VideoCodec[] { return usableVideoCodecs(combination).filter((videoCodec) => videoCodec !== VideoCodec.none); } function firstUsable(current: T, usable: T[]): T | undefined { return usable.includes(current) ? current : usable[0]; } function knownMember(values: T[], value: T, fallback: T): T { return values.includes(value) ? value : fallback; } //make sure settings are playable (e.g. might come from an older browser version with different codec support) export function playableSettings(settings: T): T { const known = { ...settings, streamingMode: knownMember(Object.values(StreamingMode), settings.streamingMode, DEFAULT_STREAMING_MODE), container: knownMember(Object.values(MediaContainer), settings.container, MediaContainer.mp4), audioCodec: knownMember(Object.values(AudioCodec), settings.audioCodec, AudioCodec.opus), ...(settings.videoCodec === undefined ? {} : { videoCodec: knownMember(Object.values(VideoCodec), settings.videoCodec, VideoCodec.none) }), ...(settings.videoBitrate === undefined ? {} : { videoBitrate: knownMember(VIDEO_BITRATES, settings.videoBitrate, 8_000_000) }), }; if (checkSupport(known).supported) return known; //outermost field first: the container decides which codecs are on offer, and the video codec decides //which audio codecs are left over const container = firstUsable(known.container, usableContainers(known)); if (container === undefined) { if (known.streamingMode === DEFAULT_STREAMING_MODE) return known; return playableSettings({ ...known, streamingMode: DEFAULT_STREAMING_MODE }); } const withContainer = { ...known, container }; const videoCodec = known.videoCodec === undefined ? undefined : firstUsable(known.videoCodec, usableVideoCodecs(withContainer)); const withVideoCodec = videoCodec === undefined ? withContainer : { ...withContainer, videoCodec }; const audioCodec = firstUsable(withVideoCodec.audioCodec, usableAudioCodecs(withVideoCodec)); //the container was only offered because some codec pair inside it works, so this cannot come up empty if (audioCodec === undefined) return known; return { ...withVideoCodec, audioCodec }; } export async function initCapabilities(): Promise { //buffered and progressive have the same support ("file"), so only one of them is probed for (const streamingMode of [StreamingMode.mse, StreamingMode.progressive]) { for (const container of Object.values(MediaContainer)) { for (const audioCodec of containerAudioCodecs(container)) { for (const videoCodec of containerVideoCodecs(container)) { //the bitrate picks the resolution ladder rung and with it the codec level, so each one the //menu offers is its own probe. an audio-only combination has none to vary - and "none" is //exactly what an absent videoCodec resolves to in probeSupport const videoBitrates = videoCodec === VideoCodec.none ? [undefined] : VIDEO_BITRATES; for (const videoBitrate of videoBitrates) { probeSupport({ container, streamingMode, audioCodec, videoCodec, videoBitrate }); } } } } } await Promise.all(pending.values()); }