import { AudioCodec, MediaContainer, VideoCodec } from "music-server-shared/types"; import { audioCodecToken, containerMimeType, containerNamesCodecs, createRFC6381String, ladderMaxWidth, type VideoStreamInfo, videoCodecToken, } from "music-server-shared/utils"; import { StreamingMode, VIDEO_BITRATES } from "./types"; export interface Support { supported: boolean; reason?: string; } //what a transcoding profile would produce, which is all it takes to decide whether a browser can play it. //deliberately the shape of AppOptions' two settings objects, so either can be passed straight in and a //single field varied with a spread - no argument threading and no positional mix-ups at the call sites export interface Combination { container: MediaContainer; streamingMode: StreamingMode; audioCodec: AudioCodec; videoCodec?: VideoCodec; videoBitrate?: number; } export const hasMediaSource = "MediaSource" in window; export const DEFAULT_STREAMING_MODE = hasMediaSource ? StreamingMode.mse : StreamingMode.progressive; //the W3C MSE Byte Stream Format Registry is a closed list: ISO BMFF, WebM, MPEG-2 TS and MPEG Audio. a //container missing from it cannot be fed to a SourceBuffer by any conforming browser, no matter how well //that same browser plays it progressively - so this is a fact about the spec, not something to probe for const MSE_BYTE_STREAM_FORMATS: MediaContainer[] = [MediaContainer.mp4, MediaContainer.webm, MediaContainer.mp3]; //audio codecs that are at least plausible in each container - ffmpeg refuses the rest outright, so there //is no browser verdict to ask for const CONTAINER_AUDIO_CODECS: Record = { [MediaContainer.mp4]: [AudioCodec.mp3, AudioCodec.opus, AudioCodec.aac, AudioCodec.flac], [MediaContainer.webm]: [AudioCodec.opus, AudioCodec.vorbis], [MediaContainer.ogg]: [AudioCodec.opus, AudioCodec.vorbis, AudioCodec.flac], [MediaContainer.mp3]: [AudioCodec.mp3], }; const CONTAINER_VIDEO_CODECS: Record = { [MediaContainer.mp4]: [VideoCodec.av1, VideoCodec.h264, VideoCodec.h265, VideoCodec.vp9], [MediaContainer.webm]: [VideoCodec.av1, VideoCodec.vp9], [MediaContainer.ogg]: [], [MediaContainer.mp3]: [], }; export function containerAudioCodecs(container: MediaContainer): AudioCodec[] { return CONTAINER_AUDIO_CODECS[container] ?? []; } export function containerVideoCodecs(container: MediaContainer): VideoCodec[] { return [...(CONTAINER_VIDEO_CODECS[container] ?? []), VideoCodec.none]; } //actual values don't seem to matter, besides for a possible "smoothnes" verdict //though any valid codec string should result in the same playability result for the same codec const NOMINAL_FRAMERATE = 30; const NOMINAL_UNSCALED_WIDTH = 3840; function nominalVideoStream(videoBitrate: number): VideoStreamInfo { const width = ladderMaxWidth(videoBitrate) ?? NOMINAL_UNSCALED_WIDTH; return { sourceWidth: width, sourceHeight: Math.round(width / (16 / 9) / 2) * 2, frameRate: NOMINAL_FRAMERATE, interlaced: false, bitrate: videoBitrate, }; } let probeElement: HTMLVideoElement | undefined; function progressivelyPlayable(mimeType: string): boolean { probeElement ??= document.createElement("video"); return probeElement.canPlayType(mimeType) !== ""; } //MediaSource.isTypeSupported is the cheap gate but a lenient one: it parses the type and checks the //codecs in isolation, so it says yes to combinations the decoder then refuses. decodingInfo asks the //actual decoder pipeline, per track, with the resolution and framerate attached const decodingCache = new Map(); const pending = new Map>(); function startDecodingProbe(key: string, configuration: MediaDecodingConfiguration) { const existing = pending.get(key); if (existing) return existing; const probe = navigator.mediaCapabilities .decodingInfo(configuration) .then((info) => { decodingCache.set(key, { supported: info.supported, smooth: info.smooth }); }) .catch(() => { //a configuration this browser cannot even parse tells us nothing beyond what the sync checks //already established, so record the permissive answer rather than disabling a working option decodingCache.set(key, { supported: true, smooth: true }); }) .finally(() => { pending.delete(key); }); pending.set(key, probe); return probe; } export function checkSupport(combination: Combination): Support { const { container, streamingMode, audioCodec } = combination; const videoCodec = combination.videoCodec ?? VideoCodec.none; const hasVideo = videoCodec !== VideoCodec.none; if (streamingMode === StreamingMode.mse) { if (!hasMediaSource) return { supported: false, reason: "this browser has no MediaSource support" }; if (!MSE_BYTE_STREAM_FORMATS.includes(container)) return { supported: false, reason: `${container} has no registered MSE byte stream format, so no browser can stream it - pick progressive or buffered`, }; //measured against -live 1, -dash 1, -default_mode infer, explicit dispositions, forced cluster //limits and both track orders: the append succeeds, updateend fires, and buffered stays empty if (container === MediaContainer.webm && hasVideo) return { supported: false, reason: "effectively unsupported by browsers in mse", }; } const video = hasVideo ? nominalVideoStream(combination.videoBitrate ?? 0) : undefined; const mimeType = createRFC6381String(container, videoCodec, audioCodec, video); if (streamingMode === StreamingMode.mse) { if (!MediaSource.isTypeSupported(mimeType)) return { supported: false, reason: `the browser rejects ${mimeType} in a SourceBuffer` }; } else if (!progressivelyPlayable(mimeType)) { return { supported: false, reason: `the browser cannot play ${mimeType}` }; } //decodingInfo needs one content type per track, which is why the tokens are built here rather than //reusing the combined string above. a container that names no codec has nothing to ask about if (!containerNamesCodecs(container)) return { supported: true }; const configuration: MediaDecodingConfiguration = { type: streamingMode === StreamingMode.mse ? "media-source" : "file", audio: { contentType: `${containerMimeType(container, false)}; codecs="${audioCodecToken(audioCodec)}"` }, ...(video ? { video: { contentType: `${containerMimeType(container, true)}; codecs="${videoCodecToken(videoCodec, video)}"`, width: video.sourceWidth, height: video.sourceHeight, bitrate: video.bitrate, framerate: video.frameRate, }, } : {}), }; //keyed on the bitrate only when there is a video track, so an audio-only probe is not re-run once per //resolution ladder rung for an answer that cannot depend on it const key = `${configuration.type}|${configuration.audio?.contentType}|${configuration.video?.contentType}|${video?.bitrate}`; const cached = decodingCache.get(key); if (!cached) { void startDecodingProbe(key, configuration); return { supported: true }; } if (!cached.supported) return { supported: false, reason: `the browser's decoders reject ${mimeType}` }; if (!cached.smooth) return { supported: true, reason: "supported, but the browser does not expect to decode this smoothly" }; return { supported: true }; } //a container is offerable when at least one codec combination inside it works in the current mode function checkContainer(combination: Combination, videoCodecs: VideoCodec[]): Support { let firstFailure: Support | undefined; for (const videoCodec of videoCodecs) { for (const audioCodec of containerAudioCodecs(combination.container)) { const support = checkSupport({ ...combination, videoCodec, audioCodec }); if (support.supported) return { supported: true }; firstFailure ??= support; } } return firstFailure ?? { supported: false, reason: `${combination.container} offers no usable codecs` }; } export function checkVideoContainerSupport(combination: Combination): Support { return checkContainer(combination, containerVideoCodecs(combination.container)); } //an audio-only profile must not be told mp4 works because mp4 video works, only because mp4 audio does export function checkAudioContainerSupport(combination: Combination): Support { return checkContainer(combination, [VideoCodec.none]); } function usableAudioCodecs(combination: Combination): AudioCodec[] { return containerAudioCodecs(combination.container).filter( (audioCodec) => checkSupport({ ...combination, audioCodec }).supported, ); } function usableVideoCodecs(combination: Combination): VideoCodec[] { return containerVideoCodecs(combination.container).filter( (videoCodec) => checkContainer(combination, [videoCodec]).supported, ); } function usableContainers(combination: Combination): MediaContainer[] { const check = combination.videoCodec === undefined ? checkAudioContainerSupport : checkVideoContainerSupport; return Object.values(MediaContainer).filter((container) => check({ ...combination, container }).supported); } //used to decide whether the video section is worth showing at all - it is not when nothing but "none" would //be selectable export function selectableVideoCodecs(combination: Combination): VideoCodec[] { return usableVideoCodecs(combination).filter((videoCodec) => videoCodec !== VideoCodec.none); } function firstUsable(current: T, usable: T[]): T | undefined { return usable.includes(current) ? current : usable[0]; } function knownMember(values: T[], value: T, fallback: T): T { return values.includes(value) ? value : fallback; } //make sure settings are playable (e.g. might come from an older browser version with different codec support) export function playableSettings(settings: T): T { const known = { ...settings, streamingMode: knownMember(Object.values(StreamingMode), settings.streamingMode, DEFAULT_STREAMING_MODE), container: knownMember(Object.values(MediaContainer), settings.container, MediaContainer.mp4), audioCodec: knownMember(Object.values(AudioCodec), settings.audioCodec, AudioCodec.opus), ...(settings.videoCodec === undefined ? {} : { videoCodec: knownMember(Object.values(VideoCodec), settings.videoCodec, VideoCodec.none) }), }; if (checkSupport(known).supported) return known; //outermost field first: the container decides which codecs are on offer, and the video codec decides //which audio codecs are left over const container = firstUsable(known.container, usableContainers(known)); if (container === undefined) { if (known.streamingMode === DEFAULT_STREAMING_MODE) return known; return playableSettings({ ...known, streamingMode: DEFAULT_STREAMING_MODE }); } const withContainer = { ...known, container }; const videoCodec = known.videoCodec === undefined ? undefined : firstUsable(known.videoCodec, usableVideoCodecs(withContainer)); const withVideoCodec = videoCodec === undefined ? withContainer : { ...withContainer, videoCodec }; const audioCodec = firstUsable(withVideoCodec.audioCodec, usableAudioCodecs(withVideoCodec)); //the container was only offered because some codec pair inside it works, so this cannot come up empty if (audioCodec === undefined) return known; return { ...withVideoCodec, audioCodec }; } export async function initCapabilities(): Promise { for (const streamingMode of Object.values(StreamingMode)) { for (const container of Object.values(MediaContainer)) { for (const audioCodec of containerAudioCodecs(container)) { for (const videoCodec of containerVideoCodecs(container)) { //the bitrate picks the resolution ladder rung and with it the codec level, so each one the //menu offers is its own probe. an audio-only combination has none to vary - and "none" is //exactly what an absent videoCodec resolves to in checkSupport const videoBitrates = videoCodec === VideoCodec.none ? [undefined] : VIDEO_BITRATES; for (const videoBitrate of videoBitrates) { checkSupport({ container, streamingMode, audioCodec, videoCodec, videoBitrate }); } } } } } //every probe started above resolves into the same map, and settling one cannot start another await Promise.all(pending.values()); }