capabilities.ts
| 1 | import { AudioCodec, MediaContainer, VideoCodec } from "music-server-shared/types"; |
| 2 | import { |
| 3 | audioCodecToken, |
| 4 | containerMimeType, |
| 5 | containerNamesCodecs, |
| 6 | createRFC6381String, |
| 7 | ladderMaxWidth, |
| 8 | type VideoStreamInfo, |
| 9 | videoCodecToken, |
| 10 | } from "music-server-shared/utils"; |
| 11 | import { StreamingMode, VIDEO_BITRATES } from "./types"; |
| 12 | |
| 13 | export interface Support { |
| 14 | supported: boolean; |
| 15 | reason?: string; |
| 16 | } |
| 17 | |
| 18 | //what a transcoding profile would produce, which is all it takes to decide whether a browser can play it. |
| 19 | //deliberately the shape of AppOptions' two settings objects, so either can be passed straight in and a |
| 20 | //single field varied with a spread - no argument threading and no positional mix-ups at the call sites |
| 21 | export interface Combination { |
| 22 | container: MediaContainer; |
| 23 | streamingMode: StreamingMode; |
| 24 | audioCodec: AudioCodec; |
| 25 | videoCodec?: VideoCodec; |
| 26 | videoBitrate?: number; |
| 27 | } |
| 28 | |
| 29 | export const hasMediaSource = "MediaSource" in window; |
| 30 | export const DEFAULT_STREAMING_MODE = hasMediaSource ? StreamingMode.mse : StreamingMode.progressive; |
| 31 | |
| 32 | //the W3C MSE Byte Stream Format Registry is a closed list: ISO BMFF, WebM, MPEG-2 TS and MPEG Audio. a |
| 33 | //container missing from it cannot be fed to a SourceBuffer by any conforming browser, no matter how well |
| 34 | //that same browser plays it progressively - so this is a fact about the spec, not something to probe for |
| 35 | const MSE_BYTE_STREAM_FORMATS: MediaContainer[] = [MediaContainer.mp4, MediaContainer.webm, MediaContainer.mp3]; |
| 36 | |
| 37 | //audio codecs that are at least plausible in each container - ffmpeg refuses the rest outright, so there |
| 38 | //is no browser verdict to ask for |
| 39 | const CONTAINER_AUDIO_CODECS: Record<MediaContainer, AudioCodec[]> = { |
| 40 | [MediaContainer.mp4]: [AudioCodec.mp3, AudioCodec.opus, AudioCodec.aac, AudioCodec.flac], |
| 41 | [MediaContainer.webm]: [AudioCodec.opus, AudioCodec.vorbis], |
| 42 | [MediaContainer.ogg]: [AudioCodec.opus, AudioCodec.vorbis, AudioCodec.flac], |
| 43 | [MediaContainer.mp3]: [AudioCodec.mp3], |
| 44 | }; |
| 45 | |
| 46 | const CONTAINER_VIDEO_CODECS: Record<MediaContainer, VideoCodec[]> = { |
| 47 | [MediaContainer.mp4]: [VideoCodec.av1, VideoCodec.h264, VideoCodec.h265, VideoCodec.vp9], |
| 48 | [MediaContainer.webm]: [VideoCodec.av1, VideoCodec.vp9], |
| 49 | [MediaContainer.ogg]: [], |
| 50 | [MediaContainer.mp3]: [], |
| 51 | }; |
| 52 | |
| 53 | export function containerAudioCodecs(container: MediaContainer): AudioCodec[] { |
| 54 | return CONTAINER_AUDIO_CODECS[container] ?? []; |
| 55 | } |
| 56 | |
| 57 | export function containerVideoCodecs(container: MediaContainer): VideoCodec[] { |
| 58 | return [...(CONTAINER_VIDEO_CODECS[container] ?? []), VideoCodec.none]; |
| 59 | } |
| 60 | |
| 61 | //actual values don't seem to matter, besides for a possible "smoothnes" verdict |
| 62 | //though any valid codec string should result in the same playability result for the same codec |
| 63 | const NOMINAL_FRAMERATE = 30; |
| 64 | const NOMINAL_UNSCALED_WIDTH = 3840; |
| 65 | |
| 66 | function nominalVideoStream(videoBitrate: number): VideoStreamInfo { |
| 67 | const width = ladderMaxWidth(videoBitrate) ?? NOMINAL_UNSCALED_WIDTH; |
| 68 | return { |
| 69 | sourceWidth: width, |
| 70 | sourceHeight: Math.round(width / (16 / 9) / 2) * 2, |
| 71 | frameRate: NOMINAL_FRAMERATE, |
| 72 | interlaced: false, |
| 73 | bitrate: videoBitrate, |
| 74 | }; |
| 75 | } |
| 76 | |
| 77 | let probeElement: HTMLVideoElement | undefined; |
| 78 | |
| 79 | function progressivelyPlayable(mimeType: string): boolean { |
| 80 | probeElement ??= document.createElement("video"); |
| 81 | return probeElement.canPlayType(mimeType) !== ""; |
| 82 | } |
| 83 | |
| 84 | //MediaSource.isTypeSupported is the cheap gate but a lenient one: it parses the type and checks the |
| 85 | //codecs in isolation, so it says yes to combinations the decoder then refuses. decodingInfo asks the |
| 86 | //actual decoder pipeline, per track, with the resolution and framerate attached |
| 87 | const decodingCache = new Map<string, { supported: boolean; smooth: boolean }>(); |
| 88 | const pending = new Map<string, Promise<void>>(); |
| 89 | |
| 90 | function startDecodingProbe(key: string, configuration: MediaDecodingConfiguration) { |
| 91 | const existing = pending.get(key); |
| 92 | if (existing) return existing; |
| 93 | const probe = navigator.mediaCapabilities |
| 94 | .decodingInfo(configuration) |
| 95 | .then((info) => { |
| 96 | decodingCache.set(key, { supported: info.supported, smooth: info.smooth }); |
| 97 | }) |
| 98 | .catch(() => { |
| 99 | //a configuration this browser cannot even parse tells us nothing beyond what the sync checks |
| 100 | //already established, so record the permissive answer rather than disabling a working option |
| 101 | decodingCache.set(key, { supported: true, smooth: true }); |
| 102 | }) |
| 103 | .finally(() => { |
| 104 | pending.delete(key); |
| 105 | }); |
| 106 | pending.set(key, probe); |
| 107 | return probe; |
| 108 | } |
| 109 | |
| 110 | export function checkSupport(combination: Combination): Support { |
| 111 | const { container, streamingMode, audioCodec } = combination; |
| 112 | const videoCodec = combination.videoCodec ?? VideoCodec.none; |
| 113 | const hasVideo = videoCodec !== VideoCodec.none; |
| 114 | if (streamingMode === StreamingMode.mse) { |
| 115 | if (!hasMediaSource) return { supported: false, reason: "this browser has no MediaSource support" }; |
| 116 | if (!MSE_BYTE_STREAM_FORMATS.includes(container)) |
| 117 | return { |
| 118 | supported: false, |
| 119 | reason: `${container} has no registered MSE byte stream format, so no browser can stream it - pick progressive or buffered`, |
| 120 | }; |
| 121 | //measured against -live 1, -dash 1, -default_mode infer, explicit dispositions, forced cluster |
| 122 | //limits and both track orders: the append succeeds, updateend fires, and buffered stays empty |
| 123 | if (container === MediaContainer.webm && hasVideo) |
| 124 | return { |
| 125 | supported: false, |
| 126 | reason: "effectively unsupported by browsers in mse", |
| 127 | }; |
| 128 | } |
| 129 | |
| 130 | const video = hasVideo ? nominalVideoStream(combination.videoBitrate ?? 0) : undefined; |
| 131 | const mimeType = createRFC6381String(container, videoCodec, audioCodec, video); |
| 132 | |
| 133 | if (streamingMode === StreamingMode.mse) { |
| 134 | if (!MediaSource.isTypeSupported(mimeType)) |
| 135 | return { supported: false, reason: `the browser rejects ${mimeType} in a SourceBuffer` }; |
| 136 | } else if (!progressivelyPlayable(mimeType)) { |
| 137 | return { supported: false, reason: `the browser cannot play ${mimeType}` }; |
| 138 | } |
| 139 | |
| 140 | //decodingInfo needs one content type per track, which is why the tokens are built here rather than |
| 141 | //reusing the combined string above. a container that names no codec has nothing to ask about |
| 142 | if (!containerNamesCodecs(container)) return { supported: true }; |
| 143 | const configuration: MediaDecodingConfiguration = { |
| 144 | type: streamingMode === StreamingMode.mse ? "media-source" : "file", |
| 145 | audio: { contentType: `${containerMimeType(container, false)}; codecs="${audioCodecToken(audioCodec)}"` }, |
| 146 | ...(video |
| 147 | ? { |
| 148 | video: { |
| 149 | contentType: `${containerMimeType(container, true)}; codecs="${videoCodecToken(videoCodec, video)}"`, |
| 150 | width: video.sourceWidth, |
| 151 | height: video.sourceHeight, |
| 152 | bitrate: video.bitrate, |
| 153 | framerate: video.frameRate, |
| 154 | }, |
| 155 | } |
| 156 | : {}), |
| 157 | }; |
| 158 | |
| 159 | //keyed on the bitrate only when there is a video track, so an audio-only probe is not re-run once per |
| 160 | //resolution ladder rung for an answer that cannot depend on it |
| 161 | const key = `${configuration.type}|${configuration.audio?.contentType}|${configuration.video?.contentType}|${video?.bitrate}`; |
| 162 | const cached = decodingCache.get(key); |
| 163 | if (!cached) { |
| 164 | void startDecodingProbe(key, configuration); |
| 165 | return { supported: true }; |
| 166 | } |
| 167 | if (!cached.supported) return { supported: false, reason: `the browser's decoders reject ${mimeType}` }; |
| 168 | if (!cached.smooth) |
| 169 | return { supported: true, reason: "supported, but the browser does not expect to decode this smoothly" }; |
| 170 | return { supported: true }; |
| 171 | } |
| 172 | |
| 173 | //a container is offerable when at least one codec combination inside it works in the current mode |
| 174 | function checkContainer(combination: Combination, videoCodecs: VideoCodec[]): Support { |
| 175 | let firstFailure: Support | undefined; |
| 176 | for (const videoCodec of videoCodecs) { |
| 177 | for (const audioCodec of containerAudioCodecs(combination.container)) { |
| 178 | const support = checkSupport({ ...combination, videoCodec, audioCodec }); |
| 179 | if (support.supported) return { supported: true }; |
| 180 | firstFailure ??= support; |
| 181 | } |
| 182 | } |
| 183 | return firstFailure ?? { supported: false, reason: `${combination.container} offers no usable codecs` }; |
| 184 | } |
| 185 | |
| 186 | export function checkVideoContainerSupport(combination: Combination): Support { |
| 187 | return checkContainer(combination, containerVideoCodecs(combination.container)); |
| 188 | } |
| 189 | |
| 190 | //an audio-only profile must not be told mp4 works because mp4 video works, only because mp4 audio does |
| 191 | export function checkAudioContainerSupport(combination: Combination): Support { |
| 192 | return checkContainer(combination, [VideoCodec.none]); |
| 193 | } |
| 194 | |
| 195 | function usableAudioCodecs(combination: Combination): AudioCodec[] { |
| 196 | return containerAudioCodecs(combination.container).filter( |
| 197 | (audioCodec) => checkSupport({ ...combination, audioCodec }).supported, |
| 198 | ); |
| 199 | } |
| 200 | |
| 201 | function usableVideoCodecs(combination: Combination): VideoCodec[] { |
| 202 | return containerVideoCodecs(combination.container).filter( |
| 203 | (videoCodec) => checkContainer(combination, [videoCodec]).supported, |
| 204 | ); |
| 205 | } |
| 206 | |
| 207 | function usableContainers(combination: Combination): MediaContainer[] { |
| 208 | const check = combination.videoCodec === undefined ? checkAudioContainerSupport : checkVideoContainerSupport; |
| 209 | return Object.values(MediaContainer).filter((container) => check({ ...combination, container }).supported); |
| 210 | } |
| 211 | |
| 212 | //used to decide whether the video section is worth showing at all - it is not when nothing but "none" would |
| 213 | //be selectable |
| 214 | export function selectableVideoCodecs(combination: Combination): VideoCodec[] { |
| 215 | return usableVideoCodecs(combination).filter((videoCodec) => videoCodec !== VideoCodec.none); |
| 216 | } |
| 217 | |
| 218 | function firstUsable<T>(current: T, usable: T[]): T | undefined { |
| 219 | return usable.includes(current) ? current : usable[0]; |
| 220 | } |
| 221 | |
| 222 | function knownMember<T extends string>(values: T[], value: T, fallback: T): T { |
| 223 | return values.includes(value) ? value : fallback; |
| 224 | } |
| 225 | |
| 226 | //make sure settings are playable (e.g. might come from an older browser version with different codec support) |
| 227 | export function playableSettings<T extends Combination>(settings: T): T { |
| 228 | const known = { |
| 229 | ...settings, |
| 230 | streamingMode: knownMember(Object.values(StreamingMode), settings.streamingMode, DEFAULT_STREAMING_MODE), |
| 231 | container: knownMember(Object.values(MediaContainer), settings.container, MediaContainer.mp4), |
| 232 | audioCodec: knownMember(Object.values(AudioCodec), settings.audioCodec, AudioCodec.opus), |
| 233 | ...(settings.videoCodec === undefined |
| 234 | ? {} |
| 235 | : { videoCodec: knownMember(Object.values(VideoCodec), settings.videoCodec, VideoCodec.none) }), |
| 236 | }; |
| 237 | if (checkSupport(known).supported) return known; |
| 238 | |
| 239 | //outermost field first: the container decides which codecs are on offer, and the video codec decides |
| 240 | //which audio codecs are left over |
| 241 | const container = firstUsable(known.container, usableContainers(known)); |
| 242 | if (container === undefined) { |
| 243 | if (known.streamingMode === DEFAULT_STREAMING_MODE) return known; |
| 244 | return playableSettings({ ...known, streamingMode: DEFAULT_STREAMING_MODE }); |
| 245 | } |
| 246 | const withContainer = { ...known, container }; |
| 247 | |
| 248 | const videoCodec = |
| 249 | known.videoCodec === undefined ? undefined : firstUsable(known.videoCodec, usableVideoCodecs(withContainer)); |
| 250 | const withVideoCodec = videoCodec === undefined ? withContainer : { ...withContainer, videoCodec }; |
| 251 | |
| 252 | const audioCodec = firstUsable(withVideoCodec.audioCodec, usableAudioCodecs(withVideoCodec)); |
| 253 | //the container was only offered because some codec pair inside it works, so this cannot come up empty |
| 254 | if (audioCodec === undefined) return known; |
| 255 | return { ...withVideoCodec, audioCodec }; |
| 256 | } |
| 257 | |
| 258 | export async function initCapabilities(): Promise<void> { |
| 259 | for (const streamingMode of Object.values(StreamingMode)) { |
| 260 | for (const container of Object.values(MediaContainer)) { |
| 261 | for (const audioCodec of containerAudioCodecs(container)) { |
| 262 | for (const videoCodec of containerVideoCodecs(container)) { |
| 263 | //the bitrate picks the resolution ladder rung and with it the codec level, so each one the |
| 264 | //menu offers is its own probe. an audio-only combination has none to vary - and "none" is |
| 265 | //exactly what an absent videoCodec resolves to in checkSupport |
| 266 | const videoBitrates = videoCodec === VideoCodec.none ? [undefined] : VIDEO_BITRATES; |
| 267 | for (const videoBitrate of videoBitrates) { |
| 268 | checkSupport({ container, streamingMode, audioCodec, videoCodec, videoBitrate }); |
| 269 | } |
| 270 | } |
| 271 | } |
| 272 | } |
| 273 | } |
| 274 | //every probe started above resolves into the same map, and settling one cannot start another |
| 275 | await Promise.all(pending.values()); |
| 276 | } |
| 277 |