ffmpeg improvements and possible ios playback improvements
Mserver/src/ffmpeg.ts
@@ -1,28 +1,119 @@
import { AudioCodec, MediaContainer, type Metadata, VideoCodec, VideoEncodingSetting } from "music-server-shared/types";
import { createRFC6381String } from "music-server-shared/utils";
import { readStream } from "./utils";
function secondsToFfmpeg(seconds: number): string {
// Ensure seconds is non-negative
if (seconds < 0) {
throw new Error("Seconds must be a non-negative number");
}
// Extract hours, minutes, seconds, and milliseconds
const hours = Math.floor(seconds / 3600);
const minutes = Math.floor((seconds % 3600) / 60);
const secs = Math.floor(seconds % 60);
const KEYFRAME_INTERVAL_SECONDS = 5;
//+frag_keyframe fragments the mp4 at video keyframes, of which an audio-only stream has none. without
//an explicit duration the entire track ends up as a single moof/mdat pair
const AUDIO_FRAGMENT_MICROSECONDS = 2_000_000;
const AUDIO_ENCODERS: Record<AudioCodec, string> = {
[AudioCodec.aac]: "aac",
[AudioCodec.mp3]: "libmp3lame",
[AudioCodec.opus]: "libopus",
[AudioCodec.vorbis]: "libvorbis",
};
const RESOLUTION_LADDER: { upToBitrate: number; width: number }[] = [
{ upToBitrate: 400_000, width: 640 },
{ upToBitrate: 800_000, width: 854 },
{ upToBitrate: 2_500_000, width: 1280 },
{ upToBitrate: 5_000_000, width: 1920 },
{ upToBitrate: 12_000_000, width: 2560 },
];
//x264 and x265 share the same preset names
const X26X_PRESETS: Record<VideoEncodingSetting, string> = {
[VideoEncodingSetting.veryfast]: "veryfast",
[VideoEncodingSetting.fast]: "fast",
[VideoEncodingSetting.balanced]: "medium",
[VideoEncodingSetting.quality]: "slow",
};
const AV1_PRESETS: Record<VideoEncodingSetting, string> = {
[VideoEncodingSetting.veryfast]: "8",
[VideoEncodingSetting.fast]: "7",
[VideoEncodingSetting.balanced]: "6",
[VideoEncodingSetting.quality]: "5",
};
//libvpx wants a deadline/cpu-used pair instead of a preset name
const VP9_PRESETS: Record<VideoEncodingSetting, string[]> = {
[VideoEncodingSetting.veryfast]: ["-deadline", "realtime", "-cpu-used", "5"],
[VideoEncodingSetting.fast]: ["-deadline", "realtime", "-cpu-used", "4"],
[VideoEncodingSetting.balanced]: ["-deadline", "good", "-cpu-used", "2"],
[VideoEncodingSetting.quality]: ["-deadline", "good", "-cpu-used", "0"],
};
function lookup<T>(table: Record<string, T>, key: string, what: string): T {
const value = table[key];
if (value === undefined) throw new Error(`Unsupported ${what}: ${key}`);
return value;
}
// Calculate the fractional part for milliseconds
const milliseconds = Math.floor((seconds - Math.floor(seconds)) * 1000);
//pick the audio track matching the earliest entry of the caller's preference list. with no match at
//all, prefer whichever track the file itself flags as default, and only then fall back to the first
function selectAudioTrack(languages: string, tracks: NonNullable<Metadata["audioTracks"]>): number {
const preferences = languages ? languages.toLocaleLowerCase().split(",") : [];
let best = -1;
let bestPreference = Number.POSITIVE_INFINITY;
for (let i = 0; i < tracks.length; i++) {
const language = tracks[i].language?.toLocaleLowerCase();
const preference = language ? preferences.indexOf(language) : -1;
if (preference >= 0 && preference < bestPreference) {
best = i;
bestPreference = preference;
if (preference === 0) break;
}
}
if (best >= 0) return best;
const defaultTrack = tracks.findIndex((track) => track.default);
return defaultTrack >= 0 ? defaultTrack : 0;
}
// Pad numbers with leading zeros
const hoursStr = hours.toString().padStart(2, "0");
const minutesStr = minutes.toString().padStart(2, "0");
const secondsStr = secs.toString().padStart(2, "0");
const millisecondsStr = milliseconds.toString().padStart(3, "0");
function videoArgsFor(
requestedVideoCodec: VideoCodec,
requestedVideoEncodingSetting: VideoEncodingSetting,
videoBitrate: number,
): string[] {
const args: string[] = [];
//TODO: support hardware encoding
switch (requestedVideoCodec) {
case VideoCodec.av1:
args.push("-c:v", "libsvtav1", "-preset", lookup(AV1_PRESETS, requestedVideoEncodingSetting, "encoding setting"));
break;
case VideoCodec.h264:
args.push("-c:v", "libx264", "-profile:v", "high");
args.push("-preset", lookup(X26X_PRESETS, requestedVideoEncodingSetting, "encoding setting"));
break;
case VideoCodec.h265:
args.push("-c:v", "libx265", "-profile:v", "main");
args.push("-preset", lookup(X26X_PRESETS, requestedVideoEncodingSetting, "encoding setting"));
args.push("-x265-params", "log-level=error");
break;
case VideoCodec.vp9:
args.push("-c:v", "libvpx-vp9", ...lookup(VP9_PRESETS, requestedVideoEncodingSetting, "encoding setting"));
args.push("-row-mt", "1", "-tile-columns", "2");
break;
default:
throw new Error(`Unsupported video codec: ${requestedVideoCodec}`);
}
return `${hoursStr}:${minutesStr}:${secondsStr}.${millisecondsStr}`;
const maxWidth = RESOLUTION_LADDER.find((rung) => videoBitrate <= rung.upToBitrate)?.width;
args.push(
"-b:v",
videoBitrate.toString(), //TODO: better bitrate selection
//make sure pixel format is browser compatible
"-pix_fmt",
"yuv420p",
//automatically deinterlace video to double fps if video is interlaced
//scale video down based on bitrate
"-vf",
`yadif=deint=1:mode=1${maxWidth ? `, scale='if(gt(iw,${maxWidth}),${maxWidth},iw)':-2` : ""}`,
"-force_key_frames",
`expr:gte(t,n_forced*${KEYFRAME_INTERVAL_SECONDS})`,
"-map",
"0:v:0",
);
return args;
}
export async function convertWithFFmpeg(
@@ -38,211 +129,52 @@ export async function convertWithFFmpeg(
metadata: Metadata,
) {
const mimeType = createRFC6381String(container, requestedVideoCodec, requestedAudioCodec);
const audioEncoder = lookup(AUDIO_ENCODERS, requestedAudioCodec, "audio codec");
const hasVideo = requestedVideoCodec !== VideoCodec.none;
let audioCodec: string;
switch (requestedAudioCodec) {
case AudioCodec.aac:
audioCodec = "aac";
break;
case AudioCodec.mp3: {
audioCodec = "libmp3lame";
break;
}
case AudioCodec.opus: {
audioCodec = "libopus";
break;
}
case AudioCodec.vorbis: {
audioCodec = "libvorbis";
break;
}
default: {
throw new Error(`Unsupported audio codec: ${requestedAudioCodec}`);
}
}
let maxWidth: number | undefined;
if (videoBitrate <= 210000) {
maxWidth = 860;
} else if (videoBitrate < 420000) {
maxWidth = 1000;
} else if (videoBitrate < 1500000) {
maxWidth = 1280;
} else if (videoBitrate < 2000000) {
maxWidth = 1920;
} else if (videoBitrate < 10000000) {
maxWidth = 3840;
}
let containerSettings: string[] | undefined;
const containerArgs: string[] = [];
if (container === MediaContainer.mp4) {
containerSettings = [
"-avoid_negative_ts",
"make_zero",
"-movflags",
"+frag_keyframe+empty_moov+default_base_moof+faststart",
];
}
const videoEncodingSettings: string[] = [];
let videoCodec: string | undefined;
let extraVideoSettings: string[] | undefined;
//only convert video if transcoding is requesting so
if (requestedVideoCodec !== VideoCodec.none) {
extraVideoSettings = [
//make sure pixel format is browser compatible
"-pix_fmt",
"yuv420p",
//automatically deinterlace video to double fps if video is interlaced
//scale video down based on bitrate
"-vf",
`yadif=deint=1:mode=1${maxWidth ? `, scale='if(gt(iw,${maxWidth}),${maxWidth},iw)':-2` : ""}`,
//one keyframe ~ 5 seconds for 30 fps video
//important because if this is too long, the frontend might remove frames relevant for current playback when discarding old data from the sourcebuffer
"-g",
"150",
];
switch (requestedVideoCodec) {
case VideoCodec.av1:
videoCodec = "libsvtav1"; //TODO: support hardware encoding
switch (requestedVideoEncodingSetting) {
case VideoEncodingSetting.veryfast:
videoEncodingSettings.push("-preset", "8");
break;
case VideoEncodingSetting.fast:
videoEncodingSettings.push("-preset", "7");
break;
case VideoEncodingSetting.balanced:
videoEncodingSettings.push("-preset", "6");
break;
case VideoEncodingSetting.quality:
videoEncodingSettings.push("-preset", "5");
break;
default: {
throw new Error(`Unsupported video encoding setting: ${requestedVideoEncodingSetting}`);
}
}
break;
case VideoCodec.h264:
videoCodec = "libx264"; //TODO: support hardware encoding
videoEncodingSettings.push("-profile", "high");
switch (requestedVideoEncodingSetting) {
case VideoEncodingSetting.veryfast:
videoEncodingSettings.push("-preset", "veryfast");
break;
case VideoEncodingSetting.fast:
videoEncodingSettings.push("-preset", "fast");
break;
case VideoEncodingSetting.balanced:
videoEncodingSettings.push("-preset", "medium");
break;
case VideoEncodingSetting.quality:
videoEncodingSettings.push("-preset", "slow");
break;
default: {
throw new Error(`Unsupported video encoding setting: ${requestedVideoEncodingSetting}`);
}
}
break;
case VideoCodec.h265:
videoCodec = "libx265"; //TODO: support hardware encoding
videoEncodingSettings.push("-profile", "main");
switch (requestedVideoEncodingSetting) {
case VideoEncodingSetting.veryfast:
videoEncodingSettings.push("-preset", "veryfast");
break;
case VideoEncodingSetting.fast:
videoEncodingSettings.push("-preset", "fast");
break;
case VideoEncodingSetting.balanced:
videoEncodingSettings.push("-preset", "medium");
break;
case VideoEncodingSetting.quality:
videoEncodingSettings.push("-preset", "slow");
break;
default: {
throw new Error(`Unsupported video encoding setting: ${requestedVideoEncodingSetting}`);
}
}
break;
case VideoCodec.vp9:
videoCodec = "libvpx-vp9";
switch (requestedVideoEncodingSetting) {
case VideoEncodingSetting.veryfast:
videoEncodingSettings.push("-deadline", "realtime", "-cpu-used", "5");
break;
case VideoEncodingSetting.fast:
videoEncodingSettings.push("-deadline", "realtime", "-cpu-used", "4");
break;
case VideoEncodingSetting.balanced:
videoEncodingSettings.push("-deadline", "good", "-cpu-used", "2");
break;
case VideoEncodingSetting.quality:
videoEncodingSettings.push("-deadline", "good", "-cpu-used", "2");
break;
default: {
throw new Error(`Unsupported video encoding setting: ${requestedVideoEncodingSetting}`);
}
}
break;
}
}
const seekParams = seekTo === undefined ? [] : ["-ss", secondsToFfmpeg(seekTo)];
let selectedAudioTrack = 0; //0 as fallback if no language matches and no default is set
const languageArray = languages.split(",").map((s) => s.toLocaleLowerCase());
if (languageArray.length > 0) {
const tracks = metadata.audioTracks || [];
let highestPriorityMatch = Number.POSITIVE_INFINITY;
// Check each track for preferred languages in priority order
for (let i = 0; i < tracks.length; i++) {
const track = tracks[i];
// Store the first default track as fallback
if (track.default && selectedAudioTrack === 0) {
selectedAudioTrack = i;
}
// Check if this track's language matches any preference
const langIndex = track.language ? languageArray.indexOf(track.language.toLocaleLowerCase()) : -1;
// If this is a higher priority match than we've seen before, take it
if (langIndex >= 0 && langIndex < highestPriorityMatch) {
selectedAudioTrack = i;
highestPriorityMatch = langIndex;
// Early exit if we already found the most preferred language (first in array)
if (highestPriorityMatch === 0) break;
}
}
//note: no +faststart - it relocates the moov atom, which needs a seekable output, and this
//muxes to a pipe
containerArgs.push("-avoid_negative_ts", "make_zero", "-movflags", "+frag_keyframe+empty_moov+default_base_moof");
if (!hasVideo) containerArgs.push("-frag_duration", AUDIO_FRAGMENT_MICROSECONDS.toString());
}
const args = ["ffmpeg"]
.concat(seekParams) //put seek first to do a fast keyframe seek
.concat(["-i", filePath])
.concat(containerSettings || [])
.concat(
videoCodec
? ["-c:v", videoCodec, "-b:v", videoBitrate.toString()] //TODO: better bitrate selection
.concat(extraVideoSettings || [])
.concat(videoEncodingSettings)
.concat(["-map", "0:v:0"])
: ["-vn"],
)
.concat(["-map", `0:a:${selectedAudioTrack.toString()}?`])
.concat(["-c:a", audioCodec, "-b:a", audioBitrate.toString(), "-ac", "2"])
.concat(["-f", container, "-"]);
const selectedAudioTrack = selectAudioTrack(languages, metadata.audioTracks || []);
const args = [
"ffmpeg",
"-nostats",
"-loglevel",
"error",
//put seek first to do a fast keyframe seek
...(seekTo === undefined ? [] : ["-ss", seekTo.toString()]),
"-i",
filePath,
...containerArgs,
...(hasVideo ? videoArgsFor(requestedVideoCodec, requestedVideoEncodingSetting, videoBitrate) : ["-vn"]),
"-map",
`0:a:${selectedAudioTrack}?`,
"-c:a",
audioEncoder,
"-b:a",
audioBitrate.toString(),
"-ac",
"2",
"-f",
container,
"-",
];
console.log(args.join(" "));
const cmd = Bun.spawn(args, {
stdin: "ignore",
stdout: "pipe",
stderr: "pipe",
onExit: async (subprocess: Bun.Subprocess<"ignore", "pipe", "pipe">, exitCode: number | null) => {
const cmd = Bun.spawn(args, { stdin: "ignore", stdout: "pipe", stderr: "pipe" });
const stderrText = new Response(cmd.stderr).text().catch(() => "");
void cmd.exited
.then(async (exitCode) => {
if (exitCode !== 0 && exitCode !== null) {
console.error("ffmpeg failed with status code", exitCode);
console.error(new TextDecoder().decode(await readStream(subprocess.stderr as ReadableStream)));
console.error(await stderrText);
}
},
});
return { cmd, mimeType };
})
.catch(() => {});
return { cmd, mimeType, stderrText };
}
Mserver/src/index.ts
@@ -27,7 +27,17 @@ import {
ServerError,
username,
} from "./shared";
import { findCover, getPathInfo, listFiles, matchesType, packWithTar, probeFile, toAvif } from "./utils";
import {
findCover,
getPathInfo,
isBelow,
listFiles,
matchesType,
packWithTar,
probeFile,
readStream,
toAvif,
} from "./utils";
//TODO: transcoding cache?
//TODO: add reasonable timeouts for caches
@@ -37,8 +47,15 @@ import { findCover, getPathInfo, listFiles, matchesType, packWithTar, probeFile,
//increase timeout to not abort when listing huge folders
const setup = new Elysia({ serve: { idleTimeout: 255 } });
async function resolveMediaFile(encodedPath: string): Promise<{ filePath: string; info: PathInfo } | ServerError> {
function resolveInRoot(encodedPath: string): string | ServerError {
const filePath = path.join(musicRoot, decodePath(encodedPath));
if (!isBelow(musicRoot, filePath)) return new ServerError(StatusMap.Forbidden, "Path outside the music root");
return filePath;
}
async function resolveMediaFile(encodedPath: string): Promise<{ filePath: string; info: PathInfo } | ServerError> {
const filePath = resolveInRoot(encodedPath);
if (filePath instanceof ServerError) return filePath;
const info = await getPathInfo(filePath);
//undefined means it is a directory rather than a file
if (!info || info instanceof ServerError)
@@ -49,6 +66,14 @@ async function resolveMediaFile(encodedPath: string): Promise<{ filePath: string
type FileHandlerContext = Context<{ params: { "*": string } }>;
function transcodeFailed(set: FileHandlerContext["set"], exitCode: number | null, stderr: string) {
set.status = "Internal Server Error";
//the first lines are the root cause; what follows is each thread unwinding and reporting the same
//failure again, so a tail would report the least informative part of it
const reason = stderr.trim().split("\n").slice(0, 3).join("\n");
return `Transcoding failed (ffmpeg exit ${exitCode})${reason ? `:\n${reason}` : ""}`;
}
const downloadHandler = async ({ params, set }: FileHandlerContext) => {
const resolved = await resolveMediaFile(params["*"]);
if (resolved instanceof ServerError) {
@@ -168,7 +193,12 @@ const app = setup
set.headers.Location = `/download/${params["*"]}`;
return "Not a media file, redirecting to normal endpoint";
}
const { cmd, mimeType } = await convertWithFFmpeg(
if (query.videoCodec && query.videoCodec !== VideoCodec.none && !videoBitrate) {
set.status = "Bad Request";
return "videoBitrate is required when videoCodec is set";
}
const { cmd, mimeType, stderrText } = await convertWithFFmpeg(
filePath,
audioBitrate,
videoBitrate || 0,
@@ -181,21 +211,37 @@ const app = setup
probe,
);
request.signal.addEventListener("abort", () => cmd.kill("SIGKILL"));
set.headers["content-type"] = mimeType;
if (query.disableChunkedTranscoding) {
const chunks: Uint8Array[] = [];
for await (const chunk of cmd.stdout) {
chunks.push(chunk);
}
const full = new Uint8Array(chunks.reduce((acc, val) => acc + val.length, 0));
let offset = 0;
for (const chunk of chunks) {
full.set(chunk, offset);
offset += chunk.length;
}
const full = await readStream(cmd.stdout);
if (full.length === 0) return transcodeFailed(set, await cmd.exited, await stderrText);
set.headers["content-type"] = mimeType;
return full;
}
return new Response(cmd.stdout);
//peek ffmpeg to check for failure and return 500
const reader = cmd.stdout.getReader();
const first = await reader.read();
if (first.done) {
reader.releaseLock();
return transcodeFailed(set, await cmd.exited, await stderrText);
}
set.headers["content-type"] = mimeType;
return new Response(
new ReadableStream<Uint8Array>({
start(controller) {
controller.enqueue(first.value);
},
async pull(controller) {
const { done, value } = await reader.read();
if (done) controller.close();
else controller.enqueue(value);
},
cancel(reason) {
void reader.cancel(reason);
},
}),
);
},
{
query: t.Object({
@@ -226,8 +272,12 @@ const app = setup
},
{ query: t.Optional(t.Object({ recursive: t.Boolean() })) },
)
.get("/isVideo/*", async ({ params }) => {
const dirPath = path.join(musicRoot, decodePath(params["*"]));
.get("/isVideo/*", async ({ params, set }) => {
const dirPath = resolveInRoot(params["*"]);
if (dirPath instanceof ServerError) {
set.status = dirPath.status;
return dirPath.error;
}
const probeData = await probeFile(dirPath).catch(() => ({}) as Metadata);
return { isVideo: probeData.videoCodec !== undefined } as IsVideoResponse;
})
@@ -235,7 +285,11 @@ const app = setup
"/cover/*",
async ({ params, set, query }) => {
set.status = "Not Found";
const dirPath = path.join(musicRoot, decodePath(params["*"]));
const dirPath = resolveInRoot(params["*"]);
if (dirPath instanceof ServerError) {
set.status = dirPath.status;
return dirPath.error;
}
const fileScanResult = await findCover(dirPath);
const setContentType = (type: string) => {
set.headers["Content-Type"] = type;
Mwebclient/src/App.tsx
@@ -500,11 +500,6 @@ const App: Component = () => {
} catch {}
}
//need both of these lines to work without logging an error
//just removeAttribute doesn't work at all
//just srcObject = null works, but logs a weird csp error
videoElement.removeAttribute("src");
videoElement.srcObject = null;
const mseURL =
options.enableTranscoding && options.useMSE && !src.startsWith("blob:")
? await getMSEUrl(src, seekTo || 0, isVideo === true)
@@ -521,6 +516,8 @@ const App: Component = () => {
return;
}
setVideoShown(false);
videoElement.removeAttribute("src");
videoElement.srcObject = null;
videoElement.src = mseURL || src;
if (mseURL && seekTo) {
ignoreSeek = true;
Mwebclient/src/components/OptionsModal.tsx
@@ -190,6 +190,8 @@ export default function OptionsModal(props: OptionsModalProps) {
<input
id="language-input"
type="text"
placeholder="en,de"
title="Two-letter language codes (ISO 639-1), separated by commas, most preferred first. Three-letter codes like 'eng' do not match."
value={newOptions.languages}
onChange={(e) => setNewOptions("languages", e.target.value)}
disabled={!newOptions.enableTranscoding}
Mwebclient/src/components/Playlist.tsx
@@ -250,7 +250,7 @@ export default function Playlist(props: PlaylistProps) {
</Show>
<img src={cover()} alt="" />
</Show>
<video ref={props.videoElement} preload="auto" />
<video ref={props.videoElement} preload="auto" playsinline />
</div>
</div>
);