import { fileTypeFromBuffer } from "file-type"; import type { Language } from "linguist-languages"; import * as linguistLangs from "linguist-languages"; import { type BundledLanguage, bundledLanguagesInfo, createHighlighter, type Highlighter, } from "shiki"; import config from "../config.ts"; import { BINARY_DETECT_BYTES, MAX_FILE_CACHE } from "../constants.ts"; let highlighter: Highlighter | null = null; let extToLangId: Map | null = null; let filenameToLangId: Map | null = null; export async function highlightStartup(): Promise { console.log( "[highlight] initializing highlighter and language detection maps...", ); // Index linguist languages by lowercase name and aliases for precise lookup. // Many shiki languages share the same tmScope (e.g. source.js has 31 entries), // so scope-based matching is unreliable — name/alias matching is used instead. const linguistByName = new Map(); for (const lang of Object.values( linguistLangs as Record, )) { linguistByName.set(lang.name.toLowerCase(), lang); for (const alias of lang.aliases ?? []) { if (!linguistByName.has(alias.toLowerCase())) { linguistByName.set(alias.toLowerCase(), lang); } } } const extMap = new Map(); const fnMap = new Map(); for (const { id, aliases } of bundledLanguagesInfo) { // Match shiki lang → linguist language by ID then aliases (no scope fallback) const linguistLang = linguistByName.get(id) ?? aliases?.reduce( (found, a) => found ?? linguistByName.get(a.toLowerCase()), undefined, ); if (!linguistLang) continue; for (const ext of linguistLang.extensions ?? []) { if (!extMap.has(ext)) extMap.set(ext, id); } for (const fn of linguistLang.filenames ?? []) { if (!fnMap.has(fn)) fnMap.set(fn, id); } } extToLangId = extMap; filenameToLangId = fnMap; // Grammars are loaded on first use by ensureLang(). Preloading all ~350 // bundled languages costs ~4s and ~265MB per worker, and a given repo // only ever touches a handful. highlighter = await createHighlighter({ themes: ["github-light", "github-dark"], langs: [], }); console.log( `[highlight] ready: ${extMap.size} extensions, ${fnMap.size} filenames`, ); } export function getHighlighter(): Highlighter { if (!highlighter) throw new Error("highlightStartup() has not been called"); return highlighter; } /** * Load `lang`'s grammar if it is not loaded yet, and return the language to * actually highlight with — falling back to "text" if the grammar is missing * or fails to load. * * Callers MUST await this before codeToHtml/codeToTokensWithThemes. Those * throw on an unloaded language, and both call sites catch that and degrade * to unhighlighted plain text — so skipping this step turns highlighting off * silently rather than loudly. */ export async function ensureLang(lang: string): Promise { const h = getHighlighter(); if (h.getLoadedLanguages().includes(lang)) return lang; try { await h.loadLanguage(lang as BundledLanguage); return lang; } catch { return "text"; } } export function hasBinaryContent(buf: Buffer): boolean { return buf.subarray(0, BINARY_DETECT_BYTES).includes(0); } export function detectLang(filename: string): string { if (!extToLangId || !filenameToLangId) return "text"; const base = filename.split("/").pop() ?? filename; const ext = base.includes(".") ? base.slice(base.lastIndexOf(".")) : ""; return ( filenameToLangId.get(base) ?? (ext ? extToLangId.get(ext) : undefined) ?? "text" ); } export type FileView = | { type: "inline"; html: string; lines: number } | { type: "download"; size: number } | { type: "binary"; size: number } | { type: "media"; mimeType: string; size: number }; const fileCache = new Map(); function shikiHtmlToTable(html: string): string { const styleMatch = html.match(/]*style="([^"]*)"/); const rawStyle = styleMatch?.[1] ?? ""; // Convert Shiki's direct background-color/color to CSS custom properties so // the CSS (not the inline style) controls the background in dark mode. const lightBg = rawStyle.match(/(?:^|;)background-color:([^;]+)/)?.[1] ?? "#fff"; const lightColor = rawStyle.match(/(?:^|;)color:([^;]+)/)?.[1] ?? ""; const darkBg = rawStyle.match(/--shiki-dark-bg:([^;]+)/)?.[1] ?? ""; const darkColor = rawStyle.match(/--shiki-dark:([^;]+)/)?.[1] ?? ""; const varParts = [`--shiki-light-bg:${lightBg}`]; if (lightColor) varParts.push(`--shiki-light:${lightColor}`); if (darkBg) varParts.push(`--shiki-dark-bg:${darkBg}`); if (darkColor) varParts.push(`--shiki-dark:${darkColor}`); const tableStyle = varParts.join(";"); const codeMatch = html.match(/([\s\S]*?)<\/code>/); const codeContent = codeMatch?.[1] ?? ""; const rawLines = codeContent.split("\n"); // Shiki adds a trailing newline, so drop the last empty entry const lines = rawLines.length > 0 && rawLines[rawLines.length - 1] === "" ? rawLines.slice(0, -1) : rawLines; const rows = lines .map((line, i) => { const n = i + 1; return `${n}${line}`; }) .join(""); return `${rows}
`; } function plainTextToTable(text: string): string { const rawLines = text.split("\n"); const lines = rawLines.length > 0 && rawLines[rawLines.length - 1] === "" ? rawLines.slice(0, -1) : rawLines; const rows = lines .map((line, i) => { const n = i + 1; const escaped = line .replace(/&/g, "&") .replace(//g, ">"); return `${n}${escaped}`; }) .join(""); return `${rows}
`; } export async function serveFile( content: Buffer, filename: string, cacheKey: string, ): Promise { const cached = fileCache.get(cacheKey); if (cached) return cached; const fileType = await fileTypeFromBuffer(content); if (fileType) { const group = fileType.mime.split("/")[0]; if (group === "image" || group === "audio" || group === "video") { return { type: "media", mimeType: fileType.mime, size: content.length, }; } } const isBinary = hasBinaryContent(content); if (isBinary) { return { type: "binary", size: content.length }; } if (content.length > config.INLINE_MAX_BYTES) { return { type: "download", size: content.length }; } const text = content.toString("utf-8"); const lineCount = text.split("\n").length; // Outside the try: a load failure must fall back to "text", not to the // unhighlighted plainTextToTable path below. const lang = await ensureLang(detectLang(filename)); let view: FileView; try { const h = getHighlighter(); const shikiHtml = h.codeToHtml(text, { lang, themes: { light: "github-light", dark: "github-dark" }, }); view = { type: "inline", html: shikiHtmlToTable(shikiHtml), lines: lineCount, }; } catch { view = { type: "inline", html: plainTextToTable(text), lines: lineCount, }; } if (cacheKey) { if (fileCache.size >= MAX_FILE_CACHE) fileCache.delete(fileCache.keys().next().value!); fileCache.set(cacheKey, view); } return view; }