show control chars in diffs
MREADME.md
@@ -6,7 +6,7 @@ Frontend works without any JS at all enabled, just required for WebAuthn (with g
## Features
- **Repository browser** — file tree, blob view, commit log, README rendering, media previews
- **Repository browser** — file tree, directly edit single files, blob view, commit log, README rendering, media previews
- **Issues** — create, comment, react
- **Patches** — submit git .patch files for review & comments. Admin can merge applicable patches directly into the repository.
- **Templates** — issue/patch templates
Mpackage.json
@@ -7,7 +7,7 @@
"css:build": "bun scripts/build-css.ts",
"db:init": "bun run src/db/init.ts",
"db:seed": "bun run src/db/seed.ts",
"test": "bun test tests/e2e.test.ts --timeout 60000",
"test": "bun test tests/highlight.test.ts && bun test tests/e2e.test.ts --timeout 60000",
"vendor:simplewebauthn": "bun build node_modules/@simplewebauthn/browser/esm/index.js --outfile public/assets/simplewebauthn-browser.js --format esm",
"vendor:jxl-polyfill": "cp node_modules/jxl-rs-polyfill/dist/auto.js public/assets/jxl-polyfill.js",
"postinstall": "bun run vendor:simplewebauthn && bun run vendor:jxl-polyfill && bun run css:build",
Msrc/services/diffHighlight.ts
@@ -181,6 +181,20 @@ function escapeHtml(s: string): string {
return s.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">");
}
// biome-ignore lint/suspicious/noControlCharactersInRegex: intentional control char rendering
const CTRL_RE = /[\x00-\x08\x0b\x0c\x0d\x0e-\x1f]/g;
function escapeHtmlAndCtrl(s: string): string {
return s
.replace(/&/g, "&")
.replace(/</g, "<")
.replace(/>/g, ">")
.replace(CTRL_RE, (ch) => {
const label = `^${String.fromCharCode(ch.charCodeAt(0) + 64)}`;
return `<span class="diff-ctrl">${label}</span>`;
});
}
// ─── Highlight a hunk ────────────────────────────────────────────────────────
async function highlightHunk(
@@ -188,15 +202,20 @@ async function highlightHunk(
lang: string,
): Promise<string[]> {
if (hunk.lines.length === 0) return [];
const code = hunk.lines.map((l) => l.content).join("\n");
// Strip trailing \r before joining with \n: Shiki normalises \r\n as a
// single newline and silently drops the \r from every line except the last.
const contents = hunk.lines.map((l) => l.content);
const trailingCR = contents.map((c) => c.endsWith("\r"));
const stripped = contents.map((c, i) => (trailingCR[i] ? c.slice(0, -1) : c));
const code = stripped.join("\n");
try {
const h = await getHighlighter();
const tokensByLine = h.codeToTokensWithThemes(code, {
lang: lang as BundledLanguage,
themes: { light: "github-light", dark: "github-dark" },
});
const lines = tokensByLine.map((lineTokens) =>
lineTokens
const lines = tokensByLine.map((lineTokens, i) => {
const html = lineTokens
.map((token) => {
const style = Object.entries(token.variants)
.map(
@@ -204,14 +223,17 @@ async function highlightHunk(
`--shiki-${theme}:${v.color ?? "inherit"}`,
)
.join(";");
return `<span style="${style}">${escapeHtml(token.content)}</span>`;
return `<span style="${style}">${escapeHtmlAndCtrl(token.content)}</span>`;
})
.join(""),
);
.join("");
return trailingCR[i]
? html + '<span class="diff-ctrl">^M</span>'
: html;
});
while (lines.length < hunk.lines.length) lines.push("");
return lines;
} catch {
return hunk.lines.map((l) => escapeHtml(l.content));
return hunk.lines.map((l) => escapeHtmlAndCtrl(l.content));
}
}
@@ -249,7 +271,7 @@ export async function highlightFile(
const highlightedHunks =
totalBytes > INLINE_MAX_BYTES
? file.hunks.map((hunk) =>
hunk.lines.map((l) => escapeHtml(l.content)),
hunk.lines.map((l) => escapeHtmlAndCtrl(l.content)),
)
: await Promise.all(
file.hunks.map((hunk) => highlightHunk(hunk, lang)),
Msrc/services/highlight.ts
@@ -71,12 +71,8 @@ export function getHighlighter(): Highlighter {
return highlighter;
}
// biome-ignore lint/suspicious/noControlCharactersInRegex: intentional binary content detection
const BINARY_RE = /[\x00-\x08\x0e-\x1f]/;
export function hasBinaryContent(buf: Buffer): boolean {
const sample = buf.subarray(0, 8000);
return BINARY_RE.test(sample.toString("binary"));
return buf.subarray(0, 8000).includes(0);
}
export function detectLang(filename: string): string {
Msrc/styles/main.css
@@ -1463,6 +1463,14 @@
font-weight: 700;
}
.diff-ctrl {
color: #f0f0f0;
background: #c0392b;
border-radius: 2px;
font-weight: bold;
padding: 0 1px;
}
/* Shiki dark-mode support in diff code cells */
@media (prefers-color-scheme: dark) {
.diff-code span[style] {
Mtests/highlight.test.ts
@@ -1,5 +1,6 @@
import { describe, test, expect, beforeAll } from "bun:test";
import { detectLang, highlightStartup } from "../src/services/highlight.ts";
import { detectLang, hasBinaryContent, highlightStartup } from "../src/services/highlight.ts";
import { parseDiff, highlightFile } from "../src/services/diffHighlight.ts";
beforeAll(async () => {
await highlightStartup();
@@ -35,3 +36,94 @@ describe("detectLang", () => {
test("falls back to text for unknown extension", () => expect(detectLang("foo.xyz")).toBe("text"));
test("falls back to text for no extension", () => expect(detectLang("LICENSE")).toBe("text"));
});
describe("hasBinaryContent", () => {
test("detects NUL byte as binary", () => {
expect(hasBinaryContent(Buffer.from([0x68, 0x65, 0x00, 0x6c, 0x6f]))).toBe(true);
});
test("detects NUL-only buffer as binary", () => {
expect(hasBinaryContent(Buffer.alloc(10, 0))).toBe(true);
});
test("plain text is not binary", () => {
expect(hasBinaryContent(Buffer.from("hello world\n"))).toBe(false);
});
test("CR LF text is not binary", () => {
expect(hasBinaryContent(Buffer.from("line1\r\nline2\r\n"))).toBe(false);
});
test("other control chars without NUL are not binary", () => {
// \x01, \x07 (BEL), \x1b (ESC) — git treats these as text
expect(hasBinaryContent(Buffer.from([0x01, 0x07, 0x1b, 0x41]))).toBe(false);
});
test("only checks first 8000 bytes", () => {
// NUL appears after the 8000-byte sample window — should not be detected
const buf = Buffer.alloc(8001, 0x41);
buf[8000] = 0x00;
expect(hasBinaryContent(buf)).toBe(false);
});
test("detects NUL within the 8000-byte sample", () => {
const buf = Buffer.alloc(8001, 0x41);
buf[7999] = 0x00;
expect(hasBinaryContent(buf)).toBe(true);
});
});
// Build a minimal unified diff string with the given added line(s).
function makeDiff(...lines: string[]): string {
return [
"diff --git a/test.txt b/test.txt",
"index 0000000..1111111 100644",
"--- a/test.txt",
"+++ b/test.txt",
`@@ -0,0 +${lines.length} @@`,
...lines.map((l) => `+${l}`),
].join("\n");
}
describe("diff control character rendering", () => {
test("CR renders as ^M with diff-ctrl span", async () => {
const files = await parseDiff(makeDiff("hello\rworld"));
const rendered = await highlightFile(files[0]!);
const html = rendered.hunks[0]!.rows[0]!.html;
expect(html).toContain('<span class="diff-ctrl">^M</span>');
});
test("trailing CR renders as ^M on every line, not just the last", async () => {
// Simulates a CRLF line-ending change where every line ends with \r.
// Shiki normalises \r\n → \n and would silently drop the \r from all
// but the last line without the pre-strip fix.
const files = await parseDiff(makeDiff("alpha\r", "beta\r", "gamma\r"));
const rendered = await highlightFile(files[0]!);
const rows = rendered.hunks[0]!.rows;
for (const row of rows) {
expect(row.html).toContain('<span class="diff-ctrl">^M</span>');
}
});
test("BEL (\\x07) renders as ^G", async () => {
const files = await parseDiff(makeDiff("ring\x07bell"));
const rendered = await highlightFile(files[0]!);
const html = rendered.hunks[0]!.rows[0]!.html;
expect(html).toContain('<span class="diff-ctrl">^G</span>');
});
test("ESC (\\x1b) renders as ^[", async () => {
const files = await parseDiff(makeDiff("\x1b[31mred\x1b[0m"));
const rendered = await highlightFile(files[0]!);
const html = rendered.hunks[0]!.rows[0]!.html;
expect(html).toContain('<span class="diff-ctrl">^[</span>');
});
test("TAB does not render as a control char span", async () => {
const files = await parseDiff(makeDiff("\thello"));
const rendered = await highlightFile(files[0]!);
const html = rendered.hunks[0]!.rows[0]!.html;
expect(html).not.toContain("diff-ctrl");
});
test("normal text has no control char spans", async () => {
const files = await parseDiff(makeDiff("hello world"));
const rendered = await highlightFile(files[0]!);
const html = rendered.hunks[0]!.rows[0]!.html;
expect(html).not.toContain("diff-ctrl");
});
});