move discovered ffmpeg presets

2025-07-03 01:22:59 -07:00 · 2025-07-03 01:22:59 -07:00 · 2320091125
commit 2320091125
parent 8d1dc0d825
8 changed files with 404 additions and 115 deletions
--- a/framework/incremental.ts
+++ b/framework/incremental.ts
@ -318,14 +318,10 @@ export class Incremental {

  async compressImpl({ algo, buffer, hash }: CompressJob) {
    let out;
-    switch (algo) {
-      case "zstd":
-        out = await zstd(buffer);
-        break;
-      case "gzip":
-        out = await gzip(buffer, { level: 9 });
-        break;
-    }
+    if (algo === "zstd") out = await zstd(buffer);
+    else if (algo === "gzip") out = await gzip(buffer, { level: 9 });
+    else algo satisfies never;
+
    let entry = this.compress.get(hash);
    if (!entry) {
      this.compress.set(
--- a/package-lock.json
+++ b/package-lock.json
@ -1,5 +1,5 @@
 {
-  "name": "paperclover.net",
+  "name": "sitegen",
  "lockfileVersion": 3,
  "requires": true,
  "packages": {
@ -22,7 +22,7 @@
        "vscode-textmate": "^9.2.0"
      },
      "devDependencies": {
-        "@types/node": "^22.15.29",
+        "@types/node": "^24.0.10",
        "typescript": "^5.8.3"
      }
    },
@ -1568,13 +1568,13 @@
      "license": "MIT"
    },
    "node_modules/@types/node": {
-      "version": "22.15.30",
-      "resolved": "https://registry.npmjs.org/@types/node/-/node-22.15.30.tgz",
-      "integrity": "sha512-6Q7lr06bEHdlfplU6YRbgG1SFBdlsfNC4/lX+SkhiTs0cpJkOElmWls8PxDFv4yY/xKb8Y6SO0OmSX4wgqTZbA==",
+      "version": "24.0.10",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-24.0.10.tgz",
+      "integrity": "sha512-ENHwaH+JIRTDIEEbDK6QSQntAYGtbvdDXnMXnZaZ6k13Du1dPMmprkEHIL7ok2Wl2aZevetwTAb5S+7yIF+enA==",
      "devOptional": true,
      "license": "MIT",
      "dependencies": {
-        "undici-types": "~6.21.0"
+        "undici-types": "~7.8.0"
      }
    },
    "node_modules/@types/unist": {
@ -4509,9 +4509,9 @@
      }
    },
    "node_modules/undici-types": {
-      "version": "6.21.0",
-      "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz",
-      "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==",
+      "version": "7.8.0",
+      "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.8.0.tgz",
+      "integrity": "sha512-9UJ2xGDvQ43tYyVMpuHlsgApydB8ZKfVYTsLDhXkFL/6gfkp+U8xTGdh8pMJv1SpZna0zxG1DwsKZsreLbXBxw==",
      "devOptional": true,
      "license": "MIT"
    },
--- a/package.json
+++ b/package.json
@ -18,7 +18,7 @@
    "vscode-textmate": "^9.2.0"
  },
  "devDependencies": {
-    "@types/node": "^22.15.29",
+    "@types/node": "^24.0.10",
    "typescript": "^5.8.3"
  },
  "imports": {
--- a/readme.md
+++ b/readme.md
@ -25,7 +25,7 @@ that assist building websites. these tools power https://paperclover.net.
    - Databases with a typed SQLite wrapper. (`import "#sitegen/sqlite"`)
    - TODO: Meta and Open Graph generation. (`export const meta`)
    - TODO: Font subsetting tools to reduce bytes downloaded by fonts.
- **Built on the battle-tested Node.js runtime.** Partial support for Deno and Bun.
+- **Built on the battle-tested Node.js runtime.**

 [1]: https://next.markojs.com

--- a/run.js
+++ b/run.js
@ -1,8 +1,26 @@
 // This file allows using Node.js in combination with
 // all available plugins. Usage: "node run <script>"
 import * as util from "node:util";
+import * as zlib from "node:zlib";
 import process from "node:process";

+if (!zlib.zstdCompress) {
+  const brand = process.versions.bun
+    ? `bun ${process.versions.bun}`
+    : process.versions.deno
+    ? `deno ${process.versions.deno}`
+    : null;
+
+  globalThis.console.error(
+    `sitegen depends on a node.js-compatibile runtime that supports zstd compression\n` +
+      `this is node.js version ${process.version}${
+        brand ? ` (${brand})` : ""
+      }\n\n` +
+      `get node.js --> https://nodejs.org/en/download/current`,
+  );
+  process.exit(1);
+}
+
 // Disable experimental warnings (Type Stripping, etc)
 const { emit: originalEmit } = process;
 const warnings = ["ExperimentalWarning"];
--- a/src/file-viewer/bin/scan3.ts
+++ b/src/file-viewer/bin/scan3.ts
@ -1,3 +1,15 @@
+// The file scanner incrementally updates an sqlite database with file
+// stats. Additionally, it runs "processors" on files, which precompute
+// expensive data such as running `ffprobe` on all media to get the
+// duration.
+//
+// Processors are also used to derive compressed and optimized assets,
+// which is how automatic JXL / AV1 encoding is done. Derived files are
+// uploaded to the clover NAS to be pulled by VPS instances for hosting.
+//
+// This is the third iteration of the scanner, hence its name "scan3";
+// Remember that any software you want to be maintainable and high
+// quality cannot be written with AI.
 const root = path.resolve("C:/media");
 const workDir = path.resolve(".clover/file-assets");

@ -258,8 +270,8 @@ const execFile: typeof execFileRaw = ((
    }
    throw e;
  })) as any;
-const ffprobe = testProgram("ffprobe", "--help");
-const ffmpeg = testProgram("ffmpeg", "--help");
+const ffprobeBin = testProgram("ffprobe", "--help");
+const ffmpegBin = testProgram("ffmpeg", "--help");

 const ffmpegOptions = [
  "-hide_banner",
@ -267,14 +279,12 @@ const ffmpegOptions = [
  "warning",
 ];

-const imageSizes = [64, 128, 256, 512, 1024, 2048];
-
 const procDuration: Process = {
  name: "calculate duration",
-  enable: ffprobe !== null,
+  enable: ffprobeBin !== null,
  include: rules.extsDuration,
  async run({ absPath, mediaFile }) {
-    const { stdout } = await execFile(ffprobe!, [
+    const { stdout } = await execFile(ffprobeBin!, [
      "-v",
      "error",
      "-show_entries",
@ -295,7 +305,7 @@ const procDuration: Process = {
 // NOTE: Never re-order the processors. Add new ones at the end.
 const procDimensions: Process = {
  name: "calculate dimensions",
-  enable: ffprobe != null,
+  enable: ffprobeBin != null,
  include: rules.extsDimensions,
  async run({ absPath, mediaFile }) {
    const ext = path.extname(absPath);
@ -370,40 +380,19 @@ const procHighlightCode: Process = {
  },
 };

-const imageSubsets = [
-  {
-    ext: ".webp",
-    // deno-fmt-disable-line
-    args: [
-      "-lossless",
-      "0",
-      "-compression_level",
-      "6",
-      "-quality",
-      "95",
-      "-method",
-      "6",
-    ],
-  },
-  {
-    ext: ".jxl",
-    args: ["-c:v", "libjxl", "-distance", "0.8", "-effort", "9"],
-  },
-];
-
 const procImageSubsets: Process = {
  name: "encode image subsets",
  include: rules.extsImage,
  depends: ["calculate dimensions"],
-  async run({ absPath, mediaFile, stat, spin }) {
+  async run({ absPath, mediaFile, spin }) {
    const { width, height } = UNWRAP(mediaFile.parseDimensions());
-    const targetSizes = imageSizes.filter((w) => w < width);
+    const targetSizes = transcodeRules.imageSizes.filter((w) => w < width);
    const baseStatus = spin.text;

    using stack = new DisposableStack();
    for (const size of targetSizes) {
      const { w, h } = resizeDimensions(width, height, size);
-      for (const { ext, args } of imageSubsets) {
+      for (const { ext, args } of transcodeRules.imagePresets) {
        spin.text = baseStatus +
          ` (${w}x${h}, ${ext.slice(1).toUpperCase()})`;

@ -413,7 +402,7 @@ const procImageSubsets: Process = {
            async (out) => {
              await fs.mkdir(path.dirname(out));
              await fs.rm(out, { force: true });
-              await execFile(ffmpeg!, [
+              await execFile(ffmpegBin!, [
                ...ffmpegOptions,
                "-i",
                absPath,
@ -433,81 +422,51 @@ const procImageSubsets: Process = {
  },
  async undo(mediaFile) {
    const { width } = UNWRAP(mediaFile.parseDimensions());
-    const targetSizes = imageSizes.filter((w) => w < width);
+    const targetSizes = transcodeRules.imageSizes.filter((w) => w < width);
    for (const size of targetSizes) {
-      for (const { ext } of imageSubsets) {
+      for (const { ext } of transcodeRules.imagePresets) {
        unproduceAsset(`${mediaFile.hash}/${size}${ext}`);
      }
    }
  },
 };

-type VideoEncodePreset = {
-  id: string;
-  codec: "av1";
-  mbit: number;
-  crf?: number;
-  maxHeight?: number;
-  audioKbit?: number;
-} | {
-  id: string;
-  codec: "vp9";
-  mbit: number;
-  maxHeight?: number;
-  audioKbit?: number;
-};
-// const av1Pass1 = [
-// '-c:v', 'libsvtav1 -b:v 2M -pass 1 -svtav1-params "rc=1:tbr=2000000" -preset 8 -f null NUL
-// ];
-// const av1BaseArgs = [
-//   ["-c:v", "libsvtav1"],
-//   ["-preset", "3"],
-//   ["-c:a", "libopus"],
-//   ["-b:a", "192k"],
-// ].flat();
-// const webmBaseArgs = [
-//   "-c:v",
-//   "libsvtav1",
-//       ["-vf",],
-// ];
-const presets: VideoEncodePreset[] = [
-  { id: "av1-max", codec: "av1", mbit: 5 },
-  { id: "av1-hq", codec: "av1", mbit: 2, maxHeight: 900 },
-  { id: "av1-md", codec: "av1", mbit: 0.5, maxHeight: 720 },
-  { id: "av1-lq", codec: "av1", mbit: 0.2, maxHeight: 640 },
-];
-const procVideos = presets.map((preset) => ({
+const procVideos = transcodeRules.videoFormats.map((preset) => ({
  name: "encode image subsets",
  include: rules.extsImage,
+  enable: ffmpegBin != null,
  async run({ absPath, mediaFile, stat, spin }) {
-    await produceAsset(`${mediaFile}/${preset.id}`, (base) => {
-      const root = path.dirname(base);
-      const pass1 = [ffmpeg!, ...ffmpegOptions, "-i", absPath];
-      if (preset.codec === "av1") {
-        pass1.push("-c:v", "libstvav1");
-        pass1.push("-b:v", preset.mbit + "M");
-        if (preset.crf) pass1.push("-crf", preset.crf.toString());
-        pass1.push(
-          "-svtav1-params",
-          ["rc=1", `tbr=${preset.mbit * 1_000_000}`].join(":"),
+    await produceAsset(`${mediaFile}/${preset.id}`, async (base) => {
+      base = path.dirname(base);
+      const args = transcodeRules.getVideoArgs(
+        preset,
+        base,
+        ["-i", absPath],
      );
-      } else if (preset.codec === "vp9") {
-        pass1.push("-c:v", "libvpx-vp9");
-      } else preset satisfies never;
-
-      if (preset.maxHeight != null) {
-        pass1.push("-vf", `scale=-2:min(${preset.maxHeight}\\,ih)`);
+      try {
+        const quality = {
+          u: "ultra-high",
+          h: "high",
+          m: "medium",
+          l: "low",
+          d: "data-saving",
+        }[preset.id[1]] ?? preset.id;
+        await ffmpeg.spawn({
+          ffmpeg: ffmpegBin!,
+          title: `${mediaFile.path.slice(1)} (${preset.codec} ${quality})`,
+          args,
+        });
+        return await collectFiles();
+      } catch (err) {
+        for (const file of await collectFiles()) {
+          // TODO: delete assets off disk
+        }
+        throw err;
      }

-      const logfile = path.join(preset.id + ".log");
-      pass1.push("-passlogfile", logfile);
-      const pass2 = pass1.slice();
-
-      pass1.push("-f", "null");
-      pass1.push(os.devNull);
-
-      pass2.push("-c:a", "libopus");
-      pass2.push("-b:a", (preset.audioKbit ?? 192) + "k");
+      async function collectFiles(): Promise<string[]> {
+        throw new Error("!");
+      }
    });
  },
 } satisfies Process));
@ -645,10 +604,11 @@ import * as path from "node:path";
 import * as child_process from "node:child_process";
 import * as util from "node:util";
 import * as crypto from "node:crypto";
-import * as os from "node:os";

 import { MediaFile, MediaFileKind } from "@/file-viewer/models/MediaFile.ts";
 import { AssetRef } from "@/file-viewer/models/AssetRef.ts";
 import { formatDate } from "@/file-viewer/format.ts";
 import * as rules from "@/file-viewer/rules.ts";
 import * as highlight from "@/file-viewer/highlight.ts";
+import * as ffmpeg from "@/file-viewer/ffmpeg.ts";
+import * as transcodeRules from "@/file-viewer/transcode-rules.ts";
--- a/src/file-viewer/ffmpeg.ts
+++ b/src/file-viewer/ffmpeg.ts
@ -0,0 +1,161 @@
+// Utilities for spawning ffmpeg and consuming its output as a `Progress`
+// A headless parser is available with `Parse`
+
+export type Line =
+  | { kind: "ignore" }
+  | { kind: "log"; level: "info" | "warn" | "error"; message: string }
+  | {
+    kind: "progress";
+    frame: number;
+    totalFrames: number;
+    speed: string | null;
+    fps: number | null;
+    rest: Record<string, string>;
+  };
+
+export const defaultExtraOptions = [
+  "-hide_banner",
+  "-stats",
+];
+
+export interface SpawnOptions {
+  args: string[];
+  title: string;
+  ffmpeg?: string;
+  progress?: Progress;
+}
+
+export async function spawn(options: SpawnOptions) {
+  const { ffmpeg = "ffmpeg", args, title } = options;
+  const proc = child_process.spawn(ffmpeg, args, {
+    stdio: ["ignore", "inherit", "pipe"],
+    env: { ...process.env, SVT_LOG: "2" },
+  });
+  const parser = new Parse();
+  const bar = options.progress ?? new Progress({ text: title });
+  let running = true;
+  const splitter = readline.createInterface({ input: proc.stderr });
+  splitter.on("line", (line) => {
+    const result = parser.onLine(line);
+    if (result.kind === "ignore") {
+      return;
+    } else if (result.kind === "log") {
+      console[result.level](result.message);
+    } else if (result.kind === "progress") {
+      if (!running) return;
+      const { frame, totalFrames, fps, speed } = result;
+      bar.value = frame;
+      bar.total = totalFrames;
+      const extras = [
+        `${fps} fps`,
+        speed,
+        parser.hlsFile,
+      ].filter(Boolean).join(", ");
+      bar.text = `${title} ${frame}/${totalFrames} ${
+        extras.length > 0 ? `(${extras})` : ""
+      }`;
+    } else result satisfies never;
+  });
+  const [code, signal] = await events.once(proc, "close");
+  if (code !== 0) {
+    const fmt = code ? `code ${code}` : `signal ${signal}`;
+    const e: any = new Error(`ffmpeg failed with ${fmt}`);
+    e.args = [ffmpeg, ...args].join(" ");
+    e.code = code;
+    e.signal = signal;
+    return e;
+  }
+}
+
+export class Parse {
+  parsingStart = true;
+  inIndentedIgnore: null | "out" | "inp" | "other" = null;
+  durationTime = 0;
+  targetFps: number | null = null;
+
+  hlsFile: string | null = null;
+  durationFrames = 0;
+
+  onLine(line: string): Line {
+    line = line.trimEnd();
+    if (/^frame=/.test(line)) {
+      if (this.parsingStart) {
+        this.parsingStart = false;
+        this.durationFrames = Math.ceil(
+          (this.targetFps ?? 25) * this.durationTime,
+        );
+      }
+      const parts = Object.fromEntries(
+        [...line.matchAll(/\b([a-z0-9]+)=\s*([^ ]+)(?= |$)/ig)].map((
+          [, k, v],
+        ) => [k, v]),
+      );
+      const { frame, fps, speed, ...rest } = parts;
+      return {
+        kind: "progress",
+        frame: Number(frame),
+        totalFrames: this.durationFrames,
+        fps: Number(fps),
+        speed,
+        rest,
+      };
+    }
+
+    if (this.parsingStart) {
+      if (this.inIndentedIgnore) {
+        if (line.startsWith(" ") || line.startsWith("\t")) {
+          line = line.trimStart();
+          if (this.inIndentedIgnore === "inp") {
+            const match = line.match(/^Duration: (\d+):(\d+):(\d+\.\d+)/);
+            if (match) {
+              const [h, m, s] = match.slice(1).map((x) => Number(x));
+              this.durationTime = Math.max(
+                this.durationTime,
+                h * 60 * 60 + m * 60 + s,
+              );
+            }
+            if (!this.targetFps) {
+              const match = line.match(/^Stream.*, (\d+) fps/);
+              if (match) this.targetFps = Number(match[1]);
+            }
+          }
+          return { kind: "ignore" };
+        }
+        this.inIndentedIgnore = null;
+      }
+      if (line === "Press [q] to stop, [?] for help") {
+        return { kind: "ignore" };
+      }
+      if (line === "Stream mapping:") {
+        this.inIndentedIgnore = "other";
+        return { kind: "ignore" };
+      }
+      if (line.startsWith("Output #") || line.startsWith("Input #")) {
+        this.inIndentedIgnore = line.slice(0, 3).toLowerCase() as "inp" | "out";
+        return { kind: "ignore" };
+      }
+    }
+
+    const hlsMatch = line.match(/^\[hls @ .*Opening '(.+)' for writing/);
+    if (hlsMatch) {
+      if (!hlsMatch[1].endsWith(".tmp")) {
+        this.hlsFile = path.basename(hlsMatch[1]);
+      }
+      return { kind: "ignore" };
+    }
+
+    let level: Extract<Line, { kind: "log" }>["level"] = "info";
+    if (line.toLowerCase().includes("err")) level = "error";
+    else if (line.toLowerCase().includes("warn")) level = "warn";
+
+    return { kind: "log", level, message: line };
+  }
+}
+
+import * as child_process from "node:child_process";
+import * as readline from "node:readline";
+import * as process from "node:process";
+import events from "node:events";
+import * as path from "node:path";
+import { Progress } from "@paperclover/console/Progress";
+
--- a/src/file-viewer/transcode-rules.ts
+++ b/src/file-viewer/transcode-rules.ts
@ -0,0 +1,154 @@
+type VideoEncodePreset = {
+  id: string;
+  codec: "av1";
+  preset: 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8;
+  mbitMax?: number;
+  crf: number;
+  maxHeight?: number;
+  audioKbit?: number;
+  depth?: 8 | 10;
+} | {
+  id: string;
+  codec: "vp9";
+  crf: number;
+  mbitMax?: number;
+  maxHeight?: number;
+  audioKbit?: number;
+};
+
+export const videoFormats = [
+  {
+    id: "au", // AV1 Ultra-High
+    codec: "av1",
+    preset: 1,
+    crf: 28,
+    depth: 10,
+  },
+  {
+    id: "vu", // VP9 Ultra-High
+    codec: "vp9",
+    crf: 30,
+  },
+  {
+    id: "ah", // AV1 High
+    preset: 2,
+    codec: "av1",
+    mbitMax: 5,
+    crf: 35,
+    depth: 10,
+  },
+  {
+    id: "am", // AV1 Medium
+    preset: 2,
+    codec: "av1",
+    mbitMax: 2,
+    crf: 40,
+    maxHeight: 900,
+  },
+  {
+    id: "vm", // VP9 Medium
+    codec: "vp9",
+    crf: 35,
+    mbitMax: 4,
+    maxHeight: 1080,
+  },
+  {
+    id: "al", // AV1 Low
+    preset: 2,
+    codec: "av1",
+    mbitMax: 1.25,
+    crf: 40,
+    maxHeight: 600,
+  },
+  {
+    id: "vl", // VP9 Low
+    codec: "vp9",
+    crf: 45,
+    mbitMax: 2,
+    maxHeight: 600,
+  },
+  {
+    id: "ad", // AV1 Data-saving
+    codec: "av1",
+    preset: 1,
+    mbitMax: 0.5,
+    crf: 10,
+    maxHeight: 360,
+  },
+  {
+    id: "vl", // VP9 Low
+    codec: "vp9",
+    crf: 10,
+    mbitMax: 0.75,
+    maxHeight: 360,
+  },
+] satisfies VideoEncodePreset[] as VideoEncodePreset[];
+
+export const imageSizes = [64, 128, 256, 512, 1024, 2048];
+export const imagePresets = [
+  {
+    ext: ".webp",
+    args: [
+      "-lossless",
+      "0",
+      "-compression_level",
+      "6",
+      "-quality",
+      "95",
+      "-method",
+      "6",
+    ],
+  },
+  // TODO: avif
+  {
+    ext: ".jxl",
+    args: ["-c:v", "libjxl", "-distance", "0.8", "-effort", "9"],
+  },
+];
+
+export function getVideoArgs(preset: VideoEncodePreset, outbase: string, input: string[]) {
+  const cmd = [...input];
+
+  if (preset.codec === "av1") {
+    cmd.push("-c:v", "libsvtav1");
+    cmd.push(
+      "-svtav1-params",
+      [
+        `preset=${preset.preset}`,
+        "keyint=2s",
+        preset.depth && `input-depth=${preset.depth}`, // EncoderBitDepth
+        `crf=${preset.crf}`, // ConstantRateFactor
+        preset.mbitMax && `mbr=${preset.mbitMax}m`, // MaxBitRate
+        "tune=1", // Tune, PSNR
+        "enable-overlays=1", // EnableOverlays
+        "fast-decode=1", // FastDecode
+        "scm=2", // ScreenContentMode, adaptive
+      ].filter(Boolean).join(":"),
+    );
+  } else if (preset.codec === "vp9") {
+    // Not much research has gone into this, since it is only going to be used on old Safari browsers.
+    cmd.push("-c:v", "libvpx-vp9");
+    cmd.push("-crf", String(preset.crf));
+    cmd.push("-b:v", preset.mbitMax ? `${preset.mbitMax * 1000}k` : "0");
+  } else preset satisfies never;
+
+  if (preset.maxHeight != null) {
+    cmd.push("-vf", `scale=-2:min(${preset.maxHeight}\\,ih)`);
+  }
+
+  cmd.push("-c:a", "libopus");
+  cmd.push("-b:a", (preset.audioKbit ?? 192) + "k");
+
+  cmd.push("-y");
+  cmd.push("-f", "hls");
+  cmd.push("-hls_list_size", "0");
+  cmd.push("-hls_segment_type", "fmp4");
+  cmd.push("-hls_time", "2");
+  cmd.push("-hls_allow_cache", "1");
+  cmd.push("-hls_fmp4_init_filename", preset.id + ".mp4");
+  cmd.push(path.join(outbase, preset.id + ".m3u8"));
+
+  return cmd;
+}
+
+import * as path from "node:path";