diff --git a/flake.nix b/flake.nix index 715bd5a..ffee12c 100644 --- a/flake.nix +++ b/flake.nix @@ -16,7 +16,10 @@ pkgs.nodejs_24 # runtime pkgs.deno # formatter (pkgs.ffmpeg.override { + withOpus = true; withSvtav1 = true; + withJxl = true; + withWebp = true; }) ]; }; diff --git a/framework/backend/entry-node.ts b/framework/backend/entry-node.ts index 5b1e877..c1a0959 100644 --- a/framework/backend/entry-node.ts +++ b/framework/backend/entry-node.ts @@ -1,35 +1,35 @@ -import "@paperclover/console/inject"; -import "#debug"; - -const protocol = "http"; - -const server = serve({ - fetch: app.fetch, -}, ({ address, port }) => { - if (address === "::") address = "::1"; - console.info(url.format({ - protocol, - hostname: address, - port, - })); -}); - -process.on("SIGINT", () => { - server.close(); - process.exit(0); -}); - -process.on("SIGTERM", () => { - server.close((err) => { - if (err) { - console.error(err); - process.exit(1); - } - process.exit(0); - }); -}); - -import app from "#backend"; -import url from "node:url"; -import { serve } from "@hono/node-server"; -import process from "node:process"; +import "@paperclover/console/inject"; +import "#debug"; + +const protocol = "http"; + +const server = serve({ + fetch: app.fetch, +}, ({ address, port }) => { + if (address === "::") address = "::1"; + console.info(url.format({ + protocol, + hostname: address, + port, + })); +}); + +process.on("SIGINT", () => { + server.close(); + process.exit(0); +}); + +process.on("SIGTERM", () => { + server.close((err) => { + if (err) { + console.error(err); + process.exit(1); + } + process.exit(0); + }); +}); + +import app from "#backend"; +import url from "node:url"; +import { serve } from "@hono/node-server"; +import process from "node:process"; diff --git a/framework/backend/entry-passthru.ts b/framework/backend/entry-passthru.ts index 1273b67..f6e5c6e 100644 --- a/framework/backend/entry-passthru.ts +++ b/framework/backend/entry-passthru.ts @@ -1,4 +1,4 @@ -import "@paperclover/console/inject"; -export default app; - -import app from "#backend"; +import "@paperclover/console/inject"; +export default app; + +import app from "#backend"; diff --git a/framework/debug.safe.ts b/framework/debug.safe.ts index 009d7a8..f959b1a 100644 --- a/framework/debug.safe.ts +++ b/framework/debug.safe.ts @@ -1,17 +1,17 @@ -globalThis.UNWRAP = (t, ...args) => { - if (t == null) { - throw new Error( - args.length > 0 ? util.format(...args) : "UNWRAP(" + t + ")", - ); - } - return t; -}; -globalThis.ASSERT = (t, ...args) => { - if (!t) { - throw new Error( - args.length > 0 ? util.format(...args) : "Assertion Failed", - ); - } -}; - -import * as util from "node:util"; +globalThis.UNWRAP = (t, ...args) => { + if (t == null) { + throw new Error( + args.length > 0 ? util.format(...args) : "UNWRAP(" + t + ")", + ); + } + return t; +}; +globalThis.ASSERT = (t, ...args) => { + if (!t) { + throw new Error( + args.length > 0 ? util.format(...args) : "Assertion Failed", + ); + } +}; + +import * as util from "node:util"; diff --git a/framework/definitions.d.ts b/framework/definitions.d.ts index 32732e8..85bee3f 100644 --- a/framework/definitions.d.ts +++ b/framework/definitions.d.ts @@ -1,4 +1,4 @@ -declare function UNWRAP(value: T | null | undefined, ...log: unknown[]): T; -declare function ASSERT(value: unknown, ...log: unknown[]): asserts value; - -type Timer = ReturnType; +declare function UNWRAP(value: T | null | undefined, ...log: unknown[]): T; +declare function ASSERT(value: unknown, ...log: unknown[]): asserts value; + +type Timer = ReturnType; diff --git a/framework/engine/jsx-runtime.ts b/framework/engine/jsx-runtime.ts index 141aaf2..973c366 100644 --- a/framework/engine/jsx-runtime.ts +++ b/framework/engine/jsx-runtime.ts @@ -1,54 +1,54 @@ -export const Fragment = ({ children }: { children: engine.Node[] }) => children; - -export function jsx( - type: string | engine.Component, - props: Record, -): engine.Element { - if (typeof type !== "function" && typeof type !== "string") { - throw new Error("Invalid component type: " + engine.inspect(type)); - } - return [engine.kElement, type, props]; -} - -export function jsxDEV( - type: string | engine.Component, - props: Record, - // Unused with the clover engine - _key: string, - // Unused with the clover engine - _isStaticChildren: boolean, - source: engine.SrcLoc, -): engine.Element { - const { fileName, lineNumber, columnNumber } = source; - - // Assert the component type is valid to render. - if (typeof type !== "function" && typeof type !== "string") { - throw new Error( - `Invalid component type at ${fileName}:${lineNumber}:${columnNumber}: ` + - engine.inspect(type) + - ". Clover SSR element must be a function or string", - ); - } - - // Construct an `ssr.Element` - return [engine.kElement, type, props, "", source]; -} - -// jsxs -export { jsx as jsxs }; - -declare global { - namespace JSX { - interface IntrinsicElements { - [name: string]: Record; - } - interface ElementChildrenAttribute { - children: Node; - } - type Element = engine.Element; - type ElementType = keyof IntrinsicElements | engine.Component; - type ElementClass = ReturnType; - } -} - -import * as engine from "./ssr.ts"; +export const Fragment = ({ children }: { children: engine.Node[] }) => children; + +export function jsx( + type: string | engine.Component, + props: Record, +): engine.Element { + if (typeof type !== "function" && typeof type !== "string") { + throw new Error("Invalid component type: " + engine.inspect(type)); + } + return [engine.kElement, type, props]; +} + +export function jsxDEV( + type: string | engine.Component, + props: Record, + // Unused with the clover engine + _key: string, + // Unused with the clover engine + _isStaticChildren: boolean, + source: engine.SrcLoc, +): engine.Element { + const { fileName, lineNumber, columnNumber } = source; + + // Assert the component type is valid to render. + if (typeof type !== "function" && typeof type !== "string") { + throw new Error( + `Invalid component type at ${fileName}:${lineNumber}:${columnNumber}: ` + + engine.inspect(type) + + ". Clover SSR element must be a function or string", + ); + } + + // Construct an `ssr.Element` + return [engine.kElement, type, props, "", source]; +} + +// jsxs +export { jsx as jsxs }; + +declare global { + namespace JSX { + interface IntrinsicElements { + [name: string]: Record; + } + interface ElementChildrenAttribute { + children: Node; + } + type Element = engine.Element; + type ElementType = keyof IntrinsicElements | engine.Component; + type ElementClass = ReturnType; + } +} + +import * as engine from "./ssr.ts"; diff --git a/framework/engine/marko-runtime.ts b/framework/engine/marko-runtime.ts index d87a2b8..238a187 100644 --- a/framework/engine/marko-runtime.ts +++ b/framework/engine/marko-runtime.ts @@ -1,147 +1,147 @@ -// This file is used to integrate Marko into the Clover Engine and Sitegen -// To use, replace the "marko/html" import with this file. -export * from "#marko/html"; - -interface BodyContentObject { - [x: PropertyKey]: unknown; - content: ServerRenderer; -} - -export const createTemplate = ( - templateId: string, - renderer: ServerRenderer, -) => { - const { render } = marko.createTemplate(templateId, renderer); - function wrap(props: Record, n: number) { - // Marko Custom Tags - const cloverAsyncMarker = { isAsync: false }; - let r: engine.Render | undefined = undefined; - try { - r = engine.getCurrentRender(); - } catch {} - // Support using Marko outside of Clover SSR - if (r) { - engine.setCurrentRender(null); - const markoResult = render.call(renderer, { - ...props, - $global: { clover: r, cloverAsyncMarker }, - }); - if (cloverAsyncMarker.isAsync) { - return markoResult.then(engine.html); - } - const rr = markoResult.toString(); - return engine.html(rr); - } else { - return renderer(props, n); - } - } - wrap.render = render; - wrap.unwrapped = renderer; - return wrap; -}; - -export const dynamicTag = ( - scopeId: number, - accessor: Accessor, - tag: unknown | string | ServerRenderer | BodyContentObject, - inputOrArgs: unknown, - content?: (() => void) | 0, - inputIsArgs?: 1, - serializeReason?: 1 | 0, -) => { - if (typeof tag === "function") { - clover: { - const unwrapped = (tag as any).unwrapped; - if (unwrapped) { - tag = unwrapped; - break clover; - } - let r: engine.Render; - try { - r = engine.getCurrentRender(); - if (!r) throw 0; - } catch { - r = marko.$global().clover as engine.Render; - } - if (!r) throw new Error("No Clover Render Active"); - const subRender = engine.initRender(r.async !== -1, r.addon); - const resolved = engine.resolveNode(subRender, [ - engine.kElement, - tag, - inputOrArgs, - ]); - - if (subRender.async > 0) { - const marker = marko.$global().cloverAsyncMarker as Async; - marker.isAsync = true; - - // Wait for async work to finish - const { resolve, reject, promise } = Promise.withResolvers(); - subRender.asyncDone = () => { - const rejections = subRender.rejections; - if (!rejections) return resolve(engine.renderNode(resolved)); - (r.rejections ??= []).push(...rejections); - return reject(new Error("Render had errors")); - }; - marko.fork( - scopeId, - accessor, - promise, - (string: string) => marko.write(string), - 0, - ); - } else { - marko.write(engine.renderNode(resolved)); - } - return; - } - } - return marko.dynamicTag( - scopeId, - accessor, - tag, - inputOrArgs, - content, - inputIsArgs, - serializeReason, - ); -}; - -export function fork( - scopeId: number, - accessor: Accessor, - promise: Promise, - callback: (data: unknown) => void, - serializeMarker?: 0 | 1, -) { - const marker = marko.$global().cloverAsyncMarker as Async; - marker.isAsync = true; - marko.fork(scopeId, accessor, promise, callback, serializeMarker); -} - -export function escapeXML(input: unknown) { - // The rationale of this check is that the default toString method - // creating `[object Object]` is universally useless to any end user. - if ( - input == null || - (typeof input === "object" && input && - // only block this if it's the default `toString` - input.toString === Object.prototype.toString) - ) { - throw new Error( - `Unexpected value in template placeholder: '` + - engine.inspect(input) + "'. " + - `To emit a literal '${input}', use \${String(value)}`, - ); - } - return marko.escapeXML(input); -} - -interface Async { - isAsync: boolean; -} - -import * as engine from "./ssr.ts"; -import type { ServerRenderer } from "marko/html/template"; -import { type Accessor } from "marko/common/types"; -import * as marko from "#marko/html"; +// This file is used to integrate Marko into the Clover Engine and Sitegen +// To use, replace the "marko/html" import with this file. +export * from "#marko/html"; + +interface BodyContentObject { + [x: PropertyKey]: unknown; + content: ServerRenderer; +} + +export const createTemplate = ( + templateId: string, + renderer: ServerRenderer, +) => { + const { render } = marko.createTemplate(templateId, renderer); + function wrap(props: Record, n: number) { + // Marko Custom Tags + const cloverAsyncMarker = { isAsync: false }; + let r: engine.Render | undefined = undefined; + try { + r = engine.getCurrentRender(); + } catch {} + // Support using Marko outside of Clover SSR + if (r) { + engine.setCurrentRender(null); + const markoResult = render.call(renderer, { + ...props, + $global: { clover: r, cloverAsyncMarker }, + }); + if (cloverAsyncMarker.isAsync) { + return markoResult.then(engine.html); + } + const rr = markoResult.toString(); + return engine.html(rr); + } else { + return renderer(props, n); + } + } + wrap.render = render; + wrap.unwrapped = renderer; + return wrap; +}; + +export const dynamicTag = ( + scopeId: number, + accessor: Accessor, + tag: unknown | string | ServerRenderer | BodyContentObject, + inputOrArgs: unknown, + content?: (() => void) | 0, + inputIsArgs?: 1, + serializeReason?: 1 | 0, +) => { + if (typeof tag === "function") { + clover: { + const unwrapped = (tag as any).unwrapped; + if (unwrapped) { + tag = unwrapped; + break clover; + } + let r: engine.Render; + try { + r = engine.getCurrentRender(); + if (!r) throw 0; + } catch { + r = marko.$global().clover as engine.Render; + } + if (!r) throw new Error("No Clover Render Active"); + const subRender = engine.initRender(r.async !== -1, r.addon); + const resolved = engine.resolveNode(subRender, [ + engine.kElement, + tag, + inputOrArgs, + ]); + + if (subRender.async > 0) { + const marker = marko.$global().cloverAsyncMarker as Async; + marker.isAsync = true; + + // Wait for async work to finish + const { resolve, reject, promise } = Promise.withResolvers(); + subRender.asyncDone = () => { + const rejections = subRender.rejections; + if (!rejections) return resolve(engine.renderNode(resolved)); + (r.rejections ??= []).push(...rejections); + return reject(new Error("Render had errors")); + }; + marko.fork( + scopeId, + accessor, + promise, + (string: string) => marko.write(string), + 0, + ); + } else { + marko.write(engine.renderNode(resolved)); + } + return; + } + } + return marko.dynamicTag( + scopeId, + accessor, + tag, + inputOrArgs, + content, + inputIsArgs, + serializeReason, + ); +}; + +export function fork( + scopeId: number, + accessor: Accessor, + promise: Promise, + callback: (data: unknown) => void, + serializeMarker?: 0 | 1, +) { + const marker = marko.$global().cloverAsyncMarker as Async; + marker.isAsync = true; + marko.fork(scopeId, accessor, promise, callback, serializeMarker); +} + +export function escapeXML(input: unknown) { + // The rationale of this check is that the default toString method + // creating `[object Object]` is universally useless to any end user. + if ( + input == null || + (typeof input === "object" && input && + // only block this if it's the default `toString` + input.toString === Object.prototype.toString) + ) { + throw new Error( + `Unexpected value in template placeholder: '` + + engine.inspect(input) + "'. " + + `To emit a literal '${input}', use \${String(value)}`, + ); + } + return marko.escapeXML(input); +} + +interface Async { + isAsync: boolean; +} + +import * as engine from "./ssr.ts"; +import type { ServerRenderer } from "marko/html/template"; +import { type Accessor } from "marko/common/types"; +import * as marko from "#marko/html"; diff --git a/framework/engine/ssr.test.tsx b/framework/engine/ssr.test.tsx index e54dc99..9b153b5 100644 --- a/framework/engine/ssr.test.tsx +++ b/framework/engine/ssr.test.tsx @@ -1,41 +1,41 @@ -import { test } from "node:test"; -import * as engine from "./ssr.ts"; - -test("sanity", (t) => t.assert.equal(engine.ssrSync("gm <3").text, "gm <3")); -test("simple tree", (t) => - t.assert.equal( - engine.ssrSync( -
-

hello world

-

haha

- {1}| - {0}| - {true}| - {false}| - {null}| - {undefined}| -
, - ).text, - '

hello world

haha

1|0|||||
', - )); -test("unescaped/escaped html", (t) => - t.assert.equal( - engine.ssrSync(
{engine.html("")}{"\"&'`<>"}
).text, - "
"&'`<>
", - )); -test("clsx built-in", (t) => - t.assert.equal( - engine.ssrSync( - <> - - - - - - - , - ).text, - '', - )); +import { test } from "node:test"; +import * as engine from "./ssr.ts"; + +test("sanity", (t) => t.assert.equal(engine.ssrSync("gm <3").text, "gm <3")); +test("simple tree", (t) => + t.assert.equal( + engine.ssrSync( +
+

hello world

+

haha

+ {1}| + {0}| + {true}| + {false}| + {null}| + {undefined}| +
, + ).text, + '

hello world

haha

1|0|||||
', + )); +test("unescaped/escaped html", (t) => + t.assert.equal( + engine.ssrSync(
{engine.html("")}{"\"&'`<>"}
).text, + "
"&'`<>
", + )); +test("clsx built-in", (t) => + t.assert.equal( + engine.ssrSync( + <> + + + + + + + , + ).text, + '', + )); diff --git a/framework/engine/ssr.ts b/framework/engine/ssr.ts index 56ca992..d417e92 100644 --- a/framework/engine/ssr.ts +++ b/framework/engine/ssr.ts @@ -1,311 +1,311 @@ -// Clover's Rendering Engine is the backbone of her website generator. It -// converts objects and components (functions returning 'Node') into HTML. The -// engine is simple and self-contained, with integrations for JSX and Marko -// (which can interop with each-other) are provided next to this file. -// -// Add-ons to the rendering engine can provide opaque data, And retrieve it -// within component calls with 'getAddonData'. For example, 'sitegen' uses this -// to track needed client scripts without introducing patches to the engine. -export type Addons = Record; - -export function ssrSync(node: Node): Result; -export function ssrSync(node: Node, addon: A): Result; -export function ssrSync(node: Node, addon: Addons = {}) { - const r = initRender(false, addon); - const resolved = resolveNode(r, node); - return { text: renderNode(resolved), addon }; -} - -export function ssrAsync(node: Node): Promise; -export function ssrAsync( - node: Node, - addon: A, -): Promise>; -export function ssrAsync(node: Node, addon: Addons = {}) { - const r = initRender(true, addon); - const resolved = resolveNode(r, node); - if (r.async === 0) { - return Promise.resolve({ text: renderNode(resolved), addon }); - } - const { resolve, reject, promise } = Promise.withResolvers(); - r.asyncDone = () => { - const rejections = r.rejections; - if (!rejections) return resolve({ text: renderNode(resolved), addon }); - if (rejections.length === 1) return reject(rejections[0]); - return reject(new AggregateError(rejections)); - }; - return promise; -} - -/** Inline HTML into a render without escaping it */ -export function html(rawText: ResolvedNode): DirectHtml { - return [kDirectHtml, rawText]; -} - -interface Result { - text: string; - addon: A; -} - -export interface Render { - /** - * Set to '-1' if rendering synchronously - * Number of async promises the render is waiting on. - */ - async: number | -1; - asyncDone: null | (() => void); - /** When components reject, those are logged here */ - rejections: unknown[] | null; - /** Add-ons to the rendering engine store state here */ - addon: Addons; -} - -export const kElement = Symbol("Element"); -export const kDirectHtml = Symbol("DirectHtml"); - -/** Node represents a webpage that can be 'rendered' into HTML. */ -export type Node = - | number - | string // Escape HTML - | Node[] // Concat - | Element // Render - | DirectHtml // Insert - | Promise // Await - // Ignore - | undefined - | null - | boolean; -export type Element = [ - tag: typeof kElement, - type: string | Component, - props: Record, - _?: "", - source?: SrcLoc, -]; -export type DirectHtml = [tag: typeof kDirectHtml, html: ResolvedNode]; -/** - * Components must return a value; 'undefined' is prohibited here - * to avoid functions that are missing a return statement. - */ -export type Component = ( - props: Record, -) => Exclude; -/** Emitted by JSX runtime */ -export interface SrcLoc { - fileName: string; - lineNumber: number; - columnNumber: number; -} - -/** - * Resolution narrows the type 'Node' into 'ResolvedNode'. Async trees are - * marked in the 'Render'. This operation performs everything besides the final - * string concatenation. This function is agnostic across async/sync modes. - */ -export function resolveNode(r: Render, node: unknown): ResolvedNode { - if (!node && node !== 0) return ""; // falsy, non numeric - if (typeof node !== "object") { - if (node === true) return ""; // booleans are ignored - if (typeof node === "string") return escapeHtml(node); - if (typeof node === "number") return String(node); // no escaping ever - throw new Error(`Cannot render ${inspect(node)} to HTML`); - } - if (node instanceof Promise) { - if (r.async === -1) { - throw new Error(`Asynchronous rendering is not supported here.`); - } - const placeholder: InsertionPoint = [null]; - r.async += 1; - node - .then((result) => void (placeholder[0] = resolveNode(r, result))) - // Intentionally catching errors in `resolveNode` - .catch((e) => (r.rejections ??= []).push(e)) - .finally(() => { - if (--r.async == 0) { - if (r.asyncDone == null) throw new Error("r.asyncDone == null"); - r.asyncDone(); - r.asyncDone = null; - } - }); - // This lie is checked with an assertion in `renderNode` - return placeholder as [ResolvedNode]; - } - if (!Array.isArray(node)) { - throw new Error(`Invalid node type: ${inspect(node)}`); - } - const type = node[0]; - if (type === kElement) { - const { 1: tag, 2: props } = node; - if (typeof tag === "function") { - currentRender = r; - try { - return resolveNode(r, tag(props)); - } catch (e) { - const { 4: src } = node; - if (e && typeof e === "object") { - } - } finally { - currentRender = null; - } - } - if (typeof tag !== "string") throw new Error("Unexpected " + typeof type); - const children = props?.children; - if (children) return [kElement, tag, props, resolveNode(r, children)]; - return node; - } - if (type === kDirectHtml) return node[1]; - return node.map((elem) => resolveNode(r, elem)); -} - -export type ResolvedNode = - | ResolvedNode[] // Concat - | ResolvedElement // Render - | string; // Direct HTML -export type ResolvedElement = [ - tag: typeof kElement, - type: string, - props: Record, - children: ResolvedNode, -]; -/** - * Async rendering is done by creating an array of one item, - * which is already a valid 'Node', but the element is written - * once the data is available. The 'Render' contains a count - * of how many async jobs are left. - */ -export type InsertionPoint = [null | ResolvedNode]; - -/** - * Convert 'ResolvedNode' into HTML text. This operation happens after all - * async work is settled. The HTML is emitted as concisely as possible. - */ -export function renderNode(node: ResolvedNode): string { - if (typeof node === "string") return node; - ASSERT(node, "Unresolved Render Node"); - const type = node[0]; - if (type === kElement) { - return renderElement(node as ResolvedElement); - } - node = node as ResolvedNode[]; // TS cannot infer. - let out = type ? renderNode(type) : ""; - let len = node.length; - for (let i = 1; i < len; i++) { - const elem = node[i]; - if (elem) out += renderNode(elem); - } - return out; -} -function renderElement(element: ResolvedElement) { - const { 1: tag, 2: props, 3: children } = element; - let out = "<" + tag; - let needSpace = true; - for (const prop in props) { - const value = props[prop]; - if (!value || typeof value === "function") continue; - let attr; - switch (prop) { - default: - attr = `${prop}=${quoteIfNeeded(escapeHtml(String(value)))}`; - break; - case "className": - // Legacy React Compat - case "class": - attr = `class=${quoteIfNeeded(escapeHtml(clsx(value as ClsxInput)))}`; - break; - case "htmlFor": - throw new Error("Do not use the `htmlFor` attribute. Use `for`"); - // Do not process these - case "children": - case "ref": - case "dangerouslySetInnerHTML": - case "key": - continue; - } - if (needSpace) out += " ", needSpace = !attr.endsWith('"'); - out += attr; - } - out += ">"; - if (children) out += renderNode(children); - if ( - tag !== "br" && tag !== "img" && tag !== "input" && tag !== "meta" && - tag !== "link" && tag !== "hr" - ) { - out += ``; - } - return out; -} -export function renderStyleAttribute(style: Record) { - let out = ``; - for (const styleName in style) { - if (out) out += ";"; - out += `${styleName.replace(/[A-Z]/g, "-$&").toLowerCase()}:${ - escapeHtml(String(style[styleName])) - }`; - } - return "style=" + quoteIfNeeded(out); -} -export function quoteIfNeeded(text: string) { - if (text.includes(" ")) return '"' + text + '"'; - return text; -} - -// -- utility functions -- - -export function initRender(allowAsync: boolean, addon: Addons): Render { - return { - async: allowAsync ? 0 : -1, - rejections: null, - asyncDone: null, - addon, - }; -} - -let currentRender: Render | null = null; -export function getCurrentRender() { - if (!currentRender) throw new Error("No Render Active"); - return currentRender; -} -export function setCurrentRender(r?: Render | null) { - currentRender = r ?? null; -} -export function getUserData(namespace: PropertyKey, def: () => T): T { - return (getCurrentRender().addon[namespace] ??= def()) as T; -} - -export function inspect(object: unknown) { - try { - return require("node:util").inspect(object); - } catch { - return typeof object; - } -} - -export type ClsxInput = string | Record | ClsxInput[]; -export function clsx(mix: ClsxInput) { - var k, y, str = ""; - if (typeof mix === "string") { - return mix; - } else if (typeof mix === "object") { - if (Array.isArray(mix)) { - for (k = 0; k < mix.length; k++) { - if (mix[k] && (y = clsx(mix[k]))) { - str && (str += " "); - str += y; - } - } - } else { - for (k in mix) { - if (mix[k]) { - str && (str += " "); - str += k; - } - } - } - } - return str; -} - -export const escapeHtml = (unsafeText: string) => - String(unsafeText) - .replace(/&/g, "&").replace(//g, ">") - .replace(/"/g, """).replace(/'/g, "'").replace(/`/g, "`"); +// Clover's Rendering Engine is the backbone of her website generator. It +// converts objects and components (functions returning 'Node') into HTML. The +// engine is simple and self-contained, with integrations for JSX and Marko +// (which can interop with each-other) are provided next to this file. +// +// Add-ons to the rendering engine can provide opaque data, And retrieve it +// within component calls with 'getAddonData'. For example, 'sitegen' uses this +// to track needed client scripts without introducing patches to the engine. +export type Addons = Record; + +export function ssrSync(node: Node): Result; +export function ssrSync(node: Node, addon: A): Result; +export function ssrSync(node: Node, addon: Addons = {}) { + const r = initRender(false, addon); + const resolved = resolveNode(r, node); + return { text: renderNode(resolved), addon }; +} + +export function ssrAsync(node: Node): Promise; +export function ssrAsync( + node: Node, + addon: A, +): Promise>; +export function ssrAsync(node: Node, addon: Addons = {}) { + const r = initRender(true, addon); + const resolved = resolveNode(r, node); + if (r.async === 0) { + return Promise.resolve({ text: renderNode(resolved), addon }); + } + const { resolve, reject, promise } = Promise.withResolvers(); + r.asyncDone = () => { + const rejections = r.rejections; + if (!rejections) return resolve({ text: renderNode(resolved), addon }); + if (rejections.length === 1) return reject(rejections[0]); + return reject(new AggregateError(rejections)); + }; + return promise; +} + +/** Inline HTML into a render without escaping it */ +export function html(rawText: ResolvedNode): DirectHtml { + return [kDirectHtml, rawText]; +} + +interface Result { + text: string; + addon: A; +} + +export interface Render { + /** + * Set to '-1' if rendering synchronously + * Number of async promises the render is waiting on. + */ + async: number | -1; + asyncDone: null | (() => void); + /** When components reject, those are logged here */ + rejections: unknown[] | null; + /** Add-ons to the rendering engine store state here */ + addon: Addons; +} + +export const kElement = Symbol("Element"); +export const kDirectHtml = Symbol("DirectHtml"); + +/** Node represents a webpage that can be 'rendered' into HTML. */ +export type Node = + | number + | string // Escape HTML + | Node[] // Concat + | Element // Render + | DirectHtml // Insert + | Promise // Await + // Ignore + | undefined + | null + | boolean; +export type Element = [ + tag: typeof kElement, + type: string | Component, + props: Record, + _?: "", + source?: SrcLoc, +]; +export type DirectHtml = [tag: typeof kDirectHtml, html: ResolvedNode]; +/** + * Components must return a value; 'undefined' is prohibited here + * to avoid functions that are missing a return statement. + */ +export type Component = ( + props: Record, +) => Exclude; +/** Emitted by JSX runtime */ +export interface SrcLoc { + fileName: string; + lineNumber: number; + columnNumber: number; +} + +/** + * Resolution narrows the type 'Node' into 'ResolvedNode'. Async trees are + * marked in the 'Render'. This operation performs everything besides the final + * string concatenation. This function is agnostic across async/sync modes. + */ +export function resolveNode(r: Render, node: unknown): ResolvedNode { + if (!node && node !== 0) return ""; // falsy, non numeric + if (typeof node !== "object") { + if (node === true) return ""; // booleans are ignored + if (typeof node === "string") return escapeHtml(node); + if (typeof node === "number") return String(node); // no escaping ever + throw new Error(`Cannot render ${inspect(node)} to HTML`); + } + if (node instanceof Promise) { + if (r.async === -1) { + throw new Error(`Asynchronous rendering is not supported here.`); + } + const placeholder: InsertionPoint = [null]; + r.async += 1; + node + .then((result) => void (placeholder[0] = resolveNode(r, result))) + // Intentionally catching errors in `resolveNode` + .catch((e) => (r.rejections ??= []).push(e)) + .finally(() => { + if (--r.async == 0) { + if (r.asyncDone == null) throw new Error("r.asyncDone == null"); + r.asyncDone(); + r.asyncDone = null; + } + }); + // This lie is checked with an assertion in `renderNode` + return placeholder as [ResolvedNode]; + } + if (!Array.isArray(node)) { + throw new Error(`Invalid node type: ${inspect(node)}`); + } + const type = node[0]; + if (type === kElement) { + const { 1: tag, 2: props } = node; + if (typeof tag === "function") { + currentRender = r; + try { + return resolveNode(r, tag(props)); + } catch (e) { + const { 4: src } = node; + if (e && typeof e === "object") { + } + } finally { + currentRender = null; + } + } + if (typeof tag !== "string") throw new Error("Unexpected " + typeof type); + const children = props?.children; + if (children) return [kElement, tag, props, resolveNode(r, children)]; + return node; + } + if (type === kDirectHtml) return node[1]; + return node.map((elem) => resolveNode(r, elem)); +} + +export type ResolvedNode = + | ResolvedNode[] // Concat + | ResolvedElement // Render + | string; // Direct HTML +export type ResolvedElement = [ + tag: typeof kElement, + type: string, + props: Record, + children: ResolvedNode, +]; +/** + * Async rendering is done by creating an array of one item, + * which is already a valid 'Node', but the element is written + * once the data is available. The 'Render' contains a count + * of how many async jobs are left. + */ +export type InsertionPoint = [null | ResolvedNode]; + +/** + * Convert 'ResolvedNode' into HTML text. This operation happens after all + * async work is settled. The HTML is emitted as concisely as possible. + */ +export function renderNode(node: ResolvedNode): string { + if (typeof node === "string") return node; + ASSERT(node, "Unresolved Render Node"); + const type = node[0]; + if (type === kElement) { + return renderElement(node as ResolvedElement); + } + node = node as ResolvedNode[]; // TS cannot infer. + let out = type ? renderNode(type) : ""; + let len = node.length; + for (let i = 1; i < len; i++) { + const elem = node[i]; + if (elem) out += renderNode(elem); + } + return out; +} +function renderElement(element: ResolvedElement) { + const { 1: tag, 2: props, 3: children } = element; + let out = "<" + tag; + let needSpace = true; + for (const prop in props) { + const value = props[prop]; + if (!value || typeof value === "function") continue; + let attr; + switch (prop) { + default: + attr = `${prop}=${quoteIfNeeded(escapeHtml(String(value)))}`; + break; + case "className": + // Legacy React Compat + case "class": + attr = `class=${quoteIfNeeded(escapeHtml(clsx(value as ClsxInput)))}`; + break; + case "htmlFor": + throw new Error("Do not use the `htmlFor` attribute. Use `for`"); + // Do not process these + case "children": + case "ref": + case "dangerouslySetInnerHTML": + case "key": + continue; + } + if (needSpace) out += " ", needSpace = !attr.endsWith('"'); + out += attr; + } + out += ">"; + if (children) out += renderNode(children); + if ( + tag !== "br" && tag !== "img" && tag !== "input" && tag !== "meta" && + tag !== "link" && tag !== "hr" + ) { + out += ``; + } + return out; +} +export function renderStyleAttribute(style: Record) { + let out = ``; + for (const styleName in style) { + if (out) out += ";"; + out += `${styleName.replace(/[A-Z]/g, "-$&").toLowerCase()}:${ + escapeHtml(String(style[styleName])) + }`; + } + return "style=" + quoteIfNeeded(out); +} +export function quoteIfNeeded(text: string) { + if (text.includes(" ")) return '"' + text + '"'; + return text; +} + +// -- utility functions -- + +export function initRender(allowAsync: boolean, addon: Addons): Render { + return { + async: allowAsync ? 0 : -1, + rejections: null, + asyncDone: null, + addon, + }; +} + +let currentRender: Render | null = null; +export function getCurrentRender() { + if (!currentRender) throw new Error("No Render Active"); + return currentRender; +} +export function setCurrentRender(r?: Render | null) { + currentRender = r ?? null; +} +export function getUserData(namespace: PropertyKey, def: () => T): T { + return (getCurrentRender().addon[namespace] ??= def()) as T; +} + +export function inspect(object: unknown) { + try { + return require("node:util").inspect(object); + } catch { + return typeof object; + } +} + +export type ClsxInput = string | Record | ClsxInput[]; +export function clsx(mix: ClsxInput) { + var k, y, str = ""; + if (typeof mix === "string") { + return mix; + } else if (typeof mix === "object") { + if (Array.isArray(mix)) { + for (k = 0; k < mix.length; k++) { + if (mix[k] && (y = clsx(mix[k]))) { + str && (str += " "); + str += y; + } + } + } else { + for (k in mix) { + if (mix[k]) { + str && (str += " "); + str += k; + } + } + } + } + return str; +} + +export const escapeHtml = (unsafeText: string) => + String(unsafeText) + .replace(/&/g, "&").replace(//g, ">") + .replace(/"/g, """).replace(/'/g, "'").replace(/`/g, "`"); diff --git a/framework/engine/suspense.test.tsx b/framework/engine/suspense.test.tsx index 96adbc4..1750c5e 100644 --- a/framework/engine/suspense.test.tsx +++ b/framework/engine/suspense.test.tsx @@ -1,40 +1,40 @@ -import { test } from "node:test"; -import { renderStreaming, Suspense } from "./suspense.ts"; - -test("sanity", async (t) => { - let resolve: () => void = null!; - - // @ts-expect-error - async function AsyncComponent() { - await new Promise((done) => resolve = done); - return ; - } - - const example = ( -
-

app shell

- - - -
(c) 2025
-
- ); - - const iterator = renderStreaming(example); - const assertContinue = (actual: unknown, value: unknown) => - t.assert.deepEqual(actual, { done: false, value }); - - assertContinue( - await iterator.next(), - "", - ); - t.assert.ok(resolve !== null), resolve(); - assertContinue( - await iterator.next(), - "", - ); - t.assert.deepEqual( - await iterator.next(), - { done: true, value: {} }, - ); -}); +import { test } from "node:test"; +import { renderStreaming, Suspense } from "./suspense.ts"; + +test("sanity", async (t) => { + let resolve: () => void = null!; + + // @ts-expect-error + async function AsyncComponent() { + await new Promise((done) => resolve = done); + return ; + } + + const example = ( +
+

app shell

+ + + +
(c) 2025
+
+ ); + + const iterator = renderStreaming(example); + const assertContinue = (actual: unknown, value: unknown) => + t.assert.deepEqual(actual, { done: false, value }); + + assertContinue( + await iterator.next(), + "", + ); + t.assert.ok(resolve !== null), resolve(); + assertContinue( + await iterator.next(), + "", + ); + t.assert.deepEqual( + await iterator.next(), + { done: true, value: {} }, + ); +}); diff --git a/framework/engine/suspense.ts b/framework/engine/suspense.ts index 9435364..a04b384 100644 --- a/framework/engine/suspense.ts +++ b/framework/engine/suspense.ts @@ -1,102 +1,102 @@ -// This file implements out-of-order HTML streaming, mimicking the React -// Suspense API. To use, place Suspense around an expensive async component -// and render the page with 'renderStreaming'. -// -// Implementation of this article: -// https://lamplightdev.com/blog/2024/01/10/streaming-html-out-of-order-without-javascript/ -// -// I would link to an article from Next.js or React, but their examples -// are too verbose and not informative to what they actually do. -const kState = Symbol("SuspenseState"); - -interface SuspenseProps { - children: ssr.Node; - fallback?: ssr.Node; -} - -interface State { - nested: boolean; - nextId: number; - completed: number; - pushChunk(name: string, node: ssr.ResolvedNode): void; -} - -export function Suspense({ children, fallback }: SuspenseProps): ssr.Node { - const state = ssr.getUserData(kState, () => { - throw new Error("Can only use with 'renderStreaming'"); - }); - if (state.nested) throw new Error(" cannot be nested"); - const parent = ssr.getCurrentRender()!; - const r = ssr.initRender(true, { [kState]: { nested: true } }); - const resolved = ssr.resolveNode(r, children); - if (r.async == 0) return ssr.html(resolved); - const name = "suspended_" + (++state.nextId); - state.nested = true; - const ip: [ssr.ResolvedNode] = [ - [ - ssr.kElement, - "slot", - { name }, - fallback ? ssr.resolveNode(parent, fallback) : "", - ], - ]; - state.nested = false; - r.asyncDone = () => { - const rejections = r.rejections; - if (rejections && rejections.length > 0) throw new Error("TODO"); - state.pushChunk?.(name, ip[0] = resolved); - }; - return ssr.html(ip); -} - -// TODO: add a User-Agent parameter, which is used to determine if a -// fallback path must be used. -// - Before ~2024 needs to use a JS implementation. -// - IE should probably bail out entirely. -export async function* renderStreaming< - T extends ssr.Addons = Record, ->( - node: ssr.Node, - addon: T = {} as T, -) { - const { - text: begin, - addon: { [kState]: state, ...addonOutput }, - } = await ssr.ssrAsync(node, { - ...addon, - [kState]: { - nested: false, - nextId: 0, - completed: 0, - pushChunk: () => {}, - } satisfies State as State, - }); - if (state.nextId === 0) { - yield begin; - return addonOutput as unknown as T; - } - let resolve: (() => void) | null = null; - let chunks: string[] = []; - state.pushChunk = (slot, node) => { - while (node.length === 1 && Array.isArray(node)) node = node[0]; - if (node[0] === ssr.kElement) { - (node as ssr.ResolvedElement)[2].slot = slot; - } else { - node = [ssr.kElement, "clover-suspense", { - style: "display:contents", - slot, - }, node]; - } - chunks.push(ssr.renderNode(node)); - resolve?.(); - }; - yield ``; - do { - await new Promise((done) => resolve = done); - yield* chunks; - chunks = []; - } while (state.nextId < state.completed); - return addonOutput as unknown as T; -} - -import * as ssr from "./ssr.ts"; +// This file implements out-of-order HTML streaming, mimicking the React +// Suspense API. To use, place Suspense around an expensive async component +// and render the page with 'renderStreaming'. +// +// Implementation of this article: +// https://lamplightdev.com/blog/2024/01/10/streaming-html-out-of-order-without-javascript/ +// +// I would link to an article from Next.js or React, but their examples +// are too verbose and not informative to what they actually do. +const kState = Symbol("SuspenseState"); + +interface SuspenseProps { + children: ssr.Node; + fallback?: ssr.Node; +} + +interface State { + nested: boolean; + nextId: number; + completed: number; + pushChunk(name: string, node: ssr.ResolvedNode): void; +} + +export function Suspense({ children, fallback }: SuspenseProps): ssr.Node { + const state = ssr.getUserData(kState, () => { + throw new Error("Can only use with 'renderStreaming'"); + }); + if (state.nested) throw new Error(" cannot be nested"); + const parent = ssr.getCurrentRender()!; + const r = ssr.initRender(true, { [kState]: { nested: true } }); + const resolved = ssr.resolveNode(r, children); + if (r.async == 0) return ssr.html(resolved); + const name = "suspended_" + (++state.nextId); + state.nested = true; + const ip: [ssr.ResolvedNode] = [ + [ + ssr.kElement, + "slot", + { name }, + fallback ? ssr.resolveNode(parent, fallback) : "", + ], + ]; + state.nested = false; + r.asyncDone = () => { + const rejections = r.rejections; + if (rejections && rejections.length > 0) throw new Error("TODO"); + state.pushChunk?.(name, ip[0] = resolved); + }; + return ssr.html(ip); +} + +// TODO: add a User-Agent parameter, which is used to determine if a +// fallback path must be used. +// - Before ~2024 needs to use a JS implementation. +// - IE should probably bail out entirely. +export async function* renderStreaming< + T extends ssr.Addons = Record, +>( + node: ssr.Node, + addon: T = {} as T, +) { + const { + text: begin, + addon: { [kState]: state, ...addonOutput }, + } = await ssr.ssrAsync(node, { + ...addon, + [kState]: { + nested: false, + nextId: 0, + completed: 0, + pushChunk: () => {}, + } satisfies State as State, + }); + if (state.nextId === 0) { + yield begin; + return addonOutput as unknown as T; + } + let resolve: (() => void) | null = null; + let chunks: string[] = []; + state.pushChunk = (slot, node) => { + while (node.length === 1 && Array.isArray(node)) node = node[0]; + if (node[0] === ssr.kElement) { + (node as ssr.ResolvedElement)[2].slot = slot; + } else { + node = [ssr.kElement, "clover-suspense", { + style: "display:contents", + slot, + }, node]; + } + chunks.push(ssr.renderNode(node)); + resolve?.(); + }; + yield ``; + do { + await new Promise((done) => resolve = done); + yield* chunks; + chunks = []; + } while (state.nextId < state.completed); + return addonOutput as unknown as T; +} + +import * as ssr from "./ssr.ts"; diff --git a/framework/esbuild-support.ts b/framework/esbuild-support.ts index eb7d5a3..3b611da 100644 --- a/framework/esbuild-support.ts +++ b/framework/esbuild-support.ts @@ -1,79 +1,79 @@ -export function virtualFiles( - map: Record, -) { - return { - name: "clover vfs", - setup(b) { - b.onResolve( - { - filter: new RegExp( - `^(?:${ - Object.keys(map).map((file) => string.escapeRegExp(file)).join( - "|", - ) - })\$`, - ), - }, - ({ path }) => ({ path, namespace: "vfs" }), - ); - b.onLoad( - { filter: /./, namespace: "vfs" }, - ({ path }) => { - const entry = map[path]; - return ({ - resolveDir: ".", - loader: "ts", - ...typeof entry === "string" ? { contents: entry } : entry, - }); - }, - ); - }, - } satisfies esbuild.Plugin; -} - -export function banFiles( - files: string[], -) { - return { - name: "clover vfs", - setup(b) { - b.onResolve( - { - filter: new RegExp( - `^(?:${ - files.map((file) => string.escapeRegExp(file)).join("|") - })\$`, - ), - }, - ({ path, importer }) => { - throw new Error( - `Loading ${path} (from ${importer}) is banned!`, - ); - }, - ); - }, - } satisfies esbuild.Plugin; -} - -export function projectRelativeResolution(root = process.cwd() + "/src") { - return { - name: "project relative resolution ('@/' prefix)", - setup(b) { - b.onResolve({ filter: /^@\// }, ({ path: id }) => { - return { - path: path.resolve(root, id.slice(2)), - }; - }); - b.onResolve({ filter: /^#/ }, ({ path: id, importer }) => { - return { - path: hot.resolveFrom(importer, id), - }; - }); - }, - } satisfies esbuild.Plugin; -} - -import * as esbuild from "esbuild"; -import * as string from "#sitegen/string"; -import * as path from "node:path"; -import * as hot from "./hot.ts"; +export function virtualFiles( + map: Record, +) { + return { + name: "clover vfs", + setup(b) { + b.onResolve( + { + filter: new RegExp( + `^(?:${ + Object.keys(map).map((file) => string.escapeRegExp(file)).join( + "|", + ) + })\$`, + ), + }, + ({ path }) => ({ path, namespace: "vfs" }), + ); + b.onLoad( + { filter: /./, namespace: "vfs" }, + ({ path }) => { + const entry = map[path]; + return ({ + resolveDir: ".", + loader: "ts", + ...typeof entry === "string" ? { contents: entry } : entry, + }); + }, + ); + }, + } satisfies esbuild.Plugin; +} + +export function banFiles( + files: string[], +) { + return { + name: "clover vfs", + setup(b) { + b.onResolve( + { + filter: new RegExp( + `^(?:${ + files.map((file) => string.escapeRegExp(file)).join("|") + })\$`, + ), + }, + ({ path, importer }) => { + throw new Error( + `Loading ${path} (from ${importer}) is banned!`, + ); + }, + ); + }, + } satisfies esbuild.Plugin; +} + +export function projectRelativeResolution(root = process.cwd() + "/src") { + return { + name: "project relative resolution ('@/' prefix)", + setup(b) { + b.onResolve({ filter: /^@\// }, ({ path: id }) => { + return { + path: path.resolve(root, id.slice(2)), + }; + }); + b.onResolve({ filter: /^#/ }, ({ path: id, importer }) => { + return { + path: hot.resolveFrom(importer, id), + }; + }); + }, + } satisfies esbuild.Plugin; +} + +import * as esbuild from "esbuild"; +import * as string from "#sitegen/string"; +import * as path from "node:path"; +import * as hot from "./hot.ts"; diff --git a/framework/hot.ts b/framework/hot.ts index 987fabd..c66dc4d 100644 --- a/framework/hot.ts +++ b/framework/hot.ts @@ -96,7 +96,9 @@ Module._resolveFilename = (...args) => { try { return require.resolve(replacedPath, { paths: [projectSrc] }); } catch (err: any) { - if (err.code === "MODULE_NOT_FOUND" && (err?.requireStack?.length ?? 0) <= 1) { + if ( + err.code === "MODULE_NOT_FOUND" && (err?.requireStack?.length ?? 0) <= 1 + ) { err.message.replace(replacedPath, args[0]); } } diff --git a/framework/lib/async.ts b/framework/lib/async.ts index 522e156..3e7aee2 100644 --- a/framework/lib/async.ts +++ b/framework/lib/async.ts @@ -1,297 +1,299 @@ -const five_minutes = 5 * 60 * 1000; - -interface QueueOptions { - name: string; - fn: (item: T, spin: Spinner) => Promise; - getItemText?: (item: T) => string; - maxJobs?: number; - passive?: boolean; -} - -// Process multiple items in parallel, queue up as many. -export class Queue { - #name: string; - #fn: (item: T, spin: Spinner) => Promise; - #maxJobs: number; - #getItemText: (item: T) => string; - #passive: boolean; - - #active: Spinner[] = []; - #queue: Array<[T] | [T, (result: R) => void, (err: unknown) => void]> = []; - - #cachedProgress: Progress<{ active: Spinner[] }> | null = null; - #done: number = 0; - #total: number = 0; - #onComplete: (() => void) | null = null; - #estimate: number | null = null; - #errors: unknown[] = []; - - constructor(options: QueueOptions) { - this.#name = options.name; - this.#fn = options.fn; - this.#maxJobs = options.maxJobs ?? 5; - this.#getItemText = options.getItemText ?? defaultGetItemText; - this.#passive = options.passive ?? false; - } - - cancel() { - const bar = this.#cachedProgress; - bar?.stop(); - this.#queue = []; - } - - get bar() { - const cached = this.#cachedProgress; - if (!cached) { - const bar = this.#cachedProgress = new Progress({ - spinner: null, - text: ({ active }) => { - const now = performance.now(); - let text = `[${this.#done}/${this.#total}] ${this.#name}`; - let n = 0; - for (const item of active) { - let itemText = "- " + item.format(now); - text += `\n` + - itemText.slice(0, Math.max(0, process.stdout.columns - 1)); - if (n > 10) { - text += `\n ... + ${active.length - n} more`; - break; - } - n++; - } - return text; - }, - props: { - active: [] as Spinner[], - }, - }); - bar.value = 0; - return bar; - } - return cached; - } - - addReturn(args: T) { - this.#total += 1; - this.updateTotal(); - if (this.#active.length >= this.#maxJobs) { - const { promise, resolve, reject } = Promise.withResolvers(); - this.#queue.push([args, resolve, reject]); - return promise; - } - return this.#run(args); - } - - add(args: T) { - return this.addReturn(args).then(() => {}, () => {}); - } - - addMany(items: T[]) { - this.#total += items.length; - this.updateTotal(); - - const runNowCount = this.#maxJobs - this.#active.length; - const runNow = items.slice(0, runNowCount); - const runLater = items.slice(runNowCount); - this.#queue.push(...runLater.reverse().map<[T]>((x) => [x])); - runNow.map((item) => this.#run(item).catch(() => {})); - } - - async #run(args: T): Promise { - const bar = this.bar; - const itemText = this.#getItemText(args); - const spinner = new Spinner(itemText); - spinner.stop(); - (spinner as any).redraw = () => (bar as any).redraw(); - const active = this.#active; - try { - active.unshift(spinner); - bar.props = { active }; - console.log(this.#name + ": " + itemText); - const result = await this.#fn(args, spinner); - this.#done++; - return result; - } catch (err) { - if (err && typeof err === "object") { - (err as any).job = itemText; - } - this.#errors.push(err); - throw err; - } finally { - active.splice(active.indexOf(spinner), 1); - bar.props = { active }; - bar.value = this.#done; - - // Process next item - const next = this.#queue.shift(); - if (next) { - const args = next[0]; - this.#run(args) - .then((result) => next[1]?.(result)) - .catch((err) => next[2]?.(err)); - } else if (this.#active.length === 0) { - if (this.#passive) { - this.bar.stop(); - this.#cachedProgress = null; - } - this.#onComplete?.(); - } - } - } - - updateTotal() { - const bar = this.bar; - bar.total = Math.max(this.#total, this.#estimate ?? 0); - } - - set estimate(e: number) { - this.#estimate = e; - if (this.#cachedProgress) { - this.updateTotal(); - } - } - - async done(o?: { method: "success" | "stop" }) { - if (this.#active.length === 0) { - this.#end(o); - return; - } - - const { promise, resolve } = Promise.withResolvers(); - this.#onComplete = resolve; - await promise; - this.#end(o); - } - - #end( - { method = this.#passive ? "stop" : "success" }: { - method?: "success" | "stop"; - } = {}, - ) { - const bar = this.#cachedProgress; - if (this.#errors.length > 0) { - if (bar) bar.stop(); - throw new AggregateError( - this.#errors, - this.#errors.length + " jobs failed in '" + this.#name + "'", - ); - } - - if (bar) bar[method](); - } - - get active(): boolean { - return this.#active.length !== 0; - } - - [Symbol.dispose]() { - if (this.active) { - this.cancel(); - } - } -} - -const cwd = process.cwd(); -function defaultGetItemText(item: unknown) { - let itemText = ""; - if (typeof item === "string") { - itemText = item; - } else if (typeof item === "object" && item !== null) { - const { path, label, id } = item as any; - itemText = label ?? path ?? id ?? JSON.stringify(item); - } else { - itemText = JSON.stringify(item); - } - - if (itemText.startsWith(cwd)) { - itemText = path.relative(cwd, itemText); - } - return itemText; -} - -export class OnceMap { - private ongoing = new Map>(); - - get(key: string, compute: () => Promise) { - if (this.ongoing.has(key)) { - return this.ongoing.get(key)!; - } - - const result = compute(); - this.ongoing.set(key, result); - return result; - } -} - -interface ARCEValue { - value: T; - [Symbol.dispose]: () => void; -} - -export function RefCountedExpirable( - init: () => Promise, - deinit: (value: T) => void, - expire: number = five_minutes, -): () => Promise> { - let refs = 0; - let item: ARCEValue | null = null; - let loading: Promise> | null = null; - let timer: ReturnType | null = null; - - function deref() { - ASSERT(item !== null); - if (--refs !== 0) return; - ASSERT(timer === null); - timer = setTimeout(() => { - ASSERT(refs === 0); - ASSERT(loading === null); - ASSERT(item !== null); - deinit(item.value); - item = null; - timer = null; - }, expire); - } - - return async function () { - if (timer !== null) { - clearTimeout(timer); - timer = null; - } - if (item !== null) { - refs++; - return item; - } - if (loading !== null) { - refs++; - return loading; - } - const p = Promise.withResolvers>(); - loading = p.promise; - try { - const value = await init(); - item = { value, [Symbol.dispose]: deref }; - refs++; - p.resolve(item); - return item; - } catch (e) { - p.reject(e); - throw e; - } finally { - loading = null; - } - }; -} - -export function once(fn: () => Promise): () => Promise { - let result: T | Promise | null = null; - return async () => { - if (result) return result; - result = await fn(); - return result; - }; -} - -import { Progress } from "@paperclover/console/Progress"; -import { Spinner } from "@paperclover/console/Spinner"; -import * as path from "node:path"; -import process from "node:process"; +const five_minutes = 5 * 60 * 1000; + +interface QueueOptions { + name: string; + fn: (item: T, spin: Spinner) => Promise; + getItemText?: (item: T) => string; + maxJobs?: number; + passive?: boolean; +} + +// Process multiple items in parallel, queue up as many. +export class Queue { + #name: string; + #fn: (item: T, spin: Spinner) => Promise; + #maxJobs: number; + #getItemText: (item: T) => string; + #passive: boolean; + + #active: Spinner[] = []; + #queue: Array<[T] | [T, (result: R) => void, (err: unknown) => void]> = []; + + #cachedProgress: Progress<{ active: Spinner[] }> | null = null; + #done: number = 0; + #total: number = 0; + #onComplete: (() => void) | null = null; + #estimate: number | null = null; + #errors: unknown[] = []; + + constructor(options: QueueOptions) { + this.#name = options.name; + this.#fn = options.fn; + this.#maxJobs = options.maxJobs ?? 5; + this.#getItemText = options.getItemText ?? defaultGetItemText; + this.#passive = options.passive ?? false; + } + + cancel() { + const bar = this.#cachedProgress; + bar?.stop(); + this.#queue = []; + } + + get bar() { + const cached = this.#cachedProgress; + if (!cached) { + const bar = this.#cachedProgress = new Progress({ + spinner: null, + text: ({ active }) => { + const now = performance.now(); + let text = `[${this.#done}/${this.#total}] ${this.#name}`; + let n = 0; + for (const item of active) { + let itemText = "- " + item.format(now); + text += `\n` + + itemText.slice(0, Math.max(0, process.stdout.columns - 1)); + if (n > 10) { + text += `\n ... + ${active.length - n} more`; + break; + } + n++; + } + return text; + }, + props: { + active: [] as Spinner[], + }, + }); + bar.value = 0; + return bar; + } + return cached; + } + + addReturn(args: T) { + this.#total += 1; + this.updateTotal(); + if (this.#active.length >= this.#maxJobs) { + const { promise, resolve, reject } = Promise.withResolvers(); + this.#queue.push([args, resolve, reject]); + return promise; + } + return this.#run(args); + } + + add(args: T) { + return this.addReturn(args).then(() => {}, () => {}); + } + + addMany(items: T[]) { + this.#total += items.length; + this.updateTotal(); + + const runNowCount = this.#maxJobs - this.#active.length; + const runNow = items.slice(0, runNowCount); + const runLater = items.slice(runNowCount); + this.#queue.push(...runLater.reverse().map<[T]>((x) => [x])); + runNow.map((item) => this.#run(item).catch(() => {})); + } + + async #run(args: T): Promise { + const bar = this.bar; + const itemText = this.#getItemText(args); + const spinner = new Spinner(itemText); + spinner.stop(); + (spinner as any).redraw = () => (bar as any).redraw(); + const active = this.#active; + try { + active.unshift(spinner); + bar.props = { active }; + // console.log(this.#name + ": " + itemText); + const result = await this.#fn(args, spinner); + this.#done++; + return result; + } catch (err) { + if (err && typeof err === "object") { + (err as any).job = itemText; + } + this.#errors.push(err); + console.error(util.inspect(err, false, Infinity, true)); + throw err; + } finally { + active.splice(active.indexOf(spinner), 1); + bar.props = { active }; + bar.value = this.#done; + + // Process next item + const next = this.#queue.shift(); + if (next) { + const args = next[0]; + this.#run(args) + .then((result) => next[1]?.(result)) + .catch((err) => next[2]?.(err)); + } else if (this.#active.length === 0) { + if (this.#passive) { + this.bar.stop(); + this.#cachedProgress = null; + } + this.#onComplete?.(); + } + } + } + + updateTotal() { + const bar = this.bar; + bar.total = Math.max(this.#total, this.#estimate ?? 0); + } + + set estimate(e: number) { + this.#estimate = e; + if (this.#cachedProgress) { + this.updateTotal(); + } + } + + async done(o?: { method: "success" | "stop" }) { + if (this.#active.length === 0) { + this.#end(o); + return; + } + + const { promise, resolve } = Promise.withResolvers(); + this.#onComplete = resolve; + await promise; + this.#end(o); + } + + #end( + { method = this.#passive ? "stop" : "success" }: { + method?: "success" | "stop"; + } = {}, + ) { + const bar = this.#cachedProgress; + if (this.#errors.length > 0) { + if (bar) bar.stop(); + throw new AggregateError( + this.#errors, + this.#errors.length + " jobs failed in '" + this.#name + "'", + ); + } + + if (bar) bar[method](); + } + + get active(): boolean { + return this.#active.length !== 0; + } + + [Symbol.dispose]() { + if (this.active) { + this.cancel(); + } + } +} + +const cwd = process.cwd(); +function defaultGetItemText(item: unknown) { + let itemText = ""; + if (typeof item === "string") { + itemText = item; + } else if (typeof item === "object" && item !== null) { + const { path, label, id } = item as any; + itemText = label ?? path ?? id ?? JSON.stringify(item); + } else { + itemText = JSON.stringify(item); + } + + if (itemText.startsWith(cwd)) { + itemText = path.relative(cwd, itemText); + } + return itemText; +} + +export class OnceMap { + private ongoing = new Map>(); + + get(key: string, compute: () => Promise) { + if (this.ongoing.has(key)) { + return this.ongoing.get(key)!; + } + + const result = compute(); + this.ongoing.set(key, result); + return result; + } +} + +interface ARCEValue { + value: T; + [Symbol.dispose]: () => void; +} + +export function RefCountedExpirable( + init: () => Promise, + deinit: (value: T) => void, + expire: number = five_minutes, +): () => Promise> { + let refs = 0; + let item: ARCEValue | null = null; + let loading: Promise> | null = null; + let timer: ReturnType | null = null; + + function deref() { + ASSERT(item !== null); + if (--refs !== 0) return; + ASSERT(timer === null); + timer = setTimeout(() => { + ASSERT(refs === 0); + ASSERT(loading === null); + ASSERT(item !== null); + deinit(item.value); + item = null; + timer = null; + }, expire); + } + + return async function () { + if (timer !== null) { + clearTimeout(timer); + timer = null; + } + if (item !== null) { + refs++; + return item; + } + if (loading !== null) { + refs++; + return loading; + } + const p = Promise.withResolvers>(); + loading = p.promise; + try { + const value = await init(); + item = { value, [Symbol.dispose]: deref }; + refs++; + p.resolve(item); + return item; + } catch (e) { + p.reject(e); + throw e; + } finally { + loading = null; + } + }; +} + +export function once(fn: () => Promise): () => Promise { + let result: T | Promise | null = null; + return async () => { + if (result) return result; + result = await fn(); + return result; + }; +} + +import { Progress } from "@paperclover/console/Progress"; +import { Spinner } from "@paperclover/console/Spinner"; +import * as path from "node:path"; +import process from "node:process"; +import * as util from "node:util"; diff --git a/framework/lib/meta.ts b/framework/lib/meta.ts index c0b65a9..420b4b5 100644 --- a/framework/lib/meta.ts +++ b/framework/lib/meta.ts @@ -1,24 +1,24 @@ -export interface Meta { - title: string; - description?: string | undefined; - openGraph?: OpenGraph; - alternates?: Alternates; -} -export interface OpenGraph { - title?: string; - description?: string | undefined; - type: string; - url: string; -} -export interface Alternates { - canonical: string; - types: { [mime: string]: AlternateType }; -} -export interface AlternateType { - url: string; - title: string; -} -export function renderMeta({ title }: Meta): string { - return `${esc(title)}`; -} -import { escapeHtml as esc } from "../engine/ssr.ts"; +export interface Meta { + title: string; + description?: string | undefined; + openGraph?: OpenGraph; + alternates?: Alternates; +} +export interface OpenGraph { + title?: string; + description?: string | undefined; + type: string; + url: string; +} +export interface Alternates { + canonical: string; + types: { [mime: string]: AlternateType }; +} +export interface AlternateType { + url: string; + title: string; +} +export function renderMeta({ title }: Meta): string { + return `${esc(title)}`; +} +import { escapeHtml as esc } from "../engine/ssr.ts"; diff --git a/framework/lib/string.ts b/framework/lib/string.ts index 239e02e..a15d412 100644 --- a/framework/lib/string.ts +++ b/framework/lib/string.ts @@ -1,3 +1,3 @@ -export function escapeRegExp(source: string) { - return source.replace(/[\$\\]/g, "\\$&"); -} +export function escapeRegExp(source: string) { + return source.replace(/[\$\\]/g, "\\$&"); +} diff --git a/framework/lib/view.ts b/framework/lib/view.ts index fe78b43..47e3f13 100644 --- a/framework/lib/view.ts +++ b/framework/lib/view.ts @@ -1,100 +1,100 @@ -// This import is generated by code 'bundle.ts' -export interface View { - component: engine.Component; - meta: - | meta.Meta - | ((props: { context?: hono.Context }) => Promise | meta.Meta); - layout?: engine.Component; - inlineCss: string; - scripts: Record; -} - -let views: Record = null!; -let scripts: Record = null!; - -// An older version of the Clover Engine supported streaming suspense -// boundaries, but those were never used. Pages will wait until they -// are fully rendered before sending. -export async function renderView( - context: hono.Context, - id: string, - props: Record, -) { - return context.html(await renderViewToString(id, { context, ...props })); -} - -export async function renderViewToString( - id: string, - props: Record, -) { - views ?? ({ views, scripts } = require("$views")); - // The view contains pre-bundled CSS and scripts, but keeps the scripts - // separate for run-time dynamic scripts. For example, the file viewer - // includes the canvas for the current page, but only the current page. - const { - component, - inlineCss, - layout, - meta: metadata, - }: View = UNWRAP(views[id], `Missing view ${id}`); - - // -- metadata -- - const renderedMetaPromise = Promise.resolve( - typeof metadata === "function" ? metadata(props) : metadata, - ).then((m) => meta.renderMeta(m)); - - // -- html -- - let page: engine.Element = [engine.kElement, component, props]; - if (layout) page = [engine.kElement, layout, { children: page }]; - const { text: body, addon: { sitegen } } = await engine.ssrAsync(page, { - sitegen: sg.initRender(), - }); - - // -- join document and send -- - return wrapDocument({ - body, - head: await renderedMetaPromise, - inlineCss, - scripts: joinScripts( - Array.from( - sitegen.scripts, - (id) => UNWRAP(scripts[id], `Missing script ${id}`), - ), - ), - }); -} - -export function provideViewData(v: typeof views, s: typeof scripts) { - views = v; - scripts = s; -} - -export function joinScripts(scriptSources: string[]) { - const { length } = scriptSources; - if (length === 0) return ""; - if (length === 1) return scriptSources[0]; - return scriptSources.map((source) => `{${source}}`).join(";"); -} - -export function wrapDocument({ - body, - head, - inlineCss, - scripts, -}: { - head: string; - body: string; - inlineCss: string; - scripts: string; -}) { - return `${head}${ - inlineCss ? `` : "" - }${body}${ - scripts ? `` : "" - }`; -} - -import * as meta from "./meta.ts"; -import type * as hono from "#hono"; -import * as engine from "../engine/ssr.ts"; -import * as sg from "./sitegen.ts"; +// This import is generated by code 'bundle.ts' +export interface View { + component: engine.Component; + meta: + | meta.Meta + | ((props: { context?: hono.Context }) => Promise | meta.Meta); + layout?: engine.Component; + inlineCss: string; + scripts: Record; +} + +let views: Record = null!; +let scripts: Record = null!; + +// An older version of the Clover Engine supported streaming suspense +// boundaries, but those were never used. Pages will wait until they +// are fully rendered before sending. +export async function renderView( + context: hono.Context, + id: string, + props: Record, +) { + return context.html(await renderViewToString(id, { context, ...props })); +} + +export async function renderViewToString( + id: string, + props: Record, +) { + views ?? ({ views, scripts } = require("$views")); + // The view contains pre-bundled CSS and scripts, but keeps the scripts + // separate for run-time dynamic scripts. For example, the file viewer + // includes the canvas for the current page, but only the current page. + const { + component, + inlineCss, + layout, + meta: metadata, + }: View = UNWRAP(views[id], `Missing view ${id}`); + + // -- metadata -- + const renderedMetaPromise = Promise.resolve( + typeof metadata === "function" ? metadata(props) : metadata, + ).then((m) => meta.renderMeta(m)); + + // -- html -- + let page: engine.Element = [engine.kElement, component, props]; + if (layout) page = [engine.kElement, layout, { children: page }]; + const { text: body, addon: { sitegen } } = await engine.ssrAsync(page, { + sitegen: sg.initRender(), + }); + + // -- join document and send -- + return wrapDocument({ + body, + head: await renderedMetaPromise, + inlineCss, + scripts: joinScripts( + Array.from( + sitegen.scripts, + (id) => UNWRAP(scripts[id], `Missing script ${id}`), + ), + ), + }); +} + +export function provideViewData(v: typeof views, s: typeof scripts) { + views = v; + scripts = s; +} + +export function joinScripts(scriptSources: string[]) { + const { length } = scriptSources; + if (length === 0) return ""; + if (length === 1) return scriptSources[0]; + return scriptSources.map((source) => `{${source}}`).join(";"); +} + +export function wrapDocument({ + body, + head, + inlineCss, + scripts, +}: { + head: string; + body: string; + inlineCss: string; + scripts: string; +}) { + return `${head}${ + inlineCss ? `` : "" + }${body}${ + scripts ? `` : "" + }`; +} + +import * as meta from "./meta.ts"; +import type * as hono from "#hono"; +import * as engine from "../engine/ssr.ts"; +import * as sg from "./sitegen.ts"; diff --git a/framework/watch.ts b/framework/watch.ts index 24ff96e..14a39b4 100644 --- a/framework/watch.ts +++ b/framework/watch.ts @@ -1,198 +1,198 @@ -// File watcher and live reloading site generator - -const debounceMilliseconds = 25; - -export async function main() { - let subprocess: child_process.ChildProcess | null = null; - - // Catch up state by running a main build. - const { incr } = await generate.main(); - // ...and watch the files that cause invals. - const watch = new Watch(rebuild); - watch.add(...incr.invals.keys()); - statusLine(); - // ... and then serve it! - serve(); - - function serve() { - if (subprocess) { - subprocess.removeListener("close", onSubprocessClose); - subprocess.kill(); - } - subprocess = child_process.fork(".clover/out/server.js", [ - "--development", - ], { - stdio: "inherit", - }); - subprocess.on("close", onSubprocessClose); - } - - function onSubprocessClose(code: number | null, signal: string | null) { - subprocess = null; - const status = code != null ? `code ${code}` : `signal ${signal}`; - console.error(`Backend process exited with ${status}`); - } - - process.on("beforeExit", () => { - subprocess?.removeListener("close", onSubprocessClose); - }); - - function rebuild(files: string[]) { - files = files.map((file) => path.relative(hot.projectRoot, file)); - const changed: string[] = []; - for (const file of files) { - let mtimeMs: number | null = null; - try { - mtimeMs = fs.statSync(file).mtimeMs; - } catch (err: any) { - if (err?.code !== "ENOENT") throw err; - } - if (incr.updateStat(file, mtimeMs)) changed.push(file); - } - if (changed.length === 0) { - console.warn("Files were modified but the 'modify' time did not change."); - return; - } - withSpinner>>({ - text: "Rebuilding", - successText: generate.successText, - failureText: () => "sitegen FAIL", - }, async (spinner) => { - console.info("---"); - console.info( - "Updated" + - (changed.length === 1 - ? " " + changed[0] - : changed.map((file) => "\n- " + file)), - ); - const result = await generate.sitegen(spinner, incr); - incr.toDisk(); // Allows picking up this state again - for (const file of watch.files) { - const relative = path.relative(hot.projectRoot, file); - if (!incr.invals.has(relative)) watch.remove(file); - } - return result; - }).then((result) => { - // Restart the server if it was changed or not running. - if ( - !subprocess || - result.inserted.some(({ kind }) => kind === "backendReplace") - ) { - serve(); - } else if ( - subprocess && - result.inserted.some(({ kind }) => kind === "asset") - ) { - subprocess.send({ type: "clover.assets.reload" }); - } - return result; - }).catch((err) => { - console.error(util.inspect(err)); - }).finally(statusLine); - } - - function statusLine() { - console.info( - `Watching ${incr.invals.size} files \x1b[36m[last change: ${ - new Date().toLocaleTimeString() - }]\x1b[39m`, - ); - } -} - -class Watch { - files = new Set(); - stale = new Set(); - onChange: (files: string[]) => void; - watchers: fs.FSWatcher[] = []; - /** Has a trailing slash */ - roots: string[] = []; - debounce: ReturnType | null = null; - - constructor(onChange: Watch["onChange"]) { - this.onChange = onChange; - } - - add(...files: string[]) { - const { roots, watchers } = this; - let newRoots: string[] = []; - for (let file of files) { - file = path.resolve(file); - if (this.files.has(file)) continue; - this.files.add(file); - // Find an existing watcher - if (roots.some((root) => file.startsWith(root))) continue; - if (newRoots.some((root) => file.startsWith(root))) continue; - newRoots.push(path.dirname(file) + path.sep); - } - if (newRoots.length === 0) return; - // Filter out directories that are already specified - newRoots = newRoots - .sort((a, b) => a.length - b.length) - .filter((dir, i, a) => { - for (let j = 0; j < i; j++) if (dir.startsWith(a[j])) return false; - return true; - }); - // Append Watches - let i = roots.length; - for (const root of newRoots) { - this.watchers.push(fs.watch( - root, - { recursive: true, encoding: "utf-8" }, - this.#handleEvent.bind(this, root), - )); - this.roots.push(root); - } - // If any new roots shadow over and old one, delete it! - while (i > 0) { - i -= 1; - const root = roots[i]; - if (newRoots.some((newRoot) => root.startsWith(newRoot))) { - watchers.splice(i, 1)[0].close(); - roots.splice(i, 1); - } - } - } - - remove(...files: string[]) { - for (const file of files) this.files.delete(path.resolve(file)); - // Find watches that are covering no files - const { roots, watchers } = this; - const existingFiles = Array.from(this.files); - let i = roots.length; - while (i > 0) { - i -= 1; - const root = roots[i]; - if (!existingFiles.some((file) => file.startsWith(root))) { - watchers.splice(i, 1)[0].close(); - roots.splice(i, 1); - } - } - } - - stop() { - for (const w of this.watchers) w.close(); - } - - #handleEvent(root: string, event: fs.WatchEventType, subPath: string | null) { - if (!subPath) return; - const file = path.join(root, subPath); - if (!this.files.has(file)) return; - this.stale.add(file); - const { debounce } = this; - if (debounce !== null) clearTimeout(debounce); - this.debounce = setTimeout(() => { - this.debounce = null; - this.onChange(Array.from(this.stale)); - this.stale.clear(); - }, debounceMilliseconds); - } -} - -import * as fs from "node:fs"; -import { withSpinner } from "@paperclover/console/Spinner"; -import * as generate from "./generate.ts"; -import * as path from "node:path"; -import * as util from "node:util"; -import * as hot from "./hot.ts"; -import * as child_process from "node:child_process"; +// File watcher and live reloading site generator + +const debounceMilliseconds = 25; + +export async function main() { + let subprocess: child_process.ChildProcess | null = null; + + // Catch up state by running a main build. + const { incr } = await generate.main(); + // ...and watch the files that cause invals. + const watch = new Watch(rebuild); + watch.add(...incr.invals.keys()); + statusLine(); + // ... and then serve it! + serve(); + + function serve() { + if (subprocess) { + subprocess.removeListener("close", onSubprocessClose); + subprocess.kill(); + } + subprocess = child_process.fork(".clover/out/server.js", [ + "--development", + ], { + stdio: "inherit", + }); + subprocess.on("close", onSubprocessClose); + } + + function onSubprocessClose(code: number | null, signal: string | null) { + subprocess = null; + const status = code != null ? `code ${code}` : `signal ${signal}`; + console.error(`Backend process exited with ${status}`); + } + + process.on("beforeExit", () => { + subprocess?.removeListener("close", onSubprocessClose); + }); + + function rebuild(files: string[]) { + files = files.map((file) => path.relative(hot.projectRoot, file)); + const changed: string[] = []; + for (const file of files) { + let mtimeMs: number | null = null; + try { + mtimeMs = fs.statSync(file).mtimeMs; + } catch (err: any) { + if (err?.code !== "ENOENT") throw err; + } + if (incr.updateStat(file, mtimeMs)) changed.push(file); + } + if (changed.length === 0) { + console.warn("Files were modified but the 'modify' time did not change."); + return; + } + withSpinner>>({ + text: "Rebuilding", + successText: generate.successText, + failureText: () => "sitegen FAIL", + }, async (spinner) => { + console.info("---"); + console.info( + "Updated" + + (changed.length === 1 + ? " " + changed[0] + : changed.map((file) => "\n- " + file)), + ); + const result = await generate.sitegen(spinner, incr); + incr.toDisk(); // Allows picking up this state again + for (const file of watch.files) { + const relative = path.relative(hot.projectRoot, file); + if (!incr.invals.has(relative)) watch.remove(file); + } + return result; + }).then((result) => { + // Restart the server if it was changed or not running. + if ( + !subprocess || + result.inserted.some(({ kind }) => kind === "backendReplace") + ) { + serve(); + } else if ( + subprocess && + result.inserted.some(({ kind }) => kind === "asset") + ) { + subprocess.send({ type: "clover.assets.reload" }); + } + return result; + }).catch((err) => { + console.error(util.inspect(err)); + }).finally(statusLine); + } + + function statusLine() { + console.info( + `Watching ${incr.invals.size} files \x1b[36m[last change: ${ + new Date().toLocaleTimeString() + }]\x1b[39m`, + ); + } +} + +class Watch { + files = new Set(); + stale = new Set(); + onChange: (files: string[]) => void; + watchers: fs.FSWatcher[] = []; + /** Has a trailing slash */ + roots: string[] = []; + debounce: ReturnType | null = null; + + constructor(onChange: Watch["onChange"]) { + this.onChange = onChange; + } + + add(...files: string[]) { + const { roots, watchers } = this; + let newRoots: string[] = []; + for (let file of files) { + file = path.resolve(file); + if (this.files.has(file)) continue; + this.files.add(file); + // Find an existing watcher + if (roots.some((root) => file.startsWith(root))) continue; + if (newRoots.some((root) => file.startsWith(root))) continue; + newRoots.push(path.dirname(file) + path.sep); + } + if (newRoots.length === 0) return; + // Filter out directories that are already specified + newRoots = newRoots + .sort((a, b) => a.length - b.length) + .filter((dir, i, a) => { + for (let j = 0; j < i; j++) if (dir.startsWith(a[j])) return false; + return true; + }); + // Append Watches + let i = roots.length; + for (const root of newRoots) { + this.watchers.push(fs.watch( + root, + { recursive: true, encoding: "utf-8" }, + this.#handleEvent.bind(this, root), + )); + this.roots.push(root); + } + // If any new roots shadow over and old one, delete it! + while (i > 0) { + i -= 1; + const root = roots[i]; + if (newRoots.some((newRoot) => root.startsWith(newRoot))) { + watchers.splice(i, 1)[0].close(); + roots.splice(i, 1); + } + } + } + + remove(...files: string[]) { + for (const file of files) this.files.delete(path.resolve(file)); + // Find watches that are covering no files + const { roots, watchers } = this; + const existingFiles = Array.from(this.files); + let i = roots.length; + while (i > 0) { + i -= 1; + const root = roots[i]; + if (!existingFiles.some((file) => file.startsWith(root))) { + watchers.splice(i, 1)[0].close(); + roots.splice(i, 1); + } + } + } + + stop() { + for (const w of this.watchers) w.close(); + } + + #handleEvent(root: string, event: fs.WatchEventType, subPath: string | null) { + if (!subPath) return; + const file = path.join(root, subPath); + if (!this.files.has(file)) return; + this.stale.add(file); + const { debounce } = this; + if (debounce !== null) clearTimeout(debounce); + this.debounce = setTimeout(() => { + this.debounce = null; + this.onChange(Array.from(this.stale)); + this.stale.clear(); + }, debounceMilliseconds); + } +} + +import * as fs from "node:fs"; +import { withSpinner } from "@paperclover/console/Spinner"; +import * as generate from "./generate.ts"; +import * as path from "node:path"; +import * as util from "node:util"; +import * as hot from "./hot.ts"; +import * as child_process from "node:child_process"; diff --git a/readme.md b/readme.md index 7fba8c2..c34ff2c 100644 --- a/readme.md +++ b/readme.md @@ -4,27 +4,27 @@ this repository contains clover's "sitegen" framework, which is a set of tools that assist building websites. these tools power https://paperclover.net. - **HTML "Server Side Rendering") engine written from scratch.** (~500 lines) - - A more practical JSX runtime (`class` instead of `className`, built-in - `clsx`, `html()` helper over `dangerouslySetInnerHTML` prop, etc). - - Integration with [Marko][1] for concisely written components. - - TODO: MDX-like compiler for content-heavy pages like blogs. - - Different languages can be used at the same time. Supports - `async function` components, ``, and custom extensions. + - A more practical JSX runtime (`class` instead of `className`, built-in + `clsx`, `html()` helper over `dangerouslySetInnerHTML` prop, etc). + - Integration with [Marko][1] for concisely written components. + - TODO: MDX-like compiler for content-heavy pages like blogs. + - Different languages can be used at the same time. Supports `async function` + components, ``, and custom extensions. - **Incremental static site generator and build system.** - - Build entire production site at start, incremental updates when pages - change; Build system state survives coding sessions. - - The only difference in development and production mode is hidden - source-maps and stripped `console.debug` calls. The site you - see locally is the same site you see deployed. - - (TODO) Tests, Lints, and Type-checking is run alongside, and only re-runs - checks when the files change. For example, changing a component re-tests - only pages that use that component and re-lints only the changed file. + - Build entire production site at start, incremental updates when pages + change; Build system state survives coding sessions. + - The only difference in development and production mode is hidden source-maps + and stripped `console.debug` calls. The site you see locally is the same + site you see deployed. + - (TODO) Tests, Lints, and Type-checking is run alongside, and only re-runs + checks when the files change. For example, changing a component re-tests + only pages that use that component and re-lints only the changed file. - **Integrated libraries for building complex, content heavy web sites.** - - Static asset serving with ETag and build-time compression. - - Dynamicly rendered pages with static client. (`#import "#sitegen/view"`) - - Databases with a typed SQLite wrapper. (`import "#sitegen/sqlite"`) - - TODO: Meta and Open Graph generation. (`export const meta`) - - TODO: Font subsetting tools to reduce bytes downloaded by fonts. + - Static asset serving with ETag and build-time compression. + - Dynamicly rendered pages with static client. (`#import "#sitegen/view"`) + - Databases with a typed SQLite wrapper. (`import "#sitegen/sqlite"`) + - TODO: Meta and Open Graph generation. (`export const meta`) + - TODO: Font subsetting tools to reduce bytes downloaded by fonts. - **Built on the battle-tested Node.js runtime.** [1]: https://next.markojs.com @@ -42,6 +42,7 @@ Included is `src`, which contains `paperclover.net`. Website highlights: ## Development minimum system requirements: + - a cpu with at least 1 core. - random access memory. - windows 7 or later, macos, or other operating system. @@ -73,4 +74,3 @@ open a shell with all needed system dependencies. ## Contributions No contributions to `src` accepted, only `framework`. - diff --git a/run.js b/run.js index 39c310f..1b509c4 100644 --- a/run.js +++ b/run.js @@ -12,7 +12,7 @@ if (!zlib.zstdCompress) { : null; globalThis.console.error( - `sitegen depends on a node.js-compatibile runtime that supports zstd compression\n` + + `sitegen depends on a node.js-compatibile runtime\n` + `this is node.js version ${process.version}${ brand ? ` (${brand})` : "" }\n\n` + diff --git a/src/admin.ts b/src/admin.ts index f23260d..dd90fda 100644 --- a/src/admin.ts +++ b/src/admin.ts @@ -1,75 +1,75 @@ -const cookieAge = 60 * 60 * 24 * 365; // 1 year - -let lastKnownToken: string | null = null; -function compareToken(token: string) { - if (token === lastKnownToken) return true; - lastKnownToken = fs.readFileSync(".clover/admin-token.txt", "utf8").trim(); - return token === lastKnownToken; -} - -export async function middleware(c: Context, next: Next) { - if (c.req.path.startsWith("/admin")) { - return adminInner(c, next); - } - return next(); -} - -export function adminInner(c: Context, next: Next) { - const token = c.req.header("Cookie")?.match(/admin-token=([^;]+)/)?.[1]; - - if (c.req.path === "/admin/login") { - const key = c.req.query("key"); - if (key) { - if (compareToken(key)) { - return c.body(null, 303, { - "Location": "/admin", - "Set-Cookie": - `admin-token=${key}; Path=/; HttpOnly; SameSite=Strict; Max-Age=${cookieAge}`, - }); - } - return serveAsset(c, "/admin/login/fail", 403); - } - if (token && compareToken(token)) { - return c.redirect("/admin", 303); - } - if (c.req.method === "POST") { - return serveAsset(c, "/admin/login/fail", 403); - } else { - return serveAsset(c, "/admin/login", 200); - } - } - - if (c.req.path === "/admin/logout") { - return c.body(null, 303, { - "Location": "/admin/login", - "Set-Cookie": - `admin-token=; Path=/; HttpOnly; SameSite=Strict; Max-Age=0`, - }); - } - - if (token && compareToken(token)) { - return next(); - } - - return c.redirect("/admin/login", 303); -} - -export function hasAdminToken(c: Context) { - const token = c.req.header("Cookie")?.match(/admin-token=([^;]+)/)?.[1]; - return token && compareToken(token); -} - -export async function main() { - const key = crypto.randomUUID(); - await fs.writeMkdir(".clover/admin-token.txt", key); - const start = ({ - win32: "start", - darwin: "open", - } as Record)[process.platform] ?? "xdg-open"; - child_process.exec(`${start} http://[::1]:3000/admin/login?key=${key}`); -} - -import * as fs from "#sitegen/fs"; -import type { Context, Next } from "hono"; -import { serveAsset } from "#sitegen/assets"; -import * as child_process from "node:child_process"; +const cookieAge = 60 * 60 * 24 * 365; // 1 year + +let lastKnownToken: string | null = null; +function compareToken(token: string) { + if (token === lastKnownToken) return true; + lastKnownToken = fs.readFileSync(".clover/admin-token.txt", "utf8").trim(); + return token === lastKnownToken; +} + +export async function middleware(c: Context, next: Next) { + if (c.req.path.startsWith("/admin")) { + return adminInner(c, next); + } + return next(); +} + +export function adminInner(c: Context, next: Next) { + const token = c.req.header("Cookie")?.match(/admin-token=([^;]+)/)?.[1]; + + if (c.req.path === "/admin/login") { + const key = c.req.query("key"); + if (key) { + if (compareToken(key)) { + return c.body(null, 303, { + "Location": "/admin", + "Set-Cookie": + `admin-token=${key}; Path=/; HttpOnly; SameSite=Strict; Max-Age=${cookieAge}`, + }); + } + return serveAsset(c, "/admin/login/fail", 403); + } + if (token && compareToken(token)) { + return c.redirect("/admin", 303); + } + if (c.req.method === "POST") { + return serveAsset(c, "/admin/login/fail", 403); + } else { + return serveAsset(c, "/admin/login", 200); + } + } + + if (c.req.path === "/admin/logout") { + return c.body(null, 303, { + "Location": "/admin/login", + "Set-Cookie": + `admin-token=; Path=/; HttpOnly; SameSite=Strict; Max-Age=0`, + }); + } + + if (token && compareToken(token)) { + return next(); + } + + return c.redirect("/admin/login", 303); +} + +export function hasAdminToken(c: Context) { + const token = c.req.header("Cookie")?.match(/admin-token=([^;]+)/)?.[1]; + return token && compareToken(token); +} + +export async function main() { + const key = crypto.randomUUID(); + await fs.writeMkdir(".clover/admin-token.txt", key); + const start = ({ + win32: "start", + darwin: "open", + } as Record)[process.platform] ?? "xdg-open"; + child_process.exec(`${start} http://[::1]:3000/admin/login?key=${key}`); +} + +import * as fs from "#sitegen/fs"; +import type { Context, Next } from "hono"; +import { serveAsset } from "#sitegen/assets"; +import * as child_process from "node:child_process"; diff --git a/src/backend.ts b/src/backend.ts index 0bf1305..474ea73 100644 --- a/src/backend.ts +++ b/src/backend.ts @@ -1,53 +1,53 @@ -// This is the main file for the backend -const app = new Hono(); -const logHttp = scoped("http", { color: "magenta" }); - -// Middleware -app.use(trimTrailingSlash()); -app.use(removeDuplicateSlashes); -app.use(logger((msg) => msg.startsWith("-->") && logHttp(msg.slice(4)))); -app.use(admin.middleware); - -// Backends -app.route("", require("./q+a/backend.ts").app); -app.route("", require("./file-viewer/backend.tsx").app); - -// Asset middleware has least precedence -app.use(assets.middleware); - -// Handlers -app.notFound(assets.notFound); -if (process.argv.includes("--development")) { - app.onError((err, c) => { - if (err instanceof HTTPException) { - // Get the custom response - return err.getResponse(); - } - - return c.text(util.inspect(err), 500); - }); -} - -export default app; - -async function removeDuplicateSlashes(c: Context, next: Next) { - const path = c.req.path; - if (/\/\/+/.test(path)) { - const normalizedPath = path.replace(/\/\/+/g, "/"); - const query = c.req.query(); - const queryString = Object.keys(query).length > 0 - ? "?" + new URLSearchParams(query).toString() - : ""; - return c.redirect(normalizedPath + queryString, 301); - } - await next(); -} - -import { type Context, Hono, type Next } from "#hono"; -import { HTTPException } from "hono/http-exception"; -import { logger } from "hono/logger"; -import { trimTrailingSlash } from "hono/trailing-slash"; -import * as assets from "#sitegen/assets"; -import * as admin from "./admin.ts"; -import { scoped } from "@paperclover/console"; -import * as util from "node:util"; +// This is the main file for the backend +const app = new Hono(); +const logHttp = scoped("http", { color: "magenta" }); + +// Middleware +app.use(trimTrailingSlash()); +app.use(removeDuplicateSlashes); +app.use(logger((msg) => msg.startsWith("-->") && logHttp(msg.slice(4)))); +app.use(admin.middleware); + +// Backends +app.route("", require("./q+a/backend.ts").app); +app.route("", require("./file-viewer/backend.tsx").app); + +// Asset middleware has least precedence +app.use(assets.middleware); + +// Handlers +app.notFound(assets.notFound); +if (process.argv.includes("--development")) { + app.onError((err, c) => { + if (err instanceof HTTPException) { + // Get the custom response + return err.getResponse(); + } + + return c.text(util.inspect(err), 500); + }); +} + +export default app; + +async function removeDuplicateSlashes(c: Context, next: Next) { + const path = c.req.path; + if (/\/\/+/.test(path)) { + const normalizedPath = path.replace(/\/\/+/g, "/"); + const query = c.req.query(); + const queryString = Object.keys(query).length > 0 + ? "?" + new URLSearchParams(query).toString() + : ""; + return c.redirect(normalizedPath + queryString, 301); + } + await next(); +} + +import { type Context, Hono, type Next } from "#hono"; +import { HTTPException } from "hono/http-exception"; +import { logger } from "hono/logger"; +import { trimTrailingSlash } from "hono/trailing-slash"; +import * as assets from "#sitegen/assets"; +import * as admin from "./admin.ts"; +import { scoped } from "@paperclover/console"; +import * as util from "node:util"; diff --git a/src/file-viewer/bin/list.ts b/src/file-viewer/bin/list.ts index ac91f6d..bfef934 100644 --- a/src/file-viewer/bin/list.ts +++ b/src/file-viewer/bin/list.ts @@ -1,8 +1,8 @@ -export function main() { - const meows = MediaFile.db.prepare(` - select * from media_files; - `).as(MediaFile).array(); - console.log(meows); -} - -import { MediaFile } from "@/file-viewer/models/MediaFile.ts"; +export function main() { + const meows = MediaFile.db.prepare(` + select * from media_files; + `).as(MediaFile).array(); + console.log(meows); +} + +import { MediaFile } from "@/file-viewer/models/MediaFile.ts"; diff --git a/src/file-viewer/bin/scan3.ts b/src/file-viewer/bin/scan3.ts index a95ed9b..e67c20e 100644 --- a/src/file-viewer/bin/scan3.ts +++ b/src/file-viewer/bin/scan3.ts @@ -1,649 +1,655 @@ -// The file scanner incrementally updates an sqlite database with file -// stats. Additionally, it runs "processors" on files, which precompute -// expensive data such as running `ffprobe` on all media to get the -// duration. -// -// Processors are also used to derive compressed and optimized assets, -// which is how automatic JXL / AV1 encoding is done. Derived files are -// uploaded to the clover NAS to be pulled by VPS instances for hosting. -// -// This is the third iteration of the scanner, hence its name "scan3"; -// Remember that any software you want to be maintainable and high -// quality cannot be written with AI. -const root = path.resolve("/Volumes/clover/Published"); -const workDir = path.resolve(".clover/file-assets"); - -export async function main() { - const start = performance.now(); - const timerSpinner = new Spinner({ - text: () => - `paper clover's scan3 [${ - ((performance.now() - start) / 1000).toFixed(1) - }s]`, - fps: 10, - }); - using _endTimerSpinner = { [Symbol.dispose]: () => timerSpinner.stop() }; - - // Read a directory or file stat and queue up changed files. - using qList = new async.Queue({ - name: "Discover Tree", - async fn(absPath: string, spin) { - const stat = await fs.stat(absPath); - - const publicPath = toPublicPath(absPath); - const mediaFile = MediaFile.getByPath(publicPath); - - if (stat.isDirectory()) { - const items = await fs.readdir(absPath); - qList.addMany(items.map((subPath) => path.join(absPath, subPath))); - - if (mediaFile) { - const deleted = mediaFile.getChildren() - .filter((child) => !items.includes(child.basename)) - .flatMap((child) => - child.kind === MediaFileKind.directory - ? child.getRecursiveFileChildren() - : child - ); - - qMeta.addMany(deleted.map((mediaFile) => ({ - absPath: path.join(root, mediaFile.path), - publicPath: mediaFile.path, - stat: null, - mediaFile, - }))); - } - - return; - } - - // All processes must be performed again if there is no file. - if ( - !mediaFile || - stat.size !== mediaFile.size || - stat.mtime.getTime() !== mediaFile.date.getTime() - ) { - qMeta.add({ absPath, publicPath, stat, mediaFile }); - return; - } - - // If the scanners changed, it may mean more processes should be run. - queueProcessors({ absPath, stat, mediaFile }); - }, - maxJobs: 24, - }); - using qMeta = new async.Queue({ - name: "Update Metadata", - async fn({ absPath, publicPath, stat, mediaFile }: UpdateMetadataJob) { - if (!stat) { - // File was deleted. - await runUndoProcessors(UNWRAP(mediaFile)); - return; - } - // TODO: run scrubLocationMetadata first - - const hash = await new Promise((resolve, reject) => { - const reader = fs.createReadStream(absPath); - reader.on("error", reject); - - const hasher = crypto.createHash("sha1").setEncoding("hex"); - hasher.on("error", reject); - hasher.on("readable", () => resolve(hasher.read())); - - reader.pipe(hasher); - }); - let date = stat.mtime; - if ( - mediaFile && - mediaFile.date.getTime() < stat.mtime.getTime() && - (Date.now() - stat.mtime.getTime()) < monthMilliseconds - ) { - date = mediaFile.date; - console.warn( - `M-time on ${publicPath} was likely corrupted. ${ - formatDate(mediaFile.date) - } -> ${formatDate(stat.mtime)}`, - ); - } - mediaFile = MediaFile.createFile({ - path: publicPath, - date, - hash, - size: stat.size, - duration: mediaFile?.duration ?? 0, - dimensions: mediaFile?.dimensions ?? "", - contents: mediaFile?.contents ?? "", - }); - await queueProcessors({ absPath, stat, mediaFile }); - }, - getItemText: (job) => - job.publicPath.slice(1) + (job.stat ? "" : " (deleted)"), - maxJobs: 10, - }); - using qProcess = new async.Queue({ - name: "Process Contents", - async fn( - { absPath, stat, mediaFile, processor, index, after }: ProcessJob, - spin, - ) { - await processor.run({ absPath, stat, mediaFile, spin }); - mediaFile.setProcessed(mediaFile.processed | (1 << (16 + index))); - for (const dependantJob of after) { - ASSERT(dependantJob.needs > 0); - dependantJob.needs -= 1; - if (dependantJob.needs == 0) qProcess.add(dependantJob); - } - }, - getItemText: ({ mediaFile, processor }) => - `${mediaFile.path.slice(1)} - ${processor.name}`, - maxJobs: 4, - }); - - function decodeProcessors(input: string) { - return input - .split(";") - .filter(Boolean) - .map(([a, b, c]) => ({ - id: a, - hash: (b.charCodeAt(0) << 8) + c.charCodeAt(0), - })); - } - - async function queueProcessors( - { absPath, stat, mediaFile }: Omit, - ) { - const ext = mediaFile.extension.toLowerCase(); - let possible = processors.filter((p) => p.include.has(ext)); - if (possible.length === 0) return; - - const hash = possible.reduce((a, b) => a ^ b.hash, 0) | 1; - ASSERT(hash <= 0xFFFF); - let processed = mediaFile.processed; - - // If the hash has changed, migrate the bitfield over. - // This also runs when the processor hash is in it's initial 0 state. - const order = decodeProcessors(mediaFile.processors); - if ((processed & 0xFFFF) !== hash) { - const previous = order.filter((_, i) => - (processed & (1 << (16 + i))) !== 0 - ); - processed = hash; - for (const { id, hash } of previous) { - const p = processors.find((p) => p.id === id); - if (!p) continue; - const index = possible.indexOf(p); - if (index !== -1 && p.hash === hash) { - processed |= 1 << (16 + index); - } else { - if (p.undo) await p.undo(mediaFile); - } - } - mediaFile.setProcessors( - processed, - possible.map((p) => - p.id + String.fromCharCode(p.hash >> 8, p.hash & 0xFF) - ).join(";"), - ); - } else { - possible = order.map(({ id }) => - UNWRAP(possible.find((p) => p.id === id)) - ); - } - - // Queue needed processors. - const jobs: ProcessJob[] = []; - for (let i = 0, { length } = possible; i < length; i += 1) { - if ((processed & (1 << (16 + i))) === 0) { - const job: ProcessJob = { - absPath, - stat, - mediaFile, - processor: possible[i], - index: i, - after: [], - needs: possible[i].depends.length, - }; - jobs.push(job); - if (job.needs === 0) qProcess.add(job); - } - } - for (const job of jobs) { - for (const dependId of job.processor.depends) { - const dependJob = jobs.find((j) => j.processor.id === dependId); - if (dependJob) { - dependJob.after.push(job); - } else { - ASSERT(job.needs > 0); - job.needs -= 1; - if (job.needs === 0) qProcess.add(job); - } - } - } - } - - async function runUndoProcessors(mediaFile: MediaFile) { - const { processed } = mediaFile; - const previous = decodeProcessors(mediaFile.processors) - .filter((_, i) => (processed & (1 << (16 + i))) !== 0); - for (const { id } of previous) { - const p = processors.find((p) => p.id === id); - if (!p) continue; - if (p.undo) { - await p.undo(mediaFile); - } - } - mediaFile.delete(); - } - - // Add the root & recursively iterate! - qList.add(root); - await qList.done(); - await qMeta.done(); - await qProcess.done(); - - console.info( - "Updated file viewer index in " + - ((performance.now() - start) / 1000).toFixed(1) + "s", - ); -} - -interface Process { - name: string; - enable?: boolean; - include: Set; - depends?: string[]; - /* Perform an action. */ - run(args: ProcessFileArgs): Promise; - /* Should detect if `run` was never even run before before undoing state */ - undo?(mediaFile: MediaFile): Promise; -} - -const execFileRaw = util.promisify(child_process.execFile); -const execFile: typeof execFileRaw = (( - ...args: Parameters -) => - execFileRaw(...args).catch((e: any) => { - if (e?.message?.startsWith?.("Command failed")) { - if (e.code > (2 ** 31)) e.code |= 0; - const code = e.signal ? `signal ${e.signal}` : `code ${e.code}`; - e.message = `${e.cmd.split(" ")[0]} failed with ${code}`; - } - throw e; - })) as any; -const ffprobeBin = testProgram("ffprobe", "--help"); -const ffmpegBin = testProgram("ffmpeg", "--help"); - -const ffmpegOptions = [ - "-hide_banner", - "-loglevel", - "warning", -]; - -const procDuration: Process = { - name: "calculate duration", - enable: ffprobeBin !== null, - include: rules.extsDuration, - async run({ absPath, mediaFile }) { - const { stdout } = await execFile(ffprobeBin!, [ - "-v", - "error", - "-show_entries", - "format=duration", - "-of", - "default=noprint_wrappers=1:nokey=1", - absPath, - ]); - - const duration = parseFloat(stdout.trim()); - if (Number.isNaN(duration)) { - throw new Error("Could not extract duration from " + stdout); - } - mediaFile.setDuration(Math.ceil(duration)); - }, -}; - -// NOTE: Never re-order the processors. Add new ones at the end. -const procDimensions: Process = { - name: "calculate dimensions", - enable: ffprobeBin != null, - include: rules.extsDimensions, - async run({ absPath, mediaFile }) { - const ext = path.extname(absPath); - - let dimensions; - - if (ext === ".svg") { - // Parse out of text data - const content = await fs.readFile(absPath, "utf8"); - const widthMatch = content.match(/width="(\d+)"/); - const heightMatch = content.match(/height="(\d+)"/); - - if (widthMatch && heightMatch) { - dimensions = `${widthMatch[1]}x${heightMatch[1]}`; - } - } else { - // Use ffprobe to observe streams - const { stdout } = await execFile("ffprobe", [ - "-v", - "error", - "-select_streams", - "v:0", - "-show_entries", - "stream=width,height", - "-of", - "csv=s=x:p=0", - absPath, - ]); - if (stdout.includes("x")) { - dimensions = stdout.trim(); - } - } - - mediaFile.setDimensions(dimensions ?? ""); - }, -}; - -const procLoadTextContents: Process = { - name: "load text content", - include: rules.extsReadContents, - async run({ absPath, mediaFile, stat }) { - if (stat.size > 1_000_000) return; - const text = await fs.readFile(absPath, "utf-8"); - mediaFile.setContents(text); - }, -}; - -const procHighlightCode: Process = { - name: "highlight source code", - include: new Set(rules.extsCode.keys()), - async run({ absPath, mediaFile, stat }) { - const language = UNWRAP( - rules.extsCode.get(path.extname(absPath).toLowerCase()), - ); - // An issue is that .ts is an overloaded extension, shared between - // 'transport stream' and 'typescript'. - // - // Filter used here is: - // - more than 1mb - // - invalid UTF-8 - if (stat.size > 1_000_000) return; - let code; - const buf = await fs.readFile(absPath); - try { - code = new TextDecoder("utf-8", { fatal: true }).decode(buf); - } catch (error) { - mediaFile.setContents(""); - return; - } - const content = await highlight.highlightCode(code, language); - mediaFile.setContents(content); - }, -}; - -const procImageSubsets: Process = { - name: "encode image subsets", - include: rules.extsImage, - depends: ["calculate dimensions"], - async run({ absPath, mediaFile, spin }) { - const { width, height } = UNWRAP(mediaFile.parseDimensions()); - const targetSizes = transcodeRules.imageSizes.filter((w) => w < width); - const baseStatus = spin.text; - - using stack = new DisposableStack(); - for (const size of targetSizes) { - const { w, h } = resizeDimensions(width, height, size); - for (const { ext, args } of transcodeRules.imagePresets) { - spin.text = baseStatus + - ` (${w}x${h}, ${ext.slice(1).toUpperCase()})`; - - stack.use( - await produceAsset( - `${mediaFile.hash}/${size}${ext}`, - async (out) => { - await fs.mkdir(path.dirname(out)); - await fs.rm(out, { force: true }); - await execFile(ffmpegBin!, [ - ...ffmpegOptions, - "-i", - absPath, - "-vf", - `scale=${w}:${h}:force_original_aspect_ratio=increase,crop=${w}:${h}`, - ...args, - out, - ]); - return [out]; - }, - ), - ); - } - } - - stack.move(); - }, - async undo(mediaFile) { - const { width } = UNWRAP(mediaFile.parseDimensions()); - const targetSizes = transcodeRules.imageSizes.filter((w) => w < width); - for (const size of targetSizes) { - for (const { ext } of transcodeRules.imagePresets) { - unproduceAsset(`${mediaFile.hash}/${size}${ext}`); - } - } - }, -}; -const qualityMap: Record = { - u: "ultra-high", - h: "high", - m: "medium", - l: "low", - d: "data-saving", -}; -const procVideos = transcodeRules.videoFormats.map((preset) => ({ - name: `encode ${preset.codec} ${UNWRAP(qualityMap[preset.id[1]])}`, - include: rules.extsVideo, - enable: ffmpegBin != null, - async run({ absPath, mediaFile, spin }) { - if ((mediaFile.duration ?? 0) < 10) return; - await produceAsset(`${mediaFile.hash}/${preset.id}`, async (base) => { - base = path.dirname(base); - await fs.mkdir(base); - - let inputArgs = ["-i", absPath]; - try { - const config = await fs.readJson( - path.join( - path.dirname(absPath), - path.basename(absPath, path.extname(absPath)) + ".json", - ), - ); - if (config.encoder && typeof config.encoder.videoSrc === "string") { - const { videoSrc, audioSrc, rate } = config.encoder; - inputArgs = [ - ...rate ? ["-r", String(rate)] : [], - "-i", - videoSrc, - ...audioSrc ? ["-i", audioSrc] : [], - ]; - } - } catch (err: any) { - if (err?.code !== "ENOENT") throw err; - } - - const args = transcodeRules.getVideoArgs( - preset, - base, - inputArgs, - ); - try { - const fakeProgress = new Progress({ text: spin.text, spinner: null }); - fakeProgress.stop(); - spin.format = (now: number) => fakeProgress.format(now); - // @ts-expect-error - fakeProgress.redraw = () => spin.redraw(); - - await ffmpeg.spawn({ - ffmpeg: ffmpegBin!, - title: fakeProgress.text, - progress: fakeProgress, - args, - }); - return await collectFiles(); - } catch (err) { - for (const file of await collectFiles()) { - try { - fs.rm(file); - } catch {} - } - throw err; - } - - async function collectFiles(): Promise { - return (await fs.readdir(base)) - .filter((basename) => basename.startsWith(preset.id)) - .map((basename) => path.join(base, basename)); - } - }); - }, -})); - -const processors = [ - procDimensions, - procDuration, - procLoadTextContents, - procHighlightCode, - procImageSubsets, - ...procVideos, -] - .map((process, id, all) => { - const strIndex = (id: number) => - String.fromCharCode("a".charCodeAt(0) + id); - return { - ...process as Process, - id: strIndex(id), - // Create a unique key. - hash: new Uint16Array( - crypto.createHash("sha1") - .update(process.run.toString()) - .digest().buffer, - ).reduce((a, b) => a ^ b), - depends: (process.depends ?? []).map((depend) => { - const index = all.findIndex((p) => p.name === depend); - if (index === -1) throw new Error(`Cannot find depend '${depend}'`); - if (index === id) throw new Error(`Cannot depend on self: '${depend}'`); - return strIndex(index); - }), - }; - }); - -function resizeDimensions(w: number, h: number, desiredWidth: number) { - ASSERT(desiredWidth < w, `${desiredWidth} < ${w}`); - return { w: desiredWidth, h: Math.floor((h / w) * desiredWidth) }; -} - -async function produceAsset( - key: string, - builder: (prefix: string) => Promise, -) { - const asset = AssetRef.putOrIncrement(key); - try { - if (asset.refs === 1) { - const paths = await builder(path.join(workDir, key)); - asset.addFiles( - paths.map((file) => - path.relative(workDir, file) - .replaceAll("\\", "/") - ), - ); - } - return { - [Symbol.dispose]: () => asset.unref(), - }; - } catch (err: any) { - if (err && typeof err === "object") err.assetKey = key; - asset.unref(); - throw err; - } -} - -async function unproduceAsset(key: string) { - const ref = AssetRef.get(key); - if (ref) { - ref.unref(); - console.log(`unref ${key}`); - // TODO: remove associated files from target - } -} - -interface UpdateMetadataJob { - absPath: string; - publicPath: string; - stat: fs.Stats | null; - mediaFile: MediaFile | null; -} - -interface ProcessFileArgs { - absPath: string; - stat: fs.Stats; - mediaFile: MediaFile; - spin: Spinner; -} - -interface ProcessJob { - absPath: string; - stat: fs.Stats; - mediaFile: MediaFile; - processor: typeof processors[0]; - index: number; - after: ProcessJob[]; - needs: number; -} - -export function skipBasename(basename: string): boolean { - // dot files must be incrementally tracked - if (basename === ".dirsort") return true; - if (basename === ".friends") return true; - - return ( - basename.startsWith(".") || - basename.startsWith("._") || - basename.startsWith(".tmp") || - basename === ".DS_Store" || - basename.toLowerCase() === "thumbs.db" || - basename.toLowerCase() === "desktop.ini" - ); -} - -export function toPublicPath(absPath: string) { - ASSERT(path.isAbsolute(absPath)); - if (absPath === root) return "/"; - return "/" + path.relative(root, absPath).replaceAll("\\", "/"); -} - -export function testProgram(name: string, helpArgument: string) { - try { - child_process.spawnSync(name, [helpArgument]); - return name; - } catch (err) { - console.warn(`Missing or corrupt executable '${name}'`); - } - return null; -} - -const monthMilliseconds = 30 * 24 * 60 * 60 * 1000; - -import { Progress } from "@paperclover/console/Progress"; -import { Spinner } from "@paperclover/console/Spinner"; -import * as async from "#sitegen/async"; -import * as fs from "#sitegen/fs"; - -import * as path from "node:path"; -import * as child_process from "node:child_process"; -import * as util from "node:util"; -import * as crypto from "node:crypto"; - -import { MediaFile, MediaFileKind } from "@/file-viewer/models/MediaFile.ts"; -import { AssetRef } from "@/file-viewer/models/AssetRef.ts"; -import { formatDate } from "@/file-viewer/format.ts"; -import * as rules from "@/file-viewer/rules.ts"; -import * as highlight from "@/file-viewer/highlight.ts"; -import * as ffmpeg from "@/file-viewer/ffmpeg.ts"; -import * as transcodeRules from "@/file-viewer/transcode-rules.ts"; +// The file scanner incrementally updates an sqlite database with file +// stats. Additionally, it runs "processors" on files, which precompute +// expensive data such as running `ffprobe` on all media to get the +// duration. +// +// Processors are also used to derive compressed and optimized assets, +// which is how automatic JXL / AV1 encoding is done. Derived files are +// uploaded to the clover NAS to be pulled by VPS instances for hosting. +// +// This is the third iteration of the scanner, hence its name "scan3"; +// Remember that any software you want to be maintainable and high +// quality cannot be written with AI. +const root = path.resolve("/Volumes/clover/Published"); +const workDir = path.resolve(".clover/file-assets"); + +export async function main() { + const start = performance.now(); + const timerSpinner = new Spinner({ + text: () => + `paper clover's scan3 [${ + ((performance.now() - start) / 1000).toFixed(1) + }s]`, + fps: 10, + }); + using _endTimerSpinner = { [Symbol.dispose]: () => timerSpinner.stop() }; + + // Read a directory or file stat and queue up changed files. + using qList = new async.Queue({ + name: "Discover Tree", + async fn(absPath: string, spin) { + const stat = await fs.stat(absPath); + + const publicPath = toPublicPath(absPath); + const mediaFile = MediaFile.getByPath(publicPath); + + if (stat.isDirectory()) { + const items = await fs.readdir(absPath); + qList.addMany(items.map((subPath) => path.join(absPath, subPath))); + + if (mediaFile) { + const deleted = mediaFile.getChildren() + .filter((child) => !items.includes(child.basename)) + .flatMap((child) => + child.kind === MediaFileKind.directory + ? child.getRecursiveFileChildren() + : child + ); + + qMeta.addMany(deleted.map((mediaFile) => ({ + absPath: path.join(root, mediaFile.path), + publicPath: mediaFile.path, + stat: null, + mediaFile, + }))); + } + + return; + } + + // All processes must be performed again if there is no file. + if ( + !mediaFile || + stat.size !== mediaFile.size || + stat.mtime.getTime() !== mediaFile.date.getTime() + ) { + qMeta.add({ absPath, publicPath, stat, mediaFile }); + return; + } + + // If the scanners changed, it may mean more processes should be run. + queueProcessors({ absPath, stat, mediaFile }); + }, + maxJobs: 24, + }); + using qMeta = new async.Queue({ + name: "Update Metadata", + async fn({ absPath, publicPath, stat, mediaFile }: UpdateMetadataJob) { + if (!stat) { + // File was deleted. + await runUndoProcessors(UNWRAP(mediaFile)); + return; + } + // TODO: run scrubLocationMetadata first + + const hash = await new Promise((resolve, reject) => { + const reader = fs.createReadStream(absPath); + reader.on("error", reject); + + const hasher = crypto.createHash("sha1").setEncoding("hex"); + hasher.on("error", reject); + hasher.on("readable", () => resolve(hasher.read())); + + reader.pipe(hasher); + }); + let date = stat.mtime; + if ( + mediaFile && + mediaFile.date.getTime() < stat.mtime.getTime() && + (Date.now() - stat.mtime.getTime()) < monthMilliseconds + ) { + date = mediaFile.date; + console.warn( + `M-time on ${publicPath} was likely corrupted. ${ + formatDate(mediaFile.date) + } -> ${formatDate(stat.mtime)}`, + ); + } + mediaFile = MediaFile.createFile({ + path: publicPath, + date, + hash, + size: stat.size, + duration: mediaFile?.duration ?? 0, + dimensions: mediaFile?.dimensions ?? "", + contents: mediaFile?.contents ?? "", + }); + await queueProcessors({ absPath, stat, mediaFile }); + }, + getItemText: (job) => + job.publicPath.slice(1) + (job.stat ? "" : " (deleted)"), + maxJobs: 10, + }); + using qProcess = new async.Queue({ + name: "Process Contents", + async fn( + { absPath, stat, mediaFile, processor, index, after }: ProcessJob, + spin, + ) { + await processor.run({ absPath, stat, mediaFile, spin }); + mediaFile.setProcessed(mediaFile.processed | (1 << (16 + index))); + for (const dependantJob of after) { + ASSERT(dependantJob.needs > 0); + dependantJob.needs -= 1; + if (dependantJob.needs == 0) qProcess.add(dependantJob); + } + }, + getItemText: ({ mediaFile, processor }) => + `${mediaFile.path.slice(1)} - ${processor.name}`, + maxJobs: 4, + }); + + function decodeProcessors(input: string) { + return input + .split(";") + .filter(Boolean) + .map(([a, b, c]) => ({ + id: a, + hash: (b.charCodeAt(0) << 8) + c.charCodeAt(0), + })); + } + + async function queueProcessors( + { absPath, stat, mediaFile }: Omit, + ) { + const ext = mediaFile.extension.toLowerCase(); + let possible = processors.filter((p) => p.include.has(ext)); + if (possible.length === 0) return; + + const hash = possible.reduce((a, b) => a ^ b.hash, 0) | 1; + ASSERT(hash <= 0xFFFF); + let processed = mediaFile.processed; + + // If the hash has changed, migrate the bitfield over. + // This also runs when the processor hash is in it's initial 0 state. + const order = decodeProcessors(mediaFile.processors); + if ((processed & 0xFFFF) !== hash) { + const previous = order.filter((_, i) => + (processed & (1 << (16 + i))) !== 0 + ); + processed = hash; + for (const { id, hash } of previous) { + const p = processors.find((p) => p.id === id); + if (!p) continue; + const index = possible.indexOf(p); + if (index !== -1 && p.hash === hash) { + processed |= 1 << (16 + index); + } else { + if (p.undo) await p.undo(mediaFile); + } + } + mediaFile.setProcessors( + processed, + possible.map((p) => + p.id + String.fromCharCode(p.hash >> 8, p.hash & 0xFF) + ).join(";"), + ); + } else { + possible = order.map(({ id }) => + UNWRAP(possible.find((p) => p.id === id)) + ); + } + + // Queue needed processors. + const jobs: ProcessJob[] = []; + for (let i = 0, { length } = possible; i < length; i += 1) { + if ((processed & (1 << (16 + i))) === 0) { + const job: ProcessJob = { + absPath, + stat, + mediaFile, + processor: possible[i], + index: i, + after: [], + needs: possible[i].depends.length, + }; + jobs.push(job); + if (job.needs === 0) qProcess.add(job); + } + } + for (const job of jobs) { + for (const dependId of job.processor.depends) { + const dependJob = jobs.find((j) => j.processor.id === dependId); + if (dependJob) { + dependJob.after.push(job); + } else { + ASSERT(job.needs > 0); + job.needs -= 1; + if (job.needs === 0) qProcess.add(job); + } + } + } + } + + async function runUndoProcessors(mediaFile: MediaFile) { + const { processed } = mediaFile; + const previous = decodeProcessors(mediaFile.processors) + .filter((_, i) => (processed & (1 << (16 + i))) !== 0); + for (const { id } of previous) { + const p = processors.find((p) => p.id === id); + if (!p) continue; + if (p.undo) { + await p.undo(mediaFile); + } + } + mediaFile.delete(); + } + + // Add the root & recursively iterate! + qList.add(root); + await qList.done(); + await qMeta.done(); + await qProcess.done(); + + console.info( + "Updated file viewer index in " + + ((performance.now() - start) / 1000).toFixed(1) + "s", + ); +} + +interface Process { + name: string; + enable?: boolean; + include: Set; + depends?: string[]; + version?: number; + /* Perform an action. */ + run(args: ProcessFileArgs): Promise; + /* Should detect if `run` was never even run before before undoing state */ + undo?(mediaFile: MediaFile): Promise; +} + +const execFileRaw = util.promisify(child_process.execFile); +const execFile: typeof execFileRaw = (( + ...args: Parameters +) => + execFileRaw(...args).catch((e: any) => { + if (e?.message?.startsWith?.("Command failed")) { + if (e.code > (2 ** 31)) e.code |= 0; + const code = e.signal ? `signal ${e.signal}` : `code ${e.code}`; + e.message = `${e.cmd.split(" ")[0]} failed with ${code}`; + } + throw e; + })) as any; +const ffprobeBin = testProgram("ffprobe", "--help"); +const ffmpegBin = testProgram("ffmpeg", "--help"); + +const ffmpegOptions = [ + "-hide_banner", + "-loglevel", + "warning", +]; + +const procDuration: Process = { + name: "calculate duration", + enable: ffprobeBin !== null, + include: rules.extsDuration, + async run({ absPath, mediaFile }) { + const { stdout } = await execFile(ffprobeBin!, [ + "-v", + "error", + "-show_entries", + "format=duration", + "-of", + "default=noprint_wrappers=1:nokey=1", + absPath, + ]); + + const duration = parseFloat(stdout.trim()); + if (Number.isNaN(duration)) { + throw new Error("Could not extract duration from " + stdout); + } + mediaFile.setDuration(Math.ceil(duration)); + }, +}; + +// NOTE: Never re-order the processors. Add new ones at the end. +const procDimensions: Process = { + name: "calculate dimensions", + enable: ffprobeBin != null, + include: rules.extsDimensions, + async run({ absPath, mediaFile }) { + const ext = path.extname(absPath); + + let dimensions; + + if (ext === ".svg") { + // Parse out of text data + const content = await fs.readFile(absPath, "utf8"); + const widthMatch = content.match(/width="(\d+)"/); + const heightMatch = content.match(/height="(\d+)"/); + + if (widthMatch && heightMatch) { + dimensions = `${widthMatch[1]}x${heightMatch[1]}`; + } + } else { + // Use ffprobe to observe streams + const { stdout } = await execFile("ffprobe", [ + "-v", + "error", + "-select_streams", + "v:0", + "-show_entries", + "stream=width,height", + "-of", + "csv=s=x:p=0", + absPath, + ]); + if (stdout.includes("x")) { + dimensions = stdout.trim(); + } + } + + mediaFile.setDimensions(dimensions ?? ""); + }, +}; + +const procLoadTextContents: Process = { + name: "load text content", + include: rules.extsReadContents, + async run({ absPath, mediaFile, stat }) { + if (stat.size > 1_000_000) return; + const text = await fs.readFile(absPath, "utf-8"); + mediaFile.setContents(text); + }, +}; + +const procHighlightCode: Process = { + name: "highlight source code", + include: new Set(rules.extsCode.keys()), + async run({ absPath, mediaFile, stat }) { + const language = UNWRAP( + rules.extsCode.get(path.extname(absPath).toLowerCase()), + ); + // An issue is that .ts is an overloaded extension, shared between + // 'transport stream' and 'typescript'. + // + // Filter used here is: + // - more than 1mb + // - invalid UTF-8 + if (stat.size > 1_000_000) return; + let code; + const buf = await fs.readFile(absPath); + try { + code = new TextDecoder("utf-8", { fatal: true }).decode(buf); + } catch (error) { + mediaFile.setContents(""); + return; + } + const content = await highlight.highlightCode(code, language); + mediaFile.setContents(content); + }, +}; + +const procImageSubsets: Process = { + name: "encode image subsets", + include: rules.extsImage, + depends: ["calculate dimensions"], + version: 2, + async run({ absPath, mediaFile, spin }) { + const { width, height } = UNWRAP(mediaFile.parseDimensions()); + const targetSizes = transcodeRules.imageSizes.filter((w) => w < width); + const baseStatus = spin.text; + + using stack = new DisposableStack(); + for (const size of targetSizes) { + const { w, h } = resizeDimensions(width, height, size); + for (const { ext, args } of transcodeRules.imagePresets) { + spin.text = baseStatus + + ` (${w}x${h}, ${ext.slice(1).toUpperCase()})`; + + stack.use( + await produceAsset( + `${mediaFile.hash}/${size}${ext}`, + async (out) => { + await fs.mkdir(path.dirname(out)); + await fs.rm(out, { force: true }); + await execFile(ffmpegBin!, [ + ...ffmpegOptions, + "-i", + absPath, + "-vf", + `scale=${w}:${h}:force_original_aspect_ratio=increase,crop=${w}:${h}`, + ...args, + out, + ]); + return [out]; + }, + ), + ); + } + } + + stack.move(); + }, + async undo(mediaFile) { + const { width } = UNWRAP(mediaFile.parseDimensions()); + const targetSizes = transcodeRules.imageSizes.filter((w) => w < width); + for (const size of targetSizes) { + for (const { ext } of transcodeRules.imagePresets) { + unproduceAsset(`${mediaFile.hash}/${size}${ext}`); + } + } + }, +}; +const qualityMap: Record = { + u: "ultra-high", + h: "high", + m: "medium", + l: "low", + d: "data-saving", +}; +const procVideos = transcodeRules.videoFormats.map((preset) => ({ + name: `encode ${preset.codec} ${UNWRAP(qualityMap[preset.id[1]])}`, + include: rules.extsVideo, + enable: ffmpegBin != null, + async run({ absPath, mediaFile, spin }) { + if ((mediaFile.duration ?? 0) < 10) return; + await produceAsset(`${mediaFile.hash}/${preset.id}`, async (base) => { + base = path.dirname(base); + await fs.mkdir(base); + + let inputArgs = ["-i", absPath]; + try { + const config = await fs.readJson( + path.join( + path.dirname(absPath), + path.basename(absPath, path.extname(absPath)) + ".json", + ), + ); + if (config.encoder && typeof config.encoder.videoSrc === "string") { + const { videoSrc, audioSrc, rate } = config.encoder; + inputArgs = [ + ...rate ? ["-r", String(rate)] : [], + "-i", + videoSrc, + ...audioSrc ? ["-i", audioSrc] : [], + ]; + } + } catch (err: any) { + if (err?.code !== "ENOENT") throw err; + } + + const args = transcodeRules.getVideoArgs( + preset, + base, + inputArgs, + ); + try { + const fakeProgress = new Progress({ text: spin.text, spinner: null }); + fakeProgress.stop(); + spin.format = (now: number) => fakeProgress.format(now); + // @ts-expect-error + fakeProgress.redraw = () => spin.redraw(); + + await ffmpeg.spawn({ + ffmpeg: ffmpegBin!, + title: fakeProgress.text, + progress: fakeProgress, + args, + cwd: base, + }); + return await collectFiles(); + } catch (err) { + for (const file of await collectFiles()) { + try { + fs.rm(file); + } catch {} + } + throw err; + } + + async function collectFiles(): Promise { + return (await fs.readdir(base)) + .filter((basename) => basename.startsWith(preset.id)) + .map((basename) => path.join(base, basename)); + } + }); + }, +})); + +const processors = [ + procDimensions, + procDuration, + procLoadTextContents, + procHighlightCode, + procImageSubsets, + ...procVideos, +] + .map((process, id, all) => { + const strIndex = (id: number) => + String.fromCharCode("a".charCodeAt(0) + id); + return { + ...process as Process, + id: strIndex(id), + // Create a unique key. + hash: new Uint16Array( + crypto.createHash("sha1") + .update( + process.run.toString() + + (process.version ? String(process.version) : ""), + ) + .digest().buffer, + ).reduce((a, b) => a ^ b), + depends: (process.depends ?? []).map((depend) => { + const index = all.findIndex((p) => p.name === depend); + if (index === -1) throw new Error(`Cannot find depend '${depend}'`); + if (index === id) throw new Error(`Cannot depend on self: '${depend}'`); + return strIndex(index); + }), + }; + }); + +function resizeDimensions(w: number, h: number, desiredWidth: number) { + ASSERT(desiredWidth < w, `${desiredWidth} < ${w}`); + return { w: desiredWidth, h: Math.floor((h / w) * desiredWidth) }; +} + +async function produceAsset( + key: string, + builder: (prefix: string) => Promise, +) { + const asset = AssetRef.putOrIncrement(key); + try { + if (asset.refs === 1) { + const paths = await builder(path.join(workDir, key)); + asset.addFiles( + paths.map((file) => + path.relative(workDir, file) + .replaceAll("\\", "/") + ), + ); + } + return { + [Symbol.dispose]: () => asset.unref(), + }; + } catch (err: any) { + if (err && typeof err === "object") err.assetKey = key; + asset.unref(); + throw err; + } +} + +async function unproduceAsset(key: string) { + const ref = AssetRef.get(key); + if (ref) { + ref.unref(); + console.log(`unref ${key}`); + // TODO: remove associated files from target + } +} + +interface UpdateMetadataJob { + absPath: string; + publicPath: string; + stat: fs.Stats | null; + mediaFile: MediaFile | null; +} + +interface ProcessFileArgs { + absPath: string; + stat: fs.Stats; + mediaFile: MediaFile; + spin: Spinner; +} + +interface ProcessJob { + absPath: string; + stat: fs.Stats; + mediaFile: MediaFile; + processor: typeof processors[0]; + index: number; + after: ProcessJob[]; + needs: number; +} + +export function skipBasename(basename: string): boolean { + // dot files must be incrementally tracked + if (basename === ".dirsort") return true; + if (basename === ".friends") return true; + + return ( + basename.startsWith(".") || + basename.startsWith("._") || + basename.startsWith(".tmp") || + basename === ".DS_Store" || + basename.toLowerCase() === "thumbs.db" || + basename.toLowerCase() === "desktop.ini" + ); +} + +export function toPublicPath(absPath: string) { + ASSERT(path.isAbsolute(absPath)); + if (absPath === root) return "/"; + return "/" + path.relative(root, absPath).replaceAll("\\", "/"); +} + +export function testProgram(name: string, helpArgument: string) { + try { + child_process.spawnSync(name, [helpArgument]); + return name; + } catch (err) { + console.warn(`Missing or corrupt executable '${name}'`); + } + return null; +} + +const monthMilliseconds = 30 * 24 * 60 * 60 * 1000; + +import { Progress } from "@paperclover/console/Progress"; +import { Spinner } from "@paperclover/console/Spinner"; +import * as async from "#sitegen/async"; +import * as fs from "#sitegen/fs"; + +import * as path from "node:path"; +import * as child_process from "node:child_process"; +import * as util from "node:util"; +import * as crypto from "node:crypto"; + +import { MediaFile, MediaFileKind } from "@/file-viewer/models/MediaFile.ts"; +import { AssetRef } from "@/file-viewer/models/AssetRef.ts"; +import { formatDate } from "@/file-viewer/format.ts"; +import * as rules from "@/file-viewer/rules.ts"; +import * as highlight from "@/file-viewer/highlight.ts"; +import * as ffmpeg from "@/file-viewer/ffmpeg.ts"; +import * as transcodeRules from "@/file-viewer/transcode-rules.ts"; diff --git a/src/file-viewer/highlight.ts b/src/file-viewer/highlight.ts index bff96ad..13f94c0 100644 --- a/src/file-viewer/highlight.ts +++ b/src/file-viewer/highlight.ts @@ -135,10 +135,7 @@ function highlightLines({ export const getRegistry = async.once(async () => { const wasmBin = await fs.readFile( - path.join( - import.meta.dirname, - "../node_modules/vscode-oniguruma/release/onig.wasm", - ), + require.resolve("vscode-oniguruma/release/onig.wasm"), ); await oniguruma.loadWASM(wasmBin); diff --git a/src/file-viewer/models/AssetRef.ts b/src/file-viewer/models/AssetRef.ts index 0b039a9..d09fe56 100644 --- a/src/file-viewer/models/AssetRef.ts +++ b/src/file-viewer/models/AssetRef.ts @@ -1,73 +1,73 @@ -const db = getDb("cache.sqlite"); -db.table( - "asset_refs", - /* SQL */ ` - create table if not exists asset_refs ( - id integer primary key autoincrement, - key text not null UNIQUE, - refs integer not null - ); - create table if not exists asset_ref_files ( - file text not null, - id integer not null, - foreign key (id) references asset_refs(id) - ); - create index asset_ref_files_id on asset_ref_files(id); -`, -); - -/** - * Uncompressed files are read directly from the media store root. Derivied - * assets like compressed files, optimized images, and streamable video are - * stored in the `derived` folder. After scanning, the derived assets are - * uploaded into the store (storage1/clofi-derived dataset on NAS). Since - * multiple files can share the same hash, the number of references is - * tracked, and the derived content is only produced once. This means if a - * file is deleted, it should only decrement a reference count; deleting it - * once all references are removed. - */ -export class AssetRef { - /** Key which aws referenced */ - id!: number; - key!: string; - refs!: number; - - unref() { - decrementQuery.run(this.key); - deleteUnreferencedQuery.run().changes > 0; - } - - addFiles(files: string[]) { - for (const file of files) { - addFileQuery.run({ id: this.id, file }); - } - } - - static get(key: string) { - return getQuery.get(key); - } - - static putOrIncrement(key: string) { - putOrIncrementQuery.get(key); - return UNWRAP(AssetRef.get(key)); - } -} - -const getQuery = db.prepare<[key: string]>(/* SQL */ ` - select * from asset_refs where key = ?; -`).as(AssetRef); -const putOrIncrementQuery = db.prepare<[key: string]>(/* SQL */ ` - insert into asset_refs (key, refs) values (?, 1) - on conflict(key) do update set refs = refs + 1; -`); -const decrementQuery = db.prepare<[key: string]>(/* SQL */ ` - update asset_refs set refs = refs - 1 where key = ? and refs > 0; -`); -const deleteUnreferencedQuery = db.prepare(/* SQL */ ` - delete from asset_refs where refs <= 0; -`); -const addFileQuery = db.prepare<[{ id: number; file: string }]>(/* SQL */ ` - insert into asset_ref_files (id, file) values ($id, $file); -`); - -import { getDb } from "#sitegen/sqlite"; +const db = getDb("cache.sqlite"); +db.table( + "asset_refs", + /* SQL */ ` + create table if not exists asset_refs ( + id integer primary key autoincrement, + key text not null UNIQUE, + refs integer not null + ); + create table if not exists asset_ref_files ( + file text not null, + id integer not null, + foreign key (id) references asset_refs(id) ON DELETE CASCADE + ); + create index asset_ref_files_id on asset_ref_files(id); +`, +); + +/** + * Uncompressed files are read directly from the media store root. Derivied + * assets like compressed files, optimized images, and streamable video are + * stored in the `derived` folder. After scanning, the derived assets are + * uploaded into the store (storage1/clofi-derived dataset on NAS). Since + * multiple files can share the same hash, the number of references is + * tracked, and the derived content is only produced once. This means if a + * file is deleted, it should only decrement a reference count; deleting it + * once all references are removed. + */ +export class AssetRef { + /** Key which aws referenced */ + id!: number; + key!: string; + refs!: number; + + unref() { + decrementQuery.run(this.key); + deleteUnreferencedQuery.run().changes > 0; + } + + addFiles(files: string[]) { + for (const file of files) { + addFileQuery.run({ id: this.id, file }); + } + } + + static get(key: string) { + return getQuery.get(key); + } + + static putOrIncrement(key: string) { + putOrIncrementQuery.get(key); + return UNWRAP(AssetRef.get(key)); + } +} + +const getQuery = db.prepare<[key: string]>(/* SQL */ ` + select * from asset_refs where key = ?; +`).as(AssetRef); +const putOrIncrementQuery = db.prepare<[key: string]>(/* SQL */ ` + insert into asset_refs (key, refs) values (?, 1) + on conflict(key) do update set refs = refs + 1; +`); +const decrementQuery = db.prepare<[key: string]>(/* SQL */ ` + update asset_refs set refs = refs - 1 where key = ? and refs > 0; +`); +const deleteUnreferencedQuery = db.prepare(/* SQL */ ` + delete from asset_refs where refs <= 0; +`); +const addFileQuery = db.prepare<[{ id: number; file: string }]>(/* SQL */ ` + insert into asset_ref_files (id, file) values ($id, $file); +`); + +import { getDb } from "#sitegen/sqlite"; diff --git a/src/file-viewer/models/FilePermissions.ts b/src/file-viewer/models/FilePermissions.ts index e9574d1..4187c6e 100644 --- a/src/file-viewer/models/FilePermissions.ts +++ b/src/file-viewer/models/FilePermissions.ts @@ -1,59 +1,59 @@ -const db = getDb("cache.sqlite"); - -db.table( - "permissions", - /* SQL */ ` - CREATE TABLE IF NOT EXISTS permissions ( - prefix TEXT PRIMARY KEY, - allow INTEGER NOT NULL - ); -`, -); -export class FilePermissions { - prefix!: string; - /** Currently set to 1 always */ - allow!: number; - - // -- static ops -- - static getByPrefix(filePath: string): number { - return getByPrefixQuery.get(filePath)?.allow ?? 0; - } - - static getExact(filePath: string): number { - return getExactQuery.get(filePath)?.allow ?? 0; - } - - static setPermissions(prefix: string, allow: number) { - if (allow) { - insertQuery.run({ prefix, allow }); - } else { - deleteQuery.run(prefix); - } - } -} - -const getByPrefixQuery = db.prepare< - [prefix: string], - Pick ->(/* SQL */ ` - SELECT allow - FROM permissions - WHERE ? GLOB prefix || '*' - ORDER BY LENGTH(prefix) DESC - LIMIT 1; -`); -const getExactQuery = db.prepare< - [file: string], - Pick ->(/* SQL */ ` - SELECT allow FROM permissions WHERE ? == prefix -`); - -const insertQuery = db.prepare<[{ prefix: string; allow: number }]>(/* SQL */ ` - REPLACE INTO permissions(prefix, allow) VALUES($prefix, $allow); -`); -const deleteQuery = db.prepare<[file: string]>(/* SQL */ ` - DELETE FROM permissions WHERE prefix = ?; -`); - -import { getDb } from "#sitegen/sqlite"; +const db = getDb("cache.sqlite"); + +db.table( + "permissions", + /* SQL */ ` + CREATE TABLE IF NOT EXISTS permissions ( + prefix TEXT PRIMARY KEY, + allow INTEGER NOT NULL + ); +`, +); +export class FilePermissions { + prefix!: string; + /** Currently set to 1 always */ + allow!: number; + + // -- static ops -- + static getByPrefix(filePath: string): number { + return getByPrefixQuery.get(filePath)?.allow ?? 0; + } + + static getExact(filePath: string): number { + return getExactQuery.get(filePath)?.allow ?? 0; + } + + static setPermissions(prefix: string, allow: number) { + if (allow) { + insertQuery.run({ prefix, allow }); + } else { + deleteQuery.run(prefix); + } + } +} + +const getByPrefixQuery = db.prepare< + [prefix: string], + Pick +>(/* SQL */ ` + SELECT allow + FROM permissions + WHERE ? GLOB prefix || '*' + ORDER BY LENGTH(prefix) DESC + LIMIT 1; +`); +const getExactQuery = db.prepare< + [file: string], + Pick +>(/* SQL */ ` + SELECT allow FROM permissions WHERE ? == prefix +`); + +const insertQuery = db.prepare<[{ prefix: string; allow: number }]>(/* SQL */ ` + REPLACE INTO permissions(prefix, allow) VALUES($prefix, $allow); +`); +const deleteQuery = db.prepare<[file: string]>(/* SQL */ ` + DELETE FROM permissions WHERE prefix = ?; +`); + +import { getDb } from "#sitegen/sqlite"; diff --git a/src/file-viewer/models/MediaFile.ts b/src/file-viewer/models/MediaFile.ts index 624c503..9d092a5 100644 --- a/src/file-viewer/models/MediaFile.ts +++ b/src/file-viewer/models/MediaFile.ts @@ -1,436 +1,436 @@ -const db = getDb("cache.sqlite"); -db.table( - "media_files", - /* SQL */ ` - create table media_files ( - id integer primary key autoincrement, - parent_id integer, - path text unique, - kind integer not null, - timestamp integer not null, - timestamp_updated integer not null default current_timestamp, - hash text not null, - size integer not null, - duration integer not null default 0, - dimensions text not null default "", - contents text not null, - dirsort text, - processed integer not null, - processors text not null default "", - foreign key (parent_id) references media_files(id) - ); - -- index for quickly looking up files by path - create index media_files_path on media_files (path); - -- index for quickly looking up children - create index media_files_parent_id on media_files (parent_id); - -- index for quickly looking up recursive file children - create index media_files_file_children on media_files (kind, path); - -- index for finding directories that need to be processed - create index media_files_directory_processed on media_files (kind, processed); -`, -); - -export enum MediaFileKind { - directory = 0, - file = 1, -} -export class MediaFile { - id!: number; - parent_id!: number | null; - /** - * Has leading slash, does not have `/file` prefix. - * @example "/2025/waterfalls/waterfalls.mp3" - */ - path!: string; - kind!: MediaFileKind; - private timestamp!: number; - private timestamp_updated!: number; - /** for mp3/mp4 files, measured in seconds */ - duration?: number; - /** for images and videos, the dimensions. Two numbers split by `x` */ - dimensions?: string; - /** - * sha1 of - * - files: the contents - * - directories: the JSON array of strings + the content of `readme.txt` - * this is used - * - to inform changes in caching mechanisms (etag, page render cache) - * - as a filename for compressed files (.clover/compressed/.{gz,zstd}) - */ - hash!: string; - /** - * Depends on the file kind. - * - * - For directories, this is the contents of `readme.txt`, if it exists. - * - Otherwise, it is an empty string. - */ - contents!: string; - /** - * For directories, if this is set, it is a JSON-encoded array of the explicit - * sorting order. Derived off of `.dirsort` files. - */ - dirsort!: string | null; - /** in bytes */ - size!: number; - /** - * 0 - not processed - * non-zero - processed - * - * file: a bit-field of the processors. - * directory: this is for re-indexing contents - */ - processed!: number; - processors!: string; - - // -- instance ops -- - get date() { - return new Date(this.timestamp); - } - get lastUpdateDate() { - return new Date(this.timestamp_updated); - } - parseDimensions() { - const dimensions = this.dimensions; - if (!dimensions) return null; - const [width, height] = dimensions.split("x").map(Number); - return { width, height }; - } - get basename() { - return path.basename(this.path); - } - get basenameWithoutExt() { - return path.basename(this.path, path.extname(this.path)); - } - get extension() { - return path.extname(this.path); - } - getChildren() { - return MediaFile.getChildren(this.id) - .filter((file) => !file.basename.startsWith(".")); - } - getPublicChildren() { - const children = MediaFile.getChildren(this.id); - if (FilePermissions.getByPrefix(this.path) == 0) { - return children.filter(({ path }) => FilePermissions.getExact(path) == 0); - } - return children; - } - getParent() { - const dirPath = this.path; - if (dirPath === "/") return null; - const parentPath = path.dirname(dirPath); - if (parentPath === dirPath) return null; - const result = MediaFile.getByPath(parentPath); - if (!result) return null; - ASSERT(result.kind === MediaFileKind.directory); - return result; - } - setProcessed(processed: number) { - setProcessedQuery.run({ id: this.id, processed }); - this.processed = processed; - } - setProcessors(processed: number, processors: string) { - setProcessorsQuery.run({ id: this.id, processed, processors }); - this.processed = processed; - this.processors = processors; - } - setDuration(duration: number) { - setDurationQuery.run({ id: this.id, duration }); - this.duration = duration; - } - setDimensions(dimensions: string) { - setDimensionsQuery.run({ id: this.id, dimensions }); - this.dimensions = dimensions; - } - setContents(contents: string) { - setContentsQuery.run({ id: this.id, contents }); - this.contents = contents; - } - getRecursiveFileChildren() { - if (this.kind !== MediaFileKind.directory) return []; - return getChildrenFilesRecursiveQuery.array(this.path + "/"); - } - delete() { - deleteCascadeQuery.run({ id: this.id }); - } - - // -- static ops -- - static getByPath(filePath: string): MediaFile | null { - const result = getByPathQuery.get(filePath); - if (result) return result; - if (filePath === "/") { - return Object.assign(new MediaFile(), { - id: 0, - parent_id: null, - path: "/", - kind: MediaFileKind.directory, - timestamp: 0, - timestamp_updated: Date.now(), - hash: "0".repeat(40), - contents: "the file scanner has not been run yet", - dirsort: null, - size: 0, - processed: 1, - }); - } - return null; - } - static createFile({ - path: filePath, - date, - hash, - size, - duration, - dimensions, - contents, - }: CreateFile) { - ASSERT( - !filePath.includes("\\") && filePath.startsWith("/"), - `Invalid path: ${filePath}`, - ); - return createFileQuery.getNonNull({ - path: filePath, - parentId: MediaFile.getOrPutDirectoryId(path.dirname(filePath)), - timestamp: date.getTime(), - timestampUpdated: Date.now(), - hash, - size, - duration, - dimensions, - contents, - }); - } - static getOrPutDirectoryId(filePath: string) { - ASSERT( - !filePath.includes("\\") && filePath.startsWith("/"), - `Invalid path: ${filePath}`, - ); - filePath = path.normalize(filePath); - const row = getDirectoryIdQuery.get(filePath); - if (row) return row.id; - let current = filePath; - let parts = []; - let parentId: null | number = null; - if (filePath === "/") { - return createDirectoryQuery.getNonNull({ - path: filePath, - parentId, - }).id; - } - // walk up the path until we find a directory that exists - do { - parts.unshift(path.basename(current)); - current = path.dirname(current); - parentId = getDirectoryIdQuery.get(current)?.id ?? null; - } while (parentId == undefined && current !== "/"); - if (parentId == undefined) { - parentId = createDirectoryQuery.getNonNull({ - path: current, - parentId, - }).id; - } - // walk back down the path, creating directories as needed - for (const part of parts) { - current = path.join(current, part); - ASSERT(parentId != undefined); - parentId = createDirectoryQuery.getNonNull({ - path: current, - parentId, - }).id; - } - return parentId; - } - static markDirectoryProcessed({ - id, - timestamp, - contents, - size, - hash, - dirsort, - }: MarkDirectoryProcessed) { - markDirectoryProcessedQuery.get({ - id, - timestamp: timestamp.getTime(), - contents, - dirsort: dirsort ? JSON.stringify(dirsort) : "", - hash, - size, - }); - } - static setProcessed(id: number, processed: number) { - setProcessedQuery.run({ id, processed }); - } - static createOrUpdateDirectory(dirPath: string) { - const id = MediaFile.getOrPutDirectoryId(dirPath); - return updateDirectoryQuery.get(id); - } - static getChildren(id: number) { - return getChildrenQuery.array(id); - } - static db = db; -} - -// Create a `file` entry with a given path, date, file hash, size, and duration -// If the file already exists, update the date and duration. -// If the file exists and the hash is different, sets `compress` to 0. -interface CreateFile { - path: string; - date: Date; - hash: string; - size: number; - duration: number; - dimensions: string; - contents: string; -} - -// Set the `processed` flag true and update the metadata for a directory -export interface MarkDirectoryProcessed { - id: number; - timestamp: Date; - contents: string; - size: number; - hash: string; - dirsort: null | string[]; -} - -export interface DirConfig { - /** Overridden sorting */ - sort: string[]; -} - -// -- queries -- - -// Get a directory ID by path, creating it if it doesn't exist -const createDirectoryQuery = db.prepare< - [{ path: string; parentId: number | null }], - { id: number } ->( - /* SQL */ ` - insert into media_files ( - path, parent_id, kind, timestamp, hash, size, - duration, dimensions, contents, dirsort, processed) - values ( - $path, $parentId, ${MediaFileKind.directory}, 0, '', 0, - 0, '', '', '', 0) - returning id; -`, -); -const getDirectoryIdQuery = db.prepare<[string], { id: number }>(/* SQL */ ` - SELECT id FROM media_files WHERE path = ? AND kind = ${MediaFileKind.directory}; -`); -const createFileQuery = db.prepare<[{ - path: string; - parentId: number; - timestamp: number; - timestampUpdated: number; - hash: string; - size: number; - duration: number; - dimensions: string; - contents: string; -}], void>(/* SQL */ ` - insert into media_files ( - path, parent_id, kind, timestamp, timestamp_updated, hash, - size, duration, dimensions, contents, processed) - values ( - $path, $parentId, ${MediaFileKind.file}, $timestamp, $timestampUpdated, - $hash, $size, $duration, $dimensions, $contents, 0) - on conflict(path) do update set - timestamp = excluded.timestamp, - timestamp_updated = excluded.timestamp_updated, - duration = excluded.duration, - size = excluded.size, - contents = excluded.contents, - processed = case - when media_files.hash != excluded.hash then 0 - else media_files.processed - end - returning *; -`).as(MediaFile); -const setProcessedQuery = db.prepare<[{ - id: number; - processed: number; -}]>(/* SQL */ ` - update media_files set processed = $processed where id = $id; -`); -const setProcessorsQuery = db.prepare<[{ - id: number; - processed: number; - processors: string; -}]>(/* SQL */ ` - update media_files set - processed = $processed, - processors = $processors - where id = $id; -`); -const setDurationQuery = db.prepare<[{ - id: number; - duration: number; -}]>(/* SQL */ ` - update media_files set duration = $duration where id = $id; -`); -const setDimensionsQuery = db.prepare<[{ - id: number; - dimensions: string; -}]>(/* SQL */ ` - update media_files set dimensions = $dimensions where id = $id; -`); -const setContentsQuery = db.prepare<[{ - id: number; - contents: string; -}]>(/* SQL */ ` - update media_files set contents = $contents where id = $id; -`); -const getByPathQuery = db.prepare<[string]>(/* SQL */ ` - select * from media_files where path = ?; -`).as(MediaFile); -const markDirectoryProcessedQuery = db.prepare<[{ - timestamp: number; - contents: string; - dirsort: string; - hash: string; - size: number; - id: number; -}]>(/* SQL */ ` - update media_files set - processed = 1, - timestamp = $timestamp, - contents = $contents, - dirsort = $dirsort, - hash = $hash, - size = $size - where id = $id; -`); -const updateDirectoryQuery = db.prepare<[id: number]>(/* SQL */ ` - update media_files set processed = 0 where id = ?; -`); - -const getChildrenQuery = db.prepare<[id: number]>(/* SQL */ ` - select * from media_files where parent_id = ?; -`).as(MediaFile); -const getChildrenFilesRecursiveQuery = db.prepare<[dir: string]>(/* SQL */ ` - select * from media_files - where path like ? || '%' - and kind = ${MediaFileKind.file} -`).as(MediaFile); -const deleteCascadeQuery = db.prepare<[{ id: number }]>(/* SQL */ ` - with recursive items as ( - select id, parent_id from media_files where id = $id - union all - select p.id, p.parent_id - from media_files p - join items c on p.id = c.parent_id - where p.parent_id is not null - and not exists ( - select 1 from media_files child - where child.parent_id = p.id - and child.id <> c.id - ) - ) - delete from media_files - where id in (select id from items) -`); - -import { getDb } from "#sitegen/sqlite"; -import * as path from "node:path/posix"; -import { FilePermissions } from "./FilePermissions.ts"; +const db = getDb("cache.sqlite"); +db.table( + "media_files", + /* SQL */ ` + create table media_files ( + id integer primary key autoincrement, + parent_id integer, + path text unique, + kind integer not null, + timestamp integer not null, + timestamp_updated integer not null default current_timestamp, + hash text not null, + size integer not null, + duration integer not null default 0, + dimensions text not null default "", + contents text not null, + dirsort text, + processed integer not null, + processors text not null default "", + foreign key (parent_id) references media_files(id) + ); + -- index for quickly looking up files by path + create index media_files_path on media_files (path); + -- index for quickly looking up children + create index media_files_parent_id on media_files (parent_id); + -- index for quickly looking up recursive file children + create index media_files_file_children on media_files (kind, path); + -- index for finding directories that need to be processed + create index media_files_directory_processed on media_files (kind, processed); +`, +); + +export enum MediaFileKind { + directory = 0, + file = 1, +} +export class MediaFile { + id!: number; + parent_id!: number | null; + /** + * Has leading slash, does not have `/file` prefix. + * @example "/2025/waterfalls/waterfalls.mp3" + */ + path!: string; + kind!: MediaFileKind; + private timestamp!: number; + private timestamp_updated!: number; + /** for mp3/mp4 files, measured in seconds */ + duration?: number; + /** for images and videos, the dimensions. Two numbers split by `x` */ + dimensions?: string; + /** + * sha1 of + * - files: the contents + * - directories: the JSON array of strings + the content of `readme.txt` + * this is used + * - to inform changes in caching mechanisms (etag, page render cache) + * - as a filename for compressed files (.clover/compressed/.{gz,zstd}) + */ + hash!: string; + /** + * Depends on the file kind. + * + * - For directories, this is the contents of `readme.txt`, if it exists. + * - Otherwise, it is an empty string. + */ + contents!: string; + /** + * For directories, if this is set, it is a JSON-encoded array of the explicit + * sorting order. Derived off of `.dirsort` files. + */ + dirsort!: string | null; + /** in bytes */ + size!: number; + /** + * 0 - not processed + * non-zero - processed + * + * file: a bit-field of the processors. + * directory: this is for re-indexing contents + */ + processed!: number; + processors!: string; + + // -- instance ops -- + get date() { + return new Date(this.timestamp); + } + get lastUpdateDate() { + return new Date(this.timestamp_updated); + } + parseDimensions() { + const dimensions = this.dimensions; + if (!dimensions) return null; + const [width, height] = dimensions.split("x").map(Number); + return { width, height }; + } + get basename() { + return path.basename(this.path); + } + get basenameWithoutExt() { + return path.basename(this.path, path.extname(this.path)); + } + get extension() { + return path.extname(this.path); + } + getChildren() { + return MediaFile.getChildren(this.id) + .filter((file) => !file.basename.startsWith(".")); + } + getPublicChildren() { + const children = MediaFile.getChildren(this.id); + if (FilePermissions.getByPrefix(this.path) == 0) { + return children.filter(({ path }) => FilePermissions.getExact(path) == 0); + } + return children; + } + getParent() { + const dirPath = this.path; + if (dirPath === "/") return null; + const parentPath = path.dirname(dirPath); + if (parentPath === dirPath) return null; + const result = MediaFile.getByPath(parentPath); + if (!result) return null; + ASSERT(result.kind === MediaFileKind.directory); + return result; + } + setProcessed(processed: number) { + setProcessedQuery.run({ id: this.id, processed }); + this.processed = processed; + } + setProcessors(processed: number, processors: string) { + setProcessorsQuery.run({ id: this.id, processed, processors }); + this.processed = processed; + this.processors = processors; + } + setDuration(duration: number) { + setDurationQuery.run({ id: this.id, duration }); + this.duration = duration; + } + setDimensions(dimensions: string) { + setDimensionsQuery.run({ id: this.id, dimensions }); + this.dimensions = dimensions; + } + setContents(contents: string) { + setContentsQuery.run({ id: this.id, contents }); + this.contents = contents; + } + getRecursiveFileChildren() { + if (this.kind !== MediaFileKind.directory) return []; + return getChildrenFilesRecursiveQuery.array(this.path + "/"); + } + delete() { + deleteCascadeQuery.run({ id: this.id }); + } + + // -- static ops -- + static getByPath(filePath: string): MediaFile | null { + const result = getByPathQuery.get(filePath); + if (result) return result; + if (filePath === "/") { + return Object.assign(new MediaFile(), { + id: 0, + parent_id: null, + path: "/", + kind: MediaFileKind.directory, + timestamp: 0, + timestamp_updated: Date.now(), + hash: "0".repeat(40), + contents: "the file scanner has not been run yet", + dirsort: null, + size: 0, + processed: 1, + }); + } + return null; + } + static createFile({ + path: filePath, + date, + hash, + size, + duration, + dimensions, + contents, + }: CreateFile) { + ASSERT( + !filePath.includes("\\") && filePath.startsWith("/"), + `Invalid path: ${filePath}`, + ); + return createFileQuery.getNonNull({ + path: filePath, + parentId: MediaFile.getOrPutDirectoryId(path.dirname(filePath)), + timestamp: date.getTime(), + timestampUpdated: Date.now(), + hash, + size, + duration, + dimensions, + contents, + }); + } + static getOrPutDirectoryId(filePath: string) { + ASSERT( + !filePath.includes("\\") && filePath.startsWith("/"), + `Invalid path: ${filePath}`, + ); + filePath = path.normalize(filePath); + const row = getDirectoryIdQuery.get(filePath); + if (row) return row.id; + let current = filePath; + let parts = []; + let parentId: null | number = null; + if (filePath === "/") { + return createDirectoryQuery.getNonNull({ + path: filePath, + parentId, + }).id; + } + // walk up the path until we find a directory that exists + do { + parts.unshift(path.basename(current)); + current = path.dirname(current); + parentId = getDirectoryIdQuery.get(current)?.id ?? null; + } while (parentId == undefined && current !== "/"); + if (parentId == undefined) { + parentId = createDirectoryQuery.getNonNull({ + path: current, + parentId, + }).id; + } + // walk back down the path, creating directories as needed + for (const part of parts) { + current = path.join(current, part); + ASSERT(parentId != undefined); + parentId = createDirectoryQuery.getNonNull({ + path: current, + parentId, + }).id; + } + return parentId; + } + static markDirectoryProcessed({ + id, + timestamp, + contents, + size, + hash, + dirsort, + }: MarkDirectoryProcessed) { + markDirectoryProcessedQuery.get({ + id, + timestamp: timestamp.getTime(), + contents, + dirsort: dirsort ? JSON.stringify(dirsort) : "", + hash, + size, + }); + } + static setProcessed(id: number, processed: number) { + setProcessedQuery.run({ id, processed }); + } + static createOrUpdateDirectory(dirPath: string) { + const id = MediaFile.getOrPutDirectoryId(dirPath); + return updateDirectoryQuery.get(id); + } + static getChildren(id: number) { + return getChildrenQuery.array(id); + } + static db = db; +} + +// Create a `file` entry with a given path, date, file hash, size, and duration +// If the file already exists, update the date and duration. +// If the file exists and the hash is different, sets `compress` to 0. +interface CreateFile { + path: string; + date: Date; + hash: string; + size: number; + duration: number; + dimensions: string; + contents: string; +} + +// Set the `processed` flag true and update the metadata for a directory +export interface MarkDirectoryProcessed { + id: number; + timestamp: Date; + contents: string; + size: number; + hash: string; + dirsort: null | string[]; +} + +export interface DirConfig { + /** Overridden sorting */ + sort: string[]; +} + +// -- queries -- + +// Get a directory ID by path, creating it if it doesn't exist +const createDirectoryQuery = db.prepare< + [{ path: string; parentId: number | null }], + { id: number } +>( + /* SQL */ ` + insert into media_files ( + path, parent_id, kind, timestamp, hash, size, + duration, dimensions, contents, dirsort, processed) + values ( + $path, $parentId, ${MediaFileKind.directory}, 0, '', 0, + 0, '', '', '', 0) + returning id; +`, +); +const getDirectoryIdQuery = db.prepare<[string], { id: number }>(/* SQL */ ` + SELECT id FROM media_files WHERE path = ? AND kind = ${MediaFileKind.directory}; +`); +const createFileQuery = db.prepare<[{ + path: string; + parentId: number; + timestamp: number; + timestampUpdated: number; + hash: string; + size: number; + duration: number; + dimensions: string; + contents: string; +}], void>(/* SQL */ ` + insert into media_files ( + path, parent_id, kind, timestamp, timestamp_updated, hash, + size, duration, dimensions, contents, processed) + values ( + $path, $parentId, ${MediaFileKind.file}, $timestamp, $timestampUpdated, + $hash, $size, $duration, $dimensions, $contents, 0) + on conflict(path) do update set + timestamp = excluded.timestamp, + timestamp_updated = excluded.timestamp_updated, + duration = excluded.duration, + size = excluded.size, + contents = excluded.contents, + processed = case + when media_files.hash != excluded.hash then 0 + else media_files.processed + end + returning *; +`).as(MediaFile); +const setProcessedQuery = db.prepare<[{ + id: number; + processed: number; +}]>(/* SQL */ ` + update media_files set processed = $processed where id = $id; +`); +const setProcessorsQuery = db.prepare<[{ + id: number; + processed: number; + processors: string; +}]>(/* SQL */ ` + update media_files set + processed = $processed, + processors = $processors + where id = $id; +`); +const setDurationQuery = db.prepare<[{ + id: number; + duration: number; +}]>(/* SQL */ ` + update media_files set duration = $duration where id = $id; +`); +const setDimensionsQuery = db.prepare<[{ + id: number; + dimensions: string; +}]>(/* SQL */ ` + update media_files set dimensions = $dimensions where id = $id; +`); +const setContentsQuery = db.prepare<[{ + id: number; + contents: string; +}]>(/* SQL */ ` + update media_files set contents = $contents where id = $id; +`); +const getByPathQuery = db.prepare<[string]>(/* SQL */ ` + select * from media_files where path = ?; +`).as(MediaFile); +const markDirectoryProcessedQuery = db.prepare<[{ + timestamp: number; + contents: string; + dirsort: string; + hash: string; + size: number; + id: number; +}]>(/* SQL */ ` + update media_files set + processed = 1, + timestamp = $timestamp, + contents = $contents, + dirsort = $dirsort, + hash = $hash, + size = $size + where id = $id; +`); +const updateDirectoryQuery = db.prepare<[id: number]>(/* SQL */ ` + update media_files set processed = 0 where id = ?; +`); + +const getChildrenQuery = db.prepare<[id: number]>(/* SQL */ ` + select * from media_files where parent_id = ?; +`).as(MediaFile); +const getChildrenFilesRecursiveQuery = db.prepare<[dir: string]>(/* SQL */ ` + select * from media_files + where path like ? || '%' + and kind = ${MediaFileKind.file} +`).as(MediaFile); +const deleteCascadeQuery = db.prepare<[{ id: number }]>(/* SQL */ ` + with recursive items as ( + select id, parent_id from media_files where id = $id + union all + select p.id, p.parent_id + from media_files p + join items c on p.id = c.parent_id + where p.parent_id is not null + and not exists ( + select 1 from media_files child + where child.parent_id = p.id + and child.id <> c.id + ) + ) + delete from media_files + where id in (select id from items) +`); + +import { getDb } from "#sitegen/sqlite"; +import * as path from "node:path/posix"; +import { FilePermissions } from "./FilePermissions.ts"; diff --git a/src/file-viewer/pages/file.404.tsx b/src/file-viewer/pages/file.404.tsx index 74cae0b..11ed09f 100644 --- a/src/file-viewer/pages/file.404.tsx +++ b/src/file-viewer/pages/file.404.tsx @@ -1,34 +1,34 @@ -import { MediaFile } from "../models/MediaFile.ts"; -import { MediaPanel } from "../views/clofi.tsx"; -import { addScript } from "#sitegen"; - -export const theme = { - bg: "#312652", - fg: "#f0f0ff", - primary: "#fabe32", -}; - -export const meta = { title: "file not found" }; - -export default function CotyledonPage() { - addScript("../scripts/canvas_cotyledon.client.ts"); - return ( -
- ); -} +import { MediaFile } from "../models/MediaFile.ts"; +import { MediaPanel } from "../views/clofi.tsx"; +import { addScript } from "#sitegen"; + +export const theme = { + bg: "#312652", + fg: "#f0f0ff", + primary: "#fabe32", +}; + +export const meta = { title: "file not found" }; + +export default function CotyledonPage() { + addScript("../scripts/canvas_cotyledon.client.ts"); + return ( +
+ +
+
+
+

this file does not exist ...

+

+ return +

+
+
+
+ ); +} diff --git a/src/file-viewer/rules.ts b/src/file-viewer/rules.ts index e4522ff..675e5b9 100644 --- a/src/file-viewer/rules.ts +++ b/src/file-viewer/rules.ts @@ -1,143 +1,147 @@ -// -- file extension rules -- - -/** Extensions that must have EXIF/etc data stripped */ -export const extScrubExif = new Set([ - ".jpg", - ".jpeg", - ".png", - ".mov", - ".mp4", - ".m4a", -]); -/** Extensions that rendered syntax-highlighted code */ -export const extsCode = new Map(Object.entries({ - ".json": "json", - ".toml": "toml", - ".ts": "ts", - ".js": "ts", - ".tsx": "tsx", - ".jsx": "tsx", - ".css": "css", - ".py": "python", - ".lua": "lua", - ".sh": "shell", - ".bat": "dosbatch", - ".ps1": "powershell", - ".cmd": "dosbatch", - ".yaml": "yaml", - ".yml": "yaml", - ".zig": "zig", - ".astro": "astro", - ".mdx": "mdx", - ".xml": "xml", - ".jsonc": "json", - ".php": "php", - ".patch": "diff", - ".diff": "diff", -})); -/** These files show an audio embed. */ -export const extsAudio = new Set([ - ".mp3", - ".flac", - ".wav", - ".ogg", - ".m4a", -]); -/** These files show a video embed. */ -export const extsVideo = new Set([ - ".mp4", - ".mkv", - ".webm", - ".avi", - ".mov", -]); -/** These files show an image embed */ -export const extsImage = new Set([ - ".jpg", - ".jpeg", - ".png", - ".gif", - ".webp", - ".avif", - ".heic", - ".svg", -]); - -/** These files populate `duration` using `ffprobe` */ -export const extsDuration = new Set([...extsAudio, ...extsVideo]); -/** These files populate `dimensions` using `ffprobe` */ -export const extsDimensions = new Set([...extsImage, ...extsVideo]); - -/** These files read file contents into `contents`, as-is */ -export const extsReadContents = new Set([".txt", ".chat"]); - -export const extsArchive = new Set([ - ".zip", - ".rar", - ".7z", - ".tar", - ".gz", - ".bz2", - ".xz", -]); - -/** - * Formats which are already compression formats, meaning a pass - * through zstd would offer little to negative benefits - */ -export const extsPreCompressed = new Set([ - ...extsAudio, - ...extsVideo, - ...extsImage, - ...extsArchive, - // TODO: are any of these NOT good for compression -]); - -export function fileIcon( - file: Pick, - dirOpen?: boolean, -) { - const { kind, basename } = file; - if (kind === MediaFileKind.directory) return dirOpen ? "dir-open" : "dir"; - - // -- special cases -- - if (file.path === "/2024/for everyone") return "snow"; - - // -- basename cases -- - if (basename === "readme.txt") return "readme"; - - // -- extension cases -- - const ext = path.extname(file.basename).toLowerCase(); - if ([".comp", ".fuse", ".setting"].includes(ext)) return "fusion"; - if ([".json", ".toml", ".yaml", ".yml"].includes(ext)) return "json"; - if (ext === ".blend") return "blend"; - if (ext === ".chat") return "chat"; - if (ext === ".html") return "webpage"; - if (ext === ".lnk") return "link"; - if (ext === ".txt" || ext === ".md") return "text"; - - // -- extension categories -- - if (extsVideo.has(ext)) return "video"; - if (extsAudio.has(ext)) return "audio"; - if (extsImage.has(ext)) return "image"; - if (extsArchive.has(ext)) return "archive"; - if (extsCode.has(ext)) return "code"; - - return "file"; -} - -// -- viewer rules -- -const pathToCanvas = new Map(Object.entries({ - "/2017": "2017", - "/2018": "2018", - "/2019": "2019", - "/2020": "2020", - "/2021": "2021", - "/2022": "2022", - "/2023": "2023", - "/2024": "2024", -})); - -import type * as highlight from "./highlight.ts"; -import { MediaFile, MediaFileKind } from "@/file-viewer/models/MediaFile.ts"; -import * as path from "node:path"; +// -- file extension rules -- + +/** Extensions that must have EXIF/etc data stripped */ +export const extScrubExif = new Set([ + ".jpg", + ".jpeg", + ".png", + ".mov", + ".mp4", + ".m4a", +]); +/** Extensions that rendered syntax-highlighted code */ +export const extsCode = new Map(Object.entries({ + ".json": "json", + ".toml": "toml", + ".ts": "ts", + ".js": "ts", + ".tsx": "tsx", + ".jsx": "tsx", + ".css": "css", + ".py": "python", + ".lua": "lua", + ".sh": "shell", + ".bat": "dosbatch", + ".ps1": "powershell", + ".cmd": "dosbatch", + ".yaml": "yaml", + ".yml": "yaml", + ".zig": "zig", + ".astro": "astro", + ".mdx": "mdx", + ".xml": "xml", + ".jsonc": "json", + ".php": "php", + ".patch": "diff", + ".diff": "diff", +})); +/** These files show an audio embed. */ +export const extsAudio = new Set([ + ".mp3", + ".flac", + ".wav", + ".ogg", + ".m4a", +]); +/** These files show a video embed. */ +export const extsVideo = new Set([ + ".mp4", + ".mkv", + ".webm", + ".avi", + ".mov", +]); +/** These files show an image embed */ +export const extsImage = new Set([ + ".jpg", + ".jpeg", + ".png", + ".webp", + ".avif", + ".heic", +]); +/** These files show an image embed, but aren't optimized */ +export const extsImageLike = new Set([ + ...extsImage, + ".svg", + ".gif", +]); + +/** These files populate `duration` using `ffprobe` */ +export const extsDuration = new Set([...extsAudio, ...extsVideo]); +/** These files populate `dimensions` using `ffprobe` */ +export const extsDimensions = new Set([...extsImage, ...extsVideo]); + +/** These files read file contents into `contents`, as-is */ +export const extsReadContents = new Set([".txt", ".chat"]); + +export const extsArchive = new Set([ + ".zip", + ".rar", + ".7z", + ".tar", + ".gz", + ".bz2", + ".xz", +]); + +/** + * Formats which are already compression formats, meaning a pass + * through zstd would offer little to negative benefits + */ +export const extsPreCompressed = new Set([ + ...extsAudio, + ...extsVideo, + ...extsImage, + ...extsArchive, + // TODO: are any of these NOT good for compression +]); + +export function fileIcon( + file: Pick, + dirOpen?: boolean, +) { + const { kind, basename } = file; + if (kind === MediaFileKind.directory) return dirOpen ? "dir-open" : "dir"; + + // -- special cases -- + if (file.path === "/2024/for everyone") return "snow"; + + // -- basename cases -- + if (basename === "readme.txt") return "readme"; + + // -- extension cases -- + const ext = path.extname(file.basename).toLowerCase(); + if ([".comp", ".fuse", ".setting"].includes(ext)) return "fusion"; + if ([".json", ".toml", ".yaml", ".yml"].includes(ext)) return "json"; + if (ext === ".blend") return "blend"; + if (ext === ".chat") return "chat"; + if (ext === ".html") return "webpage"; + if (ext === ".lnk") return "link"; + if (ext === ".txt" || ext === ".md") return "text"; + + // -- extension categories -- + if (extsVideo.has(ext)) return "video"; + if (extsAudio.has(ext)) return "audio"; + if (extsImage.has(ext)) return "image"; + if (extsArchive.has(ext)) return "archive"; + if (extsCode.has(ext)) return "code"; + + return "file"; +} + +// -- viewer rules -- +const pathToCanvas = new Map(Object.entries({ + "/2017": "2017", + "/2018": "2018", + "/2019": "2019", + "/2020": "2020", + "/2021": "2021", + "/2022": "2022", + "/2023": "2023", + "/2024": "2024", +})); + +import type * as highlight from "./highlight.ts"; +import { MediaFile, MediaFileKind } from "@/file-viewer/models/MediaFile.ts"; +import * as path from "node:path"; diff --git a/src/file-viewer/sort.ts b/src/file-viewer/sort.ts index cbbb8ba..be1b587 100644 --- a/src/file-viewer/sort.ts +++ b/src/file-viewer/sort.ts @@ -1,58 +1,58 @@ -export function splitRootDirFiles(dir: MediaFile, hasCotyledonCookie: boolean) { - const children = dir.getPublicChildren(); - let readme: MediaFile | null = null; - - const groups = { - // years 2025 and onwards - years: [] as MediaFile[], - // named categories - categories: [] as MediaFile[], - // years 2017 to 2024 - cotyledon: [] as MediaFile[], - }; - const colorMap: Record = { - years: "#a2ff91", - categories: "#9c91ff", - cotyledon: "#ff91ca", - }; - for (const child of children) { - const basename = child.basename; - if (basename === "readme.txt") { - readme = child; - continue; - } - - const year = basename.match(/^(\d{4})/); - if (year) { - const n = parseInt(year[1]); - if (n >= 2025) { - groups.years.push(child); - } else { - groups.cotyledon.push(child); - } - } else { - groups.categories.push(child); - } - } - - let sections = []; - for (const [key, files] of Object.entries(groups)) { - if (key === "cotyledon" && !hasCotyledonCookie) { - continue; - } - if (key === "years" || key === "cotyledon") { - files.sort((a, b) => { - return b.basename.localeCompare(a.basename); - }); - } else { - files.sort((a, b) => { - return a.basename.localeCompare(b.basename); - }); - } - sections.push({ key, titleColor: colorMap[key], files }); - } - - return { readme, sections }; -} - -import { MediaFile } from "./models/MediaFile.ts"; +export function splitRootDirFiles(dir: MediaFile, hasCotyledonCookie: boolean) { + const children = dir.getPublicChildren(); + let readme: MediaFile | null = null; + + const groups = { + // years 2025 and onwards + years: [] as MediaFile[], + // named categories + categories: [] as MediaFile[], + // years 2017 to 2024 + cotyledon: [] as MediaFile[], + }; + const colorMap: Record = { + years: "#a2ff91", + categories: "#9c91ff", + cotyledon: "#ff91ca", + }; + for (const child of children) { + const basename = child.basename; + if (basename === "readme.txt") { + readme = child; + continue; + } + + const year = basename.match(/^(\d{4})/); + if (year) { + const n = parseInt(year[1]); + if (n >= 2025) { + groups.years.push(child); + } else { + groups.cotyledon.push(child); + } + } else { + groups.categories.push(child); + } + } + + let sections = []; + for (const [key, files] of Object.entries(groups)) { + if (key === "cotyledon" && !hasCotyledonCookie) { + continue; + } + if (key === "years" || key === "cotyledon") { + files.sort((a, b) => { + return b.basename.localeCompare(a.basename); + }); + } else { + files.sort((a, b) => { + return a.basename.localeCompare(b.basename); + }); + } + sections.push({ key, titleColor: colorMap[key], files }); + } + + return { readme, sections }; +} + +import { MediaFile } from "./models/MediaFile.ts"; diff --git a/src/file-viewer/transcode-rules.ts b/src/file-viewer/transcode-rules.ts index 5a4b518..bb91c6f 100644 --- a/src/file-viewer/transcode-rules.ts +++ b/src/file-viewer/transcode-rules.ts @@ -99,14 +99,38 @@ export const imagePresets = [ "6", ], }, - // TODO: avif + { + ext: ".avif", + args: [ + "-c:v", + "libaom-av1", + "-crf", + "30", + "-pix_fmt", + "yuv420p10le", + ], + }, { ext: ".jxl", - args: ["-c:v", "libjxl", "-distance", "0.8", "-effort", "9"], + args: [ + "-c:v", + "libjxl", + "-distance", + "0.8", + "-effort", + "9", + "-update", + "-frames:v", + "1", + ], }, ]; -export function getVideoArgs(preset: VideoEncodePreset, outbase: string, input: string[]) { +export function getVideoArgs( + preset: VideoEncodePreset, + outbase: string, + input: string[], +) { const cmd = [...input]; if (preset.codec === "av1") { diff --git a/src/file-viewer/views/lofi.css b/src/file-viewer/views/lofi.css index 70165f3..d82b27d 100644 --- a/src/file-viewer/views/lofi.css +++ b/src/file-viewer/views/lofi.css @@ -1,43 +1,43 @@ -body { - margin: 0; - padding: 0; -} -#lofi { - padding: 32px; -} -h1 { - margin-top: 0; - font-size: 3em; - color: var(--primary); - font-family: monospace; -} -ul, li { - margin: 0; - padding: 0; - list-style-type: none; -} -ul { - padding-right: 4em; -} -li a { - display: block; - color: white; - line-height: 2em; - padding: 0 1em; - border-radius: 4px; -} -li a:hover { - background-color: rgba(255,255,255,0.2); - font-weight: bold; - text-decoration: none!important; -} -.dir a { - color: #99eeFF -} -.ext { - opacity: 0.5; -} -.meta { - margin-left: 1em; - opacity: 0.75; -} +body { + margin: 0; + padding: 0; +} +#lofi { + padding: 32px; +} +h1 { + margin-top: 0; + font-size: 3em; + color: var(--primary); + font-family: monospace; +} +ul, li { + margin: 0; + padding: 0; + list-style-type: none; +} +ul { + padding-right: 4em; +} +li a { + display: block; + color: white; + line-height: 2em; + padding: 0 1em; + border-radius: 4px; +} +li a:hover { + background-color: rgba(255, 255, 255, 0.2); + font-weight: bold; + text-decoration: none !important; +} +.dir a { + color: #99eeff; +} +.ext { + opacity: 0.5; +} +.meta { + margin-left: 1em; + opacity: 0.75; +} diff --git a/src/friend-auth.ts b/src/friend-auth.ts index e0c362b..d65c8d6 100644 --- a/src/friend-auth.ts +++ b/src/friend-auth.ts @@ -1,75 +1,75 @@ -let friendPassword = ""; -try { - friendPassword = require("./friends/hardcoded-password.ts").friendPassword; -} catch {} - -export const app = new Hono(); - -const cookieAge = 60 * 60 * 24 * 30; // 1 month - -function checkFriendsCookie(c: Context) { - const cookie = c.req.header("Cookie"); - if (!cookie) return false; - const cookies = cookie.split("; ").map((x) => x.split("=")); - return cookies.some( - (kv) => - kv[0].trim() === "friends_password" && - kv[1].trim() && - kv[1].trim() === friendPassword, - ); -} - -export function requireFriendAuth(c: Context) { - const k = c.req.query("password") || c.req.query("k"); - if (k) { - if (k === friendPassword) { - return c.body(null, 303, { - Location: "/friends", - "Set-Cookie": - `friends_password=${k}; Path=/; HttpOnly; SameSite=Strict; Max-Age=${cookieAge}`, - }); - } else { - return c.body(null, 303, { - Location: "/friends", - }); - } - } - if (checkFriendsCookie(c)) { - return undefined; - } else { - return serveAsset(c, "/friends/auth", 403); - } -} - -app.get("/friends", (c) => { - const friendAuthChallenge = requireFriendAuth(c); - if (friendAuthChallenge) return friendAuthChallenge; - return serveAsset(c, "/friends", 200); -}); - -let incorrectMap: Record = {}; -app.post("/friends", async (c) => { - const ip = c.header("X-Forwarded-For") ?? getConnInfo(c).remote.address ?? - "unknown"; - if (incorrectMap[ip]) { - return serveAsset(c, "/friends/auth/fail", 403); - } - const data = await c.req.formData(); - const k = data.get("password"); - if (k === friendPassword) { - return c.body(null, 303, { - Location: "/friends", - "Set-Cookie": - `friends_password=${k}; Path=/; HttpOnly; SameSite=Strict; Max-Age=${cookieAge}`, - }); - } - incorrectMap[ip] = true; - await setTimeout(2500); - incorrectMap[ip] = false; - return serveAsset(c, "/friends/auth/fail", 403); -}); - -import { type Context, Hono } from "hono"; -import { serveAsset } from "#sitegen/assets"; -import { setTimeout } from "node:timers/promises"; -import { getConnInfo } from "#hono/conninfo"; +let friendPassword = ""; +try { + friendPassword = require("./friends/hardcoded-password.ts").friendPassword; +} catch {} + +export const app = new Hono(); + +const cookieAge = 60 * 60 * 24 * 30; // 1 month + +function checkFriendsCookie(c: Context) { + const cookie = c.req.header("Cookie"); + if (!cookie) return false; + const cookies = cookie.split("; ").map((x) => x.split("=")); + return cookies.some( + (kv) => + kv[0].trim() === "friends_password" && + kv[1].trim() && + kv[1].trim() === friendPassword, + ); +} + +export function requireFriendAuth(c: Context) { + const k = c.req.query("password") || c.req.query("k"); + if (k) { + if (k === friendPassword) { + return c.body(null, 303, { + Location: "/friends", + "Set-Cookie": + `friends_password=${k}; Path=/; HttpOnly; SameSite=Strict; Max-Age=${cookieAge}`, + }); + } else { + return c.body(null, 303, { + Location: "/friends", + }); + } + } + if (checkFriendsCookie(c)) { + return undefined; + } else { + return serveAsset(c, "/friends/auth", 403); + } +} + +app.get("/friends", (c) => { + const friendAuthChallenge = requireFriendAuth(c); + if (friendAuthChallenge) return friendAuthChallenge; + return serveAsset(c, "/friends", 200); +}); + +let incorrectMap: Record = {}; +app.post("/friends", async (c) => { + const ip = c.header("X-Forwarded-For") ?? getConnInfo(c).remote.address ?? + "unknown"; + if (incorrectMap[ip]) { + return serveAsset(c, "/friends/auth/fail", 403); + } + const data = await c.req.formData(); + const k = data.get("password"); + if (k === friendPassword) { + return c.body(null, 303, { + Location: "/friends", + "Set-Cookie": + `friends_password=${k}; Path=/; HttpOnly; SameSite=Strict; Max-Age=${cookieAge}`, + }); + } + incorrectMap[ip] = true; + await setTimeout(2500); + incorrectMap[ip] = false; + return serveAsset(c, "/friends/auth/fail", 403); +}); + +import { type Context, Hono } from "hono"; +import { serveAsset } from "#sitegen/assets"; +import { setTimeout } from "node:timers/promises"; +import { getConnInfo } from "#hono/conninfo"; diff --git a/src/global.css b/src/global.css index bf093ee..d25b9e4 100644 --- a/src/global.css +++ b/src/global.css @@ -118,4 +118,3 @@ code { font-family: "rmo", monospace; font-size: inherit; } - diff --git a/src/pages/resume.css b/src/pages/resume.css index f3f225b..b5e1e90 100644 --- a/src/pages/resume.css +++ b/src/pages/resume.css @@ -1,47 +1,47 @@ -body,html { - overflow: hidden; -} -h1 { - color: #f09; - margin-bottom: 0; -} -.job { - padding: 18px; - margin: 1em -18px; - border: 1px solid black; -} -.job *, footer * { - margin: 0; - padding: 0; -} -.job ul { - margin-left: 1em; -} -.job li { - line-height: 1.5em; -} -.job header, footer { - display: grid; - grid-template-columns: auto max-content; - grid-template-rows: 1fr 1fr; -} -footer { - margin-top: 1.5em; -} -footer h2 { - font-size: 1em; - margin-bottom: 0.5em; -} - -.job header > em, footer > em { - margin-top: 2px; - font-size: 1.25em; -} - -header h2, header em, footer h2, footer em { - display: inline-block; -} - header em, footer em { - margin-left: 16px!important; - text-align: right; -} +body, html { + overflow: hidden; +} +h1 { + color: #f09; + margin-bottom: 0; +} +.job { + padding: 18px; + margin: 1em -18px; + border: 1px solid black; +} +.job *, footer * { + margin: 0; + padding: 0; +} +.job ul { + margin-left: 1em; +} +.job li { + line-height: 1.5em; +} +.job header, footer { + display: grid; + grid-template-columns: auto max-content; + grid-template-rows: 1fr 1fr; +} +footer { + margin-top: 1.5em; +} +footer h2 { + font-size: 1em; + margin-bottom: 0.5em; +} + +.job header > em, footer > em { + margin-top: 2px; + font-size: 1.25em; +} + +header h2, header em, footer h2, footer em { + display: inline-block; +} +header em, footer em { + margin-left: 16px !important; + text-align: right; +} diff --git a/src/pages/subscribe.client.ts b/src/pages/subscribe.client.ts index f677196..ac65460 100644 --- a/src/pages/subscribe.client.ts +++ b/src/pages/subscribe.client.ts @@ -1,97 +1,97 @@ -// @ts-nocheck -// manually obfuscated to make it very difficult to reverse engineer -// if you want to decode what the email is, visit the page! -// stops people from automatically scraping the email address -// -// Unfortunately this needs a rewrite to support Chrome without -// hardware acceleration and some Linux stuff. I will probably -// go with a proof of work alternative. -requestAnimationFrame(() => { - const hash = "SHA"; - const a = [ - { parentElement: document.getElementById("subscribe") }, - function (b) { - let c = 0, d = 0; - for (let i = 0; i < b.length; i++) { - c = (c + b[i] ^ 0xF8) % 8; - d = (c * b[i] ^ 0x82) % 193; - } - a[c + 1]()[c](d, b.buffer); - }, - function () { - const i = a[4](a[3]()); - const b = i.innerText = a.pop(); - if (a[b.indexOf("@") / 3]) { - i.href = "mailto:" + b; - } - }, - function () { - return a[a.length % 10]; - }, - function (x) { - return x.parentElement; - }, - function (b, c) { - throw new Uint8Array( - c, - 0, - 64, - c.parentElement = this[8].call(b.call(this)).location, - ); - }, - function (b, c) { - this.width = 8; - this.height = 16; - b.clearColor(0.5, 0.7, 0.9, 1.0); - b.clear(16408 ^ this.width ^ this.height); - const e = new Uint8Array(4 * this.width * this.height); - b.readPixels(0, 0, this.width, this.height, b.RGBA, b.UNSIGNED_BYTE, e); - let parent = a[this.width / 2](this); - while (parent.tagName !== "BODY") { - parent = a[2 * this.height / this.width](parent); - } - try { - let d = [hash, e.length].join("-"); - const b = a[0.25 * this.height](a.pop()(parent)).subtle.digest(d, e); - [, d] = a; - b.then(c.bind(a, a[this.width].bind(globalThis))).catch(d); - } catch (e) { - fetch(e).then(a[5]).catch(a[2]); - } - }, - function (b, c) { - const d = a.splice( - 9, - 1, - [ - a[3]().parentElement.id, - c.parentElement.hostname, - ].join(String.fromCharCode(b)), - ); - var e = new Error(); - Object.defineProperty(e, "stack", { - get() { - a[9] = d; - }, - }); - a[2].call(console.log(e)); - }, - function () { - return this; - }, - "[failed to verify your browser]", - function (a) { - a = a.parentElement.ownerDocument.defaultView; - return { parentElement: a.navigator.webdriver || a.crypto }; - }, - ]; - try { - const c = document.querySelector("canvas"); - const g = c.getContext("webgl2") || c.getContext("webgl"); - a[0].parentElement.innerText = "[...loading...]"; - g.field || requestAnimationFrame(a[6].bind(c, g, a[5])); - } catch { - a.pop(); - fetch(":").then(a[5]).catch(a[2]); - } -}); +// @ts-nocheck +// manually obfuscated to make it very difficult to reverse engineer +// if you want to decode what the email is, visit the page! +// stops people from automatically scraping the email address +// +// Unfortunately this needs a rewrite to support Chrome without +// hardware acceleration and some Linux stuff. I will probably +// go with a proof of work alternative. +requestAnimationFrame(() => { + const hash = "SHA"; + const a = [ + { parentElement: document.getElementById("subscribe") }, + function (b) { + let c = 0, d = 0; + for (let i = 0; i < b.length; i++) { + c = (c + b[i] ^ 0xF8) % 8; + d = (c * b[i] ^ 0x82) % 193; + } + a[c + 1]()[c](d, b.buffer); + }, + function () { + const i = a[4](a[3]()); + const b = i.innerText = a.pop(); + if (a[b.indexOf("@") / 3]) { + i.href = "mailto:" + b; + } + }, + function () { + return a[a.length % 10]; + }, + function (x) { + return x.parentElement; + }, + function (b, c) { + throw new Uint8Array( + c, + 0, + 64, + c.parentElement = this[8].call(b.call(this)).location, + ); + }, + function (b, c) { + this.width = 8; + this.height = 16; + b.clearColor(0.5, 0.7, 0.9, 1.0); + b.clear(16408 ^ this.width ^ this.height); + const e = new Uint8Array(4 * this.width * this.height); + b.readPixels(0, 0, this.width, this.height, b.RGBA, b.UNSIGNED_BYTE, e); + let parent = a[this.width / 2](this); + while (parent.tagName !== "BODY") { + parent = a[2 * this.height / this.width](parent); + } + try { + let d = [hash, e.length].join("-"); + const b = a[0.25 * this.height](a.pop()(parent)).subtle.digest(d, e); + [, d] = a; + b.then(c.bind(a, a[this.width].bind(globalThis))).catch(d); + } catch (e) { + fetch(e).then(a[5]).catch(a[2]); + } + }, + function (b, c) { + const d = a.splice( + 9, + 1, + [ + a[3]().parentElement.id, + c.parentElement.hostname, + ].join(String.fromCharCode(b)), + ); + var e = new Error(); + Object.defineProperty(e, "stack", { + get() { + a[9] = d; + }, + }); + a[2].call(console.log(e)); + }, + function () { + return this; + }, + "[failed to verify your browser]", + function (a) { + a = a.parentElement.ownerDocument.defaultView; + return { parentElement: a.navigator.webdriver || a.crypto }; + }, + ]; + try { + const c = document.querySelector("canvas"); + const g = c.getContext("webgl2") || c.getContext("webgl"); + a[0].parentElement.innerText = "[...loading...]"; + g.field || requestAnimationFrame(a[6].bind(c, g, a[5])); + } catch { + a.pop(); + fetch(":").then(a[5]).catch(a[2]); + } +}); diff --git a/src/q+a/artifacts.ts b/src/q+a/artifacts.ts index 188c8d7..5c7be26 100644 --- a/src/q+a/artifacts.ts +++ b/src/q+a/artifacts.ts @@ -1,89 +1,89 @@ -// Artifacts used to be a central system in the old data-driven website. -// Now, it simply refers to one of these link presets. Every project has -// one canonical URL, which the questions page can refer to with `@id`. -type Artifact = [title: string, url: string, type: ArtifactType]; -type ArtifactType = "music" | "game" | "project" | "video"; -export const artifactMap: Record = { - // 2025 - "in-the-summer": ["in the summer", "", "music"], - waterfalls: ["waterfalls", "/waterfalls", "music"], - lolzip: ["lol.zip", "", "project"], - "g-is-missing": ["g is missing", "", "music"], - "im-18-now": ["i'm 18 now", "", "music"], - "programming-comparison": [ - "thursday programming language comparison", - "", - "video", - ], - aaaaaaaaa: ["aaaaaaaaa", "", "music"], - "its-snowing": ["it's snowing", "", "video"], - // 2023 - "iphone-15-review": [ - "iphone 15 review", - "/file/2023/iphone%2015%20review/iphone-15-review.mp4", - "video", - ], - // 2022 - mayday: ["mayday", "/file/2022/mayday/mayday.mp4", "music"], - "mystery-of-life": [ - "mystery of life", - "/file/2022/mystery-of-life/mystery-of-life.mp4", - "music", - ], - // 2021 - "top-10000-bread": [ - "top 10000 bread", - "https://paperclover.net/file/2021/top-10000-bread/output.mp4", - "video", - ], - "phoenix-write-soundtrack": [ - "Phoenix, WRITE! soundtrack", - "/file/2021/phoenix-write/OST", - "music", - ], - "phoenix-write": ["Pheonix, WRITE!", "/file/2021/phoenix-write", "game"], - "money-visual-cover": [ - "money visual cover", - "/file/2021/money-visual-cover/money-visual-cover.mp4", - "video", - ], - "i-got-this-thing": [ - "i got this thing", - "/file/2021/i-got-this-thing/i-got-this-thing.mp4", - "video", - ], - // 2020 - elemental4: ["elemental 4", "/file/2020/elemental4-rewrite", "game"], - "elemental-4": ["elemental 4", "/file/2020/elemental4-rewrite", "game"], - // 2019 - "throw-soundtrack": [ - "throw soundtrack", - "/file/2019/throw/soundtrack", - "music", - ], - "elemental-lite": ["elemental lite", "/file/2019/elemental-lite-1.7", "game"], - volar: [ - "volar visual cover", - "/file/2019/volar-visual-cover/volar.mp4", - "video", - ], - wpm: [ - "how to read 500 words per minute", - "/file/2019/how-to-read-500-words-per-minute/how-to-read-500-words-per-minute.mp4", - "video", - ], - "dice-roll": [ - "thursday dice roll", - "/file/2019/thursday-dice-roll/thursday-dice-roll.mp4", - "video", - ], - "math-problem": [ - "thursday math problem", - "/file/2019/thursday-math-problem/thursday-math-problem.mp4", - "video", - ], - // 2018 - // 2017 - "hatred-island": ["hatred island", "/file/2017/hatred%20island", "game"], - "test-video-1": ["test video 1", "/file/2017/test-video1.mp4", "video"], -}; +// Artifacts used to be a central system in the old data-driven website. +// Now, it simply refers to one of these link presets. Every project has +// one canonical URL, which the questions page can refer to with `@id`. +type Artifact = [title: string, url: string, type: ArtifactType]; +type ArtifactType = "music" | "game" | "project" | "video"; +export const artifactMap: Record = { + // 2025 + "in-the-summer": ["in the summer", "", "music"], + waterfalls: ["waterfalls", "/waterfalls", "music"], + lolzip: ["lol.zip", "", "project"], + "g-is-missing": ["g is missing", "", "music"], + "im-18-now": ["i'm 18 now", "", "music"], + "programming-comparison": [ + "thursday programming language comparison", + "", + "video", + ], + aaaaaaaaa: ["aaaaaaaaa", "", "music"], + "its-snowing": ["it's snowing", "", "video"], + // 2023 + "iphone-15-review": [ + "iphone 15 review", + "/file/2023/iphone%2015%20review/iphone-15-review.mp4", + "video", + ], + // 2022 + mayday: ["mayday", "/file/2022/mayday/mayday.mp4", "music"], + "mystery-of-life": [ + "mystery of life", + "/file/2022/mystery-of-life/mystery-of-life.mp4", + "music", + ], + // 2021 + "top-10000-bread": [ + "top 10000 bread", + "https://paperclover.net/file/2021/top-10000-bread/output.mp4", + "video", + ], + "phoenix-write-soundtrack": [ + "Phoenix, WRITE! soundtrack", + "/file/2021/phoenix-write/OST", + "music", + ], + "phoenix-write": ["Pheonix, WRITE!", "/file/2021/phoenix-write", "game"], + "money-visual-cover": [ + "money visual cover", + "/file/2021/money-visual-cover/money-visual-cover.mp4", + "video", + ], + "i-got-this-thing": [ + "i got this thing", + "/file/2021/i-got-this-thing/i-got-this-thing.mp4", + "video", + ], + // 2020 + elemental4: ["elemental 4", "/file/2020/elemental4-rewrite", "game"], + "elemental-4": ["elemental 4", "/file/2020/elemental4-rewrite", "game"], + // 2019 + "throw-soundtrack": [ + "throw soundtrack", + "/file/2019/throw/soundtrack", + "music", + ], + "elemental-lite": ["elemental lite", "/file/2019/elemental-lite-1.7", "game"], + volar: [ + "volar visual cover", + "/file/2019/volar-visual-cover/volar.mp4", + "video", + ], + wpm: [ + "how to read 500 words per minute", + "/file/2019/how-to-read-500-words-per-minute/how-to-read-500-words-per-minute.mp4", + "video", + ], + "dice-roll": [ + "thursday dice roll", + "/file/2019/thursday-dice-roll/thursday-dice-roll.mp4", + "video", + ], + "math-problem": [ + "thursday math problem", + "/file/2019/thursday-math-problem/thursday-math-problem.mp4", + "video", + ], + // 2018 + // 2017 + "hatred-island": ["hatred island", "/file/2017/hatred%20island", "game"], + "test-video-1": ["test video 1", "/file/2017/test-video1.mp4", "video"], +}; diff --git a/src/q+a/backend.ts b/src/q+a/backend.ts index ce00d69..3efe087 100644 --- a/src/q+a/backend.ts +++ b/src/q+a/backend.ts @@ -1,228 +1,228 @@ -const PROXYCHECK_API_KEY = process.env.PROXYCHECK_API_KEY; - -export const app = new Hono(); - -// Main page -app.get("/q+a", async (c) => { - if (hasAdminToken(c)) { - return serveAsset(c, "/admin/q+a", 200); - } - return serveAsset(c, "/q+a", 200); -}); - -// Submit form -app.post("/q+a", async (c) => { - const form = await c.req.formData(); - let text = form.get("text"); - if (typeof text !== "string") { - return questionFailure(c, 400, "Bad Request"); - } - text = text.trim(); - const input = { - date: new Date(), - prompt: text, - sourceName: "unknown", - sourceLocation: "unknown", - sourceVPN: null, - }; - - input.date.setMilliseconds(0); - - if (text.length <= 0) { - return questionFailure(c, 400, "Content is too short", text); - } - - if (text.length > 16000) { - return questionFailure(c, 400, "Content is too long", text); - } - - // Ban patterns - if ( - text.includes("hams" + "terko" + "mbat" + ".expert") // 2025-02-18. 3 occurrences. All VPN - ) { - // To prevent known automatic spam-bots from noticing something automatic is - // happening, pretend that the question was successfully submitted. - return sendSuccess(c, new Date()); - } - - const ipAddr = c.req.header("cf-connecting-ip"); - if (ipAddr) { - input.sourceName = uniqueNamesGenerator({ - dictionaries: [adjectives, colors, animals], - separator: "-", - seed: ipAddr + PROXYCHECK_API_KEY, - }); - } - - const cfIPCountry = c.req.header("cf-ipcountry"); - if (cfIPCountry) { - input.sourceLocation = cfIPCountry; - } - - if (ipAddr && PROXYCHECK_API_KEY) { - const proxyCheck = await fetch( - `https://proxycheck.io/v2/?key=${PROXYCHECK_API_KEY}&risk=1&vpn=1`, - { - method: "POST", - body: "ips=" + ipAddr, - headers: { - "Content-Type": "application/x-www-form-urlencoded", - }, - }, - ).then((res) => res.json()); - - if (ipAddr && proxyCheck[ipAddr]) { - if (proxyCheck[ipAddr].proxy === "yes") { - input.sourceVPN = proxyCheck[ipAddr].operator?.name ?? - proxyCheck[ipAddr].organisation ?? - proxyCheck[ipAddr].provider ?? "unknown"; - } - if (Number(proxyCheck[ipAddr].risk) > 72) { - return questionFailure( - c, - 403, - "This IP address has been flagged as a high risk IP address. If you are using a VPN/Proxy, please disable it and try again.", - text, - ); - } - } - } - - const date = Question.create( - QuestionType.pending, - JSON.stringify(input), - input.date, - ); - await sendSuccess(c, date); -}); -async function sendSuccess(c: Context, date: Date) { - if (c.req.header("Accept")?.includes("application/json")) { - return c.json({ - success: true, - message: "ok", - date: date.getTime(), - id: formatQuestionId(date), - }, { status: 200 }); - } - c.res = await renderView(c, "q+a/success", { - permalink: `https://paperclover.net/q+a/${formatQuestionId(date)}`, - }); -} -// Question Permalink -app.get("/q+a/:id", async (c, next) => { - // from deadname era, the seconds used to be in the url. - // this was removed so that the url can be crafted by hand. - let id = c.req.param("id"); - if (id.length === 12 && /^\d+$/.test(id)) { - return c.redirect(`/q+a/${id.slice(0, 10)}`); - } - let image = false; - if (id.endsWith(".png")) { - image = true; - id = id.slice(0, -4); - } - - const timestamp = questionIdToTimestamp(id); - if (!timestamp) return next(); - const question = Question.getByDate(timestamp); - if (!question) return next(); - - if (image) { - return getQuestionImage(question, c.req.method === "HEAD"); - } - return renderView(c, "q+a/permalink", { question }); -}); - -// Admin -app.get("/admin/q+a", async (c) => { - return serveAsset(c, "/admin/q+a", 200); -}); -app.get("/admin/q+a/inbox", async (c) => { - return renderView(c, "q+a/backend-inbox", {}); -}); -app.delete("/admin/q+a/:id", async (c, next) => { - const id = c.req.param("id"); - const timestamp = questionIdToTimestamp(id); - if (!timestamp) return next(); - const question = Question.getByDate(timestamp); - if (!question) return next(); - const deleteFull = c.req.header("X-Delete-Full") === "true"; - if (deleteFull) { - Question.deleteByQmid(question.qmid); - } else { - Question.rejectByQmid(question.qmid); - } - return c.json({ success: true, message: "ok" }); -}); -app.patch("/admin/q+a/:id", async (c, next) => { - const id = c.req.param("id"); - const timestamp = questionIdToTimestamp(id); - if (!timestamp) return next(); - const question = Question.getByDate(timestamp); - if (!question) return next(); - const form = await c.req.raw.json(); - if (typeof form.text !== "string" || typeof form.type !== "number") { - return questionFailure(c, 400, "Bad Request"); - } - Question.updateByQmid(question.qmid, form.text, form.type); - return c.json({ success: true, message: "ok" }); -}); -app.get("/admin/q+a/:id", async (c, next) => { - const id = c.req.param("id"); - const timestamp = questionIdToTimestamp(id); - if (!timestamp) return next(); - const question = Question.getByDate(timestamp); - if (!question) return next(); - - let pendingInfo: null | PendingQuestionData = null; - if (question.type === QuestionType.pending) { - pendingInfo = JSON.parse(question.text) as PendingQuestionData; - question.text = pendingInfo.prompt.trim().split("\n").map((line) => - line.trim().length === 0 ? "" : `q: ${line.trim()}` - ).join("\n") + "\n\n"; - question.type = QuestionType.normal; - } - - return renderView(c, "q+a/editor", { - pendingInfo, - question, - }); -}); - -app.get("/q+a/things/random", async (c) => { - c.res = await renderView(c, "q+a/things-random", {}); -}); - -async function questionFailure( - c: Context, - status: ContentfulStatusCode, - message: string, - content?: string, -) { - if (c.req.header("Accept")?.includes("application/json")) { - return c.json({ success: false, message, id: null }, { status }); - } - return await renderView(c, "q+a/fail", { - error: message, - content, - }); -} - -import { type Context, Hono } from "#hono"; -import type { ContentfulStatusCode } from "hono/utils/http-status"; -import { - adjectives, - animals, - colors, - uniqueNamesGenerator, -} from "unique-names-generator"; -import { hasAdminToken } from "../admin.ts"; -import { serveAsset } from "#sitegen/assets"; -import { - PendingQuestion, - PendingQuestionData, -} from "./models/PendingQuestion.ts"; -import { Question, QuestionType } from "./models/Question.ts"; -import { renderView } from "#sitegen/view"; -import { getQuestionImage } from "./image.tsx"; -import { formatQuestionId, questionIdToTimestamp } from "./format.ts"; +const PROXYCHECK_API_KEY = process.env.PROXYCHECK_API_KEY; + +export const app = new Hono(); + +// Main page +app.get("/q+a", async (c) => { + if (hasAdminToken(c)) { + return serveAsset(c, "/admin/q+a", 200); + } + return serveAsset(c, "/q+a", 200); +}); + +// Submit form +app.post("/q+a", async (c) => { + const form = await c.req.formData(); + let text = form.get("text"); + if (typeof text !== "string") { + return questionFailure(c, 400, "Bad Request"); + } + text = text.trim(); + const input = { + date: new Date(), + prompt: text, + sourceName: "unknown", + sourceLocation: "unknown", + sourceVPN: null, + }; + + input.date.setMilliseconds(0); + + if (text.length <= 0) { + return questionFailure(c, 400, "Content is too short", text); + } + + if (text.length > 16000) { + return questionFailure(c, 400, "Content is too long", text); + } + + // Ban patterns + if ( + text.includes("hams" + "terko" + "mbat" + ".expert") // 2025-02-18. 3 occurrences. All VPN + ) { + // To prevent known automatic spam-bots from noticing something automatic is + // happening, pretend that the question was successfully submitted. + return sendSuccess(c, new Date()); + } + + const ipAddr = c.req.header("cf-connecting-ip"); + if (ipAddr) { + input.sourceName = uniqueNamesGenerator({ + dictionaries: [adjectives, colors, animals], + separator: "-", + seed: ipAddr + PROXYCHECK_API_KEY, + }); + } + + const cfIPCountry = c.req.header("cf-ipcountry"); + if (cfIPCountry) { + input.sourceLocation = cfIPCountry; + } + + if (ipAddr && PROXYCHECK_API_KEY) { + const proxyCheck = await fetch( + `https://proxycheck.io/v2/?key=${PROXYCHECK_API_KEY}&risk=1&vpn=1`, + { + method: "POST", + body: "ips=" + ipAddr, + headers: { + "Content-Type": "application/x-www-form-urlencoded", + }, + }, + ).then((res) => res.json()); + + if (ipAddr && proxyCheck[ipAddr]) { + if (proxyCheck[ipAddr].proxy === "yes") { + input.sourceVPN = proxyCheck[ipAddr].operator?.name ?? + proxyCheck[ipAddr].organisation ?? + proxyCheck[ipAddr].provider ?? "unknown"; + } + if (Number(proxyCheck[ipAddr].risk) > 72) { + return questionFailure( + c, + 403, + "This IP address has been flagged as a high risk IP address. If you are using a VPN/Proxy, please disable it and try again.", + text, + ); + } + } + } + + const date = Question.create( + QuestionType.pending, + JSON.stringify(input), + input.date, + ); + await sendSuccess(c, date); +}); +async function sendSuccess(c: Context, date: Date) { + if (c.req.header("Accept")?.includes("application/json")) { + return c.json({ + success: true, + message: "ok", + date: date.getTime(), + id: formatQuestionId(date), + }, { status: 200 }); + } + c.res = await renderView(c, "q+a/success", { + permalink: `https://paperclover.net/q+a/${formatQuestionId(date)}`, + }); +} +// Question Permalink +app.get("/q+a/:id", async (c, next) => { + // from deadname era, the seconds used to be in the url. + // this was removed so that the url can be crafted by hand. + let id = c.req.param("id"); + if (id.length === 12 && /^\d+$/.test(id)) { + return c.redirect(`/q+a/${id.slice(0, 10)}`); + } + let image = false; + if (id.endsWith(".png")) { + image = true; + id = id.slice(0, -4); + } + + const timestamp = questionIdToTimestamp(id); + if (!timestamp) return next(); + const question = Question.getByDate(timestamp); + if (!question) return next(); + + if (image) { + return getQuestionImage(question, c.req.method === "HEAD"); + } + return renderView(c, "q+a/permalink", { question }); +}); + +// Admin +app.get("/admin/q+a", async (c) => { + return serveAsset(c, "/admin/q+a", 200); +}); +app.get("/admin/q+a/inbox", async (c) => { + return renderView(c, "q+a/backend-inbox", {}); +}); +app.delete("/admin/q+a/:id", async (c, next) => { + const id = c.req.param("id"); + const timestamp = questionIdToTimestamp(id); + if (!timestamp) return next(); + const question = Question.getByDate(timestamp); + if (!question) return next(); + const deleteFull = c.req.header("X-Delete-Full") === "true"; + if (deleteFull) { + Question.deleteByQmid(question.qmid); + } else { + Question.rejectByQmid(question.qmid); + } + return c.json({ success: true, message: "ok" }); +}); +app.patch("/admin/q+a/:id", async (c, next) => { + const id = c.req.param("id"); + const timestamp = questionIdToTimestamp(id); + if (!timestamp) return next(); + const question = Question.getByDate(timestamp); + if (!question) return next(); + const form = await c.req.raw.json(); + if (typeof form.text !== "string" || typeof form.type !== "number") { + return questionFailure(c, 400, "Bad Request"); + } + Question.updateByQmid(question.qmid, form.text, form.type); + return c.json({ success: true, message: "ok" }); +}); +app.get("/admin/q+a/:id", async (c, next) => { + const id = c.req.param("id"); + const timestamp = questionIdToTimestamp(id); + if (!timestamp) return next(); + const question = Question.getByDate(timestamp); + if (!question) return next(); + + let pendingInfo: null | PendingQuestionData = null; + if (question.type === QuestionType.pending) { + pendingInfo = JSON.parse(question.text) as PendingQuestionData; + question.text = pendingInfo.prompt.trim().split("\n").map((line) => + line.trim().length === 0 ? "" : `q: ${line.trim()}` + ).join("\n") + "\n\n"; + question.type = QuestionType.normal; + } + + return renderView(c, "q+a/editor", { + pendingInfo, + question, + }); +}); + +app.get("/q+a/things/random", async (c) => { + c.res = await renderView(c, "q+a/things-random", {}); +}); + +async function questionFailure( + c: Context, + status: ContentfulStatusCode, + message: string, + content?: string, +) { + if (c.req.header("Accept")?.includes("application/json")) { + return c.json({ success: false, message, id: null }, { status }); + } + return await renderView(c, "q+a/fail", { + error: message, + content, + }); +} + +import { type Context, Hono } from "#hono"; +import type { ContentfulStatusCode } from "hono/utils/http-status"; +import { + adjectives, + animals, + colors, + uniqueNamesGenerator, +} from "unique-names-generator"; +import { hasAdminToken } from "../admin.ts"; +import { serveAsset } from "#sitegen/assets"; +import { + PendingQuestion, + PendingQuestionData, +} from "./models/PendingQuestion.ts"; +import { Question, QuestionType } from "./models/Question.ts"; +import { renderView } from "#sitegen/view"; +import { getQuestionImage } from "./image.tsx"; +import { formatQuestionId, questionIdToTimestamp } from "./format.ts"; diff --git a/src/q+a/format.ts b/src/q+a/format.ts index 79b89b4..8e5df05 100644 --- a/src/q+a/format.ts +++ b/src/q+a/format.ts @@ -1,39 +1,39 @@ -const dateFormat = new Intl.DateTimeFormat("sv", { - timeZone: "EST", - year: "numeric", - month: "2-digit", - hour: "2-digit", - day: "2-digit", - minute: "2-digit", -}); - -// YYYY-MM-DD HH:MM -export function formatQuestionTimestamp(date: Date) { - return dateFormat.format(date); -} - -// YYYY-MM-DDTHH:MM:00Z -export function formatQuestionISOTimestamp(date: Date) { - const str = dateFormat.format(date); - return `${str.slice(0, 10)}T${str.slice(11)}-05:00`; -} - -// YYMMDDHHMM -export function formatQuestionId(date: Date) { - return formatQuestionTimestamp(date).replace(/[^\d]/g, "").slice(2, 12); -} - -export function questionIdToTimestamp(id: string) { - if (id.length !== 10 || !/^\d+$/.test(id)) { - return null; - } - const date = new Date( - `20${id.slice(0, 2)}-${id.slice(2, 4)}-${id.slice(4, 6)} ${ - id.slice(6, 8) - }:${id.slice(8, 10)}:00 EST`, - ); - if (isNaN(date.getTime())) { - return null; - } - return date; -} +const dateFormat = new Intl.DateTimeFormat("sv", { + timeZone: "EST", + year: "numeric", + month: "2-digit", + hour: "2-digit", + day: "2-digit", + minute: "2-digit", +}); + +// YYYY-MM-DD HH:MM +export function formatQuestionTimestamp(date: Date) { + return dateFormat.format(date); +} + +// YYYY-MM-DDTHH:MM:00Z +export function formatQuestionISOTimestamp(date: Date) { + const str = dateFormat.format(date); + return `${str.slice(0, 10)}T${str.slice(11)}-05:00`; +} + +// YYMMDDHHMM +export function formatQuestionId(date: Date) { + return formatQuestionTimestamp(date).replace(/[^\d]/g, "").slice(2, 12); +} + +export function questionIdToTimestamp(id: string) { + if (id.length !== 10 || !/^\d+$/.test(id)) { + return null; + } + const date = new Date( + `20${id.slice(0, 2)}-${id.slice(2, 4)}-${id.slice(4, 6)} ${ + id.slice(6, 8) + }:${id.slice(8, 10)}:00 EST`, + ); + if (isNaN(date.getTime())) { + return null; + } + return date; +} diff --git a/src/q+a/image.tsx b/src/q+a/image.tsx index dac9a54..b83e961 100644 --- a/src/q+a/image.tsx +++ b/src/q+a/image.tsx @@ -1,81 +1,81 @@ -const width = 768; -const cacheImageDir = path.resolve(".clover/question_images"); - -// Cached browser session -const getBrowser = RefCountedExpirable( - () => - puppeteer.launch({ - args: ["--no-sandbox", "--disable-setuid-sandbox"], - }), - (b) => b.close(), -); - -export async function renderQuestionImage(question: Question) { - const html = await renderViewToString("q+a/image-embed", { question }); - - // this browser session will be reused if multiple images are generated - // either at the same time or within a 5-minute time span. the dispose - // symbol - using sharedBrowser = await getBrowser(); - const b = sharedBrowser.value; - - const p = await b.newPage(); - await p.setViewport({ width, height: 400 }); - await p.setContent(html); - try { - await p.waitForNetworkIdle({ idleTime: 100, timeout: 500 }); - } catch (e) {} - const height = await p.evaluate(() => { - const e = document.querySelector("main")!; - return e.getBoundingClientRect().height; - }); - const buf = await p.screenshot({ - path: "screenshot.png", - type: "png", - captureBeyondViewport: true, - clip: { x: 0, width, y: 0, height: height, scale: 1.5 }, - }); - await p.close(); - - return Buffer.from(buf); -} - -export async function getQuestionImage( - question: Question, - headOnly: boolean, -): Promise { - const hash = crypto.createHash("sha1") - .update(question.qmid + question.type + question.text) - .digest("hex"); - - const headers = { - "Content-Type": "image/png", - "Cache-Control": "public, max-age=31536000", - "ETag": `"${hash}"`, - "Last-Modified": question.date.toUTCString(), - }; - - if (headOnly) { - return new Response(null, { headers }); - } - - const cachedFilePath = path.join(cacheImageDir, `/${hash}.png`); - let buf: Buffer; - try { - buf = await fs.readFile(cachedFilePath); - } catch (e: any) { - if (e.code !== "ENOENT") throw e; - buf = await renderQuestionImage(question); - fs.writeMkdir(cachedFilePath, buf).catch(() => {}); - } - - return new Response(buf, { headers }); -} - -import * as crypto from "node:crypto"; -import * as fs from "#sitegen/fs"; -import * as path from "node:path"; -import * as puppeteer from "puppeteer"; -import { Question } from "@/q+a/models/Question.ts"; -import { RefCountedExpirable } from "#sitegen/async"; -import { renderViewToString } from "#sitegen/view"; +const width = 768; +const cacheImageDir = path.resolve(".clover/question_images"); + +// Cached browser session +const getBrowser = RefCountedExpirable( + () => + puppeteer.launch({ + args: ["--no-sandbox", "--disable-setuid-sandbox"], + }), + (b) => b.close(), +); + +export async function renderQuestionImage(question: Question) { + const html = await renderViewToString("q+a/image-embed", { question }); + + // this browser session will be reused if multiple images are generated + // either at the same time or within a 5-minute time span. the dispose + // symbol + using sharedBrowser = await getBrowser(); + const b = sharedBrowser.value; + + const p = await b.newPage(); + await p.setViewport({ width, height: 400 }); + await p.setContent(html); + try { + await p.waitForNetworkIdle({ idleTime: 100, timeout: 500 }); + } catch (e) {} + const height = await p.evaluate(() => { + const e = document.querySelector("main")!; + return e.getBoundingClientRect().height; + }); + const buf = await p.screenshot({ + path: "screenshot.png", + type: "png", + captureBeyondViewport: true, + clip: { x: 0, width, y: 0, height: height, scale: 1.5 }, + }); + await p.close(); + + return Buffer.from(buf); +} + +export async function getQuestionImage( + question: Question, + headOnly: boolean, +): Promise { + const hash = crypto.createHash("sha1") + .update(question.qmid + question.type + question.text) + .digest("hex"); + + const headers = { + "Content-Type": "image/png", + "Cache-Control": "public, max-age=31536000", + "ETag": `"${hash}"`, + "Last-Modified": question.date.toUTCString(), + }; + + if (headOnly) { + return new Response(null, { headers }); + } + + const cachedFilePath = path.join(cacheImageDir, `/${hash}.png`); + let buf: Buffer; + try { + buf = await fs.readFile(cachedFilePath); + } catch (e: any) { + if (e.code !== "ENOENT") throw e; + buf = await renderQuestionImage(question); + fs.writeMkdir(cachedFilePath, buf).catch(() => {}); + } + + return new Response(buf, { headers }); +} + +import * as crypto from "node:crypto"; +import * as fs from "#sitegen/fs"; +import * as path from "node:path"; +import * as puppeteer from "puppeteer"; +import { Question } from "@/q+a/models/Question.ts"; +import { RefCountedExpirable } from "#sitegen/async"; +import { renderViewToString } from "#sitegen/view"; diff --git a/src/q+a/scripts/editor.client.tsx b/src/q+a/scripts/editor.client.tsx index 9d9f9bf..a3c1456 100644 --- a/src/q+a/scripts/editor.client.tsx +++ b/src/q+a/scripts/editor.client.tsx @@ -1,116 +1,116 @@ -import { EditorState } from "@codemirror/state"; -import { basicSetup, EditorView } from "codemirror"; -import { ssrSync } from "#ssr"; -import type { ScriptPayload } from "@/q+a/views/editor.marko"; -import QuestionRender from "@/q+a/tags/question.marko"; - -declare const payload: ScriptPayload; -const date = new Date(payload.date); - -const main = document.getElementById("edit-grid")! as HTMLDivElement; -const preview = document.getElementById("preview")! as HTMLDivElement; - -function updatePreview(text: string) { - preview.innerHTML = ssrSync( - , - ).text; -} -updatePreview(payload.text); - -const startState = EditorState.create({ - doc: payload.text, - extensions: [ - basicSetup, - EditorView.darkTheme.of(true), - EditorView.updateListener.of((update) => { - if (update.docChanged) { - updatePreview(update.state.doc.toString()); - } - }), - EditorView.lineWrapping, - ], - // selection: EditorSelection.create([ - // EditorSelection.cursor(0), - // ], 0), -}); - -const view = new EditorView({ - state: startState, - parent: document.getElementById("editor")!, -}); -view.focus(); - -(globalThis as any).onCommitQuestion = wrapAction(async () => { - const text = view.state.doc.toString(); - const res = await fetch(`/admin/q+a/${payload.id}`, { - method: "PATCH", - headers: { - Accept: "application/json", - "Content-Type": "application/json", - }, - body: JSON.stringify({ - text, - type: payload.type, - }), - }); - if (!res.ok) { - throw new Error("Failed to update question, status: " + res.status); - } - if (location.search.includes("return=inbox")) { - location.href = "/admin/q+a/inbox"; - } else { - location.href = "/q+a#q" + payload.id; - } -}); -(globalThis as any).onDelete = wrapAction(async () => { - if (confirm("Are you sure you want to delete this question?")) { - const res = await fetch(`/admin/q+a/${payload.id}`, { - method: "DELETE", - headers: { - Accept: "application/json", - }, - }); - if (!res.ok) { - throw new Error("Failed to delete question, status: " + res.status); - } - location.href = document.referrer || "/admin/q+a"; - } -}); -(globalThis as any).onTypeChange = () => { - payload.type = parseInt( - (document.getElementById("type") as HTMLSelectElement).value, - ); - updatePreview(view.state.doc.toString()); -}; - -function wrapAction(cb: () => Promise) { - return async () => { - main.style.opacity = "0.5"; - main.style.pointerEvents = "none"; - const inputs = main.querySelectorAll("button,select,input") as NodeListOf< - HTMLButtonElement - >; - inputs.forEach((b) => { - b.disabled = true; - }); - try { - await cb(); - } catch (e: any) { - main.style.opacity = "1"; - main.style.pointerEvents = "auto"; - inputs.forEach((b) => { - b.disabled = false; - }); - alert(e.message); - } - }; -} +import { EditorState } from "@codemirror/state"; +import { basicSetup, EditorView } from "codemirror"; +import { ssrSync } from "#ssr"; +import type { ScriptPayload } from "@/q+a/views/editor.marko"; +import QuestionRender from "@/q+a/tags/question.marko"; + +declare const payload: ScriptPayload; +const date = new Date(payload.date); + +const main = document.getElementById("edit-grid")! as HTMLDivElement; +const preview = document.getElementById("preview")! as HTMLDivElement; + +function updatePreview(text: string) { + preview.innerHTML = ssrSync( + , + ).text; +} +updatePreview(payload.text); + +const startState = EditorState.create({ + doc: payload.text, + extensions: [ + basicSetup, + EditorView.darkTheme.of(true), + EditorView.updateListener.of((update) => { + if (update.docChanged) { + updatePreview(update.state.doc.toString()); + } + }), + EditorView.lineWrapping, + ], + // selection: EditorSelection.create([ + // EditorSelection.cursor(0), + // ], 0), +}); + +const view = new EditorView({ + state: startState, + parent: document.getElementById("editor")!, +}); +view.focus(); + +(globalThis as any).onCommitQuestion = wrapAction(async () => { + const text = view.state.doc.toString(); + const res = await fetch(`/admin/q+a/${payload.id}`, { + method: "PATCH", + headers: { + Accept: "application/json", + "Content-Type": "application/json", + }, + body: JSON.stringify({ + text, + type: payload.type, + }), + }); + if (!res.ok) { + throw new Error("Failed to update question, status: " + res.status); + } + if (location.search.includes("return=inbox")) { + location.href = "/admin/q+a/inbox"; + } else { + location.href = "/q+a#q" + payload.id; + } +}); +(globalThis as any).onDelete = wrapAction(async () => { + if (confirm("Are you sure you want to delete this question?")) { + const res = await fetch(`/admin/q+a/${payload.id}`, { + method: "DELETE", + headers: { + Accept: "application/json", + }, + }); + if (!res.ok) { + throw new Error("Failed to delete question, status: " + res.status); + } + location.href = document.referrer || "/admin/q+a"; + } +}); +(globalThis as any).onTypeChange = () => { + payload.type = parseInt( + (document.getElementById("type") as HTMLSelectElement).value, + ); + updatePreview(view.state.doc.toString()); +}; + +function wrapAction(cb: () => Promise) { + return async () => { + main.style.opacity = "0.5"; + main.style.pointerEvents = "none"; + const inputs = main.querySelectorAll("button,select,input") as NodeListOf< + HTMLButtonElement + >; + inputs.forEach((b) => { + b.disabled = true; + }); + try { + await cb(); + } catch (e: any) { + main.style.opacity = "1"; + main.style.pointerEvents = "auto"; + inputs.forEach((b) => { + b.disabled = false; + }); + alert(e.message); + } + }; +} diff --git a/src/q+a/simple-markdown.ts b/src/q+a/simple-markdown.ts index daedde1..ef11840 100644 --- a/src/q+a/simple-markdown.ts +++ b/src/q+a/simple-markdown.ts @@ -1,1090 +1,1090 @@ -// @ts-nocheck -// # Simple-Markdown Core -// -// This is a fork of Khan-academy's Simple-Markdown[1], initially forked in 2022 -// to add Svelte support[2], and used for paper clover's q+a markdown flavor. -// -// 1: https://github.com/Khan/perseus/tree/main/packages/simple-markdown/src -// 2: https://github.com/paperclover/svelte-simple-markdown -export type Rules = Record; -export interface ParserRule { - name: string; - match: MatchFunction; - parse: ParseFunction; - quality?: QualityFunction; -} - -export class RuleList extends Array { - constructor(input?: ArrayLike) { - super(); - if (input) { - this.push(...Array.from(input)); - } - } - - insertBefore(rule: string, newRule: ParserRule): void { - const index = this.findIndex((r) => r.name === rule); - if (index === -1) { - throw new Error(`Rule ${rule} not found`); - } - this.splice(index, 0, newRule); - } - - insertAfter(rule: string, newRule: ParserRule): void { - const index = this.findIndex((r) => r.name === rule); - if (index === -1) { - throw new Error(`Rule ${rule} not found`); - } - this.splice(index + 1, 0, newRule); - } - - toRuleObject(): Record { - const result: Record = {}; - this.forEach((rule) => { - result[rule.name] = rule; - }); - return result; - } - - add(rule: ParserRule): void { - this.push(rule); - } - - get(rule: string): ParserRule | undefined { - return this.find((r) => r.name === rule); - } - - remove(rule: string): void { - const index = this.findIndex((r) => r.name === rule); - if (index === -1) { - throw new Error(`Rule ${rule} not found`); - } - this.splice(index, 1); - } - - clone() { - return new RuleList(this); - } -} - -/** - * Creates a parser for a given set of rules, with the precedence - * specified as a list of rules. - * - * @param rules - * an object containing - * rule type -> {match, order, parse} objects - * (lower order is higher precedence) - * @param [defaultState] - * - * @returns - * The resulting parse function, with the following parameters: - * @source: the input source string to be parsed - * @state: an optional object to be threaded through parse - * calls. Allows clients to add stateful operations to - * parsing, such as keeping track of how many levels deep - * some nesting is. For an example use-case, see passage-ref - * parsing in src/widgets/passage/passage-markdown.jsx - */ -export function createParser( - ruleListInput: RuleList, - defaultState: Partial = {}, -) { - let rules = ruleListInput.toRuleObject(); - let ruleList = Object.keys(rules); - - let latestState: ParserState; - - let nestedParse = function (source: string, state?: ParserState) { - let result: ASTNode[] = []; - state = state || latestState; - latestState = state; - - while (source) { - // store the best match, it's rule, and quality: - let ruleType = null; - let rule = null; - let capture = null; - let quality = NaN; // loop control variables: - - let i = 0; - let currRuleType = ruleList[0]; - - let currRule = rules[currRuleType]; - - do { - let currCapture = currRule.match(source, state); - - if (currCapture) { - let currQuality = currRule.quality - ? currRule.quality(currCapture, state) - : 0; - - // This should always be true the first time because - // the initial quality is NaN (that's why there's the - // condition negation). - if (!(currQuality <= quality)) { - ruleType = currRuleType; - rule = currRule; - capture = currCapture; - quality = currQuality; - } - } - - // Move on to the next item. - // Note that this makes `currRule` be the next item - i++; - currRuleType = ruleList[i]; - currRule = rules[currRuleType]; - } while ( - // keep looping while we're still within the ruleList - currRule && - // if we don't have a match yet, continue - (!capture || - // or if we have a match, but the next rule is - // at the same order, and has a quality measurement - // functions, then this rule must have a quality - // measurement function (since they are sorted before - // those without), and we need to check if there is - // a better quality match - currRule.quality) - ); - - if (!rule || !capture || !ruleType) { - throw new Error( - "Could not find a matching rule for the below " + - "content. The rule with highest `order` should " + - "always match content provided to it. Check " + - "the definition of `match` for '" + - ruleList[ruleList.length - 1] + - "'. It seems to not match the following source:\n" + - source, - ); - } - - if (capture.index) { - // If present and non-zero, i.e. a non-^ regexp result: - throw new Error( - "`match` must return a capture starting at index 0 " + - "(the current parse index). Did you forget a ^ at the " + - "start of the RegExp?", - ); - } - - let parsed = rule.parse(capture, nestedParse, state); - - // We maintain the same object here so that rules can - // store references to the objects they return and - // modify them later. (oops sorry! but this adds a lot - // of power--see reflinks.) - - // We also let rules override the default type of - // their parsed node if they would like to, so that - // there can be a single output function for all links, - // even if there are several rules to parse them. - if (!parsed.type) { - parsed.type = ruleType; - } - - // Collapse text nodes - if ( - parsed.type === "text" && result[result.length - 1]?.type === "text" - ) { - result[result.length - 1].content += parsed.content; - } else { - result.push(parsed as ASTNode); - } - - state.prevCapture = capture; - source = source.substring(state.prevCapture[0].length); - } - - return result; - }; - - let outerParse = function ( - source: string, - state: ParserState = { inline: false }, - ) { - latestState = populateInitialState(state, defaultState); - - if (!latestState.inline && !latestState.disableAutoBlockNewlines) { - source = source + "\n\n"; - } - - // We store the previous capture so that match functions can - // use some limited amount of lookbehind. Lists use this to - // ensure they don't match arbitrary '- ' or '* ' in inline - // text (see the list rule for more information). This stores - // the full regex capture object, if there is one. - latestState.prevCapture = undefined; - return nestedParse(preprocess(source), latestState); - }; - - return outerParse; -} - -type Multiple = T | T[]; -type Nullable = T | null | undefined; - -export type MatchFunction = ( - source: string, - state: ParserState, -) => Nullable; - -export type Parser = (source: string, state?: ParserState) => ASTNode[]; - -export type ParseFunction = ( - source: RegExpMatchArray, - nestedParse: Parser, - state: ParserState, -) => TypeOptionalASTNode; - -export type QualityFunction = ( - capture: RegExpMatchArray, - state: ParserState, -) => number; - -export interface ParserState { - inline: boolean; - prevCapture?: RegExpMatchArray; - [key: string]: any; -} - -export interface ASTNode { - type: string; - content?: ASTNode[] | string; - [key: string]: any; -} - -export type TypeOptionalASTNode = Omit & { type?: string }; - -export interface RefNode { - type: string; - content?: Multiple; - target?: string; - title?: string; - alt?: string; -} - -/** Creates a match function for an inline scoped element from a regex */ -export function inlineRegex(regex: RegExp): MatchFunction { - return (source, state) => { - if (state.inline) { - return regex.exec(source); - } else { - return null; - } - }; -} - -/** Creates a match function for a block scoped element from a regex */ -export function blockRegex(regex: RegExp): MatchFunction { - return (source, state) => { - if (state.inline) { - return null; - } else { - return regex.exec(source); - } - }; -} - -/** Creates a match function from a regex, ignoring block/inline scope */ -export function anyScopeRegex(regex: RegExp): MatchFunction { - return (source) => { - return regex.exec(source); - }; -} - -const UNESCAPE_URL_R = /\\([^0-9A-Za-z\s])/g; -export function unescapeUrl(rawUrlString: string) { - return rawUrlString.replace(UNESCAPE_URL_R, "$1"); -} - -/** - * Parse some content with the parser `parse`, with state.inline - * set to true. Useful for block elements; not generally necessary - * to be used by inline elements (where state.inline is already true. - */ -export function parseInline( - parse: Parser, - content: string, - state: ParserState, -) { - const isCurrentlyInline = state.inline || false; - state.inline = true; - const result = parse(content, state); - state.inline = isCurrentlyInline; - return result; -} - -export function parseBlock(parse: Parser, content: string, state: ParserState) { - const isCurrentlyInline = state.inline || false; - state.inline = false; - const result = parse(content + "\n\n", state); - state.inline = isCurrentlyInline; - return result; -} - -export function parseCaptureInline( - capture: RegExpMatchArray, - parse: Parser, - state: ParserState, -) { - return { - content: parseInline(parse, capture[1], state), - }; -} - -export function ignoreCapture() { - return {}; -} - -export function sanitizeUrl(url?: string) { - if (url == null) { - return null; - } - try { - const prot = new URL(url, "https://localhost").protocol; - if ( - prot.indexOf("javascript:") === 0 || - prot.indexOf("vbscript:") === 0 || - prot.indexOf("data:") === 0 - ) { - return null; - } - } catch (e) { - // invalid URLs should throw a TypeError - // see for instance: `new URL("");` - return null; - } - return url; -} - -const CR_NEWLINE_R = /\r\n?/g; -const TAB_R = /\t/g; -const FORMFEED_R = /\f/g; - -/** - * Turn various whitespace into easy-to-process whitespace - */ -function preprocess(source: string) { - return source.replace(CR_NEWLINE_R, "\n").replace(FORMFEED_R, "").replace( - TAB_R, - " ", - ); -} - -function populateInitialState( - givenState: Partial, - defaultState: Partial, -) { - let state = givenState || {}; - - for (let prop in defaultState) { - if (Object.prototype.hasOwnProperty.call(defaultState, prop)) { - state[prop] = defaultState[prop]; - } - } - - return state as ParserState; -} - -// recognize a `*` `-`, `+`, `1.`, `2.`... list bullet -const LIST_BULLET = "(?:[*+-]|\\d+\\.)"; - -// recognize the start of a list item: -// leading space plus a bullet plus a space (` * `) -const LIST_ITEM_PREFIX = "( *)(" + LIST_BULLET + ") +"; -const LIST_ITEM_PREFIX_R = new RegExp("^" + LIST_ITEM_PREFIX); - -// recognize an individual list item: -// * hi -// this is part of the same item -// -// as is this, which is a new paragraph in the same item -// -// * but this is not part of the same item -const LIST_ITEM_R = new RegExp( - LIST_ITEM_PREFIX + "[^\\n]*(?:\\n" + "(?!\\1" + LIST_BULLET + - " )[^\\n]*)*(\n|$)", - "gm", -); -const BLOCK_END_R = /\n{2,}$/; -const INLINE_CODE_ESCAPE_BACKTICKS_R = /^ (?= *`)|(` *) $/g; - -// recognize the end of a paragraph block inside a list item: -// two or more newlines at end end of the item -const LIST_BLOCK_END_R = BLOCK_END_R; -const LIST_ITEM_END_R = / *\n+$/; - -// check whether a list item has paragraphs: if it does, -// we leave the newlines at the end -const LIST_R = new RegExp( - "^( *)(" + - LIST_BULLET + - ") " + - "[\\s\\S]+?(?:\n{2,}(?! )" + - "(?!\\1" + - LIST_BULLET + - " )\\n*" + - // the \\s*$ here is so that we can parse the inside of nested - // lists, where our content might end before we receive two `\n`s - "|\\s*\n*$)", -); -const LIST_LOOKBEHIND_R = /(?:^|\n)( *)$/; - -const TABLES = (function () { - const TABLE_ROW_SEPARATOR_TRIM = /^ *\| *| *\| *$/g; - const TABLE_CELL_END_TRIM = / *$/; - const TABLE_RIGHT_ALIGN = /^ *-+: *$/; - const TABLE_CENTER_ALIGN = /^ *:-+: *$/; - const TABLE_LEFT_ALIGN = /^ *:-+ *$/; // TODO: This needs a real type - - const parseTableAlignCapture = (alignCapture: string) => { - if (TABLE_RIGHT_ALIGN.test(alignCapture)) { - return "right"; - } else if (TABLE_CENTER_ALIGN.test(alignCapture)) { - return "center"; - } else if (TABLE_LEFT_ALIGN.test(alignCapture)) { - return "left"; - } else { - return null; - } - }; - - const parseTableAlign = (source: string, trimEndSeparators: boolean) => { - if (trimEndSeparators) { - source = source.replace(TABLE_ROW_SEPARATOR_TRIM, ""); - } - - const alignText = source.trim().split("|"); - return alignText.map(parseTableAlignCapture); - }; - - const parseTableRow = ( - source: string, - parse: Parser, - state: ParserState, - trimEndSeparators: boolean, - ) => { - const prevInTable = state.inTable; - state.inTable = true; - const tableRow = parse(source.trim(), state); - state.inTable = prevInTable; - const cells: ASTNode[][] = [[]]; - tableRow.forEach(function (node, i) { - if (node.type === "tableSeparator") { - // Filter out empty table separators at the start/end: - if (!trimEndSeparators || (i !== 0 && i !== tableRow.length - 1)) { - // Split the current row: - cells.push([]); - } - } else { - if ( - typeof node.content === "string" && - (tableRow[i + 1] == null || tableRow[i + 1].type === "tableSeparator") - ) { - node.content = node.content.replace(TABLE_CELL_END_TRIM, ""); - } - - cells[cells.length - 1].push(node); - } - }); - return cells; - }; - - /** - * @param {string} source - * @param {SimpleMarkdown.Parser} parse - * @param {SimpleMarkdown.State} state - * @param {boolean} trimEndSeparators - * @returns {SimpleMarkdown.ASTNode[][]} - */ - const parseTableCells = function ( - source: string, - parse: Parser, - state: ParserState, - trimEndSeparators: boolean, - ) { - const rowsText = source.trim().split("\n"); - return rowsText.map(function (rowText) { - return parseTableRow(rowText, parse, state, trimEndSeparators); - }); - }; - - /** - * @param {boolean} trimEndSeparators - * @returns {SimpleMarkdown.SingleNodeParseFunction} - */ - const parseTable = function (trimEndSeparators: boolean) { - return function ( - capture: RegExpMatchArray, - parse: Parser, - state: ParserState, - ) { - state.inline = true; - const header = parseTableRow(capture[1], parse, state, trimEndSeparators); - const align = parseTableAlign(capture[2], trimEndSeparators); - const cells = parseTableCells( - capture[3], - parse, - state, - trimEndSeparators, - ); - state.inline = false; - return { - type: "table", - header: header, - align: align, - cells: cells, - }; - }; - }; - - return { - parseTable: parseTable(true), - parseNpTable: parseTable(false), - TABLE_REGEX: /^ *(\|.+)\n *\|( *[-:]+[-| :]*)\n((?: *\|.*(?:\n|$))*)\n*/, - NPTABLE_REGEX: - /^ *(\S.*\|.*)\n *([-:]+ *\|[-| :]*)\n((?:.*\|.*(?:\n|$))*)\n*/, - }; -})(); - -const LINK_INSIDE = "(?:\\[[^\\]]*\\]|[^\\[\\]]|\\](?=[^\\[]*\\]))*"; -const LINK_HREF_AND_TITLE = - "\\s*?(?:\\s+['\"]([\\s\\S]*?)['\"])?\\s*"; -const AUTOLINK_MAILTO_CHECK_R = /mailto:/i; - -function parseRef( - capture: RegExpMatchArray, - state: ParserState, - refNode: RefNode, -) { - const ref = (capture[2] || capture[1]).replace(/\s+/g, " ").toLowerCase(); - - // We store information about previously seen defs on - // state._defs (_ to deconflict with client-defined - // state). If the def for this reflink/refimage has - // already been seen, we can use its target/source - // and title here: - if (state._defs && state._defs[ref]) { - const def = state._defs[ref]; - - // `refNode` can be a link or an image. Both use - // target and title properties. - refNode.target = def.target; - refNode.title = def.title; - } - - // In case we haven't seen our def yet (or if someone - // overwrites that def later on), we add this node - // to the list of ref nodes for that def. Then, when - // we find the def, we can modify this link/image AST - // node :). - // I'm sorry. - state._refs = state._refs || {}; - state._refs[ref] = state._refs[ref] || []; - - state._refs[ref].push(refNode); - - return refNode; -} - -export const defaultRules = new RuleList(); -{ - defaultRules.add({ - name: "heading", - match: blockRegex(/^ *(#{1,6})([^\n]+?)#* *(?:\n *)+\n/), - parse: function (capture, parse, state) { - return { - level: capture[1].length, - content: parseInline(parse, capture[2].trim(), state), - }; - }, - }); - - defaultRules.add({ - name: "nptable", - match: blockRegex(TABLES.NPTABLE_REGEX), - parse: TABLES.parseNpTable, - }); - - defaultRules.add({ - name: "lheading", - match: blockRegex(/^([^\n]+)\n *(=|-){3,} *(?:\n *)+\n/), - parse(capture, parse, state) { - return { - type: "heading", - level: capture[2] === "=" ? 1 : 2, - content: parseInline(parse, capture[1], state), - }; - }, - }); - - defaultRules.add({ - name: "hr", - match: blockRegex(/^( *[-*_]){3,} *(?:\n *)+\n/), - parse: ignoreCapture, - }); - - defaultRules.add({ - name: "codeBlock", - match: blockRegex(/^(?: {4}[^\n]+\n*)+(?:\n *)+\n/), - parse(capture) { - const content = capture[0].replace(/^ {4}/gm, "").replace(/\n+$/, ""); - return { - lang: undefined, - content: content, - }; - }, - }); - - defaultRules.add({ - name: "fence", - match: blockRegex( - /^ *(`{3,}|~{3,}) *(?:(\S+) *)?\n([\s\S]+?)\n?\1 *(?:\n *)+\n/, - ), - parse(capture) { - return { - type: "codeBlock", - lang: capture[2] || undefined, - content: capture[3], - }; - }, - }); - - defaultRules.add({ - name: "blockQuote", - match: blockRegex(/^( *>[^\n]+(\n[^\n]+)*\n*)+\n{2,}/), - parse(capture, parse, state) { - const content = capture[0].replace(/^ *> ?/gm, ""); - return { - content: parse(content, state), - }; - }, - }); - - defaultRules.add({ - name: "list", - - match(source, state) { - // We only want to break into a list if we are at the start of a - // line. This is to avoid parsing "hi * there" with "* there" - // becoming a part of a list. - // You might wonder, "but that's inline, so of course it wouldn't - // start a list?". You would be correct! Except that some of our - // lists can be inline, because they might be inside another list, - // in which case we can parse with inline scope, but need to allow - // nested lists inside this inline scope. - const prevCaptureStr = state.prevCapture == null - ? "" - : state.prevCapture[0]; - const isStartOfLineCapture = LIST_LOOKBEHIND_R.exec(prevCaptureStr); - const isListBlock = state._list || !state.inline; - - if (isStartOfLineCapture && isListBlock) { - source = isStartOfLineCapture[1] + source; - return LIST_R.exec(source); - } else { - return null; - } - }, - parse(capture, parse, state) { - const bullet = capture[2]; - const ordered = bullet.length > 1; - const start = ordered ? +bullet : undefined; - // We know this will match here, because of how the regexes are defined - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - const items = capture[0].replace(LIST_BLOCK_END_R, "\n").match( - LIST_ITEM_R, - )!; - - let lastItemWasAParagraph = false; - const itemContent = items.map(function (item, i) { - // We need to see how far indented this item is: - const prefixCapture = LIST_ITEM_PREFIX_R.exec(item); - const space = prefixCapture ? prefixCapture[0].length : 0; // And then we construct a regex to "unindent" the subsequent - // lines of the items by that amount: - - const spaceRegex = new RegExp("^ {1," + space + "}", "gm"); // Before processing the item, we need a couple things - - const content = item // remove indents on trailing lines: - .replace(spaceRegex, "") // remove the bullet: - .replace(LIST_ITEM_PREFIX_R, ""); // I'm not sur4 why this is necessary again? - // Handling "loose" lists, like: - // - // * this is wrapped in a paragraph - // - // * as is this - // - // * as is this - - const isLastItem = i === items.length - 1; - const containsBlocks = content.indexOf("\n\n") !== -1; // Any element in a list is a block if it contains multiple - // newlines. The last element in the list can also be a block - // if the previous item in the list was a block (this is - // because non-last items in the list can end with \n\n, but - // the last item can't, so we just "inherit" this property - // from our previous element). - - const thisItemIsAParagraph = containsBlocks || - (isLastItem && lastItemWasAParagraph); - lastItemWasAParagraph = thisItemIsAParagraph; // backup our state for restoration afterwards. We're going to - // want to set state._list to true, and state.inline depending - // on our list's looseness. - - const oldStateInline = state.inline; - const oldStateList = state._list; - state._list = true; // Parse inline if we're in a tight list, or block if we're in - // a loose list. - - let adjustedContent; - - if (thisItemIsAParagraph) { - state.inline = false; - adjustedContent = content.replace(LIST_ITEM_END_R, "\n\n"); - } else { - state.inline = true; - adjustedContent = content.replace(LIST_ITEM_END_R, ""); - } - - const result = parse(adjustedContent, state); // Restore our state before returning - - state.inline = oldStateInline; - state._list = oldStateList; - return result; - }); - - return { - ordered: ordered, - start: start, - content: itemContent, - }; - }, - }); - - defaultRules.add({ - name: "def", - // TODO: This will match without a blank line before the next - // block element, which is inconsistent with most of the rest of - // simple-markdown. - match: blockRegex( - /^ *\[([^\]]+)\]: *]*)>?(?: +["(]([^\n]+)[")])? *\n(?: *\n)*/, - ), - parse(capture, parse, state) { - const def = capture[1].replace(/\s+/g, " ").toLowerCase(); - const target = capture[2]; - const title = capture[3]; - - // Look for previous links/images using this def - // If any links/images using this def have already been declared, - // they will have added themselves to the state._refs[def] list - // (_ to deconflict with client-defined state). We look through - // that list of reflinks for this def, and modify those AST nodes - // with our newly found information now. - // Sorry :(. - if (state._refs && state._refs[def]) { - // `refNode` can be a link or an image - state._refs[def].forEach((refNode: RefNode) => { - refNode.target = target; - refNode.title = title; - }); - } - - // Add this def to our map of defs for any future links/images - // In case we haven't found any or all of the refs referring to - // this def yet, we add our def to the table of known defs, so - // that future reflinks can modify themselves appropriately with - // this information. - state._defs = state._defs || {}; - state._defs[def] = { - target: target, - title: title, - }; - - // return the relevant parsed information - // for debugging only. - return { - def: def, - target: target, - title: title, - }; - }, - }); - - defaultRules.add({ - name: "table", - match: blockRegex(TABLES.TABLE_REGEX), - parse: TABLES.parseTable, - }); - - defaultRules.add({ - name: "newline", - match: blockRegex(/^(?:\n *)*\n/), - parse: ignoreCapture, - }); - - defaultRules.add({ - name: "paragraph", - match: blockRegex(/^((?:[^\n]|\n(?! *\n))+)(?:\n *)+\n/), - parse: parseCaptureInline, - }); - - defaultRules.add({ - name: "escape", - // We don't allow escaping numbers, letters, or spaces here so that - // backslashes used in plain text still get rendered. But allowing - // escaping anything else provides a very flexible escape mechanism, - // regardless of how this grammar is extended. - match: inlineRegex(/^\\([^0-9A-Za-z\s])/), - parse(capture) { - return { - type: "text", - content: capture[1], - }; - }, - }); - - defaultRules.add({ - name: "tableSeparator", - - match(source, state) { - if (!state.inTable) { - return null; - } - - return /^ *\| */.exec(source); - }, - parse() { - return { - type: "tableSeparator", - }; - }, - }); - - defaultRules.add({ - name: "autolink", - match: inlineRegex(/^<([^: >]+:\/[^ >]+)>/), - parse(capture) { - return { - type: "link", - content: [ - { - type: "text", - content: capture[1], - }, - ], - target: capture[1], - }; - }, - }); - - defaultRules.add({ - name: "mailto", - match: inlineRegex(/^<([^ >]+@[^ >]+)>/), - parse(capture) { - const address = capture[1]; - let target = capture[1]; // Check for a `mailto:` already existing in the link: - - if (!AUTOLINK_MAILTO_CHECK_R.test(target)) { - target = "mailto:" + target; - } - - return { - type: "link", - content: [ - { - type: "text", - content: address, - }, - ], - target: target, - }; - }, - }); - - defaultRules.add({ - name: "url", - match: inlineRegex(/^(https?:\/\/[^\s<]+[^<.,:;"')\]\s])/), - parse(capture) { - return { - type: "link", - content: [ - { - type: "text", - content: capture[1], - }, - ], - target: capture[1], - title: undefined, - }; - }, - }); - - defaultRules.add({ - name: "link", - match: inlineRegex( - new RegExp( - "^\\[(" + LINK_INSIDE + ")\\]\\(" + LINK_HREF_AND_TITLE + "\\)", - ), - ), - parse(capture, parse, state) { - const link = { - content: parse(capture[1], state), - target: unescapeUrl(capture[2]), - title: capture[3], - }; - return link; - }, - }); - - defaultRules.add({ - name: "image", - match: inlineRegex( - new RegExp( - "^!\\[(" + LINK_INSIDE + ")\\]\\(" + LINK_HREF_AND_TITLE + "\\)", - ), - ), - parse: function (capture) { - const image = { - alt: capture[1], - target: unescapeUrl(capture[2]), - title: capture[3], - }; - return image; - }, - }); - - defaultRules.add({ - name: "reflink", - match: inlineRegex( - new RegExp( - // The first [part] of the link - "^\\[(" + - LINK_INSIDE + - ")\\]" + // The [ref] target of the link - "\\s*\\[([^\\]]*)\\]", - ), - ), - parse(capture, parse, state) { - return parseRef(capture, state, { - type: "link", - content: parse(capture[1], state), - }); - }, - }); - - defaultRules.add({ - name: "refimage", - match: inlineRegex( - new RegExp( - // The first [part] of the link - "^!\\[(" + - LINK_INSIDE + - ")\\]" + // The [ref] target of the link - "\\s*\\[([^\\]]*)\\]", - ), - ), - parse(capture, parse, state) { - return parseRef(capture, state, { - type: "image", - alt: capture[1], - }); - }, - }); - - defaultRules.add({ - name: "em", - /* same as strong/u */ - match: inlineRegex( - new RegExp( - // only match _s surrounding words. - "^\\b_" + - "((?:__|\\\\[\\s\\S]|[^\\\\_])+?)_" + - "\\b" + // Or match *s: - "|" + // Only match *s that are followed by a non-space: - "^\\*(?=\\S)(" + // Match at least one of: - "(?:" + // - `**`: so that bolds inside italics don't close the - // italics - "\\*\\*|" + // - escape sequence: so escaped *s don't close us - "\\\\[\\s\\S]|" + // - whitespace: followed by a non-* (we don't - // want ' *' to close an italics--it might - // start a list) - "\\s+(?:\\\\[\\s\\S]|[^\\s\\*\\\\]|\\*\\*)|" + // - non-whitespace, non-*, non-backslash characters - "[^\\s\\*\\\\]" + - ")+?" + // followed by a non-space, non-* then * - ")\\*(?!\\*)", - ), - ), - quality(capture) { - // precedence by length, `em` wins ties: - return capture[0].length + 0.2; - }, - parse(capture, parse, state) { - return { - content: parse(capture[2] || capture[1], state), - }; - }, - }); - - defaultRules.add({ - name: "strong", - /* same as em */ - match: inlineRegex(/^\*\*((?:\\[\s\S]|[^\\])+?)\*\*(?!\*)/), - quality(capture) { - // precedence by length, wins ties vs `u`: - return capture[0].length + 0.1; - }, - parse: parseCaptureInline, - }); - - defaultRules.add({ - name: "u", - /* same as em&strong; increment for next rule */ - match: inlineRegex(/^__((?:\\[\s\S]|[^\\])+?)__(?!_)/), - quality(capture) { - // precedence by length, loses all ties - return capture[0].length; - }, - parse: parseCaptureInline, - }); - - defaultRules.add({ - name: "del", - match: inlineRegex(/^~~(?=\S)((?:\\[\s\S]|~(?!~)|[^\s~\\]|\s(?!~~))+?)~~/), - parse: parseCaptureInline, - }); - - defaultRules.add({ - name: "inlineCode", - match: inlineRegex(/^(`+)([\s\S]*?[^`])\1(?!`)/), - parse(capture) { - return { - content: capture[2].replace(INLINE_CODE_ESCAPE_BACKTICKS_R, "$1"), - }; - }, - }); - - defaultRules.add({ - name: "br", - match: anyScopeRegex(/^ {2,}\n/), - parse: ignoreCapture, - }); - - defaultRules.add({ - name: "text", - // Here we look for anything followed by non-symbols, - // double newlines, or double-space-newlines - // We break on any symbol characters so that this grammar - // is easy to extend without needing to modify this regex - match: anyScopeRegex( - /^[\s\S]+?(?=[^0-9A-Za-z\s\u00c0-\uffff]|\n\n| {2,}\n|\w+:\S|$)/, - ), - parse: function (capture) { - return { - content: capture[0], - }; - }, - }); -} +// @ts-nocheck +// # Simple-Markdown Core +// +// This is a fork of Khan-academy's Simple-Markdown[1], initially forked in 2022 +// to add Svelte support[2], and used for paper clover's q+a markdown flavor. +// +// 1: https://github.com/Khan/perseus/tree/main/packages/simple-markdown/src +// 2: https://github.com/paperclover/svelte-simple-markdown +export type Rules = Record; +export interface ParserRule { + name: string; + match: MatchFunction; + parse: ParseFunction; + quality?: QualityFunction; +} + +export class RuleList extends Array { + constructor(input?: ArrayLike) { + super(); + if (input) { + this.push(...Array.from(input)); + } + } + + insertBefore(rule: string, newRule: ParserRule): void { + const index = this.findIndex((r) => r.name === rule); + if (index === -1) { + throw new Error(`Rule ${rule} not found`); + } + this.splice(index, 0, newRule); + } + + insertAfter(rule: string, newRule: ParserRule): void { + const index = this.findIndex((r) => r.name === rule); + if (index === -1) { + throw new Error(`Rule ${rule} not found`); + } + this.splice(index + 1, 0, newRule); + } + + toRuleObject(): Record { + const result: Record = {}; + this.forEach((rule) => { + result[rule.name] = rule; + }); + return result; + } + + add(rule: ParserRule): void { + this.push(rule); + } + + get(rule: string): ParserRule | undefined { + return this.find((r) => r.name === rule); + } + + remove(rule: string): void { + const index = this.findIndex((r) => r.name === rule); + if (index === -1) { + throw new Error(`Rule ${rule} not found`); + } + this.splice(index, 1); + } + + clone() { + return new RuleList(this); + } +} + +/** + * Creates a parser for a given set of rules, with the precedence + * specified as a list of rules. + * + * @param rules + * an object containing + * rule type -> {match, order, parse} objects + * (lower order is higher precedence) + * @param [defaultState] + * + * @returns + * The resulting parse function, with the following parameters: + * @source: the input source string to be parsed + * @state: an optional object to be threaded through parse + * calls. Allows clients to add stateful operations to + * parsing, such as keeping track of how many levels deep + * some nesting is. For an example use-case, see passage-ref + * parsing in src/widgets/passage/passage-markdown.jsx + */ +export function createParser( + ruleListInput: RuleList, + defaultState: Partial = {}, +) { + let rules = ruleListInput.toRuleObject(); + let ruleList = Object.keys(rules); + + let latestState: ParserState; + + let nestedParse = function (source: string, state?: ParserState) { + let result: ASTNode[] = []; + state = state || latestState; + latestState = state; + + while (source) { + // store the best match, it's rule, and quality: + let ruleType = null; + let rule = null; + let capture = null; + let quality = NaN; // loop control variables: + + let i = 0; + let currRuleType = ruleList[0]; + + let currRule = rules[currRuleType]; + + do { + let currCapture = currRule.match(source, state); + + if (currCapture) { + let currQuality = currRule.quality + ? currRule.quality(currCapture, state) + : 0; + + // This should always be true the first time because + // the initial quality is NaN (that's why there's the + // condition negation). + if (!(currQuality <= quality)) { + ruleType = currRuleType; + rule = currRule; + capture = currCapture; + quality = currQuality; + } + } + + // Move on to the next item. + // Note that this makes `currRule` be the next item + i++; + currRuleType = ruleList[i]; + currRule = rules[currRuleType]; + } while ( + // keep looping while we're still within the ruleList + currRule && + // if we don't have a match yet, continue + (!capture || + // or if we have a match, but the next rule is + // at the same order, and has a quality measurement + // functions, then this rule must have a quality + // measurement function (since they are sorted before + // those without), and we need to check if there is + // a better quality match + currRule.quality) + ); + + if (!rule || !capture || !ruleType) { + throw new Error( + "Could not find a matching rule for the below " + + "content. The rule with highest `order` should " + + "always match content provided to it. Check " + + "the definition of `match` for '" + + ruleList[ruleList.length - 1] + + "'. It seems to not match the following source:\n" + + source, + ); + } + + if (capture.index) { + // If present and non-zero, i.e. a non-^ regexp result: + throw new Error( + "`match` must return a capture starting at index 0 " + + "(the current parse index). Did you forget a ^ at the " + + "start of the RegExp?", + ); + } + + let parsed = rule.parse(capture, nestedParse, state); + + // We maintain the same object here so that rules can + // store references to the objects they return and + // modify them later. (oops sorry! but this adds a lot + // of power--see reflinks.) + + // We also let rules override the default type of + // their parsed node if they would like to, so that + // there can be a single output function for all links, + // even if there are several rules to parse them. + if (!parsed.type) { + parsed.type = ruleType; + } + + // Collapse text nodes + if ( + parsed.type === "text" && result[result.length - 1]?.type === "text" + ) { + result[result.length - 1].content += parsed.content; + } else { + result.push(parsed as ASTNode); + } + + state.prevCapture = capture; + source = source.substring(state.prevCapture[0].length); + } + + return result; + }; + + let outerParse = function ( + source: string, + state: ParserState = { inline: false }, + ) { + latestState = populateInitialState(state, defaultState); + + if (!latestState.inline && !latestState.disableAutoBlockNewlines) { + source = source + "\n\n"; + } + + // We store the previous capture so that match functions can + // use some limited amount of lookbehind. Lists use this to + // ensure they don't match arbitrary '- ' or '* ' in inline + // text (see the list rule for more information). This stores + // the full regex capture object, if there is one. + latestState.prevCapture = undefined; + return nestedParse(preprocess(source), latestState); + }; + + return outerParse; +} + +type Multiple = T | T[]; +type Nullable = T | null | undefined; + +export type MatchFunction = ( + source: string, + state: ParserState, +) => Nullable; + +export type Parser = (source: string, state?: ParserState) => ASTNode[]; + +export type ParseFunction = ( + source: RegExpMatchArray, + nestedParse: Parser, + state: ParserState, +) => TypeOptionalASTNode; + +export type QualityFunction = ( + capture: RegExpMatchArray, + state: ParserState, +) => number; + +export interface ParserState { + inline: boolean; + prevCapture?: RegExpMatchArray; + [key: string]: any; +} + +export interface ASTNode { + type: string; + content?: ASTNode[] | string; + [key: string]: any; +} + +export type TypeOptionalASTNode = Omit & { type?: string }; + +export interface RefNode { + type: string; + content?: Multiple; + target?: string; + title?: string; + alt?: string; +} + +/** Creates a match function for an inline scoped element from a regex */ +export function inlineRegex(regex: RegExp): MatchFunction { + return (source, state) => { + if (state.inline) { + return regex.exec(source); + } else { + return null; + } + }; +} + +/** Creates a match function for a block scoped element from a regex */ +export function blockRegex(regex: RegExp): MatchFunction { + return (source, state) => { + if (state.inline) { + return null; + } else { + return regex.exec(source); + } + }; +} + +/** Creates a match function from a regex, ignoring block/inline scope */ +export function anyScopeRegex(regex: RegExp): MatchFunction { + return (source) => { + return regex.exec(source); + }; +} + +const UNESCAPE_URL_R = /\\([^0-9A-Za-z\s])/g; +export function unescapeUrl(rawUrlString: string) { + return rawUrlString.replace(UNESCAPE_URL_R, "$1"); +} + +/** + * Parse some content with the parser `parse`, with state.inline + * set to true. Useful for block elements; not generally necessary + * to be used by inline elements (where state.inline is already true. + */ +export function parseInline( + parse: Parser, + content: string, + state: ParserState, +) { + const isCurrentlyInline = state.inline || false; + state.inline = true; + const result = parse(content, state); + state.inline = isCurrentlyInline; + return result; +} + +export function parseBlock(parse: Parser, content: string, state: ParserState) { + const isCurrentlyInline = state.inline || false; + state.inline = false; + const result = parse(content + "\n\n", state); + state.inline = isCurrentlyInline; + return result; +} + +export function parseCaptureInline( + capture: RegExpMatchArray, + parse: Parser, + state: ParserState, +) { + return { + content: parseInline(parse, capture[1], state), + }; +} + +export function ignoreCapture() { + return {}; +} + +export function sanitizeUrl(url?: string) { + if (url == null) { + return null; + } + try { + const prot = new URL(url, "https://localhost").protocol; + if ( + prot.indexOf("javascript:") === 0 || + prot.indexOf("vbscript:") === 0 || + prot.indexOf("data:") === 0 + ) { + return null; + } + } catch (e) { + // invalid URLs should throw a TypeError + // see for instance: `new URL("");` + return null; + } + return url; +} + +const CR_NEWLINE_R = /\r\n?/g; +const TAB_R = /\t/g; +const FORMFEED_R = /\f/g; + +/** + * Turn various whitespace into easy-to-process whitespace + */ +function preprocess(source: string) { + return source.replace(CR_NEWLINE_R, "\n").replace(FORMFEED_R, "").replace( + TAB_R, + " ", + ); +} + +function populateInitialState( + givenState: Partial, + defaultState: Partial, +) { + let state = givenState || {}; + + for (let prop in defaultState) { + if (Object.prototype.hasOwnProperty.call(defaultState, prop)) { + state[prop] = defaultState[prop]; + } + } + + return state as ParserState; +} + +// recognize a `*` `-`, `+`, `1.`, `2.`... list bullet +const LIST_BULLET = "(?:[*+-]|\\d+\\.)"; + +// recognize the start of a list item: +// leading space plus a bullet plus a space (` * `) +const LIST_ITEM_PREFIX = "( *)(" + LIST_BULLET + ") +"; +const LIST_ITEM_PREFIX_R = new RegExp("^" + LIST_ITEM_PREFIX); + +// recognize an individual list item: +// * hi +// this is part of the same item +// +// as is this, which is a new paragraph in the same item +// +// * but this is not part of the same item +const LIST_ITEM_R = new RegExp( + LIST_ITEM_PREFIX + "[^\\n]*(?:\\n" + "(?!\\1" + LIST_BULLET + + " )[^\\n]*)*(\n|$)", + "gm", +); +const BLOCK_END_R = /\n{2,}$/; +const INLINE_CODE_ESCAPE_BACKTICKS_R = /^ (?= *`)|(` *) $/g; + +// recognize the end of a paragraph block inside a list item: +// two or more newlines at end end of the item +const LIST_BLOCK_END_R = BLOCK_END_R; +const LIST_ITEM_END_R = / *\n+$/; + +// check whether a list item has paragraphs: if it does, +// we leave the newlines at the end +const LIST_R = new RegExp( + "^( *)(" + + LIST_BULLET + + ") " + + "[\\s\\S]+?(?:\n{2,}(?! )" + + "(?!\\1" + + LIST_BULLET + + " )\\n*" + + // the \\s*$ here is so that we can parse the inside of nested + // lists, where our content might end before we receive two `\n`s + "|\\s*\n*$)", +); +const LIST_LOOKBEHIND_R = /(?:^|\n)( *)$/; + +const TABLES = (function () { + const TABLE_ROW_SEPARATOR_TRIM = /^ *\| *| *\| *$/g; + const TABLE_CELL_END_TRIM = / *$/; + const TABLE_RIGHT_ALIGN = /^ *-+: *$/; + const TABLE_CENTER_ALIGN = /^ *:-+: *$/; + const TABLE_LEFT_ALIGN = /^ *:-+ *$/; // TODO: This needs a real type + + const parseTableAlignCapture = (alignCapture: string) => { + if (TABLE_RIGHT_ALIGN.test(alignCapture)) { + return "right"; + } else if (TABLE_CENTER_ALIGN.test(alignCapture)) { + return "center"; + } else if (TABLE_LEFT_ALIGN.test(alignCapture)) { + return "left"; + } else { + return null; + } + }; + + const parseTableAlign = (source: string, trimEndSeparators: boolean) => { + if (trimEndSeparators) { + source = source.replace(TABLE_ROW_SEPARATOR_TRIM, ""); + } + + const alignText = source.trim().split("|"); + return alignText.map(parseTableAlignCapture); + }; + + const parseTableRow = ( + source: string, + parse: Parser, + state: ParserState, + trimEndSeparators: boolean, + ) => { + const prevInTable = state.inTable; + state.inTable = true; + const tableRow = parse(source.trim(), state); + state.inTable = prevInTable; + const cells: ASTNode[][] = [[]]; + tableRow.forEach(function (node, i) { + if (node.type === "tableSeparator") { + // Filter out empty table separators at the start/end: + if (!trimEndSeparators || (i !== 0 && i !== tableRow.length - 1)) { + // Split the current row: + cells.push([]); + } + } else { + if ( + typeof node.content === "string" && + (tableRow[i + 1] == null || tableRow[i + 1].type === "tableSeparator") + ) { + node.content = node.content.replace(TABLE_CELL_END_TRIM, ""); + } + + cells[cells.length - 1].push(node); + } + }); + return cells; + }; + + /** + * @param {string} source + * @param {SimpleMarkdown.Parser} parse + * @param {SimpleMarkdown.State} state + * @param {boolean} trimEndSeparators + * @returns {SimpleMarkdown.ASTNode[][]} + */ + const parseTableCells = function ( + source: string, + parse: Parser, + state: ParserState, + trimEndSeparators: boolean, + ) { + const rowsText = source.trim().split("\n"); + return rowsText.map(function (rowText) { + return parseTableRow(rowText, parse, state, trimEndSeparators); + }); + }; + + /** + * @param {boolean} trimEndSeparators + * @returns {SimpleMarkdown.SingleNodeParseFunction} + */ + const parseTable = function (trimEndSeparators: boolean) { + return function ( + capture: RegExpMatchArray, + parse: Parser, + state: ParserState, + ) { + state.inline = true; + const header = parseTableRow(capture[1], parse, state, trimEndSeparators); + const align = parseTableAlign(capture[2], trimEndSeparators); + const cells = parseTableCells( + capture[3], + parse, + state, + trimEndSeparators, + ); + state.inline = false; + return { + type: "table", + header: header, + align: align, + cells: cells, + }; + }; + }; + + return { + parseTable: parseTable(true), + parseNpTable: parseTable(false), + TABLE_REGEX: /^ *(\|.+)\n *\|( *[-:]+[-| :]*)\n((?: *\|.*(?:\n|$))*)\n*/, + NPTABLE_REGEX: + /^ *(\S.*\|.*)\n *([-:]+ *\|[-| :]*)\n((?:.*\|.*(?:\n|$))*)\n*/, + }; +})(); + +const LINK_INSIDE = "(?:\\[[^\\]]*\\]|[^\\[\\]]|\\](?=[^\\[]*\\]))*"; +const LINK_HREF_AND_TITLE = + "\\s*?(?:\\s+['\"]([\\s\\S]*?)['\"])?\\s*"; +const AUTOLINK_MAILTO_CHECK_R = /mailto:/i; + +function parseRef( + capture: RegExpMatchArray, + state: ParserState, + refNode: RefNode, +) { + const ref = (capture[2] || capture[1]).replace(/\s+/g, " ").toLowerCase(); + + // We store information about previously seen defs on + // state._defs (_ to deconflict with client-defined + // state). If the def for this reflink/refimage has + // already been seen, we can use its target/source + // and title here: + if (state._defs && state._defs[ref]) { + const def = state._defs[ref]; + + // `refNode` can be a link or an image. Both use + // target and title properties. + refNode.target = def.target; + refNode.title = def.title; + } + + // In case we haven't seen our def yet (or if someone + // overwrites that def later on), we add this node + // to the list of ref nodes for that def. Then, when + // we find the def, we can modify this link/image AST + // node :). + // I'm sorry. + state._refs = state._refs || {}; + state._refs[ref] = state._refs[ref] || []; + + state._refs[ref].push(refNode); + + return refNode; +} + +export const defaultRules = new RuleList(); +{ + defaultRules.add({ + name: "heading", + match: blockRegex(/^ *(#{1,6})([^\n]+?)#* *(?:\n *)+\n/), + parse: function (capture, parse, state) { + return { + level: capture[1].length, + content: parseInline(parse, capture[2].trim(), state), + }; + }, + }); + + defaultRules.add({ + name: "nptable", + match: blockRegex(TABLES.NPTABLE_REGEX), + parse: TABLES.parseNpTable, + }); + + defaultRules.add({ + name: "lheading", + match: blockRegex(/^([^\n]+)\n *(=|-){3,} *(?:\n *)+\n/), + parse(capture, parse, state) { + return { + type: "heading", + level: capture[2] === "=" ? 1 : 2, + content: parseInline(parse, capture[1], state), + }; + }, + }); + + defaultRules.add({ + name: "hr", + match: blockRegex(/^( *[-*_]){3,} *(?:\n *)+\n/), + parse: ignoreCapture, + }); + + defaultRules.add({ + name: "codeBlock", + match: blockRegex(/^(?: {4}[^\n]+\n*)+(?:\n *)+\n/), + parse(capture) { + const content = capture[0].replace(/^ {4}/gm, "").replace(/\n+$/, ""); + return { + lang: undefined, + content: content, + }; + }, + }); + + defaultRules.add({ + name: "fence", + match: blockRegex( + /^ *(`{3,}|~{3,}) *(?:(\S+) *)?\n([\s\S]+?)\n?\1 *(?:\n *)+\n/, + ), + parse(capture) { + return { + type: "codeBlock", + lang: capture[2] || undefined, + content: capture[3], + }; + }, + }); + + defaultRules.add({ + name: "blockQuote", + match: blockRegex(/^( *>[^\n]+(\n[^\n]+)*\n*)+\n{2,}/), + parse(capture, parse, state) { + const content = capture[0].replace(/^ *> ?/gm, ""); + return { + content: parse(content, state), + }; + }, + }); + + defaultRules.add({ + name: "list", + + match(source, state) { + // We only want to break into a list if we are at the start of a + // line. This is to avoid parsing "hi * there" with "* there" + // becoming a part of a list. + // You might wonder, "but that's inline, so of course it wouldn't + // start a list?". You would be correct! Except that some of our + // lists can be inline, because they might be inside another list, + // in which case we can parse with inline scope, but need to allow + // nested lists inside this inline scope. + const prevCaptureStr = state.prevCapture == null + ? "" + : state.prevCapture[0]; + const isStartOfLineCapture = LIST_LOOKBEHIND_R.exec(prevCaptureStr); + const isListBlock = state._list || !state.inline; + + if (isStartOfLineCapture && isListBlock) { + source = isStartOfLineCapture[1] + source; + return LIST_R.exec(source); + } else { + return null; + } + }, + parse(capture, parse, state) { + const bullet = capture[2]; + const ordered = bullet.length > 1; + const start = ordered ? +bullet : undefined; + // We know this will match here, because of how the regexes are defined + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + const items = capture[0].replace(LIST_BLOCK_END_R, "\n").match( + LIST_ITEM_R, + )!; + + let lastItemWasAParagraph = false; + const itemContent = items.map(function (item, i) { + // We need to see how far indented this item is: + const prefixCapture = LIST_ITEM_PREFIX_R.exec(item); + const space = prefixCapture ? prefixCapture[0].length : 0; // And then we construct a regex to "unindent" the subsequent + // lines of the items by that amount: + + const spaceRegex = new RegExp("^ {1," + space + "}", "gm"); // Before processing the item, we need a couple things + + const content = item // remove indents on trailing lines: + .replace(spaceRegex, "") // remove the bullet: + .replace(LIST_ITEM_PREFIX_R, ""); // I'm not sur4 why this is necessary again? + // Handling "loose" lists, like: + // + // * this is wrapped in a paragraph + // + // * as is this + // + // * as is this + + const isLastItem = i === items.length - 1; + const containsBlocks = content.indexOf("\n\n") !== -1; // Any element in a list is a block if it contains multiple + // newlines. The last element in the list can also be a block + // if the previous item in the list was a block (this is + // because non-last items in the list can end with \n\n, but + // the last item can't, so we just "inherit" this property + // from our previous element). + + const thisItemIsAParagraph = containsBlocks || + (isLastItem && lastItemWasAParagraph); + lastItemWasAParagraph = thisItemIsAParagraph; // backup our state for restoration afterwards. We're going to + // want to set state._list to true, and state.inline depending + // on our list's looseness. + + const oldStateInline = state.inline; + const oldStateList = state._list; + state._list = true; // Parse inline if we're in a tight list, or block if we're in + // a loose list. + + let adjustedContent; + + if (thisItemIsAParagraph) { + state.inline = false; + adjustedContent = content.replace(LIST_ITEM_END_R, "\n\n"); + } else { + state.inline = true; + adjustedContent = content.replace(LIST_ITEM_END_R, ""); + } + + const result = parse(adjustedContent, state); // Restore our state before returning + + state.inline = oldStateInline; + state._list = oldStateList; + return result; + }); + + return { + ordered: ordered, + start: start, + content: itemContent, + }; + }, + }); + + defaultRules.add({ + name: "def", + // TODO: This will match without a blank line before the next + // block element, which is inconsistent with most of the rest of + // simple-markdown. + match: blockRegex( + /^ *\[([^\]]+)\]: *]*)>?(?: +["(]([^\n]+)[")])? *\n(?: *\n)*/, + ), + parse(capture, parse, state) { + const def = capture[1].replace(/\s+/g, " ").toLowerCase(); + const target = capture[2]; + const title = capture[3]; + + // Look for previous links/images using this def + // If any links/images using this def have already been declared, + // they will have added themselves to the state._refs[def] list + // (_ to deconflict with client-defined state). We look through + // that list of reflinks for this def, and modify those AST nodes + // with our newly found information now. + // Sorry :(. + if (state._refs && state._refs[def]) { + // `refNode` can be a link or an image + state._refs[def].forEach((refNode: RefNode) => { + refNode.target = target; + refNode.title = title; + }); + } + + // Add this def to our map of defs for any future links/images + // In case we haven't found any or all of the refs referring to + // this def yet, we add our def to the table of known defs, so + // that future reflinks can modify themselves appropriately with + // this information. + state._defs = state._defs || {}; + state._defs[def] = { + target: target, + title: title, + }; + + // return the relevant parsed information + // for debugging only. + return { + def: def, + target: target, + title: title, + }; + }, + }); + + defaultRules.add({ + name: "table", + match: blockRegex(TABLES.TABLE_REGEX), + parse: TABLES.parseTable, + }); + + defaultRules.add({ + name: "newline", + match: blockRegex(/^(?:\n *)*\n/), + parse: ignoreCapture, + }); + + defaultRules.add({ + name: "paragraph", + match: blockRegex(/^((?:[^\n]|\n(?! *\n))+)(?:\n *)+\n/), + parse: parseCaptureInline, + }); + + defaultRules.add({ + name: "escape", + // We don't allow escaping numbers, letters, or spaces here so that + // backslashes used in plain text still get rendered. But allowing + // escaping anything else provides a very flexible escape mechanism, + // regardless of how this grammar is extended. + match: inlineRegex(/^\\([^0-9A-Za-z\s])/), + parse(capture) { + return { + type: "text", + content: capture[1], + }; + }, + }); + + defaultRules.add({ + name: "tableSeparator", + + match(source, state) { + if (!state.inTable) { + return null; + } + + return /^ *\| */.exec(source); + }, + parse() { + return { + type: "tableSeparator", + }; + }, + }); + + defaultRules.add({ + name: "autolink", + match: inlineRegex(/^<([^: >]+:\/[^ >]+)>/), + parse(capture) { + return { + type: "link", + content: [ + { + type: "text", + content: capture[1], + }, + ], + target: capture[1], + }; + }, + }); + + defaultRules.add({ + name: "mailto", + match: inlineRegex(/^<([^ >]+@[^ >]+)>/), + parse(capture) { + const address = capture[1]; + let target = capture[1]; // Check for a `mailto:` already existing in the link: + + if (!AUTOLINK_MAILTO_CHECK_R.test(target)) { + target = "mailto:" + target; + } + + return { + type: "link", + content: [ + { + type: "text", + content: address, + }, + ], + target: target, + }; + }, + }); + + defaultRules.add({ + name: "url", + match: inlineRegex(/^(https?:\/\/[^\s<]+[^<.,:;"')\]\s])/), + parse(capture) { + return { + type: "link", + content: [ + { + type: "text", + content: capture[1], + }, + ], + target: capture[1], + title: undefined, + }; + }, + }); + + defaultRules.add({ + name: "link", + match: inlineRegex( + new RegExp( + "^\\[(" + LINK_INSIDE + ")\\]\\(" + LINK_HREF_AND_TITLE + "\\)", + ), + ), + parse(capture, parse, state) { + const link = { + content: parse(capture[1], state), + target: unescapeUrl(capture[2]), + title: capture[3], + }; + return link; + }, + }); + + defaultRules.add({ + name: "image", + match: inlineRegex( + new RegExp( + "^!\\[(" + LINK_INSIDE + ")\\]\\(" + LINK_HREF_AND_TITLE + "\\)", + ), + ), + parse: function (capture) { + const image = { + alt: capture[1], + target: unescapeUrl(capture[2]), + title: capture[3], + }; + return image; + }, + }); + + defaultRules.add({ + name: "reflink", + match: inlineRegex( + new RegExp( + // The first [part] of the link + "^\\[(" + + LINK_INSIDE + + ")\\]" + // The [ref] target of the link + "\\s*\\[([^\\]]*)\\]", + ), + ), + parse(capture, parse, state) { + return parseRef(capture, state, { + type: "link", + content: parse(capture[1], state), + }); + }, + }); + + defaultRules.add({ + name: "refimage", + match: inlineRegex( + new RegExp( + // The first [part] of the link + "^!\\[(" + + LINK_INSIDE + + ")\\]" + // The [ref] target of the link + "\\s*\\[([^\\]]*)\\]", + ), + ), + parse(capture, parse, state) { + return parseRef(capture, state, { + type: "image", + alt: capture[1], + }); + }, + }); + + defaultRules.add({ + name: "em", + /* same as strong/u */ + match: inlineRegex( + new RegExp( + // only match _s surrounding words. + "^\\b_" + + "((?:__|\\\\[\\s\\S]|[^\\\\_])+?)_" + + "\\b" + // Or match *s: + "|" + // Only match *s that are followed by a non-space: + "^\\*(?=\\S)(" + // Match at least one of: + "(?:" + // - `**`: so that bolds inside italics don't close the + // italics + "\\*\\*|" + // - escape sequence: so escaped *s don't close us + "\\\\[\\s\\S]|" + // - whitespace: followed by a non-* (we don't + // want ' *' to close an italics--it might + // start a list) + "\\s+(?:\\\\[\\s\\S]|[^\\s\\*\\\\]|\\*\\*)|" + // - non-whitespace, non-*, non-backslash characters + "[^\\s\\*\\\\]" + + ")+?" + // followed by a non-space, non-* then * + ")\\*(?!\\*)", + ), + ), + quality(capture) { + // precedence by length, `em` wins ties: + return capture[0].length + 0.2; + }, + parse(capture, parse, state) { + return { + content: parse(capture[2] || capture[1], state), + }; + }, + }); + + defaultRules.add({ + name: "strong", + /* same as em */ + match: inlineRegex(/^\*\*((?:\\[\s\S]|[^\\])+?)\*\*(?!\*)/), + quality(capture) { + // precedence by length, wins ties vs `u`: + return capture[0].length + 0.1; + }, + parse: parseCaptureInline, + }); + + defaultRules.add({ + name: "u", + /* same as em&strong; increment for next rule */ + match: inlineRegex(/^__((?:\\[\s\S]|[^\\])+?)__(?!_)/), + quality(capture) { + // precedence by length, loses all ties + return capture[0].length; + }, + parse: parseCaptureInline, + }); + + defaultRules.add({ + name: "del", + match: inlineRegex(/^~~(?=\S)((?:\\[\s\S]|~(?!~)|[^\s~\\]|\s(?!~~))+?)~~/), + parse: parseCaptureInline, + }); + + defaultRules.add({ + name: "inlineCode", + match: inlineRegex(/^(`+)([\s\S]*?[^`])\1(?!`)/), + parse(capture) { + return { + content: capture[2].replace(INLINE_CODE_ESCAPE_BACKTICKS_R, "$1"), + }; + }, + }); + + defaultRules.add({ + name: "br", + match: anyScopeRegex(/^ {2,}\n/), + parse: ignoreCapture, + }); + + defaultRules.add({ + name: "text", + // Here we look for anything followed by non-symbols, + // double newlines, or double-space-newlines + // We break on any symbol characters so that this grammar + // is easy to extend without needing to modify this regex + match: anyScopeRegex( + /^[\s\S]+?(?=[^0-9A-Za-z\s\u00c0-\uffff]|\n\n| {2,}\n|\w+:\S|$)/, + ), + parse: function (capture) { + return { + content: capture[0], + }; + }, + }); +} diff --git a/src/q+a/views/backend-inbox.client.ts b/src/q+a/views/backend-inbox.client.ts index b6905df..00f9b2e 100644 --- a/src/q+a/views/backend-inbox.client.ts +++ b/src/q+a/views/backend-inbox.client.ts @@ -1,74 +1,74 @@ -// @ts-ignore -globalThis.onReply = (id: string) => { - location.href = `/admin/q+a/${id}?return=inbox`; -}; -// @ts-ignore -globalThis.onDelete = async (id: string) => { - const div = document.querySelector(`[data-q="${id}"]`) as HTMLDivElement; - if (!div) return alert("Question not found"); - - // Pending State - div.style.opacity = "0.5"; - div.style.pointerEvents = "none"; - div?.querySelectorAll("button").forEach((b) => { - b.disabled = true; - }); - - try { - const resp = await fetch(`/admin/q+a/${id}`, { - method: "DELETE", - headers: { - Accept: "application/json", - }, - }); - if (resp.status !== 200) { - throw new Error("Failed to delete question, status: " + resp.status); - } - } catch (e: any) { - div.style.opacity = "1"; - div.style.pointerEvents = "auto"; - div?.querySelectorAll("button").forEach((b) => { - b.disabled = false; - }); - return alert(e.message); - } - - div.remove(); -}; -// @ts-ignore -globalThis.onDeleteFull = async (id: string) => { - const div = document.querySelector(`[data-q="${id}"]`) as HTMLDivElement; - if (!div) return alert("Question not found"); - - // Confirmation - if (!confirm("Are you sure you want to delete this question?")) return; - - // Pending State - div.style.opacity = "0.5"; - div.style.pointerEvents = "none"; - div?.querySelectorAll("button").forEach((b) => { - b.disabled = true; - }); - - try { - const resp = await fetch(`/admin/q+a/${id}`, { - method: "DELETE", - headers: { - Accept: "application/json", - "X-Delete-Full": "true", - }, - }); - if (resp.status !== 200) { - throw new Error("Failed to delete question, status: " + resp.status); - } - } catch (e: any) { - div.style.opacity = "1"; - div.style.pointerEvents = "auto"; - div?.querySelectorAll("button").forEach((b) => { - b.disabled = false; - }); - return alert(e.message); - } - - div.remove(); -}; +// @ts-ignore +globalThis.onReply = (id: string) => { + location.href = `/admin/q+a/${id}?return=inbox`; +}; +// @ts-ignore +globalThis.onDelete = async (id: string) => { + const div = document.querySelector(`[data-q="${id}"]`) as HTMLDivElement; + if (!div) return alert("Question not found"); + + // Pending State + div.style.opacity = "0.5"; + div.style.pointerEvents = "none"; + div?.querySelectorAll("button").forEach((b) => { + b.disabled = true; + }); + + try { + const resp = await fetch(`/admin/q+a/${id}`, { + method: "DELETE", + headers: { + Accept: "application/json", + }, + }); + if (resp.status !== 200) { + throw new Error("Failed to delete question, status: " + resp.status); + } + } catch (e: any) { + div.style.opacity = "1"; + div.style.pointerEvents = "auto"; + div?.querySelectorAll("button").forEach((b) => { + b.disabled = false; + }); + return alert(e.message); + } + + div.remove(); +}; +// @ts-ignore +globalThis.onDeleteFull = async (id: string) => { + const div = document.querySelector(`[data-q="${id}"]`) as HTMLDivElement; + if (!div) return alert("Question not found"); + + // Confirmation + if (!confirm("Are you sure you want to delete this question?")) return; + + // Pending State + div.style.opacity = "0.5"; + div.style.pointerEvents = "none"; + div?.querySelectorAll("button").forEach((b) => { + b.disabled = true; + }); + + try { + const resp = await fetch(`/admin/q+a/${id}`, { + method: "DELETE", + headers: { + Accept: "application/json", + "X-Delete-Full": "true", + }, + }); + if (resp.status !== 200) { + throw new Error("Failed to delete question, status: " + resp.status); + } + } catch (e: any) { + div.style.opacity = "1"; + div.style.pointerEvents = "auto"; + div?.querySelectorAll("button").forEach((b) => { + b.disabled = false; + }); + return alert(e.message); + } + + div.remove(); +}; diff --git a/src/q+a/views/editor.css b/src/q+a/views/editor.css index affd003..105f582 100644 --- a/src/q+a/views/editor.css +++ b/src/q+a/views/editor.css @@ -1,39 +1,39 @@ -#edit-grid { - display: grid; - grid-template-columns: 1fr 80ch; - grid-template-rows: 3rem 1fr; - grid-gap: 1em; - height: 100vh; - - button { - margin-right: 1rem; - } - main { - padding: 0; - margin: 0; - } -} -#topleft, #topright { - padding: 1rem; -} -#topleft { - padding-right: 0rem; -} -#topright { - padding-left: 0rem; -} -#preview { - overflow-y: auto; - e- { - margin-top: 0; - } -} -#editor { - background-color: #303030; - overflow-y: scroll; - height: 100%; -} -.cm-scroller { - overflow-y: auto; - height: 100%; -} +#edit-grid { + display: grid; + grid-template-columns: 1fr 80ch; + grid-template-rows: 3rem 1fr; + grid-gap: 1em; + height: 100vh; + + button { + margin-right: 1rem; + } + main { + padding: 0; + margin: 0; + } +} +#topleft, #topright { + padding: 1rem; +} +#topleft { + padding-right: 0rem; +} +#topright { + padding-left: 0rem; +} +#preview { + overflow-y: auto; + e- { + margin-top: 0; + } +} +#editor { + background-color: #303030; + overflow-y: scroll; + height: 100%; +} +.cm-scroller { + overflow-y: auto; + height: 100%; +} diff --git a/src/static/wurtzhome.html b/src/static/wurtzhome.html index 2d591e2..93b993b 100644 --- a/src/static/wurtzhome.html +++ b/src/static/wurtzhome.html @@ -3,7 +3,9 @@ paper clover -
+

feed

-

+

paper clover

diff --git a/src/tags/Video.tsx b/src/tags/Video.tsx index a0d6a19..9f57e9a 100644 --- a/src/tags/Video.tsx +++ b/src/tags/Video.tsx @@ -1,58 +1,58 @@ -import "./video.css"; -import * as path from "node:path"; -import { addScript } from "#sitegen"; -import { PrecomputedBlurhash } from "./blurhash.tsx"; - -export namespace Video { - export interface Props { - title: string; - width: number; - height: number; - sources: string[]; - downloads: string[]; - poster?: string; - posterHash?: string; - borderless?: boolean; - } -} - -export function Video( - { title, sources, height, poster, posterHash, width, borderless }: - Video.Props, -) { - addScript("./hls-polyfill.client.ts"); - return ( -
-
{title}
- {posterHash && } - {poster && waterfalls} - -
- ); -} -export function contentTypeFromExt(src: string) { - if (src.endsWith(".m3u8")) return "application/x-mpegURL"; - if (src.endsWith(".webm")) return "video/webm"; - if (src.endsWith(".mp4")) return "video/mp4"; - if (src.endsWith(".ogg")) return "video/ogg"; - throw new Error("Unknown video extension: " + path.extname(src)); -} -const gcd = (a: number, b: number): number => b ? gcd(b, a % b) : a; -function simplifyFraction(n: number, d: number) { - const divisor = gcd(n, d); - return `${n / divisor}/${d / divisor}`; -} +import "./video.css"; +import * as path from "node:path"; +import { addScript } from "#sitegen"; +import { PrecomputedBlurhash } from "./blurhash.tsx"; + +export namespace Video { + export interface Props { + title: string; + width: number; + height: number; + sources: string[]; + downloads: string[]; + poster?: string; + posterHash?: string; + borderless?: boolean; + } +} + +export function Video( + { title, sources, height, poster, posterHash, width, borderless }: + Video.Props, +) { + addScript("./hls-polyfill.client.ts"); + return ( +
+
{title}
+ {posterHash && } + {poster && waterfalls} + +
+ ); +} +export function contentTypeFromExt(src: string) { + if (src.endsWith(".m3u8")) return "application/x-mpegURL"; + if (src.endsWith(".webm")) return "video/webm"; + if (src.endsWith(".mp4")) return "video/mp4"; + if (src.endsWith(".ogg")) return "video/ogg"; + throw new Error("Unknown video extension: " + path.extname(src)); +} +const gcd = (a: number, b: number): number => b ? gcd(b, a % b) : a; +function simplifyFraction(n: number, d: number) { + const divisor = gcd(n, d); + return `${n / divisor}/${d / divisor}`; +} diff --git a/src/tags/video.css b/src/tags/video.css index b168df2..459b828 100644 --- a/src/tags/video.css +++ b/src/tags/video.css @@ -1,25 +1,24 @@ -.video { - border: 4px solid var(--fg); - display: flex; - flex-direction: column; - position: relative; -} - -.video > img, -.video > span { - position: absolute; - left: 0; - top: 0; - width: 100%; - height: 100%; - z-index: -1; -} - -.video figcaption { - background-color: var(--fg); - color: var(--bg); - width: 100%; - margin-top: -1px; - padding-bottom: 2px; -} - +.video { + border: 4px solid var(--fg); + display: flex; + flex-direction: column; + position: relative; +} + +.video > img, +.video > span { + position: absolute; + left: 0; + top: 0; + width: 100%; + height: 100%; + z-index: -1; +} + +.video figcaption { + background-color: var(--fg); + color: var(--bg); + width: 100%; + margin-top: -1px; + padding-bottom: 2px; +}