From a1d17a5d61cc6cf9d0bab2881d20adb4a96c092f Mon Sep 17 00:00:00 2001 From: chloe caruso Date: Mon, 9 Jun 2025 21:13:51 -0700 Subject: [PATCH] stuff --- framework/bundle.ts | 78 +++++++++++++++++++--- framework/css.ts | 30 ++++----- framework/esbuild-support.ts | 58 +++++++++++++++++ framework/{generate.tsx => generate.ts} | 56 ++++++---------- framework/hot.ts | 55 ++++++++++++---- framework/lib/sitegen.ts | 9 +++ framework/lib/string.ts | 3 + framework/lib/view.ts | 87 +++++++++++++++++++++++++ readme.md | 4 +- repl.js | 2 +- src/backend.ts | 2 + src/components/Video.tsx | 57 ++++++++++++++++ src/q+a/views/backend-inbox.marko | 7 +- src/q+a/views/fail.marko | 22 +++---- src/q+a/views/success.marko | 2 +- 15 files changed, 382 insertions(+), 90 deletions(-) create mode 100644 framework/esbuild-support.ts rename framework/{generate.tsx => generate.ts} (91%) create mode 100644 framework/lib/string.ts create mode 100644 framework/lib/view.ts diff --git a/framework/bundle.ts b/framework/bundle.ts index 8bc4295..78f72d8 100644 --- a/framework/bundle.ts +++ b/framework/bundle.ts @@ -1,5 +1,5 @@ // This file implements client-side bundling, mostly wrapping esbuild. -const plugins: esbuild.Plugin[] = [ +const clientPlugins: esbuild.Plugin[] = [ // There are currently no plugins needed by 'paperclover.net' ]; @@ -35,7 +35,7 @@ export async function bundleClientJavaScript( format: "esm", minify: !dev, outdir: "/out!", - plugins, + plugins: clientPlugins, splitting: true, write: false, metafile: true, @@ -82,23 +82,85 @@ export async function bundleClientJavaScript( await Promise.all(promises); } -export async function bundleServerJavaScript(entryPoint: string) { +type ServerPlatform = "node" | "passthru"; +export async function bundleServerJavaScript( + /** Has 'export default app;' */ + backendEntryPoint: string, + /** Views for dynamic loading */ + viewEntryPoints: FileItem[], + platform: ServerPlatform = "node", +) { + const scriptMagic = "CLOVER_CLIENT_SCRIPTS_DEFINITION"; + const viewSource = [ + ...viewEntryPoints.map((view, i) => + `import * as view${i} from ${JSON.stringify(view.file)}` + ), + `const scripts = ${scriptMagic}[-1]`, + "export const views = {", + ...viewEntryPoints.flatMap((view, i) => [ + ` ${JSON.stringify(view.id)}: {`, + ` component: view${i}.default,`, + ` meta: view${i}.meta,`, + ` layout: view${i}.layout?.default,`, + ` theme: view${i}.layout?.theme ?? view${i}.theme,`, + ` scripts: ${scriptMagic}[${i}]`, + ` },`, + ]), + "}", + ].join("\n"); + const serverPlugins: esbuild.Plugin[] = [ + virtualFiles({ + "$views": viewSource, + }), + banFiles([ + "hot.ts", + "incremental.ts", + "bundle.ts", + "generate.ts", + "css.ts", + ].map((subPath) => path.join(hot.projectRoot, "framework/" + subPath))), + { + name: "marko", + setup(b) { + b.onLoad({ filter: /\.marko$/ }, async ({ path }) => { + const src = await fs.readFile(path); + const result = await marko.compile(src, path, { + output: "html", + }); + return { + loader: "ts", + contents: result.code, + }; + }); + }, + }, + ]; const bundle = await esbuild.build({ bundle: true, chunkNames: "/js/c.[hash]", entryNames: "/js/[name]", assetNames: "/asset/[hash]", - entryPoints: [entryPoint], + entryPoints: [backendEntryPoint], + platform: "node", format: "esm", - minify: true, - outdir: "/out!", - plugins, + minify: false, + // outdir: "/out!", + outdir: ".clover/wah", + plugins: serverPlugins, splitting: true, - write: false, + write: true, + external: ["@babel/preset-typescript"], }); + console.log(bundle); + throw new Error("wahhh"); } import * as esbuild from "esbuild"; import * as path from "node:path"; import process from "node:process"; +import * as hot from "./hot.ts"; +import { banFiles, virtualFiles } from "./esbuild-support.ts"; import { Incremental } from "./incremental.ts"; +import type { FileItem } from "#sitegen"; +import * as marko from "@marko/compiler"; +import * as fs from "./lib/fs.ts"; diff --git a/framework/css.ts b/framework/css.ts index eda4d32..2192456 100644 --- a/framework/css.ts +++ b/framework/css.ts @@ -48,22 +48,8 @@ export async function bundleCssFiles( path.isAbsolute(file) ? path.relative(hot.projectRoot, file) : file ); const plugin = { - name: "clover", + name: "clover css", setup(b) { - b.onResolve( - { filter: /^\$input\$$/ }, - () => ({ path: ".", namespace: "input" }), - ); - b.onLoad( - { filter: /./, namespace: "input" }, - () => ({ - loader: "css", - contents: - cssImports.map((path) => `@import url(${JSON.stringify(path)});`) - .join("\n") + stringifyTheme(theme), - resolveDir: ".", - }), - ); b.onLoad( { filter: /\.css$/ }, async ({ path: file }) => ({ @@ -79,7 +65,18 @@ export async function bundleCssFiles( external: ["*.woff2", "*.ttf", "*.png", "*.jpeg"], metafile: true, minify: !dev, - plugins: [plugin], + plugins: [ + virtualFiles({ + "$input$": { + contents: cssImports.map((path) => + `@import url(${JSON.stringify(path)});` + ) + .join("\n") + stringifyTheme(theme), + loader: "css", + }, + }), + plugin, + ], target: ["ie11"], write: false, }); @@ -102,4 +99,5 @@ import * as esbuild from "esbuild"; import * as fs from "#sitegen/fs"; import * as hot from "./hot.ts"; import * as path from "node:path"; +import { virtualFiles } from "./esbuild-support.ts"; import { Incremental } from "./incremental.ts"; diff --git a/framework/esbuild-support.ts b/framework/esbuild-support.ts new file mode 100644 index 0000000..db91ef5 --- /dev/null +++ b/framework/esbuild-support.ts @@ -0,0 +1,58 @@ +export function virtualFiles( + map: Record, +) { + return { + name: "clover vfs", + setup(b) { + b.onResolve( + { + filter: new RegExp( + // TODO: Proper Escape + `\\$`, + ), + }, + ({ path }) => { + console.log({ path }); + return ({ path, namespace: "vfs" }); + }, + ); + b.onLoad( + { filter: /./, namespace: "vfs" }, + ({ path }) => { + const entry = map[path]; + return ({ + resolveDir: ".", + loader: "ts", + ...typeof entry === "string" ? { contents: entry } : entry, + }); + }, + ); + }, + } satisfies esbuild.Plugin; +} + +export function banFiles( + files: string[], +) { + return { + name: "clover vfs", + setup(b) { + b.onResolve( + { + filter: new RegExp( + "^(?:" + files.map((file) => string.escapeRegExp(file)).join("|") + + ")$", + ), + }, + ({ path, importer }) => { + throw new Error( + `Loading ${path} (from ${importer}) is banned!`, + ); + }, + ); + }, + } satisfies esbuild.Plugin; +} + +import * as esbuild from "esbuild"; +import * as string from "#sitegen/string"; diff --git a/framework/generate.tsx b/framework/generate.ts similarity index 91% rename from framework/generate.tsx rename to framework/generate.ts index 63964b6..66e0b4c 100644 --- a/framework/generate.tsx +++ b/framework/generate.ts @@ -7,15 +7,6 @@ export function main() { }, sitegen); } -/** - * A filesystem object associated with some ID, - * such as a page's route to it's source file. - */ -interface FileItem { - id: string; - file: string; -} - async function sitegen(status: Spinner) { const startTime = performance.now(); @@ -118,9 +109,12 @@ async function sitegen(status: Spinner) { } async function renderPage(item: FileItem) { // -- load and validate module -- - let { default: Page, meta: metadata, theme: pageTheme, layout } = require( - item.file, - ); + let { + default: Page, + meta: metadata, + theme: pageTheme, + layout, + } = require(item.file); if (!Page) throw new Error("Page is missing a 'default' export."); if (!metadata) { throw new Error("Page is missing 'meta' export with a title."); @@ -144,12 +138,11 @@ async function sitegen(status: Spinner) { () => cssQueue.add([item.id, cssImports, theme]), ); // -- html -- - let page = ; + let page = [engine.kElement, Page, {}]; if (layout?.default) { - const Layout = layout.default; - page = {page}; + page = [engine.kElement, layout.default, { children: page }]; } - const bodyPromise = await ssr.ssrAsync(page, { + const bodyPromise = engine.ssrAsync(page, { sitegen: sg.initRender(), }); @@ -198,6 +191,13 @@ async function sitegen(status: Spinner) { await pageQueue.done({ method: "stop" }); status.format = spinnerFormat; + // -- bundle backend and views -- + status.text = "Bundle backend code"; + const {} = await bundle.bundleServerJavaScript( + join("backend.ts"), + views, + ); + // -- bundle scripts -- const referencedScripts = Array.from( new Set(renderResults.flatMap((r) => r.scriptFiles)), @@ -276,7 +276,7 @@ async function sitegen(status: Spinner) { // Flush the site to disk. status.format = spinnerFormat; status.text = `Incremental Flush`; - incr.flush(); + incr.flush(); // Write outputs incr.toDisk(); // Allows picking up this state again return { elapsed: (performance.now() - startTime) / 1000 }; } @@ -285,32 +285,16 @@ function getItemText({ file }: FileItem) { return path.relative(hot.projectSrc, file).replaceAll("\\", "/"); } -function wrapDocument({ - body, - head, - inlineCss, - scripts, -}: { - head: string; - body: string; - inlineCss: string; - scripts: string; -}) { - return `${head}${ - inlineCss ? `` : "" - }${body}${ - scripts ? `` : "" - }`; -} - import { OnceMap, Queue } from "./queue.ts"; import { Incremental } from "./incremental.ts"; import * as bundle from "./bundle.ts"; import * as css from "./css.ts"; -import * as ssr from "./engine/ssr.ts"; +import * as engine from "./engine/ssr.ts"; import * as hot from "./hot.ts"; import * as fs from "#sitegen/fs"; import * as sg from "#sitegen"; +import type { FileItem } from "#sitegen"; import * as path from "node:path"; import * as meta from "#sitegen/meta"; import { Spinner, withSpinner } from "@paperclover/console/Spinner"; +import { wrapDocument } from "./lib/view.ts"; diff --git a/framework/hot.ts b/framework/hot.ts index 728845e..666701c 100644 --- a/framework/hot.ts +++ b/framework/hot.ts @@ -36,12 +36,6 @@ export interface FileStat { imports: string[]; } let fsGraph = new Map(); -export function setFsGraph(g: Map) { - if (fsGraph.size > 0) { - throw new Error("Cannot restore fsGraph when it has been written into"); - } - fsGraph = g; -} export function getFsGraph() { return fsGraph; } @@ -108,11 +102,18 @@ Module._resolveFilename = (...args) => { }; function loadEsbuild(module: NodeJS.Module, filepath: string) { - let src = fs.readFileSync(filepath, "utf8"); - return loadEsbuildCode(module, filepath, src); + return loadEsbuildCode(module, filepath, fs.readFileSync(filepath, "utf8")); } -function loadEsbuildCode(module: NodeJS.Module, filepath: string, src: string) { +interface LoadOptions { + scannedClientRefs?: string[]; +} +function loadEsbuildCode( + module: NodeJS.Module, + filepath: string, + src: string, + opt: LoadOptions = {}, +) { if (filepath === import.meta.filename) { module.exports = self; return; @@ -122,12 +123,13 @@ function loadEsbuildCode(module: NodeJS.Module, filepath: string, src: string) { if (filepath.endsWith(".ts")) loader = "ts"; else if (filepath.endsWith(".jsx")) loader = "jsx"; else if (filepath.endsWith(".js")) loader = "js"; + module.cloverClientRefs = opt.scannedClientRefs ?? extractClientScripts(src); if (src.includes("import.meta")) { src = ` import.meta.url = ${JSON.stringify(pathToFileURL(filepath).toString())}; import.meta.dirname = ${JSON.stringify(path.dirname(filepath))}; import.meta.filename = ${JSON.stringify(filepath)}; - ` + src; + `.trim().replace(/\n/g, "") + src; } src = esbuild.transformSync(src, { loader, @@ -148,7 +150,7 @@ function loadMarko(module: NodeJS.Module, filepath: string) { if (src.match(/^\s*client\s+import\s+["']/m)) { src = src.replace( /^\s*client\s+import\s+("[^"]+"|'[^']+')[^\n]+/m, - "", + (_, src) => ``, ) + '\nimport { Script as CloverScriptInclude } from "#sitegen";\n'; } @@ -206,9 +208,40 @@ export function resolveFrom(src: string, dest: string) { } } +const importRegExp = + /import\s+(\*\sas\s([a-zA-Z0-9$_]+)|{[^}]+})\s+from\s+(?:"#sitegen"|'#sitegen')/s; +const getSitegenAddScriptRegExp = /addScript(?:\s+as\s+([a-zA-Z0-9$_]+))/; +export function extractClientScripts(source: string): string[] { + // This match finds a call to 'import ... from "#sitegen"' + const importMatch = source.match(importRegExp); + if (!importMatch) return []; + const items = importMatch[1]; + let identifier = ""; + if (items.startsWith("{")) { + const clauseMatch = items.match(getSitegenAddScriptRegExp); + if (!clauseMatch) return []; // did not import + identifier = clauseMatch[1] || "addScript"; + } else if (items.startsWith("*")) { + identifier = importMatch[2] + "\\s*\\.\\s*addScript"; + } else { + throw new Error("Impossible"); + } + identifier = identifier.replaceAll("$", "\\$"); // only needed escape + const findCallsRegExp = new RegExp( + `\\b${identifier}\\s*\\(("[^"]+"|'[^']+')\\)`, + "gs", + ); + const calls = source.matchAll(findCallsRegExp); + return [...calls].map((call) => { + return JSON.parse(`"${call[1].slice(1, -1)}"`) as string; + }); +} + declare global { namespace NodeJS { interface Module { + cloverClientRefs?: string[]; + _compile( this: NodeJS.Module, content: string, diff --git a/framework/lib/sitegen.ts b/framework/lib/sitegen.ts index 5404b8c..dbe30f1 100644 --- a/framework/lib/sitegen.ts +++ b/framework/lib/sitegen.ts @@ -1,6 +1,15 @@ // Import this file with 'import * as sg from "#sitegen";' export type ScriptId = string; +/** + * A filesystem object associated with some ID, + * such as a page's route to it's source file. + */ +export interface FileItem { + id: string; + file: string; +} + const frameworkDir = path.dirname(import.meta.dirname); export interface SitegenRender { diff --git a/framework/lib/string.ts b/framework/lib/string.ts new file mode 100644 index 0000000..239e02e --- /dev/null +++ b/framework/lib/string.ts @@ -0,0 +1,3 @@ +export function escapeRegExp(source: string) { + return source.replace(/[\$\\]/g, "\\$&"); +} diff --git a/framework/lib/view.ts b/framework/lib/view.ts new file mode 100644 index 0000000..a133529 --- /dev/null +++ b/framework/lib/view.ts @@ -0,0 +1,87 @@ +// This import is generated by code 'bundle.ts' +export interface View { + component: engine.Component; + meta: + | meta.Meta + | ((props: { context?: hono.Context }) => Promise | meta.Meta); + layout?: engine.Component; + theme?: css.Theme; + inlineCss: string; + scripts: Record; +} +let views: Record = {}; + +// An older version of the Clover Engine supported streaming suspense +// boundaries, but those were never used. Pages will wait until they +// are fully rendered before sending. +async function renderView( + c: hono.Context, + id: string, + props: Record, +) { + views = require("$views").views; + // The view contains pre-bundled CSS and scripts, but keeps the scripts + // separate for run-time dynamic scripts. For example, the file viewer + // includes the canvas for the current page, but only the current page. + const { + component, + inlineCss, + layout, + meta: metadata, + scripts, + theme, + }: View = UNWRAP(views[id]); + + // -- metadata -- + const renderedMetaPromise = Promise.resolve( + typeof metadata === "function" ? metadata({ context: c }) : metadata, + ).then((m) => meta.renderMeta(m)); + + // -- html -- + let page: engine.Element = [engine.kElement, component, props]; + if (layout) page = [engine.kElement, layout, { children: page }]; + const { text: body, addon: { sitegen } } = await engine.ssrAsync(page, { + sitegen: sg.initRender(), + }); + + // -- join document and send -- + return c.html(wrapDocument({ + body, + head: await renderedMetaPromise, + inlineCss, + scripts: joinScripts( + Array.from(sitegen.scripts, (script) => scripts[script]), + ), + })); +} + +export function joinScripts(scriptSources: string[]) { + const { length } = scriptSources; + if (length === 0) return ""; + if (length === 1) return scriptSources[0]; + return scriptSources.map((source) => `{${source}}`).join(";"); +} + +export function wrapDocument({ + body, + head, + inlineCss, + scripts, +}: { + head: string; + body: string; + inlineCss: string; + scripts: string; +}) { + return `${head}${ + inlineCss ? `` : "" + }${body}${ + scripts ? `` : "" + }`; +} + +import * as meta from "./meta.ts"; +import type * as hono from "#hono"; +import * as engine from "../engine/ssr.ts"; +import type * as css from "../css.ts"; +import * as sg from "./sitegen.ts"; diff --git a/readme.md b/readme.md index e31405f..05ce24f 100644 --- a/readme.md +++ b/readme.md @@ -36,7 +36,9 @@ Included is `src`, which contains `paperclover.net`. Website highlights: minimum system requirements: - a cpu with at least 1 core. - random access memory. -- windows 7 or later, macos, or linux operating system. +- windows 7 or later, macos, or other operating system. + +my development machine, for example, is Dell Inspiron 7348 with Core i7 ``` npm install diff --git a/repl.js b/repl.js index 1fb7330..f23342d 100644 --- a/repl.js +++ b/repl.js @@ -27,7 +27,7 @@ hot.load("node:repl").start({ }); setTimeout(() => { - hot.reloadRecursive("./framework/generate.tsx"); + hot.reloadRecursive("./framework/generate.ts"); }, 100); async function evaluate(code) { diff --git a/src/backend.ts b/src/backend.ts index 3e6c20f..0252399 100644 --- a/src/backend.ts +++ b/src/backend.ts @@ -1,3 +1,5 @@ +import * as v from "#sitegen/view"; +console.log(v); const logHttp = scoped("http", { color: "magenta" }); const app = new Hono(); diff --git a/src/components/Video.tsx b/src/components/Video.tsx index e69de29..8c4ce75 100644 --- a/src/components/Video.tsx +++ b/src/components/Video.tsx @@ -0,0 +1,57 @@ +import * as path from "node:path"; +import { addScript } from "#sitegen"; +import { PrecomputedBlurhash } from "./blurhash.tsx"; +import "./Video.css"; +export namespace Video { + export interface Props { + title: string; + width: number; + height: number; + sources: string[]; + downloads: string[]; + poster?: string; + posterHash?: string; + borderless?: boolean; + } +} + function PrecomputedBlurhash({ hash }: { hash: string }) { +export function Video( + { title, sources, height, poster, posterHash, width, borderless }: + Video.Props, +) { + addScript("./video.client.ts"); + return ( +
+
{title}
+ {/* posterHash && */} + {poster && waterfalls} + +
+ ); +} +export function contentTypeFromExt(src: string) { + if (src.endsWith(".m3u8")) return "application/x-mpegURL"; + if (src.endsWith(".webm")) return "video/webm"; + if (src.endsWith(".mp4")) return "video/mp4"; + if (src.endsWith(".ogg")) return "video/ogg"; + throw new Error("Unknown video extension: " + path.extname(src)); +} +const gcd = (a: number, b: number): number => b ? gcd(b, a % b) : a; +function simplifyFraction(n: number, d: number) { + const divisor = gcd(n, d); + return `${n / divisor}/${d / divisor}`; +} diff --git a/src/q+a/views/backend-inbox.marko b/src/q+a/views/backend-inbox.marko index cddd372..aa3d62a 100644 --- a/src/q+a/views/backend-inbox.marko +++ b/src/q+a/views/backend-inbox.marko @@ -8,7 +8,6 @@ import { } from "../q+a/QuestionRender"; -