From d5ef829f015874f55216a1a3813c312c9adfc4c8 Mon Sep 17 00:00:00 2001 From: chloe caruso Date: Wed, 11 Jun 2025 00:17:58 -0700 Subject: [PATCH] fine grained incremental rebuilding --- framework/bundle.ts | 22 +-- framework/css.ts | 18 +++ framework/definitions.d.ts | 2 +- framework/generate.ts | 268 ++++++++++++++++++++++++++----------- framework/incremental.ts | 87 +++++++++--- framework/lib/view.ts | 4 +- framework/watch.ts | 4 +- 7 files changed, 288 insertions(+), 117 deletions(-) diff --git a/framework/bundle.ts b/framework/bundle.ts index a81bca1..0f79bf0 100644 --- a/framework/bundle.ts +++ b/framework/bundle.ts @@ -65,7 +65,7 @@ export async function bundleClientJavaScript( route = "/js/" + key + ".js"; incr.put({ sources, - type: "script", + kind: "script", key, value: text, }); @@ -91,16 +91,6 @@ export async function bundleServerJavaScript( platform: ServerPlatform = "node", ) { const scriptMagic = "CLOVER_CLIENT_SCRIPTS_DEFINITION"; - const viewModules = viewEntryPoints.map((view) => { - const module = require(view.file); - if (!module.meta) { - throw new Error(`${view.file} is missing 'export const meta'`); - } - if (!module.default) { - throw new Error(`${view.file} is missing a default export.`); - } - return { module, view }; - }); const viewSource = [ ...viewEntryPoints.map((view, i) => `import * as view${i} from ${JSON.stringify(view.file)}` @@ -171,6 +161,16 @@ export async function bundleServerJavaScript( write: false, metafile: true, }); + const viewModules = viewEntryPoints.map((view) => { + const module = require(view.file); + if (!module.meta) { + throw new Error(`${view.file} is missing 'export const meta'`); + } + if (!module.default) { + throw new Error(`${view.file} is missing a default export.`); + } + return { module, view }; + }); const viewData = viewModules.map(({ module, view }) => { return { id: view.id, diff --git a/framework/css.ts b/framework/css.ts index 3e82022..6a1ab22 100644 --- a/framework/css.ts +++ b/framework/css.ts @@ -5,6 +5,12 @@ export interface Theme { h1?: string; } +export const defaultTheme: Theme = { + bg: "#ffffff", + fg: "#050505", + primary: "#2e7dab", +}; + export function stringifyTheme(theme: Theme) { return [ ":root {", @@ -39,6 +45,18 @@ export interface Output { sources: string[]; } +export function styleKey( + cssImports: string[], + theme: Theme, +) { + cssImports = cssImports + .map((file) => + path.isAbsolute(file) ? path.relative(hot.projectRoot, file) : file + ) + .sort(); + return cssImports.join(":") + JSON.stringify(theme); +} + export async function bundleCssFiles( cssImports: string[], theme: Theme, diff --git a/framework/definitions.d.ts b/framework/definitions.d.ts index fa876b0..32732e8 100644 --- a/framework/definitions.d.ts +++ b/framework/definitions.d.ts @@ -1,4 +1,4 @@ -declare function UNWRAP(value: T | null | undefined): T; +declare function UNWRAP(value: T | null | undefined, ...log: unknown[]): T; declare function ASSERT(value: unknown, ...log: unknown[]): asserts value; type Timer = ReturnType; diff --git a/framework/generate.ts b/framework/generate.ts index 5a92717..3310769 100644 --- a/framework/generate.ts +++ b/framework/generate.ts @@ -1,11 +1,16 @@ +// This file contains the main site generator build process. +// By using `Incremental`'s ability to automatically purge stale +// assets, the `sitegen` function performs partial rebuilds. + export function main(incremental?: Incremental) { return withSpinner, any>({ text: "Recovering State", successText, failureText: () => "sitegen FAIL", }, async (spinner) => { - const incr = Incremental.fromDisk(); - await incr.statAllFiles(); + // const incr = Incremental.fromDisk(); + // await incr.statAllFiles(); + const incr = new Incremental(); const result = await sitegen(spinner, incr); incr.toDisk(); // Allows picking up this state again return result; @@ -94,26 +99,42 @@ export async function sitegen( scripts = scripts.filter(({ file }) => !file.match(/\.client\.[tj]sx?/)); const globalCssPath = join("global.css"); - // TODO: invalidate incremental resources + // TODO: make sure that `static` and `pages` does not overlap - // -- server side render -- + // -- inline style sheets, used and shared by pages and views -- status.text = "Building"; - const cssOnce = new OnceMap(); - const cssQueue = new Queue<[string, string[], css.Theme], css.Output>({ + const cssOnce = new OnceMap(); + const cssQueue = new Queue({ name: "Bundle", - fn: ([, files, theme]) => css.bundleCssFiles(files, theme), + async fn([, key, files, theme]: [string, string, string[], css.Theme]) { + const { text, sources } = await css.bundleCssFiles(files, theme); + incr.put({ + kind: "style", + key, + sources, + value: text, + }); + }, passive: true, getItemText: ([id]) => id, maxJobs: 2, }); - interface RenderResult { - body: string; - head: string; - css: css.Output; - clientRefs: string[]; - item: FileItem; + function ensureCssGetsBuilt( + cssImports: string[], + theme: css.Theme, + referrer: string, + ) { + const key = css.styleKey(cssImports, theme); + cssOnce.get( + key, + async () => { + incr.getArtifact("style", key) ?? + await cssQueue.add([referrer, key, cssImports, theme]); + }, + ); } - const renderResults: RenderResult[] = []; + + // -- server side render pages -- async function loadPageModule({ file }: FileItem) { require(file); } @@ -125,28 +146,27 @@ export async function sitegen( theme: pageTheme, layout, } = require(item.file); - if (!Page) throw new Error("Page is missing a 'default' export."); + if (!Page) { + throw new Error("Page is missing a 'default' export."); + } if (!metadata) { throw new Error("Page is missing 'meta' export with a title."); } + + // -- css -- if (layout?.theme) pageTheme = layout.theme; - const theme = { - bg: "#fff", - fg: "#050505", - primary: "#2e7dab", + const theme: css.Theme = { + ...css.defaultTheme, ...pageTheme, }; + const cssImports = [globalCssPath, ...hot.getCssImports(item.file)]; + ensureCssGetsBuilt(cssImports, theme, item.id); // -- metadata -- const renderedMetaPromise = Promise.resolve( typeof metadata === "function" ? metadata({ ssr: true }) : metadata, ).then((m) => meta.renderMeta(m)); - // -- css -- - const cssImports = [globalCssPath, ...hot.getCssImports(item.file)]; - const cssPromise = cssOnce.get( - cssImports.join(":") + JSON.stringify(theme), - () => cssQueue.add([item.id, cssImports, theme]), - ); + // -- html -- let page = [engine.kElement, Page, {}]; if (layout?.default) { @@ -156,9 +176,8 @@ export async function sitegen( sitegen: sg.initRender(), }); - const [{ text, addon }, cssBundle, renderedMeta] = await Promise.all([ + const [{ text, addon }, renderedMeta] = await Promise.all([ bodyPromise, - cssPromise, renderedMetaPromise, ]); if (!renderedMeta.includes("")) { @@ -167,20 +186,72 @@ export async function sitegen( "All pages need a title tag.", ); } - // The script content is not ready, allow another page to Render. The page - // contents will be rebuilt at the end. This is more convenient anyways - // because it means client scripts don't re-render the page. - renderResults.push({ - body: text, - head: renderedMeta, - css: cssBundle, - clientRefs: Array.from(addon.sitegen.scripts), - item: item, + incr.put({ + kind: "pageMetadata", + key: item.id, + // Incremental integrates with `hot.ts` + `require` + // to trace all the needed source files here. + sources: [item.file], + value: { + html: text, + meta: renderedMeta, + cssImports, + theme: theme ?? null, + clientRefs: Array.from(addon.sitegen.scripts), + }, }); } + async function prepareView(view: FileItem) { + const module = require(view.file); + if (!module.meta) { + throw new Error(`${view.file} is missing 'export const meta'`); + } + if (!module.default) { + throw new Error(`${view.file} is missing a default export.`); + } + const pageTheme = module.layout?.theme ?? module.theme; + const theme: css.Theme = { + ...css.defaultTheme, + ...pageTheme, + }; + const cssImports = hot.getCssImports(view.file) + .concat("src/global.css") + .map((file) => path.relative(hot.projectRoot, path.resolve(file))); + incr.put({ + kind: "viewMetadata", + key: view.id, + sources: [view.file], + value: { + file: path.relative(hot.projectRoot, view.file), + cssImports, + theme, + clientRefs: hot.getClientScriptRefs(view.file), + hasLayout: !!module.layout?.default, + }, + }); + } + + // Of the pages that are already built, a call to 'ensureCssGetsBuilt' is + // required so that it's (1) re-built if needed, (2) not pruned from build. + const neededPages = pages.filter((page) => { + const existing = incr.getArtifact("pageMetadata", page.id); + if (existing) { + const { cssImports, theme } = existing; + ensureCssGetsBuilt(cssImports, theme, page.id); + } + return !existing; + }); + const neededViews = views.filter((view) => { + const existing = incr.getArtifact("viewMetadata", view.id); + if (existing) { + const { cssImports, theme } = existing; + ensureCssGetsBuilt(cssImports, theme, view.id); + } + return !existing; + }); + // This is done in two passes so that a page that throws during evaluation // will report "Load Render Module" instead of "Render Static Page". - const neededPages = pages.filter((page) => incr.needsBuild("asset", page.id)); const spinnerFormat = status.format; status.format = () => ""; const moduleLoadQueue = new Queue({ @@ -190,6 +261,7 @@ export async function sitegen( maxJobs: 1, }); moduleLoadQueue.addMany(neededPages); + moduleLoadQueue.addMany(neededViews); await moduleLoadQueue.done({ method: "stop" }); const pageQueue = new Queue({ name: "Render Static Page", @@ -198,32 +270,52 @@ export async function sitegen( maxJobs: 2, }); pageQueue.addMany(neededPages); + const viewQueue = new Queue({ + name: "Build Dynamic View", + fn: prepareView, + getItemText, + maxJobs: 2, + }); + viewQueue.addMany(neededViews); await pageQueue.done({ method: "stop" }); + await viewQueue.done({ method: "stop" }); status.format = spinnerFormat; // -- bundle backend and views -- - status.text = "Bundle backend code"; - const backend = await bundle.bundleServerJavaScript( - join("backend.ts"), - views, - ); - const viewCssPromise = await Promise.all( - backend.views.map((view) => - cssOnce.get( - view.cssImports.join(":") + JSON.stringify(view.theme), - () => cssQueue.add([view.id, view.cssImports, view.theme ?? {}]), - ) - ), - ); + // status.text = "Bundle backend code"; + // const backend = await bundle.bundleServerJavaScript( + // join("backend.ts"), + // views, + // ); + // const viewCssPromise = await Promise.all( + // backend.views.map((view) => + // cssOnce.get( + // view.cssImports.join(":") + JSON.stringify(view.theme), + // () => cssQueue.add([view.id, view.cssImports, view.theme ?? {}]), + // ) + // ), + // ); // -- bundle scripts -- const referencedScripts = Array.from( - new Set([ - ...renderResults.flatMap((r) => r.clientRefs), - ...backend.views.flatMap((r) => r.clientRefs), - ]), + new Set( + [ + ...pages.map((item) => + UNWRAP( + incr.getArtifact("pageMetadata", item.id), + `Missing pageMetadata ${item.id}`, + ) + ), + ...views.map((item) => + UNWRAP( + incr.getArtifact("viewMetadata", item.id), + `Missing viewMetadata ${item.id}`, + ) + ), + ].flatMap((item) => item.clientRefs), + ), (script) => path.resolve(hot.projectSrc, script), - ); + ).filter((file) => !incr.hasArtifact("script", hot.getScriptId(file))); const extraPublicScripts = scripts.map((entry) => entry.file); const uniqueCount = new Set([ ...referencedScripts, @@ -237,7 +329,7 @@ export async function sitegen( ); // -- finalize backend bundle -- - await bundle.finalizeServerJavaScript(backend, await viewCssPromise, incr); + // await bundle.finalizeServerJavaScript(backend, await viewCssPromise, incr); // -- copy/compress static files -- async function doStaticFile(item: FileItem) { @@ -256,37 +348,51 @@ export async function sitegen( }); status.format = () => ""; staticQueue.addMany( - staticFiles.filter((file) => incr.needsBuild("asset", file.id)), + staticFiles.filter((file) => !incr.hasArtifact("asset", file.id)), ); await staticQueue.done({ method: "stop" }); status.format = spinnerFormat; + await cssQueue.done({ method: "stop" }); + // -- concatenate static rendered pages -- - status.text = `Concat ${renderResults.length} Pages`; - await Promise.all( - renderResults.map( - async ( - { item: page, body, head, css, clientRefs: scriptFiles }, - ) => { - const doc = wrapDocument({ - body, - head, - inlineCss: css.text, - scripts: scriptFiles.map( - (file) => UNWRAP(incr.out.script.get(hot.getScriptId(file))), - ).map((x) => `{${x}}`).join("\n"), - }); - await incr.putAsset({ - sources: [page.file, ...css.sources], - key: page.id, - body: doc, - headers: { - "Content-Type": "text/html", - }, - }); + status.text = `Concat Pages`; + await Promise.all(pages.map(async (page) => { + if (incr.hasArtifact("asset", page.id)) return; + const { + html, + meta, + cssImports, + theme, + clientRefs, + } = UNWRAP(incr.out.pageMetadata.get(page.id)); + const scriptIds = clientRefs.map(hot.getScriptId); + const styleKey = css.styleKey(cssImports, theme); + const style = UNWRAP( + incr.out.style.get(styleKey), + `Missing style ${styleKey}`, + ); + const doc = wrapDocument({ + body: html, + head: meta, + inlineCss: style, + scripts: scriptIds.map( + (ref) => UNWRAP(incr.out.script.get(ref), `Missing script ${ref}`), + ).map((x) => `{${x}}`).join("\n"), + }); + await incr.putAsset({ + sources: [ + page.file, + ...incr.sourcesFor("style", styleKey), + ...scriptIds.flatMap((ref) => incr.sourcesFor("script", ref)), + ], + key: page.id, + body: doc, + headers: { + "Content-Type": "text/html", }, - ), - ); + }); + })); status.format = () => ""; status.text = ``; // This will wait for all compression jobs to finish, which up diff --git a/framework/incremental.ts b/framework/incremental.ts index 8a0be90..d20dda4 100644 --- a/framework/incremental.ts +++ b/framework/incremental.ts @@ -1,27 +1,63 @@ -// `Incremental` contains multiple maps for the different parts of a site -// build, and tracks reused items across builds. It also handles emitting and -// updating the built site. This structure is self contained and serializable. +// Incremental contains multiple maps for the different kinds +// of Artifact, which contain a list of source files which +// were used to produce it. When files change, Incremental sees +// that the `mtime` is newer, and purges the referenced artifacts. type SourceId = string; // relative to project root, e.g. 'src/global.css' -type ArtifactId = string; // `${ArtifactType}#${string}` +type ArtifactId = string; // `${ArtifactType}\0${string}` type Sha1Id = string; // Sha1 hex string +// -- artifact types -- interface ArtifactMap { + /* An asset (serve with "#sitegen/asset" */ asset: Asset; + /* The bundled text of a '.client.ts' script */ + // TODO: track imports this has into `asset` script: string; + /* The bundled style tag contents. Keyed by 'css.styleKey' */ + style: string; + /* Metadata about a static page */ + pageMetadata: PageMetadata; + /* Metadata about a dynamic view */ + viewMetadata: ViewMetadata; } -type ArtifactType = keyof ArtifactMap; -interface Asset { +type ArtifactKind = keyof ArtifactMap; +export interface Asset { buffer: Buffer; headers: Record<string, string | undefined>; hash: string; } +/** + * This interface intentionally omits the *contents* + * of its scripts and styles for fine-grained rebuilds. + */ +export interface PageMetadata { + html: string; + meta: string; + cssImports: string[]; + theme: css.Theme; + clientRefs: string[]; +} +/** + * Like a page, this intentionally omits resources, + * but additionally omits the bundled server code. + */ +export interface ViewMetadata { + file: string; + // staticMeta: string | null; TODO + cssImports: string[]; + theme: css.Theme; + clientRefs: string[]; + hasLayout: boolean; +} + +// -- incremental support types -- export interface PutBase { sources: SourceId[]; key: string; } -export interface Put<T extends ArtifactType> extends PutBase { - type: T; +export interface Put<T extends ArtifactKind> extends PutBase { + kind: T; value: ArtifactMap[T]; } export interface Invalidations { @@ -37,6 +73,9 @@ export class Incremental { } = { asset: new Map(), script: new Map(), + style: new Map(), + pageMetadata: new Map(), + viewMetadata: new Map(), }; /** Tracking filesystem entries to `srcId` */ invals = new Map<SourceId, Invalidations>(); @@ -53,9 +92,16 @@ export class Incremental { getItemText: (job) => `${job.algo.toUpperCase()} ${job.label}`, }); - /** Invalidation deletes build artifacts so the check is trivial. */ - needsBuild(type: ArtifactType, key: string) { - return !this.out[type].has(key); + getArtifact<T extends ArtifactKind>(kind: T, key: string) { + return this.out[kind].get(key); + } + + hasArtifact<T extends ArtifactKind>(kind: T, key: string) { + return this.out[kind].has(key); + } + + sourcesFor(kind: ArtifactKind, key: string) { + return UNWRAP(this.sources.get(kind + "\0" + key)); } /* @@ -63,18 +109,19 @@ export class Incremental { * used to build this must be provided. 'Incremental' will trace JS * imports and file modification times tracked by 'hot.ts'. */ - put<T extends ArtifactType>({ + put<T extends ArtifactKind>({ sources, - type, + kind, key, value, }: Put<T>) { - this.out[type].set(key, value); + console.log("put " + kind + ": " + key); + this.out[kind].set(key, value); // Update sources information - ASSERT(sources.length > 0, "Missing sources for " + type + " " + key); + ASSERT(sources.length > 0, "Missing sources for " + kind + " " + key); sources = sources.map((src) => path.normalize(src)); - const fullKey = `${type}#${key}`; + const fullKey = `${kind}\0${key}`; const prevSources = this.sources.get(fullKey); const newSources = new Set( sources.map((file) => @@ -155,8 +202,9 @@ export class Incremental { ); const { files, outputs } = invalidations; for (const out of outputs) { - const [type, artifactKey] = out.split("#", 2); - this.out[type as ArtifactType].delete(artifactKey); + const [kind, artifactKey] = out.split("\0"); + this.out[kind as ArtifactKind].delete(artifactKey); + console.log("stale " + kind + ": " + artifactKey); } invalidQueue.push(...files); } @@ -179,7 +227,7 @@ export class Incremental { }, hash, }; - const a = this.put({ ...info, type: "asset", value }); + const a = this.put({ ...info, kind: "asset", value }); if (!this.compress.has(hash)) { const label = info.key; this.compress.set(hash, { @@ -412,3 +460,4 @@ import * as hot from "./hot.ts"; import * as mime from "#sitegen/mime"; import * as path from "node:path"; import { Buffer } from "node:buffer"; +import * as css from "./css.ts"; diff --git a/framework/lib/view.ts b/framework/lib/view.ts index ee672e2..0a422f6 100644 --- a/framework/lib/view.ts +++ b/framework/lib/view.ts @@ -42,10 +42,8 @@ export async function renderView( const { text: body, addon: { sitegen } } = await engine.ssrAsync(page, { sitegen: sg.initRender(), }); - console.log(sitegen); // -- join document and send -- - console.log(scripts); return c.html(wrapDocument({ body, head: await renderedMetaPromise, @@ -56,7 +54,7 @@ export async function renderView( })); } -export function provideViews(v: typeof views, s: typeof scripts) { +export function provideViewData(v: typeof views, s: typeof scripts) { views = v; scripts = s; } diff --git a/framework/watch.ts b/framework/watch.ts index af540e0..706b7ba 100644 --- a/framework/watch.ts +++ b/framework/watch.ts @@ -25,7 +25,7 @@ export async function main() { successText: generate.successText, failureText: () => "sitegen FAIL", }, async (spinner) => { - console.clear(); + console.log("---"); console.log( "Updated" + (changed.length === 1 @@ -36,7 +36,7 @@ export async function main() { incr.toDisk(); // Allows picking up this state again for (const file of watch.files) { const relative = path.relative(hot.projectRoot, file); - if (!incr.invals.has(file)) watch.remove(file); + if (!incr.invals.has(relative)) watch.remove(file); } return result; }).catch((err) => {