clo/sitegen
1
0
Fork 0
Code Issues 16 Pull requests Projects Releases Packages Wiki Activity Actions

fine grained incremental rebuilding

Browse source
This commit is contained in:
chloe caruso 2025-06-11 00:17:58 -07:00
parent 15a4600c48
commit d5ef829f01
7 changed files with 288 additions and 117 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

22
framework/bundle.ts
View file

@ -65,7 +65,7 @@ export async function bundleClientJavaScript(
route = "/js/" + key + ".js";
incr.put({
sources,
type: "script",
kind: "script",
key,
value: text,
});
@ -91,16 +91,6 @@ export async function bundleServerJavaScript(
platform: ServerPlatform = "node",
) {
const scriptMagic = "CLOVER_CLIENT_SCRIPTS_DEFINITION";
const viewModules = viewEntryPoints.map((view) => {
const module = require(view.file);
if (!module.meta) {
throw new Error(`${view.file} is missing 'export const meta'`);
}
if (!module.default) {
throw new Error(`${view.file} is missing a default export.`);
}
return { module, view };
});
const viewSource = [
...viewEntryPoints.map((view, i) =>
`import * as view${i} from ${JSON.stringify(view.file)}`
@ -171,6 +161,16 @@ export async function bundleServerJavaScript(
write: false,
metafile: true,
});
const viewModules = viewEntryPoints.map((view) => {
const module = require(view.file);
if (!module.meta) {
throw new Error(`${view.file} is missing 'export const meta'`);
}
if (!module.default) {
throw new Error(`${view.file} is missing a default export.`);
}
return { module, view };
});
const viewData = viewModules.map(({ module, view }) => {
return {
id: view.id,

18
framework/css.ts
View file

@ -5,6 +5,12 @@ export interface Theme {
h1?: string;
}
export const defaultTheme: Theme = {
bg: "#ffffff",
fg: "#050505",
primary: "#2e7dab",
};
export function stringifyTheme(theme: Theme) {
return [
":root {",
@ -39,6 +45,18 @@ export interface Output {
sources: string[];
}
export function styleKey(
cssImports: string[],
theme: Theme,
) {
cssImports = cssImports
.map((file) =>
path.isAbsolute(file) ? path.relative(hot.projectRoot, file) : file
)
.sort();
return cssImports.join(":") + JSON.stringify(theme);
}
export async function bundleCssFiles(
cssImports: string[],
theme: Theme,

2
framework/definitions.d.ts vendored
View file

@ -1,4 +1,4 @@
declare function UNWRAP<T>(value: T | null | undefined): T;
declare function UNWRAP<T>(value: T | null | undefined, ...log: unknown[]): T;
declare function ASSERT(value: unknown, ...log: unknown[]): asserts value;
type Timer = ReturnType<typeof setTimeout>;

268
framework/generate.ts
View file

@ -1,11 +1,16 @@
// This file contains the main site generator build process.
// By using `Incremental`'s ability to automatically purge stale
// assets, the `sitegen` function performs partial rebuilds.
export function main(incremental?: Incremental) {
return withSpinner<Record<string, unknown>, any>({
text: "Recovering State",
successText,
failureText: () => "sitegen FAIL",
}, async (spinner) => {
const incr = Incremental.fromDisk();
await incr.statAllFiles();
// const incr = Incremental.fromDisk();
// await incr.statAllFiles();
const incr = new Incremental();
const result = await sitegen(spinner, incr);
incr.toDisk(); // Allows picking up this state again
return result;
@ -94,26 +99,42 @@ export async function sitegen(
scripts = scripts.filter(({ file }) => !file.match(/\.client\.[tj]sx?/));
const globalCssPath = join("global.css");
// TODO: invalidate incremental resources
// TODO: make sure that `static` and `pages` does not overlap
// -- server side render --
// -- inline style sheets, used and shared by pages and views --
status.text = "Building";
const cssOnce = new OnceMap<css.Output>();
const cssQueue = new Queue<[string, string[], css.Theme], css.Output>({
const cssOnce = new OnceMap();
const cssQueue = new Queue({
name: "Bundle",
fn: ([, files, theme]) => css.bundleCssFiles(files, theme),
async fn([, key, files, theme]: [string, string, string[], css.Theme]) {
const { text, sources } = await css.bundleCssFiles(files, theme);
incr.put({
kind: "style",
key,
sources,
value: text,
});
},
passive: true,
getItemText: ([id]) => id,
maxJobs: 2,
});
interface RenderResult {
body: string;
head: string;
css: css.Output;
clientRefs: string[];
item: FileItem;
function ensureCssGetsBuilt(
cssImports: string[],
theme: css.Theme,
referrer: string,
) {
const key = css.styleKey(cssImports, theme);
cssOnce.get(
key,
async () => {
incr.getArtifact("style", key) ??
await cssQueue.add([referrer, key, cssImports, theme]);
},
);
}
const renderResults: RenderResult[] = [];
// -- server side render pages --
async function loadPageModule({ file }: FileItem) {
require(file);
}
@ -125,28 +146,27 @@ export async function sitegen(
theme: pageTheme,
layout,
} = require(item.file);
if (!Page) throw new Error("Page is missing a 'default' export.");
if (!Page) {
throw new Error("Page is missing a 'default' export.");
}
if (!metadata) {
throw new Error("Page is missing 'meta' export with a title.");
}
// -- css --
if (layout?.theme) pageTheme = layout.theme;
const theme = {
bg: "#fff",
fg: "#050505",
primary: "#2e7dab",
const theme: css.Theme = {
...css.defaultTheme,
...pageTheme,
};
const cssImports = [globalCssPath, ...hot.getCssImports(item.file)];
ensureCssGetsBuilt(cssImports, theme, item.id);
// -- metadata --
const renderedMetaPromise = Promise.resolve(
typeof metadata === "function" ? metadata({ ssr: true }) : metadata,
).then((m) => meta.renderMeta(m));
// -- css --
const cssImports = [globalCssPath, ...hot.getCssImports(item.file)];
const cssPromise = cssOnce.get(
cssImports.join(":") + JSON.stringify(theme),
() => cssQueue.add([item.id, cssImports, theme]),
);
// -- html --
let page = [engine.kElement, Page, {}];
if (layout?.default) {
@ -156,9 +176,8 @@ export async function sitegen(
sitegen: sg.initRender(),
});
const [{ text, addon }, cssBundle, renderedMeta] = await Promise.all([
const [{ text, addon }, renderedMeta] = await Promise.all([
bodyPromise,
cssPromise,
renderedMetaPromise,
]);
if (!renderedMeta.includes("<title>")) {
@ -167,20 +186,72 @@ export async function sitegen(
"All pages need a title tag.",
);
}
// The script content is not ready, allow another page to Render. The page
// contents will be rebuilt at the end. This is more convenient anyways
// because it means client scripts don't re-render the page.
renderResults.push({
body: text,
head: renderedMeta,
css: cssBundle,
clientRefs: Array.from(addon.sitegen.scripts),
item: item,
incr.put({
kind: "pageMetadata",
key: item.id,
// Incremental integrates with `hot.ts` + `require`
// to trace all the needed source files here.
sources: [item.file],
value: {
html: text,
meta: renderedMeta,
cssImports,
theme: theme ?? null,
clientRefs: Array.from(addon.sitegen.scripts),
},
});
}
async function prepareView(view: FileItem) {
const module = require(view.file);
if (!module.meta) {
throw new Error(`${view.file} is missing 'export const meta'`);
}
if (!module.default) {
throw new Error(`${view.file} is missing a default export.`);
}
const pageTheme = module.layout?.theme ?? module.theme;
const theme: css.Theme = {
...css.defaultTheme,
...pageTheme,
};
const cssImports = hot.getCssImports(view.file)
.concat("src/global.css")
.map((file) => path.relative(hot.projectRoot, path.resolve(file)));
incr.put({
kind: "viewMetadata",
key: view.id,
sources: [view.file],
value: {
file: path.relative(hot.projectRoot, view.file),
cssImports,
theme,
clientRefs: hot.getClientScriptRefs(view.file),
hasLayout: !!module.layout?.default,
},
});
}
// Of the pages that are already built, a call to 'ensureCssGetsBuilt' is
// required so that it's (1) re-built if needed, (2) not pruned from build.
const neededPages = pages.filter((page) => {
const existing = incr.getArtifact("pageMetadata", page.id);
if (existing) {
const { cssImports, theme } = existing;
ensureCssGetsBuilt(cssImports, theme, page.id);
}
return !existing;
});
const neededViews = views.filter((view) => {
const existing = incr.getArtifact("viewMetadata", view.id);
if (existing) {
const { cssImports, theme } = existing;
ensureCssGetsBuilt(cssImports, theme, view.id);
}
return !existing;
});
// This is done in two passes so that a page that throws during evaluation
// will report "Load Render Module" instead of "Render Static Page".
const neededPages = pages.filter((page) => incr.needsBuild("asset", page.id));
const spinnerFormat = status.format;
status.format = () => "";
const moduleLoadQueue = new Queue({
@ -190,6 +261,7 @@ export async function sitegen(
maxJobs: 1,
});
moduleLoadQueue.addMany(neededPages);
moduleLoadQueue.addMany(neededViews);
await moduleLoadQueue.done({ method: "stop" });
const pageQueue = new Queue({
name: "Render Static Page",
@ -198,32 +270,52 @@ export async function sitegen(
maxJobs: 2,
});
pageQueue.addMany(neededPages);
const viewQueue = new Queue({
name: "Build Dynamic View",
fn: prepareView,
getItemText,
maxJobs: 2,
});
viewQueue.addMany(neededViews);
await pageQueue.done({ method: "stop" });
await viewQueue.done({ method: "stop" });
status.format = spinnerFormat;
// -- bundle backend and views --
status.text = "Bundle backend code";
const backend = await bundle.bundleServerJavaScript(
join("backend.ts"),
views,
);
const viewCssPromise = await Promise.all(
backend.views.map((view) =>
cssOnce.get(
view.cssImports.join(":") + JSON.stringify(view.theme),
() => cssQueue.add([view.id, view.cssImports, view.theme ?? {}]),
)
),
);
// status.text = "Bundle backend code";
// const backend = await bundle.bundleServerJavaScript(
// join("backend.ts"),
// views,
// );
// const viewCssPromise = await Promise.all(
// backend.views.map((view) =>
// cssOnce.get(
// view.cssImports.join(":") + JSON.stringify(view.theme),
// () => cssQueue.add([view.id, view.cssImports, view.theme ?? {}]),
// )
// ),
// );
// -- bundle scripts --
const referencedScripts = Array.from(
new Set([
...renderResults.flatMap((r) => r.clientRefs),
...backend.views.flatMap((r) => r.clientRefs),
]),
new Set(
[
...pages.map((item) =>
UNWRAP(
incr.getArtifact("pageMetadata", item.id),
`Missing pageMetadata ${item.id}`,
)
),
...views.map((item) =>
UNWRAP(
incr.getArtifact("viewMetadata", item.id),
`Missing viewMetadata ${item.id}`,
)
),
].flatMap((item) => item.clientRefs),
),
(script) => path.resolve(hot.projectSrc, script),
);
).filter((file) => !incr.hasArtifact("script", hot.getScriptId(file)));
const extraPublicScripts = scripts.map((entry) => entry.file);
const uniqueCount = new Set([
...referencedScripts,
@ -237,7 +329,7 @@ export async function sitegen(
);
// -- finalize backend bundle --
await bundle.finalizeServerJavaScript(backend, await viewCssPromise, incr);
// await bundle.finalizeServerJavaScript(backend, await viewCssPromise, incr);
// -- copy/compress static files --
async function doStaticFile(item: FileItem) {
@ -256,37 +348,51 @@ export async function sitegen(
});
status.format = () => "";
staticQueue.addMany(
staticFiles.filter((file) => incr.needsBuild("asset", file.id)),
staticFiles.filter((file) => !incr.hasArtifact("asset", file.id)),
);
await staticQueue.done({ method: "stop" });
status.format = spinnerFormat;
await cssQueue.done({ method: "stop" });
// -- concatenate static rendered pages --
status.text = `Concat ${renderResults.length} Pages`;
await Promise.all(
renderResults.map(
async (
{ item: page, body, head, css, clientRefs: scriptFiles },
) => {
const doc = wrapDocument({
body,
head,
inlineCss: css.text,
scripts: scriptFiles.map(
(file) => UNWRAP(incr.out.script.get(hot.getScriptId(file))),
).map((x) => `{${x}}`).join("\n"),
});
await incr.putAsset({
sources: [page.file, ...css.sources],
key: page.id,
body: doc,
headers: {
"Content-Type": "text/html",
},
});
status.text = `Concat Pages`;
await Promise.all(pages.map(async (page) => {
if (incr.hasArtifact("asset", page.id)) return;
const {
html,
meta,
cssImports,
theme,
clientRefs,
} = UNWRAP(incr.out.pageMetadata.get(page.id));
const scriptIds = clientRefs.map(hot.getScriptId);
const styleKey = css.styleKey(cssImports, theme);
const style = UNWRAP(
incr.out.style.get(styleKey),
`Missing style ${styleKey}`,
);
const doc = wrapDocument({
body: html,
head: meta,
inlineCss: style,
scripts: scriptIds.map(
(ref) => UNWRAP(incr.out.script.get(ref), `Missing script ${ref}`),
).map((x) => `{${x}}`).join("\n"),
});
await incr.putAsset({
sources: [
page.file,
...incr.sourcesFor("style", styleKey),
...scriptIds.flatMap((ref) => incr.sourcesFor("script", ref)),
],
key: page.id,
body: doc,
headers: {
"Content-Type": "text/html",
},
),
);
});
}));
status.format = () => "";
status.text = ``;
// This will wait for all compression jobs to finish, which up

87
framework/incremental.ts
View file

@ -1,27 +1,63 @@
// `Incremental` contains multiple maps for the different parts of a site
// build, and tracks reused items across builds. It also handles emitting and
// updating the built site. This structure is self contained and serializable.
// Incremental contains multiple maps for the different kinds
// of Artifact, which contain a list of source files which
// were used to produce it. When files change, Incremental sees
// that the `mtime` is newer, and purges the referenced artifacts.
type SourceId = string; // relative to project root, e.g. 'src/global.css'
type ArtifactId = string; // `${ArtifactType}#${string}`
type ArtifactId = string; // `${ArtifactType}\0${string}`
type Sha1Id = string; // Sha1 hex string
// -- artifact types --
interface ArtifactMap {
/* An asset (serve with "#sitegen/asset" */
asset: Asset;
/* The bundled text of a '.client.ts' script */
// TODO: track imports this has into `asset`
script: string;
/* The bundled style tag contents. Keyed by 'css.styleKey' */
style: string;
/* Metadata about a static page */
pageMetadata: PageMetadata;
/* Metadata about a dynamic view */
viewMetadata: ViewMetadata;
}
type ArtifactType = keyof ArtifactMap;
interface Asset {
type ArtifactKind = keyof ArtifactMap;
export interface Asset {
buffer: Buffer;
headers: Record<string, string | undefined>;
hash: string;
}
/**
* This interface intentionally omits the *contents*
* of its scripts and styles for fine-grained rebuilds.
*/
export interface PageMetadata {
html: string;
meta: string;
cssImports: string[];
theme: css.Theme;
clientRefs: string[];
}
/**
* Like a page, this intentionally omits resources,
* but additionally omits the bundled server code.
*/
export interface ViewMetadata {
file: string;
// staticMeta: string | null; TODO
cssImports: string[];
theme: css.Theme;
clientRefs: string[];
hasLayout: boolean;
}
// -- incremental support types --
export interface PutBase {
sources: SourceId[];
key: string;
}
export interface Put<T extends ArtifactType> extends PutBase {
type: T;
export interface Put<T extends ArtifactKind> extends PutBase {
kind: T;
value: ArtifactMap[T];
}
export interface Invalidations {
@ -37,6 +73,9 @@ export class Incremental {
} = {
asset: new Map(),
script: new Map(),
style: new Map(),
pageMetadata: new Map(),
viewMetadata: new Map(),
};
/** Tracking filesystem entries to `srcId` */
invals = new Map<SourceId, Invalidations>();
@ -53,9 +92,16 @@ export class Incremental {
getItemText: (job) => `${job.algo.toUpperCase()} ${job.label}`,
});
/** Invalidation deletes build artifacts so the check is trivial. */
needsBuild(type: ArtifactType, key: string) {
return !this.out[type].has(key);
getArtifact<T extends ArtifactKind>(kind: T, key: string) {
return this.out[kind].get(key);
}
hasArtifact<T extends ArtifactKind>(kind: T, key: string) {
return this.out[kind].has(key);
}
sourcesFor(kind: ArtifactKind, key: string) {
return UNWRAP(this.sources.get(kind + "\0" + key));
}
/*
@ -63,18 +109,19 @@ export class Incremental {
* used to build this must be provided. 'Incremental' will trace JS
* imports and file modification times tracked by 'hot.ts'.
*/
put<T extends ArtifactType>({
put<T extends ArtifactKind>({
sources,
type,
kind,
key,
value,
}: Put<T>) {
this.out[type].set(key, value);
console.log("put " + kind + ": " + key);
this.out[kind].set(key, value);
// Update sources information
ASSERT(sources.length > 0, "Missing sources for " + type + " " + key);
ASSERT(sources.length > 0, "Missing sources for " + kind + " " + key);
sources = sources.map((src) => path.normalize(src));
const fullKey = `${type}#${key}`;
const fullKey = `${kind}\0${key}`;
const prevSources = this.sources.get(fullKey);
const newSources = new Set(
sources.map((file) =>
@ -155,8 +202,9 @@ export class Incremental {
);
const { files, outputs } = invalidations;
for (const out of outputs) {
const [type, artifactKey] = out.split("#", 2);
this.out[type as ArtifactType].delete(artifactKey);
const [kind, artifactKey] = out.split("\0");
this.out[kind as ArtifactKind].delete(artifactKey);
console.log("stale " + kind + ": " + artifactKey);
}
invalidQueue.push(...files);
}
@ -179,7 +227,7 @@ export class Incremental {
},
hash,
};
const a = this.put({ ...info, type: "asset", value });
const a = this.put({ ...info, kind: "asset", value });
if (!this.compress.has(hash)) {
const label = info.key;
this.compress.set(hash, {
@ -412,3 +460,4 @@ import * as hot from "./hot.ts";
import * as mime from "#sitegen/mime";
import * as path from "node:path";
import { Buffer } from "node:buffer";
import * as css from "./css.ts";

4
framework/lib/view.ts
View file

@ -42,10 +42,8 @@ export async function renderView(
const { text: body, addon: { sitegen } } = await engine.ssrAsync(page, {
sitegen: sg.initRender(),
});
console.log(sitegen);
// -- join document and send --
console.log(scripts);
return c.html(wrapDocument({
body,
head: await renderedMetaPromise,
@ -56,7 +54,7 @@ export async function renderView(
}));
}
export function provideViews(v: typeof views, s: typeof scripts) {
export function provideViewData(v: typeof views, s: typeof scripts) {
views = v;
scripts = s;
}

4
framework/watch.ts
View file

@ -25,7 +25,7 @@ export async function main() {
successText: generate.successText,
failureText: () => "sitegen FAIL",
}, async (spinner) => {
console.clear();
console.log("---");
console.log(
"Updated" +
(changed.length === 1
@ -36,7 +36,7 @@ export async function main() {
incr.toDisk(); // Allows picking up this state again
for (const file of watch.files) {
const relative = path.relative(hot.projectRoot, file);
if (!incr.invals.has(file)) watch.remove(file);
if (!incr.invals.has(relative)) watch.remove(file);
}
return result;
}).catch((err) => {