fine grained incremental rebuilding

framework/bundle.ts

@@ -65,7 +65,7 @@ export async function bundleClientJavaScript(
       route = "/js/" + key + ".js";
       incr.put({
         sources,
-        type: "script",
+        kind: "script",
         key,
         value: text,
       });
@@ -91,16 +91,6 @@ export async function bundleServerJavaScript(
   platform: ServerPlatform = "node",
 ) {
   const scriptMagic = "CLOVER_CLIENT_SCRIPTS_DEFINITION";
-  const viewModules = viewEntryPoints.map((view) => {
-    const module = require(view.file);
-    if (!module.meta) {
-      throw new Error(`${view.file} is missing 'export const meta'`);
-    }
-    if (!module.default) {
-      throw new Error(`${view.file} is missing a default export.`);
-    }
-    return { module, view };
-  });
   const viewSource = [
     ...viewEntryPoints.map((view, i) =>
       `import * as view${i} from ${JSON.stringify(view.file)}`
@@ -171,6 +161,16 @@ export async function bundleServerJavaScript(
     write: false,
     metafile: true,
   });
+  const viewModules = viewEntryPoints.map((view) => {
+    const module = require(view.file);
+    if (!module.meta) {
+      throw new Error(`${view.file} is missing 'export const meta'`);
+    }
+    if (!module.default) {
+      throw new Error(`${view.file} is missing a default export.`);
+    }
+    return { module, view };
+  });
   const viewData = viewModules.map(({ module, view }) => {
     return {
       id: view.id,

framework/css.ts

@@ -5,6 +5,12 @@ export interface Theme {
   h1?: string;
 }
 
+export const defaultTheme: Theme = {
+  bg: "#ffffff",
+  fg: "#050505",
+  primary: "#2e7dab",
+};
+
 export function stringifyTheme(theme: Theme) {
   return [
     ":root {",
@@ -39,6 +45,18 @@ export interface Output {
   sources: string[];
 }
 
+export function styleKey(
+  cssImports: string[],
+  theme: Theme,
+) {
+  cssImports = cssImports
+    .map((file) =>
+      path.isAbsolute(file) ? path.relative(hot.projectRoot, file) : file
+    )
+    .sort();
+  return cssImports.join(":") + JSON.stringify(theme);
+}
+
 export async function bundleCssFiles(
   cssImports: string[],
   theme: Theme,

framework/definitions.d.ts

@@ -1,4 +1,4 @@
-declare function UNWRAP<T>(value: T | null | undefined): T;
+declare function UNWRAP<T>(value: T | null | undefined, ...log: unknown[]): T;
 declare function ASSERT(value: unknown, ...log: unknown[]): asserts value;
 
 type Timer = ReturnType<typeof setTimeout>;

framework/generate.ts

@@ -1,11 +1,16 @@
+// This file contains the main site generator build process.
+// By using `Incremental`'s ability to automatically purge stale
+// assets, the `sitegen` function performs partial rebuilds.
+
 export function main(incremental?: Incremental) {
   return withSpinner<Record<string, unknown>, any>({
     text: "Recovering State",
     successText,
     failureText: () => "sitegen FAIL",
   }, async (spinner) => {
-    const incr = Incremental.fromDisk();
+    // const incr = Incremental.fromDisk();
-    await incr.statAllFiles();
+    // await incr.statAllFiles();
+    const incr = new Incremental();
     const result = await sitegen(spinner, incr);
     incr.toDisk(); // Allows picking up this state again
     return result;
@@ -94,26 +99,42 @@ export async function sitegen(
   scripts = scripts.filter(({ file }) => !file.match(/\.client\.[tj]sx?/));
   const globalCssPath = join("global.css");
 
-  // TODO: invalidate incremental resources
+  // TODO: make sure that `static` and `pages` does not overlap
 
-  // -- server side render --
+  // -- inline style sheets, used and shared by pages and views --
   status.text = "Building";
-  const cssOnce = new OnceMap<css.Output>();
+  const cssOnce = new OnceMap();
-  const cssQueue = new Queue<[string, string[], css.Theme], css.Output>({
+  const cssQueue = new Queue({
     name: "Bundle",
-    fn: ([, files, theme]) => css.bundleCssFiles(files, theme),
+    async fn([, key, files, theme]: [string, string, string[], css.Theme]) {
+      const { text, sources } = await css.bundleCssFiles(files, theme);
+      incr.put({
+        kind: "style",
+        key,
+        sources,
+        value: text,
+      });
+    },
     passive: true,
     getItemText: ([id]) => id,
     maxJobs: 2,
   });
-  interface RenderResult {
+  function ensureCssGetsBuilt(
-    body: string;
+    cssImports: string[],
-    head: string;
+    theme: css.Theme,
-    css: css.Output;
+    referrer: string,
-    clientRefs: string[];
+  ) {
-    item: FileItem;
+    const key = css.styleKey(cssImports, theme);
+    cssOnce.get(
+      key,
+      async () => {
+        incr.getArtifact("style", key) ??
+          await cssQueue.add([referrer, key, cssImports, theme]);
+      },
+    );
   }
-  const renderResults: RenderResult[] = [];
+
+  // -- server side render pages --
   async function loadPageModule({ file }: FileItem) {
     require(file);
   }
@@ -125,28 +146,27 @@ export async function sitegen(
       theme: pageTheme,
       layout,
     } = require(item.file);
-    if (!Page) throw new Error("Page is missing a 'default' export.");
+    if (!Page) {
+      throw new Error("Page is missing a 'default' export.");
+    }
     if (!metadata) {
       throw new Error("Page is missing 'meta' export with a title.");
     }
+
+    // -- css --
     if (layout?.theme) pageTheme = layout.theme;
-    const theme = {
+    const theme: css.Theme = {
-      bg: "#fff",
+      ...css.defaultTheme,
-      fg: "#050505",
-      primary: "#2e7dab",
       ...pageTheme,
     };
+    const cssImports = [globalCssPath, ...hot.getCssImports(item.file)];
+    ensureCssGetsBuilt(cssImports, theme, item.id);
 
     // -- metadata --
     const renderedMetaPromise = Promise.resolve(
       typeof metadata === "function" ? metadata({ ssr: true }) : metadata,
     ).then((m) => meta.renderMeta(m));
-    // -- css --
+
-    const cssImports = [globalCssPath, ...hot.getCssImports(item.file)];
-    const cssPromise = cssOnce.get(
-      cssImports.join(":") + JSON.stringify(theme),
-      () => cssQueue.add([item.id, cssImports, theme]),
-    );
     // -- html --
     let page = [engine.kElement, Page, {}];
     if (layout?.default) {
@@ -156,9 +176,8 @@ export async function sitegen(
       sitegen: sg.initRender(),
     });
 
-    const [{ text, addon }, cssBundle, renderedMeta] = await Promise.all([
+    const [{ text, addon }, renderedMeta] = await Promise.all([
       bodyPromise,
-      cssPromise,
       renderedMetaPromise,
     ]);
     if (!renderedMeta.includes("<title>")) {
@@ -167,20 +186,72 @@ export async function sitegen(
           "All pages need a title tag.",
       );
     }
-    // The script content is not ready, allow another page to Render. The page
+    incr.put({
-    // contents will be rebuilt at the end. This is more convenient anyways
+      kind: "pageMetadata",
-    // because it means client scripts don't re-render the page.
+      key: item.id,
-    renderResults.push({
+      // Incremental integrates with `hot.ts` + `require`
-      body: text,
+      // to trace all the needed source files here.
-      head: renderedMeta,
+      sources: [item.file],
-      css: cssBundle,
+      value: {
-      clientRefs: Array.from(addon.sitegen.scripts),
+        html: text,
-      item: item,
+        meta: renderedMeta,
+        cssImports,
+        theme: theme ?? null,
+        clientRefs: Array.from(addon.sitegen.scripts),
+      },
     });
   }
+  async function prepareView(view: FileItem) {
+    const module = require(view.file);
+    if (!module.meta) {
+      throw new Error(`${view.file} is missing 'export const meta'`);
+    }
+    if (!module.default) {
+      throw new Error(`${view.file} is missing a default export.`);
+    }
+    const pageTheme = module.layout?.theme ?? module.theme;
+    const theme: css.Theme = {
+      ...css.defaultTheme,
+      ...pageTheme,
+    };
+    const cssImports = hot.getCssImports(view.file)
+      .concat("src/global.css")
+      .map((file) => path.relative(hot.projectRoot, path.resolve(file)));
+    incr.put({
+      kind: "viewMetadata",
+      key: view.id,
+      sources: [view.file],
+      value: {
+        file: path.relative(hot.projectRoot, view.file),
+        cssImports,
+        theme,
+        clientRefs: hot.getClientScriptRefs(view.file),
+        hasLayout: !!module.layout?.default,
+      },
+    });
+  }
+
+  // Of the pages that are already built, a call to 'ensureCssGetsBuilt' is
+  // required so that it's (1) re-built if needed, (2) not pruned from build.
+  const neededPages = pages.filter((page) => {
+    const existing = incr.getArtifact("pageMetadata", page.id);
+    if (existing) {
+      const { cssImports, theme } = existing;
+      ensureCssGetsBuilt(cssImports, theme, page.id);
+    }
+    return !existing;
+  });
+  const neededViews = views.filter((view) => {
+    const existing = incr.getArtifact("viewMetadata", view.id);
+    if (existing) {
+      const { cssImports, theme } = existing;
+      ensureCssGetsBuilt(cssImports, theme, view.id);
+    }
+    return !existing;
+  });
+
   // This is done in two passes so that a page that throws during evaluation
   // will report "Load Render Module" instead of "Render Static Page".
-  const neededPages = pages.filter((page) => incr.needsBuild("asset", page.id));
   const spinnerFormat = status.format;
   status.format = () => "";
   const moduleLoadQueue = new Queue({
@@ -190,6 +261,7 @@ export async function sitegen(
     maxJobs: 1,
   });
   moduleLoadQueue.addMany(neededPages);
+  moduleLoadQueue.addMany(neededViews);
   await moduleLoadQueue.done({ method: "stop" });
   const pageQueue = new Queue({
     name: "Render Static Page",
@@ -198,32 +270,52 @@ export async function sitegen(
     maxJobs: 2,
   });
   pageQueue.addMany(neededPages);
+  const viewQueue = new Queue({
+    name: "Build Dynamic View",
+    fn: prepareView,
+    getItemText,
+    maxJobs: 2,
+  });
+  viewQueue.addMany(neededViews);
   await pageQueue.done({ method: "stop" });
+  await viewQueue.done({ method: "stop" });
   status.format = spinnerFormat;
 
   // -- bundle backend and views --
-  status.text = "Bundle backend code";
+  // status.text = "Bundle backend code";
-  const backend = await bundle.bundleServerJavaScript(
+  // const backend = await bundle.bundleServerJavaScript(
-    join("backend.ts"),
+  //   join("backend.ts"),
-    views,
+  //   views,
-  );
+  // );
-  const viewCssPromise = await Promise.all(
+  // const viewCssPromise = await Promise.all(
-    backend.views.map((view) =>
+  //   backend.views.map((view) =>
-      cssOnce.get(
+  //     cssOnce.get(
-        view.cssImports.join(":") + JSON.stringify(view.theme),
+  //       view.cssImports.join(":") + JSON.stringify(view.theme),
-        () => cssQueue.add([view.id, view.cssImports, view.theme ?? {}]),
+  //       () => cssQueue.add([view.id, view.cssImports, view.theme ?? {}]),
-      )
+  //     )
-    ),
+  //   ),
-  );
+  // );
 
   // -- bundle scripts --
   const referencedScripts = Array.from(
-    new Set([
+    new Set(
-      ...renderResults.flatMap((r) => r.clientRefs),
+      [
-      ...backend.views.flatMap((r) => r.clientRefs),
+        ...pages.map((item) =>
-    ]),
+          UNWRAP(
+            incr.getArtifact("pageMetadata", item.id),
+            `Missing pageMetadata ${item.id}`,
+          )
+        ),
+        ...views.map((item) =>
+          UNWRAP(
+            incr.getArtifact("viewMetadata", item.id),
+            `Missing viewMetadata ${item.id}`,
+          )
+        ),
+      ].flatMap((item) => item.clientRefs),
+    ),
     (script) => path.resolve(hot.projectSrc, script),
-  );
+  ).filter((file) => !incr.hasArtifact("script", hot.getScriptId(file)));
   const extraPublicScripts = scripts.map((entry) => entry.file);
   const uniqueCount = new Set([
     ...referencedScripts,
@@ -237,7 +329,7 @@ export async function sitegen(
   );
 
   // -- finalize backend bundle --
-  await bundle.finalizeServerJavaScript(backend, await viewCssPromise, incr);
+  // await bundle.finalizeServerJavaScript(backend, await viewCssPromise, incr);
 
   // -- copy/compress static files --
   async function doStaticFile(item: FileItem) {
@@ -256,37 +348,51 @@ export async function sitegen(
   });
   status.format = () => "";
   staticQueue.addMany(
-    staticFiles.filter((file) => incr.needsBuild("asset", file.id)),
+    staticFiles.filter((file) => !incr.hasArtifact("asset", file.id)),
   );
   await staticQueue.done({ method: "stop" });
   status.format = spinnerFormat;
 
+  await cssQueue.done({ method: "stop" });
+
   // -- concatenate static rendered pages --
-  status.text = `Concat ${renderResults.length} Pages`;
+  status.text = `Concat Pages`;
-  await Promise.all(
+  await Promise.all(pages.map(async (page) => {
-    renderResults.map(
+    if (incr.hasArtifact("asset", page.id)) return;
-      async (
+    const {
-        { item: page, body, head, css, clientRefs: scriptFiles },
+      html,
-      ) => {
+      meta,
-        const doc = wrapDocument({
+      cssImports,
-          body,
+      theme,
-          head,
+      clientRefs,
-          inlineCss: css.text,
+    } = UNWRAP(incr.out.pageMetadata.get(page.id));
-          scripts: scriptFiles.map(
+    const scriptIds = clientRefs.map(hot.getScriptId);
-            (file) => UNWRAP(incr.out.script.get(hot.getScriptId(file))),
+    const styleKey = css.styleKey(cssImports, theme);
-          ).map((x) => `{${x}}`).join("\n"),
+    const style = UNWRAP(
-        });
+      incr.out.style.get(styleKey),
-        await incr.putAsset({
+      `Missing style ${styleKey}`,
-          sources: [page.file, ...css.sources],
+    );
-          key: page.id,
+    const doc = wrapDocument({
-          body: doc,
+      body: html,
-          headers: {
+      head: meta,
-            "Content-Type": "text/html",
+      inlineCss: style,
-          },
+      scripts: scriptIds.map(
-        });
+        (ref) => UNWRAP(incr.out.script.get(ref), `Missing script ${ref}`),
+      ).map((x) => `{${x}}`).join("\n"),
+    });
+    await incr.putAsset({
+      sources: [
+        page.file,
+        ...incr.sourcesFor("style", styleKey),
+        ...scriptIds.flatMap((ref) => incr.sourcesFor("script", ref)),
+      ],
+      key: page.id,
+      body: doc,
+      headers: {
+        "Content-Type": "text/html",
       },
-    ),
+    });
-  );
+  }));
   status.format = () => "";
   status.text = ``;
   // This will wait for all compression jobs to finish, which up

framework/incremental.ts

@@ -1,27 +1,63 @@
-// `Incremental` contains multiple maps for the different parts of a site
+// Incremental contains multiple maps for the different kinds
-// build, and tracks reused items across builds. It also handles emitting and
+// of Artifact, which contain a list of source files which
-// updating the built site. This structure is self contained and serializable.
+// were used to produce it. When files change, Incremental sees
+// that the `mtime` is newer, and purges the referenced artifacts.
 
 type SourceId = string; // relative to project root, e.g. 'src/global.css'
-type ArtifactId = string; // `${ArtifactType}#${string}`
+type ArtifactId = string; // `${ArtifactType}\0${string}`
 type Sha1Id = string; // Sha1 hex string
 
+// -- artifact types --
 interface ArtifactMap {
+  /* An asset (serve with "#sitegen/asset" */
   asset: Asset;
+  /* The bundled text of a '.client.ts' script */
+  // TODO: track imports this has into `asset`
   script: string;
+  /* The bundled style tag contents. Keyed by 'css.styleKey' */
+  style: string;
+  /* Metadata about a static page */
+  pageMetadata: PageMetadata;
+  /* Metadata about a dynamic view */
+  viewMetadata: ViewMetadata;
 }
-type ArtifactType = keyof ArtifactMap;
+type ArtifactKind = keyof ArtifactMap;
-interface Asset {
+export interface Asset {
   buffer: Buffer;
   headers: Record<string, string | undefined>;
   hash: string;
 }
+/**
+ * This interface intentionally omits the *contents*
+ * of its scripts and styles for fine-grained rebuilds.
+ */
+export interface PageMetadata {
+  html: string;
+  meta: string;
+  cssImports: string[];
+  theme: css.Theme;
+  clientRefs: string[];
+}
+/**
+ * Like a page, this intentionally omits resources,
+ * but additionally omits the bundled server code.
+ */
+export interface ViewMetadata {
+  file: string;
+  // staticMeta: string | null; TODO
+  cssImports: string[];
+  theme: css.Theme;
+  clientRefs: string[];
+  hasLayout: boolean;
+}
+
+// -- incremental support types --
 export interface PutBase {
   sources: SourceId[];
   key: string;
 }
-export interface Put<T extends ArtifactType> extends PutBase {
+export interface Put<T extends ArtifactKind> extends PutBase {
-  type: T;
+  kind: T;
   value: ArtifactMap[T];
 }
 export interface Invalidations {
@@ -37,6 +73,9 @@ export class Incremental {
   } = {
     asset: new Map(),
     script: new Map(),
+    style: new Map(),
+    pageMetadata: new Map(),
+    viewMetadata: new Map(),
   };
   /** Tracking filesystem entries to `srcId` */
   invals = new Map<SourceId, Invalidations>();
@@ -53,9 +92,16 @@ export class Incremental {
     getItemText: (job) => `${job.algo.toUpperCase()} ${job.label}`,
   });
 
-  /** Invalidation deletes build artifacts so the check is trivial. */
+  getArtifact<T extends ArtifactKind>(kind: T, key: string) {
-  needsBuild(type: ArtifactType, key: string) {
+    return this.out[kind].get(key);
-    return !this.out[type].has(key);
+  }
+
+  hasArtifact<T extends ArtifactKind>(kind: T, key: string) {
+    return this.out[kind].has(key);
+  }
+
+  sourcesFor(kind: ArtifactKind, key: string) {
+    return UNWRAP(this.sources.get(kind + "\0" + key));
   }
 
   /*
@@ -63,18 +109,19 @@ export class Incremental {
    * used to build this must be provided. 'Incremental' will trace JS
    * imports and file modification times tracked by 'hot.ts'.
    */
-  put<T extends ArtifactType>({
+  put<T extends ArtifactKind>({
     sources,
-    type,
+    kind,
     key,
     value,
   }: Put<T>) {
-    this.out[type].set(key, value);
+    console.log("put " + kind + ": " + key);
+    this.out[kind].set(key, value);
 
     // Update sources information
-    ASSERT(sources.length > 0, "Missing sources for " + type + " " + key);
+    ASSERT(sources.length > 0, "Missing sources for " + kind + " " + key);
     sources = sources.map((src) => path.normalize(src));
-    const fullKey = `${type}#${key}`;
+    const fullKey = `${kind}\0${key}`;
     const prevSources = this.sources.get(fullKey);
     const newSources = new Set(
       sources.map((file) =>
@@ -155,8 +202,9 @@ export class Incremental {
         );
         const { files, outputs } = invalidations;
         for (const out of outputs) {
-          const [type, artifactKey] = out.split("#", 2);
+          const [kind, artifactKey] = out.split("\0");
-          this.out[type as ArtifactType].delete(artifactKey);
+          this.out[kind as ArtifactKind].delete(artifactKey);
+          console.log("stale " + kind + ": " + artifactKey);
         }
         invalidQueue.push(...files);
       }
@@ -179,7 +227,7 @@ export class Incremental {
       },
       hash,
     };
-    const a = this.put({ ...info, type: "asset", value });
+    const a = this.put({ ...info, kind: "asset", value });
     if (!this.compress.has(hash)) {
       const label = info.key;
       this.compress.set(hash, {
@@ -412,3 +460,4 @@ import * as hot from "./hot.ts";
 import * as mime from "#sitegen/mime";
 import * as path from "node:path";
 import { Buffer } from "node:buffer";
+import * as css from "./css.ts";

framework/lib/view.ts

@@ -42,10 +42,8 @@ export async function renderView(
   const { text: body, addon: { sitegen } } = await engine.ssrAsync(page, {
     sitegen: sg.initRender(),
   });
-  console.log(sitegen);
 
   // -- join document and send --
-  console.log(scripts);
   return c.html(wrapDocument({
     body,
     head: await renderedMetaPromise,
@@ -56,7 +54,7 @@ export async function renderView(
   }));
 }
 
-export function provideViews(v: typeof views, s: typeof scripts) {
+export function provideViewData(v: typeof views, s: typeof scripts) {
   views = v;
   scripts = s;
 }

framework/watch.ts

@@ -25,7 +25,7 @@ export async function main() {
       successText: generate.successText,
       failureText: () => "sitegen FAIL",
     }, async (spinner) => {
-      console.clear();
+      console.log("---");
       console.log(
         "Updated" +
           (changed.length === 1
@@ -36,7 +36,7 @@ export async function main() {
       incr.toDisk(); // Allows picking up this state again
       for (const file of watch.files) {
         const relative = path.relative(hot.projectRoot, file);
-        if (!incr.invals.has(file)) watch.remove(file);
+        if (!incr.invals.has(relative)) watch.remove(file);
       }
       return result;
     }).catch((err) => {