restore sitegen.tsx

framework/definitions.d.ts

@@ -0,0 +1,2 @@
+declare function UNWRAP<T>(value: T | null | undefined): T;
+declare function ASSERT(value: unknown, ...log: unknown[]): asserts value;

framework/fs.ts

@@ -1,12 +1,18 @@
-// File System APIs
+// File System APIs. Some custom APIs, but mostly a re-export a mix of built-in
-import { readFileSync, writeFileSync, readdirSync, statSync, existsSync, mkdirSync as nodeMkdirSync, rmSync } from 'node:fs';
+// Node.js sync+promise fs methods. For convenince.
-import { readFile, writeFile, readdir, stat, mkdir as nodeMkdir, rm } from 'node:fs/promises';
-
-// Re-export a mix of built-in Node.js sync+promise fs methods.
 export {
- readFileSync, writeFileSync, readdirSync, statSync, existsSync, rmSync ,
+  existsSync,
- readFile, writeFile, readdir, stat, mkdir, rm 
+  readdir,
-}
+  readdirSync,
+  readFile,
+  readFileSync,
+  rm,
+  rmSync,
+  stat,
+  statSync,
+  writeFile,
+  writeFileSync,
+};
 
 export function mkdir(dir: string) {
   return nodeMkdir(dir, { recursive: true });
@@ -21,5 +27,35 @@ export async function writeMkdir(file: string, contents: Buffer | string) {
   return writeFile(file, contents);
 }
 
-import * as path from 'node:path';
+export function writeMkdirSync(file: string, contents: Buffer | string) {
+  mkdirSync(path.dirname(file));
+  return writeFileSync(file, contents);
+}
 
+export function readDirRecOptionalSync(dir: string) {
+  try {
+    return readdirSync(dir, { withFileTypes: true });
+  } catch (err: any) {
+    if (err.code === "ENOENT") return [];
+    throw err;
+  }
+}
+
+import * as path from "node:path";
+import {
+  existsSync,
+  mkdirSync as nodeMkdirSync,
+  readdirSync,
+  readFileSync,
+  rmSync,
+  statSync,
+  writeFileSync,
+} from "node:fs";
+import {
+  mkdir as nodeMkdir,
+  readdir,
+  readFile,
+  rm,
+  stat,
+  writeFile,
+} from "node:fs/promises";

framework/sitegen.tsx

@@ -9,20 +9,44 @@ function main() {
   }, sitegen);
 }
 
-async function sitegen(status) {
+/**
+ * A filesystem object associated with some ID,
+ * such as a page's route to it's source file.
+ */
+interface FileItem {
+  id: string;
+  file: string;
+}
+
+async function sitegen(status: Spinner) {
   const startTime = performance.now();
 
-  let root = path.resolve(import_meta.dirname, "../src");
+  let root = path.resolve(import.meta.dirname, "../src");
-  const join = (...sub) => path.join(root, ...sub);
+  const join = (...sub: string[]) => path.join(root, ...sub);
   const incr = new Incremental();
+
+  // Sitegen reviews every defined section for resources to process
   const sections: Section[] =
     require(path.join(root, "sections.ts")).siteSections;
-  const views = [];
+
-  const staticFiles = [];
+  // Static files are compressed and served as-is.
-  let pages = [];
+  // - "{section}/static/*.png"
-  let scripts = [];
+  let staticFiles: FileItem[] = [];
+  // Pages are rendered then served as static files.
+  // - "{section}/pages/*.marko"
+  let pages: FileItem[] = [];
+  // Views are dynamically rendered pages called via backend code.
+  // - "{section}/views/*.tsx"
+  let views: FileItem[] = [];
+  // Public scripts are bundled for the client as static assets under "/js/[...]"
+  // This is used for the file viewer's canvases.
+  // Note that '.client.ts' can be placed anywhere in the file structure.
+  // - "{section}/scripts/*.client.ts"
+  let scripts: FileItem[] = [];
+  // 'backend.ts'
   const backendFiles = [];
 
+  // -- Scan for files --
   status.text = "Scanning Project";
   for (const section of sections) {
     const { root: sectionRoot } = section;
@@ -39,16 +63,21 @@ async function sitegen(status) {
       },
       { dir: sectionPath("static"), list: staticFiles, prefix: "/", ext: true },
       { dir: sectionPath("scripts"), list: scripts, prefix: rootPrefix },
-      { dir: sectionPath("views"), list: views, prefix: rootPrefix },
+      {
+        dir: sectionPath("views"),
+        list: views,
+        prefix: rootPrefix,
+        exclude: [".css", ".client.ts", ".client.tsx"],
+      },
     ];
     for (const { dir, list, prefix, exclude = [], ext = false } of kinds) {
-      const pages2 = fs.readDirRecOptional(dir);
+      const items = fs.readDirRecOptionalSync(dir);
-      page: for (const page of pages2) {
+      item: for (const item of items) {
-        if (page.isDirectory()) continue;
+        if (item.isDirectory()) continue;
-        for (const ext2 of exclude) {
+        for (const e of exclude) {
-          if (page.name.endsWith(ext2)) continue page;
+          if (item.name.endsWith(e)) continue item;
         }
-        const file = path.relative(dir, page.parentPath + "/" + page.name);
+        const file = path.relative(dir, item.parentPath + "/" + item.name);
         const trim = ext
           ? file
           : file.slice(0, -path.extname(file).length).replaceAll(".", "/");
@@ -56,7 +85,7 @@ async function sitegen(status) {
         if (prefix === "/" && id.endsWith("/index")) {
           id = id.slice(0, -"/index".length) || "/";
         }
-        list.push({ id, file: path.join(page.parentPath, page.name) });
+        list.push({ id, file: path.join(item.parentPath, item.name) });
       }
     }
     let backendFile = [
@@ -67,86 +96,110 @@ async function sitegen(status) {
   }
   scripts = scripts.filter(({ file }) => !file.match(/\.client\.[tj]sx?/));
   const globalCssPath = join("global.css");
+
+  // TODO: invalidate incremental resources
+
+  // -- server side render --
   status.text = "Building";
   const cssOnce = new OnceMap();
-  const cssQueue = new Queue({
+  const cssQueue = new Queue<[string, string[], css.Theme], string>({
     name: "Bundle",
     fn: ([, files, theme]) => css.bundleCssFiles(files, theme),
     passive: true,
     getItemText: ([id]) => id,
     maxJobs: 2,
   });
-  const ssrResults = [];
+  interface RenderResult {
-  function loadSsrModule(page) {
+    body: string;
-    require(page.file);
+    head: string;
+    inlineCss: string;
+    scriptFiles: string[];
+    item: FileItem;
   }
-  async function doSsrPage(page) {
+  const renderResults: RenderResult[] = [];
-    const module2 = require(page.file);
+  async function loadPageModule({ file }: FileItem) {
-    const Page = module2.default;
+    require(file);
-    if (!Page) {
+  }
-      throw new Error("Page is missing a 'default' export.");
+  async function renderPage(item: FileItem) {
-    }
+    // -- load and validate module --
-    const metadata = module2.meta;
+    let { default: Page, meta: metadata, theme: pageTheme, layout } = require(
+      item.file,
+    );
+    if (!Page) throw new Error("Page is missing a 'default' export.");
     if (!metadata) {
-      throw new Error("Page is missing 'meta' attribute with a title.");
+      throw new Error("Page is missing 'meta' export with a title.");
     }
+    if (layout?.theme) pageTheme = layout.theme;
     const theme = {
       bg: "#fff",
       fg: "#050505",
       primary: "#2e7dab",
-      ...module2.theme,
+      ...pageTheme,
     };
+
+    // -- metadata --
     const renderedMetaPromise = Promise.resolve(
       typeof metadata === "function" ? metadata({ ssr: true }) : metadata,
     ).then((m) => meta.resolveAndRenderMetadata(m));
-    const cssImports = [globalCssPath, ...hot.getCssImports(page.file)];
+    // -- css --
+    const cssImports = [globalCssPath, ...hot.getCssImports(item.file)];
     const cssPromise = cssOnce.get(
       cssImports.join(":") + JSON.stringify(theme),
-      () => cssQueue.add([page.id, cssImports, theme]),
+      () => cssQueue.add([item.id, cssImports, theme]),
     );
+    // -- html --
     const sitegenApi = sg.initRender();
-    const body = await (0, import_ssr.ssrAsync)(
+    const bodyPromise = await ssr.ssrAsync(<Page />, {
-      /* @__PURE__ */ (0, import_jsx_runtime.jsx)(Page, {}),
+      sitegen: sitegenApi,
-      {
+    });
-        sitegen: sitegenApi,
+
-      },
+    const [body, inlineCss, renderedMeta] = await Promise.all([
-    );
+      bodyPromise,
-    const inlineCss = await cssPromise;
+      cssPromise,
-    const renderedMeta = await renderedMetaPromise;
+      renderedMetaPromise,
+    ]);
     if (!renderedMeta.includes("<title>")) {
       throw new Error(
-        "Page is missing 'meta.title'. All pages need a title tag.",
+        "Page is missing 'meta.title'. " +
+          "All pages need a title tag.",
       );
     }
-    ssrResults.push({
+    // The script content is not ready, allow another page to Render. The page
+    // contents will be rebuilt at the end. This is more convenient anyways
+    // because it means client scripts don't re-render the page.
+    renderResults.push({
       body,
       head: renderedMeta,
       inlineCss,
       scriptFiles: Array.from(sitegenApi.scripts),
-      page,
+      item: item,
     });
   }
+  // This is done in two passes so that a page that throws during evaluation
+  // will report "Load Render Module" instead of "Render Static Page".
   const spinnerFormat = status.format;
   status.format = () => "";
-  const moduleLoadQueue = new import_queue.Queue({
+  const moduleLoadQueue = new Queue({
     name: "Load Render Module",
-    fn: loadSSRModule,
+    fn: loadPageModule,
     getItemText,
     maxJobs: 1,
   });
   moduleLoadQueue.addMany(pages);
   await moduleLoadQueue.done({ method: "stop" });
-  const pageQueue = new import_queue.Queue({
+  const pageQueue = new Queue({
-    name: "Render",
+    name: "Render Static Page",
-    fn: doSsrPage,
+    fn: renderPage,
     getItemText,
     maxJobs: 2,
   });
   pageQueue.addMany(pages);
   await pageQueue.done({ method: "stop" });
   status.format = spinnerFormat;
+
+  // -- bundle scripts --
   const referencedScripts = Array.from(
-    new Set(ssrResults.flatMap((r) => r.scriptFiles)),
+    new Set(renderResults.flatMap((r) => r.scriptFiles)),
   );
   const extraPublicScripts = scripts.map((entry) => entry.file);
   const uniqueCount = new Set([
@@ -159,15 +212,17 @@ async function sitegen(status) {
     extraPublicScripts,
     incr,
   );
-  async function doStaticFile(page) {
+
-    const body = await fs.readFile(page.file);
+  // -- copy/compress static files --
+  async function doStaticFile(item: FileItem) {
+    const body = await fs.readFile(item.file);
     incr.putAsset({
-      srcId: "static:" + page.file,
+      srcId: "static:" + item.file,
-      key: page.id,
+      key: item.id,
       body,
     });
   }
-  const staticQueue = new import_queue.Queue({
+  const staticQueue = new Queue({
     name: "Load Static",
     fn: doStaticFile,
     getItemText,
@@ -177,40 +232,68 @@ async function sitegen(status) {
   staticQueue.addMany(staticFiles);
   await staticQueue.done({ method: "stop" });
   status.format = spinnerFormat;
-  status.text = `Concat ${ssrResults.length} Pages`;
+
+  // -- concatenate static rendered pages --
+  status.text = `Concat ${renderResults.length} Pages`;
   await Promise.all(
-    ssrResults.map(async ({ page, body, head, inlineCss, scriptFiles }) => {
+    renderResults.map(
-      const doc = wrapDocument({
+      async ({ item: page, body, head, inlineCss, scriptFiles }) => {
-        body,
+        const doc = wrapDocument({
-        head,
+          body,
-        inlineCss,
+          head,
-        scripts: scriptFiles.map(
+          inlineCss,
-          (id) =>
+          scripts: scriptFiles.map(
-            UNWRAP(
+            (id) =>
-              incr.out.script.get(
+              UNWRAP(
-                path.basename(id).replace(/\.client\.[jt]sx?$/, ""),
+                incr.out.script.get(
+                  path.basename(id).replace(/\.client\.[jt]sx?$/, ""),
+                ),
               ),
-            ),
+          ).map((x) => `{${x}}`).join("\n"),
-        ).map((x) => `{${x}}`).join(""),
+        });
-      });
+        incr.putAsset({
-      incr.putAsset({
+          srcId: "page:" + page.file,
-        srcId: "page:" + page.file,
+          key: page.id,
-        key: page.id,
+          body: doc,
-        body: doc,
+          headers: {
-        headers: {
+            "Content-Type": "text/html",
-          "Content-Type": "text/html",
+          },
-        },
+        });
-      });
+      },
-    }),
+    ),
   );
   status.format = () => "";
   status.text = ``;
+  // This will wait for all compression jobs to finish, which up
+  // to this point have been left as dangling promises.
   await incr.wait();
+
+  // Flush the site to disk.
   status.format = spinnerFormat;
   status.text = `Incremental Flush`;
   incr.flush();
-  incr.serializeToDisk();
+  incr.serializeToDisk(); // Allows picking up this state again
-  return { elapsed: (performance.now() - startTime) / 1e3 };
+  return { elapsed: (performance.now() - startTime) / 1000 };
+}
+
+function getItemText({ file }: FileItem) {
+  return path.relative(hot.projectSrc, file).replaceAll("\\", "/");
+}
+
+function wrapDocument({
+  body,
+  head,
+  inlineCss,
+  scripts,
+}: {
+  head: string;
+  body: string;
+  inlineCss: string;
+  scripts: string;
+}) {
+  return `<!doctype html><head>${head}${
+    inlineCss ? `<style>${inlineCss}</style>` : ""
+  }</head><body>${body}${scripts ? `<script>${scripts}</script>` : ""}</body>`;
 }
 
 import type { Section } from "./sitegen-lib.ts";
@@ -218,5 +301,10 @@ import { OnceMap, Queue } from "./queue.ts";
 import { Incremental } from "./incremental.ts";
 import * as bundle from "./bundle.ts";
 import * as css from "./css.ts";
-import * as fs from './fs.ts';
+import * as fs from "./fs.ts";
-import { withSpinner, Spinner } from "@paperclover/console/Spinner";
+import { Spinner, withSpinner } from "@paperclover/console/Spinner";
+import * as meta from "./meta";
+import * as ssr from "./engine/ssr.ts";
+import * as sg from "./sitegen-lib.ts";
+import * as hot from "./hot.ts";
+import * as path from "node:path";

run.js

@@ -1,6 +1,7 @@
 // This file allows using Node.js in combination with
-// available plugins. Usage: "node run <script>"
+// all available plugins. Usage: "node run <script>"
 import * as path from "node:path";
+import * as util from "node:util";
 import process from "node:process";
 
 // Disable experimental warnings (Type Stripping, etc)
@@ -24,6 +25,22 @@ globalThis.console.warn = console.warn;
 globalThis.console.error = console.error;
 globalThis.console.debug = console.scoped("dbg");
 
+globalThis.UNWRAP = (t, ...args) => {
+  if (t == null) {
+    throw new Error(
+      args.length > 0 ? util.format(...args) : "UNWRAP(" + t + ")",
+    );
+  }
+  return t;
+};
+globalThis.ASSERT = (t, ...args) => {
+  if (!t) {
+    throw new Error(
+      args.length > 0 ? util.format(...args) : "Assertion Failed",
+    );
+  }
+};
+
 // Load with hooks
 if (process.argv[1].startsWith(import.meta.filename.slice(0, -".js".length))) {
   if (process.argv.length == 2) {

tsconfig.json

@@ -1,13 +1,15 @@
 {
-  "allowImportingTSExtensions": true,
+  "compilerOptions": {
-  "baseDir": ".",
+    "allowImportingTSExtensions": true,
-  "incremental": true,
+    "baseDir": ".",
-  "lib": ["dom", "esnext"],
+    "incremental": true,
-  "moduleResolution": "node18",
+    "lib": ["dom", "esnext", "esnext.iterator"],
-  "outdir": ".clover/ts",
+    "module": "nodenext",
-  "paths": { "@/*": ["src/*"] },
+    "outdir": ".clover/ts",
-  "strict": true,
+    "paths": { "@/*": ["src/*"] },
-  "target": "es2022",
+    "strict": true,
-  "jsxImportSource": "#ssr",
+    "target": "es2022",
-  "jsx": "react-jsx",
+    "jsxImportSource": "#ssr",
+    "jsx": "react-jsx"
+  }
 }