stuff

framework/bundle.ts

@@ -1,5 +1,5 @@
 // This file implements client-side bundling, mostly wrapping esbuild.
-const plugins: esbuild.Plugin[] = [
+const clientPlugins: esbuild.Plugin[] = [
   // There are currently no plugins needed by 'paperclover.net'
 ];
 
@@ -35,7 +35,7 @@ export async function bundleClientJavaScript(
     format: "esm",
     minify: !dev,
     outdir: "/out!",
-    plugins,
+    plugins: clientPlugins,
     splitting: true,
     write: false,
     metafile: true,
@@ -82,23 +82,85 @@ export async function bundleClientJavaScript(
   await Promise.all(promises);
 }
 
-export async function bundleServerJavaScript(entryPoint: string) {
+type ServerPlatform = "node" | "passthru";
+export async function bundleServerJavaScript(
+  /** Has 'export default app;' */
+  backendEntryPoint: string,
+  /** Views for dynamic loading */
+  viewEntryPoints: FileItem[],
+  platform: ServerPlatform = "node",
+) {
+  const scriptMagic = "CLOVER_CLIENT_SCRIPTS_DEFINITION";
+  const viewSource = [
+    ...viewEntryPoints.map((view, i) =>
+      `import * as view${i} from ${JSON.stringify(view.file)}`
+    ),
+    `const scripts = ${scriptMagic}[-1]`,
+    "export const views = {",
+    ...viewEntryPoints.flatMap((view, i) => [
+      `  ${JSON.stringify(view.id)}: {`,
+      `    component: view${i}.default,`,
+      `    meta: view${i}.meta,`,
+      `    layout: view${i}.layout?.default,`,
+      `    theme: view${i}.layout?.theme ?? view${i}.theme,`,
+      `    scripts: ${scriptMagic}[${i}]`,
+      `  },`,
+    ]),
+    "}",
+  ].join("\n");
+  const serverPlugins: esbuild.Plugin[] = [
+    virtualFiles({
+      "$views": viewSource,
+    }),
+    banFiles([
+      "hot.ts",
+      "incremental.ts",
+      "bundle.ts",
+      "generate.ts",
+      "css.ts",
+    ].map((subPath) => path.join(hot.projectRoot, "framework/" + subPath))),
+    {
+      name: "marko",
+      setup(b) {
+        b.onLoad({ filter: /\.marko$/ }, async ({ path }) => {
+          const src = await fs.readFile(path);
+          const result = await marko.compile(src, path, {
+            output: "html",
+          });
+          return {
+            loader: "ts",
+            contents: result.code,
+          };
+        });
+      },
+    },
+  ];
   const bundle = await esbuild.build({
     bundle: true,
     chunkNames: "/js/c.[hash]",
     entryNames: "/js/[name]",
     assetNames: "/asset/[hash]",
-    entryPoints: [entryPoint],
+    entryPoints: [backendEntryPoint],
+    platform: "node",
     format: "esm",
-    minify: true,
+    minify: false,
-    outdir: "/out!",
+    // outdir: "/out!",
-    plugins,
+    outdir: ".clover/wah",
+    plugins: serverPlugins,
     splitting: true,
-    write: false,
+    write: true,
+    external: ["@babel/preset-typescript"],
   });
+  console.log(bundle);
+  throw new Error("wahhh");
 }
 
 import * as esbuild from "esbuild";
 import * as path from "node:path";
 import process from "node:process";
+import * as hot from "./hot.ts";
+import { banFiles, virtualFiles } from "./esbuild-support.ts";
 import { Incremental } from "./incremental.ts";
+import type { FileItem } from "#sitegen";
+import * as marko from "@marko/compiler";
+import * as fs from "./lib/fs.ts";

framework/css.ts

@@ -48,22 +48,8 @@ export async function bundleCssFiles(
     path.isAbsolute(file) ? path.relative(hot.projectRoot, file) : file
   );
   const plugin = {
-    name: "clover",
+    name: "clover css",
     setup(b) {
-      b.onResolve(
-        { filter: /^\$input\$$/ },
-        () => ({ path: ".", namespace: "input" }),
-      );
-      b.onLoad(
-        { filter: /./, namespace: "input" },
-        () => ({
-          loader: "css",
-          contents:
-            cssImports.map((path) => `@import url(${JSON.stringify(path)});`)
-              .join("\n") + stringifyTheme(theme),
-          resolveDir: ".",
-        }),
-      );
       b.onLoad(
         { filter: /\.css$/ },
         async ({ path: file }) => ({
@@ -79,7 +65,18 @@ export async function bundleCssFiles(
     external: ["*.woff2", "*.ttf", "*.png", "*.jpeg"],
     metafile: true,
     minify: !dev,
-    plugins: [plugin],
+    plugins: [
+      virtualFiles({
+        "$input$": {
+          contents: cssImports.map((path) =>
+            `@import url(${JSON.stringify(path)});`
+          )
+            .join("\n") + stringifyTheme(theme),
+          loader: "css",
+        },
+      }),
+      plugin,
+    ],
     target: ["ie11"],
     write: false,
   });
@@ -102,4 +99,5 @@ import * as esbuild from "esbuild";
 import * as fs from "#sitegen/fs";
 import * as hot from "./hot.ts";
 import * as path from "node:path";
+import { virtualFiles } from "./esbuild-support.ts";
 import { Incremental } from "./incremental.ts";

framework/esbuild-support.ts

@@ -0,0 +1,58 @@
+export function virtualFiles(
+  map: Record<string, string | esbuild.OnLoadResult>,
+) {
+  return {
+    name: "clover vfs",
+    setup(b) {
+      b.onResolve(
+        {
+          filter: new RegExp(
+            // TODO: Proper Escape
+            `\\$`,
+          ),
+        },
+        ({ path }) => {
+          console.log({ path });
+          return ({ path, namespace: "vfs" });
+        },
+      );
+      b.onLoad(
+        { filter: /./, namespace: "vfs" },
+        ({ path }) => {
+          const entry = map[path];
+          return ({
+            resolveDir: ".",
+            loader: "ts",
+            ...typeof entry === "string" ? { contents: entry } : entry,
+          });
+        },
+      );
+    },
+  } satisfies esbuild.Plugin;
+}
+
+export function banFiles(
+  files: string[],
+) {
+  return {
+    name: "clover vfs",
+    setup(b) {
+      b.onResolve(
+        {
+          filter: new RegExp(
+            "^(?:" + files.map((file) => string.escapeRegExp(file)).join("|") +
+              ")$",
+          ),
+        },
+        ({ path, importer }) => {
+          throw new Error(
+            `Loading ${path} (from ${importer}) is banned!`,
+          );
+        },
+      );
+    },
+  } satisfies esbuild.Plugin;
+}
+
+import * as esbuild from "esbuild";
+import * as string from "#sitegen/string";

framework/generate.ts

@@ -7,15 +7,6 @@ export function main() {
   }, sitegen);
 }
 
-/**
- * A filesystem object associated with some ID,
- * such as a page's route to it's source file.
- */
-interface FileItem {
-  id: string;
-  file: string;
-}
-
 async function sitegen(status: Spinner) {
   const startTime = performance.now();
 
@@ -118,9 +109,12 @@ async function sitegen(status: Spinner) {
   }
   async function renderPage(item: FileItem) {
     // -- load and validate module --
-    let { default: Page, meta: metadata, theme: pageTheme, layout } = require(
+    let {
-      item.file,
+      default: Page,
-    );
+      meta: metadata,
+      theme: pageTheme,
+      layout,
+    } = require(item.file);
     if (!Page) throw new Error("Page is missing a 'default' export.");
     if (!metadata) {
       throw new Error("Page is missing 'meta' export with a title.");
@@ -144,12 +138,11 @@ async function sitegen(status: Spinner) {
       () => cssQueue.add([item.id, cssImports, theme]),
     );
     // -- html --
-    let page = <Page />;
+    let page = [engine.kElement, Page, {}];
     if (layout?.default) {
-      const Layout = layout.default;
+      page = [engine.kElement, layout.default, { children: page }];
-      page = <Layout>{page}</Layout>;
     }
-    const bodyPromise = await ssr.ssrAsync(page, {
+    const bodyPromise = engine.ssrAsync(page, {
       sitegen: sg.initRender(),
     });
 
@@ -198,6 +191,13 @@ async function sitegen(status: Spinner) {
   await pageQueue.done({ method: "stop" });
   status.format = spinnerFormat;
 
+  // -- bundle backend and views --
+  status.text = "Bundle backend code";
+  const {} = await bundle.bundleServerJavaScript(
+    join("backend.ts"),
+    views,
+  );
+
   // -- bundle scripts --
   const referencedScripts = Array.from(
     new Set(renderResults.flatMap((r) => r.scriptFiles)),
@@ -276,7 +276,7 @@ async function sitegen(status: Spinner) {
   // Flush the site to disk.
   status.format = spinnerFormat;
   status.text = `Incremental Flush`;
-  incr.flush();
+  incr.flush(); // Write outputs
   incr.toDisk(); // Allows picking up this state again
   return { elapsed: (performance.now() - startTime) / 1000 };
 }
@@ -285,32 +285,16 @@ function getItemText({ file }: FileItem) {
   return path.relative(hot.projectSrc, file).replaceAll("\\", "/");
 }
 
-function wrapDocument({
-  body,
-  head,
-  inlineCss,
-  scripts,
-}: {
-  head: string;
-  body: string;
-  inlineCss: string;
-  scripts: string;
-}) {
-  return `<!doctype html><html lang=en><head>${head}${
-    inlineCss ? `<style>${inlineCss}</style>` : ""
-  }</head><body>${body}${
-    scripts ? `<script>${scripts}</script>` : ""
-  }</body></html>`;
-}
-
 import { OnceMap, Queue } from "./queue.ts";
 import { Incremental } from "./incremental.ts";
 import * as bundle from "./bundle.ts";
 import * as css from "./css.ts";
-import * as ssr from "./engine/ssr.ts";
+import * as engine from "./engine/ssr.ts";
 import * as hot from "./hot.ts";
 import * as fs from "#sitegen/fs";
 import * as sg from "#sitegen";
+import type { FileItem } from "#sitegen";
 import * as path from "node:path";
 import * as meta from "#sitegen/meta";
 import { Spinner, withSpinner } from "@paperclover/console/Spinner";
+import { wrapDocument } from "./lib/view.ts";

framework/hot.ts

@@ -36,12 +36,6 @@ export interface FileStat {
   imports: string[];
 }
 let fsGraph = new Map<string, FileStat>();
-export function setFsGraph(g: Map<string, FileStat>) {
-  if (fsGraph.size > 0) {
-    throw new Error("Cannot restore fsGraph when it has been written into");
-  }
-  fsGraph = g;
-}
 export function getFsGraph() {
   return fsGraph;
 }
@@ -108,11 +102,18 @@ Module._resolveFilename = (...args) => {
 };
 
 function loadEsbuild(module: NodeJS.Module, filepath: string) {
-  let src = fs.readFileSync(filepath, "utf8");
+  return loadEsbuildCode(module, filepath, fs.readFileSync(filepath, "utf8"));
-  return loadEsbuildCode(module, filepath, src);
 }
 
-function loadEsbuildCode(module: NodeJS.Module, filepath: string, src: string) {
+interface LoadOptions {
+  scannedClientRefs?: string[];
+}
+function loadEsbuildCode(
+  module: NodeJS.Module,
+  filepath: string,
+  src: string,
+  opt: LoadOptions = {},
+) {
   if (filepath === import.meta.filename) {
     module.exports = self;
     return;
@@ -122,12 +123,13 @@ function loadEsbuildCode(module: NodeJS.Module, filepath: string, src: string) {
   if (filepath.endsWith(".ts")) loader = "ts";
   else if (filepath.endsWith(".jsx")) loader = "jsx";
   else if (filepath.endsWith(".js")) loader = "js";
+  module.cloverClientRefs = opt.scannedClientRefs ?? extractClientScripts(src);
   if (src.includes("import.meta")) {
     src = `
       import.meta.url = ${JSON.stringify(pathToFileURL(filepath).toString())};
       import.meta.dirname = ${JSON.stringify(path.dirname(filepath))};
       import.meta.filename = ${JSON.stringify(filepath)};
-    ` + src;
+    `.trim().replace(/\n/g, "") + src;
   }
   src = esbuild.transformSync(src, {
     loader,
@@ -148,7 +150,7 @@ function loadMarko(module: NodeJS.Module, filepath: string) {
   if (src.match(/^\s*client\s+import\s+["']/m)) {
     src = src.replace(
       /^\s*client\s+import\s+("[^"]+"|'[^']+')[^\n]+/m,
-      "<CloverScriptInclude src=$1 />",
+      (_, src) => `<CloverScriptInclude src=${src} />`,
     ) + '\nimport { Script as CloverScriptInclude } from "#sitegen";\n';
   }
 
@@ -206,9 +208,40 @@ export function resolveFrom(src: string, dest: string) {
   }
 }
 
+const importRegExp =
+  /import\s+(\*\sas\s([a-zA-Z0-9$_]+)|{[^}]+})\s+from\s+(?:"#sitegen"|'#sitegen')/s;
+const getSitegenAddScriptRegExp = /addScript(?:\s+as\s+([a-zA-Z0-9$_]+))/;
+export function extractClientScripts(source: string): string[] {
+  // This match finds a call to 'import ... from "#sitegen"'
+  const importMatch = source.match(importRegExp);
+  if (!importMatch) return [];
+  const items = importMatch[1];
+  let identifier = "";
+  if (items.startsWith("{")) {
+    const clauseMatch = items.match(getSitegenAddScriptRegExp);
+    if (!clauseMatch) return []; // did not import
+    identifier = clauseMatch[1] || "addScript";
+  } else if (items.startsWith("*")) {
+    identifier = importMatch[2] + "\\s*\\.\\s*addScript";
+  } else {
+    throw new Error("Impossible");
+  }
+  identifier = identifier.replaceAll("$", "\\$"); // only needed escape
+  const findCallsRegExp = new RegExp(
+    `\\b${identifier}\\s*\\(("[^"]+"|'[^']+')\\)`,
+    "gs",
+  );
+  const calls = source.matchAll(findCallsRegExp);
+  return [...calls].map((call) => {
+    return JSON.parse(`"${call[1].slice(1, -1)}"`) as string;
+  });
+}
+
 declare global {
   namespace NodeJS {
     interface Module {
+      cloverClientRefs?: string[];
+
       _compile(
         this: NodeJS.Module,
         content: string,

framework/lib/sitegen.ts

@@ -1,6 +1,15 @@
 // Import this file with 'import * as sg from "#sitegen";'
 export type ScriptId = string;
 
+/**
+ * A filesystem object associated with some ID,
+ * such as a page's route to it's source file.
+ */
+export interface FileItem {
+  id: string;
+  file: string;
+}
+
 const frameworkDir = path.dirname(import.meta.dirname);
 
 export interface SitegenRender {

framework/lib/string.ts

@@ -0,0 +1,3 @@
+export function escapeRegExp(source: string) {
+  return source.replace(/[\$\\]/g, "\\$&");
+}

framework/lib/view.ts

@@ -0,0 +1,87 @@
+// This import is generated by code 'bundle.ts'
+export interface View {
+  component: engine.Component;
+  meta:
+    | meta.Meta
+    | ((props: { context?: hono.Context }) => Promise<meta.Meta> | meta.Meta);
+  layout?: engine.Component;
+  theme?: css.Theme;
+  inlineCss: string;
+  scripts: Record<string, string>;
+}
+let views: Record<string, View> = {};
+
+// An older version of the Clover Engine supported streaming suspense
+// boundaries, but those were never used. Pages will wait until they
+// are fully rendered before sending.
+async function renderView(
+  c: hono.Context,
+  id: string,
+  props: Record<string, unknown>,
+) {
+  views = require("$views").views;
+  // The view contains pre-bundled CSS and scripts, but keeps the scripts
+  // separate for run-time dynamic scripts. For example, the file viewer
+  // includes the canvas for the current page, but only the current page.
+  const {
+    component,
+    inlineCss,
+    layout,
+    meta: metadata,
+    scripts,
+    theme,
+  }: View = UNWRAP(views[id]);
+
+  // -- metadata --
+  const renderedMetaPromise = Promise.resolve(
+    typeof metadata === "function" ? metadata({ context: c }) : metadata,
+  ).then((m) => meta.renderMeta(m));
+
+  // -- html --
+  let page: engine.Element = [engine.kElement, component, props];
+  if (layout) page = [engine.kElement, layout, { children: page }];
+  const { text: body, addon: { sitegen } } = await engine.ssrAsync(page, {
+    sitegen: sg.initRender(),
+  });
+
+  // -- join document and send --
+  return c.html(wrapDocument({
+    body,
+    head: await renderedMetaPromise,
+    inlineCss,
+    scripts: joinScripts(
+      Array.from(sitegen.scripts, (script) => scripts[script]),
+    ),
+  }));
+}
+
+export function joinScripts(scriptSources: string[]) {
+  const { length } = scriptSources;
+  if (length === 0) return "";
+  if (length === 1) return scriptSources[0];
+  return scriptSources.map((source) => `{${source}}`).join(";");
+}
+
+export function wrapDocument({
+  body,
+  head,
+  inlineCss,
+  scripts,
+}: {
+  head: string;
+  body: string;
+  inlineCss: string;
+  scripts: string;
+}) {
+  return `<!doctype html><html lang=en><head>${head}${
+    inlineCss ? `<style>${inlineCss}</style>` : ""
+  }</head><body>${body}${
+    scripts ? `<script>${scripts}</script>` : ""
+  }</body></html>`;
+}
+
+import * as meta from "./meta.ts";
+import type * as hono from "#hono";
+import * as engine from "../engine/ssr.ts";
+import type * as css from "../css.ts";
+import * as sg from "./sitegen.ts";

readme.md

@@ -36,7 +36,9 @@ Included is `src`, which contains `paperclover.net`. Website highlights:
 minimum system requirements:
 - a cpu with at least 1 core.
 - random access memory.
-- windows 7 or later, macos, or linux operating system.
+- windows 7 or later, macos, or other operating system.
+
+my development machine, for example, is Dell Inspiron 7348 with Core i7
 
 ```
 npm install

repl.js

@@ -27,7 +27,7 @@ hot.load("node:repl").start({
 });
 
 setTimeout(() => {
-  hot.reloadRecursive("./framework/generate.tsx");
+  hot.reloadRecursive("./framework/generate.ts");
 }, 100);
 
 async function evaluate(code) {

src/backend.ts

@@ -1,3 +1,5 @@
+import * as v from "#sitegen/view";
+console.log(v);
 const logHttp = scoped("http", { color: "magenta" });
 
 const app = new Hono();

src/components/Video.tsx

@@ -0,0 +1,57 @@
+import * as path from "node:path";
+import { addScript } from "#sitegen";
+import { PrecomputedBlurhash } from "./blurhash.tsx";
+import "./Video.css";
+export namespace Video {
+  export interface Props {
+    title: string;
+    width: number;
+    height: number;
+    sources: string[];
+    downloads: string[];
+    poster?: string;
+    posterHash?: string;
+    borderless?: boolean;
+  }
+}
+ function PrecomputedBlurhash({ hash }: { hash: string }) {
+export function Video(
+  { title, sources, height, poster, posterHash, width, borderless }:
+    Video.Props,
+) {
+  addScript("./video.client.ts");
+  return (
+    <figure class={`video ${borderless ? "borderless" : ""}`}>
+      <figcaption>{title}</figcaption>
+      {/* posterHash && <PrecomputedBlurhash hash={posterHash} /> */}
+      {poster && <img src={poster} alt="waterfalls" />}
+      <video
+        controls
+        preload="none"
+        style={`width:100%;background:transparent;aspect-ratio:${
+          simplifyFraction(width, height)
+        }`}
+        poster="data:null"
+      >
+        {sources.map((src) => (
+          <source
+            src={src}
+            type={contentTypeFromExt(src)}
+          />
+        ))}
+      </video>
+    </figure>
+  );
+}
+export function contentTypeFromExt(src: string) {
+  if (src.endsWith(".m3u8")) return "application/x-mpegURL";
+  if (src.endsWith(".webm")) return "video/webm";
+  if (src.endsWith(".mp4")) return "video/mp4";
+  if (src.endsWith(".ogg")) return "video/ogg";
+  throw new Error("Unknown video extension: " + path.extname(src));
+}
+const gcd = (a: number, b: number): number => b ? gcd(b, a % b) : a;
+function simplifyFraction(n: number, d: number) {
+  const divisor = gcd(n, d);
+  return `${n / divisor}/${d / divisor}`;
+}

src/q+a/views/backend-inbox.marko

@@ -8,7 +8,6 @@ import {
 } from "../q+a/QuestionRender";
 
 <const/questions = PendingQuestion.getAll() />
-<Script src="backend-inbox.client.ts" />
 
 <h1>inbox</h1>
 <for|q| of=questions>
@@ -16,7 +15,7 @@ import {
     data-q=q.id
     style="border-bottom: 2px solid #fff7; margin-bottom: 1rem"
   >
-    <time datetime={formatQuestionISOTimestamp(q.date)}>
+    <time datetime=formatQuestionISOTimestamp(q.date)>
       ${formatQuestionTimestamp(q.date)} ${q.id}
     </time>
     <div style="color: dodgerblue; margin-bottom: 0.25rem">
@@ -26,7 +25,7 @@ import {
     </div>
     <p style="white-space: pre-wrap">${q.prompt}</p>
     <p>
-      <button onclick=`onReply("${q.id}") style="color: lime">
+      <button onclick=`onReply("${q.id}")` style="color: lime">
         reply
       </button>
       <button onclick=`onDelete("${q.id}")` style="color: red">
@@ -38,3 +37,5 @@ import {
     </p>
   </div>
 </>
+
+client import "backend-inbox.client.ts";

src/q+a/views/fail.marko

@@ -1,4 +1,4 @@
-import * as layout from "../layouts/questions.tsx";
+import * as layout from "../layout.tsx";
 export { layout };
 export const theme = {
   ...layout.theme,
@@ -10,18 +10,14 @@ export const theme = {
 <p style="color: red">
   ${error}
 </p>
-{
+<if=content>
-  content && (
+  <br />
-    <>
+  <br />
-      <br />
+  <p>
-      <br />
+    here is a copy of what you wrote, if you want to try again:
-      <p>
+  </p>
-        here is a copy of what you wrote, if you want to try again:
+  <pre style="white-space: pre-wrap"><code>${content}</code></pre>
-      </p>
+</>
-      <pre style="white-space: pre-wrap"><code>${content}</code></pre>
-    </>
-  )
-}
 <br />
 <br />
 <p>

src/q+a/views/success.marko

@@ -3,7 +3,7 @@ export interface Input {
 }
 <const/{ permalink }=input/>
 
-import * as layout from "../layouts/questions.tsx";
+import * as layout from "../layout.tsx";
 export { layout };
 export const theme = {
   ...layout.theme,