primative backend support

framework/backend/entry-node.ts

@@ -0,0 +1,32 @@
+import "@paperclover/console/inject";
+
+const protocol = "http";
+
+const server = serve(app, ({ address, port }) => {
+  if (address === "::") address = "::1";
+  console.info(url.format({
+    protocol,
+    hostname: address,
+    port,
+  }));
+});
+
+process.on("SIGINT", () => {
+  server.close();
+  process.exit(0);
+});
+
+process.on("SIGTERM", () => {
+  server.close((err) => {
+    if (err) {
+      console.error(err);
+      process.exit(1);
+    }
+    process.exit(0);
+  });
+});
+
+import app from "#backend";
+import url from "node:url";
+import { serve } from "@hono/node-server";
+import process from "node:process";

framework/bundle.ts

@@ -49,7 +49,7 @@ export async function bundleClientJavaScript(
   const publicScriptRoutes = extraPublicScripts.map((file) =>
     path.basename(file).replace(/\.client\.[tj]sx?/, "")
   );
-  const promises: Promise<unknown>[] = [];
+  const promises: Promise<void>[] = [];
   // TODO: add a shared build hash to entrypoints, derived from all the chunk hashes.
   for (const file of bundle.outputFiles) {
     let route = file.path.replace(/^.*!/, "").replaceAll("\\", "/");
@@ -73,10 +73,23 @@ export async function bundleClientJavaScript(
       }));
     }
   }
-
-  if (promises.length > 0) {
   await Promise.all(promises);
 }
+
+export async function bundleServerJavaScript(entryPoint: string) {
+  const bundle = await esbuild.build({
+    bundle: true,
+    chunkNames: "/js/c.[hash]",
+    entryNames: "/js/[name]",
+    assetNames: "/asset/[hash]",
+    entryPoints: [entryPoint],
+    format: "esm",
+    minify: true,
+    outdir: "/out!",
+    plugins,
+    splitting: true,
+    write: false,
+  });
 }
 
 import * as esbuild from "esbuild";

framework/fs.ts

@@ -34,7 +34,7 @@ export function writeMkdirSync(file: string, contents: Buffer | string) {
 
 export function readDirRecOptionalSync(dir: string) {
   try {
-    return readdirSync(dir, { withFileTypes: true });
+    return readdirSync(dir, { recursive: true, withFileTypes: true });
   } catch (err: any) {
     if (err.code === "ENOENT") return [];
     throw err;

framework/generate.tsx

@@ -1,5 +1,3 @@
-// Sitegen! Clover's static site generator, built with love.
-
 export function main() {
   return withSpinner({
     text: "Recovering State",
@@ -26,8 +24,8 @@ async function sitegen(status: Spinner) {
   const incr = new Incremental();
 
   // Sitegen reviews every defined section for resources to process
-  const sections: Section[] =
+  const sections: sg.Section[] =
-    require(path.join(root, "sections.ts")).siteSections;
+    require(path.join(root, "site.ts")).siteSections;
 
   // Static files are compressed and served as-is.
   // - "{section}/static/*.png"
@@ -43,8 +41,6 @@ async function sitegen(status: Spinner) {
   // Note that '.client.ts' can be placed anywhere in the file structure.
   // - "{section}/scripts/*.client.ts"
   let scripts: FileItem[] = [];
-  // 'backend.ts'
-  const backendFiles = [];
 
   // -- Scan for files --
   status.text = "Scanning Project";
@@ -88,11 +84,6 @@ async function sitegen(status: Spinner) {
         list.push({ id, file: path.join(item.parentPath, item.name) });
       }
     }
-    let backendFile = [
-      sectionPath("backend.ts"),
-      sectionPath("backend.tsx"),
-    ].find((file) => fs.existsSync(file));
-    if (backendFile) backendFiles.push(backendFile);
   }
   scripts = scripts.filter(({ file }) => !file.match(/\.client\.[tj]sx?/));
   const globalCssPath = join("global.css");
@@ -215,7 +206,7 @@ async function sitegen(status: Spinner) {
   // -- copy/compress static files --
   async function doStaticFile(item: FileItem) {
     const body = await fs.readFile(item.file);
-    incr.putAsset({
+    await incr.putAsset({
       srcId: "static:" + item.file,
       key: item.id,
       body,
@@ -250,7 +241,7 @@ async function sitegen(status: Spinner) {
               ),
           ).map((x) => `{${x}}`).join("\n"),
         });
-        incr.putAsset({
+        await incr.putAsset({
           srcId: "page:" + page.file,
           key: page.id,
           body: doc,
@@ -295,7 +286,6 @@ function wrapDocument({
   }</head><body>${body}${scripts ? `<script>${scripts}</script>` : ""}</body>`;
 }
 
-import type { Section } from "./sitegen-lib.ts";
 import { OnceMap, Queue } from "./queue.ts";
 import { Incremental } from "./incremental.ts";
 import * as bundle from "./bundle.ts";
@@ -304,6 +294,6 @@ import * as fs from "./fs.ts";
 import { Spinner, withSpinner } from "@paperclover/console/Spinner";
 import * as meta from "./meta.ts";
 import * as ssr from "./engine/ssr.ts";
-import * as sg from "./sitegen-lib.ts";
+import * as sg from "#sitegen";
 import * as hot from "./hot.ts";
 import * as path from "node:path";

framework/hot.ts

@@ -146,6 +146,7 @@ function loadEsbuildCode(module: NodeJS.Module, filepath: string, src: string) {
     target: "esnext",
     jsx: "automatic",
     jsxImportSource: "#ssr",
+    sourcefile: filepath,
   }).code;
   return module._compile(src, filepath, "commonjs");
 }
@@ -162,8 +163,6 @@ function loadMarko(module: NodeJS.Module, filepath: string) {
     ) + '\nimport { Script as CloverScriptInclude } from "#sitegen";\n';
   }
 
-  console.log(src);
-  console.log("---");
   src = marko.compileSync(src, filepath).code;
   src = src.replace("marko/debug/html", "#ssr/marko");
   return loadEsbuildCode(module, filepath, src);

framework/incremental.ts

@@ -7,7 +7,6 @@
 // all its imports) to map to the build of a page, which produces an HTML file
 // plus a list of scripts.
 
-import { Buffer } from "node:buffer";
 interface ArtifactMap {
   asset: Asset;
   script: string;
@@ -126,18 +125,17 @@ export class Incremental {
       },
       hash,
     };
+    const a = this.put({ ...info, type: "asset", value });
     if (!this.compress.has(hash)) {
       const label = info.key;
       this.compress.set(hash, {
         zstd: undefined,
         gzip: undefined,
       });
-      await Promise.all([
+      this.compressQueue.add({ label, buffer, algo: "zstd", hash });
-        this.compressQueue.add({ label, buffer, algo: "zstd", hash }),
+      this.compressQueue.add({ label, buffer, algo: "gzip", hash });
-        this.compressQueue.add({ label, buffer, algo: "gzip", hash }),
-      ]);
     }
-    return this.put({ ...info, type: "asset", value });
+    return a;
   }
 
   async compressImpl({ algo, buffer, hash }: CompressJob) {
@@ -167,16 +165,18 @@ export class Incremental {
   }
 
   async wait() {
-    await this.compressQueue.done({ method: "stop" });
+    await this.compressQueue.done({ method: "success" });
   }
+
   async flush() {
+    ASSERT(!this.compressQueue.active);
     const writer = new BufferWriter();
     const asset = Object.fromEntries(
       Array.from(this.out.asset, ([key, { buffer, hash, headers }]) => {
         const raw = writer.write(buffer, hash);
         const { gzip: gzipBuf, zstd: zstdBuf } = this.compress.get(hash) ?? {};
-        const gzip = gzipBuf ? writer.write(gzipBuf, hash) : null;
+        const gzip = gzipBuf ? writer.write(gzipBuf, hash + ".gz") : null;
-        const zstd = zstdBuf ? writer.write(zstdBuf, hash) : null;
+        const zstd = zstdBuf ? writer.write(zstdBuf, hash + ".zstd") : null;
         return [key, {
           raw,
           gzip,
@@ -252,10 +252,6 @@ export interface SerializedMeta {
   script: [key: string, value: string][];
 }
 
-function never(): never {
-  throw new Error("Impossible");
-}
-
 import * as path from "node:path";
 import * as fs from "./fs.ts";
 import * as zlib from "node:zlib";
@@ -263,3 +259,4 @@ import * as util from "node:util";
 import { Queue } from "./queue.ts";
 import * as hot from "./hot.ts";
 import * as mime from "./mime.ts";
+import { Buffer } from "node:buffer";

framework/lib/assets.ts

@@ -26,7 +26,7 @@ export async function reloadSync() {
   };
 }
 
-export async function assetMiddleware(c: Context, next: Next) {
+export async function middleware(c: Context, next: Next) {
   if (!assets) await reload();
   const asset = assets!.map[c.req.path];
   if (asset) {
@@ -35,6 +35,19 @@ export async function assetMiddleware(c: Context, next: Next) {
   return next();
 }
 
+export async function notFound(c: Context) {
+  if (!assets) await reload();
+  let pathname = c.req.path;
+  do {
+    const asset = assets!.map[pathname + "/404"];
+    if (asset) return assetInner(c, asset, 404);
+    pathname = pathname.slice(0, pathname.lastIndexOf("/"));
+  } while (pathname);
+  const asset = assets!.map["/404"];
+  if (asset) return assetInner(c, asset, 404);
+  return c.text("the 'Not Found' page was not found", 404);
+}
+
 export async function serveAsset(
   c: Context,
   id: StaticPageId,
@@ -62,14 +75,13 @@ function assetInner(c: Context, asset: BuiltAsset, status: StatusCode) {
   if (ifnonematch) {
     const etag = asset.headers.ETag;
     if (etagMatches(etag, ifnonematch)) {
-      c.res = new Response(null, {
+      return c.res = new Response(null, {
         status: 304,
         statusText: "Not Modified",
         headers: {
           ETag: etag,
         },
       });
-      return;
     }
   }
   const acceptEncoding = c.req.header("Accept-Encoding") ?? "";
@@ -90,11 +102,11 @@ function assetInner(c: Context, asset: BuiltAsset, status: StatusCode) {
   } else {
     body = subarrayAsset(asset.raw);
   }
-  c.res = new Response(body, { headers, status });
+  return c.res = new Response(body, { headers, status });
 }
 
-import * as fs from "./fs.ts";
+import * as fs from "../fs.ts";
 import type { Context, Next } from "hono";
 import type { StatusCode } from "hono/utils/http-status";
-import type { BuiltAsset, BuiltAssetMap, View } from "./incremental.ts";
+import type { BuiltAsset, BuiltAssetMap, View } from "../incremental.ts";
 import { Buffer } from "node:buffer";

framework/lib/sitegen.ts

@@ -1,6 +1,8 @@
 // Import this file with 'import * as sg from "#sitegen";'
 export type ScriptId = string;
 
+const frameworkDir = path.dirname(import.meta.dirname);
+
 export interface SitegenRender {
   scripts: Set<ScriptId>;
 }
@@ -22,7 +24,7 @@ export function getRender() {
 /** Add a client-side script to the page. */
 export function addScript(id: ScriptId) {
   const srcFile: string = util.getCallSites()
-    .find((site) => !site.scriptName.startsWith(import.meta.dirname))!
+    .find((site) => !site.scriptName.startsWith(frameworkDir))!
     .scriptName;
   const filePath = hot.resolveFrom(srcFile, id);
   if (
@@ -44,6 +46,7 @@ export interface Section {
   root: string;
 }
 
-import * as ssr from "./engine/ssr.ts";
+import * as ssr from "../engine/ssr.ts";
 import * as util from "node:util";
-import * as hot from "./hot.ts";
+import * as hot from "../hot.ts";
+import * as path from "node:path";

framework/queue.ts

@@ -165,6 +165,10 @@ export class Queue<T, R> {
 
     if (bar) bar[method]();
   }
+
+  get active(): boolean {
+    return this.#active.length !== 0;
+  }
 }
 
 const cwd = process.cwd();

framework/sqlite.ts

@@ -41,11 +41,8 @@ export class WrappedDatabase {
         (key, version) values (?, ?);
       `),
     ));
-    const { changes, lastInsertRowid } = s.run(name, 1);
+    const { changes } = s.run(name, 1);
-    console.log(changes, lastInsertRowid);
+    if (changes === 1) this.node.exec(schema);
-    if (changes === 1) {
-      this.node.exec(schema);
-    }
   }
 
   prepare<Args extends unknown[] = [], Result = unknown>(

package.json

@@ -15,7 +15,9 @@
     "typescript": "^5.8.3"
   },
   "imports": {
-    "#sitegen": "./framework/sitegen-lib.ts",
+    "#backend": "./src/backend.ts",
+    "#sitegen": "./framework/lib/sitegen.ts",
+    "#sitegen/*": "./framework/lib/*.ts",
     "#sqlite": "./framework/sqlite.ts",
     "#ssr": "./framework/engine/ssr.ts",
     "#ssr/jsx-dev-runtime": "./framework/engine/jsx-runtime.ts",
@@ -26,6 +28,7 @@
       "production": "marko/production",
       "node": "marko/debug/html"
     },
+    "#hono": "hono",
     "#hono/platform": {
       "bun": "hono/bun",
       "deno": "hono/deno",

readme.md

@@ -0,0 +1 @@
+# clover sitegen framework

repl.js

@@ -27,8 +27,7 @@ hot.load("node:repl").start({
 });
 
 setTimeout(() => {
-  hot.reloadRecursive("./framework/engine/ssr.ts");
+  hot.reloadRecursive("./framework/generate.tsx");
-  hot.reloadRecursive("./framework/bundle.ts");
 }, 100);
 
 async function evaluate(code) {
@@ -41,11 +40,11 @@ async function evaluate(code) {
   if (code[0] === "=") {
     try {
       const result = await eval(code[1]);
-      console.log(inspect(result));
+      console.info(inspect(result));
     } catch (err) {
       if (err instanceof SyntaxError) {
         const result = await eval("(async() => { return " + code + " })()");
-        console.log(inspect(result));
+        console.info(inspect(result));
       } else {
         throw err;
       }

run.js

@@ -18,7 +18,7 @@ import process from "node:process";
 const hot = await import("./framework/hot.ts");
 
 const console = hot.load("@paperclover/console");
-globalThis.console.log = console.info;
+globalThis.console["log"] = console.info;
 globalThis.console.info = console.info;
 globalThis.console.warn = console.warn;
 globalThis.console.error = console.error;

src/admin.ts

@@ -0,0 +1,64 @@
+const cookieAge = 60 * 60 * 24 * 365; // 1 year
+
+let lastKnownToken: string | null = null;
+function compareToken(token: string) {
+  if (token === lastKnownToken) return true;
+  lastKnownToken = fs.readFileSync(".clover/admin-token.txt", "utf8").trim();
+  return token === lastKnownToken;
+}
+
+export async function middleware(c: Context, next: Next) {
+  if (c.req.path.startsWith("/admin")) {
+    return adminInner(c, next);
+  }
+  return next();
+}
+
+export function adminInner(c: Context, next: Next) {
+  const token = c.req.header("Cookie")?.match(/admin-token=([^;]+)/)?.[1];
+
+  if (c.req.path === "/admin/login") {
+    const key = c.req.query("key");
+    if (key) {
+      if (compareToken(key)) {
+        return c.body(null, 303, {
+          "Location": "/admin",
+          "Set-Cookie":
+            `admin-token=${key}; Path=/; HttpOnly; SameSite=Strict; Max-Age=${cookieAge}`,
+        });
+      }
+      return serveAsset(c, "/admin/login/fail", 403);
+    }
+    if (token && compareToken(token)) {
+      return c.redirect("/admin", 303);
+    }
+    if (c.req.method === "POST") {
+      return serveAsset(c, "/admin/login/fail", 403);
+    } else {
+      return serveAsset(c, "/admin/login", 200);
+    }
+  }
+
+  if (c.req.path === "/admin/logout") {
+    return c.body(null, 303, {
+      "Location": "/admin/login",
+      "Set-Cookie":
+        `admin-token=; Path=/; HttpOnly; SameSite=Strict; Max-Age=0`,
+    });
+  }
+
+  if (token && compareToken(token)) {
+    return next();
+  }
+
+  return c.redirect("/admin/login", 303);
+}
+
+export function hasAdminToken(c: Context) {
+  const token = c.req.header("Cookie")?.match(/admin-token=([^;]+)/)?.[1];
+  return token && compareToken(token);
+}
+
+import * as fs from "fs";
+import type { Context, Next } from "hono";
+import { serveAsset } from "#sitegen/assets";

src/backend.ts

@@ -0,0 +1,38 @@
+const logHttp = scoped("http", { color: "magenta" });
+
+const app = new Hono();
+
+app.notFound(assets.notFound);
+
+app.use(trimTrailingSlash());
+app.use(removeDuplicateSlashes);
+app.use(logger((msg) => msg.startsWith("-->") && logHttp(msg.slice(4))));
+app.use(admin.middleware);
+
+app.route("", require("./q+a/backend.ts").app);
+// app.route("", require("./file-viewer/backend.tsx").app);
+// app.route("", require("./friends/backend.tsx").app);
+
+app.use(assets.middleware);
+
+export default app;
+
+async function removeDuplicateSlashes(c: Context, next: Next) {
+  const path = c.req.path;
+  if (/\/\/+/.test(path)) {
+    const normalizedPath = path.replace(/\/\/+/g, "/");
+    const query = c.req.query();
+    const queryString = Object.keys(query).length > 0
+      ? "?" + new URLSearchParams(query).toString()
+      : "";
+    return c.redirect(normalizedPath + queryString, 301);
+  }
+  await next();
+}
+
+import { Hono } from "#hono";
+import { logger } from "hono/logger";
+import { trimTrailingSlash } from "hono/trailing-slash";
+import * as assets from "#sitegen/assets";
+import * as admin from "./admin.ts";
+import { scoped } from "@paperclover/console";

src/pages/404.mdx

@@ -0,0 +1,6 @@
+export const meta = { title: 'oh no,,,' };
+
+# oh dear
+
+sound the alarms
+

src/pages/index.marko

@@ -16,7 +16,6 @@ export const meta: Meta = {
     type: "website",
     url: "https://paperclover.net",
   },
-  generator: "clover",
   alternates: {
     canonical: "https://paperclover.net",
     types: {

src/q+a/backend.ts

@@ -0,0 +1,3 @@
+export const app = new Hono();
+
+import { Hono } from "#hono";

src/q+a/components/Question.marko

@@ -30,7 +30,7 @@ static const transitionDate = 1735639200000;
 </>
 
 // this singleton script will make all the '<time>' tags clickable.
-client import "./clickable-links.ts";
+client import "./clickable-links.client.ts";
 
 import type { Question } from "@/q+a/models/Question.ts";
 import { formatQuestionTimestamp, formatQuestionISOTimestamp } from "@/q+a/format.ts";