sitegen/framework/sitegen.tsx

// Sitegen! Clover's static site generator, built with love.

export function main() {
  return withSpinner({
    text: "Recovering State",
    successText: ({ elapsed }) =>
      "sitegen! update in " + elapsed.toFixed(1) + "s",
    failureText: () => "sitegen FAIL",
  }, sitegen);
}

/**
 * A filesystem object associated with some ID,
 * such as a page's route to it's source file.
 */
interface FileItem {
  id: string;
  file: string;
}

async function sitegen(status: Spinner) {
  const startTime = performance.now();

  let root = path.resolve(import.meta.dirname, "../src");
  const join = (...sub: string[]) => path.join(root, ...sub);
  const incr = new Incremental();

  // Sitegen reviews every defined section for resources to process
  const sections: Section[] =
    require(path.join(root, "sections.ts")).siteSections;

  // Static files are compressed and served as-is.
  // - "{section}/static/*.png"
  let staticFiles: FileItem[] = [];
  // Pages are rendered then served as static files.
  // - "{section}/pages/*.marko"
  let pages: FileItem[] = [];
  // Views are dynamically rendered pages called via backend code.
  // - "{section}/views/*.tsx"
  let views: FileItem[] = [];
  // Public scripts are bundled for the client as static assets under "/js/[...]"
  // This is used for the file viewer's canvases.
  // Note that '.client.ts' can be placed anywhere in the file structure.
  // - "{section}/scripts/*.client.ts"
  let scripts: FileItem[] = [];
  // 'backend.ts'
  const backendFiles = [];

  // -- Scan for files --
  status.text = "Scanning Project";
  for (const section of sections) {
    const { root: sectionRoot } = section;
    const sectionPath = (...sub: string[]) => path.join(sectionRoot, ...sub);
    const rootPrefix = root === sectionRoot
      ? ""
      : path.relative(root, sectionRoot) + "/";
    const kinds = [
      {
        dir: sectionPath("pages"),
        list: pages,
        prefix: "/",
        exclude: [".css", ".client.ts", ".client.tsx"],
      },
      { dir: sectionPath("static"), list: staticFiles, prefix: "/", ext: true },
      { dir: sectionPath("scripts"), list: scripts, prefix: rootPrefix },
      {
        dir: sectionPath("views"),
        list: views,
        prefix: rootPrefix,
        exclude: [".css", ".client.ts", ".client.tsx"],
      },
    ];
    for (const { dir, list, prefix, exclude = [], ext = false } of kinds) {
      const items = fs.readDirRecOptionalSync(dir);
      item: for (const item of items) {
        if (item.isDirectory()) continue;
        for (const e of exclude) {
          if (item.name.endsWith(e)) continue item;
        }
        const file = path.relative(dir, item.parentPath + "/" + item.name);
        const trim = ext
          ? file
          : file.slice(0, -path.extname(file).length).replaceAll(".", "/");
        let id = prefix + trim.replaceAll("\\", "/");
        if (prefix === "/" && id.endsWith("/index")) {
          id = id.slice(0, -"/index".length) || "/";
        }
        list.push({ id, file: path.join(item.parentPath, item.name) });
      }
    }
    let backendFile = [
      sectionPath("backend.ts"),
      sectionPath("backend.tsx"),
    ].find((file) => fs.existsSync(file));
    if (backendFile) backendFiles.push(backendFile);
  }
  scripts = scripts.filter(({ file }) => !file.match(/\.client\.[tj]sx?/));
  const globalCssPath = join("global.css");

  // TODO: invalidate incremental resources

  // -- server side render --
  status.text = "Building";
  const cssOnce = new OnceMap<string>();
  const cssQueue = new Queue<[string, string[], css.Theme], string>({
    name: "Bundle",
    fn: ([, files, theme]) => css.bundleCssFiles(files, theme),
    passive: true,
    getItemText: ([id]) => id,
    maxJobs: 2,
  });
  interface RenderResult {
    body: string;
    head: string;
    inlineCss: string;
    scriptFiles: string[];
    item: FileItem;
  }
  const renderResults: RenderResult[] = [];
  async function loadPageModule({ file }: FileItem) {
    require(file);
  }
  async function renderPage(item: FileItem) {
    // -- load and validate module --
    let { default: Page, meta: metadata, theme: pageTheme, layout } = require(
      item.file,
    );
    if (!Page) throw new Error("Page is missing a 'default' export.");
    if (!metadata) {
      throw new Error("Page is missing 'meta' export with a title.");
    }
    if (layout?.theme) pageTheme = layout.theme;
    const theme = {
      bg: "#fff",
      fg: "#050505",
      primary: "#2e7dab",
      ...pageTheme,
    };

    // -- metadata --
    const renderedMetaPromise = Promise.resolve(
      typeof metadata === "function" ? metadata({ ssr: true }) : metadata,
    ).then((m) => meta.renderMeta(m));
    // -- css --
    const cssImports = [globalCssPath, ...hot.getCssImports(item.file)];
    const cssPromise = cssOnce.get(
      cssImports.join(":") + JSON.stringify(theme),
      () => cssQueue.add([item.id, cssImports, theme]),
    );
    // -- html --
    const bodyPromise = await ssr.ssrAsync(<Page />, {
      sitegen: sg.initRender(),
    });

    const [{ text, addon }, inlineCss, renderedMeta] = await Promise.all([
      bodyPromise,
      cssPromise,
      renderedMetaPromise,
    ]);
    if (!renderedMeta.includes("<title>")) {
      throw new Error(
        "Page is missing 'meta.title'. " +
          "All pages need a title tag.",
      );
    }
    // The script content is not ready, allow another page to Render. The page
    // contents will be rebuilt at the end. This is more convenient anyways
    // because it means client scripts don't re-render the page.
    renderResults.push({
      body: text,
      head: renderedMeta,
      inlineCss,
      scriptFiles: Array.from(addon.sitegen.scripts),
      item: item,
    });
  }
  // This is done in two passes so that a page that throws during evaluation
  // will report "Load Render Module" instead of "Render Static Page".
  const spinnerFormat = status.format;
  status.format = () => "";
  const moduleLoadQueue = new Queue({
    name: "Load Render Module",
    fn: loadPageModule,
    getItemText,
    maxJobs: 1,
  });
  moduleLoadQueue.addMany(pages);
  await moduleLoadQueue.done({ method: "stop" });
  const pageQueue = new Queue({
    name: "Render Static Page",
    fn: renderPage,
    getItemText,
    maxJobs: 2,
  });
  pageQueue.addMany(pages);
  await pageQueue.done({ method: "stop" });
  status.format = spinnerFormat;

  // -- bundle scripts --
  const referencedScripts = Array.from(
    new Set(renderResults.flatMap((r) => r.scriptFiles)),
  );
  const extraPublicScripts = scripts.map((entry) => entry.file);
  const uniqueCount = new Set([
    ...referencedScripts,
    ...extraPublicScripts,
  ]).size;
  status.text = `Bundle ${uniqueCount} Scripts`;
  await bundle.bundleClientJavaScript(
    referencedScripts,
    extraPublicScripts,
    incr,
  );

  // -- copy/compress static files --
  async function doStaticFile(item: FileItem) {
    const body = await fs.readFile(item.file);
    incr.putAsset({
      srcId: "static:" + item.file,
      key: item.id,
      body,
    });
  }
  const staticQueue = new Queue({
    name: "Load Static",
    fn: doStaticFile,
    getItemText,
    maxJobs: 16,
  });
  status.format = () => "";
  staticQueue.addMany(staticFiles);
  await staticQueue.done({ method: "stop" });
  status.format = spinnerFormat;

  // -- concatenate static rendered pages --
  status.text = `Concat ${renderResults.length} Pages`;
  await Promise.all(
    renderResults.map(
      async ({ item: page, body, head, inlineCss, scriptFiles }) => {
        const doc = wrapDocument({
          body,
          head,
          inlineCss,
          scripts: scriptFiles.map(
            (id) =>
              UNWRAP(
                incr.out.script.get(
                  path.basename(id).replace(/\.client\.[jt]sx?$/, ""),
                ),
              ),
          ).map((x) => `{${x}}`).join("\n"),
        });
        incr.putAsset({
          srcId: "page:" + page.file,
          key: page.id,
          body: doc,
          headers: {
            "Content-Type": "text/html",
          },
        });
      },
    ),
  );
  status.format = () => "";
  status.text = ``;
  // This will wait for all compression jobs to finish, which up
  // to this point have been left as dangling promises.
  await incr.wait();

  // Flush the site to disk.
  status.format = spinnerFormat;
  status.text = `Incremental Flush`;
  incr.flush();
  incr.serializeToDisk(); // Allows picking up this state again
  return { elapsed: (performance.now() - startTime) / 1000 };
}

function getItemText({ file }: FileItem) {
  return path.relative(hot.projectSrc, file).replaceAll("\\", "/");
}

function wrapDocument({
  body,
  head,
  inlineCss,
  scripts,
}: {
  head: string;
  body: string;
  inlineCss: string;
  scripts: string;
}) {
  return `<!doctype html><head>${head}${
    inlineCss ? `<style>${inlineCss}</style>` : ""
  }</head><body>${body}${scripts ? `<script>${scripts}</script>` : ""}</body>`;
}

import type { Section } from "./sitegen-lib.ts";
import { OnceMap, Queue } from "./queue.ts";
import { Incremental } from "./incremental.ts";
import * as bundle from "./bundle.ts";
import * as css from "./css.ts";
import * as fs from "./fs.ts";
import { Spinner, withSpinner } from "@paperclover/console/Spinner";
import * as meta from "./meta.ts";
import * as ssr from "./engine/ssr.ts";
import * as sg from "./sitegen-lib.ts";
import * as hot from "./hot.ts";
import * as path from "node:path";