sitegen/framework/generate.ts

// This file contains the main site generator build process.
// By using `Incremental`'s ability to automatically purge stale
// assets, the `sitegen` function performs partial rebuilds.

export function main() {
  return withSpinner<Record<string, unknown>, any>({
    text: "Recovering State",
    successText,
    failureText: () => "sitegen FAIL",
  }, async (spinner) => {
    // const incr = Incremental.fromDisk();
    // await incr.statAllFiles();
    const incr = new Incremental();
    const result = await sitegen(spinner, incr);
    incr.toDisk(); // Allows picking up this state again
    return result;
  }) as ReturnType<typeof sitegen>;
}

export function successText({
  elapsed,
  inserted,
  referenced,
  unreferenced,
}: Awaited<ReturnType<typeof sitegen>>) {
  const s = (array: unknown[]) => array.length === 1 ? "" : "s";
  const kind = inserted.length === referenced.length ? "build" : "update";
  const status = inserted.length > 0
    ? `${kind} ${inserted.length} key${s(inserted)}`
    : unreferenced.length > 0
    ? `pruned ${unreferenced.length} key${s(unreferenced)}`
    : `checked ${referenced.length} key${s(referenced)}`;
  return `sitegen! ${status} in ${elapsed.toFixed(1)}s`;
}

export async function sitegen(
  status: Spinner,
  incr: Incremental,
) {
  const startTime = performance.now();

  let root = path.resolve(import.meta.dirname, "../src");
  const join = (...sub: string[]) => path.join(root, ...sub);

  // Sitegen reviews every defined section for resources to process
  const sections: sg.Section[] =
    require(path.join(root, "site.ts")).siteSections;

  // Static files are compressed and served as-is.
  // - "{section}/static/*.png"
  let staticFiles: FileItem[] = [];
  // Pages are rendered then served as static files.
  // - "{section}/pages/*.marko"
  let pages: FileItem[] = [];
  // Views are dynamically rendered pages called via backend code.
  // - "{section}/views/*.tsx"
  let views: FileItem[] = [];
  // Public scripts are bundled for the client as static assets under "/js/[...]"
  // This is used for the file viewer's canvases.
  // Note that '.client.ts' can be placed anywhere in the file structure.
  // - "{section}/scripts/*.client.ts"
  let scripts: FileItem[] = [];

  // -- Scan for files --
  status.text = "Scanning Project";
  for (const section of sections) {
    const { root: sectionRoot } = section;
    const sectionPath = (...sub: string[]) => path.join(sectionRoot, ...sub);
    const rootPrefix = root === sectionRoot
      ? ""
      : path.relative(root, sectionRoot) + "/";
    const kinds = [
      {
        dir: sectionPath("pages"),
        list: pages,
        prefix: "/",
        include: [".tsx", ".mdx", ".marko"],
        exclude: [".client.ts", ".client.tsx"],
      },
      {
        dir: sectionPath("static"),
        list: staticFiles,
        prefix: "/",
        ext: true,
      },
      {
        dir: sectionPath("scripts"),
        list: scripts,
        prefix: rootPrefix,
        include: [".client.ts", ".client.tsx"],
      },
      {
        dir: sectionPath("views"),
        list: views,
        prefix: rootPrefix,
        include: [".tsx", ".mdx", ".marko"],
        exclude: [".client.ts", ".client.tsx"],
      },
    ];
    for (
      const { dir, list, prefix, include = [""], exclude = [], ext = false }
        of kinds
    ) {
      const items = fs.readDirRecOptionalSync(dir);
      for (const subPath of items) {
        const file = path.join(dir, subPath);
        const stat = fs.statSync(file);
        if (stat.isDirectory()) continue;
        if (!include.some((e) => subPath.endsWith(e))) continue;
        if (exclude.some((e) => subPath.endsWith(e))) continue;
        const trim = ext
          ? subPath
          : subPath.slice(0, -path.extname(subPath).length).replaceAll(
            ".",
            "/",
          );
        let id = prefix + trim.replaceAll("\\", "/");
        if (prefix === "/" && id.endsWith("/index")) {
          id = id.slice(0, -"/index".length) || "/";
        }
        list.push({ id, file: file });
      }
    }
  }
  const globalCssPath = join("global.css");

  // TODO: make sure that `static` and `pages` does not overlap

  // -- inline style sheets, used and shared by pages and views --
  status.text = "Building";
  const cssOnce = new OnceMap();
  const cssQueue = new Queue({
    name: "Bundle",
    async fn([, key, files, theme]: [string, string, string[], css.Theme]) {
      const { text, sources } = await css.bundleCssFiles(files, theme);
      incr.put({
        kind: "style",
        key,
        sources,
        value: text,
      });
    },
    passive: true,
    getItemText: ([id]) => id,
    maxJobs: 2,
  });
  function ensureCssGetsBuilt(
    cssImports: string[],
    theme: css.Theme,
    referrer: string,
  ) {
    const key = css.styleKey(cssImports, theme);
    cssOnce.get(
      key,
      async () => {
        incr.getArtifact("style", key) ??
          await cssQueue.add([referrer, key, cssImports, theme]);
      },
    );
  }

  // -- server side render pages --
  async function loadPageModule({ file }: FileItem) {
    require(file);
  }
  async function renderPage(item: FileItem) {
    // -- load and validate module --
    let {
      default: Page,
      meta: metadata,
      theme: pageTheme,
      layout,
    } = require(item.file);
    if (!Page) {
      throw new Error("Page is missing a 'default' export.");
    }
    if (!metadata) {
      throw new Error("Page is missing 'meta' export with a title.");
    }

    // -- css --
    if (layout?.theme) pageTheme = layout.theme;
    const theme: css.Theme = {
      ...css.defaultTheme,
      ...pageTheme,
    };
    const cssImports = Array.from(
      new Set([globalCssPath, ...hot.getCssImports(item.file)]),
      (file) => path.relative(hot.projectSrc, file),
    );
    ensureCssGetsBuilt(cssImports, theme, item.id);

    // -- metadata --
    const renderedMetaPromise = Promise.resolve(
      typeof metadata === "function" ? metadata({ ssr: true }) : metadata,
    ).then((m) => meta.renderMeta(m));

    // -- html --
    let page = [engine.kElement, Page, {}];
    if (layout?.default) {
      page = [engine.kElement, layout.default, { children: page }];
    }
    const bodyPromise = engine.ssrAsync(page, {
      sitegen: sg.initRender(),
    });

    const [{ text, addon }, renderedMeta] = await Promise.all([
      bodyPromise,
      renderedMetaPromise,
    ]);
    if (!renderedMeta.includes("<title>")) {
      throw new Error(
        "Page is missing 'meta.title'. " +
          "All pages need a title tag.",
      );
    }
    incr.put({
      kind: "pageMetadata",
      key: item.id,
      // Incremental integrates with `hot.ts` + `require`
      // to trace all the needed source files here.
      sources: [item.file],
      value: {
        html: text,
        meta: renderedMeta,
        cssImports,
        theme: theme ?? null,
        clientRefs: Array.from(addon.sitegen.scripts),
      },
    });
  }
  async function prepareView(item: FileItem) {
    const module = require(item.file);
    if (!module.meta) {
      throw new Error(`${item.file} is missing 'export const meta'`);
    }
    if (!module.default) {
      throw new Error(`${item.file} is missing a default export.`);
    }
    const pageTheme = module.layout?.theme ?? module.theme;
    const theme: css.Theme = {
      ...css.defaultTheme,
      ...pageTheme,
    };
    const cssImports = Array.from(
      new Set([globalCssPath, ...hot.getCssImports(item.file)]),
      (file) => path.relative(hot.projectSrc, file),
    );
    ensureCssGetsBuilt(cssImports, theme, item.id);
    incr.put({
      kind: "viewMetadata",
      key: item.id,
      sources: [item.file],
      value: {
        file: path.relative(hot.projectRoot, item.file),
        cssImports,
        theme,
        clientRefs: hot.getClientScriptRefs(item.file),
        hasLayout: !!module.layout?.default,
      },
    });
  }

  // Of the pages that are already built, a call to 'ensureCssGetsBuilt' is
  // required so that it's (1) re-built if needed, (2) not pruned from build.
  const neededPages = pages.filter((page) => {
    const existing = incr.getArtifact("pageMetadata", page.id);
    if (existing) {
      const { cssImports, theme } = existing;
      ensureCssGetsBuilt(cssImports, theme, page.id);
    }
    return !existing;
  });
  const neededViews = views.filter((view) => {
    const existing = incr.getArtifact("viewMetadata", view.id);
    if (existing) {
      const { cssImports, theme } = existing;
      ensureCssGetsBuilt(cssImports, theme, view.id);
    }
    return !existing;
  });

  // Load the marko cache before render modules are loaded
  incr.loadMarkoCache();

  // This is done in two passes so that a page that throws during evaluation
  // will report "Load Render Module" instead of "Render Static Page".
  const spinnerFormat = status.format;
  status.format = () => "";
  const moduleLoadQueue = new Queue({
    name: "Load Render Module",
    fn: loadPageModule,
    getItemText,
    maxJobs: 1,
  });
  moduleLoadQueue.addMany(neededPages);
  moduleLoadQueue.addMany(neededViews);
  await moduleLoadQueue.done({ method: "stop" });
  const pageQueue = new Queue({
    name: "Render Static Page",
    fn: renderPage,
    getItemText,
    maxJobs: 2,
  });
  pageQueue.addMany(neededPages);
  const viewQueue = new Queue({
    name: "Build Dynamic View",
    fn: prepareView,
    getItemText,
    maxJobs: 2,
  });
  viewQueue.addMany(neededViews);
  const pageAndViews = [
    pageQueue.done({ method: "stop" }),
    viewQueue.done({ method: "stop" }),
  ];
  await Promise.allSettled(pageAndViews);
  await Promise.all(pageAndViews);
  status.format = spinnerFormat;

  // -- bundle server javascript (backend and views) --
  status.text = "Bundle JavaScript";
  incr.snapshotMarkoCache();
  const serverJavaScriptPromise = bundle.bundleServerJavaScript(incr, "node");

  // -- bundle client javascript --
  const referencedScripts = Array.from(
    new Set(
      [
        ...pages.map((item) =>
          UNWRAP(
            incr.getArtifact("pageMetadata", item.id),
            `Missing pageMetadata ${item.id}`,
          )
        ),
        ...views.map((item) =>
          UNWRAP(
            incr.getArtifact("viewMetadata", item.id),
            `Missing viewMetadata ${item.id}`,
          )
        ),
      ].flatMap((item) => item.clientRefs),
    ),
    (script) => path.resolve(hot.projectSrc, script),
  ).filter((file) => !incr.hasArtifact("script", hot.getScriptId(file)));
  const extraPublicScripts = scripts.map((entry) => entry.file);
  const clientJavaScriptPromise = bundle.bundleClientJavaScript(
    referencedScripts,
    extraPublicScripts,
    incr,
  );
  await Promise.all([
    serverJavaScriptPromise,
    clientJavaScriptPromise,
    cssQueue.done({ method: "stop" }),
  ]);
  await bundle.finalizeServerJavaScript(incr, "node");

  // -- copy/compress static files --
  async function doStaticFile(item: FileItem) {
    const body = await fs.readFile(item.file);
    await incr.putAsset({
      sources: [item.file],
      key: item.id,
      body,
    });
  }
  const staticQueue = new Queue({
    name: "Load Static",
    fn: doStaticFile,
    getItemText,
    maxJobs: 16,
  });
  status.format = () => "";
  staticQueue.addMany(
    staticFiles.filter((file) => !incr.hasArtifact("asset", file.id)),
  );
  await staticQueue.done({ method: "stop" });
  status.format = spinnerFormat;

  // -- concatenate static rendered pages --
  status.text = `Concat Pages`;
  await Promise.all(pages.map(async (page) => {
    if (incr.hasArtifact("asset", page.id)) return;
    const {
      html,
      meta,
      cssImports,
      theme,
      clientRefs,
    } = UNWRAP(incr.out.pageMetadata.get(page.id));
    const scriptIds = clientRefs.map(hot.getScriptId);
    const styleKey = css.styleKey(cssImports, theme);
    const style = UNWRAP(
      incr.out.style.get(styleKey),
      `Missing style ${styleKey}`,
    );
    const doc = wrapDocument({
      body: html,
      head: meta,
      inlineCss: style,
      scripts: scriptIds.map(
        (ref) => UNWRAP(incr.out.script.get(ref), `Missing script ${ref}`),
      ).map((x) => `{${x}}`).join("\n"),
    });
    await incr.putAsset({
      sources: [
        page.file,
        ...incr.sourcesFor("style", styleKey),
        ...scriptIds.flatMap((ref) => incr.sourcesFor("script", ref)),
      ],
      key: page.id,
      body: doc,
      headers: {
        "Content-Type": "text/html",
      },
    });
  }));
  status.format = () => "";
  status.text = ``;
  // This will wait for all compression jobs to finish, which up
  // to this point have been left as dangling promises.
  await incr.wait();

  const { inserted, referenced, unreferenced } = incr.shake();

  // Flush the site to disk.
  status.format = spinnerFormat;
  status.text = `Incremental Flush`;
  incr.flush("node"); // Write outputs
  return {
    incr,
    inserted,
    referenced,
    unreferenced,
    elapsed: (performance.now() - startTime) / 1000,
  };
}

function getItemText({ file }: FileItem) {
  return path.relative(hot.projectSrc, file).replaceAll("\\", "/");
}

import { OnceMap, Queue } from "#sitegen/async";
import { Incremental } from "./incremental.ts";
import * as bundle from "./bundle.ts";
import * as css from "./css.ts";
import * as engine from "./engine/ssr.ts";
import * as hot from "./hot.ts";
import * as fs from "#sitegen/fs";
import * as sg from "#sitegen";
import type { FileItem } from "#sitegen";
import * as path from "node:path";
import * as meta from "#sitegen/meta";
import { Spinner, withSpinner } from "@paperclover/console/Spinner";
import { wrapDocument } from "./lib/view.ts";