453 lines
13 KiB
TypeScript
453 lines
13 KiB
TypeScript
// This file contains the main site generator build process.
|
|
// By using `Incremental`'s ability to automatically purge stale
|
|
// assets, the `sitegen` function performs partial rebuilds.
|
|
|
|
export function main() {
|
|
return withSpinner<Record<string, unknown>, any>({
|
|
text: "Recovering State",
|
|
successText,
|
|
failureText: () => "sitegen FAIL",
|
|
}, async (spinner) => {
|
|
const incr = Incremental.fromDisk();
|
|
await incr.statAllFiles();
|
|
// const incr = new Incremental();
|
|
const result = await sitegen(spinner, incr);
|
|
incr.toDisk(); // Allows picking up this state again
|
|
return result;
|
|
}) as ReturnType<typeof sitegen>;
|
|
}
|
|
|
|
export function successText({
|
|
elapsed,
|
|
inserted,
|
|
referenced,
|
|
unreferenced,
|
|
}: Awaited<ReturnType<typeof sitegen>>) {
|
|
const s = (array: unknown[]) => array.length === 1 ? "" : "s";
|
|
const kind = inserted.length === referenced.length ? "build" : "update";
|
|
const status = inserted.length > 0
|
|
? `${kind} ${inserted.length} key${s(inserted)}`
|
|
: unreferenced.length > 0
|
|
? `pruned ${unreferenced.length} key${s(unreferenced)}`
|
|
: `checked ${referenced.length} key${s(referenced)}`;
|
|
return `sitegen! ${status} in ${elapsed.toFixed(1)}s`;
|
|
}
|
|
|
|
export async function sitegen(
|
|
status: Spinner,
|
|
incr: Incremental,
|
|
) {
|
|
const startTime = performance.now();
|
|
|
|
let root = path.resolve(import.meta.dirname, "../src");
|
|
const join = (...sub: string[]) => path.join(root, ...sub);
|
|
|
|
// Sitegen reviews every defined section for resources to process
|
|
const sections: sg.Section[] =
|
|
require(path.join(root, "site.ts")).siteSections;
|
|
|
|
// Static files are compressed and served as-is.
|
|
// - "{section}/static/*.png"
|
|
let staticFiles: FileItem[] = [];
|
|
// Pages are rendered then served as static files.
|
|
// - "{section}/pages/*.marko"
|
|
let pages: FileItem[] = [];
|
|
// Views are dynamically rendered pages called via backend code.
|
|
// - "{section}/views/*.tsx"
|
|
let views: FileItem[] = [];
|
|
// Public scripts are bundled for the client as static assets under "/js/[...]"
|
|
// This is used for the file viewer's canvases.
|
|
// Note that '.client.ts' can be placed anywhere in the file structure.
|
|
// - "{section}/scripts/*.client.ts"
|
|
let scripts: FileItem[] = [];
|
|
|
|
// -- Scan for files --
|
|
status.text = "Scanning Project";
|
|
for (const section of sections) {
|
|
const { root: sectionRoot } = section;
|
|
const sectionPath = (...sub: string[]) => path.join(sectionRoot, ...sub);
|
|
const rootPrefix = root === sectionRoot
|
|
? ""
|
|
: path.relative(root, sectionRoot) + "/";
|
|
const kinds = [
|
|
{
|
|
dir: sectionPath("pages"),
|
|
list: pages,
|
|
prefix: "/",
|
|
include: [".tsx", ".mdx", ".marko"],
|
|
exclude: [".client.ts", ".client.tsx"],
|
|
},
|
|
{
|
|
dir: sectionPath("static"),
|
|
list: staticFiles,
|
|
prefix: "/",
|
|
ext: true,
|
|
},
|
|
{
|
|
dir: sectionPath("scripts"),
|
|
list: scripts,
|
|
prefix: rootPrefix,
|
|
include: [".client.ts", ".client.tsx"],
|
|
},
|
|
{
|
|
dir: sectionPath("views"),
|
|
list: views,
|
|
prefix: rootPrefix,
|
|
include: [".tsx", ".mdx", ".marko"],
|
|
exclude: [".client.ts", ".client.tsx"],
|
|
},
|
|
];
|
|
for (
|
|
const { dir, list, prefix, include = [""], exclude = [], ext = false }
|
|
of kinds
|
|
) {
|
|
const items = fs.readDirRecOptionalSync(dir);
|
|
for (const subPath of items) {
|
|
const file = path.join(dir, subPath);
|
|
const stat = fs.statSync(file);
|
|
if (stat.isDirectory()) continue;
|
|
if (!include.some((e) => subPath.endsWith(e))) continue;
|
|
if (exclude.some((e) => subPath.endsWith(e))) continue;
|
|
const trim = ext
|
|
? subPath
|
|
: subPath.slice(0, -path.extname(subPath).length).replaceAll(
|
|
".",
|
|
"/",
|
|
);
|
|
let id = prefix + trim.replaceAll("\\", "/");
|
|
if (prefix === "/" && id.endsWith("/index")) {
|
|
id = id.slice(0, -"/index".length) || "/";
|
|
}
|
|
list.push({ id, file: file });
|
|
}
|
|
}
|
|
}
|
|
scripts = scripts.filter(({ file }) => !file.match(/\.client\.[tj]sx?/));
|
|
const globalCssPath = join("global.css");
|
|
|
|
// TODO: make sure that `static` and `pages` does not overlap
|
|
|
|
// -- inline style sheets, used and shared by pages and views --
|
|
status.text = "Building";
|
|
const cssOnce = new OnceMap();
|
|
const cssQueue = new Queue({
|
|
name: "Bundle",
|
|
async fn([, key, files, theme]: [string, string, string[], css.Theme]) {
|
|
const { text, sources } = await css.bundleCssFiles(files, theme);
|
|
incr.put({
|
|
kind: "style",
|
|
key,
|
|
sources,
|
|
value: text,
|
|
});
|
|
},
|
|
passive: true,
|
|
getItemText: ([id]) => id,
|
|
maxJobs: 2,
|
|
});
|
|
function ensureCssGetsBuilt(
|
|
cssImports: string[],
|
|
theme: css.Theme,
|
|
referrer: string,
|
|
) {
|
|
const key = css.styleKey(cssImports, theme);
|
|
cssOnce.get(
|
|
key,
|
|
async () => {
|
|
incr.getArtifact("style", key) ??
|
|
await cssQueue.add([referrer, key, cssImports, theme]);
|
|
},
|
|
);
|
|
}
|
|
|
|
// -- server side render pages --
|
|
async function loadPageModule({ file }: FileItem) {
|
|
require(file);
|
|
}
|
|
async function renderPage(item: FileItem) {
|
|
// -- load and validate module --
|
|
let {
|
|
default: Page,
|
|
meta: metadata,
|
|
theme: pageTheme,
|
|
layout,
|
|
} = require(item.file);
|
|
if (!Page) {
|
|
throw new Error("Page is missing a 'default' export.");
|
|
}
|
|
if (!metadata) {
|
|
throw new Error("Page is missing 'meta' export with a title.");
|
|
}
|
|
|
|
// -- css --
|
|
if (layout?.theme) pageTheme = layout.theme;
|
|
const theme: css.Theme = {
|
|
...css.defaultTheme,
|
|
...pageTheme,
|
|
};
|
|
const cssImports = Array.from(
|
|
new Set([globalCssPath, ...hot.getCssImports(item.file)]),
|
|
(file) => path.relative(hot.projectSrc, file),
|
|
);
|
|
ensureCssGetsBuilt(cssImports, theme, item.id);
|
|
|
|
// -- metadata --
|
|
const renderedMetaPromise = Promise.resolve(
|
|
typeof metadata === "function" ? metadata({ ssr: true }) : metadata,
|
|
).then((m) => meta.renderMeta(m));
|
|
|
|
// -- html --
|
|
let page = [engine.kElement, Page, {}];
|
|
if (layout?.default) {
|
|
page = [engine.kElement, layout.default, { children: page }];
|
|
}
|
|
const bodyPromise = engine.ssrAsync(page, {
|
|
sitegen: sg.initRender(),
|
|
});
|
|
|
|
const [{ text, addon }, renderedMeta] = await Promise.all([
|
|
bodyPromise,
|
|
renderedMetaPromise,
|
|
]);
|
|
if (!renderedMeta.includes("<title>")) {
|
|
throw new Error(
|
|
"Page is missing 'meta.title'. " +
|
|
"All pages need a title tag.",
|
|
);
|
|
}
|
|
incr.put({
|
|
kind: "pageMetadata",
|
|
key: item.id,
|
|
// Incremental integrates with `hot.ts` + `require`
|
|
// to trace all the needed source files here.
|
|
sources: [item.file],
|
|
value: {
|
|
html: text,
|
|
meta: renderedMeta,
|
|
cssImports,
|
|
theme: theme ?? null,
|
|
clientRefs: Array.from(addon.sitegen.scripts),
|
|
},
|
|
});
|
|
}
|
|
async function prepareView(item: FileItem) {
|
|
const module = require(item.file);
|
|
if (!module.meta) {
|
|
throw new Error(`${item.file} is missing 'export const meta'`);
|
|
}
|
|
if (!module.default) {
|
|
throw new Error(`${item.file} is missing a default export.`);
|
|
}
|
|
const pageTheme = module.layout?.theme ?? module.theme;
|
|
const theme: css.Theme = {
|
|
...css.defaultTheme,
|
|
...pageTheme,
|
|
};
|
|
const cssImports = Array.from(
|
|
new Set([globalCssPath, ...hot.getCssImports(item.file)]),
|
|
(file) => path.relative(hot.projectSrc, file),
|
|
);
|
|
ensureCssGetsBuilt(cssImports, theme, item.id);
|
|
incr.put({
|
|
kind: "viewMetadata",
|
|
key: item.id,
|
|
sources: [item.file],
|
|
value: {
|
|
file: path.relative(hot.projectRoot, item.file),
|
|
cssImports,
|
|
theme,
|
|
clientRefs: hot.getClientScriptRefs(item.file),
|
|
hasLayout: !!module.layout?.default,
|
|
},
|
|
});
|
|
}
|
|
|
|
// Of the pages that are already built, a call to 'ensureCssGetsBuilt' is
|
|
// required so that it's (1) re-built if needed, (2) not pruned from build.
|
|
const neededPages = pages.filter((page) => {
|
|
const existing = incr.getArtifact("pageMetadata", page.id);
|
|
if (existing) {
|
|
const { cssImports, theme } = existing;
|
|
ensureCssGetsBuilt(cssImports, theme, page.id);
|
|
}
|
|
return !existing;
|
|
});
|
|
const neededViews = views.filter((view) => {
|
|
const existing = incr.getArtifact("viewMetadata", view.id);
|
|
if (existing) {
|
|
const { cssImports, theme } = existing;
|
|
ensureCssGetsBuilt(cssImports, theme, view.id);
|
|
}
|
|
return !existing;
|
|
});
|
|
|
|
// Load the marko cache before render modules are loaded
|
|
incr.loadMarkoCache();
|
|
|
|
// This is done in two passes so that a page that throws during evaluation
|
|
// will report "Load Render Module" instead of "Render Static Page".
|
|
const spinnerFormat = status.format;
|
|
status.format = () => "";
|
|
const moduleLoadQueue = new Queue({
|
|
name: "Load Render Module",
|
|
fn: loadPageModule,
|
|
getItemText,
|
|
maxJobs: 1,
|
|
});
|
|
moduleLoadQueue.addMany(neededPages);
|
|
moduleLoadQueue.addMany(neededViews);
|
|
await moduleLoadQueue.done({ method: "stop" });
|
|
const pageQueue = new Queue({
|
|
name: "Render Static Page",
|
|
fn: renderPage,
|
|
getItemText,
|
|
maxJobs: 2,
|
|
});
|
|
pageQueue.addMany(neededPages);
|
|
const viewQueue = new Queue({
|
|
name: "Build Dynamic View",
|
|
fn: prepareView,
|
|
getItemText,
|
|
maxJobs: 2,
|
|
});
|
|
viewQueue.addMany(neededViews);
|
|
await pageQueue.done({ method: "stop" });
|
|
await viewQueue.done({ method: "stop" });
|
|
status.format = spinnerFormat;
|
|
|
|
// -- bundle server javascript (backend and views) --
|
|
status.text = "Bundle JavaScript";
|
|
incr.snapshotMarkoCache();
|
|
const serverJavaScriptPromise = bundle.bundleServerJavaScript(incr, "node");
|
|
|
|
// -- bundle client javascript --
|
|
const referencedScripts = Array.from(
|
|
new Set(
|
|
[
|
|
...pages.map((item) =>
|
|
UNWRAP(
|
|
incr.getArtifact("pageMetadata", item.id),
|
|
`Missing pageMetadata ${item.id}`,
|
|
)
|
|
),
|
|
...views.map((item) =>
|
|
UNWRAP(
|
|
incr.getArtifact("viewMetadata", item.id),
|
|
`Missing viewMetadata ${item.id}`,
|
|
)
|
|
),
|
|
].flatMap((item) => item.clientRefs),
|
|
),
|
|
(script) => path.resolve(hot.projectSrc, script),
|
|
).filter((file) => !incr.hasArtifact("script", hot.getScriptId(file)));
|
|
const extraPublicScripts = scripts.map((entry) => entry.file);
|
|
const clientJavaScriptPromise = bundle.bundleClientJavaScript(
|
|
referencedScripts,
|
|
extraPublicScripts,
|
|
incr,
|
|
);
|
|
await Promise.all([
|
|
serverJavaScriptPromise,
|
|
clientJavaScriptPromise,
|
|
cssQueue.done({ method: "stop" }),
|
|
]);
|
|
await bundle.finalizeServerJavaScript(incr, "node");
|
|
|
|
// -- copy/compress static files --
|
|
async function doStaticFile(item: FileItem) {
|
|
const body = await fs.readFile(item.file);
|
|
await incr.putAsset({
|
|
sources: [item.file],
|
|
key: item.id,
|
|
body,
|
|
});
|
|
}
|
|
const staticQueue = new Queue({
|
|
name: "Load Static",
|
|
fn: doStaticFile,
|
|
getItemText,
|
|
maxJobs: 16,
|
|
});
|
|
status.format = () => "";
|
|
staticQueue.addMany(
|
|
staticFiles.filter((file) => !incr.hasArtifact("asset", file.id)),
|
|
);
|
|
await staticQueue.done({ method: "stop" });
|
|
status.format = spinnerFormat;
|
|
|
|
// -- concatenate static rendered pages --
|
|
status.text = `Concat Pages`;
|
|
await Promise.all(pages.map(async (page) => {
|
|
if (incr.hasArtifact("asset", page.id)) return;
|
|
const {
|
|
html,
|
|
meta,
|
|
cssImports,
|
|
theme,
|
|
clientRefs,
|
|
} = UNWRAP(incr.out.pageMetadata.get(page.id));
|
|
const scriptIds = clientRefs.map(hot.getScriptId);
|
|
const styleKey = css.styleKey(cssImports, theme);
|
|
const style = UNWRAP(
|
|
incr.out.style.get(styleKey),
|
|
`Missing style ${styleKey}`,
|
|
);
|
|
const doc = wrapDocument({
|
|
body: html,
|
|
head: meta,
|
|
inlineCss: style,
|
|
scripts: scriptIds.map(
|
|
(ref) => UNWRAP(incr.out.script.get(ref), `Missing script ${ref}`),
|
|
).map((x) => `{${x}}`).join("\n"),
|
|
});
|
|
await incr.putAsset({
|
|
sources: [
|
|
page.file,
|
|
...incr.sourcesFor("style", styleKey),
|
|
...scriptIds.flatMap((ref) => incr.sourcesFor("script", ref)),
|
|
],
|
|
key: page.id,
|
|
body: doc,
|
|
headers: {
|
|
"Content-Type": "text/html",
|
|
},
|
|
});
|
|
}));
|
|
status.format = () => "";
|
|
status.text = ``;
|
|
// This will wait for all compression jobs to finish, which up
|
|
// to this point have been left as dangling promises.
|
|
await incr.wait();
|
|
|
|
const { inserted, referenced, unreferenced } = incr.shake();
|
|
|
|
// Flush the site to disk.
|
|
status.format = spinnerFormat;
|
|
status.text = `Incremental Flush`;
|
|
incr.flush("node"); // Write outputs
|
|
return {
|
|
incr,
|
|
inserted,
|
|
referenced,
|
|
unreferenced,
|
|
elapsed: (performance.now() - startTime) / 1000,
|
|
};
|
|
}
|
|
|
|
function getItemText({ file }: FileItem) {
|
|
return path.relative(hot.projectSrc, file).replaceAll("\\", "/");
|
|
}
|
|
|
|
import { OnceMap, Queue } from "#sitegen/async";
|
|
import { Incremental } from "./incremental.ts";
|
|
import * as bundle from "./bundle.ts";
|
|
import * as css from "./css.ts";
|
|
import * as engine from "./engine/ssr.ts";
|
|
import * as hot from "./hot.ts";
|
|
import * as fs from "#sitegen/fs";
|
|
import * as sg from "#sitegen";
|
|
import type { FileItem } from "#sitegen";
|
|
import * as path from "node:path";
|
|
import * as meta from "#sitegen/meta";
|
|
import { Spinner, withSpinner } from "@paperclover/console/Spinner";
|
|
import { wrapDocument } from "./lib/view.ts";
|