316 lines
9.7 KiB
TypeScript
316 lines
9.7 KiB
TypeScript
export function main() {
|
|
return withSpinner({
|
|
text: "Recovering State",
|
|
successText: ({ elapsed }) =>
|
|
"sitegen! update in " + elapsed.toFixed(1) + "s",
|
|
failureText: () => "sitegen FAIL",
|
|
}, sitegen);
|
|
}
|
|
|
|
async function sitegen(status: Spinner) {
|
|
const startTime = performance.now();
|
|
|
|
let root = path.resolve(import.meta.dirname, "../src");
|
|
const join = (...sub: string[]) => path.join(root, ...sub);
|
|
const incr = new Incremental();
|
|
// const incr = Incremental.fromDisk();
|
|
await incr.statAllFiles();
|
|
|
|
// Sitegen reviews every defined section for resources to process
|
|
const sections: sg.Section[] =
|
|
require(path.join(root, "site.ts")).siteSections;
|
|
|
|
// Static files are compressed and served as-is.
|
|
// - "{section}/static/*.png"
|
|
let staticFiles: FileItem[] = [];
|
|
// Pages are rendered then served as static files.
|
|
// - "{section}/pages/*.marko"
|
|
let pages: FileItem[] = [];
|
|
// Views are dynamically rendered pages called via backend code.
|
|
// - "{section}/views/*.tsx"
|
|
let views: FileItem[] = [];
|
|
// Public scripts are bundled for the client as static assets under "/js/[...]"
|
|
// This is used for the file viewer's canvases.
|
|
// Note that '.client.ts' can be placed anywhere in the file structure.
|
|
// - "{section}/scripts/*.client.ts"
|
|
let scripts: FileItem[] = [];
|
|
|
|
// -- Scan for files --
|
|
status.text = "Scanning Project";
|
|
for (const section of sections) {
|
|
const { root: sectionRoot } = section;
|
|
const sectionPath = (...sub: string[]) => path.join(sectionRoot, ...sub);
|
|
const rootPrefix = root === sectionRoot
|
|
? ""
|
|
: path.relative(root, sectionRoot) + "/";
|
|
const kinds = [
|
|
{
|
|
dir: sectionPath("pages"),
|
|
list: pages,
|
|
prefix: "/",
|
|
exclude: [".css", ".client.ts", ".client.tsx"],
|
|
},
|
|
{ dir: sectionPath("static"), list: staticFiles, prefix: "/", ext: true },
|
|
{ dir: sectionPath("scripts"), list: scripts, prefix: rootPrefix },
|
|
{
|
|
dir: sectionPath("views"),
|
|
list: views,
|
|
prefix: rootPrefix,
|
|
exclude: [".css", ".client.ts", ".client.tsx"],
|
|
},
|
|
];
|
|
for (const { dir, list, prefix, exclude = [], ext = false } of kinds) {
|
|
const items = fs.readDirRecOptionalSync(dir);
|
|
item: for (const subPath of items) {
|
|
const file = path.join(dir, subPath);
|
|
const stat = fs.statSync(file);
|
|
if (stat.isDirectory()) continue;
|
|
for (const e of exclude) {
|
|
if (subPath.endsWith(e)) continue item;
|
|
}
|
|
const trim = ext
|
|
? subPath
|
|
: subPath.slice(0, -path.extname(subPath).length).replaceAll(
|
|
".",
|
|
"/",
|
|
);
|
|
let id = prefix + trim.replaceAll("\\", "/");
|
|
if (prefix === "/" && id.endsWith("/index")) {
|
|
id = id.slice(0, -"/index".length) || "/";
|
|
}
|
|
list.push({ id, file: file });
|
|
}
|
|
}
|
|
}
|
|
scripts = scripts.filter(({ file }) => !file.match(/\.client\.[tj]sx?/));
|
|
const globalCssPath = join("global.css");
|
|
|
|
// TODO: invalidate incremental resources
|
|
|
|
// -- server side render --
|
|
status.text = "Building";
|
|
const cssOnce = new OnceMap<css.Output>();
|
|
const cssQueue = new Queue<[string, string[], css.Theme], css.Output>({
|
|
name: "Bundle",
|
|
fn: ([, files, theme]) => css.bundleCssFiles(files, theme),
|
|
passive: true,
|
|
getItemText: ([id]) => id,
|
|
maxJobs: 2,
|
|
});
|
|
interface RenderResult {
|
|
body: string;
|
|
head: string;
|
|
css: css.Output;
|
|
clientRefs: string[];
|
|
item: FileItem;
|
|
}
|
|
const renderResults: RenderResult[] = [];
|
|
async function loadPageModule({ file }: FileItem) {
|
|
require(file);
|
|
}
|
|
async function renderPage(item: FileItem) {
|
|
// -- load and validate module --
|
|
let {
|
|
default: Page,
|
|
meta: metadata,
|
|
theme: pageTheme,
|
|
layout,
|
|
} = require(item.file);
|
|
if (!Page) throw new Error("Page is missing a 'default' export.");
|
|
if (!metadata) {
|
|
throw new Error("Page is missing 'meta' export with a title.");
|
|
}
|
|
if (layout?.theme) pageTheme = layout.theme;
|
|
const theme = {
|
|
bg: "#fff",
|
|
fg: "#050505",
|
|
primary: "#2e7dab",
|
|
...pageTheme,
|
|
};
|
|
|
|
// -- metadata --
|
|
const renderedMetaPromise = Promise.resolve(
|
|
typeof metadata === "function" ? metadata({ ssr: true }) : metadata,
|
|
).then((m) => meta.renderMeta(m));
|
|
// -- css --
|
|
const cssImports = [globalCssPath, ...hot.getCssImports(item.file)];
|
|
const cssPromise = cssOnce.get(
|
|
cssImports.join(":") + JSON.stringify(theme),
|
|
() => cssQueue.add([item.id, cssImports, theme]),
|
|
);
|
|
// -- html --
|
|
let page = [engine.kElement, Page, {}];
|
|
if (layout?.default) {
|
|
page = [engine.kElement, layout.default, { children: page }];
|
|
}
|
|
const bodyPromise = engine.ssrAsync(page, {
|
|
sitegen: sg.initRender(),
|
|
});
|
|
|
|
const [{ text, addon }, cssBundle, renderedMeta] = await Promise.all([
|
|
bodyPromise,
|
|
cssPromise,
|
|
renderedMetaPromise,
|
|
]);
|
|
if (!renderedMeta.includes("<title>")) {
|
|
throw new Error(
|
|
"Page is missing 'meta.title'. " +
|
|
"All pages need a title tag.",
|
|
);
|
|
}
|
|
// The script content is not ready, allow another page to Render. The page
|
|
// contents will be rebuilt at the end. This is more convenient anyways
|
|
// because it means client scripts don't re-render the page.
|
|
renderResults.push({
|
|
body: text,
|
|
head: renderedMeta,
|
|
css: cssBundle,
|
|
clientRefs: Array.from(addon.sitegen.scripts),
|
|
item: item,
|
|
});
|
|
}
|
|
// This is done in two passes so that a page that throws during evaluation
|
|
// will report "Load Render Module" instead of "Render Static Page".
|
|
const neededPages = pages.filter((page) => incr.needsBuild("asset", page.id));
|
|
const spinnerFormat = status.format;
|
|
status.format = () => "";
|
|
const moduleLoadQueue = new Queue({
|
|
name: "Load Render Module",
|
|
fn: loadPageModule,
|
|
getItemText,
|
|
maxJobs: 1,
|
|
});
|
|
moduleLoadQueue.addMany(neededPages);
|
|
await moduleLoadQueue.done({ method: "stop" });
|
|
const pageQueue = new Queue({
|
|
name: "Render Static Page",
|
|
fn: renderPage,
|
|
getItemText,
|
|
maxJobs: 2,
|
|
});
|
|
pageQueue.addMany(neededPages);
|
|
await pageQueue.done({ method: "stop" });
|
|
status.format = spinnerFormat;
|
|
|
|
// -- bundle backend and views --
|
|
status.text = "Bundle backend code";
|
|
const backend = await bundle.bundleServerJavaScript(
|
|
join("backend.ts"),
|
|
views,
|
|
);
|
|
const viewCssPromise = await Promise.all(
|
|
backend.views.map((view) =>
|
|
cssOnce.get(
|
|
view.cssImports.join(":") + JSON.stringify(view.theme),
|
|
() => cssQueue.add([view.id, view.cssImports, view.theme ?? {}]),
|
|
)
|
|
),
|
|
);
|
|
|
|
// -- bundle scripts --
|
|
const referencedScripts = Array.from(
|
|
new Set([
|
|
...renderResults.flatMap((r) => r.clientRefs),
|
|
...backend.views.flatMap((r) => r.clientRefs),
|
|
]),
|
|
(script) => path.join(hot.projectSrc, script),
|
|
);
|
|
const extraPublicScripts = scripts.map((entry) => entry.file);
|
|
const uniqueCount = new Set([
|
|
...referencedScripts,
|
|
...extraPublicScripts,
|
|
]).size;
|
|
status.text = `Bundle ${uniqueCount} Scripts`;
|
|
await bundle.bundleClientJavaScript(
|
|
referencedScripts,
|
|
extraPublicScripts,
|
|
incr,
|
|
);
|
|
|
|
// -- finalize backend bundle --
|
|
await bundle.finalizeServerJavaScript(backend, await viewCssPromise, incr);
|
|
|
|
// -- copy/compress static files --
|
|
async function doStaticFile(item: FileItem) {
|
|
const body = await fs.readFile(item.file);
|
|
await incr.putAsset({
|
|
sources: [item.file],
|
|
key: item.id,
|
|
body,
|
|
});
|
|
}
|
|
const staticQueue = new Queue({
|
|
name: "Load Static",
|
|
fn: doStaticFile,
|
|
getItemText,
|
|
maxJobs: 16,
|
|
});
|
|
status.format = () => "";
|
|
staticQueue.addMany(
|
|
staticFiles.filter((file) => incr.needsBuild("asset", file.id)),
|
|
);
|
|
await staticQueue.done({ method: "stop" });
|
|
status.format = spinnerFormat;
|
|
|
|
// -- concatenate static rendered pages --
|
|
status.text = `Concat ${renderResults.length} Pages`;
|
|
await Promise.all(
|
|
renderResults.map(
|
|
async (
|
|
{ item: page, body, head, css, clientRefs: scriptFiles },
|
|
) => {
|
|
const doc = wrapDocument({
|
|
body,
|
|
head,
|
|
inlineCss: css.text,
|
|
scripts: scriptFiles.map(
|
|
(id) =>
|
|
UNWRAP(
|
|
incr.out.script.get(
|
|
path.basename(id).replace(/\.client\.[jt]sx?$/, ""),
|
|
),
|
|
),
|
|
).map((x) => `{${x}}`).join("\n"),
|
|
});
|
|
await incr.putAsset({
|
|
sources: [page.file, ...css.sources],
|
|
key: page.id,
|
|
body: doc,
|
|
headers: {
|
|
"Content-Type": "text/html",
|
|
},
|
|
});
|
|
},
|
|
),
|
|
);
|
|
status.format = () => "";
|
|
status.text = ``;
|
|
// This will wait for all compression jobs to finish, which up
|
|
// to this point have been left as dangling promises.
|
|
await incr.wait();
|
|
|
|
// Flush the site to disk.
|
|
status.format = spinnerFormat;
|
|
status.text = `Incremental Flush`;
|
|
incr.flush(); // Write outputs
|
|
incr.toDisk(); // Allows picking up this state again
|
|
return { elapsed: (performance.now() - startTime) / 1000 };
|
|
}
|
|
|
|
function getItemText({ file }: FileItem) {
|
|
return path.relative(hot.projectSrc, file).replaceAll("\\", "/");
|
|
}
|
|
|
|
import { OnceMap, Queue } from "./queue.ts";
|
|
import { Incremental } from "./incremental.ts";
|
|
import * as bundle from "./bundle.ts";
|
|
import * as css from "./css.ts";
|
|
import * as engine from "./engine/ssr.ts";
|
|
import * as hot from "./hot.ts";
|
|
import * as fs from "#sitegen/fs";
|
|
import * as sg from "#sitegen";
|
|
import type { FileItem } from "#sitegen";
|
|
import * as path from "node:path";
|
|
import * as meta from "#sitegen/meta";
|
|
import { Spinner, withSpinner } from "@paperclover/console/Spinner";
|
|
import { wrapDocument } from "./lib/view.ts";
|