From 49bd6bc3ffe1d3507e00bae62c12d9b045363090 Mon Sep 17 00:00:00 2001 From: Jacky Zhao Date: Tue, 8 Aug 2023 22:52:49 -0700 Subject: [PATCH] better concurrency debugging, --concurrency flag for npx quartz build --- .gitignore | 1 + content/build.md | 1 + quartz/bootstrap-cli.mjs | 4 ++++ quartz/build.ts | 21 +++------------------ quartz/ctx.ts | 1 + quartz/processors/parse.ts | 10 ++++++++-- quartz/sourcemap.ts | 19 +++++++++++++++++++ quartz/trace.ts | 30 ++++++++++++++++++++++-------- quartz/worker.ts | 3 +++ 9 files changed, 62 insertions(+), 28 deletions(-) create mode 100644 quartz/sourcemap.ts diff --git a/.gitignore b/.gitignore index 9be70c6..5177d66 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ public tsconfig.tsbuildinfo .obsidian .quartz-cache +private/ diff --git a/content/build.md b/content/build.md index 541c9d2..bce9d54 100644 --- a/content/build.md +++ b/content/build.md @@ -24,3 +24,4 @@ Once you're happy with it, let's see how to [[hosting|deploy Quartz to the web]] > - `-o` or `--output`: the output folder. This is normally just `public` > - `--serve`: run a local hot-reloading server to preview your Quartz > - `--port`: what port to run the local preview server on +> - `--concurrency`: how many threads to use to parse notes diff --git a/quartz/bootstrap-cli.mjs b/quartz/bootstrap-cli.mjs index 71d6957..adcdb9f 100755 --- a/quartz/bootstrap-cli.mjs +++ b/quartz/bootstrap-cli.mjs @@ -84,6 +84,10 @@ const BuildArgv = { default: false, describe: "show detailed bundle information", }, + concurrency: { + number: true, + describe: "how many threads to use to parse notes" + } } function escapePath(fp) { diff --git a/quartz/build.ts b/quartz/build.ts index e6dc96a..c25efbf 100644 --- a/quartz/build.ts +++ b/quartz/build.ts @@ -1,19 +1,5 @@ import sourceMapSupport from "source-map-support" -sourceMapSupport.install({ - retrieveSourceMap(source) { - // source map hack to get around query param - // import cache busting - if (source.includes(".quartz-cache")) { - let realSource = fileURLToPath(source.split("?", 2)[0] + ".map") - return { - map: fs.readFileSync(realSource, "utf8"), - } - } else { - return null - } - }, -}) - +sourceMapSupport.install(options) import path from "path" import { PerfTimer } from "./perf" import { rimraf } from "rimraf" @@ -23,14 +9,13 @@ import { parseMarkdown } from "./processors/parse" import { filterContent } from "./processors/filter" import { emitContent } from "./processors/emit" import cfg from "../quartz.config" -import { FilePath, ServerSlug, joinSegments, slugifyFilePath } from "./path" +import { FilePath, joinSegments, slugifyFilePath } from "./path" import chokidar from "chokidar" import { ProcessedContent } from "./plugins/vfile" import { Argv, BuildCtx } from "./ctx" import { glob, toPosixPath } from "./glob" import { trace } from "./trace" -import { fileURLToPath } from "url" -import fs from "fs" +import { options } from "./sourcemap" async function buildQuartz(argv: Argv, clientRefresh: () => void) { const ctx: BuildCtx = { diff --git a/quartz/ctx.ts b/quartz/ctx.ts index 8a7b803..dad5cef 100644 --- a/quartz/ctx.ts +++ b/quartz/ctx.ts @@ -7,6 +7,7 @@ export interface Argv { output: string serve: boolean port: number + concurrency?: number } export interface BuildCtx { diff --git a/quartz/processors/parse.ts b/quartz/processors/parse.ts index 52dc519..1340020 100644 --- a/quartz/processors/parse.ts +++ b/quartz/processors/parse.ts @@ -56,6 +56,8 @@ async function transpileWorkerScript() { platform: "node", format: "esm", packages: "external", + sourcemap: true, + sourcesContent: false, plugins: [ { name: "css-and-scripts-as-text", @@ -116,7 +118,7 @@ export async function parseMarkdown(ctx: BuildCtx, fps: FilePath[]): Promise { + const errString = err.toString().slice("Error:".length) + console.error(errString) + process.exit(1) + }) res = results.flat() await pool.terminate() } diff --git a/quartz/sourcemap.ts b/quartz/sourcemap.ts new file mode 100644 index 0000000..8b05c8b --- /dev/null +++ b/quartz/sourcemap.ts @@ -0,0 +1,19 @@ +import fs from "fs" +import sourceMapSupport from "source-map-support" +import { fileURLToPath } from "url" + +export const options: sourceMapSupport.Options = { + // source map hack to get around query param + // import cache busting + retrieveSourceMap(source) { + if (source.includes(".quartz-cache")) { + let realSource = fileURLToPath(source.split("?", 2)[0] + ".map") + return { + map: fs.readFileSync(realSource, "utf8"), + } + } else { + return null + } + }, +} + diff --git a/quartz/trace.ts b/quartz/trace.ts index 337ffe0..ca3a5a0 100644 --- a/quartz/trace.ts +++ b/quartz/trace.ts @@ -1,17 +1,22 @@ import chalk from "chalk" import process from "process" +import { isMainThread } from "workerpool" const rootFile = /.*at file:/ export function trace(msg: string, err: Error) { const stack = err.stack - console.log() - console.log( + + const lines: string[] = [] + + lines.push("") + lines.push( "\n" + - chalk.bgRed.black.bold(" ERROR ") + - "\n" + - chalk.red(` ${msg}`) + - (err.message.length > 0 ? `: ${err.message}` : ""), + chalk.bgRed.black.bold(" ERROR ") + + "\n" + + chalk.red(` ${msg}`) + + (err.message.length > 0 ? `: ${err.message}` : ""), ) + if (!stack) { return } @@ -23,11 +28,20 @@ export function trace(msg: string, err: Error) { } if (!line.includes("node_modules")) { - console.log(` ${line}`) + lines.push(` ${line}`) if (rootFile.test(line)) { reachedEndOfLegibleTrace = true } } } - process.exit(1) + + const traceMsg = lines.join("\n") + if (!isMainThread) { + // gather lines and throw + throw new Error(traceMsg) + } else { + // print and exit + console.error(traceMsg) + process.exit(1) + } } diff --git a/quartz/worker.ts b/quartz/worker.ts index d97c483..db11cbb 100644 --- a/quartz/worker.ts +++ b/quartz/worker.ts @@ -1,7 +1,10 @@ +import sourceMapSupport from "source-map-support" +sourceMapSupport.install(options) import cfg from "../quartz.config" import { Argv, BuildCtx } from "./ctx" import { FilePath, ServerSlug } from "./path" import { createFileParser, createProcessor } from "./processors/parse" +import { options } from "./sourcemap" // only called from worker thread export async function parseFiles(argv: Argv, fps: FilePath[], allSlugs: ServerSlug[]) {