feat(experimental): partial rebuilds (#716)

quartz/plugins/emitters/404.tsx

@@ -9,6 +9,7 @@ import { NotFound } from "../../components"
 import { defaultProcessedContent } from "../vfile"
 import { write } from "./helpers"
 import { i18n } from "../../i18n"
+import DepGraph from "../../depgraph"
 
 export const NotFoundPage: QuartzEmitterPlugin = () => {
   const opts: FullPageLayout = {
@@ -27,6 +28,9 @@ export const NotFoundPage: QuartzEmitterPlugin = () => {
     getQuartzComponents() {
       return [Head, Body, pageBody, Footer]
     },
+    async getDependencyGraph(_ctx, _content, _resources) {
+      return new DepGraph<FilePath>()
+    },
     async emit(ctx, _content, resources): Promise<FilePath[]> {
       const cfg = ctx.cfg.configuration
       const slug = "404" as FullSlug

quartz/plugins/emitters/aliases.ts

@@ -2,12 +2,17 @@ import { FilePath, FullSlug, joinSegments, resolveRelative, simplifySlug } from
 import { QuartzEmitterPlugin } from "../types"
 import path from "path"
 import { write } from "./helpers"
+import DepGraph from "../../depgraph"
 
 export const AliasRedirects: QuartzEmitterPlugin = () => ({
   name: "AliasRedirects",
   getQuartzComponents() {
     return []
   },
+  async getDependencyGraph(_ctx, _content, _resources) {
+    // TODO implement
+    return new DepGraph<FilePath>()
+  },
   async emit(ctx, content, _resources): Promise<FilePath[]> {
     const { argv } = ctx
     const fps: FilePath[] = []

quartz/plugins/emitters/assets.ts

@@ -3,6 +3,7 @@ import { QuartzEmitterPlugin } from "../types"
 import path from "path"
 import fs from "fs"
 import { glob } from "../../util/glob"
+import DepGraph from "../../depgraph"
 
 export const Assets: QuartzEmitterPlugin = () => {
   return {
@@ -10,6 +11,24 @@ export const Assets: QuartzEmitterPlugin = () => {
     getQuartzComponents() {
       return []
     },
+    async getDependencyGraph(ctx, _content, _resources) {
+      const { argv, cfg } = ctx
+      const graph = new DepGraph<FilePath>()
+
+      const fps = await glob("**", argv.directory, ["**/*.md", ...cfg.configuration.ignorePatterns])
+
+      for (const fp of fps) {
+        const ext = path.extname(fp)
+        const src = joinSegments(argv.directory, fp) as FilePath
+        const name = (slugifyFilePath(fp as FilePath, true) + ext) as FilePath
+
+        const dest = joinSegments(argv.output, name) as FilePath
+
+        graph.addEdge(src, dest)
+      }
+
+      return graph
+    },
     async emit({ argv, cfg }, _content, _resources): Promise<FilePath[]> {
       // glob all non MD/MDX/HTML files in content folder and copy it over
       const assetsPath = argv.output

quartz/plugins/emitters/cname.ts

@@ -2,6 +2,7 @@ import { FilePath, joinSegments } from "../../util/path"
 import { QuartzEmitterPlugin } from "../types"
 import fs from "fs"
 import chalk from "chalk"
+import DepGraph from "../../depgraph"
 
 export function extractDomainFromBaseUrl(baseUrl: string) {
   const url = new URL(`https://${baseUrl}`)
@@ -13,6 +14,9 @@ export const CNAME: QuartzEmitterPlugin = () => ({
   getQuartzComponents() {
     return []
   },
+  async getDependencyGraph(_ctx, _content, _resources) {
+    return new DepGraph<FilePath>()
+  },
   async emit({ argv, cfg }, _content, _resources): Promise<FilePath[]> {
     if (!cfg.configuration.baseUrl) {
       console.warn(chalk.yellow("CNAME emitter requires `baseUrl` to be set in your configuration"))

quartz/plugins/emitters/componentResources.ts

@@ -14,6 +14,7 @@ import { googleFontHref, joinStyles } from "../../util/theme"
 import { Features, transform } from "lightningcss"
 import { transform as transpile } from "esbuild"
 import { write } from "./helpers"
+import DepGraph from "../../depgraph"
 
 type ComponentResources = {
   css: string[]
@@ -149,9 +150,10 @@ function addGlobalPageResources(
       loadTime: "afterDOMReady",
       contentType: "inline",
       script: `
-        const socket = new WebSocket('${wsUrl}')
-        socket.addEventListener('message', () => document.location.reload())
-      `,
+          const socket = new WebSocket('${wsUrl}')
+          // reload(true) ensures resources like images and scripts are fetched again in firefox
+          socket.addEventListener('message', () => document.location.reload(true))
+        `,
     })
   }
 }
@@ -171,6 +173,24 @@ export const ComponentResources: QuartzEmitterPlugin<Options> = (opts?: Partial<
     getQuartzComponents() {
       return []
     },
+    async getDependencyGraph(ctx, content, _resources) {
+      // This emitter adds static resources to the `resources` parameter. One
+      // important resource this emitter adds is the code to start a websocket
+      // connection and listen to rebuild messages, which triggers a page reload.
+      // The resources parameter with the reload logic is later used by the
+      // ContentPage emitter while creating the final html page. In order for
+      // the reload logic to be included, and so for partial rebuilds to work,
+      // we need to run this emitter for all markdown files.
+      const graph = new DepGraph<FilePath>()
+
+      for (const [_tree, file] of content) {
+        const sourcePath = file.data.filePath!
+        const slug = file.data.slug!
+        graph.addEdge(sourcePath, joinSegments(ctx.argv.output, slug + ".html") as FilePath)
+      }
+
+      return graph
+    },
     async emit(ctx, _content, resources): Promise<FilePath[]> {
       const promises: Promise<FilePath>[] = []
       const cfg = ctx.cfg.configuration

quartz/plugins/emitters/contentIndex.ts

@@ -7,6 +7,7 @@ import { QuartzEmitterPlugin } from "../types"
 import { toHtml } from "hast-util-to-html"
 import { write } from "./helpers"
 import { i18n } from "../../i18n"
+import DepGraph from "../../depgraph"
 
 export type ContentIndex = Map<FullSlug, ContentDetails>
 export type ContentDetails = {
@@ -92,6 +93,26 @@ export const ContentIndex: QuartzEmitterPlugin<Partial<Options>> = (opts) => {
   opts = { ...defaultOptions, ...opts }
   return {
     name: "ContentIndex",
+    async getDependencyGraph(ctx, content, _resources) {
+      const graph = new DepGraph<FilePath>()
+
+      for (const [_tree, file] of content) {
+        const sourcePath = file.data.filePath!
+
+        graph.addEdge(
+          sourcePath,
+          joinSegments(ctx.argv.output, "static/contentIndex.json") as FilePath,
+        )
+        if (opts?.enableSiteMap) {
+          graph.addEdge(sourcePath, joinSegments(ctx.argv.output, "sitemap.xml") as FilePath)
+        }
+        if (opts?.enableRSS) {
+          graph.addEdge(sourcePath, joinSegments(ctx.argv.output, "index.xml") as FilePath)
+        }
+      }
+
+      return graph
+    },
     async emit(ctx, content, _resources) {
       const cfg = ctx.cfg.configuration
       const emitted: FilePath[] = []

quartz/plugins/emitters/contentPage.tsx

@@ -4,11 +4,12 @@ import HeaderConstructor from "../../components/Header"
 import BodyConstructor from "../../components/Body"
 import { pageResources, renderPage } from "../../components/renderPage"
 import { FullPageLayout } from "../../cfg"
-import { FilePath, pathToRoot } from "../../util/path"
+import { FilePath, joinSegments, pathToRoot } from "../../util/path"
 import { defaultContentPageLayout, sharedPageComponents } from "../../../quartz.layout"
 import { Content } from "../../components"
 import chalk from "chalk"
 import { write } from "./helpers"
+import DepGraph from "../../depgraph"
 
 export const ContentPage: QuartzEmitterPlugin<Partial<FullPageLayout>> = (userOpts) => {
   const opts: FullPageLayout = {
@@ -27,6 +28,18 @@ export const ContentPage: QuartzEmitterPlugin<Partial<FullPageLayout>> = (userOp
     getQuartzComponents() {
       return [Head, Header, Body, ...header, ...beforeBody, pageBody, ...left, ...right, Footer]
     },
+    async getDependencyGraph(ctx, content, _resources) {
+      // TODO handle transclusions
+      const graph = new DepGraph<FilePath>()
+
+      for (const [_tree, file] of content) {
+        const sourcePath = file.data.filePath!
+        const slug = file.data.slug!
+        graph.addEdge(sourcePath, joinSegments(ctx.argv.output, slug + ".html") as FilePath)
+      }
+
+      return graph
+    },
     async emit(ctx, content, resources): Promise<FilePath[]> {
       const cfg = ctx.cfg.configuration
       const fps: FilePath[] = []
@@ -60,7 +73,7 @@ export const ContentPage: QuartzEmitterPlugin<Partial<FullPageLayout>> = (userOp
         fps.push(fp)
       }
 
-      if (!containsIndex) {
+      if (!containsIndex && !ctx.argv.fastRebuild) {
         console.log(
           chalk.yellow(
             `\nWarning: you seem to be missing an \`index.md\` home page file at the root of your \`${ctx.argv.directory}\` folder. This may cause errors when deploying.`,

quartz/plugins/emitters/folderPage.tsx

@@ -19,6 +19,7 @@ import { defaultListPageLayout, sharedPageComponents } from "../../../quartz.lay
 import { FolderContent } from "../../components"
 import { write } from "./helpers"
 import { i18n } from "../../i18n"
+import DepGraph from "../../depgraph"
 
 export const FolderPage: QuartzEmitterPlugin<Partial<FullPageLayout>> = (userOpts) => {
   const opts: FullPageLayout = {
@@ -37,6 +38,13 @@ export const FolderPage: QuartzEmitterPlugin<Partial<FullPageLayout>> = (userOpt
     getQuartzComponents() {
       return [Head, Header, Body, ...header, ...beforeBody, pageBody, ...left, ...right, Footer]
     },
+    async getDependencyGraph(ctx, content, _resources) {
+      // Example graph:
+      // nested/file.md --> nested/file.html
+      //          \-------> nested/index.html
+      // TODO implement
+      return new DepGraph<FilePath>()
+    },
     async emit(ctx, content, resources): Promise<FilePath[]> {
       const fps: FilePath[] = []
       const allFiles = content.map((c) => c[1].data)

quartz/plugins/emitters/static.ts

@@ -2,12 +2,27 @@ import { FilePath, QUARTZ, joinSegments } from "../../util/path"
 import { QuartzEmitterPlugin } from "../types"
 import fs from "fs"
 import { glob } from "../../util/glob"
+import DepGraph from "../../depgraph"
 
 export const Static: QuartzEmitterPlugin = () => ({
   name: "Static",
   getQuartzComponents() {
     return []
   },
+  async getDependencyGraph({ argv, cfg }, _content, _resources) {
+    const graph = new DepGraph<FilePath>()
+
+    const staticPath = joinSegments(QUARTZ, "static")
+    const fps = await glob("**", staticPath, cfg.configuration.ignorePatterns)
+    for (const fp of fps) {
+      graph.addEdge(
+        joinSegments("static", fp) as FilePath,
+        joinSegments(argv.output, "static", fp) as FilePath,
+      )
+    }
+
+    return graph
+  },
   async emit({ argv, cfg }, _content, _resources): Promise<FilePath[]> {
     const staticPath = joinSegments(QUARTZ, "static")
     const fps = await glob("**", staticPath, cfg.configuration.ignorePatterns)

quartz/plugins/emitters/tagPage.tsx

@@ -16,6 +16,7 @@ import { defaultListPageLayout, sharedPageComponents } from "../../../quartz.lay
 import { TagContent } from "../../components"
 import { write } from "./helpers"
 import { i18n } from "../../i18n"
+import DepGraph from "../../depgraph"
 
 export const TagPage: QuartzEmitterPlugin<Partial<FullPageLayout>> = (userOpts) => {
   const opts: FullPageLayout = {
@@ -34,6 +35,10 @@ export const TagPage: QuartzEmitterPlugin<Partial<FullPageLayout>> = (userOpts)
     getQuartzComponents() {
       return [Head, Header, Body, ...header, ...beforeBody, pageBody, ...left, ...right, Footer]
     },
+    async getDependencyGraph(ctx, _content, _resources) {
+      // TODO implement
+      return new DepGraph<FilePath>()
+    },
     async emit(ctx, content, resources): Promise<FilePath[]> {
       const fps: FilePath[] = []
       const allFiles = content.map((c) => c[1].data)

quartz/plugins/types.ts

@@ -4,6 +4,7 @@ import { ProcessedContent } from "./vfile"
 import { QuartzComponent } from "../components/types"
 import { FilePath } from "../util/path"
 import { BuildCtx } from "../util/ctx"
+import DepGraph from "../depgraph"
 
 export interface PluginTypes {
   transformers: QuartzTransformerPluginInstance[]
@@ -38,4 +39,9 @@ export type QuartzEmitterPluginInstance = {
   name: string
   emit(ctx: BuildCtx, content: ProcessedContent[], resources: StaticResources): Promise<FilePath[]>
   getQuartzComponents(ctx: BuildCtx): QuartzComponent[]
+  getDependencyGraph?(
+    ctx: BuildCtx,
+    content: ProcessedContent[],
+    resources: StaticResources,
+  ): Promise<DepGraph<FilePath>>
 }