perf: incremental rebuild (--fastRebuild v2 but default) (#1841)

* checkpoint

* incremental all the things

* properly splice changes array

* smol doc update

* update docs

* make fancy logger dumb in ci

quartz/plugins/transformers/frontmatter.ts

@@ -3,12 +3,9 @@ import remarkFrontmatter from "remark-frontmatter"
 import { QuartzTransformerPlugin } from "../types"
 import yaml from "js-yaml"
 import toml from "toml"
-import { FilePath, FullSlug, joinSegments, slugifyFilePath, slugTag } from "../../util/path"
+import { FilePath, FullSlug, getFileExtension, slugifyFilePath, slugTag } from "../../util/path"
 import { QuartzPluginData } from "../vfile"
 import { i18n } from "../../i18n"
-import { Argv } from "../../util/ctx"
-import { VFile } from "vfile"
-import path from "path"
 
 export interface Options {
   delimiters: string | [string, string]
@@ -43,26 +40,24 @@ function coerceToArray(input: string | string[]): string[] | undefined {
     .map((tag: string | number) => tag.toString())
 }
 
-export function getAliasSlugs(aliases: string[], argv: Argv, file: VFile): FullSlug[] {
-  const dir = path.posix.relative(argv.directory, path.dirname(file.data.filePath!))
-  const slugs: FullSlug[] = aliases.map(
-    (alias) => path.posix.join(dir, slugifyFilePath(alias as FilePath)) as FullSlug,
-  )
-  const permalink = file.data.frontmatter?.permalink
-  if (typeof permalink === "string") {
-    slugs.push(permalink as FullSlug)
+function getAliasSlugs(aliases: string[]): FullSlug[] {
+  const res: FullSlug[] = []
+  for (const alias of aliases) {
+    const isMd = getFileExtension(alias) === "md"
+    const mockFp = isMd ? alias : alias + ".md"
+    const slug = slugifyFilePath(mockFp as FilePath)
+    res.push(slug)
   }
-  // fix any slugs that have trailing slash
-  return slugs.map((slug) =>
-    slug.endsWith("/") ? (joinSegments(slug, "index") as FullSlug) : slug,
-  )
+
+  return res
 }
 
 export const FrontMatter: QuartzTransformerPlugin<Partial<Options>> = (userOpts) => {
   const opts = { ...defaultOptions, ...userOpts }
   return {
     name: "FrontMatter",
-    markdownPlugins({ cfg, allSlugs, argv }) {
+    markdownPlugins(ctx) {
+      const { cfg, allSlugs } = ctx
       return [
         [remarkFrontmatter, ["yaml", "toml"]],
         () => {
@@ -88,9 +83,18 @@ export const FrontMatter: QuartzTransformerPlugin<Partial<Options>> = (userOpts)
             const aliases = coerceToArray(coalesceAliases(data, ["aliases", "alias"]))
             if (aliases) {
               data.aliases = aliases // frontmatter
-              const slugs = (file.data.aliases = getAliasSlugs(aliases, argv, file))
-              allSlugs.push(...slugs)
+              file.data.aliases = getAliasSlugs(aliases)
+              allSlugs.push(...file.data.aliases)
             }
+
+            if (data.permalink != null && data.permalink.toString() !== "") {
+              data.permalink = data.permalink.toString() as FullSlug
+              const aliases = file.data.aliases ?? []
+              aliases.push(data.permalink)
+              file.data.aliases = aliases
+              allSlugs.push(data.permalink)
+            }
+
             const cssclasses = coerceToArray(coalesceAliases(data, ["cssclasses", "cssclass"]))
             if (cssclasses) data.cssclasses = cssclasses
 

quartz/plugins/transformers/lastmod.ts

@@ -31,7 +31,7 @@ export const CreatedModifiedDate: QuartzTransformerPlugin<Partial<Options>> = (u
   const opts = { ...defaultOptions, ...userOpts }
   return {
     name: "CreatedModifiedDate",
-    markdownPlugins() {
+    markdownPlugins(ctx) {
       return [
         () => {
           let repo: Repository | undefined = undefined
@@ -40,8 +40,8 @@ export const CreatedModifiedDate: QuartzTransformerPlugin<Partial<Options>> = (u
             let modified: MaybeDate = undefined
             let published: MaybeDate = undefined
 
-            const fp = file.data.filePath!
-            const fullFp = path.isAbsolute(fp) ? fp : path.posix.join(file.cwd, fp)
+            const fp = file.data.relativePath!
+            const fullFp = path.posix.join(ctx.argv.directory, fp)
             for (const source of opts.priority) {
               if (source === "filesystem") {
                 const st = await fs.promises.stat(fullFp)
@@ -56,11 +56,11 @@ export const CreatedModifiedDate: QuartzTransformerPlugin<Partial<Options>> = (u
                   // Get a reference to the main git repo.
                   // It's either the same as the workdir,
                   // or 1+ level higher in case of a submodule/subtree setup
-                  repo = Repository.discover(file.cwd)
+                  repo = Repository.discover(ctx.argv.directory)
                 }
 
                 try {
-                  modified ||= await repo.getFileLatestModifiedDateAsync(file.data.filePath!)
+                  modified ||= await repo.getFileLatestModifiedDateAsync(fullFp)
                 } catch {
                   console.log(
                     chalk.yellow(

quartz/plugins/transformers/oxhugofm.ts

@@ -54,7 +54,7 @@ export const OxHugoFlavouredMarkdown: QuartzTransformerPlugin<Partial<Options>>
     textTransform(_ctx, src) {
       if (opts.wikilinks) {
         src = src.toString()
-        src = src.replaceAll(relrefRegex, (value, ...capture) => {
+        src = src.replaceAll(relrefRegex, (_value, ...capture) => {
           const [text, link] = capture
           return `[${text}](${link})`
         })
@@ -62,7 +62,7 @@ export const OxHugoFlavouredMarkdown: QuartzTransformerPlugin<Partial<Options>>
 
       if (opts.removePredefinedAnchor) {
         src = src.toString()
-        src = src.replaceAll(predefinedHeadingIdRegex, (value, ...capture) => {
+        src = src.replaceAll(predefinedHeadingIdRegex, (_value, ...capture) => {
           const [headingText] = capture
           return headingText
         })
@@ -70,7 +70,7 @@ export const OxHugoFlavouredMarkdown: QuartzTransformerPlugin<Partial<Options>>
 
       if (opts.removeHugoShortcode) {
         src = src.toString()
-        src = src.replaceAll(hugoShortcodeRegex, (value, ...capture) => {
+        src = src.replaceAll(hugoShortcodeRegex, (_value, ...capture) => {
           const [scContent] = capture
           return scContent
         })
@@ -78,7 +78,7 @@ export const OxHugoFlavouredMarkdown: QuartzTransformerPlugin<Partial<Options>>
 
       if (opts.replaceFigureWithMdImg) {
         src = src.toString()
-        src = src.replaceAll(figureTagRegex, (value, ...capture) => {
+        src = src.replaceAll(figureTagRegex, (_value, ...capture) => {
           const [src] = capture
           return `![](${src})`
         })
@@ -86,11 +86,11 @@ export const OxHugoFlavouredMarkdown: QuartzTransformerPlugin<Partial<Options>>
 
       if (opts.replaceOrgLatex) {
         src = src.toString()
-        src = src.replaceAll(inlineLatexRegex, (value, ...capture) => {
+        src = src.replaceAll(inlineLatexRegex, (_value, ...capture) => {
           const [eqn] = capture
           return `$${eqn}$`
         })
-        src = src.replaceAll(blockLatexRegex, (value, ...capture) => {
+        src = src.replaceAll(blockLatexRegex, (_value, ...capture) => {
           const [eqn] = capture
           return `$$${eqn}$$`
         })

quartz/plugins/transformers/roam.ts

@@ -1,10 +1,8 @@
 import { QuartzTransformerPlugin } from "../types"
 import { PluggableList } from "unified"
-import { SKIP, visit } from "unist-util-visit"
+import { visit } from "unist-util-visit"
 import { ReplaceFunction, findAndReplace as mdastFindReplace } from "mdast-util-find-and-replace"
 import { Root, Html, Paragraph, Text, Link, Parent } from "mdast"
-import { Node } from "unist"
-import { VFile } from "vfile"
 import { BuildVisitor } from "unist-util-visit"
 
 export interface Options {
@@ -34,21 +32,10 @@ const defaultOptions: Options = {
 const orRegex = new RegExp(/{{or:(.*?)}}/, "g")
 const TODORegex = new RegExp(/{{.*?\bTODO\b.*?}}/, "g")
 const DONERegex = new RegExp(/{{.*?\bDONE\b.*?}}/, "g")
-const videoRegex = new RegExp(/{{.*?\[\[video\]\].*?\:(.*?)}}/, "g")
-const youtubeRegex = new RegExp(
-  /{{.*?\[\[video\]\].*?(https?:\/\/(?:www\.)?youtu(?:be\.com\/watch\?v=|\.be\/)([\w\-\_]*)(&(amp;)?[\w\?=]*)?)}}/,
-  "g",
-)
 
-// const multimediaRegex = new RegExp(/{{.*?\b(video|audio)\b.*?\:(.*?)}}/, "g")
-
-const audioRegex = new RegExp(/{{.*?\[\[audio\]\].*?\:(.*?)}}/, "g")
-const pdfRegex = new RegExp(/{{.*?\[\[pdf\]\].*?\:(.*?)}}/, "g")
 const blockquoteRegex = new RegExp(/(\[\[>\]\])\s*(.*)/, "g")
 const roamHighlightRegex = new RegExp(/\^\^(.+)\^\^/, "g")
 const roamItalicRegex = new RegExp(/__(.+)__/, "g")
-const tableRegex = new RegExp(/- {{.*?\btable\b.*?}}/, "g") /* TODO */
-const attributeRegex = new RegExp(/\b\w+(?:\s+\w+)*::/, "g") /* TODO */
 
 function isSpecialEmbed(node: Paragraph): boolean {
   if (node.children.length !== 2) return false
@@ -135,7 +122,7 @@ export const RoamFlavoredMarkdown: QuartzTransformerPlugin<Partial<Options> | un
       const plugins: PluggableList = []
 
       plugins.push(() => {
-        return (tree: Root, file: VFile) => {
+        return (tree: Root) => {
           const replacements: [RegExp, ReplaceFunction][] = []
 
           // Handle special embeds (audio, video, PDF)