update-pandoc.ts

/*
* update-pandoc.ts
*
* Copyright (C) 2020-2022 Posit Software, PBC
*
*/
import { Command } from "cliffy/command/mod.ts";
import { join } from "../../../src/deno_ral/path.ts";
import { ensureDirSync } from "../../../src/deno_ral/fs.ts";
import { info } from "../../../src/deno_ral/log.ts";

import {
  Configuration,
  readConfiguration,
  withWorkingDir,
} from "../common/config.ts";
import { lines } from "../../../src/core/text.ts";
import { pandoc } from "./dependencies/pandoc.ts";
import { archiveBinaryDependency } from "./archive-binary-dependencies.ts";

import { configureDependency } from "./dependencies/dependencies.ts";
import { download, unzip } from "../util/utils.ts";
import {
  pandocListFormatDefaultExtensions,
  pandocListFormats,
} from "../../../src/core/pandoc/pandoc-formats.ts";

import {
  bgBlack,
  bold,
  brightWhite,
} from "../../../src/core/lib/external/colors.ts";

import * as ld from "../../../src/core/lodash.ts";

export function updatePandoc() {
  return new Command()
    .name("update-pandoc")
    .arguments("<version:string>")
    .description("Updates Pandoc to the specified version")
    .action(async (_args, version: string) => {
      info(`Updating Pandoc to ${version}`);

      const configuration = readConfiguration();

      // Update the configuration file
      info("  updating configuration file.");
      const configFilePath = join(
        configuration.directoryInfo.root,
        "configuration",
      );
      const configText = Deno.readTextFileSync(configFilePath);
      const configLines = lines(configText);
      const outputLines: string[] = [];
      for (const line of configLines) {
        if (line.startsWith("export PANDOC=")) {
          outputLines.push(`export PANDOC=${version}`);
        } else {
          outputLines.push(line);
        }
      }
      Deno.writeTextFileSync(configFilePath, outputLines.join("\n"));

      const pandocDependency = pandoc(version);

      // Call archive-bin-deps for this file
      await withWorkingDir(async (workingDir) => {
        await archiveBinaryDependency(pandocDependency, workingDir);

        // Configure this version of pandoc
        await configureDependency(
          pandocDependency,
          join(configuration.directoryInfo.bin, "tools"),
          configuration,
        );

        // Generate templates
        await writePandocTemplates(configuration, version, workingDir);

        // Generate variants
        await writeVariants(configuration);
      });

      // print the warning to complete the checklist
      console.log(bgBlack(brightWhite(bold(
        "\n** Remember to complete the checklist in /dev-docs/update-pandoc-checklist.md! **",
      ))));
    });
}

// Starting in Pandoc 3, we saw a number of variants that appear to be supported
// disappear from the --list-extensions command, so for the time being we're just
// hard adding them here
const kExtendedVariants: string[] = [
  "amuse",
  "attributes",
  "element_citations",
  "empty_paragraphs",
  "epub_html_exts",
  "native_numbering",
  "ntb",
  "raw_markdown",
  "sourcepos",
  "styles",
  "xrefs_name",
  "xrefs_number",
];

async function writeVariants(
  config: Configuration,
) {
  info("Generating Pandoc extensions source file...");
  info("Reading pandoc formats and extensions");
  const formats = await pandocListFormats();
  const extensions: Set<string> = new Set();
  for (const format of formats) {
    const formatExtensions = await pandocListFormatDefaultExtensions(format);
    formatExtensions.forEach((ext) => {
      const bareExtension = ext.replace(/^[\+\-]/, "");
      extensions.add(bareExtension);
    });
  }
  const extArr = Array.from(extensions.values());
  info(`  Pandoc Alone Reporting:`);
  info(`  ${formats.length} formats.`);
  info(`  ${extArr.length} extensions.`);

  extArr.push(...kExtendedVariants);
  const extended = ld.uniq(extArr);
  info(`  ${extended.length} extensions (after adding extended).`);

  const extArrExpanded = extended.toSorted().flatMap((ext: string) => {
    return [`"+${ext}"`, `"-${ext}"`];
  });

  const extensionFile = join(
    config.directoryInfo.src,
    "core",
    "pandoc",
    "format-extension.ts",
  );

  // Generate the extension list
  const tsContents = `
/*
* pandoc-extensions.ts
*
* Copyright (C) 2020-2022 Posit Software, PBC
*
*/
//
// THIS FILE IS GENERATED BY update-pandoc.ts. DO NOT EDIT MANUALLY
//

export const kPandocExtensions = [
  ${extArrExpanded.join(",\n  ")}
];
`;

  // Write the file
  Deno.writeTextFileSync(extensionFile, tsContents);
}

async function writePandocTemplates(
  config: Configuration,
  version: string,
  workingDir: string,
) {
  info("Reading pandoc templates...");
  const formatSrcDir = join(
    config.directoryInfo.src,
    "resources",
    "formats",
  );

  const srcZipUrl =
    `https://github.com/jgm/pandoc/archive/refs/tags/${version}.zip`;

  const pandocDir = `pandoc-${version}`;
  const zipFile = join(workingDir, "pandoc");
  await download(srcZipUrl, zipFile);
  await unzip(zipFile, workingDir);

  // Jats templates are multi-part templates that
  // are not properly emitted by pandoc itself, so download
  // them from source instead
  const templateDir = join(workingDir, pandocDir, "data", "templates");
  const jatsOutDir = join(
    formatSrcDir,
    "jats",
    "pandoc",
    "default-templates",
  );
  const htmlOutdir = join(
    formatSrcDir,
    "html",
    "pandoc",
  );
  const latexOutdir = join(formatSrcDir, "pdf", "pandoc");
  const revealOutdir = join(formatSrcDir, "revealjs", "pandoc");
  const beamerOutdir = join(formatSrcDir, "beamer", "pandoc");
  const asciidocOutdir = join(formatSrcDir, "asciidoc", "pandoc");
  const typstOutdir = join(formatSrcDir, "typst", "pandoc");

  const templateDirFiles: Record<string, Array<{ from: string; to?: string }>> =
    {
      [jatsOutDir]: [
        { from: "affiliations.jats" },
        { from: "article.jats_publishing" },
        { from: "default.jats_archiving" },
        { from: "default.jats_articleauthoring" },
        { from: "default.jats_publishing" },
      ],
      [htmlOutdir]: [
        { from: "default.html5", to: "html.template" },
        { from: "styles.html", to: "html.styles" },
      ],
      [revealOutdir]: [
        { from: "default.revealjs", to: "revealjs.template" },
      ],
      [latexOutdir]: [
        { from: "default.latex", to: "latex.template" },
        // Template we need to tweak
        { from: "common.latex", to: "latex.common" },
        // Template kept unchanged
        { from: "after-header-includes.latex" },
        { from: "hypersetup.latex" },
        { from: "font-settings.latex" },
        { from: "fonts.latex" },
        { from: "passoptions.latex" },
      ],
      [beamerOutdir]: [
        { from: "default.beamer", to: "beamer.template" },
        // Template we need to tweak
        { from: "common.latex", to: "latex.common" },
        // Template kept unchanged
        { from: "after-header-includes.latex" },
        { from: "hypersetup.latex" },
        { from: "font-settings.latex" },
        { from: "fonts.latex" },
        { from: "passoptions.latex" },
      ],
      [asciidocOutdir]: [
        { from: "default.asciidoc", to: "asciidoc.template" },
      ],
      [typstOutdir]: [
        { from: "default.typst", to: "typst.template" },
        { from: "template.typst" }
      ]
    };

  // Move templates
  for (const outDir of Object.keys(templateDirFiles)) {
    ensureDirSync(outDir);
    for (const file of templateDirFiles[outDir]) {
      info(`> ${file.from}`);
      Deno.copyFileSync(
        join(templateDir, file.from),
        join(outDir, file.to || file.from),
      );
    }
  }

  info("done.");
  info("");
}