import * as ld from "../../core/lodash.ts";
import { dirname, join, relative, resolve } from "../../deno_ral/path.ts";
import { warning } from "../../deno_ral/log.ts";
import { parseModule } from "observablehq/parser";
import { escape } from "../../core/lodash.ts";
import { Format, kDependencies } from "../../config/types.ts";
import { MappedExecuteResult, PandocIncludes } from "../../execute/types.ts";
import {
kCodeSummary,
kEmbedResources,
kIncludeAfterBody,
kIncludeInHeader,
kSelfContained,
} from "../../config/constants.ts";
import { RenderContext } from "../../command/render/types.ts";
import { ProjectContext } from "../../project/types.ts";
import {
isJavascriptCompatible,
isMarkdownOutput,
} from "../../config/format.ts";
import { resolveDependencies } from "../../command/render/pandoc-dependencies-html.ts";
import {
extractResourceDescriptionsFromOJSChunk,
makeSelfContainedResources,
ResourceDescription,
uniqueResources,
} from "./extract-resources.ts";
import { ojsParseError } from "./errors.ts";
import { ojsSimpleWalker } from "./ojs-tools.ts";
import {
kCellFigCap,
kCellFigSubCap,
kCellLstCap,
kCellLstLabel,
kCodeFold,
kCodeLineNumbers,
kCodeOverflow,
kEcho,
kError,
kEval,
kInclude,
kLayoutNcol,
kLayoutNrow,
kOutput,
} from "../../config/constants.ts";
import { asHtmlId } from "../../core/html.ts";
import { TempContext } from "../../core/temp.ts";
import { quartoConfig } from "../../core/quarto.ts";
import { mergeConfigs } from "../../core/config.ts";
import { formatResourcePath } from "../../core/resources.ts";
import { logError } from "../../core/log.ts";
import { breakQuartoMd, QuartoMdCell } from "../../core/lib/break-quarto-md.ts";
import { MappedString } from "../../core/mapped-text.ts";
import { languagesInMarkdown } from "../../core/pandoc/pandoc-partition.ts";
import {
pandocBlock,
pandocCode,
pandocDiv,
pandocRawStr,
} from "../../core/pandoc/codegen.ts";
import {
EitherString,
join as mappedJoin,
mappedTrim,
} from "../../core/lib/mapped-text.ts";
import { getDivAttributes } from "../../core/handlers/base.ts";
import { pathWithForwardSlashes } from "../../core/path.ts";
import { executeInlineCodeHandlerMapped } from "../../core/execute-inline.ts";
import { encodeBase64 } from "encoding/base64";
export interface OjsCompileOptions {
source: string;
format: Format;
markdown: MappedString;
libDir: string;
temp: TempContext;
project: ProjectContext;
ojsBlockLineNumbers: number[];
}
export interface OjsCompileResult {
markdown: MappedString;
filters?: string[];
includes?: PandocIncludes;
resourceFiles?: string[];
}
interface SubfigureSpec {
caption?: string;
}
const ojsHasOutputs = (parse: any) => {
let hasOutputs = false;
ojsSimpleWalker(parse, {
Cell(node: any) {
if (node.id === null) {
hasOutputs = true;
}
},
});
return hasOutputs;
};
export async function ojsCompile(
options: OjsCompileOptions,
): Promise<OjsCompileResult> {
const { markdown, project, ojsBlockLineNumbers } = options;
const output = await breakQuartoMd(markdown, true);
const hasOjsDefines = markdown.value.indexOf("ojs_define") !== -1;
if (
!isJavascriptCompatible(options.format) ||
options.format.metadata?.["ojs-engine"] === false
) {
return { markdown: markdown };
}
const languages = languagesInMarkdown(markdown.value);
if (
(options.format.metadata?.["ojs-engine"] !== true) &&
!languages.has("ojs") &&
!hasOjsDefines
) {
return { markdown: markdown };
}
const projDir = project.isSingleFile ? undefined : project.dir;
const selfContained = options.format.pandoc?.[kSelfContained] ??
options.format.pandoc?.[kEmbedResources] ?? false;
const isHtmlMarkdown = isMarkdownOutput(options.format, [
"gfm",
"commonmark",
]);
let ojsCellID = 0;
let ojsBlockIndex = 0;
const userIds: Set<string> = new Set();
const scriptContents: string[] = [];
const ojsRuntimeDir = resolve(
dirname(options.source),
options.libDir + "/ojs",
);
const docDir = dirname(options.source);
const rootDir = projDir ?? "./";
const runtimeToDoc = pathWithForwardSlashes(relative(ojsRuntimeDir, docDir));
const runtimeToRoot = pathWithForwardSlashes(
relative(ojsRuntimeDir, rootDir),
);
const docToRoot = pathWithForwardSlashes(relative(docDir, rootDir));
scriptContents.push(
`if (window.location.protocol === "file:") { alert("The OJS runtime does not work with file:// URLs. Please use a web server to view this document."); }`,
);
scriptContents.push(`window._ojs.paths.runtimeToDoc = "${runtimeToDoc}";`);
scriptContents.push(`window._ojs.paths.runtimeToRoot = "${runtimeToRoot}";`);
scriptContents.push(`window._ojs.paths.docToRoot = "${docToRoot}";`);
scriptContents.push(
`window._ojs.selfContained = ${selfContained};`,
);
interface ModuleCell {
methodName: string;
cellName?: string;
inline?: boolean;
source: string;
}
const moduleContents: ModuleCell[] = [];
function interpret(jsSrc: MappedString, inline: boolean, lenient: boolean) {
const inlineStr = inline ? "inline-" : "";
const methodName = lenient ? "interpretLenient" : "interpret";
moduleContents.push({
methodName,
cellName: `ojs-${inlineStr}cell-${ojsCellID}`,
inline,
source: jsSrc.value,
});
}
const ls: EitherString[] = [];
const resourceFiles: string[] = [];
const pageResources: ResourceDescription[] = [];
const ojsViews = new Set<string>();
const ojsIdentifiers = new Set<string>();
for (const cell of output.cells) {
const errorVal =
(cell.options?.[kError] ?? options.format.execute?.[kError] ??
false) as boolean;
const handleOJSCell = async (
cell: QuartoMdCell,
mdClassList?: string[],
) => {
const cellSrcStr = cell.source;
const bumpOjsCellIdString = () => {
ojsCellID += 1;
return `ojs-cell-${ojsCellID}`;
};
const ojsId = bumpOjsCellIdString();
const userCellId = () => {
const chooseId = (label: string) => {
const htmlLabel = asHtmlId(label as string);
if (userIds.has(htmlLabel)) {
throw new Error(`FATAL: duplicate label ${htmlLabel}`);
} else {
userIds.add(htmlLabel);
return htmlLabel;
}
};
if (cell.options?.label) {
return chooseId(cell.options.label as string);
} else if (cell.options?.[kCellLstLabel]) {
return chooseId(cell.options[kCellLstLabel] as string);
} else if (
cell.options?.[kCellFigCap] || cell.options?.[kCellFigSubCap] ||
cell.options?.[kCellLstCap]
) {
return chooseId(`fig-${ojsId}`);
} else {
return undefined;
}
};
const userId = userCellId();
const attrs = [];
const hasFigureSubCaptions = () => {
return cell.options?.[kCellFigSubCap];
};
interface SourceInfo {
start: number;
end: number;
cellType: string;
}
interface ParsedCellInfo {
info: SourceInfo[];
}
const cellStartingLoc = ojsBlockLineNumbers[ojsBlockIndex++] || 0;
if (cellStartingLoc === 0) {
warning(
"OJS block count mismatch. Line number reporting is likely to be wrong",
);
}
const handleError = (err: any, cellSrc: MappedString) => {
const div = pandocDiv({
classes: ["quarto-ojs-syntax-error"],
});
const msg = String(err).split("\n")[0].trim().replace(
/ *\(\d+:\d+\)$/,
"",
);
ojsParseError(err, cellSrc);
const preDiv = pandocCode({
classes: ["numberLines", "java"],
attrs: [
`startFrom="${cellStartingLoc - 1}"`,
`syntax-error-position="${err.pos}"`,
`source-offset="${cell.sourceOffset}"`,
],
});
preDiv.push(pandocRawStr(cell.sourceVerbatim.value.trim()));
div.push(preDiv);
const errMsgDiv = pandocDiv({
classes: ["cell-output", "cell-output-error"],
});
const calloutDiv = pandocDiv({
classes: ["callout-important"],
});
const [_heading, fullMsg] = msg.split(": ");
calloutDiv.push(
pandocRawStr(
`#### OJS Syntax Error (line ${
err.loc.line +
cellStartingLoc + cell.sourceStartLine -
1
}, column ${err.loc.column + 1})`,
),
);
calloutDiv.push(pandocRawStr(`${fullMsg}`));
errMsgDiv.push(calloutDiv);
div.push(errMsgDiv);
div.emit(ls);
};
let nCells = 0;
const parsedCells: ParsedCellInfo[] = [];
let hasOutputs = true;
try {
const parse = parseModule(cellSrcStr.value);
hasOutputs = ojsHasOutputs(parse);
let info: SourceInfo[] = [];
const flushSeqSrc = () => {
parsedCells.push({ info });
for (let i = 1; i < info.length; ++i) {
parsedCells.push({ info: [] });
}
info = [];
};
ojsSimpleWalker(parse, {
Cell(node: any) {
if (node.id && node.id.type === "ViewExpression") {
ojsViews.add(node.id.id.name);
} else if (node.id && node.id.type === "Identifier") {
ojsIdentifiers.add(node.id.name);
}
if (
node.id === null &&
node.body.type !== "ImportDeclaration"
) {
info.push({
start: node.start,
end: node.end,
cellType: "expression",
});
flushSeqSrc();
} else {
info.push({
start: node.start,
end: node.end,
cellType: "declaration",
});
}
},
});
nCells = parse.cells.length;
if (info.length > 0) {
flushSeqSrc();
}
} catch (e) {
if (e instanceof SyntaxError) {
handleError(e, cellSrcStr);
return;
} else {
logError(e);
throw new Error();
}
}
pageResources.push(
...(await extractResourceDescriptionsFromOJSChunk(
cellSrcStr,
dirname(options.source),
projDir,
)),
);
const hasManyRowsCols = () => {
const cols = cell.options?.[kLayoutNcol];
const rows = cell.options?.[kLayoutNrow];
return (Number(cols) && (Number(cols) > 1)) ||
(Number(rows) && (Number(rows) > 1)) ||
(nCells > 1);
};
const nCol = () => {
const col = cell.options
?.[kLayoutNcol] as (string | number | undefined);
if (!col) {
return 1;
}
return Number(col);
};
const nRow = () => {
const row = cell.options
?.[kLayoutNrow] as (string | number | undefined);
if (!row) {
return Math.ceil(nCells / nCol());
}
return Number(row);
};
const hasSubFigures = () => {
return hasFigureSubCaptions() ||
(hasManyRowsCols() && ((nRow() * nCol()) > 1));
};
const idPlacement = () => {
if (
hasSubFigures() ||
cell.options?.[kCellLstLabel]
) {
return "outer";
} else {
return "inner";
}
};
let outputVal: any = cell.options?.[kOutput] ??
options.format.execute[kOutput];
outputVal = outputVal ?? true;
if (outputVal === "all" || outputVal === "asis") {
attrs.push(`output="${outputVal}"`);
}
const {
classes,
attrs: otherAttrs,
} = getDivAttributes(cell.options || {});
attrs.push(...otherAttrs);
const evalVal = cell.options?.[kEval] ?? options.format.execute[kEval] ??
true;
const echoVal = cell.options?.[kEcho] ?? options.format.execute[kEcho] ??
true;
const ojsCellClasses = ["cell"];
if (!outputVal) {
ojsCellClasses.push("hidden");
}
const div = pandocDiv({
id: idPlacement() === "outer" ? userId : undefined,
classes: [
...ojsCellClasses,
...classes,
],
attrs,
});
const includeVal = cell.options?.[kInclude] ??
options.format.execute[kInclude] ?? true;
const srcClasses = mdClassList ?? ["js", "cell-code"];
const srcAttrs = [];
if (!echoVal) {
srcClasses.push("hidden");
}
if (cell.options?.[kCodeOverflow] === "wrap") {
srcClasses.push("code-overflow-wrap");
} else if (cell.options?.[kCodeOverflow] === "scroll") {
srcClasses.push("code-overflow-scroll");
}
if (
asUndefined(options.format.render?.[kCodeFold], "none") ??
cell.options?.[kCodeFold]
) {
srcAttrs.push(`${kCodeFold}="${cell.options?.[kCodeFold]}"`);
}
if (cell.options?.[kCodeSummary]) {
srcAttrs.push(
`${kCodeSummary}="${escape(cell.options?.[kCodeSummary])}"`,
);
}
if (cell.options?.[kCodeLineNumbers]) {
srcAttrs.push(
`${kCodeLineNumbers}="${cell.options?.[kCodeLineNumbers]}"`,
);
}
const srcConfig = {
classes: srcClasses.slice(),
attrs: srcAttrs.slice(),
};
if (evalVal) {
interpret(cell.source, false, errorVal);
}
const outputCellClasses = ["cell-output", "cell-output-display"];
if (!outputVal || !includeVal) {
outputCellClasses.push("hidden");
}
if (echoVal === "fenced") {
const ourAttrs = srcConfig.attrs.slice();
const ourClasses = srcConfig.classes.filter((d) => d !== "js");
ourClasses.push("java");
ourAttrs.push(
`startFrom="${cellStartingLoc - 1}"`,
`source-offset="${cell.sourceOffset}"`,
);
const srcDiv = pandocCode({
classes: ourClasses,
attrs: ourAttrs,
});
srcDiv.push(pandocRawStr(cell.sourceVerbatim.value.trim()));
div.push(srcDiv);
}
const shouldEmitSource = echoVal !== "fenced" &&
!(echoVal === false && isHtmlMarkdown);
const makeSubFigures = (specs: SubfigureSpec[]) => {
let subfigIx = 1;
const cellInfo = ([] as SourceInfo[]).concat(
...(parsedCells.map((n) => n.info)),
);
for (const spec of specs) {
const outputDiv = pandocDiv({
classes: outputCellClasses,
});
const outputInnerDiv = pandocDiv({
id: userId && `${userId}-${subfigIx}`,
});
const innerInfo = parsedCells[subfigIx - 1].info;
const ojsDiv = pandocDiv({
id: `${ojsId}-${subfigIx}`,
attrs: [`nodetype="${cellInfo[subfigIx - 1].cellType}"`],
});
if (
shouldEmitSource &&
innerInfo.length > 0 && srcConfig !== undefined
) {
const ourAttrs = srcConfig.attrs.slice();
const linesSkipped =
cellSrcStr.value.substring(0, innerInfo[0].start).split("\n")
.length;
ourAttrs.push(
`startFrom="${
cellStartingLoc + cell.sourceStartLine - 1 +
linesSkipped
}"`,
);
ourAttrs.push(`source-offset="-${innerInfo[0].start}"`);
const srcDiv = pandocCode({
attrs: ourAttrs,
classes: srcConfig.classes,
});
srcDiv.push(pandocRawStr(
cellSrcStr.value.substring(
innerInfo[0].start,
innerInfo[innerInfo.length - 1].end,
).trim(),
));
div.push(srcDiv);
}
subfigIx++;
outputDiv.push(outputInnerDiv);
outputInnerDiv.push(ojsDiv);
if (spec.caption) {
outputInnerDiv.push(pandocRawStr(spec.caption as string));
}
div.push(outputDiv);
}
};
if (!hasFigureSubCaptions() && hasManyRowsCols()) {
const cellCount = Math.max(nRow() * nCol(), nCells, 1);
const specs = [];
for (let i = 0; i < cellCount; ++i) {
specs.push({ caption: "" });
}
makeSubFigures(specs);
if (cell.options?.[kCellFigCap]) {
div.push(pandocRawStr(cell.options[kCellFigCap] as string));
}
} else if (hasFigureSubCaptions()) {
let subCap = (cell.options?.[kCellFigSubCap]) as string[] | true;
if (subCap === true) {
subCap = [""];
}
if (!Array.isArray(subCap)) {
subCap = [subCap];
}
if (
hasManyRowsCols() &&
(subCap as string[]).length !==
(nRow() * nCol())
) {
throw new Error(
"Cannot have subcaptions and multi-row/col layout with mismatched number of cells",
);
}
const specs = (subCap as string[]).map(
(caption) => ({ caption }),
);
makeSubFigures(specs);
if (cell.options?.[kCellFigCap]) {
div.push(pandocRawStr(cell.options[kCellFigCap] as string));
}
} else {
if (parsedCells.length === 0) {
throw new Error(
`Fatal: OJS cell starting on line ${cellStartingLoc} is empty. OJS cells require at least one declaration.`,
);
}
const innerInfo = parsedCells[0].info;
if (innerInfo.length > 0 && srcConfig !== undefined) {
const ourAttrs = srcConfig.attrs.slice();
ourAttrs.push(
`startFrom="${cellStartingLoc + cell.sourceStartLine - 1}"`,
);
ourAttrs.push(`source-offset="0"`);
if (shouldEmitSource) {
const srcDiv = pandocCode({
attrs: ourAttrs,
classes: srcConfig.classes,
});
srcDiv.push(
pandocRawStr(mappedTrim(cellSrcStr)),
);
div.push(srcDiv);
}
}
const outputDiv = pandocDiv({
id: idPlacement() === "inner" ? userId : undefined,
classes: outputCellClasses,
});
div.push(outputDiv);
outputDiv.push(pandocDiv({
id: ojsId,
attrs: [`nodetype="${innerInfo[0].cellType}"`],
}));
if (cell.options?.[kCellFigCap]) {
outputDiv.push(pandocRawStr(cell.options[kCellFigCap] as string));
}
}
div.emit(ls);
};
if (
cell.cell_type === "raw"
) {
ls.push(cell.sourceVerbatim);
} else if (cell.cell_type === "markdown") {
const markdown = executeInlineCodeHandlerMapped(
"ojs",
(exec) => "${" + exec + "}",
)(cell.sourceVerbatim);
ls.push(markdown);
} else if (cell.cell_type?.language === "ojs") {
await handleOJSCell(cell);
} else {
ls.push(cell.sourceVerbatim);
}
}
if (selfContained) {
const selfContainedPageResources = await makeSelfContainedResources(
pageResources,
docDir,
);
const resolver = JSON.stringify(
Object.fromEntries(Array.from(selfContainedPageResources)),
);
scriptContents.unshift(
`window._ojs.runtime.setLocalResolver(${resolver});`,
);
} else {
for (const resource of uniqueResources(pageResources)) {
resourceFiles.push(resource.filename);
}
}
const serverMetadata = options.format.metadata?.server as any;
const normalizeMetadata = (key: string, def: string[]) => {
if (
!serverMetadata ||
(serverMetadata["type"] !== "shiny") ||
!serverMetadata[key]
) {
return def;
}
if (typeof serverMetadata[key] === "string") {
return [serverMetadata[key]];
} else {
return serverMetadata[key];
}
};
const shinyInputMetadata = normalizeMetadata("ojs-export", ["viewof"]);
const shinyOutputMetadata = normalizeMetadata("ojs-import", []);
const shinyInputs = new Set<string>();
const shinyInputExcludes = new Set<string>();
if (serverMetadata?.["ojs-exports"]) {
throw new Error(
"Document metadata contains server.ojs-exports; did you mean 'ojs-export' instead?",
);
}
if (serverMetadata?.["ojs-imports"]) {
throw new Error(
"Document metadata contains server.ojs-imports; did you mean 'ojs-import' instead?",
);
}
let importAllViews = false;
let importEverything = false;
for (const shinyInput of shinyInputMetadata) {
if (shinyInput === "viewof") {
importAllViews = true;
} else if (shinyInput === "all") {
importEverything = true;
} else if (shinyInput.startsWith("~")) {
shinyInputExcludes.add(shinyInput.slice(1));
} else {
shinyInputs.add(shinyInput);
}
}
const resultSet = new Set<string>();
if (importEverything) {
for (const el of ojsViews) {
resultSet.add(el);
}
for (const el of ojsIdentifiers) {
resultSet.add(el);
}
}
if (importAllViews) {
for (const el of ojsViews) {
resultSet.add(el);
}
}
for (const el of shinyInputs) {
resultSet.add(el);
}
for (const el of shinyInputExcludes) {
resultSet.delete(el);
}
for (const el of resultSet) {
moduleContents.push({
methodName: "interpretQuiet",
source: `shinyInput('${el}')`,
});
}
for (const el of shinyOutputMetadata) {
moduleContents.push({
methodName: "interpretQuiet",
source: `${el} = shinyOutput('${el}')`,
});
}
scriptContents.push("window._ojs.runtime.interpretFromScriptTags();");
const afterBody = [
`<script type="ojs-module-contents">`,
encodeBase64(JSON.stringify({ contents: moduleContents })),
`</script>`,
`<script type="module">`,
...scriptContents,
`</script>`,
]
.join("\n");
const includeAfterBodyFile = options.temp.createFile();
Deno.writeTextFileSync(includeAfterBodyFile, afterBody);
const extras = resolveDependencies(
{
html: {
[kDependencies]: [ojsFormatDependency(selfContained)],
},
},
dirname(options.source),
options.libDir,
options.temp,
project,
);
const ojsBundleTempFiles = [];
if (selfContained) {
const ojsBundleFilename = join(
quartoConfig.sharePath(),
"formats/html/ojs/quarto-ojs-runtime.js",
);
const ojsBundle = [
`<script type="module">`,
Deno.readTextFileSync(ojsBundleFilename),
`</script>`,
];
const filename = options.temp.createFile();
Deno.writeTextFileSync(filename, ojsBundle.join("\n"));
ojsBundleTempFiles.push(filename);
}
const includeInHeader = [
...(extras?.[kIncludeInHeader] || []),
...ojsBundleTempFiles,
];
return {
markdown: mappedJoin(ls, ""),
filters: [
"ojs",
],
includes: {
[kIncludeInHeader]: includeInHeader,
[kIncludeAfterBody]: [includeAfterBodyFile],
},
resourceFiles,
};
}
export async function ojsExecuteResult(
context: RenderContext,
executeResult: MappedExecuteResult,
ojsBlockLineNumbers: number[],
) {
executeResult = {
...executeResult,
};
const { markdown, includes, filters, resourceFiles } = await ojsCompile({
source: context.target.source,
format: context.format,
markdown: executeResult.markdown,
libDir: context.libDir,
project: context.project,
temp: context.options.services.temp,
ojsBlockLineNumbers,
});
executeResult.markdown = markdown;
if (includes) {
executeResult.includes = mergeConfigs(
includes,
executeResult.includes || {},
);
}
if (filters) {
executeResult.filters = (executeResult.filters || []).concat(filters);
}
return {
executeResult,
resourceFiles: resourceFiles || [],
};
}
function asUndefined(value: any, test: any) {
if (value === test) {
return undefined;
}
return value;
}
function ojsFormatDependency(selfContained: boolean) {
const ojsResource = (resource: string) =>
formatResourcePath(
"html",
join("ojs", resource),
);
const ojsDependency = (
resource: string,
attribs?: Record<string, string>,
) => ({
name: resource,
path: ojsResource(resource),
attribs,
});
const scripts = selfContained ? [] : [
ojsDependency("quarto-ojs-runtime.js", { type: "module" }),
];
return {
name: "quarto-ojs",
stylesheets: [
ojsDependency("quarto-ojs.css"),
],
scripts,
};
}