446 lines
15 KiB
TypeScript
446 lines
15 KiB
TypeScript
import { readFileSync, existsSync, writeFileSync, write } from "fs";
|
||
|
||
import { fileURLToPath } from "url";
|
||
import path, { dirname } from "path";
|
||
|
||
import { XMLParser } from "fast-xml-parser";
|
||
|
||
import { ensureArray } from "./common";
|
||
import { json } from "stream/consumers";
|
||
|
||
export interface FunctionParam {
|
||
type: string;
|
||
name: string;
|
||
description: string;
|
||
}
|
||
|
||
export interface PythonApiDocFunction {
|
||
name: string;
|
||
definition: string;
|
||
description: string;
|
||
params: FunctionParam[];
|
||
example: string;
|
||
returns: {
|
||
type: string;
|
||
description: string;
|
||
};
|
||
}
|
||
|
||
export interface PythonApiDocModule {
|
||
name: string;
|
||
description: string;
|
||
functions: PythonApiDocFunction[];
|
||
}
|
||
|
||
export interface PythonApiDocFile {
|
||
files: PythonApiDocModule[];
|
||
}
|
||
|
||
export interface MDOutput {
|
||
filename: string;
|
||
buffer: string;
|
||
}
|
||
|
||
export class PythonApiDocParser {
|
||
private xmlParser: XMLParser;
|
||
|
||
private buffer: string;
|
||
private docPath: string;
|
||
private repoPath: string;
|
||
private fileCache: Record<string, any> = {};
|
||
private apiDoc: PythonApiDocFile;
|
||
private outputs: MDOutput[] = [];
|
||
|
||
constructor(docEntry: { buffer: string; docPath: string; repoPath: string }) {
|
||
this.buffer = docEntry.buffer;
|
||
this.docPath = docEntry.docPath;
|
||
this.repoPath = docEntry.repoPath;
|
||
this.xmlParser = new XMLParser({ ignoreAttributes: false });
|
||
this.apiDoc = { files: [] };
|
||
}
|
||
|
||
private preloadAllXmlFiles(fileList: string[]) {
|
||
for (const file of fileList) {
|
||
this.fileCache[file] = readFileSync(path.join(this.docPath, `${file}.xml`), "utf-8");
|
||
}
|
||
}
|
||
|
||
parse() {
|
||
const buffer = this.buffer;
|
||
const parser = this.xmlParser;
|
||
const obj = parser.parse(buffer);
|
||
|
||
if (!obj?.document || !obj?.document?.section) return;
|
||
|
||
const document = obj?.document;
|
||
if (!document) return;
|
||
|
||
const section = document?.section;
|
||
if (!section) return;
|
||
|
||
const compound = section?.compound;
|
||
if (!compound) return;
|
||
|
||
const compactParagraph = compound?.["compact_paragraph"];
|
||
if (!compactParagraph) return;
|
||
|
||
const bulletList = compactParagraph?.["bullet_list"];
|
||
if (!bulletList) return;
|
||
|
||
const listItems = ensureArray(bulletList?.["list_item"]);
|
||
if (!listItems) return;
|
||
|
||
const importPaths = listItems.map((li) => li?.["compact_paragraph"]?.["reference"]?.["@_refuri"]).filter(Boolean);
|
||
|
||
this.preloadAllXmlFiles(importPaths);
|
||
|
||
const moduleFiles = listItems.map((li) => li?.["compact_paragraph"]?.["reference"]?.["@_refuri"]).filter(Boolean);
|
||
const modules: PythonApiDocModule[] = [];
|
||
for (const module of moduleFiles) {
|
||
this.apiDoc.files.push(this.parseModuleFile(module));
|
||
}
|
||
}
|
||
|
||
private parseModuleFile(moduleFile: any): PythonApiDocModule {
|
||
const module = this.fileCache[moduleFile];
|
||
const parser = new XMLParser({ ignoreAttributes: false, preserveOrder: true });
|
||
const parsedModule = parser.parse(module);
|
||
const document = parsedModule?.find((d: any) => d.hasOwnProperty("document"))?.document;
|
||
if (!document) {
|
||
console.warn(`Could not find document for ${moduleFile}`);
|
||
process.exit(1);
|
||
}
|
||
const section = document?.find((s: any) => s.hasOwnProperty("section"))?.section;
|
||
if (!section) {
|
||
console.warn(`Could not find section for ${moduleFile}`);
|
||
process.exit(1);
|
||
}
|
||
const paragraphs = section?.filter(
|
||
(ele: any) => !ele.hasOwnProperty("title") && !ele.hasOwnProperty("index") && !ele.hasOwnProperty("desc")
|
||
);
|
||
let moduleDescription = "";
|
||
for (const node of paragraphs) {
|
||
if (node.hasOwnProperty("paragraph")) {
|
||
moduleDescription += this.extractTextFromNode(node.paragraph).trim() + "\n\n";
|
||
} else if (node.hasOwnProperty("literal_block")) {
|
||
moduleDescription += '```python\n' + this.extractTextFromNode(node.literal_block) + "\n```\n\n";
|
||
}
|
||
}
|
||
const functionDefinitions = section?.filter((ele: any) => ele.hasOwnProperty("desc"));
|
||
const funcs: PythonApiDocFunction[] = [];
|
||
for (const functionDescription of functionDefinitions) {
|
||
funcs.push(this.parseFunction(functionDescription?.desc));
|
||
}
|
||
return {
|
||
name: moduleFile,
|
||
description: moduleDescription,
|
||
functions: funcs
|
||
};
|
||
}
|
||
|
||
private extractTextFromNode(node: any): string {
|
||
if (typeof node === "string") {
|
||
return node;
|
||
}
|
||
|
||
if (Array.isArray(node)) {
|
||
let result = "";
|
||
for (let i = 0; i < node.length; i++) {
|
||
const currentText = this.extractTextFromNode(node[i]);
|
||
|
||
if (i > 0) {
|
||
const prevChar = result.slice(-1);
|
||
const nextChar = currentText.charAt(0);
|
||
if (
|
||
prevChar &&
|
||
nextChar &&
|
||
prevChar !== " " &&
|
||
nextChar !== " " &&
|
||
!["\n", "(", "["].includes(prevChar) &&
|
||
![".", ",", ":", ")", "]"].includes(nextChar)
|
||
) {
|
||
result += " ";
|
||
}
|
||
}
|
||
result += currentText;
|
||
}
|
||
return result;
|
||
}
|
||
|
||
if (typeof node === "object" && node !== null) {
|
||
if ("#text" in node) {
|
||
return node["#text"];
|
||
}
|
||
let result = "";
|
||
for (const key in node) {
|
||
result += this.extractTextFromNode(node[key]);
|
||
}
|
||
return result;
|
||
}
|
||
|
||
return "";
|
||
}
|
||
|
||
private methodName(moduleName: string, method: string) {
|
||
if (moduleName === "plugin") return method;
|
||
else return `${moduleName}.${method}`;
|
||
}
|
||
|
||
private parseFunction(functionDefinition: any): PythonApiDocFunction {
|
||
const desc = ensureArray(functionDefinition).find((ele: any) => ele.hasOwnProperty("desc_signature"));
|
||
const signature = desc?.["desc_signature"];
|
||
|
||
const name = desc?.[":@"]?.["@_fullname"] || "";
|
||
const moduleName = desc?.[":@"]?.["@_module"] || "";
|
||
|
||
const paramList = signature?.find((ele: any) => ele.hasOwnProperty("desc_parameterlist"))?.["desc_parameterlist"];
|
||
const parsedParams: string[] = paramList?.map((param: any) => {
|
||
const descParam = param?.["desc_parameter"]?.[0];
|
||
const sigName = descParam?.["desc_sig_name"]?.find((ele: any) => ele?.["#text"]);
|
||
return sigName?.["#text"] || "";
|
||
});
|
||
|
||
const content = ensureArray(functionDefinition).find((ele: any) =>
|
||
ele.hasOwnProperty("desc_content")
|
||
)?.desc_content;
|
||
if (!content) {
|
||
let method = this.methodName(moduleName, name);
|
||
return {
|
||
name: `${method}`,
|
||
definition: `${method}(${parsedParams.join(", ")})`,
|
||
description: "",
|
||
params: [],
|
||
example: "",
|
||
returns: {
|
||
type: "None",
|
||
description: ""
|
||
}
|
||
};
|
||
}
|
||
|
||
let description = "";
|
||
let example = "";
|
||
let currentParagraph = "";
|
||
let isExampleMode = false;
|
||
let returnType = "None";
|
||
let returnDescription = "";
|
||
let functionParams: FunctionParam[] = [];
|
||
|
||
for (const node of content) {
|
||
const key = Object.keys(node)[0];
|
||
const value = node[key];
|
||
|
||
if (key === "paragraph") {
|
||
const textContent = this.extractTextFromNode(value).trim();
|
||
|
||
if (textContent === "Example:") {
|
||
if (currentParagraph.trim()) {
|
||
description += currentParagraph.trim() + "\n\n";
|
||
currentParagraph = "";
|
||
}
|
||
isExampleMode = true;
|
||
continue;
|
||
}
|
||
|
||
if (isExampleMode) {
|
||
example += textContent + "\n\n";
|
||
} else {
|
||
example === "" && description.length > 0 && (description += "\n");
|
||
description += textContent + "\n\n";
|
||
}
|
||
} else if (key === "literal_block") {
|
||
const blockText = this.extractTextFromNode(value);
|
||
if (isExampleMode) {
|
||
example += blockText + "\n\n";
|
||
} else {
|
||
description += blockText + "\n\n";
|
||
}
|
||
} else if (key === "#text" || key === "literal") {
|
||
const part = this.extractTextFromNode(value);
|
||
if (isExampleMode) {
|
||
example += part;
|
||
} else {
|
||
description += part;
|
||
}
|
||
}
|
||
}
|
||
if (currentParagraph.trim()) {
|
||
if (isExampleMode) {
|
||
example += currentParagraph.trim() + "\n\n";
|
||
} else {
|
||
description += currentParagraph.trim() + "\n\n";
|
||
}
|
||
}
|
||
|
||
const fieldList = content?.find((ele: any) => ele.hasOwnProperty("field_list"))?.["field_list"];
|
||
|
||
if (fieldList) {
|
||
const fields = fieldList?.map((field: any) => field?.field);
|
||
for (const field of fields) {
|
||
const fieldName = field?.find((ele: any) => ele.hasOwnProperty("field_name"))?.field_name;
|
||
const fieldBody = field?.find((ele: any) => ele.hasOwnProperty("field_body"))?.field_body;
|
||
|
||
const nameText =
|
||
typeof fieldName === "string"
|
||
? fieldName
|
||
: fieldName?.["#text"] || fieldName?.find?.((e: any) => e?.["#text"])?.["#text"];
|
||
|
||
if (nameText === "Parameters") {
|
||
functionParams = this.parseFunctionParameters(fieldBody);
|
||
} else if (nameText === "Returns") {
|
||
returnDescription = this.parseFunctionReturnDescription(fieldBody);
|
||
} else if (nameText === "Return type") {
|
||
returnType = this.parseFunctionReturnType(fieldBody);
|
||
}
|
||
}
|
||
}
|
||
|
||
let method = this.methodName(moduleName, name);
|
||
return {
|
||
name: `${method}`,
|
||
definition: `${method}(${parsedParams.join(", ")})`,
|
||
description: description.trim(),
|
||
params: functionParams,
|
||
example: example.trim(),
|
||
returns: {
|
||
type: returnType,
|
||
description: returnDescription
|
||
}
|
||
};
|
||
}
|
||
|
||
private parseFunctionParameters(fieldBody: any): FunctionParam[] {
|
||
const bulletList = fieldBody?.find((ele: any) => ele.hasOwnProperty("bullet_list"))?.["bullet_list"];
|
||
const listItems = bulletList?.filter((ele: any) => ele.hasOwnProperty("list_item"));
|
||
|
||
const params: FunctionParam[] = [];
|
||
if (listItems) {
|
||
for (const listItem of listItems) {
|
||
const item = listItem?.["list_item"];
|
||
const paragraph = item?.find((ele: any) => ele.hasOwnProperty("paragraph"))?.["paragraph"];
|
||
let description = "";
|
||
let paramName = "";
|
||
let paramType: string[] = [];
|
||
let paramTypeParts: string[] = [];
|
||
for (const paragraphLine of paragraph) {
|
||
if (paragraphLine.hasOwnProperty("#text")) {
|
||
description += paragraphLine["#text"];
|
||
} else if (paragraphLine.hasOwnProperty("literal_strong")) {
|
||
paramName = paragraphLine?.["literal_strong"]?.find((ele: any) => ele.hasOwnProperty("#text"))?.["#text"];
|
||
} else if (paragraphLine.hasOwnProperty("literal_emphasis")) {
|
||
paramType.push(
|
||
paragraphLine?.["literal_emphasis"]?.find((ele: any) => ele.hasOwnProperty("#text"))?.["#text"]
|
||
);
|
||
}
|
||
}
|
||
description = description.replace(/^[()\s–-]+|[()\s–-]+$/g, "");
|
||
params.push({ name: paramName, type: paramType?.join(" "), description });
|
||
}
|
||
}
|
||
return params;
|
||
}
|
||
|
||
private parseFunctionReturnDescription(fieldBody: any) {
|
||
const paragraph = fieldBody?.find((ele: any) => ele.hasOwnProperty("paragraph"))?.["paragraph"];
|
||
return this.extractTextFromNode(paragraph);
|
||
}
|
||
|
||
private parseFunctionReturnType(fieldBody: any) {
|
||
const paragraph = fieldBody
|
||
?.find((ele: any) => ele.hasOwnProperty("paragraph"))
|
||
?.["paragraph"]?.find((ele: any) => ele.hasOwnProperty("#text"))?.["#text"];
|
||
return paragraph;
|
||
}
|
||
|
||
toMD() {
|
||
for (const module of this.apiDoc.files) {
|
||
this.outputs.push({ filename: module.name + ".md", buffer: this.writeModule(module) });
|
||
}
|
||
return this.outputs;
|
||
}
|
||
|
||
private writeModule(module: PythonApiDocModule) {
|
||
let stream = "";
|
||
stream += this.writeFrontMatter(module.name, `${module.name} Module Reference`);
|
||
stream += `${module.description}\n\n`;
|
||
if (module.functions.length > 0) {
|
||
stream += "# Functions\n\n";
|
||
for (const func of module.functions) {
|
||
stream += this.writeFunction(func);
|
||
}
|
||
}
|
||
return stream;
|
||
}
|
||
|
||
private writeFunction(func: PythonApiDocFunction) {
|
||
let stream = "";
|
||
stream += "## " + func.name + "\n\n";
|
||
if (func.definition) {
|
||
let definition = func.definition.includes('->') ? func.definition : func.definition + " -> " + func.returns.type;
|
||
stream += "```python\n" + definition + "\n```\n";
|
||
}
|
||
if (func.description) {
|
||
// Self-explanatory. Best to omit.
|
||
// stream += "#### 📄 Description";
|
||
stream += "\n\n" + func.description + "\n\n";
|
||
}
|
||
if (func.params.length > 0) {
|
||
stream += "##### Parameters\n\n";
|
||
const rows = func.params.map((param) => [`\`${param.name}\``, param.type, param.description]);
|
||
stream += this.generateMarkdownTable(["Parameter Name", "Type", "Description"], rows) + "\n\n";
|
||
}
|
||
if (func.example) {
|
||
stream += "##### Example\n\n";
|
||
stream += "```python\n" + func.example + "\n```\n";
|
||
}
|
||
if (func.returns && func.returns.type != "None") {
|
||
stream += "##### Return value\n\n";
|
||
stream +=
|
||
"`" + func.returns.type + "`" + `${func.returns.description ? `: ${func.returns.description}` : ""}` + "\n\n";
|
||
}
|
||
stream += "\n\n\n";
|
||
return stream;
|
||
}
|
||
|
||
private escapeMarkdown(text: string): string {
|
||
return text
|
||
.replace(/&/g, "&")
|
||
.replace(/</g, "<")
|
||
.replace(/>/g, ">")
|
||
.replace(/\|/g, "|")
|
||
.replace(/\[/g, "[")
|
||
.replace(/\]/g, "]");
|
||
}
|
||
|
||
private generateMarkdownTable(headers: string[], rows: string[][]): string {
|
||
const escapedRows = rows.map((row) => row.map((cell) => this.escapeMarkdown(cell)));
|
||
const escapedHeaders = headers.map((header) => this.escapeMarkdown(header));
|
||
const colWidths = headers.map((_, i) =>
|
||
Math.max(escapedHeaders[i].length, ...escapedRows.map((row) => (row[i] || "").length))
|
||
);
|
||
const pad = (text: string, width: number) => text + " ".repeat(width - text.length);
|
||
|
||
const lines = [];
|
||
lines.push("| " + escapedHeaders.map((h, i) => pad(h, colWidths[i])).join(" | ") + " |");
|
||
lines.push("| " + colWidths.map((w) => "-".repeat(w)).join(" | ") + " |");
|
||
|
||
for (const row of escapedRows) {
|
||
lines.push("| " + row.map((cell, i) => pad(cell, colWidths[i])).join(" | ") + " |");
|
||
}
|
||
|
||
return lines.join("\n");
|
||
}
|
||
|
||
private beginBlock() {
|
||
return `---\n`;
|
||
}
|
||
|
||
private endBlock() {
|
||
return `---\n\n`;
|
||
}
|
||
|
||
private writeFrontMatter(title: string, description: string) {
|
||
return `${this.beginBlock()}title: ${title}\ndescription: ${description || title}\n${this.endBlock()}`;
|
||
}
|
||
}
|