import { readFileSync, existsSync, writeFileSync, write } from "fs"; import { fileURLToPath } from "url"; import path, { dirname } from "path"; import { XMLParser } from "fast-xml-parser"; import { ensureArray } from "./common"; import { json } from "stream/consumers"; export interface FunctionParam { type: string; name: string; description: string; } export interface PythonApiDocFunction { name: string; definition: string; description: string; params: FunctionParam[]; example: string; returns: { type: string; description: string; }; } export interface PythonApiDocModule { name: string; description: string; functions: PythonApiDocFunction[]; } export interface PythonApiDocFile { files: PythonApiDocModule[]; } export interface MDOutput { filename: string; buffer: string; } export class PythonApiDocParser { private xmlParser: XMLParser; private buffer: string; private docPath: string; private repoPath: string; private fileCache: Record = {}; private apiDoc: PythonApiDocFile; private outputs: MDOutput[] = []; constructor(docEntry: { buffer: string; docPath: string; repoPath: string }) { this.buffer = docEntry.buffer; this.docPath = docEntry.docPath; this.repoPath = docEntry.repoPath; this.xmlParser = new XMLParser({ ignoreAttributes: false }); this.apiDoc = { files: [] }; } private preloadAllXmlFiles(fileList: string[]) { for (const file of fileList) { this.fileCache[file] = readFileSync(path.join(this.docPath, `${file}.xml`), "utf-8"); } } parse() { const buffer = this.buffer; const parser = this.xmlParser; const obj = parser.parse(buffer); if (!obj?.document || !obj?.document?.section) return; const document = obj?.document; if (!document) return; const section = document?.section; if (!section) return; const compound = section?.compound; if (!compound) return; const compactParagraph = compound?.["compact_paragraph"]; if (!compactParagraph) return; const bulletList = compactParagraph?.["bullet_list"]; if (!bulletList) return; const listItems = ensureArray(bulletList?.["list_item"]); if (!listItems) return; const importPaths = listItems.map((li) => li?.["compact_paragraph"]?.["reference"]?.["@_refuri"]).filter(Boolean); this.preloadAllXmlFiles(importPaths); const moduleFiles = listItems.map((li) => li?.["compact_paragraph"]?.["reference"]?.["@_refuri"]).filter(Boolean); const modules: PythonApiDocModule[] = []; for (const module of moduleFiles) { this.apiDoc.files.push(this.parseModuleFile(module)); } } private parseModuleFile(moduleFile: any): PythonApiDocModule { const module = this.fileCache[moduleFile]; const parser = new XMLParser({ ignoreAttributes: false, preserveOrder: true }); const parsedModule = parser.parse(module); const document = parsedModule?.find((d: any) => d.hasOwnProperty("document"))?.document; if (!document) { console.warn(`Could not find document for ${moduleFile}`); process.exit(1); } const section = document?.find((s: any) => s.hasOwnProperty("section"))?.section; if (!section) { console.warn(`Could not find section for ${moduleFile}`); process.exit(1); } const paragraphs = section?.filter( (ele: any) => !ele.hasOwnProperty("title") && !ele.hasOwnProperty("index") && !ele.hasOwnProperty("desc") ); let moduleDescription = ""; for (const node of paragraphs) { if (node.hasOwnProperty("paragraph")) { moduleDescription += this.extractTextFromNode(node.paragraph).trim() + "\n\n"; } else if (node.hasOwnProperty("literal_block")) { moduleDescription += '```python\n' + this.extractTextFromNode(node.literal_block) + "\n```\n\n"; } } const functionDefinitions = section?.filter((ele: any) => ele.hasOwnProperty("desc")); const funcs: PythonApiDocFunction[] = []; for (const functionDescription of functionDefinitions) { funcs.push(this.parseFunction(functionDescription?.desc)); } return { name: moduleFile, description: moduleDescription, functions: funcs }; } private extractTextFromNode(node: any): string { if (typeof node === "string") { return node; } if (Array.isArray(node)) { let result = ""; for (let i = 0; i < node.length; i++) { const currentText = this.extractTextFromNode(node[i]); if (i > 0) { const prevChar = result.slice(-1); const nextChar = currentText.charAt(0); if ( prevChar && nextChar && prevChar !== " " && nextChar !== " " && !["\n", "(", "["].includes(prevChar) && ![".", ",", ":", ")", "]"].includes(nextChar) ) { result += " "; } } result += currentText; } return result; } if (typeof node === "object" && node !== null) { if ("#text" in node) { return node["#text"]; } let result = ""; for (const key in node) { result += this.extractTextFromNode(node[key]); } return result; } return ""; } private methodName(moduleName: string, method: string) { if (moduleName === "plugin") return method; else return `${moduleName}.${method}`; } private parseFunction(functionDefinition: any): PythonApiDocFunction { const desc = ensureArray(functionDefinition).find((ele: any) => ele.hasOwnProperty("desc_signature")); const signature = desc?.["desc_signature"]; const name = desc?.[":@"]?.["@_fullname"] || ""; const moduleName = desc?.[":@"]?.["@_module"] || ""; const paramList = signature?.find((ele: any) => ele.hasOwnProperty("desc_parameterlist"))?.["desc_parameterlist"]; const parsedParams: string[] = paramList?.map((param: any) => { const descParam = param?.["desc_parameter"]?.[0]; const sigName = descParam?.["desc_sig_name"]?.find((ele: any) => ele?.["#text"]); return sigName?.["#text"] || ""; }); const content = ensureArray(functionDefinition).find((ele: any) => ele.hasOwnProperty("desc_content") )?.desc_content; if (!content) { let method = this.methodName(moduleName, name); return { name: `${method}`, definition: `${method}(${parsedParams.join(", ")})`, description: "", params: [], example: "", returns: { type: "None", description: "" } }; } let description = ""; let example = ""; let currentParagraph = ""; let isExampleMode = false; let returnType = "None"; let returnDescription = ""; let functionParams: FunctionParam[] = []; for (const node of content) { const key = Object.keys(node)[0]; const value = node[key]; if (key === "paragraph") { const textContent = this.extractTextFromNode(value).trim(); if (textContent === "Example:") { if (currentParagraph.trim()) { description += currentParagraph.trim() + "\n\n"; currentParagraph = ""; } isExampleMode = true; continue; } if (isExampleMode) { example += textContent + "\n\n"; } else { example === "" && description.length > 0 && (description += "\n"); description += textContent + "\n\n"; } } else if (key === "literal_block") { const blockText = this.extractTextFromNode(value); if (isExampleMode) { example += blockText + "\n\n"; } else { description += blockText + "\n\n"; } } else if (key === "#text" || key === "literal") { const part = this.extractTextFromNode(value); if (isExampleMode) { example += part; } else { description += part; } } } if (currentParagraph.trim()) { if (isExampleMode) { example += currentParagraph.trim() + "\n\n"; } else { description += currentParagraph.trim() + "\n\n"; } } const fieldList = content?.find((ele: any) => ele.hasOwnProperty("field_list"))?.["field_list"]; if (fieldList) { const fields = fieldList?.map((field: any) => field?.field); for (const field of fields) { const fieldName = field?.find((ele: any) => ele.hasOwnProperty("field_name"))?.field_name; const fieldBody = field?.find((ele: any) => ele.hasOwnProperty("field_body"))?.field_body; const nameText = typeof fieldName === "string" ? fieldName : fieldName?.["#text"] || fieldName?.find?.((e: any) => e?.["#text"])?.["#text"]; if (nameText === "Parameters") { functionParams = this.parseFunctionParameters(fieldBody); } else if (nameText === "Returns") { returnDescription = this.parseFunctionReturnDescription(fieldBody); } else if (nameText === "Return type") { returnType = this.parseFunctionReturnType(fieldBody); } } } let method = this.methodName(moduleName, name); return { name: `${method}`, definition: `${method}(${parsedParams.join(", ")})`, description: description.trim(), params: functionParams, example: example.trim(), returns: { type: returnType, description: returnDescription } }; } private parseFunctionParameters(fieldBody: any): FunctionParam[] { const bulletList = fieldBody?.find((ele: any) => ele.hasOwnProperty("bullet_list"))?.["bullet_list"]; const listItems = bulletList?.filter((ele: any) => ele.hasOwnProperty("list_item")); const params: FunctionParam[] = []; if (listItems) { for (const listItem of listItems) { const item = listItem?.["list_item"]; const paragraph = item?.find((ele: any) => ele.hasOwnProperty("paragraph"))?.["paragraph"]; let description = ""; let paramName = ""; let paramType: string[] = []; let paramTypeParts: string[] = []; for (const paragraphLine of paragraph) { if (paragraphLine.hasOwnProperty("#text")) { description += paragraphLine["#text"]; } else if (paragraphLine.hasOwnProperty("literal_strong")) { paramName = paragraphLine?.["literal_strong"]?.find((ele: any) => ele.hasOwnProperty("#text"))?.["#text"]; } else if (paragraphLine.hasOwnProperty("literal_emphasis")) { paramType.push( paragraphLine?.["literal_emphasis"]?.find((ele: any) => ele.hasOwnProperty("#text"))?.["#text"] ); } } description = description.replace(/^[()\s–-]+|[()\s–-]+$/g, ""); params.push({ name: paramName, type: paramType?.join(" "), description }); } } return params; } private parseFunctionReturnDescription(fieldBody: any) { const paragraph = fieldBody?.find((ele: any) => ele.hasOwnProperty("paragraph"))?.["paragraph"]; return this.extractTextFromNode(paragraph); } private parseFunctionReturnType(fieldBody: any) { const paragraph = fieldBody ?.find((ele: any) => ele.hasOwnProperty("paragraph")) ?.["paragraph"]?.find((ele: any) => ele.hasOwnProperty("#text"))?.["#text"]; return paragraph; } toMD() { for (const module of this.apiDoc.files) { this.outputs.push({ filename: module.name + ".md", buffer: this.writeModule(module) }); } return this.outputs; } private writeModule(module: PythonApiDocModule) { let stream = ""; stream += this.writeFrontMatter(module.name, `${module.name} Module Reference`); stream += `${module.description}\n\n`; if (module.functions.length > 0) { stream += "# Functions\n\n"; for (const func of module.functions) { stream += this.writeFunction(func); } } return stream; } private writeFunction(func: PythonApiDocFunction) { let stream = ""; stream += "## " + func.name + "\n\n"; if (func.definition) { let definition = func.definition.includes('->') ? func.definition : func.definition + " -> " + func.returns.type; stream += "```python\n" + definition + "\n```\n"; } if (func.description) { // Self-explanatory. Best to omit. // stream += "#### šŸ“„ Description"; stream += "\n\n" + func.description + "\n\n"; } if (func.params.length > 0) { stream += "##### Parameters\n\n"; const rows = func.params.map((param) => [`\`${param.name}\``, param.type, param.description]); stream += this.generateMarkdownTable(["Parameter Name", "Type", "Description"], rows) + "\n\n"; } if (func.example) { stream += "##### Example\n\n"; stream += "```python\n" + func.example + "\n```\n"; } if (func.returns && func.returns.type != "None") { stream += "##### Return value\n\n"; stream += "`" + func.returns.type + "`" + `${func.returns.description ? `: ${func.returns.description}` : ""}` + "\n\n"; } stream += "\n\n\n"; return stream; } private escapeMarkdown(text: string): string { return text .replace(/&/g, "&") .replace(//g, ">") .replace(/\|/g, "|") .replace(/\[/g, "[") .replace(/\]/g, "]"); } private generateMarkdownTable(headers: string[], rows: string[][]): string { const escapedRows = rows.map((row) => row.map((cell) => this.escapeMarkdown(cell))); const escapedHeaders = headers.map((header) => this.escapeMarkdown(header)); const colWidths = headers.map((_, i) => Math.max(escapedHeaders[i].length, ...escapedRows.map((row) => (row[i] || "").length)) ); const pad = (text: string, width: number) => text + " ".repeat(width - text.length); const lines = []; lines.push("| " + escapedHeaders.map((h, i) => pad(h, colWidths[i])).join(" | ") + " |"); lines.push("| " + colWidths.map((w) => "-".repeat(w)).join(" | ") + " |"); for (const row of escapedRows) { lines.push("| " + row.map((cell, i) => pad(cell, colWidths[i])).join(" | ") + " |"); } return lines.join("\n"); } private beginBlock() { return `---\n`; } private endBlock() { return `---\n\n`; } private writeFrontMatter(title: string, description: string) { return `${this.beginBlock()}title: ${title}\ndescription: ${description || title}\n${this.endBlock()}`; } }