Initial commit
This commit is contained in:
445
src/scripts/parser/apidoc/python-apidoc-parser.ts
Normal file
445
src/scripts/parser/apidoc/python-apidoc-parser.ts
Normal file
@@ -0,0 +1,445 @@
|
||||
import { readFileSync, existsSync, writeFileSync, write } from "fs";
|
||||
|
||||
import { fileURLToPath } from "url";
|
||||
import path, { dirname } from "path";
|
||||
|
||||
import { XMLParser } from "fast-xml-parser";
|
||||
|
||||
import { ensureArray } from "./common";
|
||||
import { json } from "stream/consumers";
|
||||
|
||||
export interface FunctionParam {
|
||||
type: string;
|
||||
name: string;
|
||||
description: string;
|
||||
}
|
||||
|
||||
export interface PythonApiDocFunction {
|
||||
name: string;
|
||||
definition: string;
|
||||
description: string;
|
||||
params: FunctionParam[];
|
||||
example: string;
|
||||
returns: {
|
||||
type: string;
|
||||
description: string;
|
||||
};
|
||||
}
|
||||
|
||||
export interface PythonApiDocModule {
|
||||
name: string;
|
||||
description: string;
|
||||
functions: PythonApiDocFunction[];
|
||||
}
|
||||
|
||||
export interface PythonApiDocFile {
|
||||
files: PythonApiDocModule[];
|
||||
}
|
||||
|
||||
export interface MDOutput {
|
||||
filename: string;
|
||||
buffer: string;
|
||||
}
|
||||
|
||||
export class PythonApiDocParser {
|
||||
private xmlParser: XMLParser;
|
||||
|
||||
private buffer: string;
|
||||
private docPath: string;
|
||||
private repoPath: string;
|
||||
private fileCache: Record<string, any> = {};
|
||||
private apiDoc: PythonApiDocFile;
|
||||
private outputs: MDOutput[] = [];
|
||||
|
||||
constructor(docEntry: { buffer: string; docPath: string; repoPath: string }) {
|
||||
this.buffer = docEntry.buffer;
|
||||
this.docPath = docEntry.docPath;
|
||||
this.repoPath = docEntry.repoPath;
|
||||
this.xmlParser = new XMLParser({ ignoreAttributes: false });
|
||||
this.apiDoc = { files: [] };
|
||||
}
|
||||
|
||||
private preloadAllXmlFiles(fileList: string[]) {
|
||||
for (const file of fileList) {
|
||||
this.fileCache[file] = readFileSync(path.join(this.docPath, `${file}.xml`), "utf-8");
|
||||
}
|
||||
}
|
||||
|
||||
parse() {
|
||||
const buffer = this.buffer;
|
||||
const parser = this.xmlParser;
|
||||
const obj = parser.parse(buffer);
|
||||
|
||||
if (!obj?.document || !obj?.document?.section) return;
|
||||
|
||||
const document = obj?.document;
|
||||
if (!document) return;
|
||||
|
||||
const section = document?.section;
|
||||
if (!section) return;
|
||||
|
||||
const compound = section?.compound;
|
||||
if (!compound) return;
|
||||
|
||||
const compactParagraph = compound?.["compact_paragraph"];
|
||||
if (!compactParagraph) return;
|
||||
|
||||
const bulletList = compactParagraph?.["bullet_list"];
|
||||
if (!bulletList) return;
|
||||
|
||||
const listItems = ensureArray(bulletList?.["list_item"]);
|
||||
if (!listItems) return;
|
||||
|
||||
const importPaths = listItems.map((li) => li?.["compact_paragraph"]?.["reference"]?.["@_refuri"]).filter(Boolean);
|
||||
|
||||
this.preloadAllXmlFiles(importPaths);
|
||||
|
||||
const moduleFiles = listItems.map((li) => li?.["compact_paragraph"]?.["reference"]?.["@_refuri"]).filter(Boolean);
|
||||
const modules: PythonApiDocModule[] = [];
|
||||
for (const module of moduleFiles) {
|
||||
this.apiDoc.files.push(this.parseModuleFile(module));
|
||||
}
|
||||
}
|
||||
|
||||
private parseModuleFile(moduleFile: any): PythonApiDocModule {
|
||||
const module = this.fileCache[moduleFile];
|
||||
const parser = new XMLParser({ ignoreAttributes: false, preserveOrder: true });
|
||||
const parsedModule = parser.parse(module);
|
||||
const document = parsedModule?.find((d: any) => d.hasOwnProperty("document"))?.document;
|
||||
if (!document) {
|
||||
console.warn(`Could not find document for ${moduleFile}`);
|
||||
process.exit(1);
|
||||
}
|
||||
const section = document?.find((s: any) => s.hasOwnProperty("section"))?.section;
|
||||
if (!section) {
|
||||
console.warn(`Could not find section for ${moduleFile}`);
|
||||
process.exit(1);
|
||||
}
|
||||
const paragraphs = section?.filter(
|
||||
(ele: any) => !ele.hasOwnProperty("title") && !ele.hasOwnProperty("index") && !ele.hasOwnProperty("desc")
|
||||
);
|
||||
let moduleDescription = "";
|
||||
for (const node of paragraphs) {
|
||||
if (node.hasOwnProperty("paragraph")) {
|
||||
moduleDescription += this.extractTextFromNode(node.paragraph).trim() + "\n\n";
|
||||
} else if (node.hasOwnProperty("literal_block")) {
|
||||
moduleDescription += '```python\n' + this.extractTextFromNode(node.literal_block) + "\n```\n\n";
|
||||
}
|
||||
}
|
||||
const functionDefinitions = section?.filter((ele: any) => ele.hasOwnProperty("desc"));
|
||||
const funcs: PythonApiDocFunction[] = [];
|
||||
for (const functionDescription of functionDefinitions) {
|
||||
funcs.push(this.parseFunction(functionDescription?.desc));
|
||||
}
|
||||
return {
|
||||
name: moduleFile,
|
||||
description: moduleDescription,
|
||||
functions: funcs
|
||||
};
|
||||
}
|
||||
|
||||
private extractTextFromNode(node: any): string {
|
||||
if (typeof node === "string") {
|
||||
return node;
|
||||
}
|
||||
|
||||
if (Array.isArray(node)) {
|
||||
let result = "";
|
||||
for (let i = 0; i < node.length; i++) {
|
||||
const currentText = this.extractTextFromNode(node[i]);
|
||||
|
||||
if (i > 0) {
|
||||
const prevChar = result.slice(-1);
|
||||
const nextChar = currentText.charAt(0);
|
||||
if (
|
||||
prevChar &&
|
||||
nextChar &&
|
||||
prevChar !== " " &&
|
||||
nextChar !== " " &&
|
||||
!["\n", "(", "["].includes(prevChar) &&
|
||||
![".", ",", ":", ")", "]"].includes(nextChar)
|
||||
) {
|
||||
result += " ";
|
||||
}
|
||||
}
|
||||
result += currentText;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
if (typeof node === "object" && node !== null) {
|
||||
if ("#text" in node) {
|
||||
return node["#text"];
|
||||
}
|
||||
let result = "";
|
||||
for (const key in node) {
|
||||
result += this.extractTextFromNode(node[key]);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
return "";
|
||||
}
|
||||
|
||||
private methodName(moduleName: string, method: string) {
|
||||
if (moduleName === "plugin") return method;
|
||||
else return `${moduleName}.${method}`;
|
||||
}
|
||||
|
||||
private parseFunction(functionDefinition: any): PythonApiDocFunction {
|
||||
const desc = ensureArray(functionDefinition).find((ele: any) => ele.hasOwnProperty("desc_signature"));
|
||||
const signature = desc?.["desc_signature"];
|
||||
|
||||
const name = desc?.[":@"]?.["@_fullname"] || "";
|
||||
const moduleName = desc?.[":@"]?.["@_module"] || "";
|
||||
|
||||
const paramList = signature?.find((ele: any) => ele.hasOwnProperty("desc_parameterlist"))?.["desc_parameterlist"];
|
||||
const parsedParams: string[] = paramList?.map((param: any) => {
|
||||
const descParam = param?.["desc_parameter"]?.[0];
|
||||
const sigName = descParam?.["desc_sig_name"]?.find((ele: any) => ele?.["#text"]);
|
||||
return sigName?.["#text"] || "";
|
||||
});
|
||||
|
||||
const content = ensureArray(functionDefinition).find((ele: any) =>
|
||||
ele.hasOwnProperty("desc_content")
|
||||
)?.desc_content;
|
||||
if (!content) {
|
||||
let method = this.methodName(moduleName, name);
|
||||
return {
|
||||
name: `${method}`,
|
||||
definition: `${method}(${parsedParams.join(", ")})`,
|
||||
description: "",
|
||||
params: [],
|
||||
example: "",
|
||||
returns: {
|
||||
type: "None",
|
||||
description: ""
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
let description = "";
|
||||
let example = "";
|
||||
let currentParagraph = "";
|
||||
let isExampleMode = false;
|
||||
let returnType = "None";
|
||||
let returnDescription = "";
|
||||
let functionParams: FunctionParam[] = [];
|
||||
|
||||
for (const node of content) {
|
||||
const key = Object.keys(node)[0];
|
||||
const value = node[key];
|
||||
|
||||
if (key === "paragraph") {
|
||||
const textContent = this.extractTextFromNode(value).trim();
|
||||
|
||||
if (textContent === "Example:") {
|
||||
if (currentParagraph.trim()) {
|
||||
description += currentParagraph.trim() + "\n\n";
|
||||
currentParagraph = "";
|
||||
}
|
||||
isExampleMode = true;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (isExampleMode) {
|
||||
example += textContent + "\n\n";
|
||||
} else {
|
||||
example === "" && description.length > 0 && (description += "\n");
|
||||
description += textContent + "\n\n";
|
||||
}
|
||||
} else if (key === "literal_block") {
|
||||
const blockText = this.extractTextFromNode(value);
|
||||
if (isExampleMode) {
|
||||
example += blockText + "\n\n";
|
||||
} else {
|
||||
description += blockText + "\n\n";
|
||||
}
|
||||
} else if (key === "#text" || key === "literal") {
|
||||
const part = this.extractTextFromNode(value);
|
||||
if (isExampleMode) {
|
||||
example += part;
|
||||
} else {
|
||||
description += part;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (currentParagraph.trim()) {
|
||||
if (isExampleMode) {
|
||||
example += currentParagraph.trim() + "\n\n";
|
||||
} else {
|
||||
description += currentParagraph.trim() + "\n\n";
|
||||
}
|
||||
}
|
||||
|
||||
const fieldList = content?.find((ele: any) => ele.hasOwnProperty("field_list"))?.["field_list"];
|
||||
|
||||
if (fieldList) {
|
||||
const fields = fieldList?.map((field: any) => field?.field);
|
||||
for (const field of fields) {
|
||||
const fieldName = field?.find((ele: any) => ele.hasOwnProperty("field_name"))?.field_name;
|
||||
const fieldBody = field?.find((ele: any) => ele.hasOwnProperty("field_body"))?.field_body;
|
||||
|
||||
const nameText =
|
||||
typeof fieldName === "string"
|
||||
? fieldName
|
||||
: fieldName?.["#text"] || fieldName?.find?.((e: any) => e?.["#text"])?.["#text"];
|
||||
|
||||
if (nameText === "Parameters") {
|
||||
functionParams = this.parseFunctionParameters(fieldBody);
|
||||
} else if (nameText === "Returns") {
|
||||
returnDescription = this.parseFunctionReturnDescription(fieldBody);
|
||||
} else if (nameText === "Return type") {
|
||||
returnType = this.parseFunctionReturnType(fieldBody);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let method = this.methodName(moduleName, name);
|
||||
return {
|
||||
name: `${method}`,
|
||||
definition: `${method}(${parsedParams.join(", ")})`,
|
||||
description: description.trim(),
|
||||
params: functionParams,
|
||||
example: example.trim(),
|
||||
returns: {
|
||||
type: returnType,
|
||||
description: returnDescription
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
private parseFunctionParameters(fieldBody: any): FunctionParam[] {
|
||||
const bulletList = fieldBody?.find((ele: any) => ele.hasOwnProperty("bullet_list"))?.["bullet_list"];
|
||||
const listItems = bulletList?.filter((ele: any) => ele.hasOwnProperty("list_item"));
|
||||
|
||||
const params: FunctionParam[] = [];
|
||||
if (listItems) {
|
||||
for (const listItem of listItems) {
|
||||
const item = listItem?.["list_item"];
|
||||
const paragraph = item?.find((ele: any) => ele.hasOwnProperty("paragraph"))?.["paragraph"];
|
||||
let description = "";
|
||||
let paramName = "";
|
||||
let paramType: string[] = [];
|
||||
let paramTypeParts: string[] = [];
|
||||
for (const paragraphLine of paragraph) {
|
||||
if (paragraphLine.hasOwnProperty("#text")) {
|
||||
description += paragraphLine["#text"];
|
||||
} else if (paragraphLine.hasOwnProperty("literal_strong")) {
|
||||
paramName = paragraphLine?.["literal_strong"]?.find((ele: any) => ele.hasOwnProperty("#text"))?.["#text"];
|
||||
} else if (paragraphLine.hasOwnProperty("literal_emphasis")) {
|
||||
paramType.push(
|
||||
paragraphLine?.["literal_emphasis"]?.find((ele: any) => ele.hasOwnProperty("#text"))?.["#text"]
|
||||
);
|
||||
}
|
||||
}
|
||||
description = description.replace(/^[()\s–-]+|[()\s–-]+$/g, "");
|
||||
params.push({ name: paramName, type: paramType?.join(" "), description });
|
||||
}
|
||||
}
|
||||
return params;
|
||||
}
|
||||
|
||||
private parseFunctionReturnDescription(fieldBody: any) {
|
||||
const paragraph = fieldBody?.find((ele: any) => ele.hasOwnProperty("paragraph"))?.["paragraph"];
|
||||
return this.extractTextFromNode(paragraph);
|
||||
}
|
||||
|
||||
private parseFunctionReturnType(fieldBody: any) {
|
||||
const paragraph = fieldBody
|
||||
?.find((ele: any) => ele.hasOwnProperty("paragraph"))
|
||||
?.["paragraph"]?.find((ele: any) => ele.hasOwnProperty("#text"))?.["#text"];
|
||||
return paragraph;
|
||||
}
|
||||
|
||||
toMD() {
|
||||
for (const module of this.apiDoc.files) {
|
||||
this.outputs.push({ filename: module.name + ".md", buffer: this.writeModule(module) });
|
||||
}
|
||||
return this.outputs;
|
||||
}
|
||||
|
||||
private writeModule(module: PythonApiDocModule) {
|
||||
let stream = "";
|
||||
stream += this.writeFrontMatter(module.name, `${module.name} Module Reference`);
|
||||
stream += `${module.description}\n\n`;
|
||||
if (module.functions.length > 0) {
|
||||
stream += "# Functions\n\n";
|
||||
for (const func of module.functions) {
|
||||
stream += this.writeFunction(func);
|
||||
}
|
||||
}
|
||||
return stream;
|
||||
}
|
||||
|
||||
private writeFunction(func: PythonApiDocFunction) {
|
||||
let stream = "";
|
||||
stream += "## " + func.name + "\n\n";
|
||||
if (func.definition) {
|
||||
let definition = func.definition.includes('->') ? func.definition : func.definition + " -> " + func.returns.type;
|
||||
stream += "```python\n" + definition + "\n```\n";
|
||||
}
|
||||
if (func.description) {
|
||||
// Self-explanatory. Best to omit.
|
||||
// stream += "#### 📄 Description";
|
||||
stream += "\n\n" + func.description + "\n\n";
|
||||
}
|
||||
if (func.params.length > 0) {
|
||||
stream += "##### Parameters\n\n";
|
||||
const rows = func.params.map((param) => [`\`${param.name}\``, param.type, param.description]);
|
||||
stream += this.generateMarkdownTable(["Parameter Name", "Type", "Description"], rows) + "\n\n";
|
||||
}
|
||||
if (func.example) {
|
||||
stream += "##### Example\n\n";
|
||||
stream += "```python\n" + func.example + "\n```\n";
|
||||
}
|
||||
if (func.returns && func.returns.type != "None") {
|
||||
stream += "##### Return value\n\n";
|
||||
stream +=
|
||||
"`" + func.returns.type + "`" + `${func.returns.description ? `: ${func.returns.description}` : ""}` + "\n\n";
|
||||
}
|
||||
stream += "\n\n\n";
|
||||
return stream;
|
||||
}
|
||||
|
||||
private escapeMarkdown(text: string): string {
|
||||
return text
|
||||
.replace(/&/g, "&")
|
||||
.replace(/</g, "<")
|
||||
.replace(/>/g, ">")
|
||||
.replace(/\|/g, "|")
|
||||
.replace(/\[/g, "[")
|
||||
.replace(/\]/g, "]");
|
||||
}
|
||||
|
||||
private generateMarkdownTable(headers: string[], rows: string[][]): string {
|
||||
const escapedRows = rows.map((row) => row.map((cell) => this.escapeMarkdown(cell)));
|
||||
const escapedHeaders = headers.map((header) => this.escapeMarkdown(header));
|
||||
const colWidths = headers.map((_, i) =>
|
||||
Math.max(escapedHeaders[i].length, ...escapedRows.map((row) => (row[i] || "").length))
|
||||
);
|
||||
const pad = (text: string, width: number) => text + " ".repeat(width - text.length);
|
||||
|
||||
const lines = [];
|
||||
lines.push("| " + escapedHeaders.map((h, i) => pad(h, colWidths[i])).join(" | ") + " |");
|
||||
lines.push("| " + colWidths.map((w) => "-".repeat(w)).join(" | ") + " |");
|
||||
|
||||
for (const row of escapedRows) {
|
||||
lines.push("| " + row.map((cell, i) => pad(cell, colWidths[i])).join(" | ") + " |");
|
||||
}
|
||||
|
||||
return lines.join("\n");
|
||||
}
|
||||
|
||||
private beginBlock() {
|
||||
return `---\n`;
|
||||
}
|
||||
|
||||
private endBlock() {
|
||||
return `---\n\n`;
|
||||
}
|
||||
|
||||
private writeFrontMatter(title: string, description: string) {
|
||||
return `${this.beginBlock()}title: ${title}\ndescription: ${description || title}\n${this.endBlock()}`;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user