Initial commit

src/scripts/parser/apidoc/apidoc-parser.ts

@@ -0,0 +1,33 @@
+import { CApiDocParser } from "./c-apidoc-parser";
+import { PythonApiDocParser } from "./python-apidoc-parser";
+
+export interface MDOutput {
+  filename: string;
+  buffer: string;
+}
+
+export class ApiDocParser {
+  private cApiDocParser: CApiDocParser;
+  private pythonApiDocParser: PythonApiDocParser;
+
+  constructor(
+    CApiDocEntry: { buffer: string; docPath: string; repoPath: string },
+    pythonApiDocEntryFile: { buffer: string; docPath: string; repoPath: string }
+  ) {
+    this.cApiDocParser = new CApiDocParser(CApiDocEntry);
+    this.pythonApiDocParser = new PythonApiDocParser(pythonApiDocEntryFile);
+    this.parse();
+  }
+
+  private parse() {
+    this.cApiDocParser.parse();
+    this.pythonApiDocParser.parse();
+  }
+
+  toMD() {
+    return {
+      cApiDocs: this.cApiDocParser.toMD(),
+      pythonApiDocs: this.pythonApiDocParser.toMD()
+    };
+  }
+}

src/scripts/parser/apidoc/c-apidoc-parser.ts

@@ -0,0 +1,402 @@
+import { readFileSync, existsSync, writeFileSync } from "fs";
+
+import { fileURLToPath } from "url";
+import path, { dirname } from "path";
+import { ensureArray } from "./common";
+
+import { XMLParser } from "fast-xml-parser";
+
+export interface FunctionParam {
+  type: string;
+  declname: string;
+}
+
+export interface DetailedParam {
+  paramName: string;
+  paramDescription: string;
+}
+
+export interface FunctionDetailedParamDescription {
+  description: string;
+  params: DetailedParam[];
+  returns: string;
+}
+
+export interface FunctionDoc {
+  name: string;
+  definition: string;
+  argsstring: string;
+  param: FunctionParam[];
+  briefDescription: string;
+  detailedDescription: FunctionDetailedParamDescription;
+  location: {
+    definition: {
+      file: string;
+      line: number;
+      column: number;
+    };
+    declaration: {
+      file: string;
+      line: number;
+      column: number;
+    };
+  };
+}
+
+export interface TypeDefDoc {
+  type: string;
+  definition: string;
+  argsstring: string;
+  name: string;
+  briefDescription: string;
+  detailedDescription: string;
+  location: {
+    file: string;
+    line: number;
+    column: number;
+  };
+}
+
+export interface DocFile {
+  name: string;
+  briefDescription: string;
+  detailedDescription: string;
+  functions: FunctionDoc[];
+  typedefs: TypeDefDoc[];
+  location: string;
+}
+
+export interface ApiDoc {
+  files: DocFile[];
+}
+
+export interface MDOutput {
+  filename: string;
+  buffer: string;
+}
+
+export class CApiDocParser {
+  private xmlParser: XMLParser;
+  private buffer: string;
+  private docPath: string;
+  private repoPath: string;
+  private fileCache: Record<string, any> = {};
+  private sourceFiles: Record<string, string> = {};
+  private apiDoc: ApiDoc;
+  private outputs: MDOutput[] = [];
+
+  constructor(docEntry: { buffer: string; docPath: string; repoPath: string }) {
+    this.buffer = docEntry.buffer;
+    this.docPath = docEntry.docPath;
+    this.repoPath = docEntry.repoPath;
+    this.xmlParser = new XMLParser({ ignoreAttributes: false });
+    this.apiDoc = { files: [] };
+  }
+
+  private preloadAllXmlFiles(fileList: string[]) {
+    for (const file of fileList) {
+      this.fileCache[file] = readFileSync(path.join(this.docPath, `${file}.xml`), "utf-8");
+    }
+  }
+
+  private getXmlFileNameFromRefId(refId: string): string {
+    return refId;
+  }
+
+  parse() {
+    const buffer = this.buffer;
+    const parser = this.xmlParser;
+    const obj = parser.parse(buffer);
+    const compounds = obj.doxygenindex.compound;
+    const importPaths = (compounds as any[])
+      .filter((c) => c["@_kind"] === "file")
+      .map((c) => c["@_refid"])
+      .filter((refid): refid is string => typeof refid === "string");
+    this.preloadAllXmlFiles(importPaths);
+
+    for (const compound of compounds) {
+      const compoundType = compound["@_kind"];
+      if (compoundType === "page") continue;
+      else if (compoundType === "file") {
+        const file = this.parseDocFile(compound);
+        this.apiDoc.files.push(file);
+      }
+    }
+  }
+
+  private parseDocFile(compound: any): DocFile {
+    const functions: FunctionDoc[] = [];
+    const typedefs: TypeDefDoc[] = [];
+    const refId = compound["@_refid"];
+    const fileName = this.getXmlFileNameFromRefId(refId);
+    const file = this.fileCache[fileName];
+    if (!file) {
+      console.error(`File not found: ${fileName}`);
+    }
+    const parsedFile = this.xmlParser.parse(file);
+    const compounddef = parsedFile?.doxygen?.compounddef;
+    const compoundName = compounddef?.compoundname;
+    const briefDescription = compounddef?.briefdescription || "";
+    const detailedDescription = compounddef?.detaileddescription?.para || "";
+    const location = compounddef?.location?.["@_file"] || "";
+    const sectionDefs = ensureArray(compounddef?.sectiondef);
+    for (const sectionDef of sectionDefs) {
+      const sectionType = sectionDef["@_kind"];
+      if (sectionType === "typedef") {
+        const memberDefs = ensureArray(sectionDef?.memberdef);
+        for (const memberDef of memberDefs) {
+          const typedef = this.parseTypeDef(memberDef);
+          typedefs.push(typedef);
+        }
+      } else if (sectionType === "func") {
+        const memberDefs = ensureArray(sectionDef?.memberdef);
+        for (const memberDef of memberDefs) {
+          const func = this.parseFunction(memberDef);
+          functions.push(func);
+        }
+      }
+    }
+    return {
+      name: compoundName,
+      briefDescription,
+      detailedDescription,
+      functions,
+      typedefs,
+      location
+    };
+  }
+
+  private parseFunction(memberDef: any): FunctionDoc {
+    const type = memberDef["@_kind"];
+    const functionType = memberDef?.type || "";
+    const definition = memberDef?.definition || "";
+    const argsstring = memberDef?.argsstring || "";
+    const name = memberDef?.name || "";
+    const params = Array.isArray(memberDef?.param) ? memberDef?.param : [memberDef?.param];
+    const parsedParams: FunctionParam[] = [];
+    for (const param of params) {
+      parsedParams.push(this.parseFunctionParam(param));
+    }
+    const briefDescription = memberDef?.briefDescription || "";
+    const detailedDescription: FunctionDetailedParamDescription = this.parseFunctionDetailedDescription(
+      memberDef?.detaileddescription
+    );
+    const location = memberDef?.location;
+    const parsedLocation = {
+      definition: {
+        file: location?.["@_file"] || "",
+        line: Number(location?.["@_line"]) || 0,
+        column: Number(location?.["@_column"]) || 0
+      },
+      declaration: {
+        file: location?.["@_declfile"] || "",
+        line: Number(location?.["@_declline"]) || 0,
+        column: Number(location?.["@_declcolumn"]) || 0
+      }
+    };
+    return {
+      name,
+      definition,
+      argsstring,
+      param: parsedParams,
+      briefDescription,
+      detailedDescription,
+      location: parsedLocation
+    };
+  }
+
+  private parseFunctionParam(param: any): FunctionParam {
+    return {
+      type: param?.type,
+      declname: param?.declname
+    };
+  }
+
+  private parseFunctionDetailedDescription(description: any): FunctionDetailedParamDescription {
+    let descriptionText = "";
+    const params: DetailedParam[] = [];
+    let returnsText = "void";
+
+    if (description?.para) {
+      const para = description.para;
+
+      if (para["#text"]) {
+        descriptionText = para["#text"];
+      }
+
+      if (para.parameterlist && para.parameterlist["@_kind"] === "param") {
+        const parameterList = para.parameterlist;
+        const parameterItems = ensureArray(parameterList.parameteritem);
+
+        for (const parameterItem of parameterItems) {
+          const parameterName = parameterItem?.parameternamelist?.parametername || "";
+          const parameterDescription = parameterItem?.parameterdescription?.para || "";
+          params.push({
+            paramName: parameterName,
+            paramDescription: parameterDescription
+          });
+        }
+      }
+
+      if (para.simplesect && para.simplesect["@_kind"] === "return") {
+        returnsText = para.simplesect.para || "";
+      }
+    }
+
+    return {
+      description: descriptionText,
+      params,
+      returns: returnsText
+    };
+  }
+
+  private parseTypeDef(memberDef: any): TypeDefDoc {
+    const type = memberDef["@_kind"];
+    const typeDef = memberDef?.type || "";
+    const definition: string = memberDef?.definition || "";
+    const argsstring = memberDef?.argsstring || "";
+    const name = memberDef?.name || "";
+    const briefDescription = memberDef?.briefdescription || "";
+    const detailedDescription = memberDef?.detaileddescription?.para || "";
+    const location = {
+      file: memberDef?.location?.["@_file"] || "",
+      line: Number(memberDef?.location?.["@_line"]) || 0,
+      column: Number(memberDef?.location?.["@_column"]) || 0
+    };
+    return {
+      type: typeDef,
+      name,
+      definition,
+      argsstring,
+      briefDescription,
+      detailedDescription,
+      location
+    };
+  }
+
+  toMD(): MDOutput[] {
+    for (const docFile of this.apiDoc.files) {
+      this.writeFile(docFile);
+    }
+    return this.outputs;
+  }
+
+  private writeFile(docFile: DocFile) {
+    let stream = "";
+    stream += this.writeFrontMatter(docFile.name, docFile.briefDescription || `${docFile.name} File Reference`);
+    if (docFile.typedefs.length > 0) {
+      stream += "## Typedefs\n\n";
+      for (const typedef of docFile.typedefs) {
+        stream += this.writeTypedef(typedef);
+      }
+    }
+    if (docFile.functions.length > 0) {
+      stream += "## Functions\n\n";
+      for (const func of docFile.functions) {
+        stream += this.writeFunction(func);
+      }
+    }
+    this.outputs.push({
+      filename: docFile.name.replace(/\.h$/, ".md").replace(/\.c$/, ".md"),
+      buffer: stream
+    });
+  }
+
+  private escapeMarkdown(text: string): string {
+    return text
+      .replace(/&/g, "&amp;")
+      .replace(/</g, "&lt;")
+      .replace(/>/g, "&gt;")
+      .replace(/\|/g, "&#124;")
+      .replace(/\[/g, "&#91;")
+      .replace(/\]/g, "&#93;");
+  }
+
+  private generateMarkdownTable(headers: string[], rows: string[][]): string {
+    const escapedRows = rows.map((row) => row.map((cell) => this.escapeMarkdown(cell)));
+    const escapedHeaders = headers.map((header) => this.escapeMarkdown(header));
+    const colWidths = headers.map((_, i) =>
+      Math.max(escapedHeaders[i].length, ...escapedRows.map((row) => (row[i] || "").length))
+    );
+    const pad = (text: string, width: number) => text + " ".repeat(width - text.length);
+
+    const lines = [];
+    lines.push("| " + escapedHeaders.map((h, i) => pad(h, colWidths[i])).join(" | ") + " |");
+    lines.push("| " + colWidths.map((w) => "-".repeat(w)).join(" | ") + " |");
+
+    for (const row of escapedRows) {
+      lines.push("| " + row.map((cell, i) => pad(cell, colWidths[i])).join(" | ") + " |");
+    }
+
+    return lines.join("\n");
+  }
+
+  private extractText(content: any): string {
+    if (!content) return "";
+    if (typeof content === "string") return content;
+    if (typeof content === "object") {
+      if ("#text" in content) return content["#text"];
+      return JSON.stringify(content);
+    }
+    return String(content);
+  }
+
+  private extractReturnType(definition: string): string {
+    const parts = definition.trim().split(/\s+/);
+    if (parts.length < 2) return "";
+    return parts.slice(0, -1).join(" ");
+  }
+
+  private writeFunction(func: FunctionDoc) {
+    let stream = "";
+    stream += "### " + func.name + "\n\n";
+    if (func.definition && func.argsstring) {
+      stream += "```c\n" + func.definition + func.argsstring + "\n```\n";
+    }
+    if (func.detailedDescription) {
+      if (func.detailedDescription.description) {
+        // stream += "#### 📄 Description";
+        stream += "\n\n" + func.detailedDescription.description + "\n\n";
+      }
+      if (func.detailedDescription.params.length > 0) {
+        stream += "##### Parameters\n\n";
+        const params = func.detailedDescription.params.map((param) => [
+          param.paramName,
+          this.extractText(param.paramDescription)
+        ]);
+        stream += this.generateMarkdownTable(["Parameter", "Description"], params) + "\n\n";
+      }
+
+      stream += "##### ⏎ Return Value\n\n";
+      if (func.detailedDescription.returns) {
+        stream += "`" + this.extractReturnType(func.definition) + "`: " + func.detailedDescription.returns + "\n\n";
+      }
+    }
+    return stream;
+  }
+
+  private writeTypedef(typedef: TypeDefDoc) {
+    let stream = "";
+    stream += "### " + typedef.name + "\n\n";
+    if (typedef.definition) {
+      stream += "```c\n" + typedef.definition + "\n```\n";
+    }
+    if (typedef.detailedDescription) {
+      // stream += "#### 📄 Description";
+      stream += "\n\n" + typedef.detailedDescription + "\n\n";
+    }
+    return stream;
+  }
+
+  private beginBlock() {
+    return `---\n`;
+  }
+
+  private endBlock() {
+    return `---\n\n`;
+  }
+
+  private writeFrontMatter(title: string, description: string) {
+    return `${this.beginBlock()}title: ${title}\ndescription: ${description || title}\n${this.endBlock()}`;
+  }
+}

src/scripts/parser/apidoc/common.ts

@@ -0,0 +1,9 @@
+export class GenericApiDocParser {
+  protected parse() {}
+  protected toMD() {}
+}
+
+export function ensureArray<T>(input: T | T[] | undefined): T[] {
+  if (!input) return [];
+  return Array.isArray(input) ? input : [input];
+}

src/scripts/parser/apidoc/python-apidoc-parser.ts

@@ -0,0 +1,445 @@
+import { readFileSync, existsSync, writeFileSync, write } from "fs";
+
+import { fileURLToPath } from "url";
+import path, { dirname } from "path";
+
+import { XMLParser } from "fast-xml-parser";
+
+import { ensureArray } from "./common";
+import { json } from "stream/consumers";
+
+export interface FunctionParam {
+  type: string;
+  name: string;
+  description: string;
+}
+
+export interface PythonApiDocFunction {
+  name: string;
+  definition: string;
+  description: string;
+  params: FunctionParam[];
+  example: string;
+  returns: {
+    type: string;
+    description: string;
+  };
+}
+
+export interface PythonApiDocModule {
+  name: string;
+  description: string;
+  functions: PythonApiDocFunction[];
+}
+
+export interface PythonApiDocFile {
+  files: PythonApiDocModule[];
+}
+
+export interface MDOutput {
+  filename: string;
+  buffer: string;
+}
+
+export class PythonApiDocParser {
+  private xmlParser: XMLParser;
+
+  private buffer: string;
+  private docPath: string;
+  private repoPath: string;
+  private fileCache: Record<string, any> = {};
+  private apiDoc: PythonApiDocFile;
+  private outputs: MDOutput[] = [];
+
+  constructor(docEntry: { buffer: string; docPath: string; repoPath: string }) {
+    this.buffer = docEntry.buffer;
+    this.docPath = docEntry.docPath;
+    this.repoPath = docEntry.repoPath;
+    this.xmlParser = new XMLParser({ ignoreAttributes: false });
+    this.apiDoc = { files: [] };
+  }
+
+  private preloadAllXmlFiles(fileList: string[]) {
+    for (const file of fileList) {
+      this.fileCache[file] = readFileSync(path.join(this.docPath, `${file}.xml`), "utf-8");
+    }
+  }
+
+  parse() {
+    const buffer = this.buffer;
+    const parser = this.xmlParser;
+    const obj = parser.parse(buffer);
+
+    if (!obj?.document || !obj?.document?.section) return;
+
+    const document = obj?.document;
+    if (!document) return;
+
+    const section = document?.section;
+    if (!section) return;
+
+    const compound = section?.compound;
+    if (!compound) return;
+
+    const compactParagraph = compound?.["compact_paragraph"];
+    if (!compactParagraph) return;
+
+    const bulletList = compactParagraph?.["bullet_list"];
+    if (!bulletList) return;
+
+    const listItems = ensureArray(bulletList?.["list_item"]);
+    if (!listItems) return;
+
+    const importPaths = listItems.map((li) => li?.["compact_paragraph"]?.["reference"]?.["@_refuri"]).filter(Boolean);
+
+    this.preloadAllXmlFiles(importPaths);
+
+    const moduleFiles = listItems.map((li) => li?.["compact_paragraph"]?.["reference"]?.["@_refuri"]).filter(Boolean);
+    const modules: PythonApiDocModule[] = [];
+    for (const module of moduleFiles) {
+      this.apiDoc.files.push(this.parseModuleFile(module));
+    }
+  }
+
+  private parseModuleFile(moduleFile: any): PythonApiDocModule {
+    const module = this.fileCache[moduleFile];
+    const parser = new XMLParser({ ignoreAttributes: false, preserveOrder: true });
+    const parsedModule = parser.parse(module);
+    const document = parsedModule?.find((d: any) => d.hasOwnProperty("document"))?.document;
+    if (!document) {
+      console.warn(`Could not find document for ${moduleFile}`);
+      process.exit(1);
+    }
+    const section = document?.find((s: any) => s.hasOwnProperty("section"))?.section;
+    if (!section) {
+      console.warn(`Could not find section for ${moduleFile}`);
+      process.exit(1);
+    }
+    const paragraphs = section?.filter(
+      (ele: any) => !ele.hasOwnProperty("title") && !ele.hasOwnProperty("index") && !ele.hasOwnProperty("desc")
+    );
+    let moduleDescription = "";
+    for (const node of paragraphs) {
+      if (node.hasOwnProperty("paragraph")) {
+        moduleDescription += this.extractTextFromNode(node.paragraph).trim() + "\n\n";
+      } else if (node.hasOwnProperty("literal_block")) {
+        moduleDescription += '```python\n' + this.extractTextFromNode(node.literal_block) + "\n```\n\n";
+      }
+    }
+    const functionDefinitions = section?.filter((ele: any) => ele.hasOwnProperty("desc"));
+    const funcs: PythonApiDocFunction[] = [];
+    for (const functionDescription of functionDefinitions) {
+      funcs.push(this.parseFunction(functionDescription?.desc));
+    }
+    return {
+      name: moduleFile,
+      description: moduleDescription,
+      functions: funcs
+    };
+  }
+
+  private extractTextFromNode(node: any): string {
+    if (typeof node === "string") {
+      return node;
+    }
+
+    if (Array.isArray(node)) {
+      let result = "";
+      for (let i = 0; i < node.length; i++) {
+        const currentText = this.extractTextFromNode(node[i]);
+
+        if (i > 0) {
+          const prevChar = result.slice(-1);
+          const nextChar = currentText.charAt(0);
+          if (
+            prevChar &&
+            nextChar &&
+            prevChar !== " " &&
+            nextChar !== " " &&
+            !["\n", "(", "["].includes(prevChar) &&
+            ![".", ",", ":", ")", "]"].includes(nextChar)
+          ) {
+            result += " ";
+          }
+        }
+        result += currentText;
+      }
+      return result;
+    }
+
+    if (typeof node === "object" && node !== null) {
+      if ("#text" in node) {
+        return node["#text"];
+      }
+      let result = "";
+      for (const key in node) {
+        result += this.extractTextFromNode(node[key]);
+      }
+      return result;
+    }
+
+    return "";
+  }
+
+  private methodName(moduleName: string, method: string) {
+    if (moduleName === "plugin") return method;
+    else return `${moduleName}.${method}`;
+  }
+
+  private parseFunction(functionDefinition: any): PythonApiDocFunction {
+    const desc = ensureArray(functionDefinition).find((ele: any) => ele.hasOwnProperty("desc_signature"));
+    const signature = desc?.["desc_signature"];
+
+    const name = desc?.[":@"]?.["@_fullname"] || "";
+    const moduleName = desc?.[":@"]?.["@_module"] || "";
+
+    const paramList = signature?.find((ele: any) => ele.hasOwnProperty("desc_parameterlist"))?.["desc_parameterlist"];
+    const parsedParams: string[] = paramList?.map((param: any) => {
+      const descParam = param?.["desc_parameter"]?.[0];
+      const sigName = descParam?.["desc_sig_name"]?.find((ele: any) => ele?.["#text"]);
+      return sigName?.["#text"] || "";
+    });
+
+    const content = ensureArray(functionDefinition).find((ele: any) =>
+      ele.hasOwnProperty("desc_content")
+    )?.desc_content;
+    if (!content) {
+      let method = this.methodName(moduleName, name);
+      return {
+        name: `${method}`,
+        definition: `${method}(${parsedParams.join(", ")})`,
+        description: "",
+        params: [],
+        example: "",
+        returns: {
+          type: "None",
+          description: ""
+        }
+      };
+    }
+
+    let description = "";
+    let example = "";
+    let currentParagraph = "";
+    let isExampleMode = false;
+    let returnType = "None";
+    let returnDescription = "";
+    let functionParams: FunctionParam[] = [];
+
+    for (const node of content) {
+      const key = Object.keys(node)[0];
+      const value = node[key];
+
+      if (key === "paragraph") {
+        const textContent = this.extractTextFromNode(value).trim();
+
+        if (textContent === "Example:") {
+          if (currentParagraph.trim()) {
+            description += currentParagraph.trim() + "\n\n";
+            currentParagraph = "";
+          }
+          isExampleMode = true;
+          continue;
+        }
+
+        if (isExampleMode) {
+          example += textContent + "\n\n";
+        } else {
+          example === "" && description.length > 0 && (description += "\n");
+          description += textContent + "\n\n";
+        }
+      } else if (key === "literal_block") {
+        const blockText = this.extractTextFromNode(value);
+        if (isExampleMode) {
+          example += blockText + "\n\n";
+        } else {
+          description += blockText + "\n\n";
+        }
+      } else if (key === "#text" || key === "literal") {
+        const part = this.extractTextFromNode(value);
+        if (isExampleMode) {
+          example += part;
+        } else {
+          description += part;
+        }
+      }
+    }
+    if (currentParagraph.trim()) {
+      if (isExampleMode) {
+        example += currentParagraph.trim() + "\n\n";
+      } else {
+        description += currentParagraph.trim() + "\n\n";
+      }
+    }
+
+    const fieldList = content?.find((ele: any) => ele.hasOwnProperty("field_list"))?.["field_list"];
+
+    if (fieldList) {
+      const fields = fieldList?.map((field: any) => field?.field);
+      for (const field of fields) {
+        const fieldName = field?.find((ele: any) => ele.hasOwnProperty("field_name"))?.field_name;
+        const fieldBody = field?.find((ele: any) => ele.hasOwnProperty("field_body"))?.field_body;
+
+        const nameText =
+          typeof fieldName === "string"
+            ? fieldName
+            : fieldName?.["#text"] || fieldName?.find?.((e: any) => e?.["#text"])?.["#text"];
+
+        if (nameText === "Parameters") {
+          functionParams = this.parseFunctionParameters(fieldBody);
+        } else if (nameText === "Returns") {
+          returnDescription = this.parseFunctionReturnDescription(fieldBody);
+        } else if (nameText === "Return type") {
+          returnType = this.parseFunctionReturnType(fieldBody);
+        }
+      }
+    }
+
+    let method = this.methodName(moduleName, name);
+    return {
+      name: `${method}`,
+      definition: `${method}(${parsedParams.join(", ")})`,
+      description: description.trim(),
+      params: functionParams,
+      example: example.trim(),
+      returns: {
+        type: returnType,
+        description: returnDescription
+      }
+    };
+  }
+
+  private parseFunctionParameters(fieldBody: any): FunctionParam[] {
+    const bulletList = fieldBody?.find((ele: any) => ele.hasOwnProperty("bullet_list"))?.["bullet_list"];
+    const listItems = bulletList?.filter((ele: any) => ele.hasOwnProperty("list_item"));
+
+    const params: FunctionParam[] = [];
+    if (listItems) {
+      for (const listItem of listItems) {
+        const item = listItem?.["list_item"];
+        const paragraph = item?.find((ele: any) => ele.hasOwnProperty("paragraph"))?.["paragraph"];
+        let description = "";
+        let paramName = "";
+        let paramType: string[] = [];
+        let paramTypeParts: string[] = [];
+        for (const paragraphLine of paragraph) {
+          if (paragraphLine.hasOwnProperty("#text")) {
+            description += paragraphLine["#text"];
+          } else if (paragraphLine.hasOwnProperty("literal_strong")) {
+            paramName = paragraphLine?.["literal_strong"]?.find((ele: any) => ele.hasOwnProperty("#text"))?.["#text"];
+          } else if (paragraphLine.hasOwnProperty("literal_emphasis")) {
+            paramType.push(
+              paragraphLine?.["literal_emphasis"]?.find((ele: any) => ele.hasOwnProperty("#text"))?.["#text"]
+            );
+          }
+        }
+        description = description.replace(/^[()\s–-]+|[()\s–-]+$/g, "");
+        params.push({ name: paramName, type: paramType?.join(" "), description });
+      }
+    }
+    return params;
+  }
+
+  private parseFunctionReturnDescription(fieldBody: any) {
+    const paragraph = fieldBody?.find((ele: any) => ele.hasOwnProperty("paragraph"))?.["paragraph"];
+    return this.extractTextFromNode(paragraph);
+  }
+
+  private parseFunctionReturnType(fieldBody: any) {
+    const paragraph = fieldBody
+      ?.find((ele: any) => ele.hasOwnProperty("paragraph"))
+      ?.["paragraph"]?.find((ele: any) => ele.hasOwnProperty("#text"))?.["#text"];
+    return paragraph;
+  }
+
+  toMD() {
+    for (const module of this.apiDoc.files) {
+      this.outputs.push({ filename: module.name + ".md", buffer: this.writeModule(module) });
+    }
+    return this.outputs;
+  }
+
+  private writeModule(module: PythonApiDocModule) {
+    let stream = "";
+    stream += this.writeFrontMatter(module.name, `${module.name} Module Reference`);
+    stream += `${module.description}\n\n`;
+    if (module.functions.length > 0) {
+      stream += "# Functions\n\n";
+      for (const func of module.functions) {
+        stream += this.writeFunction(func);
+      }
+    }
+    return stream;
+  }
+
+  private writeFunction(func: PythonApiDocFunction) {
+    let stream = "";
+    stream += "## " + func.name + "\n\n";
+    if (func.definition) {
+      let definition = func.definition.includes('->') ? func.definition : func.definition + " -> " + func.returns.type;
+      stream += "```python\n" + definition + "\n```\n";
+    }
+    if (func.description) {
+      // Self-explanatory. Best to omit.
+      // stream += "#### 📄 Description";
+      stream += "\n\n" + func.description + "\n\n";
+    }
+    if (func.params.length > 0) {
+      stream += "##### Parameters\n\n";
+      const rows = func.params.map((param) => [`\`${param.name}\``, param.type, param.description]);
+      stream += this.generateMarkdownTable(["Parameter Name", "Type", "Description"], rows) + "\n\n";
+    }
+    if (func.example) {
+      stream += "##### Example\n\n";
+      stream += "```python\n" + func.example + "\n```\n";
+    }
+    if (func.returns && func.returns.type != "None") {
+      stream += "##### Return value\n\n";
+      stream +=
+        "`" + func.returns.type + "`" + `${func.returns.description ? `: ${func.returns.description}` : ""}` + "\n\n";
+    }
+    stream += "\n\n\n";
+    return stream;
+  }
+
+  private escapeMarkdown(text: string): string {
+    return text
+      .replace(/&/g, "&amp;")
+      .replace(/</g, "&lt;")
+      .replace(/>/g, "&gt;")
+      .replace(/\|/g, "&#124;")
+      .replace(/\[/g, "&#91;")
+      .replace(/\]/g, "&#93;");
+  }
+
+  private generateMarkdownTable(headers: string[], rows: string[][]): string {
+    const escapedRows = rows.map((row) => row.map((cell) => this.escapeMarkdown(cell)));
+    const escapedHeaders = headers.map((header) => this.escapeMarkdown(header));
+    const colWidths = headers.map((_, i) =>
+      Math.max(escapedHeaders[i].length, ...escapedRows.map((row) => (row[i] || "").length))
+    );
+    const pad = (text: string, width: number) => text + " ".repeat(width - text.length);
+
+    const lines = [];
+    lines.push("| " + escapedHeaders.map((h, i) => pad(h, colWidths[i])).join(" | ") + " |");
+    lines.push("| " + colWidths.map((w) => "-".repeat(w)).join(" | ") + " |");
+
+    for (const row of escapedRows) {
+      lines.push("| " + row.map((cell, i) => pad(cell, colWidths[i])).join(" | ") + " |");
+    }
+
+    return lines.join("\n");
+  }
+
+  private beginBlock() {
+    return `---\n`;
+  }
+
+  private endBlock() {
+    return `---\n\n`;
+  }
+
+  private writeFrontMatter(title: string, description: string) {
+    return `${this.beginBlock()}title: ${title}\ndescription: ${description || title}\n${this.endBlock()}`;
+  }
+}

src/scripts/parser/command-parser.ts

@@ -0,0 +1,280 @@
+import type { MDOutput } from "./common";
+
+export interface CommandArg {
+  arg: {
+    name: string;
+    values: string[];
+    description: string;
+  };
+}
+
+export interface CommandGroup {
+  preamble: string; // the base command keyword
+  mainFunc: string; // the function that handles this command
+  subFuncs: string[]; // the subfunction that handles this command
+  syn: string[]; // different valid syntax patterns for the command
+  desc: string; // command description
+  args: CommandArg[]; // detailed argument descriptions for each syntax
+  examples: string[]; // usage examples for the command
+  tags: string[]; // command category tag(s)
+}
+
+const githubFunctionUrl = "https://git.jabber.space/devs/profanity/src/branch/master/src/command/cmd_defs.c";
+
+export class CommandParser {
+  private buffer: string;
+  private readonly preamblePattern: RegExp =
+    /CMD_PREAMBLE\(\s*"([^"]+)"\s*,\s*([\w_]+)\s*,\s*(\d+)\s*,\s*(\d+)\s*,\s*([^)]+)\)/ms;
+  private readonly mainFuncPattern: RegExp = /CMD_MAINFUNC\((\w+)\)/;
+  private readonly subFuncPattern: RegExp = /CMD_SUBFUNC\((\w+)\)/;
+  private readonly tagsPattern: RegExp = /CMD_TAGS\(([^)]+)\)/;
+  private readonly synPattern: RegExp = /CMD_SYN\(\s*([\s\S]*?[^\\]\")\s*\)/m;
+  private readonly descPattern: RegExp = /CMD_DESC\(([\s\S]*?[^\\]\")\s*\)/m;
+  private readonly argsPattern: RegExp = /CMD_ARGS\(([\s\S]*?)\)/m;
+  private readonly examplesPattern: RegExp = /CMD_EXAMPLES\(([\s\S]*?)\)/m;
+  private commands: CommandGroup[] = [];
+
+  constructor(buffer: string) {
+    this.buffer = buffer;
+    this.parse();
+  }
+
+  private parse() {
+    const buffer = this.buffer;
+    const commandDefsMatch = buffer.match(/command_defs\[\]\s*=\s*\{([\s\S]*)\};/);
+
+    if (!commandDefsMatch) {
+      console.error("Could not find command_defs[]");
+      process.exit(1);
+    }
+
+    const commandDefsContent = commandDefsMatch[1];
+    const commandBlocks = this.extractBlocks(commandDefsContent);
+    for (const block of commandBlocks) {
+      const group = this.parseBlock(block);
+      this.commands.push(group);
+    }
+  }
+
+  private extractBlocks(content: string) {
+    const blocks: string[] = [];
+    let pos = 0;
+    const len = content.length;
+    while (pos < len) {
+      const start = content.indexOf("{", pos);
+      if (start === -1) break;
+      let braceCount = 1;
+      let end = start + 1;
+      while (end < len && braceCount > 0) {
+        if (content[end] === "{") {
+          braceCount++;
+        } else if (content[end] === "}") {
+          braceCount--;
+        }
+        end++;
+      }
+      if (braceCount === 0) {
+        const block = content.slice(start, end);
+        blocks.push(block.trim());
+        pos = end;
+      } else {
+        break;
+      }
+    }
+    return blocks;
+  }
+
+  private extractMultilineStrings(input: string): string[] {
+    const regex = /"((?:[^"\\]|\\.)*)"/g;
+    const matches = [];
+    let match;
+    while ((match = regex.exec(input)) !== null) {
+      matches.push(match[1].replace(/\\"/g, '"'));
+    }
+    return matches;
+  }
+
+  private parseArgNameAndValues(argStr: string): {
+    name: string;
+    values: string[];
+  } {
+    return {
+      name: argStr,
+      values: []
+    };
+  }
+
+  private extractArgs = (input: string): CommandArg[] => {
+    const matches = [...input.matchAll(/\{\s*"([^"]+?)",\s*"([^"]+?)"\s*\}/gs)];
+    return matches.map((m) => {
+      const rawArg = m[1];
+      const description = m[2];
+      const { name, values } = this.parseArgNameAndValues(rawArg);
+      return {
+        arg: {
+          name,
+          values,
+          description
+        }
+      };
+    });
+  };
+  private parseBlock(block: string): CommandGroup {
+    const preambleMatch = block.match(this.preamblePattern);
+    const mainFuncMatch = block.match(this.mainFuncPattern);
+    const subFuncMatch = block.match(this.subFuncPattern);
+    const tagsMatch = block.match(this.tagsPattern);
+    const synMatch = block.match(this.synPattern);
+    const descMatch = block.match(this.descPattern);
+    const argsMatch = block.match(this.argsPattern);
+    const examplesMatch = block.match(this.examplesPattern);
+
+    return {
+      preamble: preambleMatch?.[1] || "",
+      mainFunc: mainFuncMatch?.[1] || "",
+      subFuncs: subFuncMatch?.[1].split(",") || [],
+      syn: synMatch ? this.extractMultilineStrings(synMatch[1]) : [],
+
+      desc: descMatch ? this.extractMultilineStrings(descMatch[1]).join(" ") : "",
+      args: argsMatch ? this.extractArgs(argsMatch[1]) : [],
+      examples: examplesMatch ? this.extractMultilineStrings(examplesMatch[1]) : [],
+      tags: tagsMatch ? [...tagsMatch[1].matchAll(/CMD_TAG_([A-Z_]+)/g)].map((m) => m[1]?.toLowerCase()) : []
+    };
+  }
+
+  private capitalizeFirstLetter(str: string) {
+    if (!str) return "";
+    if (str.charAt(0) === "/") {
+      if (str.length > 1) {
+        return str.charAt(0) + str.charAt(1).toUpperCase() + str.slice(2);
+      } else {
+        return str;
+      }
+    } else {
+      return str.charAt(0).toUpperCase() + str.slice(1);
+    }
+  }
+
+  toMD() {
+    let stream = "";
+    const output: MDOutput = { filename: "command-reference.md", buffer: "" };
+    stream += this.writeFrontMatter("Command Reference", "Command Reference");
+
+    // Sort commands alphabetically by command name
+    this.commands = this.commands.sort((a, b) => { if (a.preamble < b.preamble) return -1; if (a.preamble > b.preamble) return 1; return 0; });
+
+    for (const group of this.commands) {
+      if (group.preamble) {
+        stream += this.writeCommandHeader(group.preamble);
+      } else {
+        console.error("No preamble for group", group);
+        process.exit(1);
+      }
+      if (group.desc) {
+        // No need for title, it's obvious that it's a description (right under the command's title)
+        // stream += this.writeSection("📄 Description", true);
+        stream += group.desc;
+        stream += "\n\n";
+      }
+      if (group.syn) {
+        stream += this.writeSynopsisBlock(group.syn);
+      }
+      if (group.args.length > 0) {
+        stream += this.writeArgsBlock(group.args);
+      }
+      if (group.examples.length > 0) {
+        stream += this.writeExamplesBlock(group.examples);
+      }
+      stream += "\n\n";
+    }
+
+    output.buffer = stream;
+    return output;
+  }
+
+  private beginBlock() {
+    return `---\n`;
+  }
+
+  private endBlock() {
+    return `---\n\n`;
+  }
+
+  private writeSection(sectionName: string, isChild = false) {
+    return `${isChild ? "####" : "##"} ${!isChild ? "`" : ""}${sectionName}${!isChild ? "`" : ""}\n\n`;
+  }
+
+  private writeGenericBlock(blockName: string, content: string) {
+    return `**${blockName}** ${content}<br/>\n`;
+  }
+
+  private writeSynopsisBlock(syntaxList: string[]) {
+    let stream = "";
+    // No need for Synopsis section, it's self-explanatory
+    // stream += this.writeSection("🔧 Synopsis", true);
+    stream += "```bash\n";
+    stream += syntaxList.join("\n");
+    stream += "\n";
+    stream += "```\n\n";
+    return stream;
+  }
+
+  private writeExamplesBlock(exampleList: string[]) {
+    let stream = "";
+    stream += this.writeSection("💡 Examples", true);
+    stream += "```bash\n";
+    stream += exampleList.join("\n");
+    stream += "\n";
+    stream += "```";
+    stream += "\n\n";
+    return stream;
+  }
+
+  private escapeMarkdown(text: string): string {
+    return text
+      .replace(/&/g, "&amp;")
+      .replace(/</g, "&lt;")
+      .replace(/>/g, "&gt;")
+      .replace(/\|/g, "&#124;")
+      .replace(/\[/g, "&#91;")
+      .replace(/\]/g, "&#93;");
+  }
+
+  private generateMarkdownTable(headers: string[], rows: string[][], escapeMarkdownForRows=true): string {
+    const escapedRows = rows.map((row) => row.map((cell) => escapeMarkdownForRows ? this.escapeMarkdown(cell) : cell));
+    const escapedHeaders = headers.map((header) => this.escapeMarkdown(header));
+    const colWidths = headers.map((_, i) =>
+      Math.max(escapedHeaders[i].length, ...escapedRows.map((row) => (row[i] || "").length))
+    );
+    const pad = (text: string, width: number) => text + " ".repeat(width - text.length);
+
+    const lines = [];
+    lines.push("| " + escapedHeaders.map((h, i) => pad(h, colWidths[i])).join(" | ") + " |");
+    lines.push("| " + colWidths.map((w) => "-".repeat(w)).join(" | ") + " |");
+
+    for (const row of escapedRows) {
+      lines.push("| " + row.map((cell, i) => pad(cell, colWidths[i])).join(" | ") + " |");
+    }
+
+    return lines.join("\n");
+  }
+
+  private writeArgsBlock(args: CommandArg[]) {
+    let stream = "";
+    stream += this.writeSection("📝 Arguments", true);
+    const headers = ["Argument", "Description"];
+    const rows = args.map((arg) => [`\`${arg.arg.name}\``, this.escapeMarkdown(arg.arg.description)]);
+    stream += this.generateMarkdownTable(headers, rows, false);
+    stream += "\n\n";
+    return stream;
+  }
+
+  private writeFrontMatter(title: string, description: string) {
+    return `${this.beginBlock()}title: ${title}\ndescription: ${description || title
+      }\ntableOfContents:\n  minHeadingLevel: 1\n  maxHeadingLevel: 2\n${this.endBlock()}`;
+  }
+
+  private writeCommandHeader(title: string) {
+    return this.writeSection(title);
+  }
+}

src/scripts/parser/common.ts

@@ -0,0 +1,4 @@
+export interface MDOutput {
+  filename: string;
+  buffer: string;
+}