function runConversion()

in gui/tools/src/extractDocumentation.ts [595:1126]


function runConversion() {
    fs.removeSync("./conversions");
    fs.mkdirSync("./conversions");

    // We cannot load and parse all used entities in the documentation, but at least the product names
    // and some others can be used.
    const content = fs.readFileSync("../../../../documentation/mvl-shared/en/entities/mysql/product-names.ent", {
        encoding: "utf-8"
    });

    // The entities file is also an xml file, but the xml-json conversion leaves the <!ENTITY> entries out, which
    // we need here to replace values in the main xml.
    const lines = content.split("\n");
    const entities: Map<string, string> = new Map([
        ["current-series", "8.0"],
        ["previous-series", "5.7"],
        ["current-version", "8.0.22"],
        ["base-url-downloads", "/downloads"],
        ["base-url-internals", "/base"],
    ]);
    const entityRegEx = /<!ENTITY\s+(\w+)\s+(".+"|'.+')>/;
    lines.forEach((line: string) => {
        const match = line.match(entityRegEx);
        if (match) {
            entities.set(match[1], unquote(match[2]));
        }
    });

    // 1) System variables - full documentation
    let text = fs.readFileSync("../../../../documentation/refman-8.0/dba-mysqld-server-core.xml", {
        encoding: "utf-8"
    });
    let result: any = processFile(text, entities);

    // 1.1) Server system variables. Extract specific sub objects for isolated lists.
    let variableSection = findSection(result, "server-system-variables");
    if (variableSection) {
        result = sanitizeObject(variableSection);
        writeObject(result, "/Users/Mike/Downloads/server-system-variables.json");
    } else {
        throw new Error("Couldn't find section 'server-system-variables' in this file.");
    }

    // Find the item list with the variable names, by looking for an itemized list containing children having
    // a sysvar role in the first subpart (which is then also used to get the name of the variable).
    let foundVariables = false;
    for (const child of result.children) {
        if (child.name === "itemizedlist") {
            const content = child.children;
            if (content.length > 0 && content[0].length > 0) {
                const firstEntry = content[0][0]; // Take the first sub entry of the first child.
                if (firstEntry.para?.length > 0 && firstEntry.para[0].role === "sysvar") {
                    foundVariables = true;
                    const variables: any = {};
                    for (const varChild of content) {
                        // The first entry is used to get the variable name.
                        // The rest is used to collect a full description.
                        const description: string[] = [];
                        for (let i = 1; i < varChild.length; ++i) {
                            const entry = convertToMarkdown(varChild[i]);
                            if (Array.isArray(entry)) {
                                description.push(...entry);
                            }
                            else {
                                description.push(entry);
                            }
                        }

                        if (Array.isArray(varChild[0].para)) {
                            variables[unquote(varChild[0].para[0].text)] = description;
                        }
                    }
                    writeObject(variables, "./conversions/server-system-variables.json");
                }
            }
        }
    }

    if (!foundVariables) {
        console.warn("Could not find variables list in dba-mysqld-server-core.xml.");
    }

    // 1.2) InnoDB system variables - full documentation
    text = fs.readFileSync("../../../../documentation/refman-8.0/se-innodb-core.xml", {
        encoding: "utf-8"
    });
    result = processFile(text, entities);

    variableSection = findSection(result, "innodb-parameters");
    if (variableSection) {
        result = sanitizeObject(variableSection);
        writeObject(result, "/Users/Mike/Downloads/innodb-variables.json");
    } else {
        throw new Error("Couldn't find section 'innodb-parameters' in se-innodb-core.xml.");
    }

    foundVariables = false;
    for (let i = 0; i < result.children.length - 1; ++i) {
        let child = result.children[i];
        if (child.bridgehead === "InnoDB System Variables" && i < result.children.length - 1) {
            child = result.children[i + 1]; // The actual list is the next sibling of the title child.
            const content = child.children;

            // In opposition to the server variables xmls there's no "sysvar" role set for variable name entries.
            // So, we assume the title is good enough to locate the list.
            foundVariables = true;
            const variables: any = {};
            for (const varChild of content) {
                // The first entry is used to get the variable name.
                // The rest is used to collect a full description.
                const description: string[] = [];
                for (let i = 1; i < varChild.length; ++i) {
                    const entry = convertToMarkdown(varChild[i]);
                    if (Array.isArray(entry)) {
                        description.push(...entry);
                    }
                    else {
                        description.push(entry);
                    }
                }

                if (Array.isArray(varChild[0].para)) {
                    variables[unquote(varChild[0].para[0].text)] = description;
                } else {
                    variables[unquote(varChild[0].para)] = description;
                }
            }
            writeObject(variables, "./conversions/innodb-system-variables.json");
        }
    }

    if (!foundVariables) {
        console.warn("Could not find variables list in se-innodb-core.xml.");
    }

    // 3) The short variables description file.
    text = fs.readFileSync("../../../../documentation/dynamic-docs/optvar/mysqld.xml", {
        encoding: "utf-8"
    });
    result = processFile(text, entities);

    const options = result.children[0];
    if (!options || options.name !== "mysqloptions") {
        throw new Error("Couldn't find the 'mysqloptions' list in mysqld.xml.");
    }

    result = sanitizeObject(options);
    writeObject(result, "/Users/Mike/Downloads/mysqld-options.json");

    const variables: any = {};
    for (const entry of result.children) {
        if (entry.name === "mysqloption") {
            let xrefto = "";
            let name = "";
            let description = "";
            let footnote = "";

            let introduced = "";
            let deprecated = "";
            let removed = "";

            let system: IVariableInfo[] = [];
            let values: IValueInfo[] = [];

            for (const child of entry.children) {
                if (child.description) {
                    description = child.description;
                } else if (child.footnote) {
                    footnote = child.footnote;
                } else if (child.name) {
                    switch (child.name) {
                        case "xrefto": {
                            xrefto = child.attributes.id;
                            break;
                        }

                        case "seealso": {
                            // An additional external reference, which we don't use.

                            break;
                        }

                        case "description": {
                            // Appears if additional content exists for the description
                            // and hence it has not been condensed into a single "description" field.
                            description = child.children;
                            break;
                        }

                        case "types": {
                            for (const subChild of child.children) {
                                if (subChild.name === "system") {
                                    system.push({
                                        isDynamic: subChild.attributes.isdynamic === "yes",
                                        hintable: subChild.attributes.hintable === "yes",
                                        scope: subChild.attributes.scope,
                                        minVersion: convertVersion(subChild.attributes.inversion),
                                        maxVersion: convertVersion(subChild.attributes.outversion),
                                    });
                                }
                            }

                            break;
                        }

                        case "values": {
                            const type = child.attributes.vartype;
                            let defaultValue = "";
                            let minValue = "";
                            let maxValue = "";
                            const valueList: string[] = [];

                            if (child.children) {
                                for (const subChild of child.children) {
                                    if (subChild.name === "value") {
                                        if (subChild.attributes.default) {
                                            defaultValue = subChild.attributes.default;
                                        } else if (subChild.attributes.minimum) {
                                            minValue = subChild.attributes.minimum;
                                        } else if (subChild.attributes.maximum) {
                                            maxValue = subChild.attributes.maximum;
                                        }
                                    } else if (subChild.name === "choice") {
                                        valueList.push("- `" + subChild.attributes.value + "`");
                                    }
                                }
                            }

                            values.push({
                                type,
                                defaultValue,
                                minValue,
                                maxValue,
                                valueList,

                                minVersion: convertVersion(child.attributes.inversion),
                                maxVersion: convertVersion(child.attributes.outversion),
                            });

                            break;
                        }

                        case "versions": {
                            for (const subChild of child.children) {
                                if (subChild.name === "introduced") {
                                    introduced = subChild.attributes.version;
                                }

                                if (subChild.name === "deprecated") {
                                    deprecated = subChild.attributes.version;
                                }

                                if (subChild.name === "removed") {
                                    removed = subChild.attributes.version;
                                }
                            }

                            break;
                        }

                        default: {
                            name = child.name;
                            break;
                        }
                    }
                }
            }

            if (name !== "") {
                // There can be multiple values sets per variable (each with an own version range).
                // We use the highest version here.
                let value: IValueInfo = {
                    type: "",
                    defaultValue: "",
                    minValue: "",
                    maxValue: "",
                    valueList: [],

                    minVersion: 0,
                    maxVersion: 0,
                };
                if (values.length > 0) {
                    if (values.length > 1) {
                        values = values.sort((a: IValueInfo, b: IValueInfo) => a.minVersion - b.minVersion);
                    }
                    value = values[values.length - 1];
                }


                let part1 = "```mysql\ndeclare @@" + name + " " + value.type + "\n```";
                let parts = "";

                let part2 = "_**System Variable**_";
                parts = "";
                if (introduced !== "") {
                    parts += "⊕ " + introduced;
                }

                if (deprecated !== "") {
                    if (parts !== "") {
                        parts += ", ";
                    }
                    parts += "⊘ " + deprecated;
                }

                if (removed !== "") {
                    if (parts !== "") {
                        parts += ", ";
                    }
                    parts += "⊗ " + removed;
                }

                if (parts !== "") {
                    part2 += " (" + parts + ")";
                }

                part2 += "\n\n";

                parts = "";
                if (value.defaultValue !== "") {
                    parts += "default: `" + value.defaultValue + "`";
                }

                if (value.minValue !== "") {
                    if (parts !== "") {
                        parts += ", ";
                    }
                    parts += "min: `" + value.minValue + "`";
                }

                if (value.maxValue !== "") {
                    if (parts !== "") {
                        parts += ", ";
                    }
                    parts += "max: `" + value.maxValue + "`";
                }

                if (value.valueList.length > 0) {
                    if (parts !== "") {
                        parts += ", ";
                    }
                    parts += "valid values:\n\n" + value.valueList.join("\n");
                }

                if (parts !== "") {
                    part2 += parts + "\n\n";
                }

                let systemInfo = { isDynamic: false, hintable: false, scope: "", minVersion: 0, maxVersion: 0 };
                if (system.length > 0) {
                    if (system.length > 1) {
                        system = system.sort((a: IVariableInfo, b: IVariableInfo) => a.minVersion - b.minVersion);
                    }
                    systemInfo = system[system.length - 1];
                }

                if (systemInfo.scope === "both") {
                    systemInfo.scope = "global, session";
                }

                part2 += "dynamic: " + (systemInfo.isDynamic ? "yes" : "no");
                part2 += ", scope: " + systemInfo.scope;
                part2 += ", [SET_VAR hint](https://dev.mysql.com/doc/refman/8.0/en/optimizer-hints.html#" +
                    "optimizer-hints-set-var) " + (systemInfo.hintable ? "supported" : "not supported");

                let part3 = description + " [[online documentation](https://dev.mysql.com/" +
                    "doc/refman/8.0/en/server-system-variables.html#" + xrefto + ")].";
                if (footnote !== "") {
                    part3 += "\n\n> " + footnote;
                }

                variables[name] = [
                    part1,
                    part2,
                    part3
                ];
            }
        }
    }

    writeObject(variables, "./conversions/system-variables.json");

    // 4) System functions - short version.
    text = fs.readFileSync("../../../../documentation/dynamic-docs/opfunc/sql.xml", {
        encoding: "utf-8"
    });

    result = processFile(text, entities);

    const functionList = result.children[0];
    if (!functionList || functionList.name !== "opfunctions") {
        throw new Error("Couldn't find the 'opfunctions' list in mysqld.xml.");
    }

    result = sanitizeObject(functionList);
    writeObject(result, "/Users/Mike/Downloads/mysqld-functions.json");

    const functions: any = {};
    for (const entry of result.children) {
        if (entry.name === "opfunction" && entry.attributes.type === "function") {
            let name = "";

            let description = "";
            let introduced = "";
            let deprecated = "";
            let removed = "";
            let signatures = [];
            let returnValue = "";

            for (const child of entry.children) {
                if (child.description) {
                    description = child.description;
                } else if (child.display) {
                    name = child.display;
                    if (name.endsWith("()")) {
                        name = name.substr(0, name.length - 2);
                    }
                } else if (child.name) {
                    switch (child.name) {
                        case "versions": {
                            for (const subChild of child.children) {
                                if (subChild.name === "introduced") {
                                    introduced = subChild.attributes.version;
                                }

                                if (subChild.name === "deprecated") {
                                    deprecated = subChild.attributes.version;
                                }

                                if (subChild.name === "removed") {
                                    removed = subChild.attributes.version;
                                }
                            }

                            break;
                        }

                        case "arguments": {
                            // One possible call signature. Can appear multiple times.
                            const callSignature: ICallSignature = {
                                signature: "",
                                arguments: [],
                            };

                            for (const subChild of child.children) {
                                if (subChild.name === "format" && subChild.children?.length > 0) {
                                    // This string is text with embeded HTML tags, which contains often additional
                                    // information. Thus, instead of creating our own parameter string, we re-use
                                    // the one from the "format" child node.
                                    // In some cases this string can be parsed as XML, but not in all.
                                    // So we do some manual handling here by replacing known tags with text
                                    // markup.
                                    let signature: string = subChild.children[0].cdata;
                                    if (signature) {
                                        // Since this is CDATA text, it can contain unnecessary whitespaces.
                                        signature = signature.replace(/\s+/g, " ");
                                        signature = signature.replace(/<replaceable>(\w+)<\/replaceable>/g,
                                            (_: string, group: string) => "`" + group + "`");
                                        signature = signature.replace(/<literal>(\w+)<\/literal>/g,
                                            (_: string, group: string) => group);

                                        callSignature.signature = signature;
                                    }

                                }

                                if (subChild.name === "argument") {
                                    callSignature.arguments.push({
                                        name: subChild.attributes.name,
                                        type: subChild.attributes.type,
                                        description: subChild.children,
                                    });
                                }
                            }
                            signatures.push(callSignature);

                            break;
                        }

                        case "return": {
                            returnValue = child.attributes.type;

                            break;
                        }

                        default: {
                            break;
                        }
                    }
                }
            }

            if (name !== "") {
                let parts: string[] = [];
                for (const signature of signatures) {
                    parts.push("**" + name + "**(" + signature.signature + ")");

                    for (const argument of signature.arguments) {
                        parts.push("- `" + argument.name + "` (" + argument.type + ") => " + argument.description);
                    }
                    parts.push("");
                }
                parts.push("returns: " + returnValue);

                let part1 = parts.join("\n");

                parts = [];
                if (introduced !== "") {
                    parts.push("⊕ " + introduced);
                }

                if (deprecated !== "") {
                    parts.push("⊘ " + deprecated);
                }

                if (removed !== "") {
                    parts.push("⊗ " + removed);
                }

                const part2 = "_**System Function**_" + (parts.length > 0 ? " (" + parts.join(", ") + ")" : "") +
                    "\n\n" + description;

                functions[name] = [
                    part1,
                    part2,
                ];
            }
        }
    }

    writeObject(functions, "./conversions/system-functions.json");
}