in gui/tools/src/extractDocumentation.ts [595:1126]
function runConversion() {
fs.removeSync("./conversions");
fs.mkdirSync("./conversions");
// We cannot load and parse all used entities in the documentation, but at least the product names
// and some others can be used.
const content = fs.readFileSync("../../../../documentation/mvl-shared/en/entities/mysql/product-names.ent", {
encoding: "utf-8"
});
// The entities file is also an xml file, but the xml-json conversion leaves the <!ENTITY> entries out, which
// we need here to replace values in the main xml.
const lines = content.split("\n");
const entities: Map<string, string> = new Map([
["current-series", "8.0"],
["previous-series", "5.7"],
["current-version", "8.0.22"],
["base-url-downloads", "/downloads"],
["base-url-internals", "/base"],
]);
const entityRegEx = /<!ENTITY\s+(\w+)\s+(".+"|'.+')>/;
lines.forEach((line: string) => {
const match = line.match(entityRegEx);
if (match) {
entities.set(match[1], unquote(match[2]));
}
});
// 1) System variables - full documentation
let text = fs.readFileSync("../../../../documentation/refman-8.0/dba-mysqld-server-core.xml", {
encoding: "utf-8"
});
let result: any = processFile(text, entities);
// 1.1) Server system variables. Extract specific sub objects for isolated lists.
let variableSection = findSection(result, "server-system-variables");
if (variableSection) {
result = sanitizeObject(variableSection);
writeObject(result, "/Users/Mike/Downloads/server-system-variables.json");
} else {
throw new Error("Couldn't find section 'server-system-variables' in this file.");
}
// Find the item list with the variable names, by looking for an itemized list containing children having
// a sysvar role in the first subpart (which is then also used to get the name of the variable).
let foundVariables = false;
for (const child of result.children) {
if (child.name === "itemizedlist") {
const content = child.children;
if (content.length > 0 && content[0].length > 0) {
const firstEntry = content[0][0]; // Take the first sub entry of the first child.
if (firstEntry.para?.length > 0 && firstEntry.para[0].role === "sysvar") {
foundVariables = true;
const variables: any = {};
for (const varChild of content) {
// The first entry is used to get the variable name.
// The rest is used to collect a full description.
const description: string[] = [];
for (let i = 1; i < varChild.length; ++i) {
const entry = convertToMarkdown(varChild[i]);
if (Array.isArray(entry)) {
description.push(...entry);
}
else {
description.push(entry);
}
}
if (Array.isArray(varChild[0].para)) {
variables[unquote(varChild[0].para[0].text)] = description;
}
}
writeObject(variables, "./conversions/server-system-variables.json");
}
}
}
}
if (!foundVariables) {
console.warn("Could not find variables list in dba-mysqld-server-core.xml.");
}
// 1.2) InnoDB system variables - full documentation
text = fs.readFileSync("../../../../documentation/refman-8.0/se-innodb-core.xml", {
encoding: "utf-8"
});
result = processFile(text, entities);
variableSection = findSection(result, "innodb-parameters");
if (variableSection) {
result = sanitizeObject(variableSection);
writeObject(result, "/Users/Mike/Downloads/innodb-variables.json");
} else {
throw new Error("Couldn't find section 'innodb-parameters' in se-innodb-core.xml.");
}
foundVariables = false;
for (let i = 0; i < result.children.length - 1; ++i) {
let child = result.children[i];
if (child.bridgehead === "InnoDB System Variables" && i < result.children.length - 1) {
child = result.children[i + 1]; // The actual list is the next sibling of the title child.
const content = child.children;
// In opposition to the server variables xmls there's no "sysvar" role set for variable name entries.
// So, we assume the title is good enough to locate the list.
foundVariables = true;
const variables: any = {};
for (const varChild of content) {
// The first entry is used to get the variable name.
// The rest is used to collect a full description.
const description: string[] = [];
for (let i = 1; i < varChild.length; ++i) {
const entry = convertToMarkdown(varChild[i]);
if (Array.isArray(entry)) {
description.push(...entry);
}
else {
description.push(entry);
}
}
if (Array.isArray(varChild[0].para)) {
variables[unquote(varChild[0].para[0].text)] = description;
} else {
variables[unquote(varChild[0].para)] = description;
}
}
writeObject(variables, "./conversions/innodb-system-variables.json");
}
}
if (!foundVariables) {
console.warn("Could not find variables list in se-innodb-core.xml.");
}
// 3) The short variables description file.
text = fs.readFileSync("../../../../documentation/dynamic-docs/optvar/mysqld.xml", {
encoding: "utf-8"
});
result = processFile(text, entities);
const options = result.children[0];
if (!options || options.name !== "mysqloptions") {
throw new Error("Couldn't find the 'mysqloptions' list in mysqld.xml.");
}
result = sanitizeObject(options);
writeObject(result, "/Users/Mike/Downloads/mysqld-options.json");
const variables: any = {};
for (const entry of result.children) {
if (entry.name === "mysqloption") {
let xrefto = "";
let name = "";
let description = "";
let footnote = "";
let introduced = "";
let deprecated = "";
let removed = "";
let system: IVariableInfo[] = [];
let values: IValueInfo[] = [];
for (const child of entry.children) {
if (child.description) {
description = child.description;
} else if (child.footnote) {
footnote = child.footnote;
} else if (child.name) {
switch (child.name) {
case "xrefto": {
xrefto = child.attributes.id;
break;
}
case "seealso": {
// An additional external reference, which we don't use.
break;
}
case "description": {
// Appears if additional content exists for the description
// and hence it has not been condensed into a single "description" field.
description = child.children;
break;
}
case "types": {
for (const subChild of child.children) {
if (subChild.name === "system") {
system.push({
isDynamic: subChild.attributes.isdynamic === "yes",
hintable: subChild.attributes.hintable === "yes",
scope: subChild.attributes.scope,
minVersion: convertVersion(subChild.attributes.inversion),
maxVersion: convertVersion(subChild.attributes.outversion),
});
}
}
break;
}
case "values": {
const type = child.attributes.vartype;
let defaultValue = "";
let minValue = "";
let maxValue = "";
const valueList: string[] = [];
if (child.children) {
for (const subChild of child.children) {
if (subChild.name === "value") {
if (subChild.attributes.default) {
defaultValue = subChild.attributes.default;
} else if (subChild.attributes.minimum) {
minValue = subChild.attributes.minimum;
} else if (subChild.attributes.maximum) {
maxValue = subChild.attributes.maximum;
}
} else if (subChild.name === "choice") {
valueList.push("- `" + subChild.attributes.value + "`");
}
}
}
values.push({
type,
defaultValue,
minValue,
maxValue,
valueList,
minVersion: convertVersion(child.attributes.inversion),
maxVersion: convertVersion(child.attributes.outversion),
});
break;
}
case "versions": {
for (const subChild of child.children) {
if (subChild.name === "introduced") {
introduced = subChild.attributes.version;
}
if (subChild.name === "deprecated") {
deprecated = subChild.attributes.version;
}
if (subChild.name === "removed") {
removed = subChild.attributes.version;
}
}
break;
}
default: {
name = child.name;
break;
}
}
}
}
if (name !== "") {
// There can be multiple values sets per variable (each with an own version range).
// We use the highest version here.
let value: IValueInfo = {
type: "",
defaultValue: "",
minValue: "",
maxValue: "",
valueList: [],
minVersion: 0,
maxVersion: 0,
};
if (values.length > 0) {
if (values.length > 1) {
values = values.sort((a: IValueInfo, b: IValueInfo) => a.minVersion - b.minVersion);
}
value = values[values.length - 1];
}
let part1 = "```mysql\ndeclare @@" + name + " " + value.type + "\n```";
let parts = "";
let part2 = "_**System Variable**_";
parts = "";
if (introduced !== "") {
parts += "⊕ " + introduced;
}
if (deprecated !== "") {
if (parts !== "") {
parts += ", ";
}
parts += "⊘ " + deprecated;
}
if (removed !== "") {
if (parts !== "") {
parts += ", ";
}
parts += "⊗ " + removed;
}
if (parts !== "") {
part2 += " (" + parts + ")";
}
part2 += "\n\n";
parts = "";
if (value.defaultValue !== "") {
parts += "default: `" + value.defaultValue + "`";
}
if (value.minValue !== "") {
if (parts !== "") {
parts += ", ";
}
parts += "min: `" + value.minValue + "`";
}
if (value.maxValue !== "") {
if (parts !== "") {
parts += ", ";
}
parts += "max: `" + value.maxValue + "`";
}
if (value.valueList.length > 0) {
if (parts !== "") {
parts += ", ";
}
parts += "valid values:\n\n" + value.valueList.join("\n");
}
if (parts !== "") {
part2 += parts + "\n\n";
}
let systemInfo = { isDynamic: false, hintable: false, scope: "", minVersion: 0, maxVersion: 0 };
if (system.length > 0) {
if (system.length > 1) {
system = system.sort((a: IVariableInfo, b: IVariableInfo) => a.minVersion - b.minVersion);
}
systemInfo = system[system.length - 1];
}
if (systemInfo.scope === "both") {
systemInfo.scope = "global, session";
}
part2 += "dynamic: " + (systemInfo.isDynamic ? "yes" : "no");
part2 += ", scope: " + systemInfo.scope;
part2 += ", [SET_VAR hint](https://dev.mysql.com/doc/refman/8.0/en/optimizer-hints.html#" +
"optimizer-hints-set-var) " + (systemInfo.hintable ? "supported" : "not supported");
let part3 = description + " [[online documentation](https://dev.mysql.com/" +
"doc/refman/8.0/en/server-system-variables.html#" + xrefto + ")].";
if (footnote !== "") {
part3 += "\n\n> " + footnote;
}
variables[name] = [
part1,
part2,
part3
];
}
}
}
writeObject(variables, "./conversions/system-variables.json");
// 4) System functions - short version.
text = fs.readFileSync("../../../../documentation/dynamic-docs/opfunc/sql.xml", {
encoding: "utf-8"
});
result = processFile(text, entities);
const functionList = result.children[0];
if (!functionList || functionList.name !== "opfunctions") {
throw new Error("Couldn't find the 'opfunctions' list in mysqld.xml.");
}
result = sanitizeObject(functionList);
writeObject(result, "/Users/Mike/Downloads/mysqld-functions.json");
const functions: any = {};
for (const entry of result.children) {
if (entry.name === "opfunction" && entry.attributes.type === "function") {
let name = "";
let description = "";
let introduced = "";
let deprecated = "";
let removed = "";
let signatures = [];
let returnValue = "";
for (const child of entry.children) {
if (child.description) {
description = child.description;
} else if (child.display) {
name = child.display;
if (name.endsWith("()")) {
name = name.substr(0, name.length - 2);
}
} else if (child.name) {
switch (child.name) {
case "versions": {
for (const subChild of child.children) {
if (subChild.name === "introduced") {
introduced = subChild.attributes.version;
}
if (subChild.name === "deprecated") {
deprecated = subChild.attributes.version;
}
if (subChild.name === "removed") {
removed = subChild.attributes.version;
}
}
break;
}
case "arguments": {
// One possible call signature. Can appear multiple times.
const callSignature: ICallSignature = {
signature: "",
arguments: [],
};
for (const subChild of child.children) {
if (subChild.name === "format" && subChild.children?.length > 0) {
// This string is text with embeded HTML tags, which contains often additional
// information. Thus, instead of creating our own parameter string, we re-use
// the one from the "format" child node.
// In some cases this string can be parsed as XML, but not in all.
// So we do some manual handling here by replacing known tags with text
// markup.
let signature: string = subChild.children[0].cdata;
if (signature) {
// Since this is CDATA text, it can contain unnecessary whitespaces.
signature = signature.replace(/\s+/g, " ");
signature = signature.replace(/<replaceable>(\w+)<\/replaceable>/g,
(_: string, group: string) => "`" + group + "`");
signature = signature.replace(/<literal>(\w+)<\/literal>/g,
(_: string, group: string) => group);
callSignature.signature = signature;
}
}
if (subChild.name === "argument") {
callSignature.arguments.push({
name: subChild.attributes.name,
type: subChild.attributes.type,
description: subChild.children,
});
}
}
signatures.push(callSignature);
break;
}
case "return": {
returnValue = child.attributes.type;
break;
}
default: {
break;
}
}
}
}
if (name !== "") {
let parts: string[] = [];
for (const signature of signatures) {
parts.push("**" + name + "**(" + signature.signature + ")");
for (const argument of signature.arguments) {
parts.push("- `" + argument.name + "` (" + argument.type + ") => " + argument.description);
}
parts.push("");
}
parts.push("returns: " + returnValue);
let part1 = parts.join("\n");
parts = [];
if (introduced !== "") {
parts.push("⊕ " + introduced);
}
if (deprecated !== "") {
parts.push("⊘ " + deprecated);
}
if (removed !== "") {
parts.push("⊗ " + removed);
}
const part2 = "_**System Function**_" + (parts.length > 0 ? " (" + parts.join(", ") + ")" : "") +
"\n\n" + description;
functions[name] = [
part1,
part2,
];
}
}
}
writeObject(functions, "./conversions/system-functions.json");
}