powershell/plugins/plugin-tweak-model.ts (247 lines of code) (raw):

/*--------------------------------------------------------------------------------------------- * Copyright (c) Microsoft Corporation. All rights reserved. * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ import { Property, SealedChoiceSchema, codeModelSchema, CodeModel, StringSchema, ObjectSchema, GroupSchema, isObjectSchema, SchemaType, GroupProperty, ParameterLocation, Operation, Parameter, VirtualParameter, getAllProperties, ImplementationLocation, OperationGroup, Request, SchemaContext, ChoiceSchema, Schema, ConstantSchema, ConditionalValue } from '@autorest/codemodel'; import { pascalCase, deconstruct, fixLeadingNumber, serialize, KnownMediaType } from '@azure-tools/codegen'; import { items, keys, values, Dictionary, length } from '@azure-tools/linq'; import { PwshModel } from '../utils/PwshModel'; import { ModelState } from '../utils/model-state'; import { Channel, AutorestExtensionHost as Host, Session, startSession } from '@autorest/extension-base'; import { defaultCipherList } from 'constants'; import { String } from '../llcsharp/schema/string'; import { JsonType } from '../utils/schema'; export const HeaderProperty = 'HeaderProperty'; export enum HeaderPropertyType { Header = 'Header', HeaderAndBody = 'HeaderAndBody' } type State = ModelState<PwshModel>; // For now, we are not dynamically changing the service-name. Instead, we would figure out a method to change it during the creation of service readme's. export function titleToAzureServiceName(title: string): string { const titleCamelCase = pascalCase(deconstruct(title)).trim(); const serviceName = titleCamelCase // Remove: !StartsWith(Management)AndContains(Management), Client, Azure, Microsoft, APIs, API, REST .replace(/(?!^Management)(?=.*)Management|Client|Azure|Microsoft|APIs|API|REST/g, '') // Remove: EndsWith(ServiceResourceProvider), EndsWith(ResourceProvider), EndsWith(DataPlane), EndsWith(Data) .replace(/ServiceResourceProvider$|ResourceProvider$|DataPlane$|Data$/g, ''); return serviceName || titleCamelCase; } function dropDuplicatePropertiesInChildSchemas(schema: ObjectSchema, state: State, map: Map<string, Property> = new Map()) { let success = true; for (const parent of values(schema.parents?.immediate)) { //handle parents first if (!dropDuplicatePropertiesInChildSchemas(<ObjectSchema>parent, state, map)) { return false; } } for (const { key: id, value: property } of items(schema.properties)) { //see if it's in the parent. const pProp = map.get(property.serializedName); if (pProp) { //if the parent prop is the same type as the child prop //we're going to drop the child property. if (pProp.schema.type === property.schema.type) { //if it's an object type, it has to be the exact same schema type too if (pProp.schema.type != SchemaType.Object || pProp.schema === property.schema) { state.verbose(`Property '${property.serializedName}' in '${schema.language.default.name}' has a property the same as the parent, and is dropping the duplicate.`, {}); if (schema.properties) { delete schema.properties[id]; } } else { const conflict = `Property '${property.serializedName}' in '${schema.language.default.name}' has a conflict with a parent schema (allOf ${schema.parents?.immediate.joinWith(each => each.language.default.name)}.`; state.error(conflict, [], {}); success = false; } } } else { map.set(property.serializedName, property); } } return success; } export async function tweakModelV2(state: State): Promise<PwshModel> { const title = pascalCase(fixLeadingNumber(deconstruct(await state.getValue('title', state.model.info.title)))); state.setValue('title', title); const serviceName = await state.getValue('service-name', titleToAzureServiceName(title)); state.setValue('service-name', serviceName); const model = state.model; const schemas = model.schemas; // xichen: do we need other schema types? const allSchemas: Array<Schema> = [...schemas.objects ?? [], ...schemas.choices ?? [], ...schemas.sealedChoices ?? []]; model.commands = <any>{ operations: new Dictionary<any>(), parameters: new Dictionary<any>(), }; // we're going to create a schema that represents the distinct sum // of all operation PATH parameters const universalId = new ObjectSchema(`${serviceName}Identity`, ''); // xichen: Add 'universal-parameter-type' in language.default.uid, so that we can find it later universalId.language.default.uid = 'universal-parameter-type'; universalId.apiVersions = universalId.apiVersions || []; state.model.schemas.objects = state.model.schemas.objects || []; (<any>universalId.language.default).uid = 'universal-parameter-type'; state.model.schemas.objects.push(universalId); for (const group of values(model.operationGroups)) { for (const operation of values(group.operations)) { for (const response of values(operation.responses)) { // Mark returned object in response const respSchema: Schema = (<any>response).schema; if (respSchema?.type === SchemaType.Object) { respSchema.extensions = respSchema.extensions || {}; respSchema.extensions['is-return-object'] = true; } } for (const param of values(operation.parameters).where(each => each.protocol?.http?.in === ParameterLocation.Path)) { const name = param.language.default.name; const hasName = universalId.properties?.find((prop) => prop.language.default.name.toLocaleLowerCase() === name.toLocaleLowerCase()); if (!hasName) { if (!universalId.properties) { universalId.properties = []; } const newProp = new Property(name, param.language.default.description, param.schema); newProp.required = false; newProp.readOnly = false; newProp.serializedName = param.language.default.serializedName; universalId.properties.push(newProp); } } } } if (await state.getValue('azure', false)) { const idScheam = new Schema('_identity_type_', 'Resource identity path', SchemaType.String); const idProp = new Property('id', 'Resource identity path', idScheam); idProp.readOnly = false; idProp.required = false; idProp.language.default.uid = 'universal-parameter:resource identity'; if (!universalId.properties) { universalId.properties = []; } universalId.properties.push(idProp); } // xichen: do nothing in m3 logic. Comment it out // if an operation has a response that has a schema with string/binary we should make the response application/octet-stream // for (const operationGroups of values(model.operationGroups)) { // for (const operation of values(operationGroups.operations)) { // for (const response of values(operation.responses)) { // if ((response as any).schema) { // const respSchema = response as any; // if (respSchema.type === SchemaType.String && respSchema.format === StringFormat.Binary) { // // WHY WAS THIS HERE?! // // response.mimeTypes = [KnownMediaType.Stream]; // } // } // } // } // } // schemas that have parents and implement properties that are in the parent schemas // will have the property dropped in the child schema for (const schema of values(model.schemas.objects)) { if (length(schema.parents?.immediate) > 0) { if (!dropDuplicatePropertiesInChildSchemas(schema, state)) { throw new Error('Schemas are in conflict.'); } } } if (await state.getValue('use-storage-pipeline', false)) { // we're going to create new models for the reponse headers ? } else { // if an operation has a body parameter with string/binary, we should make the request application/octet-stream // === Header Schemas === // go thru the operations, find responses that have header values, and add a property to the schemas that are returned with those values for (const operationGroups of values(model.operationGroups)) { for (const operation of values(operationGroups.operations)) { for (const response of values(operation.responses)) { // for a given response, find the possible models that can be returned from the service for (const header of values(response.protocol.http?.headers)) { if (!(<any>response).schema) { // no response schema? can we fake one? // service.message{ Channel: Channel.Debug, Text: `${header.key} is in ${operation.details.default.name} but there is no response model` }); continue; } // if the method response has a schema and it's an object, we're going to add our properties to the schema object. // yes, this means that the reponse model may have properties that are undefined if the server doesn't send back the header // and other operations might add other headers that are not the same. // if the method's response is a primitive value (string, boolean, null, number) or an array, we can't modify that type obviously // in which case, we're going to add a header // work with schemas that have objects only. if ((<any>response).schema.type === SchemaType.Object) { const respSchema = <ObjectSchema>((<any>response).schema); const curHeader = <any>header; const headerKey = <string>curHeader.header; respSchema.language.default.hasHeaders = true; const property = values(getAllProperties(respSchema)).first((each) => each.language.default.name === headerKey); if (!property) { state.message({ Channel: Channel.Debug, Text: `Adding header property '${headerKey}' to model ${respSchema.language.default.name}` }); // create a property for the header value const newProperty = new Property(headerKey, curHeader.description || '', curHeader.schema); newProperty.language.default.required = false; // mark it that it's a header-only property newProperty.language.default[HeaderProperty] = HeaderPropertyType.Header; // add it to this model. if (!respSchema.properties) { respSchema.properties = []; } respSchema.properties.push(newProperty); } else { // there is a property with this name already. // was this previously declared as a header only property? if (!property.language.default[HeaderProperty]) { state.message({ Channel: Channel.Debug, Text: `Property ${headerKey} in model ${respSchema.language.default.name} can also come from the header.` }); // no.. There is duplication between header and body property. Probably because of etags. // tell it to be a header-and-body property. property.language.default[HeaderProperty] = HeaderPropertyType.HeaderAndBody; property.language.default.name = headerKey; } } } } } } } } // remove well-known header parameters from operations and add mark the operation has supporting that feature for (const operationGroups of values(model.operationGroups)) { for (const operation of values(operationGroups.operations)) { // if we have an operation with a body, and content-type is a multipart/formdata // then we should go thru the parameters of the body and look for a string/binary parameters // and remember to add another parameter for the filename of the string/binary const request = operation.requests?.[0]; request?.parameters?.filter((param) => param.schema.type !== SchemaType.Object && param.protocol.http?.in === 'body' && param.protocol.http?.style === KnownMediaType.Multipart) .forEach((param) => { for (const prop of values(getAllProperties(<ObjectSchema>param.schema))) { if (prop.schema.type === SchemaType.Binary) { prop.language.default.isNamedStream = true; } } }); // move well-known hearder parameters into details, and we can process them in the generator how we please. // operation.details.default.headerparameters = values(operation.parameters).where(p => p.in === ParameterLocation.Header && ['If-Match', 'If-None-Match'].includes(p.name)).toArray(); // remove if-match and if-none-match parameters from the operation itself. // operation.parameters = values(operation.parameters).where(p => !(p.in === ParameterLocation.Header && ['If-Match', 'If-None-Match'].includes(p.name))).toArray(); } } // identify models that are polymorphic in nature for (const schema of allSchemas) { if (schema instanceof ObjectSchema) { const objSchema = <ObjectSchema>schema; // if this actual type is polymorphic, make sure we know that. // parent class if (objSchema.discriminator) { objSchema.language.default.isPolymorphic = true; if (objSchema.children) { objSchema.language.default.polymorphicChildren = objSchema.children?.all; } } // sub class if (objSchema.discriminatorValue) { objSchema.language.default.discriminatorValue = objSchema.extensions?.['x-ms-discriminator-value']; } } } // identify parameters that are constants for (const group of values(model.operationGroups)) { for (const operation of values(group.operations)) { for (const parameter of values(operation.parameters)) { if (parameter.required) { if (parameter.schema.type === SchemaType.Choice) { const choiceSchema = <ChoiceSchema>parameter.schema; if (choiceSchema.choices.length === 1) { parameter.language.default.constantValue = choiceSchema.choices[0].value; } } else if (parameter.schema.type === SchemaType.Constant) { const constantSchema = <ConstantSchema>parameter.schema; parameter.language.default.constantValue = constantSchema.value.value; } else if (parameter.schema.type === SchemaType.SealedChoice) { const sealedChoiceSchema = <SealedChoiceSchema>parameter.schema; if (sealedChoiceSchema.choices.length === 1) { parameter.language.default.constantValue = sealedChoiceSchema.choices[0].value; if (sealedChoiceSchema.language.default.skip !== false) { sealedChoiceSchema.language.default.skip = true; } } } } else { if (parameter.schema.type === SchemaType.SealedChoice) { const sealedChoiceSchema = <SealedChoiceSchema>parameter.schema; if (sealedChoiceSchema.choices.length === 1) { sealedChoiceSchema.language.default.skip = false; } } } } } } // identify properties that are constants for (const schema of values(schemas.objects)) { for (const property of values(schema.properties)) { if (property === undefined) { continue; } if (property.required) { if (property.schema.type === SchemaType.Choice) { const choiceSchema = <ChoiceSchema>property.schema; if (choiceSchema.choices.length === 1) { // properties with an enum single value are constants // add the constant value property.language.default.constantValue = choiceSchema.choices[0].value; } } else if (property.schema.type === SchemaType.Constant) { const constantSchema = <ConstantSchema>property.schema; property.language.default.constantValue = constantSchema.value.value; } else if (property.schema.type === SchemaType.SealedChoice) { const sealedChoiceSchema = <SealedChoiceSchema>property.schema; if (sealedChoiceSchema.choices.length === 1) { property.language.default.constantValue = sealedChoiceSchema.choices[0].value; if (sealedChoiceSchema.language.default.skip !== false) { sealedChoiceSchema.language.default.skip = true; } } } } else { if (property.schema.type === SchemaType.SealedChoice) { const sealedChoiceSchema = <SealedChoiceSchema>property.schema; if (sealedChoiceSchema.choices.length === 1) { sealedChoiceSchema.language.default.skip = false; } } } } } // xichen: Do we need skip? // const enumsToSkip = new Set<string>(); // // identify properties that are constants // for (const schema of values(model.schemas)) { // for (const property of values(schema.properties)) { // if (property.details.default.required && length(property.schema.enum) === 1) { // // properties with an enum single value are constants // // add the constant value // property.details.default.constantValue = property.schema.enum[0]; // // mark as skip the generation of this model // enumsToSkip.add(property.schema.details.default.uid); // // make it a string and keep its name // property.schema = new Schema(property.schema.details.default.name, { type: property.schema.type }); // } else { // enumsToSkip.delete(property.schema.details.default.uid); // } // } // } // // mark enums that shouldn't be generated // for (const schema of values(model.schemas)) { // if (enumsToSkip.has(schema.details.default.uid)) { // schema.details.default.skip = true; // } // } return model; } // async function tweakModel(state: State): Promise<codemodel.Model> { // const title = pascalCase(fixLeadingNumber(deconstruct(await state.getValue('title', state.model.info.title)))); // state.setValue('title', title); // const serviceName = await state.getValue('service-name', titleToAzureServiceName(title)); // state.setValue('service-name', serviceName); // const model = state.model; // model.schemas = model.schemas || []; // const set = new Set<Schema>(); // const removes = new Set<string>(); // for (const key of keys(model.schemas)) { // const value = model.schemas[key]; // if (set.has(value)) { // // this schema is already in the collection. let's drop it when we're done // removes.add(key); // } else { // set.add(value); // } // } // // we're going to create a schema that represents the distinct sum // // of all operation PATH parameters // const universalId = new Schema(`${serviceName}Identity`, { // type: JsonType.Object, description: 'Resource Identity', details: { // default: { // uid: 'universal-parameter-type' // } // } // }); // model.schemas['universal-parameter-type'] = universalId; // for (const operation of values(model.http.operations)) { // for (const param of values(operation.parameters).where(each => each.in === ParameterLocation.Path)) { // const name = param.details.default.name; // if (!universalId.properties[name]) { // universalId.properties[name] = new Property(name, { // schema: param.schema, description: param.description, serializedName: name, details: { // default: { // description: param.description, // name: name, // required: false, // readOnly: false, // uid: `universal-parameter:${name}` // } // } // }); // } // } // } // if (await state.getValue('azure', false)) { // universalId.properties['id'] = new Property('id', { // schema: new Schema('_identity_type_', { type: JsonType.String, description: 'Resource identity path' }), // description: 'Resource identity path', serializedName: 'id', details: { // default: { // description: 'Resource identity path', // name: 'id', // required: false, // readOnly: false, // uid: 'universal-parameter:resource identity' // } // } // }); // } // // remove schemas that are referenced elsewhere previously. // for (const each of removes.values()) { // delete model.schemas[each]; // } // // if an operation has a response that has a schema with string/binary we should make the response application/octet-stream // for (const operation of values(model.http.operations)) { // for (const responses of values(operation.responses)) { // for (const response of responses) { // if (response.schema) { // if (response.schema.type === JsonType.String && response.schema.format === StringFormat.Binary) { // // WHY WAS THIS HERE?! // // response.mimeTypes = [KnownMediaType.Stream]; // } // } // } // } // } // // schemas that have parents and implement properties that are in the parent schemas // // will have the property dropped in the child schema // for (const schema of values(model.schemas)) { // if (length(schema.allOf) > 0) { // if (!dropDuplicatePropertiesInChildSchemas(schema, state)) { // throw new Error('Schemas are in conflict.'); // } // } // } // if (await state.getValue('use-storage-pipeline', false)) { // // we're going to create new models for the reponse headers ? // } else { // // if an operation has a body parameter with string/binary, we should make the request application/octet-stream // // === Header Schemas === // // go thru the operations, find responses that have header values, and add a property to the schemas that are returned with those values // for (const operation of values(model.http.operations)) { // for (const responses of values(operation.responses)) { // for (const response of responses) { // // for a given response, find the possible models that can be returned from the service // for (const header of values(response.headers)) { // if (!response.schema) { // // no response schema? can we fake one? // // service.message{ Channel: Channel.Debug, Text: `${header.key} is in ${operation.details.default.name} but there is no response model` }); // continue; // } // // if the method response has a schema and it's an object, we're going to add our properties to the schema object. // // yes, this means that the reponse model may have properties that are undefined if the server doesn't send back the header // // and other operations might add other headers that are not the same. // // if the method's response is a primitive value (string, boolean, null, number) or an array, we can't modify that type obviously // // in which case, we're going to add a header // // work with schemas that have objects only. // if (isSchemaObject(response.schema)) { // response.schema.details.default.hasHeaders = true; // const property = response.schema.properties[header.key]; // if (!property) { // state.message({ Channel: Channel.Debug, Text: `Adding header property '${header.key}' to model ${response.schema.details.default.name}` }); // // create a property for the header value // const newProperty = new Property(header.key, { schema: header.schema, description: header.description }); // newProperty.details.default.name = header.key; // newProperty.details.default.required = false; // // mark it that it's a header-only property // newProperty.details.default[HeaderProperty] = HeaderPropertyType.Header; // // add it to this model. // response.schema.properties[header.key] = newProperty; // } else { // // there is a property with this name already. // // was this previously declared as a header only property? // if (!property.details.default[HeaderProperty]) { // state.message({ Channel: Channel.Debug, Text: `Property ${header.key} in model ${response.schema.details.default.name} can also come from the header.` }); // // no.. There is duplication between header and body property. Probably because of etags. // // tell it to be a header-and-body property. // property.details.default[HeaderProperty] = HeaderPropertyType.HeaderAndBody; // property.details.default.name = header.key; // } // } // } // } // } // } // } // } // // remove well-known header parameters from operations and add mark the operation has supporting that feature // for (const operation of values(model.http.operations)) { // // if we have an operation with a body, and content-type is a multipart/formdata // // then we should go thru the parameters of the body and look for a string/binary parameters // // and remember to add another parameter for the filename of the string/binary // if (operation.requestBody && knownMediaType(operation.requestBody.contentType) === KnownMediaType.Multipart) { // for (const prop of values(operation.requestBody.schema.properties)) { // if (prop.schema.type === JsonType.String && prop.schema.format === 'binary') { // prop.details.default.isNamedStream = true; // } // } // } // // move well-known hearder parameters into details, and we can process them in the generator how we please. // // operation.details.default.headerparameters = values(operation.parameters).where(p => p.in === ParameterLocation.Header && ['If-Match', 'If-None-Match'].includes(p.name)).toArray(); // // remove if-match and if-none-match parameters from the operation itself. // // operation.parameters = values(operation.parameters).where(p => !(p.in === ParameterLocation.Header && ['If-Match', 'If-None-Match'].includes(p.name))).toArray(); // } // // identify models that are polymorphic in nature // for (const schema of values(model.schemas)) { // // if this actual type is polymorphic, make sure we know that. // if (schema.discriminator) { // schema.details.default.isPolymorphic = true; // } // const parents = getPolymorphicBases(schema); // if (length(parents) > 0) { // // if our parent is polymorphic, then we must have a discriminator value // schema.details.default.discriminatorValue = schema.extensions['x-ms-discriminator-value'] || schema.details.default.name; // // and make sure that all our polymorphic parents have a reference to this type. // for (const parent of getPolymorphicBases(schema)) { // parent.details.default.polymorphicChildren = parent.details.default.polymorphicChildren || new Array<Schema>(); // parent.details.default.polymorphicChildren.push(schema); // } // } // } // // identify parameters that are constants // for (const operation of values(model.http.operations)) { // for (const parameter of values(operation.parameters)) { // if (parameter.required && length(parameter.schema.enum) === 1) { // // parameters with an enum single value are constants // parameter.details.default.constantValue = parameter.schema.enum[0]; // } // } // } // const enumsToSkip = new Set<string>(); // // identify properties that are constants // for (const schema of values(model.schemas)) { // for (const property of values(schema.properties)) { // if (property.details.default.required && length(property.schema.enum) === 1) { // // properties with an enum single value are constants // // add the constant value // property.details.default.constantValue = property.schema.enum[0]; // // mark as skip the generation of this model // enumsToSkip.add(property.schema.details.default.uid); // // make it a string and keep its name // property.schema = new Schema(property.schema.details.default.name, { type: property.schema.type }); // } else { // enumsToSkip.delete(property.schema.details.default.uid); // } // } // } // // mark enums that shouldn't be generated // for (const schema of values(model.schemas)) { // if (enumsToSkip.has(schema.details.default.uid)) { // schema.details.default.skip = true; // } // } // for (const operation of values(model.http.operations)) { // for (const { key: responseCode, value: responses } of items(operation.responses)) { // for (const response of values(responses)) { // if (responseCode === 'default' || response.extensions['x-ms-error-response'] === true) { // response.details.default.isErrorResponse = true; // } // } // } // } // return model; // } // Universal version - // tweaks the code model to adjust things so that the code will generate better. export async function tweakModelPlugin(service: Host) { //const session = await startSession<PwshModel>(service, {}, codeModelSchema); const state = await new ModelState<PwshModel>(service).init(); //const result = tweakModelV2(session); await service.writeFile({ filename: 'code-model-v4-tweakcodemodel-v2.yaml', content: serialize(await tweakModelV2(state)), sourceMap: undefined, artifactType: 'code-model-v4' }); //return processCodeModel(tweakModelV2, service, 'tweakcodemodel-v2'); }