in core/src/main/java/org/apache/brooklyn/core/catalog/internal/BasicBrooklynCatalog.java [653:1066]
private void collectCatalogItemsFromItemMetadataBlock(String sourceYaml, ManagedBundle containingBundle, Map<?,?> itemMetadata, List<CatalogItemDtoAbstract<?, ?>> resultLegacyFormat, Map<RegisteredType, RegisteredType> resultNewFormat, boolean requireValidation,
Map<?,?> parentMetadata, int depth, boolean force, Boolean throwOnError) {
if (throwOnError==null) {
// default for legacy format was to throw, for new format to attempt to add and then remove
throwOnError = resultLegacyFormat!=null;
}
if (sourceYaml==null) sourceYaml = new Yaml().dump(itemMetadata);
Map<?, ?> itemMetadataWithoutItemDef = MutableMap.builder()
.putAll(itemMetadata)
.remove("item")
.remove("items")
.build();
// Parse CAMP-YAML DSL in item metadata (but not in item or items - those will be parsed only when used).
CampYamlParser parser = mgmt.getScratchpad().get(CampYamlParser.YAML_PARSER_KEY);
if (parser != null) {
itemMetadataWithoutItemDef = parser.parse((Map<String, Object>) itemMetadataWithoutItemDef);
try {
itemMetadataWithoutItemDef = (Map<String, Object>) Tasks.resolveDeepValueWithoutCoercion(itemMetadataWithoutItemDef, mgmt.getServerExecutionContext());
} catch (Exception e) {
throw Exceptions.propagate(e);
}
} else {
if (!WARNED_RE_DSL_PARSER) {
log.warn("No Camp-YAML parser registered for parsing catalog item DSL; skipping DSL-parsing (no further warnings)");
WARNED_RE_DSL_PARSER = true;
}
}
Map<Object,Object> catalogMetadata = MutableMap.<Object, Object>builder()
.putAll(parentMetadata)
.putAll(itemMetadataWithoutItemDef)
.putIfNotNull("item", itemMetadata.get("item"))
.putIfNotNull("items", itemMetadata.get("items"))
.build();
// tags we treat specially to concatenate as a set (treating as config with merge might be cleaner)
catalogMetadata.put("tags", MutableSet.copyOf(getFirstAs(parentMetadata, Collection.class, "tags").orNull())
.putAll(getFirstAs(itemMetadataWithoutItemDef, Collection.class, "tags").orNull()) );
// brooklyn.libraries we treat specially, to append the list, with the child's list preferred in classloading order
// `libraries` is supported in some places as a legacy syntax; it should always be `brooklyn.libraries` for new apps
List<?> librariesAddedHereNames = MutableList.copyOf(getFirstAs(itemMetadataWithoutItemDef, List.class, "brooklyn.libraries", "libraries").orNull());
Collection<CatalogBundle> librariesAddedHereBundles = CatalogItemDtoAbstract.parseLibraries(librariesAddedHereNames);
boolean fromInitialCatalog = containingBundle instanceof BasicManagedBundle && Boolean.TRUE.equals( ((BasicManagedBundle)containingBundle).getFromInitialCatalog() );
MutableSet<Object> librariesCombinedNames = MutableSet.of();
if (!isNoBundleOrSimpleWrappingBundle(mgmt, containingBundle)) {
// ensure containing bundle is declared, first, for search purposes
librariesCombinedNames.add(containingBundle.getVersionedName().toOsgiString());
}
librariesCombinedNames.putAll(librariesAddedHereNames);
librariesCombinedNames.putAll(getFirstAs(parentMetadata, Collection.class, "brooklyn.libraries", "libraries").orNull());
if (!librariesCombinedNames.isEmpty()) {
catalogMetadata.put("brooklyn.libraries", librariesCombinedNames);
}
Collection<CatalogBundle> libraryBundles = CatalogItemDtoAbstract.parseLibraries(librariesCombinedNames);
// TODO this may take a while if downloading; ideally the REST call would be async
// but this load is required for resolving YAML in this BOM (and if java-scanning);
// need to think through how we expect dependencies to be installed
CatalogUtils.installLibraries(mgmt, librariesAddedHereBundles, true, fromInitialCatalog);
// use resolved bundles
librariesAddedHereBundles = resolveWherePossible(mgmt, librariesAddedHereBundles);
libraryBundles = resolveWherePossible(mgmt, libraryBundles);
Boolean scanJavaAnnotations = getFirstAs(itemMetadataWithoutItemDef, Boolean.class, "scanJavaAnnotations", "scan_java_annotations").orNull();
if (scanJavaAnnotations!=null && scanJavaAnnotations) {
addLegacyScannedAnnotations(containingBundle, resultLegacyFormat, resultNewFormat, depth, catalogMetadata, librariesAddedHereBundles, libraryBundles);
}
Object items = catalogMetadata.remove("items");
Object item = catalogMetadata.remove("item");
Object url = catalogMetadata.remove("include");
if (items!=null) {
int count = 0;
for (Object ii: checkType(items, "items", List.class)) {
if (ii instanceof String) {
collectUrlReferencedCatalogItems((String) ii, containingBundle, resultLegacyFormat, resultNewFormat, requireValidation, catalogMetadata, depth+1, force, throwOnError);
} else {
Map<?,?> i = checkType(ii, "entry in items list", Map.class);
collectCatalogItemsFromItemMetadataBlock(Yamls.getTextOfYamlAtPath(sourceYaml, "items", count).getMatchedYamlTextOrWarn(),
containingBundle, i, resultLegacyFormat, resultNewFormat, requireValidation, catalogMetadata, depth+1, force, throwOnError);
}
count++;
}
}
if (url != null) {
collectUrlReferencedCatalogItems(checkType(url, "include in catalog meta", String.class), containingBundle,
resultLegacyFormat, resultNewFormat, requireValidation, catalogMetadata, depth+1, force, throwOnError);
}
if (item==null) return;
// now look at the actual item, first correcting the sourceYaml and interpreting the catalog metadata
String itemYaml = Yamls.getTextOfYamlAtPath(sourceYaml, "item").getMatchedYamlTextOrWarn();
if (itemYaml!=null) sourceYaml = itemYaml;
else sourceYaml = new Yaml().dump(item);
CatalogItemType itemType = TypeCoercions.coerce(getFirstAs(catalogMetadata, Object.class, "itemType", "item_type").orNull(), CatalogItemType.class);
String id = getFirstAs(catalogMetadata, String.class, "id").orNull();
String version = getFirstAs(catalogMetadata, String.class, "version").orNull();
if (log.isTraceEnabled()) log.trace("Installing "+id+":"+version);
if (parentMetadata.containsKey("version") && !Objects.equal(parentMetadata.get("version"), version))
log.warn("Bundle "+containingBundle+" declares version "+version+" for items overriding broader version "+parentMetadata.get("version"));
else if (!parentMetadata.containsKey("version") && containingBundle!=null && version!=null && !Objects.equal(new VersionedName("x", version).getOsgiVersionString(), containingBundle.getVersionedName().getOsgiVersionString()))
log.warn("Bundle "+containingBundle+" declares items at different version "+version);
String symbolicName = getFirstAs(catalogMetadata, String.class, "symbolicName").orNull();
String displayName = getFirstAs(catalogMetadata, String.class, "displayName").orNull();
String name = getFirstAs(catalogMetadata, String.class, "name").orNull();
String format = getFirstAs(catalogMetadata, String.class, "format").orNull();
if ("auto".equalsIgnoreCase(format)) format = null;
if ((Strings.isNonBlank(id) || Strings.isNonBlank(symbolicName)) &&
Strings.isNonBlank(displayName) &&
Strings.isNonBlank(name) && !name.equals(displayName)) {
log.warn("Name property will be ignored due to the existence of displayName and at least one of id, symbolicName");
}
CharSequence loggedId = Strings.firstNonBlank(id, symbolicName, displayName, "<unidentified>");
log.debug("Analyzing item " + loggedId + " for addition to catalog");
Exception resolutionError = null;
String itemAsString = item instanceof String ? (String) item : null;
if (itemAsString!=null && itemAsString.matches("[A-Za-z0-9]+:[^\\s]+")) {
// if sourceYaml is one word and looks like a URL, then read it as a URL first
BrooklynClassLoadingContext loader = getClassLoadingContext("catalog item url loader", parentMetadata, libraryBundles);
log.debug("Catalog load, loading referenced item at "+item+" for "+loggedId+" as part of "+(containingBundle==null ? "non-bundled load" : containingBundle.getVersionedName())+" ("+(resultNewFormat!=null ? resultNewFormat.size() : resultLegacyFormat!=null ? resultLegacyFormat.size() : "(unknown)")+" items before load)");
if (itemAsString.startsWith("http")) {
// give greater visibility to these
log.info("Loading external referenced item at "+item+" for "+loggedId+" as part of "+(containingBundle==null ? "non-bundled load" : containingBundle.getVersionedName()));
}
try {
sourceYaml = ResourceUtils.create(loader).getResourceAsString(itemAsString.trim());
item = Yamls.parseAll(sourceYaml).iterator().next();
} catch (Exception e) {
Exceptions.propagateIfFatal(e);
// don't throw, but include in list of proposed errors
resolutionError = new IllegalStateException("Unable to load '"+itemAsString+"' as URL", e);
}
}
PlanInterpreterInferringType planInterpreter = new PlanInterpreterInferringType(id, item, sourceYaml, itemType, format,
containingBundle, libraryBundles,
null, resultLegacyFormat);
Map<?, ?> itemAsMap = planInterpreter.getItem();
// the "plan yaml" includes the services: ... or brooklyn.policies: ... outer key,
// as opposed to the rawer { type: foo } map without that outer key which is valid as item input
// if symname not set, infer from: id, then name, then item id, then item name
if (Strings.isBlank(symbolicName)) {
if (Strings.isNonBlank(id)) {
if (RegisteredTypeNaming.isGoodBrooklynTypeColonVersion(id)) {
symbolicName = CatalogUtils.getSymbolicNameFromVersionedId(id);
} else if (RegisteredTypeNaming.isValidOsgiTypeColonVersion(id)) {
symbolicName = CatalogUtils.getSymbolicNameFromVersionedId(id);
log.warn("Discouraged version syntax in id '"+id+"'; version should comply with brooklyn recommendation (#.#.#-qualifier or portion) or specify symbolic name and version explicitly, not OSGi version syntax");
} else if (CatalogUtils.looksLikeVersionedId(id)) {
// use of above method is deprecated in 0.12; this block can be removed in 0.13
log.warn("Discouraged version syntax in id '"+id+"'; version should comply with brooklyn recommendation (#.#.#-qualifier or portion) or specify symbolic name and version explicitly");
symbolicName = CatalogUtils.getSymbolicNameFromVersionedId(id);
} else if (RegisteredTypeNaming.isUsableTypeColonVersion(id)) {
log.warn("Deprecated type naming syntax in id '"+id+"'; colons not allowed in type name as it is used to indicate version");
// deprecated in 0.12; from 0.13 this can change to treat part after the colon as version, also see line to set version below
// (may optionally warn or disallow if we want to require OSGi versions)
// symbolicName = CatalogUtils.getSymbolicNameFromVersionedId(id);
symbolicName = id;
} else {
symbolicName = id;
}
} else if (Strings.isNonBlank(name)) {
if (RegisteredTypeNaming.isGoodBrooklynTypeColonVersion(name) || RegisteredTypeNaming.isValidOsgiTypeColonVersion(name)) {
log.warn("Deprecated use of 'name' key to define '"+name+"'; version should be specified within 'id' key or with 'version' key, not this tag");
// deprecated in 0.12; remove in 0.13
symbolicName = CatalogUtils.getSymbolicNameFromVersionedId(name);
} else if (CatalogUtils.looksLikeVersionedId(name)) {
log.warn("Deprecated use of 'name' key to define '"+name+"'; version should be specified within 'id' key or with 'version' key, not this tag");
// deprecated in 0.12; remove in 0.13
symbolicName = CatalogUtils.getSymbolicNameFromVersionedId(name);
} else if (RegisteredTypeNaming.isUsableTypeColonVersion(name)) {
log.warn("Deprecated type naming syntax in id '"+id+"'; colons not allowed in type name as it is used to indicate version");
// deprecated in 0.12; throw error if we want in 0.13
symbolicName = name;
} else {
symbolicName = name;
}
} else {
symbolicName = setFromItemIfUnset(symbolicName, itemAsMap, "id");
symbolicName = setFromItemIfUnset(symbolicName, itemAsMap, "name");
// TODO we should let the plan transformer give us this
symbolicName = setFromItemIfUnset(symbolicName, itemAsMap, "template_name");
if (Strings.isBlank(symbolicName)) {
log.error("Can't infer catalog item symbolicName from the following plan:\n" + Sanitizer.sanitizeJsonTypes(sourceYaml));
throw new IllegalStateException("Can't infer catalog item symbolicName from catalog item metadata");
}
}
}
String versionFromId = null;
if (RegisteredTypeNaming.isGoodBrooklynTypeColonVersion(id)) {
versionFromId = CatalogUtils.getVersionFromVersionedId(id);
} else if (RegisteredTypeNaming.isValidOsgiTypeColonVersion(id)) {
versionFromId = CatalogUtils.getVersionFromVersionedId(id);
log.warn("Discouraged version syntax in id '"+id+"'; version should comply with Brooklyn recommended version syntax (#.#.#-qualifier or portion) or specify symbolic name and version explicitly, not OSGi");
} else if (CatalogUtils.looksLikeVersionedId(id)) {
log.warn("Discouraged version syntax in id '"+id+"'; version should comply with Brooklyn recommended version syntax (#.#.#-qualifier or portion) or specify symbolic name and version explicitly");
// remove in 0.13
versionFromId = CatalogUtils.getVersionFromVersionedId(id);
} else if (RegisteredTypeNaming.isUsableTypeColonVersion(id)) {
// deprecated in 0.12, with warning above; from 0.13 this can be uncommented to treat part after the colon as version
// (may optionally warn or disallow if we want to require OSGi versions)
// if comparable section above is changed, change this to:
// versionFromId = CatalogUtils.getVersionFromVersionedId(id);
}
// if version not set, infer from: id, then from name, then item version
if (versionFromId!=null) {
if (Strings.isNonBlank(version) && !versionFromId.equals(version)) {
throw new IllegalArgumentException("Discrepancy between version set in id " + versionFromId + " and version property " + version);
}
version = versionFromId;
}
if (Strings.isBlank(version)) {
if (CatalogUtils.looksLikeVersionedId(name)) {
// deprecated in 0.12, remove in 0.13
log.warn("Deprecated use of 'name' key to define '"+name+"'; version should be specified within 'id' key or with 'version' key, not this tag");
version = CatalogUtils.getVersionFromVersionedId(name);
}
if (Strings.isBlank(version)) {
version = setFromItemIfUnset(version, itemAsMap, "version");
version = setFromItemIfUnset(version, itemAsMap, "template_version");
if (Strings.isBlank(version)) {
if (log.isTraceEnabled()) log.trace("No version specified for catalog item " + symbolicName + " or BOM ancestors. Using default/bundle value.");
version = null;
}
}
}
// if not set, ID can come from symname:version, failing that, from the plan.id, failing that from the sym name
if (Strings.isBlank(id)) {
// let ID be inferred, especially from name, to support style where only "name" is specified, with inline version
if (Strings.isNonBlank(symbolicName) && Strings.isNonBlank(version)) {
id = symbolicName + ":" + version;
}
id = setFromItemIfUnset(id, itemAsMap, "id");
if (Strings.isBlank(id)) {
if (Strings.isNonBlank(symbolicName)) {
id = symbolicName;
} else {
log.error("Can't infer catalog item id from the following plan:\n" + Sanitizer.sanitizeJsonTypes(sourceYaml));
throw new IllegalStateException("Can't infer catalog item id from catalog item metadata");
}
}
}
if (Strings.isBlank(displayName)) {
if (Strings.isNonBlank(name)) displayName = name;
displayName = setFromItemIfUnset(displayName, itemAsMap, "name");
}
String description = getFirstAs(catalogMetadata, String.class, "description").orNull();
description = setFromItemIfUnset(description, itemAsMap, "description");
// icon.url is discouraged (using '.'), but kept for legacy compatibility; should deprecate this
// 2021-04: better semantics, look at this level, then in the item, then in inherited values
// this should probably be done elsewhere, and also note setFromItemIfUnset should maybe call to getFirstAs...
String catalogIconUrl = null;
catalogIconUrl = setFromItemIfUnset(catalogIconUrl, itemMetadata, "iconUrl", "icon_url", "icon.url");
catalogIconUrl = setFromItemIfUnset(catalogIconUrl, itemAsMap, "iconUrl", "icon_url", "icon.url");
catalogIconUrl = setFromItemIfUnset(catalogIconUrl, catalogMetadata, "iconUrl", "icon_url", "icon.url");
final String deprecated = getFirstAs(catalogMetadata, String.class, "deprecated").orNull();
final Boolean catalogDeprecated = Boolean.valueOf(setFromItemIfUnset(deprecated, itemAsMap, "deprecated"));
// provisional resolution - will be done again during validation, and even the kind might change, eg if there is a local bundle item
// indicating a different preferred supertype as compared with something else stored in type registry
planInterpreter.resolve();
if (!planInterpreter.isResolved()) {
// don't throw yet, we may be able to add it in an unresolved state
resolutionError = Exceptions.create("Could not resolve definition of item"
+ (Strings.isNonBlank(id) ? " '"+id+"'" : Strings.isNonBlank(symbolicName) ? " '"+symbolicName+"'" : Strings.isNonBlank(name) ? " '"+name+"'" : "")
// better not to show yaml, takes up lots of space, and with multiple plan transformers there might be multiple errors;
// some of the errors themselves may reproduce it
// (ideally in future we'll be able to return typed errors with caret position of error)
// + ":\n"+sourceYaml
, MutableList.<Exception>of().appendIfNotNull(resolutionError).appendAll(planInterpreter.getErrors()));
}
// might be null
itemType = planInterpreter.getCatalogItemType();
// run again if ID has just been learned, to catch recursive definitions and possibly other mistakes (itemType inconsistency?)
if (!Objects.equal(id, planInterpreter.itemId)) {
planInterpreter.setId(id).resolve();
if (resolutionError == null && !planInterpreter.isResolved()) {
resolutionError = new IllegalStateException("Plan resolution for " + id + " breaks after id and itemType are set; is there a recursive reference or other type inconsistency?\n" + sourceYaml);
}
}
if (throwOnError && resolutionError!=null) {
// if there was an error, throw it here
throw Exceptions.propagate(resolutionError);
}
String sourcePlanYaml = planInterpreter.getPlanYaml();
if (resultLegacyFormat==null) {
// horrible API but basically `resultLegacyFormat==null` means use the new-style,
// adding from persisted bundles to type registry (which is not persisted)
// instead of old way which persisted catalog items (and not their bundles).
// this lets us deal with forward references, with a subsequent step to validate.
Set<Object> tags = MutableSet.of().putAll(getFirstAs(catalogMetadata, Collection.class, "tags").orNull());
List<String> aliases = MutableList.of();
// could easily allow aliases to be set in catalog.bom, as done for tags above,
// but currently we don't, we only allow the official type name
Boolean catalogDisabled = null;
MutableList<Object> superTypes = MutableList.of();
if (itemType==CatalogItemType.TEMPLATE) {
tags.add(BrooklynTags.CATALOG_TEMPLATE);
itemType = CatalogItemType.APPLICATION;
}
if (itemType==CatalogItemType.APPLICATION) {
itemType = CatalogItemType.ENTITY;
superTypes.add(Application.class);
}
if (resolutionError!=null) {
if (!tags.contains(BrooklynTags.CATALOG_TEMPLATE)) {
if (requireValidation) {
throw Exceptions.propagate(resolutionError);
} else {
// normally if validation not requested (eg because we are adding multiple bundles and might have forward references,
// we add (below) as unresolved, and then do validation later; but that validation doesn't check basic problems,
// on those it makes sense to fail fast.
planInterpreter.checkResolution(true);
}
}
}
if (itemType!=null) {
// if supertype is known, set it here;
// we don't set kind (spec) because that is inferred from the supertype type
superTypes.appendIfNotNull(BrooklynObjectType.of(itemType).getInterfaceType());
}
if (version==null) {
if (containingBundle!=null) {
version = containingBundle.getVersionedName().getVersionString();
}
if (version==null) {
// use this as default version when nothing specified or inferrable from containing bundle
log.debug("No version specified for catalog item " + symbolicName + " or BOM ancestors and not available from bundle. Using default value "+BasicBrooklynCatalog.NO_VERSION+".");
version = BasicBrooklynCatalog.NO_VERSION;
}
}
if (sourcePlanYaml==null) {
// happens if unresolved and not valid yaml, replace with item yaml
// which normally has "type: " prefixed
sourcePlanYaml = planInterpreter.itemYaml;
}
Set<OsgiBundleWithUrl> searchBundles = MutableSet.<OsgiBundleWithUrl>of().putIfNotNull(containingBundle).putAll(libraryBundles);
BasicRegisteredType type = createYetUnsavedRegisteredTypeInstance(
BrooklynObjectType.of(planInterpreter.catalogItemType).getSpecType()!=null ? RegisteredTypeKind.SPEC
: planInterpreter.catalogItemType==CatalogItemType.BEAN ? RegisteredTypeKind.BEAN
: RegisteredTypeKind.UNRESOLVED,
symbolicName, version,
containingBundle, searchBundles,
displayName, description, catalogIconUrl, catalogDeprecated, sourcePlanYaml,
tags, aliases, catalogDisabled, superTypes, format);
// record original source in case it was changed
RegisteredTypes.notePlanEquivalentToThis(type, new BasicTypeImplementationPlan(format, sourceYaml));
RegisteredType replacedInstance = mgmt.getTypeRegistry().get(type.getSymbolicName(), type.getVersion());
log.debug("Analyzed " + loggedId + " as " + type + " (" + Strings.firstNonNull(planInterpreter.catalogItemType, "unresolved") + "), adding to type registry " +
(planInterpreter.catalogItemType==null ? "(despite errors at this stage, "+planInterpreter.getErrors().stream().findFirst().orElse(null)+"; may be resolved later once other items are added, or may fail later)"
: "(will re-resolve as registered type later)"));
((BasicBrooklynTypeRegistry) mgmt.getTypeRegistry()).addToLocalUnpersistedTypeRegistry(type, force);
updateResultNewFormat(resultNewFormat, replacedInstance, type);
} else {
CatalogItemDtoAbstract<?, ?> dto = createItemBuilder(itemType, symbolicName, version)
.libraries(libraryBundles)
.displayName(displayName)
.description(description)
.deprecated(catalogDeprecated)
.iconUrl(catalogIconUrl)
.plan(sourcePlanYaml)
.build();
dto.setManagementContext((ManagementContextInternal) mgmt);
log.debug("Analyzed " + loggedId + " as " + dto + " (" + planInterpreter.catalogItemType + "), adding to legacy catalog");
resultLegacyFormat.add(dto);
}
}