in src/buildstream/_project.py [0:0]
def _load_pass(self, config, output, *, ignore_unknown=False):
# Load project options
options_node = config.get_mapping("options", default={})
output.options.load(options_node)
if self.junction:
# load before user configuration
output.options.load_yaml_values(self.junction.options)
# Collect option values specified in the user configuration
overrides = self._context.get_overrides(self.name)
override_options = overrides.get_mapping("options", default={})
output.options.load_yaml_values(override_options)
if self._cli_options:
output.options.load_cli_values(self._cli_options, ignore_unknown=ignore_unknown)
# We're done modifying options, now we can use them for substitutions
output.options.resolve()
#
# Now resolve any conditionals in the remaining configuration,
# any conditionals specified for project option declarations,
# or conditionally specifying the project name; will be ignored.
#
# Specify any options that would be ignored in the restrict list
# so as to raise an appropriate error.
#
output.options.process_node(
config,
restricted=[
"min-version",
"name",
"element-path",
"junctions",
"defaults",
"fatal-warnings",
"ref-storage",
"options",
"plugins",
],
)
# Element and Source type configurations will be composited later onto
# element/source types, so we delete it from here and run our final
# assertion after.
output.element_overrides = config.get_mapping("elements", default={})
output.source_overrides = config.get_mapping("sources", default={})
config.safe_del("elements")
config.safe_del("sources")
config._assert_fully_composited()
# Load base variables
output.base_variables = config.get_mapping("variables")
# Add the project name as a default variable
output.base_variables["project-name"] = self.name
# Extend variables with automatic variables and option exports
# Initialize it as a string as all variables are processed as strings.
# Based on some testing (mainly on AWS), maximum effective
# max-jobs value seems to be around 8-10 if we have enough cores
# users should set values based on workload and build infrastructure
if self._context.build_max_jobs == 0:
# User requested automatic max-jobs
platform = self._context.platform
output.base_variables["max-jobs"] = str(platform.get_cpu_count(8))
else:
# User requested explicit max-jobs setting
output.base_variables["max-jobs"] = str(self._context.build_max_jobs)
# Export options into variables, if that was requested
output.options.export_variables(output.base_variables)
# Prepare a Variables instance for substitution of source alias and
# source mirror values.
#
# This allows substitution of any project-level variables, plus the special
# variables which allow resolving project relative directories on the host.
#
toplevel_project = self._context.get_toplevel_project()
variables_node = output.base_variables.clone()
variables_node["project-root"] = str(self._absolute_directory_path)
variables_node["toplevel-root"] = str(toplevel_project._absolute_directory_path)
variables_node["project-root-uri"] = "file://" + urllib.parse.quote(str(self._absolute_directory_path))
variables_node["toplevel-root-uri"] = "file://" + urllib.parse.quote(
str(toplevel_project._absolute_directory_path)
)
variables = Variables(variables_node)
# Override default_mirror if not set by command-line
output.default_mirror = self._default_mirror or overrides.get_str("default-mirror", default=None)
# Source url aliases
output._aliases = config.get_mapping("aliases", default={})
# First try mirrors specified in user configuration, user configuration
# is allowed to completely disable mirrors by specifying an empty list,
# so we check for a None value here too.
#
mirrors_node = overrides.get_sequence("mirrors", default=None)
if mirrors_node is None:
mirrors_node = config.get_sequence("mirrors", default=[])
else:
self._mirror_override = True
# Perform variable substitutions in source mirror definitions,
# even if the mirrors are specified in user configuration.
variables.expand(mirrors_node)
# Collect SourceMirror objects
for mirror_node in mirrors_node:
mirror = self.source_mirror_factory.create(self._context, self, mirror_node)
output.mirrors[mirror.name] = mirror
if not output.default_mirror:
output.default_mirror = mirror.name
# Perform variable substitutions in source aliases
variables.expand(output._aliases)