in esrally/mechanic/supplier.py [0:0]
def create(cfg: types.Config, sources, distribution, car, plugins=None):
logger = logging.getLogger(__name__)
if plugins is None:
plugins = []
caching_enabled = cfg.opts("source", "cache", mandatory=False, default_value=True)
revisions = _extract_revisions(cfg.opts("mechanic", "source.revision", mandatory=sources))
source_build_method = cfg.opts("mechanic", "source.build.method", mandatory=False, default_value="default")
distribution_version = cfg.opts("mechanic", "distribution.version", mandatory=False)
supply_requirements = _supply_requirements(sources, distribution, plugins, revisions, distribution_version)
build_needed = any(build for _, _, build in supply_requirements.values())
es_supplier_type, es_version, _ = supply_requirements["elasticsearch"]
src_config = cfg.all_opts("source")
suppliers = []
target_os = cfg.opts("mechanic", "target.os", mandatory=False)
target_arch = cfg.opts("mechanic", "target.arch", mandatory=False)
template_renderer = TemplateRenderer(version=es_version, os_name=target_os, arch=target_arch)
if build_needed:
es_src_dir = os.path.join(_src_dir(cfg), _config_value(src_config, "elasticsearch.src.subdir"))
if source_build_method == "docker":
builder = DockerBuilder(src_dir=es_src_dir, log_dir=paths.logs(), client=docker.from_env())
else:
raw_build_jdk = car.mandatory_var("build.jdk")
try:
build_jdk = int(raw_build_jdk)
except ValueError:
raise exceptions.SystemSetupError(f"Car config key [build.jdk] is invalid: [{raw_build_jdk}] (must be int)")
builder = Builder(
build_jdk=build_jdk,
src_dir=es_src_dir,
log_dir=paths.logs(),
)
else:
builder = None
distributions_root = os.path.join(cfg.opts("node", "root.dir"), cfg.opts("source", "distribution.dir"))
dist_cfg = {}
# car / plugin defines defaults...
dist_cfg.update(car.variables)
for plugin in plugins:
for k, v in plugin.variables.items():
dist_cfg[f"plugin_{plugin.name}_{k}"] = v
# ... but the user can override it in rally.ini
dist_cfg.update(cfg.all_opts("distributions"))
if caching_enabled:
logger.info("Enabling source artifact caching.")
max_age_days = int(cfg.opts("source", "cache.days", mandatory=False, default_value=7))
if max_age_days <= 0:
raise exceptions.SystemSetupError(f"cache.days must be a positive number but is {max_age_days}")
source_distributions_root = os.path.join(distributions_root, "src")
_prune(source_distributions_root, max_age_days)
else:
logger.info("Disabling source artifact caching.")
source_distributions_root = None
if es_supplier_type == "source":
es_src_dir = os.path.join(_src_dir(cfg), _config_value(src_config, "elasticsearch.src.subdir"))
source_supplier = ElasticsearchSourceSupplier(
revision=es_version,
es_src_dir=es_src_dir,
remote_url=cfg.opts("source", "remote.repo.url"),
car=car,
builder=builder,
template_renderer=template_renderer,
)
if caching_enabled:
es_file_resolver = ElasticsearchFileNameResolver(dist_cfg, template_renderer)
source_supplier = CachedSourceSupplier(source_distributions_root, source_supplier, es_file_resolver)
suppliers.append(source_supplier)
repo = None
else:
es_src_dir = None
repo = DistributionRepository(
name=cfg.opts("mechanic", "distribution.repository"), distribution_config=dist_cfg, template_renderer=template_renderer
)
# TODO remove the below ignore when introducing type hints
suppliers.append(ElasticsearchDistributionSupplier(repo, es_version, distributions_root)) # type: ignore[arg-type]
for plugin in plugins:
if plugin.moved_to_module:
# TODO: https://github.com/elastic/rally/issues/1622
# If it is listed as a core plugin (i.e. a user has overriden the team's path or revision), then we will build
# We still need to use the post-install hooks to configure the keystore, so don't remove from list of plugins
logger.info("Plugin [%s] is now an Elasticsearch module and no longer needs to be built from source.", plugin.name)
continue
supplier_type, plugin_version, _ = supply_requirements[plugin.name]
if supplier_type == "source":
if CorePluginSourceSupplier.can_handle(plugin):
logger.info("Adding core plugin source supplier for [%s].", plugin.name)
assert es_src_dir is not None, f"Cannot build core plugin {plugin.name} when Elasticsearch is not built from source."
plugin_supplier = CorePluginSourceSupplier(plugin, es_src_dir, builder)
elif ExternalPluginSourceSupplier.can_handle(plugin):
logger.info("Adding external plugin source supplier for [%s].", plugin.name)
plugin_supplier = ExternalPluginSourceSupplier(
plugin,
plugin_version,
_src_dir(cfg, mandatory=False),
src_config,
Builder(
src_dir=es_src_dir,
log_dir=paths.logs(),
),
)
else:
raise exceptions.RallyError(
"Plugin %s can neither be treated as core nor as external plugin. Requirements: %s"
% (plugin.name, supply_requirements[plugin.name])
)
if caching_enabled:
plugin_file_resolver = PluginFileNameResolver(plugin.name, plugin_version)
plugin_supplier = CachedSourceSupplier(source_distributions_root, plugin_supplier, plugin_file_resolver)
suppliers.append(plugin_supplier) # type: ignore[arg-type] # TODO remove this ignore when introducing type hints
else:
logger.info("Adding plugin distribution supplier for [%s].", plugin.name)
assert repo is not None, "Cannot benchmark plugin %s from a distribution version but Elasticsearch from sources" % plugin.name
# TODO remove the below ignore when introducing type hints
suppliers.append(PluginDistributionSupplier(repo, plugin)) # type: ignore[arg-type]
return CompositeSupplier(suppliers)