in src/core/src/core_logic/PatchInstaller.py [0:0]
def install_packages_in_batches(self, all_packages, all_package_versions, packages, package_versions, maintenance_window, package_manager, max_batch_size_for_packages, simulate=False):
"""
Install packages in batches.
Parameters:
all_packages (List of strings): List of all available packages to install.
all_package_versions (List of strings): Versions of the packages in the list all_packages.
packages (List of strings): List of all packages selected by user to install.
package_versions (List of strings): Versions of packages in the list packages.
maintenance_window (MaintenanceWindow): Maintenance window for the job.
package_manager (PackageManager): Package manager used.
max_batch_size_for_packages (Integer): Maximum batch size.
simulate (bool): Whether this function is called from a test run.
Returns:
installed_update_count (int): Number of packages installed through installing packages in batches.
patch_installation_successful (bool): Whether package installation succeeded for all attempted packages.
maintenance_window_batch_cutoff_reached (bool): Whether process of installing packages in batches stopped due to not enough time in maintenance window
to install packages in batch.
not_attempted_and_failed_packages (List of strings): List of packages which are (a) Not attempted due to not enough time in maintenance window to install in batch.
(b) Failed to install in batch patching.
not_attempted_and_failed_package_versions (List of strings): Versions of packages in the list not_attempted_and_failed_packages.
"""
number_of_batches = int(math.ceil(len(packages) / float(max_batch_size_for_packages)))
self.composite_logger.log("\nDividing package install in batches. \nNumber of packages to be installed: " + str(len(packages)) + "\nBatch Size: " + str(max_batch_size_for_packages) + "\nNumber of batches: " + str(number_of_batches))
installed_update_count = 0
patch_installation_successful = True
maintenance_window_batch_cutoff_reached = False
# remaining_packages are the packages which are not attempted to install due to there is not enough remaining time in maintenance window to install packages in batches.
# These packages will be attempted in sequential installation if there is enough time in maintenance window to install package sequentially.
remaining_packages = []
remaining_package_versions = []
# failed_packages are the packages which are failed to install in batch patching. These packages will be attempted again in sequential patching if there is
# enough time remaining in maintenance window.
failed_packages = []
failed_package_versions = []
for batch_index in range(0, number_of_batches):
per_batch_installation_stopwatch = Stopwatch(self.env_layer, self.telemetry_writer, self.composite_logger)
per_batch_installation_stopwatch.start()
# Extension state check
if self.lifecycle_manager is not None:
self.lifecycle_manager.lifecycle_status_check()
begin_index = batch_index * max_batch_size_for_packages
end_index = begin_index + max_batch_size_for_packages - 1
end_index = min(end_index, len(packages) - 1)
packages_in_batch = []
package_versions_in_batch = []
already_installed_packages = []
for index in range(begin_index, end_index + 1):
if packages[index] not in self.last_still_needed_packages:
# Could have got installed as dependent package of some other package. Package installation status could also have been set.
already_installed_packages.append(packages[index])
self.attempted_parent_package_install_count += 1
self.successful_parent_package_install_count += 1
else:
packages_in_batch.append(packages[index])
package_versions_in_batch.append(package_versions[index])
if len(already_installed_packages) > 0:
self.composite_logger.log("Following packages are already installed. Could have got installed as dependent package of some other package " + str(already_installed_packages))
if len(packages_in_batch) == 0:
continue
remaining_time = maintenance_window.get_remaining_time_in_minutes()
if maintenance_window.is_package_install_time_available(package_manager, remaining_time, len(packages_in_batch)) is False:
self.composite_logger.log("Stopped installing packages in batches as it is past the maintenance window cutoff time for installing in batches." +
" Batch Index: {0}, remaining time: {1}, number of packages in batch: {2}".format(batch_index, remaining_time, str(len(packages_in_batch))))
maintenance_window_batch_cutoff_reached = True
remaining_packages = packages[begin_index:]
remaining_package_versions = package_versions[begin_index:]
break
# point in time status
progress_status = self.progress_template.format(str(datetime.timedelta(minutes=remaining_time)), str(self.attempted_parent_package_install_count), str(self.successful_parent_package_install_count), str(self.failed_parent_package_install_count), str(installed_update_count - self.successful_parent_package_install_count),
"Processing batch index: " + str(batch_index) + ", Number of packages: " + str(len(packages_in_batch)) + "\nProcessing packages: " + str(packages_in_batch))
self.composite_logger.log(progress_status)
# package_and_dependencies initially conains only packages in batch. The dependencies are added in the list by method include_dependencies
package_and_dependencies = list(packages_in_batch)
package_and_dependency_versions = list(package_versions_in_batch)
self.include_dependencies(package_manager, packages_in_batch, package_versions_in_batch, all_packages, all_package_versions, packages, package_versions, package_and_dependencies, package_and_dependency_versions)
parent_packages_installed_in_batch_count = 0
parent_packages_failed_in_batch_count = 0
number_of_dependencies_installed = 0
number_of_dependencies_failed = 0
code, out, exec_cmd = package_manager.install_update_and_dependencies(package_and_dependencies, package_and_dependency_versions, simulate)
for package,version in zip(package_and_dependencies, package_and_dependency_versions):
install_result = package_manager.get_installation_status(code, out, exec_cmd, package, version, simulate)
if install_result == Constants.FAILED:
if package in packages_in_batch:
# parent package
self.status_handler.set_package_install_status(package_manager.get_product_name(str(package)), str(version), Constants.FAILED)
self.failed_parent_package_install_count += 1
patch_installation_successful = False
parent_packages_failed_in_batch_count += 1
failed_packages.append(package)
failed_package_versions.append(version)
else:
# dependent package
number_of_dependencies_failed +=1
elif install_result == Constants.INSTALLED:
self.status_handler.set_package_install_status(package_manager.get_product_name(str(package)), str(version), Constants.INSTALLED)
if package in packages_in_batch:
# parent package
self.successful_parent_package_install_count += 1
parent_packages_installed_in_batch_count += 1
else:
# dependent package
number_of_dependencies_installed += 1
if package in self.last_still_needed_packages:
index = self.last_still_needed_packages.index(package)
self.last_still_needed_packages.pop(index)
self.last_still_needed_package_versions.pop(index)
installed_update_count += 1
self.attempted_parent_package_install_count += len(packages_in_batch)
# Update reboot pending status in status_handler
self.status_handler.set_reboot_pending(self.package_manager.is_reboot_pending())
# dependency package result management fallback (not reliable enough to be used as primary, and will be removed; remember to retain last_still_needed refresh when you do that)
installed_update_count += self.perform_status_reconciliation_conditionally(package_manager, condition=(self.attempted_parent_package_install_count % Constants.PACKAGE_STATUS_REFRESH_RATE_IN_SECONDS == 0)) # reconcile status after every 10 attempted installs
per_batch_install_perf_log = "[{0}={1}][{2}={3}][{4}={5}][{6}={7}][{8}={9}][{10}={11}][{12}={13}][{14}={15}]".format(Constants.PerfLogTrackerParams.TASK, "InstallBatchOfPackages",
"PackagesInBatch", str(packages_in_batch), "PackageAndDependencies", str(package_and_dependencies), "PackageAndDependencyVersions", str(package_and_dependency_versions),
"NumberOfParentPackagesInstalled", str(parent_packages_installed_in_batch_count), "NumberOfParentPackagesFailed", str(parent_packages_failed_in_batch_count),
"NumberOfDependenciesInstalled", str(number_of_dependencies_installed), "NumberOfDependenciesFailed", str(number_of_dependencies_failed))
per_batch_installation_stopwatch.stop_and_write_telemetry(str(per_batch_install_perf_log))
# Performing reconciliation at the end to get accurate number of installed packages through this function.
installed_update_count += self.perform_status_reconciliation_conditionally(package_manager, True)
# not_attempted_and_failed_packages is the list of packages including two kind of packages:
# (a) Not attempted due to not enough time in maintenance window to install packages in batches.
# (b) Failed to install in batch patching.
# These packages are attempted in the sequential patching if there is enough time remaining in maintenance window. The non attempted packages are in
# the front of the list than failed packages and hence non attempated packages are attempted first in sequential patching than the failed packages.
not_attempted_and_failed_packages = remaining_packages + failed_packages
not_attempted_and_failed_package_versions = remaining_package_versions + failed_package_versions
return installed_update_count, patch_installation_successful, maintenance_window_batch_cutoff_reached, not_attempted_and_failed_packages, not_attempted_and_failed_package_versions