in managementnode/lib/VCL/Module/Provisioning.pm [197:460]
sub retrieve_image {
my $self = shift;
unless (ref($self) && $self->isa('VCL::Module')) {
notify($ERRORS{'CRITICAL'}, 0, "subroutine can only be called as a VCL::Module module object method");
return;
}
my $reservation_id = $self->data->get_reservation_id();
my $computer_id = $self->data->get_computer_id();
# Make sure image library functions are enabled
my $image_lib_enable = $self->data->get_management_node_image_lib_enable();
if (!$image_lib_enable) {
notify($ERRORS{'OK'}, 0, "image retrieval skipped, image library functions are disabled for this management node");
return;
}
# Get the image name from the reservation data
my $image_name = $self->data->get_image_name();
if (!$image_name) {
notify($ERRORS{'WARNING'}, 0, "failed to determine image name from reservation data");
return;
}
# Get a semaphore so only 1 process is able to retrieve the image at a time
# Do this before checking if the image exists in case another process is retrieving the image
# Wait up to 2 hours in case a large image is being retrieved
my $semaphore = $self->get_semaphore("retrieve_$image_name", (60 * 120)) || return;
# Make sure image does not already exist on this management node
if ($self->does_image_exist($image_name)) {
notify($ERRORS{'OK'}, 0, "$image_name already exists on this management node");
return 1;
}
# Get the image library partner string
my $image_lib_partners = $self->data->get_management_node_image_lib_partners();
if (!$image_lib_partners) {
notify($ERRORS{'WARNING'}, 0, "image library partners could not be determined");
return;
}
# Split up the partner list
my @partner_list = split(/,/, $image_lib_partners);
if ((scalar @partner_list) == 0) {
notify($ERRORS{'WARNING'}, 0, "image lib partners variable is not listed correctly or does not contain any information: $image_lib_partners");
return;
}
# Get the image repository path
my $image_repository_path_local;
if ($self->can('get_image_repository_path')) {
$image_repository_path_local = $self->get_image_repository_path();
}
elsif ($self->can('get_image_repository_directory_path')) {
$image_repository_path_local = $self->get_image_repository_directory_path();
}
else {
$image_repository_path_local = $self->data->get_vmhost_profile_repository_path(0);
# Add the image name as an intermediate directory
$image_repository_path_local =~ s/[\/\s]+$//;
$image_repository_path_local .= "/$image_name";
}
if (!$image_repository_path_local) {
notify($ERRORS{'WARNING'}, 0, "unable to determine the local image repository path");
return;
}
# Make sure the parent image repository path exists
my ($image_repository_directory_name, $image_repository_parent_directory_path) = fileparse($image_repository_path_local, qr/\.[^\\\/]*/);
if (!$image_repository_parent_directory_path) {
notify($ERRORS{'WARNING'}, 0, "unable to retrieve image from another management node because the path specified as the VM host profile repository could not be parsed: $image_repository_path_local");
return;
}
elsif (!$self->mn_os->file_exists($image_repository_parent_directory_path)) {
notify($ERRORS{'WARNING'}, 0, "unable to retrieve image from another management node because the path on this management node where the image files are to be copied does not exist, the path specified as the VM host profile repository path is: $image_repository_parent_directory_path, either a directory, mount point, or symbolic link must exist on this management node in this location");
return;
}
# Loop through the partners
# Find partners which have the image
# Check size for each partner
# Retrieve image from partner with largest image
# It's possible that another partner (management node) is currently copying the image from another managment node
# This should prevent copying a partial image
my %partner_info;
my $largest_partner_image_size = 0;
my @partners_with_image;
PARTNER: foreach my $partner (@partner_list) {
# Get the connection information for the partner management node
$partner_info{$partner}{hostname} = $self->data->get_management_node_hostname($partner);
$partner_info{$partner}{user} = $self->data->get_management_node_image_lib_user($partner) || 'root';
$partner_info{$partner}{identity_key} = $self->data->get_management_node_image_lib_key($partner) || '';
$partner_info{$partner}{port} = $self->data->get_management_node_ssh_port($partner) || '22';
# Call the provisioning module's get_image_repository_search_paths() subroutine
# This returns an array of strings to pass to du
my @search_paths = $self->get_image_repository_search_paths($partner);
if (@search_paths) {
notify($ERRORS{'DEBUG'}, 0, "retrieved image repository search paths for partner $partner:\n" . join("\n", @search_paths));
$partner_info{$partner}{search_paths} = \@search_paths;
}
else {
notify($ERRORS{'WARNING'}, 0, "failed to retrieve image repository search paths for partner: $partner");
next PARTNER;
}
# Run du to get the size of the image files on the partner if the image exists in any of the search paths
my $du_command = "du -ba " . join(" ", @{$partner_info{$partner}{search_paths}});
# Add 2>&1 or else STDOUT and STDERR may get mixed together (See VCL-688)
$du_command .= " 2>&1";
my ($du_exit_status, $du_output) = VCL::Module::OS::execute(
{
node => $partner,
command => $du_command,
display_output => 0,
timeout => 30,
max_attempts => 2,
port => $partner_info{$partner}{port},
user => $partner_info{$partner}{user},
identity_key => $partner_info{$partner}{identity_key},
}
);
if (!defined($du_output)) {
notify($ERRORS{'WARNING'}, 0, "failed to run SSH command to determine if image $image_name exists on $partner_info{$partner}{hostname}: $du_command");
next PARTNER;
}
# Loop through the du output lines, parse lines beginning with a number followed by a '/'
LINE: for my $line (@$du_output) {
my ($file_size, $file_path) = $line =~ /^(\d+)\s+(\/.+)/;
next if (!defined($file_size) || !defined($file_path));
my ($file_prefix, $directory_path, $file_extension) = fileparse($file_path, qr/\.[^.]*/);
if (!$file_prefix || !$directory_path || !$file_extension) {
next LINE;
}
my $file_name = "$file_prefix$file_extension";
$partner_info{$partner}{directory_paths}{$directory_path}{file_paths}{$file_path}{file_name} = $file_name;
$partner_info{$partner}{directory_paths}{$directory_path}{file_paths}{$file_path}{file_size} = $file_size;
$partner_info{$partner}{directory_paths}{$directory_path}{image_size} += $file_size;
}
if (!$partner_info{$partner}{directory_paths}) {
notify($ERRORS{'OK'}, 0, "$image_name does NOT exist on $partner_info{$partner}{hostname}, output:\n" . join("\n", @$du_output));
next PARTNER;
}
# Loop through the directories containing image files found on the partner
# The image may have been found in multiple directories
my $directory_path;
DIRECTORY_PATH: for my $check_directory_path (keys %{$partner_info{$partner}{directory_paths}}) {
if (!$directory_path) {
$directory_path = $check_directory_path;
next DIRECTORY_PATH;
}
my $file_count = scalar(keys %{$partner_info{$partner}{directory_paths}{$directory_path}{file_paths}});
my $image_size = $partner_info{$partner}{directory_paths}{$directory_path}{image_size};
my $check_file_count = scalar(keys %{$partner_info{$partner}{directory_paths}{$check_directory_path}{file_paths}});
my $check_image_size = $partner_info{$partner}{directory_paths}{$check_directory_path}{image_size};
notify($ERRORS{'DEBUG'}, 0, "found $image_name in multiple directories on $partner_info{$partner}{hostname}:\n" .
"$directory_path: file count: $file_count, image size: $image_size\n" .
"$check_directory_path: file count: $check_file_count, image size: $check_image_size"
);
# Compare the file count and image size, use the larger image
if ($check_image_size > $image_size || $check_file_count > $file_count) {
$directory_path = $check_directory_path;
}
}
# Add the info for the winning directory to the top of the hash
$partner_info{$partner}{file_paths} = $partner_info{$partner}{directory_paths}{$directory_path}{file_paths};
$partner_info{$partner}{image_size} = $partner_info{$partner}{directory_paths}{$directory_path}{image_size};
# Display the image size if any files were found
if ($partner_info{$partner}{image_size}) {
notify($ERRORS{'OK'}, 0, "$image_name exists on $partner_info{$partner}{hostname}, size: " . format_number($partner_info{$partner}{image_size}) . " bytes");
}
else {
notify($ERRORS{'DEBUG'}, 0, "$image_name does NOT exist on $partner_info{$partner}{hostname}");
next PARTNER;
}
# Check if the image size is larger than any previously found on other partners
if ($partner_info{$partner}{image_size} > $largest_partner_image_size) {
@partners_with_image = ();
}
# Check if the image size is larger than any previously found on other partners
if ($partner_info{$partner}{image_size} >= $largest_partner_image_size) {
push @partners_with_image, $partner;
$largest_partner_image_size = $partner_info{$partner}{image_size};
}
}
# Check if any partner was found
if (!@partners_with_image) {
notify($ERRORS{'WARNING'}, 0, "unable to find $image_name on other management nodes");
return;
}
notify($ERRORS{'OK'}, 0, "found $image_name on partner management nodes:\n" . join("\n", map { $partner_info{$_}{hostname} } (sort @partners_with_image)));
# Choose a random partner so that the same management node isn't used for most transfers
my $random_index = int(rand(scalar(@partners_with_image)));
my $retrieval_partner = $partners_with_image[$random_index];
my $retrieval_partner_hostname = $partner_info{$retrieval_partner}{hostname};
notify($ERRORS{'OK'}, 0, "selected random retrieval partner: $retrieval_partner_hostname:\n" . format_data($partner_info{$retrieval_partner}));
# Create the directory in the image repository
my $mkdir_command = "mkdir -pv $image_repository_path_local";
my ($mkdir_exit_status, $mkdir_output) = run_command($mkdir_command);
if (!defined($mkdir_output)) {
notify($ERRORS{'WARNING'}, 0, "failed to run command to create image repository directory: $mkdir_command");
return;
}
# Copy each file path to the image repository directory
my $image_size_total_bytes = $partner_info{$retrieval_partner}{image_size};
my $image_size_total_gb = format_number(($image_size_total_bytes / 1024 / 1024 / 1024), 1);
my $image_size_retrieved_bytes = 0;
my $image_size_retrieved_gb = 0.0;
my $file_total_count = scalar(keys %{$partner_info{$retrieval_partner}{file_paths}});
my $file_retrieved_count = 0;
for my $partner_file_path (sort {lc($a) cmp lc($b)} keys %{$partner_info{$retrieval_partner}{file_paths}}) {
$file_retrieved_count++;
my $file_name = $partner_info{$retrieval_partner}{file_paths}{$partner_file_path}{file_name};
my $local_file_path = "$image_repository_path_local/$file_name";
my $file_size_bytes = $partner_info{$retrieval_partner}{file_paths}{$partner_file_path}{file_size};
notify($ERRORS{'DEBUG'}, 0, "retrieving image file $file_retrieved_count/$file_total_count from $retrieval_partner_hostname: $partner_file_path --> $local_file_path (" . get_file_size_info_string($file_size_bytes) . ')');
if (run_scp_command("$partner_info{$retrieval_partner}{user}\@$retrieval_partner:$partner_file_path", $local_file_path, $partner_info{$retrieval_partner}{key}, $partner_info{$retrieval_partner}{port})) {
$image_size_retrieved_bytes += $file_size_bytes;
$image_size_retrieved_gb = format_number(($image_size_retrieved_bytes / 1024 / 1024 / 1024), 1);
my $retrieved_percent = format_number(int($image_size_retrieved_bytes / $image_size_total_bytes * 100), 0);
notify($ERRORS{'OK'}, 0, "retrieved image file $file_retrieved_count/$file_total_count, $retrieved_percent\% complete, $image_size_retrieved_gb/$image_size_total_gb GB");
}
else {
notify($ERRORS{'WARNING'}, 0, "failed to copy image $image_name from $retrieval_partner_hostname");
return;
}
}
# Make sure image was retrieved
if (!$self->does_image_exist($image_name)) {
notify($ERRORS{'WARNING'}, 0, "does_image_exist subroutine returned false for $image_name after it should have been retrieved");
return;
}
return 1;
}