in managementnode/lib/VCL/new.pm [637:861]
sub computer_not_being_used {
my $self = shift;
my $request_id = $self->data->get_request_id();
my $computer_id = $self->data->get_computer_id();
my $computer_short_name = $self->data->get_computer_short_name();
my $imagerevision_id = $self->data->get_imagerevision_id();
my $image_name = $self->data->get_image_name();
my $image_reloadtime = $self->data->get_image_reload_time();
my $request_state_name = $self->data->get_request_state_name();
my $attempt_limit = 24;
ATTEMPT: for (my $attempt = 1; $attempt <= $attempt_limit; $attempt++) {
if ($attempt > 2) {
notify($ERRORS{'OK'}, 0, "attempt $attempt/$attempt_limit: sleeping 5 seconds before checking if $computer_short_name is not being used");
sleep_uninterrupted(5);
}
notify($ERRORS{'OK'}, 0, "attempt $attempt/$attempt_limit: checking if $computer_short_name is not being used");
my $computer_state_name = $self->data->get_computer_state_name();
# Return 0 if computer state is deleted, vmhostinuse
if ($computer_state_name =~ /^(deleted|vmhostinuse)$/) {
notify($ERRORS{'WARNING'}, 0, "$computer_short_name is NOT available, its state is $computer_state_name");
return 0;
}
# Return 0 if computer state is maintenance and request state name is not vmhostinuse
# Allow computers to go from maintenance directly to a vmhost
if ($computer_state_name =~ /^(maintenance)$/ && $request_state_name !~ /tovmhostinuse/) {
notify($ERRORS{'WARNING'}, 0, "$computer_short_name is NOT available, its state is $computer_state_name");
return 0;
}
notify($ERRORS{'DEBUG'}, 0, "$computer_short_name state is $computer_state_name, checking if any competing reservations are active");
# Check if there is another request using this machine
# Get a hash containing all of the reservations for the computer
my $competing_request_info = get_request_by_computerid($computer_id);
# There should be at least 1 request -- the one being processed
if (!$competing_request_info) {
notify($ERRORS{'WARNING'}, 0, "failed to retrieve any requests for computer $computer_id, there should probably be at least 1");
next ATTEMPT;
}
# Loop through the competing requests
COMPETING_REQUESTS: for my $competing_request_id (sort keys %$competing_request_info) {
# Ignore the request currently being processed
next if $competing_request_id == $request_id;
my $competing_reservation_id = $competing_request_info->{$competing_request_id}{data}->get_reservation_id();
my $competing_request_state = $competing_request_info->{$competing_request_id}{data}->get_request_state_name();
my $competing_request_laststate = $competing_request_info->{$competing_request_id}{data}->get_request_laststate_name();
my $competing_imagerevision_id = $competing_request_info->{$competing_request_id}{data}->get_imagerevision_id();
my $competing_request_start = $competing_request_info->{$competing_request_id}{data}->get_request_start_time();
my $competing_request_end = $competing_request_info->{$competing_request_id}{data}->get_request_end_time();
my $competing_request_start_epoch = convert_to_epoch_seconds($competing_request_start);
my $competing_request_end_epoch = convert_to_epoch_seconds($competing_request_end);
my $now_epoch = time;
my $competing_request_info_string;
$competing_request_info_string .= "request:reservation ID: $competing_request_id:$competing_reservation_id\n";
$competing_request_info_string .= "request state: $competing_request_state/$competing_request_laststate\n";
$competing_request_info_string .= "request start time: $competing_request_start\n";
$competing_request_info_string .= "request end time: $competing_request_end";
notify($ERRORS{'DEBUG'}, 0, "checking reservation assigned to $computer_short_name:\n$competing_request_info_string");
# Check for existing image creation requests
if ($competing_request_state =~ /^(image|checkpoint)$/ || $competing_request_laststate =~ /^(image|checkpoint)$/) {
notify($ERRORS{'WARNING'}, 0, "$computer_short_name is NOT available, it is assigned to an existing imaging reservation:\n$competing_request_info_string");
next ATTEMPT;
}
# Check for any requests in the maintenance state
if ($competing_request_state =~ /^(maintenance)$/) {
notify($ERRORS{'WARNING'}, 0, "$computer_short_name is NOT available, it is assigned to an existing request in the '$competing_request_state' state:\n$competing_request_info_string");
return 0;
}
# Ignore 'complete', 'failed' requests
if ($competing_request_state =~ /^(complete|failed)$/) {
notify($ERRORS{'DEBUG'}, 0, "ignoring request in state: $competing_request_state/$competing_request_laststate");
next COMPETING_REQUESTS;
}
# Check if the other reservation assigned to computer hasn't started yet
if ($competing_request_start_epoch > $now_epoch) {
# If they overlap, let the other reservation worry about it
notify($ERRORS{'OK'}, 0, "request $competing_request_id:$competing_reservation_id start time is in the future: $competing_request_start");
next COMPETING_REQUESTS;
}
# Check if the other reservation is a 'reload' reservation for the same image revision
if ($competing_imagerevision_id eq $imagerevision_id &&
$competing_request_state =~ /^(pending|reload)$/ &&
$competing_request_laststate =~ /(reload)/) {
notify($ERRORS{'OK'}, 0, "reservation $competing_reservation_id is assigned to $computer_short_name with the same image revision: $image_name, waiting for the other reload process to complete");
my $message = "waiting for reload reservation $competing_request_id:$competing_reservation_id to finish loading $computer_short_name with $image_name";
# Wait at least 5 minutes
$image_reloadtime = 5 if $image_reloadtime < 10;
my $total_wait_seconds = (60 * $image_reloadtime);
my $attempt_delay_seconds = 10;
# Loop until other process is done
if ($self->code_loop_timeout(sub{return !reservation_being_processed(@_)}, [$competing_reservation_id], $message, $total_wait_seconds, $attempt_delay_seconds)) {
# Check if reload request finished and was already deleted
if (is_request_deleted($competing_request_id)) {
notify($ERRORS{'OK'}, 0, "reload reservation $competing_request_id:$competing_reservation_id is no longer loading $computer_short_name with $image_name, request $competing_request_id has been deleted");
}
else {
# Verified competing 'reload' is not being processed verify it is not stuck in pending/reload
notify($ERRORS{'DEBUG'}, 0, "reload reservation $competing_request_id:$competing_reservation_id is not loading $computer_short_name with $image_name, checking current state of request $competing_request_id");
my ($current_competing_request_state, $current_competing_request_laststate) = get_request_current_state_name($competing_request_id);
if (!defined($current_competing_request_state)) {
if (is_request_deleted($competing_request_id)) {
notify($ERRORS{'OK'}, 0, "reload request $competing_request_id:$competing_reservation_id which was loading $computer_short_name with $image_name was just deleted");
}
}
elsif ($current_competing_request_state eq 'pending' && $current_competing_request_laststate eq 'reload') {
notify($ERRORS{'OK'}, 0, "state of competing reload request $competing_request_id:$competing_reservation_id is $current_competing_request_state/$current_competing_request_laststate, verified it is not being processed, changing state of competing request $competing_request_id to 'complete'");
update_request_state($competing_request_id, 'complete', 'reload');
}
}
# Try again in order to retrieve a current list of competing reservations
# The list of competing reservations may have changed while waiting
notify($ERRORS{'OK'}, 0, "making another attempt to retrieve the current list of competing reservations assigned to $computer_short_name");
# It's possible for this condition to be reached on the last attempt, check one more time
if ($attempt == 5) {
$attempt_limit++;
}
next ATTEMPT;
}
else {
notify($ERRORS{'WARNING'}, 0, "reload reservation $competing_reservation_id has NOT finished loading $computer_short_name with $image_name, waited $total_wait_seconds seconds");
}
}
# Check if the other reservation assigned to computer end time has been reached
# -or-
# Reload reservation -- either for a different image or the previous check loop monitoring the reload process for the same image timed out
if ($competing_request_end_epoch <= $now_epoch ||
($competing_request_state =~ /(timeout|deleted|reload)/) ||
($competing_request_state eq 'pending' && $competing_request_laststate =~ /(timeout|deleted|reload)/)) {
# Update the competing request state to complete
# If this fails, check if the competing request has already been deleted
# Do this before checking if the reservation is being processed to prevent new processes from being created
if (update_request_state($competing_request_id, "complete", ($competing_request_state eq 'pending') ? $competing_request_laststate : $competing_request_state)) {
notify($ERRORS{'OK'}, 0, "request state set to 'complete' for competing reservation $competing_reservation_id");
}
elsif (is_request_deleted($competing_request_id)) {
notify($ERRORS{'OK'}, 0, "request state not set to 'complete' for competing reservation $competing_reservation_id because request has been deleted");
}
else {
notify($ERRORS{'CRITICAL'}, 0, "computer $computer_short_name is NOT available, failed to set request state to 'complete', competing request has NOT been deleted:\n$competing_request_info_string");
return 0;
}
# Check if the other reservation is still being processed
if (my @competing_reservation_pids = reservation_being_processed($competing_reservation_id)) {
notify($ERRORS{'OK'}, 0, "reservation $competing_reservation_id is currently being processed by PID(s): " . join(', ', @competing_reservation_pids) . ", making sure the process doesn't have any Semaphore objects open before attempting to kill it");
# Check if the competing process owns any semaphores
# This would indicate it's doing something such as retrieving an image
# Don't kill it or a partial image may be copied
my $semaphore_info = get_vcld_semaphore_info();
for my $semaphore_identifier (keys %$semaphore_info) {
for my $competing_reservation_pid (@competing_reservation_pids) {
if ($semaphore_info->{$semaphore_identifier}{reservationid} == $competing_reservation_id && $semaphore_info->{$semaphore_identifier}{pid} == $competing_reservation_pid) {
notify($ERRORS{'CRITICAL'}, 0, "computer $computer_short_name is NOT available, reservation $competing_reservation_id is still being processed and owns a semaphore with identifier '$semaphore_identifier', not killing the competing process, it may be transferring an image:\n$competing_request_info_string, semaphore info:\n" . format_data($semaphore_info->{$semaphore_identifier}));
return;
}
}
}
# Kill competing process
notify($ERRORS{'OK'}, 0, "attempting to kill process of competing reservation $competing_reservation_id assigned to $computer_short_name");
for my $competing_reservation_pid (@competing_reservation_pids) {
$self->mn_os->kill_process($competing_reservation_pid);
}
# Wait for competing process to end before verifying that it was successfully killed
sleep_uninterrupted(2);
# Verify that the competing reservation process was killed
if (reservation_being_processed($competing_reservation_id)) {
notify($ERRORS{'WARNING'}, 0, "computer $computer_short_name is NOT available, failed to kill process for competing reservation, competing reservation is still being processed:\n$competing_request_info_string");
return 0;
}
}
sleep_uninterrupted(5);
# Try again in order to retrieve a current list of competing reservations
# The list of competing reservations may have changed
# A new reload reservation may have been added by timeout/deleted processes
notify($ERRORS{'OK'}, 0, "making another attempt to retrieve the current list of competing reservations assigned to $computer_short_name");
next ATTEMPT;
}
elsif (reservation_being_processed($competing_reservation_id)) {
notify($ERRORS{'WARNING'}, 0, "computer $computer_short_name is NOT available, assigned overlapping reservations, competing reservation is currently being processed:\n$competing_request_info_string");
next ATTEMPT;
}
else {
notify($ERRORS{'WARNING'}, 0, "computer $computer_short_name is NOT available, assigned overlapping reservations, competing reservation is NOT currently being processed:\n$competing_request_info_string");
next ATTEMPT;
}
}
# Checked all competing requests and didn't find any conflicting reservations
notify($ERRORS{'OK'}, 0, "$computer_short_name is available, did not find any conflicting reservations");
return 1;
}
notify($ERRORS{'WARNING'}, 0, "computer $computer_short_name is NOT available, made $attempt_limit attempts");
return 0;
}