sub wait_for_child_reservations()

in managementnode/lib/VCL/new.pm [934:1067]
105 lines of code
15 McCabe index (conditional complexity)

sub wait_for_child_reservations {
	my $self              = shift;
	
	my $request_id        = $self->data->get_request_id();
	my @reservation_ids   = $self->data->get_reservation_ids();
	my $reservation_count = $self->data->get_reservation_count();
	
	# Set limits on how long to wait
	my $overall_timeout_minutes = 60;
	my $nochange_timeout_minutes = 30;
	my $monitor_delay_seconds = 30;
	
	my $overall_timeout_seconds = ($overall_timeout_minutes * 60);
	my $nochange_timeout_seconds = ($nochange_timeout_minutes * 60);
	
	my $monitor_start_time = time;
	my $last_change_time = $monitor_start_time;
	my $current_time;
	my $nochange_timeout_time = ($monitor_start_time + $nochange_timeout_seconds);
	my $overall_timeout_time = ($monitor_start_time + $overall_timeout_seconds);
	
	my $previous_lastcheck_info;
	my $previous_request_loadstate_names;
	
	MONITOR_LOADING: while (($current_time = time) < $nochange_timeout_time && $current_time < $overall_timeout_time) {
		my $total_elapsed_seconds      = ($current_time - $monitor_start_time);
		my $nochange_elapsed_seconds   = ($current_time - $last_change_time);
		my $nochange_remaining_seconds = ($nochange_timeout_time - $current_time);
		my $overall_remaining_seconds  = ($overall_timeout_time - $current_time);
		
		notify($ERRORS{'DEBUG'}, 0, "waiting for child reservations seconds elapsed/until no change timeout: $nochange_elapsed_seconds/$nochange_remaining_seconds, unconditional timeout: $total_elapsed_seconds/$overall_remaining_seconds");
		
		# Check if request has been deleted
		if (is_request_deleted($request_id)) {
			notify($ERRORS{'OK'}, 0, "request has been deleted, setting computer state to 'available' and exiting");
			$self->state_exit('', 'available');
		}
		
		my @reservations_ready;
		my @reservations_not_ready;
		my @reservations_failed;
		my @reservations_lastcheck_changed;
		my @reservations_loadstate_changed;
		my @reservations_unknown;
		
		# Get reservation.lastcheck value for all reservations
		my $current_lastcheck_info = get_current_reservation_lastcheck(@reservation_ids);
		$previous_lastcheck_info = $current_lastcheck_info if !$previous_lastcheck_info;
		
		# Get computerloadlog info for all reservations
		my $current_request_loadstate_names = get_request_loadstate_names($request_id);
		$previous_request_loadstate_names = $current_request_loadstate_names if !$previous_request_loadstate_names;
		
		RESERVATION_ID: for my $reservation_id (@reservation_ids) {
			if (!defined($current_request_loadstate_names->{$reservation_id})) {
				notify($ERRORS{'WARNING'}, 0, "request loadstate info does not contain a key for reservation $reservation_id:\n" . format_data($current_request_loadstate_names));
				next RESERVATION_ID;
			}
			
			my @previous_reservation_loadstate_names = @{$previous_request_loadstate_names->{$reservation_id}};
			my @current_reservation_loadstate_names = @{$current_request_loadstate_names->{$reservation_id}};
			
			if (grep {$_ eq 'failed'} @current_reservation_loadstate_names) {
				push @reservations_failed, $reservation_id;
				next RESERVATION_ID;
			}
			elsif (grep {$_ eq 'nodeready'} @current_reservation_loadstate_names) {
				push @reservations_ready, $reservation_id;
				next RESERVATION_ID;
			}
			elsif (grep {$_ eq 'begin'} @current_reservation_loadstate_names) {
				push @reservations_not_ready, $reservation_id;
			}
			else {
				push @reservations_unknown, $reservation_id;
			}
			
			if ($previous_lastcheck_info->{$reservation_id} ne $current_lastcheck_info->{$reservation_id}) {
				push @reservations_lastcheck_changed, $reservation_id;
			}
			if (scalar(@previous_reservation_loadstate_names) != scalar(@current_reservation_loadstate_names)) {
				push @reservations_loadstate_changed, $reservation_id;
			}
		}
		
		my $ready_count     = scalar @reservations_ready;
		my $not_ready_count = scalar @reservations_not_ready;
		my $unknown_count   = scalar @reservations_unknown;
		my $failed_count    = scalar @reservations_failed;
		
		notify($ERRORS{'DEBUG'}, 0, "current status of reservations:\n" .
			"ready     : $ready_count (" . join(', ', @reservations_ready) . ")\n" .
			"not ready : $not_ready_count (" . join(', ', @reservations_not_ready) . ")\n" .
			"unknown   : $unknown_count (" . join(', ', @reservations_unknown) . ")\n" .
			"failed    : $failed_count (" . join(', ', @reservations_failed) . ")"
		);
		
		if ($failed_count) {
			$self->state_exit('failed', 'available');
		}
		elsif ($ready_count == $reservation_count) {
			notify($ERRORS{'OK'}, 0, "all reservations are ready");
			return 1;
		}
		
		
		# If any changes were detected, reset the nochange timeout
		if (@reservations_lastcheck_changed || @reservations_loadstate_changed) {
			notify($ERRORS{'DEBUG'}, 0, "resetting no change timeout, detected reservation change:\n" .
				"reservation lastcheck changed: (" . join(', ', @reservations_lastcheck_changed) . ")\n" .
				"reservation loadstate changed: (" . join(', ', @reservations_loadstate_changed) . ")"
			);
			$last_change_time = $current_time;
			$nochange_timeout_time = ($last_change_time + $nochange_timeout_seconds);
		}
		
		$previous_request_loadstate_names = $current_request_loadstate_names;
		$previous_lastcheck_info = $current_lastcheck_info;
		
		if ($total_elapsed_seconds <= 30) {
			sleep_uninterrupted(3);
		}
		elsif ($total_elapsed_seconds <= 60) {
			sleep_uninterrupted(5);
		}
		else {
			sleep_uninterrupted($monitor_delay_seconds);
		}
	}
	
	# If out of main loop, waited maximum amount of time
	notify($ERRORS{'WARNING'}, 0, "waited maximum amount of time for all reservations to become ready");
	return;
} ## end sub wait_for_child_reservations