in managementnode/lib/VCL/Module/OS.pm [3312:3587]
sub execute_new {
my ($argument) = @_;
my ($computer_name, $command, $display_output, $timeout_seconds, $max_attempts, $port, $user, $password, $identity_key, $ignore_error);
my $self;
# Check if this subroutine was called as an object method
if (ref($argument) && ref($argument) =~ /VCL::Module/) {
# Subroutine was called as an object method ($self->execute)
$self = shift;
($argument) = @_;
#notify($ERRORS{'DEBUG'}, 0, "called as an object method: " . ref($self));
# Get the computer name from the reservation data
$computer_name = $self->data->get_computer_node_name();
if (!$computer_name) {
notify($ERRORS{'WARNING'}, 0, "called as an object method, failed to retrieve computer name from reservation data");
return;
}
#notify($ERRORS{'DEBUG'}, 0, "retrieved computer name from reservation data: $computer_name");
}
# Check the argument type
if (ref($argument)) {
if (ref($argument) eq 'HASH') {
#notify($ERRORS{'DEBUG'}, 0, "first argument is a hash reference:\n" . format_data($argument));
$computer_name = $argument->{node} if (!$computer_name);
$command = $argument->{command};
$display_output = $argument->{display_output};
$timeout_seconds = $argument->{timeout_seconds};
$max_attempts = $argument->{max_attempts};
$port = $argument->{port};
$user = $argument->{user};
$password = $argument->{password};
$identity_key = $argument->{identity_key};
$ignore_error = $argument->{ignore_error};
}
else {
notify($ERRORS{'WARNING'}, 0, "invalid argument reference type passed: " . ref($argument) . ", if a reference is passed as the argument it may only be a hash or VCL::Module reference");
return;
}
}
else {
# Argument is not a reference, computer name must be the first argument unless this subroutine was called as an object method
# If called as an object method, $computer_name will already be populated
if (!$computer_name) {
$computer_name = shift;
#notify($ERRORS{'DEBUG'}, 0, "first argument is a scalar, should be the computer name: $computer_name, remaining arguments:\n" . format_data(\@_));
}
else {
#notify($ERRORS{'DEBUG'}, 0, "first argument should be the command:\n" . format_data(\@_));
}
# Get the remaining arguments
($command, $display_output, $timeout_seconds, $max_attempts, $port, $user, $password, $identity_key, $ignore_error) = @_;
}
if (!$computer_name) {
notify($ERRORS{'WARNING'}, 0, "computer name could not be determined");
return;
}
if (!$command) {
notify($ERRORS{'WARNING'}, 0, "command argument was not specified");
return;
}
# Determine which string to use as the connection target
my $remote_connection_target = determine_remote_connection_target($computer_name);
my $computer_string = $computer_name;
$computer_string .= " ($remote_connection_target)" if ($remote_connection_target ne $computer_name);
$display_output = 0 unless $display_output;
$timeout_seconds = 60 unless $timeout_seconds;
$max_attempts = 3 unless $max_attempts;
# If 'ssh_user' key is set in this object, use it
# This allows OS modules to specify the username to use
if ($self && $self->{ssh_user}) {
#notify($ERRORS{'DEBUG'}, 0, "\$self->{ssh_user} is defined: $self->{ssh_user}");
$user = $self->{ssh_user};
}
elsif (!$port) {
$user = 'root';
}
# If 'ssh_port' key is set in this object, use it
# This allows OS modules to specify the port to use
if ($self && $self->{ssh_port}) {
#notify($ERRORS{'DEBUG'}, 0, "\$self->{ssh_port} is defined: $self->{ssh_port}");
$port = $self->{ssh_port};
}
elsif (!$port) {
$port = 22;
}
my $ssh_options = '-o StrictHostKeyChecking=no -o ConnectTimeout=30 -x';
# Figure out which identity key to use
# If identity key argument was supplied, it may be a single path or a comma-separated list
# If argument was not supplied, get the default management node paths
my @identity_key_paths;
if ($identity_key) {
@identity_key_paths = split(/\s*[,;]\s*/, $identity_key);
}
else {
@identity_key_paths = VCL::DataStructure::get_management_node_identity_key_paths();
}
for my $identity_key_path (@identity_key_paths) {
$ssh_options .= " -i $identity_key_path";
}
# Override the die handler
local $SIG{__DIE__} = sub{};
my $ssh;
my $attempt = 0;
my $attempt_delay = 5;
my $attempt_string = '';
ATTEMPT: while ($attempt < $max_attempts) {
if ($attempt > 0) {
$attempt_string = "attempt $attempt/$max_attempts: ";
$ssh->close() if $ssh;
delete $ENV->{net_ssh_expect}->{$remote_connection_target};
notify($ERRORS{'DEBUG'}, 0, $attempt_string . "sleeping for $attempt_delay seconds before making next attempt");
sleep $attempt_delay;
}
$attempt++;
$attempt_string = "attempt $attempt/$max_attempts: " if ($attempt > 1);
# Calling 'return' in the EVAL block doesn't exit this subroutine
# Use a flag to determine if null should be returned without making another attempt
my $return_null;
if (!$ENV->{net_ssh_expect}->{$remote_connection_target}) {
eval {
my $expect_options = {
host => $remote_connection_target,
user => $user,
port => $port,
raw_pty => 1,
no_terminal => 1,
ssh_option => $ssh_options,
#timeout => 5,
};
$ssh = Net::SSH::Expect->new(%$expect_options);
if ($ssh) {
notify($ERRORS{'DEBUG'}, 0, "created " . ref($ssh) . " object to control $computer_string, options:\n" . format_data($expect_options));
}
else {
notify($ERRORS{'WARNING'}, 0, "failed to create Net::SSH::Expect object to control $computer_string, $!, options:\n" . format_data($expect_options));
next ATTEMPT;
}
if (!$ssh->run_ssh()) {
notify($ERRORS{'WARNING'}, 0, ref($ssh) . " object failed to fork SSH process to control $computer_string, $!, options:\n" . format_data($expect_options));
next ATTEMPT;
}
#sleep_uninterrupted(1);
#$ssh->exec("stty -echo");
#$ssh->exec("stty raw -echo");
# Set the timeout counter behaviour:
# If true, sets the timeout to "inactivity timeout"
# If false sets it to "absolute timeout"
$ssh->restart_timeout_upon_receive(1);
my $initialization_output = $ssh->read_all();
if (defined($initialization_output)) {
notify($ERRORS{'DEBUG'}, 0, "SSH initialization output:\n$initialization_output") if ($display_output);
if ($initialization_output =~ /password:/i) {
if (defined($password)) {
notify($ERRORS{'WARNING'}, 0, "$attempt_string unable to connect to $computer_string, SSH is requesting a password but password authentication is not implemented, password is configured, output:\n$initialization_output");
# In EVAL block here, 'return' won't return from entire subroutine, set flag
$return_null = 1;
return;
}
else {
notify($ERRORS{'WARNING'}, 0, "$attempt_string unable to connect to $computer_string, SSH is requesting a password but password authentication is not implemented, password is not configured, output:\n$initialization_output");
$return_null = 1;
return;
}
}
}
else {
notify($ERRORS{'DEBUG'}, 0, $attempt_string . "SSH initialization output is undefined") if ($display_output);
}
};
return if ($return_null);
if ($EVAL_ERROR) {
if ($EVAL_ERROR =~ /^(\w+) at \//) {
notify($ERRORS{'DEBUG'}, 0, $attempt_string . "$1 error occurred initializing Net::SSH::Expect object for $computer_string") if ($display_output);
}
else {
notify($ERRORS{'DEBUG'}, 0, $attempt_string . "$EVAL_ERROR error occurred initializing Net::SSH::Expect object for $computer_string") if ($display_output);
}
next ATTEMPT;
}
}
else {
$ssh = $ENV->{net_ssh_expect}->{$remote_connection_target};
# Delete the stored SSH object to make sure it isn't saved if the command fails
# The SSH object will be added back to %ENV if the command completes successfully
delete $ENV->{net_ssh_expect}->{$remote_connection_target};
}
# Set the timeout
$ssh->timeout($timeout_seconds);
(my $command_formatted = $command) =~ s/\s+(;|&|&&)\s+/\n$1 /g;
notify($ERRORS{'DEBUG'}, 0, $attempt_string . "executing command on $computer_string (timeout: $timeout_seconds seconds):\n$command_formatted") if ($display_output);
my $command_start_time = time;
$ssh->send($command . ' 2>&1 ; echo exitstatus:$?');
my $ssh_wait_status;
eval {
$ssh_wait_status = $ssh->waitfor('exitstatus:[0-9]+', $timeout_seconds);
};
if ($EVAL_ERROR) {
if ($ignore_error) {
notify($ERRORS{'DEBUG'}, 0, "executed command on $computer_string: '$command', ignoring error, returning null") if ($display_output);
return;
}
elsif ($EVAL_ERROR =~ /^(\w+) at \//) {
notify($ERRORS{'WARNING'}, 0, $attempt_string . "$1 error occurred executing command on $computer_string: '$command'") if ($display_output);
}
else {
notify($ERRORS{'WARNING'}, 0, $attempt_string . "error occurred executing command on $computer_string: '$command'\nerror: $EVAL_ERROR") if ($display_output);
}
next ATTEMPT;
}
elsif (!$ssh_wait_status) {
notify($ERRORS{'WARNING'}, 0, $attempt_string . "command timed out after $timeout_seconds seconds on $computer_string: '$command'") if ($display_output);
next ATTEMPT;
}
# Need to fix this:
#2012-09-25 16:15:57|executing command on blade1a3-2 (timeout: 7200 seconds):
#2012-09-25 16:16:24|23464|1915857:2002452|image|OS.pm:execute_new(2243)|error
#SSHConnectionError Reading error type 4 found: 4:Interrupted system call at /usr/local/vcl/bin/../lib/VCL/Module/OS.pm line 2231
my $output = $ssh->before() || '';
$output =~ s/(^\s+)|(\s+$)//g;
my $exit_status_string = $ssh->match() || '';
my ($exit_status) = $exit_status_string =~ /(\d+)/;
if (!$exit_status_string || !defined($exit_status)) {
my $all_output = $ssh->read_all() || '';
notify($ERRORS{'WARNING'}, 0, $attempt_string . "failed to determine exit status from string: '$exit_status_string', output:\n$all_output");
next ATTEMPT;
}
my @output_lines = split(/\n/, $output);
map { s/[\r]+//g; } (@output_lines);
notify($ERRORS{'OK'}, 0, "executed command on $computer_string: '$command', exit status: $exit_status, output:\n$output") if ($display_output);
# Save the SSH object for later use
$ENV->{net_ssh_expect}->{$remote_connection_target} = $ssh;
return ($exit_status, \@output_lines);
}
notify($ERRORS{'WARNING'}, 0, $attempt_string . "failed to execute command on $computer_string: '$command'") if ($display_output);
return;
}