in genie-agent/src/main/java/com/netflix/genie/agent/execution/services/impl/JobSetupServiceImpl.java [525:718]
String composeScript(final JobSetupServiceProperties jobSetupProperties) {
// Assemble the content of the script
final StringBuilder sb = new StringBuilder();
// Script Header Section
sb
.append("#!/usr/bin/env bash").append(NEWLINE)
.append(NEWLINE);
sb
.append("#").append(NEWLINE)
.append("# Generated by Genie for job: ").append(this.jobId).append(NEWLINE)
.append("#").append(NEWLINE)
.append(NEWLINE);
sb
.append("# Error out if any command fails").append(NEWLINE)
.append("set -o errexit").append(NEWLINE)
.append("# Error out if any command in a pipeline fails").append(NEWLINE)
.append("set -o pipefail").append(NEWLINE)
.append("# Error out if unknown variable is used").append(NEWLINE)
.append("set -o nounset").append(NEWLINE)
.append("# Save original stdout and stderr in fd 6 and 7").append(NEWLINE)
.append("exec 6>&1").append(NEWLINE)
.append("exec 7>&2").append(NEWLINE)
.append(NEWLINE);
sb.append(NEWLINE);
// Script Signal Handling Section
sb
.append("# Trap exit signals to ensure children processes are dead before returning").append(NEWLINE)
.append("function handle_kill_request {").append(NEWLINE)
.append(" echo \"Handling $1 signal\"").append(TO_STD_ERR).append(NEWLINE)
.append(" # Update trap").append(NEWLINE)
.append(" trap wait ").append(String.join(" ", TRAPPED_SIGNALS)).append(NEWLINE)
.append(" # Send SIGTERM to all children").append(NEWLINE)
.append(" pkill -P $$ || true").append(NEWLINE)
.append(" for ((iteration=1; iteration < 30; iteration++))").append(NEWLINE)
.append(" {").append(NEWLINE)
.append(" if pkill -0 -P $$ &> /dev/null;").append(NEWLINE)
.append(" then").append(NEWLINE)
.append(" echo \"Waiting for children to terminate\"").append(TO_STD_ERR).append(NEWLINE)
.append(" sleep 1").append(NEWLINE)
.append(" else").append(NEWLINE)
.append(" echo \"All children terminated\"").append(TO_STD_ERR).append(NEWLINE)
.append(" exit 1").append(NEWLINE)
.append(" fi").append(NEWLINE)
.append(" }").append(NEWLINE)
.append(" # Reaching this point means the children did not die. Kill with SIGKILL").append(NEWLINE)
.append(" echo \"Terminating all children with SIGKILL\"").append(TO_STD_ERR).append(NEWLINE)
.append(" pkill -9 -P $$").append(NEWLINE)
.append("}").append(NEWLINE);
for (final String signal : TRAPPED_SIGNALS) {
sb.append("trap 'handle_kill_request ").append(signal).append("' ").append(signal).append(NEWLINE);
}
sb.append(NEWLINE);
// Script Local environment Section
sb.append("# Locally-generated environment variables").append(NEWLINE);
sb.append(NEWLINE);
this.localEnvironmentVariables.forEach(
envVar -> sb
.append("export ")
.append(envVar.getKey())
.append("=\"")
.append(envVar.getValue())
.append("\"")
.append(NEWLINE)
.append(NEWLINE)
);
sb.append(NEWLINE);
sb
.append("# Mark the beginnig of the setup by creating a marker file").append(NEWLINE)
.append("echo \"The job script failed during setup. ")
.append("See ${").append(SETUP_LOG_ENV_VAR).append("} for details\" ")
.append("> ${").append(SETUP_ERROR_FILE_ENV_VAR).append("}")
.append(NEWLINE);
sb.append(NEWLINE);
sb
.append("# During setup, redirect stdout and stderr of this script to a log file").append(NEWLINE)
.append("exec > ${__GENIE_SETUP_LOG_FILE}").append(NEWLINE)
.append("exec 2>&1").append(NEWLINE);
sb.append(NEWLINE);
// Script Setup Section
sb
.append("echo \"Setup start: $(date '+%Y-%m-%d %H:%M:%S')\"")
.append(NEWLINE);
sb.append(NEWLINE);
sb.append("# Server-provided environment variables").append(NEWLINE);
sb.append(NEWLINE);
this.serverEnvironmentVariables.forEach(
envVar -> sb
.append("export ")
.append(envVar.getKey())
.append("=\"")
.append(envVar.getValue())
.append("\"")
.append(NEWLINE)
.append(NEWLINE)
);
sb.append(NEWLINE);
this.setupFileReferences.forEach(
pair -> {
final String resourceDescription = pair.getLeft();
final String setupFileReference = pair.getRight();
if (setupFileReference != null) {
sb
.append("echo \"Sourcing setup script for ")
.append(resourceDescription)
.append("\"")
.append(NEWLINE)
.append("source ")
.append(setupFileReference)
.append(NEWLINE);
} else {
sb
.append("echo \"No setup script for ")
.append(resourceDescription)
.append("\"")
.append(NEWLINE);
}
sb.append(NEWLINE);
}
);
sb.append(NEWLINE);
sb
.append("echo \"Setup end: $(date '+%Y-%m-%d %H:%M:%S')\"")
.append(NEWLINE);
sb.append(NEWLINE);
sb
.append("# Setup completed successfully, delete marker file").append(NEWLINE)
.append("rm ${").append(SETUP_ERROR_FILE_ENV_VAR).append("}").append(NEWLINE);
sb.append(NEWLINE);
sb
.append("# Restore the original stdout and stderr. Close fd 6 and 7").append(NEWLINE)
.append("exec 1>&6 6>&-").append(NEWLINE)
.append("exec 2>&7 7>&-").append(NEWLINE);
sb.append(NEWLINE);
// Dump environment for debugging
sb
.append("# Dump environment post-setup")
.append(NEWLINE)
.append("env | grep -E ")
.append(jobSetupProperties.isEnvironmentDumpFilterInverted() ? GREP_INVERT_MATCH : "")
.append("--regex='").append(jobSetupProperties.getEnvironmentDumpFilterExpression()).append("'")
.append(" | sort > ")
.append("${").append(ENVIRONMENT_LOG_ENV_VAR).append("}")
.append(NEWLINE);
sb.append(NEWLINE);
// Script Executable Section
// N.B. Command *must* be last line of the script for the exit code to be propagated back correctly!
sb
.append("# Launch the command")
.append(NEWLINE)
.append(this.commandLine).append(" <&0 &").append(NEWLINE)
.append("wait %1").append(NEWLINE)
.append("exit $?").append(NEWLINE)
.append(NEWLINE);
return sb.toString();
}