in brooklyn-server/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/JcloudsLocation.java [633:1057]
protected MachineLocation obtainOnce(ConfigBag setup) throws NoMachinesAvailableException {
AccessController.Response access = getManagementContext().getAccessController().canProvisionLocation(this);
if (!access.isAllowed()) {
throw new IllegalStateException("Access controller forbids provisioning in "+this+": "+access.getMsg());
}
setCreationString(setup);
boolean waitForSshable = !"false".equalsIgnoreCase(setup.get(WAIT_FOR_SSHABLE));
boolean waitForWinRmable = !"false".equalsIgnoreCase(setup.get(WAIT_FOR_WINRM_AVAILABLE));
boolean usePortForwarding = setup.get(USE_PORT_FORWARDING);
boolean skipJcloudsSshing = Boolean.FALSE.equals(setup.get(USE_JCLOUDS_SSH_INIT)) || usePortForwarding;
JcloudsPortForwarderExtension portForwarder = setup.get(PORT_FORWARDER);
if (usePortForwarding) checkNotNull(portForwarder, "portForwarder, when use-port-forwarding enabled");
final ComputeService computeService = getConfig(COMPUTE_SERVICE_REGISTRY).findComputeService(setup, true);
CloudMachineNamer cloudMachineNamer = getCloudMachineNamer(setup);
String groupId = elvis(setup.get(GROUP_ID), cloudMachineNamer.generateNewGroupId(setup));
NodeMetadata node = null;
JcloudsMachineLocation machineLocation = null;
Duration semaphoreTimestamp = null;
Duration templateTimestamp = null;
Duration provisionTimestamp = null;
Duration usableTimestamp = null;
Duration customizedTimestamp = null;
Stopwatch provisioningStopwatch = Stopwatch.createStarted();
try {
LOG.info("Creating VM "+setup.getDescription()+" in "+this);
Semaphore machineCreationSemaphore = getMachineCreationSemaphore();
boolean acquired = machineCreationSemaphore.tryAcquire(0, TimeUnit.SECONDS);
if (!acquired) {
LOG.info("Waiting in {} for machine-creation permit ({} other queuing requests already)", new Object[] {this, machineCreationSemaphore.getQueueLength()});
Stopwatch blockStopwatch = Stopwatch.createStarted();
machineCreationSemaphore.acquire();
LOG.info("Acquired in {} machine-creation permit, after waiting {}", this, Time.makeTimeStringRounded(blockStopwatch));
} else {
LOG.debug("Acquired in {} machine-creation permit immediately", this);
}
semaphoreTimestamp = Duration.of(provisioningStopwatch);
LoginCredentials userCredentials = null;
Set<? extends NodeMetadata> nodes;
Template template;
try {
// Setup the template
template = buildTemplate(computeService, setup);
boolean expectWindows = isWindows(template, setup);
if (!skipJcloudsSshing) {
if (expectWindows) {
// TODO Was this too early to look at template.getImage? e.g. customizeTemplate could subsequently modify it.
LOG.warn("Ignoring invalid configuration for Windows provisioning of "+template.getImage()+": "+USE_JCLOUDS_SSH_INIT.getName()+" should be false");
skipJcloudsSshing = true;
} else if (waitForSshable) {
userCredentials = initTemplateForCreateUser(template, setup);
}
}
templateTimestamp = Duration.of(provisioningStopwatch);
// "Name" metadata seems to set the display name; at least in AWS
// TODO it would be nice if this salt comes from the location's ID (but we don't know that yet as the ssh machine location isn't created yet)
// TODO in softlayer we want to control the suffix of the hostname which is 3 random hex digits
template.getOptions().getUserMetadata().put("Name", cloudMachineNamer.generateNewMachineUniqueNameFromGroupId(setup, groupId));
if (setup.get(JcloudsLocationConfig.INCLUDE_BROOKLYN_USER_METADATA)) {
template.getOptions().getUserMetadata().put("brooklyn-user", System.getProperty("user.name"));
Object context = setup.get(CALLER_CONTEXT);
if (context instanceof Entity) {
Entity entity = (Entity)context;
template.getOptions().getUserMetadata().put("brooklyn-app-id", entity.getApplicationId());
template.getOptions().getUserMetadata().put("brooklyn-app-name", entity.getApplication().getDisplayName());
template.getOptions().getUserMetadata().put("brooklyn-entity-id", entity.getId());
template.getOptions().getUserMetadata().put("brooklyn-entity-name", entity.getDisplayName());
template.getOptions().getUserMetadata().put("brooklyn-server-creation-date", Time.makeDateSimpleStampString());
}
}
customizeTemplate(setup, computeService, template);
LOG.debug("jclouds using template {} / options {} to provision machine in {}",
new Object[] {template, template.getOptions(), setup.getDescription()});
if (!setup.getUnusedConfig().isEmpty())
if (LOG.isDebugEnabled())
LOG.debug("NOTE: unused flags passed to obtain VM in "+setup.getDescription()+": "
+ Sanitizer.sanitize(setup.getUnusedConfig()));
nodes = computeService.createNodesInGroup(groupId, 1, template);
provisionTimestamp = Duration.of(provisioningStopwatch);
} finally {
machineCreationSemaphore.release();
}
node = Iterables.getOnlyElement(nodes, null);
LOG.debug("jclouds created {} for {}", node, setup.getDescription());
if (node == null)
throw new IllegalStateException("No nodes returned by jclouds create-nodes in " + setup.getDescription());
boolean windows = isWindows(node, setup);
if (windows) {
int newLoginPort = node.getLoginPort() == 22 ? 5985 : node.getLoginPort();
String newLoginUser = "root".equals(node.getCredentials().getUser()) ? "Administrator" : node.getCredentials().getUser();
LOG.debug("jclouds created Windows VM {}; transforming connection details: loginPort from {} to {}; loginUser from {} to {}",
new Object[] {node, node.getLoginPort(), newLoginPort, node.getCredentials().getUser(), newLoginUser});
node = NodeMetadataBuilder.fromNodeMetadata(node)
.loginPort(newLoginPort)
.credentials(LoginCredentials.builder(node.getCredentials()).user(newLoginUser).build())
.build();
}
// FIXME How do we influence the node.getLoginPort, so it is set correctly for Windows?
// Setup port-forwarding, if required
Optional<HostAndPort> sshHostAndPortOverride;
if (usePortForwarding) {
sshHostAndPortOverride = Optional.of(portForwarder.openPortForwarding(
node,
node.getLoginPort(),
Optional.<Integer>absent(),
Protocol.TCP,
Cidr.UNIVERSAL));
} else {
sshHostAndPortOverride = Optional.absent();
}
LoginCredentials initialCredentials = node.getCredentials();
if (skipJcloudsSshing) {
boolean waitForConnectable = (windows) ? waitForWinRmable : waitForSshable;
if (waitForConnectable) {
if (windows) {
// TODO Does jclouds support any windows user setup?
initialCredentials = waitForWinRmAvailable(computeService, node, sshHostAndPortOverride, setup);
} else {
initialCredentials = waitForSshable(computeService, node, sshHostAndPortOverride, setup);
}
userCredentials = createUser(computeService, node, sshHostAndPortOverride, initialCredentials, setup);
}
}
// Figure out which login-credentials to use
LoginCredentials customCredentials = setup.get(CUSTOM_CREDENTIALS);
if (customCredentials != null) {
userCredentials = customCredentials;
//set userName and other data, from these credentials
Object oldUsername = setup.put(USER, customCredentials.getUser());
LOG.debug("node {} username {} / {} (customCredentials)", new Object[] { node, customCredentials.getUser(), oldUsername });
if (customCredentials.getOptionalPassword().isPresent()) setup.put(PASSWORD, customCredentials.getOptionalPassword().get());
if (customCredentials.getOptionalPrivateKey().isPresent()) setup.put(PRIVATE_KEY_DATA, customCredentials.getOptionalPrivateKey().get());
}
if (userCredentials == null || (!userCredentials.getOptionalPassword().isPresent() && !userCredentials.getOptionalPrivateKey().isPresent())) {
// We either don't have any userCredentials, or it is missing both a password/key.
// TODO See waitForSshable, which now handles if the node.getLoginCredentials has both a password+key
userCredentials = extractVmCredentials(setup, node, initialCredentials);
}
if (userCredentials == null) {
// TODO See waitForSshable, which now handles if the node.getLoginCredentials has both a password+key
userCredentials = extractVmCredentials(setup, node, initialCredentials);
}
if (userCredentials != null) {
node = NodeMetadataBuilder.fromNodeMetadata(node).credentials(userCredentials).build();
} else {
// only happens if something broke above...
userCredentials = LoginCredentials.fromCredentials(node.getCredentials());
}
// store the credentials, in case they have changed
setup.putIfNotNull(JcloudsLocationConfig.PASSWORD, userCredentials.getOptionalPassword().orNull());
setup.putIfNotNull(JcloudsLocationConfig.PRIVATE_KEY_DATA, userCredentials.getOptionalPrivateKey().orNull());
// Wait for the VM to be reachable over SSH
if (waitForSshable && !windows) {
waitForSshable(computeService, node, sshHostAndPortOverride, ImmutableList.of(userCredentials), setup);
} else {
LOG.debug("Skipping ssh check for {} ({}) due to config waitForSshable=false", node, setup.getDescription());
}
usableTimestamp = Duration.of(provisioningStopwatch);
// JcloudsSshMachineLocation jcloudsSshMachineLocation = null;
// WinRmMachineLocation winRmMachineLocation = null;
// Create a JcloudsSshMachineLocation, and register it
if (windows) {
machineLocation = registerWinRmMachineLocation(computeService, node, userCredentials, sshHostAndPortOverride, setup);
} else {
machineLocation = registerJcloudsSshMachineLocation(computeService, node, Optional.fromNullable(template), userCredentials, sshHostAndPortOverride, setup);
}
if (usePortForwarding && sshHostAndPortOverride.isPresent()) {
// Now that we have the sshMachineLocation, we can associate the port-forwarding address with it.
PortForwardManager portForwardManager = setup.get(PORT_FORWARDING_MANAGER);
if (portForwardManager != null) {
portForwardManager.associate(node.getId(), sshHostAndPortOverride.get(), machineLocation, node.getLoginPort());
} else {
LOG.warn("No port-forward manager for {} so could not associate {} -> {} for {}",
new Object[] {this, node.getLoginPort(), sshHostAndPortOverride, machineLocation});
}
}
if ("docker".equals(this.getProvider())) {
if (windows) {
throw new UnsupportedOperationException("Docker not supported on Windows");
}
Map<Integer, Integer> portMappings = JcloudsUtil.dockerPortMappingsFor(this, node.getId());
PortForwardManager portForwardManager = setup.get(PORT_FORWARDING_MANAGER);
if (portForwardManager != null) {
for(Integer containerPort : portMappings.keySet()) {
Integer hostPort = portMappings.get(containerPort);
String dockerHost = ((JcloudsSshMachineLocation)machineLocation).getSshHostAndPort().getHostText();
portForwardManager.associate(node.getId(), HostAndPort.fromParts(dockerHost, hostPort), machineLocation, containerPort);
}
} else {
LOG.warn("No port-forward manager for {} so could not associate docker port-mappings for {}",
this, machineLocation);
}
}
List<String> customisationForLogging = new ArrayList<String>();
// Apply same securityGroups rules to iptables, if iptables is running on the node
if (waitForSshable) {
String setupScript = setup.get(JcloudsLocationConfig.CUSTOM_MACHINE_SETUP_SCRIPT_URL);
List<String> setupScripts = setup.get(JcloudsLocationConfig.CUSTOM_MACHINE_SETUP_SCRIPT_URL_LIST);
Collection<String> allScripts = new MutableList<String>().appendIfNotNull(setupScript).appendAll(setupScripts);
for (String setupScriptItem : allScripts) {
if (Strings.isNonBlank(setupScriptItem)) {
customisationForLogging.add("custom setup script " + setupScriptItem);
String setupVarsString = setup.get(JcloudsLocationConfig.CUSTOM_MACHINE_SETUP_SCRIPT_VARS);
Map<String, String> substitutions = (setupVarsString != null)
? Splitter.on(",").withKeyValueSeparator(":").split(setupVarsString)
: ImmutableMap.<String, String>of();
String scriptContent = ResourceUtils.create(this).getResourceAsString(setupScriptItem);
String script = TemplateProcessor.processTemplateContents(scriptContent, getManagementContext(), substitutions);
if (windows) {
((WinRmMachineLocation)machineLocation).executeCommand(ImmutableList.copyOf((script.replace("\r", "").split("\n"))));
} else {
((SshMachineLocation)machineLocation).execCommands("Customizing node " + this, ImmutableList.of(script));
}
}
}
if (setup.get(JcloudsLocationConfig.MAP_DEV_RANDOM_TO_DEV_URANDOM)) {
if (windows) {
LOG.warn("Ignoring flag MAP_DEV_RANDOM_TO_DEV_URANDOM on Windows location {}", machineLocation);
} else {
customisationForLogging.add("point /dev/random to urandom");
((SshMachineLocation)machineLocation).execCommands("using urandom instead of random",
Arrays.asList("sudo mv /dev/random /dev/random-real", "sudo ln -s /dev/urandom /dev/random"));
}
}
if (setup.get(GENERATE_HOSTNAME)) {
if (windows) {
// TODO: Generate Windows Hostname
LOG.warn("Ignoring flag GENERATE_HOSTNAME on Windows location {}", machineLocation);
} else {
customisationForLogging.add("configure hostname");
((SshMachineLocation)machineLocation).execCommands("Generate hostname " + node.getName(),
Arrays.asList("sudo hostname " + node.getName(),
"sudo sed -i \"s/HOSTNAME=.*/HOSTNAME=" + node.getName() + "/g\" /etc/sysconfig/network",
"sudo bash -c \"echo 127.0.0.1 `hostname` >> /etc/hosts\"")
);
}
}
if (setup.get(OPEN_IPTABLES)) {
if (windows) {
LOG.warn("Ignoring DEPRECATED flag OPEN_IPTABLES on Windows location {}", machineLocation);
} else {
LOG.warn("Using DEPRECATED flag OPEN_IPTABLES (will not be supported in future versions) for {} at {}", machineLocation, this);
@SuppressWarnings("unchecked")
Iterable<Integer> inboundPorts = (Iterable<Integer>) setup.get(INBOUND_PORTS);
if (inboundPorts == null || Iterables.isEmpty(inboundPorts)) {
LOG.info("No ports to open in iptables (no inbound ports) for {} at {}", machineLocation, this);
} else {
customisationForLogging.add("open iptables");
List<String> iptablesRules = Lists.newArrayList();
if (isLocationFirewalldEnabled((SshMachineLocation)machineLocation)) {
for (Integer port : inboundPorts) {
iptablesRules.add(IptablesCommands.addFirewalldRule(Chain.INPUT, Protocol.TCP, port, Policy.ACCEPT));
}
} else {
iptablesRules = createIptablesRulesForNetworkInterface(inboundPorts);
iptablesRules.add(IptablesCommands.saveIptablesRules());
}
List<String> batch = Lists.newArrayList();
// Some entities, such as Riak (erlang based) have a huge range of ports, which leads to a script that
// is too large to run (fails with a broken pipe). Batch the rules into batches of 50
for (String rule : iptablesRules) {
batch.add(rule);
if (batch.size() == 50) {
((SshMachineLocation)machineLocation).execCommands("Inserting iptables rules, 50 command batch", batch);
batch.clear();
}
}
if (batch.size() > 0) {
((SshMachineLocation)machineLocation).execCommands("Inserting iptables rules", batch);
}
((SshMachineLocation)machineLocation).execCommands("List iptables rules", ImmutableList.of(IptablesCommands.listIptablesRule()));
}
}
}
if (setup.get(STOP_IPTABLES)) {
if (windows) {
LOG.warn("Ignoring DEPRECATED flag OPEN_IPTABLES on Windows location {}", machineLocation);
} else {
LOG.warn("Using DEPRECATED flag STOP_IPTABLES (will not be supported in future versions) for {} at {}", machineLocation, this);
customisationForLogging.add("stop iptables");
List<String> cmds = ImmutableList.<String>of();
if (isLocationFirewalldEnabled((SshMachineLocation)machineLocation)) {
cmds = ImmutableList.of(IptablesCommands.firewalldServiceStop(), IptablesCommands.firewalldServiceStatus());
} else {
cmds = ImmutableList.of(IptablesCommands.iptablesServiceStop(), IptablesCommands.iptablesServiceStatus());
}
((SshMachineLocation)machineLocation).execCommands("Stopping iptables", cmds);
}
}
List<String> extraKeyUrlsToAuth = setup.get(EXTRA_PUBLIC_KEY_URLS_TO_AUTH);
if (extraKeyUrlsToAuth!=null && !extraKeyUrlsToAuth.isEmpty()) {
if (windows) {
LOG.warn("Ignoring flag EXTRA_PUBLIC_KEY_URLS_TO_AUTH on Windows location", machineLocation);
} else {
List<String> extraKeyDataToAuth = MutableList.of();
for (String keyUrl : extraKeyUrlsToAuth) {
extraKeyDataToAuth.add(ResourceUtils.create().getResourceAsString(keyUrl));
}
((SshMachineLocation)machineLocation).execCommands("Authorizing ssh keys",
ImmutableList.of(new AuthorizeRSAPublicKeys(extraKeyDataToAuth).render(org.jclouds.scriptbuilder.domain.OsFamily.UNIX)));
}
}
} else {
// Otherwise we have deliberately not waited to be ssh'able, so don't try now to
// ssh to exec these commands!
}
// Apply any optional app-specific customization.
for (JcloudsLocationCustomizer customizer : getCustomizers(setup)) {
LOG.debug("Customizing machine {}, using customizer {}", machineLocation, customizer);
customizer.customize(this, computeService, machineLocation);
}
for (MachineLocationCustomizer customizer : getMachineCustomizers(setup)) {
LOG.debug("Customizing machine {}, using customizer {}", machineLocation, customizer);
customizer.customize(machineLocation);
}
customizedTimestamp = Duration.of(provisioningStopwatch);
try {
String logMessage = "Finished VM "+setup.getDescription()+" creation:"
+ " "+machineLocation.getUser()+"@"+machineLocation.getAddress()+":"+machineLocation.getPort()
+ (Boolean.TRUE.equals(setup.get(LOG_CREDENTIALS))
? "password=" + userCredentials.getOptionalPassword().or("<absent>")
+ " && key=" + userCredentials.getOptionalPrivateKey().or("<absent>")
: "")
+ " ready after "+Duration.of(provisioningStopwatch).toStringRounded()
+ " ("
+ "semaphore obtained in "+Duration.of(semaphoreTimestamp).toStringRounded()+";"
+ template+" template built in "+Duration.of(templateTimestamp).subtract(semaphoreTimestamp).toStringRounded()+";"
+ " "+node+" provisioned in "+Duration.of(provisionTimestamp).subtract(templateTimestamp).toStringRounded()+";"
+ " "+machineLocation+" connection usable in "+Duration.of(usableTimestamp).subtract(provisionTimestamp).toStringRounded()+";"
+ " and os customized in "+Duration.of(customizedTimestamp).subtract(usableTimestamp).toStringRounded()+" - "+Joiner.on(", ").join(customisationForLogging)+")";
LOG.info(logMessage);
} catch (Exception e){
// TODO Remove try-catch! @Nakomis: why did you add it? What exception happened during logging?
Exceptions.propagateIfFatal(e);
LOG.warn("Problem generating log message summarising completion of jclouds machine provisioning "+machineLocation+" by "+this, e);
}
return machineLocation;
} catch (Exception e) {
if (e instanceof RunNodesException && ((RunNodesException)e).getNodeErrors().size() > 0) {
node = Iterables.get(((RunNodesException)e).getNodeErrors().keySet(), 0);
}
// sometimes AWS nodes come up busted (eg ssh not allowed); just throw it back (and maybe try for another one)
boolean destroyNode = (node != null) && Boolean.TRUE.equals(setup.get(DESTROY_ON_FAILURE));
if (e.toString().contains("VPCResourceNotSpecified")) {
LOG.error("Detected that your EC2 account is a legacy 'classic' account, but the recommended instance type requires VPC. "
+ "You can specify the 'eu-central-1' region to avoid this problem, or you can specify a classic-compatible instance type, "
+ "or you can specify a subnet to use with 'networkName' "
+ "(taking care that the subnet auto-assigns public IP's and allows ingress on all ports, "
+ "as Brooklyn does not currently configure security groups for non-default VPC's; "
+ "or setting up Brooklyn to be in the subnet or have a jump host or other subnet access configuration). "
+ "For more information on VPC vs classic see http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-vpc.html.");
}
LOG.error("Failed to start VM for "+setup.getDescription() + (destroyNode ? " (destroying)" : "")
+ (node != null ? "; node "+node : "")
+ " after "+Duration.of(provisioningStopwatch).toStringRounded()
+ (semaphoreTimestamp != null ? " ("
+ "semaphore obtained in "+Duration.of(semaphoreTimestamp).toStringRounded()+";"
+ (templateTimestamp != null && semaphoreTimestamp != null ? " template built in "+Duration.of(templateTimestamp).subtract(semaphoreTimestamp).toStringRounded()+";" : "")
+ (provisionTimestamp != null && templateTimestamp != null ? " node provisioned in "+Duration.of(provisionTimestamp).subtract(templateTimestamp).toStringRounded()+";" : "")
+ (usableTimestamp != null && provisioningStopwatch != null ? " connection usable in "+Duration.of(usableTimestamp).subtract(provisionTimestamp).toStringRounded()+";" : "")
+ (customizedTimestamp != null && usableTimestamp != null ? " and OS customized in "+Duration.of(customizedTimestamp).subtract(usableTimestamp).toStringRounded() : "")
+ ")"
: "")
+ ": "+e.getMessage());
LOG.debug(Throwables.getStackTraceAsString(e));
if (destroyNode) {
Stopwatch destroyingStopwatch = Stopwatch.createStarted();
if (machineLocation != null) {
releaseSafely(machineLocation);
} else {
releaseNodeSafely(node);
}
LOG.info("Destroyed " + (machineLocation != null ? "machine " + machineLocation : "node " + node)
+ " in " + Duration.of(destroyingStopwatch).toStringRounded());
}
throw Exceptions.propagate(e);
}
}