in locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/JcloudsLocation.java [683:1153]
protected MachineLocation obtainOnce(ConfigBag setup) throws NoMachinesAvailableException {
AccessController.Response access = getManagementContext().getAccessController().canProvisionLocation(this);
if (!access.isAllowed()) {
throw new IllegalStateException("Access controller forbids provisioning in "+this+": "+access.getMsg());
}
Predicate<? super HostAndPort> reachablePredicate = getReachableAddressesPredicate(setup);
ConnectivityResolverOptions options = getConnectivityOptionsBuilder(setup, false).build();
// FIXME How do we influence the node.getLoginPort, so it is set correctly for Windows?
// Setup port-forwarding, if required
JcloudsPortForwarderExtension portForwarder = setup.get(PORT_FORWARDER);
if (options.usePortForwarding()) checkNotNull(portForwarder, "portForwarder, when use-port-forwarding enabled");
final ComputeService computeService = getComputeService(setup);
CloudMachineNamer cloudMachineNamer = getCloudMachineNamer(setup);
String groupId = elvis(setup.get(GROUP_ID), cloudMachineNamer.generateNewGroupId(setup));
NodeMetadata node = null;
JcloudsMachineLocation machineLocation = null;
Duration semaphoreTimestamp = null;
Duration templateTimestamp = null;
Duration provisionTimestamp = null;
Duration usableTimestamp = null;
Duration customizedTimestamp = null;
Stopwatch provisioningStopwatch = Stopwatch.createStarted();
JcloudsLocationCustomizer customizersDelegate = LocationCustomizerDelegate.newInstance(getManagementContext(), setup);
try {
LOG.info("Creating VM "+getCreationString(setup)+" in "+this);
Semaphore machineCreationSemaphore = getMachineCreationSemaphore();
boolean acquired = machineCreationSemaphore.tryAcquire(0, TimeUnit.SECONDS);
if (!acquired) {
LOG.info("Waiting in {} for machine-creation permit ({} other queuing requests already)", new Object[] {this, machineCreationSemaphore.getQueueLength()});
Stopwatch blockStopwatch = Stopwatch.createStarted();
machineCreationSemaphore.acquire();
LOG.info("Acquired in {} machine-creation permit, after waiting {}", this, Time.makeTimeStringRounded(blockStopwatch));
} else {
LOG.debug("Acquired in {} machine-creation permit immediately", this);
}
semaphoreTimestamp = Duration.of(provisioningStopwatch);
LoginCredentials userCredentials = null;
Set<? extends NodeMetadata> nodes;
Template template;
try {
// Create default network for Azure ARM if necessary
if ("azurecompute-arm".equals(this.getProvider())) {
DefaultAzureArmNetworkCreator.createDefaultNetworkAndAddToTemplateOptionsIfRequired(computeService, setup);
}
// Setup the template
template = buildTemplate(computeService, setup, ImmutableList.of(customizersDelegate));
boolean expectWindows = isWindows(template, setup);
if (!options.skipJcloudsSshing()) {
if (expectWindows) {
// TODO Was this too early to look at template.getImage? e.g. customizeTemplate could subsequently modify it.
LOG.warn("Ignoring invalid configuration for Windows provisioning of "+template.getImage()+": "+USE_JCLOUDS_SSH_INIT.getName()+" should be false");
options = options.toBuilder()
.skipJcloudsSshing(true)
.build();
} else if (options.waitForConnectable()) {
userCredentials = initTemplateForCreateUser(template, setup);
}
}
templateTimestamp = Duration.of(provisioningStopwatch);
// "Name" metadata seems to set the display name; at least in AWS
// TODO it would be nice if this salt comes from the location's ID (but we don't know that yet as the ssh machine location isn't created yet)
// TODO in softlayer we want to control the suffix of the hostname which is 3 random hex digits
template.getOptions().getUserMetadata().put("Name", cloudMachineNamer.generateNewMachineUniqueNameFromGroupId(setup, groupId));
if (setup.get(JcloudsLocationConfig.INCLUDE_BROOKLYN_USER_METADATA)) {
template.getOptions().getUserMetadata().put("brooklyn-user", System.getProperty("user.name"));
Object context = setup.get(CALLER_CONTEXT);
if (context instanceof Entity) {
Entity entity = (Entity)context;
template.getOptions().getUserMetadata().put("brooklyn-app-id", entity.getApplicationId());
template.getOptions().getUserMetadata().put("brooklyn-app-name", entity.getApplication().getDisplayName());
template.getOptions().getUserMetadata().put("brooklyn-entity-id", entity.getId());
template.getOptions().getUserMetadata().put("brooklyn-entity-name", entity.getDisplayName());
template.getOptions().getUserMetadata().put("brooklyn-server-creation-date", Time.makeDateSimpleStampString());
}
}
customizeTemplate(computeService, template, customizersDelegate);
LOG.debug("jclouds using template {} / options {} to provision machine in {}",
new Object[] {template, template.getOptions(), getCreationString(setup)});
nodes = computeService.createNodesInGroup(groupId, 1, template);
provisionTimestamp = Duration.of(provisioningStopwatch);
} finally {
machineCreationSemaphore.release();
}
node = Iterables.getOnlyElement(nodes, null);
LOG.debug("jclouds created {} for {}", node, getCreationString(setup));
if (node == null)
throw new IllegalStateException("No nodes returned by jclouds create-nodes in " + getCreationString(setup));
customizersDelegate.customize(this, node, setup);
boolean windows = isWindows(node, setup);
if (windows) {
int newLoginPort = node.getLoginPort() == 22
? (setup.get(WinRmMachineLocation.USE_HTTPS_WINRM) ? 5986 : 5985)
: node.getLoginPort();
String newLoginUser = "root".equals(node.getCredentials().getUser())
? "Administrator"
: node.getCredentials().getUser();
LOG.debug("jclouds created Windows VM {}; transforming connection details: loginPort from {} to {}; loginUser from {} to {}",
new Object[] {node, node.getLoginPort(), newLoginPort, node.getCredentials().getUser(), newLoginUser});
node = NodeMetadataBuilder.fromNodeMetadata(node)
.loginPort(newLoginPort)
.credentials(LoginCredentials.builder(node.getCredentials()).user(newLoginUser).build())
.build();
}
Optional<HostAndPort> portForwardSshOverride;
if (options.usePortForwarding()) {
portForwardSshOverride = Optional.of(portForwarder.openPortForwarding(
node,
node.getLoginPort(),
Optional.<Integer>absent(),
Protocol.TCP,
Cidr.UNIVERSAL));
} else {
portForwardSshOverride = Optional.absent();
}
options = options.toBuilder()
.isWindows(windows)
.defaultLoginPort(node.getLoginPort())
.portForwardSshOverride(portForwardSshOverride.orNull())
.initialCredentials(node.getCredentials())
.userCredentials(userCredentials)
.build();
ConnectivityResolver networkInfoCustomizer = getLocationNetworkInfoCustomizer(setup);
ManagementAddressResolveResult hostPortCred = networkInfoCustomizer.resolve(this, node, setup, options);
final HostAndPort managementHostAndPort = hostPortCred.hostAndPort();
LoginCredentials creds = hostPortCred.credentials();
LOG.info("Using host-and-port={} and user={} when connecting to {}",
new Object[]{managementHostAndPort, creds.getUser(), node});
if (options.skipJcloudsSshing() && options.waitForConnectable()) {
LoginCredentials createdCredentials = createUser(computeService, node, managementHostAndPort, creds, setup);
if (createdCredentials != null) {
userCredentials = createdCredentials;
}
}
if (userCredentials == null) {
userCredentials = creds;
}
// store the credentials, in case they have changed
putIfPresentButDifferent(setup, JcloudsLocationConfig.PASSWORD, userCredentials.getOptionalPassword().orNull());
putIfPresentButDifferent(setup, JcloudsLocationConfig.PRIVATE_KEY_DATA, userCredentials.getOptionalPrivateKey().orNull());
// Wait for the VM to be reachable over SSH
if (options.waitForConnectable() && !options.isWindows()) {
waitForSshable(computeService, node, managementHostAndPort, ImmutableList.of(userCredentials), setup);
} else {
LOG.debug("Skipping ssh check for {} ({}) due to config waitForConnectable={}, windows={}",
new Object[]{node, getCreationString(setup), options.waitForConnectable(), windows});
}
// Do not store the credentials on the node as this may leak the credentials if they
// are obtained from an external supplier
node = NodeMetadataBuilder.fromNodeMetadata(node).credentials(null).build();
usableTimestamp = Duration.of(provisioningStopwatch);
// Create a JcloudsSshMachineLocation, and register it
if (windows) {
machineLocation = registerWinRmMachineLocation(computeService, node, Optional.fromNullable(template), userCredentials, managementHostAndPort, setup);
} else {
machineLocation = registerJcloudsSshMachineLocation(computeService, node, Optional.fromNullable(template), userCredentials, managementHostAndPort, setup);
}
PortForwardManager portForwardManager = setup.get(PORT_FORWARDING_MANAGER);
if (portForwardManager == null) {
LOG.debug("No PortForwardManager, using default");
portForwardManager = (PortForwardManager) getManagementContext().getLocationRegistry().getLocationManaged(PortForwardManagerLocationResolver.PFM_GLOBAL_SPEC);
}
if (options.usePortForwarding() && portForwardSshOverride.isPresent()) {
// Now that we have the sshMachineLocation, we can associate the port-forwarding address with it.
portForwardManager.associate(node.getId(), portForwardSshOverride.get(), machineLocation, node.getLoginPort());
}
if ("docker".equals(this.getProvider())) {
if (windows) {
throw new UnsupportedOperationException("Docker not supported on Windows");
}
Map<Integer, Integer> portMappings = JcloudsUtil.dockerPortMappingsFor(this, node.getId());
for(Integer containerPort : portMappings.keySet()) {
Integer hostPort = portMappings.get(containerPort);
String dockerHost = ((JcloudsSshMachineLocation)machineLocation).getSshHostAndPort().getHost();
portForwardManager.associate(node.getId(), HostAndPort.fromParts(dockerHost, hostPort), machineLocation, containerPort);
}
}
List<String> customisationForLogging = new ArrayList<String>();
// Apply same securityGroups rules to iptables, if iptables is running on the node
if (options.waitForConnectable()) {
String setupScript = setup.get(JcloudsLocationConfig.CUSTOM_MACHINE_SETUP_SCRIPT_URL);
List<String> setupScripts = setup.get(JcloudsLocationConfig.CUSTOM_MACHINE_SETUP_SCRIPT_URL_LIST);
Collection<String> allScripts = new MutableList<String>().appendIfNotNull(setupScript).appendAll(setupScripts);
for (String setupScriptItem : allScripts) {
if (Strings.isNonBlank(setupScriptItem)) {
customisationForLogging.add("custom setup script " + setupScriptItem);
String setupVarsString = setup.get(JcloudsLocationConfig.CUSTOM_MACHINE_SETUP_SCRIPT_VARS);
Map<String, String> substitutions = (setupVarsString != null)
? Splitter.on(",").withKeyValueSeparator(":").split(setupVarsString)
: ImmutableMap.<String, String>of();
String scriptContent = ResourceUtils.create(this).getResourceAsString(setupScriptItem);
String script = TemplateProcessor.processTemplateContents("jclouds script "+setupScriptItem, scriptContent, getManagementContext(), substitutions);
if (windows) {
WinRmToolResponse resp = ((WinRmMachineLocation)machineLocation).executeCommand(ImmutableList.copyOf((script.replace("\r", "").split("\n"))));
if (resp.getStatusCode() != 0) {
throw new IllegalStateException("Command 'Customizing node " + this + "' failed with exit code " + resp.getStatusCode() + " for location " + machineLocation);
}
} else {
executeCommandThrowingOnError(
(SshMachineLocation)machineLocation,
"Customizing node " + this,
ImmutableList.of(script));
}
}
}
Boolean dontRequireTtyForSudo = setup.get(JcloudsLocationConfig.DONT_REQUIRE_TTY_FOR_SUDO);
if (Boolean.TRUE.equals(dontRequireTtyForSudo) ||
(dontRequireTtyForSudo == null && setup.get(DONT_CREATE_USER))) {
if (windows) {
LOG.warn("Ignoring flag DONT_REQUIRE_TTY_FOR_SUDO on Windows location {}", machineLocation);
} else {
customisationForLogging.add("patch /etc/sudoers to disable requiretty");
queueLocationTask("patch /etc/sudoers to disable requiretty",
SshTasks.dontRequireTtyForSudo((SshMachineLocation)machineLocation, true).newTask().asTask());
}
}
if (setup.get(JcloudsLocationConfig.MAP_DEV_RANDOM_TO_DEV_URANDOM)) {
if (windows) {
LOG.warn("Ignoring flag MAP_DEV_RANDOM_TO_DEV_URANDOM on Windows location {}", machineLocation);
} else {
customisationForLogging.add("point /dev/random to urandom");
executeCommandThrowingOnError(
(SshMachineLocation)machineLocation,
"using urandom instead of random",
Arrays.asList(
bashCommands().sudo("mv /dev/random /dev/random-real"),
bashCommands().sudo("ln -s /dev/urandom /dev/random")));
}
}
if (setup.get(GENERATE_HOSTNAME)) {
if (windows) {
// TODO: Generate Windows Hostname
LOG.warn("Ignoring flag GENERATE_HOSTNAME on Windows location {}", machineLocation);
} else {
customisationForLogging.add("configure hostname");
// also see TODO in SetHostnameCustomizer - ideally we share code between here and there
executeCommandThrowingOnError(
(SshMachineLocation)machineLocation,
"Generate hostname " + node.getName(),
ImmutableList.of(bashCommands().chainGroup(
String.format("echo '127.0.0.1 %s' | ( %s )", node.getName(), bashCommands().sudo("tee -a /etc/hosts")),
"{ " + bashCommands().sudo("sed -i \"s/HOSTNAME=.*/HOSTNAME=" + node.getName() + "/g\" /etc/sysconfig/network") + " || true ; }",
bashCommands().sudo("hostname " + node.getName()))));
}
}
if (setup.get(OPEN_IPTABLES)) {
if (windows) {
LOG.warn("Ignoring DEPRECATED flag OPEN_IPTABLES on Windows location {}", machineLocation);
} else {
LOG.warn("Using DEPRECATED flag OPEN_IPTABLES (will not be supported in future versions) for {} at {}", machineLocation, this);
Iterable<Integer> inboundPorts = Ints.asList(template.getOptions().getInboundPorts());
if (inboundPorts == null || Iterables.isEmpty(inboundPorts)) {
LOG.info("No ports to open in iptables (no inbound ports) for {} at {}", machineLocation, this);
} else {
customisationForLogging.add("open iptables");
List<String> iptablesRules = Lists.newArrayList();
if (isLocationFirewalldEnabled((SshMachineLocation)machineLocation)) {
for (Integer port : inboundPorts) {
iptablesRules.add(iptablesCommands().addFirewalldRule(Chain.INPUT, Protocol.TCP, port, Policy.ACCEPT));
}
} else {
iptablesRules = Lists.newArrayList();
for (Integer port : inboundPorts) {
iptablesRules.add(iptablesCommands().insertIptablesRule(Chain.INPUT, Protocol.TCP, port, Policy.ACCEPT));
}
iptablesRules.add(iptablesCommands().saveIptablesRules());
}
List<String> batch = Lists.newArrayList();
// Some entities, such as Riak (erlang based) have a huge range of ports, which leads to a script that
// is too large to run (fails with a broken pipe). Batch the rules into batches of 50
for (String rule : iptablesRules) {
batch.add(rule);
if (batch.size() == 50) {
executeCommandWarningOnError(
(SshMachineLocation)machineLocation,
"Inserting iptables rules, 50 command batch",
batch);
batch.clear();
}
}
if (batch.size() > 0) {
executeCommandWarningOnError(
(SshMachineLocation)machineLocation,
"Inserting iptables rules",
batch);
}
executeCommandWarningOnError(
(SshMachineLocation)machineLocation,
"List iptables rules",
ImmutableList.of(iptablesCommands().listIptablesRule()));
}
}
}
if (setup.get(STOP_IPTABLES)) {
if (windows) {
LOG.warn("Ignoring DEPRECATED flag OPEN_IPTABLES on Windows location {}", machineLocation);
} else {
LOG.warn("Using DEPRECATED flag STOP_IPTABLES (will not be supported in future versions) for {} at {}", machineLocation, this);
customisationForLogging.add("stop iptables");
List<String> cmds = ImmutableList.<String>of();
if (isLocationFirewalldEnabled((SshMachineLocation)machineLocation)) {
cmds = ImmutableList.of(iptablesCommands().firewalldServiceStop(), iptablesCommands().firewalldServiceStatus());
} else {
cmds = ImmutableList.of(iptablesCommands().iptablesServiceStop(), iptablesCommands().iptablesServiceStatus());
}
executeCommandWarningOnError(
(SshMachineLocation)machineLocation,
"Stopping iptables", cmds);
}
}
List<String> extraKeyUrlsToAuth = setup.get(EXTRA_PUBLIC_KEY_URLS_TO_AUTH);
if (extraKeyUrlsToAuth!=null && !extraKeyUrlsToAuth.isEmpty()) {
if (windows) {
LOG.warn("Ignoring flag EXTRA_PUBLIC_KEY_URLS_TO_AUTH on Windows location", machineLocation);
} else {
List<String> extraKeyDataToAuth = MutableList.of();
for (String keyUrl : extraKeyUrlsToAuth) {
extraKeyDataToAuth.add(ResourceUtils.create(this).getResourceAsString(keyUrl));
}
executeCommandThrowingOnError(
(SshMachineLocation)machineLocation,
"Authorizing ssh keys from URLs",
ImmutableList.of(new AuthorizeRSAPublicKeys(extraKeyDataToAuth).render(org.jclouds.scriptbuilder.domain.OsFamily.UNIX)));
}
}
String extraKeyDataToAuth = setup.get(EXTRA_PUBLIC_KEY_DATA_TO_AUTH);
if (extraKeyDataToAuth!=null && !extraKeyDataToAuth.isEmpty()) {
if (windows) {
LOG.warn("Ignoring flag EXTRA_PUBLIC_KEY_DATA_TO_AUTH on Windows location", machineLocation);
} else {
executeCommandThrowingOnError(
(SshMachineLocation)machineLocation,
"Authorizing ssh keys from data",
ImmutableList.of(new AuthorizeRSAPublicKeys(Collections.singletonList(extraKeyDataToAuth)).render(org.jclouds.scriptbuilder.domain.OsFamily.UNIX)));
}
}
} else {
// Otherwise we have deliberately not waited to be ssh'able, so don't try now to
// ssh to exec these commands!
}
customizersDelegate.customize(this, computeService, machineLocation);
customizedTimestamp = Duration.of(provisioningStopwatch);
String logMessage = "Finished VM "+getCreationString(setup)+" creation:"
+ " "+machineLocation.getUser()+"@"+machineLocation.getAddress()+":"+machineLocation.getPort()
+ (Boolean.TRUE.equals(setup.get(LOG_CREDENTIALS))
? "password=" + userCredentials.getOptionalPassword().or("<absent>")
+ " && key=" + userCredentials.getOptionalPrivateKey().or("<absent>")
: "")
+ " ready after "+Duration.of(provisioningStopwatch).toStringRounded()
+ " ("
+ "semaphore obtained in "+Duration.of(semaphoreTimestamp).toStringRounded()+";"
+ template+" template built in "+Duration.of(templateTimestamp).subtract(semaphoreTimestamp).toStringRounded()+";"
+ " "+node+" provisioned in "+Duration.of(provisionTimestamp).subtract(templateTimestamp).toStringRounded()+";"
+ " "+machineLocation+" connection usable in "+Duration.of(usableTimestamp).subtract(provisionTimestamp).toStringRounded()+";"
+ " and os customized in "+Duration.of(customizedTimestamp).subtract(usableTimestamp).toStringRounded()+" - "+Joiner.on(", ").join(customisationForLogging)+")";
LOG.info(logMessage);
return machineLocation;
} catch (Exception e) {
if (e instanceof RunNodesException && ((RunNodesException)e).getNodeErrors().size() > 0) {
node = Iterables.get(((RunNodesException)e).getNodeErrors().keySet(), 0);
}
// sometimes AWS nodes come up busted (eg ssh not allowed); just throw it back (and maybe try for another one)
boolean destroyNode = (node != null) && Boolean.TRUE.equals(setup.get(DESTROY_ON_FAILURE));
if (e.toString().contains("VPCResourceNotSpecified")) {
String message = "Detected that your EC2 account is a legacy 'EC2 Classic' account, "
+ "but the most appropriate hardware instance type requires 'VPC'. "
+ "One quick fix is to use the 'eu-central-1' region. "
+ "Other remedies are described at "
+ AWS_VPC_HELP_URL;
LOG.error(message);
e = new UserFacingException(message, e);
}
LOG.error("Failed to start VM for "+getCreationString(setup) + (destroyNode ? " (destroying)" : "")
+ (node != null ? "; node "+node : "")
+ " after "+Duration.of(provisioningStopwatch).toStringRounded()
+ (semaphoreTimestamp != null ? " ("
+ "semaphore obtained in "+Duration.of(semaphoreTimestamp).toStringRounded()+";"
+ (templateTimestamp != null && semaphoreTimestamp != null ? " template built in "+Duration.of(templateTimestamp).subtract(semaphoreTimestamp).toStringRounded()+";" : "")
+ (provisionTimestamp != null && templateTimestamp != null ? " node provisioned in "+Duration.of(provisionTimestamp).subtract(templateTimestamp).toStringRounded()+";" : "")
+ (usableTimestamp != null && provisioningStopwatch != null ? " connection usable in "+Duration.of(usableTimestamp).subtract(provisionTimestamp).toStringRounded()+";" : "")
+ (customizedTimestamp != null && usableTimestamp != null ? " and OS customized in "+Duration.of(customizedTimestamp).subtract(usableTimestamp).toStringRounded() : "")
+ ")"
: "")
+ ": "+e.getMessage());
LOG.debug(Throwables.getStackTraceAsString(e));
try {
customizersDelegate.preReleaseOnObtainError(this, machineLocation, e);
} catch (Exception customizerException) {
LOG.info("Got exception on calling customizer preReleaseOnObtainError, ignoring. Location is {}, machine location is {}, node is {}",
new Object[] {this, machineLocation, node, customizerException});
}
if (destroyNode) {
Stopwatch destroyingStopwatch = Stopwatch.createStarted();
if (machineLocation != null) {
releaseSafely(machineLocation);
} else {
releaseNodeSafely(node);
}
LOG.info("Destroyed " + (machineLocation != null ? "machine " + machineLocation : "node " + node)
+ " in " + Duration.of(destroyingStopwatch).toStringRounded());
try {
customizersDelegate.postReleaseOnObtainError(this, machineLocation, e);
} catch (Exception customizerException) {
LOG.debug("Got exception on calling customizer postReleaseOnObtainError, ignoring. Location is {}, machine Location is {}, node is {}",
new Object[] {this, machineLocation, node, customizerException});
}
}
throw Exceptions.propagate(e);
}
}