in src/main/java/com/amazon/jenkins/ec2fleet/EC2FleetCloud.java [390:532]
public FleetStateStats update() {
info("start");
final int currentToAdd;
final Set<String> currentInstanceIdsToTerminate;
// make snapshot of current state to work with
// this method should always work with snapshot
// as data could be modified
synchronized (this) {
currentToAdd = toAdd;
currentInstanceIdsToTerminate = new HashSet<>(instanceIdsToTerminate);
}
final Jenkins jenkins = Jenkins.getInstance();
final AmazonEC2 ec2 = Registry.getEc2Api().connect(getAwsCredentialsId(), region, endpoint);
if (currentToAdd > 0 || currentInstanceIdsToTerminate.size() > 0) {
final int targetCapacity = stats.getNumDesired() - currentInstanceIdsToTerminate.size() + toAdd;
// we do update any time even real capacity was not update like remove one add one to
// update fleet settings with NoTermination so we can terminate instances on our own
final ModifySpotFleetRequestRequest request = new ModifySpotFleetRequestRequest();
request.setSpotFleetRequestId(fleet);
request.setTargetCapacity(targetCapacity);
request.setExcessCapacityTerminationPolicy("NoTermination");
ec2.modifySpotFleetRequest(request);
info("Update fleet target capacity to %s", targetCapacity);
}
if (currentInstanceIdsToTerminate.size() > 0) {
// internally removeNode lock on queue to correctly update node list
// we do big block for all removal to avoid delay on lock waiting
// for each node
Queue.withLock(new Runnable() {
@Override
public void run() {
for (final String instanceId : currentInstanceIdsToTerminate) {
final Node node = jenkins.getNode(instanceId);
if (node != null) {
try {
jenkins.removeNode(node);
} catch (IOException e) {
warning("unable remove node %s from Jenkins, skip, just terminate EC2 instance", instanceId);
}
}
}
}
});
info("Delete terminating nodes from Jenkins %s", currentInstanceIdsToTerminate);
Registry.getEc2Api().terminateInstances(ec2, currentInstanceIdsToTerminate);
info("Instances %s were terminated with result", currentInstanceIdsToTerminate);
}
final FleetStateStats currentStats = FleetStateStats.readClusterState(ec2, getFleet(), labelString);
info("fleet instances: %s", currentStats.getInstances());
// Set up the lists of Jenkins nodes and fleet instances
// currentFleetInstances contains instances currently in the fleet
final Set<String> fleetInstances = new HashSet<>(currentStats.getInstances());
final Map<String, Instance> described = Registry.getEc2Api().describeInstances(ec2, fleetInstances);
info("described instances: %s", described.keySet());
// currentJenkinsNodes contains all registered Jenkins nodes related to this cloud
final Set<String> jenkinsInstances = new HashSet<>();
for (final Node node : jenkins.getNodes()) {
if (node instanceof EC2FleetNode && ((EC2FleetNode) node).getCloud() == this) {
jenkinsInstances.add(node.getNodeName());
}
}
info("jenkins nodes %s", jenkinsInstances);
// contains Jenkins nodes that were once fleet instances but are no longer in the fleet
final Set<String> jenkinsNodesWithInstance = new HashSet<>(jenkinsInstances);
jenkinsNodesWithInstance.removeAll(fleetInstances);
info("jenkins nodes without instance %s", jenkinsNodesWithInstance);
// terminatedFleetInstances contains fleet instances that are terminated, stopped, stopping, or shutting down
final Set<String> terminatedFleetInstances = new HashSet<>(fleetInstances);
// terminated are any current which cannot be described
terminatedFleetInstances.removeAll(described.keySet());
info("terminated instances " + terminatedFleetInstances);
// newFleetInstances contains running fleet instances that are not already Jenkins nodes
final Map<String, Instance> newFleetInstances = new HashMap<>(described);
for (final String instanceId : jenkinsInstances) newFleetInstances.remove(instanceId);
info("new instances " + newFleetInstances.keySet());
// update caches
final List<String> jenkinsNodesToRemove = new ArrayList<>();
jenkinsNodesToRemove.addAll(terminatedFleetInstances);
jenkinsNodesToRemove.addAll(jenkinsNodesWithInstance);
// Remove dying fleet instances from Jenkins
for (final String instance : jenkinsNodesToRemove) {
info("Fleet (" + getLabelString() + ") no longer has the instance " + instance + ", removing from Jenkins.");
removeNode(instance);
}
// Update the label for all Jenkins nodes in the fleet instance cache
for (final String instanceId : jenkinsInstances) {
final Node node = jenkins.getNode(instanceId);
if (node == null) continue;
if (!labelString.equals(node.getLabelString())) {
try {
info("Updating label on node %s to \"%s\".", instanceId, labelString);
node.setLabelString(labelString);
} catch (final Exception ex) {
warning(ex, "Unable to set label on node %s", instanceId);
}
}
}
// If we have new instances - create nodes for them!
if (newFleetInstances.size() > 0) {
// addNewSlave will call addNode which call queue lock
// we speed up this by getting one lock for all nodes to all
Queue.withLock(new Runnable() {
@Override
public void run() {
try {
for (final Instance instance : newFleetInstances.values()) {
addNewSlave(ec2, instance, currentStats);
}
} catch (final Exception ex) {
warning(ex, "Unable to set label on node");
}
}
});
}
// lock and update state of plugin, so terminate or provision could work with new state of world
synchronized (this) {
instanceIdsToTerminate.removeAll(currentInstanceIdsToTerminate);
// toAdd only grow outside of this method, so we can subtract
toAdd = toAdd - currentToAdd;
stats = currentStats;
}
return stats;
}