public FleetStateStats update()

in src/main/java/com/amazon/jenkins/ec2fleet/EC2FleetCloud.java [390:532]


    public FleetStateStats update() {
        info("start");

        final int currentToAdd;
        final Set<String> currentInstanceIdsToTerminate;

        // make snapshot of current state to work with
        // this method should always work with snapshot
        // as data could be modified
        synchronized (this) {
            currentToAdd = toAdd;
            currentInstanceIdsToTerminate = new HashSet<>(instanceIdsToTerminate);
        }

        final Jenkins jenkins = Jenkins.getInstance();

        final AmazonEC2 ec2 = Registry.getEc2Api().connect(getAwsCredentialsId(), region, endpoint);

        if (currentToAdd > 0 || currentInstanceIdsToTerminate.size() > 0) {
            final int targetCapacity = stats.getNumDesired() - currentInstanceIdsToTerminate.size() + toAdd;
            // we do update any time even real capacity was not update like remove one add one to
            // update fleet settings with NoTermination so we can terminate instances on our own
            final ModifySpotFleetRequestRequest request = new ModifySpotFleetRequestRequest();
            request.setSpotFleetRequestId(fleet);
            request.setTargetCapacity(targetCapacity);
            request.setExcessCapacityTerminationPolicy("NoTermination");
            ec2.modifySpotFleetRequest(request);
            info("Update fleet target capacity to %s", targetCapacity);
        }

        if (currentInstanceIdsToTerminate.size() > 0) {
            // internally removeNode lock on queue to correctly update node list
            // we do big block for all removal to avoid delay on lock waiting
            // for each node
            Queue.withLock(new Runnable() {
                @Override
                public void run() {
                    for (final String instanceId : currentInstanceIdsToTerminate) {
                        final Node node = jenkins.getNode(instanceId);
                        if (node != null) {
                            try {
                                jenkins.removeNode(node);
                            } catch (IOException e) {
                                warning("unable remove node %s from Jenkins, skip, just terminate EC2 instance", instanceId);
                            }
                        }
                    }
                }
            });
            info("Delete terminating nodes from Jenkins %s", currentInstanceIdsToTerminate);

            Registry.getEc2Api().terminateInstances(ec2, currentInstanceIdsToTerminate);
            info("Instances %s were terminated with result", currentInstanceIdsToTerminate);
        }

        final FleetStateStats currentStats = FleetStateStats.readClusterState(ec2, getFleet(), labelString);
        info("fleet instances: %s", currentStats.getInstances());

        // Set up the lists of Jenkins nodes and fleet instances
        // currentFleetInstances contains instances currently in the fleet
        final Set<String> fleetInstances = new HashSet<>(currentStats.getInstances());

        final Map<String, Instance> described = Registry.getEc2Api().describeInstances(ec2, fleetInstances);
        info("described instances: %s", described.keySet());

        // currentJenkinsNodes contains all registered Jenkins nodes related to this cloud
        final Set<String> jenkinsInstances = new HashSet<>();
        for (final Node node : jenkins.getNodes()) {
            if (node instanceof EC2FleetNode && ((EC2FleetNode) node).getCloud() == this) {
                jenkinsInstances.add(node.getNodeName());
            }
        }
        info("jenkins nodes %s", jenkinsInstances);

        // contains Jenkins nodes that were once fleet instances but are no longer in the fleet
        final Set<String> jenkinsNodesWithInstance = new HashSet<>(jenkinsInstances);
        jenkinsNodesWithInstance.removeAll(fleetInstances);
        info("jenkins nodes without instance %s", jenkinsNodesWithInstance);

        // terminatedFleetInstances contains fleet instances that are terminated, stopped, stopping, or shutting down
        final Set<String> terminatedFleetInstances = new HashSet<>(fleetInstances);
        // terminated are any current which cannot be described
        terminatedFleetInstances.removeAll(described.keySet());
        info("terminated instances " + terminatedFleetInstances);

        // newFleetInstances contains running fleet instances that are not already Jenkins nodes
        final Map<String, Instance> newFleetInstances = new HashMap<>(described);
        for (final String instanceId : jenkinsInstances) newFleetInstances.remove(instanceId);
        info("new instances " + newFleetInstances.keySet());

        // update caches
        final List<String> jenkinsNodesToRemove = new ArrayList<>();
        jenkinsNodesToRemove.addAll(terminatedFleetInstances);
        jenkinsNodesToRemove.addAll(jenkinsNodesWithInstance);
        // Remove dying fleet instances from Jenkins
        for (final String instance : jenkinsNodesToRemove) {
            info("Fleet (" + getLabelString() + ") no longer has the instance " + instance + ", removing from Jenkins.");
            removeNode(instance);
        }

        // Update the label for all Jenkins nodes in the fleet instance cache
        for (final String instanceId : jenkinsInstances) {
            final Node node = jenkins.getNode(instanceId);
            if (node == null) continue;

            if (!labelString.equals(node.getLabelString())) {
                try {
                    info("Updating label on node %s to \"%s\".", instanceId, labelString);
                    node.setLabelString(labelString);
                } catch (final Exception ex) {
                    warning(ex, "Unable to set label on node %s", instanceId);
                }
            }
        }

        // If we have new instances - create nodes for them!
        if (newFleetInstances.size() > 0) {
            // addNewSlave will call addNode which call queue lock
            // we speed up this by getting one lock for all nodes to all
            Queue.withLock(new Runnable() {
                @Override
                public void run() {
                    try {
                        for (final Instance instance : newFleetInstances.values()) {
                            addNewSlave(ec2, instance, currentStats);
                        }
                    } catch (final Exception ex) {
                        warning(ex, "Unable to set label on node");
                    }
                }
            });
        }

        // lock and update state of plugin, so terminate or provision could work with new state of world
        synchronized (this) {
            instanceIdsToTerminate.removeAll(currentInstanceIdsToTerminate);
            // toAdd only grow outside of this method, so we can subtract
            toAdd = toAdd - currentToAdd;
            stats = currentStats;
        }

        return stats;
    }