in functions/source/faz-handler/lib/core/autoscale-handler.js [528:892]
async handleSyncedCallback() {
const instanceId = this._requestInfo.instanceId,
interval = this._requestInfo.interval;
let parameters = {},
primaryIp,
isPrimary = false,
lifecycleShouldAbandon = false;
let primaryChanged = false;
parameters.instanceId = instanceId;
parameters.scalingGroupName = this.scalingGroupName;
// get selfinstance
this._selfInstance =
this._selfInstance || (await this.platform.describeInstance(parameters));
// handle hb monitor
// get self health check
this._selfHealthCheck =
this._selfHealthCheck ||
(await this.platform.getInstanceHealthCheck(
{
instanceId: this._selfInstance.instanceId
},
interval
));
// if self is already out-of-sync, skip the monitoring logics
if (this._selfHealthCheck && !this._selfHealthCheck.inSync) {
return {};
}
// get primary instance monitoring
await this.retrievePrimary();
// get the current primary instance info for comparisons later
let primaryInstanceId = (this._primaryRecord && this._primaryRecord.instanceId) || null;
let primaryElectionVote = (this._primaryRecord && this._primaryRecord.voteState) || null;
if (
this._primaryInfo &&
this._selfInstance.instanceId === this._primaryInfo.instanceId &&
this.scalingGroupName === this.primaryScalingGroupName
) {
// this instance is the current primary, skip checking primary election
// use primary health check result as self health check result
isPrimary = true;
this._selfHealthCheck = this._primaryHealthCheck;
} else if (this._selfHealthCheck && !this._selfHealthCheck.healthy) {
// this instance isn't the current primary
// if this instance is unhealth, skip primary election check
} else if (
!(this._primaryInfo && this._primaryHealthCheck && this._primaryHealthCheck.healthy)
) {
// this instance isn't the current primary
// if no primary or primary is unhealthy, try to run a primary election or check if a
// primary election is running then wait for it to end
// promiseEmitter to handle the primary election process by periodically check:
// 1. if there is a running election, then waits for its final
// 2. if there isn't a running election, then runs an election and complete it
let promiseEmitter = this.checkPrimaryElection.bind(this),
// validator set a condition to determine if the fgt needs to keep waiting or not.
validator = primaryInfo => {
// i am the new primary, don't wait, continue to finalize the election.
// should return true to end the waiting.
if (
primaryInfo &&
primaryInfo.primaryPrivateIpAddress ===
this._selfInstance.primaryPrivateIpAddress
) {
isPrimary = true;
return true;
} else if (this._primaryRecord && this._primaryRecord.voteState === 'pending') {
// i am not the new primary
// if no wait for primary election, I could become a headless instance
// may allow any instance of secondary role to come up without primary.
// They will receive the new primary ip on one of their following
// heartbeat sync callback
if (this._settings['primary-election-no-wait'] === 'true') {
return true;
} else {
// the new primary hasn't come up to finalize the election,
// I should keep on waiting.
// should return false to continue.
this._primaryRecord = null; // clear the primary record cache
return false;
}
} else if (this._primaryRecord && this._primaryRecord.voteState === 'done') {
// if the primary election is final, then no need to wait.
// should return true to end the waiting.
return true;
} else {
// no primary elected yet
// entering this syncedCallback function means i am already insync so
// i used to be assigned a primary.
// if i am not in the primary scaling group then I can't start a new
// election.
// i stay as is and hoping for someone in the primary scaling group
// triggers a primary election. Then I will be notified at some point.
if (this.scalingGroupName !== this.primaryScalingGroupName) {
return true;
} else {
// for new instance or instance in the primary scaling group
// they should keep on waiting
return false;
}
}
},
// counter to set a time based condition to end this waiting. If script execution
// time is close to its timeout (6 seconds - abount 1 inteval + 1 second), ends the
// waiting to allow for the rest of logic to run
counter = () => {
// eslint-disable-line no-unused-vars
if (Date.now() < process.env.SCRIPT_EXECUTION_EXPIRE_TIME - 6000) {
return false;
}
this.logger.warn('script execution is about to expire');
return true;
};
try {
this._primaryInfo = await CoreFunctions.waitFor(
promiseEmitter,
validator,
5000,
counter
);
// after new primary is elected, get the new primary healthcheck
// there are two possible results here:
// 1. a new instance comes up and becomes the new primary instance, its healthcheck won't
// exist yet because this instance isn't added to monitor.
// 1.1. in this case, the instance will be added to monitor.
// 2. an existing secondary instance becomes the new primary, its healthcheck exists
// because the instance in under monitoring.
// 2.1. in this case, the instance will take actions based on its healthcheck
// result.
this._primaryHealthCheck = null; // invalidate the primary health check object
// reload the primary health check object
await this.retrievePrimary();
} catch (error) {
// if error occurs, check who is holding a primary election, if it is this instance,
// terminates this election. then continue
await this.retrievePrimary(null, true);
if (
this._primaryRecord.instanceId === this._selfInstance.instanceId &&
this._primaryRecord.scalingGroupName === this._selfInstance.scalingGroupName
) {
await this.platform.removePrimaryRecord();
}
await this.removeInstance(this._selfInstance);
throw new Error(
'Failed to determine the primary instance within ' +
`${process.env.SCRIPT_EXECUTION_EXPIRE_TIME} seconds. This instance is unable` +
' to bootstrap. Please report this to administrators.'
);
}
}
// check if myself is under health check monitoring
// (primary instance itself may have got its healthcheck result in some code blocks above)
this._selfHealthCheck =
this._selfHealthCheck ||
(await this.platform.getInstanceHealthCheck(
{
instanceId: this._selfInstance.instanceId
},
interval
));
// if this instance is the primary instance and the primary election record is still pending, it will
// finalize the primary election.
if (
this._primaryInfo &&
this._selfInstance.instanceId === this._primaryInfo.instanceId &&
this.scalingGroupName === this.primaryScalingGroupName &&
this._primaryRecord &&
this._primaryRecord.voteState === 'pending'
) {
isPrimary = true;
if (
!this._selfHealthCheck ||
(this._selfHealthCheck && this._selfHealthCheck.healthy)
) {
// if election couldn't be finalized, remove the current election so someone else
// could start another election
if (!(await this.platform.finalizePrimaryElection())) {
await this.platform.removePrimaryRecord();
this._primaryRecord = null;
lifecycleShouldAbandon = true;
} else {
// primary election is finalized and now I am the new primary role
this._primaryRecord = null;
await this.retrievePrimary();
// update tags
await this.platform.updateHAAPRoleTag(this._selfInstance.instanceId);
// if enable nat gw, update route
if (this._settings['enable-integrated-egress-nat-gateway'] === 'true') {
await this.updateRouteForEgressNATGateway(this._selfInstance);
}
}
}
}
// check whether primary role has changed or not
let currentPrimaryInstanceId =
(this._primaryRecord && this._primaryRecord.instanceId) || null;
let currentPrimaryElectionVote =
(this._primaryRecord && this._primaryRecord.voteState) || null;
// no primary instance (election done) before, but have a primary (election done) now
// or
// had a pending primary election (pending) before, but now primary election is done
// no matter the primary instance id has changed or not.
if (primaryElectionVote !== 'done' && currentPrimaryElectionVote === 'done') {
primaryChanged = true;
}
// had a primary (election done) before, have a primary (election done) now,
// but they have different instance id
if (
primaryElectionVote === 'done' &&
currentPrimaryElectionVote === 'done' &&
primaryInstanceId !== currentPrimaryInstanceId
) {
primaryChanged = true;
}
// if primary role has changed and this is the new primary role,
if (primaryChanged && isPrimary) {
// update primary/secondary tags
await this.platform.updateHAAPRoleTag(this._selfInstance.instanceId);
// if enable nat gw, update route
if (this._settings['enable-integrated-egress-nat-gateway'] === 'true') {
await this.updateRouteForEgressNATGateway(this._selfInstance);
}
}
this.logger.info(
`currentPrimaryInstanceId: ${currentPrimaryInstanceId}, ` +
`currentPrimaryElectionVote: ${currentPrimaryElectionVote}, ` +
`primaryChanged: ${primaryChanged}`
);
// if no self healthcheck record found, this instance not under monitor. It's about the
// time to add it to monitor. should make sure its all lifecycle actions are complete
// while starting to monitor it.
// if this instance is not the primary, still add it to monitor but leave its primary unknown.
// if there's a primary instance, add the monitor record using this primary regardless
// the primary health status.
if (!this._selfHealthCheck) {
// check if a lifecycle event waiting
await this.completeGetConfigLifecycleAction(
this._selfInstance.instanceId,
!lifecycleShouldAbandon
);
// trigger device pre-authorization (via fortianalyzer device manager)
if (this._settings['enable-fortianalyzer-integration'] === 'true') {
await this.authorizeDevice();
}
primaryIp = this._primaryInfo ? this._primaryInfo.primaryPrivateIpAddress : null;
// if a secondary instance finds primary is pending, don't update primary ip to the health check record
if (
!isPrimary &&
this._primaryRecord &&
this._primaryRecord.voteState === 'pending' &&
this._settings['primary-election-no-wait'] === 'true'
) {
primaryIp = null;
}
await this.addInstanceToMonitor(this._selfInstance, interval, primaryIp);
let logMessagPrimaryIp =
!primaryIp && this._settings['primary-election-no-wait'] === 'true'
? ' without primary ip)'
: ` master-ip: ${primaryIp})`;
this.logger.info(
`instance (id:${this._selfInstance.instanceId}, ` +
`${logMessagPrimaryIp} is added to monitor at timestamp: ${Date.now()}.`
);
// if this newly come-up instance is the new primary, save its instance id as the
// default password into settings because all other instance will sync password from
// the primary there's a case if users never changed the primary's password, when the
// primary was torn-down, there will be no way to retrieve this original password.
// so in this case, should keep track of the update of default password.
if (
this._primaryInfo &&
this._selfInstance.instanceId === this._primaryInfo.instanceId &&
this.scalingGroupName === this.primaryScalingGroupName
) {
await this.platform.setSettingItem(
'fortigate-default-password',
this._selfInstance.instanceId,
'default password comes from the new elected primary.',
false,
false
);
}
return primaryIp
? {
'master-ip': this._primaryInfo.primaryPrivateIpAddress
}
: '';
} else if (this._selfHealthCheck && this._selfHealthCheck.healthy) {
// this instance is already in monitor. if the primary has changed (i.e.: the current
// primary is different from the one this instance is holding), and the new primary
// is in a healthy state now, notify it by sending the new primary ip to it.
// if no primary presents (reasons: waiting for the pending primary instance to become
// in-service; the primary has been purged but no new primary is elected yet.)
// keep the calling instance 'in-sync'. don't update its master-ip.
primaryIp =
this._primaryInfo && this._primaryHealthCheck && this._primaryHealthCheck.healthy
? this._primaryInfo.primaryPrivateIpAddress
: this._selfHealthCheck.primaryIp;
let now = Date.now();
await this.platform.updateInstanceHealthCheck(
this._selfHealthCheck,
interval,
primaryIp,
now
);
this.logger.info(
`hb record updated on (timestamp: ${now}, instance id:` +
`${this._selfInstance.instanceId}, ` +
`ip: ${this._selfInstance.primaryPrivateIpAddress}) health check ` +
`(${this._selfHealthCheck.healthy ? 'healthy' : 'unhealthy'}, ` +
`heartBeatLossCount: ${this._selfHealthCheck.heartBeatLossCount}, ` +
`nextHeartBeatTime: ${this._selfHealthCheck.nextHeartBeatTime}` +
`syncState: ${this._selfHealthCheck.syncState}, master-ip: ${primaryIp}).`
);
return primaryIp &&
this._selfHealthCheck &&
this._selfHealthCheck.primaryIp !== primaryIp
? {
'master-ip': this._primaryInfo.primaryPrivateIpAddress
}
: '';
} else {
this.logger.info(
'instance is unhealthy. need to remove it. healthcheck record:',
JSON.stringify(this._selfHealthCheck)
);
// for unhealthy instances, fail this instance
// if it is previously on 'in-sync' state, mark it as 'out-of-sync' so script will stop
// keeping it in sync and stop doing any other logics for it any longer.
if (this._selfHealthCheck && this._selfHealthCheck.inSync) {
// change its sync state to 'out of sync' by updating it state one last time
await this.platform.updateInstanceHealthCheck(
this._selfHealthCheck,
interval,
this._selfHealthCheck.primaryIp,
Date.now(),
true
);
// terminate it from autoscaling group
await this.removeInstance(this._selfInstance);
}
// for unhealthy instances, keep responding with action 'shutdown'
return {
action: 'shutdown'
};
}
}