in src/autoscaler-common/counters-base.js [420:495]
async function tryFlush() {
// check for if we are initialised
await pendingInit?.promise;
if (!tryFlushEnabled || !EXPORTER_PARAMETERS[exporterMode].FLUSH_ENABLED) {
// flushing disabled, do nothing!
return;
}
// Avoid simultaneous flushing!
if (flushInProgress) {
return await flushInProgress.promise;
}
flushInProgress = PromiseWithResolvers.create();
try {
// If flushed recently, wait for the min interval to pass.
const millisUntilNextForceFlush =
lastForceFlushTime +
EXPORTER_PARAMETERS[exporterMode].FLUSH_MIN_INTERVAL -
Date.now();
if (millisUntilNextForceFlush > 0) {
// wait until we can force flush again!
logger.debug('Counters.tryFlush() waiting until flushing again');
await setTimeout(millisUntilNextForceFlush);
}
// OpenTelemetry's forceFlush() will always succeed, even if the backend
// fails and reports an error...
//
// So we use the OpenTelemetry Global Error Handler installed above
// to keep a count of the number of errors reported, and if an error
// is reported during a flush, we wait a while and try again.
// Not perfect, but the best we can do.
//
// To avoid end-users seeing these errors, we supress error messages
// until the very last flush attempt.
//
// Note that if the OpenTelemetry metrics are exported to Google Cloud
// Monitoring, the first time a counter is used, it will fail to be
// exported and will need to be retried.
let attempts = EXPORTER_PARAMETERS[exporterMode].FLUSH_MAX_ATTEMPTS;
while (attempts > 0) {
const oldOpenTelemetryErrorCount = openTelemetryErrorCount;
// Suppress OTEL Diag error messages on all but the last flush attempt.
DIAG_BUNYAN_LOGGER.suppressErrors = attempts > 1;
await meterProvider.forceFlush();
DIAG_BUNYAN_LOGGER.suppressErrors = false;
lastForceFlushTime = Date.now();
if (oldOpenTelemetryErrorCount === openTelemetryErrorCount) {
// success!
return;
} else {
logger.warn('Opentelemetry reported errors during flushing, retrying.');
}
await setTimeout(EXPORTER_PARAMETERS[exporterMode].FLUSH_MIN_INTERVAL);
attempts--;
}
if (attempts <= 0) {
logger.error(
`Failed to flush counters after ${EXPORTER_PARAMETERS[exporterMode].FLUSH_MAX_ATTEMPTS} attempts - see OpenTelemetry logging`,
);
}
} catch (err) {
logger.error({err: err, message: `Error while flushing counters: ${err}`});
} finally {
// Release any waiters...
flushInProgress.resolve(null);
flushInProgress = null;
}
}