in src/WebJobs.Script.WebHost/WebJobsScriptHostService.cs [320:521]
private async Task UnsynchronizedStartHostAsync(ScriptHostStartupOperation activeOperation, int attemptCount = 0, JobHostStartupMode startupMode = JobHostStartupMode.Normal)
{
CheckFileSystem();
if (ShutdownRequested)
{
return;
}
IHost localHost = null;
var currentCancellationToken = activeOperation.CancellationTokenSource.Token;
_logger.StartupOperationStarting(activeOperation.Id);
try
{
currentCancellationToken.ThrowIfCancellationRequested();
// if we were in an error state retain that,
// otherwise move to default
if (State != ScriptHostState.Error)
{
State = ScriptHostState.Default;
}
bool isOffline = Utility.CheckAppOffline(_applicationHostOptions.CurrentValue.ScriptPath);
State = isOffline ? ScriptHostState.Offline : State;
bool hasNonTransientErrors = startupMode.HasFlag(JobHostStartupMode.HandlingNonTransientError);
bool handlingError = startupMode.HasFlag(JobHostStartupMode.HandlingError);
// If we're in a non-transient error state or offline, skip host initialization
bool skipJobHostStartup = isOffline || hasNonTransientErrors;
bool skipHostJsonConfiguration = startupMode == JobHostStartupMode.HandlingConfigurationParsingError;
string functionsExtensionVersion = _environment.GetFunctionsExtensionVersion();
_logger.Building(functionsExtensionVersion, skipJobHostStartup, skipHostJsonConfiguration, activeOperation.Id);
using (_metricsLogger.LatencyEvent(MetricEventNames.ScriptHostManagerBuildScriptHost))
{
localHost = BuildHost(skipJobHostStartup, skipHostJsonConfiguration);
}
ActiveHost = localHost;
if (!FeatureFlags.IsEnabled(ScriptConstants.FeatureFlagDisableWebHostLogForwarding, _environment))
{
// Forward logs to AppInsights/OpenTelemetry.
// These are not tracked by the AppInsights and OpenTelemetry logger provider as these are added in the script host.
var loggerProviders = ActiveHost.Services.GetServices<ILoggerProvider>();
var deferredLogProvider = ActiveHost.Services.GetService<DeferredLoggerProvider>();
if (deferredLogProvider is not null)
{
var selectedProviders = loggerProviders.Where(provider => provider is ApplicationInsightsLoggerProvider or OpenTelemetryLoggerProvider).ToArray();
_ = Task.Run(() => deferredLogProvider.ProcessBufferedLogsAsync(selectedProviders));
}
}
if (_languageWorkerOptionsChangeTokenSource is HostBuiltChangeTokenSource<LanguageWorkerOptions> { } hostBuiltChangeTokenSource)
{
hostBuiltChangeTokenSource.TriggerChange();
}
var scriptHost = (ScriptHost)ActiveHost.Services.GetService<ScriptHost>();
if (scriptHost != null)
{
scriptHost.HostInitializing += OnHostInitializing;
if (!handlingError)
{
// Services may be initialized, but we don't want set the state to Initialized as we're
// handling an error and want to retain the Error state.
scriptHost.HostInitialized += OnHostInitialized;
}
}
LogInitialization(localHost, isOffline, attemptCount, ++_hostStartCount, activeOperation.Id);
ValidateLinuxSKUConfiguration(GetHostLogger(localHost));
if (!_scriptWebHostEnvironment.InStandbyMode)
{
// At this point we know that App Insights is initialized (if being used), so we
// can dispose this early request tracking module, which forces our new one to take over.
DisposeRequestTrackingModule();
}
currentCancellationToken.ThrowIfCancellationRequested();
var hostInstanceId = GetHostInstanceId(localHost);
_logger.StartupOperationStartingHost(activeOperation.Id, hostInstanceId);
using (_metricsLogger.LatencyEvent(MetricEventNames.ScriptHostManagerStartScriptHost))
{
await localHost.StartAsync(currentCancellationToken);
}
if (!handlingError)
{
LastError = null;
if (!isOffline)
{
State = ScriptHostState.Running;
}
}
_eventManager.Publish(new HostStartEvent());
}
catch (OperationCanceledException)
{
GetHostLogger(localHost).StartupOperationWasCanceled(activeOperation.Id);
throw;
}
catch (Exception exc)
{
bool isActiveHost = ReferenceEquals(localHost, ActiveHost);
ILogger logger = GetHostLogger(localHost);
if (isActiveHost)
{
LastError = exc;
State = ScriptHostState.Error;
logger.ErrorOccurredDuringStartupOperation(activeOperation.Id, exc);
}
else
{
// Another host has been created before this host
// threw its startup exception. We want to make sure it
// doesn't control the state of the service.
logger.ErrorOccurredInactive(activeOperation.Id, exc);
}
attemptCount++;
if (attemptCount > 3)
{
_hostMetrics.AppFailure();
}
if (ShutdownHostIfUnhealthy())
{
return;
}
if (isActiveHost)
{
// We don't want to return disposed services via the Services property, so
// set this to null before calling Orphan().
ActiveHost = null;
}
var orphanTask = Orphan(localHost)
.ContinueWith(t =>
{
if (t.IsFaulted)
{
t.Exception.Handle(e => true);
}
}, TaskContinuationOptions.ExecuteSynchronously);
// Use the fallback logger now, as we cannot trust when the host
// logger will be disposed.
logger = _logger;
if (currentCancellationToken.IsCancellationRequested)
{
logger.CancellationRequested(activeOperation.Id);
currentCancellationToken.ThrowIfCancellationRequested();
}
var nextStartupAttemptMode = JobHostStartupMode.Normal;
if (exc is HostConfigurationException)
{
// Try starting the host without parsing host.json. This will start up a
// minimal host and allow the portal to see the error. Any modification will restart again.
nextStartupAttemptMode = JobHostStartupMode.HandlingConfigurationParsingError;
}
else if (exc is HostInitializationException)
{
nextStartupAttemptMode = JobHostStartupMode.HandlingInitializationError;
}
if (nextStartupAttemptMode != JobHostStartupMode.Normal)
{
logger.LogDebug($"Starting new host with '{nextStartupAttemptMode}' and parent operation id '{activeOperation.Id}'.");
Task ignore = StartHostAsync(currentCancellationToken, attemptCount, nextStartupAttemptMode, activeOperation.Id);
}
else
{
logger.LogDebug("Will start a new host after delay.");
await Utility.DelayWithBackoffAsync(attemptCount, currentCancellationToken, min: TimeSpan.FromSeconds(1), max: TimeSpan.FromMinutes(2), logger: logger);
if (currentCancellationToken.IsCancellationRequested)
{
logger.LogDebug($"Cancellation for operation '{activeOperation.Id}' requested during delay. A new host will not be started.");
currentCancellationToken.ThrowIfCancellationRequested();
}
logger.LogDebug("Starting new host after delay.");
Task ignore = StartHostAsync(currentCancellationToken, attemptCount, parentOperationId: activeOperation.Id);
}
}
}