in packages/core/src/codewhisperer/util/telemetryHelper.ts [274:427]
codewhispererSuggestionImportCount: getImportCount(_elem),
codewhispererSuggestionIndex: i,
codewhispererSuggestionState: this.getSuggestionState(i, acceptIndex, recommendationSuggestionState),
codewhispererSuggestionReferenceCount: _elem.references ? _elem.references.length : 0,
codewhispererSuggestionReferences: uniqueSuggestionReferences,
codewhispererSupplementalContextIsUtg: supplementalContextMetadata?.isUtg,
codewhispererSupplementalContextLength: supplementalContextMetadata?.contentsLength,
codewhispererSupplementalContextTimeout: supplementalContextMetadata?.isProcessTimeout,
codewhispererTriggerType: session.triggerType,
credentialStartUrl: AuthUtil.instance.startUrl,
traceId: this.traceId,
}
events.push(event)
}
// aggregate suggestion references count
const referenceCount = this.getAggregatedSuggestionReferenceCount(events)
// aggregate user decision events at requestId level
const aggregatedEvent = this.aggregateUserDecisionByRequest(events, requestIdList[0], sessionId)
if (aggregatedEvent) {
this.sessionDecisions.push(aggregatedEvent)
}
// TODO: use a ternary for this
let acceptedRecommendationContent
if (acceptIndex !== -1 && recommendations[acceptIndex] !== undefined) {
acceptedRecommendationContent = recommendations[acceptIndex].content
} else {
acceptedRecommendationContent = ''
}
// after we have all request level user decisions, aggregate them at session level and send
this.sendUserTriggerDecisionTelemetry(
sessionId,
acceptedRecommendationContent,
referenceCount,
supplementalContextMetadata
)
}
public aggregateUserDecisionByRequest(
events: CodewhispererUserDecision[],
requestId: string,
sessionId: string,
supplementalContextMetadata?: CodeWhispererSupplementalContext | undefined
) {
// the request level user decision will contain information from both the service_invocation event
// and the user_decision events for recommendations within that request
if (!events.length) {
return
}
const aggregated: CodewhispererUserTriggerDecision = {
codewhispererAutomatedTriggerType: session.autoTriggerType,
codewhispererCompletionType: events[0].codewhispererCompletionType,
codewhispererCursorOffset: session.startCursorOffset,
codewhispererFirstRequestId: requestId,
codewhispererGettingStartedTask: session.taskType,
codewhispererLanguage: events[0].codewhispererLanguage,
codewhispererLineNumber: session.startPos.line,
codewhispererSessionId: sessionId,
codewhispererSuggestionCount: events.length,
codewhispererSuggestionImportCount: events
.map((e) => e.codewhispererSuggestionImportCount || 0)
.reduce((a, b) => a + b, 0),
codewhispererSuggestionState: this.getAggregatedSuggestionState(events),
codewhispererSupplementalContextIsUtg: supplementalContextMetadata?.isUtg,
codewhispererSupplementalContextLength: supplementalContextMetadata?.contentsLength,
codewhispererSupplementalContextTimeout: supplementalContextMetadata?.isProcessTimeout,
codewhispererTriggerType: events[0].codewhispererTriggerType,
codewhispererTypeaheadLength: 0,
credentialStartUrl: events[0].credentialStartUrl,
traceId: this.traceId,
}
return aggregated
}
public sendUserTriggerDecisionTelemetry(
sessionId: string,
acceptedRecommendationContent: string,
referenceCount: number,
supplementalContextMetadata?: CodeWhispererSupplementalContext | undefined
) {
// the user trigger decision will aggregate information from request level user decisions within one session
// and add additional session level insights
if (!this.sessionDecisions.length) {
return
}
// TODO: add partial acceptance related metrics
const autoTriggerType = this.sessionDecisions[0].codewhispererAutomatedTriggerType
const language = this.sessionDecisions[0].codewhispererLanguage
const aggregatedCompletionType = this.sessionDecisions[0].codewhispererCompletionType
const aggregatedSuggestionState = this.getAggregatedSuggestionState(this.sessionDecisions)
const selectedCustomization = getSelectedCustomization()
const profile = AuthUtil.instance.regionProfileManager.activeRegionProfile
const generatedLines =
acceptedRecommendationContent.trim() === '' ? 0 : acceptedRecommendationContent.split('\n').length
const suggestionCount = this.sessionDecisions
.map((e) => e.codewhispererSuggestionCount)
.reduce((a, b) => a + b, 0)
const aggregated: CodewhispererUserTriggerDecision = {
codewhispererAutomatedTriggerType: autoTriggerType,
codewhispererCharactersAccepted: acceptedRecommendationContent.length,
codewhispererClassifierResult: this.classifierResult,
codewhispererClassifierThreshold: this.classifierThreshold,
codewhispererCompletionType: aggregatedCompletionType,
codewhispererCursorOffset: this.sessionDecisions[0].codewhispererCursorOffset,
codewhispererCustomizationArn: selectedCustomization.arn === '' ? undefined : selectedCustomization.arn,
codewhispererFeatureEvaluations: FeatureConfigProvider.instance.getFeatureConfigsTelemetry(),
codewhispererFirstRequestId: this.sessionDecisions[0].codewhispererFirstRequestId,
codewhispererGettingStartedTask: session.taskType,
codewhispererLanguage: language,
codewhispererLineNumber: this.sessionDecisions[0].codewhispererLineNumber,
codewhispererPreviousSuggestionState: this.prevTriggerDecision,
codewhispererSessionId: this.sessionDecisions[0].codewhispererSessionId,
codewhispererSuggestionCount: suggestionCount,
codewhispererSuggestionImportCount: this.sessionDecisions
.map((e) => e.codewhispererSuggestionImportCount || 0)
.reduce((a, b) => a + b, 0),
codewhispererSuggestionState: aggregatedSuggestionState,
codewhispererSupplementalContextIsUtg: supplementalContextMetadata?.isUtg,
codewhispererSupplementalContextLength: supplementalContextMetadata?.contentsLength,
// eslint-disable-next-line id-length
codewhispererSupplementalContextStrategyId: supplementalContextMetadata?.strategy,
codewhispererSupplementalContextTimeout: supplementalContextMetadata?.isProcessTimeout,
codewhispererTimeSinceLastDocumentChange: this.timeSinceLastModification
? this.timeSinceLastModification
: undefined,
codewhispererTimeSinceLastUserDecision: this.lastTriggerDecisionTime
? performance.now() - this.lastTriggerDecisionTime
: undefined,
codewhispererTimeToFirstRecommendation: session.timeToFirstRecommendation,
codewhispererTriggerCharacter: autoTriggerType === 'SpecialCharacters' ? this.triggerChar : undefined,
codewhispererTriggerType: this.sessionDecisions[0].codewhispererTriggerType,
codewhispererTypeaheadLength: this.typeAheadLength,
credentialStartUrl: this.sessionDecisions[0].credentialStartUrl,
traceId: this.traceId,
}
telemetry.codewhisperer_userTriggerDecision.emit(aggregated)
this.prevTriggerDecision = this.getAggregatedSuggestionState(this.sessionDecisions)
this.lastTriggerDecisionTime = performance.now()
// When we send a userTriggerDecision for neither Accept nor Reject, service side should not use this value
// and client side will set this value to 0.0.
let e2eLatency = session.firstSuggestionShowTime - session.invokeSuggestionStartTime
if (aggregatedSuggestionState !== 'Reject' && aggregatedSuggestionState !== 'Accept') {
e2eLatency = 0.0
}
client
.sendTelemetryEvent({
telemetryEvent: {