in source/lib/monitoring.ts [45:240]
constructor(scope: cdk.Construct, id: string, props: MonitoringProps) {
super(scope, id);
// -------------------------------------------------------------------------------------------
// Calculate Metrics
const calculateMetricsRole = new iam.Role(this, 'CalculateMetricsRole', {
assumedBy: new iam.ServicePrincipal('lambda.amazonaws.com')
});
// Declaring the policy explicitly to minimize permissions and reduce cfn_nag warnings
const calculateMetricsRolePolicy = new iam.Policy(this, 'CalculateMetricsRolePolicy', {
statements: [
iamSec.IamPermissions.lambdaLogGroup(`${cdk.Aws.STACK_NAME}-calculateMetrics`),
new iam.PolicyStatement({
resources: [
`${props.statusTable.tableArn}/stream/*`
],
actions: [
'dynamodb:ListStreams',
'dynamodb:DescribeStream',
'dynamodb:GetRecords',
'dynamodb:GetShardIterator',
'dynamodb:ListShards'
]
})
]
});
props.metricTable.grantReadWriteData(calculateMetricsRole);
calculateMetricsRolePolicy.attachToRole(calculateMetricsRole);
const statusTableEventStream = new eventsource.DynamoEventSource(props.statusTable, {
startingPosition: lambda.StartingPosition.TRIM_HORIZON,
parallelizationFactor: 1,
maxBatchingWindow: cdk.Duration.seconds(30),
batchSize: 1000
});
const calculateMetrics = new lambda.Function(this, 'CalculateMetrics', {
functionName: `${cdk.Aws.STACK_NAME}-calculateMetrics`,
runtime: lambda.Runtime.NODEJS_14_X,
handler: 'index.handler',
code: lambda.Code.fromAsset(path.join(__dirname, '../lambda/calculateMetrics')),
role: calculateMetricsRole.withoutPolicyUpdates(),
environment:
{
METRICS_TABLE: props.metricTable.tableName
},
events: [statusTableEventStream]
});
calculateMetrics.node.addDependency(calculateMetricsRolePolicy);
CfnNagSuppressor.addLambdaSuppression(calculateMetrics);
// -------------------------------------------------------------------------------------------
// Post Metrics
const postMetricsRole = new iam.Role(this, 'PostMetricsRole', {
assumedBy: new iam.ServicePrincipal('lambda.amazonaws.com')
});
// Declaring the policy granting access to the stream explicitly to minimize permissions
const postMetricsRolePolicy = new iam.Policy(this, 'PostMetricsRolePolicy', {
statements: [
new iam.PolicyStatement({
resources: [props.metricTable.tableArn],
actions: ['dynamodb:Query']
}),
new iam.PolicyStatement({
sid: 'permitPostMetrics',
effect: iam.Effect.ALLOW,
actions: ['cloudwatch:PutMetricData'],
resources: ['*'],
conditions: {
StringEquals: {
'cloudwatch:namespace': 'AmazonS3GlacierReFreezer'
}
}
}),
iamSec.IamPermissions.lambdaLogGroup(`${cdk.Aws.STACK_NAME}-postMetrics`)
]
});
postMetricsRolePolicy.attachToRole(postMetricsRole);
CfnNagSuppressor.addSuppression(postMetricsRolePolicy, 'W12', 'CloudWatch does not support metric ARNs. Using Namespace condition');
const postMetrics = new lambda.Function(this, 'PostMetrics', {
functionName: `${cdk.Aws.STACK_NAME}-postMetrics`,
runtime: lambda.Runtime.NODEJS_14_X,
handler: 'index.handler',
code: lambda.Code.fromAsset(path.join(__dirname, '../lambda/postMetrics')),
role: postMetricsRole,
environment:
{
METRICS_TABLE: props.metricTable.tableName,
STATUS_TABLE: props.statusTable.tableName,
STACK_NAME: cdk.Aws.STACK_NAME,
ARCHIVE_NOTIFICATIONS_TOPIC: props.archiveNotificationTopic.topicName
}
});
postMetrics.node.addDependency(postMetricsRolePolicy);
CfnNagSuppressor.addLambdaSuppression(postMetrics);
const postMetricSchedule = new events.Rule(this, 'PostMetricSchedule', {
schedule: {
expressionString: 'rate(1 minute)'
}
});
postMetricSchedule.addTarget(new targets.LambdaFunction(postMetrics));
// -------------------------------------------------------------------------------------------
// Dashboard
const total = Monitoring.createRefreezerMetric('ArchiveCountTotal', 'Total Archives');
const requested = Monitoring.createRefreezerMetric('ArchiveCountRequested', 'Requested from Glacier');
const staged = Monitoring.createRefreezerMetric('ArchiveCountStaged', 'Staged');
const validated = Monitoring.createRefreezerMetric('ArchiveCountValidated', 'Hashes Validated');
const copied = Monitoring.createRefreezerMetric('ArchiveCountCompleted', 'Copied to Destination');
this.dashboardName = `${cdk.Aws.STACK_NAME}-Amazon-S3-Glacier-ReFreezer`;
const dashboard = new cloudwatch.Dashboard(this, 'glacier-refreezer-dashboard',
{
dashboardName: this.dashboardName,
});
// single value
const singleValueWidget = new cloudwatch.SingleValueWidget({
width: 24,
height: 3,
title: `Amazon S3 Glacier Re:Freezer Progress Metrics : ${cdk.Aws.STACK_NAME}`,
metrics: [
total,
requested,
staged,
validated,
copied
]
});
// progress line
const graphWidget = new cloudwatch.GraphWidget({
title: 'Timeline',
width: 24,
height: 6,
view: cloudwatch.GraphWidgetView.TIME_SERIES,
left: [
total,
requested,
staged,
validated,
copied
]
});
// Log Groups and Log Widget
// Pre-creating all log groups explicitly to allow Log Insights search
// Collecting logGroupNames to include into the dashboard
const logGroupNames: string[] = [
Monitoring.createStackLogGroup(this, '/aws/vendedlogs/states', 'stageTwoOrchestrator')
];
const directoryPath = path.join(__dirname, '../lambda');
fs.readdirSync(directoryPath).map(entry => {
if (fs.lstatSync(directoryPath + '/' + entry).isDirectory()) {
logGroupNames.push(Monitoring.createStackLogGroup(this, '/aws/lambda', entry))
}
});
const logWidget = new cloudwatch.LogQueryWidget({
width: 24,
height: 6,
title: 'Errors',
logGroupNames,
view: cloudwatch.LogQueryVisualizationType.TABLE,
queryLines: [
'fields @timestamp, @message',
'filter @message like /error/ or @message like /Error/ or @message like /ERROR/',
'filter @message not like /ThrottlingException/',
'filter @message not like /Idle connections will be closed/',
'sort by @timestamp desc'
]
});
// Oldest Retrieved but not copied to Staging Bucket Archive
const sqsOldestMessageWidget = new cloudwatch.GraphWidget({
title: 'Oldest SQS Message',
width: 24,
height: 6,
view: cloudwatch.GraphWidgetView.TIME_SERIES,
left: [
Monitoring.createSqsMetric(`${cdk.Aws.STACK_NAME}-archive-notification-queue`),
Monitoring.createSqsMetric(`${cdk.Aws.STACK_NAME}-chunk-copy-queue`)
]
});
dashboard.addWidgets(singleValueWidget);
dashboard.addWidgets(graphWidget);
dashboard.addWidgets(logWidget);
dashboard.addWidgets(sqsOldestMessageWidget);
}