in static/Reliability/300_Testing_for_Resiliency_of_EC2_RDS_and_S3/Code/FailureSimulations/c#/AppResiliency/AppResiliency/AZFailover.cs [55:133]
public virtual async Task Failover()
{
try
{
Console.WriteLine($"Simulating AZ failure for {azName}");
// Find all subnets in the availability zone passed in the input
DescribeSubnetsResponse subnetsInSpecifiedAZandVpc
= await ec2Client.DescribeSubnetsAsync(new DescribeSubnetsRequest()
{
Filters = new List<Amazon.EC2.Model.Filter> {
new Amazon.EC2.Model.Filter {
Name = "vpc-id",
Values = { vpcId }
},
new Amazon.EC2.Model.Filter {
Name = "availability-zone",
Values = { azName }
}
}
});
List<string> subnetIdsInSpecifiedAZ = subnetsInSpecifiedAZandVpc.Subnets.Select(x => x.SubnetId).ToList();
// Modify the autoscaling group to remove the AZ affected which is the AZ passed in the input
// Find the autoscaling group that this is deployed into
DescribeAutoScalingGroupsResponse autoScalingGroupsResponse = await asClient.DescribeAutoScalingGroupsAsync();
if (autoScalingGroupsResponse != null && autoScalingGroupsResponse.AutoScalingGroups.Count > 0)
{
// Note: This assumes an Auto Scaling group exists; no error checking for readability
AutoScalingGroup autoScalingGroup = autoScalingGroupsResponse.AutoScalingGroups[0];
Console.WriteLine($"Updating the auto scaling group {autoScalingGroup.AutoScalingGroupName} to remove the subnet in {azName}");
UpdateAutoScalingGroupResponse updateAutoScalingGroupResponse
= await asClient.UpdateAutoScalingGroupAsync(new UpdateAutoScalingGroupRequest
{
AutoScalingGroupName = autoScalingGroup.AutoScalingGroupName,
VPCZoneIdentifier = String.Join(",", autoScalingGroup.VPCZoneIdentifier.Split(',').Where(x => !subnetIdsInSpecifiedAZ.Contains(x)))
});
}
Console.WriteLine("Creating new network ACL associations");
await BlockSubnetsInAZ(vpcId, subnetIdsInSpecifiedAZ);
Console.WriteLine("Failing over database");
//fail over rds which is in the same AZ
DescribeDBInstancesResponse describeDBInstancesResult = await rdsClient.DescribeDBInstancesAsync();
string dbInstancedId = describeDBInstancesResult.DBInstances.Where(x => String.Equals(x.DBSubnetGroup.VpcId, vpcId, StringComparison.OrdinalIgnoreCase) &&
String.Equals(x.AvailabilityZone, azName, StringComparison.OrdinalIgnoreCase) &&
x.MultiAZ &&
!x.StatusInfos.Any())?.Select(x => x.DBInstanceIdentifier).FirstOrDefault();
// we want to fail over rds if rds is present in the same az where it is affected
if (!String.IsNullOrEmpty(dbInstancedId))
{
Console.WriteLine("Rebooting dbInstanceId to secondary AZ " + dbInstancedId);
var response = await rdsClient.RebootDBInstanceAsync(new RebootDBInstanceRequest()
{
DBInstanceIdentifier = dbInstancedId,
ForceFailover = true
});
}
else
{
Console.WriteLine($"Didn't find DB in the same AZ as {azName}");
}
Console.Write("Done");
}
catch (Exception exception)
{
Console.WriteLine("Unknown exception occurred " + exception.Message);
}
}