in AmbrosiaTest/AmbrosiaTest/ActiveActive_Test.cs [963:1208]
public void AMB_ActiveActive_Kill_All_Test()
{
Utilities MyUtils = new Utilities();
string testName = "activeactivekillall";
string clientJobName = testName + "clientjob";
string serverName = testName + "server";
string ambrosiaLogDir = MyUtils.baseAmbrosiaPath + ConfigurationManager.AppSettings["AmbrosiaLogDirectory"] + "\\";
string byteSize = "13958643712";
string newPrimary = "NOW I'm Primary";
//AMB1 - primary server
string logOutputFileName_AMB1 = testName + "_AMB1.log";
AMB_Settings AMB1 = new AMB_Settings
{
AMB_ServiceName = serverName,
AMB_PortAppReceives = "1000",
AMB_PortAMBSends = "1001",
AMB_ServiceLogPath = ambrosiaLogDir,
AMB_CreateService = "A",
AMB_PauseAtStart = "N",
AMB_PersistLogs = "Y",
AMB_NewLogTriggerSize = "1000",
AMB_ActiveActive = "Y",
AMB_Version = "0"
};
MyUtils.CallAMB(AMB1, logOutputFileName_AMB1, AMB_ModeConsts.RegisterInstance);
//AMB2 - check pointer server
string logOutputFileName_AMB2 = testName + "_AMB2.log";
AMB_Settings AMB2 = new AMB_Settings
{
AMB_ServiceName = serverName,
AMB_ReplicaNumber = "1",
AMB_PortAppReceives = "2000",
AMB_PortAMBSends = "2001",
AMB_ServiceLogPath = ambrosiaLogDir,
AMB_CreateService = "A",
AMB_PauseAtStart = "N",
AMB_PersistLogs = "Y",
AMB_NewLogTriggerSize = "1000",
AMB_ActiveActive = "Y",
AMB_Version = "0"
};
MyUtils.CallAMB(AMB2, logOutputFileName_AMB2, AMB_ModeConsts.AddReplica);
//AMB3 - active secondary server
string logOutputFileName_AMB3 = testName + "_AMB3.log";
AMB_Settings AMB3 = new AMB_Settings
{
AMB_ServiceName = serverName,
AMB_ReplicaNumber = "2",
AMB_PortAppReceives = "3000",
AMB_PortAMBSends = "3001",
AMB_ServiceLogPath = ambrosiaLogDir,
AMB_CreateService = "A",
AMB_PauseAtStart = "N",
AMB_PersistLogs = "Y",
AMB_NewLogTriggerSize = "1000",
AMB_ActiveActive = "Y",
AMB_Version = "0"
};
MyUtils.CallAMB(AMB3, logOutputFileName_AMB3, AMB_ModeConsts.AddReplica);
//AMB4 - Job primary
string logOutputFileName_AMB4 = testName + "_AMB4.log";
AMB_Settings AMB4 = new AMB_Settings
{
AMB_ServiceName = clientJobName,
AMB_PortAppReceives = "4000",
AMB_PortAMBSends = "4001",
AMB_ServiceLogPath = ambrosiaLogDir,
AMB_CreateService = "A",
AMB_PauseAtStart = "N",
AMB_PersistLogs = "Y",
AMB_NewLogTriggerSize = "1000",
AMB_ActiveActive = "Y",
AMB_Version = "0"
};
MyUtils.CallAMB(AMB4, logOutputFileName_AMB4, AMB_ModeConsts.RegisterInstance);
//AMB5 - Job checkpoint
string logOutputFileName_AMB5 = testName + "_AMB5.log";
AMB_Settings AMB5 = new AMB_Settings
{
AMB_ServiceName = clientJobName,
AMB_ReplicaNumber = "1",
AMB_PortAppReceives = "5000",
AMB_PortAMBSends = "5001",
AMB_ServiceLogPath = ambrosiaLogDir,
AMB_CreateService = "A",
AMB_PauseAtStart = "N",
AMB_PersistLogs = "Y",
AMB_NewLogTriggerSize = "1000",
AMB_ActiveActive = "Y",
AMB_Version = "0"
};
MyUtils.CallAMB(AMB5, logOutputFileName_AMB5, AMB_ModeConsts.AddReplica);
//AMB6 - Job secondary
string logOutputFileName_AMB6 = testName + "_AMB6.log";
AMB_Settings AMB6 = new AMB_Settings
{
AMB_ServiceName = clientJobName,
AMB_ReplicaNumber = "2",
AMB_PortAppReceives = "6000",
AMB_PortAMBSends = "6001",
AMB_ServiceLogPath = ambrosiaLogDir,
AMB_CreateService = "A",
AMB_PauseAtStart = "N",
AMB_PersistLogs = "Y",
AMB_NewLogTriggerSize = "1000",
AMB_ActiveActive = "Y",
AMB_Version = "0"
};
MyUtils.CallAMB(AMB6, logOutputFileName_AMB6, AMB_ModeConsts.AddReplica);
//Server 1
string logOutputFileName_ImmCoord1 = testName + "_ImmCoord1.log";
int ImmCoordProcessID1 = MyUtils.StartImmCoord(serverName, 1500, logOutputFileName_ImmCoord1,true, 0);
Thread.Sleep(1000);
string logOutputFileName_Server1 = testName + "_Server1.log";
int serverProcessID1 = MyUtils.StartPerfServer("1001", "1000", clientJobName, serverName, logOutputFileName_Server1, 1, false);
//Server 2
string logOutputFileName_ImmCoord2 = testName + "_ImmCoord2.log";
int ImmCoordProcessID2 = MyUtils.StartImmCoord(serverName, 2500, logOutputFileName_ImmCoord2,true, 1);
Thread.Sleep(1000); // give a second
string logOutputFileName_Server2 = testName + "_Server2.log";
int serverProcessID2 = MyUtils.StartPerfServer("2001", "2000", clientJobName, serverName, logOutputFileName_Server2, 1, false);
//Server 3
string logOutputFileName_ImmCoord3 = testName + "_ImmCoord3.log";
int ImmCoordProcessID3 = MyUtils.StartImmCoord(serverName, 3500, logOutputFileName_ImmCoord3,true, 2);
string logOutputFileName_Server3 = testName + "_Server3.log";
int serverProcessID3 = MyUtils.StartPerfServer("3001", "3000", clientJobName, serverName, logOutputFileName_Server3, 1, false);
//Client 1
string logOutputFileName_ImmCoord4 = testName + "_ImmCoord4.log";
int ImmCoordProcessID4 = MyUtils.StartImmCoord(clientJobName, 4500, logOutputFileName_ImmCoord4, true, 0);
Thread.Sleep(1000); // give a second
string logOutputFileName_ClientJob1 = testName + "_ClientJob1.log";
int clientJobProcessID1 = MyUtils.StartPerfClientJob("4001", "4000", clientJobName, serverName, "65536", "13", logOutputFileName_ClientJob1);
//Client 2
string logOutputFileName_ImmCoord5 = testName + "_ImmCoord5.log";
int ImmCoordProcessID5 = MyUtils.StartImmCoord(clientJobName, 5500, logOutputFileName_ImmCoord5,true, 1);
Thread.Sleep(1000); // give a second
string logOutputFileName_ClientJob2 = testName + "_ClientJob2.log";
int clientJobProcessID2 = MyUtils.StartPerfClientJob("5001", "5000", clientJobName, serverName, "65536", "13", logOutputFileName_ClientJob2);
//Client 3
string logOutputFileName_ImmCoord6 = testName + "_ImmCoord6.log";
int ImmCoordProcessID6 = MyUtils.StartImmCoord(clientJobName, 6500, logOutputFileName_ImmCoord6,true, 2);
Thread.Sleep(1000); // give a second
string logOutputFileName_ClientJob3 = testName + "_ClientJob3.log";
int clientJobProcessID3 = MyUtils.StartPerfClientJob("6001", "6000", clientJobName, serverName, "65536", "13", logOutputFileName_ClientJob3);
// Give it 10 seconds to do something before killing it
Thread.Sleep(10000);
Application.DoEvents(); // if don't do this ... system sees thread as blocked thread and throws message.
//Kill all aspects - kill primary of each last
MyUtils.KillProcess(serverProcessID2);
MyUtils.KillProcess(ImmCoordProcessID2);
MyUtils.KillProcess(serverProcessID3);
MyUtils.KillProcess(ImmCoordProcessID3);
MyUtils.KillProcess(serverProcessID1);
MyUtils.KillProcess(ImmCoordProcessID1);
MyUtils.KillProcess(clientJobProcessID2);
MyUtils.KillProcess(ImmCoordProcessID5);
MyUtils.KillProcess(clientJobProcessID3);
MyUtils.KillProcess(ImmCoordProcessID6);
MyUtils.KillProcess(clientJobProcessID1);
MyUtils.KillProcess(ImmCoordProcessID4);
// at this point, the system is dead - restart
Thread.Sleep(5000);
//Restart servers
string logOutputFileName_ImmCoord1_Restarted = testName + "_ImmCoord1_Restarted.log";
int ImmCoordProcessID1_Restarted = MyUtils.StartImmCoord(serverName, 1500, logOutputFileName_ImmCoord1_Restarted,true, 0);
string logOutputFileName_Server1_Restarted = testName + "_Server1_Restarted.log";
int serverProcessID_Restarted1 = MyUtils.StartPerfServer("1001", "1000", clientJobName, serverName, logOutputFileName_Server1_Restarted, 1, false);
string logOutputFileName_ImmCoord2_Restarted = testName + "_ImmCoord2_Restarted.log";
int ImmCoordProcessID2_Restarted = MyUtils.StartImmCoord(serverName, 2500, logOutputFileName_ImmCoord2_Restarted,true, 1);
string logOutputFileName_Server2_Restarted = testName + "_Server2_Restarted.log";
int serverProcessID_Restarted2 = MyUtils.StartPerfServer("2001", "2000", clientJobName, serverName, logOutputFileName_Server2_Restarted, 1, false);
string logOutputFileName_ImmCoord3_Restarted = testName + "_ImmCoord3_Restarted.log";
int ImmCoordProcessID3_Restarted = MyUtils.StartImmCoord(serverName, 3500, logOutputFileName_ImmCoord3_Restarted,true, 2);
string logOutputFileName_Server3_Restarted = testName + "_Server3_Restarted.log";
int serverProcessID_Restarted3 = MyUtils.StartPerfServer("3001", "3000", clientJobName, serverName, logOutputFileName_Server3_Restarted, 1, false);
//Restart clients
string logOutputFileName_ImmCoord4_Restarted = testName + "_ImmCoord4_Restarted.log";
int ImmCoordProcessID4_Restarted = MyUtils.StartImmCoord(clientJobName, 4500, logOutputFileName_ImmCoord4_Restarted, true, 0);
string logOutputFileName_ClientJob1_Restarted = testName + "_ClientJob1_Restarted.log";
int clientJobProcessID_Restarted1 = MyUtils.StartPerfClientJob("4001", "4000", clientJobName, serverName, "65536", "13", logOutputFileName_ClientJob1_Restarted);
string logOutputFileName_ImmCoord5_Restarted = testName + "_ImmCoord5_Restarted.log";
int ImmCoordProcessID5_Restarted = MyUtils.StartImmCoord(clientJobName, 5500, logOutputFileName_ImmCoord5_Restarted, true, 1);
string logOutputFileName_ClientJob2_Restarted = testName + "_ClientJob2_Restarted.log";
int clientJobProcessID_Restarted2 = MyUtils.StartPerfClientJob("5001", "5000", clientJobName, serverName, "65536", "13", logOutputFileName_ClientJob2_Restarted);
string logOutputFileName_ImmCoord6_Restarted = testName + "_ImmCoord6_Restarted.log";
int ImmCoordProcessID6_Restarted = MyUtils.StartImmCoord(clientJobName, 6500, logOutputFileName_ImmCoord6_Restarted, true, 2);
string logOutputFileName_ClientJob3_Restarted = testName + "_ClientJob3_Restarted.log";
int clientJobProcessID_Restarted3 = MyUtils.StartPerfClientJob("6001", "6000", clientJobName, serverName, "65536", "13", logOutputFileName_ClientJob3_Restarted);
//Delay until finished ... looking at the primary (server1) but also verify others hit done too
bool pass = MyUtils.WaitForProcessToFinish(logOutputFileName_Server1_Restarted, byteSize, 75, false, testName, true); // Total Bytes received needs to be accurate
pass = MyUtils.WaitForProcessToFinish(logOutputFileName_Server2_Restarted, byteSize, 15, false, testName, true);
pass = MyUtils.WaitForProcessToFinish(logOutputFileName_Server3_Restarted, byteSize, 15, false, testName, true);
pass = MyUtils.WaitForProcessToFinish(logOutputFileName_ClientJob1_Restarted, byteSize, 15, false, testName, true);
pass = MyUtils.WaitForProcessToFinish(logOutputFileName_ClientJob2_Restarted, byteSize, 15, false, testName, true);
pass = MyUtils.WaitForProcessToFinish(logOutputFileName_ClientJob3_Restarted, byteSize, 15, false, testName, true);
// Stop things so file is freed up and can be opened in verify
MyUtils.KillProcess(serverProcessID_Restarted1);
MyUtils.KillProcess(serverProcessID_Restarted2);
MyUtils.KillProcess(serverProcessID_Restarted3);
MyUtils.KillProcess(clientJobProcessID_Restarted1);
MyUtils.KillProcess(clientJobProcessID_Restarted2);
MyUtils.KillProcess(clientJobProcessID_Restarted3);
MyUtils.KillProcess(ImmCoordProcessID1_Restarted);
MyUtils.KillProcess(ImmCoordProcessID2_Restarted);
MyUtils.KillProcess(ImmCoordProcessID3_Restarted);
MyUtils.KillProcess(ImmCoordProcessID4_Restarted);
MyUtils.KillProcess(ImmCoordProcessID5_Restarted);
MyUtils.KillProcess(ImmCoordProcessID6_Restarted);
// Verify cmp files for client and 3 servers
// the timing is a bit off when have so many processes so cmp files not
// really reliable. As long as they get through whole thing, that is what counts.
// Verify ImmCoord has the string to show it is primary for both server and client
pass = MyUtils.WaitForProcessToFinish(logOutputFileName_ImmCoord2_Restarted, newPrimary, 5, false, testName, true,false);
pass = MyUtils.WaitForProcessToFinish(logOutputFileName_ImmCoord5_Restarted, newPrimary, 5, false, testName, true,false);
// Verify integrity of Ambrosia logs by replaying
MyUtils.VerifyAmbrosiaLogFile(testName, Convert.ToInt64(byteSize), true, true, AMB1.AMB_Version);
}