in AmbrosiaTest/AmbrosiaTest/ActiveActive_Test.cs [722:952]
public void AMB_ActiveActive_Kill_Client_And_Server_Test()
{
Utilities MyUtils = new Utilities();
string testName = "activeactivekillclientandserver";
string clientJobName = testName + "clientjob";
string serverName = testName + "server";
string ambrosiaLogDir = MyUtils.baseAmbrosiaPath + ConfigurationManager.AppSettings["AmbrosiaLogDirectory"] + "\\";
string byteSize = "13958643712";
string newPrimary = "NOW I'm Primary";
//AMB1 - primary server
string logOutputFileName_AMB1 = testName + "_AMB1.log";
AMB_Settings AMB1 = new AMB_Settings
{
AMB_ServiceName = serverName,
AMB_PortAppReceives = "1000",
AMB_PortAMBSends = "1001",
AMB_ServiceLogPath = ambrosiaLogDir,
AMB_CreateService = "A",
AMB_PauseAtStart = "N",
AMB_PersistLogs = "Y",
AMB_NewLogTriggerSize = "1000",
AMB_ActiveActive = "Y",
AMB_Version = "0"
};
MyUtils.CallAMB(AMB1, logOutputFileName_AMB1, AMB_ModeConsts.RegisterInstance);
//AMB2 - check pointer server
string logOutputFileName_AMB2 = testName + "_AMB2.log";
AMB_Settings AMB2 = new AMB_Settings
{
AMB_ServiceName = serverName,
AMB_ReplicaNumber = "1",
AMB_PortAppReceives = "2000",
AMB_PortAMBSends = "2001",
AMB_ServiceLogPath = ambrosiaLogDir,
AMB_CreateService = "A",
AMB_PauseAtStart = "N",
AMB_PersistLogs = "Y",
AMB_NewLogTriggerSize = "1000",
AMB_ActiveActive = "Y",
AMB_Version = "0"
};
MyUtils.CallAMB(AMB2, logOutputFileName_AMB2, AMB_ModeConsts.AddReplica);
//AMB3 - active secondary server
string logOutputFileName_AMB3 = testName + "_AMB3.log";
AMB_Settings AMB3 = new AMB_Settings
{
AMB_ServiceName = serverName,
AMB_ReplicaNumber = "2",
AMB_PortAppReceives = "3000",
AMB_PortAMBSends = "3001",
AMB_ServiceLogPath = ambrosiaLogDir,
AMB_CreateService = "A",
AMB_PauseAtStart = "N",
AMB_PersistLogs = "Y",
AMB_NewLogTriggerSize = "1000",
AMB_ActiveActive = "Y",
AMB_Version = "0"
};
MyUtils.CallAMB(AMB3, logOutputFileName_AMB3, AMB_ModeConsts.AddReplica);
//AMB4 - Job primary
string logOutputFileName_AMB4 = testName + "_AMB4.log";
AMB_Settings AMB4 = new AMB_Settings
{
AMB_ServiceName = clientJobName,
AMB_PortAppReceives = "4000",
AMB_PortAMBSends = "4001",
AMB_ServiceLogPath = ambrosiaLogDir,
AMB_CreateService = "A",
AMB_PauseAtStart = "N",
AMB_PersistLogs = "Y",
AMB_NewLogTriggerSize = "1000",
AMB_ActiveActive = "Y",
AMB_Version = "0"
};
MyUtils.CallAMB(AMB4, logOutputFileName_AMB4, AMB_ModeConsts.RegisterInstance);
//AMB5 - Job checkpoint
string logOutputFileName_AMB5 = testName + "_AMB5.log";
AMB_Settings AMB5 = new AMB_Settings
{
AMB_ServiceName = clientJobName,
AMB_ReplicaNumber = "1",
AMB_PortAppReceives = "5200",
AMB_PortAMBSends = "5201",
AMB_ServiceLogPath = ambrosiaLogDir,
AMB_CreateService = "A",
AMB_PauseAtStart = "N",
AMB_PersistLogs = "Y",
AMB_NewLogTriggerSize = "1000",
AMB_ActiveActive = "Y",
AMB_Version = "0"
};
MyUtils.CallAMB(AMB5, logOutputFileName_AMB5, AMB_ModeConsts.AddReplica);
//AMB6 - Job secondary
string logOutputFileName_AMB6 = testName + "_AMB6.log";
AMB_Settings AMB6 = new AMB_Settings
{
AMB_ServiceName = clientJobName,
AMB_ReplicaNumber = "2",
AMB_PortAppReceives = "6000",
AMB_PortAMBSends = "6001",
AMB_ServiceLogPath = ambrosiaLogDir,
AMB_CreateService = "A",
AMB_PauseAtStart = "N",
AMB_PersistLogs = "Y",
AMB_NewLogTriggerSize = "1000",
AMB_ActiveActive = "Y",
AMB_Version = "0"
};
MyUtils.CallAMB(AMB6, logOutputFileName_AMB6, AMB_ModeConsts.AddReplica);
//ImmCoord1
string logOutputFileName_ImmCoord1 = testName + "_ImmCoord1.log";
int ImmCoordProcessID1 = MyUtils.StartImmCoord(serverName, 1500, logOutputFileName_ImmCoord1,true,0);
//ImmCoord2
string logOutputFileName_ImmCoord2 = testName + "_ImmCoord2.log";
int ImmCoordProcessID2 = MyUtils.StartImmCoord(serverName, 2500, logOutputFileName_ImmCoord2,true,1);
//ImmCoord3
string logOutputFileName_ImmCoord3 = testName + "_ImmCoord3.log";
int ImmCoordProcessID3 = MyUtils.StartImmCoord(serverName, 3500, logOutputFileName_ImmCoord3,true,2);
//ImmCoord4
string logOutputFileName_ImmCoord4 = testName + "_ImmCoord4.log";
int ImmCoordProcessID4 = MyUtils.StartImmCoord(clientJobName, 4500, logOutputFileName_ImmCoord4,true,0);
//ImmCoord5
string logOutputFileName_ImmCoord5 = testName + "_ImmCoord5.log";
int ImmCoordProcessID5 = MyUtils.StartImmCoord(clientJobName, 5500, logOutputFileName_ImmCoord5,true, 1);
//ImmCoord6
string logOutputFileName_ImmCoord6 = testName + "_ImmCoord6.log";
int ImmCoordProcessID6 = MyUtils.StartImmCoord(clientJobName, 6500, logOutputFileName_ImmCoord6,true, 2);
//start Client Job - primary
string logOutputFileName_ClientJob1 = testName + "_ClientJob1.log";
int clientJobProcessID1 = MyUtils.StartPerfClientJob("4001", "4000", clientJobName, serverName, "65536", "13", logOutputFileName_ClientJob1);
//start Client Job - checkpoint
string logOutputFileName_ClientJob2 = testName + "_ClientJob2.log";
int clientJobProcessID2 = MyUtils.StartPerfClientJob("5201", "5200", clientJobName, serverName, "65536", "13", logOutputFileName_ClientJob2);
//start Client Job - secondary
string logOutputFileName_ClientJob3 = testName + "_ClientJob3.log";
int clientJobProcessID3 = MyUtils.StartPerfClientJob("6001", "6000", clientJobName, serverName, "65536", "13", logOutputFileName_ClientJob3);
//Server Call - primary
string logOutputFileName_Server1 = testName + "_Server1.log";
int serverProcessID1 = MyUtils.StartPerfServer("1001", "1000", clientJobName, serverName, logOutputFileName_Server1, 1, false);
Thread.Sleep(1000); // give a second to make it a primary
//Server Call - checkpointer
string logOutputFileName_Server2 = testName + "_Server2.log";
int serverProcessID2 = MyUtils.StartPerfServer("2001", "2000", clientJobName, serverName, logOutputFileName_Server2, 1, false);
Thread.Sleep(1000); // give a second
//Server Call - active secondary
string logOutputFileName_Server3 = testName + "_Server3.log";
int serverProcessID3 = MyUtils.StartPerfServer("3001", "3000", clientJobName, serverName, logOutputFileName_Server3, 1, false);
// Give it 10 seconds to do something before killing it
Thread.Sleep(10000);
Application.DoEvents(); // if don't do this ... system sees thread as blocked thread and throws message.
//Kill primary Server and ImmCoord
MyUtils.KillProcess(serverProcessID1);
MyUtils.KillProcess(ImmCoordProcessID1);
// at this point, secondary is becoming primary server
Thread.Sleep(1000);
//Now kill primary client and ImmCoord (ImmCoord4)
MyUtils.KillProcess(clientJobProcessID1);
MyUtils.KillProcess(ImmCoordProcessID4);
// at this point, secondary client is becoming primary client
Thread.Sleep(1000);
//Restart server1 (ImmCoord1 and server1) ... this will become secondary server
string logOutputFileName_ImmCoord1_Restarted = testName + "_ImmCoord1_Restarted.log";
int ImmCoordProcessID1_Restarted = MyUtils.StartImmCoord(serverName, 1500, logOutputFileName_ImmCoord1_Restarted,true, 0);
string logOutputFileName_Server1_Restarted = testName + "_Server1_Restarted.log";
int serverProcessID_Restarted1 = MyUtils.StartPerfServer("1001", "1000", clientJobName, serverName, logOutputFileName_Server1_Restarted, 1, false);
//Restart client1 (CR4 and client1) ... this will become active secondary again
string logOutputFileName_ImmCoord4_Restarted = testName + "_ImmCoord4_Restarted.log";
int ImmCoordProcessID4_Restarted = MyUtils.StartImmCoord(clientJobName, 4500, logOutputFileName_ImmCoord4_Restarted,true, 0);
string logOutputFileName_ClientJob1_Restarted = testName + "_ClientJob1_Restarted.log";
int clientJobProcessID_Restarted1 = MyUtils.StartPerfClientJob("4001", "4000", clientJobName, serverName, "65536", "13", logOutputFileName_ClientJob1_Restarted);
//Delay until finished ... looking at the primary (server1) but also verify others hit done too
bool pass = MyUtils.WaitForProcessToFinish(logOutputFileName_Server1_Restarted, byteSize, 40, false, testName, true); // Total Bytes received needs to be accurate
pass = MyUtils.WaitForProcessToFinish(logOutputFileName_Server2, byteSize, 15, false, testName, true);
pass = MyUtils.WaitForProcessToFinish(logOutputFileName_Server3, byteSize, 15, false, testName, true);
pass = MyUtils.WaitForProcessToFinish(logOutputFileName_ClientJob1_Restarted, byteSize, 20, false, testName, true);
pass = MyUtils.WaitForProcessToFinish(logOutputFileName_ClientJob2, byteSize, 15, false, testName, true);
pass = MyUtils.WaitForProcessToFinish(logOutputFileName_ClientJob3, byteSize, 15, false, testName, true);
// Stop things so file is freed up and can be opened in verify
MyUtils.KillProcess(serverProcessID_Restarted1);
MyUtils.KillProcess(serverProcessID2);
MyUtils.KillProcess(serverProcessID3);
MyUtils.KillProcess(clientJobProcessID_Restarted1);
MyUtils.KillProcess(clientJobProcessID2);
MyUtils.KillProcess(clientJobProcessID3);
MyUtils.KillProcess(ImmCoordProcessID1_Restarted);
MyUtils.KillProcess(ImmCoordProcessID2);
MyUtils.KillProcess(ImmCoordProcessID3);
MyUtils.KillProcess(ImmCoordProcessID4_Restarted);
MyUtils.KillProcess(ImmCoordProcessID5);
MyUtils.KillProcess(ImmCoordProcessID6);
// both client and server vary too much from machine to machine to do cmp files
// Also verify ImmCoord has the string to show it is primary for both server and client
pass = MyUtils.WaitForProcessToFinish(logOutputFileName_ImmCoord3, newPrimary, 5, false, testName, true,false);
pass = MyUtils.WaitForProcessToFinish(logOutputFileName_ImmCoord6, newPrimary, 5, false, testName, true,false);
// Verify integrity of Ambrosia logs by replaying
MyUtils.VerifyAmbrosiaLogFile(testName, Convert.ToInt64(byteSize), true, true, AMB1.AMB_Version);
}