import sys import os print(os.environ['CI_SITE_CONFIG']) sys.path.append(os.environ['CI_SITE_CONFIG']) # for adding path for ci_site_config import subprocess import re import ci_site_config import common import shlex from abc import ABC, abstractmethod # abstract base class for creating abstract classes in python # A Jenkins env variable for job name is composed of the name of the jenkins job and the branch name # it is building for. for e.g. in our case jobname = 'ofi_libfabric/master' class Test: def __init__ (self, jobname, buildno, testname, core_prov, fabric, hosts, ofi_build_mode, util_prov=None): self.jobname = jobname self.buildno = buildno self.testname = testname self.core_prov = core_prov self.util_prov = "ofi_{}".format(util_prov) if util_prov != None else "" self.fabric = fabric self.hosts = hosts self.ofi_build_mode = ofi_build_mode if (len(hosts) == 2): self.server = hosts[0] self.client = hosts[1] self.nw_interface = ci_site_config.interface_map[self.fabric] self.libfab_installpath = "{}/{}/{}/{}".format(ci_site_config.install_dir, self.jobname, self.buildno, self.ofi_build_mode) self.env = [("FI_VERBS_MR_CACHE_ENABLE", "1"),\ ("FI_VERBS_INLINE_SIZE", "256")] \ if self.core_prov == "verbs" else [] class FiInfoTest(Test): def __init__(self, jobname, buildno, testname, core_prov, fabric, hosts, ofi_build_mode, util_prov=None): super().__init__(jobname, buildno, testname, core_prov, fabric, hosts, ofi_build_mode, util_prov) self.fi_info_testpath = "{}/bin".format(self.libfab_installpath) @property def cmd(self): return "{}/fi_info ".format(self.fi_info_testpath) @property def options(self): if (self.util_prov): opts = "-f -p {};{}".format(self.core_prov, self.util_prov) else: opts = "-f -p {}".format(self.core_prov) return opts def execute_cmd(self): command = self.cmd + self.options outputcmd = shlex.split(command) common.run_command(outputcmd) class Fabtest(Test): def __init__(self, jobname, buildno, testname, core_prov, fabric, hosts, ofi_build_mode, util_prov=None): super().__init__(jobname, buildno, testname, core_prov, fabric, hosts, ofi_build_mode, util_prov) self.fabtestpath = "{}/bin".format(self.libfab_installpath) self.fabtestconfigpath = "{}/share/fabtests".format(self.libfab_installpath) def get_exclude_file(self): path = self.libfab_installpath efile_path = "{}/share/fabtests/test_configs".format(path) prov = self.util_prov if self.util_prov else self.core_prov efile_old = "{path}/{prov}/{prov}.exclude".format(path=efile_path, prov=prov) if self.util_prov: efile = "{path}/{util_prov}/{core_prov}/exclude".format(path=efile_path, util_prov=self.util_prov, core_prov=self.core_prov) else: efile = "{path}/{prov}/exclude".format(path=efile_path, prov=self.core_prov) if os.path.isfile(efile): return efile elif os.path.isfile(efile_old): return efile_old else: print("Exclude file: {} not found!".format(efile)) return None @property def cmd(self): return "{}/runfabtests.sh ".format(self.fabtestpath) @property def options(self): opts = "-T 300 -vvv -p {} -S ".format(self.fabtestpath) if (self.core_prov == "verbs" and self.nw_interface): opts = "{} -s {} ".format(opts, common.get_node_name(self.server, self.nw_interface)) # include common.py opts = "{} -c {} ".format(opts, common.get_node_name(self.client, self.nw_interface)) # from common.py if (self.core_prov == "shm"): opts = "{} -s {} ".format(opts, self.server) opts = "{} -c {} ".format(opts, self.client) opts += "-N " if not re.match(".*sockets|udp|tcp.*", self.core_prov): opts = "{} -t all ".format(opts) efile = self.get_exclude_file() if efile: opts = "{} -R ".format(opts) opts = "{} -f {} ".format(opts, efile) for key,val in self.env: opts = "{options} -E {key}={value} ".format(options = opts, key=key, value=val) if self.util_prov: opts = "{options} {core};{util} ".format(options=opts, core=self.core_prov, util=self.util_prov) else: opts = "{options} {core} ".format(options=opts, core=self.core_prov) if (self.core_prov == "shm"): opts += "{} {} ".format(self.server, self.server) else: opts += "{} {} ".format(self.server, self.client) return opts @property def execute_condn(self): # fabtests works for shmem prov only for libfabric debug builds. return True if (self.core_prov != 'shm' or \ self.ofi_build_mode == 'dbg') else False def execute_cmd(self): curdir = os.getcwd() os.chdir(self.fabtestconfigpath) command = self.cmd + self.options outputcmd = shlex.split(command) common.run_command(outputcmd) os.chdir(curdir) class ShmemTest(Test): def __init__(self, jobname, buildno, testname, core_prov, fabric, hosts, ofi_build_mode, util_prov=None): super().__init__(jobname, buildno, testname, core_prov, fabric, hosts, ofi_build_mode, util_prov) #self.n - number of hosts * number of processes per host self.n = 4 # self.ppn - number of processes per node. self.ppn = 2 self.shmem_dir = "{}/shmem".format(self.libfab_installpath) @property def cmd(self): #todo: rename mpi_testpath to testpath to make it generic for shmem and mpitest return "{}/run_shmem.sh ".format(ci_site_config.mpi_testpath) def options(self, shmem_testname): if self.util_prov: prov = "{core};{util} ".format(core=self.core_prov, util=self.util_prov) else: prov = self.core_prov opts = "-n {n} -hosts {server},{client} -shmem_dir={shmemdir} \ -libfabric_path={path}/lib -prov '{provider}' -test {test} \ -server {server} -inf {inf}" \ .format(n=self.n, server=self.server, client=self.client, \ shmemdir=self.shmem_dir, path=self.libfab_installpath, \ provider=prov, test=shmem_testname, \ inf=ci_site_config.interface_map[self.fabric]) return opts @property def execute_condn(self): return True if (self.core_prov == "psm2" or self.core_prov == "sockets") \ else False def execute_cmd(self, shmem_testname): command = self.cmd + self.options(shmem_testname) outputcmd = shlex.split(command) common.run_command(outputcmd) class MpiTests(Test): def __init__(self, jobname, buildno, testname, core_prov, fabric, mpitype, hosts, ofi_build_mode, util_prov=None): super().__init__(jobname, buildno, testname, core_prov, fabric, hosts, ofi_build_mode, util_prov) self.mpi = mpitype @property def cmd(self): if (self.mpi == "impi" or self.mpi == "mpich"): self.testpath = ci_site_config.mpi_testpath return "{}/run_{}.sh ".format(self.testpath,self.mpi) elif(self.mpi =="ompi"): self.testpath = "{}/ompi/bin".format(self.libfab_installpath) return "{}/mpirun ".format(self.testpath) @property def options(self): opts = [] if (self.mpi == "impi" or self.mpi == "mpich"): opts = "-n {} -ppn {} -hosts {},{} ".format(self.n,self.ppn, self.server,self.client) if (self.mpi == "impi"): opts = "{} -mpi_root={} ".format(opts, ci_site_config.impi_root) else: opts = "{} -mpi_root={}/mpich".format(opts, self.libfab_installpath) opts = "{} -libfabric_path={}/lib ".format(opts, self.libfab_installpath) if self.util_prov: opts = "{options} -prov {core};{util} ".format(options=opts, core=self.core_prov, util=self.util_prov) else: opts = "{} -prov {} ".format(opts, self.core_prov) for key, val in self.env: opts = "{} -genv {} {} ".format(opts, key, val) elif (self.mpi == "ompi"): opts = "-np {} ".format(self.n) hosts = ",".join([":".join([host,str(self.ppn)]) \ for host in self.hosts]) opts = "{} --host {} ".format(opts, hosts) if self.util_prov: opts = "{} --mca mtl_ofi_provider_include {};{} ".format(opts, self.core_prov,self.util_prov) else: opts = "{} --mca mtl_ofi_provider_include {} ".format(opts, self.core_prov) opts += "--mca orte_base_help_aggregate 0 " opts += "--mca mtl ofi --mca pml cm -tag-output " for key,val in self.env: opts = "{} -x {}={} ".format(opts,key,val) return opts @property def mpi_gen_execute_condn(self): #Skip MPI tests for udp, verbs(core) providers. # we would still have MPI tests runnning for # verbs-rxd and verbs-rxm providers return True if (self.core_prov != "udp" and \ self.core_prov != "shm" and \ (self.core_prov != "verbs" or \ self.util_prov == "ofi_rxm" or \ self.util_prov == "ofi_rxd")) else False # IMBtests serves as an abstract class for different # types of intel MPI benchmarks. Currently we have # the mpi1 and rma tests enabled which are encapsulated # in the IMB_mpi1 and IMB_rma classes below. class IMBtests(ABC): """ This is an abstract class for IMB tests. currently IMB-MPI1 and IMB-RMA tests are supported. In future there could be more. All abstract methods must be implemented. """ @property @abstractmethod def imb_cmd(self): pass @property @abstractmethod def execute_condn(self): pass class IMBmpi1(IMBtests): def __init__(self): self.additional_tests = [ "Biband", "Uniband", "PingPongAnySource", "PingPingAnySource", "PingPongSpecificSource", "PingPingSpecificSource" ] @property def imb_cmd(self): return "{}/intel64/bin/IMB-MPI1 -include {}".format(ci_site_config.impi_root, \ ','.join(self.additional_tests)) @property def execute_condn(self): return True class IMBrma(IMBtests): def __init__(self, core_prov): self.core_prov = core_prov @property def imb_cmd(self): return "{}/intel64/bin/IMB-RMA".format(ci_site_config.impi_root) @property def execute_condn(self): return True if (self.core_prov != "verbs") else False # MpiTestIMB class inherits from the MPITests class. # It uses the same options method and class variables as all MPI tests. # It creates IMB_xxx test objects for each kind of IMB test. class MpiTestIMB(MpiTests): def __init__(self, jobname, buildno, testname, core_prov, fabric, mpitype, hosts, ofi_build_mode, util_prov=None): super().__init__(jobname, buildno, testname, core_prov, fabric, mpitype, hosts, ofi_build_mode, util_prov) self.n = 4 self.ppn = 1 self.mpi1 = IMBmpi1() self.rma = IMBrma(self.core_prov) @property def execute_condn(self): return True if (self.mpi == "impi") else False def execute_cmd(self): command = self.cmd + self.options if(self.mpi1.execute_condn): outputcmd = shlex.split(command + self.mpi1.imb_cmd) common.run_command(outputcmd) if (self.rma.execute_condn): outputcmd = shlex.split(command + self.rma.imb_cmd) common.run_command(outputcmd) class MpichTestSuite(MpiTests): def __init__(self, jobname, buildno, testname, core_prov, fabric, mpitype, hosts, ofi_build_mode, util_prov=None): super().__init__(jobname, buildno, testname, core_prov, fabric, mpitype, hosts, ofi_build_mode, util_prov) self.mpichsuitepath = "{}/{}/mpichsuite/test/mpi/" \ .format(self.libfab_installpath, self.mpi) self.pwd = os.getcwd() def testgroup(self, testgroupname): testpath = "{}/{}".format(self.mpichsuitepath, testgroupname) tests = [] with open("{}/testlist".format(testpath)) as file: for line in file: if(line[0] != '#' and line[0] != '\n'): tests.append((line.rstrip('\n')).split(' ')) return tests def options(self, nprocs, timeout=None): if (self.mpi == "impi" or self.mpi == "mpich"): if (self.mpi == "impi"): mpiroot = ci_site_config.impi_root else: mpiroot = "{}/mpich".format(self.libfab_installpath) if (self.util_prov): prov = "\"{};{}\"".format(self.core_prov, self.util_prov) else: prov = self.core_prov if (timeout != None): os.environ['MPIEXEC_TIMEOUT']=timeout opts = "-n {np} -hosts {s},{c} -mpi_root={mpiroot} \ -libfabric_path={installpath}/lib -prov {provider} " \ .format(np=nprocs, s=self.server, c=self.client, \ provider=prov, mpiroot=mpiroot, \ installpath=self.libfab_installpath) elif (self.mpi == "ompi"): print(self.mpi) return opts @property def execute_condn(self): return True if (self.mpi == 'impi' and self.core_prov != 'psm2') else False def execute_cmd(self, testgroupname): print("Running Tests: " + testgroupname) tests = [] time = None os.chdir("{}/{}".format(self.mpichsuitepath,testgroupname)) tests = self.testgroup(testgroupname) for test in tests: testname = test[0] nprocs = test[1] args = test[2:] for item in args: itemlist = item.split('=') if (itemlist[0] == 'timelimit'): time = itemlist[1] opts = self.options(nprocs, timeout=time) testcmd = self.cmd + opts +"./{}".format(testname) outputcmd = shlex.split(testcmd) common.run_command(outputcmd) os.chdir(self.pwd) class MpiTestStress(MpiTests): def __init__(self, jobname, buildno, testname, core_prov, fabric, mpitype, hosts, ofi_build_mode, util_prov=None): super().__init__(jobname, buildno, testname, core_prov, fabric, mpitype, hosts, ofi_build_mode, util_prov) if((self.core_prov == "verbs" or self.core_prov =="psm2")): self.n = 16 self.ppn = 8 else: self.n = 4 self.ppn = 2 @property def stress_cmd(self): return "{}/{}/stress/mpi_stress -dcr".format(self.libfab_installpath, self.mpi) @property def execute_condn(self): # Todo : run stress test for ompi with libfabirc-dbg builds if it works # in Jenkins for buildbot these ompi did not build with libfabric-dbg # Due to an mpich issue when the correct mpich options are enabled during # mpich builds, sttress test is failing. disabling mpich + stress tests # untill the mpich team fixes the issue. return True if (self.mpi != 'mpich' and (self.mpi != 'ompi' or \ self.ofi_build_mode != 'dbg')) else False def execute_cmd(self): command = self.cmd + self.options + self.stress_cmd outputcmd = shlex.split(command) common.run_command(outputcmd) class MpiTestOSU(MpiTests): def __init__(self, jobname, buildno, testname, core_prov, fabric, mpitype, hosts, ofi_build_mode, util_prov=None): super().__init__(jobname, buildno, testname, core_prov, fabric, mpitype, hosts, ofi_build_mode, util_prov) self.n = 4 self.ppn = 2 self.two_proc_tests = {'osu_latency', 'osu_bibw', 'osu_latency_mt', 'osu_bw','osu_get_latency', 'osu_fop_latency', 'osu_acc_latency', 'osu_get_bw', 'osu_put_latency', 'osu_put_bw', 'osu_put_bibw', 'osu_cas_latency', 'osu_get_acc_latency' } self.osu_mpi_path = "{}/{}/osu/libexec/osu-micro-benchmarks/mpi/". \ format(self.libfab_installpath,mpitype) @property def execute_condn(self): # sockets and psm2 have some issues with OSU benchmark testing. return True if (self.mpi != "ompi" or \ (self.core_prov != "sockets" and \ self.core_prov != "psm2" and \ self.ofi_build_mode!="dbg")) \ else False def execute_cmd(self): assert(self.osu_mpi_path) p = re.compile('osu_put*') for root, dirs, tests in os.walk(self.osu_mpi_path): for test in tests: if test in self.two_proc_tests: self.n=2 self.ppn=1 else: self.n=4 self.ppn=2 # for sockets provider skip 'osu_put' benchmark tests as they fail. if(self.core_prov !='sockets' or p.search(test)== None): launcher = self.cmd + self.options osu_cmd = os.path.join(root, test) command = launcher + osu_cmd outputcmd = shlex.split(command) common.run_command(outputcmd)