in pkg/gpu/nvidia/manager.go [305:331]
func (ngm *nvidiaGPUManager) isMpsHealthy() error {
var out bytes.Buffer
reader, writer := io.Pipe()
defer writer.Close()
defer reader.Close()
mpsCmd := exec.Command(mpsControlBin)
mpsCmd.Stdin = reader
mpsCmd.Stdout = &out
err := mpsCmd.Start()
if err != nil {
return fmt.Errorf("failed to start NVIDIA MPS health check command: %v", err)
}
writer.Write([]byte(mpsActiveThreadCmd))
writer.Close()
err = mpsCmd.Wait()
if err != nil {
return fmt.Errorf("failed to health check NVIDIA MPS: %v", err)
}
reader.Close()
glog.Infof("MPS is healthy, active thread percentage = %s", out.String())
return nil
}