in resctl-bench/src/run.rs [459:552]
fn minder(inner: Arc<Mutex<RunCtxInner>>) {
let mut last_status_at = SystemTime::now();
let mut last_report_at = SystemTime::now();
let mut next_at = unix_now() + 1;
'outer: loop {
let sleep_till = UNIX_EPOCH + Duration::from_secs(next_at);
'sleep: loop {
match sleep_till.duration_since(SystemTime::now()) {
Ok(dur) => {
if wait_prog_state(dur) == ProgState::Exiting {
break 'outer;
}
}
_ => break 'sleep,
}
}
next_at = unix_now() + 1;
let mut ctx = inner.lock().unwrap();
let svc = match ctx.agent_svc.as_mut() {
Some(v) => v,
None => {
debug!("minder: agent_svc is None, exiting");
break 'outer;
}
};
let mut nr_tries = 3;
'status: loop {
match svc.unit.refresh() {
Ok(()) => {
last_status_at = SystemTime::now();
if svc.unit.state == systemd::UnitState::Running {
break 'status;
}
if nr_tries > 0 {
warn!(
"minder: agent status != running ({:?}), re-verifying...",
&svc.unit.state
);
nr_tries -= 1;
continue 'status;
}
error!("minder: agent is not running ({:?})", &svc.unit.state);
ctx.minder_state = MinderState::AgentNotRunning(svc.unit.state.clone());
break 'outer;
}
Err(e) => {
if SystemTime::now().duration_since(last_status_at).unwrap()
<= MINDER_AGENT_TIMEOUT
{
warn!("minder: failed to refresh agent status ({:#})", &e);
break 'status;
}
error!(
"minder: failed to update agent status for over {}s, giving up ({:#})",
MINDER_AGENT_TIMEOUT.as_secs(),
&e
);
ctx.minder_state = MinderState::AgentTimeout;
break 'outer;
}
}
}
ctx.agent_files.refresh();
prog_kick();
let report_at = SystemTime::from(ctx.agent_files.report.data.timestamp);
if report_at > last_report_at {
last_report_at = report_at;
}
match SystemTime::now().duration_since(last_report_at) {
Ok(dur) if dur > MINDER_AGENT_TIMEOUT => {
error!(
"minder: agent report is older than {}s, giving up",
MINDER_AGENT_TIMEOUT.as_secs()
);
ctx.minder_state = MinderState::ReportTimeout;
break 'outer;
}
_ => (),
}
}
inner.lock().unwrap().agent_files.refresh();
prog_kick();
}