fn minder()

in resctl-bench/src/run.rs [459:552]


    fn minder(inner: Arc<Mutex<RunCtxInner>>) {
        let mut last_status_at = SystemTime::now();
        let mut last_report_at = SystemTime::now();
        let mut next_at = unix_now() + 1;

        'outer: loop {
            let sleep_till = UNIX_EPOCH + Duration::from_secs(next_at);
            'sleep: loop {
                match sleep_till.duration_since(SystemTime::now()) {
                    Ok(dur) => {
                        if wait_prog_state(dur) == ProgState::Exiting {
                            break 'outer;
                        }
                    }
                    _ => break 'sleep,
                }
            }
            next_at = unix_now() + 1;

            let mut ctx = inner.lock().unwrap();

            let svc = match ctx.agent_svc.as_mut() {
                Some(v) => v,
                None => {
                    debug!("minder: agent_svc is None, exiting");
                    break 'outer;
                }
            };

            let mut nr_tries = 3;
            'status: loop {
                match svc.unit.refresh() {
                    Ok(()) => {
                        last_status_at = SystemTime::now();
                        if svc.unit.state == systemd::UnitState::Running {
                            break 'status;
                        }

                        if nr_tries > 0 {
                            warn!(
                                "minder: agent status != running ({:?}), re-verifying...",
                                &svc.unit.state
                            );
                            nr_tries -= 1;
                            continue 'status;
                        }

                        error!("minder: agent is not running ({:?})", &svc.unit.state);
                        ctx.minder_state = MinderState::AgentNotRunning(svc.unit.state.clone());
                        break 'outer;
                    }
                    Err(e) => {
                        if SystemTime::now().duration_since(last_status_at).unwrap()
                            <= MINDER_AGENT_TIMEOUT
                        {
                            warn!("minder: failed to refresh agent status ({:#})", &e);
                            break 'status;
                        }

                        error!(
                            "minder: failed to update agent status for over {}s, giving up ({:#})",
                            MINDER_AGENT_TIMEOUT.as_secs(),
                            &e
                        );
                        ctx.minder_state = MinderState::AgentTimeout;
                        break 'outer;
                    }
                }
            }

            ctx.agent_files.refresh();
            prog_kick();

            let report_at = SystemTime::from(ctx.agent_files.report.data.timestamp);
            if report_at > last_report_at {
                last_report_at = report_at;
            }

            match SystemTime::now().duration_since(last_report_at) {
                Ok(dur) if dur > MINDER_AGENT_TIMEOUT => {
                    error!(
                        "minder: agent report is older than {}s, giving up",
                        MINDER_AGENT_TIMEOUT.as_secs()
                    );
                    ctx.minder_state = MinderState::ReportTimeout;
                    break 'outer;
                }
                _ => (),
            }
        }

        inner.lock().unwrap().agent_files.refresh();
        prog_kick();
    }