in folly/detail/TurnSequencer.h [117:217]
TryWaitResult tryWaitForTurn(
const uint32_t turn,
Atom<uint32_t>& spinCutoff,
const bool updateSpinCutoff,
const std::chrono::time_point<Clock, Duration>* absTime =
nullptr) noexcept {
uint32_t prevThresh = spinCutoff.load(std::memory_order_relaxed);
const uint32_t effectiveSpinCutoff =
updateSpinCutoff || prevThresh == 0 ? kMaxSpinLimit : prevThresh;
uint64_t begin = 0;
uint32_t tries;
const uint32_t sturn = turn << kTurnShift;
for (tries = 0;; ++tries) {
uint32_t state = state_.load(std::memory_order_acquire);
uint32_t current_sturn = decodeCurrentSturn(state);
if (current_sturn == sturn) {
break;
}
// wrap-safe version of (current_sturn >= sturn)
if (sturn - current_sturn >= std::numeric_limits<uint32_t>::max() / 2) {
// turn is in the past
return TryWaitResult::PAST;
}
// the first effectSpinCutoff tries are spins, after that we will
// record ourself as a waiter and block with futexWait
if (kSpinUsingHardwareClock) {
auto now = hardware_timestamp();
if (tries == 0) {
begin = now;
}
if (tries == 0 || now < begin + effectiveSpinCutoff) {
asm_volatile_pause();
continue;
}
} else {
if (tries < effectiveSpinCutoff) {
asm_volatile_pause();
continue;
}
}
uint32_t current_max_waiter_delta = decodeMaxWaitersDelta(state);
uint32_t our_waiter_delta = (sturn - current_sturn) >> kTurnShift;
uint32_t new_state;
if (our_waiter_delta <= current_max_waiter_delta) {
// state already records us as waiters, probably because this
// isn't our first time around this loop
new_state = state;
} else {
new_state = encode(current_sturn, our_waiter_delta);
if (state != new_state &&
!state_.compare_exchange_strong(state, new_state)) {
continue;
}
}
if (absTime) {
auto futexResult = detail::futexWaitUntil(
&state_, new_state, *absTime, futexChannel(turn));
if (futexResult == FutexResult::TIMEDOUT) {
return TryWaitResult::TIMEDOUT;
}
} else {
detail::futexWait(&state_, new_state, futexChannel(turn));
}
}
if (updateSpinCutoff || prevThresh == 0) {
// if we hit kMaxSpinLimit then spinning was pointless, so the right
// spinCutoff is kMinSpinLimit
uint32_t target;
uint64_t elapsed = !kSpinUsingHardwareClock || tries == 0
? tries
: hardware_timestamp() - begin;
if (tries >= kMaxSpinLimit) {
target = kMinSpinLimit;
} else {
// to account for variations, we allow ourself to spin 2*N when
// we think that N is actually required in order to succeed
target = std::min(
uint32_t{kMaxSpinLimit},
std::max(
uint32_t{kMinSpinLimit}, static_cast<uint32_t>(elapsed) * 2));
}
if (prevThresh == 0) {
// bootstrap
spinCutoff.store(target);
} else {
// try once, keep moving if CAS fails. Exponential moving average
// with alpha of 7/8
// Be careful that the quantity we add to prevThresh is signed.
spinCutoff.compare_exchange_weak(
prevThresh, prevThresh + int(target - prevThresh) / 8);
}
}
return TryWaitResult::SUCCESS;
}