in lib/model/CTokenListCategory.cc [84:197]
bool CTokenListCategory::acceptRestoreTraverser(core::CStateRestoreTraverser& traverser) {
bool expectWeight{false};
// This won't be present in pre-7.7 state,
// and for such versions it was always 0
m_OrderedCommonTokenBeginIndex = 0;
// This won't be present in pre-7.9 state.
// This value means we guess at the end.
m_BaseRawStringLen = m_BaseString.max_size();
do {
const std::string& name{traverser.name()};
if (name == BASE_STRING) {
m_BaseString = traverser.value();
} else if (name == BASE_TOKEN_ID) {
TSizeSizePr tokenAndWeight{0, 0};
if (core::CStringUtils::stringToType(traverser.value(),
tokenAndWeight.first) == false) {
LOG_ERROR(<< "Invalid base token ID in " << traverser.value());
return false;
}
m_BaseTokenIds.push_back(tokenAndWeight);
} else if (name == BASE_TOKEN_WEIGHT) {
if (m_BaseTokenIds.empty()) {
LOG_ABORT(<< "Base token weight precedes base token ID in "
<< traverser.value());
}
TSizeSizePr& tokenAndWeight = m_BaseTokenIds.back();
if (core::CStringUtils::stringToType(traverser.value(),
tokenAndWeight.second) == false) {
LOG_ERROR(<< "Invalid base token weight in " << traverser.value());
return false;
}
m_BaseWeight += tokenAndWeight.second;
} else if (name == MAX_STRING_LEN) {
if (core::CStringUtils::stringToType(traverser.value(), m_MaxStringLen) == false) {
LOG_ERROR(<< "Invalid maximum string length in " << traverser.value());
return false;
}
} else if (name == ORDERED_COMMON_TOKEN_BEGIN_INDEX) {
if (core::CStringUtils::stringToType(
traverser.value(), m_OrderedCommonTokenBeginIndex) == false) {
LOG_ERROR(<< "Invalid ordered common token start index in "
<< traverser.value());
return false;
}
} else if (name == ORDERED_COMMON_TOKEN_END_INDEX) {
if (core::CStringUtils::stringToType(
traverser.value(), m_OrderedCommonTokenEndIndex) == false) {
LOG_ERROR(<< "Invalid ordered common token end index in "
<< traverser.value());
return false;
}
} else if (name == COMMON_UNIQUE_TOKEN_ID) {
TSizeSizePr tokenAndWeight{0, 0};
if (core::CStringUtils::stringToType(traverser.value(),
tokenAndWeight.first) == false) {
LOG_ERROR(<< "Invalid common unique token ID in " << traverser.value());
return false;
}
m_CommonUniqueTokenIds.push_back(tokenAndWeight);
expectWeight = true;
} else if (name == COMMON_UNIQUE_TOKEN_WEIGHT) {
if (!expectWeight) {
LOG_ERROR(<< "Common unique token weight precedes common unique token ID in "
<< traverser.value());
return false;
}
TSizeSizePr& tokenAndWeight = m_CommonUniqueTokenIds.back();
if (core::CStringUtils::stringToType(traverser.value(),
tokenAndWeight.second) == false) {
LOG_ERROR(<< "Invalid common unique token weight in "
<< traverser.value());
return false;
}
expectWeight = false;
m_CommonUniqueTokenWeight += tokenAndWeight.second;
} else if (name == ORIG_UNIQUE_TOKEN_WEIGHT) {
if (core::CStringUtils::stringToType(traverser.value(),
m_OrigUniqueTokenWeight) == false) {
LOG_ERROR(<< "Invalid unique token weight in " << traverser.value());
return false;
}
} else if (name == NUM_MATCHES) {
if (core::CStringUtils::stringToType(traverser.value(), m_NumMatches) == false) {
LOG_ERROR(<< "Invalid number of matches in " << traverser.value());
return false;
}
} else if (name == BASE_RAW_STRING_LENGTH) {
if (core::CStringUtils::stringToType(traverser.value(), m_BaseRawStringLen) == false) {
LOG_ERROR(<< "Invalid base raw string length in " << traverser.value());
return false;
}
}
} while (traverser.next());
// Ensure that m_CommonUniqueTokenIds is sorted in ascending order.
std::sort(m_CommonUniqueTokenIds.begin(), m_CommonUniqueTokenIds.end(), CTokenIdLess{});
// m_BaseRawStringLen will only have been persisted by 7.9 and above.
// In this case the absolute maximum set at the beginning of the method
// will still be set. A reasonable compromise that will result in
// behaviour no worse than 7.8 is to set it to m_MaxStringLen.
m_BaseRawStringLen = std::min(m_BaseRawStringLen, m_MaxStringLen);
return true;
}