in src/mwcas/mwcas.cc [91:236]
void DescriptorPool::Recovery(bool enable_stats) {
MwCASMetrics::enabled = enable_stats;
auto s = MwCASMetrics::Initialize();
RAW_CHECK(s.ok(), "failed initializing metric objects");
new(&epoch_) EpochManager;
s = epoch_.Initialize();
RAW_CHECK(s.ok(), "epoch initialization failure");
RAW_CHECK(partition_count_ > 0, "invalid partition count");
partition_table_ = (DescriptorPartition *) malloc(sizeof(DescriptorPartition) * partition_count_);
RAW_CHECK(nullptr != partition_table_, "out of memory");
for (uint32_t i = 0; i < partition_count_; ++i) {
new(&partition_table_[i]) DescriptorPartition(&epoch_, this);
}
RAW_CHECK(descriptors_, "invalid descriptor array pointer");
RAW_CHECK(pool_size_ > 0, "invalid pool size");
#ifdef PMDK
auto new_pmdk_pool = reinterpret_cast<PMDKAllocator *>(Allocator::Get())->GetPool();
uint64_t adjust_offset = (uint64_t) new_pmdk_pool - pmdk_pool_;
descriptors_ = reinterpret_cast<Descriptor *>((uint64_t) descriptors_ + adjust_offset);
#else
Metadata *metadata = (Metadata*)((uint64_t)this - sizeof(Metadata));
RAW_CHECK((uint64_t)metadata->initial_address == (uint64_t)metadata,
"invalid initial address");
RAW_CHECK(metadata->descriptor_count == pool_size_,
"wrong descriptor pool size");
#endif // PMEM
// begin recovery process
// If it is an existing pool, see if it has anything in it
uint64_t in_progress_desc = 0, redo_words = 0, undo_words = 0;
if (descriptors_[0].status_ != Descriptor::kStatusInvalid) {
// Must not be a new pool which comes with everything zeroed
for (uint32_t i = 0; i < pool_size_; ++i) {
auto &desc = descriptors_[i];
if (desc.status_ == Descriptor::kStatusInvalid) {
// Must be a new pool - comes with everything zeroed but better
// find this as we look at the first descriptor.
RAW_CHECK(i == 0, "corrupted descriptor pool/data area");
break;
}
desc.assert_valid_status();
#ifdef PMDK
// Let's set the real addresses first
for (int w = 0; w < desc.count_; ++w) {
auto &word = desc.words_[w];
if((uint64_t)word.address_ == Descriptor::kAllocNullAddress) {
continue;
}
word.address_ = (uint64_t *) ((uint64_t) word.address_ + adjust_offset);
}
#endif
// Otherwise do recovery
uint32_t status = desc.status_ & ~Descriptor::kStatusDirtyFlag;
if (status == Descriptor::kStatusFinished) {
continue;
} else if (status == Descriptor::kStatusUndecided ||
status == Descriptor::kStatusFailed) {
in_progress_desc++;
for (int w = 0; w < desc.count_; ++w) {
auto &word = desc.words_[w];
if((uint64_t)word.address_ == Descriptor::kAllocNullAddress){
continue;
}
uint64_t val = Descriptor::CleanPtr(*word.address_);
#ifdef PMDK
val += adjust_offset;
#endif
if (val == (uint64_t) &desc || val == (uint64_t) &word) {
// If it's a CondCAS descriptor, then MwCAS descriptor wasn't
// installed/persisted, i.e., new value (succeeded) or old value
// (failed) wasn't installed on the field. If it's an MwCAS
// descriptor, then the final value didn't make it to the field
// (status is Undecided). In both cases we should roll back to old
// value.
*word.address_ = word.old_value_;
#ifdef PMEM
word.PersistAddress();
#endif
undo_words++;
LOG(INFO) << "Applied old value 0x" << std::hex
<< word.old_value_ << " at 0x" << word.address_;
}
}
} else {
RAW_CHECK(status == Descriptor::kStatusSucceeded, "invalid status");
in_progress_desc++;
for (int w = 0; w < desc.count_; ++w) {
auto &word = desc.words_[w];
if((uint64_t)word.address_ == Descriptor::kAllocNullAddress){
continue;
}
uint64_t val = Descriptor::CleanPtr(*word.address_);
#ifdef PMDK
val += adjust_offset;
#endif
RAW_CHECK(val != (uint64_t) &word, "invalid field value");
if (val == (uint64_t) &desc) {
*word.address_ = word.new_value_;
#ifdef PMEM
word.PersistAddress();
#endif
redo_words++;
LOG(INFO) << "Applied new value 0x" << std::hex
<< word.new_value_ << " at 0x" << word.address_;
}
}
}
for (int w = 0; w < desc.count_; ++w) {
if((uint64_t)desc.words_[w].address_ == Descriptor::kAllocNullAddress){
continue;
}
int64_t val = *desc.words_[w].address_;
RAW_CHECK((val & ~Descriptor::kDirtyFlag) !=
((int64_t) &desc | Descriptor::kMwCASFlag),
"invalid word value");
RAW_CHECK((val & ~Descriptor::kDirtyFlag) !=
((int64_t) &desc | Descriptor::kCondCASFlag),
"invalid word value");
}
}
LOG(INFO) << "Found " << in_progress_desc <<
" in-progress descriptors, rolled forward " << redo_words <<
" words, rolled back " << undo_words << " words";
}
#ifdef PMDK
// Set the new pmdk_pool addr
pmdk_pool_ = (uint64_t) reinterpret_cast<PMDKAllocator *>(Allocator::Get())->GetPool();
#endif
InitDescriptors();
}