in erts/emulator/asmjit/core/rabuilders_p.h [101:474]
Error run() noexcept {
log("[RAPass::BuildCFG]\n");
ASMJIT_PROPAGATE(prepare());
logNode(_funcNode, kRootIndentation);
logBlock(_curBlock, kRootIndentation);
BaseNode* node = _funcNode->next();
if (ASMJIT_UNLIKELY(!node))
return DebugUtils::errored(kErrorInvalidState);
_curBlock->setFirst(node);
_curBlock->setLast(node);
RAInstBuilder ib;
ZoneVector<RABlock*> blocksWithUnknownJumps;
for (;;) {
BaseNode* next = node->next();
ASMJIT_ASSERT(node->position() == 0 || node->position() == kNodePositionDidOnBefore);
if (node->isInst()) {
// Instruction | Jump | Invoke | Return
// ------------------------------------
// Handle `InstNode`, `InvokeNode`, and `FuncRetNode`. All of them
// share the same interface that provides operands that have read/write
// semantics.
if (ASMJIT_UNLIKELY(!_curBlock)) {
// Unreachable code has to be removed, we cannot allocate registers
// in such code as we cannot do proper liveness analysis in such case.
removeNode(node);
node = next;
continue;
}
_hasCode = true;
if (node->isInvoke() || node->isFuncRet()) {
if (node->position() != kNodePositionDidOnBefore) {
// Call and Reg are complicated as they may insert some surrounding
// code around them. The simplest approach is to get the previous
// node, call the `onBefore()` handlers and then check whether
// anything changed and restart if so. By restart we mean that the
// current `node` would go back to the first possible inserted node
// by `onBeforeInvoke()` or `onBeforeRet()`.
BaseNode* prev = node->prev();
if (node->type() == BaseNode::kNodeInvoke)
ASMJIT_PROPAGATE(static_cast<This*>(this)->onBeforeInvoke(node->as<InvokeNode>()));
else
ASMJIT_PROPAGATE(static_cast<This*>(this)->onBeforeRet(node->as<FuncRetNode>()));
if (prev != node->prev()) {
// If this was the first node in the block and something was
// inserted before it then we have to update the first block.
if (_curBlock->first() == node)
_curBlock->setFirst(prev->next());
node->setPosition(kNodePositionDidOnBefore);
node = prev->next();
// `onBeforeInvoke()` and `onBeforeRet()` can only insert instructions.
ASMJIT_ASSERT(node->isInst());
}
// Necessary if something was inserted after `node`, but nothing before.
next = node->next();
}
else {
// Change the position back to its original value.
node->setPosition(0);
}
}
InstNode* inst = node->as<InstNode>();
logNode(inst, kCodeIndentation);
uint32_t controlType = BaseInst::kControlNone;
ib.reset();
ASMJIT_PROPAGATE(static_cast<This*>(this)->onInst(inst, controlType, ib));
if (node->isInvoke()) {
ASMJIT_PROPAGATE(static_cast<This*>(this)->onInvoke(inst->as<InvokeNode>(), ib));
}
if (node->isFuncRet()) {
ASMJIT_PROPAGATE(static_cast<This*>(this)->onRet(inst->as<FuncRetNode>(), ib));
controlType = BaseInst::kControlReturn;
}
if (controlType == BaseInst::kControlJump) {
uint32_t fixedRegCount = 0;
for (RATiedReg& tiedReg : ib) {
RAWorkReg* workReg = _pass->workRegById(tiedReg.workId());
if (workReg->group() == BaseReg::kGroupGp) {
uint32_t useId = tiedReg.useId();
if (useId == BaseReg::kIdBad) {
useId = _pass->_scratchRegIndexes[fixedRegCount++];
tiedReg.setUseId(useId);
}
_curBlock->addExitScratchGpRegs(Support::bitMask<uint32_t>(useId));
}
}
}
ASMJIT_PROPAGATE(_pass->assignRAInst(inst, _curBlock, ib));
_blockRegStats.combineWith(ib._stats);
if (controlType != BaseInst::kControlNone) {
// Support for conditional and unconditional jumps.
if (controlType == BaseInst::kControlJump || controlType == BaseInst::kControlBranch) {
_curBlock->setLast(node);
_curBlock->addFlags(RABlock::kFlagHasTerminator);
_curBlock->makeConstructed(_blockRegStats);
if (!(inst->instOptions() & BaseInst::kOptionUnfollow)) {
// Jmp/Jcc/Call/Loop/etc...
uint32_t opCount = inst->opCount();
const Operand* opArray = inst->operands();
// Cannot jump anywhere without operands.
if (ASMJIT_UNLIKELY(!opCount))
return DebugUtils::errored(kErrorInvalidState);
if (opArray[opCount - 1].isLabel()) {
// Labels are easy for constructing the control flow.
LabelNode* labelNode;
ASMJIT_PROPAGATE(cc()->labelNodeOf(&labelNode, opArray[opCount - 1].as<Label>()));
RABlock* targetBlock = _pass->newBlockOrExistingAt(labelNode);
if (ASMJIT_UNLIKELY(!targetBlock))
return DebugUtils::errored(kErrorOutOfMemory);
targetBlock->makeTargetable();
ASMJIT_PROPAGATE(_curBlock->appendSuccessor(targetBlock));
}
else {
// Not a label - could be jump with reg/mem operand, which
// means that it can go anywhere. Such jumps must either be
// annotated so the CFG can be properly constructed, otherwise
// we assume the worst case - can jump to any basic block.
JumpAnnotation* jumpAnnotation = nullptr;
_curBlock->addFlags(RABlock::kFlagHasJumpTable);
if (inst->type() == BaseNode::kNodeJump)
jumpAnnotation = inst->as<JumpNode>()->annotation();
if (jumpAnnotation) {
uint64_t timestamp = _pass->nextTimestamp();
for (uint32_t id : jumpAnnotation->labelIds()) {
LabelNode* labelNode;
ASMJIT_PROPAGATE(cc()->labelNodeOf(&labelNode, id));
RABlock* targetBlock = _pass->newBlockOrExistingAt(labelNode);
if (ASMJIT_UNLIKELY(!targetBlock))
return DebugUtils::errored(kErrorOutOfMemory);
// Prevents adding basic-block successors multiple times.
if (!targetBlock->hasTimestamp(timestamp)) {
targetBlock->setTimestamp(timestamp);
targetBlock->makeTargetable();
ASMJIT_PROPAGATE(_curBlock->appendSuccessor(targetBlock));
}
}
ASMJIT_PROPAGATE(shareAssignmentAcrossSuccessors(_curBlock));
}
else {
ASMJIT_PROPAGATE(blocksWithUnknownJumps.append(_pass->allocator(), _curBlock));
}
}
}
if (controlType == BaseInst::kControlJump) {
// Unconditional jump makes the code after the jump unreachable,
// which will be removed instantly during the CFG construction;
// as we cannot allocate registers for instructions that are not
// part of any block. Of course we can leave these instructions
// as they are, however, that would only postpone the problem as
// assemblers can't encode instructions that use virtual registers.
_curBlock = nullptr;
}
else {
node = next;
if (ASMJIT_UNLIKELY(!node))
return DebugUtils::errored(kErrorInvalidState);
RABlock* consecutiveBlock;
if (node->type() == BaseNode::kNodeLabel) {
if (node->hasPassData()) {
consecutiveBlock = node->passData<RABlock>();
}
else {
consecutiveBlock = _pass->newBlock(node);
if (ASMJIT_UNLIKELY(!consecutiveBlock))
return DebugUtils::errored(kErrorOutOfMemory);
node->setPassData<RABlock>(consecutiveBlock);
}
}
else {
consecutiveBlock = _pass->newBlock(node);
if (ASMJIT_UNLIKELY(!consecutiveBlock))
return DebugUtils::errored(kErrorOutOfMemory);
}
_curBlock->addFlags(RABlock::kFlagHasConsecutive);
ASMJIT_PROPAGATE(_curBlock->prependSuccessor(consecutiveBlock));
_curBlock = consecutiveBlock;
_hasCode = false;
_blockRegStats.reset();
if (_curBlock->isConstructed())
break;
ASMJIT_PROPAGATE(_pass->addBlock(consecutiveBlock));
logBlock(_curBlock, kRootIndentation);
continue;
}
}
if (controlType == BaseInst::kControlReturn) {
_curBlock->setLast(node);
_curBlock->makeConstructed(_blockRegStats);
ASMJIT_PROPAGATE(_curBlock->appendSuccessor(_retBlock));
_curBlock = nullptr;
}
}
}
else if (node->type() == BaseNode::kNodeLabel) {
// Label - Basic-Block Management
// ------------------------------
if (!_curBlock) {
// If the current code is unreachable the label makes it reachable
// again. We may remove the whole block in the future if it's not
// referenced though.
_curBlock = node->passData<RABlock>();
if (_curBlock) {
// If the label has a block assigned we can either continue with
// it or skip it if the block has been constructed already.
if (_curBlock->isConstructed())
break;
}
else {
// No block assigned - create a new one and assign it.
_curBlock = _pass->newBlock(node);
if (ASMJIT_UNLIKELY(!_curBlock))
return DebugUtils::errored(kErrorOutOfMemory);
node->setPassData<RABlock>(_curBlock);
}
_curBlock->makeTargetable();
_hasCode = false;
_blockRegStats.reset();
ASMJIT_PROPAGATE(_pass->addBlock(_curBlock));
}
else {
if (node->hasPassData()) {
RABlock* consecutive = node->passData<RABlock>();
consecutive->makeTargetable();
if (_curBlock == consecutive) {
// The label currently processed is part of the current block. This
// is only possible for multiple labels that are right next to each
// other or labels that are separated by non-code nodes like directives
// and comments.
if (ASMJIT_UNLIKELY(_hasCode))
return DebugUtils::errored(kErrorInvalidState);
}
else {
// Label makes the current block constructed. There is a chance that the
// Label is not used, but we don't know that at this point. In the worst
// case there would be two blocks next to each other, it's just fine.
ASMJIT_ASSERT(_curBlock->last() != node);
_curBlock->setLast(node->prev());
_curBlock->addFlags(RABlock::kFlagHasConsecutive);
_curBlock->makeConstructed(_blockRegStats);
ASMJIT_PROPAGATE(_curBlock->appendSuccessor(consecutive));
ASMJIT_PROPAGATE(_pass->addBlock(consecutive));
_curBlock = consecutive;
_hasCode = false;
_blockRegStats.reset();
}
}
else {
// First time we see this label.
if (_hasCode) {
// Cannot continue the current block if it already contains some
// code. We need to create a new block and make it a successor.
ASMJIT_ASSERT(_curBlock->last() != node);
_curBlock->setLast(node->prev());
_curBlock->addFlags(RABlock::kFlagHasConsecutive);
_curBlock->makeConstructed(_blockRegStats);
RABlock* consecutive = _pass->newBlock(node);
if (ASMJIT_UNLIKELY(!consecutive))
return DebugUtils::errored(kErrorOutOfMemory);
consecutive->makeTargetable();
ASMJIT_PROPAGATE(_curBlock->appendSuccessor(consecutive));
ASMJIT_PROPAGATE(_pass->addBlock(consecutive));
_curBlock = consecutive;
_hasCode = false;
_blockRegStats.reset();
}
node->setPassData<RABlock>(_curBlock);
}
}
if (_curBlock && _curBlock != _lastLoggedBlock)
logBlock(_curBlock, kRootIndentation);
logNode(node, kRootIndentation);
// Unlikely: Assume that the exit label is reached only once per function.
if (ASMJIT_UNLIKELY(node->as<LabelNode>()->labelId() == _exitLabelId)) {
_curBlock->setLast(node);
_curBlock->makeConstructed(_blockRegStats);
ASMJIT_PROPAGATE(_pass->addExitBlock(_curBlock));
_curBlock = nullptr;
}
}
else {
// Other Nodes | Function Exit
// ---------------------------
logNode(node, kCodeIndentation);
if (node->type() == BaseNode::kNodeSentinel) {
if (node == _funcNode->endNode()) {
// Make sure we didn't flow here if this is the end of the function sentinel.
if (ASMJIT_UNLIKELY(_curBlock))
return DebugUtils::errored(kErrorInvalidState);
break;
}
}
else if (node->type() == BaseNode::kNodeFunc) {
// RAPass can only compile a single function at a time. If we
// encountered a function it must be the current one, bail if not.
if (ASMJIT_UNLIKELY(node != _funcNode))
return DebugUtils::errored(kErrorInvalidState);
// PASS if this is the first node.
}
else {
// PASS if this is a non-interesting or unknown node.
}
}
// Advance to the next node.
node = next;
// NOTE: We cannot encounter a NULL node, because every function must be
// terminated by a sentinel (`stop`) node. If we encountered a NULL node it
// means that something went wrong and this node list is corrupted; bail in
// such case.
if (ASMJIT_UNLIKELY(!node))
return DebugUtils::errored(kErrorInvalidState);
}
if (_pass->hasDanglingBlocks())
return DebugUtils::errored(kErrorInvalidState);
for (RABlock* block : blocksWithUnknownJumps)
handleBlockWithUnknownJump(block);
return _pass->initSharedAssignments(_sharedAssignmentsMap);
}