in src/utils/viewer/viewer.cpp [1716:1901]
void Viewer::mousePressEvent(MouseEvent& event) {
// get mouse position, appropriately scaled for Retina Displays
auto viewportPoint = getMousePosition(event.position());
if (mouseInteractionMode == MouseInteractionMode::LOOK) {
if (event.button() == MouseEvent::Button::Right) {
// if shift pressed w/right click in look mode, get object ID and
// create visualization
if (event.modifiers() & MouseEvent::Modifier::Shift) {
// cannot use the default framebuffer, so setup another framebuffer,
// also, setup the color attachment for rendering, and remove the
// visualizer for the previously picked object
objectPickingHelper_->prepareToDraw();
// redraw the scene on the object picking framebuffer
esp::gfx::RenderCamera::Flags flags =
esp::gfx::RenderCamera::Flag::UseDrawableIdAsObjectId;
if (simulator_->isFrustumCullingEnabled())
flags |= esp::gfx::RenderCamera::Flag::FrustumCulling;
for (auto& it : activeSceneGraph_->getDrawableGroups()) {
renderCamera_->draw(it.second, flags);
}
// Read the object Id - takes unscaled mouse position, and scales it in
// objectPicker
unsigned int pickedObject =
objectPickingHelper_->getObjectId(event.position(), windowSize());
// if an object is selected, create a visualizer
createPickedObjectVisualizer(pickedObject);
return;
} // drawable selection
// add primitive w/ right click if a collision object is hit by a raycast
if (simulator_->getPhysicsSimulationLibrary() !=
esp::physics::PhysicsManager::PhysicsSimulationLibrary::NoPhysics) {
auto ray = renderCamera_->unproject(viewportPoint);
esp::physics::RaycastResults raycastResults = simulator_->castRay(ray);
if (raycastResults.hasHits()) {
// If VHACD is enabled, and Ctrl + Right Click is used, voxelized
// the object clicked on.
#ifdef ESP_BUILD_WITH_VHACD
if (event.modifiers() & MouseEvent::Modifier::Ctrl) {
auto objID = raycastResults.hits[0].objectId;
displayVoxelField(objID);
return;
}
#endif
addPrimitiveObject();
auto existingObjectIDs = simulator_->getExistingObjectIDs();
// use the bounding box to create a safety margin for adding the
// object
auto rigidObjMgr = simulator_->getRigidObjectManager();
auto obj = rigidObjMgr->getObjectByID(existingObjectIDs.back());
float boundingBuffer =
obj->getSceneNode()->computeCumulativeBB().size().max() / 2.0 +
0.04;
obj->setTranslation(raycastResults.hits[0].point +
raycastResults.hits[0].normal * boundingBuffer);
obj->setRotation(esp::core::randomRotation());
}
}
// end add primitive w/ right click
} else if (event.button() == MouseEvent::Button::Left) {
// if shift-click is pressed, display semantic ID and name if exists
if (event.modifiers() & MouseEvent::Modifier::Shift) {
semanticTag_ = "";
// get semantic scene
auto semanticScene = simulator_->getSemanticScene();
// only enable for HM3D for now
if ((semanticScene) && (semanticScene->hasVertColorsDefined())) {
auto semanticObjects = semanticScene->objects();
std::string sensorId = "semantic_camera";
simulator_->drawObservation(defaultAgentId_, sensorId);
esp::sensor::Observation observation;
simulator_->getAgentObservation(defaultAgentId_, sensorId,
observation);
uint32_t desiredIdx =
(viewportPoint[0] +
(observation.buffer->shape[1] *
(observation.buffer->shape[0] - viewportPoint[1])));
uint32_t objIdx = Corrade::Containers::arrayCast<int>(
observation.buffer->data)[desiredIdx];
// TODO : Change core::buffer to magnum image, then we can simplify
// access uint32_t objIdx =
// observation.buffer->pixels<uint32_t>().flipped<0>()[viewportPoint[1]][viewportPoint[0]];
// subtract 1 to align with semanticObject array
--objIdx;
std::string tmpStr = "Unknown";
if ((objIdx >= 0) && (objIdx < semanticObjects.size())) {
tmpStr = semanticObjects[objIdx]->id();
}
semanticTag_ =
Cr::Utility::formatString("id:{}:{}", (objIdx + 1), tmpStr);
ESP_WARNING() << "Data point @ idx : " << objIdx
<< ": Object name :" << semanticTag_;
}
}
}
} else if (mouseInteractionMode == MouseInteractionMode::GRAB) {
// GRAB mode
if (simulator_->getPhysicsSimulationLibrary() !=
esp::physics::PhysicsManager::PhysicsSimulationLibrary::NoPhysics) {
auto ray = renderCamera_->unproject(viewportPoint);
esp::physics::RaycastResults raycastResults = simulator_->castRay(ray);
if (raycastResults.hasHits()) {
int hitObject = esp::ID_UNDEFINED;
Mn::Quaternion objectFrame;
Mn::Vector3 objectPivot;
int aoLink = esp::ID_UNDEFINED;
auto hitInfo = raycastResults.hits[0]; // first hit
// check if ao
if (hitInfo.objectId != esp::ID_UNDEFINED) {
// we hit an non-stage collision object
auto roMngr = simulator_->getRigidObjectManager();
auto aoMngr = simulator_->getArticulatedObjectManager();
auto ro = roMngr->getObjectByID(hitInfo.objectId);
auto ao = aoMngr->getObjectByID(hitInfo.objectId);
if (ro != nullptr) {
// grabbed an object
hitObject = hitInfo.objectId;
objectPivot = ro->getTransformation().inverted().transformPoint(
hitInfo.point);
objectFrame = ro->getRotation().inverted();
} else if (ao != nullptr) {
// grabbed the base link
hitObject = hitInfo.objectId;
objectPivot = ao->getTransformation().inverted().transformPoint(
hitInfo.point);
objectFrame = ao->getRotation().inverted();
} else {
for (auto aoHandle : aoMngr->getObjectHandlesBySubstring()) {
auto ao = aoMngr->getObjectByHandle(aoHandle);
auto linkToObjIds = ao->getLinkObjectIds();
if (linkToObjIds.count(hitInfo.objectId) > 0) {
// got a link
aoLink = linkToObjIds.at(hitInfo.objectId);
objectPivot = ao->getLinkSceneNode(aoLink)
->transformation()
.inverted()
.transformPoint(hitInfo.point);
objectFrame =
ao->getLinkSceneNode(aoLink)->rotation().inverted();
hitObject = ao->getID();
break;
}
}
} // done checking for AO
if (hitObject >= 0) {
esp::physics::RigidConstraintSettings constraintSettings;
constraintSettings.objectIdA = hitObject;
constraintSettings.linkIdA = aoLink;
constraintSettings.pivotA = objectPivot;
constraintSettings.frameA =
objectFrame.toMatrix() *
defaultAgent_->node().rotation().toMatrix();
constraintSettings.frameB =
defaultAgent_->node().rotation().toMatrix();
constraintSettings.pivotB = hitInfo.point;
// by default use a point 2 point constraint
if (event.button() == MouseEvent::Button::Right) {
constraintSettings.constraintType =
esp::physics::RigidConstraintType::Fixed;
}
mouseGrabber_ = std::make_unique<MouseGrabber>(
constraintSettings,
(hitInfo.point - renderCamera_->node().absoluteTranslation())
.length(),
*simulator_);
} else {
ESP_DEBUG() << "Oops, couldn't find the hit object. That's odd.";
}
} // end didn't hit the scene
} // end has raycast hit
} // end has physics enabled
} // end GRAB
previousMousePoint = viewportPoint;
event.setAccepted();
redraw();
}