in simple_cua_loop.py [0:0]
def handle_item(item, computer: Computer):
"""Handle each item; may cause a computer action + screenshot."""
if item["type"] == "message": # print messages
print(item["content"][0]["text"])
if item["type"] == "computer_call": # perform computer actions
action = item["action"]
action_type = action["type"]
action_args = {k: v for k, v in action.items() if k != "type"}
print(f"{action_type}({action_args})")
# give our computer environment action to perform
getattr(computer, action_type)(**action_args)
screenshot_base64 = computer.screenshot()
pending_checks = item.get("pending_safety_checks", [])
for check in pending_checks:
if not acknowledge_safety_check_callback(check["message"]):
raise ValueError(f"Safety check failed: {check['message']}")
# return value informs model of the latest screenshot
call_output = {
"type": "computer_call_output",
"call_id": item["call_id"],
"acknowledged_safety_checks": pending_checks,
"output": {
"type": "input_image",
"image_url": f"data:image/png;base64,{screenshot_base64}",
},
}
# additional URL safety checks for browser environments
if computer.get_environment() == "browser":
current_url = computer.get_current_url()
call_output["output"]["current_url"] = current_url
check_blocklisted_url(current_url)
return [call_output]
return []