project/alcatraz/alcatraz/clusters/local.py (9 lines): - line 489: # TODO footgun warning: if user provides an image that has full acr prefix but we see they have a local image of the same name (without the prefix) and without a tag to associate it with the remote image, warn the user or maybe just skip pulling... - line 1061: del language # TODO implement language support - line 1176: # TODO handle comm_open messages - line 1290: # print(success) # TODO when does this fail w/o throwing an exception?? - line 1319: # TODO clean exception for path not existing vs other problems - line 1459: del timeout # TODO implement timeout - line 1494: # TODO be mindful of edge cases here, make more robust in future - line 1860: "lsblk -o NAME,SIZE,TYPE | grep '4T' | grep 'part' | awk '{print $1}'" # TODO dont - line 2050: # TODO this leads to pyproject.toml (4 lines): - line 37: # TODO: enable the commented out checks - line 44: "B905", # TODO: a useful lint, but need ruff to release my autofix - line 45: "B028", # TODO: looks useful, disabled to minimize code change in ruff 0.8.2 upgrade - line 111: # TODO consider enable this, disabled to minimize changes when moving off isort to ruff project/paperbench/paperbench/scripts/alcatraz_services.py (3 lines): - line 41: # TODO: this is a hack because sometimes the submission dir is nested several levels deep - line 299: # TODO Don't override existing env vars if they're already set - line 300: # TODO verify the API keys passed to the solver are set project/paperbench/paperbench/nano/eval.py (2 lines): - line 704: # TODO: Move this to the `get_instances` method in `PaperBench`. - line 1015: # TODO: This is a hack to load the iterative instructions. We should refactor this project/paperbench/paperbench/rubric/tasks.py (2 lines): - line 46: # TODO: bring back error after we finalize all rubrics - line 54: # TODO: bring back error after we finalize all rubrics project/paperbench/paperbench/agents/utils.py (1 line): - line 15: # TODO: Delete this; it's essentially wrapping a few constants in a function. project/paperbench/paperbench/judge/judge.py (1 line): - line 688: # TODO we might want to pass in function signatures as well project/paperbench/paperbench/judge/judge_eval/registry.py (1 line): - line 62: # TODO change this to a more appropriate default (e.g. ~/.cache/hp/judge_eval/) at release (when we will not be e.g. committing/directly redistributing the code) project/nanoeval/nanoeval/library_config.py (1 line): - line 229: # TODO add more metrics project/paperbench/paperbench/infra/alcatraz.py (1 line): - line 86: computer: ComputerInterface, agent_log_path: str = "/home/logs/agent.log" # TODO use .env