Adding descriptive prompts for screenshot/som

src/agentlab/agents/dynamic_prompting.py

@@ -446,8 +446,12 @@ def add_screenshot(self, prompt: BaseMessage) -> BaseMessage:
         if self.flags.use_screenshot:
             if self.flags.use_som:
                 screenshot = self.obs["screenshot_som"]
+                prompt.add_text(
+                    "\n## Screenshot:\nHere is a screenshot of the page, it is annotated with bounding boxes and corresponding bids:"
+                )
             else:
                 screenshot = self.obs["screenshot"]
+                prompt.add_text("\n## Screenshot:\nHere is a screenshot of the page:")
             img_url = image_to_jpg_base64_url(screenshot)
             prompt.add_image(img_url, detail=self.flags.openai_vision_detail)
         return prompt

src/agentlab/experiments/study.py

@@ -3,6 +3,7 @@
 import pickle
 import re
 import uuid
+from abc import ABC, abstractmethod
 from dataclasses import dataclass
 from datetime import datetime
 from pathlib import Path
@@ -16,11 +17,7 @@
 from agentlab.experiments import args
 from agentlab.experiments import reproducibility_util as repro
 from agentlab.experiments.exp_utils import RESULTS_DIR, add_dependencies
-from agentlab.experiments.launch_exp import (
-    find_incomplete,
-    non_dummy_count,
-    run_experiments,
-)
+from agentlab.experiments.launch_exp import find_incomplete, non_dummy_count, run_experiments
 
 logger = logging.getLogger(__name__)
 
@@ -353,7 +350,7 @@ def _agents_on_benchmark(
             The logging level for individual jobs.
 
     Returns:
-        study: Study
+        list[ExpArgs]: The list of experiments to run.
     """
 
     if not isinstance(agents, (list, tuple)):