From 8610561e3cf0a3dce8f1c623adef46a0508a7e82 Mon Sep 17 00:00:00 2001 From: ThibaultLSDC Date: Fri, 15 Nov 2024 16:42:31 -0500 Subject: [PATCH 1/2] adding screenshot descriptions --- src/agentlab/agents/dynamic_prompting.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/agentlab/agents/dynamic_prompting.py b/src/agentlab/agents/dynamic_prompting.py index 73688f0f..a284fe37 100644 --- a/src/agentlab/agents/dynamic_prompting.py +++ b/src/agentlab/agents/dynamic_prompting.py @@ -446,8 +446,12 @@ def add_screenshot(self, prompt: BaseMessage) -> BaseMessage: if self.flags.use_screenshot: if self.flags.use_som: screenshot = self.obs["screenshot_som"] + prompt.add_text( + "\n## Screenshot:\nHere is a screenshot of the page, it is annotated with bounding boxes and corresponding bids:" + ) else: screenshot = self.obs["screenshot"] + prompt.add_text("\n## Screenshot:\nHere is a screenshot of the page:") img_url = image_to_jpg_base64_url(screenshot) prompt.add_image(img_url, detail=self.flags.openai_vision_detail) return prompt From 69c710933f43770d5cc7f28e45e3842fe714ed7f Mon Sep 17 00:00:00 2001 From: ThibaultLSDC Date: Fri, 15 Nov 2024 16:42:44 -0500 Subject: [PATCH 2/2] typo --- src/agentlab/experiments/study.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/agentlab/experiments/study.py b/src/agentlab/experiments/study.py index b42f0bb5..d9f24429 100644 --- a/src/agentlab/experiments/study.py +++ b/src/agentlab/experiments/study.py @@ -3,6 +3,7 @@ import pickle import re import uuid +from abc import ABC, abstractmethod from dataclasses import dataclass from datetime import datetime from pathlib import Path @@ -16,11 +17,7 @@ from agentlab.experiments import args from agentlab.experiments import reproducibility_util as repro from agentlab.experiments.exp_utils import RESULTS_DIR, add_dependencies -from agentlab.experiments.launch_exp import ( - find_incomplete, - non_dummy_count, - run_experiments, -) +from agentlab.experiments.launch_exp import find_incomplete, non_dummy_count, run_experiments logger = logging.getLogger(__name__) @@ -353,7 +350,7 @@ def _agents_on_benchmark( The logging level for individual jobs. Returns: - study: Study + list[ExpArgs]: The list of experiments to run. """ if not isinstance(agents, (list, tuple)):