Standardize Output Dictionaries, Mock API Calls, Math (GSM8K, SVAMP, TabMWP) for Reflexion

agential/cog/agent/react.py

@@ -96,13 +96,17 @@ def generate(
             )
 
             # Observe.
-            obs = self.strategy.generate_observation(
+            obs, external_tool_info = self.strategy.generate_observation(
                 idx=idx, action_type=action_type, query=query
             )
 
             out.append(
                 self.strategy.create_output_dict(
-                    thought=thought, action_type=action_type, query=query, obs=obs
+                    thought=thought,
+                    action_type=action_type,
+                    query=query,
+                    obs=obs,
+                    external_tool_info=external_tool_info,
                 )
             )
 

agential/cog/agent/reflexion.py

@@ -265,7 +265,7 @@ def _generate_react(
             )
 
             # Observe.
-            is_correct, obs = self.strategy.generate_observation(
+            is_correct, obs, external_tool_info = self.strategy.generate_observation(
                 step_idx=step_idx,
                 action_type=action_type,
                 query=query,
@@ -278,6 +278,7 @@ def _generate_react(
                     action_type=action_type,
                     query=query,
                     obs=obs,
+                    external_tool_info=external_tool_info,
                     is_correct=is_correct,
                 )
             )

agential/cog/eval/reflexion.py

@@ -3,7 +3,7 @@
 from agential.utils.parse import normalize_answer
 
 
-def EM(answer: str, key: str) -> bool:
+def EM(answer: str, key: str, normalize: bool = True) -> bool:
     """Compares two strings, `answer` and `key`, after normalizing them.
 
     The Exact Match grading 'metric' compares for an exact match between 2 strings
@@ -12,8 +12,14 @@ def EM(answer: str, key: str) -> bool:
     Args:
         answer (str): A string to be compared with `key`.
         key (str): A string to be compared with `answer`.
+        normalize (bool): If True, then normalize answer and key. Defaults to True.
 
     Returns:
         bool: True if the normalized `answer` and `key` match, else False.
     """
+    if answer is None:
+        return False
+
+    if not normalize:
+        return answer == key
     return normalize_answer(answer) == normalize_answer(key)