Add Fondry Project setting

madebygps · madebygps · commit a6a4a460392b · 2026-02-17T12:27:58.000-05:00
diff --git a/.env.sample b/.env.sample
@@ -9,4 +9,6 @@ OPENAI_MODEL=gpt-3.5-turbo
 # Configure for GitHub models: (GITHUB_TOKEN already exists inside Codespaces)
 GITHUB_MODEL=gpt-5-mini
 GITHUB_TOKEN=YOUR-GITHUB-PERSONAL-ACCESS-TOKEN
-OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317
+OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317
+# Optional: Set to log evaluation results to Azure AI Foundry for rich visualization
+AZURE_AI_PROJECT=https://YOUR-ACCOUNT.services.ai.azure.com/api/projects/YOUR-PROJECT
diff --git a/examples/agent_evaluation.py b/examples/agent_evaluation.py
@@ -2,6 +2,7 @@
 import json
 import logging
 import os
+import tempfile
 from typing import Annotated
 
 from agent_framework import ChatAgent, tool
@@ -13,6 +14,7 @@
     ResponseCompletenessEvaluator,
     TaskAdherenceEvaluator,
     ToolCallAccuracyEvaluator,
+    evaluate,
 )
 from azure.identity.aio import DefaultAzureCredential, get_bearer_token_provider
 from dotenv import load_dotenv
@@ -66,6 +68,10 @@
         model=os.environ.get("OPENAI_MODEL", "gpt-5-mini"),
     )
 
+# Optional: Set AZURE_AI_PROJECT in .env to log results to Azure AI Foundry.
+# Example: https://your-account.services.ai.azure.com/api/projects/your-project
+AZURE_AI_PROJECT = os.getenv("AZURE_AI_PROJECT")
+
 
 @tool
 def get_weather(
@@ -206,16 +212,29 @@ def convert_to_evaluator_messages(messages) -> list[dict]:
                     }
                 )
             elif c.type == "function_result":
+                if c.call_id:
+                    if content_items:
+                        evaluator_messages.append({"role": role, "content": content_items})
+                        content_items = []
+                    evaluator_messages.append(
+                        {
+                            "role": "tool",
+                            "tool_call_id": c.call_id,
+                            "content": [
+                                {
+                                    "type": "tool_result",
+                                    "tool_result": c.result,
+                                }
+                            ],
+                        }
+                    )
+                    continue
                 content_items.append(
                     {
                         "type": "tool_result",
                         "tool_result": c.result,
                     }
                 )
-                if c.call_id:
-                    evaluator_messages.append({"role": role, "tool_call_id": c.call_id, "content": content_items})
-                    content_items = []
-                    continue
             elif c.type == "text" and c.text:
                 content_items.append({"type": "text", "text": c.text})
         if content_items:
@@ -256,14 +275,12 @@ async def main():
     response = await agent.run(query)
     print(Panel(response.text, title="Agent Response", border_style="blue"))
 
-    # See: https://learn.microsoft.com/azure/ai-foundry/concepts/evaluation-evaluators/agent-evaluators
     eval_query = [
         {"role": "system", "content": AGENT_INSTRUCTIONS},
         {"role": "user", "content": [{"type": "text", "text": query}]},
     ]
     eval_response = convert_to_evaluator_messages(response.messages)
 
-    # ResponseCompletenessEvaluator compares the response against this ground truth
     ground_truth = (
         "A complete 3-day Tokyo trip itinerary from New York including: round-trip flight options with prices, "
         "hotel recommendations within nightly budget, hiking activities (e.g. Mt. Takao), museum visits "
@@ -273,44 +290,102 @@ async def main():
 
     logger.info("Running agent evaluators...")
 
-    # TODO: is_reasoning_model=True is needed because the GitHub Models endpoint rejects the max_tokens
-    # parameter that the SDK's prompty templates hardcode. This flag swaps it to max_completion_tokens.
-    # On Azure OpenAI this may not be necessary. Remove once the SDK updates its prompty templates.
     evaluator_kwargs = {"model_config": eval_model_config, "is_reasoning_model": True}
-    intent_evaluator = IntentResolutionEvaluator(**evaluator_kwargs)
-    completeness_evaluator = ResponseCompletenessEvaluator(**evaluator_kwargs)
-    adherence_evaluator = TaskAdherenceEvaluator(**evaluator_kwargs)
-    tool_accuracy_evaluator = ToolCallAccuracyEvaluator(**evaluator_kwargs)
-
-    intent_result = intent_evaluator(query=eval_query, response=eval_response)
-    completeness_result = completeness_evaluator(response=response.text, ground_truth=ground_truth)
-    adherence_result = adherence_evaluator(query=eval_query, response=eval_response)
-    tool_accuracy_result = tool_accuracy_evaluator(
-        query=eval_query, response=eval_response, tool_definitions=tool_definitions
-    )
-
-    # Evaluator output keys follow the pattern: {key}, {key}_result, {key}_reason
     result_keys = {
         "IntentResolution": "intent_resolution",
         "ResponseCompleteness": "response_completeness",
         "TaskAdherence": "task_adherence",
         "ToolCallAccuracy": "tool_call_accuracy",
     }
-    evaluation_results = {}
-    for name, result in [
-        ("IntentResolution", intent_result),
-        ("ResponseCompleteness", completeness_result),
-        ("TaskAdherence", adherence_result),
-        ("ToolCallAccuracy", tool_accuracy_result),
-    ]:
-        key = result_keys[name]
-        evaluation_results[name] = {
-            "score": result.get(key, "N/A"),
-            "result": result.get(f"{key}_result", "N/A"),
-            "reason": result.get(f"{key}_reason", result.get("error_message", "N/A")),
+
+    if AZURE_AI_PROJECT:
+        logger.info(f"Logging evaluation results to Azure AI project: {AZURE_AI_PROJECT}")
+
+        eval_data_row = {
+            "query": eval_query,
+            "response": eval_response,
+            "response_text": response.text,
+            "ground_truth": ground_truth,
+            "tool_definitions": tool_definitions,
         }
 
-    display_evaluation_results(evaluation_results)
+        with tempfile.NamedTemporaryFile(mode="w", suffix=".jsonl", delete=False, encoding="utf-8") as f:
+            f.write(json.dumps(eval_data_row) + "\n")
+            eval_data_file = f.name
+
+        try:
+            eval_result = evaluate(
+                data=eval_data_file,
+                evaluation_name="travel-planner-agent-eval",
+                evaluators={
+                    "intent_resolution": IntentResolutionEvaluator(**evaluator_kwargs),
+                    "response_completeness": ResponseCompletenessEvaluator(**evaluator_kwargs),
+                    "task_adherence": TaskAdherenceEvaluator(**evaluator_kwargs),
+                    "tool_call_accuracy": ToolCallAccuracyEvaluator(**evaluator_kwargs),
+                },
+                # ResponseCompletenessEvaluator expects a plain text response, not a message list,
+                # so we override its column mapping to use response_text and ground_truth.
+                # Other evaluators auto-map correctly since data keys match param names.
+                evaluator_config={
+                    "response_completeness": {
+                        "column_mapping": {
+                            "response": "${data.response_text}",
+                            "ground_truth": "${data.ground_truth}",
+                        }
+                    },
+                },
+                azure_ai_project=AZURE_AI_PROJECT,
+            )
+
+            # Parse results from the batch evaluate() output
+            evaluation_results = {}
+            rows = eval_result.get("rows", [])
+            row = rows[0] if rows else {}
+
+            for display_name, key in result_keys.items():
+                evaluation_results[display_name] = {
+                    "score": row.get(f"outputs.{key}.{key}", "N/A"),
+                    "result": row.get(f"outputs.{key}.{key}_result", "N/A"),
+                    "reason": row.get(f"outputs.{key}.{key}_reason", "N/A"),
+                }
+
+            display_evaluation_results(evaluation_results)
+
+            studio_url = eval_result.get("studio_url")
+            if studio_url:
+                print(f"\n[bold blue]View results in Azure AI Foundry:[/bold blue] {studio_url}")
+        finally:
+            os.unlink(eval_data_file)
+    else:
+        intent_evaluator = IntentResolutionEvaluator(**evaluator_kwargs)
+        completeness_evaluator = ResponseCompletenessEvaluator(**evaluator_kwargs)
+        adherence_evaluator = TaskAdherenceEvaluator(**evaluator_kwargs)
+        tool_accuracy_evaluator = ToolCallAccuracyEvaluator(**evaluator_kwargs)
+
+        intent_result = intent_evaluator(query=eval_query, response=eval_response, tool_definitions=tool_definitions)
+        completeness_result = completeness_evaluator(response=response.text, ground_truth=ground_truth)
+        adherence_result = adherence_evaluator(
+            query=eval_query, response=eval_response, tool_definitions=tool_definitions
+        )
+        tool_accuracy_result = tool_accuracy_evaluator(
+            query=eval_query, response=eval_response, tool_definitions=tool_definitions
+        )
+
+        evaluation_results = {}
+        for name, result in [
+            ("IntentResolution", intent_result),
+            ("ResponseCompleteness", completeness_result),
+            ("TaskAdherence", adherence_result),
+            ("ToolCallAccuracy", tool_accuracy_result),
+        ]:
+            key = result_keys[name]
+            evaluation_results[name] = {
+                "score": result.get(key, "N/A"),
+                "result": result.get(f"{key}_result", "N/A"),
+                "reason": result.get(f"{key}_reason", result.get("error_message", "N/A")),
+            }
+
+        display_evaluation_results(evaluation_results)
 
     if async_credential:
         await async_credential.close()
diff --git a/examples/spanish/agent_evaluation.py b/examples/spanish/agent_evaluation.py
