# toolbox.yaml — illustrative structure, verify against the current sample
name: my-toolbox
description: Order operations, customer lookup, and fulfilment tools
toolSearch:
enabled: true
# Pinned tools skip retrieval entirely and are always in context.
# Keep this list short — every pin is a permanent token cost.
pinned:
- get_order_status
- search_customers
autoPin:
enabled: true
minCallsPerWindow: 25
tools:
- name: get_order_status
source: mcp
server: orders-mcp
# Retrieval context: describe the tool the way your team describes it,
# including the vocabulary users actually type.
searchContext: >
Look up the current fulfilment state of a single order. Use when the
user mentions an order number, tracking number, "where is my package",
or asks whether something shipped.
- name: issue_refund
source: mcp
server: billing-mcp
searchContext: >
Issue a full or partial refund against a completed order. Requires an
order ID and an amount. Do not use for cancellations of unshipped
orders — use cancel_order instead.
from azure.core.settings import settings
from opentelemetry import trace
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import SimpleSpanProcessor, ConsoleSpanExporter
from azure.ai.projects.telemetry import AIProjectInstrumentor
settings.tracing_implementation = "opentelemetry"
span_exporter = ConsoleSpanExporter()
tracer_provider = TracerProvider()
tracer_provider.add_span_processor(SimpleSpanProcessor(span_exporter))
trace.set_tracer_provider(tracer_provider)
# Emits GenAI spans for every agent and model call in this process
AIProjectInstrumentor().instrument()
import os
import json
from dataclasses import dataclass, field
from statistics import mean
@dataclass
class RunResult:
config: str
input_tokens: list = field(default_factory=list)
turns: list = field(default_factory=list)
correct_tool: list = field(default_factory=list)
TASKS = [
# (prompt, tool the model should end up calling)
("Where is order 88231?", "get_order_status"),
("Refund the second item on order 88231", "issue_refund"),
("Cancel order 90114, it hasn't shipped", "cancel_order"),
("Which customers in Ohio ordered twice?", "search_customers"),
("Send the June invoice to billing@acme.com", "email_invoice"),
]
def score_run(spans, expected_tool):
"""Pull token usage and the actually-invoked tool out of collected spans."""
input_tokens = sum(
s.attributes.get("gen_ai.usage.input_tokens", 0) for s in spans
)
invoked = [
s.attributes.get("gen_ai.tool.name")
for s in spans
if s.attributes.get("gen_ai.tool.name")
]
# With Tool Search the target arrives as a call_tool argument, so unwrap it.
resolved = [
json.loads(s.attributes["gen_ai.tool.arguments"]).get("name", n)
if n == "call_tool" else n
for s, n in zip(spans, invoked)
]
return input_tokens, len(invoked), expected_tool in resolved
def report(results):
for r in results:
print(f"\n{r.config}")
print(f" mean input tokens/task : {mean(r.input_tokens):>8,.0f}")
print(f" mean turns/task : {mean(r.turns):>8.1f}")
print(f" tool selection accuracy: {mean(r.correct_tool):>8.1%}")
