{ "title": "Agent observability and evaluation platforms compared", "page": "https://prefactor.tech/compare/agent-observability-evaluation-platforms", "publisher": "Prefactor (also listed below, as a disclosed reference)", "reviewedAt": "2026-10-20", "method": "Each vendor was researched against its own documentation, pricing page, changelog and GitHub repositories on the review date; third-party sources were used only for ownership changes. 'yes' = documented by the vendor, 'partial' = limited (e.g. to model calls, one plan or a preview), 'no' = absent, 'unknown' = not stated.", "criteria": [ { "key": "llmTracing", "label": "LLM call tracing", "group": "Observe" }, { "key": "agentTracing", "label": "Multi-step agent tracing (tool calls, nested spans)", "group": "Observe" }, { "key": "costTracking", "label": "Cost tracking", "group": "Observe" }, { "key": "openTelemetry", "label": "OpenTelemetry support", "group": "Observe" }, { "key": "offlineEvals", "label": "Offline evals (datasets, experiments)", "group": "Evaluate" }, { "key": "onlineEvals", "label": "Online evals on production traffic", "group": "Evaluate" }, { "key": "llmAsJudge", "label": "LLM-as-a-judge scoring", "group": "Evaluate" }, { "key": "codeEvals", "label": "Code-based evaluators", "group": "Evaluate" }, { "key": "humanReview", "label": "Human review and annotation", "group": "Evaluate" }, { "key": "promptTooling", "label": "Prompt versioning and playground", "group": "Evaluate" }, { "key": "runtimeControls", "label": "Runtime controls on live agent actions", "group": "Act" }, { "key": "gatewayProxy", "label": "Gateway or proxy in the request path", "group": "Act" }, { "key": "openSource", "label": "Open source", "group": "Deploy" }, { "key": "selfHosting", "label": "Self-hosting", "group": "Deploy" } ], "vendors": [ { "name": "Langfuse", "shortName": "Langfuse", "slug": "langfuse", "website": "https://langfuse.com", "definition": "Langfuse is an open-source LLM engineering platform for tracing, evaluating and versioning prompts in LLM and agent applications, available as a hosted cloud service or self-hosted for free.", "ownership": "ClickHouse, Inc., which acquired Langfuse in January 2026. Langfuse said its roadmap and licence would not change and that self-hosting stays a first-class option.", "statusChange": "Acquired by ClickHouse; licence and self-hosting unchanged, per Langfuse", "bestFor": "Teams that want an open-source, self-hostable platform combining tracing, prompt versioning and offline and online evaluation for LLM and agent applications.", "capabilities": { "llmTracing": { "label": "LLM call tracing", "value": "yes", "detail": "Traces capture LLM calls and non-LLM steps such as retrieval, embeddings and API calls, with a timeline view for latency debugging. Multi-modal tracing covers text and images.", "source": "https://langfuse.com/docs" }, "agentTracing": { "label": "Multi-step agent tracing (tool calls, nested spans)", "value": "yes", "detail": "Nested observations, agent graph view (Aggregated and Expanded modes, July 2026), and sessions that group multi-turn conversations and multi-step agentic workflows.", "source": "https://langfuse.com/docs" }, "costTracking": { "label": "Cost tracking", "value": "yes", "detail": "Token usage and cost per generation, either ingested or inferred from model definitions, with custom model prices; per-user cost via userId.", "source": "https://langfuse.com/docs/observability/features/token-and-cost-tracking" }, "openTelemetry": { "label": "OpenTelemetry support", "value": "yes", "detail": "Acts as an OTLP backend at /api/public/otel (HTTP/JSON and HTTP/protobuf; gRPC not supported). OpenLLMetry and OpenLIT are named as compatible; OpenInference-style attributes are mapped, though OpenInference is not named as a supported library. Langfuse's own tracing is built on OpenTelemetry.", "source": "https://langfuse.com/integrations/native/opentelemetry" }, "offlineEvals": { "label": "Offline evals (datasets, experiments)", "value": "yes", "detail": "Datasets and experiments; since April 2026 experiments are a top-level feature that can run against dataset items, production traces or locally generated data, with side-by-side comparison and regression filters. A GitHub Action (May 2026) runs experiments in CI and fails the workflow below a score threshold.", "source": "https://langfuse.com/changelog/2026-04-13-experiments-rebuild" }, "onlineEvals": { "label": "Online evals on production traffic", "value": "yes", "detail": "Evaluator rules score live production observations automatically, with a configurable sampling rate. Evaluator backfills (September 2026) can score past observations over a window of up to six months. Evaluations run asynchronously.", "source": "https://langfuse.com/docs/evaluation/evaluation-methods/llm-as-a-judge" }, "llmAsJudge": { "label": "LLM-as-a-judge scoring", "value": "yes", "detail": "Managed LLM-as-a-Judge evaluators on live observations and on experiment runs, including categorical scores and multi-modal evaluators. A separate low-cost 'Jev-as-a-judge' classifier option was added in September 2026.", "source": "https://langfuse.com/docs/evaluation/evaluation-methods/llm-as-a-judge" }, "codeEvals": { "label": "Code-based evaluators", "value": "yes", "detail": "Code Evaluators (launched 28 May 2026): Python or TypeScript evaluate functions written in the UI, attachable to live observations or dataset experiments, stored as scores. Custom scores can also be pushed via API/SDK.", "source": "https://langfuse.com/blog/2026-05-31-langfuse-may-update" }, "humanReview": { "label": "Human review and annotation", "value": "yes", "detail": "Annotation queues let reviewers score traces, observations or sessions against defined score configs; user feedback can be collected via browser SDK, server SDKs or API.", "source": "https://langfuse.com/docs/evaluation/evaluation-methods/annotation-queues" }, "promptTooling": { "label": "Prompt versioning and playground", "value": "yes", "detail": "prompt versioning with versioning and label-based deployment without code changes, prompts linked to traces, metric comparison across versions, and a playground.", "source": "https://langfuse.com/docs" }, "runtimeControls": { "label": "Runtime controls on live agent actions", "value": "no", "detail": "Langfuse does not block or enforce guardrails on live requests. Its security docs state run-time protection comes from separate security libraries and Langfuse handles ex-post evaluation of those measures. Threshold alerts to Slack, GitHub Actions or webhooks are available, but these notify rather than block.", "source": "https://langfuse.com/docs/security-and-guardrails" }, "gatewayProxy": { "label": "Gateway or proxy in the request path", "value": "no", "detail": "Not a proxy and not required in the request path; spans are exported asynchronously from SDKs or OpenTelemetry, with an optional OTel Collector. It can ingest traces from third-party gateways such as LiteLLM Proxy and Vercel AI Gateway.", "source": "https://langfuse.com/integrations/native/opentelemetry" }, "openSource": { "label": "Open source", "value": "yes", "detail": "MIT licence for the repository except the 'ee' folders, which hold add-on features that need a licence key (e.g. Organization Creators, Instance Management API, UI customisation). GitHub's licence detector reports NOASSERTION because of this mixed licence.", "source": "https://github.com/langfuse/langfuse" }, "selfHosting": { "label": "Self-hosting", "value": "yes", "detail": "Free self-hosting via Docker Compose (testing) or production on Kubernetes (Helm), AWS, Azure or GCP; stack is Postgres, ClickHouse, Redis/Valkey and S3-compatible blob storage. Some add-on features require a licence key.", "source": "https://langfuse.com/self-hosting" } }, "openSource": "Yes. The repository is MIT-licensed, apart from add-on features in its ee folders that need a licence key. Self-hosting on Docker or Kubernetes is free.", "runtimeControls": "No. Langfuse evaluates runs after they happen. Its documentation says run-time protection comes from separate libraries, and Langfuse scores those measures afterwards.", "pricing": { "summary": "Hobby is free with 50k units a month and 30 days of data, and self-hosting is free. Core is $29 a month for 100k units, then $8 per 100k. Pro is $199 a month, and Enterprise is $2,499 a month or custom.", "source": "https://langfuse.com/pricing", "checkedAt": "2026-10-20" }, "sdks": [ "Python", "TypeScript/JavaScript", "Any language via OpenTelemetry (OTLP)" ], "frameworkIntegrations": [ "LangChain", "LangGraph", "LlamaIndex", "OpenAI Agents SDK", "Vercel AI SDK", "CrewAI", "Pydantic AI", "AutoGen", "Google ADK", "Strands Agents", "Mastra", "Claude Agent SDK", "DSPy", "LiteLLM" ], "limits": [ "No runtime blocking or guardrail enforcement; evaluations run asynchronously after the fact.", "OTLP ingest supports HTTP only; gRPC is not supported.", "Some self-hosted add-on features sit under the 'ee' folders and need a licence key.", "Langfuse Cloud becomes v4-only on 16 November 2026, which may require migration work for existing users." ] }, { "name": "LangSmith", "shortName": "LangSmith", "slug": "langsmith", "website": "https://www.langchain.com/langsmith", "definition": "LangSmith is LangChain's commercial platform for tracing, monitoring and evaluating LLM agents, with prompt versioning, agent deployment, an LLM gateway in beta and Engine, an agent that drafts fixes from production traces.", "ownership": "LangChain, Inc., a privately held company that raised a $125M Series B at a $1.25B valuation in October 2025. No acquisition has been announced.", "statusChange": null, "bestFor": "Teams building on LangChain or LangGraph, or wanting tracing, evaluation, prompt versioning and agent deployment from one vendor, with self-hosting available on an Enterprise contract.", "capabilities": { "llmTracing": { "label": "LLM call tracing", "value": "yes", "detail": "Traces LLM calls via environment variables, framework integrations, SDK decorators (@traceable) and OpenAI/Anthropic wrappers. Dashboards cover cost, tokens, latency percentiles and error rates.", "source": "https://docs.langchain.com/langsmith/home" }, "agentTracing": { "label": "Multi-step agent tracing (tool calls, nested spans)", "value": "yes", "detail": "Nested runs within traces, threads for multi-turn sessions, tool and trajectory monitoring; Trajectory view (opened to all users September 2026) combines messages across main agent and subagents.", "source": "https://www.langchain.com/langsmith" }, "costTracking": { "label": "Cost tracking", "value": "yes", "detail": "Automatic cost from token counts and a model pricing table (most OpenAI, Anthropic and Gemini models) for LangChain, @traceable and wrapper calls; manual costs on any run, including tool calls; custom price overrides. Price changes are not applied retroactively.", "source": "https://docs.langchain.com/langsmith/cost-tracking" }, "openTelemetry": { "label": "OpenTelemetry support", "value": "yes", "detail": "Ingests OTLP traces at api.smith.langchain.com/otel; maps OpenInference, OpenLLMetry and Traceloop attributes; SDK can also export to other OTel backends or OTel-only mode.", "source": "https://docs.langchain.com/langsmith/trace-with-opentelemetry" }, "offlineEvals": { "label": "Offline evals (datasets, experiments)", "value": "yes", "detail": "Datasets and experiments for comparing versions, benchmarking, unit, regression and backtesting runs; production traces can be added to datasets.", "source": "https://docs.langchain.com/langsmith/evaluation" }, "onlineEvals": { "label": "Online evals on production traffic", "value": "yes", "detail": "Online evaluators (LLM-as-judge or code) run on live traffic through automation rules with filters and sampling rates, including on multi-turn threads.", "source": "https://docs.langchain.com/langsmith/evaluation" }, "llmAsJudge": { "label": "LLM-as-a-judge scoring", "value": "yes", "detail": "LLM-as-judge evaluators offline and online (reference-free online); pairwise evaluation also supported. Pricing page lists 'Tuned Evaluators' at 0.015 LSU per run.", "source": "https://docs.langchain.com/langsmith/evaluation" }, "codeEvals": { "label": "Code-based evaluators", "value": "yes", "detail": "Rule-based code evaluators for offline and online use (e.g. format validation, safety checks, heuristics).", "source": "https://docs.langchain.com/langsmith/evaluation" }, "humanReview": { "label": "Human review and annotation", "value": "yes", "detail": "Annotation queues: single-run and pairwise queues, rubrics, required reviewer counts, assigned reviewers, item reservations; items can be added to datasets.", "source": "https://docs.langchain.com/langsmith/annotation-queues" }, "promptTooling": { "label": "Prompt versioning and playground", "value": "yes", "detail": "Prompt & Context Hub with commit-based versioning, tags, webhooks and a public hub; Playground for testing prompts against model configurations and custom endpoints.", "source": "https://docs.langchain.com/langsmith/prompt-engineering" }, "runtimeControls": { "label": "Runtime controls on live agent actions", "value": "partial", "detail": "The LLM Gateway (beta; public beta from 30 July 2026) sits in the model-call path and enforces spend caps (returns HTTP 402), model access policies and PII/secret redaction guard policies (custom regex added October 2026). It does not cover tool calls; tool and MCP gateways are described as planned. Automation rules act on recorded traces and do not block agent actions.", "source": "https://www.langchain.com/blog/introducing-llm-gateway" }, "gatewayProxy": { "label": "Gateway or proxy in the request path", "value": "no", "detail": "Not required: tracing uses an asynchronous SDK callback or OTel export, out of the request path. An optional LLM Gateway (beta) can be placed in the request path by changing the provider base_url.", "source": "https://docs.langchain.com/langsmith/llm-gateway-quickstart" }, "openSource": { "label": "Open source", "value": "partial", "detail": "The LangSmith platform is proprietary. Client SDKs are MIT-licensed (langsmith-python, langsmith-javascript; the former langsmith-sdk monorepo was archived 16 October 2026). Helm charts are Apache-2.0. The LangChain and LangGraph frameworks are open source but are separate products.", "source": "https://github.com/langchain-ai/langsmith-sdk" }, "selfHosting": { "label": "Self-hosting", "value": "yes", "detail": "Self-hosted LangSmith is an add-on to the Enterprise plan (licence key required), deployed on Kubernetes via Helm or Terraform. BYOC (LangChain-run control plane, customer data plane) is also Enterprise-only.", "source": "https://docs.langchain.com/langsmith/self-hosted" } }, "openSource": "Partly. The client SDKs are MIT-licensed and the LangChain frameworks are open source, but the LangSmith platform is proprietary. Self-hosting requires an Enterprise contract.", "runtimeControls": "Partly. The LLM Gateway, in public beta since July 2026, can cap spend, restrict models and redact personal data in model calls. It does not cover tool calls; LangChain describes tool and MCP gateways as planned.", "pricing": { "summary": "Developer is free for one seat and 5k base traces a month. Plus is $39 per seat a month with 10k base traces; the overage rate is not published. Enterprise is a custom annual contract.", "source": "https://www.langchain.com/pricing", "checkedAt": "2026-10-20" }, "sdks": [ "Python", "TypeScript", "Go", "Java" ], "frameworkIntegrations": [ "LangChain", "LangGraph", "Deep Agents", "OpenAI Agents SDK", "Claude Agent SDK", "CrewAI", "AutoGen", "Google ADK", "Mastra", "Microsoft Agent Framework", "PydanticAI", "Semantic Kernel", "Strands Agents", "Vercel AI SDK", "LiteLLM", "LiveKit", "Pipecat", "OpenTelemetry" ], "limits": [ "Platform is closed source; self-hosting and BYOC require the Enterprise plan.", "Runtime controls are limited to model calls through the LLM Gateway (beta); tool-call and MCP gateways are described as planned.", "Free tier is one seat and 5k base traces/month; base traces are retained for 14 days, extended retention capped at 180 days on SaaS.", "Per-trace overage rates are not published on the pricing page.", "Automatic cost tracking covers listed providers; others need token and cost data supplied manually." ] }, { "name": "Arize", "shortName": "Arize", "slug": "arize", "website": "https://arize.com", "definition": "Arize is an AI observability and evaluation vendor with two products: Phoenix, a source-available tool you self-host for free, and Arize AX, a commercial platform for continuous production evals, monitoring and alerting.", "ownership": "Dynatrace, which completed its acquisition of Arize on 1 October 2026. Dynatrace says Phoenix and AX will continue and that Arize capabilities will be integrated into Dynatrace over time.", "statusChange": "Acquisition by Dynatrace completed on 1 October 2026; Phoenix and AX continue", "bestFor": "Teams wanting OpenTelemetry-based tracing and evaluation with a free self-hosted option (Phoenix) and a path to a hosted platform with continuous online evals, monitors and alerting (AX), including organisations already using Dynatrace.", "capabilities": { "llmTracing": { "label": "LLM call tracing", "value": "yes", "detail": "Both products. Phoenix traces LLM applications via OpenTelemetry and accepts OTLP; AX ingests the same OpenTelemetry/OpenInference instrumentation.", "source": "https://arize.com/docs/phoenix/tracing/llm-traces" }, "agentTracing": { "label": "Multi-step agent tracing (tool calls, nested spans)", "value": "yes", "detail": "Both products trace multi-step runs, tool calls and sessions. AX adds Agent Trajectory, an interactive graph of which agents call which, set automatically for LangGraph, CrewAI, AG2 and others or via graph.node.id attributes. Phoenix added DECISION spans for routers and classifiers in October 2026.", "source": "https://arize.com/docs/ax/instrument/agent-trajectory" }, "costTracking": { "label": "Cost tracking", "value": "yes", "detail": "Both products. Phoenix calculates cost from token counts and a built-in model pricing table, with custom pricing, shown at span, trace, session, experiment and project level. AX calculates per-span and per-trace cost and added a Cost tab on every project in September 2026.", "source": "https://arize.com/docs/phoenix/tracing/how-to-tracing/cost-tracking" }, "openTelemetry": { "label": "OpenTelemetry support", "value": "yes", "detail": "Both products. Phoenix is built on OpenTelemetry and accepts OTLP; the README states the same OpenTelemetry/OpenInference instrumentation works with Phoenix and AX. Arize maintains the OpenInference semantic conventions and instrumentors.", "source": "https://github.com/Arize-ai/phoenix" }, "offlineEvals": { "label": "Offline evals (datasets, experiments)", "value": "yes", "detail": "Both products. Phoenix runs evals on datasets, experiment results and exported production traces, including server-side evaluators that run on every Playground experiment. AX offers datasets and experiments with unlimited evals on all plans.", "source": "https://arize.com/docs/phoenix/evaluation/llm-evals" }, "onlineEvals": { "label": "Online evals on production traffic", "value": "yes", "detail": "Arize AX only. AX 'tasks' run evaluators continuously against production traces at span, trace or session level, with sampling and filters. Phoenix documentation points to AX Online Evals for continuous production evaluation; no continuous online eval feature was found in Phoenix docs or 2026 release notes.", "source": "https://arize.com/docs/ax/evaluate/online-evals" }, "llmAsJudge": { "label": "LLM-as-a-judge scoring", "value": "yes", "detail": "Both products. Phoenix supports LLM-as-a-judge evaluators with rubrics. AX supports LLM judges, agent-as-a-judge (all plans since September 2026) and a 'Decision Judge' for high-volume structured evals.", "source": "https://arize.com/docs/phoenix/evaluation/llm-evals" }, "codeEvals": { "label": "Code-based evaluators", "value": "yes", "detail": "Both products. Phoenix supports deterministic code-based evaluators (exact match, regex, custom heuristics) and, from October 2026, can run Python and TypeScript code evaluators in Docker sandboxes. AX lists code evaluators alongside LLM judges for online evals.", "source": "https://arize.com/docs/phoenix/evaluation/llm-evals" }, "humanReview": { "label": "Human review and annotation", "value": "yes", "detail": "Both products. Phoenix lets users annotate traces through the UI, client or REST API. AX has annotation queues for assigning spans, traces, sessions or dataset examples to reviewers (some SDK methods marked beta).", "source": "https://arize.com/docs/phoenix/tracing/how-to-tracing/feedback-and-annotations" }, "promptTooling": { "label": "Prompt versioning and playground", "value": "yes", "detail": "Both products. AX has a Prompt Hub with versioning, diffing, rollback and environment tags, plus a Playground. Phoenix has a Playground and prompt versioning with version control and tagging.", "source": "https://arize.com/docs/ax/improve/save-and-version-prompts" }, "runtimeControls": { "label": "Runtime controls on live agent actions", "value": "partial", "detail": "Arize AX documents guards built on the Guardrails AI validator framework (dataset-embeddings jailbreak guard, RAG LLM guard) that run in the customer's code around LLM calls and can return a default response, re-ask, or raise an exception. Guard failures can be traced and alerted on. There is no documented approval workflow or policy engine for agent tool actions; the platform itself observes rather than intercepts traffic.", "source": "https://arize.com/docs/ax/security-and-settings/llm-security/guardrails" }, "gatewayProxy": { "label": "Gateway or proxy in the request path", "value": "partial", "detail": "Phoenix added an OpenAI-compatible POST /v1/chat/completions proxy in August 2026 that forwards requests to named providers using server-held keys; tool calling is not supported and the release notes do not position it for production traffic. AX has no first-party gateway; it integrates with third-party gateways such as Gravitee and Portkey for tracing.", "source": "https://arize.com/docs/phoenix/release-notes/08-2026/08-04-2026-chat-completions-proxy-ai-query-and-annotation-charts" }, "openSource": { "label": "Open source", "value": "partial", "detail": "Phoenix source code is public under the Elastic License 2.0 (ELv2), a source-available licence that prohibits offering it as a hosted or managed service; it is not an OSI-approved open-source licence. The OpenInference instrumentation libraries are Apache-2.0. Arize AX is proprietary.", "source": "https://github.com/Arize-ai/phoenix/blob/main/LICENSE" }, "selfHosting": { "label": "Self-hosting", "value": "yes", "detail": "Phoenix is self-hosted at no cost (pip, Docker, Kubernetes/Helm, one-click cloud deploys). Arize AX can be self-hosted on Kubernetes (connected, semi-restricted or air-gapped) under a commercial licence on the Enterprise plan.", "source": "https://arize.com/docs/ax/selfhosting/info/on-premise-overview" } }, "openSource": "Partly. Phoenix's code is public under the Elastic License 2.0, which is source-available rather than open source: you can self-host and modify it but not offer it to others as a hosted service. AX is commercial.", "runtimeControls": "Partly. Arize AX documents guards built on the Guardrails AI framework that run in your own code around model calls. There is no documented approval or policy layer for agent tool actions.", "pricing": { "summary": "AX Free is $0 for 25k spans a month with 15-day retention. AX Pro is $50 a month for 50k spans with 30-day retention. AX Enterprise is custom and is the only self-hosted AX option. Phoenix is free to self-host.", "source": "https://arize.com/pricing/", "checkedAt": "2026-10-20" }, "sdks": [ "Python (arize, arize-phoenix, arize-phoenix-client)", "TypeScript", "Go (Phoenix client)", "CLI (ax, @arizeai/phoenix-cli)", "REST API", "OpenInference instrumentors (Python, JavaScript/TypeScript, Java)", "MCP server (Phoenix)" ], "frameworkIntegrations": [ "OpenAI Agents SDK", "Claude Agent SDK", "LangChain / LangGraph", "LlamaIndex", "CrewAI", "DSPy", "Vercel AI SDK", "Mastra", "AutoGen / AG2", "Agno", "Google ADK", "Guardrails AI", "OpenAI", "Anthropic", "Google GenAI", "AWS Bedrock", "LiteLLM", "OpenRouter", "Portkey", "Gravitee" ], "limits": [ "Continuous online evals, monitors, alerting and automated issue detection (Signal) are AX features; Phoenix documentation points to AX for these.", "Phoenix uses the Elastic License 2.0, which is source-available rather than OSI open source and forbids offering it as a hosted service.", "Guardrails run in the customer's own code via the Guardrails AI framework; there is no documented approval or policy enforcement layer for live agent tool actions.", "AX self-hosting is limited to the Enterprise plan.", "Ownership changed in October 2026: the Dynatrace release says Arize capabilities will be integrated into Dynatrace over time, so product packaging may change." ] }, { "name": "Braintrust", "shortName": "Braintrust", "slug": "braintrust", "website": "https://www.braintrust.dev", "definition": "Braintrust is a hosted AI evaluation and observability platform covering tracing, offline experiments, online scoring, human review and prompt versioning, with an LLM gateway in preview and Loop, an in-product analysis agent.", "ownership": "Braintrust Data, Inc., an independent company that raised an $80M Series B led by ICONIQ in February 2026.", "statusChange": null, "bestFor": "Teams that want one hosted product for offline experiments, online scoring, human review and prompt iteration, with broad SDK and agent framework coverage.", "capabilities": { "llmTracing": { "label": "LLM call tracing", "value": "yes", "detail": "Auto-instrumentation for supported AI providers logs inputs, outputs, latency, token usage and costs; SDK wrappers for OpenAI, Anthropic, Gemini, Bedrock, Azure and others.", "source": "https://www.braintrust.dev/docs/instrument/trace-llm-calls" }, "agentTracing": { "label": "Multi-step agent tracing (tool calls, nested spans)", "value": "yes", "detail": "Agent framework integrations (OpenAI Agents SDK, Claude Agent SDK, LangGraph, CrewAI, Google ADK, Mastra, Pydantic AI and more) trace each run as one trace including tool calls, handoffs and nested model calls. A thread view organising traces by conversation turn entered public preview in Oct 2026.", "source": "https://www.braintrust.dev/docs/integrations/agent-frameworks/openai-agents-sdk" }, "costTracking": { "label": "Cost tracking", "value": "yes", "detail": "Token usage and cost captured per traced LLM call; usage monitoring with cost alerts; coding agent insights dashboard for cost by user, model, tool and branch (public preview, Oct 2026).", "source": "https://www.braintrust.dev/docs/instrument/trace-llm-calls" }, "openTelemetry": { "label": "OpenTelemetry support", "value": "yes", "detail": "Accepts OTLP traces and logs at /otel/v1/traces via exporter, Collector or Braintrust span processor; LLM calls converted to Braintrust LLM spans.", "source": "https://www.braintrust.dev/docs/integrations/sdk-integrations/opentelemetry/send-traces-and-logs" }, "offlineEvals": { "label": "Offline evals (datasets, experiments)", "value": "yes", "detail": "Experiments built from datasets, a task and scorers, run from SDK, CLI or UI; workflow evaluations via provider batch APIs in public preview (Sep 2026).", "source": "https://www.braintrust.dev/docs/evaluation-quickstart" }, "onlineEvals": { "label": "Online evals on production traffic", "value": "yes", "detail": "Online scoring automation rules run scorers asynchronously on production logs with configurable sampling and filters; results recorded as score spans.", "source": "https://www.braintrust.dev/docs/evaluate/score-online" }, "llmAsJudge": { "label": "LLM-as-a-judge scoring", "value": "yes", "detail": "LLM-as-a-judge scorers and classifiers defined in the UI or code; pre-built judges in the Autoevals library.", "source": "https://www.braintrust.dev/docs/evaluate/write-scorers" }, "codeEvals": { "label": "Code-based evaluators", "value": "yes", "detail": "Custom code scorers and classifiers in TypeScript or Python, defined inline in SDK code, pushed via CLI or created in the UI.", "source": "https://www.braintrust.dev/docs/evaluate/custom-code" }, "humanReview": { "label": "Human review and annotation", "value": "yes", "detail": "Categorical, continuous or free-form review scores on logs, experiments and dataset rows; assignment to reviewers, multiple reviewers, blind reviews and required scores. Starter plan limited to 1 human review score per project.", "source": "https://www.braintrust.dev/docs/annotate/human-review" }, "promptTooling": { "label": "Prompt versioning and playground", "value": "yes", "detail": "Versioned prompts with templating (Mustache or Nunjucks), playgrounds, environments, and calling prompts from code or REST API without redeploying.", "source": "https://www.braintrust.dev/docs/evaluate/prompts" }, "runtimeControls": { "label": "Runtime controls on live agent actions", "value": "no", "detail": "No inline blocking, guardrails or approval of live agent actions found in docs. Online scoring runs asynchronously after logging; alerts (Slack, PagerDuty, webhooks), Patterns and behaviour specs judge traces after the fact. Loop asks for approval only for its own edits inside Braintrust.", "source": "https://www.braintrust.dev/docs/evaluate/score-online" }, "gatewayProxy": { "label": "Gateway or proxy in the request path", "value": "yes", "detail": "Braintrust Gateway: unified API to OpenAI, Anthropic, Google, AWS and others with caching and logging. Hosted gateway is in public preview, free, with no SLA; can run inside a self-hosted data plane. The earlier AI proxy is deprecated.", "source": "https://www.braintrust.dev/docs/deploy/gateway" }, "openSource": { "label": "Open source", "value": "partial", "detail": "The platform itself is proprietary. Open: Autoevals (MIT), the AI proxy (MIT), SDKs for TypeScript and Python (Apache-2.0), Java SDK (MIT), and Terraform modules for the self-hosted data plane (Apache-2.0).", "source": "https://github.com/braintrustdata" }, "selfHosting": { "label": "Self-hosting", "value": "partial", "detail": "Enterprise only. Customers can self-host the data plane (AWS via Terraform/ECS; GCP and Azure via Kubernetes/Helm) or use BYOC; the control plane (UI, auth, metadata) is always operated by Braintrust. No fully self-contained deployment documented.", "source": "https://www.braintrust.dev/docs/admin/deployment" } }, "openSource": "Partly. The platform is proprietary. Its SDKs, the Autoevals scoring library and its Terraform deployment modules are open source.", "runtimeControls": "No. Online scoring, alerts and Patterns judge traces after they are logged. Braintrust does not document blocking, guardrails or approvals on live agent actions.", "pricing": { "summary": "Starter is free with 1 GB of processed data, 10K scores and 14-day retention. Pro is $249 a month with 5 GB and 50K scores. Every plan includes unlimited users. Enterprise, the only plan with self-hosting, is custom.", "source": "https://www.braintrust.dev/pricing", "checkedAt": "2026-10-20" }, "sdks": [ "TypeScript", "Python", "Go", "Java", "Ruby", "C#" ], "frameworkIntegrations": [ "OpenAI Agents SDK", "Claude Agent SDK", "LangChain", "LangGraph", "LlamaIndex", "Vercel AI SDK", "CrewAI", "AutoGen", "Google ADK", "Mastra", "Pydantic AI", "Strands Agents", "Microsoft Agent Framework", "DSPy", "LiteLLM", "Agno", "LiveKit Agents", "Pipecat", "Cloudflare Agents", "Temporal", "Spring AI", "OpenTelemetry / Traceloop OpenLLMetry" ], "limits": [ "No inline runtime enforcement: scoring and alerting run after the request, so it cannot block or hold a live agent action.", "Self-hosting covers only the data plane, is Enterprise-only, and still depends on the Braintrust-hosted control plane and Clerk for authentication.", "Core platform is not open source; only SDKs, Autoevals, proxy and deployment modules are.", "Hosted Gateway is in public preview with no SLA.", "Many newer features (Patterns, Topics, thread view, SQL charts, workflow evals) are in public preview; active observability pricing to be set before general availability.", "Starter retention is 14 days and Pro 30 days by default." ] }, { "name": "Helicone", "shortName": "Helicone", "slug": "helicone", "website": "https://www.helicone.ai", "definition": "Helicone is an open-source (Apache-2.0) LLM observability platform and AI gateway that logs model requests through a proxy or SDKs. It has been in maintenance mode since March 2026 and is not taking new cloud sign-ups.", "ownership": "Unclear. Mintlify acquired Helicone in March 2026 and moved it to maintenance mode. On 19 October 2026 Helicone began saying it is no longer part of Mintlify, with no new owner named.", "statusChange": "Acquired by Mintlify and moved to maintenance mode; new cloud sign-ups disabled. On 19 October 2026 Helicone said it is no longer part of Mintlify, with no new owner named", "bestFor": "Existing Helicone users or teams wanting a self-hostable, Apache-2.0 LLM request logger and gateway with cost tracking, who accept that the product has been in maintenance mode since March 2026 and that its ownership is in flux.", "capabilities": { "llmTracing": { "label": "LLM call tracing", "value": "yes", "detail": "Logs every LLM request and response via the proxy/AI Gateway or via async logging (OpenLLMetry-based SDKs and manual logger SDKs for Python, TypeScript and Go).", "source": "https://docs.helicone.ai/getting-started/integration-method/openllmetry.md" }, "agentTracing": { "label": "Multi-step agent tracing (tool calls, nested spans)", "value": "partial", "detail": "Sessions group related requests into a trace hierarchy using Helicone-Session-Id, -Path and -Name headers; tool calls and vector DB calls can be logged with the Logger SDK. The hierarchy is path-string based rather than span-based, and there is no agent-specific span model.", "source": "https://docs.helicone.ai/features/sessions.md" }, "costTracking": { "label": "Cost tracking", "value": "yes", "detail": "Per-request cost (exact via the AI Gateway's Model Registry, best-effort for direct integrations), per-session cost, slicing by custom properties, pricing for 300+ models, and cost alerts with graduated thresholds. No hard spend caps are described.", "source": "https://docs.helicone.ai/guides/cookbooks/cost-tracking.md" }, "openTelemetry": { "label": "OpenTelemetry support", "value": "partial", "detail": "Async logging is built on OpenLLMetry instrumentation via Helicone's own SDKs (@helicone/async, helicone-async). No generic OTLP ingestion endpoint is documented.", "source": "https://docs.helicone.ai/getting-started/integration-method/openllmetry.md" }, "offlineEvals": { "label": "Offline evals (datasets, experiments)", "value": "partial", "detail": "Datasets can be curated from logged requests for evaluation, and scores from external frameworks (e.g. Ragas) can be posted to requests. The docs state 'Helicone doesn't run evaluations for you - we're not an evaluation framework.' The Experiments feature for testing prompt variants against datasets was announced for removal on 1 September 2025.", "source": "https://docs.helicone.ai/features/advanced-usage/scores.md" }, "onlineEvals": { "label": "Online evals on production traffic", "value": "partial", "detail": "The open-source codebase contains online evaluators applied to live traffic (an October 2026 commit refers to 'online evals' and per-org rate limits on LLM-as-a-judge calls), but the public docs do not describe the feature and state that Helicone is not an evaluation framework. Webhooks can trigger external evaluations when requests complete.", "source": "https://github.com/Helicone/helicone/pull/5842" }, "llmAsJudge": { "label": "LLM-as-a-judge scoring", "value": "partial", "detail": "LLM-as-a-judge evaluators exist in the codebase (rate limited per org with an allowlisted model set as of October 2026), but they are undocumented in the public docs; the scores docs show LLM-as-judge implemented in the user's own code and reported to Helicone as scores.", "source": "https://github.com/Helicone/helicone/pull/5842" }, "codeEvals": { "label": "Code-based evaluators", "value": "no", "detail": "Python evaluators were turned off in October 2026: existing ones are skipped by online evals and cannot be created or edited, and the web UI no longer offers them. Code-based checks can still be run externally and posted as scores.", "source": "https://github.com/Helicone/helicone/pull/5842" }, "humanReview": { "label": "Human review and annotation", "value": "partial", "detail": "Positive/negative feedback can be attached to requests and sessions via API and is aggregated in the dashboard. No annotation queue or reviewer workflow is documented.", "source": "https://docs.helicone.ai/features/advanced-usage/feedback.md" }, "promptTooling": { "label": "Prompt versioning and playground", "value": "yes", "detail": "prompt versioning with versioning, commit history, tags, environments (production/staging/development), deployment by prompt ID via the AI Gateway, an SDK, and a Playground.", "source": "https://docs.helicone.ai/features/advanced-usage/prompts/overview.md" }, "runtimeControls": { "label": "Runtime controls on live agent actions", "value": "partial", "detail": "LLM Security can block prompt injection and jailbreak attempts (Prompt Guard, optional Llama Guard) with a PROMPT_THREAT_DETECTED error, but the docs state it works with OpenAI models only. Custom rate limits are available. No approvals or policy enforcement on agent tool actions is documented.", "source": "https://docs.helicone.ai/features/advanced-usage/llm-security.md" }, "gatewayProxy": { "label": "Gateway or proxy in the request path", "value": "yes", "detail": "AI Gateway offers one OpenAI-compatible endpoint for 100+ providers with automatic failover, caching, custom rate limits, BYOK, and managed credits at 0% markup. The original proxy is also the default integration method.", "source": "https://docs.helicone.ai/gateway/overview.md" }, "openSource": { "label": "Open source", "value": "yes", "detail": "Main platform repo Helicone/helicone is Apache-2.0 (dashboard, Jawn API, worker, self-host stack). The separate Rust AI Gateway repo Helicone/ai-gateway is GPL-3.0 and was last pushed in November 2025.", "source": "https://github.com/Helicone/helicone" }, "selfHosting": { "label": "Self-hosting", "value": "yes", "detail": "Self-hosting via manual install, Docker Compose / all-in-one Docker image, Kubernetes Helm charts, or cloud deployment. The Docker image no longer proxies LLM traffic; the AI Gateway must be deployed alongside it. Enterprise plan lists on-prem deployment.", "source": "https://docs.helicone.ai/getting-started/self-host/overview" } }, "openSource": "Yes. The main repository is Apache-2.0 and can be self-hosted. Its separate Rust AI gateway is GPL-3.0 and has not been updated since November 2025.", "runtimeControls": "Partly. Its gateway can block prompt injection and jailbreak attempts, for OpenAI models only, and apply rate limits. There are no approvals or policies on agent tool actions.", "pricing": { "summary": "Hobby is free with 10K requests, Pro is $79 a month and Team $799 a month, with Enterprise custom. New cloud sign-ups have been disabled since March 2026, so new customers cannot start on any plan.", "source": "https://www.helicone.ai/pricing", "checkedAt": "2026-10-20" }, "sdks": [ "Python (helicone-async, manual logger)", "TypeScript/JavaScript (@helicone/async, manual logger)", "Go (manual logger)", "Any OpenAI-compatible SDK via the AI Gateway / proxy base URL" ], "frameworkIntegrations": [ "OpenAI Agents SDK", "Claude Agent SDK", "OpenAI Codex", "LangChain", "LangGraph", "LlamaIndex", "CrewAI", "Vercel AI SDK", "Semantic Kernel", "DSPy", "LiteLLM", "Instructor", "Ragas", "n8n" ], "limits": [ "Maintenance mode since the March 2026 Mintlify acquisition: patches, bug fixes and model support only; changelog shows no feature entries after November 2025.", "Ownership unclear as of 19 October 2026: site states it is no longer part of Mintlify, with details pending.", "Cloud sign-ups disabled since March 2026.", "Not an evaluation framework by its own docs; Python/code evaluators were turned off in October 2026 and Experiments was deprecated in 2025.", "Agent tracing is header-based session grouping, not a span model for agent steps and tool calls.", "LLM Security blocking works with OpenAI models only.", "No generic OTLP endpoint documented.", "A May 2026 AWS account lock caused a four-day observability outage (proxy stayed up, logs queued)." ] }, { "name": "Galileo", "shortName": "Galileo", "slug": "galileo", "website": "https://galileo.ai", "definition": "Galileo is an evaluation, observability and guardrail platform for LLM and agent applications. Cisco acquired it in May 2026, and since August 2026 it has been sold as Splunk Agent Observability.", "ownership": "Cisco, which completed its acquisition of Galileo in May 2026. Since 7 August 2026 the product has been sold as Splunk Agent Observability inside Splunk Observability Cloud.", "statusChange": "Acquired by Cisco; sold as Splunk Agent Observability since 7 August 2026", "bestFor": "Teams already using, or moving to, Splunk Observability Cloud who want agent evaluation, production monitoring and runtime guardrails alongside infrastructure telemetry, including on-premises or air-gapped deployments.", "capabilities": { "llmTracing": { "label": "LLM call tracing", "value": "yes", "detail": "Python and TypeScript SDKs log sessions, traces and spans to Log Streams; wrappers and callbacks for OpenAI, LangChain and others.", "source": "https://docs.galileo.ai/sdk-api/third-party-integrations/overview" }, "agentTracing": { "label": "Multi-step agent tracing (tool calls, nested spans)", "value": "yes", "detail": "Native integrations for OpenAI Agents SDK, LangGraph, CrewAI, Google ADK, Strands, Microsoft Agent Framework and A2A multi-agent tracing (added May 2026); Agent Flow metric for agent paths.", "source": "https://docs.galileo.ai/release-notes" }, "costTracking": { "label": "Cost tracking", "value": "yes", "detail": "Admin-configured model pricing is used to calculate app and metric costs; Integration Costs and Billing Usage pages added in 2026. Splunk positions token and cost tracking by request, model, agent and workflow.", "source": "https://docs.galileo.ai/concepts/costs/model-pricing-settings" }, "openTelemetry": { "label": "OpenTelemetry support", "value": "yes", "detail": "Accepts OTLP traces at an OTel endpoint (default https://api.galileo.ai/otel/traces); supports OpenInference instrumentors (Python only) and OTel-based integrations for Mastra, Pydantic AI, Vercel AI SDK and others.", "source": "https://docs.galileo.ai/sdk-api/third-party-integrations/opentelemetry-and-openinference" }, "offlineEvals": { "label": "Offline evals (datasets, experiments)", "value": "yes", "detail": "Experiments run prompts or applications against datasets with preset and custom metrics; multi-turn experiments added June 2026.", "source": "https://docs.galileo.ai/release-notes" }, "onlineEvals": { "label": "Online evals on production traffic", "value": "yes", "detail": "Metrics configured on a Log Stream are evaluated on incoming production traces, with configurable sample rates and metadata-based sampling rules.", "source": "https://docs.galileo.ai/concepts/logging/configure-metrics/configure-metrics" }, "llmAsJudge": { "label": "LLM-as-a-judge scoring", "value": "yes", "detail": "Preset and custom LLM-as-a-judge metrics, with Autotune to adjust judge metrics from feedback; Luna-2 small models offered as lower-cost evaluators.", "source": "https://docs.galileo.ai/concepts/metrics/custom-metrics/custom-metrics-ui-llm" }, "codeEvals": { "label": "Code-based evaluators", "value": "yes", "detail": "Python code-based metrics (scorer_fn), either registered and run in a sandbox on Galileo's servers or run locally. The Splunk docs state custom code-based evaluators are available on-premises but not in the Splunk Observability Cloud SaaS edition.", "source": "https://docs.galileo.ai/concepts/metrics/custom-metrics/custom-metrics-ui-code" }, "humanReview": { "label": "Human review and annotation", "value": "yes", "detail": "Annotations on sessions, traces and spans (categories, score, star, text, thumbs); Annotation Queues for structured expert review became generally available in August 2026.", "source": "https://docs.galileo.ai/concepts/annotations/overview" }, "promptTooling": { "label": "Prompt versioning and playground", "value": "partial", "detail": "Prompts can be created, stored, templated and reused in experiments, and a console playground exists; the prompts page does not document version control.", "source": "https://docs.galileo.ai/sdk-api/experiments/prompts" }, "runtimeControls": { "label": "Runtime controls on live agent actions", "value": "yes", "detail": "Agent Control evaluates LLM and tool inputs and outputs during agent execution and can block prompt injection, PII leakage and harmful content using centrally managed rules, without agent code changes. It replaced Protect, which was deprecated in June 2026. Blocking of content is documented; approval workflows for actions are not.", "source": "https://docs.galileo.ai/concepts/agent-control/overview" }, "gatewayProxy": { "label": "Gateway or proxy in the request path", "value": "no", "detail": "No LLM gateway or proxy is listed in the documentation index; ingestion is via SDKs, integrations, OTLP and API.", "source": "https://docs.galileo.ai/_llms/documentation.txt" }, "openSource": { "label": "Open source", "value": "partial", "detail": "Platform is proprietary. Open source under Apache-2.0: Python SDK (rungalileo/galileo-python), TypeScript SDK (rungalileo/galileo-js) and Agent Control (agentcontrol/agent-control, open-sourced March 2026).", "source": "https://github.com/agentcontrol/agent-control" }, "selfHosting": { "label": "Self-hosting", "value": "yes", "detail": "Splunk Agent Observability is offered on-premises (customer-operated, including air-gapped) or as SaaS in Splunk Observability Cloud. Self-hosted is a commercial deployment, not an open-source release. Agent Control can be self-hosted from its open-source repository.", "source": "https://agent-observability-docs.splunk.com" } }, "openSource": "Partly. The platform is proprietary. Its Python and TypeScript SDKs and Agent Control, its run-time guardrail layer, are open source under Apache-2.0.", "runtimeControls": "Yes. Agent Control checks model and tool inputs and outputs during a run and can block prompt injection, personal data leaks and harmful content using rules set in one place, without changes to agent code.", "pricing": { "summary": "Splunk lists Agent Observability from $100 a month per 1.2 million spans a year, billed annually, with a Luna model allowance. No free tier is listed. On-premises deployment is available.", "source": "https://www.splunk.com/en_us/products/pricing.html", "checkedAt": "2026-10-20" }, "sdks": [ "Python", "TypeScript/JavaScript", "REST API" ], "frameworkIntegrations": [ "OpenAI SDK", "OpenAI Agents SDK", "LangChain / LangGraph", "CrewAI", "Google ADK", "Strands Agents", "Microsoft Agent Framework", "Vercel AI SDK", "Mastra", "Pydantic AI", "A2A Protocol", "AWS Bedrock", "Gemini Enterprise", "Any OpenInference-compatible framework" ], "limits": [ "Product is in transition after the Cisco acquisition: two documentation sites and branding split by onboarding date (before or after 7 August 2026).", "No self-serve free tier for Agent Observability is listed on the current pricing page; pricing is annual billing.", "The SaaS edition in Splunk Observability Cloud lacks custom code-based evaluators and Luna fine-tuning, and supports only five preset Luna evaluators.", "Core platform is proprietary; only SDKs and Agent Control are open source.", "Protect was deprecated in June 2026; existing users need to move to Agent Control.", "prompt versioning does not document version control." ] }, { "name": "HoneyHive", "shortName": "HoneyHive", "slug": "honeyhive", "website": "https://www.honeyhive.ai", "definition": "HoneyHive is a hosted agent observability and evaluation platform built on OpenTelemetry, covering tracing, offline experiments, online evaluations, human annotation, monitoring and prompt versioning.", "ownership": "HoneyHive is independent. Its last announced funding was $7.4M in total, including a seed round led by Insight Partners in April 2025.", "statusChange": null, "bestFor": "Teams, including those in regulated industries, that want a hosted, OpenTelemetry-based platform combining agent tracing, offline and online evaluation, human annotation and monitoring, with a self-hosted option on Enterprise.", "capabilities": { "llmTracing": { "label": "LLM call tracing", "value": "yes", "detail": "Distributed tracing of LLM calls, tool invocations and chain steps, with token, cost and latency capture.", "source": "https://docs.honeyhive.ai/v2/introduction/what-is-hhai" }, "agentTracing": { "label": "Multi-step agent tracing (tool calls, nested spans)", "value": "yes", "detail": "Traces, Agent Graphs, Trajectories, Threads, Timeline and (Sep 2026) Conversation views for multi-step and multi-agent runs; native integrations for agent frameworks and coding agents (Claude Code, Codex CLI, Devin).", "source": "https://docs.honeyhive.ai/v2/introduction/what-is-hhai" }, "costTracking": { "label": "Cost tracking", "value": "yes", "detail": "Cost, token and latency dashboards and alerts; session-level cost aggregation; usage reports by workspace and project (Jul 2026); per-workspace provider keys for cost attribution.", "source": "https://docs.honeyhive.ai/llms.txt" }, "openTelemetry": { "label": "OpenTelemetry support", "value": "yes", "detail": "Built on OpenTelemetry; accepts OTLP and supports OTel GenAI, OpenLLMetry and OpenInference semantic conventions, with bring-your-own instrumentor and an OpenTelemetry Collector integration.", "source": "https://docs.honeyhive.ai/v2/introduction/what-is-hhai" }, "offlineEvals": { "label": "Offline evals (datasets, experiments)", "value": "yes", "detail": "Experiments run with evaluate() against datasets to compare prompts, models and pipelines; side-by-side comparison and CI regression gating via GitHub Actions.", "source": "https://docs.honeyhive.ai/llms.txt" }, "onlineEvals": { "label": "Online evals on production traffic", "value": "yes", "detail": "Online evaluations score sampled production traces with LLM, code and human evaluators; default sampling lowered to 10% in May 2026.", "source": "https://www.honeyhive.ai/" }, "llmAsJudge": { "label": "LLM-as-a-judge scoring", "value": "yes", "detail": "LLM evaluators with custom prompts and rubrics, Jinja2 templating, versioning; evaluator calls can be routed through Portkey.", "source": "https://docs.honeyhive.ai/llms.txt" }, "codeEvals": { "label": "Code-based evaluators", "value": "yes", "detail": "Server-side Python evaluators for deterministic checks; client-side custom metrics can also be logged onto traces.", "source": "https://docs.honeyhive.ai/llms.txt" }, "humanReview": { "label": "Human review and annotation", "value": "yes", "detail": "Human evaluator fields and annotation queues route flagged traces to domain experts; queue automations added April 2026.", "source": "https://docs.honeyhive.ai/llms.txt" }, "promptTooling": { "label": "Prompt versioning and playground", "value": "yes", "detail": "Playground plus prompt versioning and deployment; deployed prompts fetched by name or version from the Python SDK or TypeScript API client.", "source": "https://docs.honeyhive.ai/llms.txt" }, "runtimeControls": { "label": "Runtime controls on live agent actions", "value": "no", "detail": "No blocking, guardrails or approvals on live agent actions are documented. 'Policies' are organisation-wide evaluation and monitoring standards, not runtime enforcement. Guardrail results from your own code can be logged as custom metrics. In the Microsoft ASSERT/ACS partnership HoneyHive is described as the observability layer while ACS enforces.", "source": "https://www.honeyhive.ai/blog/scale-agent-governance-with-microsofts-assert-and-honeyhive" }, "gatewayProxy": { "label": "Gateway or proxy in the request path", "value": "no", "detail": "No own proxy or gateway for application LLM traffic. It integrates with third-party gateways (Cloudflare AI Gateway, LiteLLM, Portkey, TrueFoundry, Vercel AI Gateway). An August 2026 changelog entry adds an OpenAI-compatible gateway as a self-hosted model provider for the platform's own use, not as an app traffic proxy.", "source": "https://docs.honeyhive.ai/llms.txt" }, "openSource": { "label": "Open source", "value": "partial", "detail": "SDKs and tooling are MIT licensed (Python SDK, TypeScript API client, CLI, OpenAPI specs, Agent Skills). The platform itself is proprietary; no open-source server is published.", "source": "https://github.com/honeyhiveai" }, "selfHosting": { "label": "Self-hosting", "value": "yes", "detail": "Enterprise only, via sales. Fully self-hosted (control plane and data plane in your environment) or hybrid (self-hosted data plane, HoneyHive-managed control plane). Documented on AWS EKS with Terraform and Helm, as BYOC or BYOVPC, managed by HoneyHive.", "source": "https://docs.honeyhive.ai/v2/setup/self-hosted" } }, "openSource": "Partly. Its SDKs and tools are MIT-licensed; the platform itself is proprietary.", "runtimeControls": "No. HoneyHive does not document blocking, guardrails or approvals on live agent actions. Its Policies feature sets evaluation and monitoring standards, not run-time enforcement.", "pricing": { "summary": "The Developer tier is free with 10,000 events a month, 30-day retention and up to 5 users. There is no published self-serve paid plan; Enterprise, including self-hosting, is quote-based.", "source": "https://www.honeyhive.ai/pricing", "checkedAt": "2026-10-20" }, "sdks": [ "Python (honeyhive, OpenTelemetry-based tracing and evals, MIT)", "TypeScript REST API client (@honeyhive/api-client, MIT; @honeyhive/logger is legacy)", "HoneyHive CLI", "OpenAPI specs for generating clients in other languages" ], "frameworkIntegrations": [ "LangChain", "LangGraph", "LlamaIndex", "CrewAI", "AutoGen", "Semantic Kernel", "DSPy", "Pydantic AI", "OpenAI Agents SDK", "Claude Agent SDK", "Google ADK", "AWS Strands", "Cursor SDK", "Databricks Agent Bricks", "Salesforce Agentforce", "Microsoft Copilot Studio", "n8n", "Microsoft ASSERT", "OpenAI", "Anthropic", "AWS Bedrock", "Azure OpenAI", "Gemini", "TypeSafe AI", "Cloudflare AI Gateway", "LiteLLM", "Portkey", "TrueFoundry", "Vercel AI Gateway", "Claude Code", "OpenAI Codex CLI", "Devin", "OpenTelemetry Collector" ], "limits": [ "No runtime blocking, guardrails or approvals on live agent actions; it observes and scores rather than enforces.", "Platform is closed source; only SDKs and tooling are open (MIT).", "No published self-serve paid tier; anything above the free 10,000 events per month requires a sales conversation.", "Self-hosting is Enterprise-only and documented for AWS only.", "Full tracing SDK is Python; the TypeScript package is a REST API client rather than an auto-instrumenting tracer (OTLP can be used instead).", "Small company (about 18-19 staff per third-party directories) with seed-stage funding last announced in April 2025." ] }, { "name": "Confident AI", "shortName": "Confident AI", "slug": "confident-ai", "website": "https://www.confident-ai.com", "definition": "Confident AI is an evaluation and observability platform built around DeepEval, its open-source evaluation framework, covering offline evals, tracing with online evals, human annotation, simulation and red teaming.", "ownership": "Confident AI is independent: a Y Combinator Winter 2025 company in San Francisco with a small team, per the YC directory.", "statusChange": null, "bestFor": "Teams that want a code-first, open-source evaluation framework (DeepEval) with a hosted platform for regression testing, online evals on production traces, human annotation and red teaming.", "capabilities": { "llmTracing": { "label": "LLM call tracing", "value": "yes", "detail": "Confident AI platform: traces made of spans, with threads grouping multi-turn conversations. Instrumented via the confident-trace SDK (Python/TypeScript), framework integrations, manual @span/withSpan, or OpenTelemetry. DeepEval also provides @observe tracing that sends to Confident AI; DeepEval alone has no UI or persistence.", "source": "https://www.confident-ai.com/docs/llm-tracing/introduction" }, "agentTracing": { "label": "Multi-step agent tracing (tool calls, nested spans)", "value": "yes", "detail": "Confident AI platform: span types include agent, tool, LLM and retriever; span trees isolate a subagent branch; tool failure analytics (Oct 2026). Integrations include OpenAI Agents, Claude Agent SDK, LangGraph, CrewAI, Google ADK, Strands, Pydantic AI and others.", "source": "https://www.confident-ai.com/docs/changelog/2026/10/9" }, "costTracking": { "label": "Cost tracking", "value": "yes", "detail": "Confident AI platform: token usage and cost on LLM spans, summed per trace. Costs set in code, via project-level model cost rules, or inferred automatically (automatic price lookup only for OpenAI, Anthropic and Gemini).", "source": "https://www.confident-ai.com/docs/llm-tracing/features/token-usage-cost" }, "openTelemetry": { "label": "OpenTelemetry support", "value": "yes", "detail": "Confident AI OTLP/HTTP endpoint (US and EU; self-hosted gets its own) reads GenAI semantic conventions, OpenInference, OpenLLMetry and Vercel AI SDK attributes. OTLP/gRPC is not supported; metrics and logs are accepted but dropped. confident-trace is an OTel-first SDK that can also export to your own collector.", "source": "https://www.confident-ai.com/docs/integrations/opentelemetry" }, "offlineEvals": { "label": "Offline evals (datasets, experiments)", "value": "yes", "detail": "DeepEval (open source, Python and TypeScript) runs test cases locally and in CI/CD with 40+ metrics, including agent, multi-turn, RAG, MCP and multimodal metrics. Confident AI stores test runs, datasets and regression reports; free tier allows 5 test runs per week.", "source": "https://github.com/confident-ai/deepeval" }, "onlineEvals": { "label": "Online evals on production traffic", "value": "yes", "detail": "Confident AI platform only: server-side evaluation tasks on traces, spans and threads as they are ingested, with filters and sample rates. Only referenceless metrics run during tracing. Not available in DeepEval alone.", "source": "https://www.confident-ai.com/docs/llm-tracing/online-evals" }, "llmAsJudge": { "label": "LLM-as-a-judge scoring", "value": "yes", "detail": "DeepEval metrics use any LLM as judge, including G-Eval and DAG custom metrics; Confident AI adds a prompt-based evaluator (Oct 2026) and LLM/HYBRID/DECISION metric modes (Oct 2026).", "source": "https://github.com/confident-ai/deepeval" }, "codeEvals": { "label": "Code-based evaluators", "value": "yes", "detail": "Confident AI 'Code-Eval': Python classes subclassing deepeval BaseMetric run in a sandbox on the platform (no external network calls) for rule-based checks such as JSON structure or regex. DeepEval custom metrics can also be written in code locally.", "source": "https://www.confident-ai.com/docs/metrics/custom-metrics/code-evals" }, "humanReview": { "label": "Human review and annotation", "value": "yes", "detail": "Confident AI platform: annotation queues for traces, spans, threads and test cases, multi-reviewer forms, end-user feedback API (thumbs or five-star), metric-to-human alignment and error analysis. Covers review of recorded data, not approval of live agent actions.", "source": "https://www.confident-ai.com/docs/human-in-the-loop/introduction" }, "promptTooling": { "label": "Prompt versioning and playground", "value": "yes", "detail": "Confident AI platform: Prompt Studio with commits, versions and environment labels, pulling prompts by label in code, side-by-side experiments comparing prompt versions and models. DeepEval README lists automatic prompt optimisation.", "source": "https://www.confident-ai.com/docs/llm-evaluation/prompt-management/version-prompts" }, "runtimeControls": { "label": "Runtime controls on live agent actions", "value": "no", "detail": "No inline blocking or approval of live agent actions is documented. Confident AI 'runtime controls' (AI governance module) assess recorded production data against thresholds over a trailing 24-hour window and report pass/fail; red-teaming trace-level detections flag traces after ingestion. Older docs described input/output 'guardrails', but that URL now redirects to the docs home page and the current docs index has no guardrails page; the DeepEval README lists guardrails only on its roadmap.", "source": "https://www.confident-ai.com/docs/ai-governance/controls/runtime-controls" }, "gatewayProxy": { "label": "Gateway or proxy in the request path", "value": "no", "detail": "No first-party LLM gateway or proxy documented. Confident AI integrates with third-party gateways (LiteLLM, Portkey, OpenRouter, Bifrost, TrueFoundry) for tracing.", "source": "https://www.confident-ai.com/docs/integrations" }, "openSource": { "label": "Open source", "value": "partial", "detail": "Apache-2.0: DeepEval (evaluation framework, Python and TypeScript) and confident-trace (OpenTelemetry tracing SDK, released Sep 2026). The Confident AI platform itself is proprietary; self-hosted images come from a private registry and need a signed licence key.", "source": "https://github.com/confident-ai/deepeval" }, "selfHosting": { "label": "Self-hosting", "value": "partial", "detail": "Confident AI platform self-hosting is Enterprise-only (licence key and private container images), deployed to AWS, GCP or Azure managed Kubernetes via Terraform and Helm. DeepEval runs locally without the platform but has no UI or persistence.", "source": "https://www.confident-ai.com/docs/self-hosting" } }, "openSource": "Partly. DeepEval and the confident-trace SDK are Apache-2.0. The Confident AI platform is proprietary, and self-hosting requires an Enterprise licence.", "runtimeControls": "No. Confident AI does not document blocking or approval of live agent actions. Its threshold checks run on recorded production data over a trailing 24-hour window.", "pricing": { "summary": "Free covers 2 seats and 1 GB-month of trace data. Starter is $200 a month per organisation with unlimited seats, and Team is $2,000 a month. Enterprise is custom.", "source": "https://www.confident-ai.com/pricing", "checkedAt": "2026-10-20" }, "sdks": [ "Python (deepeval, confident-trace)", "TypeScript/JavaScript (deepeval, confident-trace)", "OpenTelemetry for other languages (Go, Ruby, C#, Java, Rust via raw OTel SDK)", "REST API (V2, 200+ endpoints)", "MCP server" ], "frameworkIntegrations": [ "OpenAI", "Anthropic / Claude Platform", "Claude Agent SDK", "OpenAI Agents", "LangChain", "LangGraph", "Deep Agents", "Pydantic AI", "Vercel AI SDK", "CrewAI", "LlamaIndex", "Strands Agents", "LiveKit Agents", "Pipecat / Daily", "Google ADK", "smolagents", "Agno", "Spring AI", "Mastra", "AWS Bedrock AgentCore", "Microsoft Foundry", "Google Gemini Enterprise", "LiteLLM", "Portkey", "OpenRouter", "Bifrost", "TrueFoundry" ], "limits": [ "No inline blocking, guardrails or approval of live agent actions documented; 'runtime controls' assess recorded data over 24-hour windows.", "Online evals run only referenceless metrics during tracing.", "Self-hosting the platform requires an Enterprise licence; only DeepEval and confident-trace are open source.", "Automatic cost inference covers only OpenAI, Anthropic and Gemini; other providers need configured rates.", "OTLP ingestion is HTTP only (no gRPC); OTel metrics and logs are dropped.", "Free tier caps test runs at 5 per week and drops spans beyond 1 GB-month." ] }, { "name": "Datadog LLM Observability", "shortName": "Datadog", "slug": "datadog", "website": "https://www.datadoghq.com/product/llm-observability/", "definition": "Datadog Agent Observability, formerly LLM Observability, is Datadog's module for tracing, evaluating and running experiments on LLM applications and agents, alongside Datadog's APM and infrastructure data.", "ownership": "Datadog, a public company (Nasdaq: DDOG). Agent Observability is one product within the Datadog platform and needs a Datadog account.", "statusChange": null, "bestFor": "Teams already using Datadog for APM and infrastructure who want LLM and agent traces, evaluations and costs in the same platform, correlated with backend services.", "capabilities": { "llmTracing": { "label": "LLM call tracing", "value": "yes", "detail": "Each request is captured as a trace of spans covering prompts, retrieval, tool calls and LLM calls, with latency, token usage and errors per step. Auto-instrumentation for common providers and frameworks; manual SDK and HTTP API also available.", "source": "https://docs.datadoghq.com/llm_observability/" }, "agentTracing": { "label": "Multi-step agent tracing (tool calls, nested spans)", "value": "yes", "detail": "Agent monitoring visualises agent decisions such as tools used and handoffs, with complete traces for single- and multi-agent systems built on OpenAI Agents SDK, LangGraph and CrewAI (Python). Also covers MCP clients and Claude Agent SDK, Google ADK, Pydantic AI and Strands via auto-instrumentation.", "source": "https://docs.datadoghq.com/llm_observability/guide/agent_monitoring.md" }, "costTracking": { "label": "Cost tracking", "value": "yes", "detail": "Automatic per-request cost calculated from token counts and public rates for over 800 models; custom cost values can be supplied for self-hosted or unsupported models. Text-based models only.", "source": "https://docs.datadoghq.com/llm_observability/investigate/cost.md" }, "openTelemetry": { "label": "OpenTelemetry support", "value": "yes", "detail": "Accepts OTLP traces directly (no Datadog SDK or Agent required) using OpenTelemetry GenAI semantic conventions v1.37+ or OpenInference; OpenLLMetry and Langfuse instrumentation also recognised. 3-5 minute ingestion delay; not available on US government sites.", "source": "https://docs.datadoghq.com/llm_observability/instrument/otel_instrumentation.md" }, "offlineEvals": { "label": "Offline evals (datasets, experiments)", "value": "yes", "detail": "Experiments run an application against versioned datasets (which can be built from production traces) and score each record with evaluators; supports repeated runs and CI/CD automation, plus a browser Playground.", "source": "https://docs.datadoghq.com/llm_observability/investigate/evaluations/evaluation_developer_guide.md" }, "onlineEvals": { "label": "Online evals on production traffic", "value": "yes", "detail": "Published custom LLM-as-a-judge and managed evaluators run automatically on targeted production spans, with optional sampling (for example 10%) to control cost. Trace-level and session-level judges are available.", "source": "https://docs.datadoghq.com/llm_observability/investigate/evaluations/llm_as_a_judge_evaluations.md" }, "llmAsJudge": { "label": "LLM-as-a-judge scoring", "value": "yes", "detail": "Custom LLM-as-a-judge evaluators defined with natural-language prompts and templates, run with the customer's connected provider account (OpenAI, Azure OpenAI, Vertex AI, Anthropic, Amazon Bedrock or custom providers).", "source": "https://docs.datadoghq.com/llm_observability/investigate/evaluations/llm_as_a_judge_evaluations.md" }, "codeEvals": { "label": "Code-based evaluators", "value": "yes", "detail": "Class-based (BaseEvaluator) or function-based Python evaluators. They run automatically in Experiments; in production the customer runs them in application code and submits results via LLMObs.submit_evaluation() or the HTTP API. DeepEval, Pydantic and NVIDIA NeMo evaluators also supported.", "source": "https://docs.datadoghq.com/llm_observability/investigate/evaluations/evaluation_developer_guide.md" }, "humanReview": { "label": "Human review and annotation", "value": "yes", "detail": "Annotation Queues provide a structured workflow for human review of traces, used to find failure patterns, validate LLM-as-a-judge evaluators and build golden datasets. End-user feedback can also be attached to spans, traces or sessions.", "source": "https://docs.datadoghq.com/llm_observability/investigate/annotation_queues.md" }, "promptTooling": { "label": "Prompt versioning and playground", "value": "yes", "detail": "prompt versioning is a versioned prompt registry retrieved at runtime (Python, Go, JavaScript), with per-environment deployment and targeting rules; Prompt Experimentation supports A/B tests of prompt versions. Some sub-features (message placeholders, prompt composition) are in Preview.", "source": "https://docs.datadoghq.com/llm_observability/configure/prompt_management.md" }, "runtimeControls": { "label": "Runtime controls on live agent actions", "value": "partial", "detail": "Agent Observability itself monitors and evaluates but does not block. Blocking of prompts, responses and tool calls is offered by AI Guard, a separate Datadog security product billed per token scanned and stated as Limited Availability in June 2026 (coding-agent protection in Preview).", "source": "https://docs.datadoghq.com/security/ai_guard/" }, "gatewayProxy": { "label": "Gateway or proxy in the request path", "value": "no", "detail": "Datadog does not document its own LLM gateway or proxy. It provides guidance for tracing a customer-run proxy or gateway end to end, and auto-instruments LiteLLM.", "source": "https://docs.datadoghq.com/llm_observability/guide/monitor_proxy_services.md" }, "openSource": { "label": "Open source", "value": "partial", "detail": "The platform is proprietary SaaS. The instrumentation SDKs are open source: dd-trace-py is dual-licensed Apache-2.0 / BSD-3-Clause. Lapdog is built on the open-source Datadog APM test agent.", "source": "https://github.com/DataDog/dd-trace-py" }, "selfHosting": { "label": "Self-hosting", "value": "no", "detail": "No self-hosted deployment of the platform is documented; data governance docs describe only Datadog-hosted sites. Lapdog is a local development dashboard that runs without a Datadog account, not a production self-hosted option.", "source": "https://docs.datadoghq.com/llm_observability/lapdog.md" } }, "openSource": "Partly. The platform is proprietary SaaS; instrumentation SDKs such as dd-trace-py are open source. There is no self-hosted option.", "runtimeControls": "Partly. Agent Observability monitors and evaluates but does not block. Blocking prompts, responses and tool calls needs AI Guard, a separate Datadog product in limited availability, billed per token scanned.", "pricing": { "summary": "Free covers 40K LLM spans a month with 15-day retention. Pro is $160 a month billed annually for 100K LLM spans, then $3.50 per 10K. Only LLM spans are billed; tool and agent spans are not.", "source": "https://www.datadoghq.com/pricing/?product=llm-observability", "checkedAt": "2026-10-20" }, "sdks": [ "Python (ddtrace, Python 3.7+)", "Node.js (dd-trace, Node 16+)", "Java (dd-trace-java v1.51.0+)", "Go (dd-trace-go v2, experimental, manual instrumentation only)", "HTTP API (any language)", "OpenTelemetry OTLP" ], "frameworkIntegrations": [ "OpenAI and Azure OpenAI", "OpenAI Agents SDK", "Anthropic", "Claude Agent SDK", "Amazon Bedrock", "Amazon Bedrock Agents", "Google GenAI", "Google ADK", "Vertex AI", "MistralAI", "LangChain", "LangGraph", "CrewAI", "LiteLLM", "Pydantic AI", "Strands Agents (via OpenTelemetry)", "vLLM", "MCP", "Vercel AI SDK (Node.js)", "DeepEval, Pydantic Evals, NVIDIA NeMo (evaluations)" ], "limits": [ "Requires the Datadog SaaS platform; no self-hosted option.", "Not available on Datadog's US government sites.", "Blocking and guardrails need AI Guard, a separate product in Limited Availability and billed separately.", "Code evaluators run automatically only in Experiments; production code evals must be run in the application and submitted.", "Default trace retention is 15 days; longer retention is a paid add-on.", "Java SDK has a single auto-instrumented integration (OpenAI); Go SDK is experimental.", "Some features (Patterns, Bits Evals, parts of prompt versioning) were in Preview as of mid-2026." ] }, { "name": "Fiddler", "shortName": "Fiddler", "slug": "fiddler", "website": "https://www.fiddler.ai", "definition": "Fiddler is a commercial observability, evaluation and guardrails platform that covers classic ML models, LLM applications and agents in one place.", "ownership": "Fiddler is an independent, venture-backed company. It raised a $30M Series C in January 2026, bringing total funding to $100M.", "statusChange": null, "bestFor": "Enterprises, often in regulated sectors, that want one vendor for monitoring both traditional ML models and LLM/agent applications, with self-hosted deployment and low-latency guardrails.", "capabilities": { "llmTracing": { "label": "LLM call tracing", "value": "yes", "detail": "LLM calls are captured as spans with prompts, responses, token usage and latency, via native SDKs, OpenTelemetry/OTLP, or third-party AI gateways (Kong, AgentGateway, LiteLLM, Portkey).", "source": "https://docs.fiddler.ai/integrations/agentic-ai/ai-gateways.md" }, "agentTracing": { "label": "Multi-step agent tracing (tool calls, nested spans)", "value": "yes", "detail": "Agentic Observability provides hierarchical traces across agents, tools and LLM calls, with native SDKs for LangGraph, LangChain V1, Strands Agents and Google ADK, plus a generic OTel SDK for custom Python agents.", "source": "https://docs.fiddler.ai/getting-started/agentic-monitoring.md" }, "costTracking": { "label": "Cost tracking", "value": "partial", "detail": "Token usage is tracked across agent interactions. Monetary cost is stored when the trace source emits cost attributes (for example LiteLLM's gen_ai.cost.* fields); an Evaluator Cost API for judge spend was added in private preview in October 2026.", "source": "https://docs.fiddler.ai/integrations/agentic-ai/litellm-integration.md" }, "openTelemetry": { "label": "OpenTelemetry support", "value": "yes", "detail": "Accepts OTLP traces directly, provides a Fiddler OTel SDK, maps OpenTelemetry GenAI semantic conventions, supports exporting existing OTel traces and S3 ingestion of OTLP files.", "source": "https://docs.fiddler.ai/integrations/agentic-ai/opentelemetry-integration.md" }, "offlineEvals": { "label": "Offline evals (datasets, experiments)", "value": "yes", "detail": "Experiments run via the Fiddler Evals SDK against datasets/golden datasets, with 13 pre-built evaluators, custom evaluators and side-by-side experiment comparison.", "source": "https://docs.fiddler.ai/integrations/agentic-ai/evals-sdk.md" }, "onlineEvals": { "label": "Online evals on production traffic", "value": "yes", "detail": "Evaluator Rules run evaluators automatically on new production spans that match filter conditions, with optional backfill and downsampling to limit judge cost.", "source": "https://docs.fiddler.ai/evaluate-and-test/evaluator-rules.md" }, "llmAsJudge": { "label": "LLM-as-a-judge scoring", "value": "yes", "detail": "Pre-built LLM-as-a-judge evaluators and Custom Judge evaluators (Prompt Specs) using Fiddler-hosted models (e.g. fiddler/ministral3-8b, GA September 2026) or customer-supplied provider keys via the Fiddler LLM Gateway.", "source": "https://docs.fiddler.ai/changelog/product-releases.md" }, "codeEvals": { "label": "Code-based evaluators", "value": "yes", "detail": "Evals SDK supports RegexSearch/RegexMatch evaluators and EvalFn, which wraps any Python function as an evaluator. Evaluator Rules also reference rule-based functions.", "source": "https://docs.fiddler.ai/integrations/agentic-ai/evals-sdk.md" }, "humanReview": { "label": "Human review and annotation", "value": "partial", "detail": "Span-level Annotations (public preview since August 2026): numeric, boolean, categorical or free-text human scores on spans in the Explorer, also via the /v3/scores API. No dedicated review queue or labelling workflow was found in the docs.", "source": "https://docs.fiddler.ai/observability/agentic/annotations.md" }, "promptTooling": { "label": "Prompt versioning and playground", "value": "no", "detail": "No prompt versioning, versioning or playground appears in the documentation index. 'Prompt Specs' are prompt templates for LLM-as-a-judge evaluators, not application prompt versioning.", "source": "https://docs.fiddler.ai/llms.txt" }, "runtimeControls": { "label": "Runtime controls on live agent actions", "value": "partial", "detail": "Fiddler Guardrails score content in real time (safety across 11 dimensions, faithfulness, PII/PHI, secrets) for the application to block on; inline blocking or redaction is available when Fiddler is plugged into a third-party AI gateway (Kong, AgentGateway, LiteLLM, Portkey). Scope is content checks on requests and responses; no human approval step or tool-call policy engine for agent actions was found in the docs.", "source": "https://docs.fiddler.ai/protection/guardrails.md" }, "gatewayProxy": { "label": "Gateway or proxy in the request path", "value": "no", "detail": "Fiddler does not offer its own proxy for customer LLM traffic. It integrates with third-party gateways (Kong AI Gateway, AgentGateway, LiteLLM, Portkey). The 'Fiddler LLM Gateway' only manages provider credentials for Fiddler's own evaluators.", "source": "https://docs.fiddler.ai/integrations/agentic-ai/ai-gateways.md" }, "openSource": { "label": "Open source", "value": "partial", "detail": "The platform is proprietary. Client libraries are open: fiddler-client on PyPI is Apache-2.0; the Claude Code plugin (fiddler-coding-agent-plugin) is Apache-2.0; example notebooks (fiddler-examples) are MIT.", "source": "https://github.com/fiddler-labs" }, "selfHosting": { "label": "Self-hosting", "value": "yes", "detail": "Enterprise plan lists SaaS, VPC and on-premise deployment. Fiddler can also run inside a customer's AWS account as an Amazon SageMaker Partner AI App. Not available on the Developer plan (SaaS only).", "source": "https://www.fiddler.ai/pricing" } }, "openSource": "Partly. The platform is proprietary; its Python client and Claude Code plugin are Apache-2.0.", "runtimeControls": "Partly. Fiddler Guardrails score content in real time for your application to act on, and can block inline through a third-party AI gateway. There are no approvals or tool-call policies.", "pricing": { "summary": "A free plan covers guardrails only, through a demo request. The Developer plan is $0.002 per trace. Enterprise, which adds VPC and on-premises deployment, is quote-based.", "source": "https://www.fiddler.ai/pricing", "checkedAt": "2026-10-20" }, "sdks": [ "Python (fiddler-client)", "Fiddler Evals SDK (Python)", "Fiddler OTel SDK (Python)", "Fiddler LangGraph SDK (Python)", "Fiddler LangChain SDK (Python)", "Fiddler Strands SDK (Python)", "Fiddler Google ADK SDK (Python)", "REST API (v3)", "OTLP endpoint (any language via OpenTelemetry)" ], "frameworkIntegrations": [ "LangGraph", "LangChain V1", "Strands Agents", "Google ADK", "LiteLLM (SDK and proxy)", "Kong AI Gateway", "AgentGateway", "Portkey AI Gateway", "Claude Code", "OpenAI SDK and Anthropic SDK (via Python client or custom instrumentation)", "Amazon SageMaker, Databricks, MLflow, Snowflake, BigQuery, Kafka, Airflow, Datadog, PagerDuty" ], "limits": [ "Platform is closed source; only client SDKs and examples are open.", "Native agent SDKs are Python only; other languages go through raw OpenTelemetry or the REST API.", "No built-in prompt versioning or prompt playground.", "Runtime controls are content guardrails (safety, PII, secrets, faithfulness); no human approval or tool-call policy step for agent actions found in the docs.", "Inline blocking without code changes depends on a third-party AI gateway.", "Free tier and self-hosting require contact with sales; self-serve paid plan is SaaS only.", "Several recent features (Annotations, Evaluator Cost API, Claude Code plugin) are in public or private preview." ] }, { "name": "Pydantic Logfire", "shortName": "Logfire", "slug": "logfire", "website": "https://pydantic.dev/logfire", "definition": "Pydantic Logfire is an OpenTelemetry-based observability platform from the team behind Pydantic, tracing whole applications, including LLM and agent calls, with evals, prompt versioning and an AI gateway built in.", "ownership": "Pydantic, the company behind the Pydantic library and Pydantic AI. It is independent and backed by Sequoia, which led its $12.5M Series A in 2024.", "statusChange": null, "bestFor": "Python-heavy teams, especially those using Pydantic AI, who want LLM and agent traces in the same OpenTelemetry view as the rest of their application, with evals, prompt versioning and a gateway from one vendor.", "capabilities": { "llmTracing": { "label": "LLM call tracing", "value": "yes", "detail": "Traces model calls with dedicated Agents and LLMs views; integrations for OpenAI, Anthropic, Google GenAI, LiteLLM and others. Token usage and cost shown where the framework's telemetry includes those fields.", "source": "https://pydantic.dev/logfire" }, "agentTracing": { "label": "Multi-step agent tracing (tool calls, nested spans)", "value": "yes", "detail": "Traces agent runs, tool calls, retrieval and RAG steps alongside the application's HTTP and database spans. Integrations listed for Pydantic AI, LangGraph, CrewAI, AutoGen, Google ADK, smolagents, Strands, Agno, Mastra, Vercel AI SDK and others.", "source": "https://pydantic.dev/docs/logfire/integrations/" }, "costTracking": { "label": "Cost tracking", "value": "yes", "detail": "Cost is a queryable column in traces, shown where telemetry includes token data; the gateway meters spend with per-project, per-user and per-key limits.", "source": "https://pydantic.dev/logfire" }, "openTelemetry": { "label": "OpenTelemetry support", "value": "yes", "detail": "Built on OpenTelemetry; SDKs are OTel wrappers and any standard OTel SDK (Go, Java, .NET, Ruby and others) can send data. Eval results use the gen_ai.evaluation.result OTel event.", "source": "https://pydantic.dev/logfire" }, "offlineEvals": { "label": "Offline evals (datasets, experiments)", "value": "yes", "detail": "Datasets (code-defined with pydantic-evals or hosted in Logfire and editable in the UI), experiments and experiment comparison against a baseline, runnable on demand or in CI.", "source": "https://pydantic.dev/docs/logfire/evaluate/overview/" }, "onlineEvals": { "label": "Online evals on production traffic", "value": "yes", "detail": "Live Evaluations score production calls after they run. Evaluators run in the application (via pydantic-evals OnlineEvaluator / @evaluate) in the background with configurable sample rates, emitting gen_ai.evaluation.result OTel events that Logfire aggregates. They do not block or alter outputs.", "source": "https://pydantic.dev/docs/ai/evals/online-evaluation/" }, "llmAsJudge": { "label": "LLM-as-a-judge scoring", "value": "yes", "detail": "pydantic-evals LLMJudge evaluator supports pass/fail, categorical or numeric scores and records the judge's reasoning with the score.", "source": "https://pydantic.dev/docs/logfire/evaluate/overview/" }, "codeEvals": { "label": "Code-based evaluators", "value": "yes", "detail": "Scorers can be plain code (for example exact match or a PII check) defined in pydantic-evals.", "source": "https://pydantic.dev/docs/logfire/evaluate/overview/" }, "humanReview": { "label": "Human review and annotation", "value": "partial", "detail": "Run annotations (Pass/Neutral/Fail verdict with comment and tags) are in Beta and available in every project; annotation queues are a Design Partner (early-access) feature. Annotations do not feed experiment or live-eval aggregates.", "source": "https://pydantic.dev/docs/logfire/evaluate/human-review/" }, "promptTooling": { "label": "Prompt versioning and playground", "value": "yes", "detail": "prompt versioning: author templates in the UI, save immutable versions, test against saved scenarios and datasets, promote a version via labels with rollout targeting, and fetch at runtime through the SDK. A separate Prompt Playground replays prompts from captured traces.", "source": "https://pydantic.dev/docs/logfire/prompt-management/" }, "runtimeControls": { "label": "Runtime controls on live agent actions", "value": "partial", "detail": "Limited to model calls routed through the AI gateway: guardrails scan prompts and completions for secrets, PII and policy violations with per-rule observe/flag/redact/block actions, and spend caps block requests once a budget is spent. No documented approvals or policy enforcement on agent tool actions.", "source": "https://pydantic.dev/articles/logfire-ai-gateway" }, "gatewayProxy": { "label": "Gateway or proxy in the request path", "value": "yes", "detail": "Pydantic AI Gateway, managed through Logfire: single key for multiple providers (OpenAI, Anthropic, Google, Bedrock, Groq and others), BYOK or managed inference, failover and weighted load balancing, spend caps, guardrails. Available on Logfire Cloud plans and self-hosted Logfire.", "source": "https://pydantic.dev/docs/ai/gateway/" }, "openSource": { "label": "Open source", "value": "partial", "detail": "MIT licence for the SDKs (Python, JavaScript/TypeScript, Rust) and for pydantic-evals and Pydantic AI. The Logfire server/platform is closed source.", "source": "https://github.com/pydantic/logfire" }, "selfHosting": { "label": "Self-hosting", "value": "partial", "detail": "Self-hosting is available only on the Enterprise plan, deployed via the official Helm chart on Kubernetes with private container images, PostgreSQL 16+ and object storage. A dedicated single-tenant option (any GCP region) is also offered.", "source": "https://pydantic.dev/docs/logfire/reference/self-hosted/overview/" } }, "openSource": "Partly. The SDKs, pydantic-evals and Pydantic AI are MIT-licensed; the Logfire platform is closed source. Self-hosting is Enterprise only.", "runtimeControls": "Partly. Model calls routed through its AI gateway can be flagged, redacted or blocked, and spend caps stop requests over budget. Tool actions outside the gateway are not covered.", "pricing": { "summary": "Personal is free with 10M records a month. Team is $49 a month and Growth $249 a month, each with 10M records included, then $2 per extra million. Enterprise is custom.", "source": "https://pydantic.dev/pricing", "checkedAt": "2026-10-20" }, "sdks": [ "Python (logfire)", "JavaScript/TypeScript (logfire, @pydantic/logfire-node, @pydantic/logfire-cf-workers)", "Rust", "Any OpenTelemetry SDK (Go, Java, .NET, Ruby and others)" ], "frameworkIntegrations": [ "Pydantic AI", "OpenAI", "Anthropic", "Google GenAI", "OpenRouter", "LangChain", "LangGraph", "LlamaIndex", "LiteLLM", "Mirascope", "Magentic", "Instructor", "CrewAI", "AutoGen", "Google ADK", "smolagents", "Strands Agents", "Agno", "Haystack", "Semantic Kernel", "Vercel AI SDK", "Mastra", "Rig", "Letta", "MCP", "Dify", "Flowise", "Langflow", "n8n", "Open WebUI" ], "limits": [ "Platform server is closed source; only SDKs and pydantic-evals are open source.", "Self-hosting is Enterprise-only and needs Kubernetes, PostgreSQL and object storage.", "Online evaluators run in the customer's application code (pydantic-evals) and only score after the run; they cannot block outputs.", "Human review run annotations are Beta and annotation queues are early-access only.", "Runtime blocking is limited to model traffic routed through the AI gateway (guardrails, spend caps); no approvals on agent tool actions.", "Richest eval and auto-instrumentation tooling is Python-first." ] }, { "name": "Comet Opik", "shortName": "Opik", "slug": "opik", "website": "https://www.comet.com/site/products/opik/", "definition": "Opik is an open-source (Apache-2.0) tracing, evaluation and monitoring platform for LLM applications and agents from Comet, available self-hosted or as a hosted cloud.", "ownership": "Comet ML, Inc., an independent company founded in 2017 that reports $70M in total funding.", "statusChange": null, "bestFor": "Teams that want an open-source, self-hostable tracing and evaluation stack with online LLM-as-judge rules, prompt optimisation and broad framework coverage, and a low-cost cloud option.", "capabilities": { "llmTracing": { "label": "LLM call tracing", "value": "yes", "detail": "Logs LLM calls with inputs, outputs, token usage and cost through SDKs and many provider integrations.", "source": "https://www.comet.com/docs/opik/integrations/overview" }, "agentTracing": { "label": "Multi-step agent tracing (tool calls, nested spans)", "value": "yes", "detail": "Traces each agent step, including retrieval and tool calls, with thread (conversation) grouping. Integrations cover LangGraph, CrewAI, OpenAI Agents, Google ADK, Claude Agent SDK and others.", "source": "https://www.comet.com/site/products/opik/" }, "costTracking": { "label": "Cost tracking", "value": "yes", "detail": "Tracks token usage and model cost on traces in production; Cost Intelligence extends this to coding agents such as Claude Code and Codex.", "source": "https://www.comet.com/site/products/opik/" }, "openTelemetry": { "label": "OpenTelemetry support", "value": "yes", "detail": "Accepts OTLP traces at /api/v1/private/otel over HTTP only; the gRPC exporter is documented as causing errors. Supported semantic conventions are not stated on the page.", "source": "https://www.comet.com/docs/opik/integrations/opentelemetry" }, "offlineEvals": { "label": "Offline evals (datasets, experiments)", "value": "yes", "detail": "Datasets, experiments and evaluate() runs against reference data; Test Suites with plain-text assertions; resumable evaluations (evaluate_resume, June 2026).", "source": "https://www.comet.com/docs/opik/changelog" }, "onlineEvals": { "label": "Online evals on production traffic", "value": "yes", "detail": "Online evaluation rules score sampled production traces, spans and threads automatically, with alert rules for failing interactions.", "source": "https://www.comet.com/docs/opik/production/rules" }, "llmAsJudge": { "label": "LLM-as-a-judge scoring", "value": "yes", "detail": "30+ built-in metrics (hallucination, answer relevance, context precision, task completion and others), custom judge prompts, and 'Agent as a Judge' that reads traces with tools.", "source": "https://www.comet.com/docs/opik/production/rules" }, "codeEvals": { "label": "Code-based evaluators", "value": "yes", "detail": "Custom Python metrics by subclassing BaseMetric; custom Python metrics can also run as online rules on production threads.", "source": "https://www.comet.com/docs/opik/evaluation/metrics/custom_metric" }, "humanReview": { "label": "Human review and annotation", "value": "yes", "detail": "Annotation queues for subject-matter experts to score traces; since Oct 2026 queues can fill automatically from feedback-score conditions.", "source": "https://www.comet.com/docs/opik/changelog" }, "promptTooling": { "label": "Prompt versioning and playground", "value": "yes", "detail": "Prompt Library with environment-tagged versions (Opik 2.0, June 2026), Playground / Agent Playground, and a Prompt Optimizer with six optimisation algorithms.", "source": "https://www.comet.com/site/products/opik/" }, "runtimeControls": { "label": "Runtime controls on live agent actions", "value": "partial", "detail": "Guardrails (PII, topic, prompt injection, LLM judge, custom classifier) run as a blocking validate() call that fails closed and raises an exception. The application code must call them and handle the result; there are no approval or hold workflows for agent actions. PII and topic checks are English only, and self-hosted guards need a separate guardrails server (GPU recommended).", "source": "https://www.comet.com/docs/opik/production/guardrails" }, "gatewayProxy": { "label": "Gateway or proxy in the request path", "value": "partial", "detail": "An Opik LLM Gateway (OpenAI-compatible proxy) exists but is in beta and documented as for development and testing only. Opik also integrates with third-party gateways (LiteLLM, Portkey, OpenRouter, Kong AI Gateway, Helicone, TrueFoundry, Vercel AI Gateway).", "source": "https://www.comet.com/docs/opik/integrations/opik-llm-gateway" }, "openSource": { "label": "Open source", "value": "yes", "detail": "Apache License 2.0. The core platform (tracing, evaluation, UI, backend, SDKs) is in the public comet-ml/opik repository.", "source": "https://github.com/comet-ml/opik" }, "selfHosting": { "label": "Self-hosting", "value": "yes", "detail": "Local install via opik.sh / opik.ps1 with Docker (described as not production-ready) or Kubernetes for production. Self-hosted gets all Opik features except user management.", "source": "https://www.comet.com/docs/opik/self-host/overview" } }, "openSource": "Yes. The core platform, including tracing, evaluation, the UI, backend and SDKs, is Apache-2.0 and free to self-host.", "runtimeControls": "Partly. Opik guardrails run as a blocking check that your code must call and handle. There are no approval workflows for agent actions.", "pricing": { "summary": "Self-hosting is free with unlimited spans. Free Cloud covers 25k spans a month and 10 users. Pro Cloud is $19 a month for 100k spans, then $5 per 100k. Enterprise is custom.", "source": "https://www.comet.com/site/pricing/", "checkedAt": "2026-10-20" }, "sdks": [ "Python", "TypeScript", "Java", ".NET", "Ruby (via OpenTelemetry Ruby SDK)", "REST API", "MCP server" ], "frameworkIntegrations": [ "LangChain", "LangGraph", "LlamaIndex", "CrewAI", "OpenAI Agents", "Google Agent Development Kit", "Claude Agent SDK", "Microsoft Agent Framework", "Pydantic AI", "DSPy", "Haystack", "AutoGen", "AG2", "Agno", "Smolagents", "Strands Agents", "Semantic Kernel", "LiveKit Agents", "Pipecat", "Temporal", "Mastra", "Vercel AI SDK", "Spring AI", "Quarkus LangChain4j", "Dify", "Flowise", "Langflow", "n8n", "OpenAI", "Anthropic", "Bedrock", "Gemini", "Mistral", "Ollama", "LiteLLM", "OpenRouter", "Portkey" ], "limits": [ "Guardrails must be called from application code; no approval or hold workflow for agent actions.", "PII and topic guardrails support English only; self-hosted guards need a separate guardrails server, GPU recommended.", "OpenTelemetry ingestion supports HTTP only, not gRPC.", "The built-in LLM Gateway is beta and documented for development and testing, not production.", "Self-hosted open source excludes user management; the local Docker install is described as not production-ready.", "Cloud plans cap spans (25k free, 100k Pro) and default to 60-day retention.", "SDK sends anonymous feature-usage analytics by default since Sep 2026 (disable with OPIK_ANALYTICS_ENABLE=false)." ] }, { "name": "Raindrop", "shortName": "Raindrop", "slug": "raindrop", "website": "https://www.raindrop.ai", "definition": "Raindrop is a hosted monitoring platform for AI agents in production that classifies events with custom signals, groups recurring failures into issues and compares changes on live traffic.", "ownership": "Raindrop is independent. It announced a Series A led by CRV in September 2026, bringing total funding to $50M.", "statusChange": null, "bestFor": "Teams with an agent already in production who want to find recurring failures in real user interactions, track defined behaviours over time and compare changes on live traffic, rather than run pre-release eval suites.", "capabilities": { "llmTracing": { "label": "LLM call tracing", "value": "yes", "detail": "SDKs and integrations capture model calls, inputs, outputs and token usage; also accepts OTLP traces with gen_ai attributes.", "source": "https://raindrop.ai/docs/sdk/opentelemetry" }, "agentTracing": { "label": "Multi-step agent tracing (tool calls, nested spans)", "value": "yes", "detail": "Traces follow every message, tool call and decision in a run. Integrations cover agent frameworks including Claude Agent SDK, OpenAI Agents SDK, LangChain, Mastra, CrewAI, Google ADK and others (many marked beta).", "source": "https://raindrop.ai/docs/llms.txt" }, "costTracking": { "label": "Cost tracking", "value": "yes", "detail": "Costs page shows model spend by model, provider and project, calculated from captured usage and a versioned catalogue of public model rates (cache reads/writes, reasoning, tiered rates). Experiments include median cost per event.", "source": "https://raindrop.ai/docs/platform/cost-tracking" }, "openTelemetry": { "label": "OpenTelemetry support", "value": "yes", "detail": "Ingests OTLP traces over HTTP (JSON or protobuf) at https://api.raindrop.ai/v1/traces; works with the OpenTelemetry Collector. OTLP/gRPC, metrics and logs are not accepted.", "source": "https://raindrop.ai/docs/sdk/opentelemetry" }, "offlineEvals": { "label": "Offline evals (datasets, experiments)", "value": "partial", "detail": "No hosted offline eval runner documented. Workshop (local, open source) lets a coding agent turn failures into evals and re-run the agent until assertions pass; Datasets store eval sets; Simulations run tests on pull requests but are in early access.", "source": "https://raindrop.ai/docs/workshop/overview" }, "onlineEvals": { "label": "Online evals on production traffic", "value": "yes", "detail": "Signals are classifiers that run on every new event and backfill 90 days (sampled); Issue Detection groups recurring failures. Experiments compare signal rates between cohorts of production traffic.", "source": "https://raindrop.ai/docs/platform/signals" }, "llmAsJudge": { "label": "LLM-as-a-judge scoring", "value": "partial", "detail": "Signals are drafted from a plain-language description into classifiers (rd-signal-2: generated filtering code plus a task-specific classification head and in-house reasoning model for ambiguous cases). This is model-based classification of traces, but not a user-configurable general LLM-judge with custom judge prompts or models.", "source": "https://www.raindrop.ai/blog/signals-2-frontier-classification" }, "codeEvals": { "label": "Code-based evaluators", "value": "partial", "detail": "Workshop has a coding agent write code-level evals with assertions against the user's codebase and run them locally. SDKs can also send programmatic signals via trackSignal(). No hosted code-scorer feature is documented.", "source": "https://github.com/raindrop-ai/workshop" }, "humanReview": { "label": "Human review and annotation", "value": "partial", "detail": "Users label sample events as Match / Not Match when building a signal, and can annotate dataset members (Good, Bad, Needs review or custom labels). No dedicated annotation queue for reviewers is documented.", "source": "https://raindrop.ai/docs/platform/datasets" }, "promptTooling": { "label": "Prompt versioning and playground", "value": "no", "detail": "No prompt versioning, versioning or playground documented. Experiments can compare prompt versions using production traffic, and Workshop replay can rerun a trace with an adjusted system prompt locally.", "source": "https://www.raindrop.ai/blog/introducing-raindrop-2" }, "runtimeControls": { "label": "Runtime controls on live agent actions", "value": "no", "detail": "No blocking, guardrails, approvals or policy enforcement on live agent actions is documented. Raindrop analyses traces after ingestion and alerts; PII Guard redacts data at ingestion, which is not a control on agent actions. Simulations can block a merge in CI, which is pre-production only.", "source": "https://www.raindrop.ai/blog/introducing-raindrop-2" }, "gatewayProxy": { "label": "Gateway or proxy in the request path", "value": "no", "detail": "No LLM gateway or proxy offered; data arrives via SDKs, the HTTP API or OTLP. It can use total cost reported by a third-party gateway such as OpenRouter.", "source": "https://raindrop.ai/docs/platform/cost-tracking" }, "openSource": { "label": "Open source", "value": "partial", "detail": "MIT licence for Raindrop Workshop (local debugger) and several SDK/integration repos (Go, Rust, pi-agent). The hosted Raindrop Cloud platform is proprietary.", "source": "https://github.com/raindrop-ai/workshop" }, "selfHosting": { "label": "Self-hosting", "value": "partial", "detail": "Self-hosting is in beta: VPC deployment in the customer's own cloud for Enterprise, rolling out to select partners (June 2026 announcement). Workshop runs locally but is a debugger, not a self-hosted copy of the platform.", "source": "https://www.raindrop.ai/blog/introducing-raindrop-2" } }, "openSource": "Partly. Its local debugger, Workshop, and several SDKs are MIT-licensed; the Raindrop Cloud platform is proprietary.", "runtimeControls": "No. Raindrop analyses events after they arrive and alerts on issues. It does not document blocking, guardrails or approvals on live agent actions.", "pricing": { "summary": "Hobby is free with 1,000 events a month. Pro is $299 a month plus $0.003 per event up to 1M events. Enterprise is custom.", "source": "https://raindrop.ai/docs/platform/plans", "checkedAt": "2026-10-20" }, "sdks": [ "TypeScript", "Python", "Go", "Rust (beta)", "Java (beta)", "Browser JavaScript", "HTTP API", "OpenTelemetry (OTLP/HTTP)" ], "frameworkIntegrations": [ "Vercel AI SDK", "TanStack AI", "Claude Agent SDK (beta)", "Claude Managed Agents (beta)", "Cursor Agent SDK (beta)", "OpenAI Agents SDK (beta)", "OpenAI Agents API (beta)", "OpenRouter Agent SDK (beta)", "LangChain (beta)", "Deep Agents (beta)", "Mastra (beta)", "DSPy (beta)", "Pydantic AI (beta)", "Agno (beta)", "Strands Agents (beta)", "Google ADK", "CrewAI (beta)", "Pi Agent (beta)", "Eve (beta)", "Temporal (beta)", "AWS Bedrock (beta)", "Azure OpenAI (beta)", "Vertex AI (beta)", "Claude Code CLI (beta)", "Cursor (beta)", "OpenCode (beta)", "MCP servers" ], "limits": [ "No runtime controls: it detects and alerts on failures after the fact and cannot block or approve live agent actions.", "No prompt versioning or LLM gateway.", "Offline evals rely on the local Workshop tool and coding agents; the hosted pre-merge Simulations product is early access.", "Signal classifiers are built by Raindrop's own models; users cannot bring their own judge prompt or model.", "Self-hosting is beta and limited to Enterprise / select partners.", "OTLP ingestion is HTTP traces only (no gRPC, metrics or logs).", "Most framework integrations are labelled beta.", "Hobby tier capped at 1,000 events/month with 14-day retention." ] }, { "name": "Maxim AI", "shortName": "Maxim AI", "slug": "maxim-ai", "website": "https://www.getmaxim.ai/", "definition": "Maxim AI is a hosted platform for simulating, evaluating and tracing AI agents, including voice agents, sold alongside Bifrost, its Apache-2.0 AI gateway.", "ownership": "Maxim AI (H3 Labs Inc.) is independent; it raised a $3M seed round led by Elevation Capital in 2024.", "statusChange": null, "bestFor": "Teams that want pre-release simulation (including voice agents) and evaluation run by product and engineering together, with production tracing and an optional self-hosted gateway in the same vendor.", "capabilities": { "llmTracing": { "label": "LLM call tracing", "value": "yes", "detail": "SDK tracing of traces, spans, generations, retrievals, tool calls, sessions, events, errors and user feedback.", "source": "https://www.getmaxim.ai/docs/llms.txt" }, "agentTracing": { "label": "Multi-step agent tracing (tool calls, nested spans)", "value": "yes", "detail": "Visual traces for multi-agent workflows; sessions, spans and tool calls captured step by step; agent-specific evaluators such as agent trajectory, tool selection and task success.", "source": "https://www.getmaxim.ai/evals/products/agent-observability" }, "costTracking": { "label": "Cost tracking", "value": "yes", "detail": "Token usage, latency and cost logged per request with threshold alerts on cost; custom token pricing setting. Bifrost adds budgets and rate limits per virtual key, team and customer.", "source": "https://www.getmaxim.ai/evals/products/agent-observability" }, "openTelemetry": { "label": "OpenTelemetry support", "value": "yes", "detail": "Ingests OTLP over HTTP (Protobuf or JSON) at api.getmaxim.ai/v1/otel, supporting GenAI, OpenInference and AI SDK semantic conventions; forwards traces to New Relic, Snowflake or any OTel collector via data connectors. Bifrost emits OTLP traces.", "source": "https://www.getmaxim.ai/docs/tracing/opentelemetry/ingesting-via-otlp.md" }, "offlineEvals": { "label": "Offline evals (datasets, experiments)", "value": "yes", "detail": "Test runs over datasets for prompts, no-code agents and HTTP-endpoint agents, via UI or SDK, with scheduled runs and CI/CD (GitHub Actions) integration; multi-turn text and voice simulation.", "source": "https://www.getmaxim.ai/docs/llms.txt" }, "onlineEvals": { "label": "Online evals on production traffic", "value": "yes", "detail": "Automatic evaluation of production logs at session, trace and node level, with filter- and rate-based sampling; alerts to Slack and PagerDuty. Listed from the Professional plan upwards.", "source": "https://www.getmaxim.ai/evals/products/agent-observability" }, "llmAsJudge": { "label": "LLM-as-a-judge scoring", "value": "yes", "detail": "Pre-built AI evaluators (faithfulness, toxicity, task success, etc.) and custom AI evaluators with a chosen judge model, instructions and scoring scale.", "source": "https://www.getmaxim.ai/docs/library/evaluators/custom-evaluators.md" }, "codeEvals": { "label": "Code-based evaluators", "value": "yes", "detail": "Custom programmatic evaluators in JavaScript or Python defining a validate function; pre-built programmatic and statistical evaluators; API-based evaluators that call an external endpoint.", "source": "https://www.getmaxim.ai/docs/library/evaluators/custom-evaluators.md" }, "humanReview": { "label": "Human review and annotation", "value": "yes", "detail": "Human evaluators with rating guidelines; annotation queues on logs for internal or external raters (external raters by email added January 2026).", "source": "https://www.getmaxim.ai/docs/online-evals/via-ui/set-up-human-annotation-on-logs.md" }, "promptTooling": { "label": "Prompt versioning and playground", "value": "yes", "detail": "Prompt playground, versions, partials, sessions, folders and tags, deployment rules that release prompt changes without code changes, prompt optimisation, and SDK retrieval of managed prompts.", "source": "https://www.getmaxim.ai/docs/llms.txt" }, "runtimeControls": { "label": "Runtime controls on live agent actions", "value": "yes", "detail": "Only through the Bifrost gateway: guardrails check inputs, outputs and MCP tool-call arguments/results and can detect, block, redact or transform content. The pricing page lists guardrails as a Bifrost Enterprise feature. The Maxim evaluation/observability platform itself describes alerts but no blocking of live actions.", "source": "https://docs.getbifrost.ai/enterprise/guardrails" }, "gatewayProxy": { "label": "Gateway or proxy in the request path", "value": "yes", "detail": "Bifrost: OpenAI-compatible gateway with provider failover, load balancing, semantic caching, virtual keys and an MCP gateway (Code Mode and Agent Mode).", "source": "https://docs.getbifrost.ai/overview" }, "openSource": { "label": "Open source", "value": "partial", "detail": "Apache-2.0: Bifrost gateway and the Python, JS/TS and Go SDKs. The simulation, evaluation and observability platform is proprietary.", "source": "https://github.com/maximhq/bifrost" }, "selfHosting": { "label": "Self-hosting", "value": "yes", "detail": "Platform: enterprise deployment in the customer's VPC on AWS, GCP or Azure, either fully isolated ('Zero Touch') or data plane only, peered to Maxim's cloud control plane; in-VPC is listed only on the Enterprise plan. Bifrost OSS self-hosts via Docker, Kubernetes or Go binary.", "source": "https://www.getmaxim.ai/docs/self-hosting/overview.md" } }, "openSource": "Partly. The Bifrost gateway and the SDKs are Apache-2.0; the simulation, evaluation and observability platform is proprietary.", "runtimeControls": "Yes, through the Bifrost gateway: guardrails check inputs, outputs and MCP tool-call arguments and results, and can block, redact or transform content. Guardrails are a Bifrost Enterprise feature.", "pricing": { "summary": "Developer is free for up to 3 seats and 10k logs a month. Professional is $29 per seat a month and Business $49 per seat a month. Enterprise, the only plan with in-VPC hosting, is custom.", "source": "https://www.getmaxim.ai/evals/pricing", "checkedAt": "2026-10-20" }, "sdks": [ "Python", "JavaScript/TypeScript", "Go", "Java/JVM" ], "frameworkIntegrations": [ "OpenAI SDK", "OpenAI Agents SDK", "OpenAI Realtime API", "Anthropic SDK", "Google Gemini", "Google ADK", "LangChain", "LangGraph", "LlamaIndex", "CrewAI", "Agno", "Pydantic AI", "Smolagents", "LiteLLM (SDK and proxy)", "LiveKit", "ElevenLabs", "Mistral", "Groq", "Fireworks", "Together", "AWS Bedrock Agents (simulation)", "Glean agents (simulation)" ], "limits": [ "The core evaluation and observability platform is closed source; only Bifrost and the SDKs are open.", "Runtime blocking exists only in the Bifrost gateway, and guardrails are listed as a Bifrost Enterprise feature.", "Self-hosting of the evaluation platform is offered only as an enterprise in-VPC deployment.", "Free tier keeps logs for 3 days and caps at 10k logs per month.", "Public product changelog (changelog.getmaxim.ai) shows no dated 2026 entries, and the newest monthly update post on the blog covers December 2025, so recent platform changes are hard to verify.", "The getmaxim.ai homepage and /pricing now present Bifrost; the evaluation platform has moved to /evals." ] }, { "name": "W&B Weave", "shortName": "W&B Weave", "slug": "weave", "website": "https://wandb.ai/site/weave", "definition": "W&B Weave is a tracing, evaluation and monitoring toolkit for LLM applications and agents from Weights & Biases, now part of CoreWeave and sold through CoreWeave Forge.", "ownership": "CoreWeave, which completed its acquisition of Weights & Biases in May 2025. Since September 2026 W&B hosted accounts sign in through CoreWeave Forge.", "statusChange": "Weights & Biases acquired by CoreWeave; W&B hosted accounts moved to CoreWeave Forge in September 2026", "bestFor": "Teams already using Weights & Biases or CoreWeave for model training who want LLM and agent tracing, evaluations and production monitoring in the same platform.", "capabilities": { "llmTracing": { "label": "LLM call tracing", "value": "yes", "detail": "@weave.op decorator traces functions; auto-patching integrations for providers including OpenAI, Anthropic, Bedrock, Google, Mistral, Cohere and LiteLLM capture LLM calls with inputs, outputs and token usage.", "source": "https://docs.coreweave.com/products/wandb/weave/guides/integrations.md" }, "agentTracing": { "label": "Multi-step agent tracing (tool calls, nested spans)", "value": "yes", "detail": "Threads group calls into turns with nested calls; an Agents view renders OTel GenAI spans as agents, conversations, turns, LLM calls and tool calls. SDK releases in 2026 added sub-agent nesting and Claude Agent SDK sub-agent tracing. CoreWeave now recommends Agent Lens (public preview, same backend) for agent tracing and analysis.", "source": "https://docs.coreweave.com/products/wandb/weave/guides/tracking/trace-agents-otel.md" }, "costTracking": { "label": "Cost tracking", "value": "yes", "detail": "Automatic token cost calculation for supported providers, including cache tokens; custom per-model prices via add_cost. Python only: the docs state the TypeScript SDK does not support cost tracking.", "source": "https://docs.coreweave.com/products/wandb/weave/guides/tracking/costs.md" }, "openTelemetry": { "label": "OpenTelemetry support", "value": "yes", "detail": "Accepts OTLP over HTTP (protobuf) at /otel/v1/traces; understands OTel GenAI, OpenInference and OpenLLMetry conventions; a separate agents endpoint feeds the Agents view. Since SDK 0.52.43 (June 2026) OTel integrations are on by default.", "source": "https://docs.coreweave.com/products/wandb/weave/guides/tracking/otel.md" }, "offlineEvals": { "label": "Offline evals (datasets, experiments)", "value": "yes", "detail": "Evaluation API runs a model over a dataset with scorers; results can be compared to catch regressions and shared as leaderboards. EvaluationLogger supports imperative logging.", "source": "https://docs.coreweave.com/products/wandb/weave/guides/core-types/evaluations.md" }, "onlineEvals": { "label": "Online evals on production traffic", "value": "yes", "detail": "Monitors score incoming production traces automatically with LLM-as-judge signals and save results as tags; custom monitors are supported and alerts go out via automations (Slack, webhooks). Docs now describe built-in monitors as a 'previous approach' and recommend Signals under Weave for Agents.", "source": "https://docs.coreweave.com/products/wandb/weave/guides/evaluation/monitors.md" }, "llmAsJudge": { "label": "LLM-as-a-judge scoring", "value": "yes", "detail": "LLM-as-judge scorers with prompt templates, plus predefined scorers such as hallucination detection and summarisation quality.", "source": "https://docs.coreweave.com/products/wandb/weave/guides/evaluation/scorers.md" }, "codeEvals": { "label": "Code-based evaluators", "value": "yes", "detail": "Function-based scorers (Python and TypeScript) and class-based scorers inheriting weave.Scorer (Python only); local small-model scorers for toxicity and context relevance.", "source": "https://docs.coreweave.com/products/wandb/weave/guides/evaluation/scorers.md" }, "humanReview": { "label": "Human review and annotation", "value": "yes", "detail": "Feedback and annotations on calls, plus annotation queues that route selected traces to reviewers with boolean, numeric, categorical or free-text fields; annotated traces can be exported to datasets.", "source": "https://docs.coreweave.com/products/wandb/weave/guides/tracking/annotation-queues.md" }, "promptTooling": { "label": "Prompt versioning and playground", "value": "yes", "detail": "Versioned prompt objects and a Playground for editing prompts, comparing models side by side and testing against traces; prompts can be linked to the CoreWeave Registry.", "source": "https://docs.coreweave.com/products/wandb/weave/guides/tools/playground.md" }, "runtimeControls": { "label": "Runtime controls on live agent actions", "value": "partial", "detail": "Guardrails are inline scorers called from application code (Python SDK, call.apply_scorer) that can block or modify inputs and outputs before they reach users. The TypeScript SDK does not support them. No documented human approvals or policy enforcement on agent tool actions.", "source": "https://docs.coreweave.com/products/wandb/weave/guides/evaluation/guardrails.md" }, "gatewayProxy": { "label": "Gateway or proxy in the request path", "value": "unknown", "detail": "No LLM gateway or proxy for trace capture found in the fetched docs; capture is via SDK auto-patching and OTLP. CoreWeave offers separate hosted inference (Serverless Inference) which is not a proxy for third-party models.", "source": "https://docs.coreweave.com/products/wandb/weave/guides/integrations.md" }, "openSource": { "label": "Open source", "value": "partial", "detail": "The wandb/weave GitHub repository (SDK and trace server code) is Apache-2.0. Self-hosting Weave still requires a Weave-enabled licence from W&B and an existing W&B Platform installation.", "source": "https://github.com/wandb/weave" }, "selfHosting": { "label": "Self-hosting", "value": "yes", "detail": "Self-managed Weave on Kubernetes 1.29+ with ClickHouse (Altinity operator, Keeper) and S3-compatible storage; requires a Weave-enabled licence from W&B and a W&B Platform install. Dedicated Cloud endpoints are also documented.", "source": "https://docs.coreweave.com/products/wandb/weave/guides/platform/weave-self-managed.md" } }, "openSource": "Partly. The Weave repository is Apache-2.0, but self-hosting Weave needs a W&B licence and an existing W&B platform deployment.", "runtimeControls": "Partly. Guardrails are scorers called from Python application code that can block or modify inputs and outputs. The TypeScript SDK does not support them, and there are no approvals on agent actions.", "pricing": { "summary": "Forge Free includes 1 GB of Weave ingestion a month. Forge Pro starts at $60 a month with 1.5 GB, for teams under 50 people; the pricing page lists extra ingestion at $0.10 per MB. Enterprise is custom.", "source": "https://coreweave.com/forge-pricing", "checkedAt": "2026-10-20" }, "sdks": [ "Python (weave)", "TypeScript (weave npm package; fewer features than Python: no class-based scorers, guardrails or cost tracking)", "REST service API (OpenAPI spec)", "OTLP/HTTP ingestion" ], "frameworkIntegrations": [ "OpenAI Agents SDK", "Claude Agent SDK", "Claude Code", "LangChain", "LlamaIndex", "Haystack", "DSPy", "Instructor", "CrewAI", "Smolagents", "PydanticAI", "Google ADK", "AutoGen", "Verdict", "Vercel AI SDK", "Agno", "Koog", "LiteLLM", "OpenAI, Anthropic, Amazon Bedrock, Google, Azure, Mistral, Cohere, Groq, Cerebras, Together AI, OpenRouter, NVIDIA NIM, Hugging Face Hub (provider integrations)" ], "limits": [ "TypeScript SDK lags Python: no class-based scorers, guardrails or cost tracking.", "Guardrails require code changes and cover content checks on inputs and outputs, not approvals on agent tool actions.", "Agent tracing focus is moving to Agent Lens, a separate product with its own SDK and UI (public preview); built-in monitors are labelled a 'previous approach'.", "Free and Pro Weave ingestion allowances are small (1 GB and 1.5 GB per month) with per-MB overage.", "Self-hosting needs a W&B Platform install plus a Weave-enabled licence.", "Pricing, docs and login moved to CoreWeave domains in 2026; product packaging is still changing after the Forge launch." ] } ], "reference": { "name": "Prefactor", "shortName": "Prefactor", "slug": "prefactor", "website": "https://prefactor.tech", "definition": "Prefactor is an agent evaluation platform for teams building AI agents: it observes every agent run from development to production, evaluates whether the agent did its job, and shows how to improve it.", "ownership": "Prefactor Pty Ltd, an independent Australian company.", "statusChange": null, "bestFor": "Teams running agents in production who need every run scored against what the agent was meant to do, and the ability to hold or stop an action when evaluation flags it.", "capabilities": { "llmTracing": { "label": "LLM call tracing", "value": "yes", "detail": "Every model call, tool call and custom span captured as it happens.", "source": "https://prefactor.tech/platform/live-tracing" }, "agentTracing": { "label": "Multi-step agent tracing (tool calls, nested spans)", "value": "yes", "detail": "Each agent run is an instance made of spans (LLM calls, tool calls, custom operations), captured in real time.", "source": "https://prefactor.tech/platform/live-tracing" }, "costTracking": { "label": "Cost tracking", "value": "yes", "detail": "Every model call and tool invocation attributed to agent, team, task and user in real time; budgets and limits.", "source": "https://prefactor.tech/platform/cost-tracking" }, "openTelemetry": { "label": "OpenTelemetry support", "value": "yes", "detail": "OpenTelemetry ingest alongside native SDKs.", "source": "https://prefactor.tech/platform" }, "offlineEvals": { "label": "Offline evals (datasets, experiments)", "value": "yes", "detail": "Same evals run offline and against production traffic; open-source deterministic checks exit non-zero in CI.", "source": "https://prefactor.tech/open-source-evals" }, "onlineEvals": { "label": "Online evals on production traffic", "value": "yes", "detail": "Every production run scored step by step; drift tracked by comparing custom spans across versions and environments.", "source": "https://prefactor.tech/platform/quality-assessment" }, "llmAsJudge": { "label": "LLM-as-a-judge scoring", "value": "yes", "detail": "Model-based and rule-based evaluators; your own definitions run as native evals or LLM-as-judge calls.", "source": "https://prefactor.tech/product/case-studies/voice-customer-support" }, "codeEvals": { "label": "Code-based evaluators", "value": "yes", "detail": "Deterministic checks (wrong tools, loops, half-finished runs) with no judge model; custom native evals.", "source": "https://prefactor.tech/open-source-evals" }, "humanReview": { "label": "Human review and annotation", "value": "yes", "detail": "Approval routing for flagged actions; qualitative feedback captured as a custom span on the step it refers to.", "source": "https://prefactor.tech/platform/approval-routing" }, "promptTooling": { "label": "Prompt versioning and playground", "value": "no", "detail": "Not offered as a prompt playground or prompt registry.", "source": "https://prefactor.tech/platform" }, "runtimeControls": { "label": "Runtime controls on live agent actions", "value": "yes", "detail": "Declarative runtime policies block, throttle, sandbox or escalate actions before they run; approval holds; kill switch scoped to a run, agent or team.", "source": "https://prefactor.tech/platform/runtime-policies" }, "gatewayProxy": { "label": "Gateway or proxy in the request path", "value": "no", "detail": "SDK instrumentation; no gateway in the request path.", "source": "https://prefactor.tech/solutions/developers" }, "openSource": { "label": "Open source", "value": "partial", "detail": "Open-source deterministic evals package (prefactor-open-evals); the platform itself is hosted, not open source.", "source": "https://github.com/prefactordev/prefactor-open-evals" }, "selfHosting": { "label": "Self-hosting", "value": "partial", "detail": "Hosted product; enterprise deployments in the customer's region or environment.", "source": "https://prefactor.tech/platform" } }, "openSource": "Partly. Its deterministic evals package is Apache-2.0; the platform is hosted.", "runtimeControls": "Yes. Runtime policies can block, throttle, sandbox or escalate an agent's tool calls and data access before they run, approval routing holds risky actions for a person, and a kill switch stops a run, an agent or a team.", "pricing": { "summary": "Dev is free with 5,000 spans a month. Startup is $49 a month and Scaleup $199 a month. Enterprise is custom above 4 million spans a month.", "source": "https://prefactor.tech/pricing", "checkedAt": "2026-10-20" }, "sdks": [ "TypeScript", "Python" ], "frameworkIntegrations": [ "LangChain", "Claude Agent SDK", "Vercel AI SDK", "OpenClaw", "LiveKit" ], "limits": [ "No prompt playground or prompt registry.", "Platform is hosted; self-hosting only through enterprise deployments.", "SOC 2 Type II not yet certified (in progress)." ] } }