{"id":"agentbench--gh-thudm-agentbench","name":"AgentBench","url":"https://github.com/THUDM/AgentBench","page":"https://mrpeppersdev.github.io/agent-infrastructure-landscape/systems/agentbench--gh-thudm-agentbench","family":"memory-research","section":"Memory benchmarks & evaluation","sections":["Memory benchmarks & evaluation"],"tier":3,"last_verified_at":"2026-10-29","lifecycle":{"status":"active","severity":0,"note":null},"taxonomy":{"storage":[{"value":"n/a","primary":true,"reason":"not applicable — eval dataset, not a system"}],"retrieval":[{"value":"n/a","primary":true,"reason":"not applicable — eval dataset, not a system"}],"persistence":[{"value":"n/a","primary":true,"reason":"not applicable — eval dataset, not a system"}],"update":[{"value":"read-only","primary":true,"reason":null}],"unit":[{"value":"n/a","primary":true,"reason":"not applicable — eval dataset, not a system"}],"governance":[{"value":"n/a","primary":true,"reason":"not applicable — eval dataset, not a system"}],"conflict":[{"value":"n/a","primary":true,"reason":"not applicable — read-only evaluation dataset"}]},"cells":{"type":{"value":"Benchmark","citation":"https://github.com/THUDM/AgentBench","status":"real-data","tier":"T1","source_class":"repo"},"desc":{"value":"Multi-environment benchmark (ICLR'24) that evaluates LLMs acting as agents in multiple interactive environments, with code and data in the official repo.","citation":"https://github.com/THUDM/AgentBench","status":"real-data","tier":"T1","source_class":"repo"},"claims":{"value":"Abstract: The potential of Large Language Model (LLM) as agents has been widely acknowledged recently. Thus, there is an urgent need to quantitatively \\textit{evaluate LLMs as agents} on challenging tasks in interactive environments. We present AgentBench, a multi-dimensional benchmark that consists of 8 distinct environments to assess LLM-as-Agent's reasoning and decision-making abilities. Our extensive test over \\num API-based and open-sourced (OSS) LLMs shows that, while top commercial LLMs present a strong ability of acting as agents in complex environments, there is a significant disparity in performance between them and many OSS competitors that are no larger than 70B. We identify the typical reasons of failures in environments and LLMs, showing that poor long-term reasoning, decision-making, and instruction following abilities are the main obstacles for developing usable LLM agents. Improving instruction following and training on high quality multi-round alignment data could improve agent performance. And different from existing assumptions, training on code present ambivalent impacts on different agent tasks. Datasets, environments, and an integrated evaluation package","citation":"https://arxiv.org/abs/2308.03688","status":"real-data","tier":"T2","source_class":"paper"},"modalities":{"value":"searched not found","citation":"https://github.com/THUDM/AgentBench","status":"depth-floor-reached","tier":"T1","source_class":"repo"},"created":{"value":"2023-07-28","citation":"https://github.com/THUDM/AgentBench","status":"real-data","tier":"T1","source_class":"repo"},"latest-release":{"value":"last commit 2026-02-08","citation":"https://github.com/THUDM/AgentBench","status":"real-data","tier":"T1","source_class":"repo"},"license":{"value":"Apache-2.0","citation":"https://github.com/THUDM/AgentBench","status":"real-data","tier":"T1","source_class":"repo"},"gh":{"value":"https://github.com/THUDM/AgentBench (3798★)","citation":"https://github.com/THUDM/AgentBench","status":"real-data","tier":"T1","source_class":"repo"},"mindshare":{"value":"3798 GitHub stars","citation":"https://github.com/THUDM/AgentBench","status":"real-data","tier":"T1","source_class":"repo"},"citations":{"value":"searched not found","citation":"https://arxiv.org/abs/2308.03688","status":"depth-floor-reached","tier":"T2","source_class":"paper"},"funding":{"value":"searched not found","citation":"https://github.com/THUDM/AgentBench","status":"depth-floor-reached","tier":"T1","source_class":"repo"},"customers":{"value":"searched not found","citation":"https://github.com/THUDM/AgentBench","status":"depth-floor-reached","tier":"T1","source_class":"repo"},"pricing":{"value":"searched not found","citation":"https://github.com/THUDM/AgentBench","status":"depth-floor-reached","tier":"T1","source_class":"repo"},"compliance":{"value":"searched not found","citation":"https://github.com/THUDM/AgentBench","status":"depth-floor-reached","tier":"T1","source_class":"repo"},"data-handling":{"value":"searched not found","citation":"https://github.com/THUDM/AgentBench","status":"depth-floor-reached","tier":"T1","source_class":"repo"},"deployment":{"value":"searched not found","citation":"https://github.com/THUDM/AgentBench","status":"depth-floor-reached","tier":"T1","source_class":"repo"},"hq":{"value":"searched not found","citation":"https://github.com/THUDM/AgentBench","status":"depth-floor-reached","tier":"T1","source_class":"repo"},"founders":{"value":"searched not found","citation":"https://github.com/THUDM/AgentBench","status":"depth-floor-reached","tier":"T1","source_class":"repo"},"volume":{"value":"searched not found","citation":"https://github.com/THUDM/AgentBench","status":"depth-floor-reached","tier":"T1","source_class":"repo"},"perf":{"value":"searched not found","citation":"https://github.com/THUDM/AgentBench","status":"depth-floor-reached","tier":"T1","source_class":"repo"},"repro":{"value":"searched not found","citation":"https://github.com/THUDM/AgentBench","status":"depth-floor-reached","tier":"T1","source_class":"repo"},"code-release":{"value":"searched not found","citation":"https://github.com/THUDM/AgentBench","status":"depth-floor-reached","tier":"T1","source_class":"repo"},"api-surface":{"value":"searched not found","citation":"https://github.com/THUDM/AgentBench","status":"depth-floor-reached","tier":"T1","source_class":"repo"},"latency":{"value":"searched not found","citation":"https://github.com/THUDM/AgentBench","status":"depth-floor-reached","tier":"T1","source_class":"repo"},"throughput":{"value":"searched not found","citation":"https://github.com/THUDM/AgentBench","status":"depth-floor-reached","tier":"T1","source_class":"repo"},"backend-storage":{"value":"searched not found","citation":"https://github.com/THUDM/AgentBench","status":"depth-floor-reached","tier":"T1","source_class":"repo"},"multi-tenancy":{"value":"searched not found","citation":"https://github.com/THUDM/AgentBench","status":"depth-floor-reached","tier":"T1","source_class":"repo"},"encryption":{"value":"searched not found","citation":"https://github.com/THUDM/AgentBench","status":"depth-floor-reached","tier":"T1","source_class":"repo"},"sso-rbac":{"value":"searched not found","citation":"https://github.com/THUDM/AgentBench","status":"depth-floor-reached","tier":"T1","source_class":"repo"},"embedding-model":{"value":"searched not found","citation":"https://github.com/THUDM/AgentBench","status":"depth-floor-reached","tier":"T1","source_class":"repo"},"consistency":{"value":"searched not found","citation":"https://github.com/THUDM/AgentBench","status":"depth-floor-reached","tier":"T1","source_class":"repo"},"versioning":{"value":"searched not found","citation":"https://github.com/THUDM/AgentBench","status":"depth-floor-reached","tier":"T1","source_class":"repo"},"tombstoning":{"value":"searched not found","citation":"https://github.com/THUDM/AgentBench","status":"depth-floor-reached","tier":"T1","source_class":"repo"},"schema-evolution":{"value":"searched not found","citation":"https://github.com/THUDM/AgentBench","status":"depth-floor-reached","tier":"T1","source_class":"repo"},"namespace":{"value":"searched not found","citation":"https://github.com/THUDM/AgentBench","status":"depth-floor-reached","tier":"T1","source_class":"repo"},"contradiction":{"value":"searched not found","citation":"https://github.com/THUDM/AgentBench","status":"depth-floor-reached","tier":"T1","source_class":"repo"},"forgetting":{"value":"searched not found","citation":"https://github.com/THUDM/AgentBench","status":"depth-floor-reached","tier":"T1","source_class":"repo"},"mcp-support":{"value":"searched not found","citation":"https://github.com/THUDM/AgentBench","status":"depth-floor-reached","tier":"T1","source_class":"repo"},"a2a-support":{"value":"searched not found","citation":"https://github.com/THUDM/AgentBench","status":"depth-floor-reached","tier":"T1","source_class":"repo"},"otel":{"value":"searched not found","citation":"https://github.com/THUDM/AgentBench","status":"depth-floor-reached","tier":"T1","source_class":"repo"},"webhooks":{"value":"searched not found","citation":"https://github.com/THUDM/AgentBench","status":"depth-floor-reached","tier":"T1","source_class":"repo"},"import-export":{"value":"searched not found","citation":"https://github.com/THUDM/AgentBench","status":"depth-floor-reached","tier":"T1","source_class":"repo"},"integration-count":{"value":"searched not found","citation":"https://github.com/THUDM/AgentBench","status":"depth-floor-reached","tier":"T1","source_class":"repo"},"orchestration":{"value":"searched not found","citation":"https://github.com/THUDM/AgentBench","status":"depth-floor-reached","tier":"T1","source_class":"repo"},"programmatic-control":{"value":"searched not found","citation":"https://github.com/THUDM/AgentBench","status":"depth-floor-reached","tier":"T1","source_class":"repo"},"vendor-benchmarks":{"value":"searched not found","citation":"https://github.com/THUDM/AgentBench","status":"depth-floor-reached","tier":"T1","source_class":"repo"},"pricing-specifics":{"value":"searched not found","citation":"https://github.com/THUDM/AgentBench","status":"depth-floor-reached","tier":"T1","source_class":"repo"},"agent-abstraction":{"value":"searched not found","citation":"https://github.com/THUDM/AgentBench","status":"depth-floor-reached","tier":"T1","source_class":"repo"},"memory-primitives":{"value":"searched not found","citation":"https://github.com/THUDM/AgentBench","status":"depth-floor-reached","tier":"T1","source_class":"repo"},"llm-lock":{"value":"searched not found","citation":"https://github.com/THUDM/AgentBench","status":"depth-floor-reached","tier":"T1","source_class":"repo"},"runtimes":{"value":"searched not found","citation":"https://github.com/THUDM/AgentBench","status":"depth-floor-reached","tier":"T1","source_class":"repo"},"session-handling":{"value":"searched not found","citation":"https://github.com/THUDM/AgentBench","status":"depth-floor-reached","tier":"T1","source_class":"repo"},"validated-verticals":{"value":"searched not found","citation":"https://github.com/THUDM/AgentBench","status":"depth-floor-reached","tier":"T1","source_class":"repo"},"time-to-running":{"value":"searched not found","citation":"https://github.com/THUDM/AgentBench","status":"depth-floor-reached","tier":"T1","source_class":"repo"},"anti-fit":{"value":"searched not found","citation":"https://github.com/THUDM/AgentBench","status":"depth-floor-reached","tier":"T1","source_class":"repo"},"optimised-for":{"value":"searched not found","citation":"https://github.com/THUDM/AgentBench","status":"depth-floor-reached","tier":"T1","source_class":"repo"},"adjacent-infrastructure":{"value":"searched not found","citation":"https://github.com/THUDM/AgentBench","status":"depth-floor-reached","tier":"T1","source_class":"repo"},"pros":{"value":"searched not found","citation":"https://github.com/THUDM/AgentBench","status":"depth-floor-reached","tier":"T1","source_class":"repo"},"cons":{"value":"searched not found","citation":"https://github.com/THUDM/AgentBench","status":"depth-floor-reached","tier":"T1","source_class":"repo"},"links":{"value":"https://github.com/THUDM/AgentBench ; https://arxiv.org/abs/2308.03688","citation":"https://github.com/THUDM/AgentBench","status":"real-data","tier":"T1","source_class":"repo"},"obs-langsmith":{"value":"no","citation":"https://github.com/THUDM/AgentBench","status":"depth-floor-reached","tier":"T1","source_class":"repo"},"obs-opentelemetry":{"value":"no","citation":"https://github.com/THUDM/AgentBench","status":"depth-floor-reached","tier":"T1","source_class":"repo"},"obs-datadog":{"value":"no","citation":"https://github.com/THUDM/AgentBench","status":"depth-floor-reached","tier":"T1","source_class":"repo"},"obs-helicone":{"value":"no","citation":"https://github.com/THUDM/AgentBench","status":"depth-floor-reached","tier":"T1","source_class":"repo"},"obs-weave":{"value":"no","citation":"https://github.com/THUDM/AgentBench","status":"depth-floor-reached","tier":"T1","source_class":"repo"},"obs-langfuse":{"value":"no","citation":"https://github.com/THUDM/AgentBench","status":"depth-floor-reached","tier":"T1","source_class":"repo"},"obs-arize":{"value":"no","citation":"https://github.com/THUDM/AgentBench","status":"depth-floor-reached","tier":"T1","source_class":"repo"},"obs-custom":{"value":"no","citation":"https://github.com/THUDM/AgentBench","status":"depth-floor-reached","tier":"T1","source_class":"repo"},"cost-token-budget":{"value":"no","citation":"https://github.com/THUDM/AgentBench","status":"depth-floor-reached","tier":"T1","source_class":"repo"},"cost-prompt-caching":{"value":"no","citation":"https://github.com/THUDM/AgentBench","status":"depth-floor-reached","tier":"T1","source_class":"repo"},"cost-semantic-caching":{"value":"no","citation":"https://github.com/THUDM/AgentBench","status":"depth-floor-reached","tier":"T1","source_class":"repo"},"cost-batching":{"value":"no","citation":"https://github.com/THUDM/AgentBench","status":"depth-floor-reached","tier":"T1","source_class":"repo"},"cost-model-routing":{"value":"no","citation":"https://github.com/THUDM/AgentBench","status":"depth-floor-reached","tier":"T1","source_class":"repo"},"cost-streaming-only":{"value":"no","citation":"https://github.com/THUDM/AgentBench","status":"depth-floor-reached","tier":"T1","source_class":"repo"},"cost-observability-cost-attribution":{"value":"no","citation":"https://github.com/THUDM/AgentBench","status":"depth-floor-reached","tier":"T1","source_class":"repo"},"eval-langsmith-evals":{"value":"no","citation":"https://github.com/THUDM/AgentBench","status":"depth-floor-reached","tier":"T1","source_class":"repo"},"eval-braintrust":{"value":"no","citation":"https://github.com/THUDM/AgentBench","status":"depth-floor-reached","tier":"T1","source_class":"repo"},"eval-weights-and-biases-agent":{"value":"no","citation":"https://github.com/THUDM/AgentBench","status":"depth-floor-reached","tier":"T1","source_class":"repo"},"eval-helicone-evals":{"value":"no","citation":"https://github.com/THUDM/AgentBench","status":"depth-floor-reached","tier":"T1","source_class":"repo"},"eval-custom-test-harness":{"value":"no","citation":"https://github.com/THUDM/AgentBench","status":"depth-floor-reached","tier":"T1","source_class":"repo"},"eval-human-loop":{"value":"no","citation":"https://github.com/THUDM/AgentBench","status":"depth-floor-reached","tier":"T1","source_class":"repo"},"eval-production-traffic-replay":{"value":"no","citation":"https://github.com/THUDM/AgentBench","status":"depth-floor-reached","tier":"T1","source_class":"repo"},"commit-trajectory":{"value":"searched not found","citation":"https://github.com/THUDM/AgentBench","status":"depth-floor-reached","tier":"T1","source_class":"repo"},"citation-trajectory":{"value":"not applicable — non-paper row","citation":null,"status":"not-applicable","tier":"T3"},"download-trajectory":{"value":"searched not found","citation":"https://github.com/THUDM/AgentBench","status":"depth-floor-reached","tier":"T1","source_class":"repo"},"cost-input-usd-per-mtok":{"value":"not applicable — non-hosted pricing model","citation":null,"status":"not-applicable","tier":"T3"},"cost-output-usd-per-mtok":{"value":"not applicable — non-hosted pricing model","citation":null,"status":"not-applicable","tier":"T3"},"cost-tier":{"value":"searched not found","citation":"https://github.com/THUDM/AgentBench","status":"depth-floor-reached","tier":"T1","source_class":"repo"},"cost-pricing-model":{"value":"unknown","citation":null,"status":"estimate","tier":"T3"},"cost-last-verified":{"value":"searched not found","citation":"https://github.com/THUDM/AgentBench","status":"depth-floor-reached","tier":"T1","source_class":"repo"},"capability-composite-score":{"value":"(LLM unverified)","citation":null,"status":"estimate","tier":"T3"},"capability-band":{"value":"(LLM unverified)","citation":null,"status":"estimate","tier":"T3"},"capability-benchmark-sources":{"value":"(LLM unverified)","citation":null,"status":"estimate","tier":"T3"},"capability-last-verified":{"value":"(LLM unverified)","citation":null,"status":"estimate","tier":"T3"},"use-case-tags":{"value":"(LLM unverified)","citation":null,"status":"estimate","tier":"T3"},"use-case-anti-tags":{"value":"(LLM unverified)","citation":null,"status":"estimate","tier":"T3"},"adoption-accelerator":{"value":"","citation":null,"status":"no-data","tier":"T3"},"architecture-deployment-locus":{"value":"","citation":null,"status":"no-data","tier":"T3"},"licensing-bucket":{"value":"open-source","citation":"https://github.com/THUDM/AgentBench","status":"real-data","tier":"T1","source_class":"repo"},"positioning-platform-vs-product":{"value":"","citation":null,"status":"no-data","tier":"T3"},"taxonomy-three-layer-rollup":{"value":"agent-infra","citation":"https://github.com/MrPeppersDev/agent-infrastructure-landscape/blob/main/docs/schema/section-to-three-layer-rollup.json","status":"real-data","tier":"T1","source_class":"repo"},"developer-activity-opendigger":{"value":"","citation":null,"status":"no-data","tier":"T3"},"freshness-date":{"value":"2026-10-29","citation":"https://mrpeppersdev.github.io/agent-infrastructure-landscape/systems/agentbench--gh-thudm-agentbench","status":"real-data","tier":"T2","source_class":"catalog"},"compliance-certifications":{"value":"not applicable — no hosted service","citation":"https://github.com/THUDM/AgentBench","status":"not-applicable","tier":"T1","source_class":"repo"},"sdk-languages":{"value":"python","citation":"https://raw.githubusercontent.com/THUDM/AgentBench/main/README.md","status":"real-data","tier":"T2","source_class":"repo"},"protocol-support":{"value":"","citation":null,"status":"no-data","tier":"T3"},"maturity":{"value":"","citation":null,"status":"no-data","tier":"T3"}},"corroboration":{"cited_cells":12,"third_party_cells":0},"edges":{"outgoing":[],"incoming":[{"source":"memgpt-v2-agent-tools--arxiv-2310-08560","target":"agentbench--gh-thudm-agentbench","type":"cites","evidence":"S2 isInfluential citation","citation":"https://api.semanticscholar.org/graph/v1/paper/arXiv:2310.08560","isInfluential":true}]},"history":[{"date":"2026-10-30","commit":"98ac84a5b9","pr":347,"kind":"added","changes":[]}],"license":"CC-BY-4.0","source":"https://github.com/MrPeppersDev/agent-infrastructure-landscape/blob/main/data/landscape.json"}