{"id":"gaia--huggingface-co","name":"GAIA","url":"https://huggingface.co/gaia-benchmark","page":"https://mrpeppersdev.github.io/agent-infrastructure-landscape/systems/gaia--huggingface-co","family":"memory-research","section":"Memory benchmarks & evaluation","sections":["Memory benchmarks & evaluation"],"tier":3,"last_verified_at":"2026-10-29","lifecycle":{"status":"active","severity":0,"note":null},"taxonomy":{"storage":[{"value":"n/a","primary":true,"reason":"not applicable — eval dataset, not a system"}],"retrieval":[{"value":"n/a","primary":true,"reason":"not applicable — eval dataset, not a system"}],"persistence":[{"value":"n/a","primary":true,"reason":"not applicable — eval dataset, not a system"}],"update":[{"value":"read-only","primary":true,"reason":null}],"unit":[{"value":"n/a","primary":true,"reason":"not applicable — eval dataset, not a system"}],"governance":[{"value":"n/a","primary":true,"reason":"not applicable — eval dataset, not a system"}],"conflict":[{"value":"n/a","primary":true,"reason":"not applicable — read-only evaluation dataset"}]},"cells":{"type":{"value":"Benchmark","citation":"https://huggingface.co/gaia-benchmark","status":"real-data","tier":"T2","source_class":"vendor"},"desc":{"value":"Benchmark for general AI assistants made of real-world questions that require reasoning, multimodality, web browsing and tool use, with a public leaderboard on Hugging Face. Traction: ~917 likes on the Hugging Face dataset","citation":"https://huggingface.co/gaia-benchmark","status":"real-data","tier":"T2","source_class":"vendor"},"claims":{"value":"Abstract: We introduce GAIA, a benchmark for General AI Assistants that, if solved, would represent a milestone in AI research. GAIA proposes real-world questions that require a set of fundamental abilities such as reasoning, multi-modality handling, web browsing, and generally tool-use proficiency. GAIA questions are conceptually simple for humans yet challenging for most advanced AIs: we show that human respondents obtain 92\\% vs. 15\\% for GPT-4 equipped with plugins. This notable performance disparity contrasts with the recent trend of LLMs outperforming humans on tasks requiring professional skills in e.g. law or chemistry. GAIA's philosophy departs from the current trend in AI benchmarks suggesting to target tasks that are ever more difficult for humans. We posit that the advent of Artificial General Intelligence (AGI) hinges on a system's capability to exhibit similar robustness as the average human does on such questions. Using GAIA's methodology, we devise 466 questions and their answer. We release our questions while retaining answers to 300 of them to power a leader-board available at this https URL .","citation":"https://arxiv.org/abs/2311.12983","status":"real-data","tier":"T2","source_class":"paper"},"modalities":{"value":"searched not found","citation":"https://huggingface.co/gaia-benchmark","status":"depth-floor-reached","tier":"T2","source_class":"vendor"},"created":{"value":"searched not found","citation":"https://arxiv.org/abs/2311.12983","status":"depth-floor-reached","tier":"T2","source_class":"paper"},"latest-release":{"value":"searched not found","citation":"https://huggingface.co/gaia-benchmark","status":"depth-floor-reached","tier":"T2","source_class":"vendor"},"license":{"value":"searched not found","citation":"https://huggingface.co/gaia-benchmark","status":"depth-floor-reached","tier":"T2","source_class":"vendor"},"gh":{"value":"not applicable — no GitHub repo","citation":null,"status":"not-applicable","tier":"T3"},"mindshare":{"value":"searched not found","citation":"https://huggingface.co/gaia-benchmark","status":"depth-floor-reached","tier":"T2","source_class":"vendor"},"citations":{"value":"searched not found","citation":"https://arxiv.org/abs/2311.12983","status":"depth-floor-reached","tier":"T2","source_class":"paper"},"funding":{"value":"searched not found","citation":"https://huggingface.co/gaia-benchmark","status":"depth-floor-reached","tier":"T2","source_class":"vendor"},"customers":{"value":"searched not found","citation":"https://huggingface.co/gaia-benchmark","status":"depth-floor-reached","tier":"T2","source_class":"vendor"},"pricing":{"value":"searched not found","citation":"https://huggingface.co/gaia-benchmark","status":"depth-floor-reached","tier":"T2","source_class":"vendor"},"compliance":{"value":"searched not found","citation":"https://huggingface.co/gaia-benchmark","status":"depth-floor-reached","tier":"T2","source_class":"vendor"},"data-handling":{"value":"searched not found","citation":"https://huggingface.co/gaia-benchmark","status":"depth-floor-reached","tier":"T2","source_class":"vendor"},"deployment":{"value":"searched not found","citation":"https://huggingface.co/gaia-benchmark","status":"depth-floor-reached","tier":"T2","source_class":"vendor"},"hq":{"value":"searched not found","citation":"https://huggingface.co/gaia-benchmark","status":"depth-floor-reached","tier":"T2","source_class":"vendor"},"founders":{"value":"searched not found","citation":"https://huggingface.co/gaia-benchmark","status":"depth-floor-reached","tier":"T2","source_class":"vendor"},"volume":{"value":"searched not found","citation":"https://huggingface.co/gaia-benchmark","status":"depth-floor-reached","tier":"T2","source_class":"vendor"},"perf":{"value":"searched not found","citation":"https://huggingface.co/gaia-benchmark","status":"depth-floor-reached","tier":"T2","source_class":"vendor"},"repro":{"value":"searched not found","citation":"https://huggingface.co/gaia-benchmark","status":"depth-floor-reached","tier":"T2","source_class":"vendor"},"code-release":{"value":"searched not found","citation":"https://huggingface.co/gaia-benchmark","status":"depth-floor-reached","tier":"T2","source_class":"vendor"},"api-surface":{"value":"searched not found","citation":"https://huggingface.co/gaia-benchmark","status":"depth-floor-reached","tier":"T2","source_class":"vendor"},"latency":{"value":"searched not found","citation":"https://huggingface.co/gaia-benchmark","status":"depth-floor-reached","tier":"T2","source_class":"vendor"},"throughput":{"value":"searched not found","citation":"https://huggingface.co/gaia-benchmark","status":"depth-floor-reached","tier":"T2","source_class":"vendor"},"backend-storage":{"value":"searched not found","citation":"https://huggingface.co/gaia-benchmark","status":"depth-floor-reached","tier":"T2","source_class":"vendor"},"multi-tenancy":{"value":"searched not found","citation":"https://huggingface.co/gaia-benchmark","status":"depth-floor-reached","tier":"T2","source_class":"vendor"},"encryption":{"value":"searched not found","citation":"https://huggingface.co/gaia-benchmark","status":"depth-floor-reached","tier":"T2","source_class":"vendor"},"sso-rbac":{"value":"searched not found","citation":"https://huggingface.co/gaia-benchmark","status":"depth-floor-reached","tier":"T2","source_class":"vendor"},"embedding-model":{"value":"searched not found","citation":"https://huggingface.co/gaia-benchmark","status":"depth-floor-reached","tier":"T2","source_class":"vendor"},"consistency":{"value":"searched not found","citation":"https://huggingface.co/gaia-benchmark","status":"depth-floor-reached","tier":"T2","source_class":"vendor"},"versioning":{"value":"searched not found","citation":"https://huggingface.co/gaia-benchmark","status":"depth-floor-reached","tier":"T2","source_class":"vendor"},"tombstoning":{"value":"searched not found","citation":"https://huggingface.co/gaia-benchmark","status":"depth-floor-reached","tier":"T2","source_class":"vendor"},"schema-evolution":{"value":"searched not found","citation":"https://huggingface.co/gaia-benchmark","status":"depth-floor-reached","tier":"T2","source_class":"vendor"},"namespace":{"value":"searched not found","citation":"https://huggingface.co/gaia-benchmark","status":"depth-floor-reached","tier":"T2","source_class":"vendor"},"contradiction":{"value":"searched not found","citation":"https://huggingface.co/gaia-benchmark","status":"depth-floor-reached","tier":"T2","source_class":"vendor"},"forgetting":{"value":"searched not found","citation":"https://huggingface.co/gaia-benchmark","status":"depth-floor-reached","tier":"T2","source_class":"vendor"},"mcp-support":{"value":"searched not found","citation":"https://huggingface.co/gaia-benchmark","status":"depth-floor-reached","tier":"T2","source_class":"vendor"},"a2a-support":{"value":"searched not found","citation":"https://huggingface.co/gaia-benchmark","status":"depth-floor-reached","tier":"T2","source_class":"vendor"},"otel":{"value":"searched not found","citation":"https://huggingface.co/gaia-benchmark","status":"depth-floor-reached","tier":"T2","source_class":"vendor"},"webhooks":{"value":"searched not found","citation":"https://huggingface.co/gaia-benchmark","status":"depth-floor-reached","tier":"T2","source_class":"vendor"},"import-export":{"value":"searched not found","citation":"https://huggingface.co/gaia-benchmark","status":"depth-floor-reached","tier":"T2","source_class":"vendor"},"integration-count":{"value":"searched not found","citation":"https://huggingface.co/gaia-benchmark","status":"depth-floor-reached","tier":"T2","source_class":"vendor"},"orchestration":{"value":"searched not found","citation":"https://huggingface.co/gaia-benchmark","status":"depth-floor-reached","tier":"T2","source_class":"vendor"},"programmatic-control":{"value":"searched not found","citation":"https://huggingface.co/gaia-benchmark","status":"depth-floor-reached","tier":"T2","source_class":"vendor"},"vendor-benchmarks":{"value":"searched not found","citation":"https://huggingface.co/gaia-benchmark","status":"depth-floor-reached","tier":"T2","source_class":"vendor"},"pricing-specifics":{"value":"searched not found","citation":"https://huggingface.co/gaia-benchmark","status":"depth-floor-reached","tier":"T2","source_class":"vendor"},"agent-abstraction":{"value":"searched not found","citation":"https://huggingface.co/gaia-benchmark","status":"depth-floor-reached","tier":"T2","source_class":"vendor"},"memory-primitives":{"value":"searched not found","citation":"https://huggingface.co/gaia-benchmark","status":"depth-floor-reached","tier":"T2","source_class":"vendor"},"llm-lock":{"value":"searched not found","citation":"https://huggingface.co/gaia-benchmark","status":"depth-floor-reached","tier":"T2","source_class":"vendor"},"runtimes":{"value":"searched not found","citation":"https://huggingface.co/gaia-benchmark","status":"depth-floor-reached","tier":"T2","source_class":"vendor"},"session-handling":{"value":"searched not found","citation":"https://huggingface.co/gaia-benchmark","status":"depth-floor-reached","tier":"T2","source_class":"vendor"},"validated-verticals":{"value":"searched not found","citation":"https://huggingface.co/gaia-benchmark","status":"depth-floor-reached","tier":"T2","source_class":"vendor"},"time-to-running":{"value":"searched not found","citation":"https://huggingface.co/gaia-benchmark","status":"depth-floor-reached","tier":"T2","source_class":"vendor"},"anti-fit":{"value":"searched not found","citation":"https://huggingface.co/gaia-benchmark","status":"depth-floor-reached","tier":"T2","source_class":"vendor"},"optimised-for":{"value":"searched not found","citation":"https://huggingface.co/gaia-benchmark","status":"depth-floor-reached","tier":"T2","source_class":"vendor"},"adjacent-infrastructure":{"value":"searched not found","citation":"https://huggingface.co/gaia-benchmark","status":"depth-floor-reached","tier":"T2","source_class":"vendor"},"pros":{"value":"searched not found","citation":"https://huggingface.co/gaia-benchmark","status":"depth-floor-reached","tier":"T2","source_class":"vendor"},"cons":{"value":"searched not found","citation":"https://huggingface.co/gaia-benchmark","status":"depth-floor-reached","tier":"T2","source_class":"vendor"},"links":{"value":"https://huggingface.co/gaia-benchmark ; https://arxiv.org/abs/2311.12983","citation":"https://huggingface.co/gaia-benchmark","status":"real-data","tier":"T2","source_class":"vendor"},"obs-langsmith":{"value":"no","citation":"https://huggingface.co/gaia-benchmark","status":"depth-floor-reached","tier":"T2","source_class":"vendor"},"obs-opentelemetry":{"value":"no","citation":"https://huggingface.co/gaia-benchmark","status":"depth-floor-reached","tier":"T2","source_class":"vendor"},"obs-datadog":{"value":"no","citation":"https://huggingface.co/gaia-benchmark","status":"depth-floor-reached","tier":"T2","source_class":"vendor"},"obs-helicone":{"value":"no","citation":"https://huggingface.co/gaia-benchmark","status":"depth-floor-reached","tier":"T2","source_class":"vendor"},"obs-weave":{"value":"no","citation":"https://huggingface.co/gaia-benchmark","status":"depth-floor-reached","tier":"T2","source_class":"vendor"},"obs-langfuse":{"value":"no","citation":"https://huggingface.co/gaia-benchmark","status":"depth-floor-reached","tier":"T2","source_class":"vendor"},"obs-arize":{"value":"no","citation":"https://huggingface.co/gaia-benchmark","status":"depth-floor-reached","tier":"T2","source_class":"vendor"},"obs-custom":{"value":"no","citation":"https://huggingface.co/gaia-benchmark","status":"depth-floor-reached","tier":"T2","source_class":"vendor"},"cost-token-budget":{"value":"no","citation":"https://huggingface.co/gaia-benchmark","status":"depth-floor-reached","tier":"T2","source_class":"vendor"},"cost-prompt-caching":{"value":"no","citation":"https://huggingface.co/gaia-benchmark","status":"depth-floor-reached","tier":"T2","source_class":"vendor"},"cost-semantic-caching":{"value":"no","citation":"https://huggingface.co/gaia-benchmark","status":"depth-floor-reached","tier":"T2","source_class":"vendor"},"cost-batching":{"value":"no","citation":"https://huggingface.co/gaia-benchmark","status":"depth-floor-reached","tier":"T2","source_class":"vendor"},"cost-model-routing":{"value":"no","citation":"https://huggingface.co/gaia-benchmark","status":"depth-floor-reached","tier":"T2","source_class":"vendor"},"cost-streaming-only":{"value":"no","citation":"https://huggingface.co/gaia-benchmark","status":"depth-floor-reached","tier":"T2","source_class":"vendor"},"cost-observability-cost-attribution":{"value":"no","citation":"https://huggingface.co/gaia-benchmark","status":"depth-floor-reached","tier":"T2","source_class":"vendor"},"eval-langsmith-evals":{"value":"no","citation":"https://huggingface.co/gaia-benchmark","status":"depth-floor-reached","tier":"T2","source_class":"vendor"},"eval-braintrust":{"value":"no","citation":"https://huggingface.co/gaia-benchmark","status":"depth-floor-reached","tier":"T2","source_class":"vendor"},"eval-weights-and-biases-agent":{"value":"no","citation":"https://huggingface.co/gaia-benchmark","status":"depth-floor-reached","tier":"T2","source_class":"vendor"},"eval-helicone-evals":{"value":"no","citation":"https://huggingface.co/gaia-benchmark","status":"depth-floor-reached","tier":"T2","source_class":"vendor"},"eval-custom-test-harness":{"value":"no","citation":"https://huggingface.co/gaia-benchmark","status":"depth-floor-reached","tier":"T2","source_class":"vendor"},"eval-human-loop":{"value":"no","citation":"https://huggingface.co/gaia-benchmark","status":"depth-floor-reached","tier":"T2","source_class":"vendor"},"eval-production-traffic-replay":{"value":"no","citation":"https://huggingface.co/gaia-benchmark","status":"depth-floor-reached","tier":"T2","source_class":"vendor"},"commit-trajectory":{"value":"searched not found","citation":"https://huggingface.co/gaia-benchmark","status":"depth-floor-reached","tier":"T2","source_class":"vendor"},"citation-trajectory":{"value":"not applicable — non-paper row","citation":null,"status":"not-applicable","tier":"T3"},"download-trajectory":{"value":"searched not found","citation":"https://huggingface.co/gaia-benchmark","status":"depth-floor-reached","tier":"T2","source_class":"vendor"},"cost-input-usd-per-mtok":{"value":"not applicable — non-hosted pricing model","citation":null,"status":"not-applicable","tier":"T3"},"cost-output-usd-per-mtok":{"value":"not applicable — non-hosted pricing model","citation":null,"status":"not-applicable","tier":"T3"},"cost-tier":{"value":"searched not found","citation":"https://huggingface.co/gaia-benchmark","status":"depth-floor-reached","tier":"T2","source_class":"vendor"},"cost-pricing-model":{"value":"unknown","citation":null,"status":"estimate","tier":"T3"},"cost-last-verified":{"value":"searched not found","citation":"https://huggingface.co/gaia-benchmark","status":"depth-floor-reached","tier":"T2","source_class":"vendor"},"capability-composite-score":{"value":"(LLM unverified)","citation":null,"status":"estimate","tier":"T3"},"capability-band":{"value":"(LLM unverified)","citation":null,"status":"estimate","tier":"T3"},"capability-benchmark-sources":{"value":"(LLM unverified)","citation":null,"status":"estimate","tier":"T3"},"capability-last-verified":{"value":"(LLM unverified)","citation":null,"status":"estimate","tier":"T3"},"use-case-tags":{"value":"(LLM unverified)","citation":null,"status":"estimate","tier":"T3"},"use-case-anti-tags":{"value":"(LLM unverified)","citation":null,"status":"estimate","tier":"T3"},"adoption-accelerator":{"value":"","citation":null,"status":"no-data","tier":"T3"},"architecture-deployment-locus":{"value":"","citation":null,"status":"no-data","tier":"T3"},"licensing-bucket":{"value":"not applicable — research paper","citation":"https://huggingface.co/gaia-benchmark","status":"not-applicable","tier":"T2","source_class":"vendor"},"positioning-platform-vs-product":{"value":"","citation":null,"status":"no-data","tier":"T3"},"taxonomy-three-layer-rollup":{"value":"agent-infra","citation":"https://github.com/MrPeppersDev/agent-infrastructure-landscape/blob/main/docs/schema/section-to-three-layer-rollup.json","status":"real-data","tier":"T1","source_class":"repo"},"developer-activity-opendigger":{"value":"","citation":null,"status":"no-data","tier":"T3"},"freshness-date":{"value":"2026-10-29","citation":"https://mrpeppersdev.github.io/agent-infrastructure-landscape/systems/gaia--huggingface-co","status":"real-data","tier":"T2","source_class":"catalog"},"compliance-certifications":{"value":"not applicable — no hosted service","citation":"https://huggingface.co/gaia-benchmark","status":"not-applicable","tier":"T2","source_class":"vendor"},"sdk-languages":{"value":"","citation":null,"status":"no-data","tier":"T3"},"protocol-support":{"value":"","citation":null,"status":"no-data","tier":"T3"},"maturity":{"value":"","citation":null,"status":"no-data","tier":"T3"}},"corroboration":{"cited_cells":5,"third_party_cells":0},"edges":{"outgoing":[],"incoming":[{"source":"agentic-plan-caching-apc--neurips-cc","target":"gaia--huggingface-co","type":"cites","evidence":"S2 isInfluential citation","citation":"https://api.semanticscholar.org/graph/v1/paper/arXiv:2506.14852","isInfluential":true},{"source":"cola--arxiv-2503-09263","target":"gaia--huggingface-co","type":"cites","evidence":"S2 isInfluential citation","citation":"https://api.semanticscholar.org/graph/v1/paper/arXiv:2503.09263","isInfluential":true},{"source":"iterresearch--arxiv-2511-07327","target":"gaia--huggingface-co","type":"cites","evidence":"S2 isInfluential citation","citation":"https://api.semanticscholar.org/graph/v1/paper/arXiv:2511.07327","isInfluential":true},{"source":"memento--arxiv-2508-16153","target":"gaia--huggingface-co","type":"cites","evidence":"S2 isInfluential citation","citation":"https://api.semanticscholar.org/graph/v1/paper/arXiv:2508.16153","isInfluential":true},{"source":"memobrain--arxiv-2601-08079","target":"gaia--huggingface-co","type":"cites","evidence":"S2 isInfluential citation","citation":"https://api.semanticscholar.org/graph/v1/paper/arXiv:2601.08079","isInfluential":true}]},"history":[{"date":"2026-10-30","commit":"98ac84a5b9","pr":347,"kind":"added","changes":[]}],"license":"CC-BY-4.0","source":"https://github.com/MrPeppersDev/agent-infrastructure-landscape/blob/main/data/landscape.json"}