{"id":"openai-realtime-api--platform-openai-com","name":"OpenAI Realtime API","url":"https://platform.openai.com/docs/guides/realtime","page":"https://mrpeppersdev.github.io/agent-infrastructure-landscape/systems/openai-realtime-api--platform-openai-com","family":"agents","section":"Voice agent platforms","sections":["Voice agent platforms"],"tier":1,"last_verified_at":"2026-10-26","lifecycle":{"status":"active","severity":0,"note":null},"taxonomy":null,"cells":{"type":{"value":"Speech-to-speech LLM API (OpenAI Realtime)","citation":"https://platform.openai.com/docs/guides/realtime","status":"real-data","tier":"T2","source_class":"vendor"},"desc":{"value":"OpenAI's Realtime API for native speech-to-speech multimodal interaction with GPT-4o — launched Oct 2024 (DevDay). Skips the STT → LLM → TTS pipeline by passing speech directly to GPT-4o. The platform substrate that Vapi/Retell/Bland integrate as one of their LLM choices.","citation":"https://platform.openai.com/docs/guides/realtime","status":"real-data","tier":"T2","source_class":"vendor"},"claims":{"value":"Launched DevDay Oct-2024; GPT-4o speech-to-speech native; WebSocket + WebRTC support","citation":"https://platform.openai.com/docs/guides/realtime","status":"real-data","tier":"T2","source_class":"vendor"},"modalities":{"value":"voice + text (native multimodal)","citation":"https://platform.openai.com/docs/guides/realtime","status":"real-data","tier":"T2","source_class":"vendor"},"created":{"value":"2024-10","citation":"https://platform.openai.com/docs/guides/realtime","status":"real-data","tier":"T2","last_verified_at":"2026-05-14","source_class":"vendor"},"latest-release":{"value":"not applicable — not OSS","citation":null,"status":"not-applicable","tier":"T3","last_verified_at":"2026-05-14"},"license":{"value":"not applicable — not OSS","citation":null,"status":"not-applicable","tier":"T3"},"gh":{"value":"openai/openai-realtime-agents — ~6.9k stars (MIT reference app on Agents SDK), last commit n/a; API itself closed source","citation":"https://github.com/openai/openai-realtime-agents","status":"real-data","tier":"T1","last_verified_at":"2026-10-26","source_class":"repo"},"mindshare":{"value":"Highest-impact OpenAI launch Oct 2024; many voice-AI platforms adopted it","citation":"https://platform.openai.com/docs/guides/realtime","status":"real-data","tier":"T2","last_verified_at":"2026-05-14","source_class":"vendor"},"citations":{"value":"not applicable — not academic","citation":null,"status":"not-applicable","tier":"T3","last_verified_at":"2026-05-14"},"funding":{"value":"OpenAI parent — $40B round Mar-2025 ($300B val)","citation":"https://openai.com/about","status":"real-data","tier":"T2","last_verified_at":"2026-05-14","source_class":"vendor"},"customers":{"value":"API customers (broad horizontal)","citation":"https://openai.com/","status":"real-data","tier":"T2","source_class":"vendor"},"pricing":{"value":"Audio token-based: $40/$80 per 1M input/output audio tokens (Oct-2024 launch)","citation":"https://openai.com/api/pricing/","status":"real-data","tier":"T2","source_class":"vendor"},"compliance":{"value":"SOC 2 Type II; HIPAA via OpenAI Enterprise","citation":"https://openai.com/security-and-privacy/","status":"real-data","tier":"T2","source_class":"vendor"},"data-handling":{"value":"API zero-data-retention available; default 30-day abuse retention","citation":"https://openai.com/api-data-privacy","status":"real-data","tier":"T2","source_class":"vendor"},"deployment":{"value":"API (Cloud); no self-host","citation":"https://platform.openai.com/docs/guides/realtime","status":"real-data","tier":"T2","source_class":"vendor"},"hq":{"value":"US (OpenAI, San Francisco)","citation":"https://openai.com/about","status":"real-data","tier":"T2","source_class":"vendor"},"founders":{"value":"OpenAI (Sam Altman, CEO)","citation":"https://openai.com/about","status":"real-data","tier":"T2","source_class":"vendor"},"volume":{"value":"searched not found","citation":"https://www.google.com/search?q=OpenAI+Realtime+API+volume","status":"depth-floor-reached","tier":"T2","source_class":"third-party"},"perf":{"value":"Sub-500ms turn latency typical; vendor claim ~320ms","citation":"https://platform.openai.com/docs/guides/realtime","status":"real-data","tier":"T2","source_class":"vendor"},"repro":{"value":"not applicable — not a research paper","citation":null,"status":"not-applicable","tier":"T3"},"code-release":{"value":"Examples OSS — github.com/openai/openai-realtime-agents","citation":"https://github.com/openai/openai-realtime-agents","status":"real-data","tier":"T1","source_class":"repo"},"api-surface":{"value":"WebSocket + WebRTC + REST","citation":"https://platform.openai.com/docs/guides/realtime","status":"real-data","tier":"T2","source_class":"vendor"},"latency":{"value":"~320ms turn latency vendor claim","citation":"https://platform.openai.com/docs/guides/realtime","status":"real-data","tier":"T2","source_class":"vendor"},"throughput":{"value":"searched not found","citation":"https://www.google.com/search?q=OpenAI+Realtime+throughput","status":"depth-floor-reached","tier":"T2","source_class":"third-party"},"backend-storage":{"value":"Stateless session API; caller manages durability","citation":"https://platform.openai.com/docs/guides/realtime","status":"real-data","tier":"T2","source_class":"vendor"},"multi-tenancy":{"value":"API multi-tenant","citation":"https://openai.com/","status":"real-data","tier":"T2","source_class":"vendor"},"encryption":{"value":"TLS + AES-256","citation":"https://openai.com/security-and-privacy/","status":"real-data","tier":"T2","source_class":"vendor"},"sso-rbac":{"value":"via OpenAI Console / Enterprise","citation":"https://openai.com/","status":"real-data","tier":"T2","source_class":"vendor"},"embedding-model":{"value":"not applicable — not a memory product","citation":null,"status":"not-applicable","tier":"T3"},"consistency":{"value":"searched not found","citation":"https://www.google.com/search?q=OpenAI+Realtime+consistency","status":"depth-floor-reached","tier":"T2","source_class":"third-party"},"versioning":{"value":"Model snapshots (gpt-4o-realtime-preview, etc.)","citation":"https://platform.openai.com/docs/guides/realtime","status":"real-data","tier":"T2","source_class":"vendor"},"tombstoning":{"value":"not applicable — stateless","citation":null,"status":"not-applicable","tier":"T3"},"schema-evolution":{"value":"not applicable — not a memory product","citation":null,"status":"not-applicable","tier":"T3"},"namespace":{"value":"per-API-key","citation":"https://platform.openai.com/docs/guides/realtime","status":"real-data","tier":"T2","source_class":"vendor"},"contradiction":{"value":"not applicable — stateless","citation":null,"status":"not-applicable","tier":"T3"},"forgetting":{"value":"Zero retention available on API","citation":"https://openai.com/api-data-privacy","status":"real-data","tier":"T2","source_class":"vendor"},"mcp-support":{"value":"searched not found","citation":"https://www.google.com/search?q=OpenAI+Realtime+MCP","status":"depth-floor-reached","tier":"T2","source_class":"third-party"},"a2a-support":{"value":"searched not found","citation":"https://www.google.com/search?q=OpenAI+Realtime+A2A","status":"depth-floor-reached","tier":"T2","source_class":"third-party"},"otel":{"value":"searched not found","citation":"https://www.google.com/search?q=OpenAI+Realtime+OpenTelemetry","status":"depth-floor-reached","tier":"T2","source_class":"third-party"},"webhooks":{"value":"not applicable — stateless","citation":null,"status":"not-applicable","tier":"T3"},"import-export":{"value":"not applicable — stateless","citation":null,"status":"not-applicable","tier":"T3"},"integration-count":{"value":"not applicable — wrong section","citation":null,"status":"not-applicable","tier":"T3"},"orchestration":{"value":"not applicable — wrong section","citation":null,"status":"not-applicable","tier":"T3"},"programmatic-control":{"value":"not applicable — wrong section","citation":null,"status":"not-applicable","tier":"T3"},"vendor-benchmarks":{"value":"not applicable — wrong section","citation":null,"status":"not-applicable","tier":"T3","last_verified_at":"2026-05-14"},"pricing-specifics":{"value":"not applicable — wrong section","citation":null,"status":"not-applicable","tier":"T3"},"agent-abstraction":{"value":"not applicable — wrong section","citation":null,"status":"not-applicable","tier":"T3"},"memory-primitives":{"value":"not applicable — wrong section","citation":null,"status":"not-applicable","tier":"T3"},"llm-lock":{"value":"not applicable — wrong section","citation":null,"status":"not-applicable","tier":"T3"},"runtimes":{"value":"not applicable — wrong section","citation":null,"status":"not-applicable","tier":"T3"},"session-handling":{"value":"not applicable — wrong section","citation":null,"status":"not-applicable","tier":"T3"},"validated-verticals":{"value":"horizontal — any voice agent","citation":"https://platform.openai.com/docs/guides/realtime","status":"real-data","tier":"T2","source_class":"vendor"},"time-to-running":{"value":"<1hr (API key + WebRTC integration)","citation":"https://platform.openai.com/docs/guides/realtime","status":"real-data","tier":"T2","source_class":"vendor"},"anti-fit":{"value":"Stateless platform — caller builds orchestration, memory, telephony adapter","citation":"https://platform.openai.com/docs/guides/realtime","status":"real-data","tier":"T2","source_class":"vendor"},"optimised-for":{"value":"Native speech-to-speech with GPT-4o; lowest-latency LLM voice tier","citation":"https://platform.openai.com/docs/guides/realtime","status":"real-data","tier":"T2","source_class":"vendor"},"adjacent-infrastructure":{"value":"Twilio (for PSTN); WebRTC clients; LiveKit Agents","citation":"https://platform.openai.com/docs/guides/realtime","status":"real-data","tier":"T2","source_class":"vendor"},"pros":{"value":"Native speech-to-speech (skips STT/TTS); OpenAI brand + price/perf; WebRTC support.","citation":"https://platform.openai.com/docs/guides/realtime","status":"real-data","tier":"T2","source_class":"vendor"},"cons":{"value":"Audio token pricing is expensive at scale; stateless (caller builds memory/orchestration); OpenAI-only model choice.","citation":"https://platform.openai.com/docs/guides/realtime","status":"real-data","tier":"T2","source_class":"vendor"},"links":{"value":"platform.openai.com/docs/guides/realtime; github.com/openai/openai-realtime-agents","citation":"https://platform.openai.com/docs/guides/realtime","status":"real-data","tier":"T2","source_class":"vendor"},"obs-langsmith":{"value":"yes","citation":"https://docs.smith.langchain.com/observability/how_to_guides/trace_with_openai","status":"real-data","tier":"T2","last_verified_at":"2026-05-14","source_class":"third-party"},"obs-opentelemetry":{"value":"yes","citation":"https://platform.openai.com/docs/guides/observability","status":"real-data","tier":"T2","last_verified_at":"2026-05-14","source_class":"vendor"},"obs-datadog":{"value":"yes","citation":"https://docs.datadoghq.com/llm_observability/instrumentation/auto_instrumentation/?tab=python","status":"real-data","tier":"T2","last_verified_at":"2026-05-14","source_class":"third-party"},"obs-helicone":{"value":"yes","citation":"https://docs.helicone.ai/integrations/openai/python","status":"real-data","tier":"T2","last_verified_at":"2026-05-14","source_class":"third-party"},"obs-weave":{"value":"yes","citation":"https://weave-docs.wandb.ai/guides/integrations/openai","status":"real-data","tier":"T2","last_verified_at":"2026-05-14","source_class":"third-party"},"obs-langfuse":{"value":"yes","citation":"https://langfuse.com/docs/integrations/openai/python/get-started","status":"real-data","tier":"T2","last_verified_at":"2026-05-14","source_class":"third-party"},"obs-arize":{"value":"yes","citation":"https://docs.arize.com/phoenix/tracing/integrations-tracing/openai","status":"real-data","tier":"T2","last_verified_at":"2026-05-14","source_class":"third-party"},"obs-custom":{"value":"All major LLM-observability vendors auto-instrument OpenAI SDK calls","citation":"https://platform.openai.com/docs","status":"real-data","tier":"T2","last_verified_at":"2026-05-14","source_class":"vendor"},"cost-token-budget":{"value":"yes","citation":"https://platform.openai.com/docs/guides/realtime","status":"real-data","tier":"T2","last_verified_at":"2026-05-14","source_class":"vendor"},"cost-prompt-caching":{"value":"yes","citation":"https://platform.openai.com/docs/guides/prompt-caching","status":"real-data","tier":"T2","last_verified_at":"2026-05-14","source_class":"vendor"},"cost-semantic-caching":{"value":"no","citation":"https://platform.openai.com/docs/guides/realtime","status":"real-data","tier":"T2","last_verified_at":"2026-05-14","source_class":"vendor"},"cost-batching":{"value":"no","citation":"https://platform.openai.com/docs/guides/realtime","status":"real-data","tier":"T2","last_verified_at":"2026-05-14","source_class":"vendor"},"cost-model-routing":{"value":"no","citation":"https://platform.openai.com/docs/guides/realtime","status":"real-data","tier":"T2","last_verified_at":"2026-05-14","source_class":"vendor"},"cost-streaming-only":{"value":"yes","citation":"https://platform.openai.com/docs/guides/realtime","status":"real-data","tier":"T2","last_verified_at":"2026-05-14","source_class":"vendor"},"cost-observability-cost-attribution":{"value":"no","citation":"https://platform.openai.com/docs/guides/realtime","status":"real-data","tier":"T2","last_verified_at":"2026-05-14","source_class":"vendor"},"eval-langsmith-evals":{"value":"yes","citation":"https://docs.smith.langchain.com/evaluation","status":"real-data","tier":"T2","last_verified_at":"2026-05-14","source_class":"third-party"},"eval-braintrust":{"value":"yes","citation":"https://www.braintrust.dev/docs","status":"real-data","tier":"T2","last_verified_at":"2026-05-14","source_class":"third-party"},"eval-weights-and-biases-agent":{"value":"yes","citation":"https://weave-docs.wandb.ai/guides/integrations/openai","status":"real-data","tier":"T2","last_verified_at":"2026-05-14","source_class":"third-party"},"eval-helicone-evals":{"value":"yes","citation":"https://docs.helicone.ai/integrations/openai/javascript","status":"real-data","tier":"T2","last_verified_at":"2026-05-14","source_class":"third-party"},"eval-custom-test-harness":{"value":"no","citation":"https://platform.openai.com/","status":"real-data","tier":"T2","last_verified_at":"2026-05-14","source_class":"vendor"},"eval-human-loop":{"value":"no","citation":"https://platform.openai.com/","status":"real-data","tier":"T2","last_verified_at":"2026-05-14","source_class":"vendor"},"eval-production-traffic-replay":{"value":"no","citation":"https://platform.openai.com/","status":"real-data","tier":"T2","last_verified_at":"2026-05-14","source_class":"vendor"},"commit-trajectory":{"value":"[{\"ym\":\"2025-01\",\"cum\":2},{\"ym\":\"2025-05\",\"cum\":31},{\"ym\":\"2025-06\",\"cum\":37},{\"ym\":\"2025-07\",\"cum\":39},{\"ym\":\"2025-08\",\"cum\":41},{\"ym\":\"2025-09\",\"cum\":44},{\"ym\":\"2025-10\",\"cum\":45},{\"ym\":\"2026-01\",\"cum\":46}]","citation":"https://github.com/openai/openai-realtime-agents","status":"real-data","tier":"T1","last_verified_at":"2026-05-14","source_class":"repo"},"citation-trajectory":{"value":"not applicable — non-paper row","citation":null,"status":"not-applicable","tier":"T3","last_verified_at":"2026-05-14"},"download-trajectory":{"value":"not applicable — not published as a package","citation":null,"status":"not-applicable","tier":"T3","last_verified_at":"2026-05-14"},"cost-input-usd-per-mtok":{"value":"40","citation":null,"status":"estimate","tier":"T3"},"cost-output-usd-per-mtok":{"value":"80","citation":null,"status":"estimate","tier":"T3"},"cost-tier":{"value":"premium","citation":null,"status":"estimate","tier":"T3"},"cost-pricing-model":{"value":"per-token","citation":null,"status":"estimate","tier":"T3"},"cost-last-verified":{"value":"2026-06-29","citation":null,"status":"estimate","tier":"T3"},"capability-composite-score":{"value":"78","citation":null,"status":"estimate","tier":"T3"},"capability-band":{"value":"frontier","citation":null,"status":"estimate","tier":"T3"},"capability-benchmark-sources":{"value":"no benchmark of its own; capability comes from the GPT realtime model in use (see OpenAI GPT family)","citation":null,"status":"estimate","tier":"T3"},"capability-last-verified":{"value":"2026-06-29","citation":null,"status":"estimate","tier":"T3"},"use-case-tags":{"value":"latency-sensitive, scoped-agentic","citation":null,"status":"estimate","tier":"T3"},"use-case-anti-tags":{"value":"code-generation-focused, offline-capable, analytical-summarization","citation":null,"status":"estimate","tier":"T3"},"adoption-accelerator":{"value":"","citation":null,"status":"no-data","tier":"T3"},"architecture-deployment-locus":{"value":"cloud","citation":"https://developers.openai.com/api/docs/guides/realtime","status":"real-data","tier":"T2","source_class":"vendor"},"licensing-bucket":{"value":"commercial","citation":"https://openai.com/api/pricing/","status":"real-data","tier":"T2","source_class":"vendor"},"positioning-platform-vs-product":{"value":"platform","citation":"https://developers.openai.com/api/docs/guides/realtime","status":"real-data","tier":"T2","source_class":"vendor"},"taxonomy-three-layer-rollup":{"value":"agent-infra","citation":"https://github.com/MrPeppersDev/agent-infrastructure-landscape/blob/main/docs/schema/section-to-three-layer-rollup.json","status":"real-data","tier":"T1","source_class":"repo"},"developer-activity-opendigger":{"value":"{\"activity\":8.94,\"as_of\":\"2026-09\",\"openrank_12mo\":2.29,\"openrank_current\":0.6,\"participants\":4}","citation":"https://oss.open-digger.cn/github/openai/openai-realtime-agents/openrank.json","status":"real-data","tier":"T2","last_verified_at":"2026-10-28","source_class":"third-party"},"freshness-date":{"value":"2026-10-26","citation":"https://mrpeppersdev.github.io/agent-infrastructure-landscape/systems/openai-realtime-api--platform-openai-com","status":"real-data","tier":"T2","source_class":"catalog"},"compliance-certifications":{"value":"ccpa,csa-star,gdpr,iso27001,iso27017,iso27018,iso27701,iso42001,soc2-type2,soc3","citation":"https://trust.openai.com/","status":"real-data","tier":"T2","source_class":"vendor"},"sdk-languages":{"value":"csharp,go,java,python,rest,ruby,typescript","citation":"https://platform.openai.com/docs/libraries","status":"real-data","tier":"T2","source_class":"vendor"},"protocol-support":{"value":"mcp-client","citation":"https://developers.openai.com/api/docs/guides/realtime-mcp.md","status":"real-data","tier":"T2","source_class":"vendor"},"maturity":{"value":"ga","citation":"https://platform.openai.com/docs/guides/realtime","status":"real-data","tier":"T2","source_class":"vendor"}},"corroboration":{"cited_cells":66,"third_party_cells":11},"edges":{"outgoing":[],"incoming":[{"source":"helicone--helicone-ai","target":"openai-realtime-api--platform-openai-com","type":"integrates-with","evidence":"Helicone docs have a OpenAI Realtime API integration page.","citation":"https://docs.helicone.ai/integrations/openai/realtime","verified_at":"2026-10-29"},{"source":"langsmith--langchain-com","target":"openai-realtime-api--platform-openai-com","type":"integrates-with","evidence":"LangSmith docs have a tracing integration page for OpenAI Realtime API.","citation":"https://docs.langchain.com/langsmith/trace-openai-realtime","verified_at":"2026-10-29"},{"source":"livekit-agents--docs-livekit-io","target":"openai-realtime-api--platform-openai-com","type":"integrates-with","evidence":"LiveKit Agents docs have a OpenAI Realtime API model plugin page.","citation":"https://docs.livekit.io/agents/models/realtime/openai/","verified_at":"2026-10-29"},{"source":"llamaindex-framework--llamaindex-ai","target":"openai-realtime-api--platform-openai-com","type":"integrates-with","evidence":"LlamaIndex has an official voice agent integration package llama-index-voice-agents-openai in its repo.","citation":"https://github.com/run-llama/llama_index/tree/main/llama-index-integrations/voice_agents/llama-index-voice-agents-openai","verified_at":"2026-10-29"},{"source":"mastra--mastra-ai","target":"openai-realtime-api--platform-openai-com","type":"integrates-with","evidence":"Mastra repo ships the OpenAI Realtime API voice package.","citation":"https://github.com/mastra-ai/mastra/tree/main/voice/openai-realtime-api","verified_at":"2026-10-29"},{"source":"pipecat-daily--pipecat-ai","target":"openai-realtime-api--platform-openai-com","type":"integrates-with","evidence":"Pipecat docs have a OpenAI Realtime API service page.","citation":"https://docs.pipecat.ai/api-reference/server/services/s2s/openai","verified_at":"2026-10-29"},{"source":"pydantic-ai--ai-pydantic-dev","target":"openai-realtime-api--platform-openai-com","type":"integrates-with","evidence":"Pydantic AI realtime docs have an OpenAI page for gpt-realtime voice models (OpenAIRealtimeModel).","citation":"https://pydantic.dev/docs/ai/realtime/openai/","verified_at":"2026-10-29"}]},"history":[],"license":"CC-BY-4.0","source":"https://github.com/MrPeppersDev/agent-infrastructure-landscape/blob/main/data/landscape.json"}