{
  "schema_version": 1,
  "diagram_type": "architecture",
  "meta": {
    "title": "vLLM PagedAttention, KV Cache Block Allocator & Engine Architecture",
    "quality_profile": "showcase",
    "repository": {
      "url": "https://github.com/vllm-project/vllm",
      "revision": "2a61f060d342afb680f5a8937ed614b774bf9305"
    },
    "views": [
      {
        "id": "request-lifecycle",
        "label": "Prefill & Decode Lifecycle",
        "focus": ["client-req", "async-engine", "scheduler", "gpu-runner", "physical-vram"],
        "note": "Trace client prompt ingestion through chunked scheduling into GPU model runner."
      },
      {
        "id": "block-allocation",
        "label": "Virtual KV Cache Allocation",
        "focus": ["scheduler", "block-manager", "paged-attn", "physical-vram"],
        "note": "Inspect mapping of logical token sequence blocks to physical non-contiguous GPU memory."
      },
      {
        "id": "cuda-acceleration",
        "label": "CUDA Graph & Kernel Execution",
        "focus": ["gpu-runner", "cuda-graph", "paged-attn"],
        "note": "Follow static execution graph replay and PagedAttention CUDA kernel launch."
      }
    ]
  },
  "components": [
    {
      "id": "client-req",
      "type": "external",
      "label": "API Client",
      "sublabel": "OpenAI HTTP / SSE",
      "pos": [50, 270],
      "size": [140, 60],
      "tag": "v1/entry"
    },
    {
      "id": "async-engine",
      "type": "backend",
      "label": "AsyncLLM Engine",
      "sublabel": "Request Router",
      "pos": [260, 270],
      "size": [140, 60],
      "tag": "v1/engine",
      "sources": [
        {
          "path": "vllm/v1/engine/async_llm.py",
          "line": 50,
          "end_line": 120,
          "label": "AsyncLLM Engine"
        }
      ]
    },
    {
      "id": "scheduler",
      "type": "backend",
      "label": "Chunked Scheduler",
      "sublabel": "Prefill & Decode",
      "pos": [470, 270],
      "size": [150, 60],
      "tag": "Co-Scheduling",
      "sources": [
        {
          "path": "vllm/v1/core/sched/scheduler.py",
          "line": 40,
          "end_line": 95,
          "label": "Chunked Scheduler"
        }
      ]
    },
    {
      "id": "block-manager",
      "type": "database",
      "label": "KVCacheManager",
      "sublabel": "Virtual Page Table",
      "pos": [470, 110],
      "size": [150, 60],
      "tag": "Block Allocator",
      "sources": [
        {
          "path": "vllm/v1/core/kv_cache_manager.py",
          "line": 30,
          "end_line": 85,
          "label": "KVCacheManager"
        }
      ]
    },
    {
      "id": "gpu-runner",
      "type": "backend",
      "label": "GPU Model Runner",
      "sublabel": "Worker Execution",
      "pos": [690, 270],
      "size": [150, 60],
      "tag": "Worker Loop",
      "sources": [
        {
          "path": "vllm/v1/worker/gpu_model_runner.py",
          "line": 45,
          "end_line": 110,
          "label": "GPUModelRunner"
        }
      ]
    },
    {
      "id": "cuda-graph",
      "type": "cloud",
      "label": "CUDA Graph Runner",
      "sublabel": "Static Graph Replay",
      "pos": [690, 110],
      "size": [150, 60],
      "tag": "Zero Overhead"
    },
    {
      "id": "paged-attn",
      "type": "backend",
      "label": "PagedAttention",
      "sublabel": "CUDA Kernel",
      "pos": [690, 430],
      "size": [150, 60],
      "tag": "VRAM Lookup"
    },
    {
      "id": "physical-vram",
      "type": "database",
      "label": "Physical GPU HBM",
      "sublabel": "Non-Contiguous Pages",
      "pos": [910, 270],
      "size": [140, 60],
      "tag": "HBM3 / GDDR6"
    },
    {
      "id": "shared-gqa",
      "type": "database",
      "label": "Shared GQA Cache",
      "sublabel": "Grouped-Query Attn",
      "pos": [910, 430],
      "size": [140, 60],
      "tag": "8x Bandwidth"
    }
  ],
  "boundaries": [
    {
      "kind": "region",
      "label": "vLLM Host Engine Control Plane",
      "wraps": ["async-engine", "scheduler", "block-manager"]
    },
    {
      "kind": "security-group",
      "label": "GPU Acceleration Plane",
      "wraps": ["gpu-runner", "cuda-graph", "paged-attn", "physical-vram", "shared-gqa"]
    }
  ],
  "connections": [
    {
      "id": "client-to-engine",
      "from": "client-req",
      "to": "async-engine",
      "label": "requests",
      "variant": "emphasis"
    },
    {
      "id": "engine-to-sched",
      "from": "async-engine",
      "to": "scheduler",
      "label": "enqueue"
    },
    {
      "id": "sched-to-block",
      "from": "scheduler",
      "to": "block-manager",
      "label": "alloc blocks",
      "fromSide": "top",
      "toSide": "bottom"
    },
    {
      "id": "sched-to-runner",
      "from": "scheduler",
      "to": "gpu-runner",
      "label": "batch",
      "variant": "emphasis"
    },
    {
      "id": "runner-to-graph",
      "from": "gpu-runner",
      "to": "cuda-graph",
      "label": "replay graph",
      "fromSide": "top",
      "toSide": "bottom"
    },
    {
      "id": "runner-to-kernel",
      "from": "gpu-runner",
      "to": "paged-attn",
      "label": "dispatch",
      "fromSide": "bottom",
      "toSide": "top",
      "labelDy": 50
    },
    {
      "id": "runner-to-vram",
      "from": "gpu-runner",
      "to": "physical-vram",
      "label": "fetch KV"
    },
    {
      "id": "kernel-to-gqa",
      "from": "paged-attn",
      "to": "shared-gqa",
      "label": "gqa reduce"
    }
  ],
  "cards": [
    {
      "dot": "cyan",
      "title": "Non-Contiguous VRAM Paging",
      "items": [
        "Partitions GPU memory into 16-token physical blocks",
        "Eliminates 96% internal and external memory fragmentation"
      ]
    },
    {
      "dot": "emerald",
      "title": "Chunked Prefill Co-Scheduling",
      "items": [
        "Slices long prompt prefills into discrete token chunks",
        "Co-schedules prefill chunks with decode passes to saturate tensor cores"
      ]
    },
    {
      "dot": "violet",
      "title": "Static CUDA Graph Replay",
      "items": [
        "Pre-captures decode forward pass kernel sequences into GPU execution graphs",
        "Reduces CPU launch latency to near zero"
      ]
    }
  ]
}