From b116d421def49b8a69b61f9bf3bb699dd5bc2bf4 Mon Sep 17 00:00:00 2001
From: "ml-systems-publisher[bot]"
<304658946+ml-systems-publisher[bot]@users.noreply.github.com>
Date: Wed, 22 Jul 2026 07:21:34 +0000
Subject: [PATCH] Update post: How a Transformer Really Works: Attention, the
KV Cache, and Why Inference Eats Memory
---
.../.write-source.json | 162 +++++++++++-
.../AttentionExplorer.tsx | 236 ++++++------------
.../KVCacheGrower.tsx | 5 -
.../index.mdx | 24 +-
4 files changed, 239 insertions(+), 188 deletions(-)
diff --git a/src/content/posts/how-transformers-work-with-attention-and-kv-cache/.write-source.json b/src/content/posts/how-transformers-work-with-attention-and-kv-cache/.write-source.json
index fcec3df..dfa0bb5 100644
--- a/src/content/posts/how-transformers-work-with-attention-and-kv-cache/.write-source.json
+++ b/src/content/posts/how-transformers-work-with-attention-and-kv-cache/.write-source.json
@@ -129,7 +129,7 @@
},
{
"type": "text",
- "text": " of how a transformer actually works — specifically, how it generates text one token at a time. So this post opens the box. By the end you’ll understand the architecture, what Q/K/V genuinely mean, and — the payoff — exactly ",
+ "text": " of how a transformer actually works — specifically, how it generates text one token at a time. So this post opens the box. By the end you’ll understand the architecture, what Q/K/V mean, and — the payoff — exactly ",
"styles": {}
},
{
@@ -216,7 +216,7 @@
"type": "mermaid",
"props": {
"source": "flowchart TB\n P[\"`**Prompt**\n'The quick brown'`\"] --> E[\"`**Embeddings**\none vector per token`\"]\n E --> BLK\n subgraph BLK[\"Transformer Block — repeated ×N (80 in Llama 3 70B)\"]\n direction TB\n SA[\"`**Self-Attention**\ntokens mix info across the sequence\nq · k · v · o proj`\"] --> FF[\"`**Feed-Forward Network**\neach token processed on its own\ngate · up · down proj`\"]\n end\n BLK --> H[\"`**LM Head**\nscore every possible next token`\"]\n H --> NT[\"`**next token** → 'fox'`\"]\n NT -. \"append it, run the whole stack again — autoregressive\" .-> E\n\n classDef io fill:#dbeafe,stroke:#2563eb,stroke-width:1.5px,color:#0b1f44;\n classDef emb fill:#ede9fe,stroke:#7c3aed,stroke-width:1.5px,color:#2a1257;\n classDef attn fill:#fff7ed,stroke:#ea580c,stroke-width:1.5px,color:#431407;\n classDef ffn fill:#ecfdf5,stroke:#059669,stroke-width:1.5px,color:#052e16;\n classDef head fill:#fee2e2,stroke:#dc2626,stroke-width:1.5px,color:#450a0a;\n class P,NT io;\n class E emb;\n class SA attn;\n class FF ffn;\n class H head;\n style BLK fill:transparent,stroke:#94a3b8,stroke-dasharray:4 4,color:#475569;",
- "svg": "",
+ "svg": "",
"caption": "A prompt becomes vectors, flows through N identical transformer blocks, and the LM Head scores the next token — which is appended and fed back in. That dashed loop is autoregressive generation."
},
"children": []
@@ -255,7 +255,7 @@
"type": "mermaid",
"props": {
"source": "flowchart TB\n IN[\"`token vectors **in**`\"] --> Q & K & V\n subgraph SA[\"Self-Attention · mix information ACROSS tokens\"]\n direction TB\n Q[\"q_proj\"] --> AT{{\"scaled dot-product attention · softmax\"}}\n K[\"k_proj\"] --> AT\n V[\"v_proj\"] --> AT\n AT --> O[\"o_proj\"]\n end\n O --> G & U\n subgraph FF[\"Feed-Forward Network · process EACH token alone\"]\n direction TB\n G[\"gate_proj\"] --> X((\"⊗\"))\n U[\"up_proj\"] --> X\n X --> D[\"down_proj\"]\n end\n D --> OUT[\"`token vectors **out** → next block`\"]\n\n classDef io fill:#dbeafe,stroke:#2563eb,stroke-width:1.5px,color:#0b1f44;\n classDef lin fill:#f8fafc,stroke:#475569,stroke-width:1.2px,color:#0f172a;\n classDef op fill:#fef9c3,stroke:#ca8a04,stroke-width:1.2px,color:#422006;\n class IN,OUT io;\n class Q,K,V,O,G,U,D lin;\n class AT,X op;\n style SA fill:#fff7ed,stroke:#ea580c,color:#431407;\n style FF fill:#ecfdf5,stroke:#059669,color:#052e16;",
- "svg": "",
+ "svg": "",
"caption": "Inside one transformer block. Self-attention (q/k/v converge into attention, then o_proj) mixes information across tokens; the feed-forward network (gate and up multiply, then down_proj) processes each token alone. Every _proj is a learned weight matrix."
},
"children": []
@@ -400,7 +400,7 @@
"content": [
{
"type": "text",
- "text": "So the only genuinely new idea to understand is self-attention. Let’s earn it.",
+ "text": "So the only new idea to understand is self-attention. Let’s earn it.",
"styles": {}
}
],
@@ -1015,7 +1015,7 @@
"content": [
{
"type": "text",
- "text": "Attention in three steps (with real numbers)",
+ "text": "Attention in three steps",
"styles": {}
}
],
@@ -1301,8 +1301,24 @@
"content": [
{
"type": "text",
- "text": "Don’t take my word for the numbers — play with them. Drag the Query below, or click a token to make the Query mimic its Key, and watch the scores and weights recompute live:",
+ "text": "Don’t take my word for the numbers — play with them. ",
"styles": {}
+ },
+ {
+ "type": "text",
+ "text": "See it in GPT-2 example below, token by token. Below is one attention head of GPT-2 (117M) running on our sentence. Pick a token and watch what it attends to as it comes in",
+ "styles": {
+ "textColor": "rgb(246, 248, 250)",
+ "backgroundColor": "rgb(34, 134, 58)"
+ }
+ },
+ {
+ "type": "text",
+ "text": ":",
+ "styles": {
+ "textColor": "rgb(246, 248, 250)",
+ "backgroundColor": "rgb(17, 58, 27)"
+ }
}
],
"children": []
@@ -1312,14 +1328,107 @@
"type": "customComponent",
"props": {
"componentName": "AttentionExplorer",
- "source": "import { useState, useMemo, type CSSProperties, type ReactNode } from 'react';\n\n// A toy 3-dimensional attention. The MECHANISM is exactly the real one —\n// dot-product scores, softmax, value blend — just small enough to see every number.\n// Four context tokens, each with a fixed Key (its \"label\") and Value (its \"content\").\n// You control the Query. Watch attention flow to whichever Key the Query points at.\n\ntype Vec = [number, number, number];\n\nconst TOKENS = ['The', 'quick', 'brown', 'fox'] as const;\n\n// Key = \"what I contain / my label\" (near-orthogonal so a query can select one)\nconst KEYS: Vec[] = [\n [1.0, 0.1, 0.1], // The\n [0.1, 1.0, 0.1], // quick\n [0.1, 0.1, 1.0], // brown\n [0.7, 0.7, 0.2], // fox — shares direction with \"quick\", so it lights up too\n];\n\n// Value = \"my actual content to hand over\"\nconst VALUES: Vec[] = [\n [1.0, 0.0, 0.0], // The\n [0.0, 1.0, 0.0], // quick\n [0.0, 0.0, 1.0], // brown\n [0.5, 0.5, 0.5], // fox\n];\n\nconst dot = (a: Vec, b: Vec) => a[0] * b[0] + a[1] * b[1] + a[2] * b[2];\n\nfunction softmax(xs: number[]): number[] {\n const m = Math.max(...xs);\n const exps = xs.map((x) => Math.exp(x - m));\n const sum = exps.reduce((a, b) => a + b, 0);\n return exps.map((e) => e / sum);\n}\n\nconst mono: CSSProperties = { fontFamily: 'var(--font-mono)', fontSize: 12, color: 'var(--ink-2)' };\nconst fmtVec = (v: Vec) => `[${v.map((n) => n.toFixed(2)).join(', ')}]`;\n\nfunction Slider({\n label,\n value,\n onChange,\n}: {\n label: string;\n value: number;\n onChange: (v: number) => void;\n}) {\n return (\n \n );\n}\n\nexport default function AttentionExplorer(): ReactNode {\n const [q, setQ] = useState([0.2, 0.9, 0.1]);\n\n const { scores, weights, output } = useMemo(() => {\n const scores = KEYS.map((k) => dot(q, k));\n const weights = softmax(scores);\n const output: Vec = [0, 0, 0];\n VALUES.forEach((v, i) => {\n output[0] += weights[i] * v[0];\n output[1] += weights[i] * v[1];\n output[2] += weights[i] * v[2];\n });\n return { scores, weights, output };\n }, [q]);\n\n const topIdx = weights.indexOf(Math.max(...weights));\n const near = (a: number, b: number) => Math.abs(a - b) < 0.05;\n const mimics = (i: number) =>\n near(q[0], KEYS[i][0]) && near(q[1], KEYS[i][1]) && near(q[2], KEYS[i][2]);\n\n return (\n \n \n Attention is a search\n \n\n {/* Query controls */}\n
\n Query vector (what this token is looking for): {fmtVec(q)}\n
\n Try it: click quick — the query now matches quick's Key, so most attention\n flows there (and a little to fox, whose Key shares that direction). The output\n vector becomes mostly quick's Value. That is all attention does: score by Key match, blend the\n Values. Real attention divides scores by √dₖ before softmax; omitted here for legibility.\n
\n \n );\n}\n",
+ "source": "import { useState, type CSSProperties, type ReactNode } from 'react';\n\n// Real, measured attention from GPT-2 (117M) on \"The quick brown fox\" — one representative\n// head (layer 2, head 9), extracted with transformers (output_attentions=True).\n// Rows = the attending token, columns = the token attended to. Attention is causal: a token\n// attends only to itself and earlier tokens, so the upper triangle is zero. Not hand-tuned.\n\nconst TOKENS = ['The', 'quick', 'brown', 'fox'];\n\n// GPT-2 layer 2, head 9 — attention matrix (row = attending token)\nconst M = [\n [1.0, 0, 0, 0],\n [0.67, 0.33, 0, 0],\n [0.43, 0.43, 0.14, 0],\n [0.13, 0.28, 0.51, 0.07],\n];\n\nconst mono: CSSProperties = { fontFamily: 'var(--font-mono)', fontSize: 12, color: 'var(--ink-2)' };\n\nconst btn = (active: boolean): CSSProperties => ({\n ...mono,\n padding: '4px 12px',\n borderRadius: 5,\n cursor: 'pointer',\n border: `1px solid ${active ? 'var(--accent)' : 'var(--line-2)'}`,\n background: active ? 'var(--accent-soft)' : 'transparent',\n color: active ? 'var(--accent)' : 'var(--ink-2)',\n fontWeight: active ? 600 : 400,\n});\n\nexport default function AttentionExplorer(): ReactNode {\n const [at, setAt] = useState(3); // \"fox\"\n\n const row = M[at];\n const topIdx = row.indexOf(Math.max(...row));\n\n return (\n \n \n Self-attention · GPT-2 (117M)\n \n\n {/* Which token is attending */}\n
\n ⚠ Over capacity — weights + KV exceed {nodeGB} GB. Reduce users/context, use FP8, or add GPUs.\n
\n )}\n\n {/* Stats */}\n
\n \n \n \n \n
\n\n \n per token = 2 × {m.layers} layers × {m.kvHeads} kv-heads × {m.headDim} × {precBytes} B ={' '}\n {calc.perTokenKB.toFixed(0)} KB · × {ctx.toLocaleString()} tok ={' '}\n {calc.perRequestGB.toFixed(1)} GB per request\n \n\n
\n Try it: pick Llama 3 70B, 32K context — one request is ~10 GB. Now drag users up:\n the KV cache races past the weights and fills the node. Switch to FP8 and it halves.\n That single bar is why prefix caching, paged-KV, GQA, and quantized-KV all exist.\n
- Try it: click quick — the query now matches quick's Key, so most attention
- flows there (and a little to fox, whose Key shares that direction). The output
- vector becomes mostly quick's Value. That is all attention does: score by Key match, blend the
- Values. Real attention divides scores by √dₖ before softmax; omitted here for legibility.
+
+ Causal masking: a token attends to itself and earlier tokens only, never to ones that come
+ later, so those are 0 (shown as —).
);
diff --git a/src/content/posts/how-transformers-work-with-attention-and-kv-cache/KVCacheGrower.tsx b/src/content/posts/how-transformers-work-with-attention-and-kv-cache/KVCacheGrower.tsx
index c462386..588710a 100644
--- a/src/content/posts/how-transformers-work-with-attention-and-kv-cache/KVCacheGrower.tsx
+++ b/src/content/posts/how-transformers-work-with-attention-and-kv-cache/KVCacheGrower.tsx
@@ -250,11 +250,6 @@ export default function KVCacheGrower(): ReactNode {
{calc.perRequestGB.toFixed(1)} GB per request
-
- Try it: pick Llama 3 70B, 32K context — one request is ~10 GB. Now drag users up:
- the KV cache races past the weights and fills the node. Switch to FP8 and it halves.
- That single bar is why prefix caching, paged-KV, GQA, and quantized-KV all exist.
-
);
}
diff --git a/src/content/posts/how-transformers-work-with-attention-and-kv-cache/index.mdx b/src/content/posts/how-transformers-work-with-attention-and-kv-cache/index.mdx
index 629dac1..3cb31a6 100644
--- a/src/content/posts/how-transformers-work-with-attention-and-kv-cache/index.mdx
+++ b/src/content/posts/how-transformers-work-with-attention-and-kv-cache/index.mdx
@@ -3,7 +3,7 @@ title: 'How a Transformer Really Works: Attention, the KV Cache, and Why Inferen
summary: 'A from-scratch tour of what''s actually inside an LLM: how a transformer turns tokens into predictions, what Query, Key, and Value really mean, and how generating text one token at a time builds the KV cache — the growing pool of memory that makes inference so expensive.'
authors: ['dinesh']
date: '2026-07-21'
-updated: '2026-07-21'
+updated: '2026-07-22'
readMin: 10
topic: 'Architecture'
topicId: 'architecture'
@@ -16,7 +16,7 @@ import KVCacheGrower from './KVCacheGrower';
In [Neural Networks From Zero](/blog/neural-networks-from-zero) we built the machine from one neuron up and learned to size it: a 70B model is **140 GB of weights**. But running it needs _more_ memory than that — a second pool that starts near zero and grows with every token of the conversation, until one long request can cost **\~10 GB on its own**. Why does _one user’s_ conversation cost 10 GB? Why do people obsess over inference memory? Why does long context get so expensive?
-You can’t answer any of that from the outside. The cost is a _consequence_ of how a transformer actually works — specifically, how it generates text one token at a time. So this post opens the box. By the end you’ll understand the architecture, what Q/K/V genuinely mean, and — the payoff — exactly _why memory grows with every token you generate._ That growth is the root cause the whole optimization field is built around.
+You can’t answer any of that from the outside. The cost is a _consequence_ of how a transformer actually works — specifically, how it generates text one token at a time. So this post opens the box. By the end you’ll understand the architecture, what Q/K/V mean, and — the payoff — exactly _why memory grows with every token you generate._ That growth is the root cause the whole optimization field is built around.
## The map: what’s actually stacked inside
@@ -54,7 +54,7 @@ score every possible next token`"]
class H head;
style BLK fill:transparent,stroke:#94a3b8,stroke-dasharray:4 4,color:#475569;
*/}
-
+
“Layer 1, Layer 2, … Layer N” just means _this identical block, stacked N times_ — 80 times for Llama 3 70B. And each block has exactly two parts:
@@ -88,7 +88,7 @@ flowchart TB
style SA fill:#fff7ed,stroke:#ea580c,color:#431407;
style FF fill:#ecfdf5,stroke:#059669,color:#052e16;
*/}
-
+
Hold onto that one-line distinction, because it’s the soul of the architecture:
@@ -97,7 +97,7 @@ Hold onto that one-line distinction, because it’s the soul of the architecture
And here’s the reassuring part: every box with `_proj` in its name is **just a linear layer — a weight matrix**, exactly the kind you counted in the last post. Self-attention is 4 of them; the FFN is 3. That’s _all_ the learned weights are. The famous “12 × d² parameters per layer” is literally these seven matrices added up (≈4·d² for attention + ≈8·d² for the FFN). Nothing new — just assembly.
-So the only genuinely new idea to understand is self-attention. Let’s earn it.
+So the only new idea to understand is self-attention. Let’s earn it.
## Why attention has to exist
@@ -153,7 +153,7 @@ Every token produces all three, by passing its own vector through three differen
-## Attention in three steps (with real numbers)
+## Attention in three steps
Now watch one token — “fox” — update itself by attending over the four tokens “The quick brown fox”. “fox” is the 4th token, so it has query `Q4`.
@@ -221,13 +221,15 @@ So “fox” decides: pay **52%** attention to “quick”, **35%** to itself, *
That’s it — that’s attention. A weighted average of everyone’s Values, where the weights come from how well your Query matches their Keys. Do this for every token, in every block, and information flows across the sentence until each token’s vector encodes not just itself but its whole relevant context.
-Don’t take my word for the numbers — play with them. Drag the Query below, or click a token to make the Query mimic its Key, and watch the scores and weights recompute live:
+Don’t take my word for the numbers — play with them. See it in GPT-2 example below, token by token. Below is one attention head of GPT-2 (117M) running on our sentence. Pick a token and watch what it attends to as it comes in:
-
+
-Real models run attention several times in parallel — **multi-head attention**. Each _head_ gets its own slice of the vector and its own Q/K/V, so different heads can specialize: one tracks grammar, another long-range topic, another position. Llama 3 70B has **64 query heads** of dimension 128 (64 × 128 = 8192). But it deliberately shares Keys and Values across groups of heads — only **8 KV heads** — so there’s less K/V to store. That’s exactly the `8` in the KV-cache formula later, and the trick has a name: **grouped-query attention (GQA)**. It’s your first glimpse of an optimization baked right into the model to shrink the cache.
+Each token attends over itself and the tokens before it — attention is **causal**, it can't see the future — weighting them by relevance: **fox** leans on **brown** and **quick**, the words describing it. Those weights then blend each token's **Value** vector into fox's updated representation, exactly the mechanism from the diagram above. (This is just one of GPT-2's \~144 heads; [others specialize differently](https://arxiv.org/pdf/2405.00208) — some track the previous token, some fixate on the first token — and only together do they capture grammar and meaning.)
+
+In practice, attention runs several times in parallel — **multi-head attention**. Each _head_ gets its own slice of the vector and its own Q/K/V, so different heads can specialize: one tracks grammar, another long-range topic, another position. Llama 3 70B has **64 query heads** of dimension 128 (64 × 128 = 8192). But it deliberately shares Keys and Values across groups of heads — only **8 KV heads** — so there’s less K/V to store. That’s exactly the `8` in the KV-cache formula later, and the trick has a name: **grouped-query attention (GQA)**. It’s your first glimpse of an optimization baked right into the model to shrink the cache.
## The root cause: generating text one token at a time
@@ -281,10 +283,12 @@ Put real numbers on it for Llama 3 70B: `2 (K,V) × 80 layers × 8 KV-heads × 1
Here’s that formula as a live calculator — pick a model, drag the context length and the number of concurrent users, and watch the KV cache fill the GPU against the fixed weights (it’s the Red Hat slide that started this, except now you can break it yourself):
-
+
+Pick Llama 3 70B at 32K context and one request is \~10 GB; drag the user count up and the cache races past the weights until it overruns the node. Switch to **FP8** and every number halves. That single bar is why prefix caching, paged-KV, GQA, and quantized-KV all exist.
+
Drag it around and the scaling speaks for itself. Here is the same model at the context lengths people actually deploy: