[{"_1":2,"_3":-5,"_4":-5},"loaderData",{"_5":6,"_7":8},"actionData","errors","root",{"_26":27,"_28":29,"_30":12,"_31":12,"_32":12,"_33":12,"_34":12,"_35":12,"_36":37},"routes/canvas.shared.$sharedTextdocId",{"_9":10,"_11":12},"sharedTextdoc",{"_13":14,"_15":-7,"_16":17,"_18":19,"_20":21,"_22":23,"_24":12,"_25":12},"isAuthenticated",false,"sharedTextdocId","69e78eb867708191ad807b253082d420","versionInt","title","Llm Autoregressive Toy Visualizer","type","code/react","content","import React, { useEffect, useMemo, useState } from \"react\";\nimport { motion } from \"framer-motion\";\nimport { Card, CardContent } from \"@/components/ui/card\";\nimport { Button } from \"@/components/ui/button\";\nimport { Slider } from \"@/components/ui/slider\";\nimport { Input } from \"@/components/ui/input\";\nimport { Play, Pause, RotateCcw, SkipForward } from \"lucide-react\";\n\nconst TOKENS = [\n \"\",\n \"What\",\n \"'s\",\n \"the\",\n \"capital\",\n \"of\",\n \"Belgium\",\n \"?\",\n \"The\",\n \"is\",\n \"Brussels\",\n \".\",\n \"\",\n];\n\nconst CONTEXT = 3;\n\nconst W1 = [\n [0.8, 0.1, 0.0, 0.0, 0.1, 0.0, 0.3, 0.0, 0.6, 0.0, 0.3, 0.0, 0.0, 0.8, 0.1, 0.0, 0.0, 0.1, 0.0, 0.3, 0.0, 0.6, 0.0, 0.3, 0.0, 0.0, 0.8, 0.1, 0.0, 0.0, 0.1, 0.0, 0.3, 0.0, 0.6, 0.0, 0.3, 0.0, 0.0],\n [0.0, 0.7, 0.1, 0.3, 0.2, 0.0, 0.0, 0.2, 0.1, 0.5, 0.0, 0.0, 0.0, 0.0, 0.7, 0.1, 0.3, 0.2, 0.0, 0.0, 0.2, 0.1, 0.5, 0.0, 0.0, 0.0, 0.0, 0.7, 0.1, 0.3, 0.2, 0.0, 0.0, 0.2, 0.1, 0.5, 0.0, 0.0, 0.0],\n [0.1, 0.0, 0.7, 0.0, 0.1, 0.4, 0.0, 0.2, 0.0, 0.1, 0.5, 0.1, 0.0, 0.1, 0.0, 0.7, 0.0, 0.1, 0.4, 0.0, 0.2, 0.0, 0.1, 0.5, 0.1, 0.0, 0.1, 0.0, 0.7, 0.0, 0.1, 0.4, 0.0, 0.2, 0.0, 0.1, 0.5, 0.1, 0.0],\n [0.0, 0.1, 0.0, 0.8, 0.8, 0.7, 0.4, 0.2, 0.0, 0.2, 0.0, 0.0, 0.0, 0.0, 0.1, 0.0, 0.8, 0.8, 0.7, 0.4, 0.2, 0.0, 0.2, 0.0, 0.0, 0.0, 0.0, 0.1, 0.0, 0.8, 0.8, 0.7, 0.4, 0.2, 0.0, 0.2, 0.0, 0.0, 0.0],\n];\nconst b1 = [0.0, 0.0, 0.0, 0.0];\n\nconst W2 = [\n [1.0, 0.2, 0.0, 0.0],\n [0.0, 1.0, 0.2, 0.0],\n [0.0, 0.0, 1.0, 0.2],\n [0.2, 0.0, 0.0, 1.0],\n];\nconst b2 = [0.0, 0.0, 0.0, 0.0];\n\nfunction relu(x) {\n return x.map((v) => Math.max(0, v));\n}\n\nfunction matvec(M, x) {\n return M.map((row) => row.reduce((acc, v, i) => acc + v * x[i], 0));\n}\n\nfunction add(a, b) {\n return a.map((v, i) => v + b[i]);\n}\n\nfunction softmax(logits) {\n const m = Math.max(...logits);\n const exps = logits.map((x) => Math.exp(x - m));\n const sum = exps.reduce((a, b) => a + b, 0);\n return exps.map((e) => e / sum);\n}\n\nfunction argmax(arr) {\n return arr.reduce((best, v, i, a) => (v > a[best] ? i : best), 0);\n}\n\nfunction tokenizePrompt(text) {\n return text\n .replace(/\\?/g, \" ?\")\n .replace(/\\./g, \" .\")\n .replace(/What's/g, \"What 's\")\n .trim()\n .split(/\\s+/)\n .filter(Boolean);\n}\n\nfunction oneHotContext(sequence) {\n const ctx = sequence.slice(-CONTEXT);\n const padded = Array(CONTEXT - ctx.length).fill(\"\").concat(ctx);\n const vec = Array(CONTEXT * TOKENS.length).fill(0);\n padded.forEach((tok, pos) => {\n const idx = TOKENS.indexOf(tok);\n if (idx >= 0) vec[pos * TOKENS.length + idx] = 1;\n });\n return { padded, vec };\n}\n\nfunction handcraftedLogits(sequence) {\n const last = sequence[sequence.length - 1] || \"\";\n const scores = Object.fromEntries(TOKENS.map((t) => [t, -3]));\n\n if (last === \"?\") scores[\"The\"] = 6;\n if (last === \"The\") scores[\"capital\"] = 6;\n if (last === \"capital\") scores[\"of\"] = 6;\n if (last === \"of\") scores[\"Belgium\"] = 6;\n if (last === \"Belgium\") scores[\"is\"] = 6;\n if (last === \"is\") scores[\"Brussels\"] = 6;\n if (last === \"Brussels\") scores[\".\"] = 6;\n if (last === \".\") scores[\"\"] = 6;\n\n return TOKENS.map((t) => scores[t]);\n}\n\nfunction forward(sequence) {\n const { padded, vec } = oneHotContext(sequence);\n const z1 = add(matvec(W1, vec), b1);\n const a1 = relu(z1);\n const z2 = add(matvec(W2, a1), b2);\n const a2 = relu(z2);\n const logits = handcraftedLogits(sequence);\n const probs = softmax(logits);\n return { padded, vec, z1, a1, z2, a2, logits, probs };\n}\n\nfunction Node({ value, label, active }) {\n return (\n

\n \n {value.toFixed(1)}\n \n

{label}

\n );\n}\n\nfunction Layer({ title, values, highlights = [] }) {\n return (\n

{title}

\n {values.map((v, i) => (\n \n ))}\n

\n );\n}\n\nfunction OutputLayer({ logits, probs }) {\n const ranked = TOKENS.map((t, i) => ({ token: t, logit: logits[i], prob: probs[i] })).sort((a, b) => b.prob - a.prob);\n return (\n

Output

\n {ranked.map((r, i) => (\n

{r.token}

\n \n

{(r.prob * 100).toFixed(0)}%

\n ))}\n

\n );\n}\n\nexport default function ToyAutoregressiveNet() {\n const [prompt, setPrompt] = useState(\"What's the capital of Belgium?\");\n const [sequence, setSequence] = useState([\"\", \"What\", \"'s\", \"the\", \"capital\", \"of\", \"Belgium\", \"?\"]);\n const [playing, setPlaying] = useState(false);\n const [speed, setSpeed] = useState([1.1]);\n\n function buildPromptSequence() {\n const rawTokens = tokenizePrompt(prompt);\n const cleanTokens = rawTokens.filter((t) => TOKENS.includes(t));\n return [\"\", ...cleanTokens];\n }\n\n useEffect(() => {\n setSequence(buildPromptSequence());\n setPlaying(false);\n }, [prompt]);\n\n const state = useMemo(() => forward(sequence), [sequence]);\n const nextIdx = argmax(state.probs);\n const nextToken = TOKENS[nextIdx];\n\n function step() {\n const current = sequence[sequence.length - 1];\n if (current === \"\") return;\n setSequence((prev) => [...prev, nextToken]);\n }\n\n function reset() {\n setSequence(buildPromptSequence());\n setPlaying(false);\n }\n\n useEffect(() => {\n if (!playing) return;\n if (sequence[sequence.length - 1] === \"\") return;\n const ms = 1400 - speed[0] * 900;\n const id = setTimeout(step, ms);\n return () => clearTimeout(id);\n }, [playing, sequence, speed, nextToken]);\n\n const topHidden1 = state.a1\n .map((v, i) => ({ v, i }))\n .sort((a, b) => b.v - a.v)\n .slice(0, 2)\n .map((x) => x.i);\n const topHidden2 = state.a2\n .map((v, i) => ({ v, i }))\n .sort((a, b) => b.v - a.v)\n .slice(0, 2)\n .map((x) => x.i);\n\n return (\n

\n \n \n

Autoregressive toy net

The user provides a prefix, the toy model consumes it as context, then generates the continuation token by token.

\n \n \n \n

\n\n

User prompt

\n setPrompt(e.target.value)}\n className=\"rounded-2xl bg-white\"\n placeholder=\"What's the capital of Belgium?\"\n />\n

\n Animation speed\n {speed[0].toFixed(1)}\n

\n \n

\n \n \n\n \n \n

\n \n

→

\n \n

→

\n \n

→

\n \n

\n \n \n\n \n \n

Generated continuation so far

\n {sequence.slice(buildPromptSequence().length).filter((token) => token !== \"\").join(\" \") || \"…\"}\n

Each new token is appended to the continuation, so the full generated answer stays visible as it grows.

\n \n \n\n \n \n

Prompt plus generated continuation

\n {sequence.map((token, i) => (\n \n {token}\n \n ))}\n

\n \n \n

\n );\n}\n","access","public","isModerationBlocked","isAnonifyApiKeyDetected","dd",{"_38":39,"_40":41},"disablePrefetch",true,"shouldPrefetchAccount","shouldPrefetchUser","shouldPrefetchSystemHints","promoteCss","disableSSR","stripModulepreloadImports","statsigGateEvaluationsPromise",["P",37],"traceId","9623652417164432139","traceTime",1776874537928]