[{"_1":2,"_3":-5,"_4":-5},"loaderData",{"_5":6,"_7":8},"actionData","errors","root",{"_26":27,"_28":29,"_30":12,"_31":12,"_32":12,"_33":12,"_34":12,"_35":12,"_36":37},"routes/canvas.shared.$sharedTextdocId",{"_9":10,"_11":12},"sharedTextdoc",{"_13":14,"_15":-7,"_16":17,"_18":19,"_20":21,"_22":23,"_24":12,"_25":12},"isAuthenticated",false,"sharedTextdocId","6968e4cf6a188191980853cf179bda75","versionInt","title","Batch Norm Vs Saturation Playground","type","code/react","content","import React, { useMemo, useState } from \"react\";\nimport { Card, CardContent } from \"@/components/ui/card\";\nimport { Button } from \"@/components/ui/button\";\nimport { Slider } from \"@/components/ui/slider\";\nimport { Switch } from \"@/components/ui/switch\";\nimport { Label } from \"@/components/ui/label\";\nimport {\n LineChart,\n Line,\n XAxis,\n YAxis,\n Tooltip,\n CartesianGrid,\n ResponsiveContainer,\n Legend,\n} from \"recharts\";\n\n// --- Chart colors (distinct + readable) ---\nconst COLORS = {\n activation: \"#3b82f6\", // blue\n dAct: \"#22c55e\", // green\n dLdz: \"#f59e0b\", // amber\n count: \"#a855f7\", // purple\n meanGrad: \"#ef4444\", // red\n} as const;\n\n// --- Math helpers ---\nconst clamp = (x: number, lo: number, hi: number) => Math.max(lo, Math.min(hi, x));\nconst sigmoid = (x: number) => 1 / (1 + Math.exp(-x));\nconst dsigmoid = (x: number) => {\n const s = sigmoid(x);\n return s * (1 - s);\n};\nconst tanh = (x: number) => Math.tanh(x);\nconst dtanh = (x: number) => {\n const t = tanh(x);\n return 1 - t * t;\n};\n\n// Box-Muller\nfunction randn(seed: number) {\n // simple LCG for reproducibility\n let s = seed >>> 0;\n const lcg = () => (s = (1664525 * s + 1013904223) >>> 0);\n const u1 = () => (lcg() + 1) / 4294967296;\n const u2 = () => (lcg() + 1) / 4294967296;\n return () => {\n const r = Math.sqrt(-2 * Math.log(u1()));\n const theta = 2 * Math.PI * u2();\n return r * Math.cos(theta);\n };\n}\n\nfunction mean(xs: number[]) {\n return xs.reduce((a, b) => a + b, 0) / xs.length;\n}\nfunction variance(xs: number[], m: number) {\n const v = xs.reduce((a, x) => a + (x - m) * (x - m), 0) / xs.length;\n return v;\n}\n\ntype Act = \"sigmoid\" | \"tanh\";\n\nfunction actFn(act: Act) {\n return act === \"sigmoid\" ? sigmoid : tanh;\n}\nfunction dActFn(act: Act) {\n return act === \"sigmoid\" ? dsigmoid : dtanh;\n}\n\nfunction fmt(x: number) {\n if (!isFinite(x)) return \"∞\";\n const ax = Math.abs(x);\n if (ax >= 1000 || (ax > 0 && ax < 0.001)) return x.toExponential(2);\n return x.toFixed(4);\n}\n\nexport default function BatchNormSaturationPlayground() {\n // Controls\n const [act, setAct] = useState(\"sigmoid\");\n const [batchN, setBatchN] = useState(64);\n const [w, setW] = useState(6); // weight scale driving saturation\n const [b, setB] = useState(0);\n const [inputStd, setInputStd] = useState(1);\n const [useBN, setUseBN] = useState(true);\n const [gamma, setGamma] = useState(1);\n const [beta, setBeta] = useState(0);\n const [eps, setEps] = useState(1e-5);\n const [seed, setSeed] = useState(123);\n\n // We model a single neuron: z = w*x + b, y = act(z)\n // Loss: L = (y - t)^2 with fixed target t (per sample).\n // Gradient magnitude through saturation is primarily controlled by dy/dz = act'(z).\n\n const target = act === \"sigmoid\" ? 0.0 : 0.0; // same target; center makes gradients informative\n\n const {\n batch,\n stats,\n curve,\n histogram,\n } = useMemo(() => {\n const r = randn(seed);\n const N = Math.max(4, Math.round(batchN));\n\n // Sample inputs\n const xs = Array.from({ length: N }, () => r() * inputStd);\n\n // Pre-activations\n const zs = xs.map((x) => w * x + b);\n\n // Optional BatchNorm over z (like BN on pre-activation)\n let zNorm = zs.slice();\n let mu = mean(zs);\n let varz = variance(zs, mu);\n let std = Math.sqrt(varz + eps);\n if (useBN) {\n zNorm = zs.map((z) => (z - mu) / std);\n zNorm = zNorm.map((zn) => gamma * zn + beta);\n } else {\n // for display only\n std = Math.sqrt(varz + eps);\n }\n\n const f = actFn(act);\n const df = dActFn(act);\n\n const ys = zNorm.map((z) => f(z));\n\n // Loss per-sample and derivatives\n // L = (y - t)^2\n // dL/dy = 2(y - t)\n // dL/dz = dL/dy * dy/dz\n const dLdy = ys.map((y) => 2 * (y - target));\n const dydz = zNorm.map((z) => df(z));\n const dLdz = dLdy.map((g, i) => g * dydz[i]);\n\n // Summaries\n const satThreshold = act === \"sigmoid\" ? 0.01 : 0.01; // derivative below this = \"saturated\"\n const satFrac = dydz.filter((g) => g < satThreshold).length / N;\n\n // Histogram bins for z and dydz\n const bins = 20;\n const zMin = Math.min(...zNorm);\n const zMax = Math.max(...zNorm);\n const pad = (zMax - zMin) * 0.05 + 1e-9;\n const lo = zMin - pad;\n const hi = zMax + pad;\n const width = (hi - lo) / bins;\n\n const hist = Array.from({ length: bins }, (_, i) => ({\n bin: i,\n zMid: lo + (i + 0.5) * width,\n count: 0,\n meanGrad: 0,\n }));\n\n for (let i = 0; i < N; i++) {\n const idx = clamp(Math.floor((zNorm[i] - lo) / width), 0, bins - 1);\n hist[idx].count += 1;\n hist[idx].meanGrad += Math.abs(dLdz[i]);\n }\n for (const h of hist) {\n h.meanGrad = h.count > 0 ? h.meanGrad / h.count : 0;\n }\n\n // Curve over z for derivative shape\n const curvePts = 201;\n const zCurve = Array.from({ length: curvePts }, (_, i) => {\n const z = -10 + (20 * i) / (curvePts - 1);\n const y = f(z);\n const dy = df(z);\n // show \"local\" gradient factor magnitude for L=(y-t)^2 at y(z)\n const g = Math.abs(2 * (y - target) * dy);\n return {\n z,\n activation: y,\n dAct: dy,\n dLdz: g,\n };\n });\n\n const batchRows = xs.map((x, i) => ({\n i,\n x,\n zRaw: zs[i],\n z: zNorm[i],\n y: ys[i],\n dAct: dydz[i],\n dLdz: dLdz[i],\n }));\n\n const dActMean = mean(dydz);\n const dActMedian = [...dydz].sort((a, b) => a - b)[Math.floor(N / 2)];\n const absGradMean = mean(dLdz.map((g) => Math.abs(g)));\n\n return {\n batch: batchRows,\n stats: {\n N,\n zRawMean: mean(zs),\n zRawStd: Math.sqrt(variance(zs, mean(zs)) + eps),\n zUsedMean: mean(zNorm),\n zUsedStd: Math.sqrt(variance(zNorm, mean(zNorm)) + eps),\n dActMean,\n dActMedian,\n absGradMean,\n satFrac,\n satThreshold,\n },\n curve: zCurve,\n histogram: hist,\n };\n }, [act, batchN, w, b, inputStd, useBN, gamma, beta, eps, seed]);\n\n const bnLabel = useBN ? \"ON\" : \"OFF\";\n const actLabel = act === \"sigmoid\" ? \"Sigmoid\" : \"Tanh\";\n\n return (\n

\n \n \n

Saturation Playground

\n One neuron. Watch how BatchNorm keeps pre-activations near the high-gradient zone for sigmoid/tanh.\n

\n\n

\n Activation\n

\n setAct(\"sigmoid\")}\n className=\"h-8\"\n >\n Sigmoid\n \n setAct(\"tanh\")}\n className=\"h-8\"\n >\n Tanh\n \n

\n\n BatchNorm\n

\n \n {bnLabel}\n

\n\n

\n Weight scale w\n {fmt(w)}\n

\n setW(v[0])} />\n

\n Higher w pushes z = w·x + b into saturation.\n

\n\n

\n Bias b\n {fmt(b)}\n

\n setB(v[0])} />\n

\n\n

\n Input std\n {fmt(inputStd)}\n

\n setInputStd(v[0])} />\n

\n Input distribution controls spread of z.\n

\n\n

\n Batch size\n {batchN}\n

\n setBatchN(v[0])} />\n

\n Small batches = noisier BN stats (more \"regularizing\" noise).\n

\n\n

\n BN γ (scale)\n {fmt(gamma)}\n

\n setGamma(v[0])} />\n

\n BN β (shift)\n {fmt(beta)}\n

\n setBeta(v[0])} />\n

\n\n

\n Seed\n {seed}\n

\n setSeed(v[0])} />\n

\n\n

What to look for

\n Crank w or b until {actLabel} saturates. Then toggle BN.\n With BN ON, z gets centered/scaled so more samples land where the derivative is big → gradients flow.\n

\n \n \n\n

\n \n \n

Derivative shape + local gradient

\n Shows activation, d(activation)/dz, and |dL/dz| for L=(y−t)^2 (t=0).\n

\n Activation: {actLabel} · BN: {bnLabel}\n

\n\n

\n \n \n \n v.toFixed(0)} />\n \n [fmt(Number(val)), name]}\n labelFormatter={(label) => `z = ${fmt(Number(label))}`}\n />\n \n \n \n \n \n \n

\n\n

\n \n \n \n \n \n \n \n \n

\n \n \n\n \n \n

Where your batch lives

\n Histogram over z (after BN if ON). Bars show count; line shows mean |dL/dz| in each bin.\n

\n\n

\n \n \n \n Number(v).toFixed(1)}\n label={{ value: \"z\", position: \"insideBottom\", offset: -2 }}\n />\n \n \n [fmt(Number(val)), name]}\n labelFormatter={(label) => `z≈ ${fmt(Number(label))}`}\n />\n \n {/* Using LineChart for both series; counts shown as a step-line for simplicity */}\n \n \n \n \n

\n\n

\n Saturation story in one sentence: when most of your batch sits in large |z|, sigmoid/tanh derivatives shrink → backprop fades.\n BatchNorm recenters/rescales z so more samples fall near 0 where derivatives are bigger.\n

\n \n \n\n \n \n

Sample table (first 12)

x → z(raw) → z(used) → y → dAct → dL/dz

\n \n \n \n \n \n \n \n \n \n \n \n \n \n {batch.slice(0, 12).map((r) => (\n \n \n \n \n \n \n \n \n \n ))}\n \n

i	x	z(raw)	z(used)	y	dAct	dL/dz
{r.i}	{fmt(r.x)}	{fmt(r.zRaw)}	{fmt(r.z)}	{fmt(r.y)}	{fmt(r.dAct)}	{fmt(r.dLdz)}

\n \n \n

\n\n \n \n

Mini-cheat-sheet

\n Saturation: sigmoid/tanh have tiny derivatives when |z| is large.\n BatchNorm: normalize z within a batch so typical z stays near 0.\n Result: larger dAct on average → larger |dL/dz| → faster learning.\n Bonus: small batches make BN stats noisy → extra regularization-like noise.\n

\n \n \n

\n );\n}\n\nfunction Stat({ label, value }: { label: string; value: string }) {\n return (\n

{label}

{value}

\n );\n}\n","access","public","isModerationBlocked","isAnonifyApiKeyDetected","dd",{"_38":39,"_40":41},"disablePrefetch",true,"shouldPrefetchAccount","shouldPrefetchUser","shouldPrefetchSystemHints","promoteCss","disableSSR","stripModulepreloadImports","statsigGateEvaluationsPromise",["P",37],"traceId","10971248239306370843","traceTime",1780240904659]