| 1 | <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1280 720" width="1280" height="720"> |
| 2 | <defs> |
| 3 | <linearGradient id="trackGrad" x1="0%" y1="0%" x2="0%" y2="100%"> |
| 4 | <stop offset="0%" stop-color="#1A365D" /> |
| 5 | <stop offset="100%" stop-color="#3182CE" /> |
| 6 | </linearGradient> |
| 7 | <filter id="cardShadow" x="-5%" y="-15%" width="110%" height="135%"> |
| 8 | <feGaussianBlur in="SourceAlpha" stdDeviation="4" /> |
| 9 | <feOffset dx="0" dy="2" /> |
| 10 | <feFlood flood-color="#1A365D" flood-opacity="0.06" /> |
| 11 | <feComposite in2="SourceAlpha" operator="in" /> |
| 12 | <feMerge><feMergeNode /><feMergeNode in="SourceGraphic" /></feMerge> |
| 13 | </filter> |
| 14 | </defs> |
| 15 | |
| 16 | <g id="bg"> |
| 17 | <rect width="1280" height="720" fill="#FFFFFF" /> |
| 18 | </g> |
| 19 | |
| 20 | <g id="header"> |
| 21 | <rect x="60" y="56" width="6" height="40" fill="#3182CE" /> |
| 22 | <text x="84" y="88" font-family="Georgia, "Microsoft YaHei", serif" font-size="36" font-weight="700" fill="#1A365D">Five takeaways from the variation study</text> |
| 23 | <text x="84" y="118" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="13" font-weight="700" fill="#4A5568" letter-spacing="1.5">TABLE 3 ABLATIONS · WHAT THE ARCHITECTURE NEEDS TO STAY GOOD</text> |
| 24 | </g> |
| 25 | |
| 26 | <g id="track"> |
| 27 | <line x1="160" y1="172" x2="160" y2="660" stroke="url(#trackGrad)" stroke-width="4" stroke-linecap="round" /> |
| 28 | </g> |
| 29 | |
| 30 | <g id="item-1"> |
| 31 | <circle cx="160" cy="200" r="28" fill="#1A365D" fill-opacity="0.12" /> |
| 32 | <circle cx="160" cy="200" r="20" fill="#1A365D" /> |
| 33 | <text x="160" y="206" text-anchor="middle" font-family="Georgia, "Microsoft YaHei", serif" font-size="15" font-weight="700" fill="#FFFFFF">01</text> |
| 34 | <line x1="188" y1="200" x2="220" y2="200" stroke="#1A365D" stroke-width="2" stroke-dasharray="4,3" /> |
| 35 | <path fill="#FFFFFF" stroke="#E2E8F0" stroke-width="1.5" filter="url(#cardShadow)" d="M230,170 H1210 A10,10 0 0 1 1220,180 V224 A10,10 0 0 1 1210,234 H230 A10,10 0 0 1 220,224 V180 A10,10 0 0 1 230,170 Z" /> |
| 36 | <path fill="#1A365D" d="M223,170 H223 A3,3 0 0 1 226,173 V231 A3,3 0 0 1 223,234 H223 A3,3 0 0 1 220,231 V173 A3,3 0 0 1 223,170 Z" /> |
| 37 | <text x="246" y="196" font-family="Georgia, "Microsoft YaHei", serif" font-size="18" font-weight="700" fill="#1A365D">Head count has a sweet spot</text> |
| 38 | <text x="246" y="222" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="14" fill="#1A202C">Single head loses <tspan font-family="Consolas, monospace" font-weight="700" fill="#C53030">−0.9 BLEU</tspan> vs h = 8; too many heads (h = 32) also degrade. Eight is empirically right.</text> |
| 39 | </g> |
| 40 | |
| 41 | <g id="item-2"> |
| 42 | <circle cx="160" cy="296" r="28" fill="#1A365D" fill-opacity="0.12" /> |
| 43 | <circle cx="160" cy="296" r="20" fill="#1A365D" /> |
| 44 | <text x="160" y="302" text-anchor="middle" font-family="Georgia, "Microsoft YaHei", serif" font-size="15" font-weight="700" fill="#FFFFFF">02</text> |
| 45 | <line x1="188" y1="296" x2="220" y2="296" stroke="#1A365D" stroke-width="2" stroke-dasharray="4,3" /> |
| 46 | <path fill="#FFFFFF" stroke="#E2E8F0" stroke-width="1.5" filter="url(#cardShadow)" d="M230,266 H1210 A10,10 0 0 1 1220,276 V320 A10,10 0 0 1 1210,330 H230 A10,10 0 0 1 220,320 V276 A10,10 0 0 1 230,266 Z" /> |
| 47 | <path fill="#1A365D" d="M223,266 H223 A3,3 0 0 1 226,269 V327 A3,3 0 0 1 223,330 H223 A3,3 0 0 1 220,327 V269 A3,3 0 0 1 223,266 Z" /> |
| 48 | <text x="246" y="292" font-family="Georgia, "Microsoft YaHei", serif" font-size="18" font-weight="700" fill="#1A365D">Key dim matters more than value dim</text> |
| 49 | <text x="246" y="318" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="14" fill="#1A202C">Reducing <tspan font-family="Consolas, monospace" font-weight="700" fill="#3182CE">d_k</tspan> hurts quality more than reducing <tspan font-family="Consolas, monospace" font-weight="700" fill="#3182CE">d_v</tspan> — determining compatibility is the harder learning problem.</text> |
| 50 | </g> |
| 51 | |
| 52 | <g id="item-3"> |
| 53 | <circle cx="160" cy="392" r="28" fill="#3182CE" fill-opacity="0.12" /> |
| 54 | <circle cx="160" cy="392" r="20" fill="#3182CE" /> |
| 55 | <text x="160" y="398" text-anchor="middle" font-family="Georgia, "Microsoft YaHei", serif" font-size="15" font-weight="700" fill="#FFFFFF">03</text> |
| 56 | <line x1="188" y1="392" x2="220" y2="392" stroke="#3182CE" stroke-width="2" stroke-dasharray="4,3" /> |
| 57 | <path fill="#FFFFFF" stroke="#E2E8F0" stroke-width="1.5" filter="url(#cardShadow)" d="M230,362 H1210 A10,10 0 0 1 1220,372 V416 A10,10 0 0 1 1210,426 H230 A10,10 0 0 1 220,416 V372 A10,10 0 0 1 230,362 Z" /> |
| 58 | <path fill="#3182CE" d="M223,362 H223 A3,3 0 0 1 226,365 V423 A3,3 0 0 1 223,426 H223 A3,3 0 0 1 220,423 V365 A3,3 0 0 1 223,362 Z" /> |
| 59 | <text x="246" y="388" font-family="Georgia, "Microsoft YaHei", serif" font-size="18" font-weight="700" fill="#1A365D">Bigger is better</text> |
| 60 | <text x="246" y="414" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="14" fill="#1A202C">Larger <tspan font-family="Consolas, monospace" font-weight="700" fill="#3182CE">d_model</tspan> and deeper stacks consistently improve BLEU — the architecture scales monotonically with capacity.</text> |
| 61 | </g> |
| 62 | |
| 63 | <g id="item-4"> |
| 64 | <circle cx="160" cy="488" r="28" fill="#3182CE" fill-opacity="0.12" /> |
| 65 | <circle cx="160" cy="488" r="20" fill="#3182CE" /> |
| 66 | <text x="160" y="494" text-anchor="middle" font-family="Georgia, "Microsoft YaHei", serif" font-size="15" font-weight="700" fill="#FFFFFF">04</text> |
| 67 | <line x1="188" y1="488" x2="220" y2="488" stroke="#3182CE" stroke-width="2" stroke-dasharray="4,3" /> |
| 68 | <path fill="#FFFFFF" stroke="#E2E8F0" stroke-width="1.5" filter="url(#cardShadow)" d="M230,458 H1210 A10,10 0 0 1 1220,468 V512 A10,10 0 0 1 1210,522 H230 A10,10 0 0 1 220,512 V468 A10,10 0 0 1 230,458 Z" /> |
| 69 | <path fill="#3182CE" d="M223,458 H223 A3,3 0 0 1 226,461 V519 A3,3 0 0 1 223,522 H223 A3,3 0 0 1 220,519 V461 A3,3 0 0 1 223,458 Z" /> |
| 70 | <text x="246" y="484" font-family="Georgia, "Microsoft YaHei", serif" font-size="18" font-weight="700" fill="#1A365D">Dropout is essential</text> |
| 71 | <text x="246" y="510" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="14" fill="#1A202C">Removing dropout loses <tspan font-family="Consolas, monospace" font-weight="700" fill="#C53030">~0.5–1.0 BLEU</tspan>. Even p = 0.1 measurably regularizes — it isn't decorative.</text> |
| 72 | </g> |
| 73 | |
| 74 | <g id="item-5"> |
| 75 | <circle cx="160" cy="584" r="28" fill="#63B3ED" fill-opacity="0.18" /> |
| 76 | <circle cx="160" cy="584" r="20" fill="#3182CE" /> |
| 77 | <text x="160" y="590" text-anchor="middle" font-family="Georgia, "Microsoft YaHei", serif" font-size="15" font-weight="700" fill="#FFFFFF">05</text> |
| 78 | <line x1="188" y1="584" x2="220" y2="584" stroke="#3182CE" stroke-width="2" stroke-dasharray="4,3" /> |
| 79 | <path fill="#FFFFFF" stroke="#E2E8F0" stroke-width="1.5" filter="url(#cardShadow)" d="M230,554 H1210 A10,10 0 0 1 1220,564 V608 A10,10 0 0 1 1210,618 H230 A10,10 0 0 1 220,608 V564 A10,10 0 0 1 230,554 Z" /> |
| 80 | <path fill="#63B3ED" d="M223,554 H223 A3,3 0 0 1 226,557 V615 A3,3 0 0 1 223,618 H223 A3,3 0 0 1 220,615 V557 A3,3 0 0 1 223,554 Z" /> |
| 81 | <text x="246" y="580" font-family="Georgia, "Microsoft YaHei", serif" font-size="18" font-weight="700" fill="#1A365D">Label smoothing helps BLEU (and hurts perplexity)</text> |
| 82 | <text x="246" y="606" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="14" fill="#1A202C">Slightly worse perplexity, better BLEU. <tspan font-style="italic" fill="#4A5568">Optimize for the metric you ship on.</tspan></text> |
| 83 | </g> |
| 84 | |
| 85 | <g id="footer"> |
| 86 | <text x="60" y="694" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="11" fill="#A0AEC0">Source: Vaswani et al. 2017, Table 3 (§6.2 Model Variations)</text> |
| 87 | <text x="1220" y="694" text-anchor="end" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="11" fill="#A0AEC0">15</text> |
| 88 | </g> |
| 89 | </svg> |