| 1 | <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1280 720" width="1280" height="720"> |
| 2 | <defs> |
| 3 | <filter id="cardShadow" x="-10%" y="-10%" width="120%" height="120%"> |
| 4 | <feGaussianBlur in="SourceAlpha" stdDeviation="6"/> |
| 5 | <feOffset dx="0" dy="3"/> |
| 6 | <feFlood flood-color="#1A365D" flood-opacity="0.07"/> |
| 7 | <feComposite in2="SourceAlpha" operator="in"/> |
| 8 | <feMerge><feMergeNode/><feMergeNode in="SourceGraphic"/></feMerge> |
| 9 | </filter> |
| 10 | </defs> |
| 11 | |
| 12 | <g id="bg"> |
| 13 | <rect width="1280" height="720" fill="#FFFFFF"/> |
| 14 | </g> |
| 15 | |
| 16 | <g id="header"> |
| 17 | <rect x="60" y="56" width="6" height="40" fill="#3182CE"/> |
| 18 | <text x="84" y="88" font-family="Georgia, "Microsoft YaHei", serif" font-size="36" font-weight="700" fill="#1A365D">One mechanism, three roles in the Transformer</text> |
| 19 | <text x="84" y="118" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="13" font-weight="700" fill="#4A5568" letter-spacing="1.5">WHERE ATTENTION SHOWS UP IN THE ARCHITECTURE · §3.2.3</text> |
| 20 | </g> |
| 21 | |
| 22 | <g id="pillar-enc-self"> |
| 23 | <rect x="60" y="160" width="380" height="500" rx="14" fill="#FFFFFF" stroke="#E2E8F0" stroke-width="1.5" filter="url(#cardShadow)"/> |
| 24 | <path d="M 74 160 h 352 a 14 14 0 0 1 14 14 v 100 h -380 v -100 a 14 14 0 0 1 14 -14 Z" fill="#1A365D"/> |
| 25 | <use data-icon="tabler-outline/arrow-back" x="80" y="180" width="32" height="32" fill="#FFFFFF"/> |
| 26 | <text x="250" y="208" text-anchor="middle" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="14" font-weight="700" fill="#FFFFFF" fill-opacity="0.85" letter-spacing="2">ENCODER</text> |
| 27 | <text x="250" y="240" text-anchor="middle" font-family="Georgia, "Microsoft YaHei", serif" font-size="22" font-weight="700" fill="#FFFFFF">Self-Attention</text> |
| 28 | |
| 29 | <text x="80" y="312" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="12" font-weight="700" fill="#3182CE" letter-spacing="1.5">Q · K · V SOURCE</text> |
| 30 | <rect x="80" y="324" width="340" height="48" rx="6" fill="#F5F7FA"/> |
| 31 | <text x="98" y="354" font-family="Consolas, "Courier New", monospace" font-size="14" fill="#1A202C">Q, K, V ← previous encoder layer</text> |
| 32 | |
| 33 | <text x="80" y="406" font-family="Georgia, "Microsoft YaHei", serif" font-size="18" font-weight="700" fill="#1A365D">Every position attends to all</text> |
| 34 | <text x="80" y="442" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="15" fill="#1A202C">Every token in the source sequence can</text> |
| 35 | <text x="80" y="464" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="15" fill="#1A202C">attend to every other token — including</text> |
| 36 | <text x="80" y="486" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="15" fill="#1A202C">itself.</text> |
| 37 | |
| 38 | <text x="80" y="534" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="12" font-weight="700" fill="#3182CE" letter-spacing="1.5">WHAT IT'S FOR</text> |
| 39 | <text x="80" y="560" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="15" fill="#4A5568">Build rich contextualized</text> |
| 40 | <text x="80" y="582" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="15" fill="#4A5568">representations of each source</text> |
| 41 | <text x="80" y="604" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="15" fill="#4A5568">token from its surroundings.</text> |
| 42 | |
| 43 | <rect x="80" y="624" width="340" height="20" rx="4" fill="#1A365D" fill-opacity="0.08"/> |
| 44 | <text x="92" y="640" font-family="Consolas, "Courier New", monospace" font-size="12" fill="#1A365D">attention_mask = none</text> |
| 45 | </g> |
| 46 | |
| 47 | <g id="pillar-dec-masked"> |
| 48 | <rect x="450" y="160" width="380" height="500" rx="14" fill="#FFFFFF" stroke="#E2E8F0" stroke-width="1.5" filter="url(#cardShadow)"/> |
| 49 | <path d="M 464 160 h 352 a 14 14 0 0 1 14 14 v 100 h -380 v -100 a 14 14 0 0 1 14 -14 Z" fill="#3182CE"/> |
| 50 | <use data-icon="tabler-outline/lock" x="470" y="180" width="32" height="32" fill="#FFFFFF"/> |
| 51 | <text x="640" y="208" text-anchor="middle" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="14" font-weight="700" fill="#FFFFFF" fill-opacity="0.9" letter-spacing="2">DECODER</text> |
| 52 | <text x="640" y="240" text-anchor="middle" font-family="Georgia, "Microsoft YaHei", serif" font-size="22" font-weight="700" fill="#FFFFFF">Masked Self-Attention</text> |
| 53 | |
| 54 | <text x="470" y="312" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="12" font-weight="700" fill="#3182CE" letter-spacing="1.5">Q · K · V SOURCE</text> |
| 55 | <rect x="470" y="324" width="340" height="48" rx="6" fill="#F5F7FA"/> |
| 56 | <text x="488" y="354" font-family="Consolas, "Courier New", monospace" font-size="14" fill="#1A202C">Q, K, V ← previous decoder layer</text> |
| 57 | |
| 58 | <text x="470" y="406" font-family="Georgia, "Microsoft YaHei", serif" font-size="18" font-weight="700" fill="#1A365D">Only past, never future</text> |
| 59 | <text x="470" y="442" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="15" fill="#1A202C">Position i attends only to positions ≤ i.</text> |
| 60 | <text x="470" y="464" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="15" fill="#1A202C">Achieved by setting illegal positions to</text> |
| 61 | <text x="470" y="486" font-family="Consolas, "Courier New", monospace" font-size="14" fill="#1A202C">−∞ before the softmax.</text> |
| 62 | |
| 63 | <text x="470" y="534" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="12" font-weight="700" fill="#3182CE" letter-spacing="1.5">WHAT IT'S FOR</text> |
| 64 | <text x="470" y="560" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="15" fill="#4A5568">Preserve auto-regression — the</text> |
| 65 | <text x="470" y="582" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="15" fill="#4A5568">model cannot peek at tokens it</text> |
| 66 | <text x="470" y="604" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="15" fill="#4A5568">hasn't produced yet.</text> |
| 67 | |
| 68 | <rect x="470" y="624" width="340" height="20" rx="4" fill="#3182CE" fill-opacity="0.1"/> |
| 69 | <text x="482" y="640" font-family="Consolas, "Courier New", monospace" font-size="12" fill="#3182CE">attention_mask = causal triangular</text> |
| 70 | </g> |
| 71 | |
| 72 | <g id="pillar-cross"> |
| 73 | <rect x="840" y="160" width="380" height="500" rx="14" fill="#FFFFFF" stroke="#E2E8F0" stroke-width="1.5" filter="url(#cardShadow)"/> |
| 74 | <path d="M 854 160 h 352 a 14 14 0 0 1 14 14 v 100 h -380 v -100 a 14 14 0 0 1 14 -14 Z" fill="#1A365D"/> |
| 75 | <use data-icon="tabler-outline/arrows-left-right" x="860" y="180" width="32" height="32" fill="#FFFFFF"/> |
| 76 | <text x="1030" y="208" text-anchor="middle" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="14" font-weight="700" fill="#FFFFFF" fill-opacity="0.85" letter-spacing="2">ENCODER ↔ DECODER</text> |
| 77 | <text x="1030" y="240" text-anchor="middle" font-family="Georgia, "Microsoft YaHei", serif" font-size="22" font-weight="700" fill="#FFFFFF">Cross-Attention</text> |
| 78 | |
| 79 | <text x="860" y="312" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="12" font-weight="700" fill="#3182CE" letter-spacing="1.5">Q · K · V SOURCE</text> |
| 80 | <rect x="860" y="324" width="340" height="48" rx="6" fill="#F5F7FA"/> |
| 81 | <text x="878" y="346" font-family="Consolas, "Courier New", monospace" font-size="13" fill="#1A202C">Q ← previous decoder layer</text> |
| 82 | <text x="878" y="364" font-family="Consolas, "Courier New", monospace" font-size="13" fill="#3182CE">K, V ← encoder output</text> |
| 83 | |
| 84 | <text x="860" y="406" font-family="Georgia, "Microsoft YaHei", serif" font-size="18" font-weight="700" fill="#1A365D">Look back at the input</text> |
| 85 | <text x="860" y="442" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="15" fill="#1A202C">Every decoder position can attend over</text> |
| 86 | <text x="860" y="464" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="15" fill="#1A202C">all positions of the input sequence —</text> |
| 87 | <text x="860" y="486" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="15" fill="#1A202C">the classic seq2seq attention role.</text> |
| 88 | |
| 89 | <text x="860" y="534" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="12" font-weight="700" fill="#3182CE" letter-spacing="1.5">WHAT IT'S FOR</text> |
| 90 | <text x="860" y="560" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="15" fill="#4A5568">Let each output token retrieve</text> |
| 91 | <text x="860" y="582" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="15" fill="#4A5568">the relevant source information,</text> |
| 92 | <text x="860" y="604" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="15" fill="#4A5568">no matter where it sits.</text> |
| 93 | |
| 94 | <rect x="860" y="624" width="340" height="20" rx="4" fill="#1A365D" fill-opacity="0.08"/> |
| 95 | <text x="872" y="640" font-family="Consolas, "Courier New", monospace" font-size="12" fill="#1A365D">attention_mask = none</text> |
| 96 | </g> |
| 97 | |
| 98 | <g id="footer"> |
| 99 | <text x="60" y="694" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="11" fill="#A0AEC0">Source: Vaswani et al. 2017, §3.2.3 Applications of Attention</text> |
| 100 | <text x="1220" y="694" text-anchor="end" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="11" fill="#A0AEC0">09</text> |
| 101 | </g> |
| 102 | </svg> |
| 103 |