| 1 | <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1280 720" width="1280" height="720"> |
| 2 | <defs> |
| 3 | <filter id="cardShadow" x="-10%" y="-10%" width="120%" height="120%"> |
| 4 | <feGaussianBlur in="SourceAlpha" stdDeviation="6" /> |
| 5 | <feOffset dx="0" dy="3" /> |
| 6 | <feFlood flood-color="#1A365D" flood-opacity="0.07" /> |
| 7 | <feComposite in2="SourceAlpha" operator="in" /> |
| 8 | <feMerge><feMergeNode /><feMergeNode in="SourceGraphic" /></feMerge> |
| 9 | </filter> |
| 10 | </defs> |
| 11 | |
| 12 | <g id="bg"> |
| 13 | <rect width="1280" height="720" fill="#FFFFFF" /> |
| 14 | </g> |
| 15 | |
| 16 | <g id="header"> |
| 17 | <rect x="60" y="56" width="6" height="40" fill="#3182CE" /> |
| 18 | <text x="84" y="88" font-family="Georgia, "Microsoft YaHei", serif" font-size="36" font-weight="700" fill="#1A365D">One mechanism, three roles in the Transformer</text> |
| 19 | <text x="84" y="118" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="13" font-weight="700" fill="#4A5568" letter-spacing="1.5">WHERE ATTENTION SHOWS UP IN THE ARCHITECTURE · §3.2.3</text> |
| 20 | </g> |
| 21 | |
| 22 | <g id="pillar-enc-self"> |
| 23 | <path fill="#FFFFFF" stroke="#E2E8F0" stroke-width="1.5" filter="url(#cardShadow)" d="M74,160 H426 A14,14 0 0 1 440,174 V646 A14,14 0 0 1 426,660 H74 A14,14 0 0 1 60,646 V174 A14,14 0 0 1 74,160 Z" /> |
| 24 | <path d="M 74 160 h 352 a 14 14 0 0 1 14 14 v 100 h -380 v -100 a 14 14 0 0 1 14 -14 Z" fill="#1A365D" /> |
| 25 | |
| 26 | <g transform="translate(80.0, 180.0) scale(1.3333333333333333)" fill="none" stroke="#FFFFFF" stroke-width="2"> |
| 27 | <path d="M9 11l-4 4l4 4m-4 -4h11a4 4 0 0 0 0 -8h-1" /> |
| 28 | </g> |
| 29 | <text x="250" y="208" text-anchor="middle" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="14" font-weight="700" fill="#FFFFFF" fill-opacity="0.85" letter-spacing="2">ENCODER</text> |
| 30 | <text x="250" y="240" text-anchor="middle" font-family="Georgia, "Microsoft YaHei", serif" font-size="22" font-weight="700" fill="#FFFFFF">Self-Attention</text> |
| 31 | |
| 32 | <text x="80" y="312" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="12" font-weight="700" fill="#3182CE" letter-spacing="1.5">Q · K · V SOURCE</text> |
| 33 | <path fill="#F5F7FA" d="M86,324 H414 A6,6 0 0 1 420,330 V366 A6,6 0 0 1 414,372 H86 A6,6 0 0 1 80,366 V330 A6,6 0 0 1 86,324 Z" /> |
| 34 | <text x="98" y="354" font-family="Consolas, "Courier New", monospace" font-size="14" fill="#1A202C">Q, K, V ← previous encoder layer</text> |
| 35 | |
| 36 | <text x="80" y="406" font-family="Georgia, "Microsoft YaHei", serif" font-size="18" font-weight="700" fill="#1A365D">Every position attends to all</text> |
| 37 | <text x="80" y="442" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="15" fill="#1A202C">Every token in the source sequence can</text> |
| 38 | <text x="80" y="464" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="15" fill="#1A202C">attend to every other token — including</text> |
| 39 | <text x="80" y="486" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="15" fill="#1A202C">itself.</text> |
| 40 | |
| 41 | <text x="80" y="534" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="12" font-weight="700" fill="#3182CE" letter-spacing="1.5">WHAT IT'S FOR</text> |
| 42 | <text x="80" y="560" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="15" fill="#4A5568">Build rich contextualized</text> |
| 43 | <text x="80" y="582" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="15" fill="#4A5568">representations of each source</text> |
| 44 | <text x="80" y="604" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="15" fill="#4A5568">token from its surroundings.</text> |
| 45 | |
| 46 | <path fill="#1A365D" fill-opacity="0.08" d="M84,624 H416 A4,4 0 0 1 420,628 V640 A4,4 0 0 1 416,644 H84 A4,4 0 0 1 80,640 V628 A4,4 0 0 1 84,624 Z" /> |
| 47 | <text x="92" y="640" font-family="Consolas, "Courier New", monospace" font-size="12" fill="#1A365D">attention_mask = none</text> |
| 48 | </g> |
| 49 | |
| 50 | <g id="pillar-dec-masked"> |
| 51 | <path fill="#FFFFFF" stroke="#E2E8F0" stroke-width="1.5" filter="url(#cardShadow)" d="M464,160 H816 A14,14 0 0 1 830,174 V646 A14,14 0 0 1 816,660 H464 A14,14 0 0 1 450,646 V174 A14,14 0 0 1 464,160 Z" /> |
| 52 | <path d="M 464 160 h 352 a 14 14 0 0 1 14 14 v 100 h -380 v -100 a 14 14 0 0 1 14 -14 Z" fill="#3182CE" /> |
| 53 | |
| 54 | <g transform="translate(470.0, 180.0) scale(1.3333333333333333)" fill="none" stroke="#FFFFFF" stroke-width="2"> |
| 55 | <path d="M5 13a2 2 0 0 1 2 -2h10a2 2 0 0 1 2 2v6a2 2 0 0 1 -2 2h-10a2 2 0 0 1 -2 -2v-6" /> |
| 56 | <path d="M11 16a1 1 0 1 0 2 0a1 1 0 0 0 -2 0" /> |
| 57 | <path d="M8 11v-4a4 4 0 1 1 8 0v4" /> |
| 58 | </g> |
| 59 | <text x="640" y="208" text-anchor="middle" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="14" font-weight="700" fill="#FFFFFF" fill-opacity="0.9" letter-spacing="2">DECODER</text> |
| 60 | <text x="640" y="240" text-anchor="middle" font-family="Georgia, "Microsoft YaHei", serif" font-size="22" font-weight="700" fill="#FFFFFF">Masked Self-Attention</text> |
| 61 | |
| 62 | <text x="470" y="312" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="12" font-weight="700" fill="#3182CE" letter-spacing="1.5">Q · K · V SOURCE</text> |
| 63 | <path fill="#F5F7FA" d="M476,324 H804 A6,6 0 0 1 810,330 V366 A6,6 0 0 1 804,372 H476 A6,6 0 0 1 470,366 V330 A6,6 0 0 1 476,324 Z" /> |
| 64 | <text x="488" y="354" font-family="Consolas, "Courier New", monospace" font-size="14" fill="#1A202C">Q, K, V ← previous decoder layer</text> |
| 65 | |
| 66 | <text x="470" y="406" font-family="Georgia, "Microsoft YaHei", serif" font-size="18" font-weight="700" fill="#1A365D">Only past, never future</text> |
| 67 | <text x="470" y="442" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="15" fill="#1A202C">Position i attends only to positions ≤ i.</text> |
| 68 | <text x="470" y="464" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="15" fill="#1A202C">Achieved by setting illegal positions to</text> |
| 69 | <text x="470" y="486" font-family="Consolas, "Courier New", monospace" font-size="14" fill="#1A202C">−∞ before the softmax.</text> |
| 70 | |
| 71 | <text x="470" y="534" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="12" font-weight="700" fill="#3182CE" letter-spacing="1.5">WHAT IT'S FOR</text> |
| 72 | <text x="470" y="560" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="15" fill="#4A5568">Preserve auto-regression — the</text> |
| 73 | <text x="470" y="582" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="15" fill="#4A5568">model cannot peek at tokens it</text> |
| 74 | <text x="470" y="604" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="15" fill="#4A5568">hasn't produced yet.</text> |
| 75 | |
| 76 | <path fill="#3182CE" fill-opacity="0.1" d="M474,624 H806 A4,4 0 0 1 810,628 V640 A4,4 0 0 1 806,644 H474 A4,4 0 0 1 470,640 V628 A4,4 0 0 1 474,624 Z" /> |
| 77 | <text x="482" y="640" font-family="Consolas, "Courier New", monospace" font-size="12" fill="#3182CE">attention_mask = causal triangular</text> |
| 78 | </g> |
| 79 | |
| 80 | <g id="pillar-cross"> |
| 81 | <path fill="#FFFFFF" stroke="#E2E8F0" stroke-width="1.5" filter="url(#cardShadow)" d="M854,160 H1206 A14,14 0 0 1 1220,174 V646 A14,14 0 0 1 1206,660 H854 A14,14 0 0 1 840,646 V174 A14,14 0 0 1 854,160 Z" /> |
| 82 | <path d="M 854 160 h 352 a 14 14 0 0 1 14 14 v 100 h -380 v -100 a 14 14 0 0 1 14 -14 Z" fill="#1A365D" /> |
| 83 | |
| 84 | <g transform="translate(860.0, 180.0) scale(1.3333333333333333)" fill="none" stroke="#FFFFFF" stroke-width="2"> |
| 85 | <path d="M21 17l-18 0" /> |
| 86 | <path d="M6 10l-3 -3l3 -3" /> |
| 87 | <path d="M3 7l18 0" /> |
| 88 | <path d="M18 20l3 -3l-3 -3" /> |
| 89 | </g> |
| 90 | <text x="1030" y="208" text-anchor="middle" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="14" font-weight="700" fill="#FFFFFF" fill-opacity="0.85" letter-spacing="2">ENCODER ↔ DECODER</text> |
| 91 | <text x="1030" y="240" text-anchor="middle" font-family="Georgia, "Microsoft YaHei", serif" font-size="22" font-weight="700" fill="#FFFFFF">Cross-Attention</text> |
| 92 | |
| 93 | <text x="860" y="312" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="12" font-weight="700" fill="#3182CE" letter-spacing="1.5">Q · K · V SOURCE</text> |
| 94 | <path fill="#F5F7FA" d="M866,324 H1194 A6,6 0 0 1 1200,330 V366 A6,6 0 0 1 1194,372 H866 A6,6 0 0 1 860,366 V330 A6,6 0 0 1 866,324 Z" /> |
| 95 | <text x="878" y="346" font-family="Consolas, "Courier New", monospace" font-size="13" fill="#1A202C">Q ← previous decoder layer</text> |
| 96 | <text x="878" y="364" font-family="Consolas, "Courier New", monospace" font-size="13" fill="#3182CE">K, V ← encoder output</text> |
| 97 | |
| 98 | <text x="860" y="406" font-family="Georgia, "Microsoft YaHei", serif" font-size="18" font-weight="700" fill="#1A365D">Look back at the input</text> |
| 99 | <text x="860" y="442" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="15" fill="#1A202C">Every decoder position can attend over</text> |
| 100 | <text x="860" y="464" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="15" fill="#1A202C">all positions of the input sequence —</text> |
| 101 | <text x="860" y="486" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="15" fill="#1A202C">the classic seq2seq attention role.</text> |
| 102 | |
| 103 | <text x="860" y="534" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="12" font-weight="700" fill="#3182CE" letter-spacing="1.5">WHAT IT'S FOR</text> |
| 104 | <text x="860" y="560" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="15" fill="#4A5568">Let each output token retrieve</text> |
| 105 | <text x="860" y="582" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="15" fill="#4A5568">the relevant source information,</text> |
| 106 | <text x="860" y="604" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="15" fill="#4A5568">no matter where it sits.</text> |
| 107 | |
| 108 | <path fill="#1A365D" fill-opacity="0.08" d="M864,624 H1196 A4,4 0 0 1 1200,628 V640 A4,4 0 0 1 1196,644 H864 A4,4 0 0 1 860,640 V628 A4,4 0 0 1 864,624 Z" /> |
| 109 | <text x="872" y="640" font-family="Consolas, "Courier New", monospace" font-size="12" fill="#1A365D">attention_mask = none</text> |
| 110 | </g> |
| 111 | |
| 112 | <g id="footer"> |
| 113 | <text x="60" y="694" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="11" fill="#A0AEC0">Source: Vaswani et al. 2017, §3.2.3 Applications of Attention</text> |
| 114 | <text x="1220" y="694" text-anchor="end" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="11" fill="#A0AEC0">09</text> |
| 115 | </g> |
| 116 | </svg> |