| 1 | <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1280 720" width="1280" height="720"> |
| 2 | <defs> |
| 3 | <filter id="cardShadow" x="-10%" y="-10%" width="120%" height="120%"> |
| 4 | <feGaussianBlur in="SourceAlpha" stdDeviation="4"/> |
| 5 | <feOffset dx="0" dy="3"/> |
| 6 | <feFlood flood-color="#1A365D" flood-opacity="0.06"/> |
| 7 | <feComposite in2="SourceAlpha" operator="in"/> |
| 8 | <feMerge><feMergeNode/><feMergeNode in="SourceGraphic"/></feMerge> |
| 9 | </filter> |
| 10 | <clipPath id="rnnImgClip"> |
| 11 | <rect x="900" y="170" width="320" height="500" rx="14"/> |
| 12 | </clipPath> |
| 13 | </defs> |
| 14 | |
| 15 | <g id="bg"> |
| 16 | <rect width="1280" height="720" fill="#FFFFFF"/> |
| 17 | </g> |
| 18 | |
| 19 | <g id="header"> |
| 20 | <rect x="60" y="56" width="6" height="40" fill="#3182CE"/> |
| 21 | <text x="84" y="88" font-family="Georgia, "Microsoft YaHei", serif" font-size="36" font-weight="700" fill="#1A365D">What hurts in RNN/CNN sequence models</text> |
| 22 | <text x="84" y="118" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="13" font-weight="700" fill="#4A5568" letter-spacing="1.5">THREE FUNDAMENTAL CONSTRAINTS THE TRANSFORMER LIFTS</text> |
| 23 | </g> |
| 24 | |
| 25 | <g id="pillar-sequential"> |
| 26 | <rect x="60" y="170" width="265" height="500" rx="12" fill="#FFFFFF" stroke="#E2E8F0" stroke-width="1.5" filter="url(#cardShadow)"/> |
| 27 | <path d="M 72 170 h 241 a 12 12 0 0 1 12 12 v 96 h -265 v -96 a 12 12 0 0 1 12 -12 Z" fill="#1A365D" fill-opacity="0.06"/> |
| 28 | <text x="192" y="218" text-anchor="middle" font-family="Georgia, "Microsoft YaHei", serif" font-size="44" font-weight="700" fill="#1A365D">01</text> |
| 29 | <text x="192" y="252" text-anchor="middle" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="15" font-weight="800" fill="#1A365D" letter-spacing="1.5">SEQUENTIAL BOTTLENECK</text> |
| 30 | <line x1="80" y1="296" x2="305" y2="296" stroke="#E2E8F0" stroke-width="1.5"/> |
| 31 | <use data-icon="tabler-outline/hourglass" x="80" y="316" width="36" height="36" fill="#3182CE"/> |
| 32 | <text x="80" y="384" font-family="Georgia, "Microsoft YaHei", serif" font-size="18" font-weight="700" fill="#1A365D">hₜ depends on hₜ₋₁</text> |
| 33 | <text x="80" y="424" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="15" fill="#1A202C">Each hidden state must wait for</text> |
| 34 | <text x="80" y="446" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="15" fill="#1A202C">the previous one — no within-</text> |
| 35 | <text x="80" y="468" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="15" fill="#1A202C">example parallelism.</text> |
| 36 | <text x="80" y="514" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="14" font-weight="700" fill="#3182CE" letter-spacing="1">CONSEQUENCE</text> |
| 37 | <text x="80" y="542" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="15" fill="#4A5568">Training step time scales</text> |
| 38 | <text x="80" y="564" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="15" fill="#4A5568">linearly with sequence</text> |
| 39 | <text x="80" y="586" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="15" fill="#4A5568">length n.</text> |
| 40 | </g> |
| 41 | |
| 42 | <g id="pillar-pathlen"> |
| 43 | <rect x="345" y="170" width="265" height="500" rx="12" fill="#FFFFFF" stroke="#E2E8F0" stroke-width="1.5" filter="url(#cardShadow)"/> |
| 44 | <path d="M 357 170 h 241 a 12 12 0 0 1 12 12 v 96 h -265 v -96 a 12 12 0 0 1 12 -12 Z" fill="#3182CE" fill-opacity="0.08"/> |
| 45 | <text x="477" y="218" text-anchor="middle" font-family="Georgia, "Microsoft YaHei", serif" font-size="44" font-weight="700" fill="#3182CE">02</text> |
| 46 | <text x="477" y="252" text-anchor="middle" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="15" font-weight="800" fill="#1A365D" letter-spacing="1.5">LONG-RANGE PATH</text> |
| 47 | <line x1="365" y1="296" x2="590" y2="296" stroke="#E2E8F0" stroke-width="1.5"/> |
| 48 | <use data-icon="tabler-outline/arrows-horizontal" x="365" y="316" width="36" height="36" fill="#3182CE"/> |
| 49 | <text x="365" y="384" font-family="Georgia, "Microsoft YaHei", serif" font-size="18" font-weight="700" fill="#1A365D">Distant tokens, far apart</text> |
| 50 | <text x="365" y="424" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="15" fill="#1A202C">ConvS2S path grows linearly,</text> |
| 51 | <text x="365" y="446" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="15" fill="#1A202C">ByteNet logarithmically.</text> |
| 52 | <text x="365" y="468" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="15" fill="#1A202C">RNNs are even worse.</text> |
| 53 | <text x="365" y="514" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="14" font-weight="700" fill="#3182CE" letter-spacing="1">CONSEQUENCE</text> |
| 54 | <text x="365" y="542" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="15" fill="#4A5568">Hard to learn dependencies</text> |
| 55 | <text x="365" y="564" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="15" fill="#4A5568">between positions that sit</text> |
| 56 | <text x="365" y="586" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="15" fill="#4A5568">far apart in the input.</text> |
| 57 | </g> |
| 58 | |
| 59 | <g id="pillar-memory"> |
| 60 | <rect x="630" y="170" width="255" height="500" rx="12" fill="#FFFFFF" stroke="#E2E8F0" stroke-width="1.5" filter="url(#cardShadow)"/> |
| 61 | <path d="M 642 170 h 231 a 12 12 0 0 1 12 12 v 96 h -255 v -96 a 12 12 0 0 1 12 -12 Z" fill="#1A365D" fill-opacity="0.06"/> |
| 62 | <text x="757" y="218" text-anchor="middle" font-family="Georgia, "Microsoft YaHei", serif" font-size="44" font-weight="700" fill="#1A365D">03</text> |
| 63 | <text x="757" y="252" text-anchor="middle" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="15" font-weight="800" fill="#1A365D" letter-spacing="1.5">MEMORY CEILING</text> |
| 64 | <line x1="650" y1="296" x2="865" y2="296" stroke="#E2E8F0" stroke-width="1.5"/> |
| 65 | <use data-icon="tabler-outline/lock" x="650" y="316" width="36" height="36" fill="#3182CE"/> |
| 66 | <text x="650" y="384" font-family="Georgia, "Microsoft YaHei", serif" font-size="18" font-weight="700" fill="#1A365D">Batch size vs length</text> |
| 67 | <text x="650" y="424" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="15" fill="#1A202C">Long sequences exhaust</text> |
| 68 | <text x="650" y="446" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="15" fill="#1A202C">GPU memory, capping</text> |
| 69 | <text x="650" y="468" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="15" fill="#1A202C">batch size across examples.</text> |
| 70 | <text x="650" y="514" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="14" font-weight="700" fill="#3182CE" letter-spacing="1">CONSEQUENCE</text> |
| 71 | <text x="650" y="542" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="15" fill="#4A5568">Throughput drops fast as</text> |
| 72 | <text x="650" y="564" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="15" fill="#4A5568">context grows — a hard wall</text> |
| 73 | <text x="650" y="586" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="15" fill="#4A5568">at long sequence lengths.</text> |
| 74 | </g> |
| 75 | |
| 76 | <g id="rnn-visual"> |
| 77 | <image href="../images/rnn_bottleneck.png" x="900" y="170" width="320" height="500" preserveAspectRatio="xMidYMid slice" clip-path="url(#rnnImgClip)"/> |
| 78 | <text x="1060" y="624" text-anchor="middle" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="12" font-style="italic" fill="#4A5568">Sequential hidden-state chain</text> |
| 79 | <text x="1060" y="644" text-anchor="middle" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="12" font-style="italic" fill="#4A5568">— the constraint visualized</text> |
| 80 | </g> |
| 81 | |
| 82 | <g id="footer"> |
| 83 | <text x="60" y="694" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="11" fill="#A0AEC0">Source: Vaswani et al. 2017, §1 Introduction & §2 Background</text> |
| 84 | <text x="1220" y="694" text-anchor="end" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="11" fill="#A0AEC0">04</text> |
| 85 | </g> |
| 86 | </svg> |
| 87 |