| 1 | """Quantized transformer building blocks used by Echo 1.5 inference.""" |
| 2 | |
| 3 | from __future__ import annotations |
| 4 | |
| 5 | from dataclasses import dataclass |
| 6 | from pathlib import Path |
| 7 | |
| 8 | import safetensors |
| 9 | import torch |
| 10 | from torch import nn |
| 11 | |
| 12 | from ltx_core.loader.module_ops import ModuleOps |
| 13 | from ltx_core.model.transformer import LTXModel |
| 14 | from ltx_core.quantization.policy import QuantizationPolicy |
| 15 | |
| 16 | |
| 17 | _CHECKPOINT_PREFIX = "model.diffusion_model." |
| 18 | |
| 19 | |
| 20 | @dataclass(frozen=True) |
| 21 | class FP8CheckpointInfo: |
| 22 | """Transformer layers carrying direct E4M3 weights and per-tensor scales.""" |
| 23 | |
| 24 | checkpoint: str |
| 25 | module_names: tuple[str, ...] |
| 26 | |
| 27 | @property |
| 28 | def module_count(self) -> int: |
| 29 | return len(self.module_names) |
| 30 | |
| 31 | |
| 32 | def inspect_prequant_fp8_checkpoint(checkpoint_path: str | Path) -> FP8CheckpointInfo: |
| 33 | """Read only the safetensors header and discover pre-quantized Linear layers.""" |
| 34 | |
| 35 | path = Path(checkpoint_path).expanduser().resolve() |
| 36 | if not path.is_file(): |
| 37 | raise FileNotFoundError(f"FP8 checkpoint not found: {path}") |
| 38 | if path.suffix != ".safetensors": |
| 39 | raise ValueError("direct FP8 checkpoints must use the safetensors format") |
| 40 | |
| 41 | with safetensors.safe_open(str(path), framework="pt", device="cpu") as handle: |
| 42 | module_names = tuple( |
| 43 | sorted( |
| 44 | key.removeprefix(_CHECKPOINT_PREFIX).removesuffix(".weight_scale") |
| 45 | for key in handle.keys() |
| 46 | if key.startswith(_CHECKPOINT_PREFIX) and key.endswith(".weight_scale") |
| 47 | ) |
| 48 | ) |
| 49 | if not module_names: |
| 50 | raise ValueError(f"no FP8 weight_scale tensors found in {path}") |
| 51 | return FP8CheckpointInfo(checkpoint=str(path), module_names=module_names) |
| 52 | |
| 53 | |
| 54 | class PrequantFP8ScaledMMLinear(nn.Module): |
| 55 | """Linear with checkpoint-provided E4M3 weights and dynamic activation scaling.""" |
| 56 | |
| 57 | def __init__(self, linear: nn.Linear) -> None: |
| 58 | super().__init__() |
| 59 | self.in_features = linear.in_features |
| 60 | self.out_features = linear.out_features |
| 61 | self.weight = nn.Parameter( |
| 62 | torch.empty( |
| 63 | (linear.out_features, linear.in_features), |
| 64 | dtype=torch.float8_e4m3fn, |
| 65 | device=linear.weight.device, |
| 66 | ), |
| 67 | requires_grad=False, |
| 68 | ) |
| 69 | self.weight_scale = nn.Parameter( |
| 70 | torch.empty((), dtype=torch.float32, device=linear.weight.device), |
| 71 | requires_grad=False, |
| 72 | ) |
| 73 | self.bias = ( |
| 74 | nn.Parameter( |
| 75 | torch.empty( |
| 76 | (linear.out_features,), |
| 77 | dtype=linear.bias.dtype, |
| 78 | device=linear.bias.device, |
| 79 | ), |
| 80 | requires_grad=False, |
| 81 | ) |
| 82 | if linear.bias is not None |
| 83 | else None |
| 84 | ) |
| 85 | |
| 86 | def forward(self, value: torch.Tensor) -> torch.Tensor: |
| 87 | original_shape = value.shape |
| 88 | value_2d = value.reshape(-1, value.shape[-1]) |
| 89 | fp8_info = torch.finfo(torch.float8_e4m3fn) |
| 90 | max_abs = value_2d.float().abs().amax() |
| 91 | input_scale = torch.where(max_abs > 0, max_abs / fp8_info.max, torch.ones_like(max_abs)) |
| 92 | quantized_input = torch.clamp( |
| 93 | value_2d.float() / input_scale, |
| 94 | fp8_info.min, |
| 95 | fp8_info.max, |
| 96 | ).to(torch.float8_e4m3fn) |
| 97 | output = torch._scaled_mm( |
| 98 | quantized_input, |
| 99 | self.weight.t(), |
| 100 | scale_a=input_scale, |
| 101 | scale_b=self.weight_scale, |
| 102 | out_dtype=value.dtype, |
| 103 | use_fast_accum=True, |
| 104 | ) |
| 105 | # PyTorch releases that expose the amax result return a tuple here. |
| 106 | if isinstance(output, tuple): |
| 107 | output = output[0] |
| 108 | if self.bias is not None: |
| 109 | output = output + self.bias.to(output.dtype) |
| 110 | return output.reshape(*original_shape[:-1], self.out_features) |
| 111 | |
| 112 | |
| 113 | def build_prequant_fp8_policy(info: FP8CheckpointInfo) -> QuantizationPolicy: |
| 114 | """Replace exactly the Linear modules described by an FP8 checkpoint.""" |
| 115 | |
| 116 | scale_modules = frozenset(info.module_names) |
| 117 | |
| 118 | def mutate(model: nn.Module) -> nn.Module: |
| 119 | replacements: list[tuple[nn.Module, str, nn.Linear]] = [] |
| 120 | found_linears: set[str] = set() |
| 121 | for name, module in model.named_modules(): |
| 122 | if not isinstance(module, nn.Linear): |
| 123 | continue |
| 124 | found_linears.add(name) |
| 125 | if name in scale_modules: |
| 126 | parent_name, attr_name = name.rsplit(".", 1) |
| 127 | replacements.append((model.get_submodule(parent_name), attr_name, module)) |
| 128 | |
| 129 | if len(replacements) != len(scale_modules): |
| 130 | missing = sorted(scale_modules - found_linears) |
| 131 | raise ValueError( |
| 132 | "FP8 checkpoint/model layer mismatch: " |
| 133 | f"scales={len(scale_modules)} replacements={len(replacements)} " |
| 134 | f"missing_sample={missing[:10]}" |
| 135 | ) |
| 136 | for parent, attr_name, linear in replacements: |
| 137 | setattr(parent, attr_name, PrequantFP8ScaledMMLinear(linear)) |
| 138 | return model |
| 139 | |
| 140 | return QuantizationPolicy( |
| 141 | sd_ops=None, |
| 142 | module_ops=( |
| 143 | ModuleOps( |
| 144 | name="echo15_prequant_fp8_scaled_mm", |
| 145 | matcher=lambda model: isinstance(model, LTXModel), |
| 146 | mutator=mutate, |
| 147 | ), |
| 148 | ), |
| 149 | ) |
| 150 |