| 1 | from dataclasses import dataclass |
| 2 | from enum import Enum |
| 3 | |
| 4 | import torch |
| 5 | from torch._prims_common import DeviceLikeType |
| 6 | |
| 7 | |
| 8 | class PerturbationType(Enum): |
| 9 | """Types of attention perturbations for STG (Spatio-Temporal Guidance).""" |
| 10 | |
| 11 | SKIP_A2V_CROSS_ATTN = "skip_a2v_cross_attn" |
| 12 | SKIP_V2A_CROSS_ATTN = "skip_v2a_cross_attn" |
| 13 | SKIP_VIDEO_SELF_ATTN = "skip_video_self_attn" |
| 14 | SKIP_AUDIO_SELF_ATTN = "skip_audio_self_attn" |
| 15 | |
| 16 | |
| 17 | @dataclass(frozen=True) |
| 18 | class Perturbation: |
| 19 | """A single perturbation specifying which attention type to skip and in which blocks.""" |
| 20 | |
| 21 | type: PerturbationType |
| 22 | blocks: list[int] | None # None means all blocks |
| 23 | |
| 24 | def is_perturbed(self, perturbation_type: PerturbationType, block: int) -> bool: |
| 25 | if self.type != perturbation_type: |
| 26 | return False |
| 27 | |
| 28 | if self.blocks is None: |
| 29 | return True |
| 30 | |
| 31 | return block in self.blocks |
| 32 | |
| 33 | |
| 34 | @dataclass(frozen=True) |
| 35 | class PerturbationConfig: |
| 36 | """Configuration holding a list of perturbations for a single sample.""" |
| 37 | |
| 38 | perturbations: list[Perturbation] | None |
| 39 | |
| 40 | def is_perturbed(self, perturbation_type: PerturbationType, block: int) -> bool: |
| 41 | if self.perturbations is None: |
| 42 | return False |
| 43 | |
| 44 | return any(perturbation.is_perturbed(perturbation_type, block) for perturbation in self.perturbations) |
| 45 | |
| 46 | @staticmethod |
| 47 | def empty() -> "PerturbationConfig": |
| 48 | return PerturbationConfig([]) |
| 49 | |
| 50 | |
| 51 | @dataclass(frozen=True) |
| 52 | class BatchedPerturbationConfig: |
| 53 | """Perturbation configurations for a batch, with utilities for generating attention masks.""" |
| 54 | |
| 55 | perturbations: list[PerturbationConfig] |
| 56 | |
| 57 | def mask( |
| 58 | self, perturbation_type: PerturbationType, block: int, device: DeviceLikeType, dtype: torch.dtype |
| 59 | ) -> torch.Tensor: |
| 60 | mask = torch.ones((len(self.perturbations),), device=device, dtype=dtype) |
| 61 | for batch_idx, perturbation in enumerate(self.perturbations): |
| 62 | if perturbation.is_perturbed(perturbation_type, block): |
| 63 | mask[batch_idx] = 0 |
| 64 | |
| 65 | return mask |
| 66 | |
| 67 | def mask_like(self, perturbation_type: PerturbationType, block: int, values: torch.Tensor) -> torch.Tensor: |
| 68 | mask = self.mask(perturbation_type, block, values.device, values.dtype) |
| 69 | return mask.view(mask.numel(), *([1] * len(values.shape[1:]))) |
| 70 | |
| 71 | def any_in_batch(self, perturbation_type: PerturbationType, block: int) -> bool: |
| 72 | return any(perturbation.is_perturbed(perturbation_type, block) for perturbation in self.perturbations) |
| 73 | |
| 74 | def all_in_batch(self, perturbation_type: PerturbationType, block: int) -> bool: |
| 75 | return all(perturbation.is_perturbed(perturbation_type, block) for perturbation in self.perturbations) |
| 76 | |
| 77 | @staticmethod |
| 78 | def empty(batch_size: int) -> "BatchedPerturbationConfig": |
| 79 | return BatchedPerturbationConfig([PerturbationConfig.empty() for _ in range(batch_size)]) |
| 80 |