| 1 | import torch |
| 2 | from einops import rearrange |
| 3 | |
| 4 | from ltx_core.model.upsampler.pixel_shuffle import PixelShuffleND |
| 5 | from ltx_core.model.upsampler.res_block import ResBlock |
| 6 | from ltx_core.model.upsampler.spatial_rational_resampler import SpatialRationalResampler |
| 7 | from ltx_core.model.video_vae import VideoEncoder |
| 8 | |
| 9 | |
| 10 | class LatentUpsampler(torch.nn.Module): |
| 11 | """ |
| 12 | Model to upsample VAE latents spatially and/or temporally. |
| 13 | Args: |
| 14 | in_channels (`int`): Number of channels in the input latent |
| 15 | mid_channels (`int`): Number of channels in the middle layers |
| 16 | num_blocks_per_stage (`int`): Number of ResBlocks to use in each stage (pre/post upsampling) |
| 17 | dims (`int`): Number of dimensions for convolutions (2 or 3) |
| 18 | spatial_upsample (`bool`): Whether to spatially upsample the latent |
| 19 | temporal_upsample (`bool`): Whether to temporally upsample the latent |
| 20 | spatial_scale (`float`): Scale factor for spatial upsampling |
| 21 | rational_resampler (`bool`): Whether to use a rational resampler for spatial upsampling |
| 22 | """ |
| 23 | |
| 24 | def __init__( |
| 25 | self, |
| 26 | in_channels: int = 128, |
| 27 | mid_channels: int = 512, |
| 28 | num_blocks_per_stage: int = 4, |
| 29 | dims: int = 3, |
| 30 | spatial_upsample: bool = True, |
| 31 | temporal_upsample: bool = False, |
| 32 | spatial_scale: float = 2.0, |
| 33 | rational_resampler: bool = False, |
| 34 | ): |
| 35 | super().__init__() |
| 36 | |
| 37 | self.in_channels = in_channels |
| 38 | self.mid_channels = mid_channels |
| 39 | self.num_blocks_per_stage = num_blocks_per_stage |
| 40 | self.dims = dims |
| 41 | self.spatial_upsample = spatial_upsample |
| 42 | self.temporal_upsample = temporal_upsample |
| 43 | self.spatial_scale = float(spatial_scale) |
| 44 | self.rational_resampler = rational_resampler |
| 45 | |
| 46 | conv = torch.nn.Conv2d if dims == 2 else torch.nn.Conv3d |
| 47 | |
| 48 | self.initial_conv = conv(in_channels, mid_channels, kernel_size=3, padding=1) |
| 49 | self.initial_norm = torch.nn.GroupNorm(32, mid_channels) |
| 50 | self.initial_activation = torch.nn.SiLU() |
| 51 | |
| 52 | self.res_blocks = torch.nn.ModuleList([ResBlock(mid_channels, dims=dims) for _ in range(num_blocks_per_stage)]) |
| 53 | |
| 54 | if spatial_upsample and temporal_upsample: |
| 55 | self.upsampler = torch.nn.Sequential( |
| 56 | torch.nn.Conv3d(mid_channels, 8 * mid_channels, kernel_size=3, padding=1), |
| 57 | PixelShuffleND(3), |
| 58 | ) |
| 59 | elif spatial_upsample: |
| 60 | if rational_resampler: |
| 61 | self.upsampler = SpatialRationalResampler(mid_channels=mid_channels, scale=self.spatial_scale) |
| 62 | else: |
| 63 | self.upsampler = torch.nn.Sequential( |
| 64 | torch.nn.Conv2d(mid_channels, 4 * mid_channels, kernel_size=3, padding=1), |
| 65 | PixelShuffleND(2), |
| 66 | ) |
| 67 | elif temporal_upsample: |
| 68 | self.upsampler = torch.nn.Sequential( |
| 69 | torch.nn.Conv3d(mid_channels, 2 * mid_channels, kernel_size=3, padding=1), |
| 70 | PixelShuffleND(1), |
| 71 | ) |
| 72 | else: |
| 73 | raise ValueError("Either spatial_upsample or temporal_upsample must be True") |
| 74 | |
| 75 | self.post_upsample_res_blocks = torch.nn.ModuleList( |
| 76 | [ResBlock(mid_channels, dims=dims) for _ in range(num_blocks_per_stage)] |
| 77 | ) |
| 78 | |
| 79 | self.final_conv = conv(mid_channels, in_channels, kernel_size=3, padding=1) |
| 80 | |
| 81 | def forward(self, latent: torch.Tensor) -> torch.Tensor: |
| 82 | b, _, f, _, _ = latent.shape |
| 83 | |
| 84 | if self.dims == 2: |
| 85 | x = rearrange(latent, "b c f h w -> (b f) c h w") |
| 86 | x = self.initial_conv(x) |
| 87 | x = self.initial_norm(x) |
| 88 | x = self.initial_activation(x) |
| 89 | |
| 90 | for block in self.res_blocks: |
| 91 | x = block(x) |
| 92 | |
| 93 | x = self.upsampler(x) |
| 94 | |
| 95 | for block in self.post_upsample_res_blocks: |
| 96 | x = block(x) |
| 97 | |
| 98 | x = self.final_conv(x) |
| 99 | x = rearrange(x, "(b f) c h w -> b c f h w", b=b, f=f) |
| 100 | else: |
| 101 | x = self.initial_conv(latent) |
| 102 | x = self.initial_norm(x) |
| 103 | x = self.initial_activation(x) |
| 104 | |
| 105 | for block in self.res_blocks: |
| 106 | x = block(x) |
| 107 | |
| 108 | if self.temporal_upsample: |
| 109 | x = self.upsampler(x) |
| 110 | # remove the first frame after upsampling. |
| 111 | # This is done because the first frame encodes one pixel frame. |
| 112 | x = x[:, :, 1:, :, :] |
| 113 | elif isinstance(self.upsampler, SpatialRationalResampler): |
| 114 | x = self.upsampler(x) |
| 115 | else: |
| 116 | x = rearrange(x, "b c f h w -> (b f) c h w") |
| 117 | x = self.upsampler(x) |
| 118 | x = rearrange(x, "(b f) c h w -> b c f h w", b=b, f=f) |
| 119 | |
| 120 | for block in self.post_upsample_res_blocks: |
| 121 | x = block(x) |
| 122 | |
| 123 | x = self.final_conv(x) |
| 124 | |
| 125 | return x |
| 126 | |
| 127 | |
| 128 | def upsample_video(latent: torch.Tensor, video_encoder: VideoEncoder, upsampler: "LatentUpsampler") -> torch.Tensor: |
| 129 | """ |
| 130 | Apply upsampling to the latent representation using the provided upsampler, |
| 131 | with normalization and un-normalization based on the video encoder's per-channel statistics. |
| 132 | Args: |
| 133 | latent: Input latent tensor of shape [B, C, F, H, W]. |
| 134 | video_encoder: VideoEncoder with per_channel_statistics for normalization. |
| 135 | upsampler: LatentUpsampler module to perform upsampling. |
| 136 | Returns: |
| 137 | torch.Tensor: Upsampled and re-normalized latent tensor. |
| 138 | """ |
| 139 | latent = video_encoder.per_channel_statistics.un_normalize(latent) |
| 140 | latent = upsampler(latent) |
| 141 | latent = video_encoder.per_channel_statistics.normalize(latent) |
| 142 | return latent |
| 143 |