| 1 | from dataclasses import asdict, dataclass, field |
| 2 | |
| 3 | |
| 4 | @dataclass |
| 5 | class Gemma3RopeScaling: |
| 6 | factor: float = 8.0 |
| 7 | rope_type: str = "linear" |
| 8 | |
| 9 | |
| 10 | @dataclass |
| 11 | class Gemma3TextConfig: |
| 12 | attention_bias: bool = False |
| 13 | attention_dropout: float = 0.0 |
| 14 | attn_logit_softcapping: float | None = None |
| 15 | cache_implementation: str = "hybrid" |
| 16 | final_logit_softcapping: float | None = None |
| 17 | head_dim: int = 256 |
| 18 | hidden_activation: str = "gelu_pytorch_tanh" |
| 19 | hidden_size: int = 3840 |
| 20 | initializer_range: float = 0.02 |
| 21 | intermediate_size: int = 15360 |
| 22 | max_position_embeddings: int = 131072 |
| 23 | model_type: str = "gemma3_text" |
| 24 | num_attention_heads: int = 16 |
| 25 | num_hidden_layers: int = 48 |
| 26 | num_key_value_heads: int = 8 |
| 27 | query_pre_attn_scalar: int = 256 |
| 28 | rms_norm_eps: float = 1e-06 |
| 29 | rope_local_base_freq: int = 10000 |
| 30 | rope_scaling: Gemma3RopeScaling = field(default_factory=Gemma3RopeScaling) |
| 31 | rope_theta: int = 1000000 |
| 32 | sliding_window: int = 1024 |
| 33 | sliding_window_pattern: int = 6 |
| 34 | torch_dtype: str = "float32" |
| 35 | use_cache: bool = True |
| 36 | vocab_size: int = 262208 |
| 37 | |
| 38 | |
| 39 | @dataclass |
| 40 | class Gemma3VisionConfig: |
| 41 | attention_dropout: float = 0.0 |
| 42 | hidden_act: str = "gelu_pytorch_tanh" |
| 43 | hidden_size: int = 1152 |
| 44 | image_size: int = 896 |
| 45 | intermediate_size: int = 4304 |
| 46 | layer_norm_eps: float = 1e-06 |
| 47 | model_type: str = "siglip_vision_model" |
| 48 | num_attention_heads: int = 16 |
| 49 | num_channels: int = 3 |
| 50 | num_hidden_layers: int = 27 |
| 51 | patch_size: int = 14 |
| 52 | torch_dtype: str = "float32" |
| 53 | vision_use_head: bool = False |
| 54 | |
| 55 | |
| 56 | @dataclass |
| 57 | class Gemma3ConfigData: |
| 58 | architectures: list[str] = field(default_factory=lambda: ["Gemma3ForConditionalGeneration"]) |
| 59 | boi_token_index: int = 255999 |
| 60 | eoi_token_index: int = 256000 |
| 61 | eos_token_id: list[int] = field(default_factory=lambda: [1, 106]) |
| 62 | image_token_index: int = 262144 |
| 63 | initializer_range: float = 0.02 |
| 64 | mm_tokens_per_image: int = 256 |
| 65 | model_type: str = "gemma3" |
| 66 | text_config: Gemma3TextConfig = field(default_factory=Gemma3TextConfig) |
| 67 | torch_dtype: str = "bfloat16" |
| 68 | transformers_version: str = "4.51.0" |
| 69 | vision_config: Gemma3VisionConfig = field(default_factory=Gemma3VisionConfig) |
| 70 | |
| 71 | def to_dict(self) -> dict: |
| 72 | return asdict(self) |
| 73 | |
| 74 | |
| 75 | GEMMA3_CONFIG_FOR_LTX = Gemma3ConfigData() |
| 76 |