Upload 3 files

config.json

@@ -1,13 +1,19 @@
 {
   "_sliding_window_pattern": 6,
   "architectures": [
-    "Gemma3ForCausalLM"
+    "FlashHeadGemma3ForCausalLM"
   ],
   "attention_bias": false,
   "attention_dropout": 0.0,
   "attn_logit_softcapping": null,
+  "auto_map": {
+    "AutoConfig": "configuration_flash_head_gemma3_text.FlashHeadGemma3TextConfig",
+    "AutoModelForCausalLM": "modeling_flash_head_gemma3_text.FlashHeadGemma3ForCausalLM"
+  },
   "bos_token_id": 2,
   "cache_implementation": "hybrid",
+  "creation_time": 1753799722.0919366,
+  "enforce_equal_cluster_sizes": true,
   "eos_token_id": [
     1,
     106
@@ -52,7 +58,8 @@
     "sliding_attention"
   ],
   "max_position_embeddings": 32768,
-  "model_type": "gemma3_text",
+  "model_type": "flash_head_gemma3_text",
+  "n_clusters": 16384,
   "num_attention_heads": 4,
   "num_hidden_layers": 26,
   "num_key_value_heads": 1,

configuration_flash_head_gemma3_text.py

@@ -0,0 +1 @@
+from embedl.models.gemma.modeling_flash_head import FlashHeadGemma3TextConfig

modeling_flash_head_gemma3_text.py

@@ -0,0 +1 @@
+from embedl.models.gemma.modeling_flash_head import FlashHeadGemma3ForCausalLM