Upload Gemma3ForConditionalGeneration

config.json

@@ -3,6 +3,7 @@
     "Gemma3ForConditionalGeneration"
   ],
   "boi_token_index": 255999,
+  "dtype": "bfloat16",
   "eoi_token_index": 256000,
   "eos_token_id": [
     1,
@@ -13,12 +14,15 @@
   "mm_tokens_per_image": 256,
   "model_type": "gemma3",
   "quantization_config": {
+    "include_input_output_embeddings": false,
     "modules_to_not_convert": null,
     "quant_method": "torchao",
     "quant_type": {
       "default": {
         "_data": {
           "group_size": 128,
+          "int4_choose_qparams_algorithm": "hqq",
+          "int4_packing_format": "tile_packed_to_4d",
           "layout": {
             "_data": {
               "inner_k_tiles": 8
@@ -26,10 +30,6 @@
             "_type": "TensorCoreTiledLayout",
             "_version": 1
           },
-          "packing_format": {
-            "_data": "PLAIN",
-            "_type": "PackingFormat"
-          },
           "preserve_zero": null,
           "set_inductor_config": true,
           "use_hqq": false,
@@ -42,19 +42,85 @@
         "_version": 2
       }
     },
-    "quant_type_kwargs": {}
+    "quant_type_kwargs": {},
+    "untie_embedding_weights": false
   },
   "text_config": {
+    "_sliding_window_pattern": 6,
     "attention_bias": false,
     "attention_dropout": 0.0,
     "attn_logit_softcapping": null,
-    "cache_implementation": "hybrid",
+    "dtype": "bfloat16",
     "final_logit_softcapping": null,
     "head_dim": 128,
     "hidden_activation": "gelu_pytorch_tanh",
     "hidden_size": 5376,
     "initializer_range": 0.02,
     "intermediate_size": 21504,
+    "layer_types": [
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "full_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "full_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "full_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "full_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "full_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "full_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "full_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "full_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "full_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "full_attention",
+      "sliding_attention",
+      "sliding_attention"
+    ],
     "max_position_embeddings": 131072,
     "model_type": "gemma3_text",
     "num_attention_heads": 32,
@@ -69,15 +135,13 @@
     },
     "rope_theta": 1000000.0,
     "sliding_window": 1024,
-    "sliding_window_pattern": 6,
-    "torch_dtype": "bfloat16",
     "use_cache": true,
     "vocab_size": 262208
   },
-  "torch_dtype": "bfloat16",
-  "transformers_version": "4.52.0.dev0",
+  "transformers_version": "4.56.1",
   "vision_config": {
     "attention_dropout": 0.0,
+    "dtype": "bfloat16",
     "hidden_act": "gelu_pytorch_tanh",
     "hidden_size": 1152,
     "image_size": 896,
@@ -88,7 +152,6 @@
     "num_channels": 3,
     "num_hidden_layers": 27,
     "patch_size": 14,
-    "torch_dtype": "bfloat16",
     "vision_use_head": false
   }
 }

generation_config.json

@@ -9,5 +9,5 @@
   "pad_token_id": 0,
   "top_k": 64,
   "top_p": 0.95,
-  "transformers_version": "4.52.0.dev0"
+  "transformers_version": "4.56.1"
 }

pytorch_model-00001-of-00004.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:408653044aa52ff49fa0f9814868d3659abd40279dfb0abf52153d8e71969e5d
-size 4952199570
+oid sha256:6c971b60644fa3fb6e9988643dbd748f5217cd93d0d7065faa711a4874707b01
+size 4993517312

pytorch_model-00002-of-00004.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8965dabce3fdd50df83d9acfd8122ff3e6e8b0969493a3b5714f192dc3dce492
-size 4984651999
+oid sha256:2f274b6812d69241cef73232c4b1644668bb660d33ee278ca7e0ae9a74e1a91b
+size 4975728231

pytorch_model-00003-of-00004.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1d336f78c447f68412a3cff5d23fae9efe4267c36119b5c4f8053cc0bf2b2ae7
-size 4984652195
+oid sha256:01961d4ecfcca8e5d8103145e548fa8157f146dede2db5ea2fba44788ed80165
+size 4975728231

pytorch_model-00004-of-00004.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:85723169e9874620cc307c6581926afd51f0730c1c2ae25900313857ded569ad
-size 1939459663
+oid sha256:08691fb7c3930fbb501981d2c0f990589c3852da34c032a2f0a8a77053f20a80
+size 3325514970

pytorch_model.bin.index.json

@@ -1,6 +1,7 @@
 {
   "metadata": {
-    "total_size": 16860083872
+    "total_parameters": 27432406640,
+    "total_size": 18269776096
   },
   "weight_map": {
     "language_model.lm_head.weight": "pytorch_model-00001-of-00004.bin",
@@ -265,45 +266,45 @@
     "language_model.model.layers.26.self_attn.q_norm.weight": "pytorch_model-00002-of-00004.bin",
     "language_model.model.layers.26.self_attn.q_proj.weight": "pytorch_model-00002-of-00004.bin",
     "language_model.model.layers.26.self_attn.v_proj.weight": "pytorch_model-00002-of-00004.bin",
-    "language_model.model.layers.27.input_layernorm.weight": "pytorch_model-00002-of-00004.bin",
-    "language_model.model.layers.27.mlp.down_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "language_model.model.layers.27.input_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "language_model.model.layers.27.mlp.down_proj.weight": "pytorch_model-00003-of-00004.bin",
     "language_model.model.layers.27.mlp.gate_proj.weight": "pytorch_model-00002-of-00004.bin",
-    "language_model.model.layers.27.mlp.up_proj.weight": "pytorch_model-00002-of-00004.bin",
-    "language_model.model.layers.27.post_attention_layernorm.weight": "pytorch_model-00002-of-00004.bin",
-    "language_model.model.layers.27.post_feedforward_layernorm.weight": "pytorch_model-00002-of-00004.bin",
-    "language_model.model.layers.27.pre_feedforward_layernorm.weight": "pytorch_model-00002-of-00004.bin",
+    "language_model.model.layers.27.mlp.up_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "language_model.model.layers.27.post_attention_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "language_model.model.layers.27.post_feedforward_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "language_model.model.layers.27.pre_feedforward_layernorm.weight": "pytorch_model-00003-of-00004.bin",
     "language_model.model.layers.27.self_attn.k_norm.weight": "pytorch_model-00002-of-00004.bin",
     "language_model.model.layers.27.self_attn.k_proj.weight": "pytorch_model-00002-of-00004.bin",
     "language_model.model.layers.27.self_attn.o_proj.weight": "pytorch_model-00002-of-00004.bin",
     "language_model.model.layers.27.self_attn.q_norm.weight": "pytorch_model-00002-of-00004.bin",
     "language_model.model.layers.27.self_attn.q_proj.weight": "pytorch_model-00002-of-00004.bin",
     "language_model.model.layers.27.self_attn.v_proj.weight": "pytorch_model-00002-of-00004.bin",
-    "language_model.model.layers.28.input_layernorm.weight": "pytorch_model-00002-of-00004.bin",
-    "language_model.model.layers.28.mlp.down_proj.weight": "pytorch_model-00002-of-00004.bin",
-    "language_model.model.layers.28.mlp.gate_proj.weight": "pytorch_model-00002-of-00004.bin",
-    "language_model.model.layers.28.mlp.up_proj.weight": "pytorch_model-00002-of-00004.bin",
-    "language_model.model.layers.28.post_attention_layernorm.weight": "pytorch_model-00002-of-00004.bin",
-    "language_model.model.layers.28.post_feedforward_layernorm.weight": "pytorch_model-00002-of-00004.bin",
-    "language_model.model.layers.28.pre_feedforward_layernorm.weight": "pytorch_model-00002-of-00004.bin",
-    "language_model.model.layers.28.self_attn.k_norm.weight": "pytorch_model-00002-of-00004.bin",
-    "language_model.model.layers.28.self_attn.k_proj.weight": "pytorch_model-00002-of-00004.bin",
-    "language_model.model.layers.28.self_attn.o_proj.weight": "pytorch_model-00002-of-00004.bin",
-    "language_model.model.layers.28.self_attn.q_norm.weight": "pytorch_model-00002-of-00004.bin",
-    "language_model.model.layers.28.self_attn.q_proj.weight": "pytorch_model-00002-of-00004.bin",
-    "language_model.model.layers.28.self_attn.v_proj.weight": "pytorch_model-00002-of-00004.bin",
-    "language_model.model.layers.29.input_layernorm.weight": "pytorch_model-00002-of-00004.bin",
-    "language_model.model.layers.29.mlp.down_proj.weight": "pytorch_model-00002-of-00004.bin",
-    "language_model.model.layers.29.mlp.gate_proj.weight": "pytorch_model-00002-of-00004.bin",
-    "language_model.model.layers.29.mlp.up_proj.weight": "pytorch_model-00002-of-00004.bin",
-    "language_model.model.layers.29.post_attention_layernorm.weight": "pytorch_model-00002-of-00004.bin",
-    "language_model.model.layers.29.post_feedforward_layernorm.weight": "pytorch_model-00002-of-00004.bin",
-    "language_model.model.layers.29.pre_feedforward_layernorm.weight": "pytorch_model-00002-of-00004.bin",
-    "language_model.model.layers.29.self_attn.k_norm.weight": "pytorch_model-00002-of-00004.bin",
-    "language_model.model.layers.29.self_attn.k_proj.weight": "pytorch_model-00002-of-00004.bin",
-    "language_model.model.layers.29.self_attn.o_proj.weight": "pytorch_model-00002-of-00004.bin",
-    "language_model.model.layers.29.self_attn.q_norm.weight": "pytorch_model-00002-of-00004.bin",
-    "language_model.model.layers.29.self_attn.q_proj.weight": "pytorch_model-00002-of-00004.bin",
-    "language_model.model.layers.29.self_attn.v_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "language_model.model.layers.28.input_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "language_model.model.layers.28.mlp.down_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "language_model.model.layers.28.mlp.gate_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "language_model.model.layers.28.mlp.up_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "language_model.model.layers.28.post_attention_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "language_model.model.layers.28.post_feedforward_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "language_model.model.layers.28.pre_feedforward_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "language_model.model.layers.28.self_attn.k_norm.weight": "pytorch_model-00003-of-00004.bin",
+    "language_model.model.layers.28.self_attn.k_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "language_model.model.layers.28.self_attn.o_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "language_model.model.layers.28.self_attn.q_norm.weight": "pytorch_model-00003-of-00004.bin",
+    "language_model.model.layers.28.self_attn.q_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "language_model.model.layers.28.self_attn.v_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "language_model.model.layers.29.input_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "language_model.model.layers.29.mlp.down_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "language_model.model.layers.29.mlp.gate_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "language_model.model.layers.29.mlp.up_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "language_model.model.layers.29.post_attention_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "language_model.model.layers.29.post_feedforward_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "language_model.model.layers.29.pre_feedforward_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "language_model.model.layers.29.self_attn.k_norm.weight": "pytorch_model-00003-of-00004.bin",
+    "language_model.model.layers.29.self_attn.k_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "language_model.model.layers.29.self_attn.o_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "language_model.model.layers.29.self_attn.q_norm.weight": "pytorch_model-00003-of-00004.bin",
+    "language_model.model.layers.29.self_attn.q_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "language_model.model.layers.29.self_attn.v_proj.weight": "pytorch_model-00003-of-00004.bin",
     "language_model.model.layers.3.input_layernorm.weight": "pytorch_model-00001-of-00004.bin",
     "language_model.model.layers.3.mlp.down_proj.weight": "pytorch_model-00001-of-00004.bin",
     "language_model.model.layers.3.mlp.gate_proj.weight": "pytorch_model-00001-of-00004.bin",
@@ -319,17 +320,17 @@
     "language_model.model.layers.3.self_attn.v_proj.weight": "pytorch_model-00001-of-00004.bin",
     "language_model.model.layers.30.input_layernorm.weight": "pytorch_model-00003-of-00004.bin",
     "language_model.model.layers.30.mlp.down_proj.weight": "pytorch_model-00003-of-00004.bin",
-    "language_model.model.layers.30.mlp.gate_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "language_model.model.layers.30.mlp.gate_proj.weight": "pytorch_model-00003-of-00004.bin",
     "language_model.model.layers.30.mlp.up_proj.weight": "pytorch_model-00003-of-00004.bin",
     "language_model.model.layers.30.post_attention_layernorm.weight": "pytorch_model-00003-of-00004.bin",
     "language_model.model.layers.30.post_feedforward_layernorm.weight": "pytorch_model-00003-of-00004.bin",
     "language_model.model.layers.30.pre_feedforward_layernorm.weight": "pytorch_model-00003-of-00004.bin",
-    "language_model.model.layers.30.self_attn.k_norm.weight": "pytorch_model-00002-of-00004.bin",
-    "language_model.model.layers.30.self_attn.k_proj.weight": "pytorch_model-00002-of-00004.bin",
-    "language_model.model.layers.30.self_attn.o_proj.weight": "pytorch_model-00002-of-00004.bin",
-    "language_model.model.layers.30.self_attn.q_norm.weight": "pytorch_model-00002-of-00004.bin",
-    "language_model.model.layers.30.self_attn.q_proj.weight": "pytorch_model-00002-of-00004.bin",
-    "language_model.model.layers.30.self_attn.v_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "language_model.model.layers.30.self_attn.k_norm.weight": "pytorch_model-00003-of-00004.bin",
+    "language_model.model.layers.30.self_attn.k_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "language_model.model.layers.30.self_attn.o_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "language_model.model.layers.30.self_attn.q_norm.weight": "pytorch_model-00003-of-00004.bin",
+    "language_model.model.layers.30.self_attn.q_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "language_model.model.layers.30.self_attn.v_proj.weight": "pytorch_model-00003-of-00004.bin",
     "language_model.model.layers.31.input_layernorm.weight": "pytorch_model-00003-of-00004.bin",
     "language_model.model.layers.31.mlp.down_proj.weight": "pytorch_model-00003-of-00004.bin",
     "language_model.model.layers.31.mlp.gate_proj.weight": "pytorch_model-00003-of-00004.bin",
@@ -564,32 +565,32 @@
     "language_model.model.layers.47.self_attn.q_norm.weight": "pytorch_model-00003-of-00004.bin",
     "language_model.model.layers.47.self_attn.q_proj.weight": "pytorch_model-00003-of-00004.bin",
     "language_model.model.layers.47.self_attn.v_proj.weight": "pytorch_model-00003-of-00004.bin",
-    "language_model.model.layers.48.input_layernorm.weight": "pytorch_model-00003-of-00004.bin",
-    "language_model.model.layers.48.mlp.down_proj.weight": "pytorch_model-00003-of-00004.bin",
-    "language_model.model.layers.48.mlp.gate_proj.weight": "pytorch_model-00003-of-00004.bin",
-    "language_model.model.layers.48.mlp.up_proj.weight": "pytorch_model-00003-of-00004.bin",
-    "language_model.model.layers.48.post_attention_layernorm.weight": "pytorch_model-00003-of-00004.bin",
-    "language_model.model.layers.48.post_feedforward_layernorm.weight": "pytorch_model-00003-of-00004.bin",
-    "language_model.model.layers.48.pre_feedforward_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "language_model.model.layers.48.input_layernorm.weight": "pytorch_model-00004-of-00004.bin",
+    "language_model.model.layers.48.mlp.down_proj.weight": "pytorch_model-00004-of-00004.bin",
+    "language_model.model.layers.48.mlp.gate_proj.weight": "pytorch_model-00004-of-00004.bin",
+    "language_model.model.layers.48.mlp.up_proj.weight": "pytorch_model-00004-of-00004.bin",
+    "language_model.model.layers.48.post_attention_layernorm.weight": "pytorch_model-00004-of-00004.bin",
+    "language_model.model.layers.48.post_feedforward_layernorm.weight": "pytorch_model-00004-of-00004.bin",
+    "language_model.model.layers.48.pre_feedforward_layernorm.weight": "pytorch_model-00004-of-00004.bin",
     "language_model.model.layers.48.self_attn.k_norm.weight": "pytorch_model-00003-of-00004.bin",
     "language_model.model.layers.48.self_attn.k_proj.weight": "pytorch_model-00003-of-00004.bin",
     "language_model.model.layers.48.self_attn.o_proj.weight": "pytorch_model-00003-of-00004.bin",
     "language_model.model.layers.48.self_attn.q_norm.weight": "pytorch_model-00003-of-00004.bin",
     "language_model.model.layers.48.self_attn.q_proj.weight": "pytorch_model-00003-of-00004.bin",
     "language_model.model.layers.48.self_attn.v_proj.weight": "pytorch_model-00003-of-00004.bin",
-    "language_model.model.layers.49.input_layernorm.weight": "pytorch_model-00003-of-00004.bin",
-    "language_model.model.layers.49.mlp.down_proj.weight": "pytorch_model-00003-of-00004.bin",
-    "language_model.model.layers.49.mlp.gate_proj.weight": "pytorch_model-00003-of-00004.bin",
-    "language_model.model.layers.49.mlp.up_proj.weight": "pytorch_model-00003-of-00004.bin",
-    "language_model.model.layers.49.post_attention_layernorm.weight": "pytorch_model-00003-of-00004.bin",
-    "language_model.model.layers.49.post_feedforward_layernorm.weight": "pytorch_model-00003-of-00004.bin",
-    "language_model.model.layers.49.pre_feedforward_layernorm.weight": "pytorch_model-00003-of-00004.bin",
-    "language_model.model.layers.49.self_attn.k_norm.weight": "pytorch_model-00003-of-00004.bin",
-    "language_model.model.layers.49.self_attn.k_proj.weight": "pytorch_model-00003-of-00004.bin",
-    "language_model.model.layers.49.self_attn.o_proj.weight": "pytorch_model-00003-of-00004.bin",
-    "language_model.model.layers.49.self_attn.q_norm.weight": "pytorch_model-00003-of-00004.bin",
-    "language_model.model.layers.49.self_attn.q_proj.weight": "pytorch_model-00003-of-00004.bin",
-    "language_model.model.layers.49.self_attn.v_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "language_model.model.layers.49.input_layernorm.weight": "pytorch_model-00004-of-00004.bin",
+    "language_model.model.layers.49.mlp.down_proj.weight": "pytorch_model-00004-of-00004.bin",
+    "language_model.model.layers.49.mlp.gate_proj.weight": "pytorch_model-00004-of-00004.bin",
+    "language_model.model.layers.49.mlp.up_proj.weight": "pytorch_model-00004-of-00004.bin",
+    "language_model.model.layers.49.post_attention_layernorm.weight": "pytorch_model-00004-of-00004.bin",
+    "language_model.model.layers.49.post_feedforward_layernorm.weight": "pytorch_model-00004-of-00004.bin",
+    "language_model.model.layers.49.pre_feedforward_layernorm.weight": "pytorch_model-00004-of-00004.bin",
+    "language_model.model.layers.49.self_attn.k_norm.weight": "pytorch_model-00004-of-00004.bin",
+    "language_model.model.layers.49.self_attn.k_proj.weight": "pytorch_model-00004-of-00004.bin",
+    "language_model.model.layers.49.self_attn.o_proj.weight": "pytorch_model-00004-of-00004.bin",
+    "language_model.model.layers.49.self_attn.q_norm.weight": "pytorch_model-00004-of-00004.bin",
+    "language_model.model.layers.49.self_attn.q_proj.weight": "pytorch_model-00004-of-00004.bin",
+    "language_model.model.layers.49.self_attn.v_proj.weight": "pytorch_model-00004-of-00004.bin",
     "language_model.model.layers.5.input_layernorm.weight": "pytorch_model-00001-of-00004.bin",
     "language_model.model.layers.5.mlp.down_proj.weight": "pytorch_model-00001-of-00004.bin",
     "language_model.model.layers.5.mlp.gate_proj.weight": "pytorch_model-00001-of-00004.bin",
@@ -603,45 +604,45 @@
     "language_model.model.layers.5.self_attn.q_norm.weight": "pytorch_model-00001-of-00004.bin",
     "language_model.model.layers.5.self_attn.q_proj.weight": "pytorch_model-00001-of-00004.bin",
     "language_model.model.layers.5.self_attn.v_proj.weight": "pytorch_model-00001-of-00004.bin",
-    "language_model.model.layers.50.input_layernorm.weight": "pytorch_model-00003-of-00004.bin",
-    "language_model.model.layers.50.mlp.down_proj.weight": "pytorch_model-00003-of-00004.bin",
-    "language_model.model.layers.50.mlp.gate_proj.weight": "pytorch_model-00003-of-00004.bin",
-    "language_model.model.layers.50.mlp.up_proj.weight": "pytorch_model-00003-of-00004.bin",
-    "language_model.model.layers.50.post_attention_layernorm.weight": "pytorch_model-00003-of-00004.bin",
-    "language_model.model.layers.50.post_feedforward_layernorm.weight": "pytorch_model-00003-of-00004.bin",
-    "language_model.model.layers.50.pre_feedforward_layernorm.weight": "pytorch_model-00003-of-00004.bin",
-    "language_model.model.layers.50.self_attn.k_norm.weight": "pytorch_model-00003-of-00004.bin",
-    "language_model.model.layers.50.self_attn.k_proj.weight": "pytorch_model-00003-of-00004.bin",
-    "language_model.model.layers.50.self_attn.o_proj.weight": "pytorch_model-00003-of-00004.bin",
-    "language_model.model.layers.50.self_attn.q_norm.weight": "pytorch_model-00003-of-00004.bin",
-    "language_model.model.layers.50.self_attn.q_proj.weight": "pytorch_model-00003-of-00004.bin",
-    "language_model.model.layers.50.self_attn.v_proj.weight": "pytorch_model-00003-of-00004.bin",
-    "language_model.model.layers.51.input_layernorm.weight": "pytorch_model-00003-of-00004.bin",
-    "language_model.model.layers.51.mlp.down_proj.weight": "pytorch_model-00003-of-00004.bin",
-    "language_model.model.layers.51.mlp.gate_proj.weight": "pytorch_model-00003-of-00004.bin",
-    "language_model.model.layers.51.mlp.up_proj.weight": "pytorch_model-00003-of-00004.bin",
-    "language_model.model.layers.51.post_attention_layernorm.weight": "pytorch_model-00003-of-00004.bin",
-    "language_model.model.layers.51.post_feedforward_layernorm.weight": "pytorch_model-00003-of-00004.bin",
-    "language_model.model.layers.51.pre_feedforward_layernorm.weight": "pytorch_model-00003-of-00004.bin",
-    "language_model.model.layers.51.self_attn.k_norm.weight": "pytorch_model-00003-of-00004.bin",
-    "language_model.model.layers.51.self_attn.k_proj.weight": "pytorch_model-00003-of-00004.bin",
-    "language_model.model.layers.51.self_attn.o_proj.weight": "pytorch_model-00003-of-00004.bin",
-    "language_model.model.layers.51.self_attn.q_norm.weight": "pytorch_model-00003-of-00004.bin",
-    "language_model.model.layers.51.self_attn.q_proj.weight": "pytorch_model-00003-of-00004.bin",
-    "language_model.model.layers.51.self_attn.v_proj.weight": "pytorch_model-00003-of-00004.bin",
-    "language_model.model.layers.52.input_layernorm.weight": "pytorch_model-00003-of-00004.bin",
-    "language_model.model.layers.52.mlp.down_proj.weight": "pytorch_model-00003-of-00004.bin",
-    "language_model.model.layers.52.mlp.gate_proj.weight": "pytorch_model-00003-of-00004.bin",
-    "language_model.model.layers.52.mlp.up_proj.weight": "pytorch_model-00003-of-00004.bin",
-    "language_model.model.layers.52.post_attention_layernorm.weight": "pytorch_model-00003-of-00004.bin",
-    "language_model.model.layers.52.post_feedforward_layernorm.weight": "pytorch_model-00003-of-00004.bin",
-    "language_model.model.layers.52.pre_feedforward_layernorm.weight": "pytorch_model-00003-of-00004.bin",
-    "language_model.model.layers.52.self_attn.k_norm.weight": "pytorch_model-00003-of-00004.bin",
-    "language_model.model.layers.52.self_attn.k_proj.weight": "pytorch_model-00003-of-00004.bin",
-    "language_model.model.layers.52.self_attn.o_proj.weight": "pytorch_model-00003-of-00004.bin",
-    "language_model.model.layers.52.self_attn.q_norm.weight": "pytorch_model-00003-of-00004.bin",
-    "language_model.model.layers.52.self_attn.q_proj.weight": "pytorch_model-00003-of-00004.bin",
-    "language_model.model.layers.52.self_attn.v_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "language_model.model.layers.50.input_layernorm.weight": "pytorch_model-00004-of-00004.bin",
+    "language_model.model.layers.50.mlp.down_proj.weight": "pytorch_model-00004-of-00004.bin",
+    "language_model.model.layers.50.mlp.gate_proj.weight": "pytorch_model-00004-of-00004.bin",
+    "language_model.model.layers.50.mlp.up_proj.weight": "pytorch_model-00004-of-00004.bin",
+    "language_model.model.layers.50.post_attention_layernorm.weight": "pytorch_model-00004-of-00004.bin",
+    "language_model.model.layers.50.post_feedforward_layernorm.weight": "pytorch_model-00004-of-00004.bin",
+    "language_model.model.layers.50.pre_feedforward_layernorm.weight": "pytorch_model-00004-of-00004.bin",
+    "language_model.model.layers.50.self_attn.k_norm.weight": "pytorch_model-00004-of-00004.bin",
+    "language_model.model.layers.50.self_attn.k_proj.weight": "pytorch_model-00004-of-00004.bin",
+    "language_model.model.layers.50.self_attn.o_proj.weight": "pytorch_model-00004-of-00004.bin",
+    "language_model.model.layers.50.self_attn.q_norm.weight": "pytorch_model-00004-of-00004.bin",
+    "language_model.model.layers.50.self_attn.q_proj.weight": "pytorch_model-00004-of-00004.bin",
+    "language_model.model.layers.50.self_attn.v_proj.weight": "pytorch_model-00004-of-00004.bin",
+    "language_model.model.layers.51.input_layernorm.weight": "pytorch_model-00004-of-00004.bin",
+    "language_model.model.layers.51.mlp.down_proj.weight": "pytorch_model-00004-of-00004.bin",
+    "language_model.model.layers.51.mlp.gate_proj.weight": "pytorch_model-00004-of-00004.bin",
+    "language_model.model.layers.51.mlp.up_proj.weight": "pytorch_model-00004-of-00004.bin",
+    "language_model.model.layers.51.post_attention_layernorm.weight": "pytorch_model-00004-of-00004.bin",
+    "language_model.model.layers.51.post_feedforward_layernorm.weight": "pytorch_model-00004-of-00004.bin",
+    "language_model.model.layers.51.pre_feedforward_layernorm.weight": "pytorch_model-00004-of-00004.bin",
+    "language_model.model.layers.51.self_attn.k_norm.weight": "pytorch_model-00004-of-00004.bin",
+    "language_model.model.layers.51.self_attn.k_proj.weight": "pytorch_model-00004-of-00004.bin",
+    "language_model.model.layers.51.self_attn.o_proj.weight": "pytorch_model-00004-of-00004.bin",
+    "language_model.model.layers.51.self_attn.q_norm.weight": "pytorch_model-00004-of-00004.bin",
+    "language_model.model.layers.51.self_attn.q_proj.weight": "pytorch_model-00004-of-00004.bin",
+    "language_model.model.layers.51.self_attn.v_proj.weight": "pytorch_model-00004-of-00004.bin",
+    "language_model.model.layers.52.input_layernorm.weight": "pytorch_model-00004-of-00004.bin",
+    "language_model.model.layers.52.mlp.down_proj.weight": "pytorch_model-00004-of-00004.bin",
+    "language_model.model.layers.52.mlp.gate_proj.weight": "pytorch_model-00004-of-00004.bin",
+    "language_model.model.layers.52.mlp.up_proj.weight": "pytorch_model-00004-of-00004.bin",
+    "language_model.model.layers.52.post_attention_layernorm.weight": "pytorch_model-00004-of-00004.bin",
+    "language_model.model.layers.52.post_feedforward_layernorm.weight": "pytorch_model-00004-of-00004.bin",
+    "language_model.model.layers.52.pre_feedforward_layernorm.weight": "pytorch_model-00004-of-00004.bin",
+    "language_model.model.layers.52.self_attn.k_norm.weight": "pytorch_model-00004-of-00004.bin",
+    "language_model.model.layers.52.self_attn.k_proj.weight": "pytorch_model-00004-of-00004.bin",
+    "language_model.model.layers.52.self_attn.o_proj.weight": "pytorch_model-00004-of-00004.bin",
+    "language_model.model.layers.52.self_attn.q_norm.weight": "pytorch_model-00004-of-00004.bin",
+    "language_model.model.layers.52.self_attn.q_proj.weight": "pytorch_model-00004-of-00004.bin",
+    "language_model.model.layers.52.self_attn.v_proj.weight": "pytorch_model-00004-of-00004.bin",
     "language_model.model.layers.53.input_layernorm.weight": "pytorch_model-00004-of-00004.bin",
     "language_model.model.layers.53.mlp.down_proj.weight": "pytorch_model-00004-of-00004.bin",
     "language_model.model.layers.53.mlp.gate_proj.weight": "pytorch_model-00004-of-00004.bin",
@@ -649,12 +650,12 @@
     "language_model.model.layers.53.post_attention_layernorm.weight": "pytorch_model-00004-of-00004.bin",
     "language_model.model.layers.53.post_feedforward_layernorm.weight": "pytorch_model-00004-of-00004.bin",
     "language_model.model.layers.53.pre_feedforward_layernorm.weight": "pytorch_model-00004-of-00004.bin",
-    "language_model.model.layers.53.self_attn.k_norm.weight": "pytorch_model-00003-of-00004.bin",
-    "language_model.model.layers.53.self_attn.k_proj.weight": "pytorch_model-00003-of-00004.bin",
-    "language_model.model.layers.53.self_attn.o_proj.weight": "pytorch_model-00003-of-00004.bin",
-    "language_model.model.layers.53.self_attn.q_norm.weight": "pytorch_model-00003-of-00004.bin",
-    "language_model.model.layers.53.self_attn.q_proj.weight": "pytorch_model-00003-of-00004.bin",
-    "language_model.model.layers.53.self_attn.v_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "language_model.model.layers.53.self_attn.k_norm.weight": "pytorch_model-00004-of-00004.bin",
+    "language_model.model.layers.53.self_attn.k_proj.weight": "pytorch_model-00004-of-00004.bin",
+    "language_model.model.layers.53.self_attn.o_proj.weight": "pytorch_model-00004-of-00004.bin",
+    "language_model.model.layers.53.self_attn.q_norm.weight": "pytorch_model-00004-of-00004.bin",
+    "language_model.model.layers.53.self_attn.q_proj.weight": "pytorch_model-00004-of-00004.bin",
+    "language_model.model.layers.53.self_attn.v_proj.weight": "pytorch_model-00004-of-00004.bin",
     "language_model.model.layers.54.input_layernorm.weight": "pytorch_model-00004-of-00004.bin",
     "language_model.model.layers.54.mlp.down_proj.weight": "pytorch_model-00004-of-00004.bin",
     "language_model.model.layers.54.mlp.gate_proj.weight": "pytorch_model-00004-of-00004.bin",
@@ -733,13 +734,13 @@
     "language_model.model.layers.59.self_attn.q_norm.weight": "pytorch_model-00004-of-00004.bin",
     "language_model.model.layers.59.self_attn.q_proj.weight": "pytorch_model-00004-of-00004.bin",
     "language_model.model.layers.59.self_attn.v_proj.weight": "pytorch_model-00004-of-00004.bin",
-    "language_model.model.layers.6.input_layernorm.weight": "pytorch_model-00001-of-00004.bin",
-    "language_model.model.layers.6.mlp.down_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "language_model.model.layers.6.input_layernorm.weight": "pytorch_model-00002-of-00004.bin",
+    "language_model.model.layers.6.mlp.down_proj.weight": "pytorch_model-00002-of-00004.bin",
     "language_model.model.layers.6.mlp.gate_proj.weight": "pytorch_model-00001-of-00004.bin",
     "language_model.model.layers.6.mlp.up_proj.weight": "pytorch_model-00001-of-00004.bin",
-    "language_model.model.layers.6.post_attention_layernorm.weight": "pytorch_model-00001-of-00004.bin",
-    "language_model.model.layers.6.post_feedforward_layernorm.weight": "pytorch_model-00001-of-00004.bin",
-    "language_model.model.layers.6.pre_feedforward_layernorm.weight": "pytorch_model-00001-of-00004.bin",
+    "language_model.model.layers.6.post_attention_layernorm.weight": "pytorch_model-00002-of-00004.bin",
+    "language_model.model.layers.6.post_feedforward_layernorm.weight": "pytorch_model-00002-of-00004.bin",
+    "language_model.model.layers.6.pre_feedforward_layernorm.weight": "pytorch_model-00002-of-00004.bin",
     "language_model.model.layers.6.self_attn.k_norm.weight": "pytorch_model-00001-of-00004.bin",
     "language_model.model.layers.6.self_attn.k_proj.weight": "pytorch_model-00001-of-00004.bin",
     "language_model.model.layers.6.self_attn.o_proj.weight": "pytorch_model-00001-of-00004.bin",
@@ -774,17 +775,17 @@
     "language_model.model.layers.61.self_attn.v_proj.weight": "pytorch_model-00004-of-00004.bin",
     "language_model.model.layers.7.input_layernorm.weight": "pytorch_model-00002-of-00004.bin",
     "language_model.model.layers.7.mlp.down_proj.weight": "pytorch_model-00002-of-00004.bin",
-    "language_model.model.layers.7.mlp.gate_proj.weight": "pytorch_model-00001-of-00004.bin",
-    "language_model.model.layers.7.mlp.up_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "language_model.model.layers.7.mlp.gate_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "language_model.model.layers.7.mlp.up_proj.weight": "pytorch_model-00002-of-00004.bin",
     "language_model.model.layers.7.post_attention_layernorm.weight": "pytorch_model-00002-of-00004.bin",
     "language_model.model.layers.7.post_feedforward_layernorm.weight": "pytorch_model-00002-of-00004.bin",
     "language_model.model.layers.7.pre_feedforward_layernorm.weight": "pytorch_model-00002-of-00004.bin",
-    "language_model.model.layers.7.self_attn.k_norm.weight": "pytorch_model-00001-of-00004.bin",
-    "language_model.model.layers.7.self_attn.k_proj.weight": "pytorch_model-00001-of-00004.bin",
-    "language_model.model.layers.7.self_attn.o_proj.weight": "pytorch_model-00001-of-00004.bin",
-    "language_model.model.layers.7.self_attn.q_norm.weight": "pytorch_model-00001-of-00004.bin",
-    "language_model.model.layers.7.self_attn.q_proj.weight": "pytorch_model-00001-of-00004.bin",
-    "language_model.model.layers.7.self_attn.v_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "language_model.model.layers.7.self_attn.k_norm.weight": "pytorch_model-00002-of-00004.bin",
+    "language_model.model.layers.7.self_attn.k_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "language_model.model.layers.7.self_attn.o_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "language_model.model.layers.7.self_attn.q_norm.weight": "pytorch_model-00002-of-00004.bin",
+    "language_model.model.layers.7.self_attn.q_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "language_model.model.layers.7.self_attn.v_proj.weight": "pytorch_model-00002-of-00004.bin",
     "language_model.model.layers.8.input_layernorm.weight": "pytorch_model-00002-of-00004.bin",
     "language_model.model.layers.8.mlp.down_proj.weight": "pytorch_model-00002-of-00004.bin",
     "language_model.model.layers.8.mlp.gate_proj.weight": "pytorch_model-00002-of-00004.bin",