halt95 commited on
Commit
8e33458
·
verified ·
1 Parent(s): e2dfecb

Qwen3.8-Flash-Next-W4A16-Merlin weights

Browse files
.gitattributes CHANGED
@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ model.safetensors.index.json filter=lfs diff=lfs merge=lfs -text
37
+ tokenizer.json filter=lfs diff=lfs merge=lfs -text
model-plefp8-00002.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:06fd8a11abf0419a669f89397b8d70dd6ff42d401e6b2a037c65e49704faaf71
3
+ size 5200027198
model-plefp8-00003.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c6aaa1fc08e84eced3c8151ac8679ed943888eba0fcef2556963693430f95bd9
3
+ size 5200027189
model-plefp8-00004.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d94e97c96d3ea09208614da016960f3f4b429f47a044c898a51b493f42f74ba2
3
+ size 5200027190
model-plefp8-00005.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:03d5d4792e14a4ab55bae50bb459624b01786f818bbcfcaed1a5d0235af484c6
3
+ size 5200027190
model-plefp8-00006.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:586cccbc12383021bc9bc02f206d9b19fbd1672373ed9a5d91e3b0ce34c2418f
3
+ size 5200027190
model-plefp8-00007.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c4a23bcc10f3cde6b633e82c282a9a518e3a64d433628283e1bc592c94cf3d6c
3
+ size 5200027190
model-plefp8-00008.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2dc8098c0d020bff277c9cf499a6b908e17836b70a5f949dfa24793371c9a87e
3
+ size 5200027190
model-plefp8-00009.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:61de98b89bb79f386795787d7a76827a26f1e292c26edbb0a1b613da455f5a9c
3
+ size 4400023163
model.safetensors.index.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:aa458a995c677e18d7405bd3c6e2965a6af9df24b8e935090f999a064d700229
3
+ size 25005845
model_extra_tensors.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:70935e2c3a327d35e947524fa19b2ac007772af75827bd8eedc647e2e508bdb6
3
+ size 181140320
mtp-routed-experts-int4.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7f046b00fe9599aa048d99d718d61505b6825c45113cdb3a1d7f51cd994e5da0
3
+ size 1298193176
preprocessor_config.json ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "do_convert_rgb": true,
3
+ "do_normalize": true,
4
+ "do_rescale": true,
5
+ "do_resize": true,
6
+ "image_mean": [
7
+ 0.5,
8
+ 0.5,
9
+ 0.5
10
+ ],
11
+ "image_processor_type": "Qwen2VLImageProcessor",
12
+ "image_std": [
13
+ 0.5,
14
+ 0.5,
15
+ 0.5
16
+ ],
17
+ "merge_size": 2,
18
+ "patch_size": 16,
19
+ "resample": 3,
20
+ "rescale_factor": 0.00392156862745098,
21
+ "size": {
22
+ "longest_edge": 16777216,
23
+ "shortest_edge": 65536
24
+ },
25
+ "temporal_patch_size": 2
26
+ }
processor_config.json ADDED
@@ -0,0 +1,61 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "image_processor": {
3
+ "do_convert_rgb": true,
4
+ "do_normalize": true,
5
+ "do_rescale": true,
6
+ "do_resize": true,
7
+ "image_mean": [
8
+ 0.5,
9
+ 0.5,
10
+ 0.5
11
+ ],
12
+ "image_processor_type": "Qwen2VLImageProcessor",
13
+ "image_std": [
14
+ 0.5,
15
+ 0.5,
16
+ 0.5
17
+ ],
18
+ "merge_size": 2,
19
+ "patch_size": 16,
20
+ "resample": 3,
21
+ "rescale_factor": 0.00392156862745098,
22
+ "size": {
23
+ "longest_edge": 16777216,
24
+ "shortest_edge": 65536
25
+ },
26
+ "temporal_patch_size": 2
27
+ },
28
+ "processor_class": "Qwen3VLProcessor",
29
+ "video_processor": {
30
+ "do_convert_rgb": true,
31
+ "do_normalize": true,
32
+ "do_rescale": true,
33
+ "do_resize": true,
34
+ "do_sample_frames": true,
35
+ "fps": 2,
36
+ "image_mean": [
37
+ 0.5,
38
+ 0.5,
39
+ 0.5
40
+ ],
41
+ "image_std": [
42
+ 0.5,
43
+ 0.5,
44
+ 0.5
45
+ ],
46
+ "max_frames": 768,
47
+ "max_video_tokens": 768,
48
+ "merge_size": 2,
49
+ "min_frames": 4,
50
+ "patch_size": 16,
51
+ "resample": 3,
52
+ "rescale_factor": 0.00392156862745098,
53
+ "return_metadata": false,
54
+ "size": {
55
+ "longest_edge": 25165824,
56
+ "shortest_edge": 4096
57
+ },
58
+ "temporal_patch_size": 2,
59
+ "video_processor_type": "Qwen3VLVideoProcessor"
60
+ }
61
+ }
quantization_config.json ADDED
@@ -0,0 +1,3150 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bits": 4,
3
+ "data_type": "int",
4
+ "group_size": 128,
5
+ "sym": true,
6
+ "static_attention_granularity": "tensor",
7
+ "static_kv_granularity": "tensor",
8
+ "autoround_version": "0.15.0",
9
+ "block_name_to_quantize": "model.language_model.layers",
10
+ "quant_method": "auto-round",
11
+ "packing_format": "auto_round:auto_gptq",
12
+ "extra_config": {
13
+ "model.language_model.layers.0.linear_attn.in_proj_b": {
14
+ "bits": 16,
15
+ "data_type": "fp"
16
+ },
17
+ "model.language_model.layers.0.linear_attn.in_proj_a": {
18
+ "bits": 16,
19
+ "data_type": "fp"
20
+ },
21
+ "model.language_model.layers.0.mlp.shared_expert_gate": {
22
+ "bits": 16,
23
+ "data_type": "fp"
24
+ },
25
+ "model.language_model.layers.0.attn_hyper_connection.block_inject_weight": {
26
+ "bits": 16,
27
+ "data_type": "fp"
28
+ },
29
+ "model.language_model.layers.0.mlp_hyper_connection.block_inject_weight": {
30
+ "bits": 16,
31
+ "data_type": "fp"
32
+ },
33
+ "model.language_model.layers.1.linear_attn.in_proj_b": {
34
+ "bits": 16,
35
+ "data_type": "fp"
36
+ },
37
+ "model.language_model.layers.1.linear_attn.in_proj_a": {
38
+ "bits": 16,
39
+ "data_type": "fp"
40
+ },
41
+ "model.language_model.layers.1.mlp.shared_expert_gate": {
42
+ "bits": 16,
43
+ "data_type": "fp"
44
+ },
45
+ "model.language_model.layers.1.attn_hyper_connection.block_inject_weight": {
46
+ "bits": 16,
47
+ "data_type": "fp"
48
+ },
49
+ "model.language_model.layers.1.mlp_hyper_connection.block_inject_weight": {
50
+ "bits": 16,
51
+ "data_type": "fp"
52
+ },
53
+ "model.language_model.layers.2.linear_attn.in_proj_b": {
54
+ "bits": 16,
55
+ "data_type": "fp"
56
+ },
57
+ "model.language_model.layers.2.linear_attn.in_proj_a": {
58
+ "bits": 16,
59
+ "data_type": "fp"
60
+ },
61
+ "model.language_model.layers.2.mlp.shared_expert_gate": {
62
+ "bits": 16,
63
+ "data_type": "fp"
64
+ },
65
+ "model.language_model.layers.2.attn_hyper_connection.block_inject_weight": {
66
+ "bits": 16,
67
+ "data_type": "fp"
68
+ },
69
+ "model.language_model.layers.2.mlp_hyper_connection.block_inject_weight": {
70
+ "bits": 16,
71
+ "data_type": "fp"
72
+ },
73
+ "model.language_model.layers.3.mlp.shared_expert_gate": {
74
+ "bits": 16,
75
+ "data_type": "fp"
76
+ },
77
+ "model.language_model.layers.3.attn_hyper_connection.block_inject_weight": {
78
+ "bits": 16,
79
+ "data_type": "fp"
80
+ },
81
+ "model.language_model.layers.3.mlp_hyper_connection.block_inject_weight": {
82
+ "bits": 16,
83
+ "data_type": "fp"
84
+ },
85
+ "model.language_model.layers.4.linear_attn.in_proj_b": {
86
+ "bits": 16,
87
+ "data_type": "fp"
88
+ },
89
+ "model.language_model.layers.4.linear_attn.in_proj_a": {
90
+ "bits": 16,
91
+ "data_type": "fp"
92
+ },
93
+ "model.language_model.layers.4.mlp.shared_expert_gate": {
94
+ "bits": 16,
95
+ "data_type": "fp"
96
+ },
97
+ "model.language_model.layers.4.attn_hyper_connection.block_inject_weight": {
98
+ "bits": 16,
99
+ "data_type": "fp"
100
+ },
101
+ "model.language_model.layers.4.mlp_hyper_connection.block_inject_weight": {
102
+ "bits": 16,
103
+ "data_type": "fp"
104
+ },
105
+ "model.language_model.layers.5.linear_attn.in_proj_b": {
106
+ "bits": 16,
107
+ "data_type": "fp"
108
+ },
109
+ "model.language_model.layers.5.linear_attn.in_proj_a": {
110
+ "bits": 16,
111
+ "data_type": "fp"
112
+ },
113
+ "model.language_model.layers.5.mlp.shared_expert_gate": {
114
+ "bits": 16,
115
+ "data_type": "fp"
116
+ },
117
+ "model.language_model.layers.5.attn_hyper_connection.block_inject_weight": {
118
+ "bits": 16,
119
+ "data_type": "fp"
120
+ },
121
+ "model.language_model.layers.5.mlp_hyper_connection.block_inject_weight": {
122
+ "bits": 16,
123
+ "data_type": "fp"
124
+ },
125
+ "model.language_model.layers.6.linear_attn.in_proj_b": {
126
+ "bits": 16,
127
+ "data_type": "fp"
128
+ },
129
+ "model.language_model.layers.6.linear_attn.in_proj_a": {
130
+ "bits": 16,
131
+ "data_type": "fp"
132
+ },
133
+ "model.language_model.layers.6.mlp.shared_expert_gate": {
134
+ "bits": 16,
135
+ "data_type": "fp"
136
+ },
137
+ "model.language_model.layers.6.attn_hyper_connection.block_inject_weight": {
138
+ "bits": 16,
139
+ "data_type": "fp"
140
+ },
141
+ "model.language_model.layers.6.mlp_hyper_connection.block_inject_weight": {
142
+ "bits": 16,
143
+ "data_type": "fp"
144
+ },
145
+ "model.language_model.layers.7.mlp.shared_expert_gate": {
146
+ "bits": 16,
147
+ "data_type": "fp"
148
+ },
149
+ "model.language_model.layers.7.attn_hyper_connection.block_inject_weight": {
150
+ "bits": 16,
151
+ "data_type": "fp"
152
+ },
153
+ "model.language_model.layers.7.mlp_hyper_connection.block_inject_weight": {
154
+ "bits": 16,
155
+ "data_type": "fp"
156
+ },
157
+ "model.language_model.layers.8.linear_attn.in_proj_b": {
158
+ "bits": 16,
159
+ "data_type": "fp"
160
+ },
161
+ "model.language_model.layers.8.linear_attn.in_proj_a": {
162
+ "bits": 16,
163
+ "data_type": "fp"
164
+ },
165
+ "model.language_model.layers.8.mlp.shared_expert_gate": {
166
+ "bits": 16,
167
+ "data_type": "fp"
168
+ },
169
+ "model.language_model.layers.8.attn_hyper_connection.block_inject_weight": {
170
+ "bits": 16,
171
+ "data_type": "fp"
172
+ },
173
+ "model.language_model.layers.8.mlp_hyper_connection.block_inject_weight": {
174
+ "bits": 16,
175
+ "data_type": "fp"
176
+ },
177
+ "model.language_model.layers.9.linear_attn.in_proj_b": {
178
+ "bits": 16,
179
+ "data_type": "fp"
180
+ },
181
+ "model.language_model.layers.9.linear_attn.in_proj_a": {
182
+ "bits": 16,
183
+ "data_type": "fp"
184
+ },
185
+ "model.language_model.layers.9.mlp.shared_expert_gate": {
186
+ "bits": 16,
187
+ "data_type": "fp"
188
+ },
189
+ "model.language_model.layers.9.attn_hyper_connection.block_inject_weight": {
190
+ "bits": 16,
191
+ "data_type": "fp"
192
+ },
193
+ "model.language_model.layers.9.mlp_hyper_connection.block_inject_weight": {
194
+ "bits": 16,
195
+ "data_type": "fp"
196
+ },
197
+ "model.language_model.layers.10.linear_attn.in_proj_b": {
198
+ "bits": 16,
199
+ "data_type": "fp"
200
+ },
201
+ "model.language_model.layers.10.linear_attn.in_proj_a": {
202
+ "bits": 16,
203
+ "data_type": "fp"
204
+ },
205
+ "model.language_model.layers.10.mlp.shared_expert_gate": {
206
+ "bits": 16,
207
+ "data_type": "fp"
208
+ },
209
+ "model.language_model.layers.10.attn_hyper_connection.block_inject_weight": {
210
+ "bits": 16,
211
+ "data_type": "fp"
212
+ },
213
+ "model.language_model.layers.10.mlp_hyper_connection.block_inject_weight": {
214
+ "bits": 16,
215
+ "data_type": "fp"
216
+ },
217
+ "model.language_model.layers.11.mlp.shared_expert_gate": {
218
+ "bits": 16,
219
+ "data_type": "fp"
220
+ },
221
+ "model.language_model.layers.11.attn_hyper_connection.block_inject_weight": {
222
+ "bits": 16,
223
+ "data_type": "fp"
224
+ },
225
+ "model.language_model.layers.11.mlp_hyper_connection.block_inject_weight": {
226
+ "bits": 16,
227
+ "data_type": "fp"
228
+ },
229
+ "model.language_model.layers.12.linear_attn.in_proj_b": {
230
+ "bits": 16,
231
+ "data_type": "fp"
232
+ },
233
+ "model.language_model.layers.12.linear_attn.in_proj_a": {
234
+ "bits": 16,
235
+ "data_type": "fp"
236
+ },
237
+ "model.language_model.layers.12.mlp.shared_expert_gate": {
238
+ "bits": 16,
239
+ "data_type": "fp"
240
+ },
241
+ "model.language_model.layers.12.attn_hyper_connection.block_inject_weight": {
242
+ "bits": 16,
243
+ "data_type": "fp"
244
+ },
245
+ "model.language_model.layers.12.mlp_hyper_connection.block_inject_weight": {
246
+ "bits": 16,
247
+ "data_type": "fp"
248
+ },
249
+ "model.language_model.layers.13.linear_attn.in_proj_b": {
250
+ "bits": 16,
251
+ "data_type": "fp"
252
+ },
253
+ "model.language_model.layers.13.linear_attn.in_proj_a": {
254
+ "bits": 16,
255
+ "data_type": "fp"
256
+ },
257
+ "model.language_model.layers.13.mlp.shared_expert_gate": {
258
+ "bits": 16,
259
+ "data_type": "fp"
260
+ },
261
+ "model.language_model.layers.13.attn_hyper_connection.block_inject_weight": {
262
+ "bits": 16,
263
+ "data_type": "fp"
264
+ },
265
+ "model.language_model.layers.13.mlp_hyper_connection.block_inject_weight": {
266
+ "bits": 16,
267
+ "data_type": "fp"
268
+ },
269
+ "model.language_model.layers.14.linear_attn.in_proj_b": {
270
+ "bits": 16,
271
+ "data_type": "fp"
272
+ },
273
+ "model.language_model.layers.14.linear_attn.in_proj_a": {
274
+ "bits": 16,
275
+ "data_type": "fp"
276
+ },
277
+ "model.language_model.layers.14.mlp.shared_expert_gate": {
278
+ "bits": 16,
279
+ "data_type": "fp"
280
+ },
281
+ "model.language_model.layers.14.attn_hyper_connection.block_inject_weight": {
282
+ "bits": 16,
283
+ "data_type": "fp"
284
+ },
285
+ "model.language_model.layers.14.mlp_hyper_connection.block_inject_weight": {
286
+ "bits": 16,
287
+ "data_type": "fp"
288
+ },
289
+ "model.language_model.layers.15.mlp.shared_expert_gate": {
290
+ "bits": 16,
291
+ "data_type": "fp"
292
+ },
293
+ "model.language_model.layers.15.attn_hyper_connection.block_inject_weight": {
294
+ "bits": 16,
295
+ "data_type": "fp"
296
+ },
297
+ "model.language_model.layers.15.mlp_hyper_connection.block_inject_weight": {
298
+ "bits": 16,
299
+ "data_type": "fp"
300
+ },
301
+ "model.language_model.layers.16.linear_attn.in_proj_b": {
302
+ "bits": 16,
303
+ "data_type": "fp"
304
+ },
305
+ "model.language_model.layers.16.linear_attn.in_proj_a": {
306
+ "bits": 16,
307
+ "data_type": "fp"
308
+ },
309
+ "model.language_model.layers.16.mlp.shared_expert_gate": {
310
+ "bits": 16,
311
+ "data_type": "fp"
312
+ },
313
+ "model.language_model.layers.16.attn_hyper_connection.block_inject_weight": {
314
+ "bits": 16,
315
+ "data_type": "fp"
316
+ },
317
+ "model.language_model.layers.16.mlp_hyper_connection.block_inject_weight": {
318
+ "bits": 16,
319
+ "data_type": "fp"
320
+ },
321
+ "model.language_model.layers.17.linear_attn.in_proj_b": {
322
+ "bits": 16,
323
+ "data_type": "fp"
324
+ },
325
+ "model.language_model.layers.17.linear_attn.in_proj_a": {
326
+ "bits": 16,
327
+ "data_type": "fp"
328
+ },
329
+ "model.language_model.layers.17.mlp.shared_expert_gate": {
330
+ "bits": 16,
331
+ "data_type": "fp"
332
+ },
333
+ "model.language_model.layers.17.attn_hyper_connection.block_inject_weight": {
334
+ "bits": 16,
335
+ "data_type": "fp"
336
+ },
337
+ "model.language_model.layers.17.mlp_hyper_connection.block_inject_weight": {
338
+ "bits": 16,
339
+ "data_type": "fp"
340
+ },
341
+ "model.language_model.layers.18.linear_attn.in_proj_b": {
342
+ "bits": 16,
343
+ "data_type": "fp"
344
+ },
345
+ "model.language_model.layers.18.linear_attn.in_proj_a": {
346
+ "bits": 16,
347
+ "data_type": "fp"
348
+ },
349
+ "model.language_model.layers.18.mlp.shared_expert_gate": {
350
+ "bits": 16,
351
+ "data_type": "fp"
352
+ },
353
+ "model.language_model.layers.18.attn_hyper_connection.block_inject_weight": {
354
+ "bits": 16,
355
+ "data_type": "fp"
356
+ },
357
+ "model.language_model.layers.18.mlp_hyper_connection.block_inject_weight": {
358
+ "bits": 16,
359
+ "data_type": "fp"
360
+ },
361
+ "model.language_model.layers.19.mlp.shared_expert_gate": {
362
+ "bits": 16,
363
+ "data_type": "fp"
364
+ },
365
+ "model.language_model.layers.19.attn_hyper_connection.block_inject_weight": {
366
+ "bits": 16,
367
+ "data_type": "fp"
368
+ },
369
+ "model.language_model.layers.19.mlp_hyper_connection.block_inject_weight": {
370
+ "bits": 16,
371
+ "data_type": "fp"
372
+ },
373
+ "model.language_model.layers.20.linear_attn.in_proj_b": {
374
+ "bits": 16,
375
+ "data_type": "fp"
376
+ },
377
+ "model.language_model.layers.20.linear_attn.in_proj_a": {
378
+ "bits": 16,
379
+ "data_type": "fp"
380
+ },
381
+ "model.language_model.layers.20.mlp.shared_expert_gate": {
382
+ "bits": 16,
383
+ "data_type": "fp"
384
+ },
385
+ "model.language_model.layers.20.attn_hyper_connection.block_inject_weight": {
386
+ "bits": 16,
387
+ "data_type": "fp"
388
+ },
389
+ "model.language_model.layers.20.mlp_hyper_connection.block_inject_weight": {
390
+ "bits": 16,
391
+ "data_type": "fp"
392
+ },
393
+ "model.language_model.layers.21.linear_attn.in_proj_b": {
394
+ "bits": 16,
395
+ "data_type": "fp"
396
+ },
397
+ "model.language_model.layers.21.linear_attn.in_proj_a": {
398
+ "bits": 16,
399
+ "data_type": "fp"
400
+ },
401
+ "model.language_model.layers.21.mlp.shared_expert_gate": {
402
+ "bits": 16,
403
+ "data_type": "fp"
404
+ },
405
+ "model.language_model.layers.21.attn_hyper_connection.block_inject_weight": {
406
+ "bits": 16,
407
+ "data_type": "fp"
408
+ },
409
+ "model.language_model.layers.21.mlp_hyper_connection.block_inject_weight": {
410
+ "bits": 16,
411
+ "data_type": "fp"
412
+ },
413
+ "model.language_model.layers.22.linear_attn.in_proj_b": {
414
+ "bits": 16,
415
+ "data_type": "fp"
416
+ },
417
+ "model.language_model.layers.22.linear_attn.in_proj_a": {
418
+ "bits": 16,
419
+ "data_type": "fp"
420
+ },
421
+ "model.language_model.layers.22.mlp.shared_expert_gate": {
422
+ "bits": 16,
423
+ "data_type": "fp"
424
+ },
425
+ "model.language_model.layers.22.attn_hyper_connection.block_inject_weight": {
426
+ "bits": 16,
427
+ "data_type": "fp"
428
+ },
429
+ "model.language_model.layers.22.mlp_hyper_connection.block_inject_weight": {
430
+ "bits": 16,
431
+ "data_type": "fp"
432
+ },
433
+ "model.language_model.layers.23.mlp.shared_expert_gate": {
434
+ "bits": 16,
435
+ "data_type": "fp"
436
+ },
437
+ "model.language_model.layers.23.attn_hyper_connection.block_inject_weight": {
438
+ "bits": 16,
439
+ "data_type": "fp"
440
+ },
441
+ "model.language_model.layers.23.mlp_hyper_connection.block_inject_weight": {
442
+ "bits": 16,
443
+ "data_type": "fp"
444
+ },
445
+ "model.language_model.layers.24.linear_attn.in_proj_b": {
446
+ "bits": 16,
447
+ "data_type": "fp"
448
+ },
449
+ "model.language_model.layers.24.linear_attn.in_proj_a": {
450
+ "bits": 16,
451
+ "data_type": "fp"
452
+ },
453
+ "model.language_model.layers.24.mlp.shared_expert_gate": {
454
+ "bits": 16,
455
+ "data_type": "fp"
456
+ },
457
+ "model.language_model.layers.24.attn_hyper_connection.block_inject_weight": {
458
+ "bits": 16,
459
+ "data_type": "fp"
460
+ },
461
+ "model.language_model.layers.24.mlp_hyper_connection.block_inject_weight": {
462
+ "bits": 16,
463
+ "data_type": "fp"
464
+ },
465
+ "model.language_model.layers.25.linear_attn.in_proj_b": {
466
+ "bits": 16,
467
+ "data_type": "fp"
468
+ },
469
+ "model.language_model.layers.25.linear_attn.in_proj_a": {
470
+ "bits": 16,
471
+ "data_type": "fp"
472
+ },
473
+ "model.language_model.layers.25.mlp.shared_expert_gate": {
474
+ "bits": 16,
475
+ "data_type": "fp"
476
+ },
477
+ "model.language_model.layers.25.attn_hyper_connection.block_inject_weight": {
478
+ "bits": 16,
479
+ "data_type": "fp"
480
+ },
481
+ "model.language_model.layers.25.mlp_hyper_connection.block_inject_weight": {
482
+ "bits": 16,
483
+ "data_type": "fp"
484
+ },
485
+ "model.language_model.layers.26.linear_attn.in_proj_b": {
486
+ "bits": 16,
487
+ "data_type": "fp"
488
+ },
489
+ "model.language_model.layers.26.linear_attn.in_proj_a": {
490
+ "bits": 16,
491
+ "data_type": "fp"
492
+ },
493
+ "model.language_model.layers.26.mlp.shared_expert_gate": {
494
+ "bits": 16,
495
+ "data_type": "fp"
496
+ },
497
+ "model.language_model.layers.26.attn_hyper_connection.block_inject_weight": {
498
+ "bits": 16,
499
+ "data_type": "fp"
500
+ },
501
+ "model.language_model.layers.26.mlp_hyper_connection.block_inject_weight": {
502
+ "bits": 16,
503
+ "data_type": "fp"
504
+ },
505
+ "model.language_model.layers.27.mlp.shared_expert_gate": {
506
+ "bits": 16,
507
+ "data_type": "fp"
508
+ },
509
+ "model.language_model.layers.27.attn_hyper_connection.block_inject_weight": {
510
+ "bits": 16,
511
+ "data_type": "fp"
512
+ },
513
+ "model.language_model.layers.27.mlp_hyper_connection.block_inject_weight": {
514
+ "bits": 16,
515
+ "data_type": "fp"
516
+ },
517
+ "model.language_model.layers.28.linear_attn.in_proj_b": {
518
+ "bits": 16,
519
+ "data_type": "fp"
520
+ },
521
+ "model.language_model.layers.28.linear_attn.in_proj_a": {
522
+ "bits": 16,
523
+ "data_type": "fp"
524
+ },
525
+ "model.language_model.layers.28.mlp.shared_expert_gate": {
526
+ "bits": 16,
527
+ "data_type": "fp"
528
+ },
529
+ "model.language_model.layers.28.attn_hyper_connection.block_inject_weight": {
530
+ "bits": 16,
531
+ "data_type": "fp"
532
+ },
533
+ "model.language_model.layers.28.mlp_hyper_connection.block_inject_weight": {
534
+ "bits": 16,
535
+ "data_type": "fp"
536
+ },
537
+ "model.language_model.layers.29.linear_attn.in_proj_b": {
538
+ "bits": 16,
539
+ "data_type": "fp"
540
+ },
541
+ "model.language_model.layers.29.linear_attn.in_proj_a": {
542
+ "bits": 16,
543
+ "data_type": "fp"
544
+ },
545
+ "model.language_model.layers.29.mlp.shared_expert_gate": {
546
+ "bits": 16,
547
+ "data_type": "fp"
548
+ },
549
+ "model.language_model.layers.29.attn_hyper_connection.block_inject_weight": {
550
+ "bits": 16,
551
+ "data_type": "fp"
552
+ },
553
+ "model.language_model.layers.29.mlp_hyper_connection.block_inject_weight": {
554
+ "bits": 16,
555
+ "data_type": "fp"
556
+ },
557
+ "model.language_model.layers.30.linear_attn.in_proj_b": {
558
+ "bits": 16,
559
+ "data_type": "fp"
560
+ },
561
+ "model.language_model.layers.30.linear_attn.in_proj_a": {
562
+ "bits": 16,
563
+ "data_type": "fp"
564
+ },
565
+ "model.language_model.layers.30.mlp.shared_expert_gate": {
566
+ "bits": 16,
567
+ "data_type": "fp"
568
+ },
569
+ "model.language_model.layers.30.attn_hyper_connection.block_inject_weight": {
570
+ "bits": 16,
571
+ "data_type": "fp"
572
+ },
573
+ "model.language_model.layers.30.mlp_hyper_connection.block_inject_weight": {
574
+ "bits": 16,
575
+ "data_type": "fp"
576
+ },
577
+ "model.language_model.layers.31.mlp.shared_expert_gate": {
578
+ "bits": 16,
579
+ "data_type": "fp"
580
+ },
581
+ "model.language_model.layers.31.attn_hyper_connection.block_inject_weight": {
582
+ "bits": 16,
583
+ "data_type": "fp"
584
+ },
585
+ "model.language_model.layers.31.mlp_hyper_connection.block_inject_weight": {
586
+ "bits": 16,
587
+ "data_type": "fp"
588
+ },
589
+ "model.language_model.layers.32.linear_attn.in_proj_b": {
590
+ "bits": 16,
591
+ "data_type": "fp"
592
+ },
593
+ "model.language_model.layers.32.linear_attn.in_proj_a": {
594
+ "bits": 16,
595
+ "data_type": "fp"
596
+ },
597
+ "model.language_model.layers.32.mlp.shared_expert_gate": {
598
+ "bits": 16,
599
+ "data_type": "fp"
600
+ },
601
+ "model.language_model.layers.32.attn_hyper_connection.block_inject_weight": {
602
+ "bits": 16,
603
+ "data_type": "fp"
604
+ },
605
+ "model.language_model.layers.32.mlp_hyper_connection.block_inject_weight": {
606
+ "bits": 16,
607
+ "data_type": "fp"
608
+ },
609
+ "model.language_model.layers.33.linear_attn.in_proj_b": {
610
+ "bits": 16,
611
+ "data_type": "fp"
612
+ },
613
+ "model.language_model.layers.33.linear_attn.in_proj_a": {
614
+ "bits": 16,
615
+ "data_type": "fp"
616
+ },
617
+ "model.language_model.layers.33.mlp.shared_expert_gate": {
618
+ "bits": 16,
619
+ "data_type": "fp"
620
+ },
621
+ "model.language_model.layers.33.attn_hyper_connection.block_inject_weight": {
622
+ "bits": 16,
623
+ "data_type": "fp"
624
+ },
625
+ "model.language_model.layers.33.mlp_hyper_connection.block_inject_weight": {
626
+ "bits": 16,
627
+ "data_type": "fp"
628
+ },
629
+ "model.language_model.layers.34.linear_attn.in_proj_b": {
630
+ "bits": 16,
631
+ "data_type": "fp"
632
+ },
633
+ "model.language_model.layers.34.linear_attn.in_proj_a": {
634
+ "bits": 16,
635
+ "data_type": "fp"
636
+ },
637
+ "model.language_model.layers.34.mlp.shared_expert_gate": {
638
+ "bits": 16,
639
+ "data_type": "fp"
640
+ },
641
+ "model.language_model.layers.34.attn_hyper_connection.block_inject_weight": {
642
+ "bits": 16,
643
+ "data_type": "fp"
644
+ },
645
+ "model.language_model.layers.34.mlp_hyper_connection.block_inject_weight": {
646
+ "bits": 16,
647
+ "data_type": "fp"
648
+ },
649
+ "model.language_model.layers.35.mlp.shared_expert_gate": {
650
+ "bits": 16,
651
+ "data_type": "fp"
652
+ },
653
+ "model.language_model.layers.35.attn_hyper_connection.block_inject_weight": {
654
+ "bits": 16,
655
+ "data_type": "fp"
656
+ },
657
+ "model.language_model.layers.35.mlp_hyper_connection.block_inject_weight": {
658
+ "bits": 16,
659
+ "data_type": "fp"
660
+ },
661
+ "model.language_model.layers.36.linear_attn.in_proj_b": {
662
+ "bits": 16,
663
+ "data_type": "fp"
664
+ },
665
+ "model.language_model.layers.36.linear_attn.in_proj_a": {
666
+ "bits": 16,
667
+ "data_type": "fp"
668
+ },
669
+ "model.language_model.layers.36.mlp.shared_expert_gate": {
670
+ "bits": 16,
671
+ "data_type": "fp"
672
+ },
673
+ "model.language_model.layers.36.attn_hyper_connection.block_inject_weight": {
674
+ "bits": 16,
675
+ "data_type": "fp"
676
+ },
677
+ "model.language_model.layers.36.mlp_hyper_connection.block_inject_weight": {
678
+ "bits": 16,
679
+ "data_type": "fp"
680
+ },
681
+ "model.language_model.layers.37.linear_attn.in_proj_b": {
682
+ "bits": 16,
683
+ "data_type": "fp"
684
+ },
685
+ "model.language_model.layers.37.linear_attn.in_proj_a": {
686
+ "bits": 16,
687
+ "data_type": "fp"
688
+ },
689
+ "model.language_model.layers.37.mlp.shared_expert_gate": {
690
+ "bits": 16,
691
+ "data_type": "fp"
692
+ },
693
+ "model.language_model.layers.37.attn_hyper_connection.block_inject_weight": {
694
+ "bits": 16,
695
+ "data_type": "fp"
696
+ },
697
+ "model.language_model.layers.37.mlp_hyper_connection.block_inject_weight": {
698
+ "bits": 16,
699
+ "data_type": "fp"
700
+ },
701
+ "model.language_model.layers.38.linear_attn.in_proj_b": {
702
+ "bits": 16,
703
+ "data_type": "fp"
704
+ },
705
+ "model.language_model.layers.38.linear_attn.in_proj_a": {
706
+ "bits": 16,
707
+ "data_type": "fp"
708
+ },
709
+ "model.language_model.layers.38.mlp.shared_expert_gate": {
710
+ "bits": 16,
711
+ "data_type": "fp"
712
+ },
713
+ "model.language_model.layers.38.attn_hyper_connection.block_inject_weight": {
714
+ "bits": 16,
715
+ "data_type": "fp"
716
+ },
717
+ "model.language_model.layers.38.mlp_hyper_connection.block_inject_weight": {
718
+ "bits": 16,
719
+ "data_type": "fp"
720
+ },
721
+ "model.language_model.layers.39.mlp.shared_expert_gate": {
722
+ "bits": 16,
723
+ "data_type": "fp"
724
+ },
725
+ "model.language_model.layers.39.attn_hyper_connection.block_inject_weight": {
726
+ "bits": 16,
727
+ "data_type": "fp"
728
+ },
729
+ "model.language_model.layers.39.mlp_hyper_connection.block_inject_weight": {
730
+ "bits": 16,
731
+ "data_type": "fp"
732
+ },
733
+ "model.language_model.layers.40.linear_attn.in_proj_b": {
734
+ "bits": 16,
735
+ "data_type": "fp"
736
+ },
737
+ "model.language_model.layers.40.linear_attn.in_proj_a": {
738
+ "bits": 16,
739
+ "data_type": "fp"
740
+ },
741
+ "model.language_model.layers.40.mlp.shared_expert_gate": {
742
+ "bits": 16,
743
+ "data_type": "fp"
744
+ },
745
+ "model.language_model.layers.40.attn_hyper_connection.block_inject_weight": {
746
+ "bits": 16,
747
+ "data_type": "fp"
748
+ },
749
+ "model.language_model.layers.40.mlp_hyper_connection.block_inject_weight": {
750
+ "bits": 16,
751
+ "data_type": "fp"
752
+ },
753
+ "model.language_model.layers.41.linear_attn.in_proj_b": {
754
+ "bits": 16,
755
+ "data_type": "fp"
756
+ },
757
+ "model.language_model.layers.41.linear_attn.in_proj_a": {
758
+ "bits": 16,
759
+ "data_type": "fp"
760
+ },
761
+ "model.language_model.layers.41.mlp.shared_expert_gate": {
762
+ "bits": 16,
763
+ "data_type": "fp"
764
+ },
765
+ "model.language_model.layers.41.attn_hyper_connection.block_inject_weight": {
766
+ "bits": 16,
767
+ "data_type": "fp"
768
+ },
769
+ "model.language_model.layers.41.mlp_hyper_connection.block_inject_weight": {
770
+ "bits": 16,
771
+ "data_type": "fp"
772
+ },
773
+ "model.language_model.layers.42.linear_attn.in_proj_b": {
774
+ "bits": 16,
775
+ "data_type": "fp"
776
+ },
777
+ "model.language_model.layers.42.linear_attn.in_proj_a": {
778
+ "bits": 16,
779
+ "data_type": "fp"
780
+ },
781
+ "model.language_model.layers.42.mlp.shared_expert_gate": {
782
+ "bits": 16,
783
+ "data_type": "fp"
784
+ },
785
+ "model.language_model.layers.42.attn_hyper_connection.block_inject_weight": {
786
+ "bits": 16,
787
+ "data_type": "fp"
788
+ },
789
+ "model.language_model.layers.42.mlp_hyper_connection.block_inject_weight": {
790
+ "bits": 16,
791
+ "data_type": "fp"
792
+ },
793
+ "model.language_model.layers.43.mlp.shared_expert_gate": {
794
+ "bits": 16,
795
+ "data_type": "fp"
796
+ },
797
+ "model.language_model.layers.43.attn_hyper_connection.block_inject_weight": {
798
+ "bits": 16,
799
+ "data_type": "fp"
800
+ },
801
+ "model.language_model.layers.43.mlp_hyper_connection.block_inject_weight": {
802
+ "bits": 16,
803
+ "data_type": "fp"
804
+ },
805
+ "model.language_model.layers.44.linear_attn.in_proj_b": {
806
+ "bits": 16,
807
+ "data_type": "fp"
808
+ },
809
+ "model.language_model.layers.44.linear_attn.in_proj_a": {
810
+ "bits": 16,
811
+ "data_type": "fp"
812
+ },
813
+ "model.language_model.layers.44.mlp.shared_expert_gate": {
814
+ "bits": 16,
815
+ "data_type": "fp"
816
+ },
817
+ "model.language_model.layers.44.attn_hyper_connection.block_inject_weight": {
818
+ "bits": 16,
819
+ "data_type": "fp"
820
+ },
821
+ "model.language_model.layers.44.mlp_hyper_connection.block_inject_weight": {
822
+ "bits": 16,
823
+ "data_type": "fp"
824
+ },
825
+ "model.language_model.layers.45.linear_attn.in_proj_b": {
826
+ "bits": 16,
827
+ "data_type": "fp"
828
+ },
829
+ "model.language_model.layers.45.linear_attn.in_proj_a": {
830
+ "bits": 16,
831
+ "data_type": "fp"
832
+ },
833
+ "model.language_model.layers.45.mlp.shared_expert_gate": {
834
+ "bits": 16,
835
+ "data_type": "fp"
836
+ },
837
+ "model.language_model.layers.45.attn_hyper_connection.block_inject_weight": {
838
+ "bits": 16,
839
+ "data_type": "fp"
840
+ },
841
+ "model.language_model.layers.45.mlp_hyper_connection.block_inject_weight": {
842
+ "bits": 16,
843
+ "data_type": "fp"
844
+ },
845
+ "model.language_model.layers.46.linear_attn.in_proj_b": {
846
+ "bits": 16,
847
+ "data_type": "fp"
848
+ },
849
+ "model.language_model.layers.46.linear_attn.in_proj_a": {
850
+ "bits": 16,
851
+ "data_type": "fp"
852
+ },
853
+ "model.language_model.layers.46.mlp.shared_expert_gate": {
854
+ "bits": 16,
855
+ "data_type": "fp"
856
+ },
857
+ "model.language_model.layers.46.attn_hyper_connection.block_inject_weight": {
858
+ "bits": 16,
859
+ "data_type": "fp"
860
+ },
861
+ "model.language_model.layers.46.mlp_hyper_connection.block_inject_weight": {
862
+ "bits": 16,
863
+ "data_type": "fp"
864
+ },
865
+ "model.language_model.layers.47.mlp.shared_expert_gate": {
866
+ "bits": 16,
867
+ "data_type": "fp"
868
+ },
869
+ "model.language_model.layers.47.attn_hyper_connection.block_inject_weight": {
870
+ "bits": 16,
871
+ "data_type": "fp"
872
+ },
873
+ "model.language_model.layers.47.mlp_hyper_connection.block_inject_weight": {
874
+ "bits": 16,
875
+ "data_type": "fp"
876
+ },
877
+ "model.language_model.layers.0.linear_attn.out_proj": {
878
+ "bits": 16,
879
+ "data_type": "float"
880
+ },
881
+ "model.language_model.layers.0.linear_attn.in_proj_qkv": {
882
+ "bits": 16,
883
+ "data_type": "float"
884
+ },
885
+ "model.language_model.layers.0.linear_attn.in_proj_z": {
886
+ "bits": 16,
887
+ "data_type": "float"
888
+ },
889
+ "model.language_model.layers.1.linear_attn.out_proj": {
890
+ "bits": 16,
891
+ "data_type": "float"
892
+ },
893
+ "model.language_model.layers.1.linear_attn.in_proj_qkv": {
894
+ "bits": 16,
895
+ "data_type": "float"
896
+ },
897
+ "model.language_model.layers.1.linear_attn.in_proj_z": {
898
+ "bits": 16,
899
+ "data_type": "float"
900
+ },
901
+ "model.language_model.layers.2.linear_attn.out_proj": {
902
+ "bits": 16,
903
+ "data_type": "float"
904
+ },
905
+ "model.language_model.layers.2.linear_attn.in_proj_qkv": {
906
+ "bits": 16,
907
+ "data_type": "float"
908
+ },
909
+ "model.language_model.layers.2.linear_attn.in_proj_z": {
910
+ "bits": 16,
911
+ "data_type": "float"
912
+ },
913
+ "model.language_model.layers.4.linear_attn.out_proj": {
914
+ "bits": 16,
915
+ "data_type": "float"
916
+ },
917
+ "model.language_model.layers.4.linear_attn.in_proj_qkv": {
918
+ "bits": 16,
919
+ "data_type": "float"
920
+ },
921
+ "model.language_model.layers.4.linear_attn.in_proj_z": {
922
+ "bits": 16,
923
+ "data_type": "float"
924
+ },
925
+ "model.language_model.layers.5.linear_attn.out_proj": {
926
+ "bits": 16,
927
+ "data_type": "float"
928
+ },
929
+ "model.language_model.layers.5.linear_attn.in_proj_qkv": {
930
+ "bits": 16,
931
+ "data_type": "float"
932
+ },
933
+ "model.language_model.layers.5.linear_attn.in_proj_z": {
934
+ "bits": 16,
935
+ "data_type": "float"
936
+ },
937
+ "model.language_model.layers.6.linear_attn.out_proj": {
938
+ "bits": 16,
939
+ "data_type": "float"
940
+ },
941
+ "model.language_model.layers.6.linear_attn.in_proj_qkv": {
942
+ "bits": 16,
943
+ "data_type": "float"
944
+ },
945
+ "model.language_model.layers.6.linear_attn.in_proj_z": {
946
+ "bits": 16,
947
+ "data_type": "float"
948
+ },
949
+ "model.language_model.layers.8.linear_attn.out_proj": {
950
+ "bits": 16,
951
+ "data_type": "float"
952
+ },
953
+ "model.language_model.layers.8.linear_attn.in_proj_qkv": {
954
+ "bits": 16,
955
+ "data_type": "float"
956
+ },
957
+ "model.language_model.layers.8.linear_attn.in_proj_z": {
958
+ "bits": 16,
959
+ "data_type": "float"
960
+ },
961
+ "model.language_model.layers.9.linear_attn.out_proj": {
962
+ "bits": 16,
963
+ "data_type": "float"
964
+ },
965
+ "model.language_model.layers.9.linear_attn.in_proj_qkv": {
966
+ "bits": 16,
967
+ "data_type": "float"
968
+ },
969
+ "model.language_model.layers.9.linear_attn.in_proj_z": {
970
+ "bits": 16,
971
+ "data_type": "float"
972
+ },
973
+ "model.language_model.layers.10.linear_attn.out_proj": {
974
+ "bits": 16,
975
+ "data_type": "float"
976
+ },
977
+ "model.language_model.layers.10.linear_attn.in_proj_qkv": {
978
+ "bits": 16,
979
+ "data_type": "float"
980
+ },
981
+ "model.language_model.layers.10.linear_attn.in_proj_z": {
982
+ "bits": 16,
983
+ "data_type": "float"
984
+ },
985
+ "model.language_model.layers.12.linear_attn.out_proj": {
986
+ "bits": 16,
987
+ "data_type": "float"
988
+ },
989
+ "model.language_model.layers.12.linear_attn.in_proj_qkv": {
990
+ "bits": 16,
991
+ "data_type": "float"
992
+ },
993
+ "model.language_model.layers.12.linear_attn.in_proj_z": {
994
+ "bits": 16,
995
+ "data_type": "float"
996
+ },
997
+ "model.language_model.layers.13.linear_attn.out_proj": {
998
+ "bits": 16,
999
+ "data_type": "float"
1000
+ },
1001
+ "model.language_model.layers.13.linear_attn.in_proj_qkv": {
1002
+ "bits": 16,
1003
+ "data_type": "float"
1004
+ },
1005
+ "model.language_model.layers.13.linear_attn.in_proj_z": {
1006
+ "bits": 16,
1007
+ "data_type": "float"
1008
+ },
1009
+ "model.language_model.layers.14.linear_attn.out_proj": {
1010
+ "bits": 16,
1011
+ "data_type": "float"
1012
+ },
1013
+ "model.language_model.layers.14.linear_attn.in_proj_qkv": {
1014
+ "bits": 16,
1015
+ "data_type": "float"
1016
+ },
1017
+ "model.language_model.layers.14.linear_attn.in_proj_z": {
1018
+ "bits": 16,
1019
+ "data_type": "float"
1020
+ },
1021
+ "model.language_model.layers.16.linear_attn.out_proj": {
1022
+ "bits": 16,
1023
+ "data_type": "float"
1024
+ },
1025
+ "model.language_model.layers.16.linear_attn.in_proj_qkv": {
1026
+ "bits": 16,
1027
+ "data_type": "float"
1028
+ },
1029
+ "model.language_model.layers.16.linear_attn.in_proj_z": {
1030
+ "bits": 16,
1031
+ "data_type": "float"
1032
+ },
1033
+ "model.language_model.layers.17.linear_attn.out_proj": {
1034
+ "bits": 16,
1035
+ "data_type": "float"
1036
+ },
1037
+ "model.language_model.layers.17.linear_attn.in_proj_qkv": {
1038
+ "bits": 16,
1039
+ "data_type": "float"
1040
+ },
1041
+ "model.language_model.layers.17.linear_attn.in_proj_z": {
1042
+ "bits": 16,
1043
+ "data_type": "float"
1044
+ },
1045
+ "model.language_model.layers.18.linear_attn.out_proj": {
1046
+ "bits": 16,
1047
+ "data_type": "float"
1048
+ },
1049
+ "model.language_model.layers.18.linear_attn.in_proj_qkv": {
1050
+ "bits": 16,
1051
+ "data_type": "float"
1052
+ },
1053
+ "model.language_model.layers.18.linear_attn.in_proj_z": {
1054
+ "bits": 16,
1055
+ "data_type": "float"
1056
+ },
1057
+ "model.language_model.layers.20.linear_attn.out_proj": {
1058
+ "bits": 16,
1059
+ "data_type": "float"
1060
+ },
1061
+ "model.language_model.layers.20.linear_attn.in_proj_qkv": {
1062
+ "bits": 16,
1063
+ "data_type": "float"
1064
+ },
1065
+ "model.language_model.layers.20.linear_attn.in_proj_z": {
1066
+ "bits": 16,
1067
+ "data_type": "float"
1068
+ },
1069
+ "model.language_model.layers.21.linear_attn.out_proj": {
1070
+ "bits": 16,
1071
+ "data_type": "float"
1072
+ },
1073
+ "model.language_model.layers.21.linear_attn.in_proj_qkv": {
1074
+ "bits": 16,
1075
+ "data_type": "float"
1076
+ },
1077
+ "model.language_model.layers.21.linear_attn.in_proj_z": {
1078
+ "bits": 16,
1079
+ "data_type": "float"
1080
+ },
1081
+ "model.language_model.layers.22.linear_attn.out_proj": {
1082
+ "bits": 16,
1083
+ "data_type": "float"
1084
+ },
1085
+ "model.language_model.layers.22.linear_attn.in_proj_qkv": {
1086
+ "bits": 16,
1087
+ "data_type": "float"
1088
+ },
1089
+ "model.language_model.layers.22.linear_attn.in_proj_z": {
1090
+ "bits": 16,
1091
+ "data_type": "float"
1092
+ },
1093
+ "model.language_model.layers.24.linear_attn.out_proj": {
1094
+ "bits": 16,
1095
+ "data_type": "float"
1096
+ },
1097
+ "model.language_model.layers.24.linear_attn.in_proj_qkv": {
1098
+ "bits": 16,
1099
+ "data_type": "float"
1100
+ },
1101
+ "model.language_model.layers.24.linear_attn.in_proj_z": {
1102
+ "bits": 16,
1103
+ "data_type": "float"
1104
+ },
1105
+ "model.language_model.layers.25.linear_attn.out_proj": {
1106
+ "bits": 16,
1107
+ "data_type": "float"
1108
+ },
1109
+ "model.language_model.layers.25.linear_attn.in_proj_qkv": {
1110
+ "bits": 16,
1111
+ "data_type": "float"
1112
+ },
1113
+ "model.language_model.layers.25.linear_attn.in_proj_z": {
1114
+ "bits": 16,
1115
+ "data_type": "float"
1116
+ },
1117
+ "model.language_model.layers.26.linear_attn.out_proj": {
1118
+ "bits": 16,
1119
+ "data_type": "float"
1120
+ },
1121
+ "model.language_model.layers.26.linear_attn.in_proj_qkv": {
1122
+ "bits": 16,
1123
+ "data_type": "float"
1124
+ },
1125
+ "model.language_model.layers.26.linear_attn.in_proj_z": {
1126
+ "bits": 16,
1127
+ "data_type": "float"
1128
+ },
1129
+ "model.language_model.layers.28.linear_attn.out_proj": {
1130
+ "bits": 16,
1131
+ "data_type": "float"
1132
+ },
1133
+ "model.language_model.layers.28.linear_attn.in_proj_qkv": {
1134
+ "bits": 16,
1135
+ "data_type": "float"
1136
+ },
1137
+ "model.language_model.layers.28.linear_attn.in_proj_z": {
1138
+ "bits": 16,
1139
+ "data_type": "float"
1140
+ },
1141
+ "model.language_model.layers.29.linear_attn.out_proj": {
1142
+ "bits": 16,
1143
+ "data_type": "float"
1144
+ },
1145
+ "model.language_model.layers.29.linear_attn.in_proj_qkv": {
1146
+ "bits": 16,
1147
+ "data_type": "float"
1148
+ },
1149
+ "model.language_model.layers.29.linear_attn.in_proj_z": {
1150
+ "bits": 16,
1151
+ "data_type": "float"
1152
+ },
1153
+ "model.language_model.layers.30.linear_attn.out_proj": {
1154
+ "bits": 16,
1155
+ "data_type": "float"
1156
+ },
1157
+ "model.language_model.layers.30.linear_attn.in_proj_qkv": {
1158
+ "bits": 16,
1159
+ "data_type": "float"
1160
+ },
1161
+ "model.language_model.layers.30.linear_attn.in_proj_z": {
1162
+ "bits": 16,
1163
+ "data_type": "float"
1164
+ },
1165
+ "model.language_model.layers.32.linear_attn.out_proj": {
1166
+ "bits": 16,
1167
+ "data_type": "float"
1168
+ },
1169
+ "model.language_model.layers.32.linear_attn.in_proj_qkv": {
1170
+ "bits": 16,
1171
+ "data_type": "float"
1172
+ },
1173
+ "model.language_model.layers.32.linear_attn.in_proj_z": {
1174
+ "bits": 16,
1175
+ "data_type": "float"
1176
+ },
1177
+ "model.language_model.layers.33.linear_attn.out_proj": {
1178
+ "bits": 16,
1179
+ "data_type": "float"
1180
+ },
1181
+ "model.language_model.layers.33.linear_attn.in_proj_qkv": {
1182
+ "bits": 16,
1183
+ "data_type": "float"
1184
+ },
1185
+ "model.language_model.layers.33.linear_attn.in_proj_z": {
1186
+ "bits": 16,
1187
+ "data_type": "float"
1188
+ },
1189
+ "model.language_model.layers.34.linear_attn.out_proj": {
1190
+ "bits": 16,
1191
+ "data_type": "float"
1192
+ },
1193
+ "model.language_model.layers.34.linear_attn.in_proj_qkv": {
1194
+ "bits": 16,
1195
+ "data_type": "float"
1196
+ },
1197
+ "model.language_model.layers.34.linear_attn.in_proj_z": {
1198
+ "bits": 16,
1199
+ "data_type": "float"
1200
+ },
1201
+ "model.language_model.layers.36.linear_attn.out_proj": {
1202
+ "bits": 16,
1203
+ "data_type": "float"
1204
+ },
1205
+ "model.language_model.layers.36.linear_attn.in_proj_qkv": {
1206
+ "bits": 16,
1207
+ "data_type": "float"
1208
+ },
1209
+ "model.language_model.layers.36.linear_attn.in_proj_z": {
1210
+ "bits": 16,
1211
+ "data_type": "float"
1212
+ },
1213
+ "model.language_model.layers.37.linear_attn.out_proj": {
1214
+ "bits": 16,
1215
+ "data_type": "float"
1216
+ },
1217
+ "model.language_model.layers.37.linear_attn.in_proj_qkv": {
1218
+ "bits": 16,
1219
+ "data_type": "float"
1220
+ },
1221
+ "model.language_model.layers.37.linear_attn.in_proj_z": {
1222
+ "bits": 16,
1223
+ "data_type": "float"
1224
+ },
1225
+ "model.language_model.layers.38.linear_attn.out_proj": {
1226
+ "bits": 16,
1227
+ "data_type": "float"
1228
+ },
1229
+ "model.language_model.layers.38.linear_attn.in_proj_qkv": {
1230
+ "bits": 16,
1231
+ "data_type": "float"
1232
+ },
1233
+ "model.language_model.layers.38.linear_attn.in_proj_z": {
1234
+ "bits": 16,
1235
+ "data_type": "float"
1236
+ },
1237
+ "model.language_model.layers.40.linear_attn.out_proj": {
1238
+ "bits": 16,
1239
+ "data_type": "float"
1240
+ },
1241
+ "model.language_model.layers.40.linear_attn.in_proj_qkv": {
1242
+ "bits": 16,
1243
+ "data_type": "float"
1244
+ },
1245
+ "model.language_model.layers.40.linear_attn.in_proj_z": {
1246
+ "bits": 16,
1247
+ "data_type": "float"
1248
+ },
1249
+ "model.language_model.layers.41.linear_attn.out_proj": {
1250
+ "bits": 16,
1251
+ "data_type": "float"
1252
+ },
1253
+ "model.language_model.layers.41.linear_attn.in_proj_qkv": {
1254
+ "bits": 16,
1255
+ "data_type": "float"
1256
+ },
1257
+ "model.language_model.layers.41.linear_attn.in_proj_z": {
1258
+ "bits": 16,
1259
+ "data_type": "float"
1260
+ },
1261
+ "model.language_model.layers.42.linear_attn.out_proj": {
1262
+ "bits": 16,
1263
+ "data_type": "float"
1264
+ },
1265
+ "model.language_model.layers.42.linear_attn.in_proj_qkv": {
1266
+ "bits": 16,
1267
+ "data_type": "float"
1268
+ },
1269
+ "model.language_model.layers.42.linear_attn.in_proj_z": {
1270
+ "bits": 16,
1271
+ "data_type": "float"
1272
+ },
1273
+ "model.language_model.layers.44.linear_attn.out_proj": {
1274
+ "bits": 16,
1275
+ "data_type": "float"
1276
+ },
1277
+ "model.language_model.layers.44.linear_attn.in_proj_qkv": {
1278
+ "bits": 16,
1279
+ "data_type": "float"
1280
+ },
1281
+ "model.language_model.layers.44.linear_attn.in_proj_z": {
1282
+ "bits": 16,
1283
+ "data_type": "float"
1284
+ },
1285
+ "model.language_model.layers.45.linear_attn.out_proj": {
1286
+ "bits": 16,
1287
+ "data_type": "float"
1288
+ },
1289
+ "model.language_model.layers.45.linear_attn.in_proj_qkv": {
1290
+ "bits": 16,
1291
+ "data_type": "float"
1292
+ },
1293
+ "model.language_model.layers.45.linear_attn.in_proj_z": {
1294
+ "bits": 16,
1295
+ "data_type": "float"
1296
+ },
1297
+ "model.language_model.layers.46.linear_attn.out_proj": {
1298
+ "bits": 16,
1299
+ "data_type": "float"
1300
+ },
1301
+ "model.language_model.layers.46.linear_attn.in_proj_qkv": {
1302
+ "bits": 16,
1303
+ "data_type": "float"
1304
+ },
1305
+ "model.language_model.layers.46.linear_attn.in_proj_z": {
1306
+ "bits": 16,
1307
+ "data_type": "float"
1308
+ },
1309
+ "model.language_model.layers.3.self_attn.q_proj": {
1310
+ "bits": 16,
1311
+ "data_type": "float"
1312
+ },
1313
+ "model.language_model.layers.3.self_attn.k_proj": {
1314
+ "bits": 16,
1315
+ "data_type": "float"
1316
+ },
1317
+ "model.language_model.layers.3.self_attn.v_proj": {
1318
+ "bits": 16,
1319
+ "data_type": "float"
1320
+ },
1321
+ "model.language_model.layers.3.self_attn.o_proj": {
1322
+ "bits": 16,
1323
+ "data_type": "float"
1324
+ },
1325
+ "model.language_model.layers.3.self_attn.indexer.index_qk_proj": {
1326
+ "bits": 16,
1327
+ "data_type": "float"
1328
+ },
1329
+ "model.language_model.layers.7.self_attn.q_proj": {
1330
+ "bits": 16,
1331
+ "data_type": "float"
1332
+ },
1333
+ "model.language_model.layers.7.self_attn.k_proj": {
1334
+ "bits": 16,
1335
+ "data_type": "float"
1336
+ },
1337
+ "model.language_model.layers.7.self_attn.v_proj": {
1338
+ "bits": 16,
1339
+ "data_type": "float"
1340
+ },
1341
+ "model.language_model.layers.7.self_attn.o_proj": {
1342
+ "bits": 16,
1343
+ "data_type": "float"
1344
+ },
1345
+ "model.language_model.layers.7.self_attn.indexer.index_qk_proj": {
1346
+ "bits": 16,
1347
+ "data_type": "float"
1348
+ },
1349
+ "model.language_model.layers.11.self_attn.q_proj": {
1350
+ "bits": 16,
1351
+ "data_type": "float"
1352
+ },
1353
+ "model.language_model.layers.11.self_attn.k_proj": {
1354
+ "bits": 16,
1355
+ "data_type": "float"
1356
+ },
1357
+ "model.language_model.layers.11.self_attn.v_proj": {
1358
+ "bits": 16,
1359
+ "data_type": "float"
1360
+ },
1361
+ "model.language_model.layers.11.self_attn.o_proj": {
1362
+ "bits": 16,
1363
+ "data_type": "float"
1364
+ },
1365
+ "model.language_model.layers.11.self_attn.indexer.index_qk_proj": {
1366
+ "bits": 16,
1367
+ "data_type": "float"
1368
+ },
1369
+ "model.language_model.layers.15.self_attn.q_proj": {
1370
+ "bits": 16,
1371
+ "data_type": "float"
1372
+ },
1373
+ "model.language_model.layers.15.self_attn.k_proj": {
1374
+ "bits": 16,
1375
+ "data_type": "float"
1376
+ },
1377
+ "model.language_model.layers.15.self_attn.v_proj": {
1378
+ "bits": 16,
1379
+ "data_type": "float"
1380
+ },
1381
+ "model.language_model.layers.15.self_attn.o_proj": {
1382
+ "bits": 16,
1383
+ "data_type": "float"
1384
+ },
1385
+ "model.language_model.layers.15.self_attn.indexer.index_qk_proj": {
1386
+ "bits": 16,
1387
+ "data_type": "float"
1388
+ },
1389
+ "model.language_model.layers.19.self_attn.q_proj": {
1390
+ "bits": 16,
1391
+ "data_type": "float"
1392
+ },
1393
+ "model.language_model.layers.19.self_attn.k_proj": {
1394
+ "bits": 16,
1395
+ "data_type": "float"
1396
+ },
1397
+ "model.language_model.layers.19.self_attn.v_proj": {
1398
+ "bits": 16,
1399
+ "data_type": "float"
1400
+ },
1401
+ "model.language_model.layers.19.self_attn.o_proj": {
1402
+ "bits": 16,
1403
+ "data_type": "float"
1404
+ },
1405
+ "model.language_model.layers.19.self_attn.indexer.index_qk_proj": {
1406
+ "bits": 16,
1407
+ "data_type": "float"
1408
+ },
1409
+ "model.language_model.layers.23.self_attn.q_proj": {
1410
+ "bits": 16,
1411
+ "data_type": "float"
1412
+ },
1413
+ "model.language_model.layers.23.self_attn.k_proj": {
1414
+ "bits": 16,
1415
+ "data_type": "float"
1416
+ },
1417
+ "model.language_model.layers.23.self_attn.v_proj": {
1418
+ "bits": 16,
1419
+ "data_type": "float"
1420
+ },
1421
+ "model.language_model.layers.23.self_attn.o_proj": {
1422
+ "bits": 16,
1423
+ "data_type": "float"
1424
+ },
1425
+ "model.language_model.layers.23.self_attn.indexer.index_qk_proj": {
1426
+ "bits": 16,
1427
+ "data_type": "float"
1428
+ },
1429
+ "model.language_model.layers.27.self_attn.q_proj": {
1430
+ "bits": 16,
1431
+ "data_type": "float"
1432
+ },
1433
+ "model.language_model.layers.27.self_attn.k_proj": {
1434
+ "bits": 16,
1435
+ "data_type": "float"
1436
+ },
1437
+ "model.language_model.layers.27.self_attn.v_proj": {
1438
+ "bits": 16,
1439
+ "data_type": "float"
1440
+ },
1441
+ "model.language_model.layers.27.self_attn.o_proj": {
1442
+ "bits": 16,
1443
+ "data_type": "float"
1444
+ },
1445
+ "model.language_model.layers.27.self_attn.indexer.index_qk_proj": {
1446
+ "bits": 16,
1447
+ "data_type": "float"
1448
+ },
1449
+ "model.language_model.layers.31.self_attn.q_proj": {
1450
+ "bits": 16,
1451
+ "data_type": "float"
1452
+ },
1453
+ "model.language_model.layers.31.self_attn.k_proj": {
1454
+ "bits": 16,
1455
+ "data_type": "float"
1456
+ },
1457
+ "model.language_model.layers.31.self_attn.v_proj": {
1458
+ "bits": 16,
1459
+ "data_type": "float"
1460
+ },
1461
+ "model.language_model.layers.31.self_attn.o_proj": {
1462
+ "bits": 16,
1463
+ "data_type": "float"
1464
+ },
1465
+ "model.language_model.layers.31.self_attn.indexer.index_qk_proj": {
1466
+ "bits": 16,
1467
+ "data_type": "float"
1468
+ },
1469
+ "model.language_model.layers.35.self_attn.q_proj": {
1470
+ "bits": 16,
1471
+ "data_type": "float"
1472
+ },
1473
+ "model.language_model.layers.35.self_attn.k_proj": {
1474
+ "bits": 16,
1475
+ "data_type": "float"
1476
+ },
1477
+ "model.language_model.layers.35.self_attn.v_proj": {
1478
+ "bits": 16,
1479
+ "data_type": "float"
1480
+ },
1481
+ "model.language_model.layers.35.self_attn.o_proj": {
1482
+ "bits": 16,
1483
+ "data_type": "float"
1484
+ },
1485
+ "model.language_model.layers.35.self_attn.indexer.index_qk_proj": {
1486
+ "bits": 16,
1487
+ "data_type": "float"
1488
+ },
1489
+ "model.language_model.layers.39.self_attn.q_proj": {
1490
+ "bits": 16,
1491
+ "data_type": "float"
1492
+ },
1493
+ "model.language_model.layers.39.self_attn.k_proj": {
1494
+ "bits": 16,
1495
+ "data_type": "float"
1496
+ },
1497
+ "model.language_model.layers.39.self_attn.v_proj": {
1498
+ "bits": 16,
1499
+ "data_type": "float"
1500
+ },
1501
+ "model.language_model.layers.39.self_attn.o_proj": {
1502
+ "bits": 16,
1503
+ "data_type": "float"
1504
+ },
1505
+ "model.language_model.layers.39.self_attn.indexer.index_qk_proj": {
1506
+ "bits": 16,
1507
+ "data_type": "float"
1508
+ },
1509
+ "model.language_model.layers.43.self_attn.q_proj": {
1510
+ "bits": 16,
1511
+ "data_type": "float"
1512
+ },
1513
+ "model.language_model.layers.43.self_attn.k_proj": {
1514
+ "bits": 16,
1515
+ "data_type": "float"
1516
+ },
1517
+ "model.language_model.layers.43.self_attn.v_proj": {
1518
+ "bits": 16,
1519
+ "data_type": "float"
1520
+ },
1521
+ "model.language_model.layers.43.self_attn.o_proj": {
1522
+ "bits": 16,
1523
+ "data_type": "float"
1524
+ },
1525
+ "model.language_model.layers.43.self_attn.indexer.index_qk_proj": {
1526
+ "bits": 16,
1527
+ "data_type": "float"
1528
+ },
1529
+ "model.language_model.layers.47.self_attn.q_proj": {
1530
+ "bits": 16,
1531
+ "data_type": "float"
1532
+ },
1533
+ "model.language_model.layers.47.self_attn.k_proj": {
1534
+ "bits": 16,
1535
+ "data_type": "float"
1536
+ },
1537
+ "model.language_model.layers.47.self_attn.v_proj": {
1538
+ "bits": 16,
1539
+ "data_type": "float"
1540
+ },
1541
+ "model.language_model.layers.47.self_attn.o_proj": {
1542
+ "bits": 16,
1543
+ "data_type": "float"
1544
+ },
1545
+ "model.language_model.layers.47.self_attn.indexer.index_qk_proj": {
1546
+ "bits": 16,
1547
+ "data_type": "float"
1548
+ },
1549
+ "model.language_model.layers.0.attn_hyper_connection.input_mix_weight_down": {
1550
+ "bits": 16,
1551
+ "data_type": "fp"
1552
+ },
1553
+ "model.language_model.layers.0.attn_hyper_connection.input_mix_weight_up": {
1554
+ "bits": 16,
1555
+ "data_type": "fp"
1556
+ },
1557
+ "model.language_model.layers.0.mlp_hyper_connection.input_mix_weight_down": {
1558
+ "bits": 16,
1559
+ "data_type": "fp"
1560
+ },
1561
+ "model.language_model.layers.0.mlp_hyper_connection.input_mix_weight_up": {
1562
+ "bits": 16,
1563
+ "data_type": "fp"
1564
+ },
1565
+ "model.language_model.layers.1.attn_hyper_connection.input_mix_weight_down": {
1566
+ "bits": 16,
1567
+ "data_type": "fp"
1568
+ },
1569
+ "model.language_model.layers.1.attn_hyper_connection.input_mix_weight_up": {
1570
+ "bits": 16,
1571
+ "data_type": "fp"
1572
+ },
1573
+ "model.language_model.layers.1.mlp_hyper_connection.input_mix_weight_down": {
1574
+ "bits": 16,
1575
+ "data_type": "fp"
1576
+ },
1577
+ "model.language_model.layers.1.mlp_hyper_connection.input_mix_weight_up": {
1578
+ "bits": 16,
1579
+ "data_type": "fp"
1580
+ },
1581
+ "model.language_model.layers.2.attn_hyper_connection.input_mix_weight_down": {
1582
+ "bits": 16,
1583
+ "data_type": "fp"
1584
+ },
1585
+ "model.language_model.layers.2.attn_hyper_connection.input_mix_weight_up": {
1586
+ "bits": 16,
1587
+ "data_type": "fp"
1588
+ },
1589
+ "model.language_model.layers.2.mlp_hyper_connection.input_mix_weight_down": {
1590
+ "bits": 16,
1591
+ "data_type": "fp"
1592
+ },
1593
+ "model.language_model.layers.2.mlp_hyper_connection.input_mix_weight_up": {
1594
+ "bits": 16,
1595
+ "data_type": "fp"
1596
+ },
1597
+ "model.language_model.layers.3.attn_hyper_connection.input_mix_weight_down": {
1598
+ "bits": 16,
1599
+ "data_type": "fp"
1600
+ },
1601
+ "model.language_model.layers.3.attn_hyper_connection.input_mix_weight_up": {
1602
+ "bits": 16,
1603
+ "data_type": "fp"
1604
+ },
1605
+ "model.language_model.layers.3.mlp_hyper_connection.input_mix_weight_down": {
1606
+ "bits": 16,
1607
+ "data_type": "fp"
1608
+ },
1609
+ "model.language_model.layers.3.mlp_hyper_connection.input_mix_weight_up": {
1610
+ "bits": 16,
1611
+ "data_type": "fp"
1612
+ },
1613
+ "model.language_model.layers.4.attn_hyper_connection.input_mix_weight_down": {
1614
+ "bits": 16,
1615
+ "data_type": "fp"
1616
+ },
1617
+ "model.language_model.layers.4.attn_hyper_connection.input_mix_weight_up": {
1618
+ "bits": 16,
1619
+ "data_type": "fp"
1620
+ },
1621
+ "model.language_model.layers.4.mlp_hyper_connection.input_mix_weight_down": {
1622
+ "bits": 16,
1623
+ "data_type": "fp"
1624
+ },
1625
+ "model.language_model.layers.4.mlp_hyper_connection.input_mix_weight_up": {
1626
+ "bits": 16,
1627
+ "data_type": "fp"
1628
+ },
1629
+ "model.language_model.layers.5.attn_hyper_connection.input_mix_weight_down": {
1630
+ "bits": 16,
1631
+ "data_type": "fp"
1632
+ },
1633
+ "model.language_model.layers.5.attn_hyper_connection.input_mix_weight_up": {
1634
+ "bits": 16,
1635
+ "data_type": "fp"
1636
+ },
1637
+ "model.language_model.layers.5.mlp_hyper_connection.input_mix_weight_down": {
1638
+ "bits": 16,
1639
+ "data_type": "fp"
1640
+ },
1641
+ "model.language_model.layers.5.mlp_hyper_connection.input_mix_weight_up": {
1642
+ "bits": 16,
1643
+ "data_type": "fp"
1644
+ },
1645
+ "model.language_model.layers.6.attn_hyper_connection.input_mix_weight_down": {
1646
+ "bits": 16,
1647
+ "data_type": "fp"
1648
+ },
1649
+ "model.language_model.layers.6.attn_hyper_connection.input_mix_weight_up": {
1650
+ "bits": 16,
1651
+ "data_type": "fp"
1652
+ },
1653
+ "model.language_model.layers.6.mlp_hyper_connection.input_mix_weight_down": {
1654
+ "bits": 16,
1655
+ "data_type": "fp"
1656
+ },
1657
+ "model.language_model.layers.6.mlp_hyper_connection.input_mix_weight_up": {
1658
+ "bits": 16,
1659
+ "data_type": "fp"
1660
+ },
1661
+ "model.language_model.layers.7.attn_hyper_connection.input_mix_weight_down": {
1662
+ "bits": 16,
1663
+ "data_type": "fp"
1664
+ },
1665
+ "model.language_model.layers.7.attn_hyper_connection.input_mix_weight_up": {
1666
+ "bits": 16,
1667
+ "data_type": "fp"
1668
+ },
1669
+ "model.language_model.layers.7.mlp_hyper_connection.input_mix_weight_down": {
1670
+ "bits": 16,
1671
+ "data_type": "fp"
1672
+ },
1673
+ "model.language_model.layers.7.mlp_hyper_connection.input_mix_weight_up": {
1674
+ "bits": 16,
1675
+ "data_type": "fp"
1676
+ },
1677
+ "model.language_model.layers.8.attn_hyper_connection.input_mix_weight_down": {
1678
+ "bits": 16,
1679
+ "data_type": "fp"
1680
+ },
1681
+ "model.language_model.layers.8.attn_hyper_connection.input_mix_weight_up": {
1682
+ "bits": 16,
1683
+ "data_type": "fp"
1684
+ },
1685
+ "model.language_model.layers.8.mlp_hyper_connection.input_mix_weight_down": {
1686
+ "bits": 16,
1687
+ "data_type": "fp"
1688
+ },
1689
+ "model.language_model.layers.8.mlp_hyper_connection.input_mix_weight_up": {
1690
+ "bits": 16,
1691
+ "data_type": "fp"
1692
+ },
1693
+ "model.language_model.layers.9.attn_hyper_connection.input_mix_weight_down": {
1694
+ "bits": 16,
1695
+ "data_type": "fp"
1696
+ },
1697
+ "model.language_model.layers.9.attn_hyper_connection.input_mix_weight_up": {
1698
+ "bits": 16,
1699
+ "data_type": "fp"
1700
+ },
1701
+ "model.language_model.layers.9.mlp_hyper_connection.input_mix_weight_down": {
1702
+ "bits": 16,
1703
+ "data_type": "fp"
1704
+ },
1705
+ "model.language_model.layers.9.mlp_hyper_connection.input_mix_weight_up": {
1706
+ "bits": 16,
1707
+ "data_type": "fp"
1708
+ },
1709
+ "model.language_model.layers.10.attn_hyper_connection.input_mix_weight_down": {
1710
+ "bits": 16,
1711
+ "data_type": "fp"
1712
+ },
1713
+ "model.language_model.layers.10.attn_hyper_connection.input_mix_weight_up": {
1714
+ "bits": 16,
1715
+ "data_type": "fp"
1716
+ },
1717
+ "model.language_model.layers.10.mlp_hyper_connection.input_mix_weight_down": {
1718
+ "bits": 16,
1719
+ "data_type": "fp"
1720
+ },
1721
+ "model.language_model.layers.10.mlp_hyper_connection.input_mix_weight_up": {
1722
+ "bits": 16,
1723
+ "data_type": "fp"
1724
+ },
1725
+ "model.language_model.layers.11.attn_hyper_connection.input_mix_weight_down": {
1726
+ "bits": 16,
1727
+ "data_type": "fp"
1728
+ },
1729
+ "model.language_model.layers.11.attn_hyper_connection.input_mix_weight_up": {
1730
+ "bits": 16,
1731
+ "data_type": "fp"
1732
+ },
1733
+ "model.language_model.layers.11.mlp_hyper_connection.input_mix_weight_down": {
1734
+ "bits": 16,
1735
+ "data_type": "fp"
1736
+ },
1737
+ "model.language_model.layers.11.mlp_hyper_connection.input_mix_weight_up": {
1738
+ "bits": 16,
1739
+ "data_type": "fp"
1740
+ },
1741
+ "model.language_model.layers.12.attn_hyper_connection.input_mix_weight_down": {
1742
+ "bits": 16,
1743
+ "data_type": "fp"
1744
+ },
1745
+ "model.language_model.layers.12.attn_hyper_connection.input_mix_weight_up": {
1746
+ "bits": 16,
1747
+ "data_type": "fp"
1748
+ },
1749
+ "model.language_model.layers.12.mlp_hyper_connection.input_mix_weight_down": {
1750
+ "bits": 16,
1751
+ "data_type": "fp"
1752
+ },
1753
+ "model.language_model.layers.12.mlp_hyper_connection.input_mix_weight_up": {
1754
+ "bits": 16,
1755
+ "data_type": "fp"
1756
+ },
1757
+ "model.language_model.layers.13.attn_hyper_connection.input_mix_weight_down": {
1758
+ "bits": 16,
1759
+ "data_type": "fp"
1760
+ },
1761
+ "model.language_model.layers.13.attn_hyper_connection.input_mix_weight_up": {
1762
+ "bits": 16,
1763
+ "data_type": "fp"
1764
+ },
1765
+ "model.language_model.layers.13.mlp_hyper_connection.input_mix_weight_down": {
1766
+ "bits": 16,
1767
+ "data_type": "fp"
1768
+ },
1769
+ "model.language_model.layers.13.mlp_hyper_connection.input_mix_weight_up": {
1770
+ "bits": 16,
1771
+ "data_type": "fp"
1772
+ },
1773
+ "model.language_model.layers.14.attn_hyper_connection.input_mix_weight_down": {
1774
+ "bits": 16,
1775
+ "data_type": "fp"
1776
+ },
1777
+ "model.language_model.layers.14.attn_hyper_connection.input_mix_weight_up": {
1778
+ "bits": 16,
1779
+ "data_type": "fp"
1780
+ },
1781
+ "model.language_model.layers.14.mlp_hyper_connection.input_mix_weight_down": {
1782
+ "bits": 16,
1783
+ "data_type": "fp"
1784
+ },
1785
+ "model.language_model.layers.14.mlp_hyper_connection.input_mix_weight_up": {
1786
+ "bits": 16,
1787
+ "data_type": "fp"
1788
+ },
1789
+ "model.language_model.layers.15.attn_hyper_connection.input_mix_weight_down": {
1790
+ "bits": 16,
1791
+ "data_type": "fp"
1792
+ },
1793
+ "model.language_model.layers.15.attn_hyper_connection.input_mix_weight_up": {
1794
+ "bits": 16,
1795
+ "data_type": "fp"
1796
+ },
1797
+ "model.language_model.layers.15.mlp_hyper_connection.input_mix_weight_down": {
1798
+ "bits": 16,
1799
+ "data_type": "fp"
1800
+ },
1801
+ "model.language_model.layers.15.mlp_hyper_connection.input_mix_weight_up": {
1802
+ "bits": 16,
1803
+ "data_type": "fp"
1804
+ },
1805
+ "model.language_model.layers.16.attn_hyper_connection.input_mix_weight_down": {
1806
+ "bits": 16,
1807
+ "data_type": "fp"
1808
+ },
1809
+ "model.language_model.layers.16.attn_hyper_connection.input_mix_weight_up": {
1810
+ "bits": 16,
1811
+ "data_type": "fp"
1812
+ },
1813
+ "model.language_model.layers.16.mlp_hyper_connection.input_mix_weight_down": {
1814
+ "bits": 16,
1815
+ "data_type": "fp"
1816
+ },
1817
+ "model.language_model.layers.16.mlp_hyper_connection.input_mix_weight_up": {
1818
+ "bits": 16,
1819
+ "data_type": "fp"
1820
+ },
1821
+ "model.language_model.layers.17.attn_hyper_connection.input_mix_weight_down": {
1822
+ "bits": 16,
1823
+ "data_type": "fp"
1824
+ },
1825
+ "model.language_model.layers.17.attn_hyper_connection.input_mix_weight_up": {
1826
+ "bits": 16,
1827
+ "data_type": "fp"
1828
+ },
1829
+ "model.language_model.layers.17.mlp_hyper_connection.input_mix_weight_down": {
1830
+ "bits": 16,
1831
+ "data_type": "fp"
1832
+ },
1833
+ "model.language_model.layers.17.mlp_hyper_connection.input_mix_weight_up": {
1834
+ "bits": 16,
1835
+ "data_type": "fp"
1836
+ },
1837
+ "model.language_model.layers.18.attn_hyper_connection.input_mix_weight_down": {
1838
+ "bits": 16,
1839
+ "data_type": "fp"
1840
+ },
1841
+ "model.language_model.layers.18.attn_hyper_connection.input_mix_weight_up": {
1842
+ "bits": 16,
1843
+ "data_type": "fp"
1844
+ },
1845
+ "model.language_model.layers.18.mlp_hyper_connection.input_mix_weight_down": {
1846
+ "bits": 16,
1847
+ "data_type": "fp"
1848
+ },
1849
+ "model.language_model.layers.18.mlp_hyper_connection.input_mix_weight_up": {
1850
+ "bits": 16,
1851
+ "data_type": "fp"
1852
+ },
1853
+ "model.language_model.layers.19.attn_hyper_connection.input_mix_weight_down": {
1854
+ "bits": 16,
1855
+ "data_type": "fp"
1856
+ },
1857
+ "model.language_model.layers.19.attn_hyper_connection.input_mix_weight_up": {
1858
+ "bits": 16,
1859
+ "data_type": "fp"
1860
+ },
1861
+ "model.language_model.layers.19.mlp_hyper_connection.input_mix_weight_down": {
1862
+ "bits": 16,
1863
+ "data_type": "fp"
1864
+ },
1865
+ "model.language_model.layers.19.mlp_hyper_connection.input_mix_weight_up": {
1866
+ "bits": 16,
1867
+ "data_type": "fp"
1868
+ },
1869
+ "model.language_model.layers.20.attn_hyper_connection.input_mix_weight_down": {
1870
+ "bits": 16,
1871
+ "data_type": "fp"
1872
+ },
1873
+ "model.language_model.layers.20.attn_hyper_connection.input_mix_weight_up": {
1874
+ "bits": 16,
1875
+ "data_type": "fp"
1876
+ },
1877
+ "model.language_model.layers.20.mlp_hyper_connection.input_mix_weight_down": {
1878
+ "bits": 16,
1879
+ "data_type": "fp"
1880
+ },
1881
+ "model.language_model.layers.20.mlp_hyper_connection.input_mix_weight_up": {
1882
+ "bits": 16,
1883
+ "data_type": "fp"
1884
+ },
1885
+ "model.language_model.layers.21.attn_hyper_connection.input_mix_weight_down": {
1886
+ "bits": 16,
1887
+ "data_type": "fp"
1888
+ },
1889
+ "model.language_model.layers.21.attn_hyper_connection.input_mix_weight_up": {
1890
+ "bits": 16,
1891
+ "data_type": "fp"
1892
+ },
1893
+ "model.language_model.layers.21.mlp_hyper_connection.input_mix_weight_down": {
1894
+ "bits": 16,
1895
+ "data_type": "fp"
1896
+ },
1897
+ "model.language_model.layers.21.mlp_hyper_connection.input_mix_weight_up": {
1898
+ "bits": 16,
1899
+ "data_type": "fp"
1900
+ },
1901
+ "model.language_model.layers.22.attn_hyper_connection.input_mix_weight_down": {
1902
+ "bits": 16,
1903
+ "data_type": "fp"
1904
+ },
1905
+ "model.language_model.layers.22.attn_hyper_connection.input_mix_weight_up": {
1906
+ "bits": 16,
1907
+ "data_type": "fp"
1908
+ },
1909
+ "model.language_model.layers.22.mlp_hyper_connection.input_mix_weight_down": {
1910
+ "bits": 16,
1911
+ "data_type": "fp"
1912
+ },
1913
+ "model.language_model.layers.22.mlp_hyper_connection.input_mix_weight_up": {
1914
+ "bits": 16,
1915
+ "data_type": "fp"
1916
+ },
1917
+ "model.language_model.layers.23.attn_hyper_connection.input_mix_weight_down": {
1918
+ "bits": 16,
1919
+ "data_type": "fp"
1920
+ },
1921
+ "model.language_model.layers.23.attn_hyper_connection.input_mix_weight_up": {
1922
+ "bits": 16,
1923
+ "data_type": "fp"
1924
+ },
1925
+ "model.language_model.layers.23.mlp_hyper_connection.input_mix_weight_down": {
1926
+ "bits": 16,
1927
+ "data_type": "fp"
1928
+ },
1929
+ "model.language_model.layers.23.mlp_hyper_connection.input_mix_weight_up": {
1930
+ "bits": 16,
1931
+ "data_type": "fp"
1932
+ },
1933
+ "model.language_model.layers.24.attn_hyper_connection.input_mix_weight_down": {
1934
+ "bits": 16,
1935
+ "data_type": "fp"
1936
+ },
1937
+ "model.language_model.layers.24.attn_hyper_connection.input_mix_weight_up": {
1938
+ "bits": 16,
1939
+ "data_type": "fp"
1940
+ },
1941
+ "model.language_model.layers.24.mlp_hyper_connection.input_mix_weight_down": {
1942
+ "bits": 16,
1943
+ "data_type": "fp"
1944
+ },
1945
+ "model.language_model.layers.24.mlp_hyper_connection.input_mix_weight_up": {
1946
+ "bits": 16,
1947
+ "data_type": "fp"
1948
+ },
1949
+ "model.language_model.layers.25.attn_hyper_connection.input_mix_weight_down": {
1950
+ "bits": 16,
1951
+ "data_type": "fp"
1952
+ },
1953
+ "model.language_model.layers.25.attn_hyper_connection.input_mix_weight_up": {
1954
+ "bits": 16,
1955
+ "data_type": "fp"
1956
+ },
1957
+ "model.language_model.layers.25.mlp_hyper_connection.input_mix_weight_down": {
1958
+ "bits": 16,
1959
+ "data_type": "fp"
1960
+ },
1961
+ "model.language_model.layers.25.mlp_hyper_connection.input_mix_weight_up": {
1962
+ "bits": 16,
1963
+ "data_type": "fp"
1964
+ },
1965
+ "model.language_model.layers.26.attn_hyper_connection.input_mix_weight_down": {
1966
+ "bits": 16,
1967
+ "data_type": "fp"
1968
+ },
1969
+ "model.language_model.layers.26.attn_hyper_connection.input_mix_weight_up": {
1970
+ "bits": 16,
1971
+ "data_type": "fp"
1972
+ },
1973
+ "model.language_model.layers.26.mlp_hyper_connection.input_mix_weight_down": {
1974
+ "bits": 16,
1975
+ "data_type": "fp"
1976
+ },
1977
+ "model.language_model.layers.26.mlp_hyper_connection.input_mix_weight_up": {
1978
+ "bits": 16,
1979
+ "data_type": "fp"
1980
+ },
1981
+ "model.language_model.layers.27.attn_hyper_connection.input_mix_weight_down": {
1982
+ "bits": 16,
1983
+ "data_type": "fp"
1984
+ },
1985
+ "model.language_model.layers.27.attn_hyper_connection.input_mix_weight_up": {
1986
+ "bits": 16,
1987
+ "data_type": "fp"
1988
+ },
1989
+ "model.language_model.layers.27.mlp_hyper_connection.input_mix_weight_down": {
1990
+ "bits": 16,
1991
+ "data_type": "fp"
1992
+ },
1993
+ "model.language_model.layers.27.mlp_hyper_connection.input_mix_weight_up": {
1994
+ "bits": 16,
1995
+ "data_type": "fp"
1996
+ },
1997
+ "model.language_model.layers.28.attn_hyper_connection.input_mix_weight_down": {
1998
+ "bits": 16,
1999
+ "data_type": "fp"
2000
+ },
2001
+ "model.language_model.layers.28.attn_hyper_connection.input_mix_weight_up": {
2002
+ "bits": 16,
2003
+ "data_type": "fp"
2004
+ },
2005
+ "model.language_model.layers.28.mlp_hyper_connection.input_mix_weight_down": {
2006
+ "bits": 16,
2007
+ "data_type": "fp"
2008
+ },
2009
+ "model.language_model.layers.28.mlp_hyper_connection.input_mix_weight_up": {
2010
+ "bits": 16,
2011
+ "data_type": "fp"
2012
+ },
2013
+ "model.language_model.layers.29.attn_hyper_connection.input_mix_weight_down": {
2014
+ "bits": 16,
2015
+ "data_type": "fp"
2016
+ },
2017
+ "model.language_model.layers.29.attn_hyper_connection.input_mix_weight_up": {
2018
+ "bits": 16,
2019
+ "data_type": "fp"
2020
+ },
2021
+ "model.language_model.layers.29.mlp_hyper_connection.input_mix_weight_down": {
2022
+ "bits": 16,
2023
+ "data_type": "fp"
2024
+ },
2025
+ "model.language_model.layers.29.mlp_hyper_connection.input_mix_weight_up": {
2026
+ "bits": 16,
2027
+ "data_type": "fp"
2028
+ },
2029
+ "model.language_model.layers.30.attn_hyper_connection.input_mix_weight_down": {
2030
+ "bits": 16,
2031
+ "data_type": "fp"
2032
+ },
2033
+ "model.language_model.layers.30.attn_hyper_connection.input_mix_weight_up": {
2034
+ "bits": 16,
2035
+ "data_type": "fp"
2036
+ },
2037
+ "model.language_model.layers.30.mlp_hyper_connection.input_mix_weight_down": {
2038
+ "bits": 16,
2039
+ "data_type": "fp"
2040
+ },
2041
+ "model.language_model.layers.30.mlp_hyper_connection.input_mix_weight_up": {
2042
+ "bits": 16,
2043
+ "data_type": "fp"
2044
+ },
2045
+ "model.language_model.layers.31.attn_hyper_connection.input_mix_weight_down": {
2046
+ "bits": 16,
2047
+ "data_type": "fp"
2048
+ },
2049
+ "model.language_model.layers.31.attn_hyper_connection.input_mix_weight_up": {
2050
+ "bits": 16,
2051
+ "data_type": "fp"
2052
+ },
2053
+ "model.language_model.layers.31.mlp_hyper_connection.input_mix_weight_down": {
2054
+ "bits": 16,
2055
+ "data_type": "fp"
2056
+ },
2057
+ "model.language_model.layers.31.mlp_hyper_connection.input_mix_weight_up": {
2058
+ "bits": 16,
2059
+ "data_type": "fp"
2060
+ },
2061
+ "model.language_model.layers.32.attn_hyper_connection.input_mix_weight_down": {
2062
+ "bits": 16,
2063
+ "data_type": "fp"
2064
+ },
2065
+ "model.language_model.layers.32.attn_hyper_connection.input_mix_weight_up": {
2066
+ "bits": 16,
2067
+ "data_type": "fp"
2068
+ },
2069
+ "model.language_model.layers.32.mlp_hyper_connection.input_mix_weight_down": {
2070
+ "bits": 16,
2071
+ "data_type": "fp"
2072
+ },
2073
+ "model.language_model.layers.32.mlp_hyper_connection.input_mix_weight_up": {
2074
+ "bits": 16,
2075
+ "data_type": "fp"
2076
+ },
2077
+ "model.language_model.layers.33.attn_hyper_connection.input_mix_weight_down": {
2078
+ "bits": 16,
2079
+ "data_type": "fp"
2080
+ },
2081
+ "model.language_model.layers.33.attn_hyper_connection.input_mix_weight_up": {
2082
+ "bits": 16,
2083
+ "data_type": "fp"
2084
+ },
2085
+ "model.language_model.layers.33.mlp_hyper_connection.input_mix_weight_down": {
2086
+ "bits": 16,
2087
+ "data_type": "fp"
2088
+ },
2089
+ "model.language_model.layers.33.mlp_hyper_connection.input_mix_weight_up": {
2090
+ "bits": 16,
2091
+ "data_type": "fp"
2092
+ },
2093
+ "model.language_model.layers.34.attn_hyper_connection.input_mix_weight_down": {
2094
+ "bits": 16,
2095
+ "data_type": "fp"
2096
+ },
2097
+ "model.language_model.layers.34.attn_hyper_connection.input_mix_weight_up": {
2098
+ "bits": 16,
2099
+ "data_type": "fp"
2100
+ },
2101
+ "model.language_model.layers.34.mlp_hyper_connection.input_mix_weight_down": {
2102
+ "bits": 16,
2103
+ "data_type": "fp"
2104
+ },
2105
+ "model.language_model.layers.34.mlp_hyper_connection.input_mix_weight_up": {
2106
+ "bits": 16,
2107
+ "data_type": "fp"
2108
+ },
2109
+ "model.language_model.layers.35.attn_hyper_connection.input_mix_weight_down": {
2110
+ "bits": 16,
2111
+ "data_type": "fp"
2112
+ },
2113
+ "model.language_model.layers.35.attn_hyper_connection.input_mix_weight_up": {
2114
+ "bits": 16,
2115
+ "data_type": "fp"
2116
+ },
2117
+ "model.language_model.layers.35.mlp_hyper_connection.input_mix_weight_down": {
2118
+ "bits": 16,
2119
+ "data_type": "fp"
2120
+ },
2121
+ "model.language_model.layers.35.mlp_hyper_connection.input_mix_weight_up": {
2122
+ "bits": 16,
2123
+ "data_type": "fp"
2124
+ },
2125
+ "model.language_model.layers.36.attn_hyper_connection.input_mix_weight_down": {
2126
+ "bits": 16,
2127
+ "data_type": "fp"
2128
+ },
2129
+ "model.language_model.layers.36.attn_hyper_connection.input_mix_weight_up": {
2130
+ "bits": 16,
2131
+ "data_type": "fp"
2132
+ },
2133
+ "model.language_model.layers.36.mlp_hyper_connection.input_mix_weight_down": {
2134
+ "bits": 16,
2135
+ "data_type": "fp"
2136
+ },
2137
+ "model.language_model.layers.36.mlp_hyper_connection.input_mix_weight_up": {
2138
+ "bits": 16,
2139
+ "data_type": "fp"
2140
+ },
2141
+ "model.language_model.layers.37.attn_hyper_connection.input_mix_weight_down": {
2142
+ "bits": 16,
2143
+ "data_type": "fp"
2144
+ },
2145
+ "model.language_model.layers.37.attn_hyper_connection.input_mix_weight_up": {
2146
+ "bits": 16,
2147
+ "data_type": "fp"
2148
+ },
2149
+ "model.language_model.layers.37.mlp_hyper_connection.input_mix_weight_down": {
2150
+ "bits": 16,
2151
+ "data_type": "fp"
2152
+ },
2153
+ "model.language_model.layers.37.mlp_hyper_connection.input_mix_weight_up": {
2154
+ "bits": 16,
2155
+ "data_type": "fp"
2156
+ },
2157
+ "model.language_model.layers.38.attn_hyper_connection.input_mix_weight_down": {
2158
+ "bits": 16,
2159
+ "data_type": "fp"
2160
+ },
2161
+ "model.language_model.layers.38.attn_hyper_connection.input_mix_weight_up": {
2162
+ "bits": 16,
2163
+ "data_type": "fp"
2164
+ },
2165
+ "model.language_model.layers.38.mlp_hyper_connection.input_mix_weight_down": {
2166
+ "bits": 16,
2167
+ "data_type": "fp"
2168
+ },
2169
+ "model.language_model.layers.38.mlp_hyper_connection.input_mix_weight_up": {
2170
+ "bits": 16,
2171
+ "data_type": "fp"
2172
+ },
2173
+ "model.language_model.layers.39.attn_hyper_connection.input_mix_weight_down": {
2174
+ "bits": 16,
2175
+ "data_type": "fp"
2176
+ },
2177
+ "model.language_model.layers.39.attn_hyper_connection.input_mix_weight_up": {
2178
+ "bits": 16,
2179
+ "data_type": "fp"
2180
+ },
2181
+ "model.language_model.layers.39.mlp_hyper_connection.input_mix_weight_down": {
2182
+ "bits": 16,
2183
+ "data_type": "fp"
2184
+ },
2185
+ "model.language_model.layers.39.mlp_hyper_connection.input_mix_weight_up": {
2186
+ "bits": 16,
2187
+ "data_type": "fp"
2188
+ },
2189
+ "model.language_model.layers.40.attn_hyper_connection.input_mix_weight_down": {
2190
+ "bits": 16,
2191
+ "data_type": "fp"
2192
+ },
2193
+ "model.language_model.layers.40.attn_hyper_connection.input_mix_weight_up": {
2194
+ "bits": 16,
2195
+ "data_type": "fp"
2196
+ },
2197
+ "model.language_model.layers.40.mlp_hyper_connection.input_mix_weight_down": {
2198
+ "bits": 16,
2199
+ "data_type": "fp"
2200
+ },
2201
+ "model.language_model.layers.40.mlp_hyper_connection.input_mix_weight_up": {
2202
+ "bits": 16,
2203
+ "data_type": "fp"
2204
+ },
2205
+ "model.language_model.layers.41.attn_hyper_connection.input_mix_weight_down": {
2206
+ "bits": 16,
2207
+ "data_type": "fp"
2208
+ },
2209
+ "model.language_model.layers.41.attn_hyper_connection.input_mix_weight_up": {
2210
+ "bits": 16,
2211
+ "data_type": "fp"
2212
+ },
2213
+ "model.language_model.layers.41.mlp_hyper_connection.input_mix_weight_down": {
2214
+ "bits": 16,
2215
+ "data_type": "fp"
2216
+ },
2217
+ "model.language_model.layers.41.mlp_hyper_connection.input_mix_weight_up": {
2218
+ "bits": 16,
2219
+ "data_type": "fp"
2220
+ },
2221
+ "model.language_model.layers.42.attn_hyper_connection.input_mix_weight_down": {
2222
+ "bits": 16,
2223
+ "data_type": "fp"
2224
+ },
2225
+ "model.language_model.layers.42.attn_hyper_connection.input_mix_weight_up": {
2226
+ "bits": 16,
2227
+ "data_type": "fp"
2228
+ },
2229
+ "model.language_model.layers.42.mlp_hyper_connection.input_mix_weight_down": {
2230
+ "bits": 16,
2231
+ "data_type": "fp"
2232
+ },
2233
+ "model.language_model.layers.42.mlp_hyper_connection.input_mix_weight_up": {
2234
+ "bits": 16,
2235
+ "data_type": "fp"
2236
+ },
2237
+ "model.language_model.layers.43.attn_hyper_connection.input_mix_weight_down": {
2238
+ "bits": 16,
2239
+ "data_type": "fp"
2240
+ },
2241
+ "model.language_model.layers.43.attn_hyper_connection.input_mix_weight_up": {
2242
+ "bits": 16,
2243
+ "data_type": "fp"
2244
+ },
2245
+ "model.language_model.layers.43.mlp_hyper_connection.input_mix_weight_down": {
2246
+ "bits": 16,
2247
+ "data_type": "fp"
2248
+ },
2249
+ "model.language_model.layers.43.mlp_hyper_connection.input_mix_weight_up": {
2250
+ "bits": 16,
2251
+ "data_type": "fp"
2252
+ },
2253
+ "model.language_model.layers.44.attn_hyper_connection.input_mix_weight_down": {
2254
+ "bits": 16,
2255
+ "data_type": "fp"
2256
+ },
2257
+ "model.language_model.layers.44.attn_hyper_connection.input_mix_weight_up": {
2258
+ "bits": 16,
2259
+ "data_type": "fp"
2260
+ },
2261
+ "model.language_model.layers.44.mlp_hyper_connection.input_mix_weight_down": {
2262
+ "bits": 16,
2263
+ "data_type": "fp"
2264
+ },
2265
+ "model.language_model.layers.44.mlp_hyper_connection.input_mix_weight_up": {
2266
+ "bits": 16,
2267
+ "data_type": "fp"
2268
+ },
2269
+ "model.language_model.layers.45.attn_hyper_connection.input_mix_weight_down": {
2270
+ "bits": 16,
2271
+ "data_type": "fp"
2272
+ },
2273
+ "model.language_model.layers.45.attn_hyper_connection.input_mix_weight_up": {
2274
+ "bits": 16,
2275
+ "data_type": "fp"
2276
+ },
2277
+ "model.language_model.layers.45.mlp_hyper_connection.input_mix_weight_down": {
2278
+ "bits": 16,
2279
+ "data_type": "fp"
2280
+ },
2281
+ "model.language_model.layers.45.mlp_hyper_connection.input_mix_weight_up": {
2282
+ "bits": 16,
2283
+ "data_type": "fp"
2284
+ },
2285
+ "model.language_model.layers.46.attn_hyper_connection.input_mix_weight_down": {
2286
+ "bits": 16,
2287
+ "data_type": "fp"
2288
+ },
2289
+ "model.language_model.layers.46.attn_hyper_connection.input_mix_weight_up": {
2290
+ "bits": 16,
2291
+ "data_type": "fp"
2292
+ },
2293
+ "model.language_model.layers.46.mlp_hyper_connection.input_mix_weight_down": {
2294
+ "bits": 16,
2295
+ "data_type": "fp"
2296
+ },
2297
+ "model.language_model.layers.46.mlp_hyper_connection.input_mix_weight_up": {
2298
+ "bits": 16,
2299
+ "data_type": "fp"
2300
+ },
2301
+ "model.language_model.layers.47.attn_hyper_connection.input_mix_weight_down": {
2302
+ "bits": 16,
2303
+ "data_type": "fp"
2304
+ },
2305
+ "model.language_model.layers.47.attn_hyper_connection.input_mix_weight_up": {
2306
+ "bits": 16,
2307
+ "data_type": "fp"
2308
+ },
2309
+ "model.language_model.layers.47.mlp_hyper_connection.input_mix_weight_down": {
2310
+ "bits": 16,
2311
+ "data_type": "fp"
2312
+ },
2313
+ "model.language_model.layers.47.mlp_hyper_connection.input_mix_weight_up": {
2314
+ "bits": 16,
2315
+ "data_type": "fp"
2316
+ },
2317
+ "model.language_model.hyper_connection_mixer.input_mix_weight_down": {
2318
+ "bits": 16,
2319
+ "data_type": "fp"
2320
+ },
2321
+ "model.language_model.hyper_connection_mixer.input_mix_weight_up": {
2322
+ "bits": 16,
2323
+ "data_type": "fp"
2324
+ },
2325
+ "model.language_model.layers.0.mlp.shared_expert.gate_proj": {
2326
+ "bits": 16,
2327
+ "data_type": "fp"
2328
+ },
2329
+ "model.language_model.layers.0.mlp.shared_expert.up_proj": {
2330
+ "bits": 16,
2331
+ "data_type": "fp"
2332
+ },
2333
+ "model.language_model.layers.0.mlp.shared_expert.down_proj": {
2334
+ "bits": 16,
2335
+ "data_type": "fp"
2336
+ },
2337
+ "model.language_model.layers.1.mlp.shared_expert.gate_proj": {
2338
+ "bits": 16,
2339
+ "data_type": "fp"
2340
+ },
2341
+ "model.language_model.layers.1.mlp.shared_expert.up_proj": {
2342
+ "bits": 16,
2343
+ "data_type": "fp"
2344
+ },
2345
+ "model.language_model.layers.1.mlp.shared_expert.down_proj": {
2346
+ "bits": 16,
2347
+ "data_type": "fp"
2348
+ },
2349
+ "model.language_model.layers.2.mlp.shared_expert.gate_proj": {
2350
+ "bits": 16,
2351
+ "data_type": "fp"
2352
+ },
2353
+ "model.language_model.layers.2.mlp.shared_expert.up_proj": {
2354
+ "bits": 16,
2355
+ "data_type": "fp"
2356
+ },
2357
+ "model.language_model.layers.2.mlp.shared_expert.down_proj": {
2358
+ "bits": 16,
2359
+ "data_type": "fp"
2360
+ },
2361
+ "model.language_model.layers.3.mlp.shared_expert.gate_proj": {
2362
+ "bits": 16,
2363
+ "data_type": "fp"
2364
+ },
2365
+ "model.language_model.layers.3.mlp.shared_expert.up_proj": {
2366
+ "bits": 16,
2367
+ "data_type": "fp"
2368
+ },
2369
+ "model.language_model.layers.3.mlp.shared_expert.down_proj": {
2370
+ "bits": 16,
2371
+ "data_type": "fp"
2372
+ },
2373
+ "model.language_model.layers.4.mlp.shared_expert.gate_proj": {
2374
+ "bits": 16,
2375
+ "data_type": "fp"
2376
+ },
2377
+ "model.language_model.layers.4.mlp.shared_expert.up_proj": {
2378
+ "bits": 16,
2379
+ "data_type": "fp"
2380
+ },
2381
+ "model.language_model.layers.4.mlp.shared_expert.down_proj": {
2382
+ "bits": 16,
2383
+ "data_type": "fp"
2384
+ },
2385
+ "model.language_model.layers.5.mlp.shared_expert.gate_proj": {
2386
+ "bits": 16,
2387
+ "data_type": "fp"
2388
+ },
2389
+ "model.language_model.layers.5.mlp.shared_expert.up_proj": {
2390
+ "bits": 16,
2391
+ "data_type": "fp"
2392
+ },
2393
+ "model.language_model.layers.5.mlp.shared_expert.down_proj": {
2394
+ "bits": 16,
2395
+ "data_type": "fp"
2396
+ },
2397
+ "model.language_model.layers.6.mlp.shared_expert.gate_proj": {
2398
+ "bits": 16,
2399
+ "data_type": "fp"
2400
+ },
2401
+ "model.language_model.layers.6.mlp.shared_expert.up_proj": {
2402
+ "bits": 16,
2403
+ "data_type": "fp"
2404
+ },
2405
+ "model.language_model.layers.6.mlp.shared_expert.down_proj": {
2406
+ "bits": 16,
2407
+ "data_type": "fp"
2408
+ },
2409
+ "model.language_model.layers.7.mlp.shared_expert.gate_proj": {
2410
+ "bits": 16,
2411
+ "data_type": "fp"
2412
+ },
2413
+ "model.language_model.layers.7.mlp.shared_expert.up_proj": {
2414
+ "bits": 16,
2415
+ "data_type": "fp"
2416
+ },
2417
+ "model.language_model.layers.7.mlp.shared_expert.down_proj": {
2418
+ "bits": 16,
2419
+ "data_type": "fp"
2420
+ },
2421
+ "model.language_model.layers.8.mlp.shared_expert.gate_proj": {
2422
+ "bits": 16,
2423
+ "data_type": "fp"
2424
+ },
2425
+ "model.language_model.layers.8.mlp.shared_expert.up_proj": {
2426
+ "bits": 16,
2427
+ "data_type": "fp"
2428
+ },
2429
+ "model.language_model.layers.8.mlp.shared_expert.down_proj": {
2430
+ "bits": 16,
2431
+ "data_type": "fp"
2432
+ },
2433
+ "model.language_model.layers.9.mlp.shared_expert.gate_proj": {
2434
+ "bits": 16,
2435
+ "data_type": "fp"
2436
+ },
2437
+ "model.language_model.layers.9.mlp.shared_expert.up_proj": {
2438
+ "bits": 16,
2439
+ "data_type": "fp"
2440
+ },
2441
+ "model.language_model.layers.9.mlp.shared_expert.down_proj": {
2442
+ "bits": 16,
2443
+ "data_type": "fp"
2444
+ },
2445
+ "model.language_model.layers.10.mlp.shared_expert.gate_proj": {
2446
+ "bits": 16,
2447
+ "data_type": "fp"
2448
+ },
2449
+ "model.language_model.layers.10.mlp.shared_expert.up_proj": {
2450
+ "bits": 16,
2451
+ "data_type": "fp"
2452
+ },
2453
+ "model.language_model.layers.10.mlp.shared_expert.down_proj": {
2454
+ "bits": 16,
2455
+ "data_type": "fp"
2456
+ },
2457
+ "model.language_model.layers.11.mlp.shared_expert.gate_proj": {
2458
+ "bits": 16,
2459
+ "data_type": "fp"
2460
+ },
2461
+ "model.language_model.layers.11.mlp.shared_expert.up_proj": {
2462
+ "bits": 16,
2463
+ "data_type": "fp"
2464
+ },
2465
+ "model.language_model.layers.11.mlp.shared_expert.down_proj": {
2466
+ "bits": 16,
2467
+ "data_type": "fp"
2468
+ },
2469
+ "model.language_model.layers.12.mlp.shared_expert.gate_proj": {
2470
+ "bits": 16,
2471
+ "data_type": "fp"
2472
+ },
2473
+ "model.language_model.layers.12.mlp.shared_expert.up_proj": {
2474
+ "bits": 16,
2475
+ "data_type": "fp"
2476
+ },
2477
+ "model.language_model.layers.12.mlp.shared_expert.down_proj": {
2478
+ "bits": 16,
2479
+ "data_type": "fp"
2480
+ },
2481
+ "model.language_model.layers.13.mlp.shared_expert.gate_proj": {
2482
+ "bits": 16,
2483
+ "data_type": "fp"
2484
+ },
2485
+ "model.language_model.layers.13.mlp.shared_expert.up_proj": {
2486
+ "bits": 16,
2487
+ "data_type": "fp"
2488
+ },
2489
+ "model.language_model.layers.13.mlp.shared_expert.down_proj": {
2490
+ "bits": 16,
2491
+ "data_type": "fp"
2492
+ },
2493
+ "model.language_model.layers.14.mlp.shared_expert.gate_proj": {
2494
+ "bits": 16,
2495
+ "data_type": "fp"
2496
+ },
2497
+ "model.language_model.layers.14.mlp.shared_expert.up_proj": {
2498
+ "bits": 16,
2499
+ "data_type": "fp"
2500
+ },
2501
+ "model.language_model.layers.14.mlp.shared_expert.down_proj": {
2502
+ "bits": 16,
2503
+ "data_type": "fp"
2504
+ },
2505
+ "model.language_model.layers.15.mlp.shared_expert.gate_proj": {
2506
+ "bits": 16,
2507
+ "data_type": "fp"
2508
+ },
2509
+ "model.language_model.layers.15.mlp.shared_expert.up_proj": {
2510
+ "bits": 16,
2511
+ "data_type": "fp"
2512
+ },
2513
+ "model.language_model.layers.15.mlp.shared_expert.down_proj": {
2514
+ "bits": 16,
2515
+ "data_type": "fp"
2516
+ },
2517
+ "model.language_model.layers.16.mlp.shared_expert.gate_proj": {
2518
+ "bits": 16,
2519
+ "data_type": "fp"
2520
+ },
2521
+ "model.language_model.layers.16.mlp.shared_expert.up_proj": {
2522
+ "bits": 16,
2523
+ "data_type": "fp"
2524
+ },
2525
+ "model.language_model.layers.16.mlp.shared_expert.down_proj": {
2526
+ "bits": 16,
2527
+ "data_type": "fp"
2528
+ },
2529
+ "model.language_model.layers.17.mlp.shared_expert.gate_proj": {
2530
+ "bits": 16,
2531
+ "data_type": "fp"
2532
+ },
2533
+ "model.language_model.layers.17.mlp.shared_expert.up_proj": {
2534
+ "bits": 16,
2535
+ "data_type": "fp"
2536
+ },
2537
+ "model.language_model.layers.17.mlp.shared_expert.down_proj": {
2538
+ "bits": 16,
2539
+ "data_type": "fp"
2540
+ },
2541
+ "model.language_model.layers.18.mlp.shared_expert.gate_proj": {
2542
+ "bits": 16,
2543
+ "data_type": "fp"
2544
+ },
2545
+ "model.language_model.layers.18.mlp.shared_expert.up_proj": {
2546
+ "bits": 16,
2547
+ "data_type": "fp"
2548
+ },
2549
+ "model.language_model.layers.18.mlp.shared_expert.down_proj": {
2550
+ "bits": 16,
2551
+ "data_type": "fp"
2552
+ },
2553
+ "model.language_model.layers.19.mlp.shared_expert.gate_proj": {
2554
+ "bits": 16,
2555
+ "data_type": "fp"
2556
+ },
2557
+ "model.language_model.layers.19.mlp.shared_expert.up_proj": {
2558
+ "bits": 16,
2559
+ "data_type": "fp"
2560
+ },
2561
+ "model.language_model.layers.19.mlp.shared_expert.down_proj": {
2562
+ "bits": 16,
2563
+ "data_type": "fp"
2564
+ },
2565
+ "model.language_model.layers.20.mlp.shared_expert.gate_proj": {
2566
+ "bits": 16,
2567
+ "data_type": "fp"
2568
+ },
2569
+ "model.language_model.layers.20.mlp.shared_expert.up_proj": {
2570
+ "bits": 16,
2571
+ "data_type": "fp"
2572
+ },
2573
+ "model.language_model.layers.20.mlp.shared_expert.down_proj": {
2574
+ "bits": 16,
2575
+ "data_type": "fp"
2576
+ },
2577
+ "model.language_model.layers.21.mlp.shared_expert.gate_proj": {
2578
+ "bits": 16,
2579
+ "data_type": "fp"
2580
+ },
2581
+ "model.language_model.layers.21.mlp.shared_expert.up_proj": {
2582
+ "bits": 16,
2583
+ "data_type": "fp"
2584
+ },
2585
+ "model.language_model.layers.21.mlp.shared_expert.down_proj": {
2586
+ "bits": 16,
2587
+ "data_type": "fp"
2588
+ },
2589
+ "model.language_model.layers.22.mlp.shared_expert.gate_proj": {
2590
+ "bits": 16,
2591
+ "data_type": "fp"
2592
+ },
2593
+ "model.language_model.layers.22.mlp.shared_expert.up_proj": {
2594
+ "bits": 16,
2595
+ "data_type": "fp"
2596
+ },
2597
+ "model.language_model.layers.22.mlp.shared_expert.down_proj": {
2598
+ "bits": 16,
2599
+ "data_type": "fp"
2600
+ },
2601
+ "model.language_model.layers.23.mlp.shared_expert.gate_proj": {
2602
+ "bits": 16,
2603
+ "data_type": "fp"
2604
+ },
2605
+ "model.language_model.layers.23.mlp.shared_expert.up_proj": {
2606
+ "bits": 16,
2607
+ "data_type": "fp"
2608
+ },
2609
+ "model.language_model.layers.23.mlp.shared_expert.down_proj": {
2610
+ "bits": 16,
2611
+ "data_type": "fp"
2612
+ },
2613
+ "model.language_model.layers.24.mlp.shared_expert.gate_proj": {
2614
+ "bits": 16,
2615
+ "data_type": "fp"
2616
+ },
2617
+ "model.language_model.layers.24.mlp.shared_expert.up_proj": {
2618
+ "bits": 16,
2619
+ "data_type": "fp"
2620
+ },
2621
+ "model.language_model.layers.24.mlp.shared_expert.down_proj": {
2622
+ "bits": 16,
2623
+ "data_type": "fp"
2624
+ },
2625
+ "model.language_model.layers.25.mlp.shared_expert.gate_proj": {
2626
+ "bits": 16,
2627
+ "data_type": "fp"
2628
+ },
2629
+ "model.language_model.layers.25.mlp.shared_expert.up_proj": {
2630
+ "bits": 16,
2631
+ "data_type": "fp"
2632
+ },
2633
+ "model.language_model.layers.25.mlp.shared_expert.down_proj": {
2634
+ "bits": 16,
2635
+ "data_type": "fp"
2636
+ },
2637
+ "model.language_model.layers.26.mlp.shared_expert.gate_proj": {
2638
+ "bits": 16,
2639
+ "data_type": "fp"
2640
+ },
2641
+ "model.language_model.layers.26.mlp.shared_expert.up_proj": {
2642
+ "bits": 16,
2643
+ "data_type": "fp"
2644
+ },
2645
+ "model.language_model.layers.26.mlp.shared_expert.down_proj": {
2646
+ "bits": 16,
2647
+ "data_type": "fp"
2648
+ },
2649
+ "model.language_model.layers.27.mlp.shared_expert.gate_proj": {
2650
+ "bits": 16,
2651
+ "data_type": "fp"
2652
+ },
2653
+ "model.language_model.layers.27.mlp.shared_expert.up_proj": {
2654
+ "bits": 16,
2655
+ "data_type": "fp"
2656
+ },
2657
+ "model.language_model.layers.27.mlp.shared_expert.down_proj": {
2658
+ "bits": 16,
2659
+ "data_type": "fp"
2660
+ },
2661
+ "model.language_model.layers.28.mlp.shared_expert.gate_proj": {
2662
+ "bits": 16,
2663
+ "data_type": "fp"
2664
+ },
2665
+ "model.language_model.layers.28.mlp.shared_expert.up_proj": {
2666
+ "bits": 16,
2667
+ "data_type": "fp"
2668
+ },
2669
+ "model.language_model.layers.28.mlp.shared_expert.down_proj": {
2670
+ "bits": 16,
2671
+ "data_type": "fp"
2672
+ },
2673
+ "model.language_model.layers.29.mlp.shared_expert.gate_proj": {
2674
+ "bits": 16,
2675
+ "data_type": "fp"
2676
+ },
2677
+ "model.language_model.layers.29.mlp.shared_expert.up_proj": {
2678
+ "bits": 16,
2679
+ "data_type": "fp"
2680
+ },
2681
+ "model.language_model.layers.29.mlp.shared_expert.down_proj": {
2682
+ "bits": 16,
2683
+ "data_type": "fp"
2684
+ },
2685
+ "model.language_model.layers.30.mlp.shared_expert.gate_proj": {
2686
+ "bits": 16,
2687
+ "data_type": "fp"
2688
+ },
2689
+ "model.language_model.layers.30.mlp.shared_expert.up_proj": {
2690
+ "bits": 16,
2691
+ "data_type": "fp"
2692
+ },
2693
+ "model.language_model.layers.30.mlp.shared_expert.down_proj": {
2694
+ "bits": 16,
2695
+ "data_type": "fp"
2696
+ },
2697
+ "model.language_model.layers.31.mlp.shared_expert.gate_proj": {
2698
+ "bits": 16,
2699
+ "data_type": "fp"
2700
+ },
2701
+ "model.language_model.layers.31.mlp.shared_expert.up_proj": {
2702
+ "bits": 16,
2703
+ "data_type": "fp"
2704
+ },
2705
+ "model.language_model.layers.31.mlp.shared_expert.down_proj": {
2706
+ "bits": 16,
2707
+ "data_type": "fp"
2708
+ },
2709
+ "model.language_model.layers.32.mlp.shared_expert.gate_proj": {
2710
+ "bits": 16,
2711
+ "data_type": "fp"
2712
+ },
2713
+ "model.language_model.layers.32.mlp.shared_expert.up_proj": {
2714
+ "bits": 16,
2715
+ "data_type": "fp"
2716
+ },
2717
+ "model.language_model.layers.32.mlp.shared_expert.down_proj": {
2718
+ "bits": 16,
2719
+ "data_type": "fp"
2720
+ },
2721
+ "model.language_model.layers.33.mlp.shared_expert.gate_proj": {
2722
+ "bits": 16,
2723
+ "data_type": "fp"
2724
+ },
2725
+ "model.language_model.layers.33.mlp.shared_expert.up_proj": {
2726
+ "bits": 16,
2727
+ "data_type": "fp"
2728
+ },
2729
+ "model.language_model.layers.33.mlp.shared_expert.down_proj": {
2730
+ "bits": 16,
2731
+ "data_type": "fp"
2732
+ },
2733
+ "model.language_model.layers.34.mlp.shared_expert.gate_proj": {
2734
+ "bits": 16,
2735
+ "data_type": "fp"
2736
+ },
2737
+ "model.language_model.layers.34.mlp.shared_expert.up_proj": {
2738
+ "bits": 16,
2739
+ "data_type": "fp"
2740
+ },
2741
+ "model.language_model.layers.34.mlp.shared_expert.down_proj": {
2742
+ "bits": 16,
2743
+ "data_type": "fp"
2744
+ },
2745
+ "model.language_model.layers.35.mlp.shared_expert.gate_proj": {
2746
+ "bits": 16,
2747
+ "data_type": "fp"
2748
+ },
2749
+ "model.language_model.layers.35.mlp.shared_expert.up_proj": {
2750
+ "bits": 16,
2751
+ "data_type": "fp"
2752
+ },
2753
+ "model.language_model.layers.35.mlp.shared_expert.down_proj": {
2754
+ "bits": 16,
2755
+ "data_type": "fp"
2756
+ },
2757
+ "model.language_model.layers.36.mlp.shared_expert.gate_proj": {
2758
+ "bits": 16,
2759
+ "data_type": "fp"
2760
+ },
2761
+ "model.language_model.layers.36.mlp.shared_expert.up_proj": {
2762
+ "bits": 16,
2763
+ "data_type": "fp"
2764
+ },
2765
+ "model.language_model.layers.36.mlp.shared_expert.down_proj": {
2766
+ "bits": 16,
2767
+ "data_type": "fp"
2768
+ },
2769
+ "model.language_model.layers.37.mlp.shared_expert.gate_proj": {
2770
+ "bits": 16,
2771
+ "data_type": "fp"
2772
+ },
2773
+ "model.language_model.layers.37.mlp.shared_expert.up_proj": {
2774
+ "bits": 16,
2775
+ "data_type": "fp"
2776
+ },
2777
+ "model.language_model.layers.37.mlp.shared_expert.down_proj": {
2778
+ "bits": 16,
2779
+ "data_type": "fp"
2780
+ },
2781
+ "model.language_model.layers.38.mlp.shared_expert.gate_proj": {
2782
+ "bits": 16,
2783
+ "data_type": "fp"
2784
+ },
2785
+ "model.language_model.layers.38.mlp.shared_expert.up_proj": {
2786
+ "bits": 16,
2787
+ "data_type": "fp"
2788
+ },
2789
+ "model.language_model.layers.38.mlp.shared_expert.down_proj": {
2790
+ "bits": 16,
2791
+ "data_type": "fp"
2792
+ },
2793
+ "model.language_model.layers.39.mlp.shared_expert.gate_proj": {
2794
+ "bits": 16,
2795
+ "data_type": "fp"
2796
+ },
2797
+ "model.language_model.layers.39.mlp.shared_expert.up_proj": {
2798
+ "bits": 16,
2799
+ "data_type": "fp"
2800
+ },
2801
+ "model.language_model.layers.39.mlp.shared_expert.down_proj": {
2802
+ "bits": 16,
2803
+ "data_type": "fp"
2804
+ },
2805
+ "model.language_model.layers.40.mlp.shared_expert.gate_proj": {
2806
+ "bits": 16,
2807
+ "data_type": "fp"
2808
+ },
2809
+ "model.language_model.layers.40.mlp.shared_expert.up_proj": {
2810
+ "bits": 16,
2811
+ "data_type": "fp"
2812
+ },
2813
+ "model.language_model.layers.40.mlp.shared_expert.down_proj": {
2814
+ "bits": 16,
2815
+ "data_type": "fp"
2816
+ },
2817
+ "model.language_model.layers.41.mlp.shared_expert.gate_proj": {
2818
+ "bits": 16,
2819
+ "data_type": "fp"
2820
+ },
2821
+ "model.language_model.layers.41.mlp.shared_expert.up_proj": {
2822
+ "bits": 16,
2823
+ "data_type": "fp"
2824
+ },
2825
+ "model.language_model.layers.41.mlp.shared_expert.down_proj": {
2826
+ "bits": 16,
2827
+ "data_type": "fp"
2828
+ },
2829
+ "model.language_model.layers.42.mlp.shared_expert.gate_proj": {
2830
+ "bits": 16,
2831
+ "data_type": "fp"
2832
+ },
2833
+ "model.language_model.layers.42.mlp.shared_expert.up_proj": {
2834
+ "bits": 16,
2835
+ "data_type": "fp"
2836
+ },
2837
+ "model.language_model.layers.42.mlp.shared_expert.down_proj": {
2838
+ "bits": 16,
2839
+ "data_type": "fp"
2840
+ },
2841
+ "model.language_model.layers.43.mlp.shared_expert.gate_proj": {
2842
+ "bits": 16,
2843
+ "data_type": "fp"
2844
+ },
2845
+ "model.language_model.layers.43.mlp.shared_expert.up_proj": {
2846
+ "bits": 16,
2847
+ "data_type": "fp"
2848
+ },
2849
+ "model.language_model.layers.43.mlp.shared_expert.down_proj": {
2850
+ "bits": 16,
2851
+ "data_type": "fp"
2852
+ },
2853
+ "model.language_model.layers.44.mlp.shared_expert.gate_proj": {
2854
+ "bits": 16,
2855
+ "data_type": "fp"
2856
+ },
2857
+ "model.language_model.layers.44.mlp.shared_expert.up_proj": {
2858
+ "bits": 16,
2859
+ "data_type": "fp"
2860
+ },
2861
+ "model.language_model.layers.44.mlp.shared_expert.down_proj": {
2862
+ "bits": 16,
2863
+ "data_type": "fp"
2864
+ },
2865
+ "model.language_model.layers.45.mlp.shared_expert.gate_proj": {
2866
+ "bits": 16,
2867
+ "data_type": "fp"
2868
+ },
2869
+ "model.language_model.layers.45.mlp.shared_expert.up_proj": {
2870
+ "bits": 16,
2871
+ "data_type": "fp"
2872
+ },
2873
+ "model.language_model.layers.45.mlp.shared_expert.down_proj": {
2874
+ "bits": 16,
2875
+ "data_type": "fp"
2876
+ },
2877
+ "model.language_model.layers.46.mlp.shared_expert.gate_proj": {
2878
+ "bits": 16,
2879
+ "data_type": "fp"
2880
+ },
2881
+ "model.language_model.layers.46.mlp.shared_expert.up_proj": {
2882
+ "bits": 16,
2883
+ "data_type": "fp"
2884
+ },
2885
+ "model.language_model.layers.46.mlp.shared_expert.down_proj": {
2886
+ "bits": 16,
2887
+ "data_type": "fp"
2888
+ },
2889
+ "model.language_model.layers.47.mlp.shared_expert.gate_proj": {
2890
+ "bits": 16,
2891
+ "data_type": "fp"
2892
+ },
2893
+ "model.language_model.layers.47.mlp.shared_expert.up_proj": {
2894
+ "bits": 16,
2895
+ "data_type": "fp"
2896
+ },
2897
+ "model.language_model.layers.47.mlp.shared_expert.down_proj": {
2898
+ "bits": 16,
2899
+ "data_type": "fp"
2900
+ },
2901
+ "model.language_model.layers.1.ple.key_proj": {
2902
+ "bits": 16,
2903
+ "data_type": "float"
2904
+ },
2905
+ "model.language_model.layers.1.ple.value_proj": {
2906
+ "bits": 16,
2907
+ "data_type": "float"
2908
+ },
2909
+ ".*embed_tokens.*": {
2910
+ "bits": 16,
2911
+ "data_type": "float"
2912
+ },
2913
+ ".*visual.*": {
2914
+ "bits": 16,
2915
+ "data_type": "float"
2916
+ },
2917
+ ".*linear_attn.*": {
2918
+ "bits": 16,
2919
+ "data_type": "float"
2920
+ },
2921
+ ".*self_attn.*": {
2922
+ "bits": 16,
2923
+ "data_type": "float"
2924
+ },
2925
+ ".*hyper_connection.*": {
2926
+ "bits": 16,
2927
+ "data_type": "float"
2928
+ },
2929
+ ".*mlp\\.gate.*": {
2930
+ "bits": 16,
2931
+ "data_type": "float"
2932
+ },
2933
+ ".*shared_expert.*": {
2934
+ "bits": 16,
2935
+ "data_type": "float"
2936
+ },
2937
+ ".*in_proj_a.*": {
2938
+ "bits": 16,
2939
+ "data_type": "float"
2940
+ },
2941
+ ".*in_proj_b.*": {
2942
+ "bits": 16,
2943
+ "data_type": "float"
2944
+ },
2945
+ ".*ple.*": {
2946
+ "bits": 16,
2947
+ "data_type": "float"
2948
+ },
2949
+ ".*mtp.*": {
2950
+ "bits": 16,
2951
+ "data_type": "float"
2952
+ },
2953
+ ".*indexer.*": {
2954
+ "bits": 16,
2955
+ "data_type": "float"
2956
+ },
2957
+ ".*model\\.language_model\\.layers\\.0\\.mlp\\.gate.*": {
2958
+ "bits": 16,
2959
+ "data_type": "float"
2960
+ },
2961
+ ".*model\\.language_model\\.layers\\.1\\.mlp\\.gate.*": {
2962
+ "bits": 16,
2963
+ "data_type": "float"
2964
+ },
2965
+ ".*model\\.language_model\\.layers\\.2\\.mlp\\.gate.*": {
2966
+ "bits": 16,
2967
+ "data_type": "float"
2968
+ },
2969
+ ".*model\\.language_model\\.layers\\.3\\.mlp\\.gate.*": {
2970
+ "bits": 16,
2971
+ "data_type": "float"
2972
+ },
2973
+ ".*model\\.language_model\\.layers\\.4\\.mlp\\.gate.*": {
2974
+ "bits": 16,
2975
+ "data_type": "float"
2976
+ },
2977
+ ".*model\\.language_model\\.layers\\.5\\.mlp\\.gate.*": {
2978
+ "bits": 16,
2979
+ "data_type": "float"
2980
+ },
2981
+ ".*model\\.language_model\\.layers\\.6\\.mlp\\.gate.*": {
2982
+ "bits": 16,
2983
+ "data_type": "float"
2984
+ },
2985
+ ".*model\\.language_model\\.layers\\.7\\.mlp\\.gate.*": {
2986
+ "bits": 16,
2987
+ "data_type": "float"
2988
+ },
2989
+ ".*model\\.language_model\\.layers\\.8\\.mlp\\.gate.*": {
2990
+ "bits": 16,
2991
+ "data_type": "float"
2992
+ },
2993
+ ".*model\\.language_model\\.layers\\.9\\.mlp\\.gate.*": {
2994
+ "bits": 16,
2995
+ "data_type": "float"
2996
+ },
2997
+ ".*model\\.language_model\\.layers\\.10\\.mlp\\.gate.*": {
2998
+ "bits": 16,
2999
+ "data_type": "float"
3000
+ },
3001
+ ".*model\\.language_model\\.layers\\.11\\.mlp\\.gate.*": {
3002
+ "bits": 16,
3003
+ "data_type": "float"
3004
+ },
3005
+ ".*model\\.language_model\\.layers\\.12\\.mlp\\.gate.*": {
3006
+ "bits": 16,
3007
+ "data_type": "float"
3008
+ },
3009
+ ".*model\\.language_model\\.layers\\.13\\.mlp\\.gate.*": {
3010
+ "bits": 16,
3011
+ "data_type": "float"
3012
+ },
3013
+ ".*model\\.language_model\\.layers\\.14\\.mlp\\.gate.*": {
3014
+ "bits": 16,
3015
+ "data_type": "float"
3016
+ },
3017
+ ".*model\\.language_model\\.layers\\.15\\.mlp\\.gate.*": {
3018
+ "bits": 16,
3019
+ "data_type": "float"
3020
+ },
3021
+ ".*model\\.language_model\\.layers\\.16\\.mlp\\.gate.*": {
3022
+ "bits": 16,
3023
+ "data_type": "float"
3024
+ },
3025
+ ".*model\\.language_model\\.layers\\.17\\.mlp\\.gate.*": {
3026
+ "bits": 16,
3027
+ "data_type": "float"
3028
+ },
3029
+ ".*model\\.language_model\\.layers\\.18\\.mlp\\.gate.*": {
3030
+ "bits": 16,
3031
+ "data_type": "float"
3032
+ },
3033
+ ".*model\\.language_model\\.layers\\.19\\.mlp\\.gate.*": {
3034
+ "bits": 16,
3035
+ "data_type": "float"
3036
+ },
3037
+ ".*model\\.language_model\\.layers\\.20\\.mlp\\.gate.*": {
3038
+ "bits": 16,
3039
+ "data_type": "float"
3040
+ },
3041
+ ".*model\\.language_model\\.layers\\.21\\.mlp\\.gate.*": {
3042
+ "bits": 16,
3043
+ "data_type": "float"
3044
+ },
3045
+ ".*model\\.language_model\\.layers\\.22\\.mlp\\.gate.*": {
3046
+ "bits": 16,
3047
+ "data_type": "float"
3048
+ },
3049
+ ".*model\\.language_model\\.layers\\.23\\.mlp\\.gate.*": {
3050
+ "bits": 16,
3051
+ "data_type": "float"
3052
+ },
3053
+ ".*model\\.language_model\\.layers\\.24\\.mlp\\.gate.*": {
3054
+ "bits": 16,
3055
+ "data_type": "float"
3056
+ },
3057
+ ".*model\\.language_model\\.layers\\.25\\.mlp\\.gate.*": {
3058
+ "bits": 16,
3059
+ "data_type": "float"
3060
+ },
3061
+ ".*model\\.language_model\\.layers\\.26\\.mlp\\.gate.*": {
3062
+ "bits": 16,
3063
+ "data_type": "float"
3064
+ },
3065
+ ".*model\\.language_model\\.layers\\.27\\.mlp\\.gate.*": {
3066
+ "bits": 16,
3067
+ "data_type": "float"
3068
+ },
3069
+ ".*model\\.language_model\\.layers\\.28\\.mlp\\.gate.*": {
3070
+ "bits": 16,
3071
+ "data_type": "float"
3072
+ },
3073
+ ".*model\\.language_model\\.layers\\.29\\.mlp\\.gate.*": {
3074
+ "bits": 16,
3075
+ "data_type": "float"
3076
+ },
3077
+ ".*model\\.language_model\\.layers\\.30\\.mlp\\.gate.*": {
3078
+ "bits": 16,
3079
+ "data_type": "float"
3080
+ },
3081
+ ".*model\\.language_model\\.layers\\.31\\.mlp\\.gate.*": {
3082
+ "bits": 16,
3083
+ "data_type": "float"
3084
+ },
3085
+ ".*model\\.language_model\\.layers\\.32\\.mlp\\.gate.*": {
3086
+ "bits": 16,
3087
+ "data_type": "float"
3088
+ },
3089
+ ".*model\\.language_model\\.layers\\.33\\.mlp\\.gate.*": {
3090
+ "bits": 16,
3091
+ "data_type": "float"
3092
+ },
3093
+ ".*model\\.language_model\\.layers\\.34\\.mlp\\.gate.*": {
3094
+ "bits": 16,
3095
+ "data_type": "float"
3096
+ },
3097
+ ".*model\\.language_model\\.layers\\.35\\.mlp\\.gate.*": {
3098
+ "bits": 16,
3099
+ "data_type": "float"
3100
+ },
3101
+ ".*model\\.language_model\\.layers\\.36\\.mlp\\.gate.*": {
3102
+ "bits": 16,
3103
+ "data_type": "float"
3104
+ },
3105
+ ".*model\\.language_model\\.layers\\.37\\.mlp\\.gate.*": {
3106
+ "bits": 16,
3107
+ "data_type": "float"
3108
+ },
3109
+ ".*model\\.language_model\\.layers\\.38\\.mlp\\.gate.*": {
3110
+ "bits": 16,
3111
+ "data_type": "float"
3112
+ },
3113
+ ".*model\\.language_model\\.layers\\.39\\.mlp\\.gate.*": {
3114
+ "bits": 16,
3115
+ "data_type": "float"
3116
+ },
3117
+ ".*model\\.language_model\\.layers\\.40\\.mlp\\.gate.*": {
3118
+ "bits": 16,
3119
+ "data_type": "float"
3120
+ },
3121
+ ".*model\\.language_model\\.layers\\.41\\.mlp\\.gate.*": {
3122
+ "bits": 16,
3123
+ "data_type": "float"
3124
+ },
3125
+ ".*model\\.language_model\\.layers\\.42\\.mlp\\.gate.*": {
3126
+ "bits": 16,
3127
+ "data_type": "float"
3128
+ },
3129
+ ".*model\\.language_model\\.layers\\.43\\.mlp\\.gate.*": {
3130
+ "bits": 16,
3131
+ "data_type": "float"
3132
+ },
3133
+ ".*model\\.language_model\\.layers\\.44\\.mlp\\.gate.*": {
3134
+ "bits": 16,
3135
+ "data_type": "float"
3136
+ },
3137
+ ".*model\\.language_model\\.layers\\.45\\.mlp\\.gate.*": {
3138
+ "bits": 16,
3139
+ "data_type": "float"
3140
+ },
3141
+ ".*model\\.language_model\\.layers\\.46\\.mlp\\.gate.*": {
3142
+ "bits": 16,
3143
+ "data_type": "float"
3144
+ },
3145
+ ".*model\\.language_model\\.layers\\.47\\.mlp\\.gate.*": {
3146
+ "bits": 16,
3147
+ "data_type": "float"
3148
+ }
3149
+ }
3150
+ }
tokenizer.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:06b9509352d2af50381ab2247e083b80d32d5c0aba91c272ca9ff729b6a0e523
3
+ size 19989325
tokenizer_config.json ADDED
@@ -0,0 +1,33 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": false,
3
+ "audio_bos_token": "<|audio_start|>",
4
+ "audio_eos_token": "<|audio_end|>",
5
+ "audio_token": "<|audio_pad|>",
6
+ "backend": "tokenizers",
7
+ "bos_token": null,
8
+ "clean_up_tokenization_spaces": false,
9
+ "eos_token": "<|im_end|>",
10
+ "errors": "replace",
11
+ "image_token": "<|image_pad|>",
12
+ "is_local": true,
13
+ "local_files_only": false,
14
+ "model_max_length": 262144,
15
+ "model_specific_special_tokens": {
16
+ "audio_bos_token": "<|audio_start|>",
17
+ "audio_eos_token": "<|audio_end|>",
18
+ "audio_token": "<|audio_pad|>",
19
+ "image_token": "<|image_pad|>",
20
+ "video_token": "<|video_pad|>",
21
+ "vision_bos_token": "<|vision_start|>",
22
+ "vision_eos_token": "<|vision_end|>"
23
+ },
24
+ "pad_token": "<|endoftext|>",
25
+ "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
26
+ "processor_class": "Qwen3VLProcessor",
27
+ "split_special_tokens": false,
28
+ "tokenizer_class": "Qwen2Tokenizer",
29
+ "unk_token": null,
30
+ "video_token": "<|video_pad|>",
31
+ "vision_bos_token": "<|vision_start|>",
32
+ "vision_eos_token": "<|vision_end|>"
33
+ }