danielhanchen commited on
Commit
72bf21d
·
verified ·
1 Parent(s): 6a730ff

Add files using upload-large-folder tool

Browse files
.gitattributes CHANGED
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ tokenizer.json filter=lfs diff=lfs merge=lfs -text
chat_template.jinja ADDED
@@ -0,0 +1,263 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {%- macro format_parameters(properties, required) -%}
2
+ {%- set standard_keys = ['description', 'type', 'properties', 'required', 'nullable'] -%}
3
+ {%- set ns = namespace(found_first=false) -%}
4
+ {%- for key, value in properties | dictsort -%}
5
+ {%- set add_comma = false -%}
6
+ {%- if key not in standard_keys -%}
7
+ {%- if ns.found_first %},{% endif -%}
8
+ {%- set ns.found_first = true -%}
9
+ {{ key }}:{
10
+ {%- if value['description'] -%}
11
+ description:<|"|>{{ value['description'] }}<|"|>
12
+ {%- set add_comma = true -%}
13
+ {%- endif -%}
14
+ {%- if value['nullable'] %}
15
+ {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}
16
+ nullable:true
17
+ {%- endif -%}
18
+ {%- if value['type'] | upper == 'STRING' -%}
19
+ {%- if value['enum'] -%}
20
+ {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}
21
+ enum:{{ format_argument(value['enum']) }}
22
+ {%- endif -%}
23
+ {%- elif value['type'] | upper == 'OBJECT' -%}
24
+ ,properties:{
25
+ {%- if value['properties'] is defined and value['properties'] is mapping -%}
26
+ {{- format_parameters(value['properties'], value['required'] | default([])) -}}
27
+ {%- elif value is mapping -%}
28
+ {{- format_parameters(value, value['required'] | default([])) -}}
29
+ {%- endif -%}
30
+ }
31
+ {%- if value['required'] -%}
32
+ ,required:[
33
+ {%- for item in value['required'] | default([]) -%}
34
+ <|"|>{{- item -}}<|"|>
35
+ {%- if not loop.last %},{% endif -%}
36
+ {%- endfor -%}
37
+ ]
38
+ {%- endif -%}
39
+ {%- elif value['type'] | upper == 'ARRAY' -%}
40
+ {%- if value['items'] is mapping and value['items'] -%}
41
+ ,items:{
42
+ {%- set ns_items = namespace(found_first=false) -%}
43
+ {%- for item_key, item_value in value['items'] | dictsort -%}
44
+ {%- if item_value is not none -%}
45
+ {%- if ns_items.found_first %},{% endif -%}
46
+ {%- set ns_items.found_first = true -%}
47
+ {%- if item_key == 'properties' -%}
48
+ properties:{
49
+ {%- if item_value is mapping -%}
50
+ {{- format_parameters(item_value, value['items']['required'] | default([])) -}}
51
+ {%- endif -%}
52
+ }
53
+ {%- elif item_key == 'required' -%}
54
+ required:[
55
+ {%- for req_item in item_value -%}
56
+ <|"|>{{- req_item -}}<|"|>
57
+ {%- if not loop.last %},{% endif -%}
58
+ {%- endfor -%}
59
+ ]
60
+ {%- elif item_key == 'type' -%}
61
+ {%- if item_value is string -%}
62
+ type:{{ format_argument(item_value | upper) }}
63
+ {%- else -%}
64
+ type:{{ format_argument(item_value | map('upper') | list) }}
65
+ {%- endif -%}
66
+ {%- else -%}
67
+ {{ item_key }}:{{ format_argument(item_value) }}
68
+ {%- endif -%}
69
+ {%- endif -%}
70
+ {%- endfor -%}
71
+ }
72
+ {%- endif -%}
73
+ {%- endif -%}
74
+ {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}
75
+ type:<|"|>{{ value['type'] | upper }}<|"|>}
76
+ {%- endif -%}
77
+ {%- endfor -%}
78
+ {%- endmacro -%}
79
+ {%- macro format_function_declaration(tool_data) -%}
80
+ declaration:{{- tool_data['function']['name'] -}}{description:<|"|>{{- tool_data['function']['description'] -}}<|"|>
81
+ {%- set params = tool_data['function']['parameters'] -%}
82
+ {%- if params -%}
83
+ ,parameters:{
84
+ {%- if params['properties'] -%}
85
+ properties:{ {{- format_parameters(params['properties'], params['required']) -}} },
86
+ {%- endif -%}
87
+ {%- if params['required'] -%}
88
+ required:[
89
+ {%- for item in params['required'] -%}
90
+ <|"|>{{- item -}}<|"|>
91
+ {{- ',' if not loop.last -}}
92
+ {%- endfor -%}
93
+ ],
94
+ {%- endif -%}
95
+ {%- if params['type'] -%}
96
+ type:<|"|>{{- params['type'] | upper -}}<|"|>}
97
+ {%- endif -%}
98
+ {%- endif -%}
99
+ {%- if 'response' in tool_data['function'] -%}
100
+ {%- set response_declaration = tool_data['function']['response'] -%}
101
+ ,response:{
102
+ {%- if response_declaration['description'] -%}
103
+ description:<|"|>{{- response_declaration['description'] -}}<|"|>,
104
+ {%- endif -%}
105
+ {%- if response_declaration['type'] | upper == 'OBJECT' -%}
106
+ type:<|"|>{{- response_declaration['type'] | upper -}}<|"|>}
107
+ {%- endif -%}
108
+ {%- endif -%}
109
+ }
110
+ {%- endmacro -%}
111
+ {%- macro format_argument(argument, escape_keys=True) -%}
112
+ {%- if argument is string -%}
113
+ {{- '<|"|>' + argument + '<|"|>' -}}
114
+ {%- elif argument is boolean -%}
115
+ {{- 'true' if argument else 'false' -}}
116
+ {%- elif argument is mapping -%}
117
+ {{- '{' -}}
118
+ {%- set ns = namespace(found_first=false) -%}
119
+ {%- for key, value in argument | dictsort -%}
120
+ {%- if ns.found_first %},{% endif -%}
121
+ {%- set ns.found_first = true -%}
122
+ {%- if escape_keys -%}
123
+ {{- '<|"|>' + key + '<|"|>' -}}
124
+ {%- else -%}
125
+ {{- key -}}
126
+ {%- endif -%}
127
+ :{{- format_argument(value, escape_keys=escape_keys) -}}
128
+ {%- endfor -%}
129
+ {{- '}' -}}
130
+ {%- elif argument is sequence -%}
131
+ {{- '[' -}}
132
+ {%- for item in argument -%}
133
+ {{- format_argument(item, escape_keys=escape_keys) -}}
134
+ {%- if not loop.last %},{% endif -%}
135
+ {%- endfor -%}
136
+ {{- ']' -}}
137
+ {%- else -%}
138
+ {{- argument -}}
139
+ {%- endif -%}
140
+ {%- endmacro -%}
141
+ {%- macro strip_thinking(text) -%}
142
+ {%- set ns = namespace(result='') -%}
143
+ {%- for part in text.split('<channel|>') -%}
144
+ {%- if '<|channel>' in part -%}
145
+ {%- set ns.result = ns.result + part.split('<|channel>')[0] -%}
146
+ {%- else -%}
147
+ {%- set ns.result = ns.result + part -%}
148
+ {%- endif -%}
149
+ {%- endfor -%}
150
+ {{- ns.result | trim -}}
151
+ {%- endmacro -%}
152
+
153
+ {%- set ns = namespace(prev_message_type=None) -%}
154
+ {%- set loop_messages = messages -%}
155
+ {{ bos_token }}
156
+ {#- Handle System/Tool Definitions Block -#}
157
+ {%- if (enable_thinking is defined and enable_thinking) or tools or messages[0]['role'] in ['system', 'developer'] -%}
158
+ {{- '<|turn>system\n' -}}
159
+
160
+ {#- Inject Thinking token at the very top of the FIRST system turn -#}
161
+ {%- if enable_thinking is defined and enable_thinking -%}
162
+ {{- '<|think|>' -}}
163
+ {%- set ns.prev_message_type = 'think' -%}
164
+ {%- endif -%}
165
+
166
+ {%- if messages[0]['role'] in ['system', 'developer'] -%}
167
+ {{- messages[0]['content'] | trim -}}
168
+ {%- set loop_messages = messages[1:] -%}
169
+ {%- endif -%}
170
+
171
+ {%- if tools -%}
172
+ {%- for tool in tools %}
173
+ {{- '<|tool>' -}}
174
+ {{- format_function_declaration(tool) | trim -}}
175
+ {{- '<tool|>' -}}
176
+ {%- endfor %}
177
+ {%- set ns.prev_message_type = 'tool' -%}
178
+ {%- endif -%}
179
+
180
+ {{- '<turn|>\n' -}}
181
+ {%- endif %}
182
+
183
+ {#- Loop through messages -#}
184
+ {%- for message in loop_messages -%}
185
+ {%- set ns.prev_message_type = None -%}
186
+ {%- set role = 'model' if message['role'] == 'assistant' else message['role'] -%}
187
+ {{- '<|turn>' + role + '\n' }}
188
+
189
+ {%- if message['tool_calls'] -%}
190
+ {%- for tool_call in message['tool_calls'] -%}
191
+ {%- set function = tool_call['function'] -%}
192
+ {{- '<|tool_call>call:' + function['name'] + '{' -}}
193
+ {%- if function['arguments'] is mapping -%}
194
+ {%- set ns_args = namespace(found_first=false) -%}
195
+ {%- for key, value in function['arguments'] | dictsort -%}
196
+ {%- if ns_args.found_first %},{% endif -%}
197
+ {%- set ns_args.found_first = true -%}
198
+ {{- key -}}:{{- format_argument(value, escape_keys=False) -}}
199
+ {%- endfor -%}
200
+ {%- elif function['arguments'] is string -%}
201
+ {{- function['arguments'] -}}
202
+ {%- endif -%}
203
+ {{- '}<tool_call|>' -}}
204
+ {%- endfor -%}
205
+ {%- set ns.prev_message_type = 'tool_call' -%}
206
+ {%- endif -%}
207
+
208
+ {%- if message['tool_responses'] -%}
209
+ {#- Tool Response handling -#}
210
+ {%- for tool_response in message['tool_responses'] -%}
211
+ {{- '<|tool_response>' -}}
212
+ {%- if tool_response['response'] is mapping -%}
213
+ {{- 'response:' + tool_response['name'] | default('unknown') + '{' -}}
214
+ {%- for key, value in tool_response['response'] | dictsort -%}
215
+ {{- key -}}:{{- format_argument(value, escape_keys=False) -}}
216
+ {%- if not loop.last %},{% endif -%}
217
+ {%- endfor -%}
218
+ {{- '}' -}}
219
+ {%- else -%}
220
+ {{- 'response:' + tool_response['name'] | default('unknown') + '{value:' + format_argument(tool_response['response'], escape_keys=False) + '}' -}}
221
+ {%- endif -%}
222
+ {{- '<tool_response|>' -}}
223
+ {%- endfor -%}
224
+ {%- set ns.prev_message_type = 'tool_response' -%}
225
+ {%- endif -%}
226
+
227
+ {%- if message['content'] is string -%}
228
+ {%- if role == 'model' -%}
229
+ {{- strip_thinking(message['content']) -}}
230
+ {%- else -%}
231
+ {{- message['content'] | trim -}}
232
+ {%- endif -%}
233
+ {%- elif message['content'] is sequence -%}
234
+ {%- for item in message['content'] -%}
235
+ {%- if item['type'] == 'text' -%}
236
+ {%- if role == 'model' -%}
237
+ {{- strip_thinking(item['text']) -}}
238
+ {%- else -%}
239
+ {{- item['text'] | trim -}}
240
+ {%- endif -%}
241
+ {%- elif item['type'] == 'image' -%}
242
+ {{- '\n\n<|image|>\n\n' -}}
243
+ {%- set ns.prev_message_type = 'image' -%}
244
+ {%- elif item['type'] == 'audio' -%}
245
+ {{- '<|audio|>' -}}
246
+ {%- set ns.prev_message_type = 'audio' -%}
247
+ {%- elif item['type'] == 'video' -%}
248
+ {{- '\n\n<|video|>\n\n' -}}
249
+ {%- set ns.prev_message_type = 'video' -%}
250
+ {%- endif -%}
251
+ {%- endfor -%}
252
+ {%- endif -%}
253
+
254
+ {%- if not (message['tool_responses'] and not message['content']) -%}
255
+ {{- '<turn|>\n' -}}
256
+ {%- endif -%}
257
+ {%- endfor -%}
258
+
259
+ {%- if add_generation_prompt -%}
260
+ {%- if ns.prev_message_type != 'tool_response' -%}
261
+ {{- '<|turn>model\n' -}}
262
+ {%- endif -%}
263
+ {%- endif -%}
config.json ADDED
The diff for this file is too large to render. See raw diff
 
generation_config.json ADDED
@@ -0,0 +1,14 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token_id": 2,
3
+ "do_sample": true,
4
+ "eos_token_id": [
5
+ 1,
6
+ 106,
7
+ 50
8
+ ],
9
+ "pad_token_id": 0,
10
+ "temperature": 1.0,
11
+ "top_k": 64,
12
+ "top_p": 0.95,
13
+ "transformers_version": "5.5.0.dev0"
14
+ }
model-00001-of-00002.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:14dce6f7bb00e6a2806aed8825b08c93a131d501b79ed2f845798647b41257ff
3
+ size 4964690708
model-00002-of-00002.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9fcd2f2cfcfb7cd025351071a80af947c9249c1a112b7ffc32163bfab9507f99
3
+ size 3023979920
model.safetensors.index.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cc8d3a0ce36466ccc1278bf987df5f71db1719b9ca6b4118264f45cb627bfe0f
3
+ size 32169626
tokenizer_config.json ADDED
@@ -0,0 +1,94 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "audio_token": "<|audio|>",
3
+ "backend": "tokenizers",
4
+ "boa_token": "<|audio>",
5
+ "boi_token": "<|image>",
6
+ "bos_token": "<bos>",
7
+ "eoa_token": "<audio|>",
8
+ "eoc_token": "<channel|>",
9
+ "eoi_token": "<image|>",
10
+ "eos_token": "<eos>",
11
+ "eot_token": "<turn|>",
12
+ "escape_token": "<|\"|>",
13
+ "etc_token": "<tool_call|>",
14
+ "etd_token": "<tool|>",
15
+ "etr_token": "<tool_response|>",
16
+ "image_token": "<|image|>",
17
+ "is_local": true,
18
+ "mask_token": "<mask>",
19
+ "model_max_length": 1000000000000000019884624838656,
20
+ "model_specific_special_tokens": {
21
+ "audio_token": "<|audio|>",
22
+ "boa_token": "<|audio>",
23
+ "boi_token": "<|image>",
24
+ "eoa_token": "<audio|>",
25
+ "eoc_token": "<channel|>",
26
+ "eoi_token": "<image|>",
27
+ "eot_token": "<turn|>",
28
+ "escape_token": "<|\"|>",
29
+ "etc_token": "<tool_call|>",
30
+ "etd_token": "<tool|>",
31
+ "etr_token": "<tool_response|>",
32
+ "image_token": "<|image|>",
33
+ "soc_token": "<|channel>",
34
+ "sot_token": "<|turn>",
35
+ "stc_token": "<|tool_call>",
36
+ "std_token": "<|tool>",
37
+ "str_token": "<|tool_response>",
38
+ "think_token": "<|think|>",
39
+ "video_token": "<|video|>"
40
+ },
41
+ "pad_token": "<pad>",
42
+ "padding_side": "left",
43
+ "processor_class": "Gemma4Processor",
44
+ "response_schema": {
45
+ "properties": {
46
+ "content": {
47
+ "type": "string"
48
+ },
49
+ "role": {
50
+ "const": "assistant"
51
+ },
52
+ "thinking": {
53
+ "type": "string"
54
+ },
55
+ "tool_calls": {
56
+ "items": {
57
+ "properties": {
58
+ "function": {
59
+ "properties": {
60
+ "arguments": {
61
+ "additionalProperties": {},
62
+ "type": "object",
63
+ "x-parser": "gemma4-tool-call"
64
+ },
65
+ "name": {
66
+ "type": "string"
67
+ }
68
+ },
69
+ "type": "object",
70
+ "x-regex": "call\\:(?P<name>\\w+)(?P<arguments>\\{.*\\})"
71
+ },
72
+ "type": {
73
+ "const": "function"
74
+ }
75
+ },
76
+ "type": "object"
77
+ },
78
+ "type": "array",
79
+ "x-regex-iterator": "<\\|tool_call>(.*?)<tool_call\\|>"
80
+ }
81
+ },
82
+ "type": "object",
83
+ "x-regex": "(\\<\\|channel\\>thought\\n(?P<thinking>.*?)\\<channel\\|\\>)?(?P<content>(?:(?!\\<\\|tool_call\\>)(?!\\<turn\\|\\>).)+)?(?P<tool_calls>\\<\\|tool_call\\>.*\\<tool_call\\|\\>)?(?:\\<turn\\|\\>)?"
84
+ },
85
+ "soc_token": "<|channel>",
86
+ "sot_token": "<|turn>",
87
+ "stc_token": "<|tool_call>",
88
+ "std_token": "<|tool>",
89
+ "str_token": "<|tool_response>",
90
+ "think_token": "<|think|>",
91
+ "tokenizer_class": "GemmaTokenizer",
92
+ "unk_token": "<unk>",
93
+ "video_token": "<|video|>"
94
+ }