Update tokenizer_config.json
Browse filesRemove [/TOOL_CALLS] from the template ( that I "halucinated" )
- tokenizer_config.json +1 -1
tokenizer_config.json
CHANGED
|
@@ -36,7 +36,7 @@
|
|
| 36 |
},
|
| 37 |
{
|
| 38 |
"name": "tool_use",
|
| 39 |
-
"template": "{{bos_token}}{% set user_messages = messages | selectattr('role', 'equalto', 'user') | list %}{% for message in messages %}{% if message['role'] == 'user' %}{% if message == user_messages[-1] %}{{ '[AVAILABLE_TOOLS]'}}{% for tool in tools %}{{ tool }}{% endfor %}{{ '[/AVAILABLE_TOOLS]'}}{{ '[INST]' + message['content'] + '[/INST]' }}{% else %}{{ '[INST]' + message['content'] + '[/INST]' }}{% endif %}{% elif message['role'] == 'assistant' %}{{ ' ' + message['content'] + ' ' + eos_token}}{% elif message['role'] == 'tool_results' %}{{'[TOOL_RESULTS]' + message['content']|string + eos_token
|
| 40 |
}
|
| 41 |
],
|
| 42 |
"clean_up_tokenization_spaces": false,
|
|
|
|
| 36 |
},
|
| 37 |
{
|
| 38 |
"name": "tool_use",
|
| 39 |
+
"template": "{{bos_token}}{% set user_messages = messages | selectattr('role', 'equalto', 'user') | list %}{% for message in messages %}{% if message['role'] == 'user' %}{% if message == user_messages[-1] %}{{ '[AVAILABLE_TOOLS]'}}{% for tool in tools %}{{ tool }}{% endfor %}{{ '[/AVAILABLE_TOOLS]'}}{{ '[INST]' + message['content'] + '[/INST]' }}{% else %}{{ '[INST]' + message['content'] + '[/INST]' }}{% endif %}{% elif message['role'] == 'assistant' %}{{ ' ' + message['content'] + ' ' + eos_token}}{% elif message['role'] == 'tool_results' %}{{'[TOOL_RESULTS]' + message['content']|string + eos_token}}{% elif message['role'] == 'tool_calls' %}{{'[TOOL_CALLS]' + message['content']|string + eos_token+ '[/TOOL_CALLS]'}}{% endif %}{% endfor %}"
|
| 40 |
}
|
| 41 |
],
|
| 42 |
"clean_up_tokenization_spaces": false,
|