Show message dict and og tokenizer

test_template.py

@@ -7,201 +7,183 @@
 from transformers import AutoTokenizer
 
 
-def print_section(title, content):
+def print_section(title, messages, tokenizers, **tokenizer_kwargs):
     """Helper function to print formatted sections"""
     print(f"\n{'=' * 60}")
     print(f"{title}")
     print(f"{'=' * 60}")
-    print(content)
-    print()
+    print(f"\n{messages=}\n")
+    for tokenizer_name, tokenizer in tokenizers.items():
+        print(f"\n{tokenizer_name=}\n")
+        content = tokenizer.apply_chat_template(
+            messages, tokenize=False, **tokenizer_kwargs
+        )
+        print(content)
 
 
 # Initialize tokenizer
-tok = AutoTokenizer.from_pretrained(".")
+local_tokenizer = AutoTokenizer.from_pretrained(".")
+glm_tokenizer = AutoTokenizer.from_pretrained("zai-org/GLM-4.5-Air")
+tokenizers = {"Local": local_tokenizer, "GLM-4.5-Air": glm_tokenizer}
 
 # Only user message
 print_section(
     "User message only",
-    tok.apply_chat_template(
-        [{"role": "user", "content": "What is the capital of France?"}], tokenize=False
-    ),
+    [{"role": "user", "content": "What is the capital of France?"}],
+    tokenizers,
 )
 
 # User message with generation prompt
 print_section(
     "User message with generation prompt",
-    tok.apply_chat_template(
-        [
-            {"role": "user", "content": "What is the capital of France?"},
-        ],
-        tokenize=False,
-        add_generation_prompt=True,
-    ),
+    [{"role": "user", "content": "What is the capital of France?"}],
+    tokenizers,
+    add_generation_prompt=True,
 )
 
 # User message with custom system message
 print_section(
     "Custom system message",
-    tok.apply_chat_template(
-        [
-            {"role": "system", "content": "You are a helpful assistant."},
-            {"role": "user", "content": "What is the capital of France?"},
-        ],
-        tokenize=False,
-    ),
+    [
+        {"role": "system", "content": "You are a helpful assistant."},
+        {"role": "user", "content": "What is the capital of France?"},
+    ],
+    tokenizers,
 )
 
 # Single-turn with assistant response (no think)
 print_section(
     "Single-turn with assistant response (no think)",
-    tok.apply_chat_template(
-        [
-            {"role": "user", "content": "What is the capital of France?"},
-            {"role": "assistant", "content": "The capital of France is Paris."},
-        ],
-        tokenize=False,
-    ),
+    [
+        {"role": "user", "content": "What is the capital of France?"},
+        {"role": "assistant", "content": "The capital of France is Paris."},
+    ],
+    tokenizers,
 )
 
 # Single-turn with think embedded in content
 print_section(
     "Single-turn with think embedded in content",
-    tok.apply_chat_template(
-        [
-            {"role": "user", "content": "What is the capital of France?"},
-            {
-                "role": "assistant",
-                "content": "<think>The user is asking about geography. France is a country in Europe, and its capital city is Paris. This is a straightforward factual question.</think>\nThe capital of France is Paris.",
-            },
-        ],
-        tokenize=False,
-    ),
+    [
+        {"role": "user", "content": "What is the capital of France?"},
+        {
+            "role": "assistant",
+            "content": "<think>The user is asking about geography. France is a country in Europe, and its capital city is Paris. This is a straightforward factual question.</think>\nThe capital of France is Paris.",
+        },
+    ],
+    tokenizers,
 )
 
 # Single-turn with reasoning_content field
 print_section(
     "Single-turn with reasoning_content field",
-    tok.apply_chat_template(
-        [
-            {"role": "user", "content": "What is the capital of France?"},
-            {
-                "role": "assistant",
-                "content": "The capital of France is Paris.",
-                "reasoning_content": "The user is asking about geography. France is a country in Europe, and its capital city is Paris.",
-            },
-        ],
-        tokenize=False,
-    ),
+    [
+        {"role": "user", "content": "What is the capital of France?"},
+        {
+            "role": "assistant",
+            "content": "The capital of France is Paris.",
+            "reasoning_content": "The user is asking about geography. France is a country in Europe, and its capital city is Paris.",
+        },
+    ],
+    tokenizers,
 )
 
 print_section(
     "Single-turn with think section and reasoning_content field",
-    tok.apply_chat_template(
-        [
-            {"role": "user", "content": "What is the capital of France?"},
-            {
-                "role": "assistant",
-                "content": "<think>The user is asking about geography. France is a country in Europe, and its capital city is Paris. This is a straightforward factual question.</think>\nThe capital of France is Paris.",
-                "reasoning_content": "This should not be visible.",
-            },
-        ],
-        tokenize=False,
-    ),
+    [
+        {"role": "user", "content": "What is the capital of France?"},
+        {
+            "role": "assistant",
+            "content": "<think>The user is asking about geography. France is a country in Europe, and its capital city is Paris. This is a straightforward factual question.</think>\nThe capital of France is Paris.",
+            "reasoning_content": "This should not be visible.",
+        },
+    ],
+    tokenizers,
 )
 
 
 # Multi-turn and assistant response with think sections (embedded in content)
 print_section(
     "Multi-turn with think embedded in content",
-    tok.apply_chat_template(
-        [
-            {"role": "user", "content": "What is the capital of France?"},
-            {
-                "role": "assistant",
-                "content": "<think>This is a basic geography question.</think>\nThe capital of France is Paris.",
-            },
-            {"role": "user", "content": "What about Germany?"},
-            {
-                "role": "assistant",
-                "content": "<think>Another geography question. Germany's capital is Berlin.</think>\nThe capital of Germany is Berlin.",
-            },
-        ],
-        tokenize=False,
-    ),
+    [
+        {"role": "user", "content": "What is the capital of France?"},
+        {
+            "role": "assistant",
+            "content": "<think>This is a basic geography question.</think>\nThe capital of France is Paris.",
+        },
+        {"role": "user", "content": "What about Germany?"},
+        {
+            "role": "assistant",
+            "content": "<think>Another geography question. Germany's capital is Berlin.</think>\nThe capital of Germany is Berlin.",
+        },
+    ],
+    tokenizers,
 )
 
 # Multi-turn and assistant response with think sections (embedded in content)
 print_section(
     "Multi-turn with reasoning_content field",
-    tok.apply_chat_template(
-        [
-            {"role": "user", "content": "What is the capital of France?"},
-            {
-                "role": "assistant",
-                "reasoning_content": "The user is asking about geography. France is a country in Europe, and its capital city is Paris.",
-                "content": "The capital of France is Paris.",
-            },
-            {"role": "user", "content": "What about Germany?"},
-            {
-                "role": "assistant",
-                "reasoning_content": "Another geography question. Germany's capital is Berlin.",
-                "content": "The capital of Germany is Berlin.",
-            },
-        ],
-        tokenize=False,
-    ),
+    [
+        {"role": "user", "content": "What is the capital of France?"},
+        {
+            "role": "assistant",
+            "reasoning_content": "The user is asking about geography. France is a country in Europe, and its capital city is Paris.",
+            "content": "The capital of France is Paris.",
+        },
+        {"role": "user", "content": "What about Germany?"},
+        {
+            "role": "assistant",
+            "reasoning_content": "Another geography question. Germany's capital is Berlin.",
+            "content": "The capital of Germany is Berlin.",
+        },
+    ],
+    tokenizers,
 )
 
 # Assistant with only think section, no visible content
 print_section(
     "Assistant with only think section",
-    tok.apply_chat_template(
-        [
-            {
-                "role": "user",
-                "content": "Think about this problem but don't respond yet.",
-            },
-            {
-                "role": "assistant",
-                "content": "<think>The user wants me to think about something but not provide a response yet. I should just show my thinking process without any visible output.</think>",
-            },
-        ],
-        tokenize=False,
-    ),
+    [
+        {
+            "role": "user",
+            "content": "Think about this problem but don't respond yet.",
+        },
+        {
+            "role": "assistant",
+            "content": "<think>The user wants me to think about something but not provide a response yet. I should just show my thinking process without any visible output.</think>",
+        },
+    ],
+    tokenizers,
 )
 
 # Assistant with unfinished think section
 print_section(
     "Assistant with unfinished think section",
-    tok.apply_chat_template(
-        [
-            {
-                "role": "user",
-                "content": "Think about this problem but don't respond yet.",
-            },
-            {
-                "role": "assistant",
-                "content": "<think>The user wants me to think about something but not provide a response yet. I should just",
-            },
-        ],
-        tokenize=False,
-    ),
+    [
+        {
+            "role": "user",
+            "content": "Think about this problem but don't respond yet.",
+        },
+        {
+            "role": "assistant",
+            "content": "<think>The user wants me to think about something but not provide a response yet. I should just",
+        },
+    ],
+    tokenizers,
 )
 
 print_section(
     "Empty reasoning content",
-    tok.apply_chat_template(
-        [
-            {"role": "system", "content": "You are a helpful assistant."},
-            {"role": "user", "content": "Say hello"},
-            {
-                "role": "assistant",
-                "content": "Hello! How can I help you today?",
-                "reasoning_content": "",
-            },
-        ],
-        tokenize=False,
-    ),
+    [
+        {"role": "system", "content": "You are a helpful assistant."},
+        {"role": "user", "content": "Say hello"},
+        {
+            "role": "assistant",
+            "content": "Hello! How can I help you today?",
+            "reasoning_content": "",
+        },
+    ],
+    tokenizers,
 )
 
 
@@ -255,7 +237,9 @@ tools = [
 
 print_section(
     "Single-turn tool use with weather",
-    tok.apply_chat_template(tool_example, tokenize=False, tools=tools),
+    tool_example,
+    tokenizers,
+    tools=tools,
 )
 
 # ============================================================================
@@ -312,5 +296,7 @@ multi_tools = [
 
 print_section(
     "Single-turn with multiple tool calls",
-    tok.apply_chat_template(multi_tool_example, tokenize=False, tools=multi_tools),
+    multi_tool_example,
+    tokenizers,
+    tools=multi_tools,
 )