Upload character-level GPT model for WikiText-2

config.json

@@ -0,0 +1,12 @@
+{
+    "model_type": "custom_char_gpt",
+    "vocab_size": 283,
+    "n_layer": 8,
+    "n_head": 8,
+    "n_embd": 512,
+    "block_size": 512,
+    "dropout": 0.1,
+    "architectures": [
+        "GPTLanguageModel"
+    ]
+}

modeling_custom_char_gpt.py

@@ -0,0 +1,56 @@
+# ACTION REQUIRED: Manually copy model code here.
+# See warning message in notebook output.
+
+# Add imports like: 
+# import torch
+# import torch.nn as nn
+# etc.
+
+
+import torch
+import torch.nn as nn
+from torch.nn import functional as F
+import math
+
+# --- RoPE Helper Functions (Copied for model file) ---
+def precompute_freqs_cis(dim: int, end: int, theta: float = 10000.0, device='cpu'):
+    freqs_part = torch.arange(0, dim, 2)[: (dim // 2)].float() / dim
+    freqs = 1.0 / (theta ** freqs_part).to(device)
+    t = torch.arange(end, device=device)
+    freqs = torch.outer(t, freqs).float()
+    freqs_cis = torch.polar(torch.ones_like(freqs), freqs)
+    return freqs_cis
+
+def reshape_for_broadcast(freqs_cis: torch.Tensor, x: torch.Tensor):
+    ndim = x.ndim
+    assert 0 <= 1 < ndim
+    if not freqs_cis.shape == (x.shape[1], x.shape[-1]): # Check dimensions used
+         raise ValueError(f"Freqs shape {freqs_cis.shape} does not match x shape {x.shape} at dims 1 and -1")
+    shape = [1] * (ndim - 2) + list(freqs_cis.shape)
+    return freqs_cis.view(*shape)
+
+def apply_rotary_emb(xq: torch.Tensor, xk: torch.Tensor, freqs_cis: torch.Tensor):
+    xq_ = torch.view_as_complex(xq.float().reshape(*xq.shape[:-1], -1, 2))
+    xk_ = torch.view_as_complex(xk.float().reshape(*xk.shape[:-1], -1, 2))
+    freqs_cis = reshape_for_broadcast(freqs_cis, xq_)
+    xq_out_complex = xq_ * freqs_cis
+    xk_out_complex = xk_ * freqs_cis
+    xq_out = torch.view_as_real(xq_out_complex).flatten(start_dim=2)
+    xk_out = torch.view_as_real(xk_out_complex).flatten(start_dim=2)
+    return xq_out.type_as(xq), xk_out.type_as(xk)
+# --- End RoPE Helpers ---
+
+# --- Add Head Class Definition Below ---
+
+
+# --- Add MultiHeadAttention Class Definition Below ---
+
+
+# --- Add SwiGLU Class Definition Below ---
+
+
+# --- Add Block Class Definition Below ---
+
+
+# --- Add GPTLanguageModel Class Definition Below ---
+

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a1628986e6edc29e91d6a43957ee7351b2afca2a1f08642b2912524f8723bf8e
+size 169386430

tokenizer_config.json

@@ -0,0 +1,6 @@
+{
+    "model_max_length": 512,
+    "tokenizer_class": "CustomCharTokenizer",
+    "vocab_size": 283,
+    "chars": "\n !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^`abcdefghijklmnopqrstuvwxyz|~¡£¥§°±²³µ·½ÁÅÆÉÍÎÖ×ØÚÜÞàáâãäåçèéêëìíîñòóôöøúûüĀāăćčĐđėīŁłńŌōśşšūųŻžơưʻʿ̃αβγκμСавекостяاحصلنه्กงณตมยรลัาิ่์გდვზიკორსუცძწხჯ჻ḥṃṅṣṭṯảấầắễệịớửỳ‑–—‘’“”„†…′″⁄₤€₹⅓⅔→−≤☉♭♯〈〉のァアキスットプュリルヴ・動場大戦攻機殻火礮空隊～"
+}

vocab.json

@@ -0,0 +1,285 @@
+{
+    "\n": 0,
+    " ": 1,
+    "!": 2,
+    "\"": 3,
+    "#": 4,
+    "$": 5,
+    "%": 6,
+    "&": 7,
+    "'": 8,
+    "(": 9,
+    ")": 10,
+    "*": 11,
+    "+": 12,
+    ",": 13,
+    "-": 14,
+    ".": 15,
+    "/": 16,
+    "0": 17,
+    "1": 18,
+    "2": 19,
+    "3": 20,
+    "4": 21,
+    "5": 22,
+    "6": 23,
+    "7": 24,
+    "8": 25,
+    "9": 26,
+    ":": 27,
+    ";": 28,
+    "<": 29,
+    "=": 30,
+    ">": 31,
+    "?": 32,
+    "@": 33,
+    "A": 34,
+    "B": 35,
+    "C": 36,
+    "D": 37,
+    "E": 38,
+    "F": 39,
+    "G": 40,
+    "H": 41,
+    "I": 42,
+    "J": 43,
+    "K": 44,
+    "L": 45,
+    "M": 46,
+    "N": 47,
+    "O": 48,
+    "P": 49,
+    "Q": 50,
+    "R": 51,
+    "S": 52,
+    "T": 53,
+    "U": 54,
+    "V": 55,
+    "W": 56,
+    "X": 57,
+    "Y": 58,
+    "Z": 59,
+    "[": 60,
+    "\\": 61,
+    "]": 62,
+    "^": 63,
+    "`": 64,
+    "a": 65,
+    "b": 66,
+    "c": 67,
+    "d": 68,
+    "e": 69,
+    "f": 70,
+    "g": 71,
+    "h": 72,
+    "i": 73,
+    "j": 74,
+    "k": 75,
+    "l": 76,
+    "m": 77,
+    "n": 78,
+    "o": 79,
+    "p": 80,
+    "q": 81,
+    "r": 82,
+    "s": 83,
+    "t": 84,
+    "u": 85,
+    "v": 86,
+    "w": 87,
+    "x": 88,
+    "y": 89,
+    "z": 90,
+    "|": 91,
+    "~": 92,
+    "¡": 93,
+    "£": 94,
+    "¥": 95,
+    "§": 96,
+    "°": 97,
+    "±": 98,
+    "²": 99,
+    "³": 100,
+    "µ": 101,
+    "·": 102,
+    "½": 103,
+    "Á": 104,
+    "Å": 105,
+    "Æ": 106,
+    "É": 107,
+    "Í": 108,
+    "Î": 109,
+    "Ö": 110,
+    "×": 111,
+    "Ø": 112,
+    "Ú": 113,
+    "Ü": 114,
+    "Þ": 115,
+    "à": 116,
+    "á": 117,
+    "â": 118,
+    "ã": 119,
+    "ä": 120,
+    "å": 121,
+    "ç": 122,
+    "è": 123,
+    "é": 124,
+    "ê": 125,
+    "ë": 126,
+    "ì": 127,
+    "í": 128,
+    "î": 129,
+    "ñ": 130,
+    "ò": 131,
+    "ó": 132,
+    "ô": 133,
+    "ö": 134,
+    "ø": 135,
+    "ú": 136,
+    "û": 137,
+    "ü": 138,
+    "Ā": 139,
+    "ā": 140,
+    "ă": 141,
+    "ć": 142,
+    "č": 143,
+    "Đ": 144,
+    "đ": 145,
+    "ė": 146,
+    "ī": 147,
+    "Ł": 148,
+    "ł": 149,
+    "ń": 150,
+    "Ō": 151,
+    "ō": 152,
+    "ś": 153,
+    "ş": 154,
+    "š": 155,
+    "ū": 156,
+    "ų": 157,
+    "Ż": 158,
+    "ž": 159,
+    "ơ": 160,
+    "ư": 161,
+    "ʻ": 162,
+    "ʿ": 163,
+    "̃": 164,
+    "α": 165,
+    "β": 166,
+    "γ": 167,
+    "κ": 168,
+    "μ": 169,
+    "С": 170,
+    "а": 171,
+    "в": 172,
+    "е": 173,
+    "к": 174,
+    "о": 175,
+    "с": 176,
+    "т": 177,
+    "я": 178,
+    "ا": 179,
+    "ح": 180,
+    "ص": 181,
+    "ل": 182,
+    "ن": 183,
+    "ه": 184,
+    "्": 185,
+    "ก": 186,
+    "ง": 187,
+    "ณ": 188,
+    "ต": 189,
+    "ม": 190,
+    "ย": 191,
+    "ร": 192,
+    "ล": 193,
+    "ั": 194,
+    "า": 195,
+    "ิ": 196,
+    "่": 197,
+    "์": 198,
+    "გ": 199,
+    "დ": 200,
+    "ვ": 201,
+    "ზ": 202,
+    "ი": 203,
+    "კ": 204,
+    "ო": 205,
+    "რ": 206,
+    "ს": 207,
+    "უ": 208,
+    "ც": 209,
+    "ძ": 210,
+    "წ": 211,
+    "ხ": 212,
+    "ჯ": 213,
+    "჻": 214,
+    "ḥ": 215,
+    "ṃ": 216,
+    "ṅ": 217,
+    "ṣ": 218,
+    "ṭ": 219,
+    "ṯ": 220,
+    "ả": 221,
+    "ấ": 222,
+    "ầ": 223,
+    "ắ": 224,
+    "ễ": 225,
+    "ệ": 226,
+    "ị": 227,
+    "ớ": 228,
+    "ử": 229,
+    "ỳ": 230,
+    "‑": 231,
+    "–": 232,
+    "—": 233,
+    "‘": 234,
+    "’": 235,
+    "“": 236,
+    "”": 237,
+    "„": 238,
+    "†": 239,
+    "…": 240,
+    "′": 241,
+    "″": 242,
+    "⁄": 243,
+    "₤": 244,
+    "€": 245,
+    "₹": 246,
+    "⅓": 247,
+    "⅔": 248,
+    "→": 249,
+    "−": 250,
+    "≤": 251,
+    "☉": 252,
+    "♭": 253,
+    "♯": 254,
+    "〈": 255,
+    "〉": 256,
+    "の": 257,
+    "ァ": 258,
+    "ア": 259,
+    "キ": 260,
+    "ス": 261,
+    "ッ": 262,
+    "ト": 263,
+    "プ": 264,
+    "ュ": 265,
+    "リ": 266,
+    "ル": 267,
+    "ヴ": 268,
+    "・": 269,
+    "動": 270,
+    "場": 271,
+    "大": 272,
+    "戦": 273,
+    "攻": 274,
+    "機": 275,
+    "殻": 276,
+    "火": 277,
+    "礮": 278,
+    "空": 279,
+    "隊": 280,
+    "": 281,
+    "～": 282
+}