Duplicate from BlinkDL/rwkv-7-pile

Co-authored-by: BlinkDL <[email protected]>

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,23 @@
+---
+license: apache-2.0
+datasets:
+- EleutherAI/pile
+---
+
+RWKV-7 trained on the Pile w/ "20b tokenizer" (332115325534 tokens)
+
+0.1B = L12-D768, lr 8e-4 to 3e-5 cosine decay, wd 0.1, bsz 8x30x4096
+
+0.4B = L24-D1024, lr 6e-4 to 2e-5 cosine decay, wd 0.1, bsz 8x30x4096
+
+1.5B = L24-D2048, lr 5e-4 to 1.5e-5 cosine decay, wd 0.1, bsz 8x45x4096
+
+Check https://github.com/BlinkDL/RWKV-LM for details.
+
+How to run it:
+
+https://pypi.org/project/rwkv/
+
+or
+
+https://github.com/BlinkDL/RWKV-LM/tree/main/RWKV-v7

RWKV-x070-Pile-1.47B-20241210-ctx4096.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2b5dc71a921e59d2eaa947c8276d4d5fac873ea12dad47f9f5a8390ff0d8e507
+size 2930288193

RWKV-x070-Pile-164M-L33-D512-20241218-ctx4096.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c2e586c4a09ef4c0f0ba973a84f229dd87d1eb5c27a210401731a04a021f36b
+size 328874758

RWKV-x070-Pile-165M-L25-D576-20241218-ctx4096.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6bf07def143d3cd1a4f0bfef69159563f98f48cd9f4a15767825e4a3efad6b41
+size 330455994

RWKV-x070-Pile-168M-20241120-ctx4096.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5db3e76067028196aa88164a2c81f4083c3f464142ee5395463402566ca29a17
+size 335409434

RWKV-x070-Pile-421M-20241127-ctx4096.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:00bcbc502587dfc01f5f37033400c3d8a13c38da4f2c3b72ff7c8dab961a2be5
+size 842414919