architecture.svg

<svg xmlns="http://www.w3.org/2000/svg" width="640" height="684" viewBox="0 0 640 684" font-family="Helvetica, Arial, sans-serif">
<defs><marker id="a" viewBox="0 0 10 10" refX="9" refY="5" markerWidth="7" markerHeight="7" orient="auto-start-reverse"><path d="M0,0 L10,5 L0,10 z" fill="#445"/></marker></defs>
<rect width="640" height="684" fill="#ffffff"/>
<text x="320.0" y="32" text-anchor="middle" font-size="18" font-weight="bold" fill="#223">Nirca Mini</text>
<rect x="40" y="56" width="440" height="52" rx="8" fill="#eef2f7" stroke="#667" stroke-width="1.2"/>
<text x="260.0" y="77" text-anchor="middle" font-size="14" font-weight="bold" fill="#223">Input bytes</text>
<text x="260.0" y="96" text-anchor="middle" font-size="12" fill="#334">vocabulary 265: 256 bytes and 9 special tokens</text>
<rect x="40" y="134" width="440" height="52" rx="8" fill="#eef2f7" stroke="#667" stroke-width="1.2"/>
<text x="260.0" y="155" text-anchor="middle" font-size="14" font-weight="bold" fill="#223">Embedding</text>
<text x="260.0" y="174" text-anchor="middle" font-size="12" fill="#334">width 512</text>
<rect x="40" y="212" width="440" height="52" rx="8" fill="#e3ecf9" stroke="#667" stroke-width="1.2"/>
<text x="260.0" y="233" text-anchor="middle" font-size="14" font-weight="bold" fill="#223">2 transformer layers, run once</text>
<text x="260.0" y="252" text-anchor="middle" font-size="12" fill="#334">8 heads, RoPE, SwiGLU 1408</text>
<rect x="40" y="290" width="440" height="52" rx="8" fill="#f6eedf" stroke="#667" stroke-width="1.2"/>
<text x="260.0" y="311" text-anchor="middle" font-size="14" font-weight="bold" fill="#223">Merge latent with embedded input</text>
<text x="260.0" y="330" text-anchor="middle" font-size="12" fill="#334">each pass starts here</text>
<rect x="40" y="368" width="440" height="52" rx="8" fill="#f6eedf" stroke="#667" stroke-width="1.2"/>
<text x="260.0" y="389" text-anchor="middle" font-size="14" font-weight="bold" fill="#223">Recurrent block: attention + mixture of experts</text>
<text x="260.0" y="408" text-anchor="middle" font-size="12" fill="#334">32 experts, top 8, width 2048, ternary weights</text>
<rect x="40" y="446" width="440" height="52" rx="8" fill="#f6eedf" stroke="#667" stroke-width="1.2"/>
<text x="260.0" y="467" text-anchor="middle" font-size="14" font-weight="bold" fill="#223">Final norm and output head</text>
<text x="260.0" y="486" text-anchor="middle" font-size="12" fill="#334">a prediction at every pass (tied to the embedding)</text>
<line x1="260.0" y1="108" x2="260.0" y2="132" stroke="#445" stroke-width="1.5" marker-end="url(#a)"/>
<line x1="260.0" y1="186" x2="260.0" y2="210" stroke="#445" stroke-width="1.5" marker-end="url(#a)"/>
<line x1="260.0" y1="264" x2="260.0" y2="288" stroke="#445" stroke-width="1.5" marker-end="url(#a)"/>
<line x1="260.0" y1="342" x2="260.0" y2="366" stroke="#445" stroke-width="1.5" marker-end="url(#a)"/>
<line x1="260.0" y1="420" x2="260.0" y2="444" stroke="#445" stroke-width="1.5" marker-end="url(#a)"/>
<line x1="260.0" y1="498" x2="260.0" y2="522" stroke="#445" stroke-width="1.5" marker-end="url(#a)"/>
<polygon points="260.0,524 370.0,559.0 260.0,594 150.0,559.0" fill="#fbe3e3" stroke="#667" stroke-width="1.2"/>
<text x="260.0" y="556" text-anchor="middle" font-size="14" font-weight="bold" fill="#223">Halt?</text>
<text x="260.0" y="573" text-anchor="middle" font-size="12" fill="#334">learned, 1 to 24 passes</text>
<polyline points="370.0,559.0 540,559.0 540,316.0 482,316.0" fill="none" stroke="#445" stroke-width="1.5" marker-end="url(#a)"/>
<text x="548" y="437.5" font-size="12" fill="#334">no: another pass</text>
<line x1="260.0" y1="594" x2="260.0" y2="618" stroke="#445" stroke-width="1.5" marker-end="url(#a)"/>
<text x="268.0" y="611" font-size="12" fill="#334">yes</text>
<rect x="40" y="620" width="440" height="40" rx="8" fill="#e6f4e6" stroke="#667" stroke-width="1.2"/>
<text x="260.0" y="645" text-anchor="middle" font-size="14" font-weight="bold" fill="#223">Next-byte prediction</text>
</svg>