{
  "schema_version": 1,
  "revision_id": "r008",
  "checkpoint": {
    "sha256": "25b8cac7d5142c75e1be0b42fa24d980896e6f911e912c883566747a3baa56f0",
    "source_iteration": 32000,
    "tokens_processed": 15728561562
  },
  "benchmark_model": {
    "sha256": "1c1002909119b4913841ee29efd44bd86d62d81f2ae27de01602b64d4af0dbec",
    "conversion_max_absolute_logit_delta": 0.0,
    "native_parameters": 127820544,
    "packaged_parameters": 127975936
  },
  "tokenizer_sha256": "3733307577230bb4802d2c774d8e2323f7f64e4139c13712736a57daf91bdda1",
  "standard": {
    "results": {
      "blimp": {
        "sample_len": 67000,
        "acc,none": 0.7803582089552239
      },
      "lambada_openai": {
        "sample_len": 5153,
        "perplexity,none": 42.0737928479771,
        "acc,none": 0.31321560256161457
      },
      "hellaswag": {
        "sample_len": 10042,
        "acc,none": 0.2855008962358096,
        "acc_norm,none": 0.3043218482374029
      },
      "piqa": {
        "sample_len": 1838,
        "acc,none": 0.6207834602829162,
        "acc_norm,none": 0.6169749727965179
      },
      "sciq": {
        "sample_len": 1000,
        "acc,none": 0.742,
        "acc_norm,none": 0.643
      },
      "arc_easy": {
        "sample_len": 2376,
        "acc,none": 0.4692760942760943,
        "acc_norm,none": 0.4255050505050505
      },
      "arc_challenge": {
        "sample_len": 1172,
        "acc,none": 0.20733788395904437,
        "acc_norm,none": 0.24744027303754265
      }
    },
    "source": "artifacts/standard/__workspace__r008-final__inputs__model/results_2026-09-21T00-39-54.923340.json"
  },
  "tiny_ml_leaderboard": {
    "blimp_accuracy": 0.7803582089552239,
    "arc_easy_accuracy": 0.4692760942760943,
    "wikitext": {
      "name": "wikitext",
      "alias": "wikitext",
      "sample_len": 62,
      "word_perplexity,none": 34.818979881506166,
      "word_perplexity_stderr,none": "N/A",
      "byte_perplexity,none": 1.9423491827239139,
      "byte_perplexity_stderr,none": "N/A",
      "bits_per_byte,none": 0.9578025822708744,
      "bits_per_byte_stderr,none": "N/A"
    },
    "source": "artifacts/tiny-ml-wikitext/__workspace__r008-final__inputs__model/results_2026-09-21T00-40-27.995769.json"
  },
  "open_slm_leaderboard": {
    "canonical_precision": "fp32",
    "runs": {
      "fp32": {
        "scores_percent": {
          "hellaswag": 30.452101175064726,
          "arc_easy": 42.340067340067336,
          "arc_challenge": 24.573378839590443,
          "piqa": 61.099020674646354,
          "arithmark-3": 34.599999999999994
        },
        "sample_counts": {
          "hellaswag": 10042,
          "arc_easy": 2376,
          "arc_challenge": 1172,
          "piqa": 1838,
          "arithmark-3": 1000
        },
        "intelligence_index": 13.441956265520783,
        "sources": {
          "lm_eval": "artifacts/open-slm/lm-eval/fp32/__workspace__r008-final__inputs__model/results_2026-09-21T00-44-05.707661.json",
          "arithmark_3": "artifacts/open-slm/arithmark-3/fp32/_workspace_r008-final_inputs_model_arithmark-3_results.json"
        }
      },
      "bfloat16": {
        "scores_percent": {
          "hellaswag": 30.43218482374029,
          "arc_easy": 42.55050505050505,
          "arc_challenge": 24.744027303754265,
          "piqa": 61.69749727965179,
          "arithmark-3": 35.6
        },
        "sample_counts": {
          "hellaswag": 10042,
          "arc_easy": 2376,
          "arc_challenge": 1172,
          "piqa": 1838,
          "arithmark-3": 1000
        },
        "intelligence_index": 14.069660975469457,
        "sources": {
          "lm_eval": "artifacts/open-slm/lm-eval/bfloat16/__workspace__r008-final__inputs__model/results_2026-09-21T00-46-45.835602.json",
          "arithmark_3": "artifacts/open-slm/arithmark-3/bfloat16/_workspace_r008-final_inputs_model_arithmark-3_results.json"
        }
      }
    },
    "bfloat16_minus_fp32_percentage_points": {
      "hellaswag": -0.019916351324436476,
      "arc_easy": 0.21043771043771642,
      "arc_challenge": 0.17064846416382196,
      "piqa": 0.5984766050054375,
      "arithmark-3": 1.000000000000007
    }
  }
}