Akiki MiniLM-L6-v2 Clover intent classifier (seed 2)

.gitattributes

@@ -1,35 +1,2 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
 *.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
+data/train.jsonl filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,3 @@
+__pycache__/
+*.pyc
+.venv/

LICENSE

@@ -0,0 +1,176 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS

NOTICE

@@ -0,0 +1,17 @@
+Akiki MiniLM-L6-v2 Clover intent classifier
+Developed by AKIKI.AI.
+
+Project code is supplied under the Apache License, Version 2.0 (see LICENSE).
+Base model: sentence-transformers/all-MiniLM-L6-v2, revision
+1110a243fdf4706b3f48f1d95db1a4f5529b4d41, Apache-2.0.
+Original base-model card and attribution: base_model/README.md.
+https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2
+
+Clover intent data (data/) is synthetic, generated and reviewed locally by AKIKI.AI with
+cached Qwen checkpoints; see data/README.md and provenance/original/CREDITS.md.
+Labels are model-reviewed; human review is pending.
+
+provenance/original/source/ preserves the original experiment implementation for audit,
+under its internal project names. The root-level model.py is portable and needs no
+original project files. Result and manifest files copied verbatim from the experiment
+keep the internal method name 'ariadne', which is the Akiki input affine.

README.md

@@ -0,0 +1,209 @@
+---
+language:
+- en
+license: apache-2.0
+library_name: pytorch
+pipeline_tag: text-classification
+base_model: sentence-transformers/all-MiniLM-L6-v2
+base_model_relation: adapter
+tags:
+- akiki
+- clover
+- minilm
+- intent-classification
+- frozen-encoder
+- custom-code
+metrics:
+- accuracy
+- f1
+---
+
+# Akiki-MiniLM-L6-v2-clover-intent
+
+Developed by AKIKI.AI.
+
+An English six-way intent router for Clover, built on a **completely frozen**
+`sentence-transformers/all-MiniLM-L6-v2`. Only two small pieces were trained: an
+identity-initialized **384 × 384 Akiki input affine** on the word embeddings
+(147,840 parameters) and a **six-class head** (2,310 parameters), 150,150 in total.
+The 22,713,216 backbone parameters are untouched and hash-checked against the
+original.
+
+Given the current user message and, optionally, earlier conversation turns, it
+returns one of:
+
+| Token | Label | Meaning |
+|---|---|---|
+| A | chat | Conversation, explanation, how-to advice, general knowledge |
+| B | question | Inspect or report the state of the user's actual system, without changing it |
+| C | build | Create or modify software, a feature, a site or an app |
+| D | task | Perform a specified operational action (restart, move, change a setting) |
+| E | fix | Diagnose and remedy a malfunction when no specific remedy is prescribed |
+| F | unclear | The route or referent cannot be identified from message and context |
+
+> **Experimental.** Labels are synthetic and model-reviewed; human review is
+> pending. This model classifies intent only. It does not write contracts, run
+> actions, or implement Clover's refusal policy. It has not been approved for
+> serving.
+
+## Results
+
+Held-out test split: 714 examples (120 per class; 114 unclear), balanced across
+easy / medium / hard.
+
+| Method (MiniLM-L6-v2, frozen backbone) | Test accuracy | Macro F1 | Hard accuracy |
+|---|---:|---:|---:|
+| Label-description similarity, no training | 34.9% | 0.313 | 36.7% |
+| Frozen embeddings + regularized linear classifier | 92.3% | 0.923 | 91.7% |
+| Head-only training, 3-seed mean | 91.7% ± 0.3 | 0.917 | 91.1% |
+| Akiki affine + head, 3-seed mean | 96.6% ± 0.6 | 0.966 | 95.6% |
+| **This checkpoint (Akiki, seed 2)** | **97.2%** | **0.972** | **96.2%** |
+
+**Which seed, and why:** three seeds were trained. Seed 2 is released because it
+has the lowest validation NLL (0.121 vs 0.131 and 0.141), which is the
+experiment's checkpoint criterion. Test results played no part in the choice.
+Seed 2 happens to be the best on test as well, so treat the **three-seed mean
+(96.6%)** as the honest estimate of what this recipe gives. All three seeds ship
+in `checkpoints/`.
+
+Seed 2 detail: easy 99.6%, medium 95.8%, hard 96.2%; test NLL 0.082, Brier 0.043.
+
+| Class | Precision | Recall | F1 |
+|---|---:|---:|---:|
+| chat | 0.991 | 0.958 | 0.975 |
+| question | 0.967 | 0.992 | 0.979 |
+| build | 1.000 | 0.967 | 0.983 |
+| task | 0.934 | 0.950 | 0.942 |
+| fix | 0.967 | 0.992 | 0.979 |
+| unclear | 0.974 | 0.974 | 0.974 |
+
+Rows are gold labels, columns are predictions:
+
+| gold \ pred | chat | question | build | task | fix | unclear |
+|---|---:|---:|---:|---:|---:|---:|
+| chat | 115 | 2 | 0 | 1 | 0 | 2 |
+| question | 1 | 119 | 0 | 0 | 0 | 0 |
+| build | 0 | 0 | 116 | 3 | 0 | 1 |
+| task | 0 | 2 | 0 | 114 | 4 | 0 |
+| fix | 0 | 0 | 0 | 1 | 119 | 0 |
+| unclear | 0 | 0 | 0 | 3 | 0 | 111 |
+
+**Pass mark not met.** The experiment requires at least 95% accuracy **and zero
+action predictions (build/task/fix) on chat/question**. This checkpoint makes
+**1 action prediction on 240 chat/question examples** (a chat message read as
+task) and 3 on 114 unclear examples. Downstream code should therefore not treat
+a `task`/`build`/`fix` label as authorization by itself.
+
+## Usage
+
+```bash
+pip install -r requirements.txt
+python model.py 'Restart my web server.'                  # -> D
+python model.py --output label 'Restart my web server.'   # -> task
+python model.py --output json --context-file ctx.json 'yes'
+```
+
+```python
+from model import CloverIntent
+
+clf = CloverIntent.from_pretrained(".")          # loads base_model/ + adapter.safetensors, CPU by default
+clf.predict("Restart my web server.")["label"]   # 'task'
+clf.predict("yes", context=[
+    {"role": "assistant", "text": "Would you like me to check whether last night's backup ran?"},
+])["label"]                                       # 'question'
+clf.probabilities([{"message": "fix it"}])       # (1, 6) tensor, label order above
+```
+
+`predict` returns `label`, `token` (A–F) and the six `probabilities`. Everything
+loads from this repository with `local_files_only=True`; nothing is downloaded at
+run time. Download the whole repo first, e.g.
+`hf download Akiki-AI/Akiki-MiniLM-L6-v2-clover-intent --local-dir clover-intent`.
+
+### Input format
+
+The model sees exactly this text (the context block is omitted when empty):
+
+```
+Conversation context:
+assistant: Would you like me to restart the web server?
+
+Current user message: yes
+```
+
+Context turns are `{"role": ..., "text": ...}` objects. The limit is 512 tokens
+including special tokens. Longer inputs **raise an error rather than being
+truncated**. The longest training/test input is 126 tokens, so these results say
+nothing about long briefings or full chat histories.
+
+Probabilities are a plain softmax of the head logits, with no calibration fitted.
+
+## Architecture
+
+```
+tokens → MiniLM word embeddings (frozen)
+       → Akiki affine  W·x + b   (384×384, identity-initialized, FP32)
+       → + position/type embeddings → 6 BERT layers (frozen)
+       → attention-masked mean pooling → L2 normalize
+       → linear head (384 → 6) → softmax
+```
+
+`adapter.safetensors` holds `interface.proj.{weight,bias}` and `head.{weight,bias}`.
+`base_model/` is an unmodified copy of all-MiniLM-L6-v2 at revision
+`1110a243fdf4706b3f48f1d95db1a4f5529b4d41`.
+
+## Training
+
+- **Data:** 9,913 train / 707 validation / 714 test synthetic English examples
+  (`data/`, documented in `data/README.md`). Scenario groups never cross splits;
+  near-duplicates across splits were removed, protecting test first. Hard
+  examples need the conversation context to resolve the intent, using follow-up
+  phrase families kept separate between splits.
+- **Labels:** generated locally by Qwen3.8-27B-FP8 and independently classified by
+  Qwen3.5-9B; disagreements were rejected. Not human-verified.
+- **Optimization:** Adam; head lr 1e-3, affine lr 1e-4; early stopping on
+  validation NLL. Seed 2 selected epoch 8 of 11. Each fit took about 30 s on one
+  RTX PRO 6000. The full recipe is in `provenance/original/protocol/plan.json`.
+
+## Latency
+
+Warm CPU inference with FP32 PyTorch/Transformers, 8 threads, batch size 1, on a
+short contextual input (tokenization included, model loading excluded): **2.47 ms
+median, 2.62 ms p95**. Head-only measured 2.40 ms. This was measured with seed 0,
+which is architecturally identical, on the training server, not on a laptop. See
+`results/inference-comparison.json` and `results/cpu-latency.json`.
+
+## Verification
+
+`checks/packaged-inference-verification.json` re-runs this repository's
+`model.py` on CPU for all three seeds. It checks that:
+
+- the bundled backbone's tensor hash equals the frozen training backbone;
+- every test label matches the recorded GPU predictions, with probabilities
+  within 5.4e-6;
+- validation accuracy and NLL match the training history.
+
+Verify file integrity with `sha256sum -c SHA256SUMS`.
+
+## Repository contents
+
+| Path | What it is |
+|---|---|
+| `model.py` | Portable inference: `CloverIntent` class and CLI |
+| `adapter.safetensors` | Released weights (seed 2) |
+| `checkpoints/seed-{0,1,2}.safetensors` | All three trained seeds |
+| `base_model/` | Unmodified all-MiniLM-L6-v2 |
+| `model_config.json` | Labels, tokens, hashes, parameter counts |
+| `results/` | Test report, per-example predictions and uncertainty, per-seed histories, baselines, latency |
+| `data/` | Exact train/validation/test splits, dataset card, manifest, audit |
+| `checks/` | Packaged-inference verification |
+| `provenance/original/` | Original experiment protocol, reports and source, under internal project names |
+
+Files copied verbatim from the experiment (under `results/`, `data/` and
+`provenance/`) use the internal method name `ariadne`, which is the Akiki
+input affine described here.
+
+## Licence and credits
+
+Apache-2.0 (see `LICENSE` and `NOTICE`). The base model,
+[sentence-transformers/all-MiniLM-L6-v2](https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2),
+is Apache-2.0; its original card is in `base_model/README.md`.

SHA256SUMS

@@ -0,0 +1,80 @@
+ee05350c7fda0af4da116020dd7b976d731de3beed4f5e717d5bcc95ead24550  .gitattributes
+4cad133d4f0a6cf4bba15f1b8bbae48d6a55f20b94f75487c51a93cc6a5af59f  .gitignore
+a6cba85bc92e0cff7a450b1d873c0eaa2e9fc96bf472df0247a26bec77bf3ff9  LICENSE
+198b75bff2665ac699c4289c43916fb7cef607874372fa315216e7b0a38d3ffe  NOTICE
+7768ff7cd3039381491eb86efb286cde39bf7662e5719542cf66348d7cad57bc  README.md
+37b5302e645423512b6fcd6328a4f371ab8f5220457514dcd8bf2d27bdbc5a8d  adapter.safetensors
+4be450dde3b0273bb9787637cfbd28fe04a7ba6ab9d36ac48e92b11e350ffc23  base_model/1_Pooling/config.json
+dcd602d2fd35c203a247304a06fec6654a12f7941b739f9221a064fe8dc3b7f0  base_model/README.md
+953f9c0d463486b10a6871cc2fd59f223b2c70184f49815e7efbcab5d8908b41  base_model/config.json
+061ca9d39661d6c6d6de5ba27f79a1cd5770ea247f8d46412a68a498dc5ac9f3  base_model/config_sentence_transformers.json
+53aa51172d142c89d9012cce15ae4d6cc0ca6895895114379cacb4fab128d9db  base_model/model.safetensors
+84e40c8e006c9b1d6c122e02cba9b02458120b5fb0c87b746c41e0207cf642cf  base_model/modules.json
+fc1993fde0a95c24ec6c022539d41cf6e2f7c9721e5415d6fb6897472a9cd4b7  base_model/sentence_bert_config.json
+303df45a03609e4ead04bc3dc1536d0ab19b5358db685b6f3da123d05ec200e3  base_model/special_tokens_map.json
+be50c3628f2bf5bb5e3a7f17b1f74611b2561a3a27eeab05e5aa30f411572037  base_model/tokenizer.json
+acb92769e8195aabd29b7b2137a9e6d6e25c476a4f15aa4355c233426c61576b  base_model/tokenizer_config.json
+07eced375cec144d27c900241f3e339478dec958f92fddbc551f295c992038a3  base_model/vocab.txt
+5c38aa73b22d91591d644829259c9c6a790e76380fb49ff947d52fee4f6fd40a  checkpoints/seed-0.safetensors
+1b74e3473191677c943d3f36b9437213be23ab5e3cf2cf0300773c489f0fafd8  checkpoints/seed-1.safetensors
+37b5302e645423512b6fcd6328a4f371ab8f5220457514dcd8bf2d27bdbc5a8d  checkpoints/seed-2.safetensors
+6717ab4a8158c70f6ce53435b139b2537bfd14bfd71d203d41e27665b0892f7b  checks/packaged-inference-verification.json
+5ac817fc011a53b16c3826c920e6d80dba45f188234552b59f0783ddf17b959c  data/README.md
+591017f301654e0d26ad884fa37f4378acedf38e57eacaf44c409253f15e641e  data/audit-drops.jsonl
+3ed02c8fb676df20988b89adb1e04ed4fcb99565e3f6bab1e0eabc5bd2778607  data/audit.json
+857d674c7b0cf9bc9ab112fce5b6db2cdead4055c8e9c9b2901f2706bc8f7c59  data/manifest.json
+8e1fde3f1074fb216e73b879fe3905d4448faf1c7a83d1fef6b3078dc1560c28  data/test.jsonl
+2e3edef9182471d0fcd3738fdf886fdbe78fac5e6f3855373ab92a6ad0f1ca72  data/train.jsonl
+40759974effb15fe000589d5e9082ae94c7c760a9dfef40cb364d95bc74c953b  data/validation.jsonl
+b80737e8f5d49d4f506de953fff0a96744e8bb10509a5e9a0f7d970f52b9b0b4  model.py
+9168869c1b35dba0b61f2f0d8e786109c7da2ab0e79f41713d42c57d041e4e6a  model_config.json
+c90a6e3a8dfa1a48a96def4d416e816cfc671343602a4c07ba7a581d06730032  provenance/original/CREDITS.md
+53d9f11712d524f505cafce98728bb0f460f27b95c4bf8ef945a7bc38ecd2262  provenance/original/README.md
+873c0b5f9e6120d974fc5a29405a9c51b7fe57f5fc589851ffe8d628143f05df  provenance/original/final-verification.json
+2ea6b7093dcac3ad96188725cd19cf8147511d96e70c0abd353e1baf6213816f  provenance/original/minilm-l6-v2/MODEL_CARD.md
+5bbf390749bd573e2de766154ceb791e35c4b840ace6f7f7829939ce7f470ae5  provenance/original/minilm-l6-v2/model.json
+fdd46960b5ad1b4dce94e3c05a4365efb631a947fa45e375bd34e5c47ee72cb4  provenance/original/protocol/PASSMARK-CLOVER-1.md
+756f4fda69a75ee9a777b6151b599c0456cf9bc35bd0a928c1c85fd09c5e793f  provenance/original/protocol/plan.json
+3bcb85e89881c9e6c8d58503e169fd69ae55488ea1d1c4459d090c97f785644e  provenance/original/results.md
+270b5ab0ea3ad33cec13803f4ccb4ee2ecdd5f8c7152780712546d3e94c8e798  provenance/original/source/ariadne_bench/clover.py
+2d08a3d37c4c171250e5e2bcf1ad572d23ce420fff7d37e61fcccda08e5cca8c  provenance/original/source/ariadne_bench/clover_models.py
+19700fb8170b2460401900cb96ebdff22bd7bf778ca5132e98d56ba68f344187  provenance/original/source/ariadne_bench/interfaces.py
+2aef90fc4b04c759d11dc900d4ab565980c1e071fa56b492628aa5b8e4a8d46e  provenance/original/source/scripts/package_clover_minilm.py
+07ba1f84b4bed51072f943c663835b350ab2afa0f50e31e6b12e0abfe36e203c  provenance/original/source/scripts/predict_clover.py
+df8b0c3c79c627618a2ae2a12e02cb861d1369ffbbe86f5c60cd3112836c3af5  provenance/original/source/scripts/run_clover_experiments.py
+c7ea877b7e68e2dcb986b7a92d1026191eec06fff933eae220d6af0b47a21378  requirements.txt
+2ababfd7813d8cb42bc8f886d03e8dbd8e1da64a62471cfd2210ef0114856052  results/baselines/embedding-class_centroids.json
+a826d8825ade3d1e02988393369e972c6a652105ab1ec96db27c8466c9666feb  results/baselines/embedding-kmeans-seed-0.json
+e1c2415e94febcb80754e59b13fd52e28a0d4f9147ee8e5fd39a4a193eb31ac8  results/baselines/embedding-kmeans-seed-1.json
+1289a225b0df372e2a4615e4552b194e007b735db0e84d47c1d44f66f5f5e495  results/baselines/embedding-kmeans-seed-2.json
+60824d591a2f7171054e4cca12608f2b65fe5aad1eb9543dad978524e2e97013  results/baselines/embedding-knn.json
+03b4ac1dcab65ab3791f8bad5173c17de566badc3f49e74c8314bd80e8b50abc  results/baselines/embedding-linear_head.json
+d397fb275880b44b4603873a6587307dee19e04887dc52f82d68bce28d51a82f  results/baselines/frozen-baseline.json
+37c39a34166869f9c88de21fce9af9616b6494e124598e1449a2864203260b55  results/baselines/head_only-seed-0.json
+6fc13826c6f8a77c09aab46bbaffdef874b20b9473285a861013728f62241242  results/baselines/head_only-seed-1.json
+72b1edc8ffe38d518fdffd50f35d6e72318fe2df8045bab591a5f1701decc5b7  results/baselines/head_only-seed-2.json
+97e439ec22246473a9d4224ba1fe3a8ea52835c3fa151930749ec77afb402c35  results/cpu-latency.json
+7455dc220c877d297c483f3279de4dc74f76f49cfc4d069f50ebc7d747ea5fc5  results/inference-comparison.json
+13c27fe6e6eab4ef650f6a323ea652ee1f750a1039fa8a3b3b14298a53d8a841  results/inference-smoke.json
+4c0d7a9d360ad04ee82b1fed7348cb471c73bf9f3cb10778970dfc55a376d91b  results/seed-summary-minilm.json
+ca9c373b4135ab7f7912d728367f5873035a2b236e0d4ef5bb47504163c3ac0f  results/seeds/seed-0/complete.json
+74cbae8b24326b53fe7917f5b943a0d4a4e026682018d9ec963f876a835a1eaa  results/seeds/seed-0/history.json
+358d3baed5a1ad6cd0ca2e8794ed60b80677450e379ce18bf0ea4f9d38ae5827  results/seeds/seed-0/model.json
+4dfaafbe8ae351664edb49cb3a5aae7b292fe0a65a37f157f84503023e39da52  results/seeds/seed-0/test-predictions.jsonl
+1c4f65c5d4e649bdd18d2178f532a2eed7744590461b8a7a8757766e23f3177f  results/seeds/seed-0/test-report.json
+81d3aeb360106df074020edf128a1e340bf118a8203fed9b64c9eb2b6e272fc9  results/seeds/seed-0/test-uncertainty.jsonl
+2529ad5cfeeaaf836a6fb902e3039b8caa7b6e4eb3baf599b501b2d4df4c863f  results/seeds/seed-1/complete.json
+3409df997aa986ca7f80c01bdb5fcb1d89f4993f82e9e23a42d577ee06165cad  results/seeds/seed-1/history.json
+358d3baed5a1ad6cd0ca2e8794ed60b80677450e379ce18bf0ea4f9d38ae5827  results/seeds/seed-1/model.json
+83b0153d08a6e957942931166fa2729c898fa482807e4a3b67da1eddfe9ca65e  results/seeds/seed-1/test-predictions.jsonl
+3773cd70da2c750b9617c8bc8068ec88ca61c4202e646f950590a00683a17036  results/seeds/seed-1/test-report.json
+c9b586afa3e639932579183e25a6e0367299c7b373915a89780698c2e07ce230  results/seeds/seed-1/test-uncertainty.jsonl
+f533898126d28bd3364b71f12eaa6b2093470492273ccca104be3905b821ff80  results/seeds/seed-2/complete.json
+e6b741fd3fa57e2c51106cf6d5897342dfe36c4c2164190a37af63887e24279d  results/seeds/seed-2/history.json
+358d3baed5a1ad6cd0ca2e8794ed60b80677450e379ce18bf0ea4f9d38ae5827  results/seeds/seed-2/model.json
+05b3e3bb67e8973fd614fcdf52249e322c2adc10b9d7758c6638574bd1e8bf9e  results/seeds/seed-2/test-predictions.jsonl
+5dbf729da994049108220bb81213934e81265fece8019d264e6f1aa0f332f58e  results/seeds/seed-2/test-report.json
+72508270db2dbcd5a84415242a638c0fe92bb4b16810023249616a6f0f671f1f  results/seeds/seed-2/test-uncertainty.jsonl
+05b3e3bb67e8973fd614fcdf52249e322c2adc10b9d7758c6638574bd1e8bf9e  results/test-predictions.jsonl
+5dbf729da994049108220bb81213934e81265fece8019d264e6f1aa0f332f58e  results/test-report.json
+72508270db2dbcd5a84415242a638c0fe92bb4b16810023249616a6f0f671f1f  results/test-uncertainty.jsonl

adapter.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:37b5302e645423512b6fcd6328a4f371ab8f5220457514dcd8bf2d27bdbc5a8d
+size 601008

base_model/1_Pooling/config.json

@@ -0,0 +1,7 @@
+{
+  "word_embedding_dimension": 384,
+  "pooling_mode_cls_token": false,
+  "pooling_mode_mean_tokens": true,
+  "pooling_mode_max_tokens": false,
+  "pooling_mode_mean_sqrt_len_tokens": false
+}

base_model/README.md

@@ -0,0 +1,175 @@
+---
+base_model:
+  - nreimers/MiniLM-L6-H384-uncased
+language: en
+license: apache-2.0
+library_name: sentence-transformers
+tags:
+- sentence-transformers
+- feature-extraction
+- sentence-similarity
+- transformers
+datasets:
+- s2orc
+- flax-sentence-embeddings/stackexchange_xml
+- ms_marco
+- gooaq
+- yahoo_answers_topics
+- code_search_net
+- search_qa
+- eli5
+- snli
+- multi_nli
+- wikihow
+- natural_questions
+- trivia_qa
+- embedding-data/sentence-compression
+- embedding-data/flickr30k-captions
+- embedding-data/altlex
+- embedding-data/simple-wiki
+- embedding-data/QQP
+- embedding-data/SPECTER
+- embedding-data/PAQ_pairs
+- embedding-data/WikiAnswers
+pipeline_tag: sentence-similarity
+---
+
+
+# all-MiniLM-L6-v2
+This is a [sentence-transformers](https://www.SBERT.net) model: It maps sentences & paragraphs to a 384 dimensional dense vector space and can be used for tasks like clustering or semantic search.
+
+## Usage (Sentence-Transformers)
+Using this model becomes easy when you have [sentence-transformers](https://www.SBERT.net) installed:
+
+```
+pip install -U sentence-transformers
+```
+
+Then you can use the model like this:
+```python
+from sentence_transformers import SentenceTransformer
+sentences = ["This is an example sentence", "Each sentence is converted"]
+
+model = SentenceTransformer('sentence-transformers/all-MiniLM-L6-v2')
+embeddings = model.encode(sentences)
+print(embeddings)
+```
+
+## Usage (HuggingFace Transformers)
+Without [sentence-transformers](https://www.SBERT.net), you can use the model like this: First, you pass your input through the transformer model, then you have to apply the right pooling-operation on-top of the contextualized word embeddings.
+
+```python
+from transformers import AutoTokenizer, AutoModel
+import torch
+import torch.nn.functional as F
+
+#Mean Pooling - Take attention mask into account for correct averaging
+def mean_pooling(model_output, attention_mask):
+    token_embeddings = model_output[0] #First element of model_output contains all token embeddings
+    input_mask_expanded = attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float()
+    return torch.sum(token_embeddings * input_mask_expanded, 1) / torch.clamp(input_mask_expanded.sum(1), min=1e-9)
+
+
+# Sentences we want sentence embeddings for
+sentences = ['This is an example sentence', 'Each sentence is converted']
+
+# Load model from HuggingFace Hub
+tokenizer = AutoTokenizer.from_pretrained('sentence-transformers/all-MiniLM-L6-v2')
+model = AutoModel.from_pretrained('sentence-transformers/all-MiniLM-L6-v2')
+
+# Tokenize sentences
+encoded_input = tokenizer(sentences, padding=True, truncation=True, return_tensors='pt')
+
+# Compute token embeddings
+with torch.no_grad():
+    model_output = model(**encoded_input)
+
+# Perform pooling
+sentence_embeddings = mean_pooling(model_output, encoded_input['attention_mask'])
+
+# Normalize embeddings
+sentence_embeddings = F.normalize(sentence_embeddings, p=2, dim=1)
+
+print("Sentence embeddings:")
+print(sentence_embeddings)
+```
+
+------
+
+## Background
+
+The project aims to train sentence embedding models on very large sentence level datasets using a self-supervised 
+contrastive learning objective. We used the pretrained [`nreimers/MiniLM-L6-H384-uncased`](https://huggingface.co/nreimers/MiniLM-L6-H384-uncased) model and fine-tuned in on a 
+1B sentence pairs dataset. We use a contrastive learning objective: given a sentence from the pair, the model should predict which out of a set of randomly sampled other sentences, was actually paired with it in our dataset.
+
+We developed this model during the 
+[Community week using JAX/Flax for NLP & CV](https://discuss.huggingface.co/t/open-to-the-community-community-week-using-jax-flax-for-nlp-cv/7104), 
+organized by Hugging Face. We developed this model as part of the project:
+[Train the Best Sentence Embedding Model Ever with 1B Training Pairs](https://discuss.huggingface.co/t/train-the-best-sentence-embedding-model-ever-with-1b-training-pairs/7354). We benefited from efficient hardware infrastructure to run the project: 7 TPUs v3-8, as well as intervention from Googles Flax, JAX, and Cloud team member about efficient deep learning frameworks.
+
+## Intended uses
+
+Our model is intended to be used as a sentence and short paragraph encoder. Given an input text, it outputs a vector which captures 
+the semantic information. The sentence vector may be used for information retrieval, clustering or sentence similarity tasks.
+
+By default, input text longer than 256 word pieces is truncated.
+
+
+## Training procedure
+
+### Pre-training 
+
+We use the pretrained [`nreimers/MiniLM-L6-H384-uncased`](https://huggingface.co/nreimers/MiniLM-L6-H384-uncased) model. Please refer to the model card for more detailed information about the pre-training procedure.
+
+### Fine-tuning 
+
+We fine-tune the model using a contrastive objective. Formally, we compute the cosine similarity from each possible sentence pairs from the batch.
+We then apply the cross entropy loss by comparing with true pairs.
+
+#### Hyper parameters
+
+We trained our model on a TPU v3-8. We train the model during 100k steps using a batch size of 1024 (128 per TPU core).
+We use a learning rate warm up of 500. The sequence length was limited to 128 tokens. We used the AdamW optimizer with
+a 2e-5 learning rate. The full training script is accessible in this current repository: `train_script.py`.
+
+#### Training data
+
+We use the concatenation from multiple datasets to fine-tune our model. The total number of sentence pairs is above 1 billion sentences.
+We sampled each dataset given a weighted probability which configuration is detailed in the `data_config.json` file.
+
+
+| Dataset                                                  | Paper                                    | Number of training tuples  |
+|--------------------------------------------------------|:----------------------------------------:|:--------------------------:|
+| [Reddit comments (2015-2018)](https://github.com/PolyAI-LDN/conversational-datasets/tree/master/reddit) | [paper](https://arxiv.org/abs/1904.06472) | 726,484,430 |
+| [S2ORC](https://github.com/allenai/s2orc) Citation pairs (Abstracts) | [paper](https://aclanthology.org/2020.acl-main.447/) | 116,288,806 |
+| [WikiAnswers](https://github.com/afader/oqa#wikianswers-corpus) Duplicate question pairs | [paper](https://doi.org/10.1145/2623330.2623677) | 77,427,422 |
+| [PAQ](https://github.com/facebookresearch/PAQ) (Question, Answer) pairs | [paper](https://arxiv.org/abs/2102.07033) | 64,371,441 |
+| [S2ORC](https://github.com/allenai/s2orc) Citation pairs (Titles) | [paper](https://aclanthology.org/2020.acl-main.447/) | 52,603,982 |
+| [S2ORC](https://github.com/allenai/s2orc) (Title, Abstract) | [paper](https://aclanthology.org/2020.acl-main.447/) | 41,769,185 |
+| [Stack Exchange](https://huggingface.co/datasets/flax-sentence-embeddings/stackexchange_xml) (Title, Body) pairs  | - | 25,316,456 |
+| [Stack Exchange](https://huggingface.co/datasets/flax-sentence-embeddings/stackexchange_xml) (Title+Body, Answer) pairs  | - | 21,396,559 |
+| [Stack Exchange](https://huggingface.co/datasets/flax-sentence-embeddings/stackexchange_xml) (Title, Answer) pairs  | - | 21,396,559 |
+| [MS MARCO](https://microsoft.github.io/msmarco/) triplets | [paper](https://doi.org/10.1145/3404835.3462804) | 9,144,553 |
+| [GOOAQ: Open Question Answering with Diverse Answer Types](https://github.com/allenai/gooaq) | [paper](https://arxiv.org/pdf/2104.08727.pdf) | 3,012,496 |
+| [Yahoo Answers](https://www.kaggle.com/soumikrakshit/yahoo-answers-dataset) (Title, Answer) | [paper](https://proceedings.neurips.cc/paper/2015/hash/250cf8b51c773f3f8dc8b4be867a9a02-Abstract.html) | 1,198,260 |
+| [Code Search](https://huggingface.co/datasets/code_search_net) | - | 1,151,414 |
+| [COCO](https://cocodataset.org/#home) Image captions | [paper](https://link.springer.com/chapter/10.1007%2F978-3-319-10602-1_48) | 828,395|
+| [SPECTER](https://github.com/allenai/specter) citation triplets | [paper](https://doi.org/10.18653/v1/2020.acl-main.207) | 684,100 |
+| [Yahoo Answers](https://www.kaggle.com/soumikrakshit/yahoo-answers-dataset) (Question, Answer) | [paper](https://proceedings.neurips.cc/paper/2015/hash/250cf8b51c773f3f8dc8b4be867a9a02-Abstract.html) | 681,164 |
+| [Yahoo Answers](https://www.kaggle.com/soumikrakshit/yahoo-answers-dataset) (Title, Question) | [paper](https://proceedings.neurips.cc/paper/2015/hash/250cf8b51c773f3f8dc8b4be867a9a02-Abstract.html) | 659,896 |
+| [SearchQA](https://huggingface.co/datasets/search_qa) | [paper](https://arxiv.org/abs/1704.05179) | 582,261 |
+| [Eli5](https://huggingface.co/datasets/eli5) | [paper](https://doi.org/10.18653/v1/p19-1346) | 325,475 |
+| [Flickr 30k](https://shannon.cs.illinois.edu/DenotationGraph/) | [paper](https://transacl.org/ojs/index.php/tacl/article/view/229/33) | 317,695 |
+| [Stack Exchange](https://huggingface.co/datasets/flax-sentence-embeddings/stackexchange_xml) Duplicate questions (titles) | | 304,525 |
+| AllNLI ([SNLI](https://nlp.stanford.edu/projects/snli/) and [MultiNLI](https://cims.nyu.edu/~sbowman/multinli/) | [paper SNLI](https://doi.org/10.18653/v1/d15-1075), [paper MultiNLI](https://doi.org/10.18653/v1/n18-1101) | 277,230 | 
+| [Stack Exchange](https://huggingface.co/datasets/flax-sentence-embeddings/stackexchange_xml) Duplicate questions (bodies) | | 250,519 |
+| [Stack Exchange](https://huggingface.co/datasets/flax-sentence-embeddings/stackexchange_xml) Duplicate questions (titles+bodies) | | 250,460 |
+| [Sentence Compression](https://github.com/google-research-datasets/sentence-compression) | [paper](https://www.aclweb.org/anthology/D13-1155/) | 180,000 |
+| [Wikihow](https://github.com/pvl/wikihow_pairs_dataset) | [paper](https://arxiv.org/abs/1810.09305) | 128,542 |
+| [Altlex](https://github.com/chridey/altlex/) | [paper](https://aclanthology.org/P16-1135.pdf) | 112,696 |
+| [Quora Question Triplets](https://quoradata.quora.com/First-Quora-Dataset-Release-Question-Pairs) | - | 103,663 |
+| [Simple Wikipedia](https://cs.pomona.edu/~dkauchak/simplification/) | [paper](https://www.aclweb.org/anthology/P11-2117/) | 102,225 |
+| [Natural Questions (NQ)](https://ai.google.com/research/NaturalQuestions) | [paper](https://transacl.org/ojs/index.php/tacl/article/view/1455) | 100,231 |
+| [SQuAD2.0](https://rajpurkar.github.io/SQuAD-explorer/) | [paper](https://aclanthology.org/P18-2124.pdf) | 87,599 |
+| [TriviaQA](https://huggingface.co/datasets/trivia_qa) | - | 73,346 |
+| **Total** | | **1,170,060,424** |

base_model/config.json

@@ -0,0 +1,24 @@
+{
+  "_name_or_path": "nreimers/MiniLM-L6-H384-uncased",
+  "architectures": [
+    "BertModel"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "gradient_checkpointing": false,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 384,
+  "initializer_range": 0.02,
+  "intermediate_size": 1536,
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 512,
+  "model_type": "bert",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 6,
+  "pad_token_id": 0,
+  "position_embedding_type": "absolute",
+  "transformers_version": "4.8.2",
+  "type_vocab_size": 2,
+  "use_cache": true,
+  "vocab_size": 30522
+}

base_model/config_sentence_transformers.json

@@ -0,0 +1,7 @@
+{
+  "__version__": {
+    "sentence_transformers": "2.0.0",
+    "transformers": "4.6.1",
+    "pytorch": "1.8.1"
+  }
+}

base_model/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:53aa51172d142c89d9012cce15ae4d6cc0ca6895895114379cacb4fab128d9db
+size 90868376

base_model/modules.json

@@ -0,0 +1,20 @@
+[
+  {
+    "idx": 0,
+    "name": "0",
+    "path": "",
+    "type": "sentence_transformers.models.Transformer"
+  },
+  {
+    "idx": 1,
+    "name": "1",
+    "path": "1_Pooling",
+    "type": "sentence_transformers.models.Pooling"
+  },
+  {
+    "idx": 2,
+    "name": "2",
+    "path": "2_Normalize",
+    "type": "sentence_transformers.models.Normalize"
+  }
+]

base_model/sentence_bert_config.json

@@ -0,0 +1,4 @@
+{
+  "max_seq_length": 256,
+  "do_lower_case": false
+}

base_model/special_tokens_map.json

@@ -0,0 +1 @@
+{"unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": "[MASK]"}

base_model/tokenizer_config.json

@@ -0,0 +1 @@
+{"do_lower_case": true, "unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": "[MASK]", "tokenize_chinese_chars": true, "strip_accents": null, "name_or_path": "nreimers/MiniLM-L6-H384-uncased", "do_basic_tokenize": true, "never_split": null, "tokenizer_class": "BertTokenizer", "model_max_length": 512}

checkpoints/seed-0.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5c38aa73b22d91591d644829259c9c6a790e76380fb49ff947d52fee4f6fd40a
+size 601008

checkpoints/seed-1.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1b74e3473191677c943d3f36b9437213be23ab5e3cf2cf0300773c489f0fafd8
+size 601008

checkpoints/seed-2.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:37b5302e645423512b6fcd6328a4f371ab8f5220457514dcd8bf2d27bdbc5a8d
+size 601008

checks/packaged-inference-verification.json

@@ -0,0 +1,60 @@
+{
+  "device": "cpu",
+  "torch": "2.13.0+cu130",
+  "released_seed": 2,
+  "selection": {
+    "0": {
+      "epoch": 8,
+      "validation_nll": 0.14147035955480516,
+      "validation_accuracy": 0.9547383309759547
+    },
+    "1": {
+      "epoch": 7,
+      "validation_nll": 0.1305078952452114,
+      "validation_accuracy": 0.9519094766619519
+    },
+    "2": {
+      "epoch": 8,
+      "validation_nll": 0.12104468951391349,
+      "validation_accuracy": 0.9561527581329562
+    }
+  },
+  "seed-0": {
+    "frozen_backbone_sha256_matches": true,
+    "test_max_abs_probability_difference": 3.7550926208496094e-06,
+    "test_predicted_labels_match": true,
+    "test_accuracy": 0.9663865546218487,
+    "test_accuracy_recorded": 0.9663865546218487,
+    "validation_accuracy": 0.9547383309759547,
+    "validation_accuracy_recorded": 0.9547383309759547,
+    "validation_nll": 0.14147038757801056,
+    "validation_nll_recorded": 0.14147035955480516
+  },
+  "seed-1": {
+    "frozen_backbone_sha256_matches": true,
+    "test_max_abs_probability_difference": 3.933906555175781e-06,
+    "test_predicted_labels_match": true,
+    "test_accuracy": 0.9593837535014006,
+    "test_accuracy_recorded": 0.9593837535014006,
+    "validation_accuracy": 0.9519094766619519,
+    "validation_accuracy_recorded": 0.9519094766619519,
+    "validation_nll": 0.13050784170627594,
+    "validation_nll_recorded": 0.1305078952452114
+  },
+  "seed-2": {
+    "frozen_backbone_sha256_matches": true,
+    "test_max_abs_probability_difference": 5.334615707397461e-06,
+    "test_predicted_labels_match": true,
+    "test_accuracy": 0.9719887955182073,
+    "test_accuracy_recorded": 0.9719887955182073,
+    "validation_accuracy": 0.9561527581329562,
+    "validation_accuracy_recorded": 0.9561527581329562,
+    "validation_nll": 0.12104469537734985,
+    "validation_nll_recorded": 0.12104468951391349
+  },
+  "examples": {
+    "Restart my web server.": "task",
+    "yes after a restart offer": "task",
+    "yes after a backup-check offer": "question"
+  }
+}

data/README.md

@@ -0,0 +1,23 @@
+# Clover English intent dataset
+
+Developed by AKIKI.AI.
+
+Six labels: chat, question, build, task, fix, unclear. English only. Class IDs A–F follow that order.
+
+Model inputs are only `message` and `context` (earlier turns containing `role` and `text`). `label` is the target. Difficulty, domain, review reasons and all provenance fields are annotations and are excluded from the model input. The supplied renderer enforces this separation.
+
+Examples were generated locally with cached Qwen3.8-27B-FP8, independently classified by cached Qwen3.5-9B, and checked again in a fresh label-blind 27B request. Target labels and earlier votes are hidden, and input order is shuffled. Disagreements are rejected. The extra 27B vote reuses the generator checkpoint, so it is not a third distinct model. For hard examples, both models must recover the intended route from context and classify the same message without context as unclear. Hard unclear examples instead require both to find multiple unresolved referents. Decorative context is insufficient. These are synthetic, model-reviewed labels. Human review remains pending.
+
+| Split | Examples | Easy | Medium | Hard |
+|---|---:|---:|---:|---:|
+| train | 9913 | 3228 | 3325 | 3360 |
+| validation | 707 | 229 | 238 | 240 |
+| test | 714 | 235 | 239 | 240 |
+
+Scenario groups and all scenarios sharing a generation output remain in one split. Exact full-input duplicates are removed. Character n-gram similarity ≥0.92 removes lower-priority cross-split examples, protecting test first, then validation. Every retained input fits both encoder tokenizers without truncation.
+
+The longest rendered input is 126 MiniLM tokens and 127 ModernBERT tokens, including tokenizer special tokens. The experiment's 512-token encoder input cap is therefore not approached; these results do not establish long-context routing performance.
+
+Audit removed 186 examples. File hashes and label/difficulty counts are in manifest.json; drops and reasons are in audit-drops.jsonl. review-sample.jsonl is a stratified sample prepared for inspection, not evidence that human inspection has occurred.
+
+Difficulty is assigned before backbone evaluation. No backbone results were used to label difficulty or select examples. Hard current messages use a class-neutral follow-up bank, with phrase families separated between train, validation and test; the generator invents the contexts. This is a controlled contextual routing test, not an unrestricted natural-conversation sample. Model agreement and lexical filtering do not prove correctness or semantic independence.

data/audit-drops.jsonl

@@ -0,0 +1,186 @@
+{"id": "test-00042-unclear", "reason": "exact_duplicate"}
+{"id": "test-00069-unclear", "reason": "exact_duplicate"}
+{"id": "test-00075-unclear", "reason": "exact_duplicate"}
+{"id": "test-00096-unclear", "reason": "exact_duplicate"}
+{"id": "test-00112-unclear", "reason": "exact_duplicate"}
+{"id": "test-00114-unclear", "reason": "exact_duplicate"}
+{"id": "validation-00003-unclear", "reason": "exact_duplicate"}
+{"id": "validation-00006-unclear", "reason": "exact_duplicate"}
+{"id": "validation-00012-unclear", "reason": "exact_duplicate"}
+{"id": "validation-00021-unclear", "reason": "exact_duplicate"}
+{"id": "validation-00033-unclear", "reason": "exact_duplicate"}
+{"id": "validation-00037-unclear", "reason": "exact_duplicate"}
+{"id": "validation-00039-unclear", "reason": "exact_duplicate"}
+{"id": "validation-00040-unclear", "reason": "exact_duplicate"}
+{"id": "validation-00048-unclear", "reason": "exact_duplicate"}
+{"id": "validation-00063-unclear", "reason": "exact_duplicate"}
+{"id": "validation-00075-unclear", "reason": "exact_duplicate"}
+{"id": "validation-00093-unclear", "reason": "exact_duplicate"}
+{"id": "validation-00117-unclear", "reason": "exact_duplicate"}
+{"id": "train-00012-unclear", "reason": "exact_duplicate"}
+{"id": "train-00045-unclear", "reason": "exact_duplicate"}
+{"id": "train-00054-unclear", "reason": "exact_duplicate"}
+{"id": "train-00072-unclear", "reason": "exact_duplicate"}
+{"id": "train-00075-unclear", "reason": "exact_duplicate"}
+{"id": "train-00123-unclear", "reason": "exact_duplicate"}
+{"id": "train-00127-unclear", "reason": "exact_duplicate"}
+{"id": "train-00129-unclear", "reason": "exact_duplicate"}
+{"id": "train-00168-unclear", "reason": "exact_duplicate"}
+{"id": "train-00177-unclear", "reason": "exact_duplicate"}
+{"id": "train-00204-unclear", "reason": "exact_duplicate"}
+{"id": "train-00213-unclear", "reason": "exact_duplicate"}
+{"id": "train-00225-unclear", "reason": "exact_duplicate"}
+{"id": "train-00234-unclear", "reason": "exact_duplicate"}
+{"id": "train-00243-unclear", "reason": "exact_duplicate"}
+{"id": "train-00252-unclear", "reason": "exact_duplicate"}
+{"id": "train-00258-unclear", "reason": "exact_duplicate"}
+{"id": "train-00264-unclear", "reason": "exact_duplicate"}
+{"id": "train-00279-unclear", "reason": "exact_duplicate"}
+{"id": "train-00288-unclear", "reason": "exact_duplicate"}
+{"id": "train-00289-unclear", "reason": "exact_duplicate"}
+{"id": "train-00304-unclear", "reason": "exact_duplicate"}
+{"id": "train-00310-unclear", "reason": "exact_duplicate"}
+{"id": "train-00315-unclear", "reason": "exact_duplicate"}
+{"id": "train-00321-unclear", "reason": "exact_duplicate"}
+{"id": "train-00325-unclear", "reason": "exact_duplicate"}
+{"id": "train-00328-unclear", "reason": "exact_duplicate"}
+{"id": "train-00334-unclear", "reason": "exact_duplicate"}
+{"id": "train-00354-unclear", "reason": "exact_duplicate"}
+{"id": "train-00366-unclear", "reason": "exact_duplicate"}
+{"id": "train-00372-unclear", "reason": "exact_duplicate"}
+{"id": "train-00375-chat", "reason": "exact_duplicate"}
+{"id": "train-00378-unclear", "reason": "exact_duplicate"}
+{"id": "train-00381-unclear", "reason": "exact_duplicate"}
+{"id": "train-00387-unclear", "reason": "exact_duplicate"}
+{"id": "train-00396-unclear", "reason": "exact_duplicate"}
+{"id": "train-00408-unclear", "reason": "exact_duplicate"}
+{"id": "train-00420-unclear", "reason": "exact_duplicate"}
+{"id": "train-00430-chat", "reason": "exact_duplicate"}
+{"id": "train-00438-unclear", "reason": "exact_duplicate"}
+{"id": "train-00441-unclear", "reason": "exact_duplicate"}
+{"id": "train-00445-unclear", "reason": "exact_duplicate"}
+{"id": "train-00451-unclear", "reason": "exact_duplicate"}
+{"id": "train-00474-unclear", "reason": "exact_duplicate"}
+{"id": "train-00477-unclear", "reason": "exact_duplicate"}
+{"id": "train-00480-unclear", "reason": "exact_duplicate"}
+{"id": "train-00501-unclear", "reason": "exact_duplicate"}
+{"id": "train-00516-unclear", "reason": "exact_duplicate"}
+{"id": "train-00525-unclear", "reason": "exact_duplicate"}
+{"id": "train-00534-unclear", "reason": "exact_duplicate"}
+{"id": "train-00543-unclear", "reason": "exact_duplicate"}
+{"id": "train-00549-unclear", "reason": "exact_duplicate"}
+{"id": "train-00552-unclear", "reason": "exact_duplicate"}
+{"id": "train-00564-unclear", "reason": "exact_duplicate"}
+{"id": "train-00576-unclear", "reason": "exact_duplicate"}
+{"id": "train-00588-unclear", "reason": "exact_duplicate"}
+{"id": "train-00591-unclear", "reason": "exact_duplicate"}
+{"id": "train-00600-unclear", "reason": "exact_duplicate"}
+{"id": "train-00606-unclear", "reason": "exact_duplicate"}
+{"id": "train-00609-unclear", "reason": "exact_duplicate"}
+{"id": "train-00615-unclear", "reason": "exact_duplicate"}
+{"id": "train-00630-unclear", "reason": "exact_duplicate"}
+{"id": "train-00636-unclear", "reason": "exact_duplicate"}
+{"id": "train-00643-unclear", "reason": "exact_duplicate"}
+{"id": "train-00648-unclear", "reason": "exact_duplicate"}
+{"id": "train-00660-unclear", "reason": "exact_duplicate"}
+{"id": "train-00666-unclear", "reason": "exact_duplicate"}
+{"id": "train-00669-unclear", "reason": "exact_duplicate"}
+{"id": "train-00690-unclear", "reason": "exact_duplicate"}
+{"id": "train-00696-unclear", "reason": "exact_duplicate"}
+{"id": "train-00711-unclear", "reason": "exact_duplicate"}
+{"id": "train-00723-unclear", "reason": "exact_duplicate"}
+{"id": "train-00732-unclear", "reason": "exact_duplicate"}
+{"id": "train-00738-unclear", "reason": "exact_duplicate"}
+{"id": "train-00741-unclear", "reason": "exact_duplicate"}
+{"id": "train-00753-unclear", "reason": "exact_duplicate"}
+{"id": "train-00765-unclear", "reason": "exact_duplicate"}
+{"id": "train-00787-unclear", "reason": "exact_duplicate"}
+{"id": "train-00807-unclear", "reason": "exact_duplicate"}
+{"id": "train-00816-chat", "reason": "exact_duplicate"}
+{"id": "train-00816-unclear", "reason": "exact_duplicate"}
+{"id": "train-00855-task", "reason": "exact_duplicate"}
+{"id": "train-00856-unclear", "reason": "exact_duplicate"}
+{"id": "train-00876-unclear", "reason": "exact_duplicate"}
+{"id": "train-00879-unclear", "reason": "exact_duplicate"}
+{"id": "train-00891-unclear", "reason": "exact_duplicate"}
+{"id": "train-00895-unclear", "reason": "exact_duplicate"}
+{"id": "train-00907-unclear", "reason": "exact_duplicate"}
+{"id": "train-00921-unclear", "reason": "exact_duplicate"}
+{"id": "train-00973-unclear", "reason": "exact_duplicate"}
+{"id": "train-00984-chat", "reason": "exact_duplicate"}
+{"id": "train-00999-unclear", "reason": "exact_duplicate"}
+{"id": "train-01011-unclear", "reason": "exact_duplicate"}
+{"id": "train-01014-unclear", "reason": "exact_duplicate"}
+{"id": "train-01017-unclear", "reason": "exact_duplicate"}
+{"id": "train-01047-unclear", "reason": "exact_duplicate"}
+{"id": "train-01050-unclear", "reason": "exact_duplicate"}
+{"id": "train-01060-unclear", "reason": "exact_duplicate"}
+{"id": "train-01071-unclear", "reason": "exact_duplicate"}
+{"id": "train-01074-unclear", "reason": "exact_duplicate"}
+{"id": "train-01095-unclear", "reason": "exact_duplicate"}
+{"id": "train-01108-unclear", "reason": "exact_duplicate"}
+{"id": "train-01140-unclear", "reason": "exact_duplicate"}
+{"id": "train-01143-unclear", "reason": "exact_duplicate"}
+{"id": "train-01146-unclear", "reason": "exact_duplicate"}
+{"id": "train-01158-unclear", "reason": "exact_duplicate"}
+{"id": "train-01165-chat", "reason": "exact_duplicate"}
+{"id": "train-01179-unclear", "reason": "exact_duplicate"}
+{"id": "train-01191-unclear", "reason": "exact_duplicate"}
+{"id": "train-01194-unclear", "reason": "exact_duplicate"}
+{"id": "train-01206-unclear", "reason": "exact_duplicate"}
+{"id": "train-01216-unclear", "reason": "exact_duplicate"}
+{"id": "train-01221-chat", "reason": "exact_duplicate"}
+{"id": "train-01233-unclear", "reason": "exact_duplicate"}
+{"id": "train-01239-unclear", "reason": "exact_duplicate"}
+{"id": "train-01260-unclear", "reason": "exact_duplicate"}
+{"id": "train-01267-unclear", "reason": "exact_duplicate"}
+{"id": "train-01269-unclear", "reason": "exact_duplicate"}
+{"id": "train-01278-unclear", "reason": "exact_duplicate"}
+{"id": "train-01315-unclear", "reason": "exact_duplicate"}
+{"id": "train-01327-chat", "reason": "exact_duplicate"}
+{"id": "train-01335-unclear", "reason": "exact_duplicate"}
+{"id": "train-01338-chat", "reason": "exact_duplicate"}
+{"id": "train-01339-unclear", "reason": "exact_duplicate"}
+{"id": "train-01348-unclear", "reason": "exact_duplicate"}
+{"id": "train-01362-unclear", "reason": "exact_duplicate"}
+{"id": "train-01374-unclear", "reason": "exact_duplicate"}
+{"id": "train-01377-unclear", "reason": "exact_duplicate"}
+{"id": "train-01416-unclear", "reason": "exact_duplicate"}
+{"id": "train-01434-unclear", "reason": "exact_duplicate"}
+{"id": "train-01437-unclear", "reason": "exact_duplicate"}
+{"id": "train-01443-unclear", "reason": "exact_duplicate"}
+{"id": "train-01452-chat", "reason": "exact_duplicate"}
+{"id": "train-01455-chat", "reason": "exact_duplicate"}
+{"id": "train-01467-unclear", "reason": "exact_duplicate"}
+{"id": "train-01473-unclear", "reason": "exact_duplicate"}
+{"id": "train-01494-task", "reason": "exact_duplicate"}
+{"id": "train-01518-unclear", "reason": "exact_duplicate"}
+{"id": "train-01534-unclear", "reason": "exact_duplicate"}
+{"id": "train-01536-unclear", "reason": "exact_duplicate"}
+{"id": "train-01545-unclear", "reason": "exact_duplicate"}
+{"id": "train-01593-unclear", "reason": "exact_duplicate"}
+{"id": "train-01596-unclear", "reason": "exact_duplicate"}
+{"id": "train-01603-unclear", "reason": "exact_duplicate"}
+{"id": "train-01617-unclear", "reason": "exact_duplicate"}
+{"id": "train-01626-chat", "reason": "exact_duplicate"}
+{"id": "train-01641-unclear", "reason": "exact_duplicate"}
+{"id": "train-01656-unclear", "reason": "exact_duplicate"}
+{"id": "train-01662-unclear", "reason": "exact_duplicate"}
+{"id": "train-01674-unclear", "reason": "exact_duplicate"}
+{"id": "train-00003-unclear", "reason": "cross_split_near_duplicate", "similarity": 0.9422524571418762, "protected_split": "test"}
+{"id": "train-00024-unclear", "reason": "cross_split_near_duplicate", "similarity": 0.9288095831871033, "protected_split": "test"}
+{"id": "train-00192-unclear", "reason": "cross_split_near_duplicate", "similarity": 0.9634224772453308, "protected_split": "test"}
+{"id": "train-00397-unclear", "reason": "cross_split_near_duplicate", "similarity": 0.9594231843948364, "protected_split": "test"}
+{"id": "train-00520-unclear", "reason": "cross_split_near_duplicate", "similarity": 0.9355655908584595, "protected_split": "test"}
+{"id": "train-00532-unclear", "reason": "cross_split_near_duplicate", "similarity": 0.952883780002594, "protected_split": "test"}
+{"id": "train-00541-unclear", "reason": "cross_split_near_duplicate", "similarity": 0.9207755923271179, "protected_split": "test"}
+{"id": "train-00546-unclear", "reason": "cross_split_near_duplicate", "similarity": 0.9416508674621582, "protected_split": "test"}
+{"id": "train-01137-unclear", "reason": "cross_split_near_duplicate", "similarity": 0.9582526683807373, "protected_split": "test"}
+{"id": "train-01584-question", "reason": "cross_split_near_duplicate", "similarity": 0.9418998956680298, "protected_split": "test"}
+{"id": "train-00604-unclear", "reason": "cross_split_near_duplicate", "similarity": 0.9394984245300293, "protected_split": "validation"}
+{"id": "train-00963-unclear", "reason": "cross_split_near_duplicate", "similarity": 0.9514755010604858, "protected_split": "validation"}
+{"id": "train-00985-unclear", "reason": "cross_split_near_duplicate", "similarity": 0.9511113166809082, "protected_split": "validation"}
+{"id": "train-01117-task", "reason": "cross_split_near_duplicate", "similarity": 0.9571490287780762, "protected_split": "validation"}
+{"id": "train-01212-unclear", "reason": "cross_split_near_duplicate", "similarity": 0.9307024478912354, "protected_split": "validation"}
+{"id": "train-01290-unclear", "reason": "cross_split_near_duplicate", "similarity": 0.9405972957611084, "protected_split": "validation"}
+{"id": "train-01390-task", "reason": "cross_split_near_duplicate", "similarity": 0.9359763860702515, "protected_split": "validation"}

data/audit.json

@@ -0,0 +1,30 @@
+{
+  "passed": true,
+  "scenario_groups_cross_splits": 0,
+  "exact_full_input_duplicates": 0,
+  "cross_split_similarity_threshold": 0.92,
+  "pre_filter_max_similarities": {
+    "test/validation": 0.9049872756004333,
+    "test/train": 0.9634224772453308,
+    "validation/train": 0.9571490287780762
+  },
+  "dropped": 186,
+  "synthetic_only": true,
+  "human_reviewed": false,
+  "difficulty": "preassigned generation constraints, independent of evaluated backbone scores",
+  "encoder_input_audit": {
+    "minilm-l6-v2": {
+      "maximum_before_filter": 126,
+      "over_512": 0
+    },
+    "modernbert-base": {
+      "maximum_before_filter": 127,
+      "over_512": 0
+    }
+  },
+  "limitations": [
+    "Model agreement is not ground truth; human review required for release-quality claims.",
+    "Lexical near-duplicate filtering does not prove semantic or template independence.",
+    "Difficulty labels reflect the requested construction, not a human difficulty rating."
+  ]
+}

data/manifest.json

@@ -0,0 +1,186 @@
+{
+  "version": 1,
+  "language": "en",
+  "labels": [
+    "chat",
+    "question",
+    "build",
+    "task",
+    "fix",
+    "unclear"
+  ],
+  "difficulty_levels": [
+    "easy",
+    "medium",
+    "hard"
+  ],
+  "rubric_sha256": "6ff2b88f9dfd69573c07a8629fb51977c6b2c72d7e33cbf3978cf209d659f4f3",
+  "splits": {
+    "train": {
+      "file": "train.jsonl",
+      "sha256": "2e3edef9182471d0fcd3738fdf886fdbe78fac5e6f3855373ab92a6ad0f1ca72",
+      "rows": 9913,
+      "label_counts": {
+        "task": 1676,
+        "fix": 1680,
+        "build": 1680,
+        "chat": 1669,
+        "unclear": 1529,
+        "question": 1679
+      },
+      "difficulty_counts": {
+        "medium": 3325,
+        "hard": 3360,
+        "easy": 3228
+      },
+      "label_difficulty_counts": {
+        "chat": {
+          "easy": 552,
+          "medium": 557,
+          "hard": 560
+        },
+        "question": {
+          "easy": 559,
+          "medium": 560,
+          "hard": 560
+        },
+        "build": {
+          "easy": 560,
+          "medium": 560,
+          "hard": 560
+        },
+        "task": {
+          "easy": 558,
+          "medium": 558,
+          "hard": 560
+        },
+        "fix": {
+          "easy": 560,
+          "medium": 560,
+          "hard": 560
+        },
+        "unclear": {
+          "easy": 439,
+          "medium": 530,
+          "hard": 560
+        }
+      }
+    },
+    "validation": {
+      "file": "validation.jsonl",
+      "sha256": "40759974effb15fe000589d5e9082ae94c7c760a9dfef40cb364d95bc74c953b",
+      "rows": 707,
+      "label_counts": {
+        "question": 120,
+        "task": 120,
+        "fix": 120,
+        "build": 120,
+        "unclear": 107,
+        "chat": 120
+      },
+      "difficulty_counts": {
+        "easy": 229,
+        "hard": 240,
+        "medium": 238
+      },
+      "label_difficulty_counts": {
+        "chat": {
+          "easy": 40,
+          "medium": 40,
+          "hard": 40
+        },
+        "question": {
+          "easy": 40,
+          "medium": 40,
+          "hard": 40
+        },
+        "build": {
+          "easy": 40,
+          "medium": 40,
+          "hard": 40
+        },
+        "task": {
+          "easy": 40,
+          "medium": 40,
+          "hard": 40
+        },
+        "fix": {
+          "easy": 40,
+          "medium": 40,
+          "hard": 40
+        },
+        "unclear": {
+          "easy": 29,
+          "medium": 38,
+          "hard": 40
+        }
+      }
+    },
+    "test": {
+      "file": "test.jsonl",
+      "sha256": "8e1fde3f1074fb216e73b879fe3905d4448faf1c7a83d1fef6b3078dc1560c28",
+      "rows": 714,
+      "label_counts": {
+        "unclear": 114,
+        "chat": 120,
+        "build": 120,
+        "task": 120,
+        "question": 120,
+        "fix": 120
+      },
+      "difficulty_counts": {
+        "medium": 239,
+        "hard": 240,
+        "easy": 235
+      },
+      "label_difficulty_counts": {
+        "chat": {
+          "easy": 40,
+          "medium": 40,
+          "hard": 40
+        },
+        "question": {
+          "easy": 40,
+          "medium": 40,
+          "hard": 40
+        },
+        "build": {
+          "easy": 40,
+          "medium": 40,
+          "hard": 40
+        },
+        "task": {
+          "easy": 40,
+          "medium": 40,
+          "hard": 40
+        },
+        "fix": {
+          "easy": 40,
+          "medium": 40,
+          "hard": 40
+        },
+        "unclear": {
+          "easy": 35,
+          "medium": 39,
+          "hard": 40
+        }
+      }
+    }
+  },
+  "audit_sha256": "3ed02c8fb676df20988b89adb1e04ed4fcb99565e3f6bab1e0eabc5bd2778607",
+  "generator": "clover-teacher-27b",
+  "reviewer": "akiki-qwen35-9b",
+  "local_checkpoints": {
+    "generator": "Qwen/Qwen3.8-27B-FP8@017b9c7af6b5689d5dd426a76e0bc077eb5ca20a",
+    "reviewer": "Qwen/Qwen3.5-9B@c202236235762e1c871ad0ccb60c8ee5ba337b9a"
+  },
+  "generation_sources_sha256": {
+    "/mnt/storage/ariadne/scripts/build_clover_dataset.py": "35580bb3da8d9099d3a4f28c7366b498b1da26dc890307861720569636d65af5",
+    "scripts/build_clover_native_batch.py": "ab92d1c1a34e59fccfbe7d17564a0277223ce43ae5fdefe0102c7c1d9c9f816f",
+    "scripts/repair_clover_batch.py": "902630e11625ad04d3e9bf87f7301e8d59b03e80f1fc3ccf2412fb5bfd0b10b4",
+    "scripts/benchmark_clover_generation.py": "a368567a6f364c9dc1cf284fb49f67809aad1ce22de001c2c2f710c123e44144",
+    "ariadne_bench/clover.py": "270b5ab0ea3ad33cec13803f4ccb4ee2ecdd5f8c7152780712546d3e94c8e798",
+    "data/clover-intent-v1/generation-overrides.json": "bdf8a0179a856387ed16807d4f1f141390ed636747fc1476c476ec1f998a721a"
+  },
+  "complete": true
+}

data/train.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2e3edef9182471d0fcd3738fdf886fdbe78fac5e6f3855373ab92a6ad0f1ca72
+size 13057269

model.py

@@ -0,0 +1,122 @@
+"""Akiki MiniLM-L6-v2 Clover intent classifier.
+
+A frozen sentence-transformers/all-MiniLM-L6-v2 encoder, an identity-initialized
+384 x 384 Akiki input affine on its word embeddings, and a six-class head.
+Classes, in order: A=chat, B=question, C=build, D=task, E=fix, F=unclear.
+
+    from model import CloverIntent
+    clf = CloverIntent.from_pretrained(".")
+    clf.predict("Restart my web server.")
+    clf.predict("yes", context=[{"role": "assistant", "text": "Shall I restart nginx?"}])
+
+Command line:
+
+    python model.py 'Restart my web server.'
+    python model.py --context-file context.json --output json 'yes'
+"""
+
+import argparse
+import json
+from pathlib import Path
+
+import torch
+from safetensors.torch import load_file
+from torch import nn
+from torch.nn import functional as F
+
+LABELS = ("chat", "question", "build", "task", "fix", "unclear")
+MAX_LEN = 512
+HERE = Path(__file__).resolve().parent
+
+
+def render(message, context=()):
+    """The exact text seen in training: earlier turns, then the current message."""
+    prior = "\n".join(f"{turn['role']}: {turn['text']}" for turn in context)
+    return ("Conversation context:\n" + prior + "\n\n" if prior else "") + "Current user message: " + message
+
+
+class IdentityAffine(nn.Module):
+    """Akiki input affine, kept in FP32 regardless of autocast."""
+
+    def __init__(self, width):
+        super().__init__()
+        self.proj = nn.Linear(width, width)
+        nn.init.eye_(self.proj.weight)
+        nn.init.zeros_(self.proj.bias)
+
+    def forward(self, x):
+        with torch.autocast(device_type=x.device.type, enabled=False):
+            value = F.linear(x.float(), self.proj.weight.float(), self.proj.bias.float())
+        return value.to(x.dtype)
+
+
+class FrozenEncoderClassifier(nn.Module):
+    def __init__(self, encoder):
+        super().__init__()
+        width = encoder.config.hidden_size
+        self.pretrained = encoder.requires_grad_(False).eval()
+        self.interface = IdentityAffine(width)
+        self.head = nn.Linear(width, len(LABELS))
+        # The affine transforms word embeddings before position and type embeddings are added.
+        encoder.get_input_embeddings().register_forward_hook(lambda module, args, output: self.interface(output))
+
+    def features(self, input_ids, attention_mask):
+        hidden = self.pretrained(input_ids=input_ids, attention_mask=attention_mask, return_dict=True).last_hidden_state
+        mask = attention_mask.unsqueeze(-1).to(hidden.dtype)
+        return F.normalize(((hidden * mask).sum(1) / mask.sum(1).clamp_min(1)).float(), dim=-1)
+
+    def forward(self, input_ids, attention_mask):
+        return self.head(self.features(input_ids, attention_mask))
+
+
+class CloverIntent:
+    def __init__(self, tokenizer, model, device):
+        self.tokenizer, self.model, self.device = tokenizer, model, device
+
+    @classmethod
+    def from_pretrained(cls, path=HERE, adapter="adapter.safetensors", device="cpu"):
+        from transformers import AutoModel, AutoTokenizer
+        path = Path(path)
+        tokenizer = AutoTokenizer.from_pretrained(path / "base_model", local_files_only=True)
+        encoder = AutoModel.from_pretrained(path / "base_model", local_files_only=True,
+                                            dtype=torch.float32, attn_implementation="sdpa")
+        model = FrozenEncoderClassifier(encoder)
+        missing, unexpected = model.load_state_dict(load_file(path / adapter), strict=False)
+        if unexpected or any(not k.startswith("pretrained.") for k in missing):
+            raise ValueError(f"Adapter does not match the model: missing={missing}, unexpected={unexpected}")
+        return cls(tokenizer, model.to(device).eval(), torch.device(device))
+
+    def tokenize(self, texts):
+        batch = self.tokenizer(list(texts), padding=True, truncation=False, return_tensors="pt")
+        if batch["input_ids"].shape[1] > MAX_LEN:
+            raise ValueError(f"Input exceeds {MAX_LEN} tokens; truncation is disabled")
+        return {k: batch[k].to(self.device) for k in ("input_ids", "attention_mask")}
+
+    @torch.inference_mode()
+    def probabilities(self, rows):
+        """rows: dicts with 'message' and optional 'context'. Returns an (n, 6) tensor."""
+        texts = [render(r["message"], r.get("context") or ()) for r in rows]
+        return self.model(**self.tokenize(texts)).float().softmax(-1).cpu()
+
+    def predict(self, message, context=()):
+        p = self.probabilities([{"message": message, "context": context}])[0]
+        index = int(p.argmax())
+        return {"label": LABELS[index], "token": chr(65 + index),
+                "probabilities": dict(zip(LABELS, p.tolist()))}
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument("message")
+    parser.add_argument("--context-file", type=Path, help="JSON array of earlier {role, text} turns")
+    parser.add_argument("--model", type=Path, default=HERE)
+    parser.add_argument("--device", default="cpu")
+    parser.add_argument("--output", choices=["token", "label", "json"], default="token")
+    args = parser.parse_args()
+    context = json.loads(args.context_file.read_text()) if args.context_file else []
+    result = CloverIntent.from_pretrained(args.model, device=args.device).predict(args.message, context)
+    print(json.dumps(result, indent=2) if args.output == "json" else result[args.output])
+
+
+if __name__ == "__main__":
+    main()

model_config.json

@@ -0,0 +1,37 @@
+{
+  "model": "Akiki MiniLM-L6-v2 Clover intent",
+  "repo_id": "Akiki-AI/Akiki-MiniLM-L6-v2-clover-intent",
+  "labels": [
+    "chat",
+    "question",
+    "build",
+    "task",
+    "fix",
+    "unclear"
+  ],
+  "tokens": {
+    "A": "chat",
+    "B": "question",
+    "C": "build",
+    "D": "task",
+    "E": "fix",
+    "F": "unclear"
+  },
+  "base_model": "sentence-transformers/all-MiniLM-L6-v2",
+  "base_revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41",
+  "frozen_backbone_sha256": "11ac3c0bba90239eddece3b7eb97f85d85e1107b1bf82b19e2d32fc98a101aa2",
+  "base_parameters": 22713216,
+  "trainable_parameters": {
+    "input_affine": 147840,
+    "head": 2310,
+    "total": 150150
+  },
+  "adapter": "adapter.safetensors",
+  "released_seed": 2,
+  "best_epoch": 8,
+  "pooling": "attention-masked mean followed by L2 normalization",
+  "max_tokens": 512,
+  "truncation": false,
+  "input": "Conversation context:\\n<role>: <text>...\\n\\nCurrent user message: <message> (context block omitted when empty)",
+  "probabilities": "softmax of head logits; no fitted calibration"
+}

provenance/original/CREDITS.md

@@ -0,0 +1,26 @@
+# Clover experiment credits
+
+Developed by AKIKI.AI.
+
+This is an English, six-category intent-classification experiment. The small input
+interfaces and classification readouts are experiment artifacts; the pretrained
+backbones remain frozen.
+
+| Cached backbone | Provider | Local revision | Cached licence metadata |
+|---|---|---|---|
+| all-MiniLM-L6-v2 | sentence-transformers | 1110a243fdf4706b3f48f1d95db1a4f5529b4d41 | Apache-2.0 in cached model card |
+| ModernBERT-base | answerdotai | 8949b909ec900327062f0ebf497f51aef5e6f0c8 | Apache-2.0 in cached model card |
+| Laya | convaiinnovations | 55cf4c4ebb4ebe31b2550e8bdf3bd21b99753851 | No model card/licence file in this local snapshot; unverified |
+| Qwen3.5-0.8B | Qwen | 2fc06364715b967f1860aea9cf38778875588b17 | Apache-2.0 in cached model card |
+
+Synthetic data uses the existing local Qwen3.8-27B-FP8 checkpoint at revision
+017b9c7af6b5689d5dd426a76e0bc077eb5ca20a and Qwen3.5-9B at revision
+c202236235762e1c871ad0ccb60c8ee5ba337b9a for independent model review.
+
+Tools: the existing Ariadne native adapters and identity input interfaces,
+PyTorch, Hugging Face Transformers, vLLM, NumPy, SciPy, scikit-learn, optimized
+causal-conv1d and FLA kernels, Docker and Git. Model paths and runtime/kernel
+metadata are recorded with the experiment. No model artifacts are downloaded.
+
+The local cached licence metadata above is attribution information. This file
+does not assert a separate patent grant or replace the original licence texts.

provenance/original/README.md

@@ -0,0 +1,117 @@
+# Clover English intent experiment
+
+Developed by AKIKI.AI.
+
+The dataset is in `data/clover-intent-v1`. It contains six intents: chat,
+question, build, task, fix and unclear. Training, validation and test keep
+originating scenarios and generation outputs together. Each split covers easy,
+medium and hard inputs. Difficulty is defined before backbone evaluation.
+
+`generation-benchmark/README.md` records the measured batching, shared prefix,
+sample packing and MTP-three comparison. `dataset-generation.log` and
+`dataset-export.log` record the final build and audit. Model-reviewed labels are
+synthetic and require human review before release-quality acceptance claims.
+
+The comparison uses only the exact local cached MiniLM-L6-v2, ModernBERT-base,
+Laya and Qwen3.5-0.8B checkpoints. Every backbone is evaluated before fitting.
+The encoder zero-shot readout uses fixed label-description similarity; the two
+native decision models use their original frozen heads. Encoder alternatives
+include centroid similarity, k-nearest neighbours, k-means and a regularized
+linear classifier. All supervised or cluster-label fitting uses training labels;
+validation selects hyperparameters and checkpoints.
+
+Each encoder then compares head-only training with Ariadne input-affine plus
+head training. Laya and Qwen train only their input affine maps, preserving their
+native heads. Three seeds are used. All original tensors remain frozen and are
+hashed throughout. `protocol/` records the pre-training commit, data hashes,
+model paths, seeds, optimization settings and stopping rules.
+
+Run the comparison with the existing local runtime:
+
+```bash
+PYTHONPATH=.deps-causal-opt:.deps-decisions:.:scripts \
+  .venv-blackwell/bin/python -u scripts/run_clover_experiments.py
+```
+
+Live progress is in `status.json`, `progress.jsonl` and `experiment.log`.
+`results.md`, `results.json` and `seed-summary.json` collect holdout results,
+including the three difficulty tiers, confusion matrices and false actions on
+chat/question. Individual run directories contain histories, small trained
+checkpoints and per-example predictions. A completed epoch can be resumed.
+
+Inference prints one class token, with A=chat, B=question, C=build, D=task,
+E=fix and F=unclear:
+
+```bash
+PYTHONPATH=.deps-causal-opt:.deps-decisions:.:scripts \
+  .venv-blackwell/bin/python scripts/predict_clover.py \
+  --run runs/clover-intent-v1/minilm-l6-v2/ariadne-seed-0 \
+  'Restart my web server.'
+```
+
+Use `--context-file` for prior role/text turns, and `--output label` to print
+the category name instead. All model loading is offline. This experiment does
+not change a served model.
+
+`inference-smoke.json` records a contextual inference check and a warm CPU
+measurement for MiniLM Ariadne seed 0. The same message, `yes`, returned task
+after an offer to restart the web server and question after an offer to check
+a backup. With eight CPU threads, FP32 PyTorch/Transformers, batch size one,
+five warm-up calls and 100 measured calls, latency was 2.0 ms median and 3.3 ms
+p95. This includes tokenization and classification, and excludes process start
+and model loading. It measures this server CPU, not a laptop or a Rust runtime.
+
+`inference-comparison.json` compares head-only and Ariadne seed 0 on the same
+22-token context/message input, with the same eight-thread FP32 runtime.
+Across 12 paired blocks with randomized method order, 600 measured calls per
+method gave median latency of 2.40 ms for head-only and 2.47 ms for Ariadne
+(p95 2.57 ms and 2.62 ms). The input affine has 147,840 parameters, or 0.59 MB
+of FP32 weights; the shared backbone has 22,713,216 parameters and the
+six-class head has 2,310. These parameter counts describe potential weight
+storage savings, not measured process RAM: the shared benchmark wrapper
+allocates the affine for both methods and disables it for head-only.
+
+The final uniform CPU comparison is in `cpu-latency.json`. It measures all
+readouts on the same contextual request using eight threads, FP32, batch size
+one, five warm-up calls and 100 measured calls, including preparation and
+tokenization. Native choice prompts are longer than encoder inputs. Qwen uses
+the installed Transformers Torch reference functions on CPU; its CUDA-only
+causal/FLA dispatch is bypassed only in that measurement process. The accuracy
+axis in the figures uses the original GPU holdout scores, with BF16 native
+backbones. CPU latency uses seed 0 and is not a re-evaluation of CPU accuracy.
+
+`figures/` contains PNG, SVG and PDF exports of:
+
+- `frozen-to-trained`: frozen readout to Ariadne mean, with sample SD across
+  three seeds. Frozen encoder and native readouts are explicitly distinguished.
+- `latency-accuracy`: Ariadne, head-only and regularized linear classifiers.
+- `latency-accuracy-all-readouts`: additionally includes similarity, nearest
+  neighbours, clustering and untrained readouts.
+- `accuracy-by-difficulty`: easy, medium and hard means and sample SD.
+
+`figures/chart-data.csv` and `figures/chart-data.json` retain the exact plotted
+data and timing specification. Accuracy error bars represent variation across
+seeds; latency whiskers show the 5th–95th percentiles of warm calls. Neither is
+a confidence interval on human-labelled generalization.
+
+Reproduce the charts with the existing plotting environment:
+
+```bash
+.venv-monitor/bin/python scripts/plot_clover_results.py
+```
+
+`final-verification.json` records all 18 completed fits, frozen source/weight
+checks, dataset split hashes and each numerical pass mark. `error-overlap.json`
+compares errors for matched seeds across all four backbones. These analyses
+do not change fitting, data or checkpoint selection. Each model directory
+also contains a `MODEL_CARD.md`.
+
+Every evaluation's `predictions.jsonl` stores the six class probabilities.
+`scripts/analyze_clover_uncertainty.py` derives a neighbouring
+`uncertainty.jsonl` with entropy in nats, normalized entropy (0–1), maximum
+class probability and the top-two probability margin. `uncertainty-summary.json`
+aggregates these by run, difficulty and correctness. This analysis uses the
+saved probabilities, preserves original prediction files/hashes, and does not
+rerun models. Entropy measures probability spread; it is not calibrated error
+risk, and the differing readout scales make direct cross-model interpretation
+tentative. The default prediction CLI continues to print a single class token.

provenance/original/final-verification.json

@@ -0,0 +1,197 @@
+{
+  "experiment_complete": true,
+  "completed_fits": 18,
+  "expected_fits": 18,
+  "frozen_sources_match": true,
+  "source_files": 55,
+  "manifest_matches_protocol": true,
+  "split_hashes_match": {
+    "train": true,
+    "validation": true,
+    "test": true
+  },
+  "all_frozen_weights_verified": true,
+  "human_reviewed": false,
+  "runs": [
+    {
+      "run": "laya/ariadne-seed-0",
+      "accuracy": 0.957983193277311,
+      "false_actions_on_chat_question": 4,
+      "actions_on_unclear": 3,
+      "accuracy_pass": true,
+      "zero_actions_pass": false,
+      "both_numerical_passmarks": false,
+      "frozen_verified": true
+    },
+    {
+      "run": "laya/ariadne-seed-1",
+      "accuracy": 0.9677871148459384,
+      "false_actions_on_chat_question": 2,
+      "actions_on_unclear": 3,
+      "accuracy_pass": true,
+      "zero_actions_pass": false,
+      "both_numerical_passmarks": false,
+      "frozen_verified": true
+    },
+    {
+      "run": "laya/ariadne-seed-2",
+      "accuracy": 0.9509803921568627,
+      "false_actions_on_chat_question": 3,
+      "actions_on_unclear": 2,
+      "accuracy_pass": true,
+      "zero_actions_pass": false,
+      "both_numerical_passmarks": false,
+      "frozen_verified": true
+    },
+    {
+      "run": "minilm-l6-v2/ariadne-seed-0",
+      "accuracy": 0.9663865546218487,
+      "false_actions_on_chat_question": 1,
+      "actions_on_unclear": 6,
+      "accuracy_pass": true,
+      "zero_actions_pass": false,
+      "both_numerical_passmarks": false,
+      "frozen_verified": true
+    },
+    {
+      "run": "minilm-l6-v2/ariadne-seed-1",
+      "accuracy": 0.9593837535014006,
+      "false_actions_on_chat_question": 2,
+      "actions_on_unclear": 8,
+      "accuracy_pass": true,
+      "zero_actions_pass": false,
+      "both_numerical_passmarks": false,
+      "frozen_verified": true
+    },
+    {
+      "run": "minilm-l6-v2/ariadne-seed-2",
+      "accuracy": 0.9719887955182073,
+      "false_actions_on_chat_question": 1,
+      "actions_on_unclear": 3,
+      "accuracy_pass": true,
+      "zero_actions_pass": false,
+      "both_numerical_passmarks": false,
+      "frozen_verified": true
+    },
+    {
+      "run": "minilm-l6-v2/head_only-seed-0",
+      "accuracy": 0.9145658263305322,
+      "false_actions_on_chat_question": 13,
+      "actions_on_unclear": 7,
+      "accuracy_pass": false,
+      "zero_actions_pass": false,
+      "both_numerical_passmarks": false,
+      "frozen_verified": true
+    },
+    {
+      "run": "minilm-l6-v2/head_only-seed-1",
+      "accuracy": 0.9201680672268907,
+      "false_actions_on_chat_question": 12,
+      "actions_on_unclear": 6,
+      "accuracy_pass": false,
+      "zero_actions_pass": false,
+      "both_numerical_passmarks": false,
+      "frozen_verified": true
+    },
+    {
+      "run": "minilm-l6-v2/head_only-seed-2",
+      "accuracy": 0.9159663865546218,
+      "false_actions_on_chat_question": 14,
+      "actions_on_unclear": 7,
+      "accuracy_pass": false,
+      "zero_actions_pass": false,
+      "both_numerical_passmarks": false,
+      "frozen_verified": true
+    },
+    {
+      "run": "modernbert-base/ariadne-seed-0",
+      "accuracy": 0.9663865546218487,
+      "false_actions_on_chat_question": 3,
+      "actions_on_unclear": 4,
+      "accuracy_pass": true,
+      "zero_actions_pass": false,
+      "both_numerical_passmarks": false,
+      "frozen_verified": true
+    },
+    {
+      "run": "modernbert-base/ariadne-seed-1",
+      "accuracy": 0.9607843137254902,
+      "false_actions_on_chat_question": 4,
+      "actions_on_unclear": 4,
+      "accuracy_pass": true,
+      "zero_actions_pass": false,
+      "both_numerical_passmarks": false,
+      "frozen_verified": true
+    },
+    {
+      "run": "modernbert-base/ariadne-seed-2",
+      "accuracy": 0.9635854341736695,
+      "false_actions_on_chat_question": 2,
+      "actions_on_unclear": 1,
+      "accuracy_pass": true,
+      "zero_actions_pass": false,
+      "both_numerical_passmarks": false,
+      "frozen_verified": true
+    },
+    {
+      "run": "modernbert-base/head_only-seed-0",
+      "accuracy": 0.9621848739495799,
+      "false_actions_on_chat_question": 5,
+      "actions_on_unclear": 3,
+      "accuracy_pass": true,
+      "zero_actions_pass": false,
+      "both_numerical_passmarks": false,
+      "frozen_verified": true
+    },
+    {
+      "run": "modernbert-base/head_only-seed-1",
+      "accuracy": 0.9565826330532213,
+      "false_actions_on_chat_question": 5,
+      "actions_on_unclear": 3,
+      "accuracy_pass": true,
+      "zero_actions_pass": false,
+      "both_numerical_passmarks": false,
+      "frozen_verified": true
+    },
+    {
+      "run": "modernbert-base/head_only-seed-2",
+      "accuracy": 0.9537815126050421,
+      "false_actions_on_chat_question": 5,
+      "actions_on_unclear": 4,
+      "accuracy_pass": true,
+      "zero_actions_pass": false,
+      "both_numerical_passmarks": false,
+      "frozen_verified": true
+    },
+    {
+      "run": "qwen3.5-0.8B/ariadne-seed-0",
+      "accuracy": 0.9705882352941176,
+      "false_actions_on_chat_question": 1,
+      "actions_on_unclear": 4,
+      "accuracy_pass": true,
+      "zero_actions_pass": false,
+      "both_numerical_passmarks": false,
+      "frozen_verified": true
+    },
+    {
+      "run": "qwen3.5-0.8B/ariadne-seed-1",
+      "accuracy": 0.9761904761904762,
+      "false_actions_on_chat_question": 0,
+      "actions_on_unclear": 4,
+      "accuracy_pass": true,
+      "zero_actions_pass": true,
+      "both_numerical_passmarks": true,
+      "frozen_verified": true
+    },
+    {
+      "run": "qwen3.5-0.8B/ariadne-seed-2",
+      "accuracy": 0.969187675070028,
+      "false_actions_on_chat_question": 1,
+      "actions_on_unclear": 12,
+      "accuracy_pass": true,
+      "zero_actions_pass": false,
+      "both_numerical_passmarks": false,
+      "frozen_verified": true
+    }
+  ]
+}

provenance/original/minilm-l6-v2/MODEL_CARD.md

@@ -0,0 +1,65 @@
+# Clover English intent: MiniLM-L6-v2
+
+Developed by AKIKI.AI.
+
+Experimental six-category routing specialist. The cached
+`sentence-transformers/all-MiniLM-L6-v2` backbone (revision
+`1110a243fdf4706b3f48f1d95db1a4f5529b4d41`) stays frozen. Ariadne learns
+an identity-initialized input affine and a six-class readout: 150,150 trainable
+parameters, alongside 22,713,216 original backbone parameters. Original tensors
+are hashed and checked throughout training. Cached base licence: Apache-2.0;
+see `../CREDITS.md` for attribution.
+
+Inputs are an English `message` and optional earlier `context` turns with
+`role` and `text`. Output is one token: A=chat, B=question, C=build, D=task,
+E=fix, F=unclear. This specialist classifies intent; it does not generate
+contracts, execute actions, or implement the broader Clover refusal policy.
+
+The backbone and tokenizer support 512 tokens total, including rendered
+context, message and special tokens. The cached Sentence Transformers
+configuration defaults to 256; this experiment loads Transformers directly
+and explicitly allows 512. Oversized inputs raise an error; they are not
+silently truncated. The dataset's longest input is 126 tokens, so the reported
+scores do not establish performance on long briefings or full chat histories.
+
+Training uses 9,913 synthetic examples, with 707 validation and 714 test
+examples. The dataset card and files are under `data/clover-intent-v1` in
+the workspace root. Labels
+have two-model agreement; human review remains pending. Hard cases are a
+controlled context-resolution test with split-separated follow-up families.
+
+All three seeds (0, 1, 2) are retained. Validation NLL selects each checkpoint;
+test results do not select epochs or settings. See `../protocol/plan.json`
+and the committed source snapshot for the exact recipe.
+
+| Method | Test accuracy | Hard accuracy |
+|---|---:|---:|
+| Frozen description similarity, no fitted classifier | 34.9% | 36.7% |
+| Frozen embeddings + regularized linear classifier | 92.3% | 91.7% |
+| Head-only Adam training, three-seed mean | 91.7% | 91.1% |
+| Ariadne, three-seed mean | **96.6%** | **95.6%** |
+
+Ariadne accuracy is 96.6% with sample SD 0.6 percentage points across seeds;
+easy/medium/hard means are 99.4% / 94.8% / 95.6%. The three seeds make 1, 2,
+and 1 action predictions on 240 chat/question examples, and 6, 8, and 3 action
+predictions on 114 unclear examples. Thus none meets the experiment's zero
+observed actions on chat/question requirement, despite exceeding 95% accuracy.
+
+Each Ariadne fit took 28–31 seconds on the assigned RTX PRO 6000. Warm
+PyTorch/Transformers FP32 CPU inference with eight threads measured 2.47 ms
+median and 2.62 ms p95 for seed 0 on a short contextual input, including
+tokenization and excluding model loading. Head-only measured 2.40 ms median
+under the paired benchmark. See `../inference-comparison.json`; these are
+server measurements, not a laptop or Rust runtime claim.
+
+Checkpoints contain the small learned affine and head, not a copy of the
+backbone. Use the existing local cache and `scripts/predict_clover.py`:
+
+```bash
+PYTHONPATH=.deps-causal-opt:.deps-decisions:.:scripts \
+  .venv-blackwell/bin/python scripts/predict_clover.py \
+  --run runs/clover-intent-v1/minilm-l6-v2/ariadne-seed-0 \
+  --device cpu 'Restart my web server.'
+```
+
+Model loading is offline. No serving change or release has been made.

provenance/original/minilm-l6-v2/model.json

@@ -0,0 +1,15 @@
+{
+  "name": "minilm-l6-v2",
+  "path": "/mnt/models/hub/models--sentence-transformers--all-MiniLM-L6-v2/snapshots/1110a243fdf4706b3f48f1d95db1a4f5529b4d41",
+  "revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41",
+  "frozen_sha256": "11ac3c0bba90239eddece3b7eb97f85d85e1107b1bf82b19e2d32fc98a101aa2",
+  "base_parameters": 22713216,
+  "readout": "six-class head; label-description cosine prototypes before any training",
+  "input": "rendered conversation context and current user message; no metadata or labels",
+  "torch_version": "2.13.0+cu130",
+  "transformers_version": "5.17.0",
+  "cuda_runtime": "13.0",
+  "device": "cuda:1",
+  "max_len": 512,
+  "pooling": "attention-masked mean followed by L2 normalization"
+}

provenance/original/protocol/PASSMARK-CLOVER-1.md

@@ -0,0 +1,9 @@
+# Clover English intent experiment
+
+Developed by AKIKI.AI.
+
+Six categories: chat, question, build, task, fix, unclear. Output is a single class ID.
+
+Targets: at least 95% accuracy and zero observed build/task/fix classifications on pure chat/question. Report easy, medium and hard separately. These are experimental measurements on model-reviewed synthetic data, not a human-verified acceptance exam. No release or serving change is authorized.
+
+All dataset hashes, seeds, methods and stopping rules are in plan.json. Select hyperparameters and checkpoints on validation only. Holdout never enters training. Frozen backbone and original native heads remain unchanged. No model downloads.

provenance/original/protocol/plan.json

@@ -0,0 +1,291 @@
+{
+  "dataset": "/mnt/storage/ariadne/data/clover-intent-v1",
+  "dataset_manifest_sha256": "857d674c7b0cf9bc9ab112fce5b6db2cdead4055c8e9c9b2901f2706bc8f7c59",
+  "splits": {
+    "train": {
+      "file": "train.jsonl",
+      "sha256": "2e3edef9182471d0fcd3738fdf886fdbe78fac5e6f3855373ab92a6ad0f1ca72",
+      "rows": 9913,
+      "label_counts": {
+        "task": 1676,
+        "fix": 1680,
+        "build": 1680,
+        "chat": 1669,
+        "unclear": 1529,
+        "question": 1679
+      },
+      "difficulty_counts": {
+        "medium": 3325,
+        "hard": 3360,
+        "easy": 3228
+      },
+      "label_difficulty_counts": {
+        "chat": {
+          "easy": 552,
+          "medium": 557,
+          "hard": 560
+        },
+        "question": {
+          "easy": 559,
+          "medium": 560,
+          "hard": 560
+        },
+        "build": {
+          "easy": 560,
+          "medium": 560,
+          "hard": 560
+        },
+        "task": {
+          "easy": 558,
+          "medium": 558,
+          "hard": 560
+        },
+        "fix": {
+          "easy": 560,
+          "medium": 560,
+          "hard": 560
+        },
+        "unclear": {
+          "easy": 439,
+          "medium": 530,
+          "hard": 560
+        }
+      }
+    },
+    "validation": {
+      "file": "validation.jsonl",
+      "sha256": "40759974effb15fe000589d5e9082ae94c7c760a9dfef40cb364d95bc74c953b",
+      "rows": 707,
+      "label_counts": {
+        "question": 120,
+        "task": 120,
+        "fix": 120,
+        "build": 120,
+        "unclear": 107,
+        "chat": 120
+      },
+      "difficulty_counts": {
+        "easy": 229,
+        "hard": 240,
+        "medium": 238
+      },
+      "label_difficulty_counts": {
+        "chat": {
+          "easy": 40,
+          "medium": 40,
+          "hard": 40
+        },
+        "question": {
+          "easy": 40,
+          "medium": 40,
+          "hard": 40
+        },
+        "build": {
+          "easy": 40,
+          "medium": 40,
+          "hard": 40
+        },
+        "task": {
+          "easy": 40,
+          "medium": 40,
+          "hard": 40
+        },
+        "fix": {
+          "easy": 40,
+          "medium": 40,
+          "hard": 40
+        },
+        "unclear": {
+          "easy": 29,
+          "medium": 38,
+          "hard": 40
+        }
+      }
+    },
+    "test": {
+      "file": "test.jsonl",
+      "sha256": "8e1fde3f1074fb216e73b879fe3905d4448faf1c7a83d1fef6b3078dc1560c28",
+      "rows": 714,
+      "label_counts": {
+        "unclear": 114,
+        "chat": 120,
+        "build": 120,
+        "task": 120,
+        "question": 120,
+        "fix": 120
+      },
+      "difficulty_counts": {
+        "medium": 239,
+        "hard": 240,
+        "easy": 235
+      },
+      "label_difficulty_counts": {
+        "chat": {
+          "easy": 40,
+          "medium": 40,
+          "hard": 40
+        },
+        "question": {
+          "easy": 40,
+          "medium": 40,
+          "hard": 40
+        },
+        "build": {
+          "easy": 40,
+          "medium": 40,
+          "hard": 40
+        },
+        "task": {
+          "easy": 40,
+          "medium": 40,
+          "hard": 40
+        },
+        "fix": {
+          "easy": 40,
+          "medium": 40,
+          "hard": 40
+        },
+        "unclear": {
+          "easy": 35,
+          "medium": 39,
+          "hard": 40
+        }
+      }
+    }
+  },
+  "models": {
+    "minilm-l6-v2": {
+      "path": "/mnt/models/hub/models--sentence-transformers--all-MiniLM-L6-v2/snapshots/1110a243fdf4706b3f48f1d95db1a4f5529b4d41",
+      "kind": "encoder",
+      "width": 384
+    },
+    "modernbert-base": {
+      "path": "/mnt/models/hub/models--answerdotai--ModernBERT-base/snapshots/8949b909ec900327062f0ebf497f51aef5e6f0c8",
+      "kind": "encoder",
+      "width": 768
+    },
+    "laya": {
+      "path": "/mnt/storage/ariadne/.cache/huggingface/hub/models--convaiinnovations--laya/snapshots/55cf4c4ebb4ebe31b2550e8bdf3bd21b99753851",
+      "kind": "native"
+    },
+    "qwen3.5-0.8B": {
+      "path": "/mnt/models/hub/models--Qwen--Qwen3.5-0.8B/snapshots/2fc06364715b967f1860aea9cf38778875588b17",
+      "kind": "native"
+    }
+  },
+  "seeds": [
+    0,
+    1,
+    2
+  ],
+  "methods": {
+    "encoders": [
+      "description_similarity",
+      "class_centroids",
+      "knn",
+      "kmeans",
+      "linear_head",
+      "head_only",
+      "ariadne"
+    ],
+    "native": [
+      "native_frozen",
+      "ariadne"
+    ]
+  },
+  "stopping": {
+    "minimum_epochs": 10,
+    "patience": 3,
+    "selection": "lowest validation NLL; no test-based choices"
+  },
+  "optimization": {
+    "batch": 32,
+    "microbatch": 32,
+    "head_lr": 0.001,
+    "interface_lr": 0.0001,
+    "weight_decay": 0.0,
+    "clip": 1.0,
+    "head_only_feature_cache": true,
+    "feature_cache_validation": "live frozen features must match within 2e-5"
+  },
+  "seed_summary": "mean and sample standard deviation; all individual runs retained",
+  "reported_metrics": [
+    "accuracy",
+    "macro_f1",
+    "nll",
+    "brier",
+    "confusion_matrix",
+    "difficulty_tiers",
+    "actions_on_chat_question",
+    "actions_on_unclear"
+  ],
+  "passmarks": {
+    "accuracy": 0.95,
+    "false_actions_on_chat_question": 0,
+    "difficulty_tiers": [
+      "easy",
+      "medium",
+      "hard"
+    ]
+  },
+  "baseline_caveat": "Description similarity is an explicitly constructed zero-shot readout, not a native semantic head on MiniLM/ModernBERT.",
+  "label_quality": "synthetic, two-model agreement; no claim of human-labelled ground truth",
+  "output": "single class ID; Qwen uses one native letter token; JSON is artifact storage only",
+  "sources": {
+    "ariadne_bench/__init__.py": "a73c9fdf21413382cb93b523bfb3925aa3fd354e18faeead4650e86a30645411",
+    "ariadne_bench/__main__.py": "935a1c1166b0c1ea35a82256345000bf2c73ded718d77773bc27a71ecce28f7d",
+    "ariadne_bench/adapters.py": "3bd4820165be077b3b98fe83ac23e649e1ad69186ad5f2a4f5ee7addf1427cf5",
+    "ariadne_bench/arguana_monitor.py": "40702d8f1cfc6ecadd779235918c2eabfc92bf60c540136a63ec6aa651d4f415",
+    "ariadne_bench/causal_selector.py": "0b7b1c2662e13734f21ffeec5cf875c9f5733dc17e08ae05a8352e724cf46a8f",
+    "ariadne_bench/cli.py": "de267c0502489eb4bb37e4b8f6faa8da8be37a56ced270f754424a40404d1af6",
+    "ariadne_bench/clover.py": "270b5ab0ea3ad33cec13803f4ccb4ee2ecdd5f8c7152780712546d3e94c8e798",
+    "ariadne_bench/clover_models.py": "2d08a3d37c4c171250e5e2bcf1ad572d23ce420fff7d37e61fcccda08e5cca8c",
+    "ariadne_bench/datasets.py": "31c722c64895519a4635b501da88862b446384682e3e8ee884495941d6990912",
+    "ariadne_bench/decider_interface.py": "1ff921ea8c3bfe33d06dd091317fa698c8052557466b3f0d0da400d3e1340e08",
+    "ariadne_bench/embeddinggemma2_td.py": "d39dd9aa9bbfa7766f3cc7a4753c0a305fb63e13de2ac96f1f00a573357cac88",
+    "ariadne_bench/extraction_monitor.py": "218e0f3ec738a9c89ab5c2830e546c35381420aa8adda41bdd369837c5dd40a1",
+    "ariadne_bench/frozen_fp8_linear.py": "f044798c66ac93d446a77dc7e04f461730d56a6bf4daf99dc20544935c78dbdc",
+    "ariadne_bench/frozen_input_interface.py": "253f0186dac6e4a7f4b0fcf3c33449c3b3882cf8be538dca6eeca8d9e4ce9b83",
+    "ariadne_bench/full_finetune.py": "6067be8cf837d254b919c841c9a9851e1546a413dbf6324ff242e5d2f56c9734",
+    "ariadne_bench/full_input_interface.py": "a8f440a2185085c64896300a243ac52a9619c4d05a1b4e43c50c2c32aa1da996",
+    "ariadne_bench/hybrid.py": "90fbbdab9b8157a32854d27c2d93089be6dc88dfe9320426f102eae4e03d9ffd",
+    "ariadne_bench/hybrid_control.py": "1b53678bea73fe7608570b78d7215d1dff2e38d6ad3627615f26c06ae74d8b4c",
+    "ariadne_bench/interfaces.py": "19700fb8170b2460401900cb96ebdff22bd7bf778ca5132e98d56ba68f344187",
+    "ariadne_bench/metrics.py": "fc6fde20c95d5052de52d05d6fb9b6eb69b780c46966f5bf1e517738b276a4b4",
+    "ariadne_bench/minilm_fusion_td.py": "02911ff0ffbe59258bbd2ba26d11a66adbe383c59d968b18053dc33c13bc9531",
+    "ariadne_bench/minilm_retrieval.py": "b7e5eec2d9b471ee86162fb1f77ff662ebb12948a6b7bf14a527e392f3a54651",
+    "ariadne_bench/minilm_second_affine_head_td.py": "adfbfbfcec2736a75c149acab4ffd2f06880087d375e9e91fd44cb881aade36b",
+    "ariadne_bench/minilm_second_affine_td.py": "30520235f801857975cd34e37e095949d5c4b61ede155e982e2497689b4cb97d",
+    "ariadne_bench/minilm_td.py": "6bd5d04dfea12cb74005369a483bacf4beb036d5acf2f21c369765e4d36c2baa",
+    "ariadne_bench/minilm_three_pass_td.py": "5e94d643ffb7b2c02fea2e323d0a6ce90f08039587a22455eab03f5b0ce31335",
+    "ariadne_bench/minilm_two_pass_post_mean_td.py": "c34692cb6cbe6bf596d3e4c9dbc9cbd94e7c208706a0909c1bf37172c9ada924",
+    "ariadne_bench/minilm_two_pass_post_weighted_mean_td.py": "59711d070660b76494cd0f8e5d9ed19e16e49f16f783578fe1a595bc5c262a8c",
+    "ariadne_bench/minilm_two_pass_td.py": "db737ec3f55d22ed60de5ad1c078d04a4dff26ad192065e30e9b982ef861e2e9",
+    "ariadne_bench/modernbert_td.py": "d34de2ba4413fec74a0701c446b68d9c8c50dcfaf4c69cb49d750f170530a48e",
+    "ariadne_bench/portable.py": "259f633a8bb6663d0594d9639ea8d80ef493f5a63d4b11444a4003f2f9a318e8",
+    "ariadne_bench/pythia_selector.py": "cc5f46794a0cc07ae7e14731eb022d0eeed970be7b0b7c885d7b1b6f07860ad5",
+    "ariadne_bench/qwen35_moe_fast.py": "4e40340ebeb2eefb39ad58b6ff738f7190e3336ce6a85208a6cbf812b7f02b6b",
+    "ariadne_bench/qwen35_moe_fast_selector.py": "303457ed725e748cc512e3c64ba8c9f2fb1ec6744d8e9629e0f39e4668d51e83",
+    "ariadne_bench/qwen35_moe_scaling_selector.py": "1b86e32817aff48e3c54d25d9878ddc34841065a1f9bb9503adf2576d1ce645b",
+    "ariadne_bench/qwen35_scaling_selector.py": "a807f53c03cee3ba3e0d19a594d9ed9c5c078cbed141925550ff816e1c100724",
+    "ariadne_bench/qwen38_fp8_native_selector.py": "d8871ee965ef500988b147aa86b9c9beaa0566ffeafa322b2c510e670027a3e9",
+    "ariadne_bench/qwen38_fp8_scaling_selector.py": "cc7a64e814942ed3c6a6f2a1bdc1c33dfe078ca9a9524459aadb1b3ee0553bba",
+    "ariadne_bench/qwen_comparison_monitor.py": "b546c4c64ddf53e91a7b07a5753b7e81c8c3a318ba9e67f20d43263785b24531",
+    "ariadne_bench/raw_extraction.py": "e36dca0644fdf9b90262b0071a719f86a8f75d12f189ee0cba6f6d1bd51fa578",
+    "ariadne_bench/reproducibility.py": "5b0c161c596f25277b3326c645df53d5d1f2dc0272da9e77bc4a4c0d56188b6e",
+    "ariadne_bench/rotary_selector.py": "2b7a80a20688170fe7680b5d8f2e3a8b279638ccebe442ba473db12e920251c9",
+    "ariadne_bench/runner.py": "91c934177113c34052d0560938cd0ab40804339349591ce515ee1a055d986c58",
+    "ariadne_bench/scaling_selector.py": "0489fb0bab0e627595bc22fb8cec2524f1633f48d632d095860a35c35d474805",
+    "ariadne_bench/schema.py": "e4f60c57ada82b964bfbbb3e2d4b122416ab01b480484d99b3850e7c185f08c4",
+    "ariadne_bench/specializations.py": "25eada26991e9c43e3d0f5b383bbe67975ff6b7495a9d5aa81d76644b7dde010",
+    "ariadne_bench/td_monitor.py": "78766de89adb42fa8e7dafb22190ed7cd62a9a3d99399db6f37143437e5d3c5c",
+    "scripts/run_clover_experiments.py": "df8b0c3c79c627618a2ae2a12e02cb861d1369ffbbe86f5c60cd3112836c3af5",
+    "scripts/build_clover_dataset.py": "35580bb3da8d9099d3a4f28c7366b498b1da26dc890307861720569636d65af5",
+    "scripts/build_clover_native_batch.py": "ab92d1c1a34e59fccfbe7d17564a0277223ce43ae5fdefe0102c7c1d9c9f816f",
+    "scripts/repair_clover_batch.py": "902630e11625ad04d3e9bf87f7301e8d59b03e80f1fc3ccf2412fb5bfd0b10b4",
+    "scripts/benchmark_clover_generation.py": "a368567a6f364c9dc1cf284fb49f67809aad1ce22de001c2c2f710c123e44144",
+    "scripts/finish_clover_dataset.py": "bb44d35eea6939434fa7cfc541c05a20692768905e52e9aed68771f1ec0b97e9",
+    "scripts/predict_clover.py": "07ba1f84b4bed51072f943c663835b350ab2afa0f50e31e6b12e0abfe36e203c",
+    "data/clover-intent-v1/generation-overrides.json": "bdf8a0179a856387ed16807d4f1f141390ed636747fc1476c476ec1f998a721a"
+  }
+}

provenance/original/results.md

@@ -0,0 +1,67 @@
+# Clover English intent comparison
+
+Developed by AKIKI.AI. Synthetic, model-reviewed labels; human review pending.
+
+All original backbone weights remain frozen. Validation alone chooses training settings and checkpoints.
+
+Results across seeds (mean ± sample standard deviation where repeated):
+
+| Model / method | Runs | Accuracy | Macro F1 | Hard accuracy |
+|---|---:|---:|---:|---:|
+| laya / ariadne | 3 | 95.9% ± 0.8 | 0.959 ± 0.008 | 94.9% ± 2.4 |
+| laya / native_frozen | 1 | 52.4% | 0.474 | 55.8% |
+| minilm-l6-v2 / ariadne | 3 | 96.6% ± 0.6 | 0.966 ± 0.006 | 95.6% ± 0.6 |
+| minilm-l6-v2 / cosine nearest labelled examples | 1 | 78.9% | 0.786 | 79.2% |
+| minilm-l6-v2 / description_similarity | 1 | 34.9% | 0.313 | 36.7% |
+| minilm-l6-v2 / frozen embeddings + regularized linear classifier | 1 | 92.3% | 0.923 | 91.7% |
+| minilm-l6-v2 / head_only | 3 | 91.7% ± 0.3 | 0.917 ± 0.003 | 91.1% ± 0.6 |
+| minilm-l6-v2 / k-means on training embeddings; training-majority cluster labels | 3 | 51.2% ± 0.5 | 0.514 ± 0.008 | 54.7% ± 2.7 |
+| minilm-l6-v2 / training-class-centroid cosine similarity | 1 | 80.5% | 0.804 | 81.2% |
+| modernbert-base / ariadne | 3 | 96.4% ± 0.3 | 0.964 ± 0.003 | 96.0% ± 0.9 |
+| modernbert-base / cosine nearest labelled examples | 1 | 90.1% | 0.901 | 88.8% |
+| modernbert-base / description_similarity | 1 | 28.6% | 0.218 | 30.4% |
+| modernbert-base / frozen embeddings + regularized linear classifier | 1 | 95.7% | 0.956 | 95.0% |
+| modernbert-base / head_only | 3 | 95.8% ± 0.4 | 0.958 ± 0.004 | 94.9% ± 0.9 |
+| modernbert-base / k-means on training embeddings; training-majority cluster labels | 3 | 72.0% ± 3.7 | 0.724 ± 0.040 | 54.9% ± 2.1 |
+| modernbert-base / training-class-centroid cosine similarity | 1 | 80.5% | 0.802 | 61.7% |
+| qwen3.5-0.8B / ariadne | 3 | 97.2% ± 0.4 | 0.972 ± 0.004 | 96.1% ± 1.3 |
+| qwen3.5-0.8B / native_frozen | 1 | 30.5% | 0.255 | 14.6% |
+
+Individual runs:
+
+| Model / method | Accuracy | Macro F1 | Easy | Medium | Hard | Actions on chat/question | Actions on unclear |
+|---|---:|---:|---:|---:|---:|---:|---:|
+| laya/ariadne-seed-0/holdout | 95.8% | 0.958 | 98.7% | 92.9% | 95.8% | 4/240 | 3/114 |
+| laya/ariadne-seed-1/holdout | 96.8% | 0.968 | 98.7% | 95.0% | 96.7% | 2/240 | 3/114 |
+| laya/ariadne-seed-2/holdout | 95.1% | 0.951 | 99.6% | 93.7% | 92.1% | 3/240 | 2/114 |
+| laya/frozen-baseline | 52.4% | 0.474 | 54.9% | 46.4% | 55.8% | 18/240 | 76/114 |
+| minilm-l6-v2/ariadne-seed-0/holdout | 96.6% | 0.966 | 99.6% | 95.0% | 95.4% | 1/240 | 6/114 |
+| minilm-l6-v2/ariadne-seed-1/holdout | 95.9% | 0.960 | 99.1% | 93.7% | 95.0% | 2/240 | 8/114 |
+| minilm-l6-v2/ariadne-seed-2/holdout | 97.2% | 0.972 | 99.6% | 95.8% | 96.2% | 1/240 | 3/114 |
+| minilm-l6-v2/embedding-baselines/class_centroids | 80.5% | 0.804 | 81.3% | 79.1% | 81.2% | 29/240 | 20/114 |
+| minilm-l6-v2/embedding-baselines/kmeans-seed-0 | 51.7% | 0.523 | 48.5% | 49.0% | 57.5% | 105/240 | 34/114 |
+| minilm-l6-v2/embedding-baselines/kmeans-seed-1 | 50.8% | 0.512 | 49.4% | 48.5% | 54.6% | 98/240 | 34/114 |
+| minilm-l6-v2/embedding-baselines/kmeans-seed-2 | 51.0% | 0.508 | 51.5% | 49.4% | 52.1% | 85/240 | 20/114 |
+| minilm-l6-v2/embedding-baselines/knn | 78.9% | 0.786 | 81.7% | 75.7% | 79.2% | 32/240 | 12/114 |
+| minilm-l6-v2/embedding-baselines/linear_head | 92.3% | 0.923 | 94.5% | 90.8% | 91.7% | 10/240 | 5/114 |
+| minilm-l6-v2/frozen-baseline | 34.9% | 0.313 | 31.5% | 36.4% | 36.7% | 58/240 | 23/114 |
+| minilm-l6-v2/head_only-seed-0/holdout | 91.5% | 0.914 | 94.9% | 89.1% | 90.4% | 13/240 | 7/114 |
+| minilm-l6-v2/head_only-seed-1/holdout | 92.0% | 0.920 | 94.9% | 89.5% | 91.7% | 12/240 | 6/114 |
+| minilm-l6-v2/head_only-seed-2/holdout | 91.6% | 0.916 | 94.5% | 89.1% | 91.2% | 14/240 | 7/114 |
+| modernbert-base/ariadne-seed-0/holdout | 96.6% | 0.966 | 98.7% | 96.2% | 95.0% | 3/240 | 4/114 |
+| modernbert-base/ariadne-seed-1/holdout | 96.1% | 0.961 | 97.9% | 93.7% | 96.7% | 4/240 | 4/114 |
+| modernbert-base/ariadne-seed-2/holdout | 96.4% | 0.964 | 98.7% | 94.1% | 96.2% | 2/240 | 1/114 |
+| modernbert-base/embedding-baselines/class_centroids | 80.5% | 0.802 | 96.2% | 84.1% | 61.7% | 14/240 | 9/114 |
+| modernbert-base/embedding-baselines/kmeans-seed-0 | 67.8% | 0.679 | 86.0% | 65.3% | 52.5% | 78/240 | 6/114 |
+| modernbert-base/embedding-baselines/kmeans-seed-1 | 74.6% | 0.754 | 91.1% | 77.8% | 55.4% | 32/240 | 11/114 |
+| modernbert-base/embedding-baselines/kmeans-seed-2 | 73.7% | 0.740 | 88.5% | 76.2% | 56.7% | 33/240 | 9/114 |
+| modernbert-base/embedding-baselines/knn | 90.1% | 0.901 | 97.0% | 84.5% | 88.8% | 19/240 | 3/114 |
+| modernbert-base/embedding-baselines/linear_head | 95.7% | 0.956 | 98.3% | 93.7% | 95.0% | 5/240 | 4/114 |
+| modernbert-base/frozen-baseline | 28.6% | 0.218 | 23.4% | 31.8% | 30.4% | 30/240 | 3/114 |
+| modernbert-base/head_only-seed-0/holdout | 96.2% | 0.962 | 99.1% | 93.7% | 95.8% | 5/240 | 3/114 |
+| modernbert-base/head_only-seed-1/holdout | 95.7% | 0.957 | 99.1% | 93.3% | 94.6% | 5/240 | 3/114 |
+| modernbert-base/head_only-seed-2/holdout | 95.4% | 0.954 | 99.1% | 92.9% | 94.2% | 5/240 | 4/114 |
+| qwen3.5-0.8B/ariadne-seed-0/holdout | 97.1% | 0.970 | 98.3% | 96.2% | 96.7% | 1/240 | 4/114 |
+| qwen3.5-0.8B/ariadne-seed-1/holdout | 97.6% | 0.976 | 98.7% | 97.1% | 97.1% | 0/240 | 4/114 |
+| qwen3.5-0.8B/ariadne-seed-2/holdout | 96.9% | 0.969 | 98.7% | 97.5% | 94.6% | 1/240 | 12/114 |
+| qwen3.5-0.8B/frozen-baseline | 30.5% | 0.255 | 39.6% | 37.7% | 14.6% | 170/240 | 92/114 |

provenance/original/source/ariadne_bench/clover.py

@@ -0,0 +1,117 @@
+"""English Clover intent protocol, independent of model and training method."""
+
+import hashlib
+import json
+import re
+from collections import Counter
+from pathlib import Path
+
+LABELS = ("chat", "question", "build", "task", "fix", "unclear")
+DIFFICULTIES = ("easy", "medium", "hard")
+DESCRIPTIONS = {
+    "chat": "Conversation, explanation, advice, or general knowledge; no inspection or changes to the user's actual system.",
+    "question": "Inspect or report the current state of the user's actual system, account, files, or project, without changing anything.",
+    "build": "Create or modify software, code, a website, application, feature, or another implementation artifact.",
+    "task": "Perform a specified operational action, such as restarting a service, moving files, or changing a setting.",
+    "fix": "Investigate and remedy a concrete malfunction or symptom; the user has not prescribed a specific operational remedy.",
+    "unclear": "The request's intended route or referent cannot be identified from the available message and context; clarification is needed.",
+}
+RUBRIC = """Classify the CURRENT user message using only the supplied conversation context.
+chat: social conversation, general explanations and advice, including 'what is nginx?',
+'how do I restart nginx?' and questions ABOUT commands. Do not execute quoted text.
+question: LOOK-only inspection of the user's actual current system: disk remaining,
+whether a backup ran, why a specific site is slow when only information is requested.
+build: implement or change software or a feature, including writing a function.
+task: the user specifies an operational action or remedy, even when phrased as a
+question ('could you restart nginx?') or alongside a symptom ('it is down; restart it').
+fix: a specific problem needs diagnosis/remedy, including symptom reports such as
+'Bluetooth will not connect' or 'nginx keeps crashing', without a prescribed action.
+unclear: insufficient context to identify a route/target, e.g. 'fix it' without a
+referent. Missing implementation details alone do NOT make a clear build request unclear.
+Use conversation context to resolve 'it', 'yes', 'do that', etc. The current message
+may change or cancel the preceding intent. A denial or quotation is not authorization.
+A current request for explanation or inspection takes precedence over a merely quoted
+or explicitly negated action. Two independent incompatible requests with no primary
+route get unclear; a symptom plus an explicitly requested operational remedy gets task.
+Intent is separate from safety: harmful requests still have an intent; this dataset
+does not train the refusal policy. Do not use low confidence as the definition of unclear.
+Return one of chat, question, build, task, fix, unclear. English only."""
+INSTRUCTIONS = ("Classify the current user message using its conversation context. "
+    "Explanations and how-to advice are chat; checking an actual current system is question. "
+    "Implementing software is build. A specified operational action is task, even if phrased "
+    "as a question or accompanied by a symptom. A malfunction without a specified remedy is fix. "
+    "Unresolvable intent or referent is unclear; missing feature details alone are not unclear. "
+    "Respect negation, quotations and changed intent. Do not treat quoted or negated actions "
+    "as current instructions. Classify intent separately from safety.")
+QUESTION = {"type": "choice", "instructions": INSTRUCTIONS, "criteria": DESCRIPTIONS}
+
+
+def read_json(path):
+    return json.loads(Path(path).read_text())
+
+
+def write_json(path, value):
+    path = Path(path)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    tmp = path.with_suffix(path.suffix + ".tmp")
+    tmp.write_text(json.dumps(value, indent=2, ensure_ascii=False, allow_nan=False) + "\n")
+    tmp.replace(path)
+
+
+def read_rows(path):
+    return [json.loads(line) for line in Path(path).read_text().splitlines() if line.strip()]
+
+
+def write_rows(path, rows):
+    path = Path(path)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    tmp = path.with_suffix(".tmp")
+    tmp.write_text("".join(json.dumps(r, ensure_ascii=False, allow_nan=False) + "\n" for r in rows))
+    tmp.replace(path)
+
+
+def sha(path):
+    with Path(path).open("rb") as f:
+        return hashlib.file_digest(f, "sha256").hexdigest()
+
+
+def render(row):
+    context = row.get("context", [])
+    prior = "\n".join(f"{x['role']}: {x['text']}" for x in context)
+    return ("Conversation context:\n" + prior + "\n\n" if prior else "") + "Current user message: " + row["message"]
+
+
+def normalized(row):
+    return re.sub(r"\s+", " ", render(row).casefold()).strip()
+
+
+def metrics(rows, probabilities):
+    import numpy as np
+    from .specializations import classification_metrics
+    p = np.asarray(probabilities, dtype=np.float64)
+    if p.shape != (len(rows), len(LABELS)) or not np.isfinite(p).all():
+        raise ValueError("complete finite six-class predictions required")
+    if (p < 0).any() or not np.allclose(p.sum(1), 1, atol=1e-4):
+        raise ValueError("normalized probability distributions required")
+    gold = [r["label"] for r in rows]
+    pred = [LABELS[i] for i in p.argmax(1)]
+    result = classification_metrics(gold, pred, list(LABELS))
+    y = np.array([LABELS.index(g) for g in gold])
+    result["nll"] = float(-np.log(np.maximum(p[np.arange(len(y)), y], 1e-12)).mean())
+    result["brier"] = float(((p - np.eye(6)[y]) ** 2).sum(1).mean())
+    passive = [i for i,g in enumerate(gold) if g in ("chat", "question")]
+    result["false_actions"] = sum(pred[i] in ("build", "task", "fix") for i in passive)
+    result["passive_cases"] = len(passive)
+    result["false_action_rate"] = result["false_actions"] / len(passive) if passive else None
+    unresolved=[i for i,g in enumerate(gold) if g=='unclear']
+    result['unclear_cases']=len(unresolved)
+    result['actions_on_unclear']=sum(pred[i] in ('build','task','fix') for i in unresolved)
+    result['action_rate_on_unclear']=result['actions_on_unclear']/len(unresolved) if unresolved else None
+    result["by_difficulty"] = {}
+    for level in DIFFICULTIES:
+        indices = [i for i,r in enumerate(rows) if r["difficulty"] == level]
+        if indices:
+            result["by_difficulty"][level] = classification_metrics(
+                [gold[i] for i in indices], [pred[i] for i in indices], list(LABELS))
+    result["difficulty_counts"] = dict(Counter(r["difficulty"] for r in rows))
+    return result

provenance/original/source/ariadne_bench/clover_models.py

@@ -0,0 +1,210 @@
+"""Local-only Clover encoder classifier and native Laya/Qwen decision adapters."""
+
+import os
+os.environ.update(HF_HUB_OFFLINE="1", TRANSFORMERS_OFFLINE="1", HF_DATASETS_OFFLINE="1")
+
+import copy
+import hashlib
+from pathlib import Path
+
+import torch
+from torch import nn
+from torch.nn import functional as F
+
+from .clover import DESCRIPTIONS, INSTRUCTIONS, LABELS, QUESTION, render
+from .interfaces import IdentityAffine
+
+ROOT = Path(__file__).resolve().parents[1]
+MODELS = {
+    "minilm-l6-v2": {"path": "/mnt/models/hub/models--sentence-transformers--all-MiniLM-L6-v2/snapshots/1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "kind":"encoder", "width":384},
+    "modernbert-base": {"path": "/mnt/models/hub/models--answerdotai--ModernBERT-base/snapshots/8949b909ec900327062f0ebf497f51aef5e6f0c8", "kind":"encoder", "width":768},
+    "laya": {"path": str(ROOT / ".cache/huggingface/hub/models--convaiinnovations--laya/snapshots/55cf4c4ebb4ebe31b2550e8bdf3bd21b99753851"), "kind":"native"},
+    "qwen3.5-0.8B": {"path":"/mnt/models/hub/models--Qwen--Qwen3.5-0.8B/snapshots/2fc06364715b967f1860aea9cf38778875588b17", "kind":"native"},
+}
+
+
+def frozen_hash(module):
+    digest = hashlib.sha256()
+    for name,value in sorted(module.state_dict().items()):
+        digest.update(name.encode());digest.update(str((tuple(value.shape),value.dtype)).encode())
+        digest.update(value.detach().cpu().contiguous().view(torch.uint8).numpy().tobytes())
+    return digest.hexdigest()
+
+
+class FrozenEncoderClassifier(nn.Module):
+    def __init__(self, pretrained, width, modernbert=False):
+        super().__init__()
+        self.pretrained = pretrained.requires_grad_(False).eval()
+        self.modernbert = modernbert
+        self.interface = IdentityAffine(width)
+        self.head = nn.Linear(width, len(LABELS))
+        self.interface_enabled = False
+        self.hook = pretrained.get_input_embeddings().register_forward_hook(self._transform)
+        self.eval()
+
+    def _transform(self, module, args, output):
+        return self.interface(output) if self.interface_enabled else output
+
+    def train(self, mode=True):
+        super().train(mode)
+        self.pretrained.eval()
+        return self
+
+    def features(self, **batch):
+        backbone = self.pretrained.model if self.modernbert else self.pretrained
+        hidden = backbone(**batch, return_dict=True).last_hidden_state
+        mask = batch["attention_mask"].unsqueeze(-1).to(hidden.dtype)
+        return F.normalize(((hidden * mask).sum(1) / mask.sum(1).clamp_min(1)).float(), dim=-1)
+
+    def forward(self, **batch):
+        return self.head(self.features(**batch))
+
+
+class Encoder:
+    def __init__(self, name, device="cuda:1", seed=0):
+        import transformers
+        from transformers import AutoModel, AutoModelForMaskedLM, AutoTokenizer
+        torch.manual_seed(seed)
+        self.name,self.device = name,torch.device(device)
+        self.path = Path(MODELS[name]["path"]).resolve(strict=True)
+        self.tokenizer = AutoTokenizer.from_pretrained(self.path,local_files_only=True)
+        cls = AutoModelForMaskedLM if name=="modernbert-base" else AutoModel
+        pretrained = cls.from_pretrained(self.path,local_files_only=True,dtype=torch.float32,attn_implementation="sdpa")
+        self.model = FrozenEncoderClassifier(pretrained,MODELS[name]["width"],name=="modernbert-base").to(self.device)
+        self.initial_hash = frozen_hash(self.model.pretrained)
+        self.metadata = {"name":name,"path":str(self.path),"revision":self.path.name,
+                         "frozen_sha256":self.initial_hash,"base_parameters":sum(p.numel() for p in pretrained.parameters()),
+                         "readout":"six-class head; label-description cosine prototypes before any training",
+                         "input":"rendered conversation context and current user message; no metadata or labels",
+                         "torch_version":str(torch.__version__),"transformers_version":transformers.__version__,
+                         "cuda_runtime":torch.version.cuda,"device":str(self.device),
+                         "max_len":512,"pooling":"attention-masked mean followed by L2 normalization"}
+        with torch.no_grad():
+            self.prototypes = self.model.features(**self.tokenize(["Current user message: " + DESCRIPTIONS[l] for l in LABELS]))
+            self.model.head.weight.copy_(10 * self.prototypes)
+            self.model.head.bias.zero_()
+
+    def tokenize(self, texts):
+        result = self.tokenizer(texts,padding=True,truncation=False,return_tensors="pt")
+        if result["input_ids"].shape[1] > 512:
+            raise ValueError("Clover input exceeds 512 tokens; truncation is disabled")
+        return {k:v.to(self.device) for k,v in result.items() if k in ('input_ids','attention_mask')}
+
+    def batch(self,rows):
+        return self.tokenize([render(r) for r in rows])
+
+    def logits(self, rows):
+        return self.model(**self.batch(rows))
+
+    def configure(self, method):
+        self.model.interface_enabled = method=="ariadne"
+        self.model.interface.requires_grad_(method=="ariadne")
+        self.model.head.requires_grad_(True)
+        self.model.pretrained.requires_grad_(False).eval()
+        return [{"params":self.model.head.parameters(),"lr":1e-3}] + (
+            [{"params":self.model.interface.parameters(),"lr":1e-4}] if method=="ariadne" else [])
+
+    def state(self):
+        return {k:v.detach().cpu().clone() for k,v in self.model.state_dict().items() if k.startswith(("interface.","head."))}
+
+    def load(self,state):
+        self.model.load_state_dict(state,strict=False)
+
+    def verify_frozen(self):
+        assert all(not p.requires_grad and p.grad is None for p in self.model.pretrained.parameters())
+        assert frozen_hash(self.model.pretrained)==self.initial_hash
+
+    @torch.no_grad()
+    def verify_identity(self,rows):
+        self.model.interface_enabled=False;native=self.logits(rows).clone()
+        self.model.interface_enabled=True;identity=self.logits(rows).clone()
+        self.model.interface_enabled=False
+        assert torch.equal(native,identity),'identity affine changed pretrained encoder logits'
+
+
+class Native:
+    def __init__(self,name,device="cuda:1",seed=0):
+        from .adapters import create_adapter
+        self.name,self.device = name,torch.device(device)
+        path=Path(MODELS[name]["path"]).resolve(strict=True)
+        options={"device":device,"seed":seed,"max_len":2048,"batch_size":8,"add_interface":True}
+        if name=="laya":
+            options.update(laya_model=str(path),head_max_len=1024,training_scope="bridge",deterministic=True)
+            factory="ariadne_bench.frozen_input_interface:create"
+        else:
+            options.update(llm_model=str(path),base_model="Qwen/Qwen3.5-0.8B",base_revision=path.name)
+            factory="ariadne_bench.qwen35_scaling_selector:create"
+        self.adapter=create_adapter({"adapter":"python","factory":factory,"options":options})
+        self.model=self.adapter.model
+        self.initial_hash=self.adapter.frozen_state_sha256()
+        self.metadata={**self.adapter.metadata,"clover_model":name,"training":"input affine only; native decision/vocabulary head frozen",
+                       "clover_scoring_temperature":1.0,"probabilities":"softmax of raw native logits; no fitted calibration"}
+        if name=='qwen3.5-0.8B':
+            self.metadata.update(chat_template_applied=True,prompt_format='clover-native-chat-letter-v1')
+
+    def items(self,rows):
+        if self.name=="qwen3.5-0.8B":
+            # Apply the instruction checkpoint's native chat template, then gather
+            # single-letter scores from its frozen vocabulary head. No generation.
+            tokenizer=self.adapter.tokenizer
+            letters=[x[0] for x in self.adapter.candidates[:6]]
+            rubric=INSTRUCTIONS+"\n"+"\n".join(f"({letter}) {label}: {DESCRIPTIONS[label]}" for letter,label in zip(letters,LABELS))
+            items=[]
+            for row in rows:
+                prompt=tokenizer.apply_chat_template(
+                    [{"role":"system","content":rubric+"\nReturn only the answer letter."},
+                     {"role":"user","content":render(row)}], tokenize=False,add_generation_prompt=True,enable_thinking=False)
+                ids=tokenizer.encode(prompt,add_special_tokens=False)
+                if len(ids)>2048:raise ValueError("Complete Qwen prompt exceeds 2048 tokens")
+                items.append({"ids":ids,"slot":len(ids)-1,"keys":list(LABELS)})
+            return items
+        result=[]
+        for row in rows:
+            item=self.adapter.prepare(render(row),{"intent":copy.deepcopy(QUESTION)})[0]
+            if item["keys"]!=list(LABELS):raise ValueError("native label order changed")
+            if item.get("state_tokens_used")!=item.get("state_tokens_original"):
+                raise ValueError("Laya truncated the current message or context")
+            result.append(item)
+        return result
+
+    def logits(self,rows):
+        from .hybrid import collate
+        items=self.items(rows)
+        batch=self.adapter.collate(items) if hasattr(self.adapter,"collate") else collate(items,self.adapter.pad_id,self.device)
+        with self.adapter.autocast():
+            logits,_=self.model(**batch)
+        return logits.float()
+
+    def configure(self,method):
+        if method!="ariadne":raise ValueError("Native models train only their input affine")
+        return [{"params":[p for p in self.model.parameters() if p.requires_grad],"lr":1e-4}]
+
+    def state(self):
+        names=self.adapter.checkpoint_tensor_names()
+        return {k:v.detach().cpu().clone() for k,v in self.model.state_dict().items() if k in names}
+
+    def load(self,state):
+        self.model.load_state_dict(state,strict=False)
+
+    def verify_frozen(self):
+        assert self.adapter.frozen_state_sha256()==self.initial_hash
+        trained=set(self.adapter.checkpoint_tensor_names())
+        assert all(not p.requires_grad and p.grad is None for n,p in self.model.named_parameters() if n not in trained)
+
+    @torch.no_grad()
+    def verify_identity(self,rows):
+        identity=self.logits(rows).clone()
+        if self.name=='qwen3.5-0.8B':
+            self.model.disable_interface()
+            try:native=self.logits(rows).clone()
+            finally:self.model.enable_interface()
+        else:
+            embeddings=self.model.encoder.embeddings;saved=embeddings.interface
+            embeddings.interface=nn.Identity()
+            try:native=self.logits(rows).clone()
+            finally:embeddings.interface=saved
+        assert torch.equal(native,identity),'identity affine changed native decision logits'
+
+
+def load(name,device="cuda:1",seed=0):
+    return Encoder(name,device,seed) if MODELS[name]["kind"]=="encoder" else Native(name,device,seed)

provenance/original/source/ariadne_bench/interfaces.py

@@ -0,0 +1,121 @@
+"""Input transformations with explicit initialization and low-rank factors."""
+
+import math
+
+import torch
+from torch import nn
+from torch.nn import functional as F
+
+
+class IdentityAffine(nn.Module):
+    """Affine map initialized as identity, preserving input precision under AMP.
+
+    Autocasting an identity GEMM to BF16 would round FP32 embeddings before
+    BERT sees them. Keep this small interface in FP32 and return the input dtype.
+    CUDA callers must use highest float32 matmul precision (no TF32 rounding).
+    """
+
+    def __init__(self, width, residual=False):
+        super().__init__()
+        self.residual = residual
+        self.proj = nn.Linear(width, width)
+        if residual:
+            nn.init.zeros_(self.proj.weight)
+        else:
+            nn.init.eye_(self.proj.weight)
+        nn.init.zeros_(self.proj.bias)
+
+    def forward(self, x):
+        with torch.autocast(device_type=x.device.type, enabled=False):
+            value = F.linear(x.float(), self.proj.weight.float(), self.proj.bias.float())
+            if self.residual:
+                value = x.float() + value
+        return value.to(x.dtype)
+
+
+class LowRankLinear(nn.Module):
+    def __init__(self, width, rank, residual=False):
+        super().__init__()
+        if not 1 <= rank <= width:
+            raise ValueError("rank must be between one and the hidden width")
+        self.rank, self.residual = rank, residual
+        self.a = nn.Parameter(torch.empty(rank, width))
+        self.b = nn.Parameter(torch.empty(width, rank))
+        self.bias = nn.Parameter(torch.zeros(width))
+        if residual:
+            nn.init.kaiming_uniform_(self.a, a=math.sqrt(5))
+            nn.init.zeros_(self.b)
+        else:
+            # A pure rank-r map cannot start at identity. Start as an orthogonal
+            # rank-r projection instead, and record this in experiment metadata.
+            q, _ = torch.linalg.qr(torch.randn(width, rank), mode="reduced")
+            with torch.no_grad():
+                self.a.copy_(q.T)
+                self.b.copy_(q)
+
+    def forward(self, x):
+        value = F.linear(F.linear(x, self.a), self.b, self.bias)
+        return x + value if self.residual else value
+
+    def dense_weight(self):
+        weight = self.b @ self.a
+        if self.residual:
+            weight = weight + torch.eye(weight.shape[0], device=weight.device, dtype=weight.dtype)
+        return weight
+
+    @classmethod
+    def from_svd(cls, u, s, vh, bias, rank, residual=False):
+        layer = cls(u.shape[0], rank, residual=residual)
+        with torch.no_grad():
+            root = s[:rank].sqrt()
+            layer.a.copy_(root[:, None] * vh[:rank])
+            layer.b.copy_(u[:, :rank] * root[None, :])
+            layer.bias.copy_(bias)
+        return layer
+
+
+class Diagonal(nn.Module):
+    def __init__(self, width):
+        super().__init__()
+        self.scale = nn.Parameter(torch.ones(width))
+        self.bias = nn.Parameter(torch.zeros(width))
+
+    def forward(self, x):
+        return x * self.scale + self.bias
+
+
+class ResidualInput(nn.Module):
+    def __init__(self, width):
+        super().__init__()
+        self.norm = nn.LayerNorm(width)
+        self.linear = nn.Linear(width, width)
+        nn.init.zeros_(self.linear.weight)
+        nn.init.zeros_(self.linear.bias)
+
+    def forward(self, x):
+        return x + self.linear(self.norm(x))
+
+
+def make_interface(width, specification):
+    kind = specification.get("kind", "norm_linear")
+    if kind in ("exact_linear", "exact_linear_residual"):
+        return IdentityAffine(width, residual=kind == "exact_linear_residual")
+    if kind == "identity":
+        return nn.Identity()
+    if kind == "norm":
+        return nn.LayerNorm(width)
+    if kind == "diagonal":
+        return Diagonal(width)
+    if kind == "residual":
+        return ResidualInput(width)
+    if kind in ("low_rank", "low_rank_delta"):
+        return nn.Sequential(
+            nn.LayerNorm(width),
+            LowRankLinear(width, specification["rank"], residual=kind == "low_rank_delta"),
+        )
+    if kind in ("linear", "norm_linear"):
+        linear = nn.Linear(width, width)
+        nn.init.eye_(linear.weight)
+        nn.init.zeros_(linear.bias)
+        return linear if kind == "linear" else nn.Sequential(nn.LayerNorm(width), linear)
+    raise ValueError(f"unknown interface kind: {kind}")

provenance/original/source/scripts/package_clover_minilm.py

@@ -0,0 +1,154 @@
+"""Assemble the private Hugging Face package for the Clover MiniLM-L6-v2 intent model.
+
+The released adapter is Ariadne seed 2, chosen on validation NLL (the experiment's
+checkpoint criterion); test results played no part. Seeds 0 and 1 ship alongside.
+The packaged model.py is re-run on CPU and checked against the recorded predictions.
+
+    .venv-blackwell/bin/python scripts/package_clover_minilm.py
+"""
+
+import os
+os.environ.update(HF_HUB_OFFLINE='1',TRANSFORMERS_OFFLINE='1',HF_DATASETS_OFFLINE='1')
+
+import hashlib
+import importlib.util
+import json
+import shutil
+import sys
+from pathlib import Path
+
+import numpy as np
+import torch
+from safetensors.torch import save_file
+
+ROOT=Path(__file__).resolve().parents[1]
+sys.path.insert(0,str(ROOT))
+from ariadne_bench.clover import read_json,read_rows,sha,write_json
+from ariadne_bench.clover_models import MODELS,frozen_hash
+
+RUN=ROOT/'runs/clover-intent-v1';MINILM=RUN/'minilm-l6-v2';DATA=ROOT/'data/clover-intent-v1'
+OUT=ROOT/'releases/akiki-minilm-l6-v2-clover-intent'
+SEEDS=(0,1,2);RELEASED=2
+REPO_ID='Akiki-AI/Akiki-MiniLM-L6-v2-clover-intent'
+
+
+def copy(src,dst):
+    dst=OUT/dst;dst.parent.mkdir(parents=True,exist_ok=True);shutil.copy2(Path(src).resolve(strict=True),dst)
+
+
+def select_seed():
+    best={}
+    for s in SEEDS:
+        h=read_json(MINILM/f'ariadne-seed-{s}/history.json')
+        e=min(h,key=lambda x:x['validation']['nll'])
+        best[s]={'epoch':e['epoch'],'validation_nll':e['validation']['nll'],'validation_accuracy':e['validation']['accuracy']}
+        assert e['epoch']==read_json(MINILM/f'ariadne-seed-{s}/complete.json')['best_epoch']
+    chosen=min(best,key=lambda s:best[s]['validation_nll'])
+    assert chosen==RELEASED,best
+    return best
+
+
+def assemble():
+    for d in ('base_model','checkpoints','results','data','provenance','checks'):shutil.rmtree(OUT/d,ignore_errors=True)
+    base=Path(MODELS['minilm-l6-v2']['path'])
+    for f in base.rglob('*'):
+        if f.is_file():copy(f,Path('base_model')/f.relative_to(base))
+    (OUT/'checkpoints').mkdir()
+    for s in SEEDS:
+        state=torch.load(MINILM/f'ariadne-seed-{s}/best.pt',map_location='cpu',weights_only=True)
+        assert set(state)=={'interface.proj.weight','interface.proj.bias','head.weight','head.bias'}
+        save_file({k:v.contiguous() for k,v in state.items()},OUT/f'checkpoints/seed-{s}.safetensors',
+                  metadata={'seed':str(s),'source':f'runs/clover-intent-v1/minilm-l6-v2/ariadne-seed-{s}/best.pt'})
+    shutil.copy2(OUT/f'checkpoints/seed-{RELEASED}.safetensors',OUT/'adapter.safetensors')
+    for s in SEEDS:
+        src=MINILM/f'ariadne-seed-{s}'
+        for f in ('holdout/report.json','holdout/predictions.jsonl','holdout/uncertainty.jsonl','history.json','model.json','complete.json'):
+            copy(src/f,Path('results/seeds')/f'seed-{s}'/f.replace('holdout/','test-'))
+    for f in ('report.json','predictions.jsonl','uncertainty.jsonl'):copy(MINILM/f'ariadne-seed-{RELEASED}/holdout/{f}',f'results/test-{f}')
+    for f in ('frozen-baseline','head_only-seed-0','head_only-seed-1','head_only-seed-2'):
+        rep=MINILM/f/('report.json' if f=='frozen-baseline' else 'holdout/report.json');copy(rep,f'results/baselines/{f}.json')
+    for f in ('class_centroids','knn','linear_head','kmeans-seed-0','kmeans-seed-1','kmeans-seed-2'):
+        copy(MINILM/'embedding-baselines'/f/'report.json',f'results/baselines/embedding-{f}.json')
+    summary=read_json(RUN/'seed-summary.json')
+    write_json(OUT/'results/seed-summary-minilm.json',[r for r in summary if r['model']=='minilm-l6-v2'])
+    for f in ('cpu-latency.json','inference-comparison.json','inference-smoke.json'):copy(RUN/f,f'results/{f}')
+    for f in ('train.jsonl','validation.jsonl','test.jsonl','README.md','manifest.json','audit.json','audit-drops.jsonl'):copy(DATA/f,f'data/{f}')
+    for f in ('protocol/plan.json','protocol/PASSMARK-CLOVER-1.md','CREDITS.md','README.md','results.md','final-verification.json','minilm-l6-v2/MODEL_CARD.md','minilm-l6-v2/model.json'):
+        copy(RUN/f,Path('provenance/original')/f)
+    for f in ('ariadne_bench/clover.py','ariadne_bench/clover_models.py','ariadne_bench/interfaces.py','scripts/run_clover_experiments.py','scripts/predict_clover.py','scripts/package_clover_minilm.py'):
+        copy(ROOT/f,Path('provenance/original/source')/f)
+    copy(ROOT/'LICENSE','LICENSE')
+
+
+def verify(selection):
+    torch.set_num_threads(8);torch.set_float32_matmul_precision('highest')
+    spec=importlib.util.spec_from_file_location('akiki_clover_model',OUT/'model.py');module=importlib.util.module_from_spec(spec);spec.loader.exec_module(module)
+    info=read_json(MINILM/'model.json');result={'device':'cpu','torch':torch.__version__,'released_seed':RELEASED,'selection':selection}
+    test=read_rows(DATA/'test.jsonl');validation=read_rows(DATA/'validation.jsonl')
+    for s in SEEDS:
+        clf=module.CloverIntent.from_pretrained(OUT,adapter=f'checkpoints/seed-{s}.safetensors')
+        assert frozen_hash(clf.model.pretrained)==info['frozen_sha256'],'packaged backbone differs from the frozen training backbone'
+        probs=lambda rows:torch.cat([clf.probabilities(rows[i:i+64]) for i in range(0,len(rows),64)]).numpy()
+        recorded={r['id']:r for r in read_rows(MINILM/f'ariadne-seed-{s}/holdout/predictions.jsonl')}
+        p=probs(test);ref=np.array([[recorded[r['id']]['probabilities'][l] for l in module.LABELS] for r in test])
+        pred=[module.LABELS[i] for i in p.argmax(1)]
+        v=probs(validation);y=np.array([module.LABELS.index(r['label']) for r in validation])
+        best=selection[s]
+        result[f'seed-{s}']={
+            'frozen_backbone_sha256_matches':True,
+            'test_max_abs_probability_difference':float(np.abs(p-ref).max()),
+            'test_predicted_labels_match':pred==[recorded[r['id']]['predicted'] for r in test],
+            'test_accuracy':float(np.mean([a==r['label'] for a,r in zip(pred,test)])),
+            'test_accuracy_recorded':read_json(MINILM/f'ariadne-seed-{s}/holdout/report.json')['accuracy'],
+            'validation_accuracy':float((v.argmax(1)==y).mean()),'validation_accuracy_recorded':best['validation_accuracy'],
+            'validation_nll':float(-np.log(np.maximum(v[np.arange(len(y)),y],1e-12)).mean()),'validation_nll_recorded':best['validation_nll']}
+        r=result[f'seed-{s}']
+        assert r['test_predicted_labels_match'] and r['test_max_abs_probability_difference']<1e-4,r
+        assert abs(r['validation_accuracy']-best['validation_accuracy'])<1e-9 and abs(r['validation_nll']-best['validation_nll'])<1e-4,r
+    smoke=module.CloverIntent.from_pretrained(OUT)
+    result['examples']={
+        'Restart my web server.':smoke.predict('Restart my web server.')['label'],
+        'yes after a restart offer':smoke.predict('yes',[{'role':'assistant','text':'Would you like me to restart the web server?'}])['label'],
+        'yes after a backup-check offer':smoke.predict('yes',[{'role':'assistant','text':'Would you like me to check whether last night\'s backup ran?'}])['label']}
+    write_json(OUT/'checks/packaged-inference-verification.json',result)
+    return result
+
+
+def finish(check):
+    report=read_json(OUT/'results/test-report.json')
+    write_json(OUT/'model_config.json',{
+        'model':'Akiki MiniLM-L6-v2 Clover intent','repo_id':REPO_ID,'labels':list(read_json(DATA/'manifest.json')['labels']),
+        'tokens':{chr(65+i):l for i,l in enumerate(read_json(DATA/'manifest.json')['labels'])},
+        'base_model':'sentence-transformers/all-MiniLM-L6-v2','base_revision':MODELS['minilm-l6-v2']['path'].rsplit('/',1)[1],
+        'frozen_backbone_sha256':read_json(MINILM/'model.json')['frozen_sha256'],'base_parameters':22713216,
+        'trainable_parameters':{'input_affine':384*384+384,'head':6*384+6,'total':150150},
+        'adapter':'adapter.safetensors','released_seed':RELEASED,'best_epoch':report['best_epoch'],
+        'pooling':'attention-masked mean followed by L2 normalization','max_tokens':512,'truncation':False,
+        'input':'Conversation context:\\n<role>: <text>...\\n\\nCurrent user message: <message> (context block omitted when empty)',
+        'probabilities':'softmax of head logits; no fitted calibration'})
+    (OUT/'NOTICE').write_text(
+        'Akiki MiniLM-L6-v2 Clover intent classifier\nDeveloped by AKIKI.AI.\n\n'
+        'Project code is supplied under the Apache License, Version 2.0 (see LICENSE).\n'
+        'Base model: sentence-transformers/all-MiniLM-L6-v2, revision\n'
+        f"{MODELS['minilm-l6-v2']['path'].rsplit('/',1)[1]}, Apache-2.0.\n"
+        'Original base-model card and attribution: base_model/README.md.\n'
+        'https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2\n\n'
+        'Clover intent data (data/) is synthetic, generated and reviewed locally by AKIKI.AI with\n'
+        'cached Qwen checkpoints; see data/README.md and provenance/original/CREDITS.md.\n'
+        'Labels are model-reviewed; human review is pending.\n\n'
+        'provenance/original/source/ preserves the original experiment implementation for audit,\n'
+        'under its internal project names. The root-level model.py is portable and needs no\n'
+        'original project files. Result and manifest files copied verbatim from the experiment\n'
+        "keep the internal method name 'ariadne', which is the Akiki input affine.\n")
+    files=sorted(p for p in OUT.rglob('*') if p.is_file() and p.name!='SHA256SUMS' and '__pycache__' not in p.parts)
+    (OUT/'SHA256SUMS').write_text(''.join(f'{sha(p)}  {p.relative_to(OUT).as_posix()}\n' for p in files))
+    write_json(ROOT/'releases/akiki-minilm-l6-v2-clover-intent-package.json',{
+        'folder':OUT.name,'repo_id':REPO_ID,'files':len(files)+1,'bytes':sum(p.stat().st_size for p in files),
+        'adapter_sha256':sha(OUT/'adapter.safetensors'),'released_seed':RELEASED,
+        'test_accuracy':report['accuracy'],'test_macro_f1':report['macro_f1'],
+        'inference_verification':{k:v for k,v in check.items() if k.startswith('seed-')},'upload_performed':False})
+
+
+if __name__=='__main__':
+    selection=select_seed();assemble();check=verify(selection);finish(check)
+    print(json.dumps(check,indent=2))

provenance/original/source/scripts/predict_clover.py

@@ -0,0 +1,61 @@
+"""Print one Clover class token from a local experiment artifact.
+
+A=chat, B=question, C=build, D=task, E=fix, F=unclear.
+Use --output label to print the category name instead.
+"""
+
+import os
+os.environ.update(HF_HUB_OFFLINE='1',TRANSFORMERS_OFFLINE='1',HF_DATASETS_OFFLINE='1')
+
+import argparse
+from pathlib import Path
+
+import numpy as np
+import torch
+
+from ariadne_bench.clover import LABELS,read_json
+from ariadne_bench.clover_models import Encoder,load
+
+
+def main():
+    p=argparse.ArgumentParser(description=__doc__)
+    p.add_argument('--run',type=Path,required=True,help='A training-run directory or baseline report directory')
+    p.add_argument('message')
+    p.add_argument('--context-file',type=Path,help='JSON array of prior role/text conversation turns')
+    p.add_argument('--device',default='cuda:1')
+    p.add_argument('--output',choices=['token','id','label'],default='token')
+    args=p.parse_args();folder=args.run.resolve(strict=True)
+    row={'message':args.message,'context':read_json(args.context_file) if args.context_file else []}
+    if (folder/'best.pt').exists():
+        info=read_json(folder/'model.json');name=info.get('name',info.get('clover_model'))
+        model=load(name,args.device,0)
+        expected=info.get('frozen_sha256',info.get('frozen_state_sha256'))
+        if expected!=model.initial_hash:raise ValueError('Frozen backbone differs from the training artifact')
+        method='head_only' if folder.name.startswith('head_only-') else 'ariadne'
+        model.configure(method);model.load(torch.load(folder/'best.pt',map_location='cpu',weights_only=True))
+        with torch.inference_mode():index=int(model.logits([row]).argmax(-1).item())
+    else:
+        info=read_json(folder/'report.json');model=load(info['model'],args.device,0)
+        with torch.inference_mode():
+            if not isinstance(model,Encoder) or folder.name=='frozen-baseline':
+                index=int(model.logits([row]).argmax(-1).item())
+            else:
+                feature=model.model.features(**model.batch([row])).cpu().numpy()[0]
+                root=folder.parent
+                if folder.name=='class_centroids':
+                    index=int((np.load(folder/'centroids.npy')@feature).argmax())
+                elif folder.name=='knn':
+                    x=np.load(root/'train-embeddings.npy');y=np.load(root/'train-labels.npy');k=info['k']
+                    neighbors=np.argpartition(-(x@feature),k-1)[:k]
+                    index=int(np.bincount(y[neighbors],minlength=6).argmax())
+                elif folder.name=='linear_head':
+                    weights=np.load(root/'linear-head.npz');index=int((weights['weight']@feature+weights['bias']).argmax())
+                elif folder.name.startswith('kmeans-seed-'):
+                    saved=np.load(root/(folder.name+'.npz'))
+                    cluster=int(((saved['centers']-feature)**2).sum(1).argmin())
+                    index=int(saved['probabilities'][cluster].argmax())
+                else:raise ValueError('Unrecognized baseline directory')
+    print(chr(65+index) if args.output=='token' else index if args.output=='id' else LABELS[index])
+
+
+if __name__=='__main__':main()

provenance/original/source/scripts/run_clover_experiments.py

@@ -0,0 +1,373 @@
+"""Run Clover baselines and frozen-backbone specialization, entirely offline.
+
+Validation selects hyperparameters/checkpoints; holdout results never control
+training. All pretrained weights are hashed before and after each training run.
+"""
+
+import os
+os.environ.update(HF_HUB_OFFLINE="1", TRANSFORMERS_OFFLINE="1", HF_DATASETS_OFFLINE="1",
+                  CUBLAS_WORKSPACE_CONFIG=":4096:8", TOKENIZERS_PARALLELISM="false")
+
+import argparse
+import fcntl
+import gc
+import json
+import math
+import random
+import shutil
+import subprocess
+import sys
+import time
+from pathlib import Path
+
+# Reuse local CPU analysis dependencies while keeping the selected environment's
+# PyTorch/Transformers first. Qwen's installed CUDA kernels require PyTorch 2.13.
+sys.path.append(str(Path(__file__).resolve().parents[1]/'.deps-specializations'))
+
+import numpy as np
+import torch
+from torch.nn import functional as F
+
+from ariadne_bench.clover import (
+    DESCRIPTIONS, DIFFICULTIES, LABELS, RUBRIC, metrics, read_json, read_rows, render,
+    sha, write_json, write_rows,
+)
+from ariadne_bench.clover_models import MODELS, Encoder, load
+
+ROOT=Path(__file__).resolve().parents[1]
+os.environ.setdefault('TRITON_CACHE_DIR',str(ROOT/'.cache/triton-clover'))
+os.environ.setdefault('TORCHINDUCTOR_CACHE_DIR',str(ROOT/'.cache/inductor-clover'))
+
+
+def emit(out,event,**fields):
+    record={"event":event,"utc":time.strftime("%Y-%m-%dT%H:%M:%SZ",time.gmtime()),**fields}
+    with (out/"progress.jsonl").open("a") as f:f.write(json.dumps(record)+"\n")
+    write_json(out/"status.json",record)
+    print(json.dumps(record),flush=True)
+
+
+def softmax(x):
+    x=x-x.max(1,keepdims=True);p=np.exp(x);return p/p.sum(1,keepdims=True)
+
+
+def save_score(directory,rows,probabilities,metadata):
+    directory.mkdir(parents=True,exist_ok=True)
+    report={**metadata,**metrics(rows,probabilities)}
+    predictions=[{"id":r["id"],"label":r["label"],"difficulty":r["difficulty"],
+                  "predicted":LABELS[int(np.argmax(p))],"probabilities":dict(zip(LABELS,map(float,p)))}
+                 for r,p in zip(rows,probabilities)]
+    write_rows(directory/"predictions.jsonl",predictions)
+    report["predictions_sha256"]=sha(directory/"predictions.jsonl")
+    write_json(directory/"report.json",report)
+    return report
+
+
+@torch.no_grad()
+def evaluate(model,rows,batch_size):
+    model.model.eval();probabilities=[]
+    for start in range(0,len(rows),batch_size):
+        probabilities.extend(model.logits(rows[start:start+batch_size]).float().softmax(-1).cpu().tolist())
+    return np.array(probabilities)
+
+
+def protocol(args,manifest):
+    folder=args.out/"protocol";folder.mkdir(parents=True,exist_ok=True)
+    source_files=sorted((ROOT/'ariadne_bench').glob('*.py'))+[
+        ROOT/'scripts'/name for name in ['run_clover_experiments.py','build_clover_dataset.py',
+        'build_clover_native_batch.py','repair_clover_batch.py','benchmark_clover_generation.py',
+        'finish_clover_dataset.py','predict_clover.py']]
+    guidance=ROOT/'data/clover-intent-v1/generation-overrides.json'
+    if guidance.exists():source_files.append(guidance)
+    plan={"dataset":str(args.data.resolve()),"dataset_manifest_sha256":sha(args.data/"manifest.json"),
+          "splits":manifest["splits"],"models":MODELS,"seeds":args.seeds,
+          "methods":{"encoders":["description_similarity","class_centroids","knn","kmeans","linear_head","head_only","ariadne"],
+                     "native":["native_frozen","ariadne"]},
+          "stopping":{"minimum_epochs":10,"patience":3,"selection":"lowest validation NLL; no test-based choices"},
+          "optimization":{"batch":32,"microbatch":args.microbatch,"head_lr":1e-3,"interface_lr":1e-4,"weight_decay":0.0,"clip":1.0,
+                          "head_only_feature_cache":True,"feature_cache_validation":"live frozen features must match within 2e-5"},
+          "seed_summary":"mean and sample standard deviation; all individual runs retained",
+          "reported_metrics":["accuracy","macro_f1","nll","brier","confusion_matrix","difficulty_tiers",
+                              "actions_on_chat_question","actions_on_unclear"],
+          "passmarks":{"accuracy":.95,"false_actions_on_chat_question":0,"difficulty_tiers":list(DIFFICULTIES)},
+          "baseline_caveat":"Description similarity is an explicitly constructed zero-shot readout, not a native semantic head on MiniLM/ModernBERT.",
+          "label_quality":"synthetic, two-model agreement; no claim of human-labelled ground truth",
+          "output":"single class ID; Qwen uses one native letter token; JSON is artifact storage only",
+          "sources":{str(p.relative_to(ROOT)):sha(p) for p in source_files}}
+    path=folder/"plan.json"
+    if path.exists():
+        previous=read_json(path)
+        assert previous==plan,"Protocol changed after freeze; create a new experiment version"
+    else:
+        write_json(path,plan)
+        for source in source_files:
+            snapshot=folder/'sources'/source.relative_to(ROOT)
+            snapshot.parent.mkdir(parents=True,exist_ok=True)
+            shutil.copyfile(source,snapshot)
+        (folder/"PASSMARK-CLOVER-1.md").write_text(
+            "# Clover English intent experiment\n\nDeveloped by AKIKI.AI.\n\n"
+            "Six categories: chat, question, build, task, fix, unclear. Output is a single class ID.\n\n"
+            "Targets: at least 95% accuracy and zero observed build/task/fix classifications on pure chat/question. "
+            "Report easy, medium and hard separately. These are experimental measurements on model-reviewed "
+            "synthetic data, not a human-verified acceptance exam. No release or serving change is authorized.\n\n"
+            "All dataset hashes, seeds, methods and stopping rules are in plan.json. "
+            "Select hyperparameters and checkpoints on validation only. Holdout never enters training. "
+            "Frozen backbone and original native heads remain unchanged. No model downloads.\n")
+        # The workspace has no parent Git checkout. Keep the required pre-training
+        # protocol commit in a separate, self-contained experiment repository.
+        subprocess.run(["git","init","-q",str(folder)],check=True)
+        subprocess.run(["git","-C",str(folder),"add","plan.json","PASSMARK-CLOVER-1.md","sources"],check=True)
+        subprocess.run(["git","-C",str(folder),"-c","user.name=Codex","-c","user.email=codex@localhost", "commit","-qm","Freeze Clover English data hashes and comparison protocol before training"],check=True)
+    commit=subprocess.check_output(["git","-C",str(folder),"rev-parse","HEAD"],text=True).strip()
+    emit(args.out,"protocol_frozen",commit=commit,dataset=manifest["splits"])
+    return sha(path)
+
+
+def embedding_baselines(args,name,model,rows):
+    from sklearn.cluster import KMeans
+    from sklearn.linear_model import LogisticRegression
+    from sklearn.metrics import adjusted_rand_score
+    folder=args.out/name/"embedding-baselines"
+    if (folder/"complete.json").exists():return
+    folder.mkdir(parents=True,exist_ok=True)
+    vectors={}
+    for split,items in rows.items():
+        path=folder/(split+"-embeddings.npy")
+        if path.exists():vectors[split]=np.load(path)
+        else:
+            data=[];t=time.monotonic()
+            with torch.no_grad():
+                for start in range(0,len(items),args.eval_batch):
+                    data.append(model.model.features(**model.batch(items[start:start+args.eval_batch])).cpu().numpy())
+            vectors[split]=np.concatenate(data);np.save(path,vectors[split])
+            emit(args.out,"embeddings_cached",model=name,split=split,rows=len(items),seconds=round(time.monotonic()-t,1))
+        assert len(vectors[split])==len(items)
+    x=vectors["train"];v=vectors["validation"];z=vectors["test"]
+    y=np.array([LABELS.index(r["label"]) for r in rows["train"]])
+    vy=np.array([LABELS.index(r["label"]) for r in rows["validation"]])
+    ty=np.array([LABELS.index(r["label"]) for r in rows["test"]])
+    centroids=np.stack([x[y==i].mean(0) for i in range(6)])
+    centroids/=np.maximum(np.linalg.norm(centroids,axis=1,keepdims=True),1e-12)
+    (folder/'class_centroids').mkdir(parents=True,exist_ok=True)
+    np.save(folder/'class_centroids'/'centroids.npy',centroids)
+    np.save(folder/'train-labels.npy',y)
+    save_score(folder/"class_centroids",rows["test"],softmax(10*z@centroids.T),{"model":name,"method":"training-class-centroid cosine similarity","train_labels_used":True})
+    # Tune k and C on development data only; all labels are canonical indices.
+    def neighbor_prob(query,k):
+        result=[]
+        for start in range(0,len(query),128):
+            sim=query[start:start+128]@x.T
+            indices=np.argpartition(-sim,k-1,axis=1)[:,:k]
+            p=np.stack([np.bincount(y[idx],minlength=6) for idx in indices]).astype(float)
+            result.extend((p+.001)/(k+.006))
+        return np.array(result)
+    options=[]
+    for k in [1,3,5,11]:
+        p=neighbor_prob(v,k);options.append((metrics(rows["validation"],p)["macro_f1"],k))
+    _,k=max(options)
+    save_score(folder/"knn",rows["test"],neighbor_prob(z,k),{"model":name,"method":"cosine nearest labelled examples","k":k,"validation_candidates":options})
+    options=[];best=None
+    for c in [.1,1.,10.,100.]:
+        classifier=LogisticRegression(C=c,max_iter=2000,class_weight="balanced",random_state=0)
+        classifier.fit(x,y);p=classifier.predict_proba(v)
+        score=metrics(rows["validation"],p)["nll"]
+        options.append({"C":c,"validation_nll":score})
+        if best is None or score<best[0]:best=(score,c,classifier)
+    _,c,classifier=best
+    np.savez(folder/"linear-head.npz",weight=classifier.coef_,bias=classifier.intercept_)
+    save_score(folder/"linear_head",rows["test"],classifier.predict_proba(z),{"model":name,"method":"frozen embeddings + regularized linear classifier","C":c,"validation_candidates":options})
+    for seed in args.seeds:
+        candidates=[];best=None
+        for n in [6,12,24,48]:
+            km=KMeans(n_clusters=n,random_state=seed,n_init=10).fit(x)
+            # Fit clusters with no labels, then label each using TRAIN labels only.
+            count=np.stack([np.bincount(y[km.labels_==i],minlength=6) for i in range(n)]).astype(float)
+            pcluster=(count+.01)/(count.sum(1,keepdims=True)+.06)
+            p=pcluster[km.predict(v)];score=metrics(rows["validation"],p)["macro_f1"]
+            candidates.append({"clusters":n,"validation_macro_f1":score})
+            if best is None or score>best[0]:best=(score,n,km,pcluster)
+        _,n,km,pcluster=best
+        assigned=km.predict(z)
+        np.savez(folder/(f'kmeans-seed-{seed}.npz'),centers=km.cluster_centers_,probabilities=pcluster)
+        save_score(folder/("kmeans-seed-"+str(seed)),rows["test"],pcluster[assigned],{
+            "model":name,"method":"k-means on training embeddings; training-majority cluster labels",
+            "seed":seed,"clusters":n,"adjusted_rand_index":float(adjusted_rand_score(ty,assigned)),"validation_candidates":candidates})
+    write_json(folder/"complete.json",{"complete":True,"embedding_rows":{s:len(v) for s,v in vectors.items()}})
+    emit(args.out,"embedding_baselines_complete",model=name)
+
+
+def train(args,name,method,seed,rows,plan_sha):
+    folder=args.out/name/(method+"-seed-"+str(seed));folder.mkdir(parents=True,exist_ok=True)
+    if (folder/"complete.json").exists():return
+    torch.manual_seed(seed);random.seed(seed);np.random.seed(seed)
+    model=load(name,args.device,seed)
+    groups=model.configure(method)
+    optimizer=torch.optim.AdamW(groups,weight_decay=0.)
+    parameters=[p for p in model.model.parameters() if p.requires_grad]
+    feature_cache=None;cache_metadata={}
+    if method=='head_only' and isinstance(model,Encoder):
+        cache_path=args.out/name/'embedding-baselines/train-embeddings.npy'
+        values=np.load(cache_path)
+        assert values.shape==(len(rows['train']),model.model.head.in_features) and np.isfinite(values).all()
+        feature_cache=torch.from_numpy(values).to(model.device)
+        with torch.no_grad():
+            live=model.model.features(**model.batch(rows['train'][:5]))
+            torch.testing.assert_close(feature_cache[:5],live,rtol=2e-5,atol=2e-5)
+        cache_metadata={'frozen_feature_cache_sha256':sha(cache_path),
+                        'training_forward':'cached unchanged frozen features, live trainable head; validation uses full model'}
+    write_json(folder/"model.json",{**model.metadata,**cache_metadata,"trainable_parameters":sum(p.numel() for p in parameters)})
+    # Real gradient/frozen-parameter preflight, plus identity equivalence for encoders.
+    first=rows["train"][:args.microbatch]
+    initial=model.state()
+    native=model.logits(first).detach().clone()
+    loss=F.cross_entropy(model.logits(first),torch.tensor([LABELS.index(r["label"]) for r in first],device=model.device))
+    loss.backward()
+    assert all(p.grad is not None and torch.isfinite(p.grad).all() for p in parameters)
+    optimizer.step();model.verify_frozen();model.load(initial);optimizer.zero_grad(set_to_none=True)
+    assert torch.equal(native,model.logits(first).detach()),"identity restore failed"
+    optimizer=torch.optim.AdamW(groups,weight_decay=0.)
+    history=[];best=float("inf");stale=0;epoch=0
+    resume=folder/"resume.pt"
+    if resume.exists():
+        saved=torch.load(resume,map_location="cpu",weights_only=True)
+        assert saved["plan_sha256"]==plan_sha
+        model.load(saved["parameters"]);optimizer.load_state_dict(saved["optimizer"])
+        history=saved["history"];epoch=saved["epoch"];best=saved["best"];stale=saved["stale"]
+    emit(args.out,"training_started",model=name,method=method,seed=seed,trainable_parameters=sum(p.numel() for p in parameters),resumed_epoch=epoch)
+    while not (epoch>=10 and stale>=3):
+        epoch+=1;t=time.monotonic();model.model.eval()
+        order=list(range(len(rows["train"])));random.Random(seed*10000+epoch).shuffle(order)
+        train_loss=0.;train_correct=0;total=0;last_emit=time.monotonic()
+        for start in range(0,len(order),32):
+            indices=order[start:start+32];optimizer.zero_grad(set_to_none=True)
+            for micro in range(0,len(indices),args.microbatch):
+                selected=[rows["train"][i] for i in indices[micro:micro+args.microbatch]]
+                y=torch.tensor([LABELS.index(r["label"]) for r in selected],device=model.device)
+                if feature_cache is None:logits=model.logits(selected)
+                else:
+                    take=torch.tensor(indices[micro:micro+args.microbatch],device=model.device)
+                    logits=model.model.head(feature_cache.index_select(0,take))
+                loss=F.cross_entropy(logits,y)
+                (loss*len(selected)/len(indices)).backward()
+                train_loss+=loss.item()*len(selected);train_correct+=(logits.argmax(-1)==y).sum().item();total+=len(selected)
+            norm=torch.nn.utils.clip_grad_norm_(parameters,1.,error_if_nonfinite=True)
+            optimizer.step()
+            if time.monotonic()-last_emit>=45:
+                emit(args.out,"training_progress",model=name,method=method,seed=seed,epoch=epoch,examples=total,total_examples=len(order),train_loss=train_loss/total)
+                last_emit=time.monotonic()
+        model.verify_frozen()
+        p=evaluate(model,rows["validation"],args.eval_batch if isinstance(model,Encoder) else args.microbatch)
+        report=metrics(rows["validation"],p);improved=report["nll"]<best
+        if improved:
+            best=report["nll"];stale=0;torch.save(model.state(),folder/"best.pt")
+        else:stale+=1
+        row={"epoch":epoch,"train_loss":train_loss/total,"train_accuracy":train_correct/total,
+             "validation":report,"seconds":time.monotonic()-t,"improved":improved,"stale":stale}
+        history.append(row);write_json(folder/"history.json",history)
+        tmp=resume.with_suffix(".tmp")
+        torch.save({"plan_sha256":plan_sha,"parameters":model.state(),"optimizer":optimizer.state_dict(),
+                    "history":history,"epoch":epoch,"best":best,"stale":stale},tmp);tmp.replace(resume)
+        emit(args.out,"epoch_complete",model=name,method=method,seed=seed,epoch=epoch,validation_accuracy=report["accuracy"],validation_nll=report["nll"],seconds=round(row["seconds"],1),stale=stale)
+    model.load(torch.load(folder/"best.pt",map_location="cpu",weights_only=True));model.verify_frozen()
+    best_epoch=min(history,key=lambda r:r["validation"]["nll"])["epoch"]
+    p=evaluate(model,rows["test"],args.eval_batch if isinstance(model,Encoder) else args.microbatch)
+    report=save_score(folder/"holdout",rows["test"],p,{"model":name,"method":method,"seed":seed,"best_epoch":best_epoch,"epochs_completed":epoch,"frozen_weights_unchanged":True,"frozen_sha256":model.initial_hash})
+    write_json(folder/"complete.json",{"complete":True,"best_epoch":best_epoch,"holdout_accuracy":report["accuracy"],"frozen_verified":True})
+    emit(args.out,"training_complete",model=name,method=method,seed=seed,best_epoch=best_epoch,holdout_accuracy=report["accuracy"],false_actions=report["false_actions"])
+    if hasattr(model,"adapter"):model.adapter.close()
+    del model,optimizer;gc.collect();torch.cuda.empty_cache()
+
+
+def refresh_report(out):
+    reports=[]
+    for path in out.rglob("report.json"):
+        if path.is_relative_to(out/"protocol"):continue
+        report=read_json(path)
+        if "accuracy" in report:reports.append({"artifact":str(path.relative_to(out)),**report})
+    write_json(out/"results.json",reports)
+    grouped={}
+    for report in reports:
+        key=(report.get('model'),report.get('method'))
+        grouped.setdefault(key,[]).append(report)
+    summaries=[]
+    for (name,method),group in sorted(grouped.items()):
+        summary={'model':name,'method':method,'runs':len(group),'seeds':[r.get('seed') for r in group]}
+        for field in ('accuracy','macro_f1','nll','brier'):
+            values=np.array([r[field] for r in group],dtype=float)
+            summary[field]={'mean':float(values.mean()),'sample_sd':float(values.std(ddof=1)) if len(values)>1 else None}
+        summary['by_difficulty']={}
+        for level in DIFFICULTIES:
+            values=np.array([r['by_difficulty'][level]['accuracy'] for r in group],dtype=float)
+            summary['by_difficulty'][level]={'mean':float(values.mean()),'sample_sd':float(values.std(ddof=1)) if len(values)>1 else None}
+        summary['false_actions']=[r['false_actions'] for r in group]
+        summary['actions_on_unclear']=[r['actions_on_unclear'] for r in group]
+        summaries.append(summary)
+    write_json(out/'seed-summary.json',summaries)
+    lines=["# Clover English intent comparison","","Developed by AKIKI.AI. Synthetic, model-reviewed labels; human review pending.","",
+           "All original backbone weights remain frozen. Validation alone chooses training settings and checkpoints.","",
+           "Results across seeds (mean ± sample standard deviation where repeated):","",
+           "| Model / method | Runs | Accuracy | Macro F1 | Hard accuracy |",
+           "|---|---:|---:|---:|---:|"]
+    def percent(cell):
+        return f"{100*cell['mean']:.1f}%"+(f" ± {100*cell['sample_sd']:.1f}" if cell['sample_sd'] is not None else '')
+    for summary in summaries:
+        f1=summary['macro_f1']
+        f1_text=f"{f1['mean']:.3f}"+(f" ± {f1['sample_sd']:.3f}" if f1['sample_sd'] is not None else '')
+        lines.append(f"| {summary['model']} / {summary['method']} | {summary['runs']} | {percent(summary['accuracy'])} | {f1_text} | {percent(summary['by_difficulty']['hard'])} |")
+    lines.extend(["","Individual runs:","",
+           "| Model / method | Accuracy | Macro F1 | Easy | Medium | Hard | Actions on chat/question | Actions on unclear |",
+           "|---|---:|---:|---:|---:|---:|---:|---:|"])
+    for r in sorted(reports,key=lambda r:r["artifact"]):
+        levels=r["by_difficulty"]
+        cells=[f"{100*levels[d]['accuracy']:.1f}%" if d in levels else "—" for d in DIFFICULTIES]
+        lines.append(f"| {r['artifact'].replace('/report.json','')} | {100*r['accuracy']:.1f}% | {r['macro_f1']:.3f} | {' | '.join(cells)} | {r['false_actions']}/{r['passive_cases']} | {r['actions_on_unclear']}/{r['unclear_cases']} |")
+    (out/"results.md").write_text("\n".join(lines)+"\n")
+
+
+def main():
+    p=argparse.ArgumentParser(description=__doc__)
+    p.add_argument("--data",type=Path,default=ROOT/"data/clover-intent-v1")
+    p.add_argument("--out",type=Path,default=ROOT/"runs/clover-intent-v1")
+    p.add_argument("--models",nargs="+",default=list(MODELS),choices=list(MODELS))
+    p.add_argument("--seeds",nargs="+",type=int,default=[0,1,2])
+    p.add_argument("--device",default="cuda:1")
+    p.add_argument("--microbatch",type=int,default=32)
+    p.add_argument("--eval-batch",type=int,default=64)
+    args=p.parse_args();args.out.mkdir(parents=True,exist_ok=True)
+    torch.set_num_threads(8);torch.set_float32_matmul_precision("highest")
+    torch.backends.cuda.matmul.allow_tf32=False;torch.backends.cudnn.allow_tf32=False
+    from ariadne_bench.reproducibility import configure_determinism
+    configure_determinism(0)
+    with (args.out/"experiment.lock").open("a") as lock:
+        fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
+        manifest=read_json(args.data/"manifest.json");assert manifest["complete"] and read_json(args.data/"audit.json")["passed"]
+        assert sha(args.data/'audit.json')==manifest['audit_sha256']
+        rows={}
+        for split,spec in manifest["splits"].items():
+            assert sha(args.data/spec["file"])==spec["sha256"]
+            rows[split]=read_rows(args.data/spec["file"])
+        plan_sha=protocol(args,manifest)
+        # Complete all requested pre-training baselines before any specialization.
+        for name in args.models:
+            folder=args.out/name/"frozen-baseline"
+            if (folder/"report.json").exists():continue
+            emit(args.out,"baseline_started",model=name)
+            model=load(name,args.device,0);write_json(args.out/name/"model.json",model.metadata)
+            model.verify_identity(rows['validation'][:3]);model.verify_frozen()
+            probabilities=evaluate(model,rows["test"],args.eval_batch if isinstance(model,Encoder) else args.microbatch)
+            report=save_score(folder,rows["test"],probabilities,{"model":name,"method":"description_similarity" if isinstance(model,Encoder) else "native_frozen","training_examples_used":0,"identity_matches_native_logits":True})
+            emit(args.out,"baseline_complete",model=name,accuracy=report["accuracy"],by_difficulty={d:r["accuracy"] for d,r in report["by_difficulty"].items()})
+            if hasattr(model,"adapter"):model.adapter.close()
+            del model;gc.collect();torch.cuda.empty_cache();refresh_report(args.out)
+        for name in args.models:
+            if MODELS[name]["kind"]=="encoder":
+                model=load(name,args.device,0);embedding_baselines(args,name,model,rows)
+                del model;gc.collect();torch.cuda.empty_cache();refresh_report(args.out)
+        for name in args.models:
+            methods=["head_only","ariadne"] if MODELS[name]["kind"]=="encoder" else ["ariadne"]
+            for method in methods:
+                for seed in args.seeds:
+                    train(args,name,method,seed,rows,plan_sha);refresh_report(args.out)
+        refresh_report(args.out);emit(args.out,"experiment_complete",report=str(args.out/"results.md"))
+
+
+if __name__=="__main__":main()

requirements.txt

@@ -0,0 +1,4 @@
+torch==2.13.0
+transformers==5.17.0
+safetensors==0.8.0
+tokenizers==0.23.2

results/baselines/embedding-class_centroids.json

@@ -0,0 +1,419 @@
+{
+  "model": "minilm-l6-v2",
+  "method": "training-class-centroid cosine similarity",
+  "train_labels_used": true,
+  "cases": 714,
+  "accuracy": 0.8053221288515406,
+  "macro_f1": 0.8039270413110179,
+  "per_class": {
+    "chat": {
+      "precision": 0.8791208791208791,
+      "recall": 0.6666666666666666,
+      "f1": 0.7582938388625592,
+      "support": 120
+    },
+    "question": {
+      "precision": 0.8833333333333333,
+      "recall": 0.8833333333333333,
+      "f1": 0.8833333333333333,
+      "support": 120
+    },
+    "build": {
+      "precision": 0.8515625,
+      "recall": 0.9083333333333333,
+      "f1": 0.8790322580645161,
+      "support": 120
+    },
+    "task": {
+      "precision": 0.8469387755102041,
+      "recall": 0.6916666666666667,
+      "f1": 0.7614678899082569,
+      "support": 120
+    },
+    "fix": {
+      "precision": 0.6792452830188679,
+      "recall": 0.9,
+      "f1": 0.7741935483870968,
+      "support": 120
+    },
+    "unclear": {
+      "precision": 0.7542372881355932,
+      "recall": 0.7807017543859649,
+      "f1": 0.7672413793103449,
+      "support": 114
+    }
+  },
+  "confusion": {
+    "chat": {
+      "chat": 80,
+      "question": 7,
+      "build": 15,
+      "task": 2,
+      "fix": 7,
+      "unclear": 9,
+      "INVALID": 0
+    },
+    "question": {
+      "chat": 2,
+      "question": 106,
+      "build": 0,
+      "task": 0,
+      "fix": 5,
+      "unclear": 7,
+      "INVALID": 0
+    },
+    "build": {
+      "chat": 5,
+      "question": 0,
+      "build": 109,
+      "task": 3,
+      "fix": 0,
+      "unclear": 3,
+      "INVALID": 0
+    },
+    "task": {
+      "chat": 0,
+      "question": 1,
+      "build": 4,
+      "task": 83,
+      "fix": 27,
+      "unclear": 5,
+      "INVALID": 0
+    },
+    "fix": {
+      "chat": 0,
+      "question": 5,
+      "build": 0,
+      "task": 2,
+      "fix": 108,
+      "unclear": 5,
+      "INVALID": 0
+    },
+    "unclear": {
+      "chat": 4,
+      "question": 1,
+      "build": 0,
+      "task": 8,
+      "fix": 12,
+      "unclear": 89,
+      "INVALID": 0
+    }
+  },
+  "invalid": 0,
+  "nll": 0.9776460491478228,
+  "brier": 0.4748579251724496,
+  "false_actions": 29,
+  "passive_cases": 240,
+  "false_action_rate": 0.12083333333333333,
+  "unclear_cases": 114,
+  "actions_on_unclear": 20,
+  "action_rate_on_unclear": 0.17543859649122806,
+  "by_difficulty": {
+    "easy": {
+      "cases": 235,
+      "accuracy": 0.8127659574468085,
+      "macro_f1": 0.8068940909578335,
+      "per_class": {
+        "chat": {
+          "precision": 0.9545454545454546,
+          "recall": 0.525,
+          "f1": 0.6774193548387096,
+          "support": 40
+        },
+        "question": {
+          "precision": 0.8571428571428571,
+          "recall": 0.9,
+          "f1": 0.8780487804878049,
+          "support": 40
+        },
+        "build": {
+          "precision": 0.7647058823529411,
+          "recall": 0.975,
+          "f1": 0.8571428571428571,
+          "support": 40
+        },
+        "task": {
+          "precision": 0.8333333333333334,
+          "recall": 0.75,
+          "f1": 0.7894736842105263,
+          "support": 40
+        },
+        "fix": {
+          "precision": 0.7037037037037037,
+          "recall": 0.95,
+          "f1": 0.8085106382978723,
+          "support": 40
+        },
+        "unclear": {
+          "precision": 0.9,
+          "recall": 0.7714285714285715,
+          "f1": 0.8307692307692308,
+          "support": 35
+        }
+      },
+      "confusion": {
+        "chat": {
+          "chat": 21,
+          "question": 4,
+          "build": 9,
+          "task": 1,
+          "fix": 3,
+          "unclear": 2,
+          "INVALID": 0
+        },
+        "question": {
+          "chat": 1,
+          "question": 36,
+          "build": 0,
+          "task": 0,
+          "fix": 3,
+          "unclear": 0,
+          "INVALID": 0
+        },
+        "build": {
+          "chat": 0,
+          "question": 0,
+          "build": 39,
+          "task": 1,
+          "fix": 0,
+          "unclear": 0,
+          "INVALID": 0
+        },
+        "task": {
+          "chat": 0,
+          "question": 0,
+          "build": 3,
+          "task": 30,
+          "fix": 6,
+          "unclear": 1,
+          "INVALID": 0
+        },
+        "fix": {
+          "chat": 0,
+          "question": 1,
+          "build": 0,
+          "task": 1,
+          "fix": 38,
+          "unclear": 0,
+          "INVALID": 0
+        },
+        "unclear": {
+          "chat": 0,
+          "question": 1,
+          "build": 0,
+          "task": 3,
+          "fix": 4,
+          "unclear": 27,
+          "INVALID": 0
+        }
+      },
+      "invalid": 0
+    },
+    "medium": {
+      "cases": 239,
+      "accuracy": 0.7907949790794979,
+      "macro_f1": 0.7804472677829759,
+      "per_class": {
+        "chat": {
+          "precision": 1.0,
+          "recall": 0.525,
+          "f1": 0.6885245901639344,
+          "support": 40
+        },
+        "question": {
+          "precision": 0.8837209302325582,
+          "recall": 0.95,
+          "f1": 0.9156626506024096,
+          "support": 40
+        },
+        "build": {
+          "precision": 0.8809523809523809,
+          "recall": 0.925,
+          "f1": 0.9024390243902439,
+          "support": 40
+        },
+        "task": {
+          "precision": 0.7916666666666666,
+          "recall": 0.475,
+          "f1": 0.59375,
+          "support": 40
+        },
+        "fix": {
+          "precision": 0.6129032258064516,
+          "recall": 0.95,
+          "f1": 0.7450980392156863,
+          "support": 40
+        },
+        "unclear": {
+          "precision": 0.7659574468085106,
+          "recall": 0.9230769230769231,
+          "f1": 0.8372093023255814,
+          "support": 39
+        }
+      },
+      "confusion": {
+        "chat": {
+          "chat": 21,
+          "question": 3,
+          "build": 5,
+          "task": 1,
+          "fix": 3,
+          "unclear": 7,
+          "INVALID": 0
+        },
+        "question": {
+          "chat": 0,
+          "question": 38,
+          "build": 0,
+          "task": 0,
+          "fix": 0,
+          "unclear": 2,
+          "INVALID": 0
+        },
+        "build": {
+          "chat": 0,
+          "question": 0,
+          "build": 37,
+          "task": 2,
+          "fix": 0,
+          "unclear": 1,
+          "INVALID": 0
+        },
+        "task": {
+          "chat": 0,
+          "question": 1,
+          "build": 0,
+          "task": 19,
+          "fix": 20,
+          "unclear": 0,
+          "INVALID": 0
+        },
+        "fix": {
+          "chat": 0,
+          "question": 1,
+          "build": 0,
+          "task": 0,
+          "fix": 38,
+          "unclear": 1,
+          "INVALID": 0
+        },
+        "unclear": {
+          "chat": 0,
+          "question": 0,
+          "build": 0,
+          "task": 2,
+          "fix": 1,
+          "unclear": 36,
+          "INVALID": 0
+        }
+      },
+      "invalid": 0
+    },
+    "hard": {
+      "cases": 240,
+      "accuracy": 0.8125,
+      "macro_f1": 0.8136373691259903,
+      "per_class": {
+        "chat": {
+          "precision": 0.7916666666666666,
+          "recall": 0.95,
+          "f1": 0.8636363636363636,
+          "support": 40
+        },
+        "question": {
+          "precision": 0.9142857142857143,
+          "recall": 0.8,
+          "f1": 0.8533333333333334,
+          "support": 40
+        },
+        "build": {
+          "precision": 0.9428571428571428,
+          "recall": 0.825,
+          "f1": 0.88,
+          "support": 40
+        },
+        "task": {
+          "precision": 0.8947368421052632,
+          "recall": 0.85,
+          "f1": 0.8717948717948718,
+          "support": 40
+        },
+        "fix": {
+          "precision": 0.7441860465116279,
+          "recall": 0.8,
+          "f1": 0.7710843373493976,
+          "support": 40
+        },
+        "unclear": {
+          "precision": 0.6341463414634146,
+          "recall": 0.65,
+          "f1": 0.6419753086419753,
+          "support": 40
+        }
+      },
+      "confusion": {
+        "chat": {
+          "chat": 38,
+          "question": 0,
+          "build": 1,
+          "task": 0,
+          "fix": 1,
+          "unclear": 0,
+          "INVALID": 0
+        },
+        "question": {
+          "chat": 1,
+          "question": 32,
+          "build": 0,
+          "task": 0,
+          "fix": 2,
+          "unclear": 5,
+          "INVALID": 0
+        },
+        "build": {
+          "chat": 5,
+          "question": 0,
+          "build": 33,
+          "task": 0,
+          "fix": 0,
+          "unclear": 2,
+          "INVALID": 0
+        },
+        "task": {
+          "chat": 0,
+          "question": 0,
+          "build": 1,
+          "task": 34,
+          "fix": 1,
+          "unclear": 4,
+          "INVALID": 0
+        },
+        "fix": {
+          "chat": 0,
+          "question": 3,
+          "build": 0,
+          "task": 1,
+          "fix": 32,
+          "unclear": 4,
+          "INVALID": 0
+        },
+        "unclear": {
+          "chat": 4,
+          "question": 0,
+          "build": 0,
+          "task": 3,
+          "fix": 7,
+          "unclear": 26,
+          "INVALID": 0
+        }
+      },
+      "invalid": 0
+    }
+  },
+  "difficulty_counts": {
+    "medium": 239,
+    "hard": 240,
+    "easy": 235
+  },
+  "predictions_sha256": "8dc7acfec189471f677a86bf5e19e35655ba2f1e1bf77afe9678a65e1d0dbf61"
+}

results/baselines/embedding-kmeans-seed-0.json

@@ -0,0 +1,439 @@
+{
+  "model": "minilm-l6-v2",
+  "method": "k-means on training embeddings; training-majority cluster labels",
+  "seed": 0,
+  "clusters": 48,
+  "adjusted_rand_index": 0.07325460529021985,
+  "validation_candidates": [
+    {
+      "clusters": 6,
+      "validation_macro_f1": 0.4448239498166639
+    },
+    {
+      "clusters": 12,
+      "validation_macro_f1": 0.5387143732224868
+    },
+    {
+      "clusters": 24,
+      "validation_macro_f1": 0.4966756931300707
+    },
+    {
+      "clusters": 48,
+      "validation_macro_f1": 0.5649193348918948
+    }
+  ],
+  "cases": 714,
+  "accuracy": 0.5168067226890757,
+  "macro_f1": 0.5227685117233037,
+  "per_class": {
+    "chat": {
+      "precision": 0.5595238095238095,
+      "recall": 0.39166666666666666,
+      "f1": 0.46078431372549017,
+      "support": 120
+    },
+    "question": {
+      "precision": 0.45255474452554745,
+      "recall": 0.5166666666666667,
+      "f1": 0.48249027237354086,
+      "support": 120
+    },
+    "build": {
+      "precision": 0.48148148148148145,
+      "recall": 0.65,
+      "f1": 0.5531914893617021,
+      "support": 120
+    },
+    "task": {
+      "precision": 0.4406779661016949,
+      "recall": 0.43333333333333335,
+      "f1": 0.4369747899159664,
+      "support": 120
+    },
+    "fix": {
+      "precision": 0.42857142857142855,
+      "recall": 0.475,
+      "f1": 0.4505928853754941,
+      "support": 120
+    },
+    "unclear": {
+      "precision": 0.9125,
+      "recall": 0.6403508771929824,
+      "f1": 0.7525773195876289,
+      "support": 114
+    }
+  },
+  "confusion": {
+    "chat": {
+      "chat": 47,
+      "question": 14,
+      "build": 34,
+      "task": 9,
+      "fix": 14,
+      "unclear": 2,
+      "INVALID": 0
+    },
+    "question": {
+      "chat": 10,
+      "question": 62,
+      "build": 13,
+      "task": 11,
+      "fix": 24,
+      "unclear": 0,
+      "INVALID": 0
+    },
+    "build": {
+      "chat": 12,
+      "question": 11,
+      "build": 78,
+      "task": 10,
+      "fix": 9,
+      "unclear": 0,
+      "INVALID": 0
+    },
+    "task": {
+      "chat": 6,
+      "question": 23,
+      "build": 16,
+      "task": 52,
+      "fix": 19,
+      "unclear": 4,
+      "INVALID": 0
+    },
+    "fix": {
+      "chat": 5,
+      "question": 24,
+      "build": 13,
+      "task": 20,
+      "fix": 57,
+      "unclear": 1,
+      "INVALID": 0
+    },
+    "unclear": {
+      "chat": 4,
+      "question": 3,
+      "build": 8,
+      "task": 16,
+      "fix": 10,
+      "unclear": 73,
+      "INVALID": 0
+    }
+  },
+  "invalid": 0,
+  "nll": 1.2672067322258014,
+  "brier": 0.5938426790677706,
+  "false_actions": 105,
+  "passive_cases": 240,
+  "false_action_rate": 0.4375,
+  "unclear_cases": 114,
+  "actions_on_unclear": 34,
+  "action_rate_on_unclear": 0.2982456140350877,
+  "by_difficulty": {
+    "easy": {
+      "cases": 235,
+      "accuracy": 0.4851063829787234,
+      "macro_f1": 0.4885814014063359,
+      "per_class": {
+        "chat": {
+          "precision": 0.631578947368421,
+          "recall": 0.3,
+          "f1": 0.4067796610169492,
+          "support": 40
+        },
+        "question": {
+          "precision": 0.39655172413793105,
+          "recall": 0.575,
+          "f1": 0.46938775510204084,
+          "support": 40
+        },
+        "build": {
+          "precision": 0.42105263157894735,
+          "recall": 0.6,
+          "f1": 0.4948453608247423,
+          "support": 40
+        },
+        "task": {
+          "precision": 0.4642857142857143,
+          "recall": 0.325,
+          "f1": 0.38235294117647056,
+          "support": 40
+        },
+        "fix": {
+          "precision": 0.38095238095238093,
+          "recall": 0.4,
+          "f1": 0.3902439024390244,
+          "support": 40
+        },
+        "unclear": {
+          "precision": 0.8387096774193549,
+          "recall": 0.7428571428571429,
+          "f1": 0.7878787878787878,
+          "support": 35
+        }
+      },
+      "confusion": {
+        "chat": {
+          "chat": 12,
+          "question": 7,
+          "build": 14,
+          "task": 1,
+          "fix": 5,
+          "unclear": 1,
+          "INVALID": 0
+        },
+        "question": {
+          "chat": 2,
+          "question": 23,
+          "build": 6,
+          "task": 1,
+          "fix": 8,
+          "unclear": 0,
+          "INVALID": 0
+        },
+        "build": {
+          "chat": 2,
+          "question": 5,
+          "build": 24,
+          "task": 6,
+          "fix": 3,
+          "unclear": 0,
+          "INVALID": 0
+        },
+        "task": {
+          "chat": 1,
+          "question": 10,
+          "build": 7,
+          "task": 13,
+          "fix": 6,
+          "unclear": 3,
+          "INVALID": 0
+        },
+        "fix": {
+          "chat": 2,
+          "question": 12,
+          "build": 5,
+          "task": 4,
+          "fix": 16,
+          "unclear": 1,
+          "INVALID": 0
+        },
+        "unclear": {
+          "chat": 0,
+          "question": 1,
+          "build": 1,
+          "task": 3,
+          "fix": 4,
+          "unclear": 26,
+          "INVALID": 0
+        }
+      },
+      "invalid": 0
+    },
+    "medium": {
+      "cases": 239,
+      "accuracy": 0.4895397489539749,
+      "macro_f1": 0.49673073454605987,
+      "per_class": {
+        "chat": {
+          "precision": 0.42857142857142855,
+          "recall": 0.375,
+          "f1": 0.4,
+          "support": 40
+        },
+        "question": {
+          "precision": 0.41304347826086957,
+          "recall": 0.475,
+          "f1": 0.4418604651162791,
+          "support": 40
+        },
+        "build": {
+          "precision": 0.5897435897435898,
+          "recall": 0.575,
+          "f1": 0.5822784810126582,
+          "support": 40
+        },
+        "task": {
+          "precision": 0.2682926829268293,
+          "recall": 0.275,
+          "f1": 0.2716049382716049,
+          "support": 40
+        },
+        "fix": {
+          "precision": 0.4,
+          "recall": 0.45,
+          "f1": 0.4235294117647059,
+          "support": 40
+        },
+        "unclear": {
+          "precision": 0.9393939393939394,
+          "recall": 0.7948717948717948,
+          "f1": 0.8611111111111112,
+          "support": 39
+        }
+      },
+      "confusion": {
+        "chat": {
+          "chat": 15,
+          "question": 6,
+          "build": 7,
+          "task": 6,
+          "fix": 5,
+          "unclear": 1,
+          "INVALID": 0
+        },
+        "question": {
+          "chat": 4,
+          "question": 19,
+          "build": 2,
+          "task": 7,
+          "fix": 8,
+          "unclear": 0,
+          "INVALID": 0
+        },
+        "build": {
+          "chat": 8,
+          "question": 4,
+          "build": 23,
+          "task": 1,
+          "fix": 4,
+          "unclear": 0,
+          "INVALID": 0
+        },
+        "task": {
+          "chat": 4,
+          "question": 11,
+          "build": 3,
+          "task": 11,
+          "fix": 10,
+          "unclear": 1,
+          "INVALID": 0
+        },
+        "fix": {
+          "chat": 2,
+          "question": 6,
+          "build": 4,
+          "task": 10,
+          "fix": 18,
+          "unclear": 0,
+          "INVALID": 0
+        },
+        "unclear": {
+          "chat": 2,
+          "question": 0,
+          "build": 0,
+          "task": 6,
+          "fix": 0,
+          "unclear": 31,
+          "INVALID": 0
+        }
+      },
+      "invalid": 0
+    },
+    "hard": {
+      "cases": 240,
+      "accuracy": 0.575,
+      "macro_f1": 0.5733008687983755,
+      "per_class": {
+        "chat": {
+          "precision": 0.6666666666666666,
+          "recall": 0.5,
+          "f1": 0.5714285714285714,
+          "support": 40
+        },
+        "question": {
+          "precision": 0.6060606060606061,
+          "recall": 0.5,
+          "f1": 0.547945205479452,
+          "support": 40
+        },
+        "build": {
+          "precision": 0.4696969696969697,
+          "recall": 0.775,
+          "f1": 0.5849056603773585,
+          "support": 40
+        },
+        "task": {
+          "precision": 0.5714285714285714,
+          "recall": 0.7,
+          "f1": 0.6292134831460674,
+          "support": 40
+        },
+        "fix": {
+          "precision": 0.5,
+          "recall": 0.575,
+          "f1": 0.5348837209302325,
+          "support": 40
+        },
+        "unclear": {
+          "precision": 1.0,
+          "recall": 0.4,
+          "f1": 0.5714285714285714,
+          "support": 40
+        }
+      },
+      "confusion": {
+        "chat": {
+          "chat": 20,
+          "question": 1,
+          "build": 13,
+          "task": 2,
+          "fix": 4,
+          "unclear": 0,
+          "INVALID": 0
+        },
+        "question": {
+          "chat": 4,
+          "question": 20,
+          "build": 5,
+          "task": 3,
+          "fix": 8,
+          "unclear": 0,
+          "INVALID": 0
+        },
+        "build": {
+          "chat": 2,
+          "question": 2,
+          "build": 31,
+          "task": 3,
+          "fix": 2,
+          "unclear": 0,
+          "INVALID": 0
+        },
+        "task": {
+          "chat": 1,
+          "question": 2,
+          "build": 6,
+          "task": 28,
+          "fix": 3,
+          "unclear": 0,
+          "INVALID": 0
+        },
+        "fix": {
+          "chat": 1,
+          "question": 6,
+          "build": 4,
+          "task": 6,
+          "fix": 23,
+          "unclear": 0,
+          "INVALID": 0
+        },
+        "unclear": {
+          "chat": 2,
+          "question": 2,
+          "build": 7,
+          "task": 7,
+          "fix": 6,
+          "unclear": 16,
+          "INVALID": 0
+        }
+      },
+      "invalid": 0
+    }
+  },
+  "difficulty_counts": {
+    "medium": 239,
+    "hard": 240,
+    "easy": 235
+  },
+  "predictions_sha256": "daae1546df7b9abb147ca9b14879d08dba597b76853ed3abdd1ca7571917f32a"
+}

results/baselines/embedding-kmeans-seed-1.json

@@ -0,0 +1,439 @@
+{
+  "model": "minilm-l6-v2",
+  "method": "k-means on training embeddings; training-majority cluster labels",
+  "seed": 1,
+  "clusters": 48,
+  "adjusted_rand_index": 0.09704067858483406,
+  "validation_candidates": [
+    {
+      "clusters": 6,
+      "validation_macro_f1": 0.44619204513246435
+    },
+    {
+      "clusters": 12,
+      "validation_macro_f1": 0.5151510674513514
+    },
+    {
+      "clusters": 24,
+      "validation_macro_f1": 0.49806823313168463
+    },
+    {
+      "clusters": 48,
+      "validation_macro_f1": 0.5281206697704389
+    }
+  ],
+  "cases": 714,
+  "accuracy": 0.5084033613445378,
+  "macro_f1": 0.512370530501952,
+  "per_class": {
+    "chat": {
+      "precision": 0.46601941747572817,
+      "recall": 0.4,
+      "f1": 0.4304932735426009,
+      "support": 120
+    },
+    "question": {
+      "precision": 0.44696969696969696,
+      "recall": 0.49166666666666664,
+      "f1": 0.46825396825396826,
+      "support": 120
+    },
+    "build": {
+      "precision": 0.5851851851851851,
+      "recall": 0.6583333333333333,
+      "f1": 0.6196078431372549,
+      "support": 120
+    },
+    "task": {
+      "precision": 0.4017857142857143,
+      "recall": 0.375,
+      "f1": 0.3879310344827586,
+      "support": 120
+    },
+    "fix": {
+      "precision": 0.4217687074829932,
+      "recall": 0.5166666666666667,
+      "f1": 0.46441947565543074,
+      "support": 120
+    },
+    "unclear": {
+      "precision": 0.8235294117647058,
+      "recall": 0.6140350877192983,
+      "f1": 0.7035175879396985,
+      "support": 114
+    }
+  },
+  "confusion": {
+    "chat": {
+      "chat": 48,
+      "question": 16,
+      "build": 25,
+      "task": 8,
+      "fix": 20,
+      "unclear": 3,
+      "INVALID": 0
+    },
+    "question": {
+      "chat": 15,
+      "question": 59,
+      "build": 7,
+      "task": 18,
+      "fix": 20,
+      "unclear": 1,
+      "INVALID": 0
+    },
+    "build": {
+      "chat": 13,
+      "question": 12,
+      "build": 79,
+      "task": 7,
+      "fix": 9,
+      "unclear": 0,
+      "INVALID": 0
+    },
+    "task": {
+      "chat": 8,
+      "question": 23,
+      "build": 13,
+      "task": 45,
+      "fix": 28,
+      "unclear": 3,
+      "INVALID": 0
+    },
+    "fix": {
+      "chat": 13,
+      "question": 18,
+      "build": 6,
+      "task": 13,
+      "fix": 62,
+      "unclear": 8,
+      "INVALID": 0
+    },
+    "unclear": {
+      "chat": 6,
+      "question": 4,
+      "build": 5,
+      "task": 21,
+      "fix": 8,
+      "unclear": 70,
+      "INVALID": 0
+    }
+  },
+  "invalid": 0,
+  "nll": 1.2304312893348386,
+  "brier": 0.5813902944293567,
+  "false_actions": 98,
+  "passive_cases": 240,
+  "false_action_rate": 0.4083333333333333,
+  "unclear_cases": 114,
+  "actions_on_unclear": 34,
+  "action_rate_on_unclear": 0.2982456140350877,
+  "by_difficulty": {
+    "easy": {
+      "cases": 235,
+      "accuracy": 0.49361702127659574,
+      "macro_f1": 0.49739800245611443,
+      "per_class": {
+        "chat": {
+          "precision": 0.5384615384615384,
+          "recall": 0.35,
+          "f1": 0.42424242424242425,
+          "support": 40
+        },
+        "question": {
+          "precision": 0.3620689655172414,
+          "recall": 0.525,
+          "f1": 0.42857142857142855,
+          "support": 40
+        },
+        "build": {
+          "precision": 0.5333333333333333,
+          "recall": 0.6,
+          "f1": 0.5647058823529412,
+          "support": 40
+        },
+        "task": {
+          "precision": 0.4166666666666667,
+          "recall": 0.25,
+          "f1": 0.3125,
+          "support": 40
+        },
+        "fix": {
+          "precision": 0.41509433962264153,
+          "recall": 0.55,
+          "f1": 0.4731182795698925,
+          "support": 40
+        },
+        "unclear": {
+          "precision": 0.8620689655172413,
+          "recall": 0.7142857142857143,
+          "f1": 0.78125,
+          "support": 35
+        }
+      },
+      "confusion": {
+        "chat": {
+          "chat": 14,
+          "question": 8,
+          "build": 9,
+          "task": 3,
+          "fix": 6,
+          "unclear": 0,
+          "INVALID": 0
+        },
+        "question": {
+          "chat": 4,
+          "question": 21,
+          "build": 3,
+          "task": 4,
+          "fix": 7,
+          "unclear": 1,
+          "INVALID": 0
+        },
+        "build": {
+          "chat": 4,
+          "question": 5,
+          "build": 24,
+          "task": 3,
+          "fix": 4,
+          "unclear": 0,
+          "INVALID": 0
+        },
+        "task": {
+          "chat": 1,
+          "question": 12,
+          "build": 4,
+          "task": 10,
+          "fix": 12,
+          "unclear": 1,
+          "INVALID": 0
+        },
+        "fix": {
+          "chat": 3,
+          "question": 9,
+          "build": 3,
+          "task": 1,
+          "fix": 22,
+          "unclear": 2,
+          "INVALID": 0
+        },
+        "unclear": {
+          "chat": 0,
+          "question": 3,
+          "build": 2,
+          "task": 3,
+          "fix": 2,
+          "unclear": 25,
+          "INVALID": 0
+        }
+      },
+      "invalid": 0
+    },
+    "medium": {
+      "cases": 239,
+      "accuracy": 0.48535564853556484,
+      "macro_f1": 0.48565144203591504,
+      "per_class": {
+        "chat": {
+          "precision": 0.325,
+          "recall": 0.325,
+          "f1": 0.325,
+          "support": 40
+        },
+        "question": {
+          "precision": 0.4186046511627907,
+          "recall": 0.45,
+          "f1": 0.43373493975903615,
+          "support": 40
+        },
+        "build": {
+          "precision": 0.6388888888888888,
+          "recall": 0.575,
+          "f1": 0.6052631578947368,
+          "support": 40
+        },
+        "task": {
+          "precision": 0.2857142857142857,
+          "recall": 0.25,
+          "f1": 0.26666666666666666,
+          "support": 40
+        },
+        "fix": {
+          "precision": 0.45652173913043476,
+          "recall": 0.525,
+          "f1": 0.4883720930232558,
+          "support": 40
+        },
+        "unclear": {
+          "precision": 0.7948717948717948,
+          "recall": 0.7948717948717948,
+          "f1": 0.7948717948717948,
+          "support": 39
+        }
+      },
+      "confusion": {
+        "chat": {
+          "chat": 13,
+          "question": 7,
+          "build": 6,
+          "task": 5,
+          "fix": 6,
+          "unclear": 3,
+          "INVALID": 0
+        },
+        "question": {
+          "chat": 6,
+          "question": 18,
+          "build": 2,
+          "task": 8,
+          "fix": 6,
+          "unclear": 0,
+          "INVALID": 0
+        },
+        "build": {
+          "chat": 7,
+          "question": 5,
+          "build": 23,
+          "task": 3,
+          "fix": 2,
+          "unclear": 0,
+          "INVALID": 0
+        },
+        "task": {
+          "chat": 6,
+          "question": 9,
+          "build": 2,
+          "task": 10,
+          "fix": 11,
+          "unclear": 2,
+          "INVALID": 0
+        },
+        "fix": {
+          "chat": 6,
+          "question": 4,
+          "build": 2,
+          "task": 4,
+          "fix": 21,
+          "unclear": 3,
+          "INVALID": 0
+        },
+        "unclear": {
+          "chat": 2,
+          "question": 0,
+          "build": 1,
+          "task": 5,
+          "fix": 0,
+          "unclear": 31,
+          "INVALID": 0
+        }
+      },
+      "invalid": 0
+    },
+    "hard": {
+      "cases": 240,
+      "accuracy": 0.5458333333333333,
+      "macro_f1": 0.5417277585950407,
+      "per_class": {
+        "chat": {
+          "precision": 0.5675675675675675,
+          "recall": 0.525,
+          "f1": 0.5454545454545454,
+          "support": 40
+        },
+        "question": {
+          "precision": 0.6451612903225806,
+          "recall": 0.5,
+          "f1": 0.5633802816901409,
+          "support": 40
+        },
+        "build": {
+          "precision": 0.5925925925925926,
+          "recall": 0.8,
+          "f1": 0.6808510638297872,
+          "support": 40
+        },
+        "task": {
+          "precision": 0.4716981132075472,
+          "recall": 0.625,
+          "f1": 0.5376344086021505,
+          "support": 40
+        },
+        "fix": {
+          "precision": 0.3958333333333333,
+          "recall": 0.475,
+          "f1": 0.4318181818181818,
+          "support": 40
+        },
+        "unclear": {
+          "precision": 0.8235294117647058,
+          "recall": 0.35,
+          "f1": 0.49122807017543857,
+          "support": 40
+        }
+      },
+      "confusion": {
+        "chat": {
+          "chat": 21,
+          "question": 1,
+          "build": 10,
+          "task": 0,
+          "fix": 8,
+          "unclear": 0,
+          "INVALID": 0
+        },
+        "question": {
+          "chat": 5,
+          "question": 20,
+          "build": 2,
+          "task": 6,
+          "fix": 7,
+          "unclear": 0,
+          "INVALID": 0
+        },
+        "build": {
+          "chat": 2,
+          "question": 2,
+          "build": 32,
+          "task": 1,
+          "fix": 3,
+          "unclear": 0,
+          "INVALID": 0
+        },
+        "task": {
+          "chat": 1,
+          "question": 2,
+          "build": 7,
+          "task": 25,
+          "fix": 5,
+          "unclear": 0,
+          "INVALID": 0
+        },
+        "fix": {
+          "chat": 4,
+          "question": 5,
+          "build": 1,
+          "task": 8,
+          "fix": 19,
+          "unclear": 3,
+          "INVALID": 0
+        },
+        "unclear": {
+          "chat": 4,
+          "question": 1,
+          "build": 2,
+          "task": 13,
+          "fix": 6,
+          "unclear": 14,
+          "INVALID": 0
+        }
+      },
+      "invalid": 0
+    }
+  },
+  "difficulty_counts": {
+    "medium": 239,
+    "hard": 240,
+    "easy": 235
+  },
+  "predictions_sha256": "1ba3d3f923ef3ca3d7c9457518467c77480715331c0bf09efe97ff9beba42463"
+}

results/baselines/embedding-kmeans-seed-2.json

@@ -0,0 +1,439 @@
+{
+  "model": "minilm-l6-v2",
+  "method": "k-means on training embeddings; training-majority cluster labels",
+  "seed": 2,
+  "clusters": 48,
+  "adjusted_rand_index": 0.08624731089370681,
+  "validation_candidates": [
+    {
+      "clusters": 6,
+      "validation_macro_f1": 0.44224637568678826
+    },
+    {
+      "clusters": 12,
+      "validation_macro_f1": 0.47702334836973365
+    },
+    {
+      "clusters": 24,
+      "validation_macro_f1": 0.5096566658977267
+    },
+    {
+      "clusters": 48,
+      "validation_macro_f1": 0.534471064892484
+    }
+  ],
+  "cases": 714,
+  "accuracy": 0.5098039215686274,
+  "macro_f1": 0.5078508804654348,
+  "per_class": {
+    "chat": {
+      "precision": 0.45794392523364486,
+      "recall": 0.4083333333333333,
+      "f1": 0.43171806167400884,
+      "support": 120
+    },
+    "question": {
+      "precision": 0.4460431654676259,
+      "recall": 0.5166666666666667,
+      "f1": 0.47876447876447875,
+      "support": 120
+    },
+    "build": {
+      "precision": 0.5572519083969466,
+      "recall": 0.6083333333333333,
+      "f1": 0.5816733067729084,
+      "support": 120
+    },
+    "task": {
+      "precision": 0.43373493975903615,
+      "recall": 0.3,
+      "f1": 0.35467980295566504,
+      "support": 120
+    },
+    "fix": {
+      "precision": 0.39855072463768115,
+      "recall": 0.4583333333333333,
+      "f1": 0.4263565891472868,
+      "support": 120
+    },
+    "unclear": {
+      "precision": 0.7672413793103449,
+      "recall": 0.7807017543859649,
+      "f1": 0.7739130434782608,
+      "support": 114
+    }
+  },
+  "confusion": {
+    "chat": {
+      "chat": 49,
+      "question": 21,
+      "build": 27,
+      "task": 6,
+      "fix": 15,
+      "unclear": 2,
+      "INVALID": 0
+    },
+    "question": {
+      "chat": 17,
+      "question": 62,
+      "build": 7,
+      "task": 11,
+      "fix": 19,
+      "unclear": 4,
+      "INVALID": 0
+    },
+    "build": {
+      "chat": 14,
+      "question": 15,
+      "build": 73,
+      "task": 11,
+      "fix": 7,
+      "unclear": 0,
+      "INVALID": 0
+    },
+    "task": {
+      "chat": 9,
+      "question": 18,
+      "build": 14,
+      "task": 36,
+      "fix": 34,
+      "unclear": 9,
+      "INVALID": 0
+    },
+    "fix": {
+      "chat": 16,
+      "question": 20,
+      "build": 6,
+      "task": 11,
+      "fix": 55,
+      "unclear": 12,
+      "INVALID": 0
+    },
+    "unclear": {
+      "chat": 2,
+      "question": 3,
+      "build": 4,
+      "task": 8,
+      "fix": 8,
+      "unclear": 89,
+      "INVALID": 0
+    }
+  },
+  "invalid": 0,
+  "nll": 1.229046056763782,
+  "brier": 0.5827884372993704,
+  "false_actions": 85,
+  "passive_cases": 240,
+  "false_action_rate": 0.3541666666666667,
+  "unclear_cases": 114,
+  "actions_on_unclear": 20,
+  "action_rate_on_unclear": 0.17543859649122806,
+  "by_difficulty": {
+    "easy": {
+      "cases": 235,
+      "accuracy": 0.5148936170212766,
+      "macro_f1": 0.5112951342836657,
+      "per_class": {
+        "chat": {
+          "precision": 0.5277777777777778,
+          "recall": 0.475,
+          "f1": 0.5,
+          "support": 40
+        },
+        "question": {
+          "precision": 0.4583333333333333,
+          "recall": 0.55,
+          "f1": 0.5,
+          "support": 40
+        },
+        "build": {
+          "precision": 0.5116279069767442,
+          "recall": 0.55,
+          "f1": 0.5301204819277109,
+          "support": 40
+        },
+        "task": {
+          "precision": 0.3,
+          "recall": 0.15,
+          "f1": 0.2,
+          "support": 40
+        },
+        "fix": {
+          "precision": 0.4074074074074074,
+          "recall": 0.55,
+          "f1": 0.46808510638297873,
+          "support": 40
+        },
+        "unclear": {
+          "precision": 0.8823529411764706,
+          "recall": 0.8571428571428571,
+          "f1": 0.8695652173913043,
+          "support": 35
+        }
+      },
+      "confusion": {
+        "chat": {
+          "chat": 19,
+          "question": 6,
+          "build": 10,
+          "task": 0,
+          "fix": 4,
+          "unclear": 1,
+          "INVALID": 0
+        },
+        "question": {
+          "chat": 5,
+          "question": 22,
+          "build": 3,
+          "task": 3,
+          "fix": 7,
+          "unclear": 0,
+          "INVALID": 0
+        },
+        "build": {
+          "chat": 3,
+          "question": 5,
+          "build": 22,
+          "task": 7,
+          "fix": 3,
+          "unclear": 0,
+          "INVALID": 0
+        },
+        "task": {
+          "chat": 3,
+          "question": 8,
+          "build": 4,
+          "task": 6,
+          "fix": 16,
+          "unclear": 3,
+          "INVALID": 0
+        },
+        "fix": {
+          "chat": 6,
+          "question": 7,
+          "build": 3,
+          "task": 2,
+          "fix": 22,
+          "unclear": 0,
+          "INVALID": 0
+        },
+        "unclear": {
+          "chat": 0,
+          "question": 0,
+          "build": 1,
+          "task": 2,
+          "fix": 2,
+          "unclear": 30,
+          "INVALID": 0
+        }
+      },
+      "invalid": 0
+    },
+    "medium": {
+      "cases": 239,
+      "accuracy": 0.49372384937238495,
+      "macro_f1": 0.48830145932976254,
+      "per_class": {
+        "chat": {
+          "precision": 0.35294117647058826,
+          "recall": 0.3,
+          "f1": 0.32432432432432434,
+          "support": 40
+        },
+        "question": {
+          "precision": 0.40816326530612246,
+          "recall": 0.5,
+          "f1": 0.449438202247191,
+          "support": 40
+        },
+        "build": {
+          "precision": 0.6486486486486487,
+          "recall": 0.6,
+          "f1": 0.6233766233766234,
+          "support": 40
+        },
+        "task": {
+          "precision": 0.23333333333333334,
+          "recall": 0.175,
+          "f1": 0.2,
+          "support": 40
+        },
+        "fix": {
+          "precision": 0.42857142857142855,
+          "recall": 0.525,
+          "f1": 0.47191011235955055,
+          "support": 40
+        },
+        "unclear": {
+          "precision": 0.85,
+          "recall": 0.8717948717948718,
+          "f1": 0.8607594936708861,
+          "support": 39
+        }
+      },
+      "confusion": {
+        "chat": {
+          "chat": 12,
+          "question": 11,
+          "build": 5,
+          "task": 6,
+          "fix": 5,
+          "unclear": 1,
+          "INVALID": 0
+        },
+        "question": {
+          "chat": 5,
+          "question": 20,
+          "build": 2,
+          "task": 7,
+          "fix": 5,
+          "unclear": 1,
+          "INVALID": 0
+        },
+        "build": {
+          "chat": 6,
+          "question": 6,
+          "build": 24,
+          "task": 2,
+          "fix": 2,
+          "unclear": 0,
+          "INVALID": 0
+        },
+        "task": {
+          "chat": 6,
+          "question": 5,
+          "build": 3,
+          "task": 7,
+          "fix": 16,
+          "unclear": 3,
+          "INVALID": 0
+        },
+        "fix": {
+          "chat": 5,
+          "question": 6,
+          "build": 3,
+          "task": 4,
+          "fix": 21,
+          "unclear": 1,
+          "INVALID": 0
+        },
+        "unclear": {
+          "chat": 0,
+          "question": 1,
+          "build": 0,
+          "task": 4,
+          "fix": 0,
+          "unclear": 34,
+          "INVALID": 0
+        }
+      },
+      "invalid": 0
+    },
+    "hard": {
+      "cases": 240,
+      "accuracy": 0.5208333333333334,
+      "macro_f1": 0.5181061704750312,
+      "per_class": {
+        "chat": {
+          "precision": 0.4864864864864865,
+          "recall": 0.45,
+          "f1": 0.4675324675324675,
+          "support": 40
+        },
+        "question": {
+          "precision": 0.47619047619047616,
+          "recall": 0.5,
+          "f1": 0.4878048780487805,
+          "support": 40
+        },
+        "build": {
+          "precision": 0.5294117647058824,
+          "recall": 0.675,
+          "f1": 0.5934065934065934,
+          "support": 40
+        },
+        "task": {
+          "precision": 0.696969696969697,
+          "recall": 0.575,
+          "f1": 0.6301369863013698,
+          "support": 40
+        },
+        "fix": {
+          "precision": 0.34285714285714286,
+          "recall": 0.3,
+          "f1": 0.32,
+          "support": 40
+        },
+        "unclear": {
+          "precision": 0.5952380952380952,
+          "recall": 0.625,
+          "f1": 0.6097560975609756,
+          "support": 40
+        }
+      },
+      "confusion": {
+        "chat": {
+          "chat": 18,
+          "question": 4,
+          "build": 12,
+          "task": 0,
+          "fix": 6,
+          "unclear": 0,
+          "INVALID": 0
+        },
+        "question": {
+          "chat": 7,
+          "question": 20,
+          "build": 2,
+          "task": 1,
+          "fix": 7,
+          "unclear": 3,
+          "INVALID": 0
+        },
+        "build": {
+          "chat": 5,
+          "question": 4,
+          "build": 27,
+          "task": 2,
+          "fix": 2,
+          "unclear": 0,
+          "INVALID": 0
+        },
+        "task": {
+          "chat": 0,
+          "question": 5,
+          "build": 7,
+          "task": 23,
+          "fix": 2,
+          "unclear": 3,
+          "INVALID": 0
+        },
+        "fix": {
+          "chat": 5,
+          "question": 7,
+          "build": 0,
+          "task": 5,
+          "fix": 12,
+          "unclear": 11,
+          "INVALID": 0
+        },
+        "unclear": {
+          "chat": 2,
+          "question": 2,
+          "build": 3,
+          "task": 2,
+          "fix": 6,
+          "unclear": 25,
+          "INVALID": 0
+        }
+      },
+      "invalid": 0
+    }
+  },
+  "difficulty_counts": {
+    "medium": 239,
+    "hard": 240,
+    "easy": 235
+  },
+  "predictions_sha256": "6d6fce21c582abab8bf4fa4df51cdadc84ed2eca9c12fde94feb8ac25226e713"
+}