========================================================================================== RoPE ATTENTION vs CONSTELLATION RELAY Input dim: 128, Sequence length: 2000 Device: cuda ========================================================================================== Baseline: CV=0.0590 eff_dim=120.0 ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ TEST 1: Single pass — all architectures ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ arch params CV_out CV_norm cos_orig eff_dim vanilla 65,792 0.0616 0.0598 0.989555 119.2 rope_std 65,792 0.0591 0.0595 0.994694 119.9 rope_ntk 65,792 0.0600 0.0621 0.992254 119.7 relay 19,112 0.0579 0.0588 0.995458 119.2 ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ TEST 2: Depth sweep — 16 layers, all architectures ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ vanilla: depth CV CV_n eff_d cos_orig 1 0.0590 0.0613 119.2 0.988720 2 0.0812 0.0835 118.6 0.906975 4 0.0782 0.0772 118.0 0.258551 8 0.0681 0.0678 118.0 0.124103 12 0.0672 0.0670 118.0 0.084707 16 0.0656 0.0673 118.0 0.070113 rope_std: depth CV CV_n eff_d cos_orig 1 0.0599 0.0604 119.8 0.994533 2 0.0772 0.0784 119.6 0.937518 4 0.0715 0.0726 119.5 0.270651 8 0.0627 0.0633 119.5 0.124870 12 0.0637 0.0644 119.5 0.086343 16 0.0641 0.0635 119.5 0.072248 rope_ntk: depth CV CV_n eff_d cos_orig 1 0.0607 0.0605 119.6 0.991296 2 0.0852 0.0831 119.3 0.909574 4 0.0745 0.0759 119.1 0.255422 8 0.0662 0.0664 119.1 0.123437 12 0.0633 0.0614 119.1 0.092321 16 0.0623 0.0634 119.1 0.076500 relay: depth CV CV_n eff_d cos_orig 1 0.0581 0.0578 119.2 0.995456 2 0.0586 0.0577 119.2 0.994940 4 0.0592 0.0575 119.1 0.994511 8 0.0605 0.0578 119.1 0.994254 12 0.0587 0.0580 119.1 0.994219 16 0.0583 0.0591 119.1 0.994187 ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ TEST 3: Interleaved — [attn type] → relay → [attn type] → relay → ... ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ vanilla + relay interleaved: step type CV_n eff_d cos_orig 1 attn 0.0599 119.2 0.988902 2 relay 0.0607 118.5 0.984072 4 relay 0.0618 118.4 0.982892 8 relay 0.0613 118.4 0.981810 12 relay 0.0623 118.3 0.981172 16 relay 0.0629 118.3 0.981130 rope_std + relay interleaved: step type CV_n eff_d cos_orig 1 attn 0.0619 119.8 0.994483 2 relay 0.0598 119.1 0.989662 4 relay 0.0607 119.0 0.988010 8 relay 0.0612 119.0 0.987173 12 relay 0.0607 119.0 0.987077 16 relay 0.0613 119.0 0.986923 rope_ntk + relay interleaved: step type CV_n eff_d cos_orig 1 attn 0.0612 119.7 0.992953 2 relay 0.0596 119.0 0.988482 4 relay 0.0615 118.9 0.986365 8 relay 0.0612 118.9 0.985944 12 relay 0.0612 118.9 0.986218 16 relay 0.0600 118.9 0.986233 ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ TEST 4: Throughput ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ arch ms params vanilla 0.51 65,792 rope_std 0.56 65,792 rope_ntk 0.56 65,792 relay 0.59 19,112 ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ TEST 5: Clustered input (10 clusters, d=128) ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ spread CV_base vanilla rope_std rope_ntk relay 0.1 0.1545 0.1683 0.1736 0.1619 0.1489 0.3 0.0610 0.0804 0.0848 0.0822 0.0627 0.5 0.0595 0.0693 0.0722 0.0704 0.0583 1.0 0.0593 0.0619 0.0594 0.0623 0.0574 ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ TEST 6: RoPE base frequency sweep Does the rotation frequency affect geometric preservation? ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ base CV_n cos_orig eff_dim 100 0.0601 0.993803 120.0 500 0.0597 0.995451 119.9 1000 0.0609 0.995856 120.0 5000 0.0596 0.994216 119.8 10000 0.0622 0.994004 119.8 50000 0.0597 0.993237 119.8 100000 0.0589 0.994278 119.7 500000 0.0600 0.994765 119.6 ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ TEST 7: NTK scale factor sweep (base=10000) ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ scale CV_n cos_orig eff_dim 1.0 0.0602 0.993739 119.8 2.0 0.0591 0.995000 119.7 4.0 0.0599 0.994255 119.7 8.0 0.0600 0.993167 119.7 16.0 0.0607 0.992647 119.7 32.0 0.0584 0.993814 119.7 64.0 0.0607 0.993126 119.6 ========================================================================================== SUMMARY — cos_to_orig at depth 16 ========================================================================================== Compare the depth-16 cos_to_orig from Test 2 across all architectures: vanilla attention: (see Test 2) RoPE standard: (see Test 2) RoPE NTK: (see Test 2) constellation relay: (see Test 2) And the interleaved results from Test 3: vanilla + relay: (see Test 3) rope_std + relay: (see Test 3) rope_ntk + relay: (see Test 3) ========================================================================================== DONE ==========================================================================================