27630 stories 2000/27630 (424918 tokens) 4000/27630 (850028 tokens) 6000/27630 (1276653 tokens) 8000/27630 (1703505 tokens) 10000/27630 (2129443 tokens) 12000/27630 (2551094 tokens) 14000/27630 (2971406 tokens) 16000/27630 (3392767 tokens) 18000/27630 (3818407 tokens) 20000/27630 (4251049 tokens) 22000/27630 (4679628 tokens) 24000/27630 (5109598 tokens) 26000/27630 (5540190 tokens) wrote ../data/tokens.npy: 5885589 tokens === FROZEN BASELINES === val[frozen exact T=512 (RoPE extrapolated)] CE = 2.6080 (3s) val[frozen kmeans-raw m=64] CE = 4.2283 (14s) val[frozen kmeans-reindex m=64] CE = 3.9313 (21s) === TUNE RAW === val[frozen kmeans-raw (before)] CE = 4.2283 (15s) step 0 loss 4.2118 lr 1.00e-05 8s step 10 loss 3.4479 lr 6.00e-05 90s step 20 loss 3.1825 lr 9.88e-05 167s step 30 loss 3.0716 lr 9.74e-05 241s step 40 loss 2.9751 lr 9.55e-05 313s step 50 loss 3.0526 lr 9.30e-05 386s step 60 loss 2.9354 lr 9.01e-05 461s step 70 loss 2.8348 lr 8.68e-05 533s step 80 loss 3.0630 lr 8.31e-05 609s step 90 loss 2.7585 lr 7.90e-05 683s step 100 loss 2.9197 lr 7.45e-05 757s val[kmeans-raw @ step 100] CE = 2.8765 (13s) step 110 loss 2.9657 lr 6.99e-05 845s step 120 loss 2.7240 lr 6.50e-05 922s step 130 loss 2.7540 lr 5.99e-05 998s step 140 loss 2.8152 lr 5.47e-05 1075s step 150 loss 2.8801 lr 4.95e-05 1149s step 160 loss 2.6784 lr 4.43e-05 1222s step 170 loss 2.6437 lr 3.91e-05 1294s step 180 loss 2.7983 lr 3.41e-05 1373s step 190 loss 2.8824 lr 2.92e-05 1447s step 200 loss 2.7135 lr 2.45e-05 2324s val[kmeans-raw @ step 200] CE = 2.7445 (14s) step 210 loss 2.8994 lr 2.02e-05 2414s step 220 loss 2.5447 lr 1.62e-05 2490s step 230 loss 2.8946 lr 1.25e-05 2567s step 240 loss 2.6490 lr 9.24e-06 2643s step 250 loss 2.6658 lr 6.44e-06 2720s step 260 loss 2.8923 lr 4.11e-06 2817s step 270 loss 2.7757 lr 2.29e-06 2930s step 280 loss 2.8411 lr 9.86e-07 3045s step 290 loss 2.6668 lr 2.22e-07 3159s val[tuned kmeans-raw (final)] CE = 2.7131 (23s) exported ../models/stories15M-raw.bin === TUNE REINDEX === val[frozen kmeans-reindex (before)] CE = 3.9313 (34s) step 0 loss 3.9785 lr 1.00e-05 15s step 10 loss 3.2042 lr 6.00e-05 161s step 20 loss 2.9119 lr 9.88e-05 293s step 30 loss 2.8596 lr 9.74e-05 391s step 40 loss 2.7752 lr 9.55e-05 489s step 50 loss 2.7981 lr 9.30e-05 590s step 60 loss 2.7069 lr 9.01e-05 697s step 70 loss 2.5204 lr 8.68e-05 798s step 80 loss 2.8117 lr 8.31e-05 3809s step 90 loss 2.5159 lr 7.90e-05 3909s step 100 loss 2.6274 lr 7.45e-05 4011s val[kmeans-reindex @ step 100] CE = 2.6342 (20s) step 110 loss 2.7148 lr 6.99e-05 4135s step 120 loss 2.4818 lr 6.50e-05 4252s step 130 loss 2.4556 lr 5.99e-05 4363s step 140 loss 2.5012 lr 5.47e-05 4484s step 150 loss 2.6366 lr 4.95e-05 4592s step 160 loss 2.4226 lr 4.43e-05 4699s step 170 loss 2.4394 lr 3.91e-05 8759s step 180 loss 2.5796 lr 3.41e-05 18660s step 190 loss 2.6130 lr 2.92e-05 25830s step 200 loss 2.4913 lr 2.45e-05 26049s val[kmeans-reindex @ step 200] CE = 2.5107 (40s) step 210 loss 2.6061 lr 2.02e-05 36076s step 220 loss 2.3664 lr 1.62e-05 36198s step 230 loss 2.6222 lr 1.25e-05 36302s step 240 loss 2.4536 lr 9.24e-06 36404s step 250 loss 2.4442 lr 6.44e-06 36511s step 260 loss 2.7062 lr 4.11e-06 36617s step 270 loss 2.4886 lr 2.29e-06 36719s step 280 loss 2.6760 lr 9.86e-07 36818s step 290 loss 2.3969 lr 2.22e-07 36919s val[tuned kmeans-reindex (final)] CE = 2.4927 (20s) exported ../models/stories15M-reindex.bin === ALL DONE ===