{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 500,
  "global_step": 2030,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0004926108374384236,
      "grad_norm": 0.7725405549175612,
      "kl": 0.0,
      "learning_rate": 0.0,
      "logits/chosen": -94463020.52173913,
      "logits/rejected": -108540403.51219513,
      "logps/chosen": -343.6521739130435,
      "logps/rejected": -394.9268292682927,
      "loss": 0.5583,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.0009852216748768472,
      "grad_norm": 0.7243943399689341,
      "kl": 0.0,
      "learning_rate": 2.0000000000000002e-07,
      "logits/chosen": -74667349.33333333,
      "logits/rejected": -78223769.6,
      "logps/chosen": -238.33333333333334,
      "logps/rejected": -352.4,
      "loss": 0.5613,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 2
    },
    {
      "epoch": 0.001477832512315271,
      "grad_norm": 0.7039018749647754,
      "kl": 0.0234375,
      "learning_rate": 4.0000000000000003e-07,
      "logits/chosen": -112575119.36,
      "logits/rejected": -86144551.38461539,
      "logps/chosen": -336.32,
      "logps/rejected": -329.43589743589746,
      "loss": 0.5638,
      "rewards/chosen": 2.8049188232421876,
      "rewards/margins": 2.807130173903245,
      "rewards/rejected": -0.0022113506610576925,
      "step": 3
    },
    {
      "epoch": 0.0019704433497536944,
      "grad_norm": 0.7589659865122234,
      "kl": 0.03515625,
      "learning_rate": 6.000000000000001e-07,
      "logits/chosen": -80440758.85714285,
      "logits/rejected": -106255701.33333333,
      "logps/chosen": -282.0,
      "logps/rejected": -389.3333333333333,
      "loss": 0.5724,
      "rewards/chosen": 0.010645185198102678,
      "rewards/margins": -0.00690581306578621,
      "rewards/rejected": 0.017550998263888888,
      "step": 4
    },
    {
      "epoch": 0.0024630541871921183,
      "grad_norm": 0.9310889817961356,
      "kl": 0.248046875,
      "learning_rate": 8.000000000000001e-07,
      "logits/chosen": -92861890.56,
      "logits/rejected": -97436908.3076923,
      "logps/chosen": -281.6,
      "logps/rejected": -484.9230769230769,
      "loss": 0.563,
      "rewards/chosen": 2.5313290405273436,
      "rewards/margins": 2.5308657993414463,
      "rewards/rejected": 0.0004632411858974359,
      "step": 5
    },
    {
      "epoch": 0.002955665024630542,
      "grad_norm": 0.7314630037335983,
      "kl": 0.0546875,
      "learning_rate": 1.0000000000000002e-06,
      "logits/chosen": -82643323.25925925,
      "logits/rejected": -84452877.83783785,
      "logps/chosen": -361.48148148148147,
      "logps/rejected": -401.2972972972973,
      "loss": 0.5679,
      "rewards/chosen": 0.02662037037037037,
      "rewards/margins": 0.01515235939064064,
      "rewards/rejected": 0.01146801097972973,
      "step": 6
    },
    {
      "epoch": 0.0034482758620689655,
      "grad_norm": 1.4042147890059478,
      "kl": 0.0078125,
      "learning_rate": 1.2000000000000002e-06,
      "logits/chosen": -96297795.91836734,
      "logits/rejected": -93017982.37974684,
      "logps/chosen": -353.6326530612245,
      "logps/rejected": -389.67088607594934,
      "loss": 0.5557,
      "rewards/chosen": 2.5828259526466835,
      "rewards/margins": 2.6090818861909875,
      "rewards/rejected": -0.0262559335443038,
      "step": 7
    },
    {
      "epoch": 0.003940886699507389,
      "grad_norm": 0.7770018499992601,
      "kl": 0.166015625,
      "learning_rate": 1.4000000000000001e-06,
      "logits/chosen": -96668720.76190476,
      "logits/rejected": -94615694.88372093,
      "logps/chosen": -300.57142857142856,
      "logps/rejected": -392.93023255813955,
      "loss": 0.5525,
      "rewards/chosen": 2.387027922130766,
      "rewards/margins": 2.3833970400450104,
      "rewards/rejected": 0.003630882085755814,
      "step": 8
    },
    {
      "epoch": 0.004433497536945813,
      "grad_norm": 0.7619256534553093,
      "kl": 0.1103515625,
      "learning_rate": 1.6000000000000001e-06,
      "logits/chosen": -79991369.14285715,
      "logits/rejected": -92762397.76744185,
      "logps/chosen": -227.04761904761904,
      "logps/rejected": -370.9767441860465,
      "loss": 0.5562,
      "rewards/chosen": -0.02492559523809524,
      "rewards/margins": -0.027041244084388846,
      "rewards/rejected": 0.0021156488462936046,
      "step": 9
    },
    {
      "epoch": 0.0049261083743842365,
      "grad_norm": 0.6317336067331906,
      "kl": 0.03515625,
      "learning_rate": 1.8000000000000001e-06,
      "logits/chosen": -103845181.79310344,
      "logits/rejected": -75617309.25714286,
      "logps/chosen": -296.2758620689655,
      "logps/rejected": -327.77142857142854,
      "loss": 0.5726,
      "rewards/chosen": -0.004099878771551724,
      "rewards/margins": 0.011504194889162562,
      "rewards/rejected": -0.015604073660714285,
      "step": 10
    },
    {
      "epoch": 0.00541871921182266,
      "grad_norm": 0.7322469592402041,
      "kl": 0.0703125,
      "learning_rate": 2.0000000000000003e-06,
      "logits/chosen": -84346430.43902439,
      "logits/rejected": -93817420.50574712,
      "logps/chosen": -303.609756097561,
      "logps/rejected": -342.0689655172414,
      "loss": 0.5505,
      "rewards/chosen": 2.4509563911251906,
      "rewards/margins": 2.4577249794872595,
      "rewards/rejected": -0.006768588362068966,
      "step": 11
    },
    {
      "epoch": 0.005911330049261084,
      "grad_norm": 0.7724780558761759,
      "kl": 0.0,
      "learning_rate": 2.2e-06,
      "logits/chosen": -104513804.59016393,
      "logits/rejected": -86390142.08955224,
      "logps/chosen": -345.57377049180326,
      "logps/rejected": -389.25373134328356,
      "loss": 0.5774,
      "rewards/chosen": 0.0034099641393442624,
      "rewards/margins": 0.010413520575911427,
      "rewards/rejected": -0.0070035564365671646,
      "step": 12
    },
    {
      "epoch": 0.0064039408866995075,
      "grad_norm": 0.8104643277644997,
      "kl": 0.0,
      "learning_rate": 2.4000000000000003e-06,
      "logits/chosen": -90596966.4,
      "logits/rejected": -88618115.28205128,
      "logps/chosen": -282.24,
      "logps/rejected": -396.71794871794873,
      "loss": 0.5642,
      "rewards/chosen": 2.529853515625,
      "rewards/margins": 2.527925430689103,
      "rewards/rejected": 0.001928084935897436,
      "step": 13
    },
    {
      "epoch": 0.006896551724137931,
      "grad_norm": 0.8109007591713548,
      "kl": 0.0,
      "learning_rate": 2.6e-06,
      "logits/chosen": -82676184.61538461,
      "logits/rejected": -83775703.57894737,
      "logps/chosen": -416.61538461538464,
      "logps/rejected": -370.10526315789474,
      "loss": 0.5632,
      "rewards/chosen": 0.006742037259615385,
      "rewards/margins": 0.025817129776062753,
      "rewards/rejected": -0.01907509251644737,
      "step": 14
    },
    {
      "epoch": 0.007389162561576354,
      "grad_norm": 0.6764157650546082,
      "kl": 0.091796875,
      "learning_rate": 2.8000000000000003e-06,
      "logits/chosen": -70433617.17073171,
      "logits/rejected": -98252776.45977011,
      "logps/chosen": -274.9268292682927,
      "logps/rejected": -320.367816091954,
      "loss": 0.5497,
      "rewards/chosen": 3.2483888951743523,
      "rewards/margins": 3.271489649484697,
      "rewards/rejected": -0.023100754310344827,
      "step": 15
    },
    {
      "epoch": 0.007881773399014778,
      "grad_norm": 0.6839051832080609,
      "kl": 0.046875,
      "learning_rate": 3e-06,
      "logits/chosen": -88857106.96296297,
      "logits/rejected": -88420462.7027027,
      "logps/chosen": -280.2962962962963,
      "logps/rejected": -380.97297297297297,
      "loss": 0.5654,
      "rewards/chosen": 0.006370261863425926,
      "rewards/margins": 0.030441207809371873,
      "rewards/rejected": -0.024070945945945946,
      "step": 16
    },
    {
      "epoch": 0.008374384236453201,
      "grad_norm": 0.9318811437956562,
      "kl": 0.0,
      "learning_rate": 3.2000000000000003e-06,
      "logits/chosen": -80191097.90476191,
      "logits/rejected": -100175586.23255815,
      "logps/chosen": -283.8095238095238,
      "logps/rejected": -440.5581395348837,
      "loss": 0.5501,
      "rewards/chosen": -0.018624441964285716,
      "rewards/margins": 0.017917164140365444,
      "rewards/rejected": -0.03654160610465116,
      "step": 17
    },
    {
      "epoch": 0.008866995073891626,
      "grad_norm": 0.6700493297866319,
      "kl": 0.0,
      "learning_rate": 3.4000000000000005e-06,
      "logits/chosen": -92006966.46808511,
      "logits/rejected": -78345456.19753087,
      "logps/chosen": -280.8510638297872,
      "logps/rejected": -318.0246913580247,
      "loss": 0.5586,
      "rewards/chosen": -0.02820083942819149,
      "rewards/margins": 0.004520032485388754,
      "rewards/rejected": -0.032720871913580245,
      "step": 18
    },
    {
      "epoch": 0.00935960591133005,
      "grad_norm": 0.6991054270972118,
      "kl": 0.0,
      "learning_rate": 3.6000000000000003e-06,
      "logits/chosen": -104552559.7090909,
      "logits/rejected": -98365047.23287672,
      "logps/chosen": -343.56363636363636,
      "logps/rejected": -346.3013698630137,
      "loss": 0.5695,
      "rewards/chosen": -0.032356400923295454,
      "rewards/margins": 0.007669284008211395,
      "rewards/rejected": -0.04002568493150685,
      "step": 19
    },
    {
      "epoch": 0.009852216748768473,
      "grad_norm": 0.6848023735743262,
      "kl": 0.0,
      "learning_rate": 3.8000000000000005e-06,
      "logits/chosen": -67108864.0,
      "logits/rejected": -96253899.48717949,
      "logps/chosen": -191.04,
      "logps/rejected": -379.4871794871795,
      "loss": 0.5574,
      "rewards/chosen": 2.2927310180664064,
      "rewards/margins": 2.3326699203099963,
      "rewards/rejected": -0.039938902243589744,
      "step": 20
    },
    {
      "epoch": 0.010344827586206896,
      "grad_norm": 0.7926843387167746,
      "kl": 0.0,
      "learning_rate": 4.000000000000001e-06,
      "logits/chosen": -85983232.0,
      "logits/rejected": -109051904.0,
      "logps/chosen": -198.88,
      "logps/rejected": -362.2564102564103,
      "loss": 0.5606,
      "rewards/chosen": -0.00783203125,
      "rewards/margins": 0.02451034154647436,
      "rewards/rejected": -0.03234237279647436,
      "step": 21
    },
    {
      "epoch": 0.01083743842364532,
      "grad_norm": 0.7995665443898083,
      "kl": 0.0,
      "learning_rate": 4.2000000000000004e-06,
      "logits/chosen": -75963505.77777778,
      "logits/rejected": -116533635.45945945,
      "logps/chosen": -298.962962962963,
      "logps/rejected": -399.56756756756755,
      "loss": 0.5636,
      "rewards/chosen": -0.04568820529513889,
      "rewards/margins": 0.03871846700215841,
      "rewards/rejected": -0.0844066722972973,
      "step": 22
    },
    {
      "epoch": 0.011330049261083743,
      "grad_norm": 0.7678950627217043,
      "kl": 0.0,
      "learning_rate": 4.4e-06,
      "logits/chosen": -95782651.34545454,
      "logits/rejected": -87678190.46575342,
      "logps/chosen": -325.23636363636365,
      "logps/rejected": -376.54794520547944,
      "loss": 0.5616,
      "rewards/chosen": 1.7402904163707387,
      "rewards/margins": 1.8265489780145743,
      "rewards/rejected": -0.08625856164383562,
      "step": 23
    },
    {
      "epoch": 0.011822660098522168,
      "grad_norm": 0.916378100197572,
      "kl": 0.0,
      "learning_rate": 4.600000000000001e-06,
      "logits/chosen": -80303445.33333333,
      "logits/rejected": -121425100.8,
      "logps/chosen": -295.3333333333333,
      "logps/rejected": -460.0,
      "loss": 0.5473,
      "rewards/chosen": -0.044820149739583336,
      "rewards/margins": 0.08457438151041666,
      "rewards/rejected": -0.12939453125,
      "step": 24
    },
    {
      "epoch": 0.012315270935960592,
      "grad_norm": 0.831275713182495,
      "kl": 0.0,
      "learning_rate": 4.800000000000001e-06,
      "logits/chosen": -77070336.0,
      "logits/rejected": -93637836.8,
      "logps/chosen": -386.0,
      "logps/rejected": -363.6,
      "loss": 0.542,
      "rewards/chosen": 0.018890380859375,
      "rewards/margins": 0.132659912109375,
      "rewards/rejected": -0.11376953125,
      "step": 25
    },
    {
      "epoch": 0.012807881773399015,
      "grad_norm": 0.812637172761167,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -88806321.23076923,
      "logits/rejected": -91391676.63157895,
      "logps/chosen": -329.53846153846155,
      "logps/rejected": -381.89473684210526,
      "loss": 0.5491,
      "rewards/chosen": -0.030949519230769232,
      "rewards/margins": 0.1211886386639676,
      "rewards/rejected": -0.15213815789473684,
      "step": 26
    },
    {
      "epoch": 0.013300492610837438,
      "grad_norm": 0.7677912743347989,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -115143631.23809524,
      "logits/rejected": -85739377.11627907,
      "logps/chosen": -293.14285714285717,
      "logps/rejected": -345.30232558139534,
      "loss": 0.5339,
      "rewards/chosen": 7.043703351702009,
      "rewards/margins": 7.189960619143869,
      "rewards/rejected": -0.14625726744186046,
      "step": 27
    },
    {
      "epoch": 0.013793103448275862,
      "grad_norm": 0.7435954981942442,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -81985536.0,
      "logits/rejected": -99483648.0,
      "logps/chosen": -356.25,
      "logps/rejected": -421.0,
      "loss": 0.5697,
      "rewards/chosen": -0.068267822265625,
      "rewards/margins": 0.103118896484375,
      "rewards/rejected": -0.17138671875,
      "step": 28
    },
    {
      "epoch": 0.014285714285714285,
      "grad_norm": 0.746804331927697,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -83711317.33333333,
      "logits/rejected": -76021760.0,
      "logps/chosen": -264.8333333333333,
      "logps/rejected": -318.0,
      "loss": 0.5363,
      "rewards/chosen": -0.06380208333333333,
      "rewards/margins": 0.15260416666666665,
      "rewards/rejected": -0.21640625,
      "step": 29
    },
    {
      "epoch": 0.014778325123152709,
      "grad_norm": 0.8574708616035411,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -89908670.35897435,
      "logits/rejected": -83226301.84269662,
      "logps/chosen": -290.87179487179486,
      "logps/rejected": -373.9325842696629,
      "loss": 0.5162,
      "rewards/chosen": 9.602936573517628,
      "rewards/margins": 9.842753989247965,
      "rewards/rejected": -0.23981741573033707,
      "step": 30
    },
    {
      "epoch": 0.015270935960591134,
      "grad_norm": 0.8526952335033704,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -110712149.33333333,
      "logits/rejected": -90701824.0,
      "logps/chosen": -324.6666666666667,
      "logps/rejected": -403.6,
      "loss": 0.527,
      "rewards/chosen": -0.062744140625,
      "rewards/margins": 0.22006835937500002,
      "rewards/rejected": -0.2828125,
      "step": 31
    },
    {
      "epoch": 0.015763546798029555,
      "grad_norm": 0.7985100167732029,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -80390826.66666667,
      "logits/rejected": -91707890.16216215,
      "logps/chosen": -315.85185185185185,
      "logps/rejected": -383.13513513513516,
      "loss": 0.5321,
      "rewards/chosen": -0.050238715277777776,
      "rewards/margins": 0.2664842576951952,
      "rewards/rejected": -0.31672297297297297,
      "step": 32
    },
    {
      "epoch": 0.01625615763546798,
      "grad_norm": 0.9348083034368446,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -111369808.84210527,
      "logits/rejected": -93113548.8,
      "logps/chosen": -416.42105263157896,
      "logps/rejected": -413.15555555555557,
      "loss": 0.4868,
      "rewards/chosen": 6.768390053196957,
      "rewards/margins": 7.151028942085846,
      "rewards/rejected": -0.38263888888888886,
      "step": 33
    },
    {
      "epoch": 0.016748768472906402,
      "grad_norm": 0.9139524355460118,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -76450722.9090909,
      "logits/rejected": -78593267.8095238,
      "logps/chosen": -336.3636363636364,
      "logps/rejected": -381.3333333333333,
      "loss": 0.4982,
      "rewards/chosen": 2.906952597878196,
      "rewards/margins": 3.3623097407353386,
      "rewards/rejected": -0.45535714285714285,
      "step": 34
    },
    {
      "epoch": 0.017241379310344827,
      "grad_norm": 0.8673199758741379,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -92939638.63414635,
      "logits/rejected": -89189222.98850575,
      "logps/chosen": -365.4634146341463,
      "logps/rejected": -385.8390804597701,
      "loss": 0.4922,
      "rewards/chosen": 3.065965605945122,
      "rewards/margins": 3.5192702036462715,
      "rewards/rejected": -0.45330459770114945,
      "step": 35
    },
    {
      "epoch": 0.017733990147783252,
      "grad_norm": 0.9177722127688166,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -95374825.73913044,
      "logits/rejected": -87978083.90243903,
      "logps/chosen": -362.2608695652174,
      "logps/rejected": -417.9512195121951,
      "loss": 0.4814,
      "rewards/chosen": -0.09515115489130435,
      "rewards/margins": 0.510031771937964,
      "rewards/rejected": -0.6051829268292683,
      "step": 36
    },
    {
      "epoch": 0.018226600985221674,
      "grad_norm": 0.7378081734983335,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -75497472.0,
      "logits/rejected": -81899304.42105263,
      "logps/chosen": -259.2307692307692,
      "logps/rejected": -348.2105263157895,
      "loss": 0.5079,
      "rewards/chosen": 1.821783212515024,
      "rewards/margins": 2.424579265146603,
      "rewards/rejected": -0.602796052631579,
      "step": 37
    },
    {
      "epoch": 0.0187192118226601,
      "grad_norm": 0.7584988314480431,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -89578349.71428572,
      "logits/rejected": -95769941.33333333,
      "logps/chosen": -334.57142857142856,
      "logps/rejected": -420.44444444444446,
      "loss": 0.5208,
      "rewards/chosen": -0.22209821428571427,
      "rewards/margins": 0.3946552579365079,
      "rewards/rejected": -0.6167534722222222,
      "step": 38
    },
    {
      "epoch": 0.01921182266009852,
      "grad_norm": 0.7413986705732822,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -88656072.78431372,
      "logits/rejected": -92819402.80519481,
      "logps/chosen": -371.45098039215685,
      "logps/rejected": -346.5974025974026,
      "loss": 0.5036,
      "rewards/chosen": -0.2262561274509804,
      "rewards/margins": 0.4385165998217469,
      "rewards/rejected": -0.6647727272727273,
      "step": 39
    },
    {
      "epoch": 0.019704433497536946,
      "grad_norm": 0.6975443146870118,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -80941143.14893617,
      "logits/rejected": -110916039.1111111,
      "logps/chosen": -261.1063829787234,
      "logps/rejected": -422.71604938271605,
      "loss": 0.4758,
      "rewards/chosen": -0.21575797872340424,
      "rewards/margins": 0.5836247373259784,
      "rewards/rejected": -0.7993827160493827,
      "step": 40
    },
    {
      "epoch": 0.02019704433497537,
      "grad_norm": 0.6804390663086125,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -117924470.15384616,
      "logits/rejected": -90839794.5263158,
      "logps/chosen": -329.53846153846155,
      "logps/rejected": -381.2631578947368,
      "loss": 0.4893,
      "rewards/chosen": 2.36469239455003,
      "rewards/margins": 3.2651858156026616,
      "rewards/rejected": -0.9004934210526315,
      "step": 41
    },
    {
      "epoch": 0.020689655172413793,
      "grad_norm": 0.695657556821512,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -107310872.1509434,
      "logits/rejected": -104913524.05333333,
      "logps/chosen": -340.52830188679246,
      "logps/rejected": -477.8666666666667,
      "loss": 0.4708,
      "rewards/chosen": 2.225431334297612,
      "rewards/margins": 3.3804313342976124,
      "rewards/rejected": -1.155,
      "step": 42
    },
    {
      "epoch": 0.021182266009852218,
      "grad_norm": 0.6669213375116103,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -88564342.15384616,
      "logits/rejected": -99669908.21052632,
      "logps/chosen": -292.0,
      "logps/rejected": -473.6842105263158,
      "loss": 0.4536,
      "rewards/chosen": -0.24661959134615385,
      "rewards/margins": 0.9968014612854251,
      "rewards/rejected": -1.243421052631579,
      "step": 43
    },
    {
      "epoch": 0.02167487684729064,
      "grad_norm": 0.6980261500683331,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -87168578.7826087,
      "logits/rejected": -86852782.82926829,
      "logps/chosen": -286.4347826086956,
      "logps/rejected": -351.219512195122,
      "loss": 0.4657,
      "rewards/chosen": -0.44650135869565216,
      "rewards/margins": 0.599230348621421,
      "rewards/rejected": -1.045731707317073,
      "step": 44
    },
    {
      "epoch": 0.022167487684729065,
      "grad_norm": 0.5975142037852935,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -96633474.50980392,
      "logits/rejected": -91838916.15584415,
      "logps/chosen": -276.078431372549,
      "logps/rejected": -400.6233766233766,
      "loss": 0.4672,
      "rewards/chosen": -0.40349264705882354,
      "rewards/margins": 0.874104755538579,
      "rewards/rejected": -1.2775974025974026,
      "step": 45
    },
    {
      "epoch": 0.022660098522167486,
      "grad_norm": 0.6397523526128902,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -97867093.33333333,
      "logits/rejected": -86109566.45783132,
      "logps/chosen": -306.4888888888889,
      "logps/rejected": -390.93975903614455,
      "loss": 0.413,
      "rewards/chosen": 2.4276945326063366,
      "rewards/margins": 3.8659475446545293,
      "rewards/rejected": -1.4382530120481927,
      "step": 46
    },
    {
      "epoch": 0.02315270935960591,
      "grad_norm": 0.585749344206562,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -94337460.4590164,
      "logits/rejected": -95404765.6119403,
      "logps/chosen": -315.0163934426229,
      "logps/rejected": -417.910447761194,
      "loss": 0.4875,
      "rewards/chosen": 1.5378739403896644,
      "rewards/margins": 3.021082895613545,
      "rewards/rejected": -1.4832089552238805,
      "step": 47
    },
    {
      "epoch": 0.023645320197044337,
      "grad_norm": 0.545628675142517,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -96235975.1111111,
      "logits/rejected": -98641944.6746988,
      "logps/chosen": -372.6222222222222,
      "logps/rejected": -405.5903614457831,
      "loss": 0.435,
      "rewards/chosen": -0.5722222222222222,
      "rewards/margins": 0.9940428380187416,
      "rewards/rejected": -1.5662650602409638,
      "step": 48
    },
    {
      "epoch": 0.02413793103448276,
      "grad_norm": 0.5684520500626891,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -77639244.25531915,
      "logits/rejected": -102734557.2345679,
      "logps/chosen": -257.70212765957444,
      "logps/rejected": -469.7283950617284,
      "loss": 0.4176,
      "rewards/chosen": -0.4910239361702128,
      "rewards/margins": 1.2975563107433676,
      "rewards/rejected": -1.7885802469135803,
      "step": 49
    },
    {
      "epoch": 0.024630541871921183,
      "grad_norm": 0.5596855167499316,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -79419922.96296297,
      "logits/rejected": -77807173.1891892,
      "logps/chosen": -363.25925925925924,
      "logps/rejected": -444.97297297297297,
      "loss": 0.4492,
      "rewards/chosen": -0.5428240740740741,
      "rewards/margins": 0.8349249823554022,
      "rewards/rejected": -1.3777490564294763,
      "step": 50
    },
    {
      "epoch": 0.025123152709359605,
      "grad_norm": 0.5708688728605908,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -89478485.33333333,
      "logits/rejected": -94618563.76470588,
      "logps/chosen": -328.53333333333336,
      "logps/rejected": -386.8235294117647,
      "loss": 0.4942,
      "rewards/chosen": 1.8541152954101563,
      "rewards/margins": 3.681321177763097,
      "rewards/rejected": -1.8272058823529411,
      "step": 51
    },
    {
      "epoch": 0.02561576354679803,
      "grad_norm": 0.48756961071488325,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -98239920.35555555,
      "logits/rejected": -89545863.71084337,
      "logps/chosen": -321.4222222222222,
      "logps/rejected": -352.3855421686747,
      "loss": 0.4218,
      "rewards/chosen": 2.884007093641493,
      "rewards/margins": 4.655091430990891,
      "rewards/rejected": -1.7710843373493976,
      "step": 52
    },
    {
      "epoch": 0.026108374384236452,
      "grad_norm": 0.5932515246958694,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -100740968.2962963,
      "logits/rejected": -128436390.05405405,
      "logps/chosen": -392.8888888888889,
      "logps/rejected": -497.72972972972974,
      "loss": 0.4579,
      "rewards/chosen": -0.6857638888888888,
      "rewards/margins": 0.11576133017783419,
      "rewards/rejected": -0.801525219066723,
      "step": 53
    },
    {
      "epoch": 0.026600985221674877,
      "grad_norm": 0.4657679378054746,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -105285590.20408164,
      "logits/rejected": -91319023.79746835,
      "logps/chosen": -414.6938775510204,
      "logps/rejected": -411.9493670886076,
      "loss": 0.4423,
      "rewards/chosen": -0.8903061224489796,
      "rewards/margins": -0.6260882145796165,
      "rewards/rejected": -0.2642179078693631,
      "step": 54
    },
    {
      "epoch": 0.027093596059113302,
      "grad_norm": 0.48769576022223715,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -88403022.76923077,
      "logits/rejected": -86902208.71910113,
      "logps/chosen": -240.6153846153846,
      "logps/rejected": -389.75280898876406,
      "loss": 0.4046,
      "rewards/chosen": 5.487836006360176,
      "rewards/margins": 7.201319152427592,
      "rewards/rejected": -1.7134831460674158,
      "step": 55
    },
    {
      "epoch": 0.027586206896551724,
      "grad_norm": 0.6342743295072918,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -84872975.05882353,
      "logits/rejected": -88243798.44155844,
      "logps/chosen": -305.2549019607843,
      "logps/rejected": -403.94805194805195,
      "loss": 0.4715,
      "rewards/chosen": -0.8265931372549019,
      "rewards/margins": 1.0857445250827604,
      "rewards/rejected": -1.9123376623376624,
      "step": 56
    },
    {
      "epoch": 0.02807881773399015,
      "grad_norm": 0.5429719051894184,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -99413928.85106383,
      "logits/rejected": -94242386.17283951,
      "logps/chosen": -333.9574468085106,
      "logps/rejected": -402.962962962963,
      "loss": 0.3995,
      "rewards/chosen": -0.5103058510638298,
      "rewards/margins": 1.8245706921460467,
      "rewards/rejected": -2.3348765432098766,
      "step": 57
    },
    {
      "epoch": 0.02857142857142857,
      "grad_norm": 0.5006607265210148,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -119957094.4,
      "logits/rejected": -96038806.97435898,
      "logps/chosen": -410.24,
      "logps/rejected": -395.4871794871795,
      "loss": 0.4583,
      "rewards/chosen": 2.1074755859375,
      "rewards/margins": 4.133116611578526,
      "rewards/rejected": -2.0256410256410255,
      "step": 58
    },
    {
      "epoch": 0.029064039408866996,
      "grad_norm": 0.5451032499212339,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -84934656.0,
      "logits/rejected": -103284736.0,
      "logps/chosen": -296.3333333333333,
      "logps/rejected": -428.8,
      "loss": 0.4001,
      "rewards/chosen": 2.1965506871541343,
      "rewards/margins": 4.662175687154134,
      "rewards/rejected": -2.465625,
      "step": 59
    },
    {
      "epoch": 0.029556650246305417,
      "grad_norm": 0.5559742746056191,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -98129237.33333333,
      "logits/rejected": -86402662.4,
      "logps/chosen": -316.3333333333333,
      "logps/rejected": -375.6,
      "loss": 0.4206,
      "rewards/chosen": -0.634765625,
      "rewards/margins": 1.437109375,
      "rewards/rejected": -2.071875,
      "step": 60
    },
    {
      "epoch": 0.030049261083743842,
      "grad_norm": 0.5561508148520591,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -95646579.4509804,
      "logits/rejected": -99682809.35064936,
      "logps/chosen": -343.52941176470586,
      "logps/rejected": -398.54545454545456,
      "loss": 0.4293,
      "rewards/chosen": -0.6211703431372549,
      "rewards/margins": 1.646686799719888,
      "rewards/rejected": -2.267857142857143,
      "step": 61
    },
    {
      "epoch": 0.030541871921182268,
      "grad_norm": 0.4720636694813662,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -89348429.39534883,
      "logits/rejected": -97110473.78823529,
      "logps/chosen": -343.8139534883721,
      "logps/rejected": -383.2470588235294,
      "loss": 0.4145,
      "rewards/chosen": -0.9171511627906976,
      "rewards/margins": 1.2975547195622437,
      "rewards/rejected": -2.2147058823529413,
      "step": 62
    },
    {
      "epoch": 0.03103448275862069,
      "grad_norm": 0.5481966941890423,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -89331316.77192983,
      "logits/rejected": -89143728.67605634,
      "logps/chosen": -339.0877192982456,
      "logps/rejected": -411.0422535211268,
      "loss": 0.4634,
      "rewards/chosen": 1.5696065802323191,
      "rewards/margins": 4.006226298542178,
      "rewards/rejected": -2.436619718309859,
      "step": 63
    },
    {
      "epoch": 0.03152709359605911,
      "grad_norm": 0.5273407928346874,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -96945617.45454545,
      "logits/rejected": -94571568.76190476,
      "logps/chosen": -296.0,
      "logps/rejected": -403.8095238095238,
      "loss": 0.4054,
      "rewards/chosen": 2.437475551258434,
      "rewards/margins": 4.666642217925101,
      "rewards/rejected": -2.2291666666666665,
      "step": 64
    },
    {
      "epoch": 0.03201970443349754,
      "grad_norm": 0.5509554198796726,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -85546325.33333333,
      "logits/rejected": -89443532.8,
      "logps/chosen": -347.0,
      "logps/rejected": -407.6,
      "loss": 0.4092,
      "rewards/chosen": 2.3254515329996743,
      "rewards/margins": 4.731701532999674,
      "rewards/rejected": -2.40625,
      "step": 65
    },
    {
      "epoch": 0.03251231527093596,
      "grad_norm": 0.6330695583600935,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -105556650.66666667,
      "logits/rejected": -105951766.26086956,
      "logps/chosen": -439.55555555555554,
      "logps/rejected": -428.17391304347825,
      "loss": 0.3447,
      "rewards/chosen": -0.6956380208333334,
      "rewards/margins": 1.8097967617753623,
      "rewards/rejected": -2.505434782608696,
      "step": 66
    },
    {
      "epoch": 0.03300492610837438,
      "grad_norm": 0.5720825146139848,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -70640909.4736842,
      "logits/rejected": -94165078.53521127,
      "logps/chosen": -262.3157894736842,
      "logps/rejected": -431.32394366197184,
      "loss": 0.4598,
      "rewards/chosen": -0.5537280701754386,
      "rewards/margins": 1.9110606622189275,
      "rewards/rejected": -2.464788732394366,
      "step": 67
    },
    {
      "epoch": 0.033497536945812804,
      "grad_norm": 0.4472108090062024,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -72089600.0,
      "logits/rejected": -95420416.0,
      "logps/chosen": -247.5,
      "logps/rejected": -494.4,
      "loss": 0.3897,
      "rewards/chosen": -0.6484375,
      "rewards/margins": 1.7546875000000002,
      "rewards/rejected": -2.403125,
      "step": 68
    },
    {
      "epoch": 0.03399014778325123,
      "grad_norm": 0.445522321441771,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -87508433.45454545,
      "logits/rejected": -90876586.66666667,
      "logps/chosen": -302.54545454545456,
      "logps/rejected": -410.6666666666667,
      "loss": 0.3981,
      "rewards/chosen": -0.7137784090909091,
      "rewards/margins": 1.78026920995671,
      "rewards/rejected": -2.494047619047619,
      "step": 69
    },
    {
      "epoch": 0.034482758620689655,
      "grad_norm": 0.5025674189857072,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -99386768.69565217,
      "logits/rejected": -108233503.2195122,
      "logps/chosen": -396.17391304347825,
      "logps/rejected": -474.9268292682927,
      "loss": 0.3993,
      "rewards/chosen": -0.852921195652174,
      "rewards/margins": 2.098298316542948,
      "rewards/rejected": -2.951219512195122,
      "step": 70
    },
    {
      "epoch": 0.034975369458128076,
      "grad_norm": 0.43442581793287155,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -94476697.6,
      "logits/rejected": -104190324.36363636,
      "logps/chosen": -290.6,
      "logps/rejected": -384.0,
      "loss": 0.3737,
      "rewards/chosen": -0.59609375,
      "rewards/margins": 1.77890625,
      "rewards/rejected": -2.375,
      "step": 71
    },
    {
      "epoch": 0.035467980295566505,
      "grad_norm": 0.5445677764684267,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -75726252.21818182,
      "logits/rejected": -93998374.57534246,
      "logps/chosen": -277.8181818181818,
      "logps/rejected": -436.6027397260274,
      "loss": 0.4151,
      "rewards/chosen": 1.823251065340909,
      "rewards/margins": 4.487634626984745,
      "rewards/rejected": -2.664383561643836,
      "step": 72
    },
    {
      "epoch": 0.03596059113300493,
      "grad_norm": 0.6201651982666507,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -79920556.21818182,
      "logits/rejected": -98135222.35616438,
      "logps/chosen": -313.0181818181818,
      "logps/rejected": -427.83561643835617,
      "loss": 0.41,
      "rewards/chosen": 1.836005332253196,
      "rewards/margins": 4.589429989787442,
      "rewards/rejected": -2.7534246575342465,
      "step": 73
    },
    {
      "epoch": 0.03645320197044335,
      "grad_norm": 0.4497273845631509,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -82810617.43589744,
      "logits/rejected": -92463196.04494382,
      "logps/chosen": -320.0,
      "logps/rejected": -446.92134831460675,
      "loss": 0.3495,
      "rewards/chosen": 2.9158262595152245,
      "rewards/margins": 5.609646484234325,
      "rewards/rejected": -2.693820224719101,
      "step": 74
    },
    {
      "epoch": 0.03694581280788178,
      "grad_norm": 0.5054740403389032,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -77944149.33333333,
      "logits/rejected": -94791270.4,
      "logps/chosen": -252.5,
      "logps/rejected": -418.0,
      "loss": 0.3935,
      "rewards/chosen": 2.4113852183024087,
      "rewards/margins": 4.911385218302408,
      "rewards/rejected": -2.5,
      "step": 75
    },
    {
      "epoch": 0.0374384236453202,
      "grad_norm": 0.599703048149177,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -98222973.67272727,
      "logits/rejected": -96756273.09589042,
      "logps/chosen": -356.94545454545454,
      "logps/rejected": -437.041095890411,
      "loss": 0.398,
      "rewards/chosen": -0.4029829545454545,
      "rewards/margins": 2.4771540317559153,
      "rewards/rejected": -2.8801369863013697,
      "step": 76
    },
    {
      "epoch": 0.03793103448275862,
      "grad_norm": 0.5297717313967738,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -89774237.53846154,
      "logits/rejected": -111921690.94736843,
      "logps/chosen": -320.3076923076923,
      "logps/rejected": -453.05263157894734,
      "loss": 0.4026,
      "rewards/chosen": 1.8797215681809645,
      "rewards/margins": 4.488274199759912,
      "rewards/rejected": -2.6085526315789473,
      "step": 77
    },
    {
      "epoch": 0.03842364532019704,
      "grad_norm": 0.48233982376989754,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -93936582.03773585,
      "logits/rejected": -90037725.86666666,
      "logps/chosen": -253.58490566037736,
      "logps/rejected": -402.3466666666667,
      "loss": 0.3942,
      "rewards/chosen": -0.30638266509433965,
      "rewards/margins": 2.1636173349056604,
      "rewards/rejected": -2.47,
      "step": 78
    },
    {
      "epoch": 0.03891625615763547,
      "grad_norm": 0.4656744743329345,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -106779989.33333333,
      "logits/rejected": -101711872.0,
      "logps/chosen": -339.3333333333333,
      "logps/rejected": -476.8,
      "loss": 0.3656,
      "rewards/chosen": 5.447848002115886,
      "rewards/margins": 8.172848002115886,
      "rewards/rejected": -2.725,
      "step": 79
    },
    {
      "epoch": 0.03940886699507389,
      "grad_norm": 0.4932205695461123,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -94371840.0,
      "logits/rejected": -104061213.16455697,
      "logps/chosen": -326.2040816326531,
      "logps/rejected": -483.24050632911394,
      "loss": 0.4167,
      "rewards/chosen": 1.8787689208984375,
      "rewards/margins": 4.771173984189577,
      "rewards/rejected": -2.892405063291139,
      "step": 80
    },
    {
      "epoch": 0.039901477832512314,
      "grad_norm": 0.4896810345752916,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -108745003.70731707,
      "logits/rejected": -117440512.0,
      "logps/chosen": -342.0487804878049,
      "logps/rejected": -498.02298850574715,
      "loss": 0.3458,
      "rewards/chosen": 2.422771732981612,
      "rewards/margins": 5.172771732981612,
      "rewards/rejected": -2.75,
      "step": 81
    },
    {
      "epoch": 0.04039408866995074,
      "grad_norm": 0.44843074531829613,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -82636712.85106383,
      "logits/rejected": -101284674.37037037,
      "logps/chosen": -267.9148936170213,
      "logps/rejected": -472.8888888888889,
      "loss": 0.3575,
      "rewards/chosen": 2.9463861343708446,
      "rewards/margins": 5.813670084988129,
      "rewards/rejected": -2.867283950617284,
      "step": 82
    },
    {
      "epoch": 0.040886699507389164,
      "grad_norm": 0.47812313937054174,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -88901008.69565217,
      "logits/rejected": -94422990.04878049,
      "logps/chosen": -269.39130434782606,
      "logps/rejected": -423.4146341463415,
      "loss": 0.3582,
      "rewards/chosen": 2.418779456097147,
      "rewards/margins": 5.310547748780074,
      "rewards/rejected": -2.8917682926829267,
      "step": 83
    },
    {
      "epoch": 0.041379310344827586,
      "grad_norm": 0.4613572479837589,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -85660593.23076923,
      "logits/rejected": -111031238.47191012,
      "logps/chosen": -287.1794871794872,
      "logps/rejected": -382.92134831460675,
      "loss": 0.3416,
      "rewards/chosen": 2.7370992807241588,
      "rewards/margins": 3.1166342271229475,
      "rewards/rejected": -0.3795349463987886,
      "step": 84
    },
    {
      "epoch": 0.04187192118226601,
      "grad_norm": 0.8842816394578003,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -98368299.47169812,
      "logits/rejected": -86682282.66666667,
      "logps/chosen": -357.1320754716981,
      "logps/rejected": -416.0,
      "loss": 0.3931,
      "rewards/chosen": 7.28466105910967,
      "rewards/margins": 10.09466105910967,
      "rewards/rejected": -2.81,
      "step": 85
    },
    {
      "epoch": 0.042364532019704436,
      "grad_norm": 0.5245494057824104,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -96984081.96491228,
      "logits/rejected": -91270418.02816902,
      "logps/chosen": -353.4035087719298,
      "logps/rejected": -372.7323943661972,
      "loss": 0.4543,
      "rewards/chosen": -0.6225328947368421,
      "rewards/margins": 1.884509358784285,
      "rewards/rejected": -2.507042253521127,
      "step": 86
    },
    {
      "epoch": 0.04285714285714286,
      "grad_norm": 0.4270581172518662,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -76479117.61702128,
      "logits/rejected": -103045246.41975309,
      "logps/chosen": -301.6170212765957,
      "logps/rejected": -436.9382716049383,
      "loss": 0.3591,
      "rewards/chosen": 1.774014412088597,
      "rewards/margins": 4.832656387397239,
      "rewards/rejected": -3.058641975308642,
      "step": 87
    },
    {
      "epoch": 0.04334975369458128,
      "grad_norm": 0.4410532179319701,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -95698609.63265306,
      "logits/rejected": -85478853.67088607,
      "logps/chosen": -298.44897959183675,
      "logps/rejected": -444.75949367088606,
      "loss": 0.3837,
      "rewards/chosen": -0.6044323979591837,
      "rewards/margins": 2.123415703306639,
      "rewards/rejected": -2.7278481012658227,
      "step": 88
    },
    {
      "epoch": 0.04384236453201971,
      "grad_norm": 0.6151829361767176,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -88167765.33333333,
      "logits/rejected": -84620083.2,
      "logps/chosen": -251.66666666666666,
      "logps/rejected": -426.4,
      "loss": 0.3416,
      "rewards/chosen": -0.11336263020833333,
      "rewards/margins": 2.7991373697916666,
      "rewards/rejected": -2.9125,
      "step": 89
    },
    {
      "epoch": 0.04433497536945813,
      "grad_norm": 0.4007597899987483,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -70638217.36585365,
      "logits/rejected": -87646490.48275863,
      "logps/chosen": -252.6829268292683,
      "logps/rejected": -421.14942528735634,
      "loss": 0.3273,
      "rewards/chosen": 3.083173054020579,
      "rewards/margins": 6.091793743675751,
      "rewards/rejected": -3.0086206896551726,
      "step": 90
    },
    {
      "epoch": 0.04482758620689655,
      "grad_norm": 0.4605796014706463,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -92023029.76,
      "logits/rejected": -86467190.15384616,
      "logps/chosen": -306.88,
      "logps/rejected": -374.56410256410254,
      "loss": 0.3882,
      "rewards/chosen": 1.8921554565429688,
      "rewards/margins": 4.517155456542969,
      "rewards/rejected": -2.625,
      "step": 91
    },
    {
      "epoch": 0.04532019704433497,
      "grad_norm": 0.48085235435750817,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -81222130.16216215,
      "logits/rejected": -90891950.41758242,
      "logps/chosen": -225.0810810810811,
      "logps/rejected": -379.7802197802198,
      "loss": 0.3452,
      "rewards/chosen": 3.5184948895428634,
      "rewards/margins": 6.1998135708615445,
      "rewards/rejected": -2.681318681318681,
      "step": 92
    },
    {
      "epoch": 0.0458128078817734,
      "grad_norm": 0.48145538166447344,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -90296242.71698113,
      "logits/rejected": -82655750.82666667,
      "logps/chosen": -349.8867924528302,
      "logps/rejected": -400.64,
      "loss": 0.3652,
      "rewards/chosen": 2.3613597941848465,
      "rewards/margins": 5.401359794184847,
      "rewards/rejected": -3.04,
      "step": 93
    },
    {
      "epoch": 0.04630541871921182,
      "grad_norm": 0.4697493709206804,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -83886080.0,
      "logits/rejected": -96468992.0,
      "logps/chosen": -211.71428571428572,
      "logps/rejected": -414.13953488372096,
      "loss": 0.3107,
      "rewards/chosen": 3.6178835914248513,
      "rewards/margins": 6.68474405654113,
      "rewards/rejected": -3.066860465116279,
      "step": 94
    },
    {
      "epoch": 0.046798029556650245,
      "grad_norm": 0.42217225371662637,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -82742178.9090909,
      "logits/rejected": -88479841.52380952,
      "logps/chosen": -318.54545454545456,
      "logps/rejected": -378.2857142857143,
      "loss": 0.366,
      "rewards/chosen": 5.030813043767756,
      "rewards/margins": 7.980217805672518,
      "rewards/rejected": -2.949404761904762,
      "step": 95
    },
    {
      "epoch": 0.04729064039408867,
      "grad_norm": 0.45104986519223395,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -87757745.23076923,
      "logits/rejected": -94813345.68421052,
      "logps/chosen": -275.38461538461536,
      "logps/rejected": -413.89473684210526,
      "loss": 0.3767,
      "rewards/chosen": 2.1972186748798075,
      "rewards/margins": 5.039323938037702,
      "rewards/rejected": -2.8421052631578947,
      "step": 96
    },
    {
      "epoch": 0.047783251231527095,
      "grad_norm": 0.4422736145180365,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -90876586.66666667,
      "logits/rejected": -83361792.0,
      "logps/chosen": -304.0,
      "logps/rejected": -409.2,
      "loss": 0.3635,
      "rewards/chosen": -0.7023111979166666,
      "rewards/margins": 2.4976888020833337,
      "rewards/rejected": -3.2,
      "step": 97
    },
    {
      "epoch": 0.04827586206896552,
      "grad_norm": 0.5145440996994145,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -79178187.75510204,
      "logits/rejected": -128059003.1392405,
      "logps/chosen": -291.265306122449,
      "logps/rejected": -483.6455696202532,
      "loss": 0.3506,
      "rewards/chosen": 2.3606041110291773,
      "rewards/margins": 5.629591452801329,
      "rewards/rejected": -3.268987341772152,
      "step": 98
    },
    {
      "epoch": 0.04876847290640394,
      "grad_norm": 0.45249099432141876,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -103945794.7826087,
      "logits/rejected": -95241390.82926829,
      "logps/chosen": -325.2173913043478,
      "logps/rejected": -464.390243902439,
      "loss": 0.3622,
      "rewards/chosen": -0.44548233695652173,
      "rewards/margins": 2.749639614262991,
      "rewards/rejected": -3.1951219512195124,
      "step": 99
    },
    {
      "epoch": 0.04926108374384237,
      "grad_norm": 0.4377256144851661,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -90511173.81818181,
      "logits/rejected": -100563431.61904761,
      "logps/chosen": -318.6363636363636,
      "logps/rejected": -472.76190476190476,
      "loss": 0.3101,
      "rewards/chosen": -0.3631036931818182,
      "rewards/margins": 3.363086783008658,
      "rewards/rejected": -3.7261904761904763,
      "step": 100
    },
    {
      "epoch": 0.04975369458128079,
      "grad_norm": 0.4251841243662302,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -92365868.52173913,
      "logits/rejected": -103016198.24390244,
      "logps/chosen": -328.69565217391306,
      "logps/rejected": -412.4878048780488,
      "loss": 0.3428,
      "rewards/chosen": 6.120626698369565,
      "rewards/margins": 9.471236454467126,
      "rewards/rejected": -3.350609756097561,
      "step": 101
    },
    {
      "epoch": 0.05024630541871921,
      "grad_norm": 0.4689931552167608,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -83361792.0,
      "logits/rejected": -97307852.8,
      "logps/chosen": -292.0,
      "logps/rejected": -457.6,
      "loss": 0.3615,
      "rewards/chosen": -0.572265625,
      "rewards/margins": 2.699609375,
      "rewards/rejected": -3.271875,
      "step": 102
    },
    {
      "epoch": 0.05073891625615764,
      "grad_norm": 0.5305416383192328,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -83960978.28571428,
      "logits/rejected": -82837504.0,
      "logps/chosen": -287.42857142857144,
      "logps/rejected": -404.0,
      "loss": 0.3903,
      "rewards/chosen": -0.34095982142857145,
      "rewards/margins": 0.542695090884254,
      "rewards/rejected": -0.8836549123128256,
      "step": 103
    },
    {
      "epoch": 0.05123152709359606,
      "grad_norm": 0.5689595473041731,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -104358278.09523809,
      "logits/rejected": -93640275.34883721,
      "logps/chosen": -415.6190476190476,
      "logps/rejected": -389.95348837209303,
      "loss": 0.3194,
      "rewards/chosen": 6.805296398344494,
      "rewards/margins": 9.973901049507283,
      "rewards/rejected": -3.1686046511627906,
      "step": 104
    },
    {
      "epoch": 0.05172413793103448,
      "grad_norm": 0.43715904838696906,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -106867370.66666667,
      "logits/rejected": -91855257.6,
      "logps/chosen": -396.6666666666667,
      "logps/rejected": -412.8,
      "loss": 0.3593,
      "rewards/chosen": -0.328125,
      "rewards/margins": 2.840625,
      "rewards/rejected": -3.16875,
      "step": 105
    },
    {
      "epoch": 0.052216748768472904,
      "grad_norm": 0.4529292695378069,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -88080384.0,
      "logits/rejected": -90665913.8630137,
      "logps/chosen": -308.3636363636364,
      "logps/rejected": -422.5753424657534,
      "loss": 0.3589,
      "rewards/chosen": -0.16235795454545454,
      "rewards/margins": 3.3958612235367376,
      "rewards/rejected": -3.558219178082192,
      "step": 106
    },
    {
      "epoch": 0.05270935960591133,
      "grad_norm": 0.41034030383336506,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -92798976.0,
      "logits/rejected": -85668659.2,
      "logps/chosen": -311.0,
      "logps/rejected": -421.6,
      "loss": 0.3524,
      "rewards/chosen": -0.3053385416666667,
      "rewards/margins": 3.0352864583333337,
      "rewards/rejected": -3.340625,
      "step": 107
    },
    {
      "epoch": 0.053201970443349754,
      "grad_norm": 0.46303257678557386,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -83886080.0,
      "logits/rejected": -83785012.43373494,
      "logps/chosen": -282.6666666666667,
      "logps/rejected": -438.7469879518072,
      "loss": 0.3481,
      "rewards/chosen": 2.907855902777778,
      "rewards/margins": 6.070506505187416,
      "rewards/rejected": -3.1626506024096384,
      "step": 108
    },
    {
      "epoch": 0.053694581280788176,
      "grad_norm": 0.3878300173471027,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -84064561.0212766,
      "logits/rejected": -86889408.79012346,
      "logps/chosen": -344.51063829787233,
      "logps/rejected": -431.01234567901236,
      "loss": 0.3303,
      "rewards/chosen": -0.5322473404255319,
      "rewards/margins": 2.936888462043604,
      "rewards/rejected": -3.4691358024691357,
      "step": 109
    },
    {
      "epoch": 0.054187192118226604,
      "grad_norm": 0.4686839871836073,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -91963998.81481482,
      "logits/rejected": -99983138.5945946,
      "logps/chosen": -280.2962962962963,
      "logps/rejected": -403.02702702702703,
      "loss": 0.4029,
      "rewards/chosen": -0.7326388888888888,
      "rewards/margins": 2.8247935435435436,
      "rewards/rejected": -3.5574324324324325,
      "step": 110
    },
    {
      "epoch": 0.054679802955665026,
      "grad_norm": 0.4219734060718876,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -85468836.2264151,
      "logits/rejected": -88024459.94666667,
      "logps/chosen": -268.6792452830189,
      "logps/rejected": -422.4,
      "loss": 0.3666,
      "rewards/chosen": -0.3912146226415094,
      "rewards/margins": 3.162118710691824,
      "rewards/rejected": -3.5533333333333332,
      "step": 111
    },
    {
      "epoch": 0.05517241379310345,
      "grad_norm": 0.3777637007817232,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -80915114.66666667,
      "logits/rejected": -92169830.4,
      "logps/chosen": -264.6666666666667,
      "logps/rejected": -406.8,
      "loss": 0.3552,
      "rewards/chosen": 2.194291432698568,
      "rewards/margins": 5.397416432698568,
      "rewards/rejected": -3.203125,
      "step": 112
    },
    {
      "epoch": 0.05566502463054187,
      "grad_norm": 0.402146631981609,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -74490839.04,
      "logits/rejected": -85284181.33333333,
      "logps/chosen": -273.92,
      "logps/rejected": -352.4102564102564,
      "loss": 0.3811,
      "rewards/chosen": -0.418046875,
      "rewards/margins": 2.5434915865384617,
      "rewards/rejected": -2.9615384615384617,
      "step": 113
    },
    {
      "epoch": 0.0561576354679803,
      "grad_norm": 0.4971403236388034,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -96179729.65517241,
      "logits/rejected": -95270619.42857143,
      "logps/chosen": -352.82758620689657,
      "logps/rejected": -402.2857142857143,
      "loss": 0.3753,
      "rewards/chosen": 2.1757359997979524,
      "rewards/margins": 5.700735999797953,
      "rewards/rejected": -3.525,
      "step": 114
    },
    {
      "epoch": 0.05665024630541872,
      "grad_norm": 0.49365246596832674,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -84808826.88,
      "logits/rejected": -95070890.66666667,
      "logps/chosen": -287.52,
      "logps/rejected": -406.56410256410254,
      "loss": 0.3737,
      "rewards/chosen": 2.2399830627441406,
      "rewards/margins": 2.243112288254958,
      "rewards/rejected": -0.0031292255108173075,
      "step": 115
    },
    {
      "epoch": 0.05714285714285714,
      "grad_norm": 0.4500225384582829,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -77471262.11764705,
      "logits/rejected": -98048664.93506494,
      "logps/chosen": -297.0980392156863,
      "logps/rejected": -461.7142857142857,
      "loss": 0.3135,
      "rewards/chosen": -0.050551470588235295,
      "rewards/margins": 3.468929048892284,
      "rewards/rejected": -3.5194805194805197,
      "step": 116
    },
    {
      "epoch": 0.05763546798029557,
      "grad_norm": 0.49725912200746214,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -95858911.41818182,
      "logits/rejected": -82851868.05479452,
      "logps/chosen": -355.7818181818182,
      "logps/rejected": -359.013698630137,
      "loss": 0.4062,
      "rewards/chosen": -0.50625,
      "rewards/margins": 2.4081335616438357,
      "rewards/rejected": -2.914383561643836,
      "step": 117
    },
    {
      "epoch": 0.05812807881773399,
      "grad_norm": 0.4369258632181929,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -101974016.0,
      "logits/rejected": -83886080.0,
      "logps/chosen": -217.66666666666666,
      "logps/rejected": -366.8,
      "loss": 0.3385,
      "rewards/chosen": -0.13675944010416666,
      "rewards/margins": 3.0601155598958334,
      "rewards/rejected": -3.196875,
      "step": 118
    },
    {
      "epoch": 0.05862068965517241,
      "grad_norm": 0.5012880440998103,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -93540892.98113208,
      "logits/rejected": -74714535.25333333,
      "logps/chosen": -402.1132075471698,
      "logps/rejected": -399.36,
      "loss": 0.36,
      "rewards/chosen": -0.18985849056603774,
      "rewards/margins": 3.466808176100629,
      "rewards/rejected": -3.6566666666666667,
      "step": 119
    },
    {
      "epoch": 0.059113300492610835,
      "grad_norm": 0.46921543190742365,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -95462359.04,
      "logits/rejected": -104319868.71794872,
      "logps/chosen": -288.32,
      "logps/rejected": -452.5128205128205,
      "loss": 0.3266,
      "rewards/chosen": -0.121875,
      "rewards/margins": 3.692227564102564,
      "rewards/rejected": -3.8141025641025643,
      "step": 120
    },
    {
      "epoch": 0.05960591133004926,
      "grad_norm": 0.36582811550689726,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -107761348.92307693,
      "logits/rejected": -92086179.95505618,
      "logps/chosen": -384.8205128205128,
      "logps/rejected": -431.82022471910113,
      "loss": 0.2843,
      "rewards/chosen": -0.25,
      "rewards/margins": 3.6713483146067416,
      "rewards/rejected": -3.9213483146067416,
      "step": 121
    },
    {
      "epoch": 0.060098522167487685,
      "grad_norm": 0.37720882606630113,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -92363928.5106383,
      "logits/rejected": -90099863.7037037,
      "logps/chosen": -314.21276595744683,
      "logps/rejected": -466.962962962963,
      "loss": 0.3359,
      "rewards/chosen": -0.29970079787234044,
      "rewards/margins": 3.5212868564486475,
      "rewards/rejected": -3.8209876543209877,
      "step": 122
    },
    {
      "epoch": 0.06059113300492611,
      "grad_norm": 0.4742222557614187,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -81565826.72340426,
      "logits/rejected": -92999629.43209876,
      "logps/chosen": -296.0,
      "logps/rejected": -461.4320987654321,
      "loss": 0.3132,
      "rewards/chosen": -0.22377825797872342,
      "rewards/margins": 3.671283470416338,
      "rewards/rejected": -3.8950617283950617,
      "step": 123
    },
    {
      "epoch": 0.061083743842364535,
      "grad_norm": 0.4292837146161033,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -80056498.08695652,
      "logits/rejected": -92070087.80487806,
      "logps/chosen": -338.7826086956522,
      "logps/rejected": -391.8048780487805,
      "loss": 0.345,
      "rewards/chosen": -0.005519701086956522,
      "rewards/margins": 3.308504689156946,
      "rewards/rejected": -3.3140243902439024,
      "step": 124
    },
    {
      "epoch": 0.06157635467980296,
      "grad_norm": 0.49404153301421727,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -127027492.57142857,
      "logits/rejected": -101828380.44444445,
      "logps/chosen": -251.42857142857142,
      "logps/rejected": -444.44444444444446,
      "loss": 0.3569,
      "rewards/chosen": -0.14369419642857142,
      "rewards/margins": 4.064639136904762,
      "rewards/rejected": -4.208333333333333,
      "step": 125
    },
    {
      "epoch": 0.06206896551724138,
      "grad_norm": 0.3926891664082588,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -79447921.11627907,
      "logits/rejected": -109644041.03529412,
      "logps/chosen": -264.93023255813955,
      "logps/rejected": -439.3411764705882,
      "loss": 0.2801,
      "rewards/chosen": 0.15261627906976744,
      "rewards/margins": 4.029086867305062,
      "rewards/rejected": -3.876470588235294,
      "step": 126
    },
    {
      "epoch": 0.06256157635467981,
      "grad_norm": 0.41010245186645045,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -88639624.53333333,
      "logits/rejected": -93689633.92771085,
      "logps/chosen": -361.9555555555556,
      "logps/rejected": -466.89156626506025,
      "loss": 0.3369,
      "rewards/chosen": 2.7208426581488716,
      "rewards/margins": 6.991926995498269,
      "rewards/rejected": -4.271084337349397,
      "step": 127
    },
    {
      "epoch": 0.06305418719211822,
      "grad_norm": 0.43980704132267007,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -78704880.94117647,
      "logits/rejected": -98157607.89610389,
      "logps/chosen": -326.5882352941176,
      "logps/rejected": -457.5584415584416,
      "loss": 0.3152,
      "rewards/chosen": -0.07674632352941177,
      "rewards/margins": 4.351825105042017,
      "rewards/rejected": -4.428571428571429,
      "step": 128
    },
    {
      "epoch": 0.06354679802955665,
      "grad_norm": 0.4673748528331846,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -71878856.78431372,
      "logits/rejected": -93146231.68831168,
      "logps/chosen": -253.80392156862746,
      "logps/rejected": -404.7792207792208,
      "loss": 0.38,
      "rewards/chosen": -0.41942401960784315,
      "rewards/margins": 3.3760305258467023,
      "rewards/rejected": -3.7954545454545454,
      "step": 129
    },
    {
      "epoch": 0.06403940886699508,
      "grad_norm": 0.3718270439691944,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -91636424.3478261,
      "logits/rejected": -102606997.85365854,
      "logps/chosen": -353.39130434782606,
      "logps/rejected": -402.3414634146341,
      "loss": 0.289,
      "rewards/chosen": 5.675248519234035,
      "rewards/margins": 9.69049242167306,
      "rewards/rejected": -4.015243902439025,
      "step": 130
    },
    {
      "epoch": 0.0645320197044335,
      "grad_norm": 0.4304884057998777,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -85843421.86666666,
      "logits/rejected": -83178607.03614458,
      "logps/chosen": -290.4888888888889,
      "logps/rejected": -478.4578313253012,
      "loss": 0.2985,
      "rewards/chosen": 0.12482638888888889,
      "rewards/margins": 4.251332412985274,
      "rewards/rejected": -4.126506024096385,
      "step": 131
    },
    {
      "epoch": 0.06502463054187192,
      "grad_norm": 0.48329133807875063,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -94318066.87179486,
      "logits/rejected": -96139102.92134832,
      "logps/chosen": -336.8205128205128,
      "logps/rejected": -433.2584269662921,
      "loss": 0.2655,
      "rewards/chosen": 4.092061360677083,
      "rewards/margins": 8.26621866404787,
      "rewards/rejected": -4.174157303370786,
      "step": 132
    },
    {
      "epoch": 0.06551724137931035,
      "grad_norm": 0.47826324205269277,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -91587097.18032786,
      "logits/rejected": -90772250.74626866,
      "logps/chosen": -381.11475409836066,
      "logps/rejected": -428.4179104477612,
      "loss": 0.3489,
      "rewards/chosen": 2.255064166960169,
      "rewards/margins": 6.45282536099002,
      "rewards/rejected": -4.197761194029851,
      "step": 133
    },
    {
      "epoch": 0.06600985221674877,
      "grad_norm": 0.4975989229775862,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -106017726.63829787,
      "logits/rejected": -90410552.8888889,
      "logps/chosen": -375.82978723404256,
      "logps/rejected": -393.87654320987656,
      "loss": 0.3328,
      "rewards/chosen": -0.2237367021276596,
      "rewards/margins": 3.6836707052797477,
      "rewards/rejected": -3.9074074074074074,
      "step": 134
    },
    {
      "epoch": 0.0665024630541872,
      "grad_norm": 0.4590176988726528,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -100567970.9090909,
      "logits/rejected": -88180248.38095239,
      "logps/chosen": -291.27272727272725,
      "logps/rejected": -397.7142857142857,
      "loss": 0.2949,
      "rewards/chosen": 0.02556818181818182,
      "rewards/margins": 4.251758658008658,
      "rewards/rejected": -4.226190476190476,
      "step": 135
    },
    {
      "epoch": 0.06699507389162561,
      "grad_norm": 0.49281652614958893,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -105025372.16,
      "logits/rejected": -72486176.82051282,
      "logps/chosen": -310.72,
      "logps/rejected": -362.2564102564103,
      "loss": 0.3344,
      "rewards/chosen": 1.9943649291992187,
      "rewards/margins": 5.949493134327424,
      "rewards/rejected": -3.9551282051282053,
      "step": 136
    },
    {
      "epoch": 0.06748768472906404,
      "grad_norm": 0.43171135675632283,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -72501540.57142857,
      "logits/rejected": -87730858.66666667,
      "logps/chosen": -285.7142857142857,
      "logps/rejected": -418.6666666666667,
      "loss": 0.3722,
      "rewards/chosen": -0.49386160714285715,
      "rewards/margins": 3.6693328373015874,
      "rewards/rejected": -4.163194444444445,
      "step": 137
    },
    {
      "epoch": 0.06798029556650247,
      "grad_norm": 0.42278669675212227,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -76357842.05128205,
      "logits/rejected": -92745958.11235955,
      "logps/chosen": -267.28205128205127,
      "logps/rejected": -437.2134831460674,
      "loss": 0.2612,
      "rewards/chosen": 0.0625,
      "rewards/margins": 4.483848314606742,
      "rewards/rejected": -4.421348314606742,
      "step": 138
    },
    {
      "epoch": 0.06847290640394088,
      "grad_norm": 0.5518155053549193,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -87289005.88679245,
      "logits/rejected": -95070890.66666667,
      "logps/chosen": -313.0566037735849,
      "logps/rejected": -397.6533333333333,
      "loss": 0.3455,
      "rewards/chosen": 2.4187794811320753,
      "rewards/margins": 4.28040423699145,
      "rewards/rejected": -1.861624755859375,
      "step": 139
    },
    {
      "epoch": 0.06896551724137931,
      "grad_norm": 0.3546383864638669,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -112012589.1764706,
      "logits/rejected": -93791776.68085106,
      "logps/chosen": -372.70588235294116,
      "logps/rejected": -445.6170212765957,
      "loss": 0.2598,
      "rewards/chosen": 11.872976864085478,
      "rewards/margins": 16.809147076851435,
      "rewards/rejected": -4.9361702127659575,
      "step": 140
    },
    {
      "epoch": 0.06945812807881774,
      "grad_norm": 0.4520474916921425,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -94197077.33333333,
      "logits/rejected": -104438169.6,
      "logps/chosen": -304.3333333333333,
      "logps/rejected": -474.4,
      "loss": 0.2904,
      "rewards/chosen": 0.00537109375,
      "rewards/margins": 3.91162109375,
      "rewards/rejected": -3.90625,
      "step": 141
    },
    {
      "epoch": 0.06995073891625615,
      "grad_norm": 0.4791833873093016,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -75780870.91891892,
      "logits/rejected": -84900087.56043956,
      "logps/chosen": -282.5945945945946,
      "logps/rejected": -429.7142857142857,
      "loss": 0.2742,
      "rewards/chosen": 12.897696830130911,
      "rewards/margins": 17.018575951010032,
      "rewards/rejected": -4.1208791208791204,
      "step": 142
    },
    {
      "epoch": 0.07044334975369458,
      "grad_norm": 0.5006685890953549,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -82609552.69565217,
      "logits/rejected": -82965379.12195122,
      "logps/chosen": -306.60869565217394,
      "logps/rejected": -395.3170731707317,
      "loss": 0.3018,
      "rewards/chosen": -0.10054347826086957,
      "rewards/margins": 4.442139448568399,
      "rewards/rejected": -4.5426829268292686,
      "step": 143
    },
    {
      "epoch": 0.07093596059113301,
      "grad_norm": 0.5415002678367673,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -93000625.23076923,
      "logits/rejected": -83996456.42105263,
      "logps/chosen": -368.15384615384613,
      "logps/rejected": -396.63157894736844,
      "loss": 0.3218,
      "rewards/chosen": 5.58347907433143,
      "rewards/margins": 9.51440012696301,
      "rewards/rejected": -3.9309210526315788,
      "step": 144
    },
    {
      "epoch": 0.07142857142857142,
      "grad_norm": 0.5323923271473462,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -78821260.0754717,
      "logits/rejected": -80754333.01333334,
      "logps/chosen": -273.35849056603774,
      "logps/rejected": -415.14666666666665,
      "loss": 0.343,
      "rewards/chosen": 0.0660377358490566,
      "rewards/margins": 2.7501881264740566,
      "rewards/rejected": -2.684150390625,
      "step": 145
    },
    {
      "epoch": 0.07192118226600985,
      "grad_norm": 0.5335841891983689,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -63295860.36363637,
      "logits/rejected": -105456786.28571428,
      "logps/chosen": -301.6363636363636,
      "logps/rejected": -485.3333333333333,
      "loss": 0.2646,
      "rewards/chosen": 0.12198153409090909,
      "rewards/margins": 5.794600581709957,
      "rewards/rejected": -5.6726190476190474,
      "step": 146
    },
    {
      "epoch": 0.07241379310344828,
      "grad_norm": 0.4385595843131795,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -76412592.87272727,
      "logits/rejected": -86873803.39726028,
      "logps/chosen": -244.94545454545454,
      "logps/rejected": -421.26027397260276,
      "loss": 0.3356,
      "rewards/chosen": 0.15227272727272728,
      "rewards/margins": 5.090628891656289,
      "rewards/rejected": -4.938356164383562,
      "step": 147
    },
    {
      "epoch": 0.0729064039408867,
      "grad_norm": 0.44513991390458224,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -68817654.51851852,
      "logits/rejected": -96809070.7027027,
      "logps/chosen": -217.77777777777777,
      "logps/rejected": -495.56756756756755,
      "loss": 0.3077,
      "rewards/chosen": 0.20493344907407407,
      "rewards/margins": 2.489078494998905,
      "rewards/rejected": -2.284145045924831,
      "step": 148
    },
    {
      "epoch": 0.07339901477832513,
      "grad_norm": 0.49931498703866184,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -84626251.29411764,
      "logits/rejected": -102515326.33766234,
      "logps/chosen": -184.7843137254902,
      "logps/rejected": -455.06493506493507,
      "loss": 0.3079,
      "rewards/chosen": 2.0849872663909315,
      "rewards/margins": 6.974597656001321,
      "rewards/rejected": -4.8896103896103895,
      "step": 149
    },
    {
      "epoch": 0.07389162561576355,
      "grad_norm": 0.5132856042074379,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -89633829.92592593,
      "logits/rejected": -101570172.54054055,
      "logps/chosen": -370.3703703703704,
      "logps/rejected": -447.56756756756755,
      "loss": 0.3596,
      "rewards/chosen": -0.4626736111111111,
      "rewards/margins": 4.537326388888889,
      "rewards/rejected": -5.0,
      "step": 150
    },
    {
      "epoch": 0.07438423645320197,
      "grad_norm": 0.4597720111952633,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -102713844.62222221,
      "logits/rejected": -90253336.6746988,
      "logps/chosen": -355.2,
      "logps/rejected": -427.95180722891564,
      "loss": 0.2838,
      "rewards/chosen": 2.7486070421006943,
      "rewards/margins": 8.049811861377803,
      "rewards/rejected": -5.301204819277109,
      "step": 151
    },
    {
      "epoch": 0.0748768472906404,
      "grad_norm": 0.4572870850189307,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -85135447.14893617,
      "logits/rejected": -92378251.06172839,
      "logps/chosen": -414.29787234042556,
      "logps/rejected": -427.85185185185185,
      "loss": 0.3028,
      "rewards/chosen": -0.6617353723404256,
      "rewards/margins": 4.171597960992908,
      "rewards/rejected": -4.833333333333333,
      "step": 152
    },
    {
      "epoch": 0.07536945812807881,
      "grad_norm": 0.5268703885761351,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -71846874.07407407,
      "logits/rejected": -83999439.56756757,
      "logps/chosen": -255.55555555555554,
      "logps/rejected": -436.3243243243243,
      "loss": 0.3546,
      "rewards/chosen": -0.4752604166666667,
      "rewards/margins": 4.9571720157657655,
      "rewards/rejected": -5.4324324324324325,
      "step": 153
    },
    {
      "epoch": 0.07586206896551724,
      "grad_norm": 0.39202839145130397,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -74960991.25581396,
      "logits/rejected": -104512186.72941177,
      "logps/chosen": -320.3720930232558,
      "logps/rejected": -508.2352941176471,
      "loss": 0.2327,
      "rewards/chosen": 2.919940150061319,
      "rewards/margins": 8.98464603241426,
      "rewards/rejected": -6.064705882352941,
      "step": 154
    },
    {
      "epoch": 0.07635467980295567,
      "grad_norm": 0.4872856855558592,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -100570089.24444444,
      "logits/rejected": -93083228.53012048,
      "logps/chosen": -271.46666666666664,
      "logps/rejected": -434.50602409638554,
      "loss": 0.2919,
      "rewards/chosen": 2.4159437391493057,
      "rewards/margins": 7.102690727101113,
      "rewards/rejected": -4.686746987951807,
      "step": 155
    },
    {
      "epoch": 0.07684729064039408,
      "grad_norm": 0.5086954431408451,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -87996497.92,
      "logits/rejected": -106255701.33333333,
      "logps/chosen": -339.84,
      "logps/rejected": -487.38461538461536,
      "loss": 0.339,
      "rewards/chosen": 4.1130810546875,
      "rewards/margins": 9.856670798277243,
      "rewards/rejected": -5.743589743589744,
      "step": 156
    },
    {
      "epoch": 0.07733990147783251,
      "grad_norm": 0.6635727012208371,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -73126778.43478261,
      "logits/rejected": -91047086.82926829,
      "logps/chosen": -347.4782608695652,
      "logps/rejected": -388.6829268292683,
      "loss": 0.3423,
      "rewards/chosen": 4.797486014988111,
      "rewards/margins": 9.858461624744209,
      "rewards/rejected": -5.060975609756097,
      "step": 157
    },
    {
      "epoch": 0.07783251231527094,
      "grad_norm": 0.45553919406189447,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -74239180.8,
      "logits/rejected": -103567044.92307693,
      "logps/chosen": -225.28,
      "logps/rejected": -474.2564102564103,
      "loss": 0.2998,
      "rewards/chosen": 2.2293516540527345,
      "rewards/margins": 7.966531141232222,
      "rewards/rejected": -5.737179487179487,
      "step": 158
    },
    {
      "epoch": 0.07832512315270936,
      "grad_norm": 0.47076742141605366,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -85337954.46153846,
      "logits/rejected": -86645490.5263158,
      "logps/chosen": -356.61538461538464,
      "logps/rejected": -506.94736842105266,
      "loss": 0.3232,
      "rewards/chosen": -0.11899038461538461,
      "rewards/margins": 5.657325404858299,
      "rewards/rejected": -5.776315789473684,
      "step": 159
    },
    {
      "epoch": 0.07881773399014778,
      "grad_norm": 0.5360682239054377,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -83065455.30434783,
      "logits/rejected": -86341282.34146342,
      "logps/chosen": -356.5217391304348,
      "logps/rejected": -469.0731707317073,
      "loss": 0.2746,
      "rewards/chosen": 0.29245923913043476,
      "rewards/margins": 5.774166556203605,
      "rewards/rejected": -5.4817073170731705,
      "step": 160
    },
    {
      "epoch": 0.07931034482758621,
      "grad_norm": 0.6368435418411478,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -78320561.23076923,
      "logits/rejected": -96468992.0,
      "logps/chosen": -283.6923076923077,
      "logps/rejected": -502.7368421052632,
      "loss": 0.3133,
      "rewards/chosen": -0.6487379807692307,
      "rewards/margins": 5.9104725455465585,
      "rewards/rejected": -6.559210526315789,
      "step": 161
    },
    {
      "epoch": 0.07980295566502463,
      "grad_norm": 0.4034544177213964,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -77694488.38095239,
      "logits/rejected": -82032782.88372093,
      "logps/chosen": -309.0952380952381,
      "logps/rejected": -427.16279069767444,
      "loss": 0.2655,
      "rewards/chosen": 2.21535401117234,
      "rewards/margins": 8.040935406521177,
      "rewards/rejected": -5.825581395348837,
      "step": 162
    },
    {
      "epoch": 0.08029556650246306,
      "grad_norm": 0.5106496866229862,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -69457674.24,
      "logits/rejected": -93780335.58974358,
      "logps/chosen": -280.32,
      "logps/rejected": -460.71794871794873,
      "loss": 0.304,
      "rewards/chosen": -0.295,
      "rewards/margins": 6.435769230769231,
      "rewards/rejected": -6.730769230769231,
      "step": 163
    },
    {
      "epoch": 0.08078817733990148,
      "grad_norm": 0.5223366963691325,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -82627788.8,
      "logits/rejected": -86037005.12820514,
      "logps/chosen": -282.24,
      "logps/rejected": -456.61538461538464,
      "loss": 0.3029,
      "rewards/chosen": -0.98484375,
      "rewards/margins": 4.861310096153845,
      "rewards/rejected": -5.846153846153846,
      "step": 164
    },
    {
      "epoch": 0.0812807881773399,
      "grad_norm": 0.6879738910035819,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -73686295.27272727,
      "logits/rejected": -95170755.04761904,
      "logps/chosen": -286.1818181818182,
      "logps/rejected": -465.5238095238095,
      "loss": 0.2924,
      "rewards/chosen": 5.615004106001421,
      "rewards/margins": 10.870956486953801,
      "rewards/rejected": -5.255952380952381,
      "step": 165
    },
    {
      "epoch": 0.08177339901477833,
      "grad_norm": 0.49977965523884904,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -82153650.08695652,
      "logits/rejected": -93604589.26829268,
      "logps/chosen": -354.0869565217391,
      "logps/rejected": -497.9512195121951,
      "loss": 0.2465,
      "rewards/chosen": 0.1358695652173913,
      "rewards/margins": 7.068796394485684,
      "rewards/rejected": -6.932926829268292,
      "step": 166
    },
    {
      "epoch": 0.08226600985221674,
      "grad_norm": 0.4491311873689934,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -86882011.42857143,
      "logits/rejected": -95354050.43037975,
      "logps/chosen": -369.6326530612245,
      "logps/rejected": -409.5189873417722,
      "loss": 0.3048,
      "rewards/chosen": -0.4499362244897959,
      "rewards/margins": 3.7968992185481794,
      "rewards/rejected": -4.246835443037975,
      "step": 167
    },
    {
      "epoch": 0.08275862068965517,
      "grad_norm": 0.46621923932255205,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -89178892.1904762,
      "logits/rejected": -80179485.76744185,
      "logps/chosen": -331.04761904761904,
      "logps/rejected": -400.0,
      "loss": 0.237,
      "rewards/chosen": 3.1794796898251487,
      "rewards/margins": 8.8771541084298,
      "rewards/rejected": -5.6976744186046515,
      "step": 168
    },
    {
      "epoch": 0.0832512315270936,
      "grad_norm": 0.5350679305871537,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -94867530.47272727,
      "logits/rejected": -79289582.46575342,
      "logps/chosen": -385.74545454545455,
      "logps/rejected": -460.71232876712327,
      "loss": 0.3336,
      "rewards/chosen": -0.8909090909090909,
      "rewards/margins": 5.081693648816937,
      "rewards/rejected": -5.972602739726027,
      "step": 169
    },
    {
      "epoch": 0.08374384236453201,
      "grad_norm": 0.5147267683595977,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -83886080.0,
      "logits/rejected": -82732646.4,
      "logps/chosen": -307.6666666666667,
      "logps/rejected": -455.6,
      "loss": 0.2711,
      "rewards/chosen": 0.23299153645833334,
      "rewards/margins": 6.114241536458333,
      "rewards/rejected": -5.88125,
      "step": 170
    },
    {
      "epoch": 0.08423645320197044,
      "grad_norm": 0.440337117354319,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -82543902.72,
      "logits/rejected": -85714366.35897435,
      "logps/chosen": -397.44,
      "logps/rejected": -483.6923076923077,
      "loss": 0.2557,
      "rewards/chosen": 6.273861694335937,
      "rewards/margins": 12.177707848182092,
      "rewards/rejected": -5.903846153846154,
      "step": 171
    },
    {
      "epoch": 0.08472906403940887,
      "grad_norm": 0.5116310261805157,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -68634065.45454545,
      "logits/rejected": -89114595.94520548,
      "logps/chosen": -280.72727272727275,
      "logps/rejected": -464.2191780821918,
      "loss": 0.3491,
      "rewards/chosen": -0.7221590909090909,
      "rewards/margins": 3.414180070584916,
      "rewards/rejected": -4.136339161494007,
      "step": 172
    },
    {
      "epoch": 0.08522167487684729,
      "grad_norm": 0.4710547174638074,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -67360522.24,
      "logits/rejected": -94425613.12820514,
      "logps/chosen": -266.88,
      "logps/rejected": -450.87179487179486,
      "loss": 0.3005,
      "rewards/chosen": 2.150704345703125,
      "rewards/margins": 7.054550499549279,
      "rewards/rejected": -4.903846153846154,
      "step": 173
    },
    {
      "epoch": 0.08571428571428572,
      "grad_norm": 0.4999460172858985,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -73972270.54545455,
      "logits/rejected": -88579705.90476191,
      "logps/chosen": -214.36363636363637,
      "logps/rejected": -423.23809523809524,
      "loss": 0.2654,
      "rewards/chosen": 2.724389509721236,
      "rewards/margins": 8.022008557340284,
      "rewards/rejected": -5.2976190476190474,
      "step": 174
    },
    {
      "epoch": 0.08620689655172414,
      "grad_norm": 0.4985034059123771,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -105313502.60869566,
      "logits/rejected": -83067679.2195122,
      "logps/chosen": -301.2173913043478,
      "logps/rejected": -432.390243902439,
      "loss": 0.2661,
      "rewards/chosen": -0.5197010869565217,
      "rewards/margins": 5.108347693531283,
      "rewards/rejected": -5.628048780487805,
      "step": 175
    },
    {
      "epoch": 0.08669950738916256,
      "grad_norm": 0.5483669178099033,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -89841991.68,
      "logits/rejected": -87865291.48717949,
      "logps/chosen": -276.16,
      "logps/rejected": -495.1794871794872,
      "loss": 0.3182,
      "rewards/chosen": 1.988096923828125,
      "rewards/margins": 8.481686667417868,
      "rewards/rejected": -6.493589743589744,
      "step": 176
    },
    {
      "epoch": 0.08719211822660099,
      "grad_norm": 0.475616686744915,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -72778941.62962963,
      "logits/rejected": -77878022.91891892,
      "logps/chosen": -254.8148148148148,
      "logps/rejected": -396.5405405405405,
      "loss": 0.3307,
      "rewards/chosen": -0.3237847222222222,
      "rewards/margins": 4.845134196696697,
      "rewards/rejected": -5.168918918918919,
      "step": 177
    },
    {
      "epoch": 0.08768472906403942,
      "grad_norm": 0.4841533872470013,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -81788928.0,
      "logits/rejected": -83047219.2,
      "logps/chosen": -249.93103448275863,
      "logps/rejected": -402.2857142857143,
      "loss": 0.346,
      "rewards/chosen": -0.19019396551724138,
      "rewards/margins": 4.563377463054188,
      "rewards/rejected": -4.753571428571429,
      "step": 178
    },
    {
      "epoch": 0.08817733990147783,
      "grad_norm": 0.43656628358298055,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -73765042.08695652,
      "logits/rejected": -90535586.34146342,
      "logps/chosen": -272.3478260869565,
      "logps/rejected": -435.5121951219512,
      "loss": 0.2649,
      "rewards/chosen": 6.3160572881283965,
      "rewards/margins": 12.206301190567421,
      "rewards/rejected": -5.890243902439025,
      "step": 179
    },
    {
      "epoch": 0.08866995073891626,
      "grad_norm": 0.48443406097827024,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -83321462.15384616,
      "logits/rejected": -90067159.57894737,
      "logps/chosen": -275.38461538461536,
      "logps/rejected": -419.7894736842105,
      "loss": 0.2995,
      "rewards/chosen": -0.8499474158653846,
      "rewards/margins": 4.965842057818826,
      "rewards/rejected": -5.815789473684211,
      "step": 180
    },
    {
      "epoch": 0.08916256157635467,
      "grad_norm": 0.676151855899301,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -96057785.7254902,
      "logits/rejected": -71629996.88311689,
      "logps/chosen": -340.70588235294116,
      "logps/rejected": -426.38961038961037,
      "loss": 0.3429,
      "rewards/chosen": -0.4145986519607843,
      "rewards/margins": 4.572414335052202,
      "rewards/rejected": -4.987012987012987,
      "step": 181
    },
    {
      "epoch": 0.0896551724137931,
      "grad_norm": 0.41170662951160203,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -86099740.44444445,
      "logits/rejected": -72493443.45945945,
      "logps/chosen": -217.4814814814815,
      "logps/rejected": -373.6216216216216,
      "loss": 0.2912,
      "rewards/chosen": 0.08940972222222222,
      "rewards/margins": 5.3731935060060065,
      "rewards/rejected": -5.283783783783784,
      "step": 182
    },
    {
      "epoch": 0.09014778325123153,
      "grad_norm": 0.4241515599955532,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -85942111.37254901,
      "logits/rejected": -90967372.46753247,
      "logps/chosen": -350.4313725490196,
      "logps/rejected": -462.961038961039,
      "loss": 0.3059,
      "rewards/chosen": 2.2635070202397367,
      "rewards/margins": 9.399870656603373,
      "rewards/rejected": -7.136363636363637,
      "step": 183
    },
    {
      "epoch": 0.09064039408866995,
      "grad_norm": 0.4569268317902183,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -94977683.91111112,
      "logits/rejected": -82572201.63855422,
      "logps/chosen": -357.3333333333333,
      "logps/rejected": -423.71084337349396,
      "loss": 0.2568,
      "rewards/chosen": 3.3353695339626737,
      "rewards/margins": 9.666694835167492,
      "rewards/rejected": -6.331325301204819,
      "step": 184
    },
    {
      "epoch": 0.09113300492610837,
      "grad_norm": 0.4634832657601789,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -81742324.62222221,
      "logits/rejected": -95609917.68674698,
      "logps/chosen": -356.26666666666665,
      "logps/rejected": -474.6024096385542,
      "loss": 0.2471,
      "rewards/chosen": 8.322793918185765,
      "rewards/margins": 14.684239701318294,
      "rewards/rejected": -6.36144578313253,
      "step": 185
    },
    {
      "epoch": 0.0916256157635468,
      "grad_norm": 0.47464291748235016,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -79691776.0,
      "logits/rejected": -78683019.34177215,
      "logps/chosen": -243.26530612244898,
      "logps/rejected": -410.73417721518985,
      "loss": 0.2849,
      "rewards/chosen": -0.29063695790816324,
      "rewards/margins": 5.449869371205761,
      "rewards/rejected": -5.7405063291139244,
      "step": 186
    },
    {
      "epoch": 0.09211822660098522,
      "grad_norm": 0.41098832009133235,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -64778695.11111111,
      "logits/rejected": -84088215.13253012,
      "logps/chosen": -241.77777777777777,
      "logps/rejected": -461.49397590361446,
      "loss": 0.2464,
      "rewards/chosen": 0.23194444444444445,
      "rewards/margins": 7.4970046854082995,
      "rewards/rejected": -7.265060240963855,
      "step": 187
    },
    {
      "epoch": 0.09261083743842365,
      "grad_norm": 0.5199096641749047,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -86745832.72727273,
      "logits/rejected": -82288249.90476191,
      "logps/chosen": -219.27272727272728,
      "logps/rejected": -409.9047619047619,
      "loss": 0.2716,
      "rewards/chosen": -0.26171875,
      "rewards/margins": 5.833519345238095,
      "rewards/rejected": -6.095238095238095,
      "step": 188
    },
    {
      "epoch": 0.09310344827586207,
      "grad_norm": 0.36782143601872164,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -69659454.27027027,
      "logits/rejected": -89785760.35164835,
      "logps/chosen": -303.35135135135135,
      "logps/rejected": -425.4945054945055,
      "loss": 0.2064,
      "rewards/chosen": 0.48015202702702703,
      "rewards/margins": 7.364767411642412,
      "rewards/rejected": -6.884615384615385,
      "step": 189
    },
    {
      "epoch": 0.09359605911330049,
      "grad_norm": 0.6305960743146983,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -98621332.21052632,
      "logits/rejected": -86495869.15555556,
      "logps/chosen": -329.2631578947368,
      "logps/rejected": -450.84444444444443,
      "loss": 0.2454,
      "rewards/chosen": 3.661106310392681,
      "rewards/margins": 10.294439643726015,
      "rewards/rejected": -6.633333333333334,
      "step": 190
    },
    {
      "epoch": 0.09408866995073892,
      "grad_norm": 0.5342746234937424,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -75826437.01960784,
      "logits/rejected": -105892558.12987013,
      "logps/chosen": -325.3333333333333,
      "logps/rejected": -500.3636363636364,
      "loss": 0.2747,
      "rewards/chosen": 0.7933517156862745,
      "rewards/margins": 3.5246761468703247,
      "rewards/rejected": -2.73132443118405,
      "step": 191
    },
    {
      "epoch": 0.09458128078817735,
      "grad_norm": 0.49115164657337596,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -62557597.957446806,
      "logits/rejected": -84611021.43209876,
      "logps/chosen": -239.48936170212767,
      "logps/rejected": -454.320987654321,
      "loss": 0.2689,
      "rewards/chosen": -0.4328457446808511,
      "rewards/margins": 5.690611045442606,
      "rewards/rejected": -6.1234567901234565,
      "step": 192
    },
    {
      "epoch": 0.09507389162561576,
      "grad_norm": 0.4593521428312989,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -77882468.39215687,
      "logits/rejected": -86718596.98701298,
      "logps/chosen": -287.37254901960785,
      "logps/rejected": -407.27272727272725,
      "loss": 0.2886,
      "rewards/chosen": -0.16130514705882354,
      "rewards/margins": 6.539993554239878,
      "rewards/rejected": -6.701298701298701,
      "step": 193
    },
    {
      "epoch": 0.09556650246305419,
      "grad_norm": 0.4618591351006865,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -83187029.33333333,
      "logits/rejected": -89653248.0,
      "logps/chosen": -318.6666666666667,
      "logps/rejected": -500.8,
      "loss": 0.2597,
      "rewards/chosen": 2.2569910685221353,
      "rewards/margins": 10.019491068522136,
      "rewards/rejected": -7.7625,
      "step": 194
    },
    {
      "epoch": 0.0960591133004926,
      "grad_norm": 0.4179662851040029,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -70004931.04761904,
      "logits/rejected": -88373009.86046511,
      "logps/chosen": -333.7142857142857,
      "logps/rejected": -496.3720930232558,
      "loss": 0.2393,
      "rewards/chosen": -0.2585565476190476,
      "rewards/margins": 7.904234150055371,
      "rewards/rejected": -8.162790697674419,
      "step": 195
    },
    {
      "epoch": 0.09655172413793103,
      "grad_norm": 0.41514531795497783,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -68476571.82608695,
      "logits/rejected": -77952674.34146342,
      "logps/chosen": -271.82608695652175,
      "logps/rejected": -429.6585365853659,
      "loss": 0.2391,
      "rewards/chosen": 3.8612080449643345,
      "rewards/margins": 10.812427557159456,
      "rewards/rejected": -6.951219512195122,
      "step": 196
    },
    {
      "epoch": 0.09704433497536946,
      "grad_norm": 0.5264374392237376,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -76171556.57142857,
      "logits/rejected": -75788743.1111111,
      "logps/chosen": -276.0,
      "logps/rejected": -395.1111111111111,
      "loss": 0.3374,
      "rewards/chosen": -0.31808035714285715,
      "rewards/margins": 5.966641865079366,
      "rewards/rejected": -6.284722222222222,
      "step": 197
    },
    {
      "epoch": 0.09753694581280788,
      "grad_norm": 0.6482756371482139,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -89630452.86956522,
      "logits/rejected": -80919377.17073171,
      "logps/chosen": -334.95652173913044,
      "logps/rejected": -451.1219512195122,
      "loss": 0.2296,
      "rewards/chosen": 0.14673913043478262,
      "rewards/margins": 7.573568398727465,
      "rewards/rejected": -7.426829268292683,
      "step": 198
    },
    {
      "epoch": 0.0980295566502463,
      "grad_norm": 0.4581599604967598,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -83172155.91489361,
      "logits/rejected": -83057575.50617284,
      "logps/chosen": -348.25531914893617,
      "logps/rejected": -486.71604938271605,
      "loss": 0.2951,
      "rewards/chosen": 0.969375935006649,
      "rewards/margins": 8.932338897969611,
      "rewards/rejected": -7.962962962962963,
      "step": 199
    },
    {
      "epoch": 0.09852216748768473,
      "grad_norm": 0.49178414737848397,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -87624481.39130434,
      "logits/rejected": -100151795.51219513,
      "logps/chosen": -342.60869565217394,
      "logps/rejected": -500.2926829268293,
      "loss": 0.2754,
      "rewards/chosen": 5.512853539508322,
      "rewards/margins": 12.964073051703444,
      "rewards/rejected": -7.451219512195122,
      "step": 200
    },
    {
      "epoch": 0.09901477832512315,
      "grad_norm": 0.4559951704514162,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -63426060.487804875,
      "logits/rejected": -93335316.59770115,
      "logps/chosen": -244.4878048780488,
      "logps/rejected": -500.9655172413793,
      "loss": 0.2173,
      "rewards/chosen": 0.04496951219512195,
      "rewards/margins": 7.780601696103168,
      "rewards/rejected": -7.735632183908046,
      "step": 201
    },
    {
      "epoch": 0.09950738916256158,
      "grad_norm": 0.5141291157492974,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -84666415.62790698,
      "logits/rejected": -84872975.05882353,
      "logps/chosen": -321.1162790697674,
      "logps/rejected": -475.8588235294118,
      "loss": 0.2236,
      "rewards/chosen": 3.271086936773256,
      "rewards/margins": 7.57253741931002,
      "rewards/rejected": -4.301450482536764,
      "step": 202
    },
    {
      "epoch": 0.1,
      "grad_norm": 0.5022331064772415,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -87536677.92592593,
      "logits/rejected": -86833428.75675675,
      "logps/chosen": -324.74074074074076,
      "logps/rejected": -451.02702702702703,
      "loss": 0.3174,
      "rewards/chosen": -0.5543981481481481,
      "rewards/margins": 5377636.202358608,
      "rewards/rejected": -5377636.756756756,
      "step": 203
    },
    {
      "epoch": 0.10049261083743842,
      "grad_norm": 0.4116977410646617,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -80248163.26530612,
      "logits/rejected": -96203529.721519,
      "logps/chosen": -334.53061224489795,
      "logps/rejected": -527.3924050632911,
      "loss": 0.2536,
      "rewards/chosen": 3.7544082330197703,
      "rewards/margins": 11.387319625424833,
      "rewards/rejected": -7.632911392405063,
      "step": 204
    },
    {
      "epoch": 0.10098522167487685,
      "grad_norm": 0.3714534053938671,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -72637719.27272727,
      "logits/rejected": -83087164.95238096,
      "logps/chosen": -296.3636363636364,
      "logps/rejected": -413.7142857142857,
      "loss": 0.2537,
      "rewards/chosen": -0.5042613636363636,
      "rewards/margins": 6.971929112554113,
      "rewards/rejected": -7.476190476190476,
      "step": 205
    },
    {
      "epoch": 0.10147783251231528,
      "grad_norm": 0.4870568437056009,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -98266550.85714285,
      "logits/rejected": -81821191.87692308,
      "logps/chosen": -264.8888888888889,
      "logps/rejected": -453.9076923076923,
      "loss": 0.3336,
      "rewards/chosen": 1.8437737358940973,
      "rewards/margins": 8.16685065897102,
      "rewards/rejected": -6.323076923076923,
      "step": 206
    },
    {
      "epoch": 0.10197044334975369,
      "grad_norm": 0.51695689070544,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -75272777.14285715,
      "logits/rejected": -83420046.22222222,
      "logps/chosen": -322.57142857142856,
      "logps/rejected": -473.3333333333333,
      "loss": 0.2763,
      "rewards/chosen": 2.9190761021205356,
      "rewards/margins": 9.384353879898313,
      "rewards/rejected": -6.465277777777778,
      "step": 207
    },
    {
      "epoch": 0.10246305418719212,
      "grad_norm": 0.4901387173844053,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -75086265.7254902,
      "logits/rejected": -80508848.20779221,
      "logps/chosen": -273.2549019607843,
      "logps/rejected": -420.57142857142856,
      "loss": 0.2715,
      "rewards/chosen": 2.7634420955882355,
      "rewards/margins": 9.321883654029794,
      "rewards/rejected": -6.558441558441558,
      "step": 208
    },
    {
      "epoch": 0.10295566502463054,
      "grad_norm": 0.4131432819229727,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -71093452.8,
      "logits/rejected": -94741925.64705883,
      "logps/chosen": -254.8,
      "logps/rejected": -432.47058823529414,
      "loss": 0.2729,
      "rewards/chosen": 1.6726366678873699,
      "rewards/margins": 8.312342550240311,
      "rewards/rejected": -6.639705882352941,
      "step": 209
    },
    {
      "epoch": 0.10344827586206896,
      "grad_norm": 0.4608217341499813,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -109870304.7804878,
      "logits/rejected": -80318511.08045977,
      "logps/chosen": -341.8536585365854,
      "logps/rejected": -397.2413793103448,
      "loss": 0.2245,
      "rewards/chosen": 0.7096036585365854,
      "rewards/margins": 6.284316302214747,
      "rewards/rejected": -5.574712643678161,
      "step": 210
    },
    {
      "epoch": 0.10394088669950739,
      "grad_norm": 0.591712202518733,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -71816756.24489796,
      "logits/rejected": -82452583.69620253,
      "logps/chosen": -279.3469387755102,
      "logps/rejected": -533.873417721519,
      "loss": 0.2657,
      "rewards/chosen": 4.403705830476722,
      "rewards/margins": 11.245477982375455,
      "rewards/rejected": -6.841772151898734,
      "step": 211
    },
    {
      "epoch": 0.10443349753694581,
      "grad_norm": 0.39943732007637434,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -83214991.36,
      "logits/rejected": -90984132.92307693,
      "logps/chosen": -345.92,
      "logps/rejected": -478.35897435897436,
      "loss": 0.2457,
      "rewards/chosen": 0.50453125,
      "rewards/margins": 7.267351762820513,
      "rewards/rejected": -6.762820512820513,
      "step": 212
    },
    {
      "epoch": 0.10492610837438424,
      "grad_norm": 0.44352404388359323,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -78624134.98181818,
      "logits/rejected": -78025545.64383562,
      "logps/chosen": -300.5090909090909,
      "logps/rejected": -456.7671232876712,
      "loss": 0.3026,
      "rewards/chosen": -0.4125,
      "rewards/margins": 5.464212328767124,
      "rewards/rejected": -5.876712328767123,
      "step": 213
    },
    {
      "epoch": 0.10541871921182266,
      "grad_norm": 0.5178539483687342,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -88517290.66666667,
      "logits/rejected": -87136665.6,
      "logps/chosen": -286.6666666666667,
      "logps/rejected": -519.2,
      "loss": 0.2767,
      "rewards/chosen": -1.0201822916666667,
      "rewards/margins": 5.723567708333333,
      "rewards/rejected": -6.74375,
      "step": 214
    },
    {
      "epoch": 0.10591133004926108,
      "grad_norm": 0.4805146215020015,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -114120021.33333333,
      "logits/rejected": -86088089.6,
      "logps/chosen": -396.0,
      "logps/rejected": -462.0,
      "loss": 0.2735,
      "rewards/chosen": 2.0995470682779946,
      "rewards/margins": 9.080797068277995,
      "rewards/rejected": -6.98125,
      "step": 215
    },
    {
      "epoch": 0.10640394088669951,
      "grad_norm": 0.5164091291327072,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -69784540.68965517,
      "logits/rejected": -98026876.34285714,
      "logps/chosen": -285.51724137931035,
      "logps/rejected": -442.51428571428573,
      "loss": 0.3162,
      "rewards/chosen": -0.3407866379310345,
      "rewards/margins": 7.030641933497536,
      "rewards/rejected": -7.371428571428571,
      "step": 216
    },
    {
      "epoch": 0.10689655172413794,
      "grad_norm": 0.5269727647222272,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -76626707.6923077,
      "logits/rejected": -81457798.73684211,
      "logps/chosen": -308.0,
      "logps/rejected": -403.36842105263156,
      "loss": 0.3008,
      "rewards/chosen": -0.26352163461538464,
      "rewards/margins": 5.414109944331983,
      "rewards/rejected": -5.677631578947368,
      "step": 217
    },
    {
      "epoch": 0.10738916256157635,
      "grad_norm": 0.5303995089839261,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -72672736.65306123,
      "logits/rejected": -86222148.05063291,
      "logps/chosen": -288.9795918367347,
      "logps/rejected": -426.9367088607595,
      "loss": 0.2746,
      "rewards/chosen": 2.372424067283163,
      "rewards/margins": 8.378753181207214,
      "rewards/rejected": -6.006329113924051,
      "step": 218
    },
    {
      "epoch": 0.10788177339901478,
      "grad_norm": 0.6231118001910895,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -85563801.6,
      "logits/rejected": -105374705.97260274,
      "logps/chosen": -336.8727272727273,
      "logps/rejected": -526.027397260274,
      "loss": 0.3087,
      "rewards/chosen": -0.19495738636363635,
      "rewards/margins": 6.462576860211706,
      "rewards/rejected": -6.657534246575342,
      "step": 219
    },
    {
      "epoch": 0.10837438423645321,
      "grad_norm": 0.48952707564429015,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -69283688.2962963,
      "logits/rejected": -84622917.1891892,
      "logps/chosen": -236.0,
      "logps/rejected": -440.2162162162162,
      "loss": 0.2854,
      "rewards/chosen": 1.9581789087366175,
      "rewards/margins": 9.093314043871752,
      "rewards/rejected": -7.135135135135135,
      "step": 220
    },
    {
      "epoch": 0.10886699507389162,
      "grad_norm": 0.4360071239486091,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -71093452.8,
      "logits/rejected": -84839330.9090909,
      "logps/chosen": -296.0,
      "logps/rejected": -454.90909090909093,
      "loss": 0.243,
      "rewards/chosen": 6.10458984375,
      "rewards/margins": 12.72958984375,
      "rewards/rejected": -6.625,
      "step": 221
    },
    {
      "epoch": 0.10935960591133005,
      "grad_norm": 0.5879821716397607,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -73400320.0,
      "logits/rejected": -88080384.0,
      "logps/chosen": -295.1111111111111,
      "logps/rejected": -389.9130434782609,
      "loss": 0.2131,
      "rewards/chosen": 2.093701468573676,
      "rewards/margins": 8.615440599008458,
      "rewards/rejected": -6.521739130434782,
      "step": 222
    },
    {
      "epoch": 0.10985221674876847,
      "grad_norm": 0.3883357980983041,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -71838611.0638298,
      "logits/rejected": -107187768.8888889,
      "logps/chosen": -254.29787234042553,
      "logps/rejected": -537.283950617284,
      "loss": 0.2682,
      "rewards/chosen": 2.396605146692154,
      "rewards/margins": 8.902777986198327,
      "rewards/rejected": -6.506172839506172,
      "step": 223
    },
    {
      "epoch": 0.1103448275862069,
      "grad_norm": 0.4142722257125457,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -97517568.0,
      "logits/rejected": -90940136.72727273,
      "logps/chosen": -406.8,
      "logps/rejected": -503.27272727272725,
      "loss": 0.2251,
      "rewards/chosen": -0.351171875,
      "rewards/margins": 6.870419034090909,
      "rewards/rejected": -7.221590909090909,
      "step": 224
    },
    {
      "epoch": 0.11083743842364532,
      "grad_norm": 0.724423885555267,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -86067118.08,
      "logits/rejected": -83025709.94871795,
      "logps/chosen": -368.96,
      "logps/rejected": -464.4102564102564,
      "loss": 0.2485,
      "rewards/chosen": 6.251170654296875,
      "rewards/margins": 12.751170654296875,
      "rewards/rejected": -6.5,
      "step": 225
    },
    {
      "epoch": 0.11133004926108374,
      "grad_norm": 0.4622610205535244,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -98643816.2962963,
      "logits/rejected": -89554058.37837838,
      "logps/chosen": -301.9259259259259,
      "logps/rejected": -435.4594594594595,
      "loss": 0.2694,
      "rewards/chosen": 2.5127376980251737,
      "rewards/margins": 9.120845806133282,
      "rewards/rejected": -6.608108108108108,
      "step": 226
    },
    {
      "epoch": 0.11182266009852217,
      "grad_norm": 0.4407075034613784,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -73540130.13333334,
      "logits/rejected": -82875404.3373494,
      "logps/chosen": -305.77777777777777,
      "logps/rejected": -435.27710843373495,
      "loss": 0.2483,
      "rewards/chosen": -0.18472222222222223,
      "rewards/margins": 5.899615127175368,
      "rewards/rejected": -6.0843373493975905,
      "step": 227
    },
    {
      "epoch": 0.1123152709359606,
      "grad_norm": 0.44991015057902484,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -78188157.58490565,
      "logits/rejected": -92162839.89333333,
      "logps/chosen": -271.8490566037736,
      "logps/rejected": -475.73333333333335,
      "loss": 0.2948,
      "rewards/chosen": -0.46860259433962265,
      "rewards/margins": 6.538064072327044,
      "rewards/rejected": -7.006666666666667,
      "step": 228
    },
    {
      "epoch": 0.11280788177339901,
      "grad_norm": 0.44036164149299617,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -82674392.17777778,
      "logits/rejected": -84593552.96385542,
      "logps/chosen": -318.5777777777778,
      "logps/rejected": -460.72289156626505,
      "loss": 0.2816,
      "rewards/chosen": 5.460606214735243,
      "rewards/margins": 11.334100190638857,
      "rewards/rejected": -5.873493975903615,
      "step": 229
    },
    {
      "epoch": 0.11330049261083744,
      "grad_norm": 0.41448534513313073,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -84626251.29411764,
      "logits/rejected": -85955996.25974026,
      "logps/chosen": -315.921568627451,
      "logps/rejected": -459.2207792207792,
      "loss": 0.267,
      "rewards/chosen": -0.07889093137254902,
      "rewards/margins": 6.089940237458619,
      "rewards/rejected": -6.1688311688311686,
      "step": 230
    },
    {
      "epoch": 0.11379310344827587,
      "grad_norm": 0.4228280085136401,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -97867093.33333333,
      "logits/rejected": -83785012.43373494,
      "logps/chosen": -433.6,
      "logps/rejected": -464.1927710843373,
      "loss": 0.2832,
      "rewards/chosen": 6.538808865017361,
      "rewards/margins": 12.779772720439048,
      "rewards/rejected": -6.240963855421687,
      "step": 231
    },
    {
      "epoch": 0.11428571428571428,
      "grad_norm": 0.5114175872187416,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -81336601.09803921,
      "logits/rejected": -96959235.32467532,
      "logps/chosen": -433.88235294117646,
      "logps/rejected": -512.0,
      "loss": 0.2882,
      "rewards/chosen": 2.257891486672794,
      "rewards/margins": 8.446203174984483,
      "rewards/rejected": -6.188311688311688,
      "step": 232
    },
    {
      "epoch": 0.11477832512315271,
      "grad_norm": 0.5344925668858636,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -70875177.79591836,
      "logits/rejected": -89726250.12658228,
      "logps/chosen": -303.6734693877551,
      "logps/rejected": -499.8481012658228,
      "loss": 0.2369,
      "rewards/chosen": 2.963911718251754,
      "rewards/margins": 9.381633237239095,
      "rewards/rejected": -6.417721518987341,
      "step": 233
    },
    {
      "epoch": 0.11527093596059114,
      "grad_norm": 0.4212814215691416,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -66346263.27272727,
      "logits/rejected": -114452788.60273972,
      "logps/chosen": -311.8545454545455,
      "logps/rejected": -431.3424657534247,
      "loss": 0.2718,
      "rewards/chosen": 2.6937286376953127,
      "rewards/margins": 5.899208089750108,
      "rewards/rejected": -3.2054794520547945,
      "step": 234
    },
    {
      "epoch": 0.11576354679802955,
      "grad_norm": 0.45248092327477596,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -83571507.2,
      "logits/rejected": -83314129.45454545,
      "logps/chosen": -299.4,
      "logps/rejected": -473.45454545454544,
      "loss": 0.2203,
      "rewards/chosen": 6.558524322509766,
      "rewards/margins": 13.024433413418857,
      "rewards/rejected": -6.465909090909091,
      "step": 235
    },
    {
      "epoch": 0.11625615763546798,
      "grad_norm": 0.42737719593513696,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -80166602.86792453,
      "logits/rejected": -88136308.05333333,
      "logps/chosen": -270.188679245283,
      "logps/rejected": -421.12,
      "loss": 0.2684,
      "rewards/chosen": -0.0294811320754717,
      "rewards/margins": 5.670518867924528,
      "rewards/rejected": -5.7,
      "step": 236
    },
    {
      "epoch": 0.1167487684729064,
      "grad_norm": 0.4699309910927715,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -74555893.55102041,
      "logits/rejected": -77727355.1392405,
      "logps/chosen": -338.61224489795916,
      "logps/rejected": -436.65822784810126,
      "loss": 0.2887,
      "rewards/chosen": 2.1827031349649233,
      "rewards/margins": 7.828272755218087,
      "rewards/rejected": -5.6455696202531644,
      "step": 237
    },
    {
      "epoch": 0.11724137931034483,
      "grad_norm": 0.4395380388407768,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -77944149.33333333,
      "logits/rejected": -74703955.02702703,
      "logps/chosen": -371.25925925925924,
      "logps/rejected": -385.72972972972974,
      "loss": 0.2798,
      "rewards/chosen": 3.0043182373046875,
      "rewards/margins": 8.984047967034417,
      "rewards/rejected": -5.97972972972973,
      "step": 238
    },
    {
      "epoch": 0.11773399014778325,
      "grad_norm": 0.4081476928149675,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -78514427.50877193,
      "logits/rejected": -97473261.97183098,
      "logps/chosen": -323.9298245614035,
      "logps/rejected": -491.71830985915494,
      "loss": 0.2886,
      "rewards/chosen": 0.3170230263157895,
      "rewards/margins": 6.556459646034099,
      "rewards/rejected": -6.23943661971831,
      "step": 239
    },
    {
      "epoch": 0.11822660098522167,
      "grad_norm": 0.4324353875663478,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -67795204.65454546,
      "logits/rejected": -85494854.1369863,
      "logps/chosen": -277.8181818181818,
      "logps/rejected": -449.3150684931507,
      "loss": 0.2663,
      "rewards/chosen": -0.11534090909090909,
      "rewards/margins": 6.692878268991283,
      "rewards/rejected": -6.808219178082192,
      "step": 240
    },
    {
      "epoch": 0.1187192118226601,
      "grad_norm": 0.4973261116604227,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -90520706.32727273,
      "logits/rejected": -96296623.34246576,
      "logps/chosen": -371.4909090909091,
      "logps/rejected": -452.82191780821915,
      "loss": 0.3166,
      "rewards/chosen": -0.5755681818181818,
      "rewards/margins": 5.369637297633873,
      "rewards/rejected": -5.945205479452055,
      "step": 241
    },
    {
      "epoch": 0.11921182266009853,
      "grad_norm": 0.39029921229006187,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -68367155.2,
      "logits/rejected": -94902444.72289157,
      "logps/chosen": -254.04444444444445,
      "logps/rejected": -473.4457831325301,
      "loss": 0.2322,
      "rewards/chosen": 0.024826388888888887,
      "rewards/margins": 6.627236027443106,
      "rewards/rejected": -6.602409638554217,
      "step": 242
    },
    {
      "epoch": 0.11970443349753694,
      "grad_norm": 0.3891278872462206,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -71489581.51111111,
      "logits/rejected": -85402093.49397591,
      "logps/chosen": -319.2888888888889,
      "logps/rejected": -446.4578313253012,
      "loss": 0.2165,
      "rewards/chosen": 3.5653028700086806,
      "rewards/margins": 9.559278773623138,
      "rewards/rejected": -5.993975903614458,
      "step": 243
    },
    {
      "epoch": 0.12019704433497537,
      "grad_norm": 0.4022388673073596,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -55700357.12,
      "logits/rejected": -77863489.64102565,
      "logps/chosen": -220.48,
      "logps/rejected": -436.5128205128205,
      "loss": 0.247,
      "rewards/chosen": 0.4921875,
      "rewards/margins": 6.376802884615385,
      "rewards/rejected": -5.884615384615385,
      "step": 244
    },
    {
      "epoch": 0.1206896551724138,
      "grad_norm": 0.4336577719481423,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -62672580.92307692,
      "logits/rejected": -83775703.57894737,
      "logps/chosen": -168.92307692307693,
      "logps/rejected": -463.57894736842104,
      "loss": 0.26,
      "rewards/chosen": 0.42397836538461536,
      "rewards/margins": 7.009504681174088,
      "rewards/rejected": -6.5855263157894735,
      "step": 245
    },
    {
      "epoch": 0.12118226600985221,
      "grad_norm": 0.38635215097088527,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -92536832.0,
      "logits/rejected": -97098137.6,
      "logps/chosen": -402.6666666666667,
      "logps/rejected": -472.0,
      "loss": 0.1926,
      "rewards/chosen": 5.646500905354817,
      "rewards/margins": 12.490250905354817,
      "rewards/rejected": -6.84375,
      "step": 246
    },
    {
      "epoch": 0.12167487684729064,
      "grad_norm": 0.5498350632006187,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -78326188.65116279,
      "logits/rejected": -80925394.8235294,
      "logps/chosen": -359.8139534883721,
      "logps/rejected": -376.09411764705885,
      "loss": 0.2502,
      "rewards/chosen": 0.6947674418604651,
      "rewards/margins": 5.8300615595075245,
      "rewards/rejected": -5.135294117647059,
      "step": 247
    },
    {
      "epoch": 0.12216748768472907,
      "grad_norm": 0.6113175925122105,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -82816943.68627451,
      "logits/rejected": -77730802.7012987,
      "logps/chosen": -386.5098039215686,
      "logps/rejected": -414.3376623376623,
      "loss": 0.2932,
      "rewards/chosen": 2.2352788588579964,
      "rewards/margins": 8.52748665106579,
      "rewards/rejected": -6.292207792207792,
      "step": 248
    },
    {
      "epoch": 0.12266009852216748,
      "grad_norm": 0.444330448981823,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -71224030.18867925,
      "logits/rejected": -98314485.76,
      "logps/chosen": -237.58490566037736,
      "logps/rejected": -481.28,
      "loss": 0.2653,
      "rewards/chosen": 0.08785377358490566,
      "rewards/margins": 7.014520440251572,
      "rewards/rejected": -6.926666666666667,
      "step": 249
    },
    {
      "epoch": 0.12315270935960591,
      "grad_norm": 0.3932483236594659,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -66835322.43478261,
      "logits/rejected": -88489584.3902439,
      "logps/chosen": -229.91304347826087,
      "logps/rejected": -442.5365853658537,
      "loss": 0.2498,
      "rewards/chosen": 0.10122282608695653,
      "rewards/margins": 6.3085398992576875,
      "rewards/rejected": -6.2073170731707314,
      "step": 250
    },
    {
      "epoch": 0.12364532019704433,
      "grad_norm": 0.41606799226131563,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -109526730.86792453,
      "logits/rejected": -74546763.09333333,
      "logps/chosen": -414.79245283018867,
      "logps/rejected": -410.88,
      "loss": 0.2435,
      "rewards/chosen": 1.2040094339622642,
      "rewards/margins": 7.0773427672955975,
      "rewards/rejected": -5.873333333333333,
      "step": 251
    },
    {
      "epoch": 0.12413793103448276,
      "grad_norm": 0.47613931654065456,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -64196152.88888889,
      "logits/rejected": -98282745.08108108,
      "logps/chosen": -335.1111111111111,
      "logps/rejected": -457.94594594594594,
      "loss": 0.2528,
      "rewards/chosen": 2.9045308430989585,
      "rewards/margins": 8.668044356612471,
      "rewards/rejected": -5.763513513513513,
      "step": 252
    },
    {
      "epoch": 0.12463054187192119,
      "grad_norm": 0.49863568836801325,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -69844279.6521739,
      "logits/rejected": -88387284.29268293,
      "logps/chosen": -277.9130434782609,
      "logps/rejected": -449.9512195121951,
      "loss": 0.2387,
      "rewards/chosen": 0.38247282608695654,
      "rewards/margins": 6.699545996818664,
      "rewards/rejected": -6.317073170731708,
      "step": 253
    },
    {
      "epoch": 0.12512315270935961,
      "grad_norm": 0.45024223806734,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -80514188.5490196,
      "logits/rejected": -75170643.11688311,
      "logps/chosen": -234.35294117647058,
      "logps/rejected": -481.6623376623377,
      "loss": 0.2524,
      "rewards/chosen": 0.22849647671568626,
      "rewards/margins": 7.397327645546855,
      "rewards/rejected": -7.1688311688311686,
      "step": 254
    },
    {
      "epoch": 0.12561576354679804,
      "grad_norm": 0.46281431892189123,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -82609552.69565217,
      "logits/rejected": -72837669.46341464,
      "logps/chosen": -255.82608695652175,
      "logps/rejected": -406.6341463414634,
      "loss": 0.2566,
      "rewards/chosen": -0.09578804347826086,
      "rewards/margins": 5.873724151643691,
      "rewards/rejected": -5.969512195121951,
      "step": 255
    },
    {
      "epoch": 0.12610837438423644,
      "grad_norm": 0.467010219698562,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -65145572.765957445,
      "logits/rejected": -97763530.27160494,
      "logps/chosen": -273.70212765957444,
      "logps/rejected": -511.2098765432099,
      "loss": 0.2361,
      "rewards/chosen": 3.09272603785738,
      "rewards/margins": 10.043343321807997,
      "rewards/rejected": -6.950617283950617,
      "step": 256
    },
    {
      "epoch": 0.12660098522167487,
      "grad_norm": 0.5909183706530793,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -55266123.294117644,
      "logits/rejected": -97612893.0909091,
      "logps/chosen": -173.33333333333334,
      "logps/rejected": -479.5844155844156,
      "loss": 0.2545,
      "rewards/chosen": -0.27389705882352944,
      "rewards/margins": 6.602726317799847,
      "rewards/rejected": -6.876623376623376,
      "step": 257
    },
    {
      "epoch": 0.1270935960591133,
      "grad_norm": 0.5778065686314343,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -83796839.4893617,
      "logits/rejected": -80054246.71604939,
      "logps/chosen": -365.6170212765957,
      "logps/rejected": -425.48148148148147,
      "loss": 0.2316,
      "rewards/chosen": 8.227979294797207,
      "rewards/margins": 15.005757072574983,
      "rewards/rejected": -6.777777777777778,
      "step": 258
    },
    {
      "epoch": 0.12758620689655173,
      "grad_norm": 0.4503196623661979,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -69439032.8888889,
      "logits/rejected": -94902444.72289157,
      "logps/chosen": -321.6,
      "logps/rejected": -480.7710843373494,
      "loss": 0.2259,
      "rewards/chosen": 2.6993231879340276,
      "rewards/margins": 9.006552103596679,
      "rewards/rejected": -6.307228915662651,
      "step": 259
    },
    {
      "epoch": 0.12807881773399016,
      "grad_norm": 0.8294845005099252,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -89448091.82608695,
      "logits/rejected": -94320689.95121951,
      "logps/chosen": -319.6521739130435,
      "logps/rejected": -462.8292682926829,
      "loss": 0.2098,
      "rewards/chosen": 0.8655740489130435,
      "rewards/margins": 7.420452097693531,
      "rewards/rejected": -6.554878048780488,
      "step": 260
    },
    {
      "epoch": 0.12857142857142856,
      "grad_norm": 0.7602056752120422,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -89552069.61403508,
      "logits/rejected": -103026284.16901408,
      "logps/chosen": -378.6666666666667,
      "logps/rejected": -449.80281690140845,
      "loss": 0.3098,
      "rewards/chosen": -0.03728070175438596,
      "rewards/margins": 6.4063812700766,
      "rewards/rejected": -6.443661971830986,
      "step": 261
    },
    {
      "epoch": 0.129064039408867,
      "grad_norm": 0.6372924032684402,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -87791121.65517241,
      "logits/rejected": -79452101.48571429,
      "logps/chosen": -254.06896551724137,
      "logps/rejected": -445.7142857142857,
      "loss": 0.2619,
      "rewards/chosen": 1.1298491379310345,
      "rewards/margins": 4.138737600542641,
      "rewards/rejected": -3.008888462611607,
      "step": 262
    },
    {
      "epoch": 0.12955665024630542,
      "grad_norm": 0.5728676005700551,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -94511650.13333334,
      "logits/rejected": -95407782.55421686,
      "logps/chosen": -313.24444444444447,
      "logps/rejected": -467.66265060240966,
      "loss": 0.2415,
      "rewards/chosen": -0.3095486111111111,
      "rewards/margins": 7.2205718708166,
      "rewards/rejected": -7.530120481927711,
      "step": 263
    },
    {
      "epoch": 0.13004926108374384,
      "grad_norm": 0.410364081271538,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -74152129.20754717,
      "logits/rejected": -82040586.24,
      "logps/chosen": -267.47169811320754,
      "logps/rejected": -445.44,
      "loss": 0.2632,
      "rewards/chosen": 2.8676135944870285,
      "rewards/margins": 10.127613594487029,
      "rewards/rejected": -7.26,
      "step": 264
    },
    {
      "epoch": 0.13054187192118227,
      "grad_norm": 0.49821188239896275,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -91768478.89655173,
      "logits/rejected": -81848846.62857144,
      "logps/chosen": -426.2068965517241,
      "logps/rejected": -395.8857142857143,
      "loss": 0.2825,
      "rewards/chosen": 0.3537176724137931,
      "rewards/margins": 4.105444095404865,
      "rewards/rejected": -3.7517264229910716,
      "step": 265
    },
    {
      "epoch": 0.1310344827586207,
      "grad_norm": 0.6576917771091381,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -68649628.73469388,
      "logits/rejected": -82930415.79746835,
      "logps/chosen": -279.51020408163265,
      "logps/rejected": -514.8354430379746,
      "loss": 0.2809,
      "rewards/chosen": -0.8195153061224489,
      "rewards/margins": 6.579218871092741,
      "rewards/rejected": -7.39873417721519,
      "step": 266
    },
    {
      "epoch": 0.1315270935960591,
      "grad_norm": 0.4349543524560866,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -78938952.20512821,
      "logits/rejected": -93971260.40449437,
      "logps/chosen": -359.79487179487177,
      "logps/rejected": -517.7528089887641,
      "loss": 0.2301,
      "rewards/chosen": -0.7251602564102564,
      "rewards/margins": 6.286075698645924,
      "rewards/rejected": -7.01123595505618,
      "step": 267
    },
    {
      "epoch": 0.13201970443349753,
      "grad_norm": 0.42453774447166764,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -61923179.054545455,
      "logits/rejected": -81128181.47945206,
      "logps/chosen": -227.78181818181818,
      "logps/rejected": -449.3150684931507,
      "loss": 0.3099,
      "rewards/chosen": -0.7335227272727273,
      "rewards/margins": 5.814422478206725,
      "rewards/rejected": -6.5479452054794525,
      "step": 268
    },
    {
      "epoch": 0.13251231527093596,
      "grad_norm": 0.5345971287959294,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -77412262.95652173,
      "logits/rejected": -108335803.31707317,
      "logps/chosen": -333.9130434782609,
      "logps/rejected": -504.9756097560976,
      "loss": 0.2398,
      "rewards/chosen": 3.8736227284307065,
      "rewards/margins": 10.53825687477217,
      "rewards/rejected": -6.664634146341464,
      "step": 269
    },
    {
      "epoch": 0.1330049261083744,
      "grad_norm": 0.41096916136669065,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -73949574.09523809,
      "logits/rejected": -79984401.86046511,
      "logps/chosen": -211.14285714285714,
      "logps/rejected": -428.6511627906977,
      "loss": 0.2345,
      "rewards/chosen": 6.560151599702381,
      "rewards/margins": 12.58340741365587,
      "rewards/rejected": -6.023255813953488,
      "step": 270
    },
    {
      "epoch": 0.13349753694581282,
      "grad_norm": 0.49112743870979075,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -70212648.96,
      "logits/rejected": -98243505.23076923,
      "logps/chosen": -314.88,
      "logps/rejected": -489.43589743589746,
      "loss": 0.2648,
      "rewards/chosen": 0.02625,
      "rewards/margins": 7.372403846153846,
      "rewards/rejected": -7.346153846153846,
      "step": 271
    },
    {
      "epoch": 0.13399014778325122,
      "grad_norm": 0.4651755464628304,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -82561562.94736843,
      "logits/rejected": -82172913.57746479,
      "logps/chosen": -340.7719298245614,
      "logps/rejected": -520.112676056338,
      "loss": 0.2604,
      "rewards/chosen": 2.681961862664474,
      "rewards/margins": 9.787595665481374,
      "rewards/rejected": -7.105633802816901,
      "step": 272
    },
    {
      "epoch": 0.13448275862068965,
      "grad_norm": 0.4745157813519903,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -67287345.0212766,
      "logits/rejected": -85853778.17283951,
      "logps/chosen": -354.8936170212766,
      "logps/rejected": -485.1358024691358,
      "loss": 0.2496,
      "rewards/chosen": 2.8005556147149266,
      "rewards/margins": 9.899321046813693,
      "rewards/rejected": -7.098765432098766,
      "step": 273
    },
    {
      "epoch": 0.13497536945812807,
      "grad_norm": 0.42606235067591725,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -81430877.65853658,
      "logits/rejected": -90731955.49425288,
      "logps/chosen": -372.2926829268293,
      "logps/rejected": -411.95402298850576,
      "loss": 0.2141,
      "rewards/chosen": 0.9222560975609756,
      "rewards/margins": 7.450991729744883,
      "rewards/rejected": -6.528735632183908,
      "step": 274
    },
    {
      "epoch": 0.1354679802955665,
      "grad_norm": 0.8048641346417821,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -110498215.72413793,
      "logits/rejected": -75737146.51428571,
      "logps/chosen": -401.37931034482756,
      "logps/rejected": -426.9714285714286,
      "loss": 0.3409,
      "rewards/chosen": -0.5767780172413793,
      "rewards/margins": 6.0160791256157635,
      "rewards/rejected": -6.5928571428571425,
      "step": 275
    },
    {
      "epoch": 0.13596059113300493,
      "grad_norm": 0.6419149197293943,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -78493403.42857143,
      "logits/rejected": -78789204.25316456,
      "logps/chosen": -344.48979591836735,
      "logps/rejected": -494.5822784810127,
      "loss": 0.249,
      "rewards/chosen": 0.5309311224489796,
      "rewards/margins": 7.277766565486955,
      "rewards/rejected": -6.746835443037975,
      "step": 276
    },
    {
      "epoch": 0.13645320197044336,
      "grad_norm": 0.5320359816230755,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -62518870.943396226,
      "logits/rejected": -102229169.49333334,
      "logps/chosen": -183.69811320754718,
      "logps/rejected": -526.5066666666667,
      "loss": 0.2767,
      "rewards/chosen": 1.886762798957105,
      "rewards/margins": 8.566762798957104,
      "rewards/rejected": -6.68,
      "step": 277
    },
    {
      "epoch": 0.13694581280788176,
      "grad_norm": 0.47426307256424866,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -82662741.33333333,
      "logits/rejected": -73610035.2,
      "logps/chosen": -312.8333333333333,
      "logps/rejected": -426.0,
      "loss": 0.2473,
      "rewards/chosen": 0.32421875,
      "rewards/margins": 6.39921875,
      "rewards/rejected": -6.075,
      "step": 278
    },
    {
      "epoch": 0.1374384236453202,
      "grad_norm": 0.5470050383745936,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -95333034.66666667,
      "logits/rejected": -87031808.0,
      "logps/chosen": -402.8333333333333,
      "logps/rejected": -419.2,
      "loss": 0.247,
      "rewards/chosen": 0.5559895833333334,
      "rewards/margins": 6.037239583333333,
      "rewards/rejected": -5.48125,
      "step": 279
    },
    {
      "epoch": 0.13793103448275862,
      "grad_norm": 0.5105631179258332,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -80421220.17391305,
      "logits/rejected": -92990788.68292683,
      "logps/chosen": -284.17391304347825,
      "logps/rejected": -452.6829268292683,
      "loss": 0.1983,
      "rewards/chosen": 1.218070652173913,
      "rewards/margins": 4.298837655675246,
      "rewards/rejected": -3.0807670035013337,
      "step": 280
    },
    {
      "epoch": 0.13842364532019705,
      "grad_norm": 0.9531436210108849,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -82422950.69767442,
      "logits/rejected": -88623176.28235294,
      "logps/chosen": -327.4418604651163,
      "logps/rejected": -482.63529411764705,
      "loss": 0.2115,
      "rewards/chosen": 3.235634737236555,
      "rewards/margins": 8.71210532547185,
      "rewards/rejected": -5.476470588235294,
      "step": 281
    },
    {
      "epoch": 0.13891625615763548,
      "grad_norm": 0.8108541787257343,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -73222595.2542373,
      "logits/rejected": -78415248.69565217,
      "logps/chosen": -313.76271186440675,
      "logps/rejected": -410.4347826086956,
      "loss": 0.3498,
      "rewards/chosen": -0.1965042372881356,
      "rewards/margins": 4.8832059076394,
      "rewards/rejected": -5.079710144927536,
      "step": 282
    },
    {
      "epoch": 0.1394088669950739,
      "grad_norm": 1.0261451842367517,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -80111206.4,
      "logits/rejected": -71303168.0,
      "logps/chosen": -390.2,
      "logps/rejected": -449.45454545454544,
      "loss": 0.219,
      "rewards/chosen": 6.150386047363281,
      "rewards/margins": 11.974249683726917,
      "rewards/rejected": -5.823863636363637,
      "step": 283
    },
    {
      "epoch": 0.1399014778325123,
      "grad_norm": 0.517207876684435,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -88967640.61538461,
      "logits/rejected": -86645490.5263158,
      "logps/chosen": -286.46153846153845,
      "logps/rejected": -414.7368421052632,
      "loss": 0.2682,
      "rewards/chosen": 3.0708741408128004,
      "rewards/margins": 8.998505719760168,
      "rewards/rejected": -5.927631578947368,
      "step": 284
    },
    {
      "epoch": 0.14039408866995073,
      "grad_norm": 0.48167235429167454,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -75771013.56521739,
      "logits/rejected": -74372170.92682926,
      "logps/chosen": -296.69565217391306,
      "logps/rejected": -426.1463414634146,
      "loss": 0.2485,
      "rewards/chosen": 0.04483695652173913,
      "rewards/margins": 5.374105249204666,
      "rewards/rejected": -5.329268292682927,
      "step": 285
    },
    {
      "epoch": 0.14088669950738916,
      "grad_norm": 0.6162899950787905,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -63082332.16,
      "logits/rejected": -94318066.87179486,
      "logps/chosen": -225.28,
      "logps/rejected": -513.6410256410256,
      "loss": 0.2778,
      "rewards/chosen": 2.7064556884765625,
      "rewards/margins": 8.501327483348357,
      "rewards/rejected": -5.794871794871795,
      "step": 286
    },
    {
      "epoch": 0.1413793103448276,
      "grad_norm": 0.42110592512763784,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -72997021.53846154,
      "logits/rejected": -80795540.21052632,
      "logps/chosen": -274.7692307692308,
      "logps/rejected": -470.3157894736842,
      "loss": 0.2608,
      "rewards/chosen": 2.8707339947040262,
      "rewards/margins": 8.8312603104935,
      "rewards/rejected": -5.9605263157894735,
      "step": 287
    },
    {
      "epoch": 0.14187192118226602,
      "grad_norm": 0.4366554569165783,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -68388482.16949153,
      "logits/rejected": -76348490.20289855,
      "logps/chosen": -205.96610169491527,
      "logps/rejected": -402.0869565217391,
      "loss": 0.2921,
      "rewards/chosen": 0.2232521186440678,
      "rewards/margins": 2.0737299177694073,
      "rewards/rejected": -1.8504777991253396,
      "step": 288
    },
    {
      "epoch": 0.14236453201970442,
      "grad_norm": 0.5453900205197487,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -60978727.384615384,
      "logits/rejected": -97929929.34831461,
      "logps/chosen": -211.48717948717947,
      "logps/rejected": -552.2696629213483,
      "loss": 0.2363,
      "rewards/chosen": 0.26282051282051283,
      "rewards/margins": 6.139225007202535,
      "rewards/rejected": -5.876404494382022,
      "step": 289
    },
    {
      "epoch": 0.14285714285714285,
      "grad_norm": 0.5216450403305182,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -74467292.07017544,
      "logits/rejected": -87253338.14084508,
      "logps/chosen": -293.89473684210526,
      "logps/rejected": -402.92957746478874,
      "loss": 0.2759,
      "rewards/chosen": 0.484375,
      "rewards/margins": 5.878741197183099,
      "rewards/rejected": -5.394366197183099,
      "step": 290
    },
    {
      "epoch": 0.14334975369458128,
      "grad_norm": 0.4870581794068908,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -58541774.97872341,
      "logits/rejected": -77879422.41975309,
      "logps/chosen": -246.12765957446808,
      "logps/rejected": -419.55555555555554,
      "loss": 0.211,
      "rewards/chosen": 6.190145289644282,
      "rewards/margins": 11.918540351372677,
      "rewards/rejected": -5.728395061728395,
      "step": 291
    },
    {
      "epoch": 0.1438423645320197,
      "grad_norm": 0.4836797120913992,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -97932121.30232558,
      "logits/rejected": -83392632.47058824,
      "logps/chosen": -253.02325581395348,
      "logps/rejected": -411.8588235294118,
      "loss": 0.2174,
      "rewards/chosen": 1.0813953488372092,
      "rewards/margins": 7.099042407660739,
      "rewards/rejected": -6.017647058823529,
      "step": 292
    },
    {
      "epoch": 0.14433497536945813,
      "grad_norm": 0.3700720771412705,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -53991383.84313726,
      "logits/rejected": -74190156.46753247,
      "logps/chosen": -174.58823529411765,
      "logps/rejected": -431.7922077922078,
      "loss": 0.2114,
      "rewards/chosen": 2.567973118202359,
      "rewards/margins": 8.717323767553008,
      "rewards/rejected": -6.14935064935065,
      "step": 293
    },
    {
      "epoch": 0.14482758620689656,
      "grad_norm": 0.4336125585942127,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -62411243.52,
      "logits/rejected": -83617214.35897435,
      "logps/chosen": -258.88,
      "logps/rejected": -445.53846153846155,
      "loss": 0.2167,
      "rewards/chosen": 3.2955767822265627,
      "rewards/margins": 10.193012679662461,
      "rewards/rejected": -6.897435897435898,
      "step": 294
    },
    {
      "epoch": 0.14532019704433496,
      "grad_norm": 0.48952254799350176,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -61117001.14285714,
      "logits/rejected": -79320128.81012659,
      "logps/chosen": -249.9591836734694,
      "logps/rejected": -448.8101265822785,
      "loss": 0.2836,
      "rewards/chosen": -0.579719387755102,
      "rewards/margins": 6.23673630844743,
      "rewards/rejected": -6.8164556962025316,
      "step": 295
    },
    {
      "epoch": 0.1458128078817734,
      "grad_norm": 0.4515206508244321,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -64520889.19148936,
      "logits/rejected": -93517444.74074075,
      "logps/chosen": -327.48936170212767,
      "logps/rejected": -478.0246913580247,
      "loss": 0.2651,
      "rewards/chosen": -0.6628989361702128,
      "rewards/margins": 5.528459088521146,
      "rewards/rejected": -6.191358024691358,
      "step": 296
    },
    {
      "epoch": 0.14630541871921182,
      "grad_norm": 0.4053074226221189,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -54525952.0,
      "logits/rejected": -73972270.54545455,
      "logps/chosen": -237.9,
      "logps/rejected": -437.8181818181818,
      "loss": 0.2048,
      "rewards/chosen": 7.226457214355468,
      "rewards/margins": 14.976457214355468,
      "rewards/rejected": -7.75,
      "step": 297
    },
    {
      "epoch": 0.14679802955665025,
      "grad_norm": 0.5572159179844524,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -72421649.06666666,
      "logits/rejected": -72162242.31325302,
      "logps/chosen": -289.6,
      "logps/rejected": -456.48192771084337,
      "loss": 0.261,
      "rewards/chosen": -0.4892361111111111,
      "rewards/margins": 6.5348602744310575,
      "rewards/rejected": -7.024096385542169,
      "step": 298
    },
    {
      "epoch": 0.14729064039408868,
      "grad_norm": 0.44594644640126496,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -67011322.04651163,
      "logits/rejected": -75398782.49411765,
      "logps/chosen": -306.9767441860465,
      "logps/rejected": -434.0705882352941,
      "loss": 0.2095,
      "rewards/chosen": 3.408422070880269,
      "rewards/margins": 11.243716188527328,
      "rewards/rejected": -7.8352941176470585,
      "step": 299
    },
    {
      "epoch": 0.1477832512315271,
      "grad_norm": 0.5074754182344536,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -81418842.35294117,
      "logits/rejected": -83559251.11688311,
      "logps/chosen": -354.5098039215686,
      "logps/rejected": -452.15584415584414,
      "loss": 0.2319,
      "rewards/chosen": 3.36583724676394,
      "rewards/margins": 11.01518789611459,
      "rewards/rejected": -7.64935064935065,
      "step": 300
    },
    {
      "epoch": 0.1482758620689655,
      "grad_norm": 0.5108823680991963,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -81788928.0,
      "logits/rejected": -91865487.6097561,
      "logps/chosen": -259.82608695652175,
      "logps/rejected": -472.5853658536585,
      "loss": 0.2423,
      "rewards/chosen": 0.09442934782608696,
      "rewards/margins": 6.0323030020753245,
      "rewards/rejected": -5.937873654249238,
      "step": 301
    },
    {
      "epoch": 0.14876847290640394,
      "grad_norm": 0.704928199638884,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -74482721.03225806,
      "logits/rejected": -84775780.84848484,
      "logps/chosen": -309.6774193548387,
      "logps/rejected": -519.7575757575758,
      "loss": 0.2542,
      "rewards/chosen": 0.5584677419354839,
      "rewards/margins": 10.255437438905181,
      "rewards/rejected": -9.696969696969697,
      "step": 302
    },
    {
      "epoch": 0.14926108374384237,
      "grad_norm": 0.4194669496047058,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -80303445.33333333,
      "logits/rejected": -85249228.8,
      "logps/chosen": -269.6666666666667,
      "logps/rejected": -468.8,
      "loss": 0.2399,
      "rewards/chosen": 3.3458360036214194,
      "rewards/margins": 10.88958600362142,
      "rewards/rejected": -7.54375,
      "step": 303
    },
    {
      "epoch": 0.1497536945812808,
      "grad_norm": 0.6565230959810632,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -85902572.3076923,
      "logits/rejected": -98455767.57894737,
      "logps/chosen": -403.6923076923077,
      "logps/rejected": -531.3684210526316,
      "loss": 0.2498,
      "rewards/chosen": 2.2436614403357873,
      "rewards/margins": 11.269977229809472,
      "rewards/rejected": -9.026315789473685,
      "step": 304
    },
    {
      "epoch": 0.15024630541871922,
      "grad_norm": 0.5816936250879824,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -67979379.9245283,
      "logits/rejected": -85675649.70666666,
      "logps/chosen": -229.73584905660377,
      "logps/rejected": -459.0933333333333,
      "loss": 0.2578,
      "rewards/chosen": 0.15330188679245282,
      "rewards/margins": 9.366635220125787,
      "rewards/rejected": -9.213333333333333,
      "step": 305
    },
    {
      "epoch": 0.15073891625615762,
      "grad_norm": 0.40355910617705815,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -51566637.51111111,
      "logits/rejected": -95812052.81927711,
      "logps/chosen": -275.73333333333335,
      "logps/rejected": -478.0722891566265,
      "loss": 0.2354,
      "rewards/chosen": -1.16875,
      "rewards/margins": 8.13245481927711,
      "rewards/rejected": -9.301204819277109,
      "step": 306
    },
    {
      "epoch": 0.15123152709359605,
      "grad_norm": 0.5736397329944254,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -69537145.26315789,
      "logits/rejected": -83531631.77464789,
      "logps/chosen": -300.63157894736844,
      "logps/rejected": -507.0422535211268,
      "loss": 0.2893,
      "rewards/chosen": 1.8760920742101836,
      "rewards/margins": 10.66482446857638,
      "rewards/rejected": -8.788732394366198,
      "step": 307
    },
    {
      "epoch": 0.15172413793103448,
      "grad_norm": 0.4722722360844266,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -67061201.45454545,
      "logits/rejected": -84984588.1904762,
      "logps/chosen": -240.36363636363637,
      "logps/rejected": -514.2857142857143,
      "loss": 0.1992,
      "rewards/chosen": 0.8338068181818182,
      "rewards/margins": 9.184997294372295,
      "rewards/rejected": -8.351190476190476,
      "step": 308
    },
    {
      "epoch": 0.1522167487684729,
      "grad_norm": 0.4241053266067081,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -71215786.66666667,
      "logits/rejected": -84095795.2,
      "logps/chosen": -255.0,
      "logps/rejected": -460.8,
      "loss": 0.2366,
      "rewards/chosen": 0.14192708333333334,
      "rewards/margins": 4.724688466389973,
      "rewards/rejected": -4.58276138305664,
      "step": 309
    },
    {
      "epoch": 0.15270935960591134,
      "grad_norm": 0.6140443486563215,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -112110250.66666667,
      "logits/rejected": -74973184.0,
      "logps/chosen": -296.3333333333333,
      "logps/rejected": -476.8,
      "loss": 0.2397,
      "rewards/chosen": 0.192626953125,
      "rewards/margins": 10.442626953125,
      "rewards/rejected": -10.25,
      "step": 310
    },
    {
      "epoch": 0.15320197044334977,
      "grad_norm": 0.6533912157423709,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -76277807.62790698,
      "logits/rejected": -83590011.48235294,
      "logps/chosen": -273.86046511627904,
      "logps/rejected": -513.1294117647059,
      "loss": 0.2142,
      "rewards/chosen": 0.3417514534883721,
      "rewards/margins": 11.035869100547195,
      "rewards/rejected": -10.694117647058823,
      "step": 311
    },
    {
      "epoch": 0.15369458128078817,
      "grad_norm": 0.6225993138266218,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -93809189.46341464,
      "logits/rejected": -73569056.36781609,
      "logps/chosen": -424.9756097560976,
      "logps/rejected": -427.0344827586207,
      "loss": 0.2135,
      "rewards/chosen": 3.834535924399771,
      "rewards/margins": 13.466719832445749,
      "rewards/rejected": -9.632183908045977,
      "step": 312
    },
    {
      "epoch": 0.1541871921182266,
      "grad_norm": 0.42582351566965554,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -64285774.76923077,
      "logits/rejected": -98124638.31578948,
      "logps/chosen": -226.6153846153846,
      "logps/rejected": -464.42105263157896,
      "loss": 0.2574,
      "rewards/chosen": 0.3557692307692308,
      "rewards/margins": 9.211032388663966,
      "rewards/rejected": -8.855263157894736,
      "step": 313
    },
    {
      "epoch": 0.15467980295566502,
      "grad_norm": 0.5162956027783581,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -85641835.1627907,
      "logits/rejected": -82405737.41176471,
      "logps/chosen": -369.1162790697674,
      "logps/rejected": -457.0352941176471,
      "loss": 0.2462,
      "rewards/chosen": 6.169246939725654,
      "rewards/margins": 14.333952822078594,
      "rewards/rejected": -8.16470588235294,
      "step": 314
    },
    {
      "epoch": 0.15517241379310345,
      "grad_norm": 0.5150900203010241,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -68234165.07317074,
      "logits/rejected": -87742911.26436782,
      "logps/chosen": -357.6585365853659,
      "logps/rejected": -518.9885057471264,
      "loss": 0.2063,
      "rewards/chosen": 4.4470129245665015,
      "rewards/margins": 13.573449706175698,
      "rewards/rejected": -9.126436781609195,
      "step": 315
    },
    {
      "epoch": 0.15566502463054188,
      "grad_norm": 0.4933994074590794,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -97779712.0,
      "logits/rejected": -82103500.8,
      "logps/chosen": -364.0,
      "logps/rejected": -501.2,
      "loss": 0.2645,
      "rewards/chosen": -0.3297526041666667,
      "rewards/margins": 8.613997395833334,
      "rewards/rejected": -8.94375,
      "step": 316
    },
    {
      "epoch": 0.15615763546798028,
      "grad_norm": 0.4888937100897653,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -75826437.01960784,
      "logits/rejected": -108180360.31168832,
      "logps/chosen": -326.5882352941176,
      "logps/rejected": -464.6233766233766,
      "loss": 0.2206,
      "rewards/chosen": 2.522672167011336,
      "rewards/margins": 11.379815024154194,
      "rewards/rejected": -8.857142857142858,
      "step": 317
    },
    {
      "epoch": 0.1566502463054187,
      "grad_norm": 0.4410152931587118,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -80996670.57777777,
      "logits/rejected": -81056188.14457831,
      "logps/chosen": -267.02222222222224,
      "logps/rejected": -460.33734939759034,
      "loss": 0.243,
      "rewards/chosen": 2.3569678412543404,
      "rewards/margins": 11.91118470872422,
      "rewards/rejected": -9.55421686746988,
      "step": 318
    },
    {
      "epoch": 0.15714285714285714,
      "grad_norm": 0.4342769764280174,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -80645026.9090909,
      "logits/rejected": -84485266.28571428,
      "logps/chosen": -274.54545454545456,
      "logps/rejected": -528.7619047619048,
      "loss": 0.1916,
      "rewards/chosen": 0.3103693181818182,
      "rewards/margins": 11.393702651515152,
      "rewards/rejected": -11.083333333333334,
      "step": 319
    },
    {
      "epoch": 0.15763546798029557,
      "grad_norm": 0.9307649697898638,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -72966426.48275863,
      "logits/rejected": -83286893.71428572,
      "logps/chosen": -346.48275862068965,
      "logps/rejected": -522.5142857142857,
      "loss": 0.2923,
      "rewards/chosen": 2.050897006330819,
      "rewards/margins": 12.12232557775939,
      "rewards/rejected": -10.071428571428571,
      "step": 320
    },
    {
      "epoch": 0.158128078817734,
      "grad_norm": 0.4606339182308189,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -76952638.69387755,
      "logits/rejected": -87814921.721519,
      "logps/chosen": -283.1020408163265,
      "logps/rejected": -476.3544303797468,
      "loss": 0.234,
      "rewards/chosen": -0.6122448979591837,
      "rewards/margins": 8.80547662102816,
      "rewards/rejected": -9.417721518987342,
      "step": 321
    },
    {
      "epoch": 0.15862068965517243,
      "grad_norm": 0.612992364517869,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -73765042.08695652,
      "logits/rejected": -77850374.24390244,
      "logps/chosen": -321.7391304347826,
      "logps/rejected": -504.1951219512195,
      "loss": 0.2297,
      "rewards/chosen": -0.45991847826086957,
      "rewards/margins": 10.064471765641569,
      "rewards/rejected": -10.524390243902438,
      "step": 322
    },
    {
      "epoch": 0.15911330049261083,
      "grad_norm": 0.5008572717752803,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -75274370.72340426,
      "logits/rejected": -84507458.37037037,
      "logps/chosen": -312.3404255319149,
      "logps/rejected": -451.55555555555554,
      "loss": 0.216,
      "rewards/chosen": 0.35638297872340424,
      "rewards/margins": 8.677370633044392,
      "rewards/rejected": -8.320987654320987,
      "step": 323
    },
    {
      "epoch": 0.15960591133004925,
      "grad_norm": 0.40842362477161853,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -75839864.1632653,
      "logits/rejected": -69338747.1392405,
      "logps/chosen": -278.0408163265306,
      "logps/rejected": -413.9746835443038,
      "loss": 0.277,
      "rewards/chosen": 1.8161459163743623,
      "rewards/margins": 8.99969022017183,
      "rewards/rejected": -7.1835443037974684,
      "step": 324
    },
    {
      "epoch": 0.16009852216748768,
      "grad_norm": 0.5463973395895309,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -112788647.56363636,
      "logits/rejected": -82047480.98630136,
      "logps/chosen": -368.2909090909091,
      "logps/rejected": -514.1917808219179,
      "loss": 0.2741,
      "rewards/chosen": -0.014204545454545454,
      "rewards/margins": 10.520042029887922,
      "rewards/rejected": -10.534246575342467,
      "step": 325
    },
    {
      "epoch": 0.1605911330049261,
      "grad_norm": 0.48173699704138845,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -85029981.0909091,
      "logits/rejected": -75597336.38095239,
      "logps/chosen": -342.54545454545456,
      "logps/rejected": -439.6190476190476,
      "loss": 0.2086,
      "rewards/chosen": 0.44105113636363635,
      "rewards/margins": 8.96486066017316,
      "rewards/rejected": -8.523809523809524,
      "step": 326
    },
    {
      "epoch": 0.16108374384236454,
      "grad_norm": 0.46673340719689577,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -65084027.5862069,
      "logits/rejected": -79871531.88571429,
      "logps/chosen": -274.48275862068965,
      "logps/rejected": -494.62857142857143,
      "loss": 0.2512,
      "rewards/chosen": 0.28953394396551724,
      "rewards/margins": 9.775248229679804,
      "rewards/rejected": -9.485714285714286,
      "step": 327
    },
    {
      "epoch": 0.16157635467980297,
      "grad_norm": 0.42672048602554963,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -68581332.42553191,
      "logits/rejected": -86268030.41975309,
      "logps/chosen": -247.48936170212767,
      "logps/rejected": -503.30864197530866,
      "loss": 0.2152,
      "rewards/chosen": 0.4276097074468085,
      "rewards/margins": 10.34118995436039,
      "rewards/rejected": -9.91358024691358,
      "step": 328
    },
    {
      "epoch": 0.16206896551724137,
      "grad_norm": 0.5887275096208967,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -88866816.0,
      "logits/rejected": -69940019.2,
      "logps/chosen": -332.6666666666667,
      "logps/rejected": -460.0,
      "loss": 0.2084,
      "rewards/chosen": 0.7102864583333334,
      "rewards/margins": 9.872786458333334,
      "rewards/rejected": -9.1625,
      "step": 329
    },
    {
      "epoch": 0.1625615763546798,
      "grad_norm": 0.5779820906415946,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -66509677.71428572,
      "logits/rejected": -83536554.66666667,
      "logps/chosen": -245.42857142857142,
      "logps/rejected": -480.0,
      "loss": 0.2936,
      "rewards/chosen": -1.0412946428571428,
      "rewards/margins": 7.847594246031747,
      "rewards/rejected": -8.88888888888889,
      "step": 330
    },
    {
      "epoch": 0.16305418719211823,
      "grad_norm": 0.5332462700310877,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -81274532.2264151,
      "logits/rejected": -70184686.93333334,
      "logps/chosen": -310.64150943396226,
      "logps/rejected": -439.04,
      "loss": 0.2679,
      "rewards/chosen": -0.5058962264150944,
      "rewards/margins": 7.8674371069182385,
      "rewards/rejected": -8.373333333333333,
      "step": 331
    },
    {
      "epoch": 0.16354679802955666,
      "grad_norm": 0.5603473011038073,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -83207589.64705883,
      "logits/rejected": -76568358.12765957,
      "logps/chosen": -293.88235294117646,
      "logps/rejected": -450.0425531914894,
      "loss": 0.1668,
      "rewards/chosen": 4.532152961282169,
      "rewards/margins": 12.798110408090679,
      "rewards/rejected": -8.26595744680851,
      "step": 332
    },
    {
      "epoch": 0.16403940886699508,
      "grad_norm": 0.5369993477157926,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -70482543.30434783,
      "logits/rejected": -86955082.92682926,
      "logps/chosen": -278.0869565217391,
      "logps/rejected": -547.9024390243902,
      "loss": 0.2232,
      "rewards/chosen": 3.2710246210512905,
      "rewards/margins": 13.636878279587876,
      "rewards/rejected": -10.365853658536585,
      "step": 333
    },
    {
      "epoch": 0.16453201970443349,
      "grad_norm": 0.6333843528658143,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -93323264.0,
      "logits/rejected": -77594624.0,
      "logps/chosen": -294.9230769230769,
      "logps/rejected": -461.89473684210526,
      "loss": 0.2305,
      "rewards/chosen": 2.1968124096210184,
      "rewards/margins": 10.992865041199966,
      "rewards/rejected": -8.796052631578947,
      "step": 334
    },
    {
      "epoch": 0.16502463054187191,
      "grad_norm": 0.5136330831213423,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -76429539.55555555,
      "logits/rejected": -75440792.21621622,
      "logps/chosen": -333.6296296296296,
      "logps/rejected": -460.97297297297297,
      "loss": 0.2562,
      "rewards/chosen": -0.1814236111111111,
      "rewards/margins": 8.663170983483484,
      "rewards/rejected": -8.844594594594595,
      "step": 335
    },
    {
      "epoch": 0.16551724137931034,
      "grad_norm": 0.5679434228317332,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -81208864.68085106,
      "logits/rejected": -75911724.24691358,
      "logps/chosen": -363.2340425531915,
      "logps/rejected": -466.962962962963,
      "loss": 0.2496,
      "rewards/chosen": -0.944813829787234,
      "rewards/margins": 8.573704688731285,
      "rewards/rejected": -9.518518518518519,
      "step": 336
    },
    {
      "epoch": 0.16600985221674877,
      "grad_norm": 0.42415606852745674,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -81971289.04347827,
      "logits/rejected": -84295280.3902439,
      "logps/chosen": -270.2608695652174,
      "logps/rejected": -532.2926829268292,
      "loss": 0.1908,
      "rewards/chosen": 3.1786459217900815,
      "rewards/margins": 13.48352397057057,
      "rewards/rejected": -10.304878048780488,
      "step": 337
    },
    {
      "epoch": 0.1665024630541872,
      "grad_norm": 0.41148290729825676,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -64172851.2,
      "logits/rejected": -87650198.97435898,
      "logps/chosen": -217.28,
      "logps/rejected": -476.71794871794873,
      "loss": 0.2475,
      "rewards/chosen": 1.5202943420410155,
      "rewards/margins": 11.148499470246144,
      "rewards/rejected": -9.628205128205128,
      "step": 338
    },
    {
      "epoch": 0.16699507389162563,
      "grad_norm": 0.47425054144651263,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -77894217.14285715,
      "logits/rejected": -74647992.70886075,
      "logps/chosen": -270.6938775510204,
      "logps/rejected": -448.8101265822785,
      "loss": 0.2119,
      "rewards/chosen": 3.058014538823342,
      "rewards/margins": 11.602318336291695,
      "rewards/rejected": -8.544303797468354,
      "step": 339
    },
    {
      "epoch": 0.16748768472906403,
      "grad_norm": 0.369310842057698,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -60719866.04651163,
      "logits/rejected": -95728820.70588236,
      "logps/chosen": -282.4186046511628,
      "logps/rejected": -491.67058823529413,
      "loss": 0.1854,
      "rewards/chosen": 1.164607558139535,
      "rewards/margins": 9.729313440492476,
      "rewards/rejected": -8.564705882352941,
      "step": 340
    },
    {
      "epoch": 0.16798029556650246,
      "grad_norm": 0.7725868047189061,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -84132803.76470588,
      "logits/rejected": -94017775.37662338,
      "logps/chosen": -441.0980392156863,
      "logps/rejected": -487.8961038961039,
      "loss": 0.2351,
      "rewards/chosen": 1.4586362651750153,
      "rewards/margins": 10.731363537902288,
      "rewards/rejected": -9.272727272727273,
      "step": 341
    },
    {
      "epoch": 0.1684729064039409,
      "grad_norm": 0.42575644714946004,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -78193810.28571428,
      "logits/rejected": -75985181.76744185,
      "logps/chosen": -306.85714285714283,
      "logps/rejected": -499.3488372093023,
      "loss": 0.1824,
      "rewards/chosen": 3.1941481090727306,
      "rewards/margins": 12.973217876514592,
      "rewards/rejected": -9.779069767441861,
      "step": 342
    },
    {
      "epoch": 0.16896551724137931,
      "grad_norm": 0.48251902795007284,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -65130418.716981135,
      "logits/rejected": -81760965.97333333,
      "logps/chosen": -256.60377358490564,
      "logps/rejected": -476.16,
      "loss": 0.2497,
      "rewards/chosen": 0.2474941037735849,
      "rewards/margins": 8.987494103773585,
      "rewards/rejected": -8.74,
      "step": 343
    },
    {
      "epoch": 0.16945812807881774,
      "grad_norm": 0.429497380300187,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -92798976.0,
      "logits/rejected": -84095795.2,
      "logps/chosen": -398.0,
      "logps/rejected": -465.2,
      "loss": 0.218,
      "rewards/chosen": 8.809876759847006,
      "rewards/margins": 17.666126759847003,
      "rewards/rejected": -8.85625,
      "step": 344
    },
    {
      "epoch": 0.16995073891625614,
      "grad_norm": 0.496693306852464,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -66302267.07692308,
      "logits/rejected": -82561562.94736843,
      "logps/chosen": -255.84615384615384,
      "logps/rejected": -504.42105263157896,
      "loss": 0.2601,
      "rewards/chosen": 3.04264890230619,
      "rewards/margins": 11.950543639148295,
      "rewards/rejected": -8.907894736842104,
      "step": 345
    },
    {
      "epoch": 0.17044334975369457,
      "grad_norm": 0.42315064593352253,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -62015780.571428575,
      "logits/rejected": -92740721.77777778,
      "logps/chosen": -248.0,
      "logps/rejected": -462.22222222222223,
      "loss": 0.2669,
      "rewards/chosen": -0.35714285714285715,
      "rewards/margins": 5.511947752937439,
      "rewards/rejected": -5.8690906100802955,
      "step": 346
    },
    {
      "epoch": 0.170935960591133,
      "grad_norm": 0.5505104708545758,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -73642299.07692307,
      "logits/rejected": -78919141.05263157,
      "logps/chosen": -249.3846153846154,
      "logps/rejected": -489.2631578947368,
      "loss": 0.2427,
      "rewards/chosen": 4.290128267728365,
      "rewards/margins": 12.592759846675733,
      "rewards/rejected": -8.302631578947368,
      "step": 347
    },
    {
      "epoch": 0.17142857142857143,
      "grad_norm": 0.4248005685771568,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -72939969.56097561,
      "logits/rejected": -73906529.10344827,
      "logps/chosen": -327.4146341463415,
      "logps/rejected": -504.64367816091954,
      "loss": 0.2383,
      "rewards/chosen": 3.717975895579268,
      "rewards/margins": 11.988090838108004,
      "rewards/rejected": -8.270114942528735,
      "step": 348
    },
    {
      "epoch": 0.17192118226600986,
      "grad_norm": 0.4192287030617427,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -82737639.61904761,
      "logits/rejected": -79886859.90697674,
      "logps/chosen": -271.42857142857144,
      "logps/rejected": -467.3488372093023,
      "loss": 0.1908,
      "rewards/chosen": 3.30047607421875,
      "rewards/margins": 12.288848167242007,
      "rewards/rejected": -8.988372093023257,
      "step": 349
    },
    {
      "epoch": 0.1724137931034483,
      "grad_norm": 0.6317666870418222,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -76763676.98113208,
      "logits/rejected": -93505017.17333333,
      "logps/chosen": -312.75471698113205,
      "logps/rejected": -491.94666666666666,
      "loss": 0.3266,
      "rewards/chosen": -1.178287146226415,
      "rewards/margins": 6.7150461871069185,
      "rewards/rejected": -7.8933333333333335,
      "step": 350
    },
    {
      "epoch": 0.1729064039408867,
      "grad_norm": 0.4055374026769498,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -77815376.84210527,
      "logits/rejected": -75916902.4,
      "logps/chosen": -343.1578947368421,
      "logps/rejected": -513.7777777777778,
      "loss": 0.1953,
      "rewards/chosen": 0.5674342105263158,
      "rewards/margins": 9.211878654970759,
      "rewards/rejected": -8.644444444444444,
      "step": 351
    },
    {
      "epoch": 0.17339901477832512,
      "grad_norm": 0.3496655884582423,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -59513772.972972974,
      "logits/rejected": -81120604.83516483,
      "logps/chosen": -242.16216216216216,
      "logps/rejected": -469.45054945054943,
      "loss": 0.1952,
      "rewards/chosen": 7.78793664880701,
      "rewards/margins": 15.595628956499318,
      "rewards/rejected": -7.8076923076923075,
      "step": 352
    },
    {
      "epoch": 0.17389162561576355,
      "grad_norm": 0.5111945153566491,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -72929530.7755102,
      "logits/rejected": -120095134.78481013,
      "logps/chosen": -271.6734693877551,
      "logps/rejected": -490.126582278481,
      "loss": 0.2321,
      "rewards/chosen": 2.512111741669324,
      "rewards/margins": 8.537428197365527,
      "rewards/rejected": -6.025316455696203,
      "step": 353
    },
    {
      "epoch": 0.17438423645320197,
      "grad_norm": 0.3725506748383311,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -76814288.37209302,
      "logits/rejected": -93261583.05882353,
      "logps/chosen": -291.90697674418607,
      "logps/rejected": -460.0470588235294,
      "loss": 0.1915,
      "rewards/chosen": 1.208575581395349,
      "rewards/margins": 8.302693228454173,
      "rewards/rejected": -7.094117647058823,
      "step": 354
    },
    {
      "epoch": 0.1748768472906404,
      "grad_norm": 0.5220892804601454,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -73437112.14035088,
      "logits/rejected": -96055469.07042253,
      "logps/chosen": -327.0175438596491,
      "logps/rejected": -477.2957746478873,
      "loss": 0.2481,
      "rewards/chosen": 0.5307017543859649,
      "rewards/margins": 8.425067951569064,
      "rewards/rejected": -7.894366197183099,
      "step": 355
    },
    {
      "epoch": 0.17536945812807883,
      "grad_norm": 0.42926755036330017,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -68756626.28571428,
      "logits/rejected": -92079604.09302326,
      "logps/chosen": -253.42857142857142,
      "logps/rejected": -482.9767441860465,
      "loss": 0.1936,
      "rewards/chosen": 3.1814658755347844,
      "rewards/margins": 11.076814712744087,
      "rewards/rejected": -7.895348837209302,
      "step": 356
    },
    {
      "epoch": 0.17586206896551723,
      "grad_norm": 0.4234120688213137,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -75241721.75609756,
      "logits/rejected": -78004412.32183908,
      "logps/chosen": -341.4634146341463,
      "logps/rejected": -500.2298850574713,
      "loss": 0.2142,
      "rewards/chosen": 1.950829482660061,
      "rewards/margins": 9.651978907947417,
      "rewards/rejected": -7.7011494252873565,
      "step": 357
    },
    {
      "epoch": 0.17635467980295566,
      "grad_norm": 0.41093836636408376,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -53170475.70731708,
      "logits/rejected": -86200178.7586207,
      "logps/chosen": -278.6341463414634,
      "logps/rejected": -473.01149425287355,
      "loss": 0.208,
      "rewards/chosen": 3.047394170993712,
      "rewards/margins": 10.277279228464977,
      "rewards/rejected": -7.2298850574712645,
      "step": 358
    },
    {
      "epoch": 0.1768472906403941,
      "grad_norm": 0.4807984937304573,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -67863838.72,
      "logits/rejected": -80229507.28205128,
      "logps/chosen": -323.36,
      "logps/rejected": -441.84615384615387,
      "loss": 0.2412,
      "rewards/chosen": 4.06433349609375,
      "rewards/margins": 11.057923239683493,
      "rewards/rejected": -6.993589743589744,
      "step": 359
    },
    {
      "epoch": 0.17733990147783252,
      "grad_norm": 0.3963406668549604,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -72775636.42553191,
      "logits/rejected": -75497472.0,
      "logps/chosen": -346.21276595744683,
      "logps/rejected": -435.75308641975306,
      "loss": 0.2413,
      "rewards/chosen": -0.20079787234042554,
      "rewards/margins": 7.7127823745731545,
      "rewards/rejected": -7.91358024691358,
      "step": 360
    },
    {
      "epoch": 0.17783251231527095,
      "grad_norm": 0.4247698699267358,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -58605865.89090909,
      "logits/rejected": -81817656.10958904,
      "logps/chosen": -239.70909090909092,
      "logps/rejected": -458.0821917808219,
      "loss": 0.245,
      "rewards/chosen": 0.5681818181818182,
      "rewards/margins": 9.088729763387297,
      "rewards/rejected": -8.520547945205479,
      "step": 361
    },
    {
      "epoch": 0.17832512315270935,
      "grad_norm": 0.4612659951584332,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -82192226.46153846,
      "logits/rejected": -89570465.68421052,
      "logps/chosen": -267.84615384615387,
      "logps/rejected": -501.05263157894734,
      "loss": 0.2442,
      "rewards/chosen": -0.08173076923076923,
      "rewards/margins": 8.694585020242915,
      "rewards/rejected": -8.776315789473685,
      "step": 362
    },
    {
      "epoch": 0.17881773399014778,
      "grad_norm": 0.4227429010354994,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -111310375.38461539,
      "logits/rejected": -100332166.73684211,
      "logps/chosen": -338.46153846153845,
      "logps/rejected": -522.1052631578947,
      "loss": 0.2644,
      "rewards/chosen": -1.0378605769230769,
      "rewards/margins": 5.409507844129555,
      "rewards/rejected": -6.447368421052632,
      "step": 363
    },
    {
      "epoch": 0.1793103448275862,
      "grad_norm": 0.481977844469611,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -90215666.03636363,
      "logits/rejected": -94687849.20547946,
      "logps/chosen": -271.7090909090909,
      "logps/rejected": -477.8082191780822,
      "loss": 0.2285,
      "rewards/chosen": 2.528425181995739,
      "rewards/margins": 8.084873922497664,
      "rewards/rejected": -5.556448740501926,
      "step": 364
    },
    {
      "epoch": 0.17980295566502463,
      "grad_norm": 0.3653055730042976,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -69905066.66666667,
      "logits/rejected": -106220748.8,
      "logps/chosen": -236.33333333333334,
      "logps/rejected": -516.8,
      "loss": 0.2277,
      "rewards/chosen": 3.436952590942383,
      "rewards/margins": 12.161952590942382,
      "rewards/rejected": -8.725,
      "step": 365
    },
    {
      "epoch": 0.18029556650246306,
      "grad_norm": 0.5224975050680682,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -65090849.81132075,
      "logits/rejected": -76336332.8,
      "logps/chosen": -241.20754716981133,
      "logps/rejected": -465.49333333333334,
      "loss": 0.2415,
      "rewards/chosen": 2.5026253754237913,
      "rewards/margins": 11.675958708757125,
      "rewards/rejected": -9.173333333333334,
      "step": 366
    },
    {
      "epoch": 0.1807881773399015,
      "grad_norm": 0.3999464276632189,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -58363293.957446806,
      "logits/rejected": -84921710.61728396,
      "logps/chosen": -240.3404255319149,
      "logps/rejected": -448.39506172839504,
      "loss": 0.2209,
      "rewards/chosen": 8.425279982546543,
      "rewards/margins": 17.672193562793456,
      "rewards/rejected": -9.246913580246913,
      "step": 367
    },
    {
      "epoch": 0.1812807881773399,
      "grad_norm": 0.3905725340654214,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -71203303.61904761,
      "logits/rejected": -95883740.27906977,
      "logps/chosen": -230.28571428571428,
      "logps/rejected": -514.2325581395348,
      "loss": 0.162,
      "rewards/chosen": 1.2663690476190477,
      "rewards/margins": 11.115206256921374,
      "rewards/rejected": -9.848837209302326,
      "step": 368
    },
    {
      "epoch": 0.18177339901477832,
      "grad_norm": 0.44016954392173535,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -70484767.2195122,
      "logits/rejected": -89382064.55172414,
      "logps/chosen": -307.5121951219512,
      "logps/rejected": -530.3908045977012,
      "loss": 0.1794,
      "rewards/chosen": 1.1539634146341464,
      "rewards/margins": 9.498791000841043,
      "rewards/rejected": -8.344827586206897,
      "step": 369
    },
    {
      "epoch": 0.18226600985221675,
      "grad_norm": 0.4942591205667238,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -68844438.06896552,
      "logits/rejected": -91795338.97142857,
      "logps/chosen": -264.55172413793105,
      "logps/rejected": -460.8,
      "loss": 0.2322,
      "rewards/chosen": 1.3081896551724137,
      "rewards/margins": 9.586761083743841,
      "rewards/rejected": -8.278571428571428,
      "step": 370
    },
    {
      "epoch": 0.18275862068965518,
      "grad_norm": 0.5176557977403649,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -61364491.13043478,
      "logits/rejected": -97082792.58536585,
      "logps/chosen": -223.56521739130434,
      "logps/rejected": -468.2926829268293,
      "loss": 0.2477,
      "rewards/chosen": -0.09035326086956522,
      "rewards/margins": 7.318183324496288,
      "rewards/rejected": -7.408536585365853,
      "step": 371
    },
    {
      "epoch": 0.1832512315270936,
      "grad_norm": 0.5913036083943561,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -81747807.37254901,
      "logits/rejected": -77131616.41558442,
      "logps/chosen": -294.9019607843137,
      "logps/rejected": -465.87012987012986,
      "loss": 0.2346,
      "rewards/chosen": 0.6599264705882353,
      "rewards/margins": 9.166419977081741,
      "rewards/rejected": -8.506493506493506,
      "step": 372
    },
    {
      "epoch": 0.183743842364532,
      "grad_norm": 0.4317239190422468,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -62767391.438596494,
      "logits/rejected": -89320952.7887324,
      "logps/chosen": -268.35087719298247,
      "logps/rejected": -490.8169014084507,
      "loss": 0.2781,
      "rewards/chosen": 1.769724126447711,
      "rewards/margins": 9.917611450391373,
      "rewards/rejected": -8.147887323943662,
      "step": 373
    },
    {
      "epoch": 0.18423645320197043,
      "grad_norm": 0.42230484544532837,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -58971914.24,
      "logits/rejected": -72271084.3076923,
      "logps/chosen": -293.12,
      "logps/rejected": -443.8974358974359,
      "loss": 0.2153,
      "rewards/chosen": 2.4618719482421874,
      "rewards/margins": 10.692641179011417,
      "rewards/rejected": -8.23076923076923,
      "step": 374
    },
    {
      "epoch": 0.18472906403940886,
      "grad_norm": 0.42686156980738627,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -83382763.52,
      "logits/rejected": -78508767.17948718,
      "logps/chosen": -259.52,
      "logps/rejected": -500.5128205128205,
      "loss": 0.2402,
      "rewards/chosen": -0.01875,
      "rewards/margins": 9.096634615384614,
      "rewards/rejected": -9.115384615384615,
      "step": 375
    },
    {
      "epoch": 0.1852216748768473,
      "grad_norm": 0.46221036198119014,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -75147946.66666667,
      "logits/rejected": -81349989.20930232,
      "logps/chosen": -277.3333333333333,
      "logps/rejected": -479.25581395348837,
      "loss": 0.2137,
      "rewards/chosen": -0.09151785714285714,
      "rewards/margins": 8.420110049833886,
      "rewards/rejected": -8.511627906976743,
      "step": 376
    },
    {
      "epoch": 0.18571428571428572,
      "grad_norm": 0.7284688789908284,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -86027852.25531915,
      "logits/rejected": -91963998.81481482,
      "logps/chosen": -393.8723404255319,
      "logps/rejected": -523.0617283950618,
      "loss": 0.2369,
      "rewards/chosen": 3.2034691343916224,
      "rewards/margins": 12.055320986243473,
      "rewards/rejected": -8.851851851851851,
      "step": 377
    },
    {
      "epoch": 0.18620689655172415,
      "grad_norm": 0.4482111187902806,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -79132535.46666667,
      "logits/rejected": -84391417.83132531,
      "logps/chosen": -321.06666666666666,
      "logps/rejected": -458.0240963855422,
      "loss": 0.2432,
      "rewards/chosen": 2.7442596435546873,
      "rewards/margins": 11.202090968855892,
      "rewards/rejected": -8.457831325301205,
      "step": 378
    },
    {
      "epoch": 0.18669950738916255,
      "grad_norm": 0.4513658896189183,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -71790877.76744185,
      "logits/rejected": -74609266.44705883,
      "logps/chosen": -348.27906976744185,
      "logps/rejected": -440.09411764705885,
      "loss": 0.1846,
      "rewards/chosen": 3.346701688544695,
      "rewards/margins": 12.229054629721166,
      "rewards/rejected": -8.882352941176471,
      "step": 379
    },
    {
      "epoch": 0.18719211822660098,
      "grad_norm": 0.4509511578559119,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -68856490.66666667,
      "logits/rejected": -75812044.8,
      "logps/chosen": -292.0,
      "logps/rejected": -430.0,
      "loss": 0.2186,
      "rewards/chosen": 0.62548828125,
      "rewards/margins": 8.96298828125,
      "rewards/rejected": -8.3375,
      "step": 380
    },
    {
      "epoch": 0.1876847290640394,
      "grad_norm": 0.451132521290662,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -70391362.7826087,
      "logits/rejected": -80817077.07317074,
      "logps/chosen": -338.0869565217391,
      "logps/rejected": -467.5121951219512,
      "loss": 0.1974,
      "rewards/chosen": 8.641939246136209,
      "rewards/margins": 17.36145144125816,
      "rewards/rejected": -8.71951219512195,
      "step": 381
    },
    {
      "epoch": 0.18817733990147784,
      "grad_norm": 0.43729007103295287,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -59300319.319148935,
      "logits/rejected": -80261372.83950618,
      "logps/chosen": -255.48936170212767,
      "logps/rejected": -413.2345679012346,
      "loss": 0.2188,
      "rewards/chosen": 0.3324468085106383,
      "rewards/margins": 7.46824927764644,
      "rewards/rejected": -7.135802469135802,
      "step": 382
    },
    {
      "epoch": 0.18866995073891626,
      "grad_norm": 0.3826508709679253,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -73400320.0,
      "logits/rejected": -87844748.94382022,
      "logps/chosen": -268.1025641025641,
      "logps/rejected": -520.629213483146,
      "loss": 0.1814,
      "rewards/chosen": 3.0100128956330128,
      "rewards/margins": 12.71787806417234,
      "rewards/rejected": -9.707865168539326,
      "step": 383
    },
    {
      "epoch": 0.1891625615763547,
      "grad_norm": 0.40649617145391936,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -83145908.70588236,
      "logits/rejected": -106437272.93506494,
      "logps/chosen": -298.6666666666667,
      "logps/rejected": -478.75324675324674,
      "loss": 0.2206,
      "rewards/chosen": 0.008272058823529412,
      "rewards/margins": 9.501778552330023,
      "rewards/rejected": -9.493506493506494,
      "step": 384
    },
    {
      "epoch": 0.1896551724137931,
      "grad_norm": 0.34192227651146445,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -82009680.84210527,
      "logits/rejected": -89198865.06666666,
      "logps/chosen": -417.6842105263158,
      "logps/rejected": -475.73333333333335,
      "loss": 0.1604,
      "rewards/chosen": -0.07072368421052631,
      "rewards/margins": 9.27372076023392,
      "rewards/rejected": -9.344444444444445,
      "step": 385
    },
    {
      "epoch": 0.19014778325123152,
      "grad_norm": 0.5238403993536404,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -65095598.08,
      "logits/rejected": -80659692.3076923,
      "logps/chosen": -239.04,
      "logps/rejected": -492.71794871794873,
      "loss": 0.2569,
      "rewards/chosen": 0.03484375,
      "rewards/margins": 9.009202724358975,
      "rewards/rejected": -8.974358974358974,
      "step": 386
    },
    {
      "epoch": 0.19064039408866995,
      "grad_norm": 0.39319993051791097,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -68283269.12,
      "logits/rejected": -86682282.66666667,
      "logps/chosen": -271.36,
      "logps/rejected": -523.0769230769231,
      "loss": 0.2436,
      "rewards/chosen": 0.129375,
      "rewards/margins": 9.167836538461538,
      "rewards/rejected": -9.038461538461538,
      "step": 387
    },
    {
      "epoch": 0.19113300492610838,
      "grad_norm": 0.4707907034805312,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -76211396.08510639,
      "logits/rejected": -88649980.83950618,
      "logps/chosen": -344.8510638297872,
      "logps/rejected": -477.2345679012346,
      "loss": 0.2011,
      "rewards/chosen": 1.2579787234042554,
      "rewards/margins": 10.350571315996849,
      "rewards/rejected": -9.092592592592593,
      "step": 388
    },
    {
      "epoch": 0.1916256157635468,
      "grad_norm": 0.43582791796264664,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -77594624.0,
      "logits/rejected": -87626945.72972973,
      "logps/chosen": -300.14814814814815,
      "logps/rejected": -550.918918918919,
      "loss": 0.2212,
      "rewards/chosen": 0.5271990740740741,
      "rewards/margins": 10.5542261011011,
      "rewards/rejected": -10.027027027027026,
      "step": 389
    },
    {
      "epoch": 0.1921182266009852,
      "grad_norm": 0.507712498463174,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -68981321.14285715,
      "logits/rejected": -88313400.8888889,
      "logps/chosen": -242.42857142857142,
      "logps/rejected": -578.6666666666666,
      "loss": 0.2558,
      "rewards/chosen": 0.17243303571428573,
      "rewards/margins": 10.241877480158731,
      "rewards/rejected": -10.069444444444445,
      "step": 390
    },
    {
      "epoch": 0.19261083743842364,
      "grad_norm": 0.5965289429410375,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -93017429.33333333,
      "logits/rejected": -82627788.8,
      "logps/chosen": -409.0,
      "logps/rejected": -503.2,
      "loss": 0.2029,
      "rewards/chosen": 3.624966621398926,
      "rewards/margins": 14.787466621398925,
      "rewards/rejected": -11.1625,
      "step": 391
    },
    {
      "epoch": 0.19310344827586207,
      "grad_norm": 0.37017280514694084,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -82021944.8888889,
      "logits/rejected": -89950133.97590362,
      "logps/chosen": -335.2888888888889,
      "logps/rejected": -475.7590361445783,
      "loss": 0.2276,
      "rewards/chosen": 0.3509548611111111,
      "rewards/margins": 9.471436788821954,
      "rewards/rejected": -9.120481927710843,
      "step": 392
    },
    {
      "epoch": 0.1935960591133005,
      "grad_norm": 0.5139445819810803,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -68671447.84313725,
      "logits/rejected": -80890148.57142857,
      "logps/chosen": -293.96078431372547,
      "logps/rejected": -537.3506493506494,
      "loss": 0.2766,
      "rewards/chosen": -0.8094362745098039,
      "rewards/margins": 8.24251177743825,
      "rewards/rejected": -9.051948051948052,
      "step": 393
    },
    {
      "epoch": 0.19408866995073892,
      "grad_norm": 0.4544920556262221,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -78361874.73170732,
      "logits/rejected": -70869274.48275863,
      "logps/chosen": -362.9268292682927,
      "logps/rejected": -446.8965517241379,
      "loss": 0.2274,
      "rewards/chosen": -1.3511814024390243,
      "rewards/margins": 7.27525537917017,
      "rewards/rejected": -8.626436781609195,
      "step": 394
    },
    {
      "epoch": 0.19458128078817735,
      "grad_norm": 0.4718089148535332,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -84410368.0,
      "logits/rejected": -104018739.2,
      "logps/chosen": -414.0,
      "logps/rejected": -528.4,
      "loss": 0.2105,
      "rewards/chosen": 1.4737141927083333,
      "rewards/margins": 5.361887868245443,
      "rewards/rejected": -3.8881736755371095,
      "step": 395
    },
    {
      "epoch": 0.19507389162561575,
      "grad_norm": 0.4820597965062455,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -70410762.89361702,
      "logits/rejected": -81607692.64197531,
      "logps/chosen": -314.8936170212766,
      "logps/rejected": -450.7654320987654,
      "loss": 0.1994,
      "rewards/chosen": 6.589707394863697,
      "rewards/margins": 15.18229998745629,
      "rewards/rejected": -8.592592592592593,
      "step": 396
    },
    {
      "epoch": 0.19556650246305418,
      "grad_norm": 0.4039103602940966,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -74144470.70967741,
      "logits/rejected": -75306821.81818181,
      "logps/chosen": -281.2903225806452,
      "logps/rejected": -488.72727272727275,
      "loss": 0.2671,
      "rewards/chosen": 2.7454541114068802,
      "rewards/margins": 10.889393505346275,
      "rewards/rejected": -8.143939393939394,
      "step": 397
    },
    {
      "epoch": 0.1960591133004926,
      "grad_norm": 0.4388403831204051,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -64180764.981132075,
      "logits/rejected": -80530636.8,
      "logps/chosen": -270.188679245283,
      "logps/rejected": -541.8666666666667,
      "loss": 0.2272,
      "rewards/chosen": 3.106120559404481,
      "rewards/margins": 12.492787226071147,
      "rewards/rejected": -9.386666666666667,
      "step": 398
    },
    {
      "epoch": 0.19655172413793104,
      "grad_norm": 0.38521002727257025,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -71138685.49019608,
      "logits/rejected": -69069837.2987013,
      "logps/chosen": -302.4313725490196,
      "logps/rejected": -448.0,
      "loss": 0.2887,
      "rewards/chosen": 1.6408016728419883,
      "rewards/margins": 10.095347127387443,
      "rewards/rejected": -8.454545454545455,
      "step": 399
    },
    {
      "epoch": 0.19704433497536947,
      "grad_norm": 0.4127185205763299,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -69430710.85714285,
      "logits/rejected": -87381333.33333333,
      "logps/chosen": -329.7142857142857,
      "logps/rejected": -445.3333333333333,
      "loss": 0.2347,
      "rewards/chosen": 1.1339285714285714,
      "rewards/margins": 9.050595238095239,
      "rewards/rejected": -7.916666666666667,
      "step": 400
    },
    {
      "epoch": 0.19753694581280787,
      "grad_norm": 0.4189581065504769,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -63438848.0,
      "logits/rejected": -81054924.8,
      "logps/chosen": -281.0,
      "logps/rejected": -490.4,
      "loss": 0.182,
      "rewards/chosen": 1.26171875,
      "rewards/margins": 10.01171875,
      "rewards/rejected": -8.75,
      "step": 401
    },
    {
      "epoch": 0.1980295566502463,
      "grad_norm": 0.3669084671113357,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -68474451.34883721,
      "logits/rejected": -80530636.8,
      "logps/chosen": -299.90697674418607,
      "logps/rejected": -477.36470588235295,
      "loss": 0.1846,
      "rewards/chosen": 4.68167540084484,
      "rewards/margins": 13.611087165550723,
      "rewards/rejected": -8.929411764705883,
      "step": 402
    },
    {
      "epoch": 0.19852216748768473,
      "grad_norm": 0.445479374865107,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -70232281.87234043,
      "logits/rejected": -85698433.58024691,
      "logps/chosen": -344.8510638297872,
      "logps/rejected": -469.3333333333333,
      "loss": 0.2384,
      "rewards/chosen": -0.7872340425531915,
      "rewards/margins": 7.546099290780142,
      "rewards/rejected": -8.333333333333334,
      "step": 403
    },
    {
      "epoch": 0.19901477832512315,
      "grad_norm": 0.3204920872533464,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -74117766.73684211,
      "logits/rejected": -94604856.8888889,
      "logps/chosen": -328.2105263157895,
      "logps/rejected": -441.6,
      "loss": 0.1639,
      "rewards/chosen": 4.286207901804071,
      "rewards/margins": 13.67509679069296,
      "rewards/rejected": -9.38888888888889,
      "step": 404
    },
    {
      "epoch": 0.19950738916256158,
      "grad_norm": 0.40003071063391754,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -60817408.0,
      "logits/rejected": -82371470.22222222,
      "logps/chosen": -198.85714285714286,
      "logps/rejected": -519.5555555555555,
      "loss": 0.2481,
      "rewards/chosen": -0.0625,
      "rewards/margins": 10.777777777777779,
      "rewards/rejected": -10.840277777777779,
      "step": 405
    },
    {
      "epoch": 0.2,
      "grad_norm": 0.38946750736101754,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -66713174.943396226,
      "logits/rejected": -69960990.72,
      "logps/chosen": -336.9056603773585,
      "logps/rejected": -415.14666666666665,
      "loss": 0.2719,
      "rewards/chosen": 2.536533571639151,
      "rewards/margins": 10.57653357163915,
      "rewards/rejected": -8.04,
      "step": 406
    },
    {
      "epoch": 0.2004926108374384,
      "grad_norm": 0.42814820033886863,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -63137661.27659574,
      "logits/rejected": -73219084.64197531,
      "logps/chosen": -269.9574468085106,
      "logps/rejected": -448.0,
      "loss": 0.2224,
      "rewards/chosen": -0.12400265957446809,
      "rewards/margins": 9.30809610585763,
      "rewards/rejected": -9.432098765432098,
      "step": 407
    },
    {
      "epoch": 0.20098522167487684,
      "grad_norm": 0.4762711549217207,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -80968303.30434783,
      "logits/rejected": -85727481.75609756,
      "logps/chosen": -301.5652173913044,
      "logps/rejected": -496.390243902439,
      "loss": 0.2262,
      "rewards/chosen": 0.4741847826086957,
      "rewards/margins": 8.888818928950158,
      "rewards/rejected": -8.414634146341463,
      "step": 408
    },
    {
      "epoch": 0.20147783251231527,
      "grad_norm": 0.44872196760712846,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -58473532.23529412,
      "logits/rejected": -78874703.79220779,
      "logps/chosen": -298.98039215686276,
      "logps/rejected": -425.97402597402595,
      "loss": 0.2283,
      "rewards/chosen": 0.3431372549019608,
      "rewards/margins": 9.239241151005858,
      "rewards/rejected": -8.896103896103897,
      "step": 409
    },
    {
      "epoch": 0.2019704433497537,
      "grad_norm": 0.38739735129418446,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -80073076.36363636,
      "logits/rejected": -79192454.09523809,
      "logps/chosen": -308.0,
      "logps/rejected": -492.1904761904762,
      "loss": 0.1619,
      "rewards/chosen": 6.187475724653765,
      "rewards/margins": 16.47319001036805,
      "rewards/rejected": -10.285714285714286,
      "step": 410
    },
    {
      "epoch": 0.20246305418719213,
      "grad_norm": 0.43178414543959764,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -69367335.38461539,
      "logits/rejected": -71082415.15789473,
      "logps/chosen": -263.53846153846155,
      "logps/rejected": -441.2631578947368,
      "loss": 0.2683,
      "rewards/chosen": 1.8437271118164062,
      "rewards/margins": 10.093727111816406,
      "rewards/rejected": -8.25,
      "step": 411
    },
    {
      "epoch": 0.20295566502463055,
      "grad_norm": 0.469420342868114,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -76734253.94871795,
      "logits/rejected": -77853822.56179775,
      "logps/chosen": -403.4871794871795,
      "logps/rejected": -454.8314606741573,
      "loss": 0.2127,
      "rewards/chosen": -0.7668269230769231,
      "rewards/margins": 7.851150605012965,
      "rewards/rejected": -8.617977528089888,
      "step": 412
    },
    {
      "epoch": 0.20344827586206896,
      "grad_norm": 0.49939558493564495,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -76000788.48,
      "logits/rejected": -74422009.43589744,
      "logps/chosen": -389.12,
      "logps/rejected": -466.87179487179486,
      "loss": 0.2315,
      "rewards/chosen": 0.224375,
      "rewards/margins": 9.570528846153847,
      "rewards/rejected": -9.346153846153847,
      "step": 413
    },
    {
      "epoch": 0.20394088669950738,
      "grad_norm": 0.49813241887448495,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -76546048.0,
      "logits/rejected": -74681912.8888889,
      "logps/chosen": -241.85714285714286,
      "logps/rejected": -474.6666666666667,
      "loss": 0.2754,
      "rewards/chosen": 0.109375,
      "rewards/margins": 6.831597222222222,
      "rewards/rejected": -6.722222222222222,
      "step": 414
    },
    {
      "epoch": 0.2044334975369458,
      "grad_norm": 0.38223237660370396,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -71481649.0212766,
      "logits/rejected": -76843791.80246913,
      "logps/chosen": -341.78723404255317,
      "logps/rejected": -439.30864197530866,
      "loss": 0.2159,
      "rewards/chosen": 0.5518617021276596,
      "rewards/margins": 9.625935776201734,
      "rewards/rejected": -9.074074074074074,
      "step": 415
    },
    {
      "epoch": 0.20492610837438424,
      "grad_norm": 0.3689999941346616,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -73347891.2,
      "logits/rejected": -76736698.18181819,
      "logps/chosen": -406.4,
      "logps/rejected": -518.5454545454545,
      "loss": 0.1658,
      "rewards/chosen": 4.90270767211914,
      "rewards/margins": 15.15270767211914,
      "rewards/rejected": -10.25,
      "step": 416
    },
    {
      "epoch": 0.20541871921182267,
      "grad_norm": 0.5765097224556388,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -80965046.85714285,
      "logits/rejected": -74914929.77777778,
      "logps/chosen": -283.7142857142857,
      "logps/rejected": -500.44444444444446,
      "loss": 0.2603,
      "rewards/chosen": 3.00335693359375,
      "rewards/margins": 12.906134711371529,
      "rewards/rejected": -9.902777777777779,
      "step": 417
    },
    {
      "epoch": 0.20591133004926107,
      "grad_norm": 0.5067321898598187,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -87077398.26086956,
      "logits/rejected": -78566474.92682926,
      "logps/chosen": -322.0869565217391,
      "logps/rejected": -451.5121951219512,
      "loss": 0.2423,
      "rewards/chosen": -0.3845108695652174,
      "rewards/margins": 8.469147667020149,
      "rewards/rejected": -8.853658536585366,
      "step": 418
    },
    {
      "epoch": 0.2064039408866995,
      "grad_norm": 0.4969155929628195,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -72447069.0909091,
      "logits/rejected": -79059757.5890411,
      "logps/chosen": -315.92727272727274,
      "logps/rejected": -428.71232876712327,
      "loss": 0.2534,
      "rewards/chosen": -0.7113636363636363,
      "rewards/margins": 8.466718555417186,
      "rewards/rejected": -9.178082191780822,
      "step": 419
    },
    {
      "epoch": 0.20689655172413793,
      "grad_norm": 0.45272670998566567,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -73947403.13043478,
      "logits/rejected": -73451470.04878049,
      "logps/chosen": -322.0869565217391,
      "logps/rejected": -455.0243902439024,
      "loss": 0.173,
      "rewards/chosen": 1.3451086956521738,
      "rewards/margins": 10.479255037115589,
      "rewards/rejected": -9.134146341463415,
      "step": 420
    },
    {
      "epoch": 0.20738916256157636,
      "grad_norm": 0.38575306945387805,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -72637719.27272727,
      "logits/rejected": -77095302.09523809,
      "logps/chosen": -306.90909090909093,
      "logps/rejected": -476.1904761904762,
      "loss": 0.1988,
      "rewards/chosen": 6.47064902565696,
      "rewards/margins": 15.684934739942673,
      "rewards/rejected": -9.214285714285714,
      "step": 421
    },
    {
      "epoch": 0.20788177339901479,
      "grad_norm": 0.4870077729817272,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -59669909.73584906,
      "logits/rejected": -75161927.68,
      "logps/chosen": -265.35849056603774,
      "logps/rejected": -450.9866666666667,
      "loss": 0.2309,
      "rewards/chosen": 0.6515330188679245,
      "rewards/margins": 9.064866352201259,
      "rewards/rejected": -8.413333333333334,
      "step": 422
    },
    {
      "epoch": 0.20837438423645321,
      "grad_norm": 0.44743549636949825,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -61341696.0,
      "logits/rejected": -76126617.6,
      "logps/chosen": -286.3333333333333,
      "logps/rejected": -524.0,
      "loss": 0.1913,
      "rewards/chosen": 3.671851476033529,
      "rewards/margins": 13.32185147603353,
      "rewards/rejected": -9.65,
      "step": 423
    },
    {
      "epoch": 0.20886699507389161,
      "grad_norm": 0.5962262805338212,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -66016597.333333336,
      "logits/rejected": -88080384.0,
      "logps/chosen": -333.3333333333333,
      "logps/rejected": -507.2,
      "loss": 0.2494,
      "rewards/chosen": 2.3854497273763022,
      "rewards/margins": 11.166699727376303,
      "rewards/rejected": -8.78125,
      "step": 424
    },
    {
      "epoch": 0.20935960591133004,
      "grad_norm": 0.5867182371063389,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -76648849.56862745,
      "logits/rejected": -93146231.68831168,
      "logps/chosen": -332.2352941176471,
      "logps/rejected": -462.961038961039,
      "loss": 0.2653,
      "rewards/chosen": -1.4840686274509804,
      "rewards/margins": 6.996450853068501,
      "rewards/rejected": -8.480519480519481,
      "step": 425
    },
    {
      "epoch": 0.20985221674876847,
      "grad_norm": 0.6721361579395668,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -78786187.63636364,
      "logits/rejected": -71502896.76190476,
      "logps/chosen": -280.3636363636364,
      "logps/rejected": -408.0,
      "loss": 0.208,
      "rewards/chosen": -1.2911931818181819,
      "rewards/margins": 7.613568722943723,
      "rewards/rejected": -8.904761904761905,
      "step": 426
    },
    {
      "epoch": 0.2103448275862069,
      "grad_norm": 0.5058023746761569,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -62540068.571428575,
      "logits/rejected": -69206016.0,
      "logps/chosen": -247.71428571428572,
      "logps/rejected": -461.3333333333333,
      "loss": 0.2517,
      "rewards/chosen": 2.5733885083879744,
      "rewards/margins": 11.295610730610196,
      "rewards/rejected": -8.722222222222221,
      "step": 427
    },
    {
      "epoch": 0.21083743842364533,
      "grad_norm": 0.4935171266620077,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -75235328.0,
      "logits/rejected": -63386419.2,
      "logps/chosen": -296.0,
      "logps/rejected": -434.0,
      "loss": 0.2327,
      "rewards/chosen": 4.966154098510742,
      "rewards/margins": 14.078654098510743,
      "rewards/rejected": -9.1125,
      "step": 428
    },
    {
      "epoch": 0.21133004926108373,
      "grad_norm": 0.9012997152167697,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -69851293.53846154,
      "logits/rejected": -74504084.21052632,
      "logps/chosen": -256.7692307692308,
      "logps/rejected": -433.6842105263158,
      "loss": 0.2377,
      "rewards/chosen": 1.1071213942307692,
      "rewards/margins": 6.883437183704453,
      "rewards/rejected": -5.776315789473684,
      "step": 429
    },
    {
      "epoch": 0.21182266009852216,
      "grad_norm": 0.5446621041820667,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -59419306.666666664,
      "logits/rejected": -70883737.6,
      "logps/chosen": -256.6666666666667,
      "logps/rejected": -443.2,
      "loss": 0.2295,
      "rewards/chosen": 0.5716145833333334,
      "rewards/margins": 8.971614583333334,
      "rewards/rejected": -8.4,
      "step": 430
    },
    {
      "epoch": 0.2123152709359606,
      "grad_norm": 0.4034654552129997,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -65011712.0,
      "logits/rejected": -66479718.4,
      "logps/chosen": -323.3333333333333,
      "logps/rejected": -434.4,
      "loss": 0.2305,
      "rewards/chosen": 2.5344913800557456,
      "rewards/margins": 10.859491380055745,
      "rewards/rejected": -8.325,
      "step": 431
    },
    {
      "epoch": 0.21280788177339902,
      "grad_norm": 0.45852600853097647,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -73035597.91304348,
      "logits/rejected": -85215981.26829268,
      "logps/chosen": -214.08695652173913,
      "logps/rejected": -522.1463414634146,
      "loss": 0.224,
      "rewards/chosen": 0.002547554347826087,
      "rewards/margins": 9.563523164103923,
      "rewards/rejected": -9.560975609756097,
      "step": 432
    },
    {
      "epoch": 0.21330049261083744,
      "grad_norm": 0.4057382245606136,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -72648510.79245283,
      "logits/rejected": -91715447.46666667,
      "logps/chosen": -283.77358490566036,
      "logps/rejected": -492.37333333333333,
      "loss": 0.2355,
      "rewards/chosen": 2.511167418281987,
      "rewards/margins": 9.36450075161532,
      "rewards/rejected": -6.8533333333333335,
      "step": 433
    },
    {
      "epoch": 0.21379310344827587,
      "grad_norm": 0.47730407508338496,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -75934378.66666667,
      "logits/rejected": -82103500.8,
      "logps/chosen": -295.3333333333333,
      "logps/rejected": -450.4,
      "loss": 0.2353,
      "rewards/chosen": 2.434894561767578,
      "rewards/margins": 11.072394561767577,
      "rewards/rejected": -8.6375,
      "step": 434
    },
    {
      "epoch": 0.21428571428571427,
      "grad_norm": 0.6839567034230547,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -72089600.0,
      "logits/rejected": -82418073.6,
      "logps/chosen": -350.0,
      "logps/rejected": -530.4,
      "loss": 0.2247,
      "rewards/chosen": 2.7258872985839844,
      "rewards/margins": 11.113387298583984,
      "rewards/rejected": -8.3875,
      "step": 435
    },
    {
      "epoch": 0.2147783251231527,
      "grad_norm": 0.5055524409479584,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -63730119.11111111,
      "logits/rejected": -77822026.02409638,
      "logps/chosen": -258.1333333333333,
      "logps/rejected": -518.9397590361446,
      "loss": 0.2387,
      "rewards/chosen": 2.494937472873264,
      "rewards/margins": 10.470841087331095,
      "rewards/rejected": -7.975903614457831,
      "step": 436
    },
    {
      "epoch": 0.21527093596059113,
      "grad_norm": 0.47602781960285556,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -72443724.3508772,
      "logits/rejected": -84594976.45070423,
      "logps/chosen": -293.89473684210526,
      "logps/rejected": -531.830985915493,
      "loss": 0.2191,
      "rewards/chosen": 4.36074213396039,
      "rewards/margins": 13.438206922692784,
      "rewards/rejected": -9.077464788732394,
      "step": 437
    },
    {
      "epoch": 0.21576354679802956,
      "grad_norm": 0.5688684768911707,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -67871464.72727273,
      "logits/rejected": -77106246.1369863,
      "logps/chosen": -323.4909090909091,
      "logps/rejected": -476.93150684931504,
      "loss": 0.2757,
      "rewards/chosen": 1.9227351795543324,
      "rewards/margins": 9.991228330239263,
      "rewards/rejected": -8.068493150684931,
      "step": 438
    },
    {
      "epoch": 0.216256157635468,
      "grad_norm": 0.5284998747977449,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -70117821.2173913,
      "logits/rejected": -77031973.46341464,
      "logps/chosen": -314.4347826086956,
      "logps/rejected": -460.4878048780488,
      "loss": 0.227,
      "rewards/chosen": 0.38790760869565216,
      "rewards/margins": 8.253761267232237,
      "rewards/rejected": -7.865853658536586,
      "step": 439
    },
    {
      "epoch": 0.21674876847290642,
      "grad_norm": 0.499739614554008,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -64267561.29032258,
      "logits/rejected": -81378145.64948453,
      "logps/chosen": -297.6774193548387,
      "logps/rejected": -507.3814432989691,
      "loss": 0.1702,
      "rewards/chosen": 3.7772216796875,
      "rewards/margins": 11.735984566285438,
      "rewards/rejected": -7.958762886597938,
      "step": 440
    },
    {
      "epoch": 0.21724137931034482,
      "grad_norm": 0.7750830863317114,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -72085156.88135593,
      "logits/rejected": -73673861.56521739,
      "logps/chosen": -344.6779661016949,
      "logps/rejected": -496.69565217391306,
      "loss": 0.2321,
      "rewards/chosen": 1.0190677966101696,
      "rewards/margins": 6.736459100957996,
      "rewards/rejected": -5.717391304347826,
      "step": 441
    },
    {
      "epoch": 0.21773399014778325,
      "grad_norm": 0.44649569771642944,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -48779755.52,
      "logits/rejected": -78938952.20512821,
      "logps/chosen": -222.4,
      "logps/rejected": -484.1025641025641,
      "loss": 0.2153,
      "rewards/chosen": 3.0887469482421874,
      "rewards/margins": 11.729772589267828,
      "rewards/rejected": -8.64102564102564,
      "step": 442
    },
    {
      "epoch": 0.21822660098522167,
      "grad_norm": 0.43295452771044524,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -73651978.24,
      "logits/rejected": -87757745.23076923,
      "logps/chosen": -259.2,
      "logps/rejected": -479.1794871794872,
      "loss": 0.2151,
      "rewards/chosen": 2.74716796875,
      "rewards/margins": 10.631783353365385,
      "rewards/rejected": -7.884615384615385,
      "step": 443
    },
    {
      "epoch": 0.2187192118226601,
      "grad_norm": 0.5188284512833804,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -66523612.27906977,
      "logits/rejected": -88228418.25882353,
      "logps/chosen": -248.5581395348837,
      "logps/rejected": -448.7529411764706,
      "loss": 0.1626,
      "rewards/chosen": 6.655483511991279,
      "rewards/margins": 15.290777629638338,
      "rewards/rejected": -8.635294117647058,
      "step": 444
    },
    {
      "epoch": 0.21921182266009853,
      "grad_norm": 0.47104630104165773,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -59544137.14285714,
      "logits/rejected": -84818147.55555555,
      "logps/chosen": -251.14285714285714,
      "logps/rejected": -518.6666666666666,
      "loss": 0.213,
      "rewards/chosen": 4.82475335257394,
      "rewards/margins": 14.06086446368505,
      "rewards/rejected": -9.23611111111111,
      "step": 445
    },
    {
      "epoch": 0.21970443349753693,
      "grad_norm": 0.4221639811874944,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -57322154.666666664,
      "logits/rejected": -96528066.70422535,
      "logps/chosen": -229.05263157894737,
      "logps/rejected": -532.7323943661971,
      "loss": 0.2478,
      "rewards/chosen": 2.6578864382024396,
      "rewards/margins": 9.41844981848413,
      "rewards/rejected": -6.76056338028169,
      "step": 446
    },
    {
      "epoch": 0.22019704433497536,
      "grad_norm": 0.42029823755713674,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -69038243.84,
      "logits/rejected": -70765436.71794872,
      "logps/chosen": -304.32,
      "logps/rejected": -493.94871794871796,
      "loss": 0.1984,
      "rewards/chosen": 1.62375,
      "rewards/margins": 10.482724358974359,
      "rewards/rejected": -8.85897435897436,
      "step": 447
    },
    {
      "epoch": 0.2206896551724138,
      "grad_norm": 0.4651097043037803,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -57713623.04,
      "logits/rejected": -81520062.35897435,
      "logps/chosen": -258.88,
      "logps/rejected": -457.84615384615387,
      "loss": 0.2248,
      "rewards/chosen": 2.237513427734375,
      "rewards/margins": 11.365718555939502,
      "rewards/rejected": -9.128205128205128,
      "step": 448
    },
    {
      "epoch": 0.22118226600985222,
      "grad_norm": 0.4703626492966765,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -50613956.92307692,
      "logits/rejected": -62252301.473684214,
      "logps/chosen": -291.38461538461536,
      "logps/rejected": -405.4736842105263,
      "loss": 0.2281,
      "rewards/chosen": 5.13403555063101,
      "rewards/margins": 12.423509234841536,
      "rewards/rejected": -7.2894736842105265,
      "step": 449
    },
    {
      "epoch": 0.22167487684729065,
      "grad_norm": 0.5023653209958856,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -78216002.37037037,
      "logits/rejected": -76744427.24324325,
      "logps/chosen": -306.6666666666667,
      "logps/rejected": -540.972972972973,
      "loss": 0.2391,
      "rewards/chosen": 0.7860966435185185,
      "rewards/margins": 9.286096643518519,
      "rewards/rejected": -8.5,
      "step": 450
    },
    {
      "epoch": 0.22216748768472908,
      "grad_norm": 0.4680392189915183,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -65793003.921568625,
      "logits/rejected": -81271448.93506494,
      "logps/chosen": -285.96078431372547,
      "logps/rejected": -443.42857142857144,
      "loss": 0.2647,
      "rewards/chosen": 0.06571691176470588,
      "rewards/margins": 7.6566260026737964,
      "rewards/rejected": -7.590909090909091,
      "step": 451
    },
    {
      "epoch": 0.22266009852216748,
      "grad_norm": 0.5043565535059356,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -43384832.0,
      "logits/rejected": -78643200.0,
      "logps/chosen": -184.5,
      "logps/rejected": -509.6,
      "loss": 0.2342,
      "rewards/chosen": 0.669921875,
      "rewards/margins": 9.094921875,
      "rewards/rejected": -8.425,
      "step": 452
    },
    {
      "epoch": 0.2231527093596059,
      "grad_norm": 0.41429999992722766,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -57228947.91111111,
      "logits/rejected": -72364377.44578314,
      "logps/chosen": -238.22222222222223,
      "logps/rejected": -501.2048192771084,
      "loss": 0.1882,
      "rewards/chosen": 3.1218943277994793,
      "rewards/margins": 11.808641315751286,
      "rewards/rejected": -8.686746987951807,
      "step": 453
    },
    {
      "epoch": 0.22364532019704433,
      "grad_norm": 0.4959491315492185,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -77755943.38461539,
      "logits/rejected": -83003068.63157895,
      "logps/chosen": -396.61538461538464,
      "logps/rejected": -440.42105263157896,
      "loss": 0.2147,
      "rewards/chosen": 2.5105250431941104,
      "rewards/margins": 10.576314516878321,
      "rewards/rejected": -8.06578947368421,
      "step": 454
    },
    {
      "epoch": 0.22413793103448276,
      "grad_norm": 0.4867923413078301,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -61462685.538461536,
      "logits/rejected": -94371840.0,
      "logps/chosen": -284.3076923076923,
      "logps/rejected": -448.8421052631579,
      "loss": 0.2303,
      "rewards/chosen": 0.7061298076923077,
      "rewards/margins": 9.541656123481783,
      "rewards/rejected": -8.835526315789474,
      "step": 455
    },
    {
      "epoch": 0.2246305418719212,
      "grad_norm": 0.4479429213187825,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -68802717.53846154,
      "logits/rejected": -67108864.0,
      "logps/chosen": -399.38461538461536,
      "logps/rejected": -460.2105263157895,
      "loss": 0.2184,
      "rewards/chosen": 4.675224304199219,
      "rewards/margins": 13.517329567357114,
      "rewards/rejected": -8.842105263157896,
      "step": 456
    },
    {
      "epoch": 0.22512315270935962,
      "grad_norm": 0.4431438552514659,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -69072155.23404256,
      "logits/rejected": -77775859.35802469,
      "logps/chosen": -350.97872340425533,
      "logps/rejected": -475.25925925925924,
      "loss": 0.2019,
      "rewards/chosen": 2.982691338721742,
      "rewards/margins": 11.278987635018037,
      "rewards/rejected": -8.296296296296296,
      "step": 457
    },
    {
      "epoch": 0.22561576354679802,
      "grad_norm": 0.512787822353586,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -67960832.0,
      "logits/rejected": -81920000.0,
      "logps/chosen": -276.25,
      "logps/rejected": -570.0,
      "loss": 0.2855,
      "rewards/chosen": -0.465576171875,
      "rewards/margins": 9.425048828125,
      "rewards/rejected": -9.890625,
      "step": 458
    },
    {
      "epoch": 0.22610837438423645,
      "grad_norm": 0.4633772443987741,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -67969234.05128205,
      "logits/rejected": -76345758.20224719,
      "logps/chosen": -273.2307692307692,
      "logps/rejected": -465.97752808988764,
      "loss": 0.2051,
      "rewards/chosen": 9.721147586137821,
      "rewards/margins": 18.21552960860973,
      "rewards/rejected": -8.49438202247191,
      "step": 459
    },
    {
      "epoch": 0.22660098522167488,
      "grad_norm": 0.4272419343095174,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -73819750.4,
      "logits/rejected": -74475782.56410256,
      "logps/chosen": -313.92,
      "logps/rejected": -468.9230769230769,
      "loss": 0.2084,
      "rewards/chosen": 3.3990521240234375,
      "rewards/margins": 11.956744431715746,
      "rewards/rejected": -8.557692307692308,
      "step": 460
    },
    {
      "epoch": 0.2270935960591133,
      "grad_norm": 0.42887712036275244,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -65524349.155555554,
      "logits/rejected": -90051201.54216868,
      "logps/chosen": -279.8222222222222,
      "logps/rejected": -524.3373493975904,
      "loss": 0.1925,
      "rewards/chosen": 0.6701388888888888,
      "rewards/margins": 10.850861780455155,
      "rewards/rejected": -10.180722891566266,
      "step": 461
    },
    {
      "epoch": 0.22758620689655173,
      "grad_norm": 0.3937278492548046,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -49869775.23809524,
      "logits/rejected": -74424510.51162791,
      "logps/chosen": -311.8095238095238,
      "logps/rejected": -443.16279069767444,
      "loss": 0.2305,
      "rewards/chosen": 3.7573620024181547,
      "rewards/margins": 11.79805967683676,
      "rewards/rejected": -8.040697674418604,
      "step": 462
    },
    {
      "epoch": 0.22807881773399014,
      "grad_norm": 0.38166469574915646,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -68320551.82222222,
      "logits/rejected": -99854755.46987952,
      "logps/chosen": -249.95555555555555,
      "logps/rejected": -561.3493975903615,
      "loss": 0.2259,
      "rewards/chosen": 2.2433987087673612,
      "rewards/margins": 12.809663769008324,
      "rewards/rejected": -10.566265060240964,
      "step": 463
    },
    {
      "epoch": 0.22857142857142856,
      "grad_norm": 0.5059648773892569,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -64018324.21052632,
      "logits/rejected": -60256198.30985916,
      "logps/chosen": -276.7719298245614,
      "logps/rejected": -472.7887323943662,
      "loss": 0.2311,
      "rewards/chosen": 2.690240692674068,
      "rewards/margins": 12.211367453237449,
      "rewards/rejected": -9.52112676056338,
      "step": 464
    },
    {
      "epoch": 0.229064039408867,
      "grad_norm": 0.42214341657185556,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -75240677.87755102,
      "logits/rejected": -83461340.35443038,
      "logps/chosen": -269.7142857142857,
      "logps/rejected": -525.7721518987341,
      "loss": 0.1886,
      "rewards/chosen": 3.7665679308832907,
      "rewards/margins": 12.76656793088329,
      "rewards/rejected": -9.0,
      "step": 465
    },
    {
      "epoch": 0.22955665024630542,
      "grad_norm": 0.4375907606686047,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -81318138.7755102,
      "logits/rejected": -77727355.1392405,
      "logps/chosen": -349.0612244897959,
      "logps/rejected": -503.0886075949367,
      "loss": 0.2387,
      "rewards/chosen": 5.239799655213648,
      "rewards/margins": 14.632204718504788,
      "rewards/rejected": -9.39240506329114,
      "step": 466
    },
    {
      "epoch": 0.23004926108374385,
      "grad_norm": 0.5590472686545093,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -65311305.14285714,
      "logits/rejected": -70931520.81012659,
      "logps/chosen": -322.9387755102041,
      "logps/rejected": -506.32911392405066,
      "loss": 0.2294,
      "rewards/chosen": 3.280457165776467,
      "rewards/margins": 12.166533115143555,
      "rewards/rejected": -8.886075949367088,
      "step": 467
    },
    {
      "epoch": 0.23054187192118228,
      "grad_norm": 0.4562243300270902,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -61911574.26086956,
      "logits/rejected": -73349169.95121951,
      "logps/chosen": -259.82608695652175,
      "logps/rejected": -476.0975609756098,
      "loss": 0.2245,
      "rewards/chosen": 2.1888965109120244,
      "rewards/margins": 10.957189193838854,
      "rewards/rejected": -8.768292682926829,
      "step": 468
    },
    {
      "epoch": 0.23103448275862068,
      "grad_norm": 0.5672373773994582,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -77433304.61538461,
      "logits/rejected": -90619041.68421052,
      "logps/chosen": -296.3076923076923,
      "logps/rejected": -455.1578947368421,
      "loss": 0.2435,
      "rewards/chosen": 0.5673076923076923,
      "rewards/margins": 8.67914979757085,
      "rewards/rejected": -8.111842105263158,
      "step": 469
    },
    {
      "epoch": 0.2315270935960591,
      "grad_norm": 0.41533331719683897,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -52656751.30434783,
      "logits/rejected": -65881262.82926829,
      "logps/chosen": -236.17391304347825,
      "logps/rejected": -452.6829268292683,
      "loss": 0.2087,
      "rewards/chosen": 3.2520207944123642,
      "rewards/margins": 11.654459818802607,
      "rewards/rejected": -8.402439024390244,
      "step": 470
    },
    {
      "epoch": 0.23201970443349754,
      "grad_norm": 0.4226421593911158,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -57949873.63265306,
      "logits/rejected": -71568630.278481,
      "logps/chosen": -223.0204081632653,
      "logps/rejected": -510.37974683544303,
      "loss": 0.2011,
      "rewards/chosen": 0.8686224489795918,
      "rewards/margins": 9.75469839834668,
      "rewards/rejected": -8.886075949367088,
      "step": 471
    },
    {
      "epoch": 0.23251231527093597,
      "grad_norm": 0.5288387825214991,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -64540922.77551021,
      "logits/rejected": -77302615.49367088,
      "logps/chosen": -302.85714285714283,
      "logps/rejected": -540.7594936708861,
      "loss": 0.2252,
      "rewards/chosen": -0.4320790816326531,
      "rewards/margins": 10.859060158873676,
      "rewards/rejected": -11.291139240506329,
      "step": 472
    },
    {
      "epoch": 0.2330049261083744,
      "grad_norm": 0.5044564143493349,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -69564066.34146342,
      "logits/rejected": -71351378.3908046,
      "logps/chosen": -343.4146341463415,
      "logps/rejected": -584.8275862068965,
      "loss": 0.1982,
      "rewards/chosen": 6.6615660132431405,
      "rewards/margins": 16.604094748875326,
      "rewards/rejected": -9.942528735632184,
      "step": 473
    },
    {
      "epoch": 0.2334975369458128,
      "grad_norm": 0.3797390244753592,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -98095354.7755102,
      "logits/rejected": -75178917.26582278,
      "logps/chosen": -287.6734693877551,
      "logps/rejected": -476.75949367088606,
      "loss": 0.2185,
      "rewards/chosen": 2.931768923389668,
      "rewards/margins": 11.74189550566815,
      "rewards/rejected": -8.810126582278482,
      "step": 474
    },
    {
      "epoch": 0.23399014778325122,
      "grad_norm": 0.478449645037186,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -79467081.14285715,
      "logits/rejected": -72293489.77777778,
      "logps/chosen": -387.42857142857144,
      "logps/rejected": -448.8888888888889,
      "loss": 0.2347,
      "rewards/chosen": 3.335894448416574,
      "rewards/margins": 12.912283337305464,
      "rewards/rejected": -9.57638888888889,
      "step": 475
    },
    {
      "epoch": 0.23448275862068965,
      "grad_norm": 0.5974089961360364,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -83361792.0,
      "logits/rejected": -73400320.0,
      "logps/chosen": -338.57142857142856,
      "logps/rejected": -468.0,
      "loss": 0.2843,
      "rewards/chosen": -0.41964285714285715,
      "rewards/margins": 8.205357142857142,
      "rewards/rejected": -8.625,
      "step": 476
    },
    {
      "epoch": 0.23497536945812808,
      "grad_norm": 0.4815505347390793,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -70448772.74074075,
      "logits/rejected": -86663389.4054054,
      "logps/chosen": -298.0740740740741,
      "logps/rejected": -458.81081081081084,
      "loss": 0.2169,
      "rewards/chosen": 1.0089699074074074,
      "rewards/margins": 10.644105042542542,
      "rewards/rejected": -9.635135135135135,
      "step": 477
    },
    {
      "epoch": 0.2354679802955665,
      "grad_norm": 0.412109187469162,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -79845226.14634146,
      "logits/rejected": -79932827.95402299,
      "logps/chosen": -257.1707317073171,
      "logps/rejected": -447.264367816092,
      "loss": 0.1843,
      "rewards/chosen": 3.012180235327744,
      "rewards/margins": 11.920226212339237,
      "rewards/rejected": -8.908045977011493,
      "step": 478
    },
    {
      "epoch": 0.23596059113300494,
      "grad_norm": 0.5391052325601268,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -78835795.59183674,
      "logits/rejected": -70506781.16455697,
      "logps/chosen": -373.55102040816325,
      "logps/rejected": -405.0632911392405,
      "loss": 0.1701,
      "rewards/chosen": 3.322747055365115,
      "rewards/margins": 11.96831667561828,
      "rewards/rejected": -8.645569620253164,
      "step": 479
    },
    {
      "epoch": 0.23645320197044334,
      "grad_norm": 0.5098532606298425,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -77426851.84,
      "logits/rejected": -92543553.64102565,
      "logps/chosen": -268.8,
      "logps/rejected": -444.3076923076923,
      "loss": 0.2706,
      "rewards/chosen": -0.73421875,
      "rewards/margins": 7.688858173076923,
      "rewards/rejected": -8.423076923076923,
      "step": 480
    },
    {
      "epoch": 0.23694581280788177,
      "grad_norm": 0.5719130686968628,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -66720502.518518515,
      "logits/rejected": -77197865.51351352,
      "logps/chosen": -298.962962962963,
      "logps/rejected": -435.4594594594595,
      "loss": 0.2788,
      "rewards/chosen": -0.47800925925925924,
      "rewards/margins": 7.528747497497497,
      "rewards/rejected": -8.006756756756756,
      "step": 481
    },
    {
      "epoch": 0.2374384236453202,
      "grad_norm": 0.4708029496440373,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -74038583.6521739,
      "logits/rejected": -74883671.41463415,
      "logps/chosen": -359.6521739130435,
      "logps/rejected": -526.0487804878048,
      "loss": 0.2035,
      "rewards/chosen": 3.270383088485054,
      "rewards/margins": 13.38013918604603,
      "rewards/rejected": -10.109756097560975,
      "step": 482
    },
    {
      "epoch": 0.23793103448275862,
      "grad_norm": 0.4965925464729482,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -72351744.0,
      "logits/rejected": -81998643.2,
      "logps/chosen": -296.6666666666667,
      "logps/rejected": -499.2,
      "loss": 0.2503,
      "rewards/chosen": 2.1386801401774087,
      "rewards/margins": 11.326180140177408,
      "rewards/rejected": -9.1875,
      "step": 483
    },
    {
      "epoch": 0.23842364532019705,
      "grad_norm": 0.42149268777012533,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -58891452.08163265,
      "logits/rejected": -79426313.721519,
      "logps/chosen": -340.2448979591837,
      "logps/rejected": -497.4177215189873,
      "loss": 0.2044,
      "rewards/chosen": 0.22321428571428573,
      "rewards/margins": 10.438404159132007,
      "rewards/rejected": -10.215189873417721,
      "step": 484
    },
    {
      "epoch": 0.23891625615763548,
      "grad_norm": 0.4929322755287035,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -82998823.38461539,
      "logits/rejected": -77263494.73684211,
      "logps/chosen": -371.38461538461536,
      "logps/rejected": -513.6842105263158,
      "loss": 0.2389,
      "rewards/chosen": 1.4843292236328125,
      "rewards/margins": 12.773802907843338,
      "rewards/rejected": -11.289473684210526,
      "step": 485
    },
    {
      "epoch": 0.23940886699507388,
      "grad_norm": 0.4589290364706024,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -54619158.755555555,
      "logits/rejected": -71252634.21686748,
      "logps/chosen": -296.53333333333336,
      "logps/rejected": -492.33734939759034,
      "loss": 0.1889,
      "rewards/chosen": 5.833280436197916,
      "rewards/margins": 16.31520814704129,
      "rewards/rejected": -10.481927710843374,
      "step": 486
    },
    {
      "epoch": 0.2399014778325123,
      "grad_norm": 0.4477626765050539,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -48993040.34042553,
      "logits/rejected": -76636665.67901234,
      "logps/chosen": -251.91489361702128,
      "logps/rejected": -501.3333333333333,
      "loss": 0.2453,
      "rewards/chosen": 2.2144085498566324,
      "rewards/margins": 11.609470278251694,
      "rewards/rejected": -9.395061728395062,
      "step": 487
    },
    {
      "epoch": 0.24039408866995074,
      "grad_norm": 0.48014128024970737,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -70170705.92,
      "logits/rejected": -74422009.43589744,
      "logps/chosen": -312.96,
      "logps/rejected": -484.1025641025641,
      "loss": 0.2042,
      "rewards/chosen": 3.8643634033203127,
      "rewards/margins": 15.133594172551083,
      "rewards/rejected": -11.26923076923077,
      "step": 488
    },
    {
      "epoch": 0.24088669950738917,
      "grad_norm": 0.4525098621568746,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -87334729.95555556,
      "logits/rejected": -78327363.85542169,
      "logps/chosen": -277.68888888888887,
      "logps/rejected": -471.90361445783134,
      "loss": 0.2194,
      "rewards/chosen": -0.4076388888888889,
      "rewards/margins": 8.592361111111112,
      "rewards/rejected": -9.0,
      "step": 489
    },
    {
      "epoch": 0.2413793103448276,
      "grad_norm": 0.46439674597741953,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -77499298.9090909,
      "logits/rejected": -81689063.61904761,
      "logps/chosen": -230.0,
      "logps/rejected": -490.6666666666667,
      "loss": 0.1608,
      "rewards/chosen": 1.5482954545454546,
      "rewards/margins": 12.405438311688313,
      "rewards/rejected": -10.857142857142858,
      "step": 490
    },
    {
      "epoch": 0.241871921182266,
      "grad_norm": 0.4139008701936525,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -73246869.85365854,
      "logits/rejected": -80607773.42528735,
      "logps/chosen": -283.1219512195122,
      "logps/rejected": -553.1954022988506,
      "loss": 0.2134,
      "rewards/chosen": 2.998739475157203,
      "rewards/margins": 12.768854417685938,
      "rewards/rejected": -9.770114942528735,
      "step": 491
    },
    {
      "epoch": 0.24236453201970443,
      "grad_norm": 0.39219246779862915,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -62185115.82608695,
      "logits/rejected": -83681479.80487806,
      "logps/chosen": -221.56521739130434,
      "logps/rejected": -555.7073170731708,
      "loss": 0.2359,
      "rewards/chosen": 2.593047100564708,
      "rewards/margins": 11.54426661275983,
      "rewards/rejected": -8.951219512195122,
      "step": 492
    },
    {
      "epoch": 0.24285714285714285,
      "grad_norm": 0.41973786386451817,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -82208358.4,
      "logits/rejected": -67412066.69879518,
      "logps/chosen": -293.15555555555557,
      "logps/rejected": -454.1686746987952,
      "loss": 0.1971,
      "rewards/chosen": 0.3298611111111111,
      "rewards/margins": 9.35395749665328,
      "rewards/rejected": -9.024096385542169,
      "step": 493
    },
    {
      "epoch": 0.24334975369458128,
      "grad_norm": 0.4163839274003641,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -59077218.042553194,
      "logits/rejected": -103977313.97530864,
      "logps/chosen": -301.27659574468083,
      "logps/rejected": -431.01234567901236,
      "loss": 0.2326,
      "rewards/chosen": -0.17553191489361702,
      "rewards/margins": 7.676319936958235,
      "rewards/rejected": -7.851851851851852,
      "step": 494
    },
    {
      "epoch": 0.2438423645320197,
      "grad_norm": 0.4427120046266015,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -75415230.74509804,
      "logits/rejected": -69178780.25974026,
      "logps/chosen": -291.1372549019608,
      "logps/rejected": -476.2597402597403,
      "loss": 0.2292,
      "rewards/chosen": -0.5870098039215687,
      "rewards/margins": 9.075327858416093,
      "rewards/rejected": -9.662337662337663,
      "step": 495
    },
    {
      "epoch": 0.24433497536945814,
      "grad_norm": 0.4312384047349302,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -74368236.3076923,
      "logits/rejected": -75387095.57894737,
      "logps/chosen": -276.3076923076923,
      "logps/rejected": -516.2105263157895,
      "loss": 0.2197,
      "rewards/chosen": 0.5384615384615384,
      "rewards/margins": 10.288461538461538,
      "rewards/rejected": -9.75,
      "step": 496
    },
    {
      "epoch": 0.24482758620689654,
      "grad_norm": 0.4579475139152946,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -62646838.4680851,
      "logits/rejected": -70526445.03703703,
      "logps/chosen": -300.25531914893617,
      "logps/rejected": -434.17283950617286,
      "loss": 0.2439,
      "rewards/chosen": 5.593774024476397,
      "rewards/margins": 13.815996246698617,
      "rewards/rejected": -8.222222222222221,
      "step": 497
    },
    {
      "epoch": 0.24532019704433497,
      "grad_norm": 0.4126240146526365,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -58877542.4,
      "logits/rejected": -75974097.45454545,
      "logps/chosen": -227.4,
      "logps/rejected": -538.9090909090909,
      "loss": 0.1619,
      "rewards/chosen": 0.8125,
      "rewards/margins": 11.380681818181818,
      "rewards/rejected": -10.568181818181818,
      "step": 498
    },
    {
      "epoch": 0.2458128078817734,
      "grad_norm": 0.37265356270140826,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -75586712.5106383,
      "logits/rejected": -80675625.08641975,
      "logps/chosen": -342.1276595744681,
      "logps/rejected": -497.77777777777777,
      "loss": 0.2103,
      "rewards/chosen": 3.6196756565824466,
      "rewards/margins": 12.878934915841706,
      "rewards/rejected": -9.25925925925926,
      "step": 499
    },
    {
      "epoch": 0.24630541871921183,
      "grad_norm": 0.5267379523646621,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -67183762.28571428,
      "logits/rejected": -61283441.777777776,
      "logps/chosen": -298.85714285714283,
      "logps/rejected": -435.1111111111111,
      "loss": 0.2415,
      "rewards/chosen": 0.30680629185267855,
      "rewards/margins": 9.834584069630457,
      "rewards/rejected": -9.527777777777779,
      "step": 500
    },
    {
      "epoch": 0.24679802955665026,
      "grad_norm": 0.4754248104051504,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -61656268.8,
      "logits/rejected": -84638903.79487179,
      "logps/chosen": -268.48,
      "logps/rejected": -496.8205128205128,
      "loss": 0.2766,
      "rewards/chosen": -0.37828125,
      "rewards/margins": 8.53197516025641,
      "rewards/rejected": -8.91025641025641,
      "step": 501
    },
    {
      "epoch": 0.24729064039408866,
      "grad_norm": 0.41515558566673605,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -63587231.39622641,
      "logits/rejected": -66661471.57333333,
      "logps/chosen": -344.45283018867923,
      "logps/rejected": -462.08,
      "loss": 0.197,
      "rewards/chosen": 1.267688679245283,
      "rewards/margins": 9.267061237839034,
      "rewards/rejected": -7.99937255859375,
      "step": 502
    },
    {
      "epoch": 0.24778325123152709,
      "grad_norm": 0.37246332475857347,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -57850705.17073171,
      "logits/rejected": -90828376.27586207,
      "logps/chosen": -255.609756097561,
      "logps/rejected": -498.7586206896552,
      "loss": 0.1936,
      "rewards/chosen": 2.7133971423637577,
      "rewards/margins": 13.954776452708586,
      "rewards/rejected": -11.241379310344827,
      "step": 503
    },
    {
      "epoch": 0.2482758620689655,
      "grad_norm": 0.384307752119197,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -54884002.34146342,
      "logits/rejected": -80222090.29885058,
      "logps/chosen": -349.6585365853659,
      "logps/rejected": -544.3678160919541,
      "loss": 0.1746,
      "rewards/chosen": 4.999469664038681,
      "rewards/margins": 15.079929434153623,
      "rewards/rejected": -10.080459770114942,
      "step": 504
    },
    {
      "epoch": 0.24876847290640394,
      "grad_norm": 0.4720389269280284,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -66959067.428571425,
      "logits/rejected": -77186844.44444445,
      "logps/chosen": -317.7142857142857,
      "logps/rejected": -498.22222222222223,
      "loss": 0.2339,
      "rewards/chosen": 0.7103794642857143,
      "rewards/margins": 10.140935019841269,
      "rewards/rejected": -9.430555555555555,
      "step": 505
    },
    {
      "epoch": 0.24926108374384237,
      "grad_norm": 0.41935212385273163,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -67644307.0638298,
      "logits/rejected": -70008629.72839506,
      "logps/chosen": -321.02127659574467,
      "logps/rejected": -470.51851851851853,
      "loss": 0.1791,
      "rewards/chosen": 0.14278590425531915,
      "rewards/margins": 6.049023847928159,
      "rewards/rejected": -5.906237943672839,
      "step": 506
    },
    {
      "epoch": 0.2497536945812808,
      "grad_norm": 0.43929885369132776,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -72701269.33333333,
      "logits/rejected": -73375053.10843374,
      "logps/chosen": -316.8,
      "logps/rejected": -454.93975903614455,
      "loss": 0.2142,
      "rewards/chosen": 0.4791666666666667,
      "rewards/margins": 9.696034136546183,
      "rewards/rejected": -9.216867469879517,
      "step": 507
    },
    {
      "epoch": 0.25024630541871923,
      "grad_norm": 0.44823980476587405,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -49632597.333333336,
      "logits/rejected": -70339611.67567568,
      "logps/chosen": -212.14814814814815,
      "logps/rejected": -486.05405405405406,
      "loss": 0.2183,
      "rewards/chosen": 0.5219907407407407,
      "rewards/margins": 10.278747497497497,
      "rewards/rejected": -9.756756756756756,
      "step": 508
    },
    {
      "epoch": 0.25073891625615763,
      "grad_norm": 0.39329765923689736,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -103721642.66666667,
      "logits/rejected": -66794291.2,
      "logps/chosen": -302.3333333333333,
      "logps/rejected": -444.8,
      "loss": 0.2073,
      "rewards/chosen": 0.4557291666666667,
      "rewards/margins": 10.005729166666667,
      "rewards/rejected": -9.55,
      "step": 509
    },
    {
      "epoch": 0.2512315270935961,
      "grad_norm": 0.4585928095855213,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -70404388.57142857,
      "logits/rejected": -93789297.77777778,
      "logps/chosen": -315.7142857142857,
      "logps/rejected": -560.4444444444445,
      "loss": 0.2528,
      "rewards/chosen": -0.16263253348214285,
      "rewards/margins": 10.309589688740079,
      "rewards/rejected": -10.472222222222221,
      "step": 510
    },
    {
      "epoch": 0.2517241379310345,
      "grad_norm": 0.3972846951801192,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -62425224.53333333,
      "logits/rejected": -74633938.8235294,
      "logps/chosen": -256.26666666666665,
      "logps/rejected": -538.3529411764706,
      "loss": 0.2225,
      "rewards/chosen": 2.8515393575032553,
      "rewards/margins": 14.939774651620901,
      "rewards/rejected": -12.088235294117647,
      "step": 511
    },
    {
      "epoch": 0.2522167487684729,
      "grad_norm": 0.3234723254869255,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -64073512.421052635,
      "logits/rejected": -66829243.733333334,
      "logps/chosen": -277.2631578947368,
      "logps/rejected": -511.2888888888889,
      "loss": 0.159,
      "rewards/chosen": -0.14152446546052633,
      "rewards/margins": 11.080697756761696,
      "rewards/rejected": -11.222222222222221,
      "step": 512
    },
    {
      "epoch": 0.25270935960591134,
      "grad_norm": 0.4885682925543118,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -56581160.96,
      "logits/rejected": -78293674.66666667,
      "logps/chosen": -259.68,
      "logps/rejected": -514.0512820512821,
      "loss": 0.196,
      "rewards/chosen": 0.58125,
      "rewards/margins": 10.401762820512822,
      "rewards/rejected": -9.820512820512821,
      "step": 513
    },
    {
      "epoch": 0.25320197044334974,
      "grad_norm": 0.5004510865938641,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -68351620.74074075,
      "logits/rejected": -71756606.27027027,
      "logps/chosen": -243.25925925925927,
      "logps/rejected": -486.05405405405406,
      "loss": 0.2376,
      "rewards/chosen": 2.1423475477430554,
      "rewards/margins": 13.723428628824136,
      "rewards/rejected": -11.58108108108108,
      "step": 514
    },
    {
      "epoch": 0.2536945812807882,
      "grad_norm": 0.39294872287794497,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -58642583.7037037,
      "logits/rejected": -79691776.0,
      "logps/chosen": -255.40740740740742,
      "logps/rejected": -518.918918918919,
      "loss": 0.2724,
      "rewards/chosen": -0.9171006944444444,
      "rewards/margins": 9.25857498123123,
      "rewards/rejected": -10.175675675675675,
      "step": 515
    },
    {
      "epoch": 0.2541871921182266,
      "grad_norm": 0.4310237856184267,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -59885340.44444445,
      "logits/rejected": -79351697.2972973,
      "logps/chosen": -294.81481481481484,
      "logps/rejected": -536.6486486486486,
      "loss": 0.2409,
      "rewards/chosen": 0.2222222222222222,
      "rewards/margins": 5.59035700434321,
      "rewards/rejected": -5.368134782120988,
      "step": 516
    },
    {
      "epoch": 0.254679802955665,
      "grad_norm": 0.4212510526878625,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -77964709.64705883,
      "logits/rejected": -72011297.24675325,
      "logps/chosen": -445.4901960784314,
      "logps/rejected": -512.4155844155844,
      "loss": 0.2056,
      "rewards/chosen": 1.2549019607843137,
      "rewards/margins": 9.656268355619947,
      "rewards/rejected": -8.401366394835634,
      "step": 517
    },
    {
      "epoch": 0.25517241379310346,
      "grad_norm": 0.44879928939559904,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -62379116.93617021,
      "logits/rejected": -83782516.93827161,
      "logps/chosen": -292.4255319148936,
      "logps/rejected": -499.358024691358,
      "loss": 0.1924,
      "rewards/chosen": 2.941502347905585,
      "rewards/margins": 12.941502347905585,
      "rewards/rejected": -10.0,
      "step": 518
    },
    {
      "epoch": 0.25566502463054186,
      "grad_norm": 0.4903625997536651,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -61027123.2,
      "logits/rejected": -70550344.20512821,
      "logps/chosen": -383.68,
      "logps/rejected": -472.61538461538464,
      "loss": 0.2229,
      "rewards/chosen": 0.5634375,
      "rewards/margins": 10.217283653846152,
      "rewards/rejected": -9.653846153846153,
      "step": 519
    },
    {
      "epoch": 0.2561576354679803,
      "grad_norm": 0.6272520078243976,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -72095425.42222223,
      "logits/rejected": -71202100.43373494,
      "logps/chosen": -345.4222222222222,
      "logps/rejected": -461.87951807228916,
      "loss": 0.1725,
      "rewards/chosen": 0.9347222222222222,
      "rewards/margins": 11.778095716198125,
      "rewards/rejected": -10.843373493975903,
      "step": 520
    },
    {
      "epoch": 0.2566502463054187,
      "grad_norm": 0.45067789958837756,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -63800024.177777775,
      "logits/rejected": -65491782.93975904,
      "logps/chosen": -317.8666666666667,
      "logps/rejected": -477.68674698795184,
      "loss": 0.1905,
      "rewards/chosen": -0.014583333333333334,
      "rewards/margins": 9.623970883534138,
      "rewards/rejected": -9.63855421686747,
      "step": 521
    },
    {
      "epoch": 0.2571428571428571,
      "grad_norm": 0.4390077544525934,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -56349562.43478261,
      "logits/rejected": -69461766.24390244,
      "logps/chosen": -219.82608695652175,
      "logps/rejected": -483.1219512195122,
      "loss": 0.2415,
      "rewards/chosen": -0.7432065217391305,
      "rewards/margins": 8.98850079533404,
      "rewards/rejected": -9.731707317073171,
      "step": 522
    },
    {
      "epoch": 0.2576354679802956,
      "grad_norm": 0.38333333334949377,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -61361114.074074075,
      "logits/rejected": -91934609.2972973,
      "logps/chosen": -333.9259259259259,
      "logps/rejected": -520.6486486486486,
      "loss": 0.2542,
      "rewards/chosen": -1.48828125,
      "rewards/margins": 10.187394425675675,
      "rewards/rejected": -11.675675675675675,
      "step": 523
    },
    {
      "epoch": 0.258128078817734,
      "grad_norm": 0.43305137610958433,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -51224053.10638298,
      "logits/rejected": -68506965.33333333,
      "logps/chosen": -212.4255319148936,
      "logps/rejected": -470.91358024691357,
      "loss": 0.2177,
      "rewards/chosen": 0.1675531914893617,
      "rewards/margins": 10.649034672970842,
      "rewards/rejected": -10.481481481481481,
      "step": 524
    },
    {
      "epoch": 0.25862068965517243,
      "grad_norm": 0.5438663319929314,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -66041222.981818184,
      "logits/rejected": -69407112.76712328,
      "logps/chosen": -306.90909090909093,
      "logps/rejected": -484.3835616438356,
      "loss": 0.252,
      "rewards/chosen": 4.459687389026989,
      "rewards/margins": 12.672016156150276,
      "rewards/rejected": -8.212328767123287,
      "step": 525
    },
    {
      "epoch": 0.25911330049261083,
      "grad_norm": 0.40659042588632655,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -56337128.72727273,
      "logits/rejected": -90976451.04761904,
      "logps/chosen": -269.8181818181818,
      "logps/rejected": -518.0952380952381,
      "loss": 0.1925,
      "rewards/chosen": 0.5696022727272727,
      "rewards/margins": 11.10531655844156,
      "rewards/rejected": -10.535714285714286,
      "step": 526
    },
    {
      "epoch": 0.2596059113300493,
      "grad_norm": 0.48856812435829244,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -74812687.6734694,
      "logits/rejected": -83886080.0,
      "logps/chosen": -286.3673469387755,
      "logps/rejected": -558.1772151898734,
      "loss": 0.2294,
      "rewards/chosen": 0.6052295918367347,
      "rewards/margins": 11.478647313355722,
      "rewards/rejected": -10.873417721518987,
      "step": 527
    },
    {
      "epoch": 0.2600985221674877,
      "grad_norm": 0.47704723228551466,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -64520889.19148936,
      "logits/rejected": -72701269.33333333,
      "logps/chosen": -263.82978723404256,
      "logps/rejected": -513.5802469135803,
      "loss": 0.2291,
      "rewards/chosen": -0.052194148936170214,
      "rewards/margins": 10.799657702915681,
      "rewards/rejected": -10.851851851851851,
      "step": 528
    },
    {
      "epoch": 0.2605911330049261,
      "grad_norm": 0.38145764046204816,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -61117001.14285714,
      "logits/rejected": -86434517.87341772,
      "logps/chosen": -233.79591836734693,
      "logps/rejected": -538.7341772151899,
      "loss": 0.1855,
      "rewards/chosen": 1.1926020408163265,
      "rewards/margins": 8.9141210281581,
      "rewards/rejected": -7.7215189873417724,
      "step": 529
    },
    {
      "epoch": 0.26108374384236455,
      "grad_norm": 0.34885314920536403,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -48663458.90909091,
      "logits/rejected": -77994081.52380952,
      "logps/chosen": -183.0909090909091,
      "logps/rejected": -506.6666666666667,
      "loss": 0.176,
      "rewards/chosen": 0.9190340909090909,
      "rewards/margins": 11.28808170995671,
      "rewards/rejected": -10.369047619047619,
      "step": 530
    },
    {
      "epoch": 0.26157635467980295,
      "grad_norm": 0.44576618206288754,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -53633546.89361702,
      "logits/rejected": -78086548.54320988,
      "logps/chosen": -241.19148936170214,
      "logps/rejected": -494.22222222222223,
      "loss": 0.205,
      "rewards/chosen": 1.1888297872340425,
      "rewards/margins": 10.657965589703178,
      "rewards/rejected": -9.469135802469136,
      "step": 531
    },
    {
      "epoch": 0.2620689655172414,
      "grad_norm": 0.3836935109416018,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -82429724.44444445,
      "logits/rejected": -79600595.47826087,
      "logps/chosen": -244.88888888888889,
      "logps/rejected": -433.39130434782606,
      "loss": 0.1738,
      "rewards/chosen": 2.939240985446506,
      "rewards/margins": 11.97184968109868,
      "rewards/rejected": -9.032608695652174,
      "step": 532
    },
    {
      "epoch": 0.2625615763546798,
      "grad_norm": 0.3883142733489655,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -63089322.666666664,
      "logits/rejected": -71303168.0,
      "logps/chosen": -295.3333333333333,
      "logps/rejected": -457.2,
      "loss": 0.192,
      "rewards/chosen": 3.5592673619588218,
      "rewards/margins": 12.253017361958822,
      "rewards/rejected": -8.69375,
      "step": 533
    },
    {
      "epoch": 0.2630541871921182,
      "grad_norm": 0.4419144126712791,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -58108586.666666664,
      "logits/rejected": -60240691.2,
      "logps/chosen": -286.6666666666667,
      "logps/rejected": -400.4,
      "loss": 0.2193,
      "rewards/chosen": 0.6223958333333334,
      "rewards/margins": 9.184895833333334,
      "rewards/rejected": -8.5625,
      "step": 534
    },
    {
      "epoch": 0.26354679802955666,
      "grad_norm": 0.4231946013808008,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -77059180.9361702,
      "logits/rejected": -88753543.90123457,
      "logps/chosen": -281.8723404255319,
      "logps/rejected": -584.6913580246913,
      "loss": 0.2024,
      "rewards/chosen": 0.723404255319149,
      "rewards/margins": 11.056737588652483,
      "rewards/rejected": -10.333333333333334,
      "step": 535
    },
    {
      "epoch": 0.26403940886699506,
      "grad_norm": 0.43619839857391335,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -79971396.26666667,
      "logits/rejected": -74789999.03614458,
      "logps/chosen": -346.31111111111113,
      "logps/rejected": -528.1927710843373,
      "loss": 0.1815,
      "rewards/chosen": 0.5615451388888889,
      "rewards/margins": 11.139858391900937,
      "rewards/rejected": -10.578313253012048,
      "step": 536
    },
    {
      "epoch": 0.2645320197044335,
      "grad_norm": 0.46542940385955234,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -54084446.315789476,
      "logits/rejected": -63032709.40845071,
      "logps/chosen": -320.0,
      "logps/rejected": -484.9577464788732,
      "loss": 0.2098,
      "rewards/chosen": 1.1260964912280702,
      "rewards/margins": 10.316237336298492,
      "rewards/rejected": -9.190140845070422,
      "step": 537
    },
    {
      "epoch": 0.2650246305418719,
      "grad_norm": 0.4864373194402953,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -61754433.36170213,
      "logits/rejected": -91860435.75308642,
      "logps/chosen": -350.29787234042556,
      "logps/rejected": -484.34567901234567,
      "loss": 0.1571,
      "rewards/chosen": 1.738031914893617,
      "rewards/margins": 11.491118334646703,
      "rewards/rejected": -9.753086419753087,
      "step": 538
    },
    {
      "epoch": 0.2655172413793103,
      "grad_norm": 0.443012450738382,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -48637794.461538464,
      "logits/rejected": -71965426.5263158,
      "logps/chosen": -263.6923076923077,
      "logps/rejected": -413.4736842105263,
      "loss": 0.21,
      "rewards/chosen": 0.7427884615384616,
      "rewards/margins": 10.04542004048583,
      "rewards/rejected": -9.302631578947368,
      "step": 539
    },
    {
      "epoch": 0.2660098522167488,
      "grad_norm": 0.3995365638031227,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -70753913.90476191,
      "logits/rejected": -75790097.86046511,
      "logps/chosen": -288.95238095238096,
      "logps/rejected": -493.3953488372093,
      "loss": 0.1869,
      "rewards/chosen": 7.635290236700149,
      "rewards/margins": 17.274825120421077,
      "rewards/rejected": -9.63953488372093,
      "step": 540
    },
    {
      "epoch": 0.2665024630541872,
      "grad_norm": 0.37284898193778043,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -53187344.34042553,
      "logits/rejected": -71251386.4691358,
      "logps/chosen": -226.04255319148936,
      "logps/rejected": -480.39506172839504,
      "loss": 0.2012,
      "rewards/chosen": 3.2971311528631984,
      "rewards/margins": 11.728067989765204,
      "rewards/rejected": -8.430936836902006,
      "step": 541
    },
    {
      "epoch": 0.26699507389162563,
      "grad_norm": 0.4560095509212947,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -58589184.0,
      "logits/rejected": -68472012.8,
      "logps/chosen": -300.0,
      "logps/rejected": -433.2,
      "loss": 0.2318,
      "rewards/chosen": -0.6861979166666666,
      "rewards/margins": 5.526302083333333,
      "rewards/rejected": -6.2125,
      "step": 542
    },
    {
      "epoch": 0.26748768472906403,
      "grad_norm": 0.44010762231194295,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -60856976.905660376,
      "logits/rejected": -77063345.49333334,
      "logps/chosen": -270.188679245283,
      "logps/rejected": -492.37333333333333,
      "loss": 0.2517,
      "rewards/chosen": 1.8773792194870282,
      "rewards/margins": 12.064045886153696,
      "rewards/rejected": -10.186666666666667,
      "step": 543
    },
    {
      "epoch": 0.26798029556650244,
      "grad_norm": 0.47214064691796487,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -82580306.11320755,
      "logits/rejected": -85563801.6,
      "logps/chosen": -322.8679245283019,
      "logps/rejected": -485.5466666666667,
      "loss": 0.2523,
      "rewards/chosen": -0.652122641509434,
      "rewards/margins": 8.667877358490566,
      "rewards/rejected": -9.32,
      "step": 544
    },
    {
      "epoch": 0.2684729064039409,
      "grad_norm": 0.5573823750206949,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -71706466.46153846,
      "logits/rejected": -69868274.5263158,
      "logps/chosen": -221.23076923076923,
      "logps/rejected": -492.63157894736844,
      "loss": 0.2243,
      "rewards/chosen": 3.2018954937274637,
      "rewards/margins": 12.24136917793799,
      "rewards/rejected": -9.039473684210526,
      "step": 545
    },
    {
      "epoch": 0.2689655172413793,
      "grad_norm": 0.41314050097540567,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -70645237.96078432,
      "logits/rejected": -73863327.58441558,
      "logps/chosen": -260.8627450980392,
      "logps/rejected": -467.1168831168831,
      "loss": 0.2265,
      "rewards/chosen": -0.12415747549019608,
      "rewards/margins": 9.187530836198114,
      "rewards/rejected": -9.311688311688311,
      "step": 546
    },
    {
      "epoch": 0.26945812807881775,
      "grad_norm": 0.5197862183081222,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -80717050.31111111,
      "logits/rejected": -79742309.78313252,
      "logps/chosen": -308.7111111111111,
      "logps/rejected": -507.3734939759036,
      "loss": 0.2283,
      "rewards/chosen": 3.0596754286024304,
      "rewards/margins": 13.071723621373515,
      "rewards/rejected": -10.012048192771084,
      "step": 547
    },
    {
      "epoch": 0.26995073891625615,
      "grad_norm": 0.4918653330784461,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -94323069.02325581,
      "logits/rejected": -79247673.22352941,
      "logps/chosen": -373.5813953488372,
      "logps/rejected": -487.15294117647056,
      "loss": 0.2038,
      "rewards/chosen": 0.12790697674418605,
      "rewards/margins": 9.339671682626538,
      "rewards/rejected": -9.211764705882352,
      "step": 548
    },
    {
      "epoch": 0.2704433497536946,
      "grad_norm": 0.4445409364955147,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -74705214.57777777,
      "logits/rejected": -66148722.12048193,
      "logps/chosen": -276.0888888888889,
      "logps/rejected": -441.06024096385545,
      "loss": 0.1749,
      "rewards/chosen": 3.6854112413194446,
      "rewards/margins": 12.22757991601824,
      "rewards/rejected": -8.542168674698795,
      "step": 549
    },
    {
      "epoch": 0.270935960591133,
      "grad_norm": 0.3860413933649138,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -55622190.54545455,
      "logits/rejected": -78593267.8095238,
      "logps/chosen": -236.36363636363637,
      "logps/rejected": -506.6666666666667,
      "loss": 0.1869,
      "rewards/chosen": 1.1107954545454546,
      "rewards/margins": 10.670319264069265,
      "rewards/rejected": -9.55952380952381,
      "step": 550
    },
    {
      "epoch": 0.2714285714285714,
      "grad_norm": 0.4844942897507497,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -59517173.76,
      "logits/rejected": -72378630.56410256,
      "logps/chosen": -304.96,
      "logps/rejected": -501.3333333333333,
      "loss": 0.2124,
      "rewards/chosen": 3.594355163574219,
      "rewards/margins": 12.902047471266528,
      "rewards/rejected": -9.307692307692308,
      "step": 551
    },
    {
      "epoch": 0.27192118226600986,
      "grad_norm": 0.3739187282438958,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -75790097.86046511,
      "logits/rejected": -81714910.87058823,
      "logps/chosen": -430.51162790697674,
      "logps/rejected": -495.43529411764706,
      "loss": 0.1967,
      "rewards/chosen": 0.02616279069767442,
      "rewards/margins": 9.832045143638851,
      "rewards/rejected": -9.805882352941177,
      "step": 552
    },
    {
      "epoch": 0.27241379310344827,
      "grad_norm": 0.481706043586712,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -69793218.56,
      "logits/rejected": -69044696.61538461,
      "logps/chosen": -311.36,
      "logps/rejected": -501.3333333333333,
      "loss": 0.2231,
      "rewards/chosen": 0.385625,
      "rewards/margins": 9.840753205128204,
      "rewards/rejected": -9.455128205128204,
      "step": 553
    },
    {
      "epoch": 0.2729064039408867,
      "grad_norm": 0.46125463633298336,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -69987307.92156863,
      "logits/rejected": -75279586.07792208,
      "logps/chosen": -344.94117647058823,
      "logps/rejected": -521.974025974026,
      "loss": 0.2198,
      "rewards/chosen": -0.42892156862745096,
      "rewards/margins": 10.66198752228164,
      "rewards/rejected": -11.090909090909092,
      "step": 554
    },
    {
      "epoch": 0.2733990147783251,
      "grad_norm": 0.5550550189773641,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -70346572.3508772,
      "logits/rejected": -77978609.57746479,
      "logps/chosen": -280.70175438596493,
      "logps/rejected": -541.7464788732394,
      "loss": 0.2371,
      "rewards/chosen": 0.6181469298245614,
      "rewards/margins": 6.683223730963426,
      "rewards/rejected": -6.065076801138864,
      "step": 555
    },
    {
      "epoch": 0.2738916256157635,
      "grad_norm": 0.5115206913045063,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -61982492.44444445,
      "logits/rejected": -69602774.48648648,
      "logps/chosen": -266.3703703703704,
      "logps/rejected": -442.81081081081084,
      "loss": 0.1989,
      "rewards/chosen": 3.0879714400679976,
      "rewards/margins": 11.452836304932863,
      "rewards/rejected": -8.364864864864865,
      "step": 556
    },
    {
      "epoch": 0.274384236453202,
      "grad_norm": 0.4676091595584576,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -86245376.0,
      "logits/rejected": -71093452.8,
      "logps/chosen": -349.0,
      "logps/rejected": -491.6,
      "loss": 0.2021,
      "rewards/chosen": 3.43559201558431,
      "rewards/margins": 12.78559201558431,
      "rewards/rejected": -9.35,
      "step": 557
    },
    {
      "epoch": 0.2748768472906404,
      "grad_norm": 0.4990499878922012,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -65626394.48275862,
      "logits/rejected": -69625446.4,
      "logps/chosen": -272.0,
      "logps/rejected": -471.3142857142857,
      "loss": 0.2981,
      "rewards/chosen": -1.443426724137931,
      "rewards/margins": 8.599430418719212,
      "rewards/rejected": -10.042857142857143,
      "step": 558
    },
    {
      "epoch": 0.27536945812807884,
      "grad_norm": 0.4283002230644313,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -92810131.0638298,
      "logits/rejected": -74668967.50617284,
      "logps/chosen": -329.70212765957444,
      "logps/rejected": -508.8395061728395,
      "loss": 0.1803,
      "rewards/chosen": 3.1775506202210773,
      "rewards/margins": 12.609649385653174,
      "rewards/rejected": -9.432098765432098,
      "step": 559
    },
    {
      "epoch": 0.27586206896551724,
      "grad_norm": 0.48582387129316024,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -57744836.46511628,
      "logits/rejected": -70661686.21176471,
      "logps/chosen": -329.30232558139534,
      "logps/rejected": -479.6235294117647,
      "loss": 0.1886,
      "rewards/chosen": 2.914239395496457,
      "rewards/margins": 12.102474689614104,
      "rewards/rejected": -9.188235294117646,
      "step": 560
    },
    {
      "epoch": 0.27635467980295564,
      "grad_norm": 0.44219539860358614,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -58309049.7254902,
      "logits/rejected": -86064939.22077923,
      "logps/chosen": -253.80392156862746,
      "logps/rejected": -496.6233766233766,
      "loss": 0.2185,
      "rewards/chosen": -0.042279411764705885,
      "rewards/margins": 10.529149159663865,
      "rewards/rejected": -10.571428571428571,
      "step": 561
    },
    {
      "epoch": 0.2768472906403941,
      "grad_norm": 0.565801382102556,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -59978547.2,
      "logits/rejected": -85499273.84615384,
      "logps/chosen": -224.0,
      "logps/rejected": -482.46153846153845,
      "loss": 0.2331,
      "rewards/chosen": -0.39921875,
      "rewards/margins": 9.421294070512822,
      "rewards/rejected": -9.820512820512821,
      "step": 562
    },
    {
      "epoch": 0.2773399014778325,
      "grad_norm": 0.605643105891297,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -62958250.666666664,
      "logits/rejected": -69468160.0,
      "logps/chosen": -233.83333333333334,
      "logps/rejected": -541.6,
      "loss": 0.2416,
      "rewards/chosen": 3.7859776814778647,
      "rewards/margins": 11.39788564046224,
      "rewards/rejected": -7.611907958984375,
      "step": 563
    },
    {
      "epoch": 0.27783251231527095,
      "grad_norm": 0.7314895090055761,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -66796522.212765954,
      "logits/rejected": -73011958.51851852,
      "logps/chosen": -251.5744680851064,
      "logps/rejected": -494.22222222222223,
      "loss": 0.2041,
      "rewards/chosen": 0.7087765957446809,
      "rewards/margins": 9.807542027843446,
      "rewards/rejected": -9.098765432098766,
      "step": 564
    },
    {
      "epoch": 0.27832512315270935,
      "grad_norm": 0.5268637947035827,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -65583662.54545455,
      "logits/rejected": -69505609.14285715,
      "logps/chosen": -348.72727272727275,
      "logps/rejected": -499.04761904761904,
      "loss": 0.184,
      "rewards/chosen": 3.166410272771662,
      "rewards/margins": 11.666410272771662,
      "rewards/rejected": -8.5,
      "step": 565
    },
    {
      "epoch": 0.2788177339901478,
      "grad_norm": 1.9884820006930797,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -71428997.12,
      "logits/rejected": -67485275.8974359,
      "logps/chosen": -243.84,
      "logps/rejected": -455.79487179487177,
      "loss": 0.2412,
      "rewards/chosen": -0.24,
      "rewards/margins": 7.490769230769231,
      "rewards/rejected": -7.730769230769231,
      "step": 566
    },
    {
      "epoch": 0.2793103448275862,
      "grad_norm": 0.42214946767335126,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -75581358.08,
      "logits/rejected": -73023908.1025641,
      "logps/chosen": -267.2,
      "logps/rejected": -507.0769230769231,
      "loss": 0.1933,
      "rewards/chosen": 0.89,
      "rewards/margins": 11.12076923076923,
      "rewards/rejected": -10.23076923076923,
      "step": 567
    },
    {
      "epoch": 0.2798029556650246,
      "grad_norm": 0.6320527687801526,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -64225280.0,
      "logits/rejected": -88394956.8,
      "logps/chosen": -318.0,
      "logps/rejected": -532.0,
      "loss": 0.2102,
      "rewards/chosen": 1.044921875,
      "rewards/margins": 9.694921875,
      "rewards/rejected": -8.65,
      "step": 568
    },
    {
      "epoch": 0.28029556650246307,
      "grad_norm": 0.462371642177571,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -47567220.36363637,
      "logits/rejected": -67658118.09523809,
      "logps/chosen": -244.0,
      "logps/rejected": -448.3809523809524,
      "loss": 0.2,
      "rewards/chosen": 3.0269574252041904,
      "rewards/margins": 12.146005044251808,
      "rewards/rejected": -9.119047619047619,
      "step": 569
    },
    {
      "epoch": 0.28078817733990147,
      "grad_norm": 0.6515801670583563,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -45767258.35294118,
      "logits/rejected": -73536498.7012987,
      "logps/chosen": -211.6078431372549,
      "logps/rejected": -524.4675324675325,
      "loss": 0.2198,
      "rewards/chosen": 0.9172794117647058,
      "rewards/margins": 9.527669022154317,
      "rewards/rejected": -8.61038961038961,
      "step": 570
    },
    {
      "epoch": 0.2812807881773399,
      "grad_norm": 0.49514223870111057,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -64181589.333333336,
      "logits/rejected": -79272345.6,
      "logps/chosen": -271.6666666666667,
      "logps/rejected": -458.4,
      "loss": 0.2156,
      "rewards/chosen": 0.8501790364583334,
      "rewards/margins": 10.087679036458335,
      "rewards/rejected": -9.2375,
      "step": 571
    },
    {
      "epoch": 0.2817733990147783,
      "grad_norm": 0.5365451847435575,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -52741141.787234046,
      "logits/rejected": -70785352.69135803,
      "logps/chosen": -223.48936170212767,
      "logps/rejected": -525.0370370370371,
      "loss": 0.1565,
      "rewards/chosen": 4.1722236795628325,
      "rewards/margins": 13.196915037587523,
      "rewards/rejected": -9.024691358024691,
      "step": 572
    },
    {
      "epoch": 0.2822660098522167,
      "grad_norm": 0.5852630895684319,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -63535938.37037037,
      "logits/rejected": -82525765.1891892,
      "logps/chosen": -307.25925925925924,
      "logps/rejected": -484.3243243243243,
      "loss": 0.2551,
      "rewards/chosen": 0.02546296296296296,
      "rewards/margins": 8.768706206206208,
      "rewards/rejected": -8.743243243243244,
      "step": 573
    },
    {
      "epoch": 0.2827586206896552,
      "grad_norm": 0.47731845990830296,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -67310513.23076923,
      "logits/rejected": -68598945.68421052,
      "logps/chosen": -265.0769230769231,
      "logps/rejected": -416.8421052631579,
      "loss": 0.2131,
      "rewards/chosen": 5.207295931302584,
      "rewards/margins": 13.042822247092058,
      "rewards/rejected": -7.8355263157894735,
      "step": 574
    },
    {
      "epoch": 0.2832512315270936,
      "grad_norm": 0.5473030720853506,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -87411080.17021276,
      "logits/rejected": -80364935.90123457,
      "logps/chosen": -304.0,
      "logps/rejected": -461.4320987654321,
      "loss": 0.2464,
      "rewards/chosen": 2.1964635646089596,
      "rewards/margins": 10.912512947325009,
      "rewards/rejected": -8.716049382716049,
      "step": 575
    },
    {
      "epoch": 0.28374384236453204,
      "grad_norm": 0.6747053058450018,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -68968602.56603773,
      "logits/rejected": -80530636.8,
      "logps/chosen": -294.0377358490566,
      "logps/rejected": -499.2,
      "loss": 0.2205,
      "rewards/chosen": 0.777122641509434,
      "rewards/margins": 9.197122641509434,
      "rewards/rejected": -8.42,
      "step": 576
    },
    {
      "epoch": 0.28423645320197044,
      "grad_norm": 0.5264723584063021,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -59688172.307692304,
      "logits/rejected": -58620391.61904762,
      "logps/chosen": -325.9076923076923,
      "logps/rejected": -467.8095238095238,
      "loss": 0.2372,
      "rewards/chosen": 1.2701923076923076,
      "rewards/margins": 9.452731990231989,
      "rewards/rejected": -8.182539682539682,
      "step": 577
    },
    {
      "epoch": 0.28472906403940884,
      "grad_norm": 0.7739658620755226,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -69641273.96226415,
      "logits/rejected": -85563801.6,
      "logps/chosen": -354.7169811320755,
      "logps/rejected": -548.2666666666667,
      "loss": 0.2099,
      "rewards/chosen": 6.474024934588738,
      "rewards/margins": 15.367358267922071,
      "rewards/rejected": -8.893333333333333,
      "step": 578
    },
    {
      "epoch": 0.2852216748768473,
      "grad_norm": 0.41376353623939943,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -60351374.222222224,
      "logits/rejected": -70343026.12048192,
      "logps/chosen": -254.04444444444445,
      "logps/rejected": -492.5301204819277,
      "loss": 0.178,
      "rewards/chosen": 3.7270941840277776,
      "rewards/margins": 12.582515870774765,
      "rewards/rejected": -8.855421686746988,
      "step": 579
    },
    {
      "epoch": 0.2857142857142857,
      "grad_norm": 0.5567204655078561,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -62598008.75471698,
      "logits/rejected": -77510737.92,
      "logps/chosen": -348.9811320754717,
      "logps/rejected": -446.29333333333335,
      "loss": 0.2289,
      "rewards/chosen": 1.921618767504422,
      "rewards/margins": 5.8632366060460885,
      "rewards/rejected": -3.9416178385416667,
      "step": 580
    },
    {
      "epoch": 0.28620689655172415,
      "grad_norm": 0.516787362400974,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -52578596.571428575,
      "logits/rejected": -73254007.06976745,
      "logps/chosen": -187.8095238095238,
      "logps/rejected": -438.3255813953488,
      "loss": 0.1834,
      "rewards/chosen": 0.5230654761904762,
      "rewards/margins": 8.185856173864895,
      "rewards/rejected": -7.662790697674419,
      "step": 581
    },
    {
      "epoch": 0.28669950738916256,
      "grad_norm": 0.4021348377008341,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -62144177.63265306,
      "logits/rejected": -79107758.98734178,
      "logps/chosen": -291.265306122449,
      "logps/rejected": -421.67088607594934,
      "loss": 0.2208,
      "rewards/chosen": 2.523922044403699,
      "rewards/margins": 10.080884069720156,
      "rewards/rejected": -7.556962025316456,
      "step": 582
    },
    {
      "epoch": 0.287192118226601,
      "grad_norm": 0.4967051137610648,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -66497194.666666664,
      "logits/rejected": -73400320.0,
      "logps/chosen": -254.33333333333334,
      "logps/rejected": -426.8,
      "loss": 0.2163,
      "rewards/chosen": 0.6619466145833334,
      "rewards/margins": 7.924446614583333,
      "rewards/rejected": -7.2625,
      "step": 583
    },
    {
      "epoch": 0.2876847290640394,
      "grad_norm": 0.5756094222027017,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -77911175.24528302,
      "logits/rejected": -87241523.2,
      "logps/chosen": -303.8490566037736,
      "logps/rejected": -552.1066666666667,
      "loss": 0.2234,
      "rewards/chosen": 2.5411325130822524,
      "rewards/margins": 7.349413763082253,
      "rewards/rejected": -4.80828125,
      "step": 584
    },
    {
      "epoch": 0.2881773399014778,
      "grad_norm": 0.45741507649570984,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -61106670.344827585,
      "logits/rejected": -83047219.2,
      "logps/chosen": -279.44827586206895,
      "logps/rejected": -483.65714285714284,
      "loss": 0.2249,
      "rewards/chosen": 0.5635775862068966,
      "rewards/margins": 6.570720443349754,
      "rewards/rejected": -6.007142857142857,
      "step": 585
    },
    {
      "epoch": 0.28866995073891627,
      "grad_norm": 0.4030391534271517,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -84327585.68421052,
      "logits/rejected": -84165700.26666667,
      "logps/chosen": -350.3157894736842,
      "logps/rejected": -521.2444444444444,
      "loss": 0.1764,
      "rewards/chosen": 2.8766601964047083,
      "rewards/margins": 10.78777130751582,
      "rewards/rejected": -7.911111111111111,
      "step": 586
    },
    {
      "epoch": 0.28916256157635467,
      "grad_norm": 0.5780606586344836,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -68531931.42857143,
      "logits/rejected": -73866353.77777778,
      "logps/chosen": -272.2857142857143,
      "logps/rejected": -520.4444444444445,
      "loss": 0.2183,
      "rewards/chosen": 3.204768862043108,
      "rewards/margins": 12.718657750931998,
      "rewards/rejected": -9.51388888888889,
      "step": 587
    },
    {
      "epoch": 0.2896551724137931,
      "grad_norm": 0.4610573981738896,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -66060288.0,
      "logits/rejected": -69405744.76190476,
      "logps/chosen": -258.90909090909093,
      "logps/rejected": -455.6190476190476,
      "loss": 0.2041,
      "rewards/chosen": 0.23082386363636365,
      "rewards/margins": 8.433204816017316,
      "rewards/rejected": -8.202380952380953,
      "step": 588
    },
    {
      "epoch": 0.29014778325123153,
      "grad_norm": 0.48097654279953533,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -64259902.79245283,
      "logits/rejected": -75106003.62666667,
      "logps/chosen": -261.58490566037733,
      "logps/rejected": -519.68,
      "loss": 0.2133,
      "rewards/chosen": 3.590224068119841,
      "rewards/margins": 13.79022406811984,
      "rewards/rejected": -10.2,
      "step": 589
    },
    {
      "epoch": 0.29064039408866993,
      "grad_norm": 0.7200901262456861,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -79051966.91525424,
      "logits/rejected": -66379419.82608695,
      "logps/chosen": -300.06779661016947,
      "logps/rejected": -450.5507246376812,
      "loss": 0.2215,
      "rewards/chosen": 0.8449417372881356,
      "rewards/margins": 9.584072172070744,
      "rewards/rejected": -8.73913043478261,
      "step": 590
    },
    {
      "epoch": 0.2911330049261084,
      "grad_norm": 0.516199756437163,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -68399419.07692307,
      "logits/rejected": -86314361.26315789,
      "logps/chosen": -249.23076923076923,
      "logps/rejected": -542.3157894736842,
      "loss": 0.2278,
      "rewards/chosen": 4.8106830303485575,
      "rewards/margins": 14.047525135611714,
      "rewards/rejected": -9.236842105263158,
      "step": 591
    },
    {
      "epoch": 0.2916256157635468,
      "grad_norm": 0.479036231990338,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -67986741.58139534,
      "logits/rejected": -75398782.49411765,
      "logps/chosen": -363.16279069767444,
      "logps/rejected": -504.47058823529414,
      "loss": 0.1854,
      "rewards/chosen": 5.930968173714572,
      "rewards/margins": 14.378026997243984,
      "rewards/rejected": -8.447058823529412,
      "step": 592
    },
    {
      "epoch": 0.29211822660098524,
      "grad_norm": 0.5438706948613773,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -62360595.320754714,
      "logits/rejected": -75161927.68,
      "logps/chosen": -356.22641509433964,
      "logps/rejected": -471.46666666666664,
      "loss": 0.2394,
      "rewards/chosen": -0.27476415094339623,
      "rewards/margins": 8.925235849056603,
      "rewards/rejected": -9.2,
      "step": 593
    },
    {
      "epoch": 0.29261083743842364,
      "grad_norm": 0.47529816594624974,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -57746578.28571428,
      "logits/rejected": -70312846.22222222,
      "logps/chosen": -258.57142857142856,
      "logps/rejected": -487.55555555555554,
      "loss": 0.2249,
      "rewards/chosen": 0.5248325892857143,
      "rewards/margins": 9.934554811507935,
      "rewards/rejected": -9.409722222222221,
      "step": 594
    },
    {
      "epoch": 0.29310344827586204,
      "grad_norm": 0.49204260457043125,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -72079890.96296297,
      "logits/rejected": -83886080.0,
      "logps/chosen": -240.59259259259258,
      "logps/rejected": -583.7837837837837,
      "loss": 0.2334,
      "rewards/chosen": 0.6689814814814815,
      "rewards/margins": 8.31763013013013,
      "rewards/rejected": -7.648648648648648,
      "step": 595
    },
    {
      "epoch": 0.2935960591133005,
      "grad_norm": 0.5622078428146405,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -59635376.872727275,
      "logits/rejected": -79864144.65753424,
      "logps/chosen": -253.38181818181818,
      "logps/rejected": -532.1643835616438,
      "loss": 0.2277,
      "rewards/chosen": 0.878125,
      "rewards/margins": 11.234289383561645,
      "rewards/rejected": -10.356164383561644,
      "step": 596
    },
    {
      "epoch": 0.2940886699507389,
      "grad_norm": 0.5158168039135648,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -57093155.31034483,
      "logits/rejected": -78493403.42857143,
      "logps/chosen": -326.62068965517244,
      "logps/rejected": -520.2285714285714,
      "loss": 0.2381,
      "rewards/chosen": 0.15396012931034483,
      "rewards/margins": 9.66824584359606,
      "rewards/rejected": -9.514285714285714,
      "step": 597
    },
    {
      "epoch": 0.29458128078817736,
      "grad_norm": 0.45362366271925425,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -72887682.84444444,
      "logits/rejected": -90556539.37349397,
      "logps/chosen": -306.84444444444443,
      "logps/rejected": -599.9036144578313,
      "loss": 0.1981,
      "rewards/chosen": -0.5072916666666667,
      "rewards/margins": 12.167407128514055,
      "rewards/rejected": -12.674698795180722,
      "step": 598
    },
    {
      "epoch": 0.29507389162561576,
      "grad_norm": 0.6395260374506546,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -81089877.33333333,
      "logits/rejected": -74817314.5945946,
      "logps/chosen": -343.7037037037037,
      "logps/rejected": -497.72972972972974,
      "loss": 0.2562,
      "rewards/chosen": 3.7638267234519676,
      "rewards/margins": 12.182745642370886,
      "rewards/rejected": -8.41891891891892,
      "step": 599
    },
    {
      "epoch": 0.2955665024630542,
      "grad_norm": 0.5275322387663373,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -76062089.84615384,
      "logits/rejected": -70861662.31578948,
      "logps/chosen": -344.61538461538464,
      "logps/rejected": -518.7368421052631,
      "loss": 0.2116,
      "rewards/chosen": 3.991619403545673,
      "rewards/margins": 13.465303614071988,
      "rewards/rejected": -9.473684210526315,
      "step": 600
    },
    {
      "epoch": 0.2960591133004926,
      "grad_norm": 0.4384817709889149,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -74361514.66666667,
      "logits/rejected": -73610035.2,
      "logps/chosen": -236.5,
      "logps/rejected": -480.8,
      "loss": 0.2134,
      "rewards/chosen": 1.9407466252644856,
      "rewards/margins": 12.565746625264486,
      "rewards/rejected": -10.625,
      "step": 601
    },
    {
      "epoch": 0.296551724137931,
      "grad_norm": 0.5259918491826775,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -67889199.62790698,
      "logits/rejected": -94840615.15294118,
      "logps/chosen": -311.4418604651163,
      "logps/rejected": -596.3294117647059,
      "loss": 0.1751,
      "rewards/chosen": 0.3953488372093023,
      "rewards/margins": 11.077701778385773,
      "rewards/rejected": -10.68235294117647,
      "step": 602
    },
    {
      "epoch": 0.2970443349753695,
      "grad_norm": 0.43272194930809754,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -65450650.79069767,
      "logits/rejected": -67207553.50588235,
      "logps/chosen": -322.6046511627907,
      "logps/rejected": -542.1176470588235,
      "loss": 0.1616,
      "rewards/chosen": 0.8815406976744186,
      "rewards/margins": 11.76389363885089,
      "rewards/rejected": -10.882352941176471,
      "step": 603
    },
    {
      "epoch": 0.2975369458128079,
      "grad_norm": 0.5110946102088519,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -72501540.57142857,
      "logits/rejected": -88313400.8888889,
      "logps/chosen": -312.2857142857143,
      "logps/rejected": -591.5555555555555,
      "loss": 0.1959,
      "rewards/chosen": 1.6729910714285714,
      "rewards/margins": 12.672991071428571,
      "rewards/rejected": -11.0,
      "step": 604
    },
    {
      "epoch": 0.29802955665024633,
      "grad_norm": 0.5044894760207637,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -46087411.809523806,
      "logits/rejected": -73741716.8372093,
      "logps/chosen": -218.66666666666666,
      "logps/rejected": -481.48837209302326,
      "loss": 0.1829,
      "rewards/chosen": 4.25889405750093,
      "rewards/margins": 9.029144954575784,
      "rewards/rejected": -4.770250897074854,
      "step": 605
    },
    {
      "epoch": 0.29852216748768473,
      "grad_norm": 0.4912000219836262,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -74774316.13793103,
      "logits/rejected": -97068178.28571428,
      "logps/chosen": -240.27586206896552,
      "logps/rejected": -567.3142857142857,
      "loss": 0.2733,
      "rewards/chosen": 0.12931034482758622,
      "rewards/margins": 9.472167487684729,
      "rewards/rejected": -9.342857142857143,
      "step": 606
    },
    {
      "epoch": 0.29901477832512313,
      "grad_norm": 0.40301313271317485,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -70371100.44444445,
      "logits/rejected": -68299139.45945945,
      "logps/chosen": -315.25925925925924,
      "logps/rejected": -434.5945945945946,
      "loss": 0.2323,
      "rewards/chosen": -0.19560185185185186,
      "rewards/margins": 10.6557494994995,
      "rewards/rejected": -10.85135135135135,
      "step": 607
    },
    {
      "epoch": 0.2995073891625616,
      "grad_norm": 0.43668497327484695,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -67536854.20408164,
      "logits/rejected": -75922211.64556962,
      "logps/chosen": -269.7142857142857,
      "logps/rejected": -550.0759493670886,
      "loss": 0.2298,
      "rewards/chosen": 0.5605867346938775,
      "rewards/margins": 9.839067747352106,
      "rewards/rejected": -9.278481012658228,
      "step": 608
    },
    {
      "epoch": 0.3,
      "grad_norm": 0.42488061363700474,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -63479177.84615385,
      "logits/rejected": -80685163.78947368,
      "logps/chosen": -302.15384615384613,
      "logps/rejected": -558.3157894736842,
      "loss": 0.2134,
      "rewards/chosen": 0.4670973557692308,
      "rewards/margins": 10.598676303137651,
      "rewards/rejected": -10.131578947368421,
      "step": 609
    },
    {
      "epoch": 0.30049261083743845,
      "grad_norm": 0.41178886031539585,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -64219454.57777778,
      "logits/rejected": -68827012.62650603,
      "logps/chosen": -249.24444444444444,
      "logps/rejected": -512.7710843373494,
      "loss": 0.1675,
      "rewards/chosen": 0.8513888888888889,
      "rewards/margins": 11.116449129852745,
      "rewards/rejected": -10.265060240963855,
      "step": 610
    },
    {
      "epoch": 0.30098522167487685,
      "grad_norm": 0.5790707734004943,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -63014424.38095238,
      "logits/rejected": -87787758.13953489,
      "logps/chosen": -185.42857142857142,
      "logps/rejected": -557.3953488372093,
      "loss": 0.171,
      "rewards/chosen": 3.8771780104864213,
      "rewards/margins": 14.423689638393398,
      "rewards/rejected": -10.546511627906977,
      "step": 611
    },
    {
      "epoch": 0.30147783251231525,
      "grad_norm": 0.41110775986469694,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -60945805.06122449,
      "logits/rejected": -62330542.98734177,
      "logps/chosen": -340.57142857142856,
      "logps/rejected": -490.9367088607595,
      "loss": 0.1979,
      "rewards/chosen": 0.9011479591836735,
      "rewards/margins": 10.559375807284939,
      "rewards/rejected": -9.658227848101266,
      "step": 612
    },
    {
      "epoch": 0.3019704433497537,
      "grad_norm": 0.5095215179018655,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -73120699.73333333,
      "logits/rejected": -71426529.88235295,
      "logps/chosen": -277.6,
      "logps/rejected": -480.47058823529414,
      "loss": 0.2473,
      "rewards/chosen": -0.052083333333333336,
      "rewards/margins": 9.315563725490195,
      "rewards/rejected": -9.367647058823529,
      "step": 613
    },
    {
      "epoch": 0.3024630541871921,
      "grad_norm": 0.4804084321909975,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -63963136.0,
      "logits/rejected": -72246886.4,
      "logps/chosen": -290.3333333333333,
      "logps/rejected": -486.4,
      "loss": 0.2095,
      "rewards/chosen": -0.15592447916666666,
      "rewards/margins": 8.944075520833334,
      "rewards/rejected": -9.1,
      "step": 614
    },
    {
      "epoch": 0.30295566502463056,
      "grad_norm": 0.46387906982826743,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -56741810.716981135,
      "logits/rejected": -71918332.58666667,
      "logps/chosen": -328.45283018867923,
      "logps/rejected": -491.52,
      "loss": 0.2223,
      "rewards/chosen": 0.5153301886792453,
      "rewards/margins": 9.035330188679245,
      "rewards/rejected": -8.52,
      "step": 615
    },
    {
      "epoch": 0.30344827586206896,
      "grad_norm": 0.6345982608852073,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -59909685.49253731,
      "logits/rejected": -81548271.21311475,
      "logps/chosen": -266.9850746268657,
      "logps/rejected": -467.40983606557376,
      "loss": 0.2994,
      "rewards/chosen": 0.8339552238805971,
      "rewards/margins": 4.36872124911113,
      "rewards/rejected": -3.5347660252305326,
      "step": 616
    },
    {
      "epoch": 0.30394088669950736,
      "grad_norm": 0.5204305430168332,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -77687830.75555556,
      "logits/rejected": -73880390.93975903,
      "logps/chosen": -257.9555555555556,
      "logps/rejected": -488.86746987951807,
      "loss": 0.2173,
      "rewards/chosen": 0.5447916666666667,
      "rewards/margins": 8.94238202811245,
      "rewards/rejected": -8.397590361445783,
      "step": 617
    },
    {
      "epoch": 0.3044334975369458,
      "grad_norm": 0.6454107360270729,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -68775830.97435898,
      "logits/rejected": -66826101.93258427,
      "logps/chosen": -273.64102564102564,
      "logps/rejected": -440.08988764044943,
      "loss": 0.1575,
      "rewards/chosen": 1.0064102564102564,
      "rewards/margins": 9.377196773264188,
      "rewards/rejected": -8.370786516853933,
      "step": 618
    },
    {
      "epoch": 0.3049261083743842,
      "grad_norm": 0.497191756261105,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -75974097.45454545,
      "logits/rejected": -65810627.047619045,
      "logps/chosen": -359.27272727272725,
      "logps/rejected": -457.14285714285717,
      "loss": 0.2076,
      "rewards/chosen": 5.277223413640803,
      "rewards/margins": 13.705794842212232,
      "rewards/rejected": -8.428571428571429,
      "step": 619
    },
    {
      "epoch": 0.3054187192118227,
      "grad_norm": 0.5437026090522965,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -67583690.86792453,
      "logits/rejected": -75217851.73333333,
      "logps/chosen": -278.33962264150944,
      "logps/rejected": -499.2,
      "loss": 0.1945,
      "rewards/chosen": 1.2004716981132075,
      "rewards/margins": 11.467138364779874,
      "rewards/rejected": -10.266666666666667,
      "step": 620
    },
    {
      "epoch": 0.3059113300492611,
      "grad_norm": 0.5144046802304202,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -67108864.0,
      "logits/rejected": -68143075.94520548,
      "logps/chosen": -257.8909090909091,
      "logps/rejected": -419.06849315068496,
      "loss": 0.2271,
      "rewards/chosen": 1.0857954545454545,
      "rewards/margins": 10.017302303860523,
      "rewards/rejected": -8.931506849315069,
      "step": 621
    },
    {
      "epoch": 0.30640394088669953,
      "grad_norm": 0.48043479506235387,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -53374574.431372546,
      "logits/rejected": -84539737.76623377,
      "logps/chosen": -278.5882352941176,
      "logps/rejected": -535.2727272727273,
      "loss": 0.2204,
      "rewards/chosen": 0.34681372549019607,
      "rewards/margins": 10.35980073847721,
      "rewards/rejected": -10.012987012987013,
      "step": 622
    },
    {
      "epoch": 0.30689655172413793,
      "grad_norm": 0.45347400697286083,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -62739797.333333336,
      "logits/rejected": -70569164.8,
      "logps/chosen": -293.6666666666667,
      "logps/rejected": -464.0,
      "loss": 0.1773,
      "rewards/chosen": 3.5323959986368814,
      "rewards/margins": 12.582395998636882,
      "rewards/rejected": -9.05,
      "step": 623
    },
    {
      "epoch": 0.30738916256157633,
      "grad_norm": 0.4656251541017435,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -83361792.0,
      "logits/rejected": -64592281.6,
      "logps/chosen": -242.66666666666666,
      "logps/rejected": -489.6,
      "loss": 0.1996,
      "rewards/chosen": 5.829828262329102,
      "rewards/margins": 14.5923282623291,
      "rewards/rejected": -8.7625,
      "step": 624
    },
    {
      "epoch": 0.3078817733990148,
      "grad_norm": 0.4213223525711944,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -60600461.24137931,
      "logits/rejected": -69086178.74285714,
      "logps/chosen": -306.2068965517241,
      "logps/rejected": -432.0,
      "loss": 0.2087,
      "rewards/chosen": 0.9633620689655172,
      "rewards/margins": 9.50621921182266,
      "rewards/rejected": -8.542857142857143,
      "step": 625
    },
    {
      "epoch": 0.3083743842364532,
      "grad_norm": 0.4306918750077963,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -68658932.86956522,
      "logits/rejected": -70689367.41463415,
      "logps/chosen": -313.5652173913044,
      "logps/rejected": -518.6341463414634,
      "loss": 0.1903,
      "rewards/chosen": 0.39877717391304346,
      "rewards/margins": 9.081704003181336,
      "rewards/rejected": -8.682926829268293,
      "step": 626
    },
    {
      "epoch": 0.30886699507389165,
      "grad_norm": 0.4754484222567208,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -60842374.0952381,
      "logits/rejected": -64865399.069767445,
      "logps/chosen": -327.3333333333333,
      "logps/rejected": -435.3488372093023,
      "loss": 0.181,
      "rewards/chosen": 3.6361214773995534,
      "rewards/margins": 12.729144733213507,
      "rewards/rejected": -9.093023255813954,
      "step": 627
    },
    {
      "epoch": 0.30935960591133005,
      "grad_norm": 0.4468152467202844,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -50181851.428571425,
      "logits/rejected": -65011712.0,
      "logps/chosen": -297.0,
      "logps/rejected": -439.55555555555554,
      "loss": 0.2278,
      "rewards/chosen": 1.1446707589285714,
      "rewards/margins": 10.464115203373016,
      "rewards/rejected": -9.319444444444445,
      "step": 628
    },
    {
      "epoch": 0.30985221674876845,
      "grad_norm": 0.45959845477047034,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -85887906.9090909,
      "logits/rejected": -77794352.76190476,
      "logps/chosen": -392.3636363636364,
      "logps/rejected": -453.7142857142857,
      "loss": 0.1945,
      "rewards/chosen": 3.2925675132057886,
      "rewards/margins": 12.11399608463436,
      "rewards/rejected": -8.821428571428571,
      "step": 629
    },
    {
      "epoch": 0.3103448275862069,
      "grad_norm": 0.5105797185429546,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -68157440.0,
      "logits/rejected": -76325295.15789473,
      "logps/chosen": -399.38461538461536,
      "logps/rejected": -480.2105263157895,
      "loss": 0.2185,
      "rewards/chosen": 2.5940352219801683,
      "rewards/margins": 11.172982590401222,
      "rewards/rejected": -8.578947368421053,
      "step": 630
    },
    {
      "epoch": 0.3108374384236453,
      "grad_norm": 0.5376846676935044,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -82458231.82978724,
      "logits/rejected": -68558746.86419754,
      "logps/chosen": -352.3404255319149,
      "logps/rejected": -430.22222222222223,
      "loss": 0.2594,
      "rewards/chosen": -0.7659574468085106,
      "rewards/margins": 7.6167586025742064,
      "rewards/rejected": -8.382716049382717,
      "step": 631
    },
    {
      "epoch": 0.31133004926108376,
      "grad_norm": 0.40636209767369363,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -55346576.69565217,
      "logits/rejected": -65676662.63414634,
      "logps/chosen": -345.39130434782606,
      "logps/rejected": -474.1463414634146,
      "loss": 0.1649,
      "rewards/chosen": 0.5484035326086957,
      "rewards/margins": 9.463037678950158,
      "rewards/rejected": -8.914634146341463,
      "step": 632
    },
    {
      "epoch": 0.31182266009852216,
      "grad_norm": 0.4845234363071494,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -71913248.58181818,
      "logits/rejected": -70556237.15068494,
      "logps/chosen": -288.8727272727273,
      "logps/rejected": -522.082191780822,
      "loss": 0.2318,
      "rewards/chosen": -0.011931818181818182,
      "rewards/margins": 9.303136674968867,
      "rewards/rejected": -9.315068493150685,
      "step": 633
    },
    {
      "epoch": 0.31231527093596056,
      "grad_norm": 0.4665174226558288,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -61402659.72093023,
      "logits/rejected": -78458157.1764706,
      "logps/chosen": -282.7906976744186,
      "logps/rejected": -534.5882352941177,
      "loss": 0.1626,
      "rewards/chosen": 3.202777241551599,
      "rewards/margins": 12.896894888610422,
      "rewards/rejected": -9.694117647058823,
      "step": 634
    },
    {
      "epoch": 0.312807881773399,
      "grad_norm": 0.42066573328779233,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -44858081.28,
      "logits/rejected": -66678678.974358976,
      "logps/chosen": -244.8,
      "logps/rejected": -444.71794871794873,
      "loss": 0.1874,
      "rewards/chosen": 1.1825,
      "rewards/margins": 8.797884615384614,
      "rewards/rejected": -7.615384615384615,
      "step": 635
    },
    {
      "epoch": 0.3133004926108374,
      "grad_norm": 0.48699482091905943,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -67439993.26315789,
      "logits/rejected": -65337935.64444444,
      "logps/chosen": -339.7894736842105,
      "logps/rejected": -479.2888888888889,
      "loss": 0.1833,
      "rewards/chosen": 0.21217105263157895,
      "rewards/margins": 10.02328216374269,
      "rewards/rejected": -9.811111111111112,
      "step": 636
    },
    {
      "epoch": 0.3137931034482759,
      "grad_norm": 0.5207228061257093,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -75658791.38461539,
      "logits/rejected": -69537145.26315789,
      "logps/chosen": -274.61538461538464,
      "logps/rejected": -530.5263157894736,
      "loss": 0.2648,
      "rewards/chosen": 3.0156390850360575,
      "rewards/margins": 7.3317706768329325,
      "rewards/rejected": -4.316131591796875,
      "step": 637
    },
    {
      "epoch": 0.3142857142857143,
      "grad_norm": 0.553422734992621,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -74734871.27272727,
      "logits/rejected": -65511033.9047619,
      "logps/chosen": -422.90909090909093,
      "logps/rejected": -489.14285714285717,
      "loss": 0.1896,
      "rewards/chosen": 5.712737343528054,
      "rewards/margins": 15.319880200670912,
      "rewards/rejected": -9.607142857142858,
      "step": 638
    },
    {
      "epoch": 0.31477832512315274,
      "grad_norm": 0.4102922545094117,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -72244746.44897959,
      "logits/rejected": -69763486.78481013,
      "logps/chosen": -245.22448979591837,
      "logps/rejected": -494.17721518987344,
      "loss": 0.2234,
      "rewards/chosen": 0.9706807039221939,
      "rewards/margins": 10.185870577339914,
      "rewards/rejected": -9.215189873417721,
      "step": 639
    },
    {
      "epoch": 0.31527093596059114,
      "grad_norm": 0.43753477238107447,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -63182281.5319149,
      "logits/rejected": -70992478.81481482,
      "logps/chosen": -353.36170212765956,
      "logps/rejected": -499.358024691358,
      "loss": 0.2037,
      "rewards/chosen": 3.7220692736037235,
      "rewards/margins": 14.38873594027039,
      "rewards/rejected": -10.666666666666666,
      "step": 640
    },
    {
      "epoch": 0.31576354679802954,
      "grad_norm": 0.4535969697829465,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -62315373.71428572,
      "logits/rejected": -65635548.35443038,
      "logps/chosen": -324.8979591836735,
      "logps/rejected": -493.36708860759495,
      "loss": 0.165,
      "rewards/chosen": 1.590561224489796,
      "rewards/margins": 11.843725781451822,
      "rewards/rejected": -10.253164556962025,
      "step": 641
    },
    {
      "epoch": 0.316256157635468,
      "grad_norm": 0.7622629963478382,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -71667890.08695652,
      "logits/rejected": -68745665.56097561,
      "logps/chosen": -365.5652173913044,
      "logps/rejected": -515.5121951219512,
      "loss": 0.1922,
      "rewards/chosen": 1.357336956521739,
      "rewards/margins": 11.515873541887592,
      "rewards/rejected": -10.158536585365853,
      "step": 642
    },
    {
      "epoch": 0.3167487684729064,
      "grad_norm": 0.4348647778100496,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -79008982.3255814,
      "logits/rejected": -70562996.70588236,
      "logps/chosen": -300.27906976744185,
      "logps/rejected": -495.43529411764706,
      "loss": 0.1993,
      "rewards/chosen": 5.518894993981649,
      "rewards/margins": 15.060071464569884,
      "rewards/rejected": -9.541176470588235,
      "step": 643
    },
    {
      "epoch": 0.31724137931034485,
      "grad_norm": 0.47736038990077795,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -92711594.66666667,
      "logits/rejected": -67685580.8,
      "logps/chosen": -452.6666666666667,
      "logps/rejected": -424.4,
      "loss": 0.1995,
      "rewards/chosen": 0.53515625,
      "rewards/margins": 10.64140625,
      "rewards/rejected": -10.10625,
      "step": 644
    },
    {
      "epoch": 0.31773399014778325,
      "grad_norm": 0.5322538243749402,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -82547472.34042554,
      "logits/rejected": -65037602.7654321,
      "logps/chosen": -337.36170212765956,
      "logps/rejected": -482.3703703703704,
      "loss": 0.1917,
      "rewards/chosen": 3.4687577917220747,
      "rewards/margins": 12.19715285345047,
      "rewards/rejected": -8.728395061728396,
      "step": 645
    },
    {
      "epoch": 0.31822660098522165,
      "grad_norm": 0.4821799066071639,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -68923707.07692307,
      "logits/rejected": -63080124.631578945,
      "logps/chosen": -305.53846153846155,
      "logps/rejected": -501.05263157894734,
      "loss": 0.2377,
      "rewards/chosen": -0.19831730769230768,
      "rewards/margins": 7.760963409053169,
      "rewards/rejected": -7.959280716745477,
      "step": 646
    },
    {
      "epoch": 0.3187192118226601,
      "grad_norm": 0.5135660947436075,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -61220706.461538464,
      "logits/rejected": -68102251.78947368,
      "logps/chosen": -252.15384615384616,
      "logps/rejected": -469.89473684210526,
      "loss": 0.187,
      "rewards/chosen": 2.610269986666166,
      "rewards/margins": 11.682638407718798,
      "rewards/rejected": -9.072368421052632,
      "step": 647
    },
    {
      "epoch": 0.3192118226600985,
      "grad_norm": 0.5189140041855136,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -65431142.4,
      "logits/rejected": -73476120.6746988,
      "logps/chosen": -394.31111111111113,
      "logps/rejected": -499.66265060240966,
      "loss": 0.2264,
      "rewards/chosen": 3.241653781467014,
      "rewards/margins": 13.386232094720025,
      "rewards/rejected": -10.144578313253012,
      "step": 648
    },
    {
      "epoch": 0.31970443349753697,
      "grad_norm": 0.4414939179231532,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -70548193.28,
      "logits/rejected": -68399419.07692307,
      "logps/chosen": -303.84,
      "logps/rejected": -499.6923076923077,
      "loss": 0.199,
      "rewards/chosen": 0.49375,
      "rewards/margins": 10.26298076923077,
      "rewards/rejected": -9.76923076923077,
      "step": 649
    },
    {
      "epoch": 0.32019704433497537,
      "grad_norm": 0.5514980046189086,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -66662661.44680851,
      "logits/rejected": -59755886.61728395,
      "logps/chosen": -333.27659574468083,
      "logps/rejected": -501.7283950617284,
      "loss": 0.1479,
      "rewards/chosen": 5.160255269801363,
      "rewards/margins": 15.666428109307535,
      "rewards/rejected": -10.506172839506172,
      "step": 650
    },
    {
      "epoch": 0.32068965517241377,
      "grad_norm": 0.5008651193377749,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -74629684.96551724,
      "logits/rejected": -93772653.71428572,
      "logps/chosen": -325.51724137931035,
      "logps/rejected": -554.5142857142857,
      "loss": 0.2196,
      "rewards/chosen": 2.0595796519312364,
      "rewards/margins": 13.931008223359807,
      "rewards/rejected": -11.871428571428572,
      "step": 651
    },
    {
      "epoch": 0.3211822660098522,
      "grad_norm": 0.4005577996859094,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -66699663.6097561,
      "logits/rejected": -78293674.66666667,
      "logps/chosen": -279.4146341463415,
      "logps/rejected": -493.60919540229884,
      "loss": 0.1664,
      "rewards/chosen": 1.8723492506073742,
      "rewards/margins": 13.883843503480938,
      "rewards/rejected": -12.011494252873563,
      "step": 652
    },
    {
      "epoch": 0.3216748768472906,
      "grad_norm": 0.4471878033166764,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -60862028.25531915,
      "logits/rejected": -64416224.39506173,
      "logps/chosen": -335.3191489361702,
      "logps/rejected": -473.679012345679,
      "loss": 0.1542,
      "rewards/chosen": 1.940159574468085,
      "rewards/margins": 11.224110191752036,
      "rewards/rejected": -9.283950617283951,
      "step": 653
    },
    {
      "epoch": 0.3221674876847291,
      "grad_norm": 0.4967483890943799,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -60817408.0,
      "logits/rejected": -60602315.48717949,
      "logps/chosen": -214.56,
      "logps/rejected": -466.87179487179486,
      "loss": 0.2113,
      "rewards/chosen": 5.359307861328125,
      "rewards/margins": 15.551615553635816,
      "rewards/rejected": -10.192307692307692,
      "step": 654
    },
    {
      "epoch": 0.3226600985221675,
      "grad_norm": 0.4164275208733055,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -61152952.32,
      "logits/rejected": -80121961.02564102,
      "logps/chosen": -301.44,
      "logps/rejected": -521.025641025641,
      "loss": 0.1762,
      "rewards/chosen": 1.563125,
      "rewards/margins": 11.960560897435897,
      "rewards/rejected": -10.397435897435898,
      "step": 655
    },
    {
      "epoch": 0.32315270935960594,
      "grad_norm": 0.43748676198882225,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -52768878.7027027,
      "logits/rejected": -80383144.79120879,
      "logps/chosen": -311.7837837837838,
      "logps/rejected": -526.7692307692307,
      "loss": 0.1573,
      "rewards/chosen": 6.295590787320523,
      "rewards/margins": 17.3725138642436,
      "rewards/rejected": -11.076923076923077,
      "step": 656
    },
    {
      "epoch": 0.32364532019704434,
      "grad_norm": 0.5597599885364618,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -71003574.85714285,
      "logits/rejected": -55924053.333333336,
      "logps/chosen": -272.2857142857143,
      "logps/rejected": -420.8888888888889,
      "loss": 0.2775,
      "rewards/chosen": 0.39536830357142855,
      "rewards/margins": 10.339812748015873,
      "rewards/rejected": -9.944444444444445,
      "step": 657
    },
    {
      "epoch": 0.32413793103448274,
      "grad_norm": 0.38857998168621677,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -65788434.96296296,
      "logits/rejected": -62404441.94594595,
      "logps/chosen": -308.74074074074076,
      "logps/rejected": -454.9189189189189,
      "loss": 0.1606,
      "rewards/chosen": 1.728587962962963,
      "rewards/margins": 14.093452827827829,
      "rewards/rejected": -12.364864864864865,
      "step": 658
    },
    {
      "epoch": 0.3246305418719212,
      "grad_norm": 0.4768398995035029,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -53259747.018867925,
      "logits/rejected": -71806484.48,
      "logps/chosen": -296.45283018867923,
      "logps/rejected": -443.73333333333335,
      "loss": 0.2348,
      "rewards/chosen": 2.752291409474499,
      "rewards/margins": 12.825624742807832,
      "rewards/rejected": -10.073333333333334,
      "step": 659
    },
    {
      "epoch": 0.3251231527093596,
      "grad_norm": 0.6367315576096225,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -70746780.73469388,
      "logits/rejected": -79744868.4556962,
      "logps/chosen": -328.81632653061223,
      "logps/rejected": -524.9620253164557,
      "loss": 0.1969,
      "rewards/chosen": 0.5826690051020408,
      "rewards/margins": 10.380137359532421,
      "rewards/rejected": -9.79746835443038,
      "step": 660
    },
    {
      "epoch": 0.32561576354679805,
      "grad_norm": 0.38689631749906633,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -59688172.307692304,
      "logits/rejected": -69757898.10526316,
      "logps/chosen": -248.0,
      "logps/rejected": -417.2631578947368,
      "loss": 0.2079,
      "rewards/chosen": 0.5193810096153846,
      "rewards/margins": 7.935443214076733,
      "rewards/rejected": -7.4160622044613485,
      "step": 661
    },
    {
      "epoch": 0.32610837438423645,
      "grad_norm": 0.392250126797889,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -69905066.66666667,
      "logits/rejected": -69233251.74025974,
      "logps/chosen": -395.6078431372549,
      "logps/rejected": -510.3376623376623,
      "loss": 0.1898,
      "rewards/chosen": 3.5030768899356617,
      "rewards/margins": 14.606972993831764,
      "rewards/rejected": -11.103896103896103,
      "step": 662
    },
    {
      "epoch": 0.32660098522167486,
      "grad_norm": 0.3867937520775157,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -62265441.52380952,
      "logits/rejected": -82861889.48837209,
      "logps/chosen": -305.14285714285717,
      "logps/rejected": -496.3720930232558,
      "loss": 0.1735,
      "rewards/chosen": 3.0580328078497026,
      "rewards/margins": 13.360358389245052,
      "rewards/rejected": -10.30232558139535,
      "step": 663
    },
    {
      "epoch": 0.3270935960591133,
      "grad_norm": 0.5846763806889463,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -33816576.0,
      "logits/rejected": -69415731.2,
      "logps/chosen": -247.33333333333334,
      "logps/rejected": -468.4,
      "loss": 0.2292,
      "rewards/chosen": 5.2284698486328125,
      "rewards/margins": 16.56596984863281,
      "rewards/rejected": -11.3375,
      "step": 664
    },
    {
      "epoch": 0.3275862068965517,
      "grad_norm": 0.43082780850990665,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -54775612.952380955,
      "logits/rejected": -63889979.53488372,
      "logps/chosen": -310.85714285714283,
      "logps/rejected": -468.09302325581393,
      "loss": 0.179,
      "rewards/chosen": 3.1480204264322915,
      "rewards/margins": 13.7294157752695,
      "rewards/rejected": -10.581395348837209,
      "step": 665
    },
    {
      "epoch": 0.32807881773399017,
      "grad_norm": 0.6334299355438661,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -75726252.21818182,
      "logits/rejected": -85035204.38356164,
      "logps/chosen": -215.85454545454544,
      "logps/rejected": -452.82191780821915,
      "loss": 0.2363,
      "rewards/chosen": 3.1050681374289772,
      "rewards/margins": 12.105068137428978,
      "rewards/rejected": -9.0,
      "step": 666
    },
    {
      "epoch": 0.32857142857142857,
      "grad_norm": 0.4192761007156039,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -53477376.0,
      "logits/rejected": -69978650.94736843,
      "logps/chosen": -250.46153846153845,
      "logps/rejected": -483.36842105263156,
      "loss": 0.1916,
      "rewards/chosen": 3.3401773892916164,
      "rewards/margins": 13.73491423139688,
      "rewards/rejected": -10.394736842105264,
      "step": 667
    },
    {
      "epoch": 0.32906403940886697,
      "grad_norm": 0.3820791733598647,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -67021482.666666664,
      "logits/rejected": -59139686.4,
      "logps/chosen": -305.5,
      "logps/rejected": -455.2,
      "loss": 0.2359,
      "rewards/chosen": -0.5651041666666666,
      "rewards/margins": 9.047395833333335,
      "rewards/rejected": -9.6125,
      "step": 668
    },
    {
      "epoch": 0.3295566502463054,
      "grad_norm": 0.47568587520853334,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -60769745.45454545,
      "logits/rejected": -75697200.76190476,
      "logps/chosen": -350.1818181818182,
      "logps/rejected": -521.1428571428571,
      "loss": 0.1907,
      "rewards/chosen": 3.331362637606534,
      "rewards/margins": 12.7480293042732,
      "rewards/rejected": -9.416666666666666,
      "step": 669
    },
    {
      "epoch": 0.33004926108374383,
      "grad_norm": 0.43950065685599143,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -61758986.448979594,
      "logits/rejected": -67321233.82278481,
      "logps/chosen": -340.57142857142856,
      "logps/rejected": -467.44303797468353,
      "loss": 0.2349,
      "rewards/chosen": -0.8858418367346939,
      "rewards/margins": 8.823018922758978,
      "rewards/rejected": -9.708860759493671,
      "step": 670
    },
    {
      "epoch": 0.3305418719211823,
      "grad_norm": 0.49562576146432,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -49987852.59016393,
      "logits/rejected": -60660904.11940298,
      "logps/chosen": -230.81967213114754,
      "logps/rejected": -448.95522388059703,
      "loss": 0.2567,
      "rewards/chosen": 0.14600409836065573,
      "rewards/margins": 10.377347381942746,
      "rewards/rejected": -10.23134328358209,
      "step": 671
    },
    {
      "epoch": 0.3310344827586207,
      "grad_norm": 0.5120522408908604,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -68879792.35555555,
      "logits/rejected": -63318830.26506024,
      "logps/chosen": -395.73333333333335,
      "logps/rejected": -469.2048192771084,
      "loss": 0.1641,
      "rewards/chosen": 4.5479454888237845,
      "rewards/margins": 13.186499705691254,
      "rewards/rejected": -8.63855421686747,
      "step": 672
    },
    {
      "epoch": 0.3315270935960591,
      "grad_norm": 0.4706010205613421,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -67566424.43636364,
      "logits/rejected": -88195296.43835616,
      "logps/chosen": -237.0909090909091,
      "logps/rejected": -537.4246575342465,
      "loss": 0.2269,
      "rewards/chosen": 1.190909090909091,
      "rewards/margins": 11.410087173100873,
      "rewards/rejected": -10.219178082191782,
      "step": 673
    },
    {
      "epoch": 0.33201970443349754,
      "grad_norm": 0.4914023442690481,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -55610685.79310345,
      "logits/rejected": -71482923.88571429,
      "logps/chosen": -260.9655172413793,
      "logps/rejected": -496.9142857142857,
      "loss": 0.2128,
      "rewards/chosen": 3.0624721132475754,
      "rewards/margins": 12.24818639896186,
      "rewards/rejected": -9.185714285714285,
      "step": 674
    },
    {
      "epoch": 0.33251231527093594,
      "grad_norm": 0.3818314902786052,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -60622324.093023255,
      "logits/rejected": -72536786.8235294,
      "logps/chosen": -262.8837209302326,
      "logps/rejected": -513.1294117647059,
      "loss": 0.1793,
      "rewards/chosen": 2.16786389018214,
      "rewards/margins": 12.76786389018214,
      "rewards/rejected": -10.6,
      "step": 675
    },
    {
      "epoch": 0.3330049261083744,
      "grad_norm": 0.5066583147772753,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -50964750.49056604,
      "logits/rejected": -62411243.52,
      "logps/chosen": -348.07547169811323,
      "logps/rejected": -488.1066666666667,
      "loss": 0.2292,
      "rewards/chosen": 0.6400353773584906,
      "rewards/margins": 11.426702044025157,
      "rewards/rejected": -10.786666666666667,
      "step": 676
    },
    {
      "epoch": 0.3334975369458128,
      "grad_norm": 0.41894416730216444,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -63096921.04347826,
      "logits/rejected": -63016860.09756097,
      "logps/chosen": -273.9130434782609,
      "logps/rejected": -499.1219512195122,
      "loss": 0.207,
      "rewards/chosen": 2.7445764956266983,
      "rewards/margins": 12.16530820294377,
      "rewards/rejected": -9.420731707317072,
      "step": 677
    },
    {
      "epoch": 0.33399014778325126,
      "grad_norm": 0.4804819799862797,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -60293120.0,
      "logits/rejected": -77128590.22222222,
      "logps/chosen": -253.14285714285714,
      "logps/rejected": -466.22222222222223,
      "loss": 0.2391,
      "rewards/chosen": 2.9988741193498885,
      "rewards/margins": 12.332207452683223,
      "rewards/rejected": -9.333333333333334,
      "step": 678
    },
    {
      "epoch": 0.33448275862068966,
      "grad_norm": 0.5098952980480504,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -67182448.28070176,
      "logits/rejected": -81464017.12676056,
      "logps/chosen": -286.3157894736842,
      "logps/rejected": -435.38028169014086,
      "loss": 0.24,
      "rewards/chosen": -0.1288377192982456,
      "rewards/margins": 8.040176365208797,
      "rewards/rejected": -8.169014084507042,
      "step": 679
    },
    {
      "epoch": 0.33497536945812806,
      "grad_norm": 0.5400118807483358,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -60488442.98039216,
      "logits/rejected": -78765760.83116883,
      "logps/chosen": -273.88235294117646,
      "logps/rejected": -536.1038961038961,
      "loss": 0.2247,
      "rewards/chosen": -0.5355392156862745,
      "rewards/margins": 10.425499745352687,
      "rewards/rejected": -10.96103896103896,
      "step": 680
    },
    {
      "epoch": 0.3354679802955665,
      "grad_norm": 0.5601822748068336,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -63031068.44444445,
      "logits/rejected": -67278903.35135135,
      "logps/chosen": -349.6296296296296,
      "logps/rejected": -426.81081081081084,
      "loss": 0.2092,
      "rewards/chosen": 0.6134259259259259,
      "rewards/margins": 10.680993493493494,
      "rewards/rejected": -10.067567567567568,
      "step": 681
    },
    {
      "epoch": 0.3359605911330049,
      "grad_norm": 0.48108686767972053,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -55734480.27118644,
      "logits/rejected": -67108864.0,
      "logps/chosen": -288.0,
      "logps/rejected": -503.6521739130435,
      "loss": 0.2269,
      "rewards/chosen": 2.6859042927370234,
      "rewards/margins": 12.70039704636021,
      "rewards/rejected": -10.014492753623188,
      "step": 682
    },
    {
      "epoch": 0.33645320197044337,
      "grad_norm": 0.44843272601759127,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -53576298.26415094,
      "logits/rejected": -63865268.906666666,
      "logps/chosen": -238.49056603773585,
      "logps/rejected": -513.28,
      "loss": 0.2069,
      "rewards/chosen": 2.613787093252506,
      "rewards/margins": 13.880453759919174,
      "rewards/rejected": -11.266666666666667,
      "step": 683
    },
    {
      "epoch": 0.3369458128078818,
      "grad_norm": 0.5028309744524054,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -63304727.81395349,
      "logits/rejected": -69526756.89411765,
      "logps/chosen": -275.3488372093023,
      "logps/rejected": -526.6823529411764,
      "loss": 0.1956,
      "rewards/chosen": -0.13517441860465115,
      "rewards/margins": 9.735413816689467,
      "rewards/rejected": -9.870588235294118,
      "step": 684
    },
    {
      "epoch": 0.3374384236453202,
      "grad_norm": 0.45495292666997356,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -51581015.14893617,
      "logits/rejected": -59134508.24691358,
      "logps/chosen": -305.36170212765956,
      "logps/rejected": -466.962962962963,
      "loss": 0.2073,
      "rewards/chosen": 0.7124335106382979,
      "rewards/margins": 11.885273016811139,
      "rewards/rejected": -11.17283950617284,
      "step": 685
    },
    {
      "epoch": 0.33793103448275863,
      "grad_norm": 0.4616956342885635,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -58478276.92307692,
      "logits/rejected": -55133022.315789476,
      "logps/chosen": -264.9230769230769,
      "logps/rejected": -409.6842105263158,
      "loss": 0.2227,
      "rewards/chosen": -0.15564903846153846,
      "rewards/margins": 8.607508856275304,
      "rewards/rejected": -8.763157894736842,
      "step": 686
    },
    {
      "epoch": 0.33842364532019703,
      "grad_norm": 0.42243613223720716,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -57124596.86956522,
      "logits/rejected": -73349169.95121951,
      "logps/chosen": -229.2173913043478,
      "logps/rejected": -525.2682926829268,
      "loss": 0.1843,
      "rewards/chosen": 1.080163043478261,
      "rewards/margins": 12.397236214209968,
      "rewards/rejected": -11.317073170731707,
      "step": 687
    },
    {
      "epoch": 0.3389162561576355,
      "grad_norm": 0.40529022333410053,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -70861662.31578948,
      "logits/rejected": -74658611.2,
      "logps/chosen": -355.7894736842105,
      "logps/rejected": -525.5111111111111,
      "loss": 0.1715,
      "rewards/chosen": 0.78125,
      "rewards/margins": 10.503472222222221,
      "rewards/rejected": -9.722222222222221,
      "step": 688
    },
    {
      "epoch": 0.3394088669950739,
      "grad_norm": 0.4493713545510213,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -65340677.01960784,
      "logits/rejected": -66890978.077922076,
      "logps/chosen": -304.94117647058823,
      "logps/rejected": -507.012987012987,
      "loss": 0.2176,
      "rewards/chosen": -0.46109068627450983,
      "rewards/margins": 10.370078144894322,
      "rewards/rejected": -10.831168831168831,
      "step": 689
    },
    {
      "epoch": 0.3399014778325123,
      "grad_norm": 0.42457699174346963,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -61318900.86956522,
      "logits/rejected": -63732960.780487806,
      "logps/chosen": -278.60869565217394,
      "logps/rejected": -501.8536585365854,
      "loss": 0.2091,
      "rewards/chosen": -0.48006538722826086,
      "rewards/margins": 11.032129734722957,
      "rewards/rejected": -11.512195121951219,
      "step": 690
    },
    {
      "epoch": 0.34039408866995075,
      "grad_norm": 0.4917424996243405,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -68478432.65306123,
      "logits/rejected": -65728460.151898734,
      "logps/chosen": -323.59183673469386,
      "logps/rejected": -486.0759493670886,
      "loss": 0.2129,
      "rewards/chosen": 0.21428571428571427,
      "rewards/margins": 12.416817359855333,
      "rewards/rejected": -12.20253164556962,
      "step": 691
    },
    {
      "epoch": 0.34088669950738915,
      "grad_norm": 0.48834753192631497,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -54525952.0,
      "logits/rejected": -55613364.14814815,
      "logps/chosen": -288.8510638297872,
      "logps/rejected": -459.4567901234568,
      "loss": 0.2587,
      "rewards/chosen": 7.542570073553857,
      "rewards/margins": 18.04874291306003,
      "rewards/rejected": -10.506172839506172,
      "step": 692
    },
    {
      "epoch": 0.3413793103448276,
      "grad_norm": 0.57332490268912,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -76021760.0,
      "logits/rejected": -83781222.4,
      "logps/chosen": -325.6666666666667,
      "logps/rejected": -515.2,
      "loss": 0.1891,
      "rewards/chosen": 4.174468358357747,
      "rewards/margins": 16.436968358357746,
      "rewards/rejected": -12.2625,
      "step": 693
    },
    {
      "epoch": 0.341871921182266,
      "grad_norm": 0.578000177897179,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -53892849.50943396,
      "logits/rejected": -80027320.32,
      "logps/chosen": -264.1509433962264,
      "logps/rejected": -532.9066666666666,
      "loss": 0.2111,
      "rewards/chosen": 0.5707547169811321,
      "rewards/margins": 11.530754716981132,
      "rewards/rejected": -10.96,
      "step": 694
    },
    {
      "epoch": 0.34236453201970446,
      "grad_norm": 0.5381359632152715,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -55538370.20689655,
      "logits/rejected": -65131549.25714286,
      "logps/chosen": -307.58620689655174,
      "logps/rejected": -515.2,
      "loss": 0.2572,
      "rewards/chosen": 0.28987068965517243,
      "rewards/margins": 12.075584975369459,
      "rewards/rejected": -11.785714285714286,
      "step": 695
    },
    {
      "epoch": 0.34285714285714286,
      "grad_norm": 0.49302070814313603,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -50250988.307692304,
      "logits/rejected": -70530533.05263157,
      "logps/chosen": -324.61538461538464,
      "logps/rejected": -559.578947368421,
      "loss": 0.2667,
      "rewards/chosen": 1.1721710791954627,
      "rewards/margins": 9.679098569429838,
      "rewards/rejected": -8.506927490234375,
      "step": 696
    },
    {
      "epoch": 0.34334975369458126,
      "grad_norm": 0.5982324487091989,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -64576454.03773585,
      "logits/rejected": -64200813.22666667,
      "logps/chosen": -348.6792452830189,
      "logps/rejected": -473.17333333333335,
      "loss": 0.2146,
      "rewards/chosen": 0.36556603773584906,
      "rewards/margins": 11.018899371069184,
      "rewards/rejected": -10.653333333333334,
      "step": 697
    },
    {
      "epoch": 0.3438423645320197,
      "grad_norm": 0.44189961981847803,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -57113926.80851064,
      "logits/rejected": -67005300.938271604,
      "logps/chosen": -357.78723404255317,
      "logps/rejected": -512.7901234567901,
      "loss": 0.1977,
      "rewards/chosen": 8.378917937583111,
      "rewards/margins": 20.057930283262124,
      "rewards/rejected": -11.679012345679013,
      "step": 698
    },
    {
      "epoch": 0.3443349753694581,
      "grad_norm": 0.5094006642147564,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -56693009.06666667,
      "logits/rejected": -68404163.76470588,
      "logps/chosen": -301.8666666666667,
      "logps/rejected": -521.8823529411765,
      "loss": 0.2276,
      "rewards/chosen": 0.6864583333333333,
      "rewards/margins": 12.362928921568626,
      "rewards/rejected": -11.676470588235293,
      "step": 699
    },
    {
      "epoch": 0.3448275862068966,
      "grad_norm": 0.4759664145142234,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -53938749.44,
      "logits/rejected": -64420207.58974359,
      "logps/chosen": -239.04,
      "logps/rejected": -466.05128205128204,
      "loss": 0.1821,
      "rewards/chosen": 1.22875,
      "rewards/margins": 12.446698717948717,
      "rewards/rejected": -11.217948717948717,
      "step": 700
    },
    {
      "epoch": 0.345320197044335,
      "grad_norm": 0.5785413273001965,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -50089668.92307692,
      "logits/rejected": -67936687.15789473,
      "logps/chosen": -294.15384615384613,
      "logps/rejected": -553.2631578947369,
      "loss": 0.217,
      "rewards/chosen": 0.3786057692307692,
      "rewards/margins": 11.707553137651823,
      "rewards/rejected": -11.328947368421053,
      "step": 701
    },
    {
      "epoch": 0.3458128078817734,
      "grad_norm": 0.42077993098453353,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -47090594.90909091,
      "logits/rejected": -71303168.0,
      "logps/chosen": -237.0909090909091,
      "logps/rejected": -522.6666666666666,
      "loss": 0.1962,
      "rewards/chosen": -0.05397727272727273,
      "rewards/margins": 11.39840367965368,
      "rewards/rejected": -11.452380952380953,
      "step": 702
    },
    {
      "epoch": 0.34630541871921183,
      "grad_norm": 0.4580024603081531,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -66965876.36363637,
      "logits/rejected": -72201947.42857143,
      "logps/chosen": -284.0,
      "logps/rejected": -550.8571428571429,
      "loss": 0.1829,
      "rewards/chosen": 4.115085948597301,
      "rewards/margins": 14.48413356764492,
      "rewards/rejected": -10.369047619047619,
      "step": 703
    },
    {
      "epoch": 0.34679802955665023,
      "grad_norm": 0.5124735839166231,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -54571542.26086956,
      "logits/rejected": -76111272.58536585,
      "logps/chosen": -279.6521739130435,
      "logps/rejected": -446.0487804878049,
      "loss": 0.2,
      "rewards/chosen": 0.14334239130434784,
      "rewards/margins": 9.753098488865323,
      "rewards/rejected": -9.609756097560975,
      "step": 704
    },
    {
      "epoch": 0.3472906403940887,
      "grad_norm": 0.4444215486326398,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -67975078.95652173,
      "logits/rejected": -78054974.43902439,
      "logps/chosen": -312.3478260869565,
      "logps/rejected": -512.390243902439,
      "loss": 0.1845,
      "rewards/chosen": 0.28719429347826086,
      "rewards/margins": 11.750608927624603,
      "rewards/rejected": -11.463414634146341,
      "step": 705
    },
    {
      "epoch": 0.3477832512315271,
      "grad_norm": 0.4488876513067928,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -73667230.25454545,
      "logits/rejected": -66074652.05479452,
      "logps/chosen": -322.90909090909093,
      "logps/rejected": -502.5753424657534,
      "loss": 0.2291,
      "rewards/chosen": 0.79140625,
      "rewards/margins": 11.462639126712329,
      "rewards/rejected": -10.67123287671233,
      "step": 706
    },
    {
      "epoch": 0.3482758620689655,
      "grad_norm": 0.47840713543506164,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -69320406.10909091,
      "logits/rejected": -63833859.50684932,
      "logps/chosen": -350.9818181818182,
      "logps/rejected": -475.6164383561644,
      "loss": 0.2375,
      "rewards/chosen": 4.8207630504261365,
      "rewards/margins": 13.82761236549463,
      "rewards/rejected": -9.006849315068493,
      "step": 707
    },
    {
      "epoch": 0.34876847290640395,
      "grad_norm": 0.5791484455155855,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -60237344.680851065,
      "logits/rejected": -71251386.4691358,
      "logps/chosen": -340.7659574468085,
      "logps/rejected": -500.9382716049383,
      "loss": 0.2006,
      "rewards/chosen": 1.9696460480385638,
      "rewards/margins": 12.167176912236096,
      "rewards/rejected": -10.197530864197532,
      "step": 708
    },
    {
      "epoch": 0.34926108374384235,
      "grad_norm": 0.4758063878008899,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -57609999.058823526,
      "logits/rejected": -64821061.81818182,
      "logps/chosen": -310.5882352941176,
      "logps/rejected": -510.75324675324674,
      "loss": 0.22,
      "rewards/chosen": 1.2916666666666667,
      "rewards/margins": 11.252705627705627,
      "rewards/rejected": -9.96103896103896,
      "step": 709
    },
    {
      "epoch": 0.3497536945812808,
      "grad_norm": 0.428678309172454,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -68395752.72727273,
      "logits/rejected": -79591911.61904761,
      "logps/chosen": -320.1818181818182,
      "logps/rejected": -534.8571428571429,
      "loss": 0.174,
      "rewards/chosen": 0.7720170454545454,
      "rewards/margins": 11.224397997835498,
      "rewards/rejected": -10.452380952380953,
      "step": 710
    },
    {
      "epoch": 0.3502463054187192,
      "grad_norm": 0.5004627621865979,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -58506260.89795918,
      "logits/rejected": -63551669.468354434,
      "logps/chosen": -261.8775510204082,
      "logps/rejected": -453.26582278481015,
      "loss": 0.1896,
      "rewards/chosen": 0.5522959183673469,
      "rewards/margins": 2040758.42571364,
      "rewards/rejected": -2040757.8734177216,
      "step": 711
    },
    {
      "epoch": 0.35073891625615766,
      "grad_norm": 0.4534973855437003,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -74145974.04444444,
      "logits/rejected": -74436262.55421686,
      "logps/chosen": -305.24444444444447,
      "logps/rejected": -491.1807228915663,
      "loss": 0.1819,
      "rewards/chosen": 3.6523651123046874,
      "rewards/margins": 14.568027762907098,
      "rewards/rejected": -10.91566265060241,
      "step": 712
    },
    {
      "epoch": 0.35123152709359606,
      "grad_norm": 0.4430317906843456,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -53099888.64,
      "logits/rejected": -75551245.12820514,
      "logps/chosen": -259.84,
      "logps/rejected": -491.0769230769231,
      "loss": 0.2005,
      "rewards/chosen": 0.64875,
      "rewards/margins": 10.546185897435898,
      "rewards/rejected": -9.897435897435898,
      "step": 713
    },
    {
      "epoch": 0.35172413793103446,
      "grad_norm": 0.39693875500769954,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -53221057.42222222,
      "logits/rejected": -76002809.83132531,
      "logps/chosen": -266.84444444444443,
      "logps/rejected": -506.6024096385542,
      "loss": 0.158,
      "rewards/chosen": 5.074294026692709,
      "rewards/margins": 14.556221737536083,
      "rewards/rejected": -9.481927710843374,
      "step": 714
    },
    {
      "epoch": 0.3522167487684729,
      "grad_norm": 0.5215421771873008,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -46967466.666666664,
      "logits/rejected": -75602329.6,
      "logps/chosen": -281.0,
      "logps/rejected": -516.0,
      "loss": 0.2002,
      "rewards/chosen": 2.3593422571818032,
      "rewards/margins": 12.534342257181804,
      "rewards/rejected": -10.175,
      "step": 715
    },
    {
      "epoch": 0.3527093596059113,
      "grad_norm": 0.5106983556222133,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -40976384.0,
      "logits/rejected": -55626956.8,
      "logps/chosen": -242.83333333333334,
      "logps/rejected": -452.8,
      "loss": 0.2194,
      "rewards/chosen": 1.0227864583333333,
      "rewards/margins": 9.429036458333334,
      "rewards/rejected": -8.40625,
      "step": 716
    },
    {
      "epoch": 0.3532019704433498,
      "grad_norm": 0.45254026040519996,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -67566424.43636364,
      "logits/rejected": -56077269.91780822,
      "logps/chosen": -268.8,
      "logps/rejected": -418.63013698630135,
      "loss": 0.223,
      "rewards/chosen": 3.165941273082386,
      "rewards/margins": 12.371420725137181,
      "rewards/rejected": -9.205479452054794,
      "step": 717
    },
    {
      "epoch": 0.3536945812807882,
      "grad_norm": 0.46748101819352533,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -59612661.10638298,
      "logits/rejected": -63587719.90123457,
      "logps/chosen": -308.0851063829787,
      "logps/rejected": -482.3703703703704,
      "loss": 0.2069,
      "rewards/chosen": 0.3497340425531915,
      "rewards/margins": 10.337388363540846,
      "rewards/rejected": -9.987654320987655,
      "step": 718
    },
    {
      "epoch": 0.3541871921182266,
      "grad_norm": 0.43970584453652434,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -60406201.7254902,
      "logits/rejected": -66237320.31168831,
      "logps/chosen": -218.19607843137254,
      "logps/rejected": -426.38961038961037,
      "loss": 0.2198,
      "rewards/chosen": 0.9686734068627451,
      "rewards/margins": 10.488153926343264,
      "rewards/rejected": -9.519480519480519,
      "step": 719
    },
    {
      "epoch": 0.35467980295566504,
      "grad_norm": 0.47664316911279675,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -73443119.02040817,
      "logits/rejected": -68011435.74683544,
      "logps/chosen": -274.61224489795916,
      "logps/rejected": -521.7215189873418,
      "loss": 0.2145,
      "rewards/chosen": 2.987236801458865,
      "rewards/margins": 13.923945662218358,
      "rewards/rejected": -10.936708860759493,
      "step": 720
    },
    {
      "epoch": 0.35517241379310344,
      "grad_norm": 0.5404793087377677,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -72083503.62790698,
      "logits/rejected": -66121968.941176474,
      "logps/chosen": -288.3720930232558,
      "logps/rejected": -529.3176470588236,
      "loss": 0.2042,
      "rewards/chosen": -0.6282703488372093,
      "rewards/margins": 10.689376709986322,
      "rewards/rejected": -11.31764705882353,
      "step": 721
    },
    {
      "epoch": 0.3556650246305419,
      "grad_norm": 0.5556399193753436,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -57322154.666666664,
      "logits/rejected": -65691071.09859155,
      "logps/chosen": -242.52631578947367,
      "logps/rejected": -492.16901408450707,
      "loss": 0.2806,
      "rewards/chosen": 0.9133924517715186,
      "rewards/margins": 11.166913578532082,
      "rewards/rejected": -10.253521126760564,
      "step": 722
    },
    {
      "epoch": 0.3561576354679803,
      "grad_norm": 0.5142432119468734,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -81614165.33333333,
      "logits/rejected": -62180556.8,
      "logps/chosen": -310.5,
      "logps/rejected": -469.6,
      "loss": 0.2207,
      "rewards/chosen": -0.15852864583333334,
      "rewards/margins": 10.003971354166666,
      "rewards/rejected": -10.1625,
      "step": 723
    },
    {
      "epoch": 0.3566502463054187,
      "grad_norm": 0.43513803442297855,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -65195212.8,
      "logits/rejected": -70445242.18181819,
      "logps/chosen": -216.6,
      "logps/rejected": -468.3636363636364,
      "loss": 0.1932,
      "rewards/chosen": 2.354915237426758,
      "rewards/margins": 8.464234057339755,
      "rewards/rejected": -6.109318819912997,
      "step": 724
    },
    {
      "epoch": 0.35714285714285715,
      "grad_norm": 0.4883474703997419,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -54975341.71428572,
      "logits/rejected": -67840428.65116279,
      "logps/chosen": -239.9047619047619,
      "logps/rejected": -517.2093023255813,
      "loss": 0.1904,
      "rewards/chosen": 7.151710146949405,
      "rewards/margins": 19.05868689113545,
      "rewards/rejected": -11.906976744186046,
      "step": 725
    },
    {
      "epoch": 0.35763546798029555,
      "grad_norm": 0.37601293223664184,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -63342550.20408163,
      "logits/rejected": -65356812.962025315,
      "logps/chosen": -259.9183673469388,
      "logps/rejected": -524.5569620253165,
      "loss": 0.2037,
      "rewards/chosen": 0.34183673469387754,
      "rewards/margins": 12.189938000516662,
      "rewards/rejected": -11.848101265822784,
      "step": 726
    },
    {
      "epoch": 0.358128078817734,
      "grad_norm": 0.6987163039674783,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -69777966.54545455,
      "logits/rejected": -84920291.94520548,
      "logps/chosen": -281.6,
      "logps/rejected": -580.3835616438356,
      "loss": 0.2204,
      "rewards/chosen": 0.7147727272727272,
      "rewards/margins": 11.52299190535492,
      "rewards/rejected": -10.808219178082192,
      "step": 727
    },
    {
      "epoch": 0.3586206896551724,
      "grad_norm": 0.40709698410032336,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -68506965.33333333,
      "logits/rejected": -61171472.62337662,
      "logps/chosen": -255.84313725490196,
      "logps/rejected": -481.6623376623377,
      "loss": 0.2013,
      "rewards/chosen": -0.30330882352941174,
      "rewards/margins": 11.930457410236823,
      "rewards/rejected": -12.233766233766234,
      "step": 728
    },
    {
      "epoch": 0.35911330049261087,
      "grad_norm": 0.4548516189196922,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -60245457.45454545,
      "logits/rejected": -76129490.4109589,
      "logps/chosen": -343.56363636363636,
      "logps/rejected": -576.4383561643835,
      "loss": 0.1949,
      "rewards/chosen": 1.1522727272727273,
      "rewards/margins": 13.261861768368618,
      "rewards/rejected": -12.10958904109589,
      "step": 729
    },
    {
      "epoch": 0.35960591133004927,
      "grad_norm": 0.4323871041492519,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -80717050.31111111,
      "logits/rejected": -85705296.19277108,
      "logps/chosen": -233.6,
      "logps/rejected": -592.1927710843373,
      "loss": 0.1688,
      "rewards/chosen": 4.095783487955729,
      "rewards/margins": 15.878916018076211,
      "rewards/rejected": -11.783132530120483,
      "step": 730
    },
    {
      "epoch": 0.36009852216748767,
      "grad_norm": 0.41859246271729866,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -78014054.4,
      "logits/rejected": -69044696.61538461,
      "logps/chosen": -251.92,
      "logps/rejected": -517.7435897435897,
      "loss": 0.1697,
      "rewards/chosen": 3.546241455078125,
      "rewards/margins": 16.0206004294371,
      "rewards/rejected": -12.474358974358974,
      "step": 731
    },
    {
      "epoch": 0.3605911330049261,
      "grad_norm": 0.43991617935121624,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -54896037.64705882,
      "logits/rejected": -62996267.22077922,
      "logps/chosen": -218.66666666666666,
      "logps/rejected": -520.3116883116883,
      "loss": 0.1976,
      "rewards/chosen": 3.4350454293045343,
      "rewards/margins": 15.876603870862976,
      "rewards/rejected": -12.441558441558442,
      "step": 732
    },
    {
      "epoch": 0.3610837438423645,
      "grad_norm": 0.4211580235270674,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -48790883.26530612,
      "logits/rejected": -59144995.64556962,
      "logps/chosen": -255.26530612244898,
      "logps/rejected": -516.0506329113924,
      "loss": 0.176,
      "rewards/chosen": 3.0462559291294644,
      "rewards/margins": 15.957648334192754,
      "rewards/rejected": -12.91139240506329,
      "step": 733
    },
    {
      "epoch": 0.361576354679803,
      "grad_norm": 0.7302405265983951,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -66394939.91489362,
      "logits/rejected": -82850449.38271604,
      "logps/chosen": -312.17021276595744,
      "logps/rejected": -533.3333333333334,
      "loss": 0.1882,
      "rewards/chosen": -0.1924867021276596,
      "rewards/margins": 13.622328112687155,
      "rewards/rejected": -13.814814814814815,
      "step": 734
    },
    {
      "epoch": 0.3620689655172414,
      "grad_norm": 0.4758391438087997,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -50956331.57446808,
      "logits/rejected": -72908395.45679012,
      "logps/chosen": -202.46808510638297,
      "logps/rejected": -509.6296296296296,
      "loss": 0.2114,
      "rewards/chosen": 0.4933510638297872,
      "rewards/margins": 11.443968347780404,
      "rewards/rejected": -10.950617283950617,
      "step": 735
    },
    {
      "epoch": 0.3625615763546798,
      "grad_norm": 0.49031354049040826,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -53700843.016393445,
      "logits/rejected": -57906435.82089552,
      "logps/chosen": -262.55737704918033,
      "logps/rejected": -490.5074626865672,
      "loss": 0.2367,
      "rewards/chosen": 0.5418481045081968,
      "rewards/margins": 10.855280940329093,
      "rewards/rejected": -10.313432835820896,
      "step": 736
    },
    {
      "epoch": 0.36305418719211824,
      "grad_norm": 0.5879185717741717,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -43999862.15384615,
      "logits/rejected": -68874886.73684211,
      "logps/chosen": -289.84615384615387,
      "logps/rejected": -551.578947368421,
      "loss": 0.2451,
      "rewards/chosen": 1.7037558922400842,
      "rewards/margins": 13.440597997503241,
      "rewards/rejected": -11.736842105263158,
      "step": 737
    },
    {
      "epoch": 0.36354679802955664,
      "grad_norm": 0.4098162698178407,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -64793258.666666664,
      "logits/rejected": -56885248.0,
      "logps/chosen": -266.8333333333333,
      "logps/rejected": -488.0,
      "loss": 0.2087,
      "rewards/chosen": -0.4518229166666667,
      "rewards/margins": 10.310677083333333,
      "rewards/rejected": -10.7625,
      "step": 738
    },
    {
      "epoch": 0.3640394088669951,
      "grad_norm": 1.0062789556590384,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -70604117.33333333,
      "logits/rejected": -78331461.1891892,
      "logps/chosen": -426.0740740740741,
      "logps/rejected": -515.8918918918919,
      "loss": 0.2085,
      "rewards/chosen": 0.7679398148148148,
      "rewards/margins": 14.119291166166166,
      "rewards/rejected": -13.35135135135135,
      "step": 739
    },
    {
      "epoch": 0.3645320197044335,
      "grad_norm": 0.39959721641054347,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -58458112.0,
      "logits/rejected": -58982400.0,
      "logps/chosen": -361.0,
      "logps/rejected": -464.8,
      "loss": 0.2186,
      "rewards/chosen": 5.878904978434245,
      "rewards/margins": 16.491404978434247,
      "rewards/rejected": -10.6125,
      "step": 740
    },
    {
      "epoch": 0.3650246305418719,
      "grad_norm": 0.4970997086411604,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -64229723.118644066,
      "logits/rejected": -70087427.71014492,
      "logps/chosen": -281.49152542372883,
      "logps/rejected": -500.8695652173913,
      "loss": 0.2203,
      "rewards/chosen": 0.3776483050847458,
      "rewards/margins": 12.88489468189634,
      "rewards/rejected": -12.507246376811594,
      "step": 741
    },
    {
      "epoch": 0.36551724137931035,
      "grad_norm": 0.503776730580904,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -54209400.75471698,
      "logits/rejected": -54861496.32,
      "logps/chosen": -315.47169811320754,
      "logps/rejected": -471.8933333333333,
      "loss": 0.181,
      "rewards/chosen": 0.9905660377358491,
      "rewards/margins": 13.283899371069182,
      "rewards/rejected": -12.293333333333333,
      "step": 742
    },
    {
      "epoch": 0.36600985221674875,
      "grad_norm": 0.4293487007233558,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -59319442.28571428,
      "logits/rejected": -66790309.26582278,
      "logps/chosen": -367.0204081632653,
      "logps/rejected": -492.55696202531647,
      "loss": 0.1909,
      "rewards/chosen": 2.3833443777901784,
      "rewards/margins": 13.889673491714229,
      "rewards/rejected": -11.50632911392405,
      "step": 743
    },
    {
      "epoch": 0.3665024630541872,
      "grad_norm": 0.4477627138913976,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -63629498.18181818,
      "logits/rejected": -57521883.428571425,
      "logps/chosen": -377.8181818181818,
      "logps/rejected": -482.6666666666667,
      "loss": 0.1917,
      "rewards/chosen": 4.386303988370028,
      "rewards/margins": 16.374399226465265,
      "rewards/rejected": -11.988095238095237,
      "step": 744
    },
    {
      "epoch": 0.3669950738916256,
      "grad_norm": 0.3856855766997373,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -67895296.0,
      "logits/rejected": -66298600.72727273,
      "logps/chosen": -390.2,
      "logps/rejected": -561.4545454545455,
      "loss": 0.142,
      "rewards/chosen": 4.089832305908203,
      "rewards/margins": 16.06710503318093,
      "rewards/rejected": -11.977272727272727,
      "step": 745
    },
    {
      "epoch": 0.367487684729064,
      "grad_norm": 0.3991767317521714,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -59512513.42222222,
      "logits/rejected": -67513134.26506025,
      "logps/chosen": -234.84444444444443,
      "logps/rejected": -565.9759036144578,
      "loss": 0.1914,
      "rewards/chosen": 3.924851820203993,
      "rewards/margins": 17.069430133457004,
      "rewards/rejected": -13.144578313253012,
      "step": 746
    },
    {
      "epoch": 0.36798029556650247,
      "grad_norm": 0.5143145989823862,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -59069781.333333336,
      "logits/rejected": -72666316.8,
      "logps/chosen": -327.6666666666667,
      "logps/rejected": -520.8,
      "loss": 0.2049,
      "rewards/chosen": 2.89975643157959,
      "rewards/margins": 16.27475643157959,
      "rewards/rejected": -13.375,
      "step": 747
    },
    {
      "epoch": 0.36847290640394087,
      "grad_norm": 0.5282583632040089,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -73470225.06666666,
      "logits/rejected": -70994763.29411764,
      "logps/chosen": -313.6,
      "logps/rejected": -529.4117647058823,
      "loss": 0.2345,
      "rewards/chosen": 0.2864583333333333,
      "rewards/margins": 12.198223039215687,
      "rewards/rejected": -11.911764705882353,
      "step": 748
    },
    {
      "epoch": 0.3689655172413793,
      "grad_norm": 0.478245819965396,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -57555171.55555555,
      "logits/rejected": -55092749.83783784,
      "logps/chosen": -351.1111111111111,
      "logps/rejected": -506.81081081081084,
      "loss": 0.2495,
      "rewards/chosen": -0.9762731481481481,
      "rewards/margins": 10.415618743743742,
      "rewards/rejected": -11.391891891891891,
      "step": 749
    },
    {
      "epoch": 0.3694581280788177,
      "grad_norm": 0.4217109182417153,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -65440674.90909091,
      "logits/rejected": -68107507.8095238,
      "logps/chosen": -223.0909090909091,
      "logps/rejected": -492.95238095238096,
      "loss": 0.1618,
      "rewards/chosen": 0.29829545454545453,
      "rewards/margins": 12.334009740259742,
      "rewards/rejected": -12.035714285714286,
      "step": 750
    },
    {
      "epoch": 0.3699507389162562,
      "grad_norm": 0.5755247493323583,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -73738570.32258065,
      "logits/rejected": -61834208.96969697,
      "logps/chosen": -338.3225806451613,
      "logps/rejected": -534.7878787878788,
      "loss": 0.2304,
      "rewards/chosen": -0.24407762096774194,
      "rewards/margins": 13.34683146994135,
      "rewards/rejected": -13.590909090909092,
      "step": 751
    },
    {
      "epoch": 0.3704433497536946,
      "grad_norm": 0.39542112908418975,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -49359797.07317073,
      "logits/rejected": -78775778.57471265,
      "logps/chosen": -316.0975609756098,
      "logps/rejected": -507.58620689655174,
      "loss": 0.1835,
      "rewards/chosen": 5.5122263838605186,
      "rewards/margins": 17.949007993055922,
      "rewards/rejected": -12.436781609195402,
      "step": 752
    },
    {
      "epoch": 0.370935960591133,
      "grad_norm": 0.443592392981004,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -58541774.97872341,
      "logits/rejected": -70526445.03703703,
      "logps/chosen": -352.3404255319149,
      "logps/rejected": -541.2345679012345,
      "loss": 0.1855,
      "rewards/chosen": 3.0205909242021276,
      "rewards/margins": 16.563800800745337,
      "rewards/rejected": -13.54320987654321,
      "step": 753
    },
    {
      "epoch": 0.37142857142857144,
      "grad_norm": 0.47026820129302827,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -59744446.511627905,
      "logits/rejected": -79938499.76470588,
      "logps/chosen": -306.2325581395349,
      "logps/rejected": -560.9411764705883,
      "loss": 0.2053,
      "rewards/chosen": 2.9440520530523258,
      "rewards/margins": 16.661699111875855,
      "rewards/rejected": -13.717647058823529,
      "step": 754
    },
    {
      "epoch": 0.37192118226600984,
      "grad_norm": 0.402789936151447,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -43569402.77551021,
      "logits/rejected": -64029501.56962025,
      "logps/chosen": -265.7959183673469,
      "logps/rejected": -487.2911392405063,
      "loss": 0.2149,
      "rewards/chosen": -0.20918367346938777,
      "rewards/margins": 9.816132782226815,
      "rewards/rejected": -10.025316455696203,
      "step": 755
    },
    {
      "epoch": 0.3724137931034483,
      "grad_norm": 0.43982857664826797,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -67985854.83636364,
      "logits/rejected": -67108864.0,
      "logps/chosen": -304.58181818181816,
      "logps/rejected": -471.2328767123288,
      "loss": 0.2608,
      "rewards/chosen": 1.6701643510298296,
      "rewards/margins": 12.122219145550376,
      "rewards/rejected": -10.452054794520548,
      "step": 756
    },
    {
      "epoch": 0.3729064039408867,
      "grad_norm": 0.43187136903893625,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -45308237.39534884,
      "logits/rejected": -61582251.67058823,
      "logps/chosen": -200.0,
      "logps/rejected": -526.3058823529411,
      "loss": 0.1746,
      "rewards/chosen": 3.8317161382630816,
      "rewards/margins": 16.46701025591014,
      "rewards/rejected": -12.635294117647058,
      "step": 757
    },
    {
      "epoch": 0.3733990147783251,
      "grad_norm": 0.3921705202822449,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -54569642.666666664,
      "logits/rejected": -59611545.6,
      "logps/chosen": -334.0,
      "logps/rejected": -570.0,
      "loss": 0.1933,
      "rewards/chosen": -0.4339192708333333,
      "rewards/margins": 12.691080729166666,
      "rewards/rejected": -13.125,
      "step": 758
    },
    {
      "epoch": 0.37389162561576356,
      "grad_norm": 0.495908233703316,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -49241128.96,
      "logits/rejected": -76895573.33333333,
      "logps/chosen": -296.0,
      "logps/rejected": -613.7435897435897,
      "loss": 0.1707,
      "rewards/chosen": 1.379375,
      "rewards/margins": 10.481338923527645,
      "rewards/rejected": -9.101963923527645,
      "step": 759
    },
    {
      "epoch": 0.37438423645320196,
      "grad_norm": 0.42378835120727376,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -69655405.71428572,
      "logits/rejected": -69497287.1111111,
      "logps/chosen": -397.7142857142857,
      "logps/rejected": -527.1111111111111,
      "loss": 0.2207,
      "rewards/chosen": 1.89117431640625,
      "rewards/margins": 13.64117431640625,
      "rewards/rejected": -11.75,
      "step": 760
    },
    {
      "epoch": 0.3748768472906404,
      "grad_norm": 0.46457560102081963,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -58992109.03703704,
      "logits/rejected": -65238431.13513514,
      "logps/chosen": -267.25925925925924,
      "logps/rejected": -497.2972972972973,
      "loss": 0.2121,
      "rewards/chosen": 0.90625,
      "rewards/margins": 13.67652027027027,
      "rewards/rejected": -12.77027027027027,
      "step": 761
    },
    {
      "epoch": 0.3753694581280788,
      "grad_norm": 0.40739787464300836,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -61129749.787234046,
      "logits/rejected": -77568733.2345679,
      "logps/chosen": -337.02127659574467,
      "logps/rejected": -473.28395061728395,
      "loss": 0.1448,
      "rewards/chosen": 4.253354985663232,
      "rewards/margins": 16.920021652329897,
      "rewards/rejected": -12.666666666666666,
      "step": 762
    },
    {
      "epoch": 0.3758620689655172,
      "grad_norm": 0.4781557587098213,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -58529605.81818182,
      "logits/rejected": -70004931.04761904,
      "logps/chosen": -250.45454545454547,
      "logps/rejected": -540.1904761904761,
      "loss": 0.1968,
      "rewards/chosen": 1.8266885930841619,
      "rewards/margins": 14.421926688322257,
      "rewards/rejected": -12.595238095238095,
      "step": 763
    },
    {
      "epoch": 0.37635467980295567,
      "grad_norm": 0.5664762006787253,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -53477376.0,
      "logits/rejected": -61638032.69565217,
      "logps/chosen": -235.55555555555554,
      "logps/rejected": -496.0,
      "loss": 0.1564,
      "rewards/chosen": 1.3663194444444444,
      "rewards/margins": 12.24675422705314,
      "rewards/rejected": -10.880434782608695,
      "step": 764
    },
    {
      "epoch": 0.3768472906403941,
      "grad_norm": 0.6618836413515298,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -46615641.8245614,
      "logits/rejected": -62855485.295774646,
      "logps/chosen": -255.859649122807,
      "logps/rejected": -482.7042253521127,
      "loss": 0.1981,
      "rewards/chosen": 3.6746052524499726,
      "rewards/margins": 16.67460525244997,
      "rewards/rejected": -13.0,
      "step": 765
    },
    {
      "epoch": 0.37733990147783253,
      "grad_norm": 0.4485751335103215,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -46443178.666666664,
      "logits/rejected": -68839014.4,
      "logps/chosen": -206.91666666666666,
      "logps/rejected": -508.8,
      "loss": 0.2051,
      "rewards/chosen": 5.398414611816406,
      "rewards/margins": 19.273414611816406,
      "rewards/rejected": -13.875,
      "step": 766
    },
    {
      "epoch": 0.37783251231527093,
      "grad_norm": 0.4634679551763814,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -59943594.666666664,
      "logits/rejected": -60764979.2,
      "logps/chosen": -338.3333333333333,
      "logps/rejected": -503.6,
      "loss": 0.2092,
      "rewards/chosen": 3.6230411529541016,
      "rewards/margins": 15.610541152954102,
      "rewards/rejected": -11.9875,
      "step": 767
    },
    {
      "epoch": 0.3783251231527094,
      "grad_norm": 0.5042342659369549,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -56373443.047619045,
      "logits/rejected": -70230206.51162791,
      "logps/chosen": -265.14285714285717,
      "logps/rejected": -518.6976744186046,
      "loss": 0.1912,
      "rewards/chosen": 3.353385561988467,
      "rewards/margins": 15.958036724779165,
      "rewards/rejected": -12.604651162790697,
      "step": 768
    },
    {
      "epoch": 0.3788177339901478,
      "grad_norm": 0.5547710544409407,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -60183385.302325584,
      "logits/rejected": -67997069.55294117,
      "logps/chosen": -349.3953488372093,
      "logps/rejected": -547.3882352941176,
      "loss": 0.1742,
      "rewards/chosen": 6.343795421511628,
      "rewards/margins": 19.814383656805745,
      "rewards/rejected": -13.470588235294118,
      "step": 769
    },
    {
      "epoch": 0.3793103448275862,
      "grad_norm": 0.5794741853567806,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -46279523.79661017,
      "logits/rejected": -71485529.04347827,
      "logps/chosen": -286.64406779661016,
      "logps/rejected": -537.9710144927536,
      "loss": 0.2624,
      "rewards/chosen": 0.2071636652542373,
      "rewards/margins": 11.540496998587571,
      "rewards/rejected": -11.333333333333334,
      "step": 770
    },
    {
      "epoch": 0.37980295566502464,
      "grad_norm": 0.4918979930470207,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -55756012.307692304,
      "logits/rejected": -84713903.15789473,
      "logps/chosen": -347.0769230769231,
      "logps/rejected": -508.2105263157895,
      "loss": 0.2304,
      "rewards/chosen": 1.0090191180889423,
      "rewards/margins": 14.9958612233521,
      "rewards/rejected": -13.986842105263158,
      "step": 771
    },
    {
      "epoch": 0.38029556650246304,
      "grad_norm": 0.6072850372499744,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -72807646.60869566,
      "logits/rejected": -78668775.02439025,
      "logps/chosen": -433.39130434782606,
      "logps/rejected": -562.7317073170732,
      "loss": 0.2223,
      "rewards/chosen": -0.11005434782608696,
      "rewards/margins": 12.243604188759278,
      "rewards/rejected": -12.353658536585366,
      "step": 772
    },
    {
      "epoch": 0.3807881773399015,
      "grad_norm": 0.49385275965782205,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -41332959.41818182,
      "logits/rejected": -66879039.12328767,
      "logps/chosen": -248.72727272727272,
      "logps/rejected": -486.13698630136986,
      "loss": 0.2149,
      "rewards/chosen": 2.9215462424538354,
      "rewards/margins": 14.921546242453836,
      "rewards/rejected": -12.0,
      "step": 773
    },
    {
      "epoch": 0.3812807881773399,
      "grad_norm": 0.522332489652123,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -49129621.853658535,
      "logits/rejected": -69133700.4137931,
      "logps/chosen": -271.609756097561,
      "logps/rejected": -503.17241379310343,
      "loss": 0.1647,
      "rewards/chosen": 8.26778281607279,
      "rewards/margins": 20.716058678141756,
      "rewards/rejected": -12.448275862068966,
      "step": 774
    },
    {
      "epoch": 0.3817733990147783,
      "grad_norm": 0.557088429441143,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -63405382.80851064,
      "logits/rejected": -64519787.45679013,
      "logps/chosen": -218.2127659574468,
      "logps/rejected": -531.7530864197531,
      "loss": 0.2124,
      "rewards/chosen": 2.6934863151387964,
      "rewards/margins": 15.644103599089414,
      "rewards/rejected": -12.950617283950617,
      "step": 775
    },
    {
      "epoch": 0.38226600985221676,
      "grad_norm": 0.4403545614584043,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -65057302.26086956,
      "logits/rejected": -65983562.92682927,
      "logps/chosen": -274.4347826086956,
      "logps/rejected": -524.4878048780488,
      "loss": 0.156,
      "rewards/chosen": 1.578804347826087,
      "rewards/margins": 14.456853128313892,
      "rewards/rejected": -12.878048780487806,
      "step": 776
    },
    {
      "epoch": 0.38275862068965516,
      "grad_norm": 0.38174142826286533,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -35237672.421052635,
      "logits/rejected": -64033041.06666667,
      "logps/chosen": -220.94736842105263,
      "logps/rejected": -550.4,
      "loss": 0.1642,
      "rewards/chosen": 0.5049342105263158,
      "rewards/margins": 13.371600877192982,
      "rewards/rejected": -12.866666666666667,
      "step": 777
    },
    {
      "epoch": 0.3832512315270936,
      "grad_norm": 0.5795890574787211,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -54382964.36363637,
      "logits/rejected": -60917272.38095238,
      "logps/chosen": -225.27272727272728,
      "logps/rejected": -546.2857142857143,
      "loss": 0.1873,
      "rewards/chosen": 0.32173295454545453,
      "rewards/margins": 12.036018668831169,
      "rewards/rejected": -11.714285714285714,
      "step": 778
    },
    {
      "epoch": 0.383743842364532,
      "grad_norm": 0.586636597220489,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -55312384.0,
      "logits/rejected": -65710762.666666664,
      "logps/chosen": -325.42857142857144,
      "logps/rejected": -435.55555555555554,
      "loss": 0.2432,
      "rewards/chosen": -0.21651785714285715,
      "rewards/margins": 11.325148809523808,
      "rewards/rejected": -11.541666666666666,
      "step": 779
    },
    {
      "epoch": 0.3842364532019704,
      "grad_norm": 0.45208014697890175,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -69730304.0,
      "logits/rejected": -85144371.2,
      "logps/chosen": -324.6666666666667,
      "logps/rejected": -520.0,
      "loss": 0.204,
      "rewards/chosen": 0.439453125,
      "rewards/margins": 11.914453125,
      "rewards/rejected": -11.475,
      "step": 780
    },
    {
      "epoch": 0.3847290640394089,
      "grad_norm": 0.38606390430531184,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -69303557.95348836,
      "logits/rejected": -72388752.56470588,
      "logps/chosen": -370.6046511627907,
      "logps/rejected": -540.6117647058824,
      "loss": 0.1472,
      "rewards/chosen": 7.3059742062590844,
      "rewards/margins": 17.105974206259084,
      "rewards/rejected": -9.8,
      "step": 781
    },
    {
      "epoch": 0.3852216748768473,
      "grad_norm": 0.49113886223692266,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -56438061.176470585,
      "logits/rejected": -63513746.28571428,
      "logps/chosen": -278.11764705882354,
      "logps/rejected": -447.16883116883116,
      "loss": 0.1953,
      "rewards/chosen": 0.8100490196078431,
      "rewards/margins": 10.92693213649096,
      "rewards/rejected": -10.116883116883116,
      "step": 782
    },
    {
      "epoch": 0.38571428571428573,
      "grad_norm": 0.585678333889905,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -56409108.89795918,
      "logits/rejected": -65197535.594936706,
      "logps/chosen": -281.7959183673469,
      "logps/rejected": -509.56962025316454,
      "loss": 0.2219,
      "rewards/chosen": 7.249911561304209,
      "rewards/margins": 16.395481181557372,
      "rewards/rejected": -9.145569620253164,
      "step": 783
    },
    {
      "epoch": 0.38620689655172413,
      "grad_norm": 0.531925704911731,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -66498783.41818182,
      "logits/rejected": -56594375.89041096,
      "logps/chosen": -392.43636363636364,
      "logps/rejected": -488.32876712328766,
      "loss": 0.2521,
      "rewards/chosen": 0.0011363636363636363,
      "rewards/margins": 10.206615815691158,
      "rewards/rejected": -10.205479452054794,
      "step": 784
    },
    {
      "epoch": 0.3866995073891626,
      "grad_norm": 0.4310185618899825,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -59042894.76923077,
      "logits/rejected": -69559468.58426966,
      "logps/chosen": -328.61538461538464,
      "logps/rejected": -393.34831460674155,
      "loss": 0.1931,
      "rewards/chosen": 3.2796497834034457,
      "rewards/margins": 13.08863854744839,
      "rewards/rejected": -9.808988764044944,
      "step": 785
    },
    {
      "epoch": 0.387192118226601,
      "grad_norm": 0.5566754711298803,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -51439577.358490564,
      "logits/rejected": -66549623.46666667,
      "logps/chosen": -284.37735849056605,
      "logps/rejected": -472.32,
      "loss": 0.2378,
      "rewards/chosen": 0.40035377358490565,
      "rewards/margins": 10.520353773584905,
      "rewards/rejected": -10.12,
      "step": 786
    },
    {
      "epoch": 0.3876847290640394,
      "grad_norm": 0.5112701285842626,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -43538699.13043478,
      "logits/rejected": -75599772.09756097,
      "logps/chosen": -275.30434782608694,
      "logps/rejected": -501.8536585365854,
      "loss": 0.1832,
      "rewards/chosen": 1.2425271739130435,
      "rewards/margins": 10.364478393425237,
      "rewards/rejected": -9.121951219512194,
      "step": 787
    },
    {
      "epoch": 0.38817733990147785,
      "grad_norm": 0.5100204007330281,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -51380224.0,
      "logits/rejected": -61408912.41025641,
      "logps/chosen": -285.44,
      "logps/rejected": -470.56410256410254,
      "loss": 0.2145,
      "rewards/chosen": 3.0324951171875,
      "rewards/margins": 12.045315630008012,
      "rewards/rejected": -9.012820512820513,
      "step": 788
    },
    {
      "epoch": 0.38866995073891625,
      "grad_norm": 0.672073912203716,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -55255396.17391305,
      "logits/rejected": -76980823.41463415,
      "logps/chosen": -299.4782608695652,
      "logps/rejected": -511.219512195122,
      "loss": 0.2009,
      "rewards/chosen": 0.07472826086956522,
      "rewards/margins": 11.105216065747614,
      "rewards/rejected": -11.03048780487805,
      "step": 789
    },
    {
      "epoch": 0.3891625615763547,
      "grad_norm": 0.42689841553855024,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -43269809.63265306,
      "logits/rejected": -61056324.05063291,
      "logps/chosen": -261.55102040816325,
      "logps/rejected": -460.55696202531647,
      "loss": 0.2259,
      "rewards/chosen": -0.2771843112244898,
      "rewards/margins": 9.305094169788168,
      "rewards/rejected": -9.582278481012658,
      "step": 790
    },
    {
      "epoch": 0.3896551724137931,
      "grad_norm": 0.4034017595913464,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -51713861.81818182,
      "logits/rejected": -69206016.0,
      "logps/chosen": -335.27272727272725,
      "logps/rejected": -552.3809523809524,
      "loss": 0.1465,
      "rewards/chosen": 3.6227056329900567,
      "rewards/margins": 12.83699134727577,
      "rewards/rejected": -9.214285714285714,
      "step": 791
    },
    {
      "epoch": 0.3901477832512315,
      "grad_norm": 0.5156415122906529,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -55719159.172413796,
      "logits/rejected": -61176919.77142857,
      "logps/chosen": -297.37931034482756,
      "logps/rejected": -464.45714285714286,
      "loss": 0.2528,
      "rewards/chosen": 0.011129445043103448,
      "rewards/margins": 9.511129445043103,
      "rewards/rejected": -9.5,
      "step": 792
    },
    {
      "epoch": 0.39064039408866996,
      "grad_norm": 0.4562523054223254,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -64861915.428571425,
      "logits/rejected": -60232156.27906977,
      "logps/chosen": -351.04761904761904,
      "logps/rejected": -480.0,
      "loss": 0.1569,
      "rewards/chosen": 6.357915969122024,
      "rewards/margins": 15.62535782958714,
      "rewards/rejected": -9.267441860465116,
      "step": 793
    },
    {
      "epoch": 0.39113300492610836,
      "grad_norm": 0.4306353267004907,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -79866538.66666667,
      "logits/rejected": -77804339.2,
      "logps/chosen": -353.3333333333333,
      "logps/rejected": -544.0,
      "loss": 0.1933,
      "rewards/chosen": 2.912435531616211,
      "rewards/margins": 13.04993553161621,
      "rewards/rejected": -10.1375,
      "step": 794
    },
    {
      "epoch": 0.3916256157635468,
      "grad_norm": 0.5159559709896869,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -42159986.75862069,
      "logits/rejected": -63393909.02857143,
      "logps/chosen": -247.44827586206895,
      "logps/rejected": -469.9428571428571,
      "loss": 0.2468,
      "rewards/chosen": 2.1486911115975214,
      "rewards/margins": 9.062976825883236,
      "rewards/rejected": -6.914285714285715,
      "step": 795
    },
    {
      "epoch": 0.3921182266009852,
      "grad_norm": 0.474770939943981,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -60407095.652173914,
      "logits/rejected": -51124473.75609756,
      "logps/chosen": -350.60869565217394,
      "logps/rejected": -524.4878048780488,
      "loss": 0.2319,
      "rewards/chosen": 5.875015922214674,
      "rewards/margins": 15.277454946604918,
      "rewards/rejected": -9.402439024390244,
      "step": 796
    },
    {
      "epoch": 0.3926108374384236,
      "grad_norm": 0.46349223677222934,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -65377881.396825396,
      "logits/rejected": -54429160.36923077,
      "logps/chosen": -339.8095238095238,
      "logps/rejected": -462.7692307692308,
      "loss": 0.1864,
      "rewards/chosen": 1.8353174603174602,
      "rewards/margins": 11.135317460317461,
      "rewards/rejected": -9.3,
      "step": 797
    },
    {
      "epoch": 0.3931034482758621,
      "grad_norm": 0.40878992013258914,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -59462997.333333336,
      "logits/rejected": -56675532.8,
      "logps/chosen": -280.0,
      "logps/rejected": -482.0,
      "loss": 0.1604,
      "rewards/chosen": 1.0924479166666667,
      "rewards/margins": 11.667447916666665,
      "rewards/rejected": -10.575,
      "step": 798
    },
    {
      "epoch": 0.3935960591133005,
      "grad_norm": 0.5085015677484773,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -47049149.217391305,
      "logits/rejected": -78873375.2195122,
      "logps/chosen": -262.2608695652174,
      "logps/rejected": -507.3170731707317,
      "loss": 0.2009,
      "rewards/chosen": 3.1324784652046533,
      "rewards/margins": 12.876380904229045,
      "rewards/rejected": -9.74390243902439,
      "step": 799
    },
    {
      "epoch": 0.39408866995073893,
      "grad_norm": 0.49994068792831103,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -41506133.333333336,
      "logits/rejected": -70254592.0,
      "logps/chosen": -253.33333333333334,
      "logps/rejected": -503.2,
      "loss": 0.2287,
      "rewards/chosen": 0.21419270833333334,
      "rewards/margins": 9.889192708333335,
      "rewards/rejected": -9.675,
      "step": 800
    },
    {
      "epoch": 0.39458128078817734,
      "grad_norm": 0.4738635150118653,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -59894661.12,
      "logits/rejected": -60817408.0,
      "logps/chosen": -422.08,
      "logps/rejected": -519.3846153846154,
      "loss": 0.2096,
      "rewards/chosen": 2.475306396484375,
      "rewards/margins": 12.35992178109976,
      "rewards/rejected": -9.884615384615385,
      "step": 801
    },
    {
      "epoch": 0.39507389162561574,
      "grad_norm": 0.36883890209575726,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -52848230.4,
      "logits/rejected": -73066682.18181819,
      "logps/chosen": -296.0,
      "logps/rejected": -423.6363636363636,
      "loss": 0.1956,
      "rewards/chosen": 4.729658889770508,
      "rewards/margins": 14.081931617043235,
      "rewards/rejected": -9.352272727272727,
      "step": 802
    },
    {
      "epoch": 0.3955665024630542,
      "grad_norm": 0.5046992460673937,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -55647928.32,
      "logits/rejected": -69851293.53846154,
      "logps/chosen": -303.68,
      "logps/rejected": -501.3333333333333,
      "loss": 0.2392,
      "rewards/chosen": -0.179375,
      "rewards/margins": 10.41036858974359,
      "rewards/rejected": -10.58974358974359,
      "step": 803
    },
    {
      "epoch": 0.3960591133004926,
      "grad_norm": 0.508127888540938,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -54754732.21818182,
      "logits/rejected": -63029472.43835616,
      "logps/chosen": -211.63636363636363,
      "logps/rejected": -541.3698630136986,
      "loss": 0.2356,
      "rewards/chosen": 0.675,
      "rewards/margins": 10.222945205479453,
      "rewards/rejected": -9.547945205479452,
      "step": 804
    },
    {
      "epoch": 0.39655172413793105,
      "grad_norm": 0.5007065527763213,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -47443117.88679245,
      "logits/rejected": -60789445.97333334,
      "logps/chosen": -336.0,
      "logps/rejected": -460.8,
      "loss": 0.2343,
      "rewards/chosen": 0.3870872641509434,
      "rewards/margins": 5.567439233551985,
      "rewards/rejected": -5.180351969401042,
      "step": 805
    },
    {
      "epoch": 0.39704433497536945,
      "grad_norm": 0.42149226332898165,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -48111134.11764706,
      "logits/rejected": -70595038.75324675,
      "logps/chosen": -248.47058823529412,
      "logps/rejected": -502.4415584415584,
      "loss": 0.1699,
      "rewards/chosen": 3.6329402549594056,
      "rewards/margins": 13.139433761452912,
      "rewards/rejected": -9.506493506493506,
      "step": 806
    },
    {
      "epoch": 0.3975369458128079,
      "grad_norm": 0.4815580567118331,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -66528800.680851065,
      "logits/rejected": -65503636.54320987,
      "logps/chosen": -288.0,
      "logps/rejected": -508.04938271604937,
      "loss": 0.2357,
      "rewards/chosen": -0.9694148936170213,
      "rewards/margins": 9.166387575518781,
      "rewards/rejected": -10.135802469135802,
      "step": 807
    },
    {
      "epoch": 0.3980295566502463,
      "grad_norm": 0.48988091011970514,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -81030383.65957446,
      "logits/rejected": -68248057.67901234,
      "logps/chosen": -299.2340425531915,
      "logps/rejected": -485.1358024691358,
      "loss": 0.1902,
      "rewards/chosen": 0.9454787234042553,
      "rewards/margins": 2702431.760293538,
      "rewards/rejected": -2702430.814814815,
      "step": 808
    },
    {
      "epoch": 0.3985221674876847,
      "grad_norm": 0.5466184241574649,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -44332817.86046512,
      "logits/rejected": -66121968.941176474,
      "logps/chosen": -300.27906976744185,
      "logps/rejected": -493.1764705882353,
      "loss": 0.1739,
      "rewards/chosen": 0.7505450581395349,
      "rewards/margins": 10.727015646374829,
      "rewards/rejected": -9.976470588235294,
      "step": 809
    },
    {
      "epoch": 0.39901477832512317,
      "grad_norm": 0.6062986011258861,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -56846205.27659574,
      "logits/rejected": -57115028.54320987,
      "logps/chosen": -348.93617021276594,
      "logps/rejected": -470.91358024691357,
      "loss": 0.1922,
      "rewards/chosen": 0.4242021276595745,
      "rewards/margins": 10.103214473338587,
      "rewards/rejected": -9.679012345679013,
      "step": 810
    },
    {
      "epoch": 0.39950738916256157,
      "grad_norm": 0.4048419022851712,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -69987307.92156863,
      "logits/rejected": -60299928.935064934,
      "logps/chosen": -372.078431372549,
      "logps/rejected": -488.31168831168833,
      "loss": 0.1725,
      "rewards/chosen": 1.5379901960784315,
      "rewards/margins": 11.226301884390121,
      "rewards/rejected": -9.688311688311689,
      "step": 811
    },
    {
      "epoch": 0.4,
      "grad_norm": 0.4717027352458002,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -40475033.6,
      "logits/rejected": -70377953.88235295,
      "logps/chosen": -187.46666666666667,
      "logps/rejected": -480.47058823529414,
      "loss": 0.2619,
      "rewards/chosen": 0.021158854166666668,
      "rewards/margins": 9.388805912990195,
      "rewards/rejected": -9.367647058823529,
      "step": 812
    },
    {
      "epoch": 0.4004926108374384,
      "grad_norm": 0.4049754825016535,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -70828341.13207547,
      "logits/rejected": -69345826.13333334,
      "logps/chosen": -311.8490566037736,
      "logps/rejected": -481.28,
      "loss": 0.2213,
      "rewards/chosen": 0.22302476415094338,
      "rewards/margins": 10.676358097484277,
      "rewards/rejected": -10.453333333333333,
      "step": 813
    },
    {
      "epoch": 0.4009852216748768,
      "grad_norm": 0.45027493359892445,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -54708313.04347826,
      "logits/rejected": -60561657.75609756,
      "logps/chosen": -282.0869565217391,
      "logps/rejected": -481.9512195121951,
      "loss": 0.2052,
      "rewards/chosen": -0.34646739130434784,
      "rewards/margins": 8.71450821845175,
      "rewards/rejected": -9.060975609756097,
      "step": 814
    },
    {
      "epoch": 0.4014778325123153,
      "grad_norm": 0.5433337665482778,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -67895296.0,
      "logits/rejected": -66759338.666666664,
      "logps/chosen": -215.71428571428572,
      "logps/rejected": -532.4444444444445,
      "loss": 0.1833,
      "rewards/chosen": 5.5401475088936945,
      "rewards/margins": 14.901258620004805,
      "rewards/rejected": -9.36111111111111,
      "step": 815
    },
    {
      "epoch": 0.4019704433497537,
      "grad_norm": 0.5364777569714381,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -62996801.25490196,
      "logits/rejected": -72501540.57142857,
      "logps/chosen": -330.3529411764706,
      "logps/rejected": -548.987012987013,
      "loss": 0.2197,
      "rewards/chosen": 0.45002297794117646,
      "rewards/margins": 9.281191809110007,
      "rewards/rejected": -8.831168831168831,
      "step": 816
    },
    {
      "epoch": 0.40246305418719214,
      "grad_norm": 0.49074221265334284,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -54900443.428571425,
      "logits/rejected": -64312661.333333336,
      "logps/chosen": -348.2857142857143,
      "logps/rejected": -528.0,
      "loss": 0.1982,
      "rewards/chosen": 3.464813777378627,
      "rewards/margins": 12.409258221823071,
      "rewards/rejected": -8.944444444444445,
      "step": 817
    },
    {
      "epoch": 0.40295566502463054,
      "grad_norm": 0.4121837714733238,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -50203250.93877551,
      "logits/rejected": -67958343.29113925,
      "logps/chosen": -255.51020408163265,
      "logps/rejected": -443.54430379746833,
      "loss": 0.1657,
      "rewards/chosen": 1.2678571428571428,
      "rewards/margins": 9.913426763110307,
      "rewards/rejected": -8.645569620253164,
      "step": 818
    },
    {
      "epoch": 0.40344827586206894,
      "grad_norm": 0.5565297552270749,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -55441072.872727275,
      "logits/rejected": -65385177.42465753,
      "logps/chosen": -279.56363636363636,
      "logps/rejected": -448.8767123287671,
      "loss": 0.2312,
      "rewards/chosen": 0.6386363636363637,
      "rewards/margins": 8.775622665006226,
      "rewards/rejected": -8.136986301369863,
      "step": 819
    },
    {
      "epoch": 0.4039408866995074,
      "grad_norm": 0.407084758638987,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -66182215.44186047,
      "logits/rejected": -63556041.78823529,
      "logps/chosen": -280.5581395348837,
      "logps/rejected": -487.15294117647056,
      "loss": 0.1844,
      "rewards/chosen": 1.087936046511628,
      "rewards/margins": 9.41734781121751,
      "rewards/rejected": -8.329411764705883,
      "step": 820
    },
    {
      "epoch": 0.4044334975369458,
      "grad_norm": 0.4834024264099697,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -52051312.64,
      "logits/rejected": -63936249.43589743,
      "logps/chosen": -194.24,
      "logps/rejected": -477.94871794871796,
      "loss": 0.2029,
      "rewards/chosen": 2.8224871826171873,
      "rewards/margins": 11.19428205441206,
      "rewards/rejected": -8.371794871794872,
      "step": 821
    },
    {
      "epoch": 0.40492610837438425,
      "grad_norm": 0.5364251326611913,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -55593593.018181816,
      "logits/rejected": -57973325.15068493,
      "logps/chosen": -261.23636363636365,
      "logps/rejected": -423.45205479452056,
      "loss": 0.2219,
      "rewards/chosen": 1.269904119318182,
      "rewards/margins": 8.8520959001401,
      "rewards/rejected": -7.582191780821918,
      "step": 822
    },
    {
      "epoch": 0.40541871921182265,
      "grad_norm": 0.4475198087552982,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -52894833.777777776,
      "logits/rejected": -69837688.28915663,
      "logps/chosen": -339.2,
      "logps/rejected": -510.4578313253012,
      "loss": 0.1439,
      "rewards/chosen": 2.5944444444444446,
      "rewards/margins": 11.702878179384204,
      "rewards/rejected": -9.108433734939759,
      "step": 823
    },
    {
      "epoch": 0.4059113300492611,
      "grad_norm": 0.5124843131224265,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -54279228.23529412,
      "logits/rejected": -65964962.90909091,
      "logps/chosen": -332.2352941176471,
      "logps/rejected": -511.16883116883116,
      "loss": 0.2274,
      "rewards/chosen": 2.7656462426279105,
      "rewards/margins": 10.492918969900638,
      "rewards/rejected": -7.7272727272727275,
      "step": 824
    },
    {
      "epoch": 0.4064039408866995,
      "grad_norm": 0.48534515945835155,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -49239381.333333336,
      "logits/rejected": -73662464.0,
      "logps/chosen": -227.33333333333334,
      "logps/rejected": -457.6,
      "loss": 0.2016,
      "rewards/chosen": 3.040990193684896,
      "rewards/margins": 5.8345181783040365,
      "rewards/rejected": -2.7935279846191405,
      "step": 825
    },
    {
      "epoch": 0.4068965517241379,
      "grad_norm": 0.42983786878934094,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -65536000.0,
      "logits/rejected": -66689433.6,
      "logps/chosen": -269.3333333333333,
      "logps/rejected": -446.4,
      "loss": 0.1928,
      "rewards/chosen": 5.666662216186523,
      "rewards/margins": 14.291662216186523,
      "rewards/rejected": -8.625,
      "step": 826
    },
    {
      "epoch": 0.40738916256157637,
      "grad_norm": 0.5667901493847409,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -58245429.13207547,
      "logits/rejected": -59615040.85333333,
      "logps/chosen": -336.3018867924528,
      "logps/rejected": -497.92,
      "loss": 0.2358,
      "rewards/chosen": 0.4834905660377358,
      "rewards/margins": 9.803490566037736,
      "rewards/rejected": -9.32,
      "step": 827
    },
    {
      "epoch": 0.40788177339901477,
      "grad_norm": 0.380204500733942,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -44564480.0,
      "logits/rejected": -68806558.47619048,
      "logps/chosen": -283.45454545454544,
      "logps/rejected": -480.76190476190476,
      "loss": 0.1663,
      "rewards/chosen": 2.859360088001598,
      "rewards/margins": 11.573645802287311,
      "rewards/rejected": -8.714285714285714,
      "step": 828
    },
    {
      "epoch": 0.4083743842364532,
      "grad_norm": 0.5069603382118502,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -62518870.943396226,
      "logits/rejected": -66829243.733333334,
      "logps/chosen": -368.3018867924528,
      "logps/rejected": -488.1066666666667,
      "loss": 0.2087,
      "rewards/chosen": 0.5825471698113207,
      "rewards/margins": 11.289213836477987,
      "rewards/rejected": -10.706666666666667,
      "step": 829
    },
    {
      "epoch": 0.4088669950738916,
      "grad_norm": 0.503910012312923,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -41581462.06896552,
      "logits/rejected": -58180988.342857145,
      "logps/chosen": -276.13793103448273,
      "logps/rejected": -498.2857142857143,
      "loss": 0.2375,
      "rewards/chosen": 0.2510775862068966,
      "rewards/margins": 7.1725061576354685,
      "rewards/rejected": -6.921428571428572,
      "step": 830
    },
    {
      "epoch": 0.40935960591133,
      "grad_norm": 0.505163877156002,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -64877851.234042555,
      "logits/rejected": -64778695.11111111,
      "logps/chosen": -274.72340425531917,
      "logps/rejected": -513.1851851851852,
      "loss": 0.1894,
      "rewards/chosen": 0.7127659574468085,
      "rewards/margins": 12.132519043866562,
      "rewards/rejected": -11.419753086419753,
      "step": 831
    },
    {
      "epoch": 0.4098522167487685,
      "grad_norm": 0.521395733737519,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -58989121.64102564,
      "logits/rejected": -61170860.584269665,
      "logps/chosen": -296.4102564102564,
      "logps/rejected": -433.6179775280899,
      "loss": 0.1761,
      "rewards/chosen": -0.7645858373397436,
      "rewards/margins": 9.49384112895239,
      "rewards/rejected": -10.258426966292134,
      "step": 832
    },
    {
      "epoch": 0.4103448275862069,
      "grad_norm": 0.4938440422716046,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -66204210.196078435,
      "logits/rejected": -65038947.74025974,
      "logps/chosen": -248.47058823529412,
      "logps/rejected": -411.012987012987,
      "loss": 0.2446,
      "rewards/chosen": -0.3284313725490196,
      "rewards/margins": 3.3370503929998843,
      "rewards/rejected": -3.665481765548904,
      "step": 833
    },
    {
      "epoch": 0.41083743842364534,
      "grad_norm": 0.43664829359418633,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -60817408.0,
      "logits/rejected": -56901039.807228915,
      "logps/chosen": -261.3333333333333,
      "logps/rejected": -427.95180722891564,
      "loss": 0.1665,
      "rewards/chosen": 4.844452582465278,
      "rewards/margins": 12.049271859573711,
      "rewards/rejected": -7.204819277108434,
      "step": 834
    },
    {
      "epoch": 0.41133004926108374,
      "grad_norm": 0.556807236240192,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -65202362.18181818,
      "logits/rejected": -63214153.14285714,
      "logps/chosen": -299.6363636363636,
      "logps/rejected": -463.6190476190476,
      "loss": 0.1665,
      "rewards/chosen": 7.0809409401633525,
      "rewards/margins": 17.890464749687162,
      "rewards/rejected": -10.80952380952381,
      "step": 835
    },
    {
      "epoch": 0.41182266009852214,
      "grad_norm": 0.4621122358402364,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -58205166.03508772,
      "logits/rejected": -55648371.38028169,
      "logps/chosen": -267.50877192982455,
      "logps/rejected": -434.92957746478874,
      "loss": 0.1976,
      "rewards/chosen": 0.8245614035087719,
      "rewards/margins": 11.183716333086236,
      "rewards/rejected": -10.359154929577464,
      "step": 836
    },
    {
      "epoch": 0.4123152709359606,
      "grad_norm": 0.4400073242172143,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -48234496.0,
      "logits/rejected": -66189564.49315068,
      "logps/chosen": -273.1636363636364,
      "logps/rejected": -536.5479452054794,
      "loss": 0.2272,
      "rewards/chosen": 2.6136402476917615,
      "rewards/margins": 10.955458861240562,
      "rewards/rejected": -8.341818613548801,
      "step": 837
    },
    {
      "epoch": 0.412807881773399,
      "grad_norm": 0.608653759725693,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -55341511.11111111,
      "logits/rejected": -64738170.43478261,
      "logps/chosen": -232.0,
      "logps/rejected": -470.95652173913044,
      "loss": 0.1569,
      "rewards/chosen": 7.582455952962239,
      "rewards/margins": 18.73462986600572,
      "rewards/rejected": -11.152173913043478,
      "step": 838
    },
    {
      "epoch": 0.41330049261083746,
      "grad_norm": 0.4723125882205793,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -66238348.0754717,
      "logits/rejected": -63473800.53333333,
      "logps/chosen": -374.33962264150944,
      "logps/rejected": -457.38666666666666,
      "loss": 0.195,
      "rewards/chosen": 2.9416296617040096,
      "rewards/margins": 14.82162966170401,
      "rewards/rejected": -11.88,
      "step": 839
    },
    {
      "epoch": 0.41379310344827586,
      "grad_norm": 0.48816683533948724,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -65758155.93220339,
      "logits/rejected": -72762056.3478261,
      "logps/chosen": -280.9491525423729,
      "logps/rejected": -578.7826086956521,
      "loss": 0.2128,
      "rewards/chosen": -0.08315677966101695,
      "rewards/margins": 8.060410291783729,
      "rewards/rejected": -8.143567071444746,
      "step": 840
    },
    {
      "epoch": 0.4142857142857143,
      "grad_norm": 0.4318902978613888,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -69008171.47169812,
      "logits/rejected": -65822610.77333333,
      "logps/chosen": -324.22641509433964,
      "logps/rejected": -488.1066666666667,
      "loss": 0.2064,
      "rewards/chosen": -0.3738207547169811,
      "rewards/margins": 12.212845911949685,
      "rewards/rejected": -12.586666666666666,
      "step": 841
    },
    {
      "epoch": 0.4147783251231527,
      "grad_norm": 0.41798760725838496,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -60415825.70212766,
      "logits/rejected": -70940697.28395061,
      "logps/chosen": -327.82978723404256,
      "logps/rejected": -541.2345679012345,
      "loss": 0.1675,
      "rewards/chosen": 1.5731382978723405,
      "rewards/margins": 14.844743236143945,
      "rewards/rejected": -13.271604938271604,
      "step": 842
    },
    {
      "epoch": 0.4152709359605911,
      "grad_norm": 0.3654318473103204,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -51233911.069767445,
      "logits/rejected": -72142028.8,
      "logps/chosen": -328.5581395348837,
      "logps/rejected": -529.6941176470589,
      "loss": 0.1354,
      "rewards/chosen": 1.447674418604651,
      "rewards/margins": 11.388850889192886,
      "rewards/rejected": -9.941176470588236,
      "step": 843
    },
    {
      "epoch": 0.41576354679802957,
      "grad_norm": 0.46344083500555433,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -60010811.07692308,
      "logits/rejected": -72958814.31578948,
      "logps/chosen": -298.7692307692308,
      "logps/rejected": -493.4736842105263,
      "loss": 0.2015,
      "rewards/chosen": -0.109375,
      "rewards/margins": 12.377467105263158,
      "rewards/rejected": -12.486842105263158,
      "step": 844
    },
    {
      "epoch": 0.41625615763546797,
      "grad_norm": 0.6905595173825354,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -73781620.36363636,
      "logits/rejected": -55924053.333333336,
      "logps/chosen": -408.72727272727275,
      "logps/rejected": -501.3333333333333,
      "loss": 0.2,
      "rewards/chosen": 3.553762609308416,
      "rewards/margins": 16.672810228356035,
      "rewards/rejected": -13.119047619047619,
      "step": 845
    },
    {
      "epoch": 0.41674876847290643,
      "grad_norm": 0.3926955102158075,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -86037005.12820514,
      "logits/rejected": -71256040.98876405,
      "logps/chosen": -490.6666666666667,
      "logps/rejected": -550.8314606741573,
      "loss": 0.1477,
      "rewards/chosen": 1.9006410256410255,
      "rewards/margins": 15.114124171708442,
      "rewards/rejected": -13.213483146067416,
      "step": 846
    },
    {
      "epoch": 0.41724137931034483,
      "grad_norm": 0.5792110184274998,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -65516581.925925925,
      "logits/rejected": -69942853.1891892,
      "logps/chosen": -334.51851851851853,
      "logps/rejected": -592.4324324324324,
      "loss": 0.1585,
      "rewards/chosen": 1.6435185185185186,
      "rewards/margins": 15.386761761761763,
      "rewards/rejected": -13.743243243243244,
      "step": 847
    },
    {
      "epoch": 0.41773399014778323,
      "grad_norm": 0.43923656836698627,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -67021482.666666664,
      "logits/rejected": -54106521.6,
      "logps/chosen": -250.5,
      "logps/rejected": -451.2,
      "loss": 0.2051,
      "rewards/chosen": 2.710937182108561,
      "rewards/margins": 14.073437182108561,
      "rewards/rejected": -11.3625,
      "step": 848
    },
    {
      "epoch": 0.4182266009852217,
      "grad_norm": 0.4700502374465844,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -59681450.666666664,
      "logits/rejected": -70621593.6,
      "logps/chosen": -340.0,
      "logps/rejected": -516.4,
      "loss": 0.2097,
      "rewards/chosen": 0.7923324902852377,
      "rewards/margins": 13.842332490285239,
      "rewards/rejected": -13.05,
      "step": 849
    },
    {
      "epoch": 0.4187192118226601,
      "grad_norm": 0.531054101691075,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -71303168.0,
      "logits/rejected": -59645470.11764706,
      "logps/chosen": -333.8666666666667,
      "logps/rejected": -459.7647058823529,
      "loss": 0.2325,
      "rewards/chosen": 1.321875,
      "rewards/margins": 1301117.792463235,
      "rewards/rejected": -1301116.4705882352,
      "step": 850
    },
    {
      "epoch": 0.41921182266009854,
      "grad_norm": 0.43710512789572886,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -74299099.42857143,
      "logits/rejected": -57555171.55555555,
      "logps/chosen": -278.42857142857144,
      "logps/rejected": -485.3333333333333,
      "loss": 0.1843,
      "rewards/chosen": 3.496619633265904,
      "rewards/margins": 12.162630172002883,
      "rewards/rejected": -8.666010538736979,
      "step": 851
    },
    {
      "epoch": 0.41970443349753694,
      "grad_norm": 0.4368761012697483,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -58929971.2,
      "logits/rejected": -83314129.45454545,
      "logps/chosen": -246.6,
      "logps/rejected": -574.5454545454545,
      "loss": 0.153,
      "rewards/chosen": 6.835113525390625,
      "rewards/margins": 18.835113525390625,
      "rewards/rejected": -12.0,
      "step": 852
    },
    {
      "epoch": 0.42019704433497534,
      "grad_norm": 0.6677431249192434,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -48802474.666666664,
      "logits/rejected": -59192115.2,
      "logps/chosen": -250.0,
      "logps/rejected": -411.6,
      "loss": 0.2216,
      "rewards/chosen": 3.4856859842936196,
      "rewards/margins": 15.085685984293619,
      "rewards/rejected": -11.6,
      "step": 853
    },
    {
      "epoch": 0.4206896551724138,
      "grad_norm": 0.5219605158151402,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -55627845.423728816,
      "logits/rejected": -59935996.28985507,
      "logps/chosen": -285.5593220338983,
      "logps/rejected": -499.0144927536232,
      "loss": 0.1989,
      "rewards/chosen": 2.8196509409759005,
      "rewards/margins": 14.631245143874452,
      "rewards/rejected": -11.81159420289855,
      "step": 854
    },
    {
      "epoch": 0.4211822660098522,
      "grad_norm": 0.47249816619137797,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -56964500.8372093,
      "logits/rejected": -67799690.54117647,
      "logps/chosen": -296.1860465116279,
      "logps/rejected": -575.2470588235294,
      "loss": 0.1663,
      "rewards/chosen": 0.44440406976744184,
      "rewards/margins": 13.303227599179207,
      "rewards/rejected": -12.858823529411765,
      "step": 855
    },
    {
      "epoch": 0.42167487684729066,
      "grad_norm": 0.4488910531226941,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -58443273.66037736,
      "logits/rejected": -69122129.92,
      "logps/chosen": -339.47169811320754,
      "logps/rejected": -509.0133333333333,
      "loss": 0.2131,
      "rewards/chosen": 2.7623429208431602,
      "rewards/margins": 14.349009587509826,
      "rewards/rejected": -11.586666666666666,
      "step": 856
    },
    {
      "epoch": 0.42216748768472906,
      "grad_norm": 0.487422194379068,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -49218873.469387755,
      "logits/rejected": -58030054.07594936,
      "logps/chosen": -268.57142857142856,
      "logps/rejected": -466.63291139240505,
      "loss": 0.1666,
      "rewards/chosen": 1.3360969387755102,
      "rewards/margins": 10.893058964091965,
      "rewards/rejected": -9.556962025316455,
      "step": 857
    },
    {
      "epoch": 0.42266009852216746,
      "grad_norm": 0.4787283357064719,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -47269806.08,
      "logits/rejected": -58182524.71794872,
      "logps/chosen": -334.88,
      "logps/rejected": -475.0769230769231,
      "loss": 0.2057,
      "rewards/chosen": 2.1356333923339843,
      "rewards/margins": 12.148453905154497,
      "rewards/rejected": -10.012820512820513,
      "step": 858
    },
    {
      "epoch": 0.4231527093596059,
      "grad_norm": 0.4717889366804385,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -67062260.62222222,
      "logits/rejected": -64885377.54216868,
      "logps/chosen": -353.77777777777777,
      "logps/rejected": -500.04819277108436,
      "loss": 0.1624,
      "rewards/chosen": 1.604861111111111,
      "rewards/margins": 12.821728580990628,
      "rewards/rejected": -11.216867469879517,
      "step": 859
    },
    {
      "epoch": 0.4236453201970443,
      "grad_norm": 0.5331857322806236,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -54603624.2962963,
      "logits/rejected": -62687840.86486486,
      "logps/chosen": -258.0740740740741,
      "logps/rejected": -433.2972972972973,
      "loss": 0.2473,
      "rewards/chosen": 2.054970070167824,
      "rewards/margins": 11.94010520530296,
      "rewards/rejected": -9.885135135135135,
      "step": 860
    },
    {
      "epoch": 0.4241379310344828,
      "grad_norm": 0.40855030369923345,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -48758784.0,
      "logits/rejected": -68576870.4,
      "logps/chosen": -305.0,
      "logps/rejected": -523.2,
      "loss": 0.1894,
      "rewards/chosen": 2.8766256968180337,
      "rewards/margins": 13.564125696818033,
      "rewards/rejected": -10.6875,
      "step": 861
    },
    {
      "epoch": 0.4246305418719212,
      "grad_norm": 0.515738294024321,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -68649628.73469388,
      "logits/rejected": -88239661.3670886,
      "logps/chosen": -309.8775510204082,
      "logps/rejected": -574.379746835443,
      "loss": 0.1689,
      "rewards/chosen": 1.371173469387755,
      "rewards/margins": 12.978768406096616,
      "rewards/rejected": -11.60759493670886,
      "step": 862
    },
    {
      "epoch": 0.42512315270935963,
      "grad_norm": 0.5472203987865952,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -66809270.85714286,
      "logits/rejected": -68571993.30232558,
      "logps/chosen": -350.85714285714283,
      "logps/rejected": -544.0,
      "loss": 0.1849,
      "rewards/chosen": 3.226165771484375,
      "rewards/margins": 10.26826583507449,
      "rewards/rejected": -7.042100063590116,
      "step": 863
    },
    {
      "epoch": 0.42561576354679803,
      "grad_norm": 0.36678365644403077,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -67225372.44444445,
      "logits/rejected": -61546852.17391305,
      "logps/chosen": -231.77777777777777,
      "logps/rejected": -488.3478260869565,
      "loss": 0.1597,
      "rewards/chosen": 4.5208630032009545,
      "rewards/margins": 14.39042822059226,
      "rewards/rejected": -9.869565217391305,
      "step": 864
    },
    {
      "epoch": 0.42610837438423643,
      "grad_norm": 0.4600772716757483,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -55039540.24489796,
      "logits/rejected": -70294411.34177215,
      "logps/chosen": -219.10204081632654,
      "logps/rejected": -533.0632911392405,
      "loss": 0.1858,
      "rewards/chosen": 0.32979910714285715,
      "rewards/margins": 11.582963664104883,
      "rewards/rejected": -11.253164556962025,
      "step": 865
    },
    {
      "epoch": 0.4266009852216749,
      "grad_norm": 0.4394293105199626,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -42884618.448979594,
      "logits/rejected": -63923316.658227846,
      "logps/chosen": -341.2244897959184,
      "logps/rejected": -445.9746835443038,
      "loss": 0.1972,
      "rewards/chosen": 1.425701530612245,
      "rewards/margins": 11.096587606561613,
      "rewards/rejected": -9.670886075949367,
      "step": 866
    },
    {
      "epoch": 0.4270935960591133,
      "grad_norm": 0.43593872040482884,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -58205166.03508772,
      "logits/rejected": -59015629.52112676,
      "logps/chosen": -268.35087719298247,
      "logps/rejected": -444.84507042253523,
      "loss": 0.2632,
      "rewards/chosen": -0.14583333333333334,
      "rewards/margins": 9.178110328638496,
      "rewards/rejected": -9.32394366197183,
      "step": 867
    },
    {
      "epoch": 0.42758620689655175,
      "grad_norm": 0.4449620296437249,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -78531649.36170213,
      "logits/rejected": -64001972.14814815,
      "logps/chosen": -280.17021276595744,
      "logps/rejected": -497.77777777777777,
      "loss": 0.1748,
      "rewards/chosen": 0.7433510638297872,
      "rewards/margins": 10.372980693459416,
      "rewards/rejected": -9.62962962962963,
      "step": 868
    },
    {
      "epoch": 0.42807881773399015,
      "grad_norm": 0.48706644024785484,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -74706093.88679245,
      "logits/rejected": -64536357.54666667,
      "logps/chosen": -251.0188679245283,
      "logps/rejected": -516.6933333333334,
      "loss": 0.2105,
      "rewards/chosen": 1.0790094339622642,
      "rewards/margins": 12.959009433962265,
      "rewards/rejected": -11.88,
      "step": 869
    },
    {
      "epoch": 0.42857142857142855,
      "grad_norm": 0.46522108532221723,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -66463586.461538464,
      "logits/rejected": -63687194.94736842,
      "logps/chosen": -351.0769230769231,
      "logps/rejected": -495.57894736842104,
      "loss": 0.2206,
      "rewards/chosen": 0.3389423076923077,
      "rewards/margins": 10.944205465587045,
      "rewards/rejected": -10.605263157894736,
      "step": 870
    },
    {
      "epoch": 0.429064039408867,
      "grad_norm": 0.36316874971751273,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -45687954.28571428,
      "logits/rejected": -76033952.74418604,
      "logps/chosen": -268.0,
      "logps/rejected": -559.6279069767442,
      "loss": 0.1788,
      "rewards/chosen": 6.059168134416852,
      "rewards/margins": 17.640563483254063,
      "rewards/rejected": -11.581395348837209,
      "step": 871
    },
    {
      "epoch": 0.4295566502463054,
      "grad_norm": 0.47238009046238705,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -56261526.06896552,
      "logits/rejected": -61716187.428571425,
      "logps/chosen": -272.82758620689657,
      "logps/rejected": -522.0571428571428,
      "loss": 0.217,
      "rewards/chosen": 2.3556050267712823,
      "rewards/margins": 13.48417645534271,
      "rewards/rejected": -11.128571428571428,
      "step": 872
    },
    {
      "epoch": 0.43004926108374386,
      "grad_norm": 0.40766193103804454,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -84065835.88571429,
      "logits/rejected": -76579873.03225806,
      "logps/chosen": -353.14285714285717,
      "logps/rejected": -465.89247311827955,
      "loss": 0.1697,
      "rewards/chosen": 5.512539236886161,
      "rewards/margins": 15.534044613230247,
      "rewards/rejected": -10.021505376344086,
      "step": 873
    },
    {
      "epoch": 0.43054187192118226,
      "grad_norm": 0.38625902546115637,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -64801996.8,
      "logits/rejected": -60975258.15053763,
      "logps/chosen": -305.14285714285717,
      "logps/rejected": -491.35483870967744,
      "loss": 0.131,
      "rewards/chosen": 7.4893118722098215,
      "rewards/margins": 18.510817248553906,
      "rewards/rejected": -11.021505376344086,
      "step": 874
    },
    {
      "epoch": 0.43103448275862066,
      "grad_norm": 0.46173556654100634,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -64574805.333333336,
      "logits/rejected": -66270003.2,
      "logps/chosen": -277.5,
      "logps/rejected": -552.8,
      "loss": 0.1835,
      "rewards/chosen": 0.9856770833333334,
      "rewards/margins": 11.873177083333333,
      "rewards/rejected": -10.8875,
      "step": 875
    },
    {
      "epoch": 0.4315270935960591,
      "grad_norm": 0.46683183961530517,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -52242386.48888889,
      "logits/rejected": -76053343.61445783,
      "logps/chosen": -216.53333333333333,
      "logps/rejected": -506.9879518072289,
      "loss": 0.1922,
      "rewards/chosen": 0.050347222222222224,
      "rewards/margins": 9.941913487282463,
      "rewards/rejected": -9.891566265060241,
      "step": 876
    },
    {
      "epoch": 0.4320197044334975,
      "grad_norm": 0.4659374009299693,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -59506688.0,
      "logits/rejected": -59454259.2,
      "logps/chosen": -282.6666666666667,
      "logps/rejected": -524.0,
      "loss": 0.1761,
      "rewards/chosen": 3.21941343943278,
      "rewards/margins": 15.106913439432779,
      "rewards/rejected": -11.8875,
      "step": 877
    },
    {
      "epoch": 0.432512315270936,
      "grad_norm": 0.5046492379991776,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -58774029.12820513,
      "logits/rejected": -63527211.14606742,
      "logps/chosen": -290.46153846153845,
      "logps/rejected": -473.1685393258427,
      "loss": 0.176,
      "rewards/chosen": -0.5280448717948718,
      "rewards/margins": 10.550606813598387,
      "rewards/rejected": -11.07865168539326,
      "step": 878
    },
    {
      "epoch": 0.4330049261083744,
      "grad_norm": 0.46243882581056267,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -41623908.17391305,
      "logits/rejected": -82453878.63414635,
      "logps/chosen": -284.8695652173913,
      "logps/rejected": -490.1463414634146,
      "loss": 0.1967,
      "rewards/chosen": 1.2819324990977412,
      "rewards/margins": 12.928273962512375,
      "rewards/rejected": -11.646341463414634,
      "step": 879
    },
    {
      "epoch": 0.43349753694581283,
      "grad_norm": 0.27695710474506896,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -58851328.0,
      "logits/rejected": -63526229.333333336,
      "logps/chosen": -332.0,
      "logps/rejected": -431.3333333333333,
      "loss": 0.1229,
      "rewards/chosen": 3.657168388366699,
      "rewards/margins": 14.511335055033365,
      "rewards/rejected": -10.854166666666666,
      "step": 880
    },
    {
      "epoch": 0.43399014778325123,
      "grad_norm": 0.47244023032380356,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -56187846.03773585,
      "logits/rejected": -53295622.82666667,
      "logps/chosen": -313.35849056603774,
      "logps/rejected": -486.82666666666665,
      "loss": 0.2047,
      "rewards/chosen": -0.12853773584905662,
      "rewards/margins": 12.044795597484278,
      "rewards/rejected": -12.173333333333334,
      "step": 881
    },
    {
      "epoch": 0.43448275862068964,
      "grad_norm": 0.4895510846069611,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -43683229.957446806,
      "logits/rejected": -68662309.92592593,
      "logps/chosen": -254.29787234042553,
      "logps/rejected": -466.5679012345679,
      "loss": 0.2,
      "rewards/chosen": 1.7765980172664562,
      "rewards/margins": 11.912400486402259,
      "rewards/rejected": -10.135802469135802,
      "step": 882
    },
    {
      "epoch": 0.4349753694581281,
      "grad_norm": 0.5389381521134301,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -55967744.0,
      "logits/rejected": -65431142.4,
      "logps/chosen": -264.6666666666667,
      "logps/rejected": -497.2,
      "loss": 0.1887,
      "rewards/chosen": 3.474604924519857,
      "rewards/margins": 14.824604924519857,
      "rewards/rejected": -11.35,
      "step": 883
    },
    {
      "epoch": 0.4354679802955665,
      "grad_norm": 0.48378079195613594,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -56623104.0,
      "logits/rejected": -57400699.68539326,
      "logps/chosen": -339.6923076923077,
      "logps/rejected": -451.59550561797755,
      "loss": 0.1539,
      "rewards/chosen": 7.701896471854968,
      "rewards/margins": 18.567065011180812,
      "rewards/rejected": -10.865168539325843,
      "step": 884
    },
    {
      "epoch": 0.43596059113300495,
      "grad_norm": 0.47595561010955045,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -49618616.32,
      "logits/rejected": -59741945.43589743,
      "logps/chosen": -296.64,
      "logps/rejected": -463.1794871794872,
      "loss": 0.1778,
      "rewards/chosen": 3.5937432861328125,
      "rewards/margins": 15.080922773312299,
      "rewards/rejected": -11.487179487179487,
      "step": 885
    },
    {
      "epoch": 0.43645320197044335,
      "grad_norm": 0.507879236307451,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -49941480.18604651,
      "logits/rejected": -71155133.74117647,
      "logps/chosen": -338.6046511627907,
      "logps/rejected": -563.2,
      "loss": 0.1903,
      "rewards/chosen": -0.5784883720930233,
      "rewards/margins": 12.75092339261286,
      "rewards/rejected": -13.329411764705883,
      "step": 886
    },
    {
      "epoch": 0.43694581280788175,
      "grad_norm": 0.4846533552141126,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -58765846.26086956,
      "logits/rejected": -77952674.34146342,
      "logps/chosen": -250.6086956521739,
      "logps/rejected": -481.9512195121951,
      "loss": 0.1865,
      "rewards/chosen": 0.06929347826086957,
      "rewards/margins": 12.642464209968185,
      "rewards/rejected": -12.573170731707316,
      "step": 887
    },
    {
      "epoch": 0.4374384236453202,
      "grad_norm": 0.47934929109216107,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -52679546.43478261,
      "logits/rejected": -58004155.317073174,
      "logps/chosen": -257.39130434782606,
      "logps/rejected": -478.8292682926829,
      "loss": 0.1858,
      "rewards/chosen": 0.348335597826087,
      "rewards/margins": 11.836140475874869,
      "rewards/rejected": -11.487804878048781,
      "step": 888
    },
    {
      "epoch": 0.4379310344827586,
      "grad_norm": 0.502353648742191,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -50733230.29787234,
      "logits/rejected": -75083219.75308642,
      "logps/chosen": -284.25531914893617,
      "logps/rejected": -593.3827160493827,
      "loss": 0.2272,
      "rewards/chosen": 0.9114165610455452,
      "rewards/margins": 10.379382899780113,
      "rewards/rejected": -9.467966338734568,
      "step": 889
    },
    {
      "epoch": 0.43842364532019706,
      "grad_norm": 0.4946573810660453,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -99684625.06666666,
      "logits/rejected": -80550850.31325302,
      "logps/chosen": -297.9555555555556,
      "logps/rejected": -591.4216867469879,
      "loss": 0.1783,
      "rewards/chosen": 9.953458658854167,
      "rewards/margins": 22.929362273311998,
      "rewards/rejected": -12.975903614457831,
      "step": 890
    },
    {
      "epoch": 0.43891625615763546,
      "grad_norm": 0.5697663398968825,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -64760053.76,
      "logits/rejected": -54095766.974358976,
      "logps/chosen": -269.92,
      "logps/rejected": -460.3076923076923,
      "loss": 0.2007,
      "rewards/chosen": 0.42,
      "rewards/margins": 11.368717948717949,
      "rewards/rejected": -10.948717948717949,
      "step": 891
    },
    {
      "epoch": 0.43940886699507387,
      "grad_norm": 0.43701766798995245,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -45698405.20930233,
      "logits/rejected": -55216778.54117647,
      "logps/chosen": -240.93023255813952,
      "logps/rejected": -443.4823529411765,
      "loss": 0.1775,
      "rewards/chosen": 3.1845671187999636,
      "rewards/margins": 14.57280241291761,
      "rewards/rejected": -11.388235294117647,
      "step": 892
    },
    {
      "epoch": 0.4399014778325123,
      "grad_norm": 0.507262317281793,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -51000951.82978723,
      "logits/rejected": -63380593.777777776,
      "logps/chosen": -269.9574468085106,
      "logps/rejected": -507.25925925925924,
      "loss": 0.2178,
      "rewards/chosen": 2.2336737450132977,
      "rewards/margins": 14.542315720321941,
      "rewards/rejected": -12.308641975308642,
      "step": 893
    },
    {
      "epoch": 0.4403940886699507,
      "grad_norm": 0.4407041823226262,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -53295014.95652174,
      "logits/rejected": -63068010.146341465,
      "logps/chosen": -228.69565217391303,
      "logps/rejected": -536.9756097560976,
      "loss": 0.1915,
      "rewards/chosen": 3.5557018777598506,
      "rewards/margins": 17.128872609467166,
      "rewards/rejected": -13.573170731707316,
      "step": 894
    },
    {
      "epoch": 0.4408866995073892,
      "grad_norm": 0.5529923369503469,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -69749722.07407407,
      "logits/rejected": -68242459.67567568,
      "logps/chosen": -357.9259259259259,
      "logps/rejected": -582.918918918919,
      "loss": 0.2112,
      "rewards/chosen": 1.32377850567853,
      "rewards/margins": 17.296751478651505,
      "rewards/rejected": -15.972972972972974,
      "step": 895
    },
    {
      "epoch": 0.4413793103448276,
      "grad_norm": 0.4288238014234191,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -55009910.15384615,
      "logits/rejected": -61865984.0,
      "logps/chosen": -400.3076923076923,
      "logps/rejected": -479.1578947368421,
      "loss": 0.2193,
      "rewards/chosen": -0.6430288461538461,
      "rewards/margins": 13.09381325910931,
      "rewards/rejected": -13.736842105263158,
      "step": 896
    },
    {
      "epoch": 0.44187192118226604,
      "grad_norm": 0.44241436810302065,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -54311956.89795918,
      "logits/rejected": -63392392.101265825,
      "logps/chosen": -329.7959183673469,
      "logps/rejected": -536.3037974683544,
      "loss": 0.1997,
      "rewards/chosen": 2.078412114357462,
      "rewards/margins": 13.85056401309164,
      "rewards/rejected": -11.772151898734178,
      "step": 897
    },
    {
      "epoch": 0.44236453201970444,
      "grad_norm": 0.48064493823832366,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -52832098.461538464,
      "logits/rejected": -63356065.684210524,
      "logps/chosen": -345.84615384615387,
      "logps/rejected": -516.6315789473684,
      "loss": 0.2384,
      "rewards/chosen": -1.66796875,
      "rewards/margins": 12.14782072368421,
      "rewards/rejected": -13.81578947368421,
      "step": 898
    },
    {
      "epoch": 0.44285714285714284,
      "grad_norm": 0.5303577895385799,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -62846909.93548387,
      "logits/rejected": -65710762.666666664,
      "logps/chosen": -370.06451612903226,
      "logps/rejected": -475.1515151515151,
      "loss": 0.248,
      "rewards/chosen": -0.9974798387096774,
      "rewards/margins": 10.85100500977517,
      "rewards/rejected": -11.848484848484848,
      "step": 899
    },
    {
      "epoch": 0.4433497536945813,
      "grad_norm": 0.5044665332627942,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -61708697.6,
      "logits/rejected": -59864157.09090909,
      "logps/chosen": -336.4,
      "logps/rejected": -570.1818181818181,
      "loss": 0.1662,
      "rewards/chosen": 6.355458068847656,
      "rewards/margins": 20.46909443248402,
      "rewards/rejected": -14.113636363636363,
      "step": 900
    },
    {
      "epoch": 0.4438423645320197,
      "grad_norm": 0.4010038252595966,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -69625446.4,
      "logits/rejected": -52482573.12820513,
      "logps/chosen": -289.28,
      "logps/rejected": -466.46153846153845,
      "loss": 0.1685,
      "rewards/chosen": 1.1840625,
      "rewards/margins": 14.82508814102564,
      "rewards/rejected": -13.64102564102564,
      "step": 901
    },
    {
      "epoch": 0.44433497536945815,
      "grad_norm": 0.45153007502228754,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -53040469.333333336,
      "logits/rejected": -83047219.2,
      "logps/chosen": -299.3333333333333,
      "logps/rejected": -566.4,
      "loss": 0.1904,
      "rewards/chosen": 1.0703125,
      "rewards/margins": 13.3453125,
      "rewards/rejected": -12.275,
      "step": 902
    },
    {
      "epoch": 0.44482758620689655,
      "grad_norm": 0.3795613680109965,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -35363739.60784314,
      "logits/rejected": -62369845.1948052,
      "logps/chosen": -225.72549019607843,
      "logps/rejected": -474.1818181818182,
      "loss": 0.1883,
      "rewards/chosen": 3.2163014131433822,
      "rewards/margins": 15.592924789766759,
      "rewards/rejected": -12.376623376623376,
      "step": 903
    },
    {
      "epoch": 0.44532019704433495,
      "grad_norm": 0.40118712686652813,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -56303972.17391305,
      "logits/rejected": -65472062.43902439,
      "logps/chosen": -271.6521739130435,
      "logps/rejected": -528.780487804878,
      "loss": 0.1954,
      "rewards/chosen": 6.384516176970108,
      "rewards/margins": 19.299150323311572,
      "rewards/rejected": -12.914634146341463,
      "step": 904
    },
    {
      "epoch": 0.4458128078817734,
      "grad_norm": 0.5417611185810587,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -51118080.0,
      "logits/rejected": -57933824.0,
      "logps/chosen": -244.25,
      "logps/rejected": -504.5,
      "loss": 0.215,
      "rewards/chosen": 0.501953125,
      "rewards/margins": 14.798828125,
      "rewards/rejected": -14.296875,
      "step": 905
    },
    {
      "epoch": 0.4463054187192118,
      "grad_norm": 0.419853362151523,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -42547987.692307696,
      "logits/rejected": -58775444.21052632,
      "logps/chosen": -292.61538461538464,
      "logps/rejected": -486.3157894736842,
      "loss": 0.2266,
      "rewards/chosen": -0.5994591346153846,
      "rewards/margins": 12.150540865384615,
      "rewards/rejected": -12.75,
      "step": 906
    },
    {
      "epoch": 0.44679802955665027,
      "grad_norm": 0.456517930818572,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -53632720.59259259,
      "logits/rejected": -71869965.83783785,
      "logps/chosen": -327.4074074074074,
      "logps/rejected": -539.6756756756756,
      "loss": 0.2365,
      "rewards/chosen": -1.4212962962962963,
      "rewards/margins": 11.930055055055055,
      "rewards/rejected": -13.35135135135135,
      "step": 907
    },
    {
      "epoch": 0.44729064039408867,
      "grad_norm": 0.4397472462836469,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -46323757.51111111,
      "logits/rejected": -59478262.746987954,
      "logps/chosen": -305.77777777777777,
      "logps/rejected": -520.4819277108434,
      "loss": 0.2014,
      "rewards/chosen": 3.2062547471788196,
      "rewards/margins": 18.194206554407735,
      "rewards/rejected": -14.987951807228916,
      "step": 908
    },
    {
      "epoch": 0.44778325123152707,
      "grad_norm": 0.5117704311468145,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -59097743.36,
      "logits/rejected": -69259789.12820514,
      "logps/chosen": -359.04,
      "logps/rejected": -544.0,
      "loss": 0.2331,
      "rewards/chosen": 2.210646057128906,
      "rewards/margins": 15.120902467385317,
      "rewards/rejected": -12.91025641025641,
      "step": 909
    },
    {
      "epoch": 0.4482758620689655,
      "grad_norm": 0.5404431319557834,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -51341387.85185185,
      "logits/rejected": -58833615.567567565,
      "logps/chosen": -333.037037037037,
      "logps/rejected": -460.97297297297297,
      "loss": 0.245,
      "rewards/chosen": -1.0462962962962963,
      "rewards/margins": 12.58883883883884,
      "rewards/rejected": -13.635135135135135,
      "step": 910
    },
    {
      "epoch": 0.4487684729064039,
      "grad_norm": 0.4024909969403619,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -61801785.469387755,
      "logits/rejected": -64825888.405063294,
      "logps/chosen": -247.83673469387756,
      "logps/rejected": -596.253164556962,
      "loss": 0.1878,
      "rewards/chosen": 0.3048469387755102,
      "rewards/margins": 15.520036812193231,
      "rewards/rejected": -15.215189873417721,
      "step": 911
    },
    {
      "epoch": 0.4492610837438424,
      "grad_norm": 1.292496648395119,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -64971382.15384615,
      "logits/rejected": -59879208.421052635,
      "logps/chosen": -355.38461538461536,
      "logps/rejected": -583.578947368421,
      "loss": 0.1609,
      "rewards/chosen": 0.9675480769230769,
      "rewards/margins": 16.29649544534413,
      "rewards/rejected": -15.328947368421053,
      "step": 912
    },
    {
      "epoch": 0.4497536945812808,
      "grad_norm": 0.49943436827993937,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -48758784.0,
      "logits/rejected": -56360960.0,
      "logps/chosen": -362.0,
      "logps/rejected": -505.2,
      "loss": 0.2306,
      "rewards/chosen": 1.1386864980061848,
      "rewards/margins": 15.126186498006186,
      "rewards/rejected": -13.9875,
      "step": 913
    },
    {
      "epoch": 0.45024630541871924,
      "grad_norm": 0.5606166984459698,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -63061728.561403506,
      "logits/rejected": -71893915.04225352,
      "logps/chosen": -393.5438596491228,
      "logps/rejected": -540.3943661971831,
      "loss": 0.1915,
      "rewards/chosen": 1.080592105263158,
      "rewards/margins": 14.193268161601186,
      "rewards/rejected": -13.112676056338028,
      "step": 914
    },
    {
      "epoch": 0.45073891625615764,
      "grad_norm": 0.4809246183737587,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -52790377.93103448,
      "logits/rejected": -93952409.6,
      "logps/chosen": -280.2758620689655,
      "logps/rejected": -597.0285714285715,
      "loss": 0.2198,
      "rewards/chosen": 1.2171336206896552,
      "rewards/margins": 13.388562192118226,
      "rewards/rejected": -12.17142857142857,
      "step": 915
    },
    {
      "epoch": 0.45123152709359604,
      "grad_norm": 0.4474490855390664,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -53141040.30188679,
      "logits/rejected": -59111724.373333335,
      "logps/chosen": -291.92452830188677,
      "logps/rejected": -483.84,
      "loss": 0.1811,
      "rewards/chosen": 0.7424823113207547,
      "rewards/margins": 12.609148977987422,
      "rewards/rejected": -11.866666666666667,
      "step": 916
    },
    {
      "epoch": 0.4517241379310345,
      "grad_norm": 0.41734649139970853,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -52297728.0,
      "logits/rejected": -71041024.0,
      "logps/chosen": -292.5,
      "logps/rejected": -510.8,
      "loss": 0.2174,
      "rewards/chosen": 3.8501771291097007,
      "rewards/margins": 16.7001771291097,
      "rewards/rejected": -12.85,
      "step": 917
    },
    {
      "epoch": 0.4522167487684729,
      "grad_norm": 0.5397047603289519,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -50160949.58139535,
      "logits/rejected": -60003219.57647059,
      "logps/chosen": -269.95348837209303,
      "logps/rejected": -513.5058823529412,
      "loss": 0.1779,
      "rewards/chosen": -0.06340843023255814,
      "rewards/margins": 12.124826863885088,
      "rewards/rejected": -12.188235294117646,
      "step": 918
    },
    {
      "epoch": 0.45270935960591135,
      "grad_norm": 0.4011865297027065,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -60770804.62222222,
      "logits/rejected": -57911715.469879515,
      "logps/chosen": -371.9111111111111,
      "logps/rejected": -475.3734939759036,
      "loss": 0.1987,
      "rewards/chosen": -0.265625,
      "rewards/margins": 10.553652108433734,
      "rewards/rejected": -10.819277108433734,
      "step": 919
    },
    {
      "epoch": 0.45320197044334976,
      "grad_norm": 0.4836130323902945,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -56784423.384615384,
      "logits/rejected": -76380483.36842105,
      "logps/chosen": -229.23076923076923,
      "logps/rejected": -490.5263157894737,
      "loss": 0.2128,
      "rewards/chosen": 0.3954326923076923,
      "rewards/margins": 10.13227479757085,
      "rewards/rejected": -9.736842105263158,
      "step": 920
    },
    {
      "epoch": 0.45369458128078816,
      "grad_norm": 0.4924246013883259,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -50452637.538461536,
      "logits/rejected": -58333938.526315786,
      "logps/chosen": -272.3076923076923,
      "logps/rejected": -501.89473684210526,
      "loss": 0.2186,
      "rewards/chosen": 0.39242788461538464,
      "rewards/margins": 11.497691042510121,
      "rewards/rejected": -11.105263157894736,
      "step": 921
    },
    {
      "epoch": 0.4541871921182266,
      "grad_norm": 0.5137012005421238,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -52685594.12244898,
      "logits/rejected": -63233114.73417722,
      "logps/chosen": -244.89795918367346,
      "logps/rejected": -478.7848101265823,
      "loss": 0.2108,
      "rewards/chosen": -0.475765306122449,
      "rewards/margins": 10.245753681219323,
      "rewards/rejected": -10.721518987341772,
      "step": 922
    },
    {
      "epoch": 0.454679802955665,
      "grad_norm": 0.5176985469836175,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -74522052.46511628,
      "logits/rejected": -64444247.34117647,
      "logps/chosen": -243.34883720930233,
      "logps/rejected": -482.63529411764705,
      "loss": 0.2039,
      "rewards/chosen": 2.4051671582599017,
      "rewards/margins": 12.79340245237755,
      "rewards/rejected": -10.388235294117647,
      "step": 923
    },
    {
      "epoch": 0.45517241379310347,
      "grad_norm": 0.49153204384501753,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -67720533.33333333,
      "logits/rejected": -66060288.0,
      "logps/chosen": -279.6666666666667,
      "logps/rejected": -475.6,
      "loss": 0.1906,
      "rewards/chosen": 0.5998942057291666,
      "rewards/margins": 11.274894205729167,
      "rewards/rejected": -10.675,
      "step": 924
    },
    {
      "epoch": 0.45566502463054187,
      "grad_norm": 0.6879881358021728,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -65842659.018867925,
      "logits/rejected": -72365725.01333334,
      "logps/chosen": -257.50943396226415,
      "logps/rejected": -512.4266666666666,
      "loss": 0.2131,
      "rewards/chosen": 2.725222605579304,
      "rewards/margins": 13.085222605579304,
      "rewards/rejected": -10.36,
      "step": 925
    },
    {
      "epoch": 0.45615763546798027,
      "grad_norm": 0.5861568277492721,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -55775319.14893617,
      "logits/rejected": -54267044.345679015,
      "logps/chosen": -282.5531914893617,
      "logps/rejected": -510.0246913580247,
      "loss": 0.1798,
      "rewards/chosen": 0.456781914893617,
      "rewards/margins": 11.604930063041765,
      "rewards/rejected": -11.148148148148149,
      "step": 926
    },
    {
      "epoch": 0.4566502463054187,
      "grad_norm": 0.4176144271898173,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -56417500.8627451,
      "logits/rejected": -57685297.87012987,
      "logps/chosen": -326.9019607843137,
      "logps/rejected": -441.76623376623377,
      "loss": 0.2219,
      "rewards/chosen": -0.30051317401960786,
      "rewards/margins": 9.374811501305068,
      "rewards/rejected": -9.675324675324676,
      "step": 927
    },
    {
      "epoch": 0.45714285714285713,
      "grad_norm": 0.43537379317427083,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -72806786.41509435,
      "logits/rejected": -66102231.04,
      "logps/chosen": -403.92452830188677,
      "logps/rejected": -542.72,
      "loss": 0.1944,
      "rewards/chosen": 3.040702387971698,
      "rewards/margins": 14.800702387971697,
      "rewards/rejected": -11.76,
      "step": 928
    },
    {
      "epoch": 0.4576354679802956,
      "grad_norm": 0.5698939122647655,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -56157070.222222224,
      "logits/rejected": -57416620.972972974,
      "logps/chosen": -299.55555555555554,
      "logps/rejected": -493.4054054054054,
      "loss": 0.2372,
      "rewards/chosen": 1.425351036919488,
      "rewards/margins": 13.101026712595164,
      "rewards/rejected": -11.675675675675675,
      "step": 929
    },
    {
      "epoch": 0.458128078817734,
      "grad_norm": 0.5966004882293177,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -52428800.0,
      "logits/rejected": -83320555.86516854,
      "logps/chosen": -250.66666666666666,
      "logps/rejected": -548.6741573033707,
      "loss": 0.1608,
      "rewards/chosen": 1.294871794871795,
      "rewards/margins": 14.778017862287525,
      "rewards/rejected": -13.48314606741573,
      "step": 930
    },
    {
      "epoch": 0.4586206896551724,
      "grad_norm": 0.5569604754530777,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -56425259.47169811,
      "logits/rejected": -62019775.14666667,
      "logps/chosen": -314.2641509433962,
      "logps/rejected": -535.8933333333333,
      "loss": 0.2293,
      "rewards/chosen": 0.6615566037735849,
      "rewards/margins": 14.834889937106919,
      "rewards/rejected": -14.173333333333334,
      "step": 931
    },
    {
      "epoch": 0.45911330049261084,
      "grad_norm": 0.6178464800619976,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -62839661.71428572,
      "logits/rejected": -63089322.666666664,
      "logps/chosen": -268.57142857142856,
      "logps/rejected": -541.3333333333334,
      "loss": 0.2393,
      "rewards/chosen": 1.5888631003243583,
      "rewards/margins": 16.422196433657692,
      "rewards/rejected": -14.833333333333334,
      "step": 932
    },
    {
      "epoch": 0.45960591133004924,
      "grad_norm": 0.5294601310609366,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -55487146.666666664,
      "logits/rejected": -70149734.4,
      "logps/chosen": -350.8333333333333,
      "logps/rejected": -538.4,
      "loss": 0.201,
      "rewards/chosen": 3.112663904825846,
      "rewards/margins": 17.175163904825848,
      "rewards/rejected": -14.0625,
      "step": 933
    },
    {
      "epoch": 0.4600985221674877,
      "grad_norm": 0.5654349109004201,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -32392496.43243243,
      "logits/rejected": -72086719.2967033,
      "logps/chosen": -221.6216216216216,
      "logps/rejected": -579.5164835164835,
      "loss": 0.1801,
      "rewards/chosen": 7.217036066828547,
      "rewards/margins": 22.096156945949428,
      "rewards/rejected": -14.87912087912088,
      "step": 934
    },
    {
      "epoch": 0.4605911330049261,
      "grad_norm": 0.4816021779601834,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -50113194.666666664,
      "logits/rejected": -68576870.4,
      "logps/chosen": -313.6666666666667,
      "logps/rejected": -579.2,
      "loss": 0.2007,
      "rewards/chosen": -1.3450520833333333,
      "rewards/margins": 15.829947916666667,
      "rewards/rejected": -17.175,
      "step": 935
    },
    {
      "epoch": 0.46108374384236456,
      "grad_norm": 0.5979389104342263,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -50613956.92307692,
      "logits/rejected": -66998487.578947365,
      "logps/chosen": -247.84615384615384,
      "logps/rejected": -523.7894736842105,
      "loss": 0.2152,
      "rewards/chosen": 2.048677004300631,
      "rewards/margins": 16.509203320090105,
      "rewards/rejected": -14.460526315789474,
      "step": 936
    },
    {
      "epoch": 0.46157635467980296,
      "grad_norm": 0.48708982899222236,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -68992020.89795919,
      "logits/rejected": -70294411.34177215,
      "logps/chosen": -394.1224489795918,
      "logps/rejected": -602.7341772151899,
      "loss": 0.2192,
      "rewards/chosen": 0.5758888088926977,
      "rewards/margins": 16.056901467120547,
      "rewards/rejected": -15.481012658227849,
      "step": 937
    },
    {
      "epoch": 0.46206896551724136,
      "grad_norm": 0.5853974819373513,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -61887383.510204084,
      "logits/rejected": -63710946.835443035,
      "logps/chosen": -434.2857142857143,
      "logps/rejected": -603.5443037974684,
      "loss": 0.2002,
      "rewards/chosen": -0.046875,
      "rewards/margins": 16.877175632911392,
      "rewards/rejected": -16.924050632911392,
      "step": 938
    },
    {
      "epoch": 0.4625615763546798,
      "grad_norm": 0.6411798355965045,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -49838200.47058824,
      "logits/rejected": -70159266.9090909,
      "logps/chosen": -280.7843137254902,
      "logps/rejected": -580.1558441558442,
      "loss": 0.2392,
      "rewards/chosen": 0.7316176470588235,
      "rewards/margins": 14.329020244461422,
      "rewards/rejected": -13.597402597402597,
      "step": 939
    },
    {
      "epoch": 0.4630541871921182,
      "grad_norm": 0.4453424977733073,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -66395832.32,
      "logits/rejected": -59257987.28205128,
      "logps/chosen": -299.2,
      "logps/rejected": -520.2051282051282,
      "loss": 0.2176,
      "rewards/chosen": 2.015946044921875,
      "rewards/margins": 16.259535788511617,
      "rewards/rejected": -14.243589743589743,
      "step": 940
    },
    {
      "epoch": 0.4635467980295567,
      "grad_norm": 0.5064465526486419,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -44082135.04,
      "logits/rejected": -63990022.56410257,
      "logps/chosen": -242.24,
      "logps/rejected": -564.5128205128206,
      "loss": 0.2166,
      "rewards/chosen": 0.5774813842773437,
      "rewards/margins": 16.359532666328626,
      "rewards/rejected": -15.782051282051283,
      "step": 941
    },
    {
      "epoch": 0.4640394088669951,
      "grad_norm": 0.5489939489356387,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -55552217.872340426,
      "logits/rejected": -65141165.82716049,
      "logps/chosen": -296.8510638297872,
      "logps/rejected": -580.7407407407408,
      "loss": 0.2024,
      "rewards/chosen": 0.12632978723404256,
      "rewards/margins": 13.755959416863673,
      "rewards/rejected": -13.62962962962963,
      "step": 942
    },
    {
      "epoch": 0.4645320197044335,
      "grad_norm": 0.40669959974435066,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -71392408.5106383,
      "logits/rejected": -71613857.18518518,
      "logps/chosen": -411.2340425531915,
      "logps/rejected": -475.65432098765433,
      "loss": 0.1539,
      "rewards/chosen": 6.666933749584442,
      "rewards/margins": 20.72866214464617,
      "rewards/rejected": -14.061728395061728,
      "step": 943
    },
    {
      "epoch": 0.46502463054187193,
      "grad_norm": 0.41847092485668036,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -42502280.53333333,
      "logits/rejected": -63672566.746987954,
      "logps/chosen": -198.4,
      "logps/rejected": -524.3373493975904,
      "loss": 0.1951,
      "rewards/chosen": 0.13958333333333334,
      "rewards/margins": 14.332354417670683,
      "rewards/rejected": -14.19277108433735,
      "step": 944
    },
    {
      "epoch": 0.46551724137931033,
      "grad_norm": 0.4350738733873682,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -53171541.333333336,
      "logits/rejected": -72299315.2,
      "logps/chosen": -324.8333333333333,
      "logps/rejected": -494.4,
      "loss": 0.1766,
      "rewards/chosen": -0.018229166666666668,
      "rewards/margins": 12.681770833333333,
      "rewards/rejected": -12.7,
      "step": 945
    },
    {
      "epoch": 0.4660098522167488,
      "grad_norm": 0.48709789065111714,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -52615213.51111111,
      "logits/rejected": -60539472.192771085,
      "logps/chosen": -316.44444444444446,
      "logps/rejected": -528.1927710843373,
      "loss": 0.2122,
      "rewards/chosen": 0.19583333333333333,
      "rewards/margins": 14.484989959839357,
      "rewards/rejected": -14.289156626506024,
      "step": 946
    },
    {
      "epoch": 0.4665024630541872,
      "grad_norm": 0.4457332618536397,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -58468597.76,
      "logits/rejected": -52590119.384615384,
      "logps/chosen": -311.68,
      "logps/rejected": -539.0769230769231,
      "loss": 0.1951,
      "rewards/chosen": 4.512518310546875,
      "rewards/margins": 19.768928566957133,
      "rewards/rejected": -15.256410256410257,
      "step": 947
    },
    {
      "epoch": 0.4669950738916256,
      "grad_norm": 0.5333098398251631,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -67807914.66666667,
      "logits/rejected": -66882144.86486486,
      "logps/chosen": -352.5925925925926,
      "logps/rejected": -561.2972972972973,
      "loss": 0.2308,
      "rewards/chosen": 2.2118745026765048,
      "rewards/margins": 12.9139379858374,
      "rewards/rejected": -10.702063483160895,
      "step": 948
    },
    {
      "epoch": 0.46748768472906405,
      "grad_norm": 0.4850443026895136,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -49620749.016949154,
      "logits/rejected": -67048076.98550725,
      "logps/chosen": -291.52542372881356,
      "logps/rejected": -487.8840579710145,
      "loss": 0.243,
      "rewards/chosen": -0.9721927966101694,
      "rewards/margins": 8.895997802914287,
      "rewards/rejected": -9.868190599524457,
      "step": 949
    },
    {
      "epoch": 0.46798029556650245,
      "grad_norm": 0.5510607147080441,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -62681543.11111111,
      "logits/rejected": -64331554.5945946,
      "logps/chosen": -298.962962962963,
      "logps/rejected": -531.027027027027,
      "loss": 0.2299,
      "rewards/chosen": -0.3680555555555556,
      "rewards/margins": 9.806340856237096,
      "rewards/rejected": -10.174396411792651,
      "step": 950
    },
    {
      "epoch": 0.4684729064039409,
      "grad_norm": 0.41345217537743156,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -48282158.54545455,
      "logits/rejected": -74898285.71428572,
      "logps/chosen": -259.27272727272725,
      "logps/rejected": -552.3809523809524,
      "loss": 0.1695,
      "rewards/chosen": 0.4463778409090909,
      "rewards/margins": 14.87494926948052,
      "rewards/rejected": -14.428571428571429,
      "step": 951
    },
    {
      "epoch": 0.4689655172413793,
      "grad_norm": 0.443078508113594,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -73400320.0,
      "logits/rejected": -62180556.8,
      "logps/chosen": -417.3333333333333,
      "logps/rejected": -529.6,
      "loss": 0.1905,
      "rewards/chosen": 2.6172107060750327,
      "rewards/margins": 15.617210706075033,
      "rewards/rejected": -13.0,
      "step": 952
    },
    {
      "epoch": 0.46945812807881776,
      "grad_norm": 0.42181899766575265,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -75163834.18181819,
      "logits/rejected": -61216865.52380952,
      "logps/chosen": -281.6363636363636,
      "logps/rejected": -513.5238095238095,
      "loss": 0.1635,
      "rewards/chosen": 2.402357968417081,
      "rewards/margins": 17.14045320651232,
      "rewards/rejected": -14.738095238095237,
      "step": 953
    },
    {
      "epoch": 0.46995073891625616,
      "grad_norm": 0.44085973441976983,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -54459021.61702128,
      "logits/rejected": -60118357.333333336,
      "logps/chosen": -268.25531914893617,
      "logps/rejected": -511.60493827160496,
      "loss": 0.1598,
      "rewards/chosen": 4.181521801238365,
      "rewards/margins": 12.295606304035433,
      "rewards/rejected": -8.114084502797068,
      "step": 954
    },
    {
      "epoch": 0.47044334975369456,
      "grad_norm": 0.5244917352095784,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -74982382.03508772,
      "logits/rejected": -67640536.33802816,
      "logps/chosen": -350.87719298245617,
      "logps/rejected": -530.9295774647887,
      "loss": 0.1945,
      "rewards/chosen": 0.4906798245614035,
      "rewards/margins": 13.025891092167038,
      "rewards/rejected": -12.535211267605634,
      "step": 955
    },
    {
      "epoch": 0.470935960591133,
      "grad_norm": 0.4231441725652608,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -46507429.64705882,
      "logits/rejected": -57303997.50649351,
      "logps/chosen": -246.58823529411765,
      "logps/rejected": -503.27272727272725,
      "loss": 0.2045,
      "rewards/chosen": -0.8366268382352942,
      "rewards/margins": 12.358178356569901,
      "rewards/rejected": -13.194805194805195,
      "step": 956
    },
    {
      "epoch": 0.4714285714285714,
      "grad_norm": 0.6935691276722671,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -57470888.85106383,
      "logits/rejected": -68662309.92592593,
      "logps/chosen": -280.51063829787233,
      "logps/rejected": -520.6913580246913,
      "loss": 0.1486,
      "rewards/chosen": 1.4534574468085106,
      "rewards/margins": 15.330000656685053,
      "rewards/rejected": -13.876543209876543,
      "step": 957
    },
    {
      "epoch": 0.4719211822660099,
      "grad_norm": 0.44440172145488105,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -45881593.75609756,
      "logits/rejected": -66723180.873563215,
      "logps/chosen": -318.8292682926829,
      "logps/rejected": -559.080459770115,
      "loss": 0.1713,
      "rewards/chosen": 6.3484065358231705,
      "rewards/margins": 19.463349064558802,
      "rewards/rejected": -13.114942528735632,
      "step": 958
    },
    {
      "epoch": 0.4724137931034483,
      "grad_norm": 0.40895934904949494,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -53615867.16981132,
      "logits/rejected": -58832104.10666667,
      "logps/chosen": -359.39622641509436,
      "logps/rejected": -563.2,
      "loss": 0.2126,
      "rewards/chosen": 5.597924574366155,
      "rewards/margins": 18.717924574366155,
      "rewards/rejected": -13.12,
      "step": 959
    },
    {
      "epoch": 0.4729064039408867,
      "grad_norm": 0.3896074969666703,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -62646838.4680851,
      "logits/rejected": -72494143.20987654,
      "logps/chosen": -268.4255319148936,
      "logps/rejected": -596.5432098765432,
      "loss": 0.1574,
      "rewards/chosen": 3.14100030127992,
      "rewards/margins": 19.955815116094733,
      "rewards/rejected": -16.814814814814813,
      "step": 960
    },
    {
      "epoch": 0.47339901477832513,
      "grad_norm": 0.40758325168736337,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -55624460.190476194,
      "logits/rejected": -64328918.325581394,
      "logps/chosen": -331.6190476190476,
      "logps/rejected": -518.6976744186046,
      "loss": 0.1708,
      "rewards/chosen": -0.39694940476190477,
      "rewards/margins": 12.47514361849391,
      "rewards/rejected": -12.872093023255815,
      "step": 961
    },
    {
      "epoch": 0.47389162561576353,
      "grad_norm": 0.40263274499320895,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -56245616.64,
      "logits/rejected": -52912758.15384615,
      "logps/chosen": -299.84,
      "logps/rejected": -493.94871794871796,
      "loss": 0.1616,
      "rewards/chosen": 6.166275634765625,
      "rewards/margins": 18.204737173227162,
      "rewards/rejected": -12.038461538461538,
      "step": 962
    },
    {
      "epoch": 0.474384236453202,
      "grad_norm": 0.5202136114282185,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -52607281.02127659,
      "logits/rejected": -65037602.7654321,
      "logps/chosen": -268.59574468085106,
      "logps/rejected": -529.3827160493827,
      "loss": 0.1947,
      "rewards/chosen": 0.11170212765957446,
      "rewards/margins": 13.222813238770685,
      "rewards/rejected": -13.11111111111111,
      "step": 963
    },
    {
      "epoch": 0.4748768472906404,
      "grad_norm": 0.7078735918963831,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -66786225.23076923,
      "logits/rejected": -62804183.578947365,
      "logps/chosen": -369.84615384615387,
      "logps/rejected": -526.3157894736842,
      "loss": 0.197,
      "rewards/chosen": 1.3317307692307692,
      "rewards/margins": 16.33173076923077,
      "rewards/rejected": -15.0,
      "step": 964
    },
    {
      "epoch": 0.4753694581280788,
      "grad_norm": 0.36616829815663654,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -60626757.81818182,
      "logits/rejected": -61516458.666666664,
      "logps/chosen": -277.8181818181818,
      "logps/rejected": -504.0,
      "loss": 0.1883,
      "rewards/chosen": -1.4261363636363635,
      "rewards/margins": 12.323863636363637,
      "rewards/rejected": -13.75,
      "step": 965
    },
    {
      "epoch": 0.47586206896551725,
      "grad_norm": 0.47694689818374186,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -52815117.473684214,
      "logits/rejected": -59838737.06666667,
      "logps/chosen": -265.2631578947368,
      "logps/rejected": -499.9111111111111,
      "loss": 0.2153,
      "rewards/chosen": -0.22430098684210525,
      "rewards/margins": 11.43125456871345,
      "rewards/rejected": -11.655555555555555,
      "step": 966
    },
    {
      "epoch": 0.47635467980295565,
      "grad_norm": 0.4550882197746433,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -51686058.666666664,
      "logits/rejected": -61603840.0,
      "logps/chosen": -271.3333333333333,
      "logps/rejected": -509.6,
      "loss": 0.1743,
      "rewards/chosen": 0.9231770833333334,
      "rewards/margins": 13.873177083333333,
      "rewards/rejected": -12.95,
      "step": 967
    },
    {
      "epoch": 0.4768472906403941,
      "grad_norm": 0.6075212621405741,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -63495021.71428572,
      "logits/rejected": -59128035.55555555,
      "logps/chosen": -261.42857142857144,
      "logps/rejected": -547.5555555555555,
      "loss": 0.2199,
      "rewards/chosen": 2.1779986790248325,
      "rewards/margins": 15.69188756791372,
      "rewards/rejected": -13.51388888888889,
      "step": 968
    },
    {
      "epoch": 0.4773399014778325,
      "grad_norm": 0.41395935187878863,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -47613117.62962963,
      "logits/rejected": -53902474.37837838,
      "logps/chosen": -336.0,
      "logps/rejected": -500.7567567567568,
      "loss": 0.1732,
      "rewards/chosen": 4.066010934335214,
      "rewards/margins": 17.647092015416295,
      "rewards/rejected": -13.58108108108108,
      "step": 969
    },
    {
      "epoch": 0.47783251231527096,
      "grad_norm": 0.4943451920218669,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -48584021.333333336,
      "logits/rejected": -55073035.13043478,
      "logps/chosen": -268.1111111111111,
      "logps/rejected": -538.4347826086956,
      "loss": 0.156,
      "rewards/chosen": 3.229994879828559,
      "rewards/margins": 17.947386184176384,
      "rewards/rejected": -14.717391304347826,
      "step": 970
    },
    {
      "epoch": 0.47832512315270936,
      "grad_norm": 0.47248756022818833,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -44282171.07692308,
      "logits/rejected": -71634297.26315789,
      "logps/chosen": -282.46153846153845,
      "logps/rejected": -557.4736842105264,
      "loss": 0.2401,
      "rewards/chosen": -0.4519230769230769,
      "rewards/margins": 13.15334008097166,
      "rewards/rejected": -13.605263157894736,
      "step": 971
    },
    {
      "epoch": 0.47881773399014776,
      "grad_norm": 0.429957387473118,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -65206795.906976745,
      "logits/rejected": -66467382.21176471,
      "logps/chosen": -325.5813953488372,
      "logps/rejected": -570.7294117647059,
      "loss": 0.1622,
      "rewards/chosen": 6.3873106490734015,
      "rewards/margins": 21.328487119661638,
      "rewards/rejected": -14.941176470588236,
      "step": 972
    },
    {
      "epoch": 0.4793103448275862,
      "grad_norm": 0.41290441195958555,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -56972629.333333336,
      "logits/rejected": -65536000.0,
      "logps/chosen": -382.0,
      "logps/rejected": -549.6,
      "loss": 0.1626,
      "rewards/chosen": 1.67578125,
      "rewards/margins": 16.82578125,
      "rewards/rejected": -15.15,
      "step": 973
    },
    {
      "epoch": 0.4798029556650246,
      "grad_norm": 0.42838785106478133,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -71684468.36363636,
      "logits/rejected": -59069781.333333336,
      "logps/chosen": -356.72727272727275,
      "logps/rejected": -537.9047619047619,
      "loss": 0.166,
      "rewards/chosen": 0.75,
      "rewards/margins": 13.25,
      "rewards/rejected": -12.5,
      "step": 974
    },
    {
      "epoch": 0.4802955665024631,
      "grad_norm": 0.4260208805339078,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -47609812.42553192,
      "logits/rejected": -66332141.03703704,
      "logps/chosen": -284.25531914893617,
      "logps/rejected": -530.1728395061729,
      "loss": 0.1685,
      "rewards/chosen": -0.7074468085106383,
      "rewards/margins": 11.687614919884423,
      "rewards/rejected": -12.395061728395062,
      "step": 975
    },
    {
      "epoch": 0.4807881773399015,
      "grad_norm": 0.5744662778537948,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -69164895.37254901,
      "logits/rejected": -66182848.83116883,
      "logps/chosen": -363.29411764705884,
      "logps/rejected": -610.9090909090909,
      "loss": 0.1987,
      "rewards/chosen": 4.677102481617647,
      "rewards/margins": 20.20957001408518,
      "rewards/rejected": -15.532467532467532,
      "step": 976
    },
    {
      "epoch": 0.4812807881773399,
      "grad_norm": 0.42862734180344136,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -46650932.24489796,
      "logits/rejected": -55428523.74683544,
      "logps/chosen": -285.55102040816325,
      "logps/rejected": -484.0506329113924,
      "loss": 0.186,
      "rewards/chosen": 0.005341198979591837,
      "rewards/margins": 13.815467781258073,
      "rewards/rejected": -13.810126582278482,
      "step": 977
    },
    {
      "epoch": 0.48177339901477834,
      "grad_norm": 0.6020115344904227,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -48908580.571428575,
      "logits/rejected": -56040561.777777776,
      "logps/chosen": -260.0,
      "logps/rejected": -552.0,
      "loss": 0.2338,
      "rewards/chosen": -0.34151785714285715,
      "rewards/margins": 14.991815476190476,
      "rewards/rejected": -15.333333333333334,
      "step": 978
    },
    {
      "epoch": 0.48226600985221674,
      "grad_norm": 0.417605327788783,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -42028638.04081633,
      "logits/rejected": -56437280.405063294,
      "logps/chosen": -392.16326530612247,
      "logps/rejected": -512.8101265822785,
      "loss": 0.1712,
      "rewards/chosen": -0.22959183673469388,
      "rewards/margins": 14.175471454404548,
      "rewards/rejected": -14.405063291139241,
      "step": 979
    },
    {
      "epoch": 0.4827586206896552,
      "grad_norm": 0.54954306075501,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -50401553.06666667,
      "logits/rejected": -66923821.176470585,
      "logps/chosen": -363.73333333333335,
      "logps/rejected": -501.1764705882353,
      "loss": 0.2238,
      "rewards/chosen": -0.11770833333333333,
      "rewards/margins": 15.852879901960785,
      "rewards/rejected": -15.970588235294118,
      "step": 980
    },
    {
      "epoch": 0.4832512315270936,
      "grad_norm": 0.5226954238216002,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -46530560.0,
      "logits/rejected": -57514393.6,
      "logps/chosen": -280.0,
      "logps/rejected": -502.4,
      "loss": 0.2057,
      "rewards/chosen": 2.4977219899495444,
      "rewards/margins": 15.160221989949545,
      "rewards/rejected": -12.6625,
      "step": 981
    },
    {
      "epoch": 0.483743842364532,
      "grad_norm": 0.4754715883214664,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -63486510.54545455,
      "logits/rejected": -61915916.190476194,
      "logps/chosen": -348.3636363636364,
      "logps/rejected": -537.1428571428571,
      "loss": 0.1486,
      "rewards/chosen": 2.34375,
      "rewards/margins": 18.772321428571427,
      "rewards/rejected": -16.428571428571427,
      "step": 982
    },
    {
      "epoch": 0.48423645320197045,
      "grad_norm": 0.38863020512562324,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -52557197.06122449,
      "logits/rejected": -65516090.32911392,
      "logps/chosen": -298.7755102040816,
      "logps/rejected": -587.3417721518987,
      "loss": 0.1933,
      "rewards/chosen": -1.4885204081632653,
      "rewards/margins": 14.929201110824078,
      "rewards/rejected": -16.417721518987342,
      "step": 983
    },
    {
      "epoch": 0.48472906403940885,
      "grad_norm": 0.5498064024288197,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -65649975.652173914,
      "logits/rejected": -73911820.48780487,
      "logps/chosen": -340.8695652173913,
      "logps/rejected": -585.3658536585366,
      "loss": 0.1543,
      "rewards/chosen": 1.6878396739130435,
      "rewards/margins": 14.931742112937433,
      "rewards/rejected": -13.24390243902439,
      "step": 984
    },
    {
      "epoch": 0.4852216748768473,
      "grad_norm": 0.5938783456230349,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -67722664.58536585,
      "logits/rejected": -85910916.4137931,
      "logps/chosen": -350.4390243902439,
      "logps/rejected": -545.8390804597701,
      "loss": 0.1545,
      "rewards/chosen": 3.504557167611471,
      "rewards/margins": 18.987315788301125,
      "rewards/rejected": -15.482758620689655,
      "step": 985
    },
    {
      "epoch": 0.4857142857142857,
      "grad_norm": 0.3440731978870511,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -74067595.63636364,
      "logits/rejected": -66959067.428571425,
      "logps/chosen": -432.3636363636364,
      "logps/rejected": -553.1428571428571,
      "loss": 0.1659,
      "rewards/chosen": 2.795464949174361,
      "rewards/margins": 17.41451256822198,
      "rewards/rejected": -14.619047619047619,
      "step": 986
    },
    {
      "epoch": 0.4862068965517241,
      "grad_norm": 0.6362860091897966,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -50855936.0,
      "logits/rejected": -54892953.6,
      "logps/chosen": -256.3333333333333,
      "logps/rejected": -454.4,
      "loss": 0.1705,
      "rewards/chosen": 1.34765625,
      "rewards/margins": 13.91015625,
      "rewards/rejected": -12.5625,
      "step": 987
    },
    {
      "epoch": 0.48669950738916257,
      "grad_norm": 0.5171540301062826,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -38005302.4680851,
      "logits/rejected": -67782023.90123457,
      "logps/chosen": -258.72340425531917,
      "logps/rejected": -515.1604938271605,
      "loss": 0.1939,
      "rewards/chosen": 5.202674703395113,
      "rewards/margins": 18.32613149351857,
      "rewards/rejected": -13.123456790123457,
      "step": 988
    },
    {
      "epoch": 0.48719211822660097,
      "grad_norm": 0.37182814804449726,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -53990508.93617021,
      "logits/rejected": -61827147.85185185,
      "logps/chosen": -241.70212765957447,
      "logps/rejected": -538.074074074074,
      "loss": 0.1456,
      "rewards/chosen": 0.7925531914893617,
      "rewards/margins": 15.780207512477016,
      "rewards/rejected": -14.987654320987655,
      "step": 989
    },
    {
      "epoch": 0.4876847290640394,
      "grad_norm": 0.5526921514311751,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -55458019.55555555,
      "logits/rejected": -67044336.24615385,
      "logps/chosen": -265.14285714285717,
      "logps/rejected": -627.2,
      "loss": 0.2241,
      "rewards/chosen": -0.05109126984126984,
      "rewards/margins": 15.656601037851036,
      "rewards/rejected": -15.707692307692307,
      "step": 990
    },
    {
      "epoch": 0.4881773399014778,
      "grad_norm": 0.44090943023743034,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -65011712.0,
      "logits/rejected": -60001848.88888889,
      "logps/chosen": -372.2857142857143,
      "logps/rejected": -633.7777777777778,
      "loss": 0.1994,
      "rewards/chosen": 2.349931444440569,
      "rewards/margins": 13.988820333329459,
      "rewards/rejected": -11.63888888888889,
      "step": 991
    },
    {
      "epoch": 0.4886699507389163,
      "grad_norm": 0.48106862026255154,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -60817408.0,
      "logits/rejected": -61609665.42222222,
      "logps/chosen": -273.2631578947368,
      "logps/rejected": -552.5333333333333,
      "loss": 0.1686,
      "rewards/chosen": 4.329733597604852,
      "rewards/margins": 18.48528915316041,
      "rewards/rejected": -14.155555555555555,
      "step": 992
    },
    {
      "epoch": 0.4891625615763547,
      "grad_norm": 0.419289425487524,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -54364632.615384616,
      "logits/rejected": -55436557.473684214,
      "logps/chosen": -317.38461538461536,
      "logps/rejected": -593.6842105263158,
      "loss": 0.2011,
      "rewards/chosen": 3.466260763315054,
      "rewards/margins": 15.137313394894,
      "rewards/rejected": -11.671052631578947,
      "step": 993
    },
    {
      "epoch": 0.4896551724137931,
      "grad_norm": 0.49526578098260615,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -50720009.481481485,
      "logits/rejected": -53732435.027027026,
      "logps/chosen": -258.962962962963,
      "logps/rejected": -458.3783783783784,
      "loss": 0.2258,
      "rewards/chosen": 2.1105389065212674,
      "rewards/margins": 15.759187555169916,
      "rewards/rejected": -13.64864864864865,
      "step": 994
    },
    {
      "epoch": 0.49014778325123154,
      "grad_norm": 0.4435540106179835,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -60467882.666666664,
      "logits/rejected": -55941529.6,
      "logps/chosen": -249.0,
      "logps/rejected": -516.0,
      "loss": 0.196,
      "rewards/chosen": 4.728901545206706,
      "rewards/margins": 20.253901545206705,
      "rewards/rejected": -15.525,
      "step": 995
    },
    {
      "epoch": 0.49064039408866994,
      "grad_norm": 0.4821271760238665,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -49390069.551020406,
      "logits/rejected": -67692881.01265822,
      "logps/chosen": -290.61224489795916,
      "logps/rejected": -533.0632911392405,
      "loss": 0.1778,
      "rewards/chosen": 1.861614616549745,
      "rewards/margins": 16.671741198828226,
      "rewards/rejected": -14.810126582278482,
      "step": 996
    },
    {
      "epoch": 0.4911330049261084,
      "grad_norm": 0.4441664892034326,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -65734867.862068966,
      "logits/rejected": -68247317.94285715,
      "logps/chosen": -430.3448275862069,
      "logps/rejected": -552.6857142857143,
      "loss": 0.2176,
      "rewards/chosen": -0.2349137931034483,
      "rewards/margins": 16.679371921182266,
      "rewards/rejected": -16.914285714285715,
      "step": 997
    },
    {
      "epoch": 0.4916256157635468,
      "grad_norm": 0.5825103589308603,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -52611161.04347826,
      "logits/rejected": -58976006.24390244,
      "logps/chosen": -317.2173913043478,
      "logps/rejected": -544.0,
      "loss": 0.2007,
      "rewards/chosen": 2.7785120425016983,
      "rewards/margins": 16.376073018111455,
      "rewards/rejected": -13.597560975609756,
      "step": 998
    },
    {
      "epoch": 0.4921182266009852,
      "grad_norm": 0.4971808848595372,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -59594069.333333336,
      "logits/rejected": -61341696.0,
      "logps/chosen": -364.3333333333333,
      "logps/rejected": -569.6,
      "loss": 0.1939,
      "rewards/chosen": 4.6302235921223955,
      "rewards/margins": 18.080223592122394,
      "rewards/rejected": -13.45,
      "step": 999
    },
    {
      "epoch": 0.49261083743842365,
      "grad_norm": 0.4719768845123845,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -50624273.86046512,
      "logits/rejected": -66763450.729411766,
      "logps/chosen": -314.04651162790697,
      "logps/rejected": -533.8352941176471,
      "loss": 0.188,
      "rewards/chosen": 0.24563953488372092,
      "rewards/margins": 13.269168946648426,
      "rewards/rejected": -13.023529411764706,
      "step": 1000
    },
    {
      "epoch": 0.49310344827586206,
      "grad_norm": 0.5777977534934168,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -47064930.461538464,
      "logits/rejected": -65398029.473684214,
      "logps/chosen": -283.38461538461536,
      "logps/rejected": -570.9473684210526,
      "loss": 0.2191,
      "rewards/chosen": 5.594274080716646,
      "rewards/margins": 20.80480039650612,
      "rewards/rejected": -15.210526315789474,
      "step": 1001
    },
    {
      "epoch": 0.4935960591133005,
      "grad_norm": 0.4783721675597589,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -44142492.09756097,
      "logits/rejected": -67301705.56321838,
      "logps/chosen": -236.29268292682926,
      "logps/rejected": -578.9425287356322,
      "loss": 0.1503,
      "rewards/chosen": 3.7797159334508383,
      "rewards/margins": 20.14753202540486,
      "rewards/rejected": -16.367816091954023,
      "step": 1002
    },
    {
      "epoch": 0.4940886699507389,
      "grad_norm": 0.5068800553626509,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -66472228.571428575,
      "logits/rejected": -61982492.44444445,
      "logps/chosen": -269.57142857142856,
      "logps/rejected": -545.7777777777778,
      "loss": 0.2268,
      "rewards/chosen": -0.7354910714285714,
      "rewards/margins": 14.23673115079365,
      "rewards/rejected": -14.972222222222221,
      "step": 1003
    },
    {
      "epoch": 0.4945812807881773,
      "grad_norm": 0.5013319677292009,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -56756964.765957445,
      "logits/rejected": -73011958.51851852,
      "logps/chosen": -465.02127659574467,
      "logps/rejected": -572.8395061728395,
      "loss": 0.1754,
      "rewards/chosen": 1.485621675531915,
      "rewards/margins": 16.720189576766483,
      "rewards/rejected": -15.234567901234568,
      "step": 1004
    },
    {
      "epoch": 0.49507389162561577,
      "grad_norm": 0.46815376636842526,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -66490032.26229508,
      "logits/rejected": -54776358.20895522,
      "logps/chosen": -352.26229508196724,
      "logps/rejected": -490.9850746268657,
      "loss": 0.2906,
      "rewards/chosen": -2.4385245901639343,
      "rewards/margins": 12.845057499388304,
      "rewards/rejected": -15.283582089552239,
      "step": 1005
    },
    {
      "epoch": 0.49556650246305417,
      "grad_norm": 0.4840313614792693,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -70432652.0754717,
      "logits/rejected": -61180914.346666664,
      "logps/chosen": -341.7358490566038,
      "logps/rejected": -572.5866666666667,
      "loss": 0.2279,
      "rewards/chosen": -1.099941037735849,
      "rewards/margins": 14.90005896226415,
      "rewards/rejected": -16.0,
      "step": 1006
    },
    {
      "epoch": 0.4960591133004926,
      "grad_norm": 0.5771124396581636,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -62657765.87755102,
      "logits/rejected": -60684676.8607595,
      "logps/chosen": -229.55102040816325,
      "logps/rejected": -557.367088607595,
      "loss": 0.2109,
      "rewards/chosen": -1.6084183673469388,
      "rewards/margins": 13.98651834151382,
      "rewards/rejected": -15.594936708860759,
      "step": 1007
    },
    {
      "epoch": 0.496551724137931,
      "grad_norm": 0.5081031428336948,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -54484008.96,
      "logits/rejected": -71410714.25641026,
      "logps/chosen": -242.56,
      "logps/rejected": -626.0512820512821,
      "loss": 0.1698,
      "rewards/chosen": 3.5949835205078124,
      "rewards/margins": 19.56934249486679,
      "rewards/rejected": -15.974358974358974,
      "step": 1008
    },
    {
      "epoch": 0.4970443349753695,
      "grad_norm": 0.4603245931965185,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -62165577.14285714,
      "logits/rejected": -71108084.09302326,
      "logps/chosen": -292.57142857142856,
      "logps/rejected": -505.6744186046512,
      "loss": 0.1508,
      "rewards/chosen": 0.9181547619047619,
      "rewards/margins": 14.918154761904763,
      "rewards/rejected": -14.0,
      "step": 1009
    },
    {
      "epoch": 0.4975369458128079,
      "grad_norm": 0.5383690593474273,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -51189573.81818182,
      "logits/rejected": -71303168.0,
      "logps/chosen": -260.1818181818182,
      "logps/rejected": -612.5714285714286,
      "loss": 0.1655,
      "rewards/chosen": -0.1590909090909091,
      "rewards/margins": 16.59090909090909,
      "rewards/rejected": -16.75,
      "step": 1010
    },
    {
      "epoch": 0.4980295566502463,
      "grad_norm": 2.7411780832014414,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -56666794.666666664,
      "logits/rejected": -57357107.2,
      "logps/chosen": -310.3333333333333,
      "logps/rejected": -570.4,
      "loss": 0.1835,
      "rewards/chosen": -0.9814453125,
      "rewards/margins": 16.5435546875,
      "rewards/rejected": -17.525,
      "step": 1011
    },
    {
      "epoch": 0.49852216748768474,
      "grad_norm": 1.6053555962689512,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -54777610.24,
      "logits/rejected": -61032500.51282051,
      "logps/chosen": -310.4,
      "logps/rejected": -561.2307692307693,
      "loss": 0.1837,
      "rewards/chosen": 0.8975,
      "rewards/margins": 15.30775641025641,
      "rewards/rejected": -14.41025641025641,
      "step": 1012
    },
    {
      "epoch": 0.49901477832512314,
      "grad_norm": 0.42319163588304354,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -51460883.692307696,
      "logits/rejected": -60793844.494382024,
      "logps/chosen": -225.64102564102564,
      "logps/rejected": -586.7865168539325,
      "loss": 0.1578,
      "rewards/chosen": 2.120965810922476,
      "rewards/margins": 18.525460192944948,
      "rewards/rejected": -16.40449438202247,
      "step": 1013
    },
    {
      "epoch": 0.4995073891625616,
      "grad_norm": 0.41941255193010674,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -55651253.07317073,
      "logits/rejected": -57997100.137931034,
      "logps/chosen": -291.5121951219512,
      "logps/rejected": -556.8735632183908,
      "loss": 0.1463,
      "rewards/chosen": -0.3795731707317073,
      "rewards/margins": 14.850311886739558,
      "rewards/rejected": -15.229885057471265,
      "step": 1014
    },
    {
      "epoch": 0.5,
      "grad_norm": 0.43994361993201003,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -53477376.0,
      "logits/rejected": -66260016.76190476,
      "logps/chosen": -290.0,
      "logps/rejected": -571.4285714285714,
      "loss": 0.161,
      "rewards/chosen": 0.09730113636363637,
      "rewards/margins": 14.62111066017316,
      "rewards/rejected": -14.523809523809524,
      "step": 1015
    },
    {
      "epoch": 0.5004926108374385,
      "grad_norm": 0.4351518816990407,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -59544137.14285714,
      "logits/rejected": -77478115.55555555,
      "logps/chosen": -308.57142857142856,
      "logps/rejected": -561.7777777777778,
      "loss": 0.2234,
      "rewards/chosen": 2.3655052185058594,
      "rewards/margins": 16.448838551839195,
      "rewards/rejected": -14.083333333333334,
      "step": 1016
    },
    {
      "epoch": 0.5009852216748768,
      "grad_norm": 0.5442747423566496,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -63158414.88372093,
      "logits/rejected": -62865215.24705882,
      "logps/chosen": -336.0,
      "logps/rejected": -499.2,
      "loss": 0.1371,
      "rewards/chosen": 6.287119310955669,
      "rewards/margins": 13.948810774650521,
      "rewards/rejected": -7.6616914636948525,
      "step": 1017
    },
    {
      "epoch": 0.5014778325123153,
      "grad_norm": 0.9429005936270607,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -52509459.692307696,
      "logits/rejected": -65453217.684210524,
      "logps/chosen": -299.38461538461536,
      "logps/rejected": -535.1578947368421,
      "loss": 0.216,
      "rewards/chosen": -0.5249399038461539,
      "rewards/margins": 14.185586411943321,
      "rewards/rejected": -14.710526315789474,
      "step": 1018
    },
    {
      "epoch": 0.5019704433497537,
      "grad_norm": 0.5543835060221721,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -61103383.27272727,
      "logits/rejected": -54725680.76190476,
      "logps/chosen": -309.09090909090907,
      "logps/rejected": -489.14285714285717,
      "loss": 0.1923,
      "rewards/chosen": 3.3144281560724433,
      "rewards/margins": 19.016809108453394,
      "rewards/rejected": -15.702380952380953,
      "step": 1019
    },
    {
      "epoch": 0.5024630541871922,
      "grad_norm": 0.5411533637493398,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -50462720.0,
      "logits/rejected": -58929971.2,
      "logps/chosen": -295.0,
      "logps/rejected": -569.6,
      "loss": 0.2005,
      "rewards/chosen": -1.8125,
      "rewards/margins": 14.462499999999999,
      "rewards/rejected": -16.275,
      "step": 1020
    },
    {
      "epoch": 0.5029556650246305,
      "grad_norm": 0.5225058641752904,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -61117001.14285714,
      "logits/rejected": -70620374.3255814,
      "logps/chosen": -329.9047619047619,
      "logps/rejected": -608.0,
      "loss": 0.1657,
      "rewards/chosen": 0.125,
      "rewards/margins": 15.101744186046512,
      "rewards/rejected": -14.976744186046512,
      "step": 1021
    },
    {
      "epoch": 0.503448275862069,
      "grad_norm": 0.41971504244691843,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -60703017.89090909,
      "logits/rejected": -69292200.32876712,
      "logps/chosen": -384.58181818181816,
      "logps/rejected": -579.5068493150685,
      "loss": 0.1819,
      "rewards/chosen": 3.0108373468572442,
      "rewards/margins": 17.161522278364092,
      "rewards/rejected": -14.150684931506849,
      "step": 1022
    },
    {
      "epoch": 0.5039408866995074,
      "grad_norm": 0.525208979020738,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -64545678.222222224,
      "logits/rejected": -68069005.87951808,
      "logps/chosen": -301.5111111111111,
      "logps/rejected": -558.2650602409639,
      "loss": 0.1598,
      "rewards/chosen": 3.6159796820746526,
      "rewards/margins": 19.037666429062604,
      "rewards/rejected": -15.421686746987952,
      "step": 1023
    },
    {
      "epoch": 0.5044334975369458,
      "grad_norm": 0.5711538800673839,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -64137898.666666664,
      "logits/rejected": -72037171.2,
      "logps/chosen": -405.5,
      "logps/rejected": -487.6,
      "loss": 0.2,
      "rewards/chosen": 3.501985549926758,
      "rewards/margins": 15.376985549926758,
      "rewards/rejected": -11.875,
      "step": 1024
    },
    {
      "epoch": 0.5049261083743842,
      "grad_norm": 0.5252494607184918,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -62830673.92,
      "logits/rejected": -54310859.48717949,
      "logps/chosen": -338.88,
      "logps/rejected": -479.5897435897436,
      "loss": 0.2225,
      "rewards/chosen": 5.133796997070313,
      "rewards/margins": 18.633796997070313,
      "rewards/rejected": -13.5,
      "step": 1025
    },
    {
      "epoch": 0.5054187192118227,
      "grad_norm": 0.44121163723425283,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -74099370.66666667,
      "logits/rejected": -60741607.32530121,
      "logps/chosen": -272.7111111111111,
      "logps/rejected": -513.5421686746988,
      "loss": 0.2035,
      "rewards/chosen": 6.2659898546006945,
      "rewards/margins": 17.39851997508262,
      "rewards/rejected": -11.132530120481928,
      "step": 1026
    },
    {
      "epoch": 0.505911330049261,
      "grad_norm": 0.47515035324989824,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -56343483.733333334,
      "logits/rejected": -69433418.02409638,
      "logps/chosen": -228.0888888888889,
      "logps/rejected": -537.4457831325301,
      "loss": 0.1371,
      "rewards/chosen": 3.1062469482421875,
      "rewards/margins": 16.93757224944701,
      "rewards/rejected": -13.831325301204819,
      "step": 1027
    },
    {
      "epoch": 0.5064039408866995,
      "grad_norm": 0.49753914068058674,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -61290958.451612905,
      "logits/rejected": -67235964.12121212,
      "logps/chosen": -348.9032258064516,
      "logps/rejected": -486.7878787878788,
      "loss": 0.2432,
      "rewards/chosen": 0.7736895161290323,
      "rewards/margins": 12.62217436461388,
      "rewards/rejected": -11.848484848484848,
      "step": 1028
    },
    {
      "epoch": 0.506896551724138,
      "grad_norm": 0.5673982048198487,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -67713811.6923077,
      "logits/rejected": -68543757.4736842,
      "logps/chosen": -271.84615384615387,
      "logps/rejected": -483.36842105263156,
      "loss": 0.2151,
      "rewards/chosen": 0.7403846153846154,
      "rewards/margins": 12.003542510121457,
      "rewards/rejected": -11.263157894736842,
      "step": 1029
    },
    {
      "epoch": 0.5073891625615764,
      "grad_norm": 0.5405730564854793,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -41866779.92727273,
      "logits/rejected": -78829932.71232876,
      "logps/chosen": -270.25454545454545,
      "logps/rejected": -500.16438356164383,
      "loss": 0.2009,
      "rewards/chosen": 2.463076504794034,
      "rewards/margins": 11.25759705273924,
      "rewards/rejected": -8.794520547945206,
      "step": 1030
    },
    {
      "epoch": 0.5078817733990147,
      "grad_norm": 0.5386942661978696,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -90489877.78723404,
      "logits/rejected": -59289852.83950617,
      "logps/chosen": -316.93617021276594,
      "logps/rejected": -522.2716049382716,
      "loss": 0.1524,
      "rewards/chosen": 1.5639128989361701,
      "rewards/margins": 13.02070302239296,
      "rewards/rejected": -11.45679012345679,
      "step": 1031
    },
    {
      "epoch": 0.5083743842364532,
      "grad_norm": 0.5292513889371561,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -49385873.568627454,
      "logits/rejected": -54852780.883116886,
      "logps/chosen": -337.2549019607843,
      "logps/rejected": -436.3636363636364,
      "loss": 0.2228,
      "rewards/chosen": 0.5980392156862745,
      "rewards/margins": 10.779857397504456,
      "rewards/rejected": -10.181818181818182,
      "step": 1032
    },
    {
      "epoch": 0.5088669950738917,
      "grad_norm": 0.577785383563319,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -48883614.47619048,
      "logits/rejected": -55257516.65116279,
      "logps/chosen": -268.76190476190476,
      "logps/rejected": -464.3720930232558,
      "loss": 0.2018,
      "rewards/chosen": -0.48214285714285715,
      "rewards/margins": 9.866694352159469,
      "rewards/rejected": -10.348837209302326,
      "step": 1033
    },
    {
      "epoch": 0.50935960591133,
      "grad_norm": 0.4101476557768549,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -47403548.981132075,
      "logits/rejected": -50723116.373333335,
      "logps/chosen": -273.20754716981133,
      "logps/rejected": -440.32,
      "loss": 0.1836,
      "rewards/chosen": 1.0622051886792452,
      "rewards/margins": 11.955538522012578,
      "rewards/rejected": -10.893333333333333,
      "step": 1034
    },
    {
      "epoch": 0.5098522167487685,
      "grad_norm": 0.4272323349761279,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -44468182.20408163,
      "logits/rejected": -62330542.98734177,
      "logps/chosen": -255.83673469387756,
      "logps/rejected": -602.7341772151899,
      "loss": 0.1681,
      "rewards/chosen": 1.2933673469387754,
      "rewards/margins": 10.951595195040042,
      "rewards/rejected": -9.658227848101266,
      "step": 1035
    },
    {
      "epoch": 0.5103448275862069,
      "grad_norm": 0.5394255002133063,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -45739608.27586207,
      "logits/rejected": -63573664.91428571,
      "logps/chosen": -334.0689655172414,
      "logps/rejected": -542.1714285714286,
      "loss": 0.2063,
      "rewards/chosen": 1.0106411637931034,
      "rewards/margins": 13.696355449507388,
      "rewards/rejected": -12.685714285714285,
      "step": 1036
    },
    {
      "epoch": 0.5108374384236454,
      "grad_norm": 0.4474294905502573,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -44780363.294117644,
      "logits/rejected": -57957655.27272727,
      "logps/chosen": -274.8235294117647,
      "logps/rejected": -520.3116883116883,
      "loss": 0.1878,
      "rewards/chosen": 0.4803921568627451,
      "rewards/margins": 12.9349376114082,
      "rewards/rejected": -12.454545454545455,
      "step": 1037
    },
    {
      "epoch": 0.5113300492610837,
      "grad_norm": 0.4890851662892079,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -43802778.56603774,
      "logits/rejected": -67444408.32,
      "logps/chosen": -221.28301886792454,
      "logps/rejected": -581.12,
      "loss": 0.1753,
      "rewards/chosen": 1.3455188679245282,
      "rewards/margins": 13.625518867924528,
      "rewards/rejected": -12.28,
      "step": 1038
    },
    {
      "epoch": 0.5118226600985222,
      "grad_norm": 0.40206832395225967,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -60691578.88,
      "logits/rejected": -60333449.84615385,
      "logps/chosen": -327.68,
      "logps/rejected": -457.02564102564105,
      "loss": 0.1939,
      "rewards/chosen": 0.9046875,
      "rewards/margins": 12.853405448717949,
      "rewards/rejected": -11.948717948717949,
      "step": 1039
    },
    {
      "epoch": 0.5123152709359606,
      "grad_norm": 0.42196547763403935,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -54911143.18367347,
      "logits/rejected": -60897046.6835443,
      "logps/chosen": -320.3265306122449,
      "logps/rejected": -524.1518987341772,
      "loss": 0.1407,
      "rewards/chosen": 1.7957589285714286,
      "rewards/margins": 14.365379181735985,
      "rewards/rejected": -12.569620253164556,
      "step": 1040
    },
    {
      "epoch": 0.512807881773399,
      "grad_norm": 0.7594179507465254,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -79176686.03508772,
      "logits/rejected": -62205663.549295776,
      "logps/chosen": -353.6842105263158,
      "logps/rejected": -488.5633802816901,
      "loss": 0.2014,
      "rewards/chosen": 1.3464912280701755,
      "rewards/margins": 14.332406721027922,
      "rewards/rejected": -12.985915492957746,
      "step": 1041
    },
    {
      "epoch": 0.5133004926108374,
      "grad_norm": 0.4621964453819777,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -57298458.0338983,
      "logits/rejected": -65163679.53623188,
      "logps/chosen": -353.08474576271186,
      "logps/rejected": -540.2898550724638,
      "loss": 0.23,
      "rewards/chosen": 0.3628177966101695,
      "rewards/margins": 14.406296057479734,
      "rewards/rejected": -14.043478260869565,
      "step": 1042
    },
    {
      "epoch": 0.5137931034482759,
      "grad_norm": 0.42706605009890874,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -40010370.50980392,
      "logits/rejected": -70486095.79220779,
      "logps/chosen": -224.31372549019608,
      "logps/rejected": -571.4285714285714,
      "loss": 0.1972,
      "rewards/chosen": 0.7028186274509803,
      "rewards/margins": 16.209312133944486,
      "rewards/rejected": -15.506493506493506,
      "step": 1043
    },
    {
      "epoch": 0.5142857142857142,
      "grad_norm": 0.5431391853391443,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -50049339.07692308,
      "logits/rejected": -64294265.2631579,
      "logps/chosen": -403.38461538461536,
      "logps/rejected": -613.0526315789474,
      "loss": 0.2283,
      "rewards/chosen": 0.2548076923076923,
      "rewards/margins": 15.37322874493927,
      "rewards/rejected": -15.118421052631579,
      "step": 1044
    },
    {
      "epoch": 0.5147783251231527,
      "grad_norm": 0.5797274637669607,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -64621544.18604651,
      "logits/rejected": -56598431.62352941,
      "logps/chosen": -283.90697674418607,
      "logps/rejected": -555.6705882352941,
      "loss": 0.1773,
      "rewards/chosen": 2.882278797238372,
      "rewards/margins": 17.30580820900308,
      "rewards/rejected": -14.423529411764706,
      "step": 1045
    },
    {
      "epoch": 0.5152709359605911,
      "grad_norm": 0.4920993329680365,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -54335301.81818182,
      "logits/rejected": -55824188.952380955,
      "logps/chosen": -404.3636363636364,
      "logps/rejected": -485.3333333333333,
      "loss": 0.1441,
      "rewards/chosen": 2.0255681818181817,
      "rewards/margins": 14.70413961038961,
      "rewards/rejected": -12.678571428571429,
      "step": 1046
    },
    {
      "epoch": 0.5157635467980296,
      "grad_norm": 0.4633562727863738,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -48574574.7027027,
      "logits/rejected": -64712118.85714286,
      "logps/chosen": -360.86486486486484,
      "logps/rejected": -542.2417582417582,
      "loss": 0.1593,
      "rewards/chosen": 6.556610107421875,
      "rewards/margins": 19.853313404125174,
      "rewards/rejected": -13.296703296703297,
      "step": 1047
    },
    {
      "epoch": 0.516256157635468,
      "grad_norm": 0.568524986570345,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -47989828.266666666,
      "logits/rejected": -57424956.23529412,
      "logps/chosen": -262.0,
      "logps/rejected": -583.5294117647059,
      "loss": 0.2108,
      "rewards/chosen": 3.5140467325846356,
      "rewards/margins": 18.55816437964346,
      "rewards/rejected": -15.044117647058824,
      "step": 1048
    },
    {
      "epoch": 0.5167487684729064,
      "grad_norm": 0.477106581500512,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -55826186.24,
      "logits/rejected": -57967432.20512821,
      "logps/chosen": -385.92,
      "logps/rejected": -517.7435897435897,
      "loss": 0.2003,
      "rewards/chosen": -0.19625,
      "rewards/margins": 14.137083333333335,
      "rewards/rejected": -14.333333333333334,
      "step": 1049
    },
    {
      "epoch": 0.5172413793103449,
      "grad_norm": 0.44921810679091806,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -52575968.561403506,
      "logits/rejected": -56120969.01408451,
      "logps/chosen": -283.2280701754386,
      "logps/rejected": -506.59154929577466,
      "loss": 0.2239,
      "rewards/chosen": 1.0953947368421053,
      "rewards/margins": 14.940465159377318,
      "rewards/rejected": -13.845070422535212,
      "step": 1050
    },
    {
      "epoch": 0.5177339901477832,
      "grad_norm": 0.4967439306274774,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -57191082.666666664,
      "logits/rejected": -62809702.4,
      "logps/chosen": -277.3333333333333,
      "logps/rejected": -589.6,
      "loss": 0.186,
      "rewards/chosen": 2.8869450887044272,
      "rewards/margins": 18.149445088704425,
      "rewards/rejected": -15.2625,
      "step": 1051
    },
    {
      "epoch": 0.5182266009852217,
      "grad_norm": 0.4043694152254172,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -73770405.64705883,
      "logits/rejected": -64330818.49350649,
      "logps/chosen": -341.3333333333333,
      "logps/rejected": -492.05194805194805,
      "loss": 0.1674,
      "rewards/chosen": 1.1004901960784315,
      "rewards/margins": 14.217373312961548,
      "rewards/rejected": -13.116883116883116,
      "step": 1052
    },
    {
      "epoch": 0.5187192118226601,
      "grad_norm": 0.5349015651419408,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -50260154.18181818,
      "logits/rejected": -65411169.52380952,
      "logps/chosen": -278.0,
      "logps/rejected": -508.95238095238096,
      "loss": 0.2192,
      "rewards/chosen": -0.6732954545454546,
      "rewards/margins": 11.505275974025974,
      "rewards/rejected": -12.178571428571429,
      "step": 1053
    },
    {
      "epoch": 0.5192118226600986,
      "grad_norm": 0.435220029666445,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -54255351.741935484,
      "logits/rejected": -61141709.8556701,
      "logps/chosen": -382.96774193548384,
      "logps/rejected": -574.680412371134,
      "loss": 0.1304,
      "rewards/chosen": -1.1824596774193548,
      "rewards/margins": 13.394859910209512,
      "rewards/rejected": -14.577319587628866,
      "step": 1054
    },
    {
      "epoch": 0.5197044334975369,
      "grad_norm": 0.4467637178163414,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -61015252.52830189,
      "logits/rejected": -65375218.346666664,
      "logps/chosen": -306.7169811320755,
      "logps/rejected": -572.5866666666667,
      "loss": 0.2014,
      "rewards/chosen": 1.0548349056603774,
      "rewards/margins": 16.94816823899371,
      "rewards/rejected": -15.893333333333333,
      "step": 1055
    },
    {
      "epoch": 0.5201970443349754,
      "grad_norm": 0.445185885052298,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -62151959.27272727,
      "logits/rejected": -62465170.28571428,
      "logps/chosen": -278.1818181818182,
      "logps/rejected": -526.4761904761905,
      "loss": 0.1816,
      "rewards/chosen": 3.312517686323686,
      "rewards/margins": 14.693470067276067,
      "rewards/rejected": -11.380952380952381,
      "step": 1056
    },
    {
      "epoch": 0.5206896551724138,
      "grad_norm": 0.5761222726797774,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -77972798.95081967,
      "logits/rejected": -56717006.32835821,
      "logps/chosen": -364.0655737704918,
      "logps/rejected": -510.089552238806,
      "loss": 0.2317,
      "rewards/chosen": 2.500790455302254,
      "rewards/margins": 12.036687325929584,
      "rewards/rejected": -9.535896870627331,
      "step": 1057
    },
    {
      "epoch": 0.5211822660098522,
      "grad_norm": 0.51957585767659,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -58270866.28571428,
      "logits/rejected": -62524392.18604651,
      "logps/chosen": -348.57142857142856,
      "logps/rejected": -520.1860465116279,
      "loss": 0.1949,
      "rewards/chosen": 2.514916192917597,
      "rewards/margins": 17.63119526268504,
      "rewards/rejected": -15.116279069767442,
      "step": 1058
    },
    {
      "epoch": 0.5216748768472906,
      "grad_norm": 0.5018956056326654,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -49682529.52380952,
      "logits/rejected": -51111983.62790698,
      "logps/chosen": -245.71428571428572,
      "logps/rejected": -482.9767441860465,
      "loss": 0.165,
      "rewards/chosen": 3.8853817894345237,
      "rewards/margins": 16.59468411501592,
      "rewards/rejected": -12.709302325581396,
      "step": 1059
    },
    {
      "epoch": 0.5221674876847291,
      "grad_norm": 0.4796343241420566,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -47185920.0,
      "logits/rejected": -56755835.1392405,
      "logps/chosen": -281.46938775510205,
      "logps/rejected": -515.2405063291139,
      "loss": 0.1904,
      "rewards/chosen": 1.1058805816027584,
      "rewards/margins": 14.257779315779974,
      "rewards/rejected": -13.151898734177216,
      "step": 1060
    },
    {
      "epoch": 0.5226600985221674,
      "grad_norm": 0.4239072960935138,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -41859153.92,
      "logits/rejected": -58182524.71794872,
      "logps/chosen": -244.48,
      "logps/rejected": -544.8205128205128,
      "loss": 0.195,
      "rewards/chosen": 4.55248779296875,
      "rewards/margins": 16.91146215194311,
      "rewards/rejected": -12.35897435897436,
      "step": 1061
    },
    {
      "epoch": 0.5231527093596059,
      "grad_norm": 0.4158513822590812,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -51263715.55555555,
      "logits/rejected": -60236269.4939759,
      "logps/chosen": -300.8,
      "logps/rejected": -532.433734939759,
      "loss": 0.1843,
      "rewards/chosen": -0.9876736111111111,
      "rewards/margins": 12.024374581659973,
      "rewards/rejected": -13.012048192771084,
      "step": 1062
    },
    {
      "epoch": 0.5236453201970444,
      "grad_norm": 0.5426186647305322,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -62215509.333333336,
      "logits/rejected": -66523612.27906977,
      "logps/chosen": -305.9047619047619,
      "logps/rejected": -541.0232558139535,
      "loss": 0.1992,
      "rewards/chosen": 0.07366071428571429,
      "rewards/margins": 12.92249792358804,
      "rewards/rejected": -12.848837209302326,
      "step": 1063
    },
    {
      "epoch": 0.5241379310344828,
      "grad_norm": 0.4773342430405913,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -48355485.538461536,
      "logits/rejected": -54360387.368421055,
      "logps/chosen": -251.69230769230768,
      "logps/rejected": -489.2631578947368,
      "loss": 0.182,
      "rewards/chosen": 1.5315504807692308,
      "rewards/margins": 14.649971533400809,
      "rewards/rejected": -13.118421052631579,
      "step": 1064
    },
    {
      "epoch": 0.5246305418719212,
      "grad_norm": 0.5320161644199546,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -57755566.08,
      "logits/rejected": -59042894.76923077,
      "logps/chosen": -288.64,
      "logps/rejected": -519.3846153846154,
      "loss": 0.1651,
      "rewards/chosen": 1.00125,
      "rewards/margins": 11.50125,
      "rewards/rejected": -10.5,
      "step": 1065
    },
    {
      "epoch": 0.5251231527093596,
      "grad_norm": 0.44613249151033424,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -46583546.55319149,
      "logits/rejected": -57581062.32098766,
      "logps/chosen": -278.468085106383,
      "logps/rejected": -514.3703703703703,
      "loss": 0.1682,
      "rewards/chosen": 0.8417553191489362,
      "rewards/margins": 13.755335566062517,
      "rewards/rejected": -12.91358024691358,
      "step": 1066
    },
    {
      "epoch": 0.5256157635467981,
      "grad_norm": 0.5685146313652627,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -65263370.24,
      "logits/rejected": -56891969.64102564,
      "logps/chosen": -289.28,
      "logps/rejected": -530.8717948717949,
      "loss": 0.1847,
      "rewards/chosen": 3.4382373046875,
      "rewards/margins": 17.39977576622596,
      "rewards/rejected": -13.961538461538462,
      "step": 1067
    },
    {
      "epoch": 0.5261083743842364,
      "grad_norm": 0.44279031192962287,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -47977701.87755102,
      "logits/rejected": -52773900.962025315,
      "logps/chosen": -368.0,
      "logps/rejected": -509.56962025316454,
      "loss": 0.1611,
      "rewards/chosen": 4.2085512122329405,
      "rewards/margins": 17.740196781853193,
      "rewards/rejected": -13.531645569620252,
      "step": 1068
    },
    {
      "epoch": 0.5266009852216749,
      "grad_norm": 0.4806845209638956,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -65875245.176470585,
      "logits/rejected": -56269039.37662338,
      "logps/chosen": -377.72549019607845,
      "logps/rejected": -528.6233766233767,
      "loss": 0.15,
      "rewards/chosen": 2.047794117647059,
      "rewards/margins": 10.98162963027896,
      "rewards/rejected": -8.9338355126319,
      "step": 1069
    },
    {
      "epoch": 0.5270935960591133,
      "grad_norm": 0.44837752796310526,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -32865661.49019608,
      "logits/rejected": -58284484.15584416,
      "logps/chosen": -221.1764705882353,
      "logps/rejected": -576.8311688311688,
      "loss": 0.2009,
      "rewards/chosen": 1.7573297538009345,
      "rewards/margins": 16.04304403951522,
      "rewards/rejected": -14.285714285714286,
      "step": 1070
    },
    {
      "epoch": 0.5275862068965518,
      "grad_norm": 0.465247041813713,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -53099888.64,
      "logits/rejected": -54579725.12820513,
      "logps/chosen": -267.68,
      "logps/rejected": -537.4358974358975,
      "loss": 0.1761,
      "rewards/chosen": 0.40125,
      "rewards/margins": 14.183301282051282,
      "rewards/rejected": -13.782051282051283,
      "step": 1071
    },
    {
      "epoch": 0.5280788177339901,
      "grad_norm": 0.48063877404085165,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -54316236.8,
      "logits/rejected": -56354238.35897436,
      "logps/chosen": -304.32,
      "logps/rejected": -526.7692307692307,
      "loss": 0.2259,
      "rewards/chosen": 2.0124945068359374,
      "rewards/margins": 14.268904763246194,
      "rewards/rejected": -12.256410256410257,
      "step": 1072
    },
    {
      "epoch": 0.5285714285714286,
      "grad_norm": 0.5812739727546389,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -56840050.75862069,
      "logits/rejected": -52368881.37142857,
      "logps/chosen": -321.37931034482756,
      "logps/rejected": -506.51428571428573,
      "loss": 0.2558,
      "rewards/chosen": 0.5593031521501213,
      "rewards/margins": 11.58787458072155,
      "rewards/rejected": -11.028571428571428,
      "step": 1073
    },
    {
      "epoch": 0.529064039408867,
      "grad_norm": 0.37350858993709257,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -49682529.52380952,
      "logits/rejected": -57159584.744186044,
      "logps/chosen": -328.76190476190476,
      "logps/rejected": -532.8372093023256,
      "loss": 0.1722,
      "rewards/chosen": 4.74554443359375,
      "rewards/margins": 16.792056061500727,
      "rewards/rejected": -12.046511627906977,
      "step": 1074
    },
    {
      "epoch": 0.5295566502463054,
      "grad_norm": 0.5006242536659558,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -42949672.96,
      "logits/rejected": -60709861.743589744,
      "logps/chosen": -277.76,
      "logps/rejected": -525.1282051282051,
      "loss": 0.1928,
      "rewards/chosen": 3.196197814941406,
      "rewards/margins": 15.337223455967047,
      "rewards/rejected": -12.14102564102564,
      "step": 1075
    },
    {
      "epoch": 0.5300492610837438,
      "grad_norm": 0.5747525009795468,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -52001602.37037037,
      "logits/rejected": -67789021.4054054,
      "logps/chosen": -297.48148148148147,
      "logps/rejected": -480.43243243243245,
      "loss": 0.1962,
      "rewards/chosen": 1.1979166666666667,
      "rewards/margins": 12.103322072072071,
      "rewards/rejected": -10.905405405405405,
      "step": 1076
    },
    {
      "epoch": 0.5305418719211823,
      "grad_norm": 0.4320157476445995,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -58384711.68,
      "logits/rejected": -58989121.64102564,
      "logps/chosen": -261.44,
      "logps/rejected": -462.7692307692308,
      "loss": 0.1562,
      "rewards/chosen": 5.82561767578125,
      "rewards/margins": 17.18459203475561,
      "rewards/rejected": -11.35897435897436,
      "step": 1077
    },
    {
      "epoch": 0.5310344827586206,
      "grad_norm": 0.5635163488517809,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -59695675.53488372,
      "logits/rejected": -59213703.52941176,
      "logps/chosen": -332.09302325581393,
      "logps/rejected": -509.74117647058824,
      "loss": 0.2268,
      "rewards/chosen": 2.7107096827307413,
      "rewards/margins": 14.157768506260153,
      "rewards/rejected": -11.447058823529412,
      "step": 1078
    },
    {
      "epoch": 0.5315270935960591,
      "grad_norm": 0.509810424273507,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -63351466.666666664,
      "logits/rejected": -62180556.8,
      "logps/chosen": -358.0,
      "logps/rejected": -519.2,
      "loss": 0.18,
      "rewards/chosen": 2.9433250427246094,
      "rewards/margins": 16.593325042724608,
      "rewards/rejected": -13.65,
      "step": 1079
    },
    {
      "epoch": 0.5320197044334976,
      "grad_norm": 0.40100473625169697,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -44239920.76190476,
      "logits/rejected": -63109643.906976745,
      "logps/chosen": -254.47619047619048,
      "logps/rejected": -514.2325581395348,
      "loss": 0.1479,
      "rewards/chosen": 4.541603451683407,
      "rewards/margins": 16.367184847032245,
      "rewards/rejected": -11.825581395348838,
      "step": 1080
    },
    {
      "epoch": 0.532512315270936,
      "grad_norm": 0.4392303094223186,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -70604117.33333333,
      "logits/rejected": -71303168.0,
      "logps/chosen": -320.0,
      "logps/rejected": -528.3720930232558,
      "loss": 0.1902,
      "rewards/chosen": 3.4844022478376115,
      "rewards/margins": 15.84486736411668,
      "rewards/rejected": -12.36046511627907,
      "step": 1081
    },
    {
      "epoch": 0.5330049261083744,
      "grad_norm": 0.5208659791035742,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -56539217.92,
      "logits/rejected": -74045597.53846154,
      "logps/chosen": -286.56,
      "logps/rejected": -481.2307692307692,
      "loss": 0.1961,
      "rewards/chosen": 2.988748779296875,
      "rewards/margins": 14.488748779296875,
      "rewards/rejected": -11.5,
      "step": 1082
    },
    {
      "epoch": 0.5334975369458128,
      "grad_norm": 0.45513719493129645,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -62552982.06896552,
      "logits/rejected": -65610898.28571428,
      "logps/chosen": -296.55172413793105,
      "logps/rejected": -532.1142857142858,
      "loss": 0.24,
      "rewards/chosen": -0.5808189655172413,
      "rewards/margins": 11.89060960591133,
      "rewards/rejected": -12.471428571428572,
      "step": 1083
    },
    {
      "epoch": 0.5339901477832513,
      "grad_norm": 0.5227538342898496,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -48748084.24489796,
      "logits/rejected": -60631584.405063294,
      "logps/chosen": -202.44897959183675,
      "logps/rejected": -505.5189873417722,
      "loss": 0.1745,
      "rewards/chosen": 0.7487244897959183,
      "rewards/margins": 12.318344742960475,
      "rewards/rejected": -11.569620253164556,
      "step": 1084
    },
    {
      "epoch": 0.5344827586206896,
      "grad_norm": 0.39545988322808423,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -52728393.14285714,
      "logits/rejected": -69604209.41772152,
      "logps/chosen": -286.3673469387755,
      "logps/rejected": -549.2658227848101,
      "loss": 0.1696,
      "rewards/chosen": 3.318263462611607,
      "rewards/margins": 15.280288779067304,
      "rewards/rejected": -11.962025316455696,
      "step": 1085
    },
    {
      "epoch": 0.5349753694581281,
      "grad_norm": 0.4291267145359297,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -39727181.283018865,
      "logits/rejected": -61236838.4,
      "logps/chosen": -253.58490566037736,
      "logps/rejected": -516.2666666666667,
      "loss": 0.207,
      "rewards/chosen": -0.05660377358490566,
      "rewards/margins": 12.863396226415095,
      "rewards/rejected": -12.92,
      "step": 1086
    },
    {
      "epoch": 0.5354679802955665,
      "grad_norm": 0.46833917619391413,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -58300825.6,
      "logits/rejected": -63022106.256410256,
      "logps/chosen": -313.6,
      "logps/rejected": -534.974358974359,
      "loss": 0.2039,
      "rewards/chosen": -0.533125,
      "rewards/margins": 11.543798076923077,
      "rewards/rejected": -12.076923076923077,
      "step": 1087
    },
    {
      "epoch": 0.5359605911330049,
      "grad_norm": 0.5042690587126082,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -57058361.96226415,
      "logits/rejected": -48933546.666666664,
      "logps/chosen": -308.52830188679246,
      "logps/rejected": -432.2133333333333,
      "loss": 0.171,
      "rewards/chosen": 1.8797169811320755,
      "rewards/margins": 13.559716981132075,
      "rewards/rejected": -11.68,
      "step": 1088
    },
    {
      "epoch": 0.5364532019704433,
      "grad_norm": 0.43543056785067996,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -67602311.52941176,
      "logits/rejected": -78438931.94805194,
      "logps/chosen": -347.921568627451,
      "logps/rejected": -600.1038961038961,
      "loss": 0.216,
      "rewards/chosen": 3.4877068014705883,
      "rewards/margins": 16.40978472354851,
      "rewards/rejected": -12.922077922077921,
      "step": 1089
    },
    {
      "epoch": 0.5369458128078818,
      "grad_norm": 0.39925882238207694,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -53477376.0,
      "logits/rejected": -63913203.809523806,
      "logps/chosen": -331.27272727272725,
      "logps/rejected": -462.4761904761905,
      "loss": 0.1675,
      "rewards/chosen": 5.342366998845881,
      "rewards/margins": 16.675700332179215,
      "rewards/rejected": -11.333333333333334,
      "step": 1090
    },
    {
      "epoch": 0.5374384236453202,
      "grad_norm": 0.4586702265054993,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -56533863.4893617,
      "logits/rejected": -74151152.19753087,
      "logps/chosen": -343.48936170212767,
      "logps/rejected": -534.9135802469136,
      "loss": 0.182,
      "rewards/chosen": 0.35804521276595747,
      "rewards/margins": 12.802489657210401,
      "rewards/rejected": -12.444444444444445,
      "step": 1091
    },
    {
      "epoch": 0.5379310344827586,
      "grad_norm": 0.5192420819618433,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -62988144.28070176,
      "logits/rejected": -54821325.52112676,
      "logps/chosen": -345.82456140350877,
      "logps/rejected": -511.5492957746479,
      "loss": 0.217,
      "rewards/chosen": 1.2221765350877194,
      "rewards/margins": 10.898232873115889,
      "rewards/rejected": -9.67605633802817,
      "step": 1092
    },
    {
      "epoch": 0.538423645320197,
      "grad_norm": 0.5358645920150152,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -60491184.35555556,
      "logits/rejected": -65087512.67469879,
      "logps/chosen": -271.64444444444445,
      "logps/rejected": -475.7590361445783,
      "loss": 0.19,
      "rewards/chosen": 2.750685119628906,
      "rewards/margins": 14.0639381316771,
      "rewards/rejected": -11.313253012048193,
      "step": 1093
    },
    {
      "epoch": 0.5389162561576355,
      "grad_norm": 0.537220793999,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -62739797.333333336,
      "logits/rejected": -65116569.6,
      "logps/chosen": -323.0,
      "logps/rejected": -528.0,
      "loss": 0.2351,
      "rewards/chosen": 2.327489217122396,
      "rewards/margins": 14.277489217122396,
      "rewards/rejected": -11.95,
      "step": 1094
    },
    {
      "epoch": 0.5394088669950738,
      "grad_norm": 0.48259979330002656,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -48392771.62264151,
      "logits/rejected": -59782813.013333336,
      "logps/chosen": -302.49056603773585,
      "logps/rejected": -564.0533333333333,
      "loss": 0.207,
      "rewards/chosen": 3.2467092837927476,
      "rewards/margins": 12.846709283792748,
      "rewards/rejected": -9.6,
      "step": 1095
    },
    {
      "epoch": 0.5399014778325123,
      "grad_norm": 0.5728852147530287,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -60010811.07692308,
      "logits/rejected": -68323004.63157895,
      "logps/chosen": -307.38461538461536,
      "logps/rejected": -607.1578947368421,
      "loss": 0.2115,
      "rewards/chosen": 0.3858173076923077,
      "rewards/margins": 14.030554149797572,
      "rewards/rejected": -13.644736842105264,
      "step": 1096
    },
    {
      "epoch": 0.5403940886699508,
      "grad_norm": 0.5048266143760464,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -53694322.75862069,
      "logits/rejected": -72022191.54285714,
      "logps/chosen": -280.55172413793105,
      "logps/rejected": -457.6,
      "loss": 0.219,
      "rewards/chosen": 0.5797413793103449,
      "rewards/margins": 12.09402709359606,
      "rewards/rejected": -11.514285714285714,
      "step": 1097
    },
    {
      "epoch": 0.5408866995073892,
      "grad_norm": 0.4923155020408852,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -48286924.8,
      "logits/rejected": -65869637.81818182,
      "logps/chosen": -234.6,
      "logps/rejected": -552.0,
      "loss": 0.1746,
      "rewards/chosen": -0.2390625,
      "rewards/margins": 11.954119318181819,
      "rewards/rejected": -12.193181818181818,
      "step": 1098
    },
    {
      "epoch": 0.5413793103448276,
      "grad_norm": 0.5669630715088992,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -41780329.93103448,
      "logits/rejected": -60278140.342857145,
      "logps/chosen": -342.8965517241379,
      "logps/rejected": -499.2,
      "loss": 0.2443,
      "rewards/chosen": -0.15732758620689655,
      "rewards/margins": 10.928386699507389,
      "rewards/rejected": -11.085714285714285,
      "step": 1099
    },
    {
      "epoch": 0.541871921182266,
      "grad_norm": 0.5005247273946413,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -61646514.60465116,
      "logits/rejected": -55907605.08235294,
      "logps/chosen": -317.2093023255814,
      "logps/rejected": -476.61176470588236,
      "loss": 0.1715,
      "rewards/chosen": 5.6246458541515265,
      "rewards/margins": 18.012881148269173,
      "rewards/rejected": -12.388235294117647,
      "step": 1100
    },
    {
      "epoch": 0.5423645320197045,
      "grad_norm": 0.4672213885991642,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -60517814.85714286,
      "logits/rejected": -56818187.906976745,
      "logps/chosen": -297.5238095238095,
      "logps/rejected": -486.69767441860466,
      "loss": 0.1868,
      "rewards/chosen": 1.8400310334705172,
      "rewards/margins": 14.537705452075167,
      "rewards/rejected": -12.69767441860465,
      "step": 1101
    },
    {
      "epoch": 0.5428571428571428,
      "grad_norm": 0.4163303785925366,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -55947749.9661017,
      "logits/rejected": -58233959.88405797,
      "logps/chosen": -339.52542372881356,
      "logps/rejected": -471.18840579710144,
      "loss": 0.2157,
      "rewards/chosen": 0.4891419491525424,
      "rewards/margins": 12.344214412920657,
      "rewards/rejected": -11.855072463768115,
      "step": 1102
    },
    {
      "epoch": 0.5433497536945813,
      "grad_norm": 0.6083986092382424,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -77259079.68,
      "logits/rejected": -63452291.28205128,
      "logps/chosen": -399.36,
      "logps/rejected": -538.2564102564103,
      "loss": 0.2223,
      "rewards/chosen": 0.9775546264648437,
      "rewards/margins": 6.3524462499374,
      "rewards/rejected": -5.374891623472556,
      "step": 1103
    },
    {
      "epoch": 0.5438423645320197,
      "grad_norm": 0.4762888268222311,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -54324302.76923077,
      "logits/rejected": -58830632.421052635,
      "logps/chosen": -275.0769230769231,
      "logps/rejected": -493.05263157894734,
      "loss": 0.2302,
      "rewards/chosen": -1.1442307692307692,
      "rewards/margins": 10.526821862348179,
      "rewards/rejected": -11.671052631578947,
      "step": 1104
    },
    {
      "epoch": 0.5443349753694581,
      "grad_norm": 0.46716117480797,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -53886142.915254235,
      "logits/rejected": -65589188.63768116,
      "logps/chosen": -286.10169491525426,
      "logps/rejected": -579.7101449275362,
      "loss": 0.1849,
      "rewards/chosen": 1.4078389830508475,
      "rewards/margins": 13.726679562760992,
      "rewards/rejected": -12.318840579710145,
      "step": 1105
    },
    {
      "epoch": 0.5448275862068965,
      "grad_norm": 0.534783473652758,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -55981118.69387755,
      "logits/rejected": -58295516.35443038,
      "logps/chosen": -205.71428571428572,
      "logps/rejected": -486.0759493670886,
      "loss": 0.1949,
      "rewards/chosen": 0.43494897959183676,
      "rewards/margins": 13.510898346680445,
      "rewards/rejected": -13.075949367088608,
      "step": 1106
    },
    {
      "epoch": 0.545320197044335,
      "grad_norm": 0.4898474134675568,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -58605865.89090909,
      "logits/rejected": -55617620.16438356,
      "logps/chosen": -314.76363636363635,
      "logps/rejected": -500.6027397260274,
      "loss": 0.2263,
      "rewards/chosen": -0.4647727272727273,
      "rewards/margins": 12.384542341220424,
      "rewards/rejected": -12.849315068493151,
      "step": 1107
    },
    {
      "epoch": 0.5458128078817734,
      "grad_norm": 0.45899251222058113,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -51080630.85714286,
      "logits/rejected": -52038632.18604651,
      "logps/chosen": -246.66666666666666,
      "logps/rejected": -452.4651162790698,
      "loss": 0.1524,
      "rewards/chosen": 4.357141767229352,
      "rewards/margins": 16.252490604438655,
      "rewards/rejected": -11.895348837209303,
      "step": 1108
    },
    {
      "epoch": 0.5463054187192118,
      "grad_norm": 0.5269016641472818,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -74332387.55555555,
      "logits/rejected": -66882144.86486486,
      "logps/chosen": -307.55555555555554,
      "logps/rejected": -546.5945945945946,
      "loss": 0.2119,
      "rewards/chosen": 0.0005787037037037037,
      "rewards/margins": 12.041119244244245,
      "rewards/rejected": -12.04054054054054,
      "step": 1109
    },
    {
      "epoch": 0.5467980295566502,
      "grad_norm": 0.4558682752420919,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -45330747.07692308,
      "logits/rejected": -68488569.26315789,
      "logps/chosen": -207.69230769230768,
      "logps/rejected": -577.6842105263158,
      "loss": 0.1899,
      "rewards/chosen": 2.8954444298377404,
      "rewards/margins": 17.369128640364057,
      "rewards/rejected": -14.473684210526315,
      "step": 1110
    },
    {
      "epoch": 0.5472906403940887,
      "grad_norm": 0.41602787427311544,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -57514393.6,
      "logits/rejected": -60340782.54545455,
      "logps/chosen": -382.4,
      "logps/rejected": -549.8181818181819,
      "loss": 0.1714,
      "rewards/chosen": -0.290625,
      "rewards/margins": 14.323011363636363,
      "rewards/rejected": -14.613636363636363,
      "step": 1111
    },
    {
      "epoch": 0.547783251231527,
      "grad_norm": 0.5138689292922286,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -58046171.428571425,
      "logits/rejected": -61283441.777777776,
      "logps/chosen": -362.2857142857143,
      "logps/rejected": -531.1111111111111,
      "loss": 0.2129,
      "rewards/chosen": 0.014508928571428572,
      "rewards/margins": 13.695064484126984,
      "rewards/rejected": -13.680555555555555,
      "step": 1112
    },
    {
      "epoch": 0.5482758620689655,
      "grad_norm": 0.5224280413315137,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -52773108.53731343,
      "logits/rejected": -70134263.60655738,
      "logps/chosen": -366.8059701492537,
      "logps/rejected": -550.2950819672132,
      "loss": 0.2486,
      "rewards/chosen": -0.28043376865671643,
      "rewards/margins": 13.22776295265476,
      "rewards/rejected": -13.508196721311476,
      "step": 1113
    },
    {
      "epoch": 0.548768472906404,
      "grad_norm": 0.4528003778412072,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -39196769.52380952,
      "logits/rejected": -59256736.744186044,
      "logps/chosen": -262.4761904761905,
      "logps/rejected": -483.3488372093023,
      "loss": 0.167,
      "rewards/chosen": 4.357873099190848,
      "rewards/margins": 16.206710308493175,
      "rewards/rejected": -11.848837209302326,
      "step": 1114
    },
    {
      "epoch": 0.5492610837438424,
      "grad_norm": 0.6080930539621097,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -47302428.44444445,
      "logits/rejected": -51805322.37837838,
      "logps/chosen": -347.55555555555554,
      "logps/rejected": -425.0810810810811,
      "loss": 0.2361,
      "rewards/chosen": 0.44212962962962965,
      "rewards/margins": 13.13131881881882,
      "rewards/rejected": -12.68918918918919,
      "step": 1115
    },
    {
      "epoch": 0.5497536945812808,
      "grad_norm": 0.49764562754161373,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -63543705.6,
      "logits/rejected": -66107950.54545455,
      "logps/chosen": -293.6,
      "logps/rejected": -530.1818181818181,
      "loss": 0.1815,
      "rewards/chosen": -0.6421875,
      "rewards/margins": 12.198721590909091,
      "rewards/rejected": -12.840909090909092,
      "step": 1116
    },
    {
      "epoch": 0.5502463054187192,
      "grad_norm": 0.5251032764830523,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -52149179.733333334,
      "logits/rejected": -52605668.240963854,
      "logps/chosen": -287.46666666666664,
      "logps/rejected": -405.9759036144578,
      "loss": 0.1634,
      "rewards/chosen": 3.1978888617621526,
      "rewards/margins": 14.270178018388657,
      "rewards/rejected": -11.072289156626505,
      "step": 1117
    },
    {
      "epoch": 0.5507389162561577,
      "grad_norm": 0.4214389968009757,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -42863740.87804878,
      "logits/rejected": -70869274.48275863,
      "logps/chosen": -227.90243902439025,
      "logps/rejected": -576.735632183908,
      "loss": 0.1573,
      "rewards/chosen": 7.721779427877286,
      "rewards/margins": 21.353963335923265,
      "rewards/rejected": -13.632183908045977,
      "step": 1118
    },
    {
      "epoch": 0.551231527093596,
      "grad_norm": 0.5017526279082771,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -63983696.313725494,
      "logits/rejected": -63949518.12987013,
      "logps/chosen": -306.03921568627453,
      "logps/rejected": -543.5844155844156,
      "loss": 0.1875,
      "rewards/chosen": 2.8106635598575367,
      "rewards/margins": 15.239234988428965,
      "rewards/rejected": -12.428571428571429,
      "step": 1119
    },
    {
      "epoch": 0.5517241379310345,
      "grad_norm": 0.43692760944804315,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -52518040.5106383,
      "logits/rejected": -66901737.87654321,
      "logps/chosen": -295.1489361702128,
      "logps/rejected": -584.6913580246913,
      "loss": 0.16,
      "rewards/chosen": 0.6502659574468085,
      "rewards/margins": 13.625574599422118,
      "rewards/rejected": -12.975308641975309,
      "step": 1120
    },
    {
      "epoch": 0.5522167487684729,
      "grad_norm": 0.42908289842138314,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -55419183.40740741,
      "logits/rejected": -60874087.783783786,
      "logps/chosen": -313.6296296296296,
      "logps/rejected": -524.5405405405405,
      "loss": 0.2073,
      "rewards/chosen": 0.5677083333333334,
      "rewards/margins": 13.135275900900902,
      "rewards/rejected": -12.567567567567568,
      "step": 1121
    },
    {
      "epoch": 0.5527093596059113,
      "grad_norm": 0.4969418718066682,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -39929774.08,
      "logits/rejected": -64527753.84615385,
      "logps/chosen": -212.16,
      "logps/rejected": -475.0769230769231,
      "loss": 0.1823,
      "rewards/chosen": -0.33,
      "rewards/margins": 11.40076923076923,
      "rewards/rejected": -11.73076923076923,
      "step": 1122
    },
    {
      "epoch": 0.5532019704433497,
      "grad_norm": 0.5169934389251244,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -55297545.66037736,
      "logits/rejected": -62075699.2,
      "logps/chosen": -256.75471698113205,
      "logps/rejected": -538.0266666666666,
      "loss": 0.1684,
      "rewards/chosen": 2.740559128095519,
      "rewards/margins": 12.513892461428853,
      "rewards/rejected": -9.773333333333333,
      "step": 1123
    },
    {
      "epoch": 0.5536945812807882,
      "grad_norm": 0.40666799127102266,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -49946456.81632653,
      "logits/rejected": -64188778.93670886,
      "logps/chosen": -260.2448979591837,
      "logps/rejected": -465.82278481012656,
      "loss": 0.2047,
      "rewards/chosen": 0.998405612244898,
      "rewards/margins": 11.31486130844743,
      "rewards/rejected": -10.316455696202532,
      "step": 1124
    },
    {
      "epoch": 0.5541871921182266,
      "grad_norm": 0.5112933380269145,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -45068207.686274506,
      "logits/rejected": -70159266.9090909,
      "logps/chosen": -283.921568627451,
      "logps/rejected": -521.974025974026,
      "loss": 0.2047,
      "rewards/chosen": -0.2916666666666667,
      "rewards/margins": 10.43560606060606,
      "rewards/rejected": -10.727272727272727,
      "step": 1125
    },
    {
      "epoch": 0.554679802955665,
      "grad_norm": 0.4749429617845848,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -47535445.333333336,
      "logits/rejected": -48339353.6,
      "logps/chosen": -258.9166666666667,
      "logps/rejected": -455.6,
      "loss": 0.2098,
      "rewards/chosen": -0.5133463541666666,
      "rewards/margins": 9.649153645833334,
      "rewards/rejected": -10.1625,
      "step": 1126
    },
    {
      "epoch": 0.5551724137931034,
      "grad_norm": 0.572715649001079,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -62632251.07692308,
      "logits/rejected": -59603267.368421055,
      "logps/chosen": -296.3076923076923,
      "logps/rejected": -515.3684210526316,
      "loss": 0.2106,
      "rewards/chosen": 0.6634615384615384,
      "rewards/margins": 11.29504048582996,
      "rewards/rejected": -10.631578947368421,
      "step": 1127
    },
    {
      "epoch": 0.5556650246305419,
      "grad_norm": 0.4686344359448969,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -41085114.18181818,
      "logits/rejected": -84900830.96774194,
      "logps/chosen": -346.42424242424244,
      "logps/rejected": -589.4193548387096,
      "loss": 0.1896,
      "rewards/chosen": 2.106060606060606,
      "rewards/margins": 15.138318670576735,
      "rewards/rejected": -13.03225806451613,
      "step": 1128
    },
    {
      "epoch": 0.5561576354679802,
      "grad_norm": 0.5332992311351586,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -78040826.55319148,
      "logits/rejected": -56441868.641975306,
      "logps/chosen": -358.468085106383,
      "logps/rejected": -541.2345679012345,
      "loss": 0.2116,
      "rewards/chosen": 0.32779255319148937,
      "rewards/margins": 12.574706133438402,
      "rewards/rejected": -12.246913580246913,
      "step": 1129
    },
    {
      "epoch": 0.5566502463054187,
      "grad_norm": 0.4107928770280995,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -59652323.55555555,
      "logits/rejected": -61550147.855421685,
      "logps/chosen": -246.04444444444445,
      "logps/rejected": -495.03614457831327,
      "loss": 0.1563,
      "rewards/chosen": 0.6875,
      "rewards/margins": 12.699548192771084,
      "rewards/rejected": -12.012048192771084,
      "step": 1130
    },
    {
      "epoch": 0.5571428571428572,
      "grad_norm": 0.41718908285392114,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -50382798.048780486,
      "logits/rejected": -57466785.83908046,
      "logps/chosen": -350.4390243902439,
      "logps/rejected": -473.01149425287355,
      "loss": 0.1823,
      "rewards/chosen": 3.2721326874523626,
      "rewards/margins": 13.421557974808685,
      "rewards/rejected": -10.149425287356323,
      "step": 1131
    },
    {
      "epoch": 0.5576354679802956,
      "grad_norm": 0.43479516358415693,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -60867340.190476194,
      "logits/rejected": -56720645.95348837,
      "logps/chosen": -265.9047619047619,
      "logps/rejected": -488.1860465116279,
      "loss": 0.172,
      "rewards/chosen": 5.851970490955171,
      "rewards/margins": 16.46824956072261,
      "rewards/rejected": -10.616279069767442,
      "step": 1132
    },
    {
      "epoch": 0.558128078817734,
      "grad_norm": 0.518922442067822,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -49202412.307692304,
      "logits/rejected": -71523920.84210527,
      "logps/chosen": -259.6923076923077,
      "logps/rejected": -510.3157894736842,
      "loss": 0.1981,
      "rewards/chosen": 4.530643463134766,
      "rewards/margins": 15.714853989450555,
      "rewards/rejected": -11.18421052631579,
      "step": 1133
    },
    {
      "epoch": 0.5586206896551724,
      "grad_norm": 0.4197312038182414,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -86293921.18518518,
      "logits/rejected": -69942853.1891892,
      "logps/chosen": -210.37037037037038,
      "logps/rejected": -517.1891891891892,
      "loss": 0.2051,
      "rewards/chosen": 0.9377893518518519,
      "rewards/margins": 8.478329892392392,
      "rewards/rejected": -7.54054054054054,
      "step": 1134
    },
    {
      "epoch": 0.5591133004926109,
      "grad_norm": 0.5106910842144342,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -57420021.76,
      "logits/rejected": -60118357.333333336,
      "logps/chosen": -330.24,
      "logps/rejected": -505.43589743589746,
      "loss": 0.2339,
      "rewards/chosen": 2.809354248046875,
      "rewards/margins": 14.47602091471354,
      "rewards/rejected": -11.666666666666666,
      "step": 1135
    },
    {
      "epoch": 0.5596059113300492,
      "grad_norm": 0.42880438473092486,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -47102033.92,
      "logits/rejected": -60602315.48717949,
      "logps/chosen": -328.0,
      "logps/rejected": -518.1538461538462,
      "loss": 0.1968,
      "rewards/chosen": 5.29971435546875,
      "rewards/margins": 16.19715025290465,
      "rewards/rejected": -10.897435897435898,
      "step": 1136
    },
    {
      "epoch": 0.5600985221674877,
      "grad_norm": 0.4317124266494348,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -55355058.60465116,
      "logits/rejected": -56055639.34117647,
      "logps/chosen": -251.1627906976744,
      "logps/rejected": -490.16470588235296,
      "loss": 0.1736,
      "rewards/chosen": 3.626239155614099,
      "rewards/margins": 13.685062685025864,
      "rewards/rejected": -10.058823529411764,
      "step": 1137
    },
    {
      "epoch": 0.5605911330049261,
      "grad_norm": 0.4197439552239474,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -39131963.91489362,
      "logits/rejected": -57373936.197530866,
      "logps/chosen": -211.74468085106383,
      "logps/rejected": -485.9259259259259,
      "loss": 0.1871,
      "rewards/chosen": 1.1373005319148937,
      "rewards/margins": 7.794744108303783,
      "rewards/rejected": -6.657443576388889,
      "step": 1138
    },
    {
      "epoch": 0.5610837438423645,
      "grad_norm": 0.6104998579463203,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -56623104.0,
      "logits/rejected": -63380593.777777776,
      "logps/chosen": -336.3404255319149,
      "logps/rejected": -557.0370370370371,
      "loss": 0.1809,
      "rewards/chosen": 0.9502992021276596,
      "rewards/margins": 7.624635953429743,
      "rewards/rejected": -6.674336751302083,
      "step": 1139
    },
    {
      "epoch": 0.5615763546798029,
      "grad_norm": 0.6431923514995186,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -50606684.32786885,
      "logits/rejected": -71115363.34328358,
      "logps/chosen": -272.5245901639344,
      "logps/rejected": -505.7910447761194,
      "loss": 0.1893,
      "rewards/chosen": 1.8698770491803278,
      "rewards/margins": 12.705697944702717,
      "rewards/rejected": -10.835820895522389,
      "step": 1140
    },
    {
      "epoch": 0.5620689655172414,
      "grad_norm": 0.5300333621625216,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -62914560.0,
      "logits/rejected": -63495309.78461538,
      "logps/chosen": -288.76190476190476,
      "logps/rejected": -523.3230769230769,
      "loss": 0.2228,
      "rewards/chosen": 0.8988095238095238,
      "rewards/margins": 7.954504103887649,
      "rewards/rejected": -7.055694580078125,
      "step": 1141
    },
    {
      "epoch": 0.5625615763546798,
      "grad_norm": 0.46860575664963683,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -51401623.510204084,
      "logits/rejected": -52667716.05063291,
      "logps/chosen": -272.0,
      "logps/rejected": -469.46835443037975,
      "loss": 0.1854,
      "rewards/chosen": 0.26658163265306123,
      "rewards/margins": 12.25392340480496,
      "rewards/rejected": -11.987341772151899,
      "step": 1142
    },
    {
      "epoch": 0.5630541871921182,
      "grad_norm": 0.4331667232000017,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -47810072.38095238,
      "logits/rejected": -63694895.62790698,
      "logps/chosen": -290.6666666666667,
      "logps/rejected": -465.1162790697674,
      "loss": 0.1844,
      "rewards/chosen": 1.9248609996977306,
      "rewards/margins": 13.471372627604708,
      "rewards/rejected": -11.546511627906977,
      "step": 1143
    },
    {
      "epoch": 0.5635467980295567,
      "grad_norm": 0.5639102050780369,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -50331648.0,
      "logits/rejected": -58770789.78313253,
      "logps/chosen": -337.4222222222222,
      "logps/rejected": -541.3012048192771,
      "loss": 0.1562,
      "rewards/chosen": 5.177836778428819,
      "rewards/margins": 14.040730491954799,
      "rewards/rejected": -8.86289371352598,
      "step": 1144
    },
    {
      "epoch": 0.5640394088669951,
      "grad_norm": 0.5297200241411447,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -60212460.307692304,
      "logits/rejected": -67826310.73684211,
      "logps/chosen": -268.0,
      "logps/rejected": -586.9473684210526,
      "loss": 0.1865,
      "rewards/chosen": 1.2043644831730769,
      "rewards/margins": 13.928048693699392,
      "rewards/rejected": -12.723684210526315,
      "step": 1145
    },
    {
      "epoch": 0.5645320197044335,
      "grad_norm": 0.4568418350050017,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -51435412.21052632,
      "logits/rejected": -65956907.26760563,
      "logps/chosen": -308.7719298245614,
      "logps/rejected": -508.84507042253523,
      "loss": 0.1875,
      "rewards/chosen": 1.469298245614035,
      "rewards/margins": 14.342537682233754,
      "rewards/rejected": -12.873239436619718,
      "step": 1146
    },
    {
      "epoch": 0.5650246305418719,
      "grad_norm": 1.4480251222692284,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -69206016.0,
      "logits/rejected": -57288054.63414634,
      "logps/chosen": -361.39130434782606,
      "logps/rejected": -515.9024390243902,
      "loss": 0.192,
      "rewards/chosen": 7.351940652598506,
      "rewards/margins": 19.851940652598508,
      "rewards/rejected": -12.5,
      "step": 1147
    },
    {
      "epoch": 0.5655172413793104,
      "grad_norm": 0.4033919003524666,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -59034828.8,
      "logits/rejected": -69777966.54545455,
      "logps/chosen": -234.0,
      "logps/rejected": -592.7272727272727,
      "loss": 0.1628,
      "rewards/chosen": 2.2676017761230467,
      "rewards/margins": 15.767601776123048,
      "rewards/rejected": -13.5,
      "step": 1148
    },
    {
      "epoch": 0.5660098522167488,
      "grad_norm": 0.5311534711576843,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -49246914.20689655,
      "logits/rejected": -55784243.2,
      "logps/chosen": -244.13793103448276,
      "logps/rejected": -573.2571428571429,
      "loss": 0.22,
      "rewards/chosen": 0.6869612068965517,
      "rewards/margins": 10.68562017732066,
      "rewards/rejected": -9.998658970424108,
      "step": 1149
    },
    {
      "epoch": 0.5665024630541872,
      "grad_norm": 0.4299602569962629,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -54351189.333333336,
      "logits/rejected": -52900659.2,
      "logps/chosen": -320.6666666666667,
      "logps/rejected": -478.0,
      "loss": 0.2269,
      "rewards/chosen": -0.9358723958333334,
      "rewards/margins": 9.614127604166667,
      "rewards/rejected": -10.55,
      "step": 1150
    },
    {
      "epoch": 0.5669950738916256,
      "grad_norm": 0.4851700440242502,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -53004488.78431372,
      "logits/rejected": -55070666.8051948,
      "logps/chosen": -324.3921568627451,
      "logps/rejected": -486.64935064935065,
      "loss": 0.1796,
      "rewards/chosen": 2.417584587545956,
      "rewards/margins": 14.248753418714788,
      "rewards/rejected": -11.831168831168831,
      "step": 1151
    },
    {
      "epoch": 0.5674876847290641,
      "grad_norm": 0.4831011522466132,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -52040438.518518515,
      "logits/rejected": -61440885.62162162,
      "logps/chosen": -235.55555555555554,
      "logps/rejected": -522.3783783783783,
      "loss": 0.2001,
      "rewards/chosen": 1.0584490740740742,
      "rewards/margins": 6.048205477816684,
      "rewards/rejected": -4.98975640374261,
      "step": 1152
    },
    {
      "epoch": 0.5679802955665024,
      "grad_norm": 0.5573355051903316,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -53576298.26415094,
      "logits/rejected": -56371445.76,
      "logps/chosen": -306.41509433962267,
      "logps/rejected": -441.17333333333335,
      "loss": 0.2057,
      "rewards/chosen": -0.1494693396226415,
      "rewards/margins": 10.823863993710692,
      "rewards/rejected": -10.973333333333333,
      "step": 1153
    },
    {
      "epoch": 0.5684729064039409,
      "grad_norm": 0.5156405374897425,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -64897321.89090909,
      "logits/rejected": -69234744.10958904,
      "logps/chosen": -367.41818181818184,
      "logps/rejected": -471.67123287671234,
      "loss": 0.2096,
      "rewards/chosen": 2.9954845081676136,
      "rewards/margins": 13.557128343784052,
      "rewards/rejected": -10.561643835616438,
      "step": 1154
    },
    {
      "epoch": 0.5689655172413793,
      "grad_norm": 0.510464681005792,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -54274293.76,
      "logits/rejected": -61193819.897435896,
      "logps/chosen": -243.36,
      "logps/rejected": -520.2051282051282,
      "loss": 0.1929,
      "rewards/chosen": 0.97125,
      "rewards/margins": 9.534547369541265,
      "rewards/rejected": -8.563297369541266,
      "step": 1155
    },
    {
      "epoch": 0.5694581280788177,
      "grad_norm": 0.6464182360758514,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -54442065.92,
      "logits/rejected": -55655187.692307696,
      "logps/chosen": -318.72,
      "logps/rejected": -500.5128205128205,
      "loss": 0.203,
      "rewards/chosen": 2.85440185546875,
      "rewards/margins": 12.700555701622596,
      "rewards/rejected": -9.846153846153847,
      "step": 1156
    },
    {
      "epoch": 0.5699507389162561,
      "grad_norm": 0.5335730346017434,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -67382405.56521739,
      "logits/rejected": -52684550.24390244,
      "logps/chosen": -308.5217391304348,
      "logps/rejected": -477.2682926829268,
      "loss": 0.1659,
      "rewards/chosen": 1.0679347826086956,
      "rewards/margins": 12.21427624602333,
      "rewards/rejected": -11.146341463414634,
      "step": 1157
    },
    {
      "epoch": 0.5704433497536946,
      "grad_norm": 0.40789601006597237,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -59601059.84,
      "logits/rejected": -63828703.179487176,
      "logps/chosen": -257.92,
      "logps/rejected": -573.5384615384615,
      "loss": 0.16,
      "rewards/chosen": 3.282861328125,
      "rewards/margins": 14.872604917868589,
      "rewards/rejected": -11.58974358974359,
      "step": 1158
    },
    {
      "epoch": 0.570935960591133,
      "grad_norm": 0.47401212282059463,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -60118357.333333336,
      "logits/rejected": -97612893.0909091,
      "logps/chosen": -307.7647058823529,
      "logps/rejected": -580.987012987013,
      "loss": 0.1832,
      "rewards/chosen": 1.2181372549019607,
      "rewards/margins": 12.880474917239624,
      "rewards/rejected": -11.662337662337663,
      "step": 1159
    },
    {
      "epoch": 0.5714285714285714,
      "grad_norm": 0.4797008332716378,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -52607281.02127659,
      "logits/rejected": -51419060.14814815,
      "logps/chosen": -274.8936170212766,
      "logps/rejected": -417.18518518518516,
      "loss": 0.2008,
      "rewards/chosen": -0.3271276595744681,
      "rewards/margins": 10.154353821907012,
      "rewards/rejected": -10.481481481481481,
      "step": 1160
    },
    {
      "epoch": 0.5719211822660099,
      "grad_norm": 0.42432458029783837,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -36678760.489795916,
      "logits/rejected": -47464655.39240506,
      "logps/chosen": -245.55102040816325,
      "logps/rejected": -481.62025316455697,
      "loss": 0.1768,
      "rewards/chosen": 0.26403061224489793,
      "rewards/margins": 10.086815422371481,
      "rewards/rejected": -9.822784810126583,
      "step": 1161
    },
    {
      "epoch": 0.5724137931034483,
      "grad_norm": 0.5934128699604491,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -56623104.0,
      "logits/rejected": -62533259.63636363,
      "logps/chosen": -291.7647058823529,
      "logps/rejected": -499.94805194805195,
      "loss": 0.2238,
      "rewards/chosen": 2.3908852969898895,
      "rewards/margins": 14.053222959327552,
      "rewards/rejected": -11.662337662337663,
      "step": 1162
    },
    {
      "epoch": 0.5729064039408867,
      "grad_norm": 0.49459473961258793,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -59841147.5862069,
      "logits/rejected": -51829613.71428572,
      "logps/chosen": -276.9655172413793,
      "logps/rejected": -498.74285714285713,
      "loss": 0.2274,
      "rewards/chosen": 0.2413793103448276,
      "rewards/margins": 12.68423645320197,
      "rewards/rejected": -12.442857142857143,
      "step": 1163
    },
    {
      "epoch": 0.5733990147783251,
      "grad_norm": 0.5752126929861624,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -48557134.76923077,
      "logits/rejected": -53974069.89473684,
      "logps/chosen": -257.53846153846155,
      "logps/rejected": -494.3157894736842,
      "loss": 0.2395,
      "rewards/chosen": 2.7049146798940806,
      "rewards/margins": 12.402283100946713,
      "rewards/rejected": -9.697368421052632,
      "step": 1164
    },
    {
      "epoch": 0.5738916256157636,
      "grad_norm": 0.3997198569530107,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -60406201.7254902,
      "logits/rejected": -63786103.68831169,
      "logps/chosen": -299.29411764705884,
      "logps/rejected": -514.9090909090909,
      "loss": 0.149,
      "rewards/chosen": 2.382912430108762,
      "rewards/margins": 11.759535806732138,
      "rewards/rejected": -9.376623376623376,
      "step": 1165
    },
    {
      "epoch": 0.574384236453202,
      "grad_norm": 0.5653796692630289,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -60436107.63636363,
      "logits/rejected": -50381580.190476194,
      "logps/chosen": -329.45454545454544,
      "logps/rejected": -484.1904761904762,
      "loss": 0.2089,
      "rewards/chosen": -0.3302556818181818,
      "rewards/margins": 9.24117288961039,
      "rewards/rejected": -9.571428571428571,
      "step": 1166
    },
    {
      "epoch": 0.5748768472906404,
      "grad_norm": 0.49308157110456025,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -49340267.054545455,
      "logits/rejected": -50331648.0,
      "logps/chosen": -303.7090909090909,
      "logps/rejected": -448.43835616438355,
      "loss": 0.2056,
      "rewards/chosen": 5.494342595880682,
      "rewards/margins": 16.480643965743695,
      "rewards/rejected": -10.986301369863014,
      "step": 1167
    },
    {
      "epoch": 0.5753694581280788,
      "grad_norm": 0.46559556947935915,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -46806647.82978723,
      "logits/rejected": -48726420.54320987,
      "logps/chosen": -290.3829787234043,
      "logps/rejected": -446.81481481481484,
      "loss": 0.2133,
      "rewards/chosen": 3.1489400660738034,
      "rewards/margins": 13.025483275950346,
      "rewards/rejected": -9.876543209876543,
      "step": 1168
    },
    {
      "epoch": 0.5758620689655173,
      "grad_norm": 0.5202056559976711,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -56337128.72727273,
      "logits/rejected": -59469238.85714286,
      "logps/chosen": -271.6363636363636,
      "logps/rejected": -484.57142857142856,
      "loss": 0.1505,
      "rewards/chosen": 3.4344839616255327,
      "rewards/margins": 13.898769675911247,
      "rewards/rejected": -10.464285714285714,
      "step": 1169
    },
    {
      "epoch": 0.5763546798029556,
      "grad_norm": 0.4546140921202635,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -38059425.18518519,
      "logits/rejected": -53619075.45945946,
      "logps/chosen": -254.22222222222223,
      "logps/rejected": -459.2432432432432,
      "loss": 0.2126,
      "rewards/chosen": 2.4791847511574074,
      "rewards/margins": 10.3710766430493,
      "rewards/rejected": -7.891891891891892,
      "step": 1170
    },
    {
      "epoch": 0.5768472906403941,
      "grad_norm": 0.43961419002535856,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -55962889.481481485,
      "logits/rejected": -56623104.0,
      "logps/chosen": -324.14814814814815,
      "logps/rejected": -477.4054054054054,
      "loss": 0.1789,
      "rewards/chosen": 1.306712962962963,
      "rewards/margins": 11.421577827827829,
      "rewards/rejected": -10.114864864864865,
      "step": 1171
    },
    {
      "epoch": 0.5773399014778325,
      "grad_norm": 0.40684590363451845,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -46849584.30188679,
      "logits/rejected": -68618813.44,
      "logps/chosen": -298.2641509433962,
      "logps/rejected": -529.92,
      "loss": 0.1791,
      "rewards/chosen": 5.13683448647553,
      "rewards/margins": 14.81683448647553,
      "rewards/rejected": -9.68,
      "step": 1172
    },
    {
      "epoch": 0.5778325123152709,
      "grad_norm": 0.49830296811905544,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -43538699.13043478,
      "logits/rejected": -66188163.12195122,
      "logps/chosen": -266.2608695652174,
      "logps/rejected": -473.3658536585366,
      "loss": 0.2362,
      "rewards/chosen": -0.36141304347826086,
      "rewards/margins": 8.83370890774125,
      "rewards/rejected": -9.195121951219512,
      "step": 1173
    },
    {
      "epoch": 0.5783251231527093,
      "grad_norm": 0.5093250837180837,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -54937892.571428575,
      "logits/rejected": -54991985.777777776,
      "logps/chosen": -346.85714285714283,
      "logps/rejected": -476.8888888888889,
      "loss": 0.2081,
      "rewards/chosen": 2.4513457162039622,
      "rewards/margins": 11.243012382870628,
      "rewards/rejected": -8.791666666666666,
      "step": 1174
    },
    {
      "epoch": 0.5788177339901478,
      "grad_norm": 0.40723587233095876,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -55424731.428571425,
      "logits/rejected": -53599303.44186047,
      "logps/chosen": -256.1904761904762,
      "logps/rejected": -462.8837209302326,
      "loss": 0.1521,
      "rewards/chosen": 1.0422247023809523,
      "rewards/margins": 11.391061911683279,
      "rewards/rejected": -10.348837209302326,
      "step": 1175
    },
    {
      "epoch": 0.5793103448275863,
      "grad_norm": 0.45994669852886766,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -58427630.13953488,
      "logits/rejected": -62371767.71764706,
      "logps/chosen": -314.4186046511628,
      "logps/rejected": -533.0823529411765,
      "loss": 0.1644,
      "rewards/chosen": 5.303045228470204,
      "rewards/margins": 15.903045228470203,
      "rewards/rejected": -10.6,
      "step": 1176
    },
    {
      "epoch": 0.5798029556650246,
      "grad_norm": 0.4749144132782511,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -50240467.47826087,
      "logits/rejected": -56009303.414634146,
      "logps/chosen": -270.2608695652174,
      "logps/rejected": -508.8780487804878,
      "loss": 0.156,
      "rewards/chosen": 5.580882860266644,
      "rewards/margins": 17.19063895782762,
      "rewards/rejected": -11.609756097560975,
      "step": 1177
    },
    {
      "epoch": 0.5802955665024631,
      "grad_norm": 0.49428713463592466,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -62871760.97959184,
      "logits/rejected": -46508991.18987342,
      "logps/chosen": -290.2857142857143,
      "logps/rejected": -458.53164556962025,
      "loss": 0.1945,
      "rewards/chosen": 0.41541772959183676,
      "rewards/margins": 11.883772159971585,
      "rewards/rejected": -11.468354430379748,
      "step": 1178
    },
    {
      "epoch": 0.5807881773399015,
      "grad_norm": 0.510308980405452,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -47710208.0,
      "logits/rejected": -56899045.05263158,
      "logps/chosen": -318.7692307692308,
      "logps/rejected": -484.2105263157895,
      "loss": 0.1632,
      "rewards/chosen": 1.9555288461538463,
      "rewards/margins": 11.797634109311742,
      "rewards/rejected": -9.842105263157896,
      "step": 1179
    },
    {
      "epoch": 0.5812807881773399,
      "grad_norm": 0.4599625150179965,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -53937277.75438596,
      "logits/rejected": -52990009.69014084,
      "logps/chosen": -303.719298245614,
      "logps/rejected": -451.6056338028169,
      "loss": 0.2037,
      "rewards/chosen": 0.40789473684210525,
      "rewards/margins": 10.605077835433654,
      "rewards/rejected": -10.19718309859155,
      "step": 1180
    },
    {
      "epoch": 0.5817733990147783,
      "grad_norm": 0.4013851386720798,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -48669753.96226415,
      "logits/rejected": -54973344.42666667,
      "logps/chosen": -263.0943396226415,
      "logps/rejected": -525.6533333333333,
      "loss": 0.1569,
      "rewards/chosen": 1.1058372641509433,
      "rewards/margins": 13.145837264150943,
      "rewards/rejected": -12.04,
      "step": 1181
    },
    {
      "epoch": 0.5822660098522168,
      "grad_norm": 0.39564176928169303,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -66698551.652173914,
      "logits/rejected": -61175458.34146342,
      "logps/chosen": -372.5217391304348,
      "logps/rejected": -558.829268292683,
      "loss": 0.1783,
      "rewards/chosen": 3.3872448465098506,
      "rewards/margins": 16.570171675778145,
      "rewards/rejected": -13.182926829268293,
      "step": 1182
    },
    {
      "epoch": 0.5827586206896552,
      "grad_norm": 0.5931794515084092,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -49797079.84313726,
      "logits/rejected": -53654408.31168831,
      "logps/chosen": -246.58823529411765,
      "logps/rejected": -506.1818181818182,
      "loss": 0.1767,
      "rewards/chosen": 1.5407475490196079,
      "rewards/margins": 14.410877419149479,
      "rewards/rejected": -12.87012987012987,
      "step": 1183
    },
    {
      "epoch": 0.5832512315270936,
      "grad_norm": 0.408325337346025,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -41622641.777777776,
      "logits/rejected": -68469178.8108108,
      "logps/chosen": -261.037037037037,
      "logps/rejected": -550.4864864864865,
      "loss": 0.1754,
      "rewards/chosen": 1.0083912037037037,
      "rewards/margins": 16.751634446946948,
      "rewards/rejected": -15.743243243243244,
      "step": 1184
    },
    {
      "epoch": 0.583743842364532,
      "grad_norm": 0.53308968311824,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -42816853.333333336,
      "logits/rejected": -60660121.6,
      "logps/chosen": -276.3333333333333,
      "logps/rejected": -565.6,
      "loss": 0.187,
      "rewards/chosen": -0.11360677083333333,
      "rewards/margins": 12.823893229166666,
      "rewards/rejected": -12.9375,
      "step": 1185
    },
    {
      "epoch": 0.5842364532019705,
      "grad_norm": 0.6451257633224206,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -39699575.069767445,
      "logits/rejected": -68293138.07058823,
      "logps/chosen": -202.7906976744186,
      "logps/rejected": -579.0117647058823,
      "loss": 0.1495,
      "rewards/chosen": 3.354769684547602,
      "rewards/margins": 17.14300497866525,
      "rewards/rejected": -13.788235294117648,
      "step": 1186
    },
    {
      "epoch": 0.5847290640394088,
      "grad_norm": 0.4157869444035907,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -56737494.10909091,
      "logits/rejected": -66304476.93150685,
      "logps/chosen": -316.2181818181818,
      "logps/rejected": -504.1095890410959,
      "loss": 0.1866,
      "rewards/chosen": 0.06079545454545455,
      "rewards/margins": 14.047096824408468,
      "rewards/rejected": -13.986301369863014,
      "step": 1187
    },
    {
      "epoch": 0.5852216748768473,
      "grad_norm": 0.3993555467518719,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -45134358.26086956,
      "logits/rejected": -60970858.146341465,
      "logps/chosen": -291.4782608695652,
      "logps/rejected": -556.4878048780488,
      "loss": 0.1658,
      "rewards/chosen": 0.47690217391304346,
      "rewards/margins": 14.30617046659597,
      "rewards/rejected": -13.829268292682928,
      "step": 1188
    },
    {
      "epoch": 0.5857142857142857,
      "grad_norm": 0.6289208770780652,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -51179432.85106383,
      "logits/rejected": -66487485.62962963,
      "logps/chosen": -244.25531914893617,
      "logps/rejected": -550.716049382716,
      "loss": 0.1995,
      "rewards/chosen": 0.3836436170212766,
      "rewards/margins": 11.91450781455214,
      "rewards/rejected": -11.530864197530864,
      "step": 1189
    },
    {
      "epoch": 0.5862068965517241,
      "grad_norm": 0.5245371593247741,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -47427899.07692308,
      "logits/rejected": -54139634.526315786,
      "logps/chosen": -282.61538461538464,
      "logps/rejected": -500.2105263157895,
      "loss": 0.2064,
      "rewards/chosen": 2.5198499239408054,
      "rewards/margins": 14.375113081835542,
      "rewards/rejected": -11.855263157894736,
      "step": 1190
    },
    {
      "epoch": 0.5866995073891625,
      "grad_norm": 0.45810225615963945,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -48006544.69565217,
      "logits/rejected": -57543804.87804878,
      "logps/chosen": -224.8695652173913,
      "logps/rejected": -487.0243902439024,
      "loss": 0.1906,
      "rewards/chosen": 0.3498641304347826,
      "rewards/margins": 7.157680054358182,
      "rewards/rejected": -6.8078159239234,
      "step": 1191
    },
    {
      "epoch": 0.587192118226601,
      "grad_norm": 0.5081750305421365,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -57631350.15384615,
      "logits/rejected": -73786637.4736842,
      "logps/chosen": -272.0,
      "logps/rejected": -526.3157894736842,
      "loss": 0.204,
      "rewards/chosen": 3.1814539982722354,
      "rewards/margins": 14.234085577219604,
      "rewards/rejected": -11.052631578947368,
      "step": 1192
    },
    {
      "epoch": 0.5876847290640395,
      "grad_norm": 0.3820091436187533,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -71303168.0,
      "logits/rejected": -49950347.63636363,
      "logps/chosen": -408.7843137254902,
      "logps/rejected": -517.4025974025974,
      "loss": 0.2093,
      "rewards/chosen": -0.4362745098039216,
      "rewards/margins": 14.628660555131145,
      "rewards/rejected": -15.064935064935066,
      "step": 1193
    },
    {
      "epoch": 0.5881773399014778,
      "grad_norm": 0.4499321581712659,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -56289466.18181818,
      "logits/rejected": -61316729.9047619,
      "logps/chosen": -393.09090909090907,
      "logps/rejected": -495.23809523809524,
      "loss": 0.2069,
      "rewards/chosen": 2.1026840209960938,
      "rewards/margins": 14.590779259091331,
      "rewards/rejected": -12.488095238095237,
      "step": 1194
    },
    {
      "epoch": 0.5886699507389163,
      "grad_norm": 0.4140996180983965,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -46804619.63636363,
      "logits/rejected": -62614966.85714286,
      "logps/chosen": -227.0909090909091,
      "logps/rejected": -533.3333333333334,
      "loss": 0.1674,
      "rewards/chosen": 3.432557539506392,
      "rewards/margins": 17.003986110934964,
      "rewards/rejected": -13.571428571428571,
      "step": 1195
    },
    {
      "epoch": 0.5891625615763547,
      "grad_norm": 0.42475283787863133,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -76650905.6,
      "logits/rejected": -64344436.36363637,
      "logps/chosen": -413.6,
      "logps/rejected": -505.45454545454544,
      "loss": 0.1903,
      "rewards/chosen": 0.16796875,
      "rewards/margins": 12.599786931818182,
      "rewards/rejected": -12.431818181818182,
      "step": 1196
    },
    {
      "epoch": 0.5896551724137931,
      "grad_norm": 0.428039607823997,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -59786604.47457627,
      "logits/rejected": -54222016.927536234,
      "logps/chosen": -272.8135593220339,
      "logps/rejected": -549.1014492753624,
      "loss": 0.2474,
      "rewards/chosen": 1.7500201726363878,
      "rewards/margins": 16.3587158248103,
      "rewards/rejected": -14.608695652173912,
      "step": 1197
    },
    {
      "epoch": 0.5901477832512315,
      "grad_norm": 0.40374418682556534,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -42820917.58139535,
      "logits/rejected": -55315468.04705882,
      "logps/chosen": -305.1162790697674,
      "logps/rejected": -493.9294117647059,
      "loss": 0.1751,
      "rewards/chosen": 3.29142548317133,
      "rewards/margins": 15.49142548317133,
      "rewards/rejected": -12.2,
      "step": 1198
    },
    {
      "epoch": 0.59064039408867,
      "grad_norm": 0.48851508532864935,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -56371445.76,
      "logits/rejected": -76895573.33333333,
      "logps/chosen": -397.12,
      "logps/rejected": -564.1025641025641,
      "loss": 0.2289,
      "rewards/chosen": 1.052838134765625,
      "rewards/margins": 14.706684288611779,
      "rewards/rejected": -13.653846153846153,
      "step": 1199
    },
    {
      "epoch": 0.5911330049261084,
      "grad_norm": 0.47181307661098243,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -62736078.97872341,
      "logits/rejected": -50435211.061728396,
      "logps/chosen": -396.59574468085106,
      "logps/rejected": -511.60493827160496,
      "loss": 0.1508,
      "rewards/chosen": 2.0425531914893615,
      "rewards/margins": 15.178355660625163,
      "rewards/rejected": -13.135802469135802,
      "step": 1200
    },
    {
      "epoch": 0.5916256157635468,
      "grad_norm": 0.5169778500026613,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -50009009.23076923,
      "logits/rejected": -70420156.63157895,
      "logps/chosen": -269.53846153846155,
      "logps/rejected": -561.6842105263158,
      "loss": 0.218,
      "rewards/chosen": -0.80078125,
      "rewards/margins": 12.186060855263158,
      "rewards/rejected": -12.986842105263158,
      "step": 1201
    },
    {
      "epoch": 0.5921182266009852,
      "grad_norm": 0.49947843310646384,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -58720256.0,
      "logits/rejected": -53918881.684210524,
      "logps/chosen": -338.15384615384613,
      "logps/rejected": -501.89473684210526,
      "loss": 0.1594,
      "rewards/chosen": 1.3028846153846154,
      "rewards/margins": 14.171305668016194,
      "rewards/rejected": -12.868421052631579,
      "step": 1202
    },
    {
      "epoch": 0.5926108374384237,
      "grad_norm": 0.46738570253214257,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -52079274.666666664,
      "logits/rejected": -60768637.02325582,
      "logps/chosen": -225.52380952380952,
      "logps/rejected": -535.0697674418604,
      "loss": 0.1816,
      "rewards/chosen": 0.7916666666666666,
      "rewards/margins": 11.826550387596898,
      "rewards/rejected": -11.034883720930232,
      "step": 1203
    },
    {
      "epoch": 0.593103448275862,
      "grad_norm": 0.41500913403294726,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -49140084.36363637,
      "logits/rejected": -56123782.0952381,
      "logps/chosen": -278.3636363636364,
      "logps/rejected": -493.3333333333333,
      "loss": 0.1518,
      "rewards/chosen": 4.339540654962713,
      "rewards/margins": 17.184778750200806,
      "rewards/rejected": -12.845238095238095,
      "step": 1204
    },
    {
      "epoch": 0.5935960591133005,
      "grad_norm": 0.4398259107797025,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -54070049.39130435,
      "logits/rejected": -66290463.219512194,
      "logps/chosen": -304.69565217391306,
      "logps/rejected": -567.4146341463414,
      "loss": 0.1635,
      "rewards/chosen": 0.7615489130434783,
      "rewards/margins": 13.932280620360551,
      "rewards/rejected": -13.170731707317072,
      "step": 1205
    },
    {
      "epoch": 0.594088669950739,
      "grad_norm": 0.3671621115819196,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -54233326.13953488,
      "logits/rejected": -57832050.44705883,
      "logps/chosen": -335.4418604651163,
      "logps/rejected": -519.1529411764706,
      "loss": 0.1395,
      "rewards/chosen": 1.1090116279069768,
      "rewards/margins": 13.485482216142271,
      "rewards/rejected": -12.376470588235295,
      "step": 1206
    },
    {
      "epoch": 0.5945812807881773,
      "grad_norm": 0.4359894409968083,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -40616270.36734694,
      "logits/rejected": -72365017.11392406,
      "logps/chosen": -221.55102040816325,
      "logps/rejected": -475.9493670886076,
      "loss": 0.1701,
      "rewards/chosen": 1.6721938775510203,
      "rewards/margins": 14.127890080082667,
      "rewards/rejected": -12.455696202531646,
      "step": 1207
    },
    {
      "epoch": 0.5950738916256157,
      "grad_norm": 0.5063034158042833,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -58495561.14285714,
      "logits/rejected": -63380593.777777776,
      "logps/chosen": -242.28571428571428,
      "logps/rejected": -565.7777777777778,
      "loss": 0.193,
      "rewards/chosen": -0.08816964285714286,
      "rewards/margins": 14.911830357142858,
      "rewards/rejected": -15.0,
      "step": 1208
    },
    {
      "epoch": 0.5955665024630542,
      "grad_norm": 0.5973803077531763,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -77081986.84444444,
      "logits/rejected": -70140890.9879518,
      "logps/chosen": -382.5777777777778,
      "logps/rejected": -538.2168674698795,
      "loss": 0.1626,
      "rewards/chosen": -0.3076388888888889,
      "rewards/margins": 14.535734605087015,
      "rewards/rejected": -14.843373493975903,
      "step": 1209
    },
    {
      "epoch": 0.5960591133004927,
      "grad_norm": 0.500590008042594,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -59936604.16,
      "logits/rejected": -63129652.51282051,
      "logps/chosen": -311.68,
      "logps/rejected": -540.7179487179487,
      "loss": 0.2025,
      "rewards/chosen": 2.7812481689453126,
      "rewards/margins": 16.204325092022238,
      "rewards/rejected": -13.423076923076923,
      "step": 1210
    },
    {
      "epoch": 0.596551724137931,
      "grad_norm": 0.5115592710892428,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -111246597.95348836,
      "logits/rejected": -57535981.92941176,
      "logps/chosen": -256.0,
      "logps/rejected": -557.9294117647058,
      "loss": 0.1509,
      "rewards/chosen": 9.702808468840843,
      "rewards/margins": 23.761631998252607,
      "rewards/rejected": -14.058823529411764,
      "step": 1211
    },
    {
      "epoch": 0.5970443349753695,
      "grad_norm": 0.4140412475063112,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -41826531.55555555,
      "logits/rejected": -65794985.638554215,
      "logps/chosen": -218.48888888888888,
      "logps/rejected": -561.3493975903615,
      "loss": 0.1552,
      "rewards/chosen": 6.3402899848090275,
      "rewards/margins": 20.12342251492951,
      "rewards/rejected": -13.783132530120483,
      "step": 1212
    },
    {
      "epoch": 0.5975369458128079,
      "grad_norm": 0.48655576769911546,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -54386141.86666667,
      "logits/rejected": -71656904.48192771,
      "logps/chosen": -334.93333333333334,
      "logps/rejected": -542.0722891566265,
      "loss": 0.1957,
      "rewards/chosen": -1.7930555555555556,
      "rewards/margins": 10.821402275769746,
      "rewards/rejected": -12.614457831325302,
      "step": 1213
    },
    {
      "epoch": 0.5980295566502463,
      "grad_norm": 0.5313101423268677,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -43138416.64,
      "logits/rejected": -57859885.94871795,
      "logps/chosen": -226.56,
      "logps/rejected": -482.87179487179486,
      "loss": 0.1631,
      "rewards/chosen": 3.3744070434570315,
      "rewards/margins": 15.374407043457031,
      "rewards/rejected": -12.0,
      "step": 1214
    },
    {
      "epoch": 0.5985221674876847,
      "grad_norm": 0.4236569656002468,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -56265053.65853658,
      "logits/rejected": -59009518.344827585,
      "logps/chosen": -305.5609756097561,
      "logps/rejected": -511.264367816092,
      "loss": 0.1784,
      "rewards/chosen": 4.214960237828697,
      "rewards/margins": 17.168983226334444,
      "rewards/rejected": -12.954022988505747,
      "step": 1215
    },
    {
      "epoch": 0.5990147783251232,
      "grad_norm": 0.47090625768064265,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -46051745.95918367,
      "logits/rejected": -66896494.17721519,
      "logps/chosen": -248.81632653061226,
      "logps/rejected": -518.4810126582279,
      "loss": 0.2133,
      "rewards/chosen": -0.6779336734693877,
      "rewards/margins": 11.80307898475846,
      "rewards/rejected": -12.481012658227849,
      "step": 1216
    },
    {
      "epoch": 0.5995073891625615,
      "grad_norm": 0.4406513000437246,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -56583535.094339624,
      "logits/rejected": -57434002.77333333,
      "logps/chosen": -305.20754716981133,
      "logps/rejected": -507.73333333333335,
      "loss": 0.1609,
      "rewards/chosen": 3.3260952931530072,
      "rewards/margins": 15.646095293153007,
      "rewards/rejected": -12.32,
      "step": 1217
    },
    {
      "epoch": 0.6,
      "grad_norm": 0.4168540626212657,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -54617132.52173913,
      "logits/rejected": -65114012.09756097,
      "logps/chosen": -335.4782608695652,
      "logps/rejected": -499.5121951219512,
      "loss": 0.1463,
      "rewards/chosen": 4.706485914147419,
      "rewards/margins": 15.316242011708393,
      "rewards/rejected": -10.609756097560975,
      "step": 1218
    },
    {
      "epoch": 0.6004926108374384,
      "grad_norm": 0.4995848196525996,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -44564480.0,
      "logits/rejected": -49341326.222222224,
      "logps/chosen": -261.14285714285717,
      "logps/rejected": -466.22222222222223,
      "loss": 0.1928,
      "rewards/chosen": 1.3207310267857142,
      "rewards/margins": 13.306842137896824,
      "rewards/rejected": -11.98611111111111,
      "step": 1219
    },
    {
      "epoch": 0.6009852216748769,
      "grad_norm": 0.521960854051218,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -56139145.84615385,
      "logits/rejected": -56899045.05263158,
      "logps/chosen": -351.38461538461536,
      "logps/rejected": -489.2631578947368,
      "loss": 0.2388,
      "rewards/chosen": -0.5222355769230769,
      "rewards/margins": 7.344941482852828,
      "rewards/rejected": -7.867177059775905,
      "step": 1220
    },
    {
      "epoch": 0.6014778325123152,
      "grad_norm": 0.4320146018760964,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -55924053.333333336,
      "logits/rejected": -62257620.81927711,
      "logps/chosen": -346.31111111111113,
      "logps/rejected": -535.1325301204819,
      "loss": 0.1628,
      "rewards/chosen": 1.336111111111111,
      "rewards/margins": 14.203580990629183,
      "rewards/rejected": -12.867469879518072,
      "step": 1221
    },
    {
      "epoch": 0.6019704433497537,
      "grad_norm": 0.45528050868254627,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -72106332.59574468,
      "logits/rejected": -58564911.40740741,
      "logps/chosen": -300.93617021276594,
      "logps/rejected": -545.1851851851852,
      "loss": 0.1544,
      "rewards/chosen": 4.093771427235705,
      "rewards/margins": 17.476487476618423,
      "rewards/rejected": -13.382716049382717,
      "step": 1222
    },
    {
      "epoch": 0.6024630541871921,
      "grad_norm": 0.5376470908313353,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -45643896.47058824,
      "logits/rejected": -66999921.03896104,
      "logps/chosen": -322.19607843137254,
      "logps/rejected": -497.038961038961,
      "loss": 0.1945,
      "rewards/chosen": 3.4847088982077206,
      "rewards/margins": 16.419773833272654,
      "rewards/rejected": -12.935064935064934,
      "step": 1223
    },
    {
      "epoch": 0.6029556650246305,
      "grad_norm": 0.5126858190218033,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -44459622.4,
      "logits/rejected": -59634399.179487176,
      "logps/chosen": -262.72,
      "logps/rejected": -502.15384615384613,
      "loss": 0.1876,
      "rewards/chosen": 2.4150102233886717,
      "rewards/margins": 14.748343556722006,
      "rewards/rejected": -12.333333333333334,
      "step": 1224
    },
    {
      "epoch": 0.603448275862069,
      "grad_norm": 0.42352581116072124,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -49818059.75510204,
      "logits/rejected": -75922211.64556962,
      "logps/chosen": -338.9387755102041,
      "logps/rejected": -592.2025316455696,
      "loss": 0.158,
      "rewards/chosen": 1.4610969387755102,
      "rewards/margins": 14.866160229914751,
      "rewards/rejected": -13.405063291139241,
      "step": 1225
    },
    {
      "epoch": 0.6039408866995074,
      "grad_norm": 0.5210787592291699,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -73035597.91304348,
      "logits/rejected": -62582084.682926826,
      "logps/chosen": -414.95652173913044,
      "logps/rejected": -551.8048780487804,
      "loss": 0.1855,
      "rewards/chosen": 0.529891304347826,
      "rewards/margins": 12.627452279957582,
      "rewards/rejected": -12.097560975609756,
      "step": 1226
    },
    {
      "epoch": 0.6044334975369459,
      "grad_norm": 0.4374663778665087,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -67437829.01960784,
      "logits/rejected": -70540567.27272727,
      "logps/chosen": -322.19607843137254,
      "logps/rejected": -575.1688311688312,
      "loss": 0.1958,
      "rewards/chosen": 4.2832471062155335,
      "rewards/margins": 17.53000035296878,
      "rewards/rejected": -13.246753246753247,
      "step": 1227
    },
    {
      "epoch": 0.6049261083743842,
      "grad_norm": 0.33349889090302215,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -45327080.72727273,
      "logits/rejected": -51929478.0952381,
      "logps/chosen": -227.27272727272728,
      "logps/rejected": -472.0,
      "loss": 0.1536,
      "rewards/chosen": 3.3544259504838423,
      "rewards/margins": 15.390140236198128,
      "rewards/rejected": -12.035714285714286,
      "step": 1228
    },
    {
      "epoch": 0.6054187192118227,
      "grad_norm": 0.471930419199345,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -60211564.08888889,
      "logits/rejected": -61651215.421686746,
      "logps/chosen": -270.75555555555553,
      "logps/rejected": -468.8192771084337,
      "loss": 0.1949,
      "rewards/chosen": 6.387434895833334,
      "rewards/margins": 18.146471040411647,
      "rewards/rejected": -11.759036144578314,
      "step": 1229
    },
    {
      "epoch": 0.6059113300492611,
      "grad_norm": 0.46455396929063564,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -62168905.95555556,
      "logits/rejected": -53970080.38554217,
      "logps/chosen": -337.06666666666666,
      "logps/rejected": -461.10843373493975,
      "loss": 0.1536,
      "rewards/chosen": 4.1184068467881945,
      "rewards/margins": 16.76900925642675,
      "rewards/rejected": -12.650602409638553,
      "step": 1230
    },
    {
      "epoch": 0.6064039408866995,
      "grad_norm": 0.47876053199309904,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -51187628.408163264,
      "logits/rejected": -66312477.164556965,
      "logps/chosen": -227.42857142857142,
      "logps/rejected": -546.8354430379746,
      "loss": 0.184,
      "rewards/chosen": 1.8335773701570472,
      "rewards/margins": 15.65636218028363,
      "rewards/rejected": -13.822784810126583,
      "step": 1231
    },
    {
      "epoch": 0.6068965517241379,
      "grad_norm": 0.49911019834244275,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -41541457.70212766,
      "logits/rejected": -52817161.481481485,
      "logps/chosen": -270.8085106382979,
      "logps/rejected": -517.5308641975308,
      "loss": 0.1714,
      "rewards/chosen": 5.534595895320811,
      "rewards/margins": 18.38644774717266,
      "rewards/rejected": -12.851851851851851,
      "step": 1232
    },
    {
      "epoch": 0.6073891625615764,
      "grad_norm": 0.4487154199238094,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -61385386.666666664,
      "logits/rejected": -59087257.6,
      "logps/chosen": -316.6666666666667,
      "logps/rejected": -500.8,
      "loss": 0.1611,
      "rewards/chosen": 4.220031102498372,
      "rewards/margins": 11.460544713338216,
      "rewards/rejected": -7.240513610839844,
      "step": 1233
    },
    {
      "epoch": 0.6078817733990147,
      "grad_norm": 0.4456456153120844,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -47934902.85714286,
      "logits/rejected": -57526492.55384615,
      "logps/chosen": -371.8095238095238,
      "logps/rejected": -559.2615384615384,
      "loss": 0.2018,
      "rewards/chosen": 1.4841269841269842,
      "rewards/margins": 14.699511599511599,
      "rewards/rejected": -13.215384615384615,
      "step": 1234
    },
    {
      "epoch": 0.6083743842364532,
      "grad_norm": 0.47702718314941545,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -62817018.04651163,
      "logits/rejected": -68589206.58823529,
      "logps/chosen": -403.3488372093023,
      "logps/rejected": -517.2705882352941,
      "loss": 0.1492,
      "rewards/chosen": 4.07741919229197,
      "rewards/margins": 17.11271330993903,
      "rewards/rejected": -13.035294117647059,
      "step": 1235
    },
    {
      "epoch": 0.6088669950738916,
      "grad_norm": 0.5226996281612895,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -47005131.03448276,
      "logits/rejected": -65850572.8,
      "logps/chosen": -234.75862068965517,
      "logps/rejected": -511.54285714285714,
      "loss": 0.1805,
      "rewards/chosen": 1.5765086206896552,
      "rewards/margins": 11.632979219182959,
      "rewards/rejected": -10.056470598493304,
      "step": 1236
    },
    {
      "epoch": 0.6093596059113301,
      "grad_norm": 0.4954610184543275,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -53203834.43478261,
      "logits/rejected": -71916968.58536585,
      "logps/chosen": -338.7826086956522,
      "logps/rejected": -537.7560975609756,
      "loss": 0.2065,
      "rewards/chosen": 2.7846527099609375,
      "rewards/margins": 14.857823441668254,
      "rewards/rejected": -12.073170731707316,
      "step": 1237
    },
    {
      "epoch": 0.6098522167487684,
      "grad_norm": 0.6309951537892893,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -49970070.06896552,
      "logits/rejected": -74598692.57142857,
      "logps/chosen": -314.48275862068965,
      "logps/rejected": -479.54285714285714,
      "loss": 0.218,
      "rewards/chosen": 3.563074572332974,
      "rewards/margins": 15.091646000904403,
      "rewards/rejected": -11.528571428571428,
      "step": 1238
    },
    {
      "epoch": 0.6103448275862069,
      "grad_norm": 0.46450342443189896,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -44920600.1509434,
      "logits/rejected": -66549623.46666667,
      "logps/chosen": -279.8490566037736,
      "logps/rejected": -581.12,
      "loss": 0.1718,
      "rewards/chosen": 1.2995283018867925,
      "rewards/margins": 11.672861635220126,
      "rewards/rejected": -10.373333333333333,
      "step": 1239
    },
    {
      "epoch": 0.6108374384236454,
      "grad_norm": 0.40204058588942043,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -54065601.56097561,
      "logits/rejected": -61178985.93103448,
      "logps/chosen": -284.8780487804878,
      "logps/rejected": -530.0229885057471,
      "loss": 0.1382,
      "rewards/chosen": 6.703526938833842,
      "rewards/margins": 19.243756823891314,
      "rewards/rejected": -12.540229885057471,
      "step": 1240
    },
    {
      "epoch": 0.6113300492610837,
      "grad_norm": 0.4744965748935197,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -77594624.0,
      "logits/rejected": -55973985.52380952,
      "logps/chosen": -274.90909090909093,
      "logps/rejected": -500.95238095238096,
      "loss": 0.1513,
      "rewards/chosen": 4.285557833584872,
      "rewards/margins": 16.57127211929916,
      "rewards/rejected": -12.285714285714286,
      "step": 1241
    },
    {
      "epoch": 0.6118226600985222,
      "grad_norm": 0.48949078644362154,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -66596226.844444446,
      "logits/rejected": -66603526.1686747,
      "logps/chosen": -316.44444444444446,
      "logps/rejected": -523.566265060241,
      "loss": 0.1827,
      "rewards/chosen": 3.350690714518229,
      "rewards/margins": 15.62779914825317,
      "rewards/rejected": -12.27710843373494,
      "step": 1242
    },
    {
      "epoch": 0.6123152709359606,
      "grad_norm": 0.6666174494820656,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -43740598.85714286,
      "logits/rejected": -83142785.62025316,
      "logps/chosen": -287.3469387755102,
      "logps/rejected": -588.1518987341772,
      "loss": 0.1961,
      "rewards/chosen": 2.559290360431282,
      "rewards/margins": 13.065619474355332,
      "rewards/rejected": -10.50632911392405,
      "step": 1243
    },
    {
      "epoch": 0.6128078817733991,
      "grad_norm": 0.574889791262013,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -72244746.44897959,
      "logits/rejected": -64613518.58227848,
      "logps/chosen": -315.7551020408163,
      "logps/rejected": -527.3924050632911,
      "loss": 0.1894,
      "rewards/chosen": 0.4403698979591837,
      "rewards/margins": 13.19986356884526,
      "rewards/rejected": -12.759493670886076,
      "step": 1244
    },
    {
      "epoch": 0.6133004926108374,
      "grad_norm": 0.41952925294634985,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -43893879.069767445,
      "logits/rejected": -73474337.12941177,
      "logps/chosen": -239.8139534883721,
      "logps/rejected": -587.2941176470588,
      "loss": 0.1453,
      "rewards/chosen": 1.5472383720930232,
      "rewards/margins": 14.006061901504788,
      "rewards/rejected": -12.458823529411765,
      "step": 1245
    },
    {
      "epoch": 0.6137931034482759,
      "grad_norm": 0.42669155969223277,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -55380347.25925926,
      "logits/rejected": -59740492.10810811,
      "logps/chosen": -239.7037037037037,
      "logps/rejected": -535.3513513513514,
      "loss": 0.1575,
      "rewards/chosen": 3.91607666015625,
      "rewards/margins": 16.80796855204814,
      "rewards/rejected": -12.891891891891891,
      "step": 1246
    },
    {
      "epoch": 0.6142857142857143,
      "grad_norm": 1.624475846099554,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -42763664.69565217,
      "logits/rejected": -56674254.048780486,
      "logps/chosen": -298.0869565217391,
      "logps/rejected": -538.5365853658536,
      "loss": 0.1632,
      "rewards/chosen": 1.4442934782608696,
      "rewards/margins": 14.480878844114528,
      "rewards/rejected": -13.036585365853659,
      "step": 1247
    },
    {
      "epoch": 0.6147783251231527,
      "grad_norm": 0.4353622472698109,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -58720256.0,
      "logits/rejected": -68376909.39534883,
      "logps/chosen": -311.6190476190476,
      "logps/rejected": -494.8837209302326,
      "loss": 0.1522,
      "rewards/chosen": 3.9322996593656994,
      "rewards/margins": 15.490439194249419,
      "rewards/rejected": -11.55813953488372,
      "step": 1248
    },
    {
      "epoch": 0.6152709359605911,
      "grad_norm": 0.6245477119923204,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -61351976.15686274,
      "logits/rejected": -59210499.32467532,
      "logps/chosen": -353.2549019607843,
      "logps/rejected": -490.38961038961037,
      "loss": 0.1859,
      "rewards/chosen": 3.4865193086511947,
      "rewards/margins": 16.083921906053792,
      "rewards/rejected": -12.597402597402597,
      "step": 1249
    },
    {
      "epoch": 0.6157635467980296,
      "grad_norm": 0.4350944444706493,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -58195968.0,
      "logits/rejected": -67382405.56521739,
      "logps/chosen": -273.1111111111111,
      "logps/rejected": -505.7391304347826,
      "loss": 0.1476,
      "rewards/chosen": 7.2751922607421875,
      "rewards/margins": 19.525192260742188,
      "rewards/rejected": -12.25,
      "step": 1250
    },
    {
      "epoch": 0.6162561576354679,
      "grad_norm": 0.458079535551526,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -42157521.45454545,
      "logits/rejected": -53826901.333333336,
      "logps/chosen": -213.45454545454547,
      "logps/rejected": -510.4761904761905,
      "loss": 0.171,
      "rewards/chosen": 6.541828502308238,
      "rewards/margins": 18.56563802611776,
      "rewards/rejected": -12.023809523809524,
      "step": 1251
    },
    {
      "epoch": 0.6167487684729064,
      "grad_norm": 0.5159439888783491,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -58929971.2,
      "logits/rejected": -67108864.0,
      "logps/chosen": -339.2,
      "logps/rejected": -520.9411764705883,
      "loss": 0.1988,
      "rewards/chosen": 1.7947916666666666,
      "rewards/margins": 14.588909313725491,
      "rewards/rejected": -12.794117647058824,
      "step": 1252
    },
    {
      "epoch": 0.6172413793103448,
      "grad_norm": 0.5075829776742236,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -57294192.64,
      "logits/rejected": -72217311.17948718,
      "logps/chosen": -288.96,
      "logps/rejected": -545.6410256410256,
      "loss": 0.1633,
      "rewards/chosen": 1.43375,
      "rewards/margins": 14.305544871794872,
      "rewards/rejected": -12.871794871794872,
      "step": 1253
    },
    {
      "epoch": 0.6177339901477833,
      "grad_norm": 0.5803311887898046,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -55268693.333333336,
      "logits/rejected": -47919923.2,
      "logps/chosen": -262.6666666666667,
      "logps/rejected": -440.8,
      "loss": 0.1902,
      "rewards/chosen": 3.7794322967529297,
      "rewards/margins": 14.71693229675293,
      "rewards/rejected": -10.9375,
      "step": 1254
    },
    {
      "epoch": 0.6182266009852216,
      "grad_norm": 0.4536110564940902,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -61793668.4137931,
      "logits/rejected": -62854641.37142857,
      "logps/chosen": -279.44827586206895,
      "logps/rejected": -562.2857142857143,
      "loss": 0.1871,
      "rewards/chosen": 2.7575531005859375,
      "rewards/margins": 16.614695957728795,
      "rewards/rejected": -13.857142857142858,
      "step": 1255
    },
    {
      "epoch": 0.6187192118226601,
      "grad_norm": 0.4569648501170869,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -66140947.692307696,
      "logits/rejected": -54714460.04494382,
      "logps/chosen": -381.94871794871796,
      "logps/rejected": -454.1123595505618,
      "loss": 0.1753,
      "rewards/chosen": 3.3052618564703526,
      "rewards/margins": 15.45132927220069,
      "rewards/rejected": -12.146067415730338,
      "step": 1256
    },
    {
      "epoch": 0.6192118226600986,
      "grad_norm": 0.6105985848710347,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -60004226.6122449,
      "logits/rejected": -64135686.48101266,
      "logps/chosen": -344.81632653061223,
      "logps/rejected": -469.46835443037975,
      "loss": 0.1381,
      "rewards/chosen": 2.3494897959183674,
      "rewards/margins": 13.552021441487987,
      "rewards/rejected": -11.20253164556962,
      "step": 1257
    },
    {
      "epoch": 0.6197044334975369,
      "grad_norm": 0.44292257144503966,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -62961163.37777778,
      "logits/rejected": -60135201.927710846,
      "logps/chosen": -313.6,
      "logps/rejected": -506.6024096385542,
      "loss": 0.1948,
      "rewards/chosen": 1.1840274386935763,
      "rewards/margins": 13.400894908573093,
      "rewards/rejected": -12.216867469879517,
      "step": 1258
    },
    {
      "epoch": 0.6201970443349754,
      "grad_norm": 0.4619007476622154,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -41549824.0,
      "logits/rejected": -54735667.2,
      "logps/chosen": -286.3333333333333,
      "logps/rejected": -544.0,
      "loss": 0.2043,
      "rewards/chosen": 3.161478042602539,
      "rewards/margins": 16.59897804260254,
      "rewards/rejected": -13.4375,
      "step": 1259
    },
    {
      "epoch": 0.6206896551724138,
      "grad_norm": 0.5639843996715164,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -68157440.0,
      "logits/rejected": -66906728.86746988,
      "logps/chosen": -301.68888888888887,
      "logps/rejected": -490.4096385542169,
      "loss": 0.1523,
      "rewards/chosen": 2.325,
      "rewards/margins": 12.867168674698796,
      "rewards/rejected": -10.542168674698795,
      "step": 1260
    },
    {
      "epoch": 0.6211822660098523,
      "grad_norm": 0.47477422604477554,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -59077218.042553194,
      "logits/rejected": -51211934.02469136,
      "logps/chosen": -369.02127659574467,
      "logps/rejected": -547.1604938271605,
      "loss": 0.2096,
      "rewards/chosen": 2.5158314806349735,
      "rewards/margins": 12.577559875696702,
      "rewards/rejected": -10.061728395061728,
      "step": 1261
    },
    {
      "epoch": 0.6216748768472906,
      "grad_norm": 0.5164891224153447,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -51936611.26530612,
      "logits/rejected": -68595452.75949366,
      "logps/chosen": -227.42857142857142,
      "logps/rejected": -528.6075949367089,
      "loss": 0.1917,
      "rewards/chosen": 2.7534758898676657,
      "rewards/margins": 15.740817662019564,
      "rewards/rejected": -12.987341772151899,
      "step": 1262
    },
    {
      "epoch": 0.6221674876847291,
      "grad_norm": 0.4800675972880958,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -52543190.10909091,
      "logits/rejected": -56881656.98630137,
      "logps/chosen": -280.8727272727273,
      "logps/rejected": -576.8767123287671,
      "loss": 0.1983,
      "rewards/chosen": 4.723692737926136,
      "rewards/margins": 19.23054205299463,
      "rewards/rejected": -14.506849315068493,
      "step": 1263
    },
    {
      "epoch": 0.6226600985221675,
      "grad_norm": 0.5463853524117752,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -48234496.0,
      "logits/rejected": -62403059.512195125,
      "logps/chosen": -300.17391304347825,
      "logps/rejected": -619.7073170731708,
      "loss": 0.1398,
      "rewards/chosen": 1.921875,
      "rewards/margins": 15.104801829268293,
      "rewards/rejected": -13.182926829268293,
      "step": 1264
    },
    {
      "epoch": 0.6231527093596059,
      "grad_norm": 0.5392061548200745,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -49102283.03448276,
      "logits/rejected": -58480581.48571429,
      "logps/chosen": -249.79310344827587,
      "logps/rejected": -494.1714285714286,
      "loss": 0.1958,
      "rewards/chosen": 0.9962284482758621,
      "rewards/margins": 13.853371305418719,
      "rewards/rejected": -12.857142857142858,
      "step": 1265
    },
    {
      "epoch": 0.6236453201970443,
      "grad_norm": 0.48093830989413466,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -57774481.568627454,
      "logits/rejected": -63949518.12987013,
      "logps/chosen": -341.01960784313724,
      "logps/rejected": -498.2857142857143,
      "loss": 0.2043,
      "rewards/chosen": 5.613963407628677,
      "rewards/margins": 16.685391979057247,
      "rewards/rejected": -11.071428571428571,
      "step": 1266
    },
    {
      "epoch": 0.6241379310344828,
      "grad_norm": 0.4304586243938718,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -60626757.81818182,
      "logits/rejected": -59369374.47619048,
      "logps/chosen": -340.3636363636364,
      "logps/rejected": -565.3333333333334,
      "loss": 0.1613,
      "rewards/chosen": 2.3309804742986504,
      "rewards/margins": 15.83098047429865,
      "rewards/rejected": -13.5,
      "step": 1267
    },
    {
      "epoch": 0.6246305418719211,
      "grad_norm": 0.5060123335500619,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -63639030.69090909,
      "logits/rejected": -63833859.50684932,
      "logps/chosen": -357.8181818181818,
      "logps/rejected": -544.0,
      "loss": 0.2186,
      "rewards/chosen": 0.8909090909090909,
      "rewards/margins": 11.986799501867996,
      "rewards/rejected": -11.095890410958905,
      "step": 1268
    },
    {
      "epoch": 0.6251231527093596,
      "grad_norm": 0.45099911271035487,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -44910707.9245283,
      "logits/rejected": -61013142.18666667,
      "logps/chosen": -275.0188679245283,
      "logps/rejected": -485.97333333333336,
      "loss": 0.1731,
      "rewards/chosen": 1.1132075471698113,
      "rewards/margins": 13.939874213836477,
      "rewards/rejected": -12.826666666666666,
      "step": 1269
    },
    {
      "epoch": 0.625615763546798,
      "grad_norm": 0.48071161407940055,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -55752588.0754717,
      "logits/rejected": -62634939.733333334,
      "logps/chosen": -232.9056603773585,
      "logps/rejected": -520.1066666666667,
      "loss": 0.1681,
      "rewards/chosen": 5.253557313163325,
      "rewards/margins": 17.573557313163327,
      "rewards/rejected": -12.32,
      "step": 1270
    },
    {
      "epoch": 0.6261083743842365,
      "grad_norm": 0.5414220483665906,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -66381280.653061226,
      "logits/rejected": -67480511.18987341,
      "logps/chosen": -390.53061224489795,
      "logps/rejected": -537.1139240506329,
      "loss": 0.2056,
      "rewards/chosen": 2.800684403400032,
      "rewards/margins": 14.889291998336741,
      "rewards/rejected": -12.08860759493671,
      "step": 1271
    },
    {
      "epoch": 0.6266009852216748,
      "grad_norm": 0.6024093035245592,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -51608175.30434783,
      "logits/rejected": -61175458.34146342,
      "logps/chosen": -333.9130434782609,
      "logps/rejected": -470.6341463414634,
      "loss": 0.1497,
      "rewards/chosen": 1.783967391304348,
      "rewards/margins": 13.47908934252386,
      "rewards/rejected": -11.695121951219512,
      "step": 1272
    },
    {
      "epoch": 0.6270935960591133,
      "grad_norm": 0.4431305623748162,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -53360867.55555555,
      "logits/rejected": -63651397.18918919,
      "logps/chosen": -294.51851851851853,
      "logps/rejected": -515.4594594594595,
      "loss": 0.1957,
      "rewards/chosen": 0.9508101851851852,
      "rewards/margins": 12.058918293293294,
      "rewards/rejected": -11.108108108108109,
      "step": 1273
    },
    {
      "epoch": 0.6275862068965518,
      "grad_norm": 0.4774236405033744,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -53621298.196078435,
      "logits/rejected": -61498301.50649351,
      "logps/chosen": -272.0,
      "logps/rejected": -470.02597402597405,
      "loss": 0.2002,
      "rewards/chosen": 3.6887446384803924,
      "rewards/margins": 14.104329054064808,
      "rewards/rejected": -10.415584415584416,
      "step": 1274
    },
    {
      "epoch": 0.6280788177339901,
      "grad_norm": 0.46106025930928396,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -51316025.469387755,
      "logits/rejected": -67321233.82278481,
      "logps/chosen": -252.08163265306123,
      "logps/rejected": -484.8607594936709,
      "loss": 0.1798,
      "rewards/chosen": 3.2397784797512754,
      "rewards/margins": 14.847373416460137,
      "rewards/rejected": -11.60759493670886,
      "step": 1275
    },
    {
      "epoch": 0.6285714285714286,
      "grad_norm": 0.5722840390945911,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -42970216.489795916,
      "logits/rejected": -74329437.97468354,
      "logps/chosen": -273.9591836734694,
      "logps/rejected": -526.5822784810126,
      "loss": 0.1754,
      "rewards/chosen": 1.2442602040816326,
      "rewards/margins": 13.117677925600619,
      "rewards/rejected": -11.873417721518987,
      "step": 1276
    },
    {
      "epoch": 0.629064039408867,
      "grad_norm": 0.4190315967037116,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -51511296.0,
      "logits/rejected": -56499742.11764706,
      "logps/chosen": -310.4186046511628,
      "logps/rejected": -446.49411764705883,
      "loss": 0.2059,
      "rewards/chosen": -1.2049418604651163,
      "rewards/margins": 8.583293433652532,
      "rewards/rejected": -9.788235294117648,
      "step": 1277
    },
    {
      "epoch": 0.6295566502463055,
      "grad_norm": 0.3985101654974627,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -49989255.8367347,
      "logits/rejected": -53570287.79746836,
      "logps/chosen": -315.42857142857144,
      "logps/rejected": -493.36708860759495,
      "loss": 0.1669,
      "rewards/chosen": 1.105389030612245,
      "rewards/margins": 10.966148524283131,
      "rewards/rejected": -9.860759493670885,
      "step": 1278
    },
    {
      "epoch": 0.6300492610837438,
      "grad_norm": 0.38085559161601057,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -56665903.02040816,
      "logits/rejected": -54260489.721518986,
      "logps/chosen": -348.734693877551,
      "logps/rejected": -469.46835443037975,
      "loss": 0.1547,
      "rewards/chosen": 2.313775510204082,
      "rewards/margins": 11.617572978558512,
      "rewards/rejected": -9.30379746835443,
      "step": 1279
    },
    {
      "epoch": 0.6305418719211823,
      "grad_norm": 0.6830089214405191,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -63233691.82608695,
      "logits/rejected": -65165162.146341465,
      "logps/chosen": -234.95652173913044,
      "logps/rejected": -511.219512195122,
      "loss": 0.1679,
      "rewards/chosen": 0.3967391304347826,
      "rewards/margins": 10.908934252386002,
      "rewards/rejected": -10.512195121951219,
      "step": 1280
    },
    {
      "epoch": 0.6310344827586207,
      "grad_norm": 0.5348793952979759,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -52428800.0,
      "logits/rejected": -57303261.4054054,
      "logps/chosen": -313.48148148148147,
      "logps/rejected": -410.81081081081084,
      "loss": 0.1918,
      "rewards/chosen": 3.310210616500289,
      "rewards/margins": 12.850751157040829,
      "rewards/rejected": -9.54054054054054,
      "step": 1281
    },
    {
      "epoch": 0.6315270935960591,
      "grad_norm": 0.5128158621692759,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -64280147.34883721,
      "logits/rejected": -67799690.54117647,
      "logps/chosen": -390.3255813953488,
      "logps/rejected": -515.0117647058823,
      "loss": 0.1846,
      "rewards/chosen": 3.7545300860737645,
      "rewards/margins": 14.131000674309059,
      "rewards/rejected": -10.376470588235295,
      "step": 1282
    },
    {
      "epoch": 0.6320197044334975,
      "grad_norm": 0.48789233675350563,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -45388361.14285714,
      "logits/rejected": -64429169.777777776,
      "logps/chosen": -228.64285714285714,
      "logps/rejected": -497.3333333333333,
      "loss": 0.2485,
      "rewards/chosen": -0.3046875,
      "rewards/margins": 9.55642361111111,
      "rewards/rejected": -9.86111111111111,
      "step": 1283
    },
    {
      "epoch": 0.632512315270936,
      "grad_norm": 0.4135684470847331,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -50481444.571428575,
      "logits/rejected": -59477560.88888889,
      "logps/chosen": -284.2857142857143,
      "logps/rejected": -464.44444444444446,
      "loss": 0.168,
      "rewards/chosen": 1.6545758928571428,
      "rewards/margins": 8.834474472772508,
      "rewards/rejected": -7.179898579915364,
      "step": 1284
    },
    {
      "epoch": 0.6330049261083743,
      "grad_norm": 0.40631671255585944,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -55848069.56521739,
      "logits/rejected": -61277758.43902439,
      "logps/chosen": -275.4782608695652,
      "logps/rejected": -462.0487804878049,
      "loss": 0.167,
      "rewards/chosen": 6.380458665930706,
      "rewards/margins": 16.23411720251607,
      "rewards/rejected": -9.853658536585366,
      "step": 1285
    },
    {
      "epoch": 0.6334975369458128,
      "grad_norm": 0.4978765120072639,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -68265913.37931034,
      "logits/rejected": -59319442.28571428,
      "logps/chosen": -328.2758620689655,
      "logps/rejected": -504.6857142857143,
      "loss": 0.1721,
      "rewards/chosen": 1.978448275862069,
      "rewards/margins": 13.135591133004926,
      "rewards/rejected": -11.157142857142857,
      "step": 1286
    },
    {
      "epoch": 0.6339901477832512,
      "grad_norm": 0.4378797036924801,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -54951050.37837838,
      "logits/rejected": -64804301.36263736,
      "logps/chosen": -249.72972972972974,
      "logps/rejected": -507.7802197802198,
      "loss": 0.1376,
      "rewards/chosen": 1.260135135135135,
      "rewards/margins": 12.040354915354916,
      "rewards/rejected": -10.780219780219781,
      "step": 1287
    },
    {
      "epoch": 0.6344827586206897,
      "grad_norm": 0.49149283283200085,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -64006826.666666664,
      "logits/rejected": -68681728.0,
      "logps/chosen": -400.3333333333333,
      "logps/rejected": -521.6,
      "loss": 0.1524,
      "rewards/chosen": 3.1282984415690103,
      "rewards/margins": 14.35329844156901,
      "rewards/rejected": -11.225,
      "step": 1288
    },
    {
      "epoch": 0.634975369458128,
      "grad_norm": 0.5584399007390265,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -60576751.21311475,
      "logits/rejected": -57155217.19402985,
      "logps/chosen": -345.7049180327869,
      "logps/rejected": -476.17910447761193,
      "loss": 0.2246,
      "rewards/chosen": 2.490779188812756,
      "rewards/margins": 13.595256800753054,
      "rewards/rejected": -11.104477611940299,
      "step": 1289
    },
    {
      "epoch": 0.6354679802955665,
      "grad_norm": 0.4252397502350599,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -55793997.39534884,
      "logits/rejected": -73128923.85882352,
      "logps/chosen": -415.6279069767442,
      "logps/rejected": -557.9294117647058,
      "loss": 0.155,
      "rewards/chosen": 2.4352289244186047,
      "rewards/margins": 14.788170100889193,
      "rewards/rejected": -12.352941176470589,
      "step": 1290
    },
    {
      "epoch": 0.635960591133005,
      "grad_norm": 0.5681191478856813,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -48653926.4,
      "logits/rejected": -64925527.67123288,
      "logps/chosen": -328.72727272727275,
      "logps/rejected": -569.8630136986301,
      "loss": 0.1907,
      "rewards/chosen": 1.7397727272727272,
      "rewards/margins": 14.602786425902865,
      "rewards/rejected": -12.863013698630137,
      "step": 1291
    },
    {
      "epoch": 0.6364532019704433,
      "grad_norm": 0.38944888601491084,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -56451907.91836735,
      "logits/rejected": -59463550.37974683,
      "logps/chosen": -416.6530612244898,
      "logps/rejected": -535.493670886076,
      "loss": 0.2004,
      "rewards/chosen": 0.24457908163265307,
      "rewards/margins": 13.61166768922759,
      "rewards/rejected": -13.367088607594937,
      "step": 1292
    },
    {
      "epoch": 0.6369458128078818,
      "grad_norm": 0.44602573267657786,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -58554691.368421055,
      "logits/rejected": -65897176.177777775,
      "logps/chosen": -383.57894736842104,
      "logps/rejected": -531.2,
      "loss": 0.1559,
      "rewards/chosen": 7.21047652395148,
      "rewards/margins": 19.532698746173704,
      "rewards/rejected": -12.322222222222223,
      "step": 1293
    },
    {
      "epoch": 0.6374384236453202,
      "grad_norm": 0.4987085718130619,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -49370453.333333336,
      "logits/rejected": -54211379.2,
      "logps/chosen": -214.0,
      "logps/rejected": -459.2,
      "loss": 0.1779,
      "rewards/chosen": 3.5429763793945312,
      "rewards/margins": 16.280476379394532,
      "rewards/rejected": -12.7375,
      "step": 1294
    },
    {
      "epoch": 0.6379310344827587,
      "grad_norm": 0.448778437744243,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -54945382.4,
      "logits/rejected": -75446938.21686748,
      "logps/chosen": -312.8888888888889,
      "logps/rejected": -617.6385542168674,
      "loss": 0.1627,
      "rewards/chosen": 0.35,
      "rewards/margins": 16.84397590361446,
      "rewards/rejected": -16.49397590361446,
      "step": 1295
    },
    {
      "epoch": 0.638423645320197,
      "grad_norm": 0.5199550966969767,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -49384547.09677419,
      "logits/rejected": -65520112.484848484,
      "logps/chosen": -345.2903225806452,
      "logps/rejected": -537.2121212121212,
      "loss": 0.2069,
      "rewards/chosen": 0.782258064516129,
      "rewards/margins": 14.888318670576734,
      "rewards/rejected": -14.106060606060606,
      "step": 1296
    },
    {
      "epoch": 0.6389162561576355,
      "grad_norm": 0.44691597688028095,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -50331648.0,
      "logits/rejected": -73326302.87058823,
      "logps/chosen": -305.48837209302326,
      "logps/rejected": -514.2588235294118,
      "loss": 0.146,
      "rewards/chosen": 4.305241074672965,
      "rewards/margins": 19.222888133496497,
      "rewards/rejected": -14.91764705882353,
      "step": 1297
    },
    {
      "epoch": 0.6394088669950739,
      "grad_norm": 0.5759450571577931,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -58098877.62962963,
      "logits/rejected": -57813379.45945946,
      "logps/chosen": -323.55555555555554,
      "logps/rejected": -485.18918918918916,
      "loss": 0.2544,
      "rewards/chosen": -1.6180555555555556,
      "rewards/margins": 11.841403903903904,
      "rewards/rejected": -13.45945945945946,
      "step": 1298
    },
    {
      "epoch": 0.6399014778325123,
      "grad_norm": 0.6266464742499648,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -63913203.809523806,
      "logits/rejected": -62011171.44615385,
      "logps/chosen": -318.984126984127,
      "logps/rejected": -560.2461538461539,
      "loss": 0.2069,
      "rewards/chosen": -0.12053571428571429,
      "rewards/margins": 17.141002747252745,
      "rewards/rejected": -17.26153846153846,
      "step": 1299
    },
    {
      "epoch": 0.6403940886699507,
      "grad_norm": 0.4106922568103984,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -60089230.222222224,
      "logits/rejected": -60087963.82608695,
      "logps/chosen": -266.22222222222223,
      "logps/rejected": -580.8695652173913,
      "loss": 0.1298,
      "rewards/chosen": 1.4118923611111112,
      "rewards/margins": 17.65102279589372,
      "rewards/rejected": -16.23913043478261,
      "step": 1300
    },
    {
      "epoch": 0.6408866995073892,
      "grad_norm": 0.43562775757621813,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -59601059.84,
      "logits/rejected": -62591921.23076923,
      "logps/chosen": -366.08,
      "logps/rejected": -539.8974358974359,
      "loss": 0.1975,
      "rewards/chosen": 1.4249687194824219,
      "rewards/margins": 17.37368666820037,
      "rewards/rejected": -15.948717948717949,
      "step": 1301
    },
    {
      "epoch": 0.6413793103448275,
      "grad_norm": 0.5184313461789309,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -47437578.24,
      "logits/rejected": -67216410.25641026,
      "logps/chosen": -310.08,
      "logps/rejected": -632.6153846153846,
      "loss": 0.1801,
      "rewards/chosen": 4.098763427734375,
      "rewards/margins": 20.316712145683095,
      "rewards/rejected": -16.21794871794872,
      "step": 1302
    },
    {
      "epoch": 0.641871921182266,
      "grad_norm": 0.46948138460024813,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -57632843.85185185,
      "logits/rejected": -73116921.08108108,
      "logps/chosen": -305.77777777777777,
      "logps/rejected": -572.5405405405405,
      "loss": 0.2236,
      "rewards/chosen": 0.37214265046296297,
      "rewards/margins": 18.5072777855981,
      "rewards/rejected": -18.135135135135137,
      "step": 1303
    },
    {
      "epoch": 0.6423645320197044,
      "grad_norm": 0.46374813152669425,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -50554749.27659574,
      "logits/rejected": -64726913.58024691,
      "logps/chosen": -310.63829787234044,
      "logps/rejected": -580.7407407407408,
      "loss": 0.1826,
      "rewards/chosen": 0.20212765957446807,
      "rewards/margins": 18.844102968216443,
      "rewards/rejected": -18.641975308641975,
      "step": 1304
    },
    {
      "epoch": 0.6428571428571429,
      "grad_norm": 0.5194292835264159,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -51030698.666666664,
      "logits/rejected": -64277708.8,
      "logps/chosen": -416.6666666666667,
      "logps/rejected": -595.2,
      "loss": 0.1927,
      "rewards/chosen": 5.169939676920573,
      "rewards/margins": 22.294939676920574,
      "rewards/rejected": -17.125,
      "step": 1305
    },
    {
      "epoch": 0.6433497536945813,
      "grad_norm": 0.47860492857403,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -54651781.12,
      "logits/rejected": -63022106.256410256,
      "logps/chosen": -300.8,
      "logps/rejected": -572.3076923076923,
      "loss": 0.1594,
      "rewards/chosen": 0.11,
      "rewards/margins": 17.39205128205128,
      "rewards/rejected": -17.28205128205128,
      "step": 1306
    },
    {
      "epoch": 0.6438423645320197,
      "grad_norm": 0.5735305685849673,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -63695851.921568625,
      "logits/rejected": -61661715.948051944,
      "logps/chosen": -336.62745098039215,
      "logps/rejected": -559.3766233766233,
      "loss": 0.2236,
      "rewards/chosen": -1.6029411764705883,
      "rewards/margins": 12.630825057295645,
      "rewards/rejected": -14.233766233766234,
      "step": 1307
    },
    {
      "epoch": 0.6443349753694582,
      "grad_norm": 0.49611868274020754,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -50907336.78431372,
      "logits/rejected": -71902354.28571428,
      "logps/chosen": -265.4117647058824,
      "logps/rejected": -613.4025974025974,
      "loss": 0.19,
      "rewards/chosen": 0.8333333333333334,
      "rewards/margins": 19.508658008658006,
      "rewards/rejected": -18.675324675324674,
      "step": 1308
    },
    {
      "epoch": 0.6448275862068965,
      "grad_norm": 0.4595231463938075,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -49567431.59322034,
      "logits/rejected": -60969375.53623188,
      "logps/chosen": -343.0508474576271,
      "logps/rejected": -549.5652173913044,
      "loss": 0.2442,
      "rewards/chosen": -2.152542372881356,
      "rewards/margins": 14.861950380741831,
      "rewards/rejected": -17.014492753623188,
      "step": 1309
    },
    {
      "epoch": 0.645320197044335,
      "grad_norm": 0.7760875612450625,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -58632874.666666664,
      "logits/rejected": -70569164.8,
      "logps/chosen": -246.16666666666666,
      "logps/rejected": -647.2,
      "loss": 0.1995,
      "rewards/chosen": -0.3333333333333333,
      "rewards/margins": 15.404166666666667,
      "rewards/rejected": -15.7375,
      "step": 1310
    },
    {
      "epoch": 0.6458128078817734,
      "grad_norm": 0.4571622334348678,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -56522279.384615384,
      "logits/rejected": -66943299.368421055,
      "logps/chosen": -314.7692307692308,
      "logps/rejected": -623.1578947368421,
      "loss": 0.203,
      "rewards/chosen": -1.6117788461538463,
      "rewards/margins": 14.651379048582996,
      "rewards/rejected": -16.263157894736842,
      "step": 1311
    },
    {
      "epoch": 0.6463054187192119,
      "grad_norm": 0.4432942814200351,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -59502244.881355934,
      "logits/rejected": -57747663.768115945,
      "logps/chosen": -284.20338983050846,
      "logps/rejected": -535.1884057971015,
      "loss": 0.1986,
      "rewards/chosen": 2.166062694484905,
      "rewards/margins": 19.310990230716786,
      "rewards/rejected": -17.144927536231883,
      "step": 1312
    },
    {
      "epoch": 0.6467980295566502,
      "grad_norm": 0.51372848015245,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -44430359.81395349,
      "logits/rejected": -63457352.28235294,
      "logps/chosen": -199.8139534883721,
      "logps/rejected": -481.88235294117646,
      "loss": 0.1862,
      "rewards/chosen": 6.0036670773528344,
      "rewards/margins": 19.921314136176363,
      "rewards/rejected": -13.91764705882353,
      "step": 1313
    },
    {
      "epoch": 0.6472906403940887,
      "grad_norm": 0.5425404142865471,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -65173031.384615384,
      "logits/rejected": -52373611.78947368,
      "logps/chosen": -388.61538461538464,
      "logps/rejected": -502.3157894736842,
      "loss": 0.2144,
      "rewards/chosen": 3.402508075420673,
      "rewards/margins": 17.284087022789095,
      "rewards/rejected": -13.881578947368421,
      "step": 1314
    },
    {
      "epoch": 0.6477832512315271,
      "grad_norm": 0.46564644128606475,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -56250276.97777778,
      "logits/rejected": -62409222.1686747,
      "logps/chosen": -306.1333333333333,
      "logps/rejected": -575.2289156626506,
      "loss": 0.1654,
      "rewards/chosen": 4.352208794487847,
      "rewards/margins": 18.376305180030016,
      "rewards/rejected": -14.024096385542169,
      "step": 1315
    },
    {
      "epoch": 0.6482758620689655,
      "grad_norm": 0.46153420728437716,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -57652614.981818184,
      "logits/rejected": -61363242.08219178,
      "logps/chosen": -303.1272727272727,
      "logps/rejected": -494.90410958904107,
      "loss": 0.2036,
      "rewards/chosen": 1.7426261208274147,
      "rewards/margins": 14.180982285210977,
      "rewards/rejected": -12.438356164383562,
      "step": 1316
    },
    {
      "epoch": 0.6487684729064039,
      "grad_norm": 0.43873830646310513,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -47353692.16,
      "logits/rejected": -54041993.84615385,
      "logps/chosen": -221.12,
      "logps/rejected": -532.5128205128206,
      "loss": 0.1944,
      "rewards/chosen": 6.306906127929688,
      "rewards/margins": 22.332547153570715,
      "rewards/rejected": -16.025641025641026,
      "step": 1317
    },
    {
      "epoch": 0.6492610837438424,
      "grad_norm": 0.4318005171281947,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -48522340.39215686,
      "logits/rejected": -70649510.23376623,
      "logps/chosen": -296.7843137254902,
      "logps/rejected": -505.76623376623377,
      "loss": 0.1669,
      "rewards/chosen": 1.0067401960784315,
      "rewards/margins": 12.876870066208301,
      "rewards/rejected": -11.87012987012987,
      "step": 1318
    },
    {
      "epoch": 0.6497536945812807,
      "grad_norm": 0.5061302284748611,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -40916774.127659574,
      "logits/rejected": -60739735.7037037,
      "logps/chosen": -268.4255319148936,
      "logps/rejected": -519.1111111111111,
      "loss": 0.1571,
      "rewards/chosen": 3.630292851874169,
      "rewards/margins": 17.494490382738366,
      "rewards/rejected": -13.864197530864198,
      "step": 1319
    },
    {
      "epoch": 0.6502463054187192,
      "grad_norm": 0.39851590933998554,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -60068425.14285714,
      "logits/rejected": -55924053.333333336,
      "logps/chosen": -269.42857142857144,
      "logps/rejected": -552.8888888888889,
      "loss": 0.1657,
      "rewards/chosen": 1.2533482142857142,
      "rewards/margins": 15.906125992063492,
      "rewards/rejected": -14.652777777777779,
      "step": 1320
    },
    {
      "epoch": 0.6507389162561577,
      "grad_norm": 0.49218282058499574,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -49239381.333333336,
      "logits/rejected": -66112716.8,
      "logps/chosen": -277.3333333333333,
      "logps/rejected": -584.0,
      "loss": 0.192,
      "rewards/chosen": -0.23372395833333334,
      "rewards/margins": 14.841276041666665,
      "rewards/rejected": -15.075,
      "step": 1321
    },
    {
      "epoch": 0.6512315270935961,
      "grad_norm": 0.5007669029737002,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -58338955.63636363,
      "logits/rejected": -56673036.190476194,
      "logps/chosen": -282.72727272727275,
      "logps/rejected": -527.2380952380952,
      "loss": 0.1893,
      "rewards/chosen": 0.7116477272727273,
      "rewards/margins": 14.009266774891774,
      "rewards/rejected": -13.297619047619047,
      "step": 1322
    },
    {
      "epoch": 0.6517241379310345,
      "grad_norm": 0.4611802321536815,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -49554925.03703704,
      "logits/rejected": -57303261.4054054,
      "logps/chosen": -198.66666666666666,
      "logps/rejected": -517.6216216216217,
      "loss": 0.2002,
      "rewards/chosen": 3.3495240388093173,
      "rewards/margins": 17.98465917394445,
      "rewards/rejected": -14.635135135135135,
      "step": 1323
    },
    {
      "epoch": 0.6522167487684729,
      "grad_norm": 0.49876635451348883,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -57470030.76923077,
      "logits/rejected": -69206016.0,
      "logps/chosen": -272.3076923076923,
      "logps/rejected": -577.6842105263158,
      "loss": 0.2057,
      "rewards/chosen": 0.6262019230769231,
      "rewards/margins": 15.941991396761134,
      "rewards/rejected": -15.31578947368421,
      "step": 1324
    },
    {
      "epoch": 0.6527093596059114,
      "grad_norm": 0.4471227946683958,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -61740154.88,
      "logits/rejected": -51568429.94871795,
      "logps/chosen": -271.84,
      "logps/rejected": -547.2820512820513,
      "loss": 0.1805,
      "rewards/chosen": 1.9450111389160156,
      "rewards/margins": 16.13731883122371,
      "rewards/rejected": -14.192307692307692,
      "step": 1325
    },
    {
      "epoch": 0.6532019704433497,
      "grad_norm": 0.41368821162353864,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -49941480.18604651,
      "logits/rejected": -54920710.02352941,
      "logps/chosen": -273.5813953488372,
      "logps/rejected": -485.6470588235294,
      "loss": 0.1562,
      "rewards/chosen": 6.473866307458212,
      "rewards/margins": 21.50916042510527,
      "rewards/rejected": -15.035294117647059,
      "step": 1326
    },
    {
      "epoch": 0.6536945812807882,
      "grad_norm": 0.32975284298267343,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -56480116.36363637,
      "logits/rejected": -62215509.333333336,
      "logps/chosen": -264.0,
      "logps/rejected": -557.7142857142857,
      "loss": 0.126,
      "rewards/chosen": 4.585956920276988,
      "rewards/margins": 21.633575967896036,
      "rewards/rejected": -17.047619047619047,
      "step": 1327
    },
    {
      "epoch": 0.6541871921182266,
      "grad_norm": 0.5148923879830405,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -50412307.692307696,
      "logits/rejected": -63797571.368421055,
      "logps/chosen": -242.46153846153845,
      "logps/rejected": -552.421052631579,
      "loss": 0.201,
      "rewards/chosen": 0.3239182692307692,
      "rewards/margins": 15.613391953441296,
      "rewards/rejected": -15.289473684210526,
      "step": 1328
    },
    {
      "epoch": 0.6546798029556651,
      "grad_norm": 0.5021595377684083,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -49404999.44186047,
      "logits/rejected": -63457352.28235294,
      "logps/chosen": -330.4186046511628,
      "logps/rejected": -577.5058823529412,
      "loss": 0.1438,
      "rewards/chosen": 4.730348099109738,
      "rewards/margins": 21.883289275580324,
      "rewards/rejected": -17.152941176470588,
      "step": 1329
    },
    {
      "epoch": 0.6551724137931034,
      "grad_norm": 0.5194437009306517,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -44903725.176470585,
      "logits/rejected": -47880431.37662338,
      "logps/chosen": -250.35294117647058,
      "logps/rejected": -492.05194805194805,
      "loss": 0.1902,
      "rewards/chosen": 1.359375,
      "rewards/margins": 15.878855519480519,
      "rewards/rejected": -14.519480519480519,
      "step": 1330
    },
    {
      "epoch": 0.6556650246305419,
      "grad_norm": 0.33210972544267525,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -37249414.0952381,
      "logits/rejected": -67303947.90697674,
      "logps/chosen": -195.42857142857142,
      "logps/rejected": -588.6511627906976,
      "loss": 0.1436,
      "rewards/chosen": 3.7272774832589284,
      "rewards/margins": 20.355184460003112,
      "rewards/rejected": -16.627906976744185,
      "step": 1331
    },
    {
      "epoch": 0.6561576354679803,
      "grad_norm": 0.45667920939323964,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -63501762.56,
      "logits/rejected": -58290070.974358976,
      "logps/chosen": -348.48,
      "logps/rejected": -510.35897435897436,
      "loss": 0.1591,
      "rewards/chosen": 3.0756625366210937,
      "rewards/margins": 18.83207279303135,
      "rewards/rejected": -15.756410256410257,
      "step": 1332
    },
    {
      "epoch": 0.6566502463054187,
      "grad_norm": 0.6164213276316141,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -80641429.73584905,
      "logits/rejected": -48653926.4,
      "logps/chosen": -426.8679245283019,
      "logps/rejected": -496.2133333333333,
      "loss": 0.2472,
      "rewards/chosen": -0.6375294811320755,
      "rewards/margins": 14.015803852201259,
      "rewards/rejected": -14.653333333333334,
      "step": 1333
    },
    {
      "epoch": 0.6571428571428571,
      "grad_norm": 0.4332796326175513,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -60346618.77551021,
      "logits/rejected": -70984613.26582278,
      "logps/chosen": -384.3265306122449,
      "logps/rejected": -508.75949367088606,
      "loss": 0.1971,
      "rewards/chosen": 0.02295918367346939,
      "rewards/margins": 14.93435158873676,
      "rewards/rejected": -14.91139240506329,
      "step": 1334
    },
    {
      "epoch": 0.6576354679802956,
      "grad_norm": 0.5833183719516665,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -58977050.12244898,
      "logits/rejected": -52349161.3164557,
      "logps/chosen": -275.265306122449,
      "logps/rejected": -512.4050632911392,
      "loss": 0.1954,
      "rewards/chosen": -0.2911352040816326,
      "rewards/margins": 14.265826821234823,
      "rewards/rejected": -14.556962025316455,
      "step": 1335
    },
    {
      "epoch": 0.6581280788177339,
      "grad_norm": 0.5495340947344706,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -54525952.0,
      "logits/rejected": -65256833.66233766,
      "logps/chosen": -338.19607843137254,
      "logps/rejected": -596.7792207792207,
      "loss": 0.185,
      "rewards/chosen": -0.37254901960784315,
      "rewards/margins": 15.549528902470078,
      "rewards/rejected": -15.922077922077921,
      "step": 1336
    },
    {
      "epoch": 0.6586206896551724,
      "grad_norm": 0.5327707281507243,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -66642830.222222224,
      "logits/rejected": -59117014.48648649,
      "logps/chosen": -306.962962962963,
      "logps/rejected": -502.05405405405406,
      "loss": 0.2272,
      "rewards/chosen": -1.5679976851851851,
      "rewards/margins": 14.540110422922924,
      "rewards/rejected": -16.10810810810811,
      "step": 1337
    },
    {
      "epoch": 0.6591133004926109,
      "grad_norm": 0.47244062148258137,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -43546744.47058824,
      "logits/rejected": -60245457.45454545,
      "logps/chosen": -237.1764705882353,
      "logps/rejected": -625.038961038961,
      "loss": 0.1702,
      "rewards/chosen": 2.417531630572151,
      "rewards/margins": 21.690258903299423,
      "rewards/rejected": -19.272727272727273,
      "step": 1338
    },
    {
      "epoch": 0.6596059113300493,
      "grad_norm": 0.4514545750738939,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -44782933.333333336,
      "logits/rejected": -62442700.8,
      "logps/chosen": -219.5,
      "logps/rejected": -536.0,
      "loss": 0.1756,
      "rewards/chosen": -0.5286458333333334,
      "rewards/margins": 16.346354166666668,
      "rewards/rejected": -16.875,
      "step": 1339
    },
    {
      "epoch": 0.6600985221674877,
      "grad_norm": 0.6048480086712542,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -67592822.15384616,
      "logits/rejected": -55077834.10526316,
      "logps/chosen": -433.2307692307692,
      "logps/rejected": -552.421052631579,
      "loss": 0.2027,
      "rewards/chosen": 2.0925686176006613,
      "rewards/margins": 19.776779143916453,
      "rewards/rejected": -17.68421052631579,
      "step": 1340
    },
    {
      "epoch": 0.6605911330049261,
      "grad_norm": 0.4424855501775865,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -43302305.18518519,
      "logits/rejected": -63027919.567567565,
      "logps/chosen": -262.51851851851853,
      "logps/rejected": -544.8648648648649,
      "loss": 0.1753,
      "rewards/chosen": 1.0734953703703705,
      "rewards/margins": 18.168089964964963,
      "rewards/rejected": -17.094594594594593,
      "step": 1341
    },
    {
      "epoch": 0.6610837438423646,
      "grad_norm": 0.752817406360772,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -61734912.0,
      "logits/rejected": -54158950.4,
      "logps/chosen": -379.6666666666667,
      "logps/rejected": -517.6,
      "loss": 0.1626,
      "rewards/chosen": 0.35546875,
      "rewards/margins": 16.46796875,
      "rewards/rejected": -16.1125,
      "step": 1342
    },
    {
      "epoch": 0.6615763546798029,
      "grad_norm": 0.4182003784586825,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -46804619.63636363,
      "logits/rejected": -66210084.571428575,
      "logps/chosen": -377.45454545454544,
      "logps/rejected": -556.952380952381,
      "loss": 0.1708,
      "rewards/chosen": -0.28462357954545453,
      "rewards/margins": 15.858233563311687,
      "rewards/rejected": -16.142857142857142,
      "step": 1343
    },
    {
      "epoch": 0.6620689655172414,
      "grad_norm": 0.5574854512686669,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -51046586.18181818,
      "logits/rejected": -53127850.666666664,
      "logps/chosen": -283.6363636363636,
      "logps/rejected": -547.047619047619,
      "loss": 0.2166,
      "rewards/chosen": -1.28125,
      "rewards/margins": 13.46875,
      "rewards/rejected": -14.75,
      "step": 1344
    },
    {
      "epoch": 0.6625615763546798,
      "grad_norm": 0.47599159722446777,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -58478276.92307692,
      "logits/rejected": -53035870.315789476,
      "logps/chosen": -300.9230769230769,
      "logps/rejected": -534.3157894736842,
      "loss": 0.1899,
      "rewards/chosen": 0.6826923076923077,
      "rewards/margins": 15.551113360323887,
      "rewards/rejected": -14.868421052631579,
      "step": 1345
    },
    {
      "epoch": 0.6630541871921182,
      "grad_norm": 0.4777896125123047,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -54675748.571428575,
      "logits/rejected": -63613610.666666664,
      "logps/chosen": -300.0,
      "logps/rejected": -564.0,
      "loss": 0.1936,
      "rewards/chosen": 0.6316964285714286,
      "rewards/margins": 16.520585317460316,
      "rewards/rejected": -15.88888888888889,
      "step": 1346
    },
    {
      "epoch": 0.6635467980295566,
      "grad_norm": 1.1611353010959262,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -46474386.28571428,
      "logits/rejected": -59652323.55555555,
      "logps/chosen": -253.57142857142858,
      "logps/rejected": -528.4444444444445,
      "loss": 0.1746,
      "rewards/chosen": 1.3297991071428572,
      "rewards/margins": 20.2742435515873,
      "rewards/rejected": -18.944444444444443,
      "step": 1347
    },
    {
      "epoch": 0.6640394088669951,
      "grad_norm": 0.4948404367932326,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -39089937.86046512,
      "logits/rejected": -57980084.705882356,
      "logps/chosen": -296.5581395348837,
      "logps/rejected": -530.0705882352942,
      "loss": 0.1391,
      "rewards/chosen": 1.3386627906976745,
      "rewards/margins": 14.985721614227085,
      "rewards/rejected": -13.647058823529411,
      "step": 1348
    },
    {
      "epoch": 0.6645320197044335,
      "grad_norm": 0.46532957005932035,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -76612978.38297872,
      "logits/rejected": -45464184.09876543,
      "logps/chosen": -344.8510638297872,
      "logps/rejected": -455.9012345679012,
      "loss": 0.1849,
      "rewards/chosen": -0.39029255319148937,
      "rewards/margins": 15.510942014709745,
      "rewards/rejected": -15.901234567901234,
      "step": 1349
    },
    {
      "epoch": 0.6650246305418719,
      "grad_norm": 0.3885249729960594,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -84090680.19512194,
      "logits/rejected": -59780884.59770115,
      "logps/chosen": -414.0487804878049,
      "logps/rejected": -523.7701149425287,
      "loss": 0.1375,
      "rewards/chosen": 2.211890243902439,
      "rewards/margins": 18.25786725539669,
      "rewards/rejected": -16.04597701149425,
      "step": 1350
    },
    {
      "epoch": 0.6655172413793103,
      "grad_norm": 0.4304623890747988,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -45321784.88888889,
      "logits/rejected": -57189901.83783784,
      "logps/chosen": -278.51851851851853,
      "logps/rejected": -589.8378378378378,
      "loss": 0.171,
      "rewards/chosen": 2.994815402560764,
      "rewards/margins": 20.508328916074277,
      "rewards/rejected": -17.513513513513512,
      "step": 1351
    },
    {
      "epoch": 0.6660098522167488,
      "grad_norm": 0.48895925398739976,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -47098538.666666664,
      "logits/rejected": -59139686.4,
      "logps/chosen": -294.3333333333333,
      "logps/rejected": -590.4,
      "loss": 0.2097,
      "rewards/chosen": -1.97265625,
      "rewards/margins": 13.30234375,
      "rewards/rejected": -15.275,
      "step": 1352
    },
    {
      "epoch": 0.6665024630541871,
      "grad_norm": 0.522145922295499,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -48444211.2,
      "logits/rejected": -43145818.35294118,
      "logps/chosen": -296.53333333333336,
      "logps/rejected": -523.2941176470588,
      "loss": 0.1956,
      "rewards/chosen": 1.541155497233073,
      "rewards/margins": 15.319869845521216,
      "rewards/rejected": -13.778714348288144,
      "step": 1353
    },
    {
      "epoch": 0.6669950738916256,
      "grad_norm": 0.5517856365081376,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -57564682.448979594,
      "logits/rejected": -70506781.16455697,
      "logps/chosen": -301.7142857142857,
      "logps/rejected": -491.746835443038,
      "loss": 0.2215,
      "rewards/chosen": 2.3036144023038903,
      "rewards/margins": 16.442854908633006,
      "rewards/rejected": -14.139240506329115,
      "step": 1354
    },
    {
      "epoch": 0.6674876847290641,
      "grad_norm": 0.518867599804882,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -44356743.24528302,
      "logits/rejected": -56091825.49333333,
      "logps/chosen": -254.03773584905662,
      "logps/rejected": -585.3866666666667,
      "loss": 0.1779,
      "rewards/chosen": 0.5117924528301887,
      "rewards/margins": 18.16512578616352,
      "rewards/rejected": -17.653333333333332,
      "step": 1355
    },
    {
      "epoch": 0.6679802955665025,
      "grad_norm": 0.41960224057084206,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -57521883.428571425,
      "logits/rejected": -51941090.23255814,
      "logps/chosen": -301.14285714285717,
      "logps/rejected": -538.046511627907,
      "loss": 0.1204,
      "rewards/chosen": 4.607942127046131,
      "rewards/margins": 22.421895615418222,
      "rewards/rejected": -17.813953488372093,
      "step": 1356
    },
    {
      "epoch": 0.6684729064039409,
      "grad_norm": 0.4524940924690705,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -35367594.666666664,
      "logits/rejected": -50620006.4,
      "logps/chosen": -250.16666666666666,
      "logps/rejected": -593.6,
      "loss": 0.1955,
      "rewards/chosen": -0.5891927083333334,
      "rewards/margins": 16.94830729166667,
      "rewards/rejected": -17.5375,
      "step": 1357
    },
    {
      "epoch": 0.6689655172413793,
      "grad_norm": 0.4304866151482297,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -68932474.43478261,
      "logits/rejected": -50843148.487804875,
      "logps/chosen": -292.8695652173913,
      "logps/rejected": -526.829268292683,
      "loss": 0.1906,
      "rewards/chosen": -1.4079483695652173,
      "rewards/margins": 14.665222362142101,
      "rewards/rejected": -16.073170731707318,
      "step": 1358
    },
    {
      "epoch": 0.6694581280788178,
      "grad_norm": 0.41807989338303203,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -49847689.84615385,
      "logits/rejected": -71413544.42105263,
      "logps/chosen": -285.38461538461536,
      "logps/rejected": -489.2631578947368,
      "loss": 0.209,
      "rewards/chosen": 3.409303225003756,
      "rewards/margins": 14.225092698687966,
      "rewards/rejected": -10.81578947368421,
      "step": 1359
    },
    {
      "epoch": 0.6699507389162561,
      "grad_norm": 0.5928858486140149,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -46730877.58490566,
      "logits/rejected": -62075699.2,
      "logps/chosen": -291.92452830188677,
      "logps/rejected": -513.28,
      "loss": 0.2094,
      "rewards/chosen": 1.876306785727447,
      "rewards/margins": 15.142973452394115,
      "rewards/rejected": -13.266666666666667,
      "step": 1360
    },
    {
      "epoch": 0.6704433497536946,
      "grad_norm": 0.4299665525912451,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -53146575.23809524,
      "logits/rejected": -67303947.90697674,
      "logps/chosen": -266.6666666666667,
      "logps/rejected": -553.6744186046511,
      "loss": 0.1996,
      "rewards/chosen": 3.838557652064732,
      "rewards/margins": 16.838557652064733,
      "rewards/rejected": -13.0,
      "step": 1361
    },
    {
      "epoch": 0.670935960591133,
      "grad_norm": 0.5015308500834302,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -47928661.333333336,
      "logits/rejected": -56727961.6,
      "logps/chosen": -314.5,
      "logps/rejected": -478.4,
      "loss": 0.1825,
      "rewards/chosen": 0.5572916666666666,
      "rewards/margins": 14.732291666666667,
      "rewards/rejected": -14.175,
      "step": 1362
    },
    {
      "epoch": 0.6714285714285714,
      "grad_norm": 0.481152613098509,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -54953942.20408163,
      "logits/rejected": -47570840.30379747,
      "logps/chosen": -372.57142857142856,
      "logps/rejected": -524.1518987341772,
      "loss": 0.2445,
      "rewards/chosen": 0.6843342683753189,
      "rewards/margins": 14.899524141793039,
      "rewards/rejected": -14.215189873417721,
      "step": 1363
    },
    {
      "epoch": 0.6719211822660098,
      "grad_norm": 0.5277853008539152,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -49202412.307692304,
      "logits/rejected": -48289684.21052632,
      "logps/chosen": -315.0769230769231,
      "logps/rejected": -474.5263157894737,
      "loss": 0.2045,
      "rewards/chosen": 7.567979665902945,
      "rewards/margins": 21.067979665902946,
      "rewards/rejected": -13.5,
      "step": 1364
    },
    {
      "epoch": 0.6724137931034483,
      "grad_norm": 0.5244669504066632,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -75449809.45454545,
      "logits/rejected": -64362593.52380952,
      "logps/chosen": -276.0,
      "logps/rejected": -500.57142857142856,
      "loss": 0.163,
      "rewards/chosen": 8.69176552512429,
      "rewards/margins": 23.32271790607667,
      "rewards/rejected": -14.630952380952381,
      "step": 1365
    },
    {
      "epoch": 0.6729064039408867,
      "grad_norm": 0.502078612929415,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -46032486.4,
      "logits/rejected": -53286725.81818182,
      "logps/chosen": -194.8,
      "logps/rejected": -499.6363636363636,
      "loss": 0.2,
      "rewards/chosen": -0.1984375,
      "rewards/margins": 13.369744318181818,
      "rewards/rejected": -13.568181818181818,
      "step": 1366
    },
    {
      "epoch": 0.6733990147783251,
      "grad_norm": 0.646329193465149,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -46423319.27272727,
      "logits/rejected": -51480088.38095238,
      "logps/chosen": -237.63636363636363,
      "logps/rejected": -527.2380952380952,
      "loss": 0.1699,
      "rewards/chosen": 0.8760653409090909,
      "rewards/margins": 15.090351055194805,
      "rewards/rejected": -14.214285714285714,
      "step": 1367
    },
    {
      "epoch": 0.6738916256157635,
      "grad_norm": 0.46797541454856423,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -65788434.96296296,
      "logits/rejected": -56623104.0,
      "logps/chosen": -376.0,
      "logps/rejected": -529.2972972972973,
      "loss": 0.1726,
      "rewards/chosen": 3.701967027452257,
      "rewards/margins": 18.566831892317122,
      "rewards/rejected": -14.864864864864865,
      "step": 1368
    },
    {
      "epoch": 0.674384236453202,
      "grad_norm": 0.5376788152109091,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -40434113.56097561,
      "logits/rejected": -61612879.448275864,
      "logps/chosen": -252.29268292682926,
      "logps/rejected": -584.0919540229885,
      "loss": 0.1875,
      "rewards/chosen": 3.2329120170779344,
      "rewards/margins": 20.38233730443426,
      "rewards/rejected": -17.149425287356323,
      "step": 1369
    },
    {
      "epoch": 0.6748768472906403,
      "grad_norm": 0.45842237589029317,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -68336465.17073171,
      "logits/rejected": -63444874.29885057,
      "logps/chosen": -316.8780487804878,
      "logps/rejected": -480.0,
      "loss": 0.1716,
      "rewards/chosen": -0.42835365853658536,
      "rewards/margins": 10.631878920486404,
      "rewards/rejected": -11.060232579022989,
      "step": 1370
    },
    {
      "epoch": 0.6753694581280788,
      "grad_norm": 0.5437018351471934,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -36356989.67272727,
      "logits/rejected": -59869380.38356164,
      "logps/chosen": -271.7090909090909,
      "logps/rejected": -589.5890410958904,
      "loss": 0.2003,
      "rewards/chosen": 3.2875294078480115,
      "rewards/margins": 20.43821433935486,
      "rewards/rejected": -17.15068493150685,
      "step": 1371
    },
    {
      "epoch": 0.6758620689655173,
      "grad_norm": 0.436359984477189,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -43128386.782608695,
      "logits/rejected": -61380058.53658537,
      "logps/chosen": -250.08695652173913,
      "logps/rejected": -542.439024390244,
      "loss": 0.1945,
      "rewards/chosen": 0.5618206521739131,
      "rewards/margins": 15.927674310710497,
      "rewards/rejected": -15.365853658536585,
      "step": 1372
    },
    {
      "epoch": 0.6763546798029557,
      "grad_norm": 0.5266439547403126,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -46705322.666666664,
      "logits/rejected": -60397977.6,
      "logps/chosen": -336.3333333333333,
      "logps/rejected": -591.2,
      "loss": 0.1905,
      "rewards/chosen": 2.9160133997599282,
      "rewards/margins": 18.36601339975993,
      "rewards/rejected": -15.45,
      "step": 1373
    },
    {
      "epoch": 0.6768472906403941,
      "grad_norm": 0.49214587626688355,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -44724976.32653061,
      "logits/rejected": -57074389.87341772,
      "logps/chosen": -281.3061224489796,
      "logps/rejected": -550.8860759493671,
      "loss": 0.1946,
      "rewards/chosen": 3.020439303651148,
      "rewards/margins": 19.387527911246085,
      "rewards/rejected": -16.367088607594937,
      "step": 1374
    },
    {
      "epoch": 0.6773399014778325,
      "grad_norm": 0.6108553587464125,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -66483397.614035085,
      "logits/rejected": -57834135.43661972,
      "logps/chosen": -360.42105263157896,
      "logps/rejected": -658.0281690140845,
      "loss": 0.2187,
      "rewards/chosen": -1.6491228070175439,
      "rewards/margins": 17.900172967630343,
      "rewards/rejected": -19.549295774647888,
      "step": 1375
    },
    {
      "epoch": 0.677832512315271,
      "grad_norm": 0.4663823771739578,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -47069411.55555555,
      "logits/rejected": -68015740.54054055,
      "logps/chosen": -270.0740740740741,
      "logps/rejected": -588.972972972973,
      "loss": 0.1783,
      "rewards/chosen": 3.351309882269965,
      "rewards/margins": 19.418877449837534,
      "rewards/rejected": -16.06756756756757,
      "step": 1376
    },
    {
      "epoch": 0.6783251231527093,
      "grad_norm": 0.48581075784788025,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -45481984.0,
      "logits/rejected": -57619251.2,
      "logps/chosen": -239.0,
      "logps/rejected": -571.2,
      "loss": 0.1829,
      "rewards/chosen": 0.14192708333333334,
      "rewards/margins": 15.616927083333334,
      "rewards/rejected": -15.475,
      "step": 1377
    },
    {
      "epoch": 0.6788177339901478,
      "grad_norm": 0.49786474133993275,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -62723909.81818182,
      "logits/rejected": -57971273.14285714,
      "logps/chosen": -237.45454545454547,
      "logps/rejected": -547.8095238095239,
      "loss": 0.1613,
      "rewards/chosen": 4.068156849254262,
      "rewards/margins": 19.99672827782569,
      "rewards/rejected": -15.928571428571429,
      "step": 1378
    },
    {
      "epoch": 0.6793103448275862,
      "grad_norm": 0.4794051018208231,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -66368692.705882356,
      "logits/rejected": -63840575.16883117,
      "logps/chosen": -282.3529411764706,
      "logps/rejected": -549.4025974025974,
      "loss": 0.1713,
      "rewards/chosen": 4.628117280847886,
      "rewards/margins": 20.42032507305568,
      "rewards/rejected": -15.792207792207792,
      "step": 1379
    },
    {
      "epoch": 0.6798029556650246,
      "grad_norm": 0.4020506889400936,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -47098538.666666664,
      "logits/rejected": -60712550.4,
      "logps/chosen": -195.16666666666666,
      "logps/rejected": -576.0,
      "loss": 0.1682,
      "rewards/chosen": 0.45703125,
      "rewards/margins": 11.743350219726562,
      "rewards/rejected": -11.286318969726562,
      "step": 1380
    },
    {
      "epoch": 0.680295566502463,
      "grad_norm": 0.4871864510904372,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -58195968.0,
      "logits/rejected": -63343522.90909091,
      "logps/chosen": -287.2,
      "logps/rejected": -572.3636363636364,
      "loss": 0.1859,
      "rewards/chosen": 2.8132843017578124,
      "rewards/margins": 16.926920665394174,
      "rewards/rejected": -14.113636363636363,
      "step": 1381
    },
    {
      "epoch": 0.6807881773399015,
      "grad_norm": 0.47926510776290276,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -53314264.177777775,
      "logits/rejected": -61903884.3373494,
      "logps/chosen": -246.04444444444445,
      "logps/rejected": -635.3734939759037,
      "loss": 0.1639,
      "rewards/chosen": 6.056943088107639,
      "rewards/margins": 22.322003329071492,
      "rewards/rejected": -16.265060240963855,
      "step": 1382
    },
    {
      "epoch": 0.68128078817734,
      "grad_norm": 0.5463678742295157,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -53382050.90909091,
      "logits/rejected": -51530020.571428575,
      "logps/chosen": -350.90909090909093,
      "logps/rejected": -503.6190476190476,
      "loss": 0.1855,
      "rewards/chosen": 5.857654224742543,
      "rewards/margins": 20.310035177123495,
      "rewards/rejected": -14.452380952380953,
      "step": 1383
    },
    {
      "epoch": 0.6817733990147783,
      "grad_norm": 0.5075058231755483,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -41487997.58490566,
      "logits/rejected": -59111724.373333335,
      "logps/chosen": -308.22641509433964,
      "logps/rejected": -561.0666666666667,
      "loss": 0.233,
      "rewards/chosen": -1.471698113207547,
      "rewards/margins": 13.43496855345912,
      "rewards/rejected": -14.906666666666666,
      "step": 1384
    },
    {
      "epoch": 0.6822660098522167,
      "grad_norm": 0.4472242114995643,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -44502276.33898305,
      "logits/rejected": -54890674.08695652,
      "logps/chosen": -266.8474576271187,
      "logps/rejected": -501.3333333333333,
      "loss": 0.2083,
      "rewards/chosen": -0.607521186440678,
      "rewards/margins": 14.160594755588308,
      "rewards/rejected": -14.768115942028986,
      "step": 1385
    },
    {
      "epoch": 0.6827586206896552,
      "grad_norm": 0.7068705816919141,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -78306864.3018868,
      "logits/rejected": -50331648.0,
      "logps/chosen": -331.47169811320754,
      "logps/rejected": -506.02666666666664,
      "loss": 0.1921,
      "rewards/chosen": 1.0713443396226414,
      "rewards/margins": 15.791344339622642,
      "rewards/rejected": -14.72,
      "step": 1386
    },
    {
      "epoch": 0.6832512315270935,
      "grad_norm": 0.43826255198193076,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -48576888.1632653,
      "logits/rejected": -50862572.55696203,
      "logps/chosen": -352.0,
      "logps/rejected": -484.45569620253167,
      "loss": 0.21,
      "rewards/chosen": 3.4416401142976722,
      "rewards/margins": 16.985943911766025,
      "rewards/rejected": -13.544303797468354,
      "step": 1387
    },
    {
      "epoch": 0.683743842364532,
      "grad_norm": 0.4949590263725294,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -56023917.71428572,
      "logits/rejected": -54154304.81012658,
      "logps/chosen": -314.44897959183675,
      "logps/rejected": -514.0253164556962,
      "loss": 0.2011,
      "rewards/chosen": 1.0758928571428572,
      "rewards/margins": 16.480956148282097,
      "rewards/rejected": -15.405063291139241,
      "step": 1388
    },
    {
      "epoch": 0.6842364532019705,
      "grad_norm": 0.49102294096117444,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -37131926.5882353,
      "logits/rejected": -64385289.97402597,
      "logps/chosen": -319.52941176470586,
      "logps/rejected": -527.7922077922078,
      "loss": 0.1725,
      "rewards/chosen": 0.3125,
      "rewards/margins": 17.6112012987013,
      "rewards/rejected": -17.2987012987013,
      "step": 1389
    },
    {
      "epoch": 0.6847290640394089,
      "grad_norm": 0.4818083398464857,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -57440637.83050848,
      "logits/rejected": -57079006.60869565,
      "logps/chosen": -314.8474576271187,
      "logps/rejected": -553.7391304347826,
      "loss": 0.2073,
      "rewards/chosen": -0.02754237288135593,
      "rewards/margins": 15.769559076394007,
      "rewards/rejected": -15.797101449275363,
      "step": 1390
    },
    {
      "epoch": 0.6852216748768473,
      "grad_norm": 0.47464701528700876,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -52285812.36363637,
      "logits/rejected": -66260016.76190476,
      "logps/chosen": -270.54545454545456,
      "logps/rejected": -588.952380952381,
      "loss": 0.1803,
      "rewards/chosen": 2.8990943215110083,
      "rewards/margins": 19.672903845320533,
      "rewards/rejected": -16.773809523809526,
      "step": 1391
    },
    {
      "epoch": 0.6857142857142857,
      "grad_norm": 0.5256096244506482,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -55337752.77419355,
      "logits/rejected": -58275405.57575758,
      "logps/chosen": -371.61290322580646,
      "logps/rejected": -567.7575757575758,
      "loss": 0.2091,
      "rewards/chosen": 1.1045911235194052,
      "rewards/margins": 19.740954759883042,
      "rewards/rejected": -18.636363636363637,
      "step": 1392
    },
    {
      "epoch": 0.6862068965517242,
      "grad_norm": 0.360207379902082,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -51353337.43589743,
      "logits/rejected": -71208913.9775281,
      "logps/chosen": -342.56410256410254,
      "logps/rejected": -587.5056179775281,
      "loss": 0.178,
      "rewards/chosen": 1.7492157373672876,
      "rewards/margins": 17.861575287929085,
      "rewards/rejected": -16.1123595505618,
      "step": 1393
    },
    {
      "epoch": 0.6866995073891625,
      "grad_norm": 0.5245295088299957,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -48192552.96,
      "logits/rejected": -59849491.692307696,
      "logps/chosen": -319.04,
      "logps/rejected": -558.7692307692307,
      "loss": 0.2053,
      "rewards/chosen": 0.815,
      "rewards/margins": 16.04576923076923,
      "rewards/rejected": -15.23076923076923,
      "step": 1394
    },
    {
      "epoch": 0.687192118226601,
      "grad_norm": 0.4304967264011274,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -58759092.14814815,
      "logits/rejected": -59797171.89189189,
      "logps/chosen": -338.3703703703704,
      "logps/rejected": -549.1891891891892,
      "loss": 0.1426,
      "rewards/chosen": 2.0324074074074074,
      "rewards/margins": 18.437812812812815,
      "rewards/rejected": -16.405405405405407,
      "step": 1395
    },
    {
      "epoch": 0.6876847290640394,
      "grad_norm": 0.39854965860800323,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -50524806.7368421,
      "logits/rejected": -50052027.733333334,
      "logps/chosen": -460.2105263157895,
      "logps/rejected": -540.4444444444445,
      "loss": 0.1441,
      "rewards/chosen": 7.444102237099095,
      "rewards/margins": 23.68854668154354,
      "rewards/rejected": -16.244444444444444,
      "step": 1396
    },
    {
      "epoch": 0.6881773399014778,
      "grad_norm": 0.568844144506398,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -59698926.93333333,
      "logits/rejected": -70140890.9879518,
      "logps/chosen": -418.4888888888889,
      "logps/rejected": -606.0722891566265,
      "loss": 0.1504,
      "rewards/chosen": 3.5798573811848957,
      "rewards/margins": 20.20636340528128,
      "rewards/rejected": -16.626506024096386,
      "step": 1397
    },
    {
      "epoch": 0.6886699507389162,
      "grad_norm": 0.49188802651599267,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -44077641.14285714,
      "logits/rejected": -56215324.44444445,
      "logps/chosen": -178.42857142857142,
      "logps/rejected": -502.22222222222223,
      "loss": 0.1643,
      "rewards/chosen": 3.6361683436802457,
      "rewards/margins": 20.566723899235804,
      "rewards/rejected": -16.930555555555557,
      "step": 1398
    },
    {
      "epoch": 0.6891625615763547,
      "grad_norm": 0.5200787054649146,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -44778078.81481481,
      "logits/rejected": -64614953.51351351,
      "logps/chosen": -243.55555555555554,
      "logps/rejected": -635.6756756756756,
      "loss": 0.184,
      "rewards/chosen": 0.96875,
      "rewards/margins": 17.86064189189189,
      "rewards/rejected": -16.89189189189189,
      "step": 1399
    },
    {
      "epoch": 0.6896551724137931,
      "grad_norm": 0.4201570550894576,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -55210736.32653061,
      "logits/rejected": -54579044.4556962,
      "logps/chosen": -310.53061224489795,
      "logps/rejected": -555.746835443038,
      "loss": 0.1392,
      "rewards/chosen": 2.0095663265306123,
      "rewards/margins": 18.515895440454663,
      "rewards/rejected": -16.50632911392405,
      "step": 1400
    },
    {
      "epoch": 0.6901477832512315,
      "grad_norm": 0.4587920117970231,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -56058486.15384615,
      "logits/rejected": -60596655.15789474,
      "logps/chosen": -307.2307692307692,
      "logps/rejected": -513.2631578947369,
      "loss": 0.2199,
      "rewards/chosen": 0.9303213266225961,
      "rewards/margins": 16.27242658978049,
      "rewards/rejected": -15.342105263157896,
      "step": 1401
    },
    {
      "epoch": 0.69064039408867,
      "grad_norm": 0.4533109370805213,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -30943272.156862747,
      "logits/rejected": -72556012.05194806,
      "logps/chosen": -204.54901960784315,
      "logps/rejected": -563.5324675324675,
      "loss": 0.2339,
      "rewards/chosen": 0.19161254284428617,
      "rewards/margins": 16.022781374013118,
      "rewards/rejected": -15.831168831168831,
      "step": 1402
    },
    {
      "epoch": 0.6911330049261084,
      "grad_norm": 0.504667395331523,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -49904450.37037037,
      "logits/rejected": -77311225.08108108,
      "logps/chosen": -287.4074074074074,
      "logps/rejected": -559.5675675675676,
      "loss": 0.1908,
      "rewards/chosen": 2.4502393934461804,
      "rewards/margins": 12.791312094565269,
      "rewards/rejected": -10.341072701119089,
      "step": 1403
    },
    {
      "epoch": 0.6916256157635468,
      "grad_norm": 0.5620433567949668,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -70378999.3220339,
      "logits/rejected": -64251874.31884058,
      "logps/chosen": -349.2881355932203,
      "logps/rejected": -527.768115942029,
      "loss": 0.2253,
      "rewards/chosen": -0.2934322033898305,
      "rewards/margins": 15.677582289363793,
      "rewards/rejected": -15.971014492753623,
      "step": 1404
    },
    {
      "epoch": 0.6921182266009852,
      "grad_norm": 0.5248619677466053,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -65195672.701754384,
      "logits/rejected": -55766520.788732395,
      "logps/chosen": -264.140350877193,
      "logps/rejected": -611.1549295774648,
      "loss": 0.2352,
      "rewards/chosen": 0.3072916666666667,
      "rewards/margins": 13.419967723004694,
      "rewards/rejected": -13.112676056338028,
      "step": 1405
    },
    {
      "epoch": 0.6926108374384237,
      "grad_norm": 0.5448243672393424,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -40165792.54237288,
      "logits/rejected": -65589188.63768116,
      "logps/chosen": -254.3728813559322,
      "logps/rejected": -550.4927536231884,
      "loss": 0.1838,
      "rewards/chosen": 1.31885593220339,
      "rewards/margins": 12.971029845246868,
      "rewards/rejected": -11.652173913043478,
      "step": 1406
    },
    {
      "epoch": 0.6931034482758621,
      "grad_norm": 0.3520386559791265,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -50506410.666666664,
      "logits/rejected": -63229132.8,
      "logps/chosen": -333.0,
      "logps/rejected": -592.8,
      "loss": 0.1803,
      "rewards/chosen": -0.3092447916666667,
      "rewards/margins": 13.490755208333335,
      "rewards/rejected": -13.8,
      "step": 1407
    },
    {
      "epoch": 0.6935960591133005,
      "grad_norm": 0.42696495700699444,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -47867494.4,
      "logits/rejected": -63724823.27272727,
      "logps/chosen": -266.2,
      "logps/rejected": -564.0,
      "loss": 0.1365,
      "rewards/chosen": 4.453911590576172,
      "rewards/margins": 19.64709340875799,
      "rewards/rejected": -15.193181818181818,
      "step": 1408
    },
    {
      "epoch": 0.6940886699507389,
      "grad_norm": 0.4128821929482779,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -64412525.71428572,
      "logits/rejected": -59729012.658227846,
      "logps/chosen": -315.42857142857144,
      "logps/rejected": -544.8101265822785,
      "loss": 0.183,
      "rewards/chosen": -0.11479591836734694,
      "rewards/margins": 15.821912942392146,
      "rewards/rejected": -15.936708860759493,
      "step": 1409
    },
    {
      "epoch": 0.6945812807881774,
      "grad_norm": 0.4551702144991465,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -54842503.24528302,
      "logits/rejected": -64424509.44,
      "logps/chosen": -302.49056603773585,
      "logps/rejected": -569.1733333333333,
      "loss": 0.1741,
      "rewards/chosen": 5.644485185731132,
      "rewards/margins": 21.5111518523978,
      "rewards/rejected": -15.866666666666667,
      "step": 1410
    },
    {
      "epoch": 0.6950738916256157,
      "grad_norm": 0.47412947099159375,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -56710485.333333336,
      "logits/rejected": -61027123.2,
      "logps/chosen": -357.0,
      "logps/rejected": -606.4,
      "loss": 0.1617,
      "rewards/chosen": 0.8255208333333334,
      "rewards/margins": 18.175520833333334,
      "rewards/rejected": -17.35,
      "step": 1411
    },
    {
      "epoch": 0.6955665024630542,
      "grad_norm": 0.47070793447038584,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -47613910.20408163,
      "logits/rejected": -46031159.088607594,
      "logps/chosen": -314.7755102040816,
      "logps/rejected": -499.0379746835443,
      "loss": 0.1715,
      "rewards/chosen": 1.1135204081632653,
      "rewards/margins": 18.910988762593643,
      "rewards/rejected": -17.79746835443038,
      "step": 1412
    },
    {
      "epoch": 0.6960591133004926,
      "grad_norm": 0.4954941571173225,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -55292219.07692308,
      "logits/rejected": -54194822.7368421,
      "logps/chosen": -269.2307692307692,
      "logps/rejected": -544.8421052631579,
      "loss": 0.2066,
      "rewards/chosen": 2.665273226224459,
      "rewards/margins": 17.836325857803406,
      "rewards/rejected": -15.171052631578947,
      "step": 1413
    },
    {
      "epoch": 0.696551724137931,
      "grad_norm": 0.47054810423174187,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -52803291.428571425,
      "logits/rejected": -48292750.222222224,
      "logps/chosen": -269.14285714285717,
      "logps/rejected": -531.5555555555555,
      "loss": 0.1978,
      "rewards/chosen": 2.0390625,
      "rewards/margins": 17.51128472222222,
      "rewards/rejected": -15.472222222222221,
      "step": 1414
    },
    {
      "epoch": 0.6970443349753694,
      "grad_norm": 0.5557941468670056,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -55277761.20754717,
      "logits/rejected": -52400837.97333334,
      "logps/chosen": -328.1509433962264,
      "logps/rejected": -521.3866666666667,
      "loss": 0.2221,
      "rewards/chosen": -1.0984669811320755,
      "rewards/margins": 11.728199685534591,
      "rewards/rejected": -12.826666666666666,
      "step": 1415
    },
    {
      "epoch": 0.6975369458128079,
      "grad_norm": 0.5576980991439802,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -53776969.14285714,
      "logits/rejected": -77245098.66666667,
      "logps/chosen": -225.42857142857142,
      "logps/rejected": -592.0,
      "loss": 0.1962,
      "rewards/chosen": 2.7533460344587053,
      "rewards/margins": 20.322790478903148,
      "rewards/rejected": -17.569444444444443,
      "step": 1416
    },
    {
      "epoch": 0.6980295566502464,
      "grad_norm": 0.49435970899564363,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -51971239.56363636,
      "logits/rejected": -93481268.60273972,
      "logps/chosen": -285.6727272727273,
      "logps/rejected": -528.2191780821918,
      "loss": 0.2156,
      "rewards/chosen": 1.1142045454545455,
      "rewards/margins": 15.484067559153175,
      "rewards/rejected": -14.36986301369863,
      "step": 1417
    },
    {
      "epoch": 0.6985221674876847,
      "grad_norm": 0.4828117964676111,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -36395119.70909091,
      "logits/rejected": -58030781.36986301,
      "logps/chosen": -225.45454545454547,
      "logps/rejected": -486.5753424657534,
      "loss": 0.2062,
      "rewards/chosen": -0.014772727272727272,
      "rewards/margins": 14.122213574097135,
      "rewards/rejected": -14.136986301369863,
      "step": 1418
    },
    {
      "epoch": 0.6990147783251232,
      "grad_norm": 0.8735990100004302,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -56890014.25454546,
      "logits/rejected": -53951389.80821918,
      "logps/chosen": -216.14545454545456,
      "logps/rejected": -459.83561643835617,
      "loss": 0.1685,
      "rewards/chosen": 2.929567649147727,
      "rewards/margins": 11.21723888202444,
      "rewards/rejected": -8.287671232876713,
      "step": 1419
    },
    {
      "epoch": 0.6995073891625616,
      "grad_norm": 0.3969786505325502,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -58662001.777777776,
      "logits/rejected": -51015501.91304348,
      "logps/chosen": -290.44444444444446,
      "logps/rejected": -485.2173913043478,
      "loss": 0.1408,
      "rewards/chosen": 3.9722696940104165,
      "rewards/margins": 17.25487838966259,
      "rewards/rejected": -13.282608695652174,
      "step": 1420
    },
    {
      "epoch": 0.7,
      "grad_norm": 0.5436777725106279,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -55306287.62790698,
      "logits/rejected": -55118089.035294116,
      "logps/chosen": -297.30232558139534,
      "logps/rejected": -536.8470588235294,
      "loss": 0.1723,
      "rewards/chosen": 1.5755817502043967,
      "rewards/margins": 9.706703935406601,
      "rewards/rejected": -8.131122185202205,
      "step": 1421
    },
    {
      "epoch": 0.7004926108374384,
      "grad_norm": 0.5414849011648222,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -45725403.428571425,
      "logits/rejected": -62128128.0,
      "logps/chosen": -324.0,
      "logps/rejected": -545.7777777777778,
      "loss": 0.1832,
      "rewards/chosen": 1.1950334821428572,
      "rewards/margins": 16.6394779265873,
      "rewards/rejected": -15.444444444444445,
      "step": 1422
    },
    {
      "epoch": 0.7009852216748769,
      "grad_norm": 0.4895489961437864,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -61533508.682926826,
      "logits/rejected": -66048235.40229885,
      "logps/chosen": -265.3658536585366,
      "logps/rejected": -542.1609195402299,
      "loss": 0.1783,
      "rewards/chosen": 6.914547803925305,
      "rewards/margins": 22.15592711427013,
      "rewards/rejected": -15.241379310344827,
      "step": 1423
    },
    {
      "epoch": 0.7014778325123153,
      "grad_norm": 0.47110566452309255,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -59372703.28888889,
      "logits/rejected": -54677553.34939759,
      "logps/chosen": -330.6666666666667,
      "logps/rejected": -536.6746987951807,
      "loss": 0.2056,
      "rewards/chosen": 1.6090662638346354,
      "rewards/margins": 16.76569276985873,
      "rewards/rejected": -15.156626506024097,
      "step": 1424
    },
    {
      "epoch": 0.7019704433497537,
      "grad_norm": 0.4793051598513379,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -60973578.89361702,
      "logits/rejected": -56648994.7654321,
      "logps/chosen": -385.02127659574467,
      "logps/rejected": -470.91358024691357,
      "loss": 0.1693,
      "rewards/chosen": 7.11629924368351,
      "rewards/margins": 20.51136097207857,
      "rewards/rejected": -13.395061728395062,
      "step": 1425
    },
    {
      "epoch": 0.7024630541871921,
      "grad_norm": 0.5601405512321483,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -48986305.20754717,
      "logits/rejected": -53910787.413333334,
      "logps/chosen": -302.79245283018867,
      "logps/rejected": -557.2266666666667,
      "loss": 0.215,
      "rewards/chosen": 1.4500450278228183,
      "rewards/margins": 16.836711694489484,
      "rewards/rejected": -15.386666666666667,
      "step": 1426
    },
    {
      "epoch": 0.7029556650246306,
      "grad_norm": 0.5430656576060344,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -52817161.481481485,
      "logits/rejected": -61724284.54054054,
      "logps/chosen": -294.6666666666667,
      "logps/rejected": -570.8108108108108,
      "loss": 0.1751,
      "rewards/chosen": 1.0671296296296295,
      "rewards/margins": 16.87794044044044,
      "rewards/rejected": -15.81081081081081,
      "step": 1427
    },
    {
      "epoch": 0.7034482758620689,
      "grad_norm": 0.4150836887243653,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -53141040.30188679,
      "logits/rejected": -61460534.61333334,
      "logps/chosen": -307.0188679245283,
      "logps/rejected": -496.2133333333333,
      "loss": 0.1717,
      "rewards/chosen": 3.7141101765182785,
      "rewards/margins": 19.460776843184945,
      "rewards/rejected": -15.746666666666666,
      "step": 1428
    },
    {
      "epoch": 0.7039408866995074,
      "grad_norm": 0.39854151171814567,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -35564202.666666664,
      "logits/rejected": -56486333.217391305,
      "logps/chosen": -310.22222222222223,
      "logps/rejected": -572.5217391304348,
      "loss": 0.131,
      "rewards/chosen": 3.0243252648247614,
      "rewards/margins": 20.241716569172585,
      "rewards/rejected": -17.217391304347824,
      "step": 1429
    },
    {
      "epoch": 0.7044334975369458,
      "grad_norm": 0.6355358758696302,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -53638695.384615384,
      "logits/rejected": -44978391.578947365,
      "logps/chosen": -302.7692307692308,
      "logps/rejected": -497.2631578947368,
      "loss": 0.1856,
      "rewards/chosen": 0.3545673076923077,
      "rewards/margins": 15.380883097165993,
      "rewards/rejected": -15.026315789473685,
      "step": 1430
    },
    {
      "epoch": 0.7049261083743842,
      "grad_norm": 0.4716794347107994,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -56489243.234042555,
      "logits/rejected": -60532609.58024691,
      "logps/chosen": -411.06382978723406,
      "logps/rejected": -576.0,
      "loss": 0.1812,
      "rewards/chosen": 0.6040558510638298,
      "rewards/margins": 16.850969431310745,
      "rewards/rejected": -16.246913580246915,
      "step": 1431
    },
    {
      "epoch": 0.7054187192118226,
      "grad_norm": 0.48839806687377035,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -62015780.571428575,
      "logits/rejected": -67108864.0,
      "logps/chosen": -332.40816326530614,
      "logps/rejected": -586.5316455696203,
      "loss": 0.1825,
      "rewards/chosen": 0.41517857142857145,
      "rewards/margins": 15.668343128390596,
      "rewards/rejected": -15.253164556962025,
      "step": 1432
    },
    {
      "epoch": 0.7059113300492611,
      "grad_norm": 0.47832925955491534,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -52519980.52173913,
      "logits/rejected": -55395502.82926829,
      "logps/chosen": -344.69565217391306,
      "logps/rejected": -512.0,
      "loss": 0.1801,
      "rewards/chosen": -1.5584239130434783,
      "rewards/margins": 14.148893160127253,
      "rewards/rejected": -15.707317073170731,
      "step": 1433
    },
    {
      "epoch": 0.7064039408866996,
      "grad_norm": 0.6091210685444179,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -59077218.042553194,
      "logits/rejected": -68248057.67901234,
      "logps/chosen": -353.02127659574467,
      "logps/rejected": -565.7283950617284,
      "loss": 0.1711,
      "rewards/chosen": 3.10754849048371,
      "rewards/margins": 17.947054663323215,
      "rewards/rejected": -14.839506172839506,
      "step": 1434
    },
    {
      "epoch": 0.7068965517241379,
      "grad_norm": 0.41378501967741305,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -63001941.333333336,
      "logits/rejected": -68472012.8,
      "logps/chosen": -330.3333333333333,
      "logps/rejected": -529.6,
      "loss": 0.1528,
      "rewards/chosen": 1.4127604166666667,
      "rewards/margins": 15.637760416666666,
      "rewards/rejected": -14.225,
      "step": 1435
    },
    {
      "epoch": 0.7073891625615764,
      "grad_norm": 0.4656051185065364,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -76409277.2173913,
      "logits/rejected": -52326499.90243903,
      "logps/chosen": -344.3478260869565,
      "logps/rejected": -520.5853658536586,
      "loss": 0.1653,
      "rewards/chosen": 0.297554347826087,
      "rewards/margins": 10.698836681572276,
      "rewards/rejected": -10.401282333746188,
      "step": 1436
    },
    {
      "epoch": 0.7078817733990148,
      "grad_norm": 0.4580155925711123,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -46510171.02222222,
      "logits/rejected": -53565810.12048193,
      "logps/chosen": -316.0888888888889,
      "logps/rejected": -522.7951807228916,
      "loss": 0.1869,
      "rewards/chosen": 2.301399400499132,
      "rewards/margins": 17.168869280017205,
      "rewards/rejected": -14.867469879518072,
      "step": 1437
    },
    {
      "epoch": 0.7083743842364532,
      "grad_norm": 0.5508615434635092,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -44413019.02222222,
      "logits/rejected": -58113850.60240964,
      "logps/chosen": -307.9111111111111,
      "logps/rejected": -531.277108433735,
      "loss": 0.1875,
      "rewards/chosen": 3.8041781955295138,
      "rewards/margins": 18.59935891842108,
      "rewards/rejected": -14.795180722891565,
      "step": 1438
    },
    {
      "epoch": 0.7088669950738916,
      "grad_norm": 0.47948947060223374,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -71787126.15384616,
      "logits/rejected": -59603267.368421055,
      "logps/chosen": -313.84615384615387,
      "logps/rejected": -559.1578947368421,
      "loss": 0.1988,
      "rewards/chosen": 0.0546875,
      "rewards/margins": 14.436266447368421,
      "rewards/rejected": -14.381578947368421,
      "step": 1439
    },
    {
      "epoch": 0.7093596059113301,
      "grad_norm": 0.439709336329677,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -66409813.333333336,
      "logits/rejected": -63651397.18918919,
      "logps/chosen": -261.037037037037,
      "logps/rejected": -490.81081081081084,
      "loss": 0.1693,
      "rewards/chosen": 0.4285300925925926,
      "rewards/margins": 14.442043606106107,
      "rewards/rejected": -14.013513513513514,
      "step": 1440
    },
    {
      "epoch": 0.7098522167487685,
      "grad_norm": 0.6407965952620597,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -58291293.09090909,
      "logits/rejected": -59319442.28571428,
      "logps/chosen": -204.0,
      "logps/rejected": -520.3809523809524,
      "loss": 0.1355,
      "rewards/chosen": 3.210237676447088,
      "rewards/margins": 17.495951962161374,
      "rewards/rejected": -14.285714285714286,
      "step": 1441
    },
    {
      "epoch": 0.7103448275862069,
      "grad_norm": 0.47130731580007623,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -67871464.72727273,
      "logits/rejected": -66559609.9047619,
      "logps/chosen": -434.1818181818182,
      "logps/rejected": -613.3333333333334,
      "loss": 0.1612,
      "rewards/chosen": 7.23224570534446,
      "rewards/margins": 20.33938856248732,
      "rewards/rejected": -13.107142857142858,
      "step": 1442
    },
    {
      "epoch": 0.7108374384236453,
      "grad_norm": 0.5811173912022892,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -76498385.45454545,
      "logits/rejected": -62614966.85714286,
      "logps/chosen": -381.09090909090907,
      "logps/rejected": -538.6666666666666,
      "loss": 0.16,
      "rewards/chosen": 8.50633378462358,
      "rewards/margins": 23.292048070337867,
      "rewards/rejected": -14.785714285714286,
      "step": 1443
    },
    {
      "epoch": 0.7113300492610838,
      "grad_norm": 0.5745768074430554,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -65885525.333333336,
      "logits/rejected": -60869836.8,
      "logps/chosen": -277.6666666666667,
      "logps/rejected": -528.8,
      "loss": 0.1854,
      "rewards/chosen": 0.5260416666666666,
      "rewards/margins": 14.301041666666666,
      "rewards/rejected": -13.775,
      "step": 1444
    },
    {
      "epoch": 0.7118226600985221,
      "grad_norm": 0.43460676660559955,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -31840905.36585366,
      "logits/rejected": -57322154.666666664,
      "logps/chosen": -255.02439024390245,
      "logps/rejected": -496.55172413793105,
      "loss": 0.1502,
      "rewards/chosen": 0.5007621951219512,
      "rewards/margins": 12.92604955144379,
      "rewards/rejected": -12.425287356321839,
      "step": 1445
    },
    {
      "epoch": 0.7123152709359606,
      "grad_norm": 0.35250790101641816,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -51811990.5882353,
      "logits/rejected": -60626757.81818182,
      "logps/chosen": -266.3529411764706,
      "logps/rejected": -500.3636363636364,
      "loss": 0.1781,
      "rewards/chosen": -0.37714460784313725,
      "rewards/margins": 13.038439807741279,
      "rewards/rejected": -13.415584415584416,
      "step": 1446
    },
    {
      "epoch": 0.712807881773399,
      "grad_norm": 0.439657251330718,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -55507597.61702128,
      "logits/rejected": -49606706.56790123,
      "logps/chosen": -212.25531914893617,
      "logps/rejected": -504.0987654320988,
      "loss": 0.1416,
      "rewards/chosen": 1.9574468085106382,
      "rewards/margins": 16.15497767270817,
      "rewards/rejected": -14.197530864197532,
      "step": 1447
    },
    {
      "epoch": 0.7133004926108374,
      "grad_norm": 0.4945193569404091,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -73869419.78947368,
      "logits/rejected": -51263715.55555555,
      "logps/chosen": -342.7368421052632,
      "logps/rejected": -497.77777777777777,
      "loss": 0.1639,
      "rewards/chosen": 2.93300367656507,
      "rewards/margins": 16.555225898787292,
      "rewards/rejected": -13.622222222222222,
      "step": 1448
    },
    {
      "epoch": 0.7137931034482758,
      "grad_norm": 0.441852981514593,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -45497534.915254235,
      "logits/rejected": -50574796.057971016,
      "logps/chosen": -269.8305084745763,
      "logps/rejected": -494.3768115942029,
      "loss": 0.1907,
      "rewards/chosen": 2.1904043423927435,
      "rewards/margins": 18.262868110508684,
      "rewards/rejected": -16.07246376811594,
      "step": 1449
    },
    {
      "epoch": 0.7142857142857143,
      "grad_norm": 0.44348505320942694,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -55372878.76923077,
      "logits/rejected": -52042482.526315786,
      "logps/chosen": -261.53846153846155,
      "logps/rejected": -552.0,
      "loss": 0.1727,
      "rewards/chosen": 2.9110594529371996,
      "rewards/margins": 19.46369103188457,
      "rewards/rejected": -16.55263157894737,
      "step": 1450
    },
    {
      "epoch": 0.7147783251231528,
      "grad_norm": 0.42404514896624507,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -38221623.21568628,
      "logits/rejected": -65910491.428571425,
      "logps/chosen": -269.1764705882353,
      "logps/rejected": -527.3766233766233,
      "loss": 0.1471,
      "rewards/chosen": 2.836411120844822,
      "rewards/margins": 12.271812145601317,
      "rewards/rejected": -9.435401024756494,
      "step": 1451
    },
    {
      "epoch": 0.7152709359605911,
      "grad_norm": 0.5259124995481929,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -56187846.03773585,
      "logits/rejected": -56315521.70666666,
      "logps/chosen": -313.9622641509434,
      "logps/rejected": -572.5866666666667,
      "loss": 0.1822,
      "rewards/chosen": 0.8349056603773585,
      "rewards/margins": 17.52823899371069,
      "rewards/rejected": -16.69333333333333,
      "step": 1452
    },
    {
      "epoch": 0.7157635467980296,
      "grad_norm": 0.5064431141902217,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -64682746.98039216,
      "logits/rejected": -65311305.14285714,
      "logps/chosen": -318.4313725490196,
      "logps/rejected": -502.85714285714283,
      "loss": 0.1707,
      "rewards/chosen": 3.081510057636336,
      "rewards/margins": 13.678912655038934,
      "rewards/rejected": -10.597402597402597,
      "step": 1453
    },
    {
      "epoch": 0.716256157635468,
      "grad_norm": 0.4914316295634671,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -48833682.28571428,
      "logits/rejected": -61458204.44444445,
      "logps/chosen": -255.71428571428572,
      "logps/rejected": -536.0,
      "loss": 0.2232,
      "rewards/chosen": 2.1978465488978793,
      "rewards/margins": 16.989513215564546,
      "rewards/rejected": -14.791666666666666,
      "step": 1454
    },
    {
      "epoch": 0.7167487684729064,
      "grad_norm": 0.42325495011096037,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -60183385.302325584,
      "logits/rejected": -50825095.52941176,
      "logps/chosen": -282.4186046511628,
      "logps/rejected": -489.4117647058824,
      "loss": 0.15,
      "rewards/chosen": 4.469481712163881,
      "rewards/margins": 19.198893476869763,
      "rewards/rejected": -14.729411764705882,
      "step": 1455
    },
    {
      "epoch": 0.7172413793103448,
      "grad_norm": 0.4663171544651088,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -43536875.52,
      "logits/rejected": -56999515.897435896,
      "logps/chosen": -283.84,
      "logps/rejected": -486.15384615384613,
      "loss": 0.2005,
      "rewards/chosen": 5.023120422363281,
      "rewards/margins": 17.18978708902995,
      "rewards/rejected": -12.166666666666666,
      "step": 1456
    },
    {
      "epoch": 0.7177339901477833,
      "grad_norm": 0.4415208083658387,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -69799549.58490565,
      "logits/rejected": -61292762.45333333,
      "logps/chosen": -351.0943396226415,
      "logps/rejected": -544.0,
      "loss": 0.165,
      "rewards/chosen": 2.5937759111512384,
      "rewards/margins": 18.953775911151236,
      "rewards/rejected": -16.36,
      "step": 1457
    },
    {
      "epoch": 0.7182266009852217,
      "grad_norm": 0.5078567383691056,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -57759061.333333336,
      "logits/rejected": -62390272.0,
      "logps/chosen": -314.6666666666667,
      "logps/rejected": -558.4,
      "loss": 0.2067,
      "rewards/chosen": -0.7369791666666666,
      "rewards/margins": 10.738020833333334,
      "rewards/rejected": -11.475,
      "step": 1458
    },
    {
      "epoch": 0.7187192118226601,
      "grad_norm": 0.4212538546741305,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -59871633.568627454,
      "logits/rejected": -66782035.116883114,
      "logps/chosen": -378.3529411764706,
      "logps/rejected": -556.8831168831168,
      "loss": 0.1679,
      "rewards/chosen": 0.9632352941176471,
      "rewards/margins": 15.32687165775401,
      "rewards/rejected": -14.363636363636363,
      "step": 1459
    },
    {
      "epoch": 0.7192118226600985,
      "grad_norm": 0.5201518174165123,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -65230165.333333336,
      "logits/rejected": -70621593.6,
      "logps/chosen": -317.6666666666667,
      "logps/rejected": -596.8,
      "loss": 0.1481,
      "rewards/chosen": 3.4915409088134766,
      "rewards/margins": 19.591540908813478,
      "rewards/rejected": -16.1,
      "step": 1460
    },
    {
      "epoch": 0.719704433497537,
      "grad_norm": 0.4828229275673242,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -74898285.71428572,
      "logits/rejected": -75302388.09302326,
      "logps/chosen": -277.14285714285717,
      "logps/rejected": -628.0930232558139,
      "loss": 0.1678,
      "rewards/chosen": -0.32849702380952384,
      "rewards/margins": 15.229642511074196,
      "rewards/rejected": -15.55813953488372,
      "step": 1461
    },
    {
      "epoch": 0.7201970443349753,
      "grad_norm": 0.5359132037913797,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -50674818.32727273,
      "logits/rejected": -57858412.71232877,
      "logps/chosen": -324.07272727272726,
      "logps/rejected": -547.945205479452,
      "loss": 0.2182,
      "rewards/chosen": 0.6778409090909091,
      "rewards/margins": 16.28058063511831,
      "rewards/rejected": -15.602739726027398,
      "step": 1462
    },
    {
      "epoch": 0.7206896551724138,
      "grad_norm": 0.5586939831213632,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -65710762.666666664,
      "logits/rejected": -57239913.4117647,
      "logps/chosen": -383.46666666666664,
      "logps/rejected": -535.5294117647059,
      "loss": 0.2146,
      "rewards/chosen": 1.098828125,
      "rewards/margins": 16.319416360294117,
      "rewards/rejected": -15.220588235294118,
      "step": 1463
    },
    {
      "epoch": 0.7211822660098522,
      "grad_norm": 0.5289809291548156,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -59598133.58139535,
      "logits/rejected": -60200598.5882353,
      "logps/chosen": -390.69767441860466,
      "logps/rejected": -505.9764705882353,
      "loss": 0.1332,
      "rewards/chosen": 8.92959168899891,
      "rewards/margins": 24.24723874782244,
      "rewards/rejected": -15.31764705882353,
      "step": 1464
    },
    {
      "epoch": 0.7216748768472906,
      "grad_norm": 0.508731061321178,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -47185920.0,
      "logits/rejected": -64097568.820512824,
      "logps/chosen": -217.28,
      "logps/rejected": -550.5641025641025,
      "loss": 0.1554,
      "rewards/chosen": 5.479371337890625,
      "rewards/margins": 20.59475595327524,
      "rewards/rejected": -15.115384615384615,
      "step": 1465
    },
    {
      "epoch": 0.722167487684729,
      "grad_norm": 0.43055605621493626,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -57921340.952380955,
      "logits/rejected": -53452990.511627905,
      "logps/chosen": -254.28571428571428,
      "logps/rejected": -546.2325581395348,
      "loss": 0.1755,
      "rewards/chosen": 1.9784407842726934,
      "rewards/margins": 16.920301249388974,
      "rewards/rejected": -14.94186046511628,
      "step": 1466
    },
    {
      "epoch": 0.7226600985221675,
      "grad_norm": 0.4278568292073719,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -81252447.25581396,
      "logits/rejected": -45939964.988235295,
      "logps/chosen": -324.83720930232556,
      "logps/rejected": -517.2705882352941,
      "loss": 0.1741,
      "rewards/chosen": 6.125068132267442,
      "rewards/margins": 20.92506813226744,
      "rewards/rejected": -14.8,
      "step": 1467
    },
    {
      "epoch": 0.723152709359606,
      "grad_norm": 0.5900432259492333,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -49706964.42553192,
      "logits/rejected": -59134508.24691358,
      "logps/chosen": -345.531914893617,
      "logps/rejected": -611.1604938271605,
      "loss": 0.2087,
      "rewards/chosen": 1.1382978723404256,
      "rewards/margins": 14.95311268715524,
      "rewards/rejected": -13.814814814814815,
      "step": 1468
    },
    {
      "epoch": 0.7236453201970443,
      "grad_norm": 0.6369423965428387,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -58186435.49090909,
      "logits/rejected": -49125067.39726027,
      "logps/chosen": -346.1818181818182,
      "logps/rejected": -443.6164383561644,
      "loss": 0.2112,
      "rewards/chosen": 1.2,
      "rewards/margins": 7.674246926503639,
      "rewards/rejected": -6.474246926503639,
      "step": 1469
    },
    {
      "epoch": 0.7241379310344828,
      "grad_norm": 0.4916350872874596,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -50160451.91836735,
      "logits/rejected": -63073837.36708861,
      "logps/chosen": -298.2857142857143,
      "logps/rejected": -504.7088607594937,
      "loss": 0.1595,
      "rewards/chosen": 1.367984693877551,
      "rewards/margins": 15.836339124257298,
      "rewards/rejected": -14.468354430379748,
      "step": 1470
    },
    {
      "epoch": 0.7246305418719212,
      "grad_norm": 0.45865779365637277,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -55424731.428571425,
      "logits/rejected": -52428800.0,
      "logps/chosen": -330.6666666666667,
      "logps/rejected": -565.5813953488372,
      "loss": 0.1231,
      "rewards/chosen": 2.517857142857143,
      "rewards/margins": 16.552740863787374,
      "rewards/rejected": -14.034883720930232,
      "step": 1471
    },
    {
      "epoch": 0.7251231527093596,
      "grad_norm": 0.5416313469165649,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -68812800.0,
      "logits/rejected": -59594069.333333336,
      "logps/chosen": -330.0,
      "logps/rejected": -553.7777777777778,
      "loss": 0.2635,
      "rewards/chosen": -1.1785714285714286,
      "rewards/margins": 11.21031746031746,
      "rewards/rejected": -12.38888888888889,
      "step": 1472
    },
    {
      "epoch": 0.725615763546798,
      "grad_norm": 0.45155515412508446,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -57913659.07692308,
      "logits/rejected": -66060288.0,
      "logps/chosen": -312.9230769230769,
      "logps/rejected": -579.3684210526316,
      "loss": 0.1469,
      "rewards/chosen": 7.139501131497896,
      "rewards/margins": 23.455290605182107,
      "rewards/rejected": -16.31578947368421,
      "step": 1473
    },
    {
      "epoch": 0.7261083743842365,
      "grad_norm": 0.5102810805822175,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -42620245.333333336,
      "logits/rejected": -45927628.8,
      "logps/chosen": -323.0,
      "logps/rejected": -464.8,
      "loss": 0.1629,
      "rewards/chosen": 0.6783854166666666,
      "rewards/margins": 14.790885416666667,
      "rewards/rejected": -14.1125,
      "step": 1474
    },
    {
      "epoch": 0.7266009852216748,
      "grad_norm": 0.4798371183672393,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -53386195.47826087,
      "logits/rejected": -56571953.951219514,
      "logps/chosen": -289.39130434782606,
      "logps/rejected": -533.0731707317074,
      "loss": 0.1485,
      "rewards/chosen": 5.401499872622282,
      "rewards/margins": 20.291743775061306,
      "rewards/rejected": -14.890243902439025,
      "step": 1475
    },
    {
      "epoch": 0.7270935960591133,
      "grad_norm": 0.41932597388043413,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -46096223.37254902,
      "logits/rejected": -46055636.77922078,
      "logps/chosen": -271.05882352941177,
      "logps/rejected": -552.7272727272727,
      "loss": 0.182,
      "rewards/chosen": 1.0376838235294117,
      "rewards/margins": 15.661060446906035,
      "rewards/rejected": -14.623376623376624,
      "step": 1476
    },
    {
      "epoch": 0.7275862068965517,
      "grad_norm": 0.4826612182088675,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -43515904.0,
      "logits/rejected": -54892953.6,
      "logps/chosen": -271.0,
      "logps/rejected": -536.8,
      "loss": 0.172,
      "rewards/chosen": 0.8251953125,
      "rewards/margins": 15.0126953125,
      "rewards/rejected": -14.1875,
      "step": 1477
    },
    {
      "epoch": 0.7280788177339902,
      "grad_norm": 0.42816927365616536,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -46899944.72727273,
      "logits/rejected": -67051407.78082192,
      "logps/chosen": -227.92727272727274,
      "logps/rejected": -591.7808219178082,
      "loss": 0.165,
      "rewards/chosen": 1.0920454545454545,
      "rewards/margins": 15.379716687422167,
      "rewards/rejected": -14.287671232876713,
      "step": 1478
    },
    {
      "epoch": 0.7285714285714285,
      "grad_norm": 0.49308617900668755,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -49717847.414634146,
      "logits/rejected": -52645746.75862069,
      "logps/chosen": -229.5609756097561,
      "logps/rejected": -518.6206896551724,
      "loss": 0.1384,
      "rewards/chosen": 3.661564524580793,
      "rewards/margins": 18.696047283201484,
      "rewards/rejected": -15.03448275862069,
      "step": 1479
    },
    {
      "epoch": 0.729064039408867,
      "grad_norm": 0.47115122343892196,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -40689863.80487805,
      "logits/rejected": -60359409.287356324,
      "logps/chosen": -231.02439024390245,
      "logps/rejected": -545.1034482758621,
      "loss": 0.1529,
      "rewards/chosen": 3.839934186237614,
      "rewards/margins": 20.460623841410026,
      "rewards/rejected": -16.620689655172413,
      "step": 1480
    },
    {
      "epoch": 0.7295566502463054,
      "grad_norm": 0.423382644199708,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -63402269.76744186,
      "logits/rejected": -64542936.847058825,
      "logps/chosen": -429.3953488372093,
      "logps/rejected": -568.4705882352941,
      "loss": 0.1907,
      "rewards/chosen": 1.4949248557867005,
      "rewards/margins": 12.07869740289148,
      "rewards/rejected": -10.583772547104779,
      "step": 1481
    },
    {
      "epoch": 0.7300492610837438,
      "grad_norm": 0.47174853664241223,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -36165979.7735849,
      "logits/rejected": -52121217.70666666,
      "logps/chosen": -183.39622641509433,
      "logps/rejected": -568.32,
      "loss": 0.2147,
      "rewards/chosen": 2.604488732679835,
      "rewards/margins": 20.337822066013167,
      "rewards/rejected": -17.733333333333334,
      "step": 1482
    },
    {
      "epoch": 0.7305418719211823,
      "grad_norm": 0.4845914241241128,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -74365675.68253969,
      "logits/rejected": -65882836.676923074,
      "logps/chosen": -319.4920634920635,
      "logps/rejected": -591.7538461538461,
      "loss": 0.213,
      "rewards/chosen": 1.0248015873015872,
      "rewards/margins": 7.049738204639327,
      "rewards/rejected": -6.02493661733774,
      "step": 1483
    },
    {
      "epoch": 0.7310344827586207,
      "grad_norm": 0.705590676661556,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -40484151.652173914,
      "logits/rejected": -63068010.146341465,
      "logps/chosen": -250.6086956521739,
      "logps/rejected": -598.6341463414634,
      "loss": 0.1872,
      "rewards/chosen": 4.42864990234375,
      "rewards/margins": 19.13596697551448,
      "rewards/rejected": -14.707317073170731,
      "step": 1484
    },
    {
      "epoch": 0.7315270935960592,
      "grad_norm": 0.5623491058235867,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -43697799.8367347,
      "logits/rejected": -59569735.29113924,
      "logps/chosen": -306.2857142857143,
      "logps/rejected": -578.4303797468355,
      "loss": 0.1747,
      "rewards/chosen": 1.308673469387755,
      "rewards/margins": 18.739053216223198,
      "rewards/rejected": -17.430379746835442,
      "step": 1485
    },
    {
      "epoch": 0.7320197044334975,
      "grad_norm": 0.46295809176884195,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -58133053.44,
      "logits/rejected": -53880674.461538464,
      "logps/chosen": -288.32,
      "logps/rejected": -583.7948717948718,
      "loss": 0.1789,
      "rewards/chosen": -0.7521875,
      "rewards/margins": 17.83755608974359,
      "rewards/rejected": -18.58974358974359,
      "step": 1486
    },
    {
      "epoch": 0.732512315270936,
      "grad_norm": 0.538332661640476,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -48662486.20408163,
      "logits/rejected": -60525399.49367089,
      "logps/chosen": -290.61224489795916,
      "logps/rejected": -547.6455696202531,
      "loss": 0.1879,
      "rewards/chosen": 1.2748895761918049,
      "rewards/margins": 18.97109210783737,
      "rewards/rejected": -17.696202531645568,
      "step": 1487
    },
    {
      "epoch": 0.7330049261083744,
      "grad_norm": 0.43588618640268323,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -46406209.64102564,
      "logits/rejected": -56646667.505617976,
      "logps/chosen": -282.6666666666667,
      "logps/rejected": -570.2471910112359,
      "loss": 0.1362,
      "rewards/chosen": 2.3060897435897436,
      "rewards/margins": 18.48586502448862,
      "rewards/rejected": -16.179775280898877,
      "step": 1488
    },
    {
      "epoch": 0.7334975369458128,
      "grad_norm": 0.481197268045728,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -50973633.30612245,
      "logits/rejected": -58295516.35443038,
      "logps/chosen": -283.1020408163265,
      "logps/rejected": -598.6835443037975,
      "loss": 0.1816,
      "rewards/chosen": 0.8469576154436383,
      "rewards/margins": 19.65708419772212,
      "rewards/rejected": -18.810126582278482,
      "step": 1489
    },
    {
      "epoch": 0.7339901477832512,
      "grad_norm": 0.4270875898702156,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -55952015.36,
      "logits/rejected": -51622203.07692308,
      "logps/chosen": -204.64,
      "logps/rejected": -535.7948717948718,
      "loss": 0.1706,
      "rewards/chosen": 8.86875732421875,
      "rewards/margins": 25.932859888321317,
      "rewards/rejected": -17.064102564102566,
      "step": 1490
    },
    {
      "epoch": 0.7344827586206897,
      "grad_norm": 0.5935826951397605,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -53920108.08888889,
      "logits/rejected": -63167228.915662654,
      "logps/chosen": -330.31111111111113,
      "logps/rejected": -663.9036144578313,
      "loss": 0.1442,
      "rewards/chosen": 4.244460381401909,
      "rewards/margins": 23.232412188630825,
      "rewards/rejected": -18.987951807228917,
      "step": 1491
    },
    {
      "epoch": 0.734975369458128,
      "grad_norm": 0.6273528446783334,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -41213595.82608695,
      "logits/rejected": -64039861.07317073,
      "logps/chosen": -233.2173913043478,
      "logps/rejected": -613.4634146341464,
      "loss": 0.1883,
      "rewards/chosen": 0.3023097826086957,
      "rewards/margins": 15.777919538706257,
      "rewards/rejected": -15.475609756097562,
      "step": 1492
    },
    {
      "epoch": 0.7354679802955665,
      "grad_norm": 0.6648729371685592,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -44148665.37931035,
      "logits/rejected": -58360744.22857143,
      "logps/chosen": -247.17241379310346,
      "logps/rejected": -560.4571428571429,
      "loss": 0.2559,
      "rewards/chosen": -0.4040948275862069,
      "rewards/margins": 16.11019088669951,
      "rewards/rejected": -16.514285714285716,
      "step": 1493
    },
    {
      "epoch": 0.7359605911330049,
      "grad_norm": 0.5780383477641916,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -52942388.24489796,
      "logits/rejected": -57711499.341772154,
      "logps/chosen": -305.3061224489796,
      "logps/rejected": -571.1392405063291,
      "loss": 0.1802,
      "rewards/chosen": 0.6875,
      "rewards/margins": 19.269778481012658,
      "rewards/rejected": -18.582278481012658,
      "step": 1494
    },
    {
      "epoch": 0.7364532019704434,
      "grad_norm": 0.5356148062176879,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -57013271.81395349,
      "logits/rejected": -60842080.37647059,
      "logps/chosen": -300.27906976744185,
      "logps/rejected": -567.7176470588236,
      "loss": 0.1868,
      "rewards/chosen": 4.679501200831214,
      "rewards/margins": 20.255971789066507,
      "rewards/rejected": -15.576470588235294,
      "step": 1495
    },
    {
      "epoch": 0.7369458128078817,
      "grad_norm": 0.4860200005852432,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -59148246.20408163,
      "logits/rejected": -55056876.55696203,
      "logps/chosen": -319.0204081632653,
      "logps/rejected": -567.0886075949367,
      "loss": 0.1966,
      "rewards/chosen": 0.6415816326530612,
      "rewards/margins": 17.73018922758977,
      "rewards/rejected": -17.088607594936708,
      "step": 1496
    },
    {
      "epoch": 0.7374384236453202,
      "grad_norm": 0.5309704224456763,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -51170508.8,
      "logits/rejected": -58777712.21917808,
      "logps/chosen": -226.3272727272727,
      "logps/rejected": -525.1506849315068,
      "loss": 0.1832,
      "rewards/chosen": 3.181257213245739,
      "rewards/margins": 20.00317502146492,
      "rewards/rejected": -16.82191780821918,
      "step": 1497
    },
    {
      "epoch": 0.7379310344827587,
      "grad_norm": 0.40974967480939767,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -25631857.777777776,
      "logits/rejected": -70090357.20481928,
      "logps/chosen": -299.3777777777778,
      "logps/rejected": -532.8192771084338,
      "loss": 0.1655,
      "rewards/chosen": 3.537525092230903,
      "rewards/margins": 21.27246485126705,
      "rewards/rejected": -17.734939759036145,
      "step": 1498
    },
    {
      "epoch": 0.738423645320197,
      "grad_norm": 0.4903830612816885,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -50578371.76470588,
      "logits/rejected": -54961723.84415584,
      "logps/chosen": -329.0980392156863,
      "logps/rejected": -571.012987012987,
      "loss": 0.1908,
      "rewards/chosen": 0.03676470588235294,
      "rewards/margins": 16.880920550038194,
      "rewards/rejected": -16.844155844155843,
      "step": 1499
    },
    {
      "epoch": 0.7389162561576355,
      "grad_norm": 0.46987683529068813,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -50575502.88372093,
      "logits/rejected": -53637746.44705883,
      "logps/chosen": -300.6511627906977,
      "logps/rejected": -585.035294117647,
      "loss": 0.1455,
      "rewards/chosen": -0.7630813953488372,
      "rewards/margins": 16.695742134062925,
      "rewards/rejected": -17.458823529411763,
      "step": 1500
    },
    {
      "epoch": 0.7394088669950739,
      "grad_norm": 0.5275685572406184,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -46719886.222222224,
      "logits/rejected": -67732341.62162162,
      "logps/chosen": -273.18518518518516,
      "logps/rejected": -511.13513513513516,
      "loss": 0.213,
      "rewards/chosen": 1.5288300690827545,
      "rewards/margins": 19.582884123136807,
      "rewards/rejected": -18.054054054054053,
      "step": 1501
    },
    {
      "epoch": 0.7399014778325124,
      "grad_norm": 0.47173426465042706,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -60523806.72,
      "logits/rejected": -59419306.666666664,
      "logps/chosen": -365.12,
      "logps/rejected": -562.8717948717949,
      "loss": 0.1694,
      "rewards/chosen": -0.9375,
      "rewards/margins": 16.806089743589745,
      "rewards/rejected": -17.743589743589745,
      "step": 1502
    },
    {
      "epoch": 0.7403940886699507,
      "grad_norm": 0.5561948867911692,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -59889821.538461536,
      "logits/rejected": -71247979.78947368,
      "logps/chosen": -307.6923076923077,
      "logps/rejected": -606.3157894736842,
      "loss": 0.1918,
      "rewards/chosen": 0.6176382211538461,
      "rewards/margins": 14.6965855895749,
      "rewards/rejected": -14.078947368421053,
      "step": 1503
    },
    {
      "epoch": 0.7408866995073892,
      "grad_norm": 0.47420439346699716,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -59791142.127659574,
      "logits/rejected": -79950683.65432099,
      "logps/chosen": -221.61702127659575,
      "logps/rejected": -602.8641975308642,
      "loss": 0.181,
      "rewards/chosen": -0.02127659574468085,
      "rewards/margins": 16.336748095613345,
      "rewards/rejected": -16.358024691358025,
      "step": 1504
    },
    {
      "epoch": 0.7413793103448276,
      "grad_norm": 0.4479154230515311,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -48663458.90909091,
      "logits/rejected": -56523239.61904762,
      "logps/chosen": -222.54545454545453,
      "logps/rejected": -555.047619047619,
      "loss": 0.1707,
      "rewards/chosen": -0.10049715909090909,
      "rewards/margins": 16.28045522186147,
      "rewards/rejected": -16.38095238095238,
      "step": 1505
    },
    {
      "epoch": 0.741871921182266,
      "grad_norm": 0.5196482482553662,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -42683211.294117644,
      "logits/rejected": -59373913.766233765,
      "logps/chosen": -312.62745098039215,
      "logps/rejected": -548.1558441558442,
      "loss": 0.2109,
      "rewards/chosen": 1.913578706629136,
      "rewards/margins": 20.744747537797966,
      "rewards/rejected": -18.83116883116883,
      "step": 1506
    },
    {
      "epoch": 0.7423645320197044,
      "grad_norm": 0.5531109756731873,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -41441350.62068965,
      "logits/rejected": -61955861.942857146,
      "logps/chosen": -217.93103448275863,
      "logps/rejected": -596.1142857142858,
      "loss": 0.1916,
      "rewards/chosen": 1.6670258620689655,
      "rewards/margins": 20.295597290640394,
      "rewards/rejected": -18.62857142857143,
      "step": 1507
    },
    {
      "epoch": 0.7428571428571429,
      "grad_norm": 0.4424114766405649,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -45760932.102564104,
      "logits/rejected": -60558209.43820225,
      "logps/chosen": -215.7948717948718,
      "logps/rejected": -586.0674157303371,
      "loss": 0.1536,
      "rewards/chosen": -0.8798076923076923,
      "rewards/margins": 18.87300129645635,
      "rewards/rejected": -19.752808988764045,
      "step": 1508
    },
    {
      "epoch": 0.7433497536945812,
      "grad_norm": 0.5011788349530316,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -39462754.461538464,
      "logits/rejected": -54194822.7368421,
      "logps/chosen": -288.3076923076923,
      "logps/rejected": -500.63157894736844,
      "loss": 0.1892,
      "rewards/chosen": 0.16105769230769232,
      "rewards/margins": 17.38474190283401,
      "rewards/rejected": -17.223684210526315,
      "step": 1509
    },
    {
      "epoch": 0.7438423645320197,
      "grad_norm": 0.5624977918730771,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -53933278.60869565,
      "logits/rejected": -65881262.82926829,
      "logps/chosen": -238.95652173913044,
      "logps/rejected": -608.0,
      "loss": 0.1483,
      "rewards/chosen": 1.9123641304347827,
      "rewards/margins": 15.09709521846832,
      "rewards/rejected": -13.184731088033537,
      "step": 1510
    },
    {
      "epoch": 0.7443349753694581,
      "grad_norm": 0.4715515884301109,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -48036116.75675676,
      "logits/rejected": -55493868.307692304,
      "logps/chosen": -316.7567567567568,
      "logps/rejected": -515.1648351648352,
      "loss": 0.1459,
      "rewards/chosen": 7.320145890519425,
      "rewards/margins": 22.836629407002942,
      "rewards/rejected": -15.516483516483516,
      "step": 1511
    },
    {
      "epoch": 0.7448275862068966,
      "grad_norm": 0.6486116146584476,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -49562692.266666666,
      "logits/rejected": -60015555.76470588,
      "logps/chosen": -253.6,
      "logps/rejected": -604.2352941176471,
      "loss": 0.1897,
      "rewards/chosen": 0.9041666666666667,
      "rewards/margins": 21.507107843137252,
      "rewards/rejected": -20.602941176470587,
      "step": 1512
    },
    {
      "epoch": 0.7453201970443349,
      "grad_norm": 0.3710086638736267,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -43950951.4893617,
      "logits/rejected": -54888422.71604938,
      "logps/chosen": -303.1489361702128,
      "logps/rejected": -491.4567901234568,
      "loss": 0.1703,
      "rewards/chosen": -1.1409574468085106,
      "rewards/margins": 14.180030207512477,
      "rewards/rejected": -15.320987654320987,
      "step": 1513
    },
    {
      "epoch": 0.7458128078817734,
      "grad_norm": 0.49429197599931457,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -49920441.7254902,
      "logits/rejected": -54525952.0,
      "logps/chosen": -331.29411764705884,
      "logps/rejected": -548.5714285714286,
      "loss": 0.2097,
      "rewards/chosen": -1.8137254901960784,
      "rewards/margins": 17.926534250063664,
      "rewards/rejected": -19.74025974025974,
      "step": 1514
    },
    {
      "epoch": 0.7463054187192119,
      "grad_norm": 0.5098580426341796,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -39929774.08,
      "logits/rejected": -55708960.820512824,
      "logps/chosen": -205.12,
      "logps/rejected": -588.3076923076923,
      "loss": 0.1935,
      "rewards/chosen": 2.5837799072265626,
      "rewards/margins": 22.301728625175283,
      "rewards/rejected": -19.71794871794872,
      "step": 1515
    },
    {
      "epoch": 0.7467980295566502,
      "grad_norm": 0.5076700876504692,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -44350881.18518519,
      "logits/rejected": -59287053.83783784,
      "logps/chosen": -285.9259259259259,
      "logps/rejected": -596.3243243243244,
      "loss": 0.1906,
      "rewards/chosen": 1.0486111111111112,
      "rewards/margins": 20.643205705705704,
      "rewards/rejected": -19.594594594594593,
      "step": 1516
    },
    {
      "epoch": 0.7472906403940887,
      "grad_norm": 0.4950600020291534,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -41943040.0,
      "logits/rejected": -50486992.59259259,
      "logps/chosen": -305.02127659574467,
      "logps/rejected": -553.0864197530864,
      "loss": 0.219,
      "rewards/chosen": 1.2194609946392951,
      "rewards/margins": 19.663905439083738,
      "rewards/rejected": -18.444444444444443,
      "step": 1517
    },
    {
      "epoch": 0.7477832512315271,
      "grad_norm": 0.46827541102882503,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -49477252.74074074,
      "logits/rejected": -76971146.37837838,
      "logps/chosen": -304.0,
      "logps/rejected": -697.9459459459459,
      "loss": 0.1705,
      "rewards/chosen": 2.013893410011574,
      "rewards/margins": 21.986866382984545,
      "rewards/rejected": -19.972972972972972,
      "step": 1518
    },
    {
      "epoch": 0.7482758620689656,
      "grad_norm": 0.5457579965418795,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -50941728.58181818,
      "logits/rejected": -59639555.50684932,
      "logps/chosen": -311.1272727272727,
      "logps/rejected": -596.1643835616438,
      "loss": 0.2283,
      "rewards/chosen": -2.834090909090909,
      "rewards/margins": 17.796046077210462,
      "rewards/rejected": -20.63013698630137,
      "step": 1519
    },
    {
      "epoch": 0.7487684729064039,
      "grad_norm": 0.5090581377037062,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -51030698.666666664,
      "logits/rejected": -56143033.06024096,
      "logps/chosen": -342.22222222222223,
      "logps/rejected": -593.7349397590361,
      "loss": 0.1576,
      "rewards/chosen": 3.4430345323350693,
      "rewards/margins": 18.574421898204797,
      "rewards/rejected": -15.13138736586973,
      "step": 1520
    },
    {
      "epoch": 0.7492610837438424,
      "grad_norm": 0.4329372761321115,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -60390210.37037037,
      "logits/rejected": -61214166.48648649,
      "logps/chosen": -347.85185185185185,
      "logps/rejected": -614.918918918919,
      "loss": 0.1702,
      "rewards/chosen": 0.9780092592592593,
      "rewards/margins": 22.32936061061061,
      "rewards/rejected": -21.35135135135135,
      "step": 1521
    },
    {
      "epoch": 0.7497536945812808,
      "grad_norm": 0.43556908270717165,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -35742764.52173913,
      "logits/rejected": -50229347.90243903,
      "logps/chosen": -214.2608695652174,
      "logps/rejected": -540.8780487804878,
      "loss": 0.1611,
      "rewards/chosen": 0.5329483695652174,
      "rewards/margins": 18.630509345174975,
      "rewards/rejected": -18.097560975609756,
      "step": 1522
    },
    {
      "epoch": 0.7502463054187192,
      "grad_norm": 0.5888102878506728,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -25123024.979591835,
      "logits/rejected": -67215048.9113924,
      "logps/chosen": -266.61224489795916,
      "logps/rejected": -549.6708860759494,
      "loss": 0.1627,
      "rewards/chosen": 1.2015306122448979,
      "rewards/margins": 14.478811009297905,
      "rewards/rejected": -13.277280397053007,
      "step": 1523
    },
    {
      "epoch": 0.7507389162561576,
      "grad_norm": 0.5383028189548088,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -44496094.60869565,
      "logits/rejected": -51814999.414634146,
      "logps/chosen": -268.3478260869565,
      "logps/rejected": -616.5853658536586,
      "loss": 0.1635,
      "rewards/chosen": 6.521717900815218,
      "rewards/margins": 25.887571559351805,
      "rewards/rejected": -19.365853658536587,
      "step": 1524
    },
    {
      "epoch": 0.7512315270935961,
      "grad_norm": 0.48286809137348774,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -59223572.48,
      "logits/rejected": -53181623.79487179,
      "logps/chosen": -293.12,
      "logps/rejected": -570.2564102564103,
      "loss": 0.1591,
      "rewards/chosen": 0.4196875,
      "rewards/margins": 20.676097756410254,
      "rewards/rejected": -20.256410256410255,
      "step": 1525
    },
    {
      "epoch": 0.7517241379310344,
      "grad_norm": 0.49039264339615213,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -39546294.85714286,
      "logits/rejected": -52667716.05063291,
      "logps/chosen": -296.16326530612247,
      "logps/rejected": -597.0632911392405,
      "loss": 0.2004,
      "rewards/chosen": -2.683673469387755,
      "rewards/margins": 16.607465771118573,
      "rewards/rejected": -19.29113924050633,
      "step": 1526
    },
    {
      "epoch": 0.7522167487684729,
      "grad_norm": 0.3731846477176045,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -45088768.0,
      "logits/rejected": -46433412.51764706,
      "logps/chosen": -268.4651162790698,
      "logps/rejected": -558.6823529411764,
      "loss": 0.1487,
      "rewards/chosen": 0.6359011627906976,
      "rewards/margins": 22.18884233926129,
      "rewards/rejected": -21.55294117647059,
      "step": 1527
    },
    {
      "epoch": 0.7527093596059113,
      "grad_norm": 0.5870771569653082,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -50751078.4,
      "logits/rejected": -52966531.28205128,
      "logps/chosen": -237.44,
      "logps/rejected": -593.2307692307693,
      "loss": 0.2069,
      "rewards/chosen": 0.89375,
      "rewards/margins": 20.93221153846154,
      "rewards/rejected": -20.03846153846154,
      "step": 1528
    },
    {
      "epoch": 0.7532019704433498,
      "grad_norm": 0.5318476330832187,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -45726137.7254902,
      "logits/rejected": -66782035.116883114,
      "logps/chosen": -253.9607843137255,
      "logps/rejected": -590.961038961039,
      "loss": 0.224,
      "rewards/chosen": -1.3529411764705883,
      "rewards/margins": 18.90679908326967,
      "rewards/rejected": -20.25974025974026,
      "step": 1529
    },
    {
      "epoch": 0.7536945812807881,
      "grad_norm": 0.43432623275206406,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -78738525.0909091,
      "logits/rejected": -65710762.666666664,
      "logps/chosen": -343.27272727272725,
      "logps/rejected": -646.8571428571429,
      "loss": 0.1542,
      "rewards/chosen": 1.3025568181818181,
      "rewards/margins": 22.20731872294372,
      "rewards/rejected": -20.904761904761905,
      "step": 1530
    },
    {
      "epoch": 0.7541871921182266,
      "grad_norm": 0.4886224046862866,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -40772536.55813953,
      "logits/rejected": -54229883.48235294,
      "logps/chosen": -274.6046511627907,
      "logps/rejected": -609.8823529411765,
      "loss": 0.1822,
      "rewards/chosen": 5.5181391516397165,
      "rewards/margins": 28.365197975169128,
      "rewards/rejected": -22.847058823529412,
      "step": 1531
    },
    {
      "epoch": 0.7546798029556651,
      "grad_norm": 0.4567565020286927,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -49702502.4,
      "logits/rejected": -48758784.0,
      "logps/chosen": -273.8,
      "logps/rejected": -563.6363636363636,
      "loss": 0.1266,
      "rewards/chosen": 1.498828125,
      "rewards/margins": 22.83973721590909,
      "rewards/rejected": -21.34090909090909,
      "step": 1532
    },
    {
      "epoch": 0.7551724137931034,
      "grad_norm": 0.4927219578420668,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -52813991.18367347,
      "logits/rejected": -67852158.37974684,
      "logps/chosen": -346.44897959183675,
      "logps/rejected": -681.3164556962025,
      "loss": 0.1894,
      "rewards/chosen": 0.12183956224091198,
      "rewards/margins": 21.185130701481416,
      "rewards/rejected": -21.063291139240505,
      "step": 1533
    },
    {
      "epoch": 0.7556650246305419,
      "grad_norm": 0.512554516576634,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -52519980.52173913,
      "logits/rejected": -50178197.853658535,
      "logps/chosen": -306.7826086956522,
      "logps/rejected": -605.6585365853658,
      "loss": 0.1873,
      "rewards/chosen": -1.3702067499575408,
      "rewards/margins": 20.776134713457093,
      "rewards/rejected": -22.146341463414632,
      "step": 1534
    },
    {
      "epoch": 0.7561576354679803,
      "grad_norm": 0.4331011717680631,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -44515018.86792453,
      "logits/rejected": -56147749.54666667,
      "logps/chosen": -286.9433962264151,
      "logps/rejected": -637.44,
      "loss": 0.1688,
      "rewards/chosen": -0.8779363092386497,
      "rewards/margins": 16.836998586594685,
      "rewards/rejected": -17.714934895833334,
      "step": 1535
    },
    {
      "epoch": 0.7566502463054188,
      "grad_norm": 0.5597797506976947,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -68178000.31372549,
      "logits/rejected": -54253594.597402595,
      "logps/chosen": -357.96078431372547,
      "logps/rejected": -640.8311688311688,
      "loss": 0.1699,
      "rewards/chosen": -0.3238357843137255,
      "rewards/margins": 24.325514865036922,
      "rewards/rejected": -24.649350649350648,
      "step": 1536
    },
    {
      "epoch": 0.7571428571428571,
      "grad_norm": 0.40081019787965594,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -36635961.469387755,
      "logits/rejected": -49322891.341772154,
      "logps/chosen": -241.9591836734694,
      "logps/rejected": -593.8227848101266,
      "loss": 0.1644,
      "rewards/chosen": -0.7474489795918368,
      "rewards/margins": 22.163943425471455,
      "rewards/rejected": -22.911392405063292,
      "step": 1537
    },
    {
      "epoch": 0.7576354679802956,
      "grad_norm": 0.5264356102986425,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -52473420.25531915,
      "logits/rejected": -70578226.56790124,
      "logps/chosen": -297.8723404255319,
      "logps/rejected": -651.8518518518518,
      "loss": 0.1848,
      "rewards/chosen": -0.5039893617021277,
      "rewards/margins": 23.323171132125033,
      "rewards/rejected": -23.82716049382716,
      "step": 1538
    },
    {
      "epoch": 0.758128078817734,
      "grad_norm": 0.5912700660456777,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -54073625.09803922,
      "logits/rejected": -64167404.051948056,
      "logps/chosen": -260.70588235294116,
      "logps/rejected": -627.5324675324675,
      "loss": 0.1668,
      "rewards/chosen": -0.19117647058823528,
      "rewards/margins": 24.717914438502675,
      "rewards/rejected": -24.90909090909091,
      "step": 1539
    },
    {
      "epoch": 0.7586206896551724,
      "grad_norm": 0.5291070264269125,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -55499629.71428572,
      "logits/rejected": -62623288.88888889,
      "logps/chosen": -300.2857142857143,
      "logps/rejected": -592.8888888888889,
      "loss": 0.1871,
      "rewards/chosen": -0.47544642857142855,
      "rewards/margins": 21.607886904761905,
      "rewards/rejected": -22.083333333333332,
      "step": 1540
    },
    {
      "epoch": 0.7591133004926108,
      "grad_norm": 0.46783624878207924,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -45407899.82608695,
      "logits/rejected": -61584658.73170732,
      "logps/chosen": -308.17391304347825,
      "logps/rejected": -575.219512195122,
      "loss": 0.1726,
      "rewards/chosen": 1.8865804257600203,
      "rewards/margins": 20.496336523320995,
      "rewards/rejected": -18.609756097560975,
      "step": 1541
    },
    {
      "epoch": 0.7596059113300493,
      "grad_norm": 0.45447928117850145,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -52830382.29787234,
      "logits/rejected": -60687954.17283951,
      "logps/chosen": -299.9148936170213,
      "logps/rejected": -624.9876543209876,
      "loss": 0.1563,
      "rewards/chosen": -0.47606382978723405,
      "rewards/margins": 23.820232466509065,
      "rewards/rejected": -24.296296296296298,
      "step": 1542
    },
    {
      "epoch": 0.7600985221674876,
      "grad_norm": 0.5182078491399895,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -35241271.652173914,
      "logits/rejected": -55497802.92682927,
      "logps/chosen": -345.2173913043478,
      "logps/rejected": -638.439024390244,
      "loss": 0.1583,
      "rewards/chosen": 3.846458103345788,
      "rewards/margins": 26.748897127736033,
      "rewards/rejected": -22.902439024390244,
      "step": 1543
    },
    {
      "epoch": 0.7605911330049261,
      "grad_norm": 0.5353282496322738,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -62914560.0,
      "logits/rejected": -54421094.4,
      "logps/chosen": -380.6666666666667,
      "logps/rejected": -606.4,
      "loss": 0.1897,
      "rewards/chosen": -0.0377051035563151,
      "rewards/margins": 22.412294896443683,
      "rewards/rejected": -22.45,
      "step": 1544
    },
    {
      "epoch": 0.7610837438423645,
      "grad_norm": 0.49636855147323905,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -58888028.16,
      "logits/rejected": -62645694.35897436,
      "logps/chosen": -425.92,
      "logps/rejected": -598.1538461538462,
      "loss": 0.2087,
      "rewards/chosen": 1.232416229248047,
      "rewards/margins": 21.4888264856583,
      "rewards/rejected": -20.256410256410255,
      "step": 1545
    },
    {
      "epoch": 0.761576354679803,
      "grad_norm": 0.4136354470427946,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -40787466.448979594,
      "logits/rejected": -61268693.87341772,
      "logps/chosen": -278.2040816326531,
      "logps/rejected": -634.3291139240506,
      "loss": 0.192,
      "rewards/chosen": -2.020408163265306,
      "rewards/margins": 17.498579178506848,
      "rewards/rejected": -19.518987341772153,
      "step": 1546
    },
    {
      "epoch": 0.7620689655172413,
      "grad_norm": 0.4765611398735563,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -27555601.860465117,
      "logits/rejected": -62618491.48235294,
      "logps/chosen": -221.58139534883722,
      "logps/rejected": -561.6941176470589,
      "loss": 0.1556,
      "rewards/chosen": 0.34011627906976744,
      "rewards/margins": 20.50482216142271,
      "rewards/rejected": -20.16470588235294,
      "step": 1547
    },
    {
      "epoch": 0.7625615763546798,
      "grad_norm": 0.5592488316807515,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -49406433.88235294,
      "logits/rejected": -60681229.2987013,
      "logps/chosen": -260.54901960784315,
      "logps/rejected": -675.7402597402597,
      "loss": 0.1859,
      "rewards/chosen": 1.965405782063802,
      "rewards/margins": 24.35501617167419,
      "rewards/rejected": -22.38961038961039,
      "step": 1548
    },
    {
      "epoch": 0.7630541871921183,
      "grad_norm": 0.5157440249692516,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -45043177.73913044,
      "logits/rejected": -61789258.92682927,
      "logps/chosen": -296.0,
      "logps/rejected": -584.5853658536586,
      "loss": 0.182,
      "rewards/chosen": 4.889323027237602,
      "rewards/margins": 24.060054734554676,
      "rewards/rejected": -19.170731707317074,
      "step": 1549
    },
    {
      "epoch": 0.7635467980295566,
      "grad_norm": 0.5343621534632549,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -52943889.96491228,
      "logits/rejected": -52458337.35211267,
      "logps/chosen": -287.43859649122805,
      "logps/rejected": -521.0140845070423,
      "loss": 0.1809,
      "rewards/chosen": 0.6820175438596491,
      "rewards/margins": 20.03413021991599,
      "rewards/rejected": -19.35211267605634,
      "step": 1550
    },
    {
      "epoch": 0.7640394088669951,
      "grad_norm": 0.5372727607905413,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -56139145.84615385,
      "logits/rejected": -55740092.631578945,
      "logps/chosen": -313.84615384615387,
      "logps/rejected": -486.3157894736842,
      "loss": 0.2022,
      "rewards/chosen": 2.7263835026667667,
      "rewards/margins": 20.06848876582466,
      "rewards/rejected": -17.342105263157894,
      "step": 1551
    },
    {
      "epoch": 0.7645320197044335,
      "grad_norm": 0.408153232977436,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -43690666.666666664,
      "logits/rejected": -60712550.4,
      "logps/chosen": -263.3333333333333,
      "logps/rejected": -566.4,
      "loss": 0.1549,
      "rewards/chosen": 3.4352283477783203,
      "rewards/margins": 20.96022834777832,
      "rewards/rejected": -17.525,
      "step": 1552
    },
    {
      "epoch": 0.765024630541872,
      "grad_norm": 0.6808217829072382,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -46548550.2745098,
      "logits/rejected": -59973100.051948056,
      "logps/chosen": -287.05882352941177,
      "logps/rejected": -621.7142857142857,
      "loss": 0.1428,
      "rewards/chosen": 2.124387254901961,
      "rewards/margins": 22.669841800356508,
      "rewards/rejected": -20.545454545454547,
      "step": 1553
    },
    {
      "epoch": 0.7655172413793103,
      "grad_norm": 0.6679437109889456,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -44128523.13043478,
      "logits/rejected": -57339204.682926826,
      "logps/chosen": -358.95652173913044,
      "logps/rejected": -565.8536585365854,
      "loss": 0.1582,
      "rewards/chosen": 2.763797262440557,
      "rewards/margins": 20.861358238050315,
      "rewards/rejected": -18.097560975609756,
      "step": 1554
    },
    {
      "epoch": 0.7660098522167488,
      "grad_norm": 0.45259473596095867,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -53327579.428571425,
      "logits/rejected": -53145548.151898734,
      "logps/chosen": -279.0204081632653,
      "logps/rejected": -546.8354430379746,
      "loss": 0.1565,
      "rewards/chosen": 1.9764204998405612,
      "rewards/margins": 16.913129360600056,
      "rewards/rejected": -14.936708860759493,
      "step": 1555
    },
    {
      "epoch": 0.7665024630541872,
      "grad_norm": 0.440052564696523,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -47370962.823529415,
      "logits/rejected": -51148720.20779221,
      "logps/chosen": -265.0980392156863,
      "logps/rejected": -533.6103896103896,
      "loss": 0.1856,
      "rewards/chosen": 0.4215686274509804,
      "rewards/margins": 16.473516679399033,
      "rewards/rejected": -16.051948051948052,
      "step": 1556
    },
    {
      "epoch": 0.7669950738916256,
      "grad_norm": 0.45609284513027054,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -46260705.88235294,
      "logits/rejected": -66509677.71428572,
      "logps/chosen": -256.3137254901961,
      "logps/rejected": -596.7792207792207,
      "loss": 0.1848,
      "rewards/chosen": 0.17892156862745098,
      "rewards/margins": 16.81528520499109,
      "rewards/rejected": -16.636363636363637,
      "step": 1557
    },
    {
      "epoch": 0.767487684729064,
      "grad_norm": 0.5085099332153549,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -64901335.578947365,
      "logits/rejected": -54880400.225352116,
      "logps/chosen": -352.56140350877195,
      "logps/rejected": -533.1830985915493,
      "loss": 0.1899,
      "rewards/chosen": 1.075657894736842,
      "rewards/margins": 17.82213676797628,
      "rewards/rejected": -16.746478873239436,
      "step": 1558
    },
    {
      "epoch": 0.7679802955665025,
      "grad_norm": 0.5338539059261982,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -59828185.358490564,
      "logits/rejected": -62411243.52,
      "logps/chosen": -296.75471698113205,
      "logps/rejected": -568.32,
      "loss": 0.2175,
      "rewards/chosen": -0.23496462264150944,
      "rewards/margins": 13.96503537735849,
      "rewards/rejected": -14.2,
      "step": 1559
    },
    {
      "epoch": 0.7684729064039408,
      "grad_norm": 0.5460672541806998,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -71629391.64444445,
      "logits/rejected": -60893208.67469879,
      "logps/chosen": -299.3777777777778,
      "logps/rejected": -546.6987951807229,
      "loss": 0.1554,
      "rewards/chosen": 1.3381944444444445,
      "rewards/margins": 17.675543842034806,
      "rewards/rejected": -16.337349397590362,
      "step": 1560
    },
    {
      "epoch": 0.7689655172413793,
      "grad_norm": 0.43654922803782414,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -57629736.96,
      "logits/rejected": -56623104.0,
      "logps/chosen": -279.84,
      "logps/rejected": -571.0769230769231,
      "loss": 0.1952,
      "rewards/chosen": -1.5575,
      "rewards/margins": 16.49378205128205,
      "rewards/rejected": -18.05128205128205,
      "step": 1561
    },
    {
      "epoch": 0.7694581280788177,
      "grad_norm": 0.567026520998757,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -45500708.571428575,
      "logits/rejected": -60467882.666666664,
      "logps/chosen": -281.7142857142857,
      "logps/rejected": -512.8888888888889,
      "loss": 0.2002,
      "rewards/chosen": 2.1155106680733815,
      "rewards/margins": 15.26828844585116,
      "rewards/rejected": -13.152777777777779,
      "step": 1562
    },
    {
      "epoch": 0.7699507389162562,
      "grad_norm": 0.4129209173158797,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -47990641.11627907,
      "logits/rejected": -53094954.16470588,
      "logps/chosen": -201.67441860465115,
      "logps/rejected": -525.5529411764705,
      "loss": 0.1684,
      "rewards/chosen": 1.1824355014534884,
      "rewards/margins": 18.31184726615937,
      "rewards/rejected": -17.129411764705882,
      "step": 1563
    },
    {
      "epoch": 0.7704433497536946,
      "grad_norm": 0.5473810658849552,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -46640660.48,
      "logits/rejected": -57214608.41025641,
      "logps/chosen": -294.4,
      "logps/rejected": -514.4615384615385,
      "loss": 0.2147,
      "rewards/chosen": -1.6925,
      "rewards/margins": 13.102371794871797,
      "rewards/rejected": -14.794871794871796,
      "step": 1564
    },
    {
      "epoch": 0.770935960591133,
      "grad_norm": 0.43650543756586385,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -44040192.0,
      "logits/rejected": -62817018.04651163,
      "logps/chosen": -198.1904761904762,
      "logps/rejected": -612.4651162790698,
      "loss": 0.179,
      "rewards/chosen": 2.5479867117745534,
      "rewards/margins": 21.827056479216413,
      "rewards/rejected": -19.27906976744186,
      "step": 1565
    },
    {
      "epoch": 0.7714285714285715,
      "grad_norm": 0.5257215011893909,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -43078997.333333336,
      "logits/rejected": -58510540.8,
      "logps/chosen": -318.3333333333333,
      "logps/rejected": -524.8,
      "loss": 0.1529,
      "rewards/chosen": -0.4026692708333333,
      "rewards/margins": 16.33483072916667,
      "rewards/rejected": -16.7375,
      "step": 1566
    },
    {
      "epoch": 0.7719211822660098,
      "grad_norm": 0.5119546684765898,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -54646941.538461536,
      "logits/rejected": -52318423.578947365,
      "logps/chosen": -264.61538461538464,
      "logps/rejected": -565.8947368421053,
      "loss": 0.2099,
      "rewards/chosen": -0.9930138221153846,
      "rewards/margins": 13.598471452350076,
      "rewards/rejected": -14.59148527446546,
      "step": 1567
    },
    {
      "epoch": 0.7724137931034483,
      "grad_norm": 0.5549967919139162,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -49830155.13043478,
      "logits/rejected": -52940300.487804875,
      "logps/chosen": -313.39130434782606,
      "logps/rejected": -568.1951219512196,
      "loss": 0.1678,
      "rewards/chosen": 2.6372640858525815,
      "rewards/margins": 20.95433725658429,
      "rewards/rejected": -18.317073170731707,
      "step": 1568
    },
    {
      "epoch": 0.7729064039408867,
      "grad_norm": 0.45704471372459204,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -51358824.489795916,
      "logits/rejected": -74647992.70886075,
      "logps/chosen": -344.48979591836735,
      "logps/rejected": -530.632911392405,
      "loss": 0.1862,
      "rewards/chosen": 2.0957221206353633,
      "rewards/margins": 17.703317057344222,
      "rewards/rejected": -15.60759493670886,
      "step": 1569
    },
    {
      "epoch": 0.7733990147783252,
      "grad_norm": 0.4419218506279478,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -46628166.80851064,
      "logits/rejected": -64105535.209876545,
      "logps/chosen": -330.3829787234043,
      "logps/rejected": -599.7037037037037,
      "loss": 0.184,
      "rewards/chosen": 1.4534191375083112,
      "rewards/margins": 19.03366605108856,
      "rewards/rejected": -17.580246913580247,
      "step": 1570
    },
    {
      "epoch": 0.7738916256157635,
      "grad_norm": 0.4077042762636408,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -58220934.0952381,
      "logits/rejected": -53696845.39534884,
      "logps/chosen": -286.4761904761905,
      "logps/rejected": -572.2790697674419,
      "loss": 0.13,
      "rewards/chosen": 6.609367007300968,
      "rewards/margins": 23.58611119334748,
      "rewards/rejected": -16.976744186046513,
      "step": 1571
    },
    {
      "epoch": 0.774384236453202,
      "grad_norm": 0.5183212449663355,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -49111486.836363636,
      "logits/rejected": -63316753.53424658,
      "logps/chosen": -252.8,
      "logps/rejected": -647.8904109589041,
      "loss": 0.1885,
      "rewards/chosen": 0.7943181818181818,
      "rewards/margins": 18.71212640099626,
      "rewards/rejected": -17.91780821917808,
      "step": 1572
    },
    {
      "epoch": 0.7748768472906404,
      "grad_norm": 0.4251519617661679,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -60630162.28571428,
      "logits/rejected": -58254222.222222224,
      "logps/chosen": -266.85714285714283,
      "logps/rejected": -551.1111111111111,
      "loss": 0.1672,
      "rewards/chosen": 0.3994140625,
      "rewards/margins": 17.204969618055557,
      "rewards/rejected": -16.805555555555557,
      "step": 1573
    },
    {
      "epoch": 0.7753694581280788,
      "grad_norm": 0.506317519944243,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -36458180.92307692,
      "logits/rejected": -60762219.78947368,
      "logps/chosen": -239.69230769230768,
      "logps/rejected": -622.3157894736842,
      "loss": 0.2032,
      "rewards/chosen": 0.3467548076923077,
      "rewards/margins": 17.583596912955464,
      "rewards/rejected": -17.236842105263158,
      "step": 1574
    },
    {
      "epoch": 0.7758620689655172,
      "grad_norm": 0.6156444545104844,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -66230986.41860465,
      "logits/rejected": -51910680.09411765,
      "logps/chosen": -291.90697674418607,
      "logps/rejected": -575.2470588235294,
      "loss": 0.1751,
      "rewards/chosen": 2.9651666685592297,
      "rewards/margins": 21.12987255091217,
      "rewards/rejected": -18.16470588235294,
      "step": 1575
    },
    {
      "epoch": 0.7763546798029557,
      "grad_norm": 0.3760841763032648,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -54866030.7027027,
      "logits/rejected": -64804301.36263736,
      "logps/chosen": -359.35135135135135,
      "logps/rejected": -538.021978021978,
      "loss": 0.1461,
      "rewards/chosen": 6.269512279613598,
      "rewards/margins": 23.610171620272936,
      "rewards/rejected": -17.34065934065934,
      "step": 1576
    },
    {
      "epoch": 0.776847290640394,
      "grad_norm": 0.4231947542615105,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -50514009.04347826,
      "logits/rejected": -43682141.65853658,
      "logps/chosen": -340.17391304347825,
      "logps/rejected": -552.9756097560976,
      "loss": 0.1379,
      "rewards/chosen": 1.6548913043478262,
      "rewards/margins": 19.97196447507953,
      "rewards/rejected": -18.317073170731707,
      "step": 1577
    },
    {
      "epoch": 0.7773399014778325,
      "grad_norm": 0.5215387943155237,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -46468473.2631579,
      "logits/rejected": -54762250.81690141,
      "logps/chosen": -290.2456140350877,
      "logps/rejected": -506.59154929577466,
      "loss": 0.2081,
      "rewards/chosen": -0.581140350877193,
      "rewards/margins": 11.221676550531257,
      "rewards/rejected": -11.80281690140845,
      "step": 1578
    },
    {
      "epoch": 0.777832512315271,
      "grad_norm": 0.4729946343340475,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -49654442.666666664,
      "logits/rejected": -62023270.4,
      "logps/chosen": -299.3333333333333,
      "logps/rejected": -532.0,
      "loss": 0.1613,
      "rewards/chosen": 1.1119791666666667,
      "rewards/margins": 18.53697916666667,
      "rewards/rejected": -17.425,
      "step": 1579
    },
    {
      "epoch": 0.7783251231527094,
      "grad_norm": 0.4186039038314913,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -51296337.92,
      "logits/rejected": -61408912.41025641,
      "logps/chosen": -244.16,
      "logps/rejected": -612.1025641025641,
      "loss": 0.1612,
      "rewards/chosen": 2.0893768310546874,
      "rewards/margins": 22.204761446439303,
      "rewards/rejected": -20.115384615384617,
      "step": 1580
    },
    {
      "epoch": 0.7788177339901478,
      "grad_norm": 0.5341316599391712,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -57170187.13043478,
      "logits/rejected": -58822556.09756097,
      "logps/chosen": -220.69565217391303,
      "logps/rejected": -539.3170731707318,
      "loss": 0.1589,
      "rewards/chosen": 4.352587492569633,
      "rewards/margins": 20.42575822427695,
      "rewards/rejected": -16.073170731707318,
      "step": 1581
    },
    {
      "epoch": 0.7793103448275862,
      "grad_norm": 0.548477164720469,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -72526506.66666667,
      "logits/rejected": -56675532.8,
      "logps/chosen": -382.3333333333333,
      "logps/rejected": -569.6,
      "loss": 0.1783,
      "rewards/chosen": -0.4850260416666667,
      "rewards/margins": 18.314973958333333,
      "rewards/rejected": -18.8,
      "step": 1582
    },
    {
      "epoch": 0.7798029556650247,
      "grad_norm": 0.46596433864081205,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -33599052.25531915,
      "logits/rejected": -53542102.913580246,
      "logps/chosen": -250.38297872340425,
      "logps/rejected": -563.358024691358,
      "loss": 0.1909,
      "rewards/chosen": 4.696794225814495,
      "rewards/margins": 21.363460892481164,
      "rewards/rejected": -16.666666666666668,
      "step": 1583
    },
    {
      "epoch": 0.780295566502463,
      "grad_norm": 0.4126402561785756,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -55418357.10638298,
      "logits/rejected": -61412895.60493827,
      "logps/chosen": -324.7659574468085,
      "logps/rejected": -570.4691358024692,
      "loss": 0.1482,
      "rewards/chosen": -0.3723404255319149,
      "rewards/margins": 13.84871233295574,
      "rewards/rejected": -14.221052758487655,
      "step": 1584
    },
    {
      "epoch": 0.7807881773399015,
      "grad_norm": 0.5215797470087216,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -36617377.684210524,
      "logits/rejected": -60770804.62222222,
      "logps/chosen": -269.2631578947368,
      "logps/rejected": -614.4,
      "loss": 0.1488,
      "rewards/chosen": -0.8562911184210527,
      "rewards/margins": 17.36593110380117,
      "rewards/rejected": -18.22222222222222,
      "step": 1585
    },
    {
      "epoch": 0.7812807881773399,
      "grad_norm": 0.4017024102264542,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -55636208.941176474,
      "logits/rejected": -52830382.29787234,
      "logps/chosen": -327.29411764705884,
      "logps/rejected": -554.2127659574468,
      "loss": 0.1308,
      "rewards/chosen": 2.844705918255974,
      "rewards/margins": 20.823429322511295,
      "rewards/rejected": -17.97872340425532,
      "step": 1586
    },
    {
      "epoch": 0.7817733990147783,
      "grad_norm": 0.49496068661812137,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -55383877.81818182,
      "logits/rejected": -59469238.85714286,
      "logps/chosen": -261.8181818181818,
      "logps/rejected": -640.7619047619048,
      "loss": 0.1545,
      "rewards/chosen": 2.3828107660466973,
      "rewards/margins": 21.739953623189557,
      "rewards/rejected": -19.357142857142858,
      "step": 1587
    },
    {
      "epoch": 0.7822660098522167,
      "grad_norm": 0.5724688340041225,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -55272882.84931507,
      "logits/rejected": -61084318.25454546,
      "logps/chosen": -252.27397260273972,
      "logps/rejected": -525.3818181818182,
      "loss": 0.2221,
      "rewards/chosen": 1.9982876712328768,
      "rewards/margins": 18.998287671232877,
      "rewards/rejected": -17.0,
      "step": 1588
    },
    {
      "epoch": 0.7827586206896552,
      "grad_norm": 0.5184471626421919,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -57024686.29787234,
      "logits/rejected": -67160645.5308642,
      "logps/chosen": -320.3404255319149,
      "logps/rejected": -675.5555555555555,
      "loss": 0.1965,
      "rewards/chosen": 1.38531039623504,
      "rewards/margins": 20.866791877716523,
      "rewards/rejected": -19.48148148148148,
      "step": 1589
    },
    {
      "epoch": 0.7832512315270936,
      "grad_norm": 0.4816869174729474,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -54312578.97674418,
      "logits/rejected": -52206748.61176471,
      "logps/chosen": -315.90697674418607,
      "logps/rejected": -547.3882352941176,
      "loss": 0.2043,
      "rewards/chosen": -0.28200619719749276,
      "rewards/margins": 16.165052626331917,
      "rewards/rejected": -16.44705882352941,
      "step": 1590
    },
    {
      "epoch": 0.783743842364532,
      "grad_norm": 0.5052064456922017,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -55981118.69387755,
      "logits/rejected": -51924421.67088608,
      "logps/chosen": -344.16326530612247,
      "logps/rejected": -531.4430379746835,
      "loss": 0.171,
      "rewards/chosen": -1.1307397959183674,
      "rewards/margins": 14.5528045078791,
      "rewards/rejected": -15.683544303797468,
      "step": 1591
    },
    {
      "epoch": 0.7842364532019704,
      "grad_norm": 0.40964753862032,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -47928661.333333336,
      "logits/rejected": -67423436.8,
      "logps/chosen": -210.25,
      "logps/rejected": -520.0,
      "loss": 0.1797,
      "rewards/chosen": -0.5807291666666666,
      "rewards/margins": 19.36927083333333,
      "rewards/rejected": -19.95,
      "step": 1592
    },
    {
      "epoch": 0.7847290640394089,
      "grad_norm": 0.5380109606452669,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -34136974.222222224,
      "logits/rejected": -54020614.1686747,
      "logps/chosen": -233.95555555555555,
      "logps/rejected": -568.289156626506,
      "loss": 0.1699,
      "rewards/chosen": 0.88125,
      "rewards/margins": 19.218599397590364,
      "rewards/rejected": -18.337349397590362,
      "step": 1593
    },
    {
      "epoch": 0.7852216748768472,
      "grad_norm": 0.4975255982789124,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -54263808.0,
      "logits/rejected": -61341696.0,
      "logps/chosen": -242.75,
      "logps/rejected": -617.0,
      "loss": 0.1925,
      "rewards/chosen": 0.3662109375,
      "rewards/margins": 19.6943359375,
      "rewards/rejected": -19.328125,
      "step": 1594
    },
    {
      "epoch": 0.7857142857142857,
      "grad_norm": 0.4649254205886746,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -51475549.09090909,
      "logits/rejected": -55824188.952380955,
      "logps/chosen": -321.8181818181818,
      "logps/rejected": -552.3809523809524,
      "loss": 0.1812,
      "rewards/chosen": 3.67683237249201,
      "rewards/margins": 22.41492761058725,
      "rewards/rejected": -18.738095238095237,
      "step": 1595
    },
    {
      "epoch": 0.7862068965517242,
      "grad_norm": 0.4633551763717643,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -49045658.56603774,
      "logits/rejected": -53854863.36,
      "logps/chosen": -291.0188679245283,
      "logps/rejected": -599.04,
      "loss": 0.1623,
      "rewards/chosen": 1.3974056603773586,
      "rewards/margins": 21.690738993710692,
      "rewards/rejected": -20.293333333333333,
      "step": 1596
    },
    {
      "epoch": 0.7866995073891626,
      "grad_norm": 0.5570500189395763,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -46089681.45454545,
      "logits/rejected": -59369374.47619048,
      "logps/chosen": -291.45454545454544,
      "logps/rejected": -675.047619047619,
      "loss": 0.1734,
      "rewards/chosen": 2.723033905029297,
      "rewards/margins": 25.080176762172155,
      "rewards/rejected": -22.357142857142858,
      "step": 1597
    },
    {
      "epoch": 0.787192118226601,
      "grad_norm": 0.5544520118345799,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -48821698.56,
      "logits/rejected": -51299564.307692304,
      "logps/chosen": -300.32,
      "logps/rejected": -575.1794871794872,
      "loss": 0.1525,
      "rewards/chosen": 1.7140625,
      "rewards/margins": 21.637139423076924,
      "rewards/rejected": -19.923076923076923,
      "step": 1598
    },
    {
      "epoch": 0.7876847290640394,
      "grad_norm": 0.5080721150371861,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -54220117.333333336,
      "logits/rejected": -42886758.4,
      "logps/chosen": -414.5,
      "logps/rejected": -542.8,
      "loss": 0.1575,
      "rewards/chosen": 0.4329427083333333,
      "rewards/margins": 19.170442708333333,
      "rewards/rejected": -18.7375,
      "step": 1599
    },
    {
      "epoch": 0.7881773399014779,
      "grad_norm": 0.5385117226115996,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -43013015.510204084,
      "logits/rejected": -51712051.848101266,
      "logps/chosen": -324.2448979591837,
      "logps/rejected": -553.3164556962025,
      "loss": 0.2037,
      "rewards/chosen": 1.1368781887755102,
      "rewards/margins": 19.681181986243864,
      "rewards/rejected": -18.544303797468356,
      "step": 1600
    },
    {
      "epoch": 0.7886699507389162,
      "grad_norm": 0.5738442615630684,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -48097725.217391305,
      "logits/rejected": -54423651.90243903,
      "logps/chosen": -302.95652173913044,
      "logps/rejected": -611.1219512195122,
      "loss": 0.1991,
      "rewards/chosen": -2.2635869565217392,
      "rewards/margins": 18.51690084835631,
      "rewards/rejected": -20.78048780487805,
      "step": 1601
    },
    {
      "epoch": 0.7891625615763547,
      "grad_norm": 0.48551884666233697,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -52755023.64444444,
      "logits/rejected": -68170073.44578314,
      "logps/chosen": -329.24444444444447,
      "logps/rejected": -645.3975903614457,
      "loss": 0.1494,
      "rewards/chosen": 0.11180555555555556,
      "rewards/margins": 19.943130856760376,
      "rewards/rejected": -19.83132530120482,
      "step": 1602
    },
    {
      "epoch": 0.7896551724137931,
      "grad_norm": 0.46231419991875267,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -53229530.763636366,
      "logits/rejected": -64695702.79452055,
      "logps/chosen": -329.0181818181818,
      "logps/rejected": -656.6575342465753,
      "loss": 0.1961,
      "rewards/chosen": -0.6090909090909091,
      "rewards/margins": 10.887345641101728,
      "rewards/rejected": -11.496436550192637,
      "step": 1603
    },
    {
      "epoch": 0.7901477832512315,
      "grad_norm": 0.5642271121981629,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -51985171.692307696,
      "logits/rejected": -44040192.0,
      "logps/chosen": -280.3076923076923,
      "logps/rejected": -560.8421052631579,
      "loss": 0.1961,
      "rewards/chosen": 1.2313431959885817,
      "rewards/margins": 20.178711617041213,
      "rewards/rejected": -18.94736842105263,
      "step": 1604
    },
    {
      "epoch": 0.7906403940886699,
      "grad_norm": 0.3680265217648469,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -54035129.19148936,
      "logits/rejected": -45464184.09876543,
      "logps/chosen": -273.70212765957444,
      "logps/rejected": -526.2222222222222,
      "loss": 0.1855,
      "rewards/chosen": -2.0904255319148937,
      "rewards/margins": 17.44043866561597,
      "rewards/rejected": -19.530864197530864,
      "step": 1605
    },
    {
      "epoch": 0.7911330049261084,
      "grad_norm": 0.7332546416870258,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -52276279.85454545,
      "logits/rejected": -56536919.67123288,
      "logps/chosen": -499.4909090909091,
      "logps/rejected": -589.1506849315068,
      "loss": 0.1914,
      "rewards/chosen": 1.835776034268466,
      "rewards/margins": 21.34262534933696,
      "rewards/rejected": -19.506849315068493,
      "step": 1606
    },
    {
      "epoch": 0.7916256157635468,
      "grad_norm": 0.4048219711634421,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -44040192.0,
      "logits/rejected": -65925855.179487176,
      "logps/chosen": -326.72,
      "logps/rejected": -554.6666666666666,
      "loss": 0.2096,
      "rewards/chosen": -1.96806640625,
      "rewards/margins": 17.416548978365384,
      "rewards/rejected": -19.384615384615383,
      "step": 1607
    },
    {
      "epoch": 0.7921182266009852,
      "grad_norm": 0.43583991180017856,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -58631015.4893617,
      "logits/rejected": -54059918.222222224,
      "logps/chosen": -373.1063829787234,
      "logps/rejected": -555.4567901234568,
      "loss": 0.161,
      "rewards/chosen": 7.019358533494016,
      "rewards/margins": 26.056395570531052,
      "rewards/rejected": -19.037037037037038,
      "step": 1608
    },
    {
      "epoch": 0.7926108374384236,
      "grad_norm": 0.48709977817766,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -49561265.63265306,
      "logits/rejected": -60206844.75949367,
      "logps/chosen": -332.0816326530612,
      "logps/rejected": -623.7974683544304,
      "loss": 0.1883,
      "rewards/chosen": 4.066356425382653,
      "rewards/margins": 24.01572351399025,
      "rewards/rejected": -19.949367088607595,
      "step": 1609
    },
    {
      "epoch": 0.7931034482758621,
      "grad_norm": 0.46797786549886594,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -49619407.69811321,
      "logits/rejected": -56874762.24,
      "logps/chosen": -316.6792452830189,
      "logps/rejected": -688.64,
      "loss": 0.1977,
      "rewards/chosen": 3.1645124903265036,
      "rewards/margins": 23.53784582365984,
      "rewards/rejected": -20.373333333333335,
      "step": 1610
    },
    {
      "epoch": 0.7935960591133004,
      "grad_norm": 0.49474912300264845,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -48440099.1372549,
      "logits/rejected": -67816993.24675325,
      "logps/chosen": -326.7450980392157,
      "logps/rejected": -494.54545454545456,
      "loss": 0.1921,
      "rewards/chosen": -0.016544117647058824,
      "rewards/margins": 16.97046886936593,
      "rewards/rejected": -16.98701298701299,
      "step": 1611
    },
    {
      "epoch": 0.7940886699507389,
      "grad_norm": 0.49469878727155875,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -58958568.72727273,
      "logits/rejected": -50131919.23809524,
      "logps/chosen": -260.0,
      "logps/rejected": -531.047619047619,
      "loss": 0.1675,
      "rewards/chosen": 5.949582186612216,
      "rewards/margins": 24.568629805659835,
      "rewards/rejected": -18.61904761904762,
      "step": 1612
    },
    {
      "epoch": 0.7945812807881774,
      "grad_norm": 0.5185648024566005,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -58283349.333333336,
      "logits/rejected": -51589939.2,
      "logps/chosen": -275.0,
      "logps/rejected": -562.4,
      "loss": 0.2065,
      "rewards/chosen": 1.86656920115153,
      "rewards/margins": 17.916569201151532,
      "rewards/rejected": -16.05,
      "step": 1613
    },
    {
      "epoch": 0.7950738916256158,
      "grad_norm": 0.5424337897580888,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -50580462.644067794,
      "logits/rejected": -58720256.0,
      "logps/chosen": -288.54237288135596,
      "logps/rejected": -594.5507246376811,
      "loss": 0.2013,
      "rewards/chosen": 1.0737552966101696,
      "rewards/margins": 19.566508919798576,
      "rewards/rejected": -18.492753623188406,
      "step": 1614
    },
    {
      "epoch": 0.7955665024630542,
      "grad_norm": 0.5472999823186717,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -68701146.07407407,
      "logits/rejected": -56963182.7027027,
      "logps/chosen": -345.48148148148147,
      "logps/rejected": -494.27027027027026,
      "loss": 0.1987,
      "rewards/chosen": 0.7060185185185185,
      "rewards/margins": 15.462775275275275,
      "rewards/rejected": -14.756756756756756,
      "step": 1615
    },
    {
      "epoch": 0.7960591133004926,
      "grad_norm": 0.4762730827289831,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -54220117.333333336,
      "logits/rejected": -52271513.6,
      "logps/chosen": -249.5,
      "logps/rejected": -561.6,
      "loss": 0.1631,
      "rewards/chosen": 1.2395833333333333,
      "rewards/margins": 20.664583333333333,
      "rewards/rejected": -19.425,
      "step": 1616
    },
    {
      "epoch": 0.7965517241379311,
      "grad_norm": 0.4876287234977255,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -66554899.320754714,
      "logits/rejected": -59950585.17333333,
      "logps/chosen": -361.9622641509434,
      "logps/rejected": -528.64,
      "loss": 0.2245,
      "rewards/chosen": 0.046875,
      "rewards/margins": 15.060208333333334,
      "rewards/rejected": -15.013333333333334,
      "step": 1617
    },
    {
      "epoch": 0.7970443349753694,
      "grad_norm": 0.5117333500294224,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -42542226.28571428,
      "logits/rejected": -52916509.76744186,
      "logps/chosen": -298.85714285714283,
      "logps/rejected": -484.4651162790698,
      "loss": 0.1716,
      "rewards/chosen": 0.12165178571428571,
      "rewards/margins": 17.8890936461794,
      "rewards/rejected": -17.767441860465116,
      "step": 1618
    },
    {
      "epoch": 0.7975369458128079,
      "grad_norm": 0.5875809380441211,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -55542752.96969697,
      "logits/rejected": -53342075.87096774,
      "logps/chosen": -341.09090909090907,
      "logps/rejected": -513.5483870967741,
      "loss": 0.2385,
      "rewards/chosen": 0.3697916666666667,
      "rewards/margins": 11.723864483576948,
      "rewards/rejected": -11.354072816910282,
      "step": 1619
    },
    {
      "epoch": 0.7980295566502463,
      "grad_norm": 0.47270082024159316,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -60037072.37209302,
      "logits/rejected": -58276153.22352941,
      "logps/chosen": -314.04651162790697,
      "logps/rejected": -588.8,
      "loss": 0.1625,
      "rewards/chosen": 1.9070257142532703,
      "rewards/margins": 21.130555126017974,
      "rewards/rejected": -19.223529411764705,
      "step": 1620
    },
    {
      "epoch": 0.7985221674876847,
      "grad_norm": 0.4425011729164274,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -37573973.333333336,
      "logits/rejected": -51013222.4,
      "logps/chosen": -279.0,
      "logps/rejected": -553.6,
      "loss": 0.1732,
      "rewards/chosen": -0.4540201822916667,
      "rewards/margins": 18.18347981770833,
      "rewards/rejected": -18.6375,
      "step": 1621
    },
    {
      "epoch": 0.7990147783251231,
      "grad_norm": 0.5455681722805954,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -60238883.31034483,
      "logits/rejected": -58840093.25714286,
      "logps/chosen": -292.6896551724138,
      "logps/rejected": -580.1142857142858,
      "loss": 0.2125,
      "rewards/chosen": -0.4396551724137931,
      "rewards/margins": 17.217487684729065,
      "rewards/rejected": -17.65714285714286,
      "step": 1622
    },
    {
      "epoch": 0.7995073891625616,
      "grad_norm": 0.465229547255741,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -61382025.84615385,
      "logits/rejected": -57064609.684210524,
      "logps/chosen": -341.84615384615387,
      "logps/rejected": -653.4736842105264,
      "loss": 0.2068,
      "rewards/chosen": 2.9372699444110575,
      "rewards/margins": 24.305690997042635,
      "rewards/rejected": -21.36842105263158,
      "step": 1623
    },
    {
      "epoch": 0.8,
      "grad_norm": 0.5173562629441438,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -75132749.91304348,
      "logits/rejected": -56981154.34146342,
      "logps/chosen": -327.30434782608694,
      "logps/rejected": -634.5365853658536,
      "loss": 0.1653,
      "rewards/chosen": 4.592385996942935,
      "rewards/margins": 23.494825021333178,
      "rewards/rejected": -18.902439024390244,
      "step": 1624
    },
    {
      "epoch": 0.8004926108374384,
      "grad_norm": 0.4713149055784955,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -77444827.42857143,
      "logits/rejected": -55208745.674418606,
      "logps/chosen": -307.04761904761904,
      "logps/rejected": -671.2558139534884,
      "loss": 0.19,
      "rewards/chosen": 2.5148864019484747,
      "rewards/margins": 23.654421285669404,
      "rewards/rejected": -21.13953488372093,
      "step": 1625
    },
    {
      "epoch": 0.8009852216748768,
      "grad_norm": 0.48755063557966505,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -46505265.403508775,
      "logits/rejected": -50390722.704225354,
      "logps/chosen": -264.70175438596493,
      "logps/rejected": -580.5070422535211,
      "loss": 0.1896,
      "rewards/chosen": 0.02905701754385965,
      "rewards/margins": 14.770180270734881,
      "rewards/rejected": -14.741123253191022,
      "step": 1626
    },
    {
      "epoch": 0.8014778325123153,
      "grad_norm": 0.6004144253456709,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -46022953.89090909,
      "logits/rejected": -54353583.34246575,
      "logps/chosen": -299.3454545454546,
      "logps/rejected": -508.4931506849315,
      "loss": 0.2021,
      "rewards/chosen": 1.6926166881214488,
      "rewards/margins": 18.185767373052954,
      "rewards/rejected": -16.493150684931507,
      "step": 1627
    },
    {
      "epoch": 0.8019704433497536,
      "grad_norm": 0.43687190411924326,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -47431331.404255316,
      "logits/rejected": -57684625.38271605,
      "logps/chosen": -253.4468085106383,
      "logps/rejected": -566.5185185185185,
      "loss": 0.1689,
      "rewards/chosen": 0.8610372340425532,
      "rewards/margins": 16.66350636984502,
      "rewards/rejected": -15.802469135802468,
      "step": 1628
    },
    {
      "epoch": 0.8024630541871921,
      "grad_norm": 0.6370363457415312,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -38242183.52941176,
      "logits/rejected": -45374743.27272727,
      "logps/chosen": -285.5686274509804,
      "logps/rejected": -520.3116883116883,
      "loss": 0.1797,
      "rewards/chosen": 0.5686274509803921,
      "rewards/margins": 15.399796282149223,
      "rewards/rejected": -14.831168831168831,
      "step": 1629
    },
    {
      "epoch": 0.8029556650246306,
      "grad_norm": 0.5018950142692682,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -56851884.21818182,
      "logits/rejected": -72969398.35616438,
      "logps/chosen": -258.90909090909093,
      "logps/rejected": -620.7123287671233,
      "loss": 0.1871,
      "rewards/chosen": -0.865909090909091,
      "rewards/margins": 20.284775840597757,
      "rewards/rejected": -21.15068493150685,
      "step": 1630
    },
    {
      "epoch": 0.803448275862069,
      "grad_norm": 0.5268451593823502,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -58050952.17021277,
      "logits/rejected": -56493650.17283951,
      "logps/chosen": -370.21276595744683,
      "logps/rejected": -617.8765432098766,
      "loss": 0.1509,
      "rewards/chosen": 2.0944148936170213,
      "rewards/margins": 23.254908720777514,
      "rewards/rejected": -21.160493827160494,
      "step": 1631
    },
    {
      "epoch": 0.8039408866995074,
      "grad_norm": 0.48636153001091875,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -39235807.41818182,
      "logits/rejected": -57283850.52054795,
      "logps/chosen": -330.4727272727273,
      "logps/rejected": -605.8082191780821,
      "loss": 0.1829,
      "rewards/chosen": -0.8170454545454545,
      "rewards/margins": 20.90898194271482,
      "rewards/rejected": -21.726027397260275,
      "step": 1632
    },
    {
      "epoch": 0.8044334975369458,
      "grad_norm": 0.5126699348988752,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -58274053.44680851,
      "logits/rejected": -56907902.41975309,
      "logps/chosen": -279.48936170212767,
      "logps/rejected": -681.0864197530864,
      "loss": 0.1698,
      "rewards/chosen": -0.7632978723404256,
      "rewards/margins": 22.570035460992905,
      "rewards/rejected": -23.333333333333332,
      "step": 1633
    },
    {
      "epoch": 0.8049261083743843,
      "grad_norm": 0.4705456559824377,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -44414683.428571425,
      "logits/rejected": -54234680.88888889,
      "logps/chosen": -284.57142857142856,
      "logps/rejected": -579.5555555555555,
      "loss": 0.1731,
      "rewards/chosen": 0.26674107142857145,
      "rewards/margins": 21.877852182539684,
      "rewards/rejected": -21.61111111111111,
      "step": 1634
    },
    {
      "epoch": 0.8054187192118226,
      "grad_norm": 0.5066280891728557,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -50716839.18367347,
      "logits/rejected": -58560978.63291139,
      "logps/chosen": -231.0204081632653,
      "logps/rejected": -624.6075949367089,
      "loss": 0.1935,
      "rewards/chosen": 1.1020348996532208,
      "rewards/margins": 22.874186798387395,
      "rewards/rejected": -21.772151898734176,
      "step": 1635
    },
    {
      "epoch": 0.8059113300492611,
      "grad_norm": 0.44076154040561716,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -62263719.72413793,
      "logits/rejected": -56443348.114285715,
      "logps/chosen": -250.75862068965517,
      "logps/rejected": -551.3142857142857,
      "loss": 0.1931,
      "rewards/chosen": -0.6651400862068966,
      "rewards/margins": 14.915164740802032,
      "rewards/rejected": -15.580304827008929,
      "step": 1636
    },
    {
      "epoch": 0.8064039408866995,
      "grad_norm": 0.5151042890495043,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -57135741.155555554,
      "logits/rejected": -64278972.144578315,
      "logps/chosen": -327.8222222222222,
      "logps/rejected": -719.4216867469879,
      "loss": 0.1698,
      "rewards/chosen": 3.9639105902777776,
      "rewards/margins": 27.819332277024767,
      "rewards/rejected": -23.85542168674699,
      "step": 1637
    },
    {
      "epoch": 0.8068965517241379,
      "grad_norm": 0.5430823076035921,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -51530020.571428575,
      "logits/rejected": -56118725.67088608,
      "logps/chosen": -222.69387755102042,
      "logps/rejected": -599.493670886076,
      "loss": 0.1854,
      "rewards/chosen": 0.3239795918367347,
      "rewards/margins": 22.298663136140533,
      "rewards/rejected": -21.974683544303797,
      "step": 1638
    },
    {
      "epoch": 0.8073891625615763,
      "grad_norm": 0.4627902714678917,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -61322277.925925925,
      "logits/rejected": -57303261.4054054,
      "logps/chosen": -262.0740740740741,
      "logps/rejected": -634.8108108108108,
      "loss": 0.2132,
      "rewards/chosen": 0.3605702718098958,
      "rewards/margins": 22.549759460999084,
      "rewards/rejected": -22.18918918918919,
      "step": 1639
    },
    {
      "epoch": 0.8078817733990148,
      "grad_norm": 0.49783536152295105,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -62762039.85454545,
      "logits/rejected": -66189564.49315068,
      "logps/chosen": -364.8,
      "logps/rejected": -621.5890410958904,
      "loss": 0.2184,
      "rewards/chosen": -3.0545454545454547,
      "rewards/margins": 12.493399750933998,
      "rewards/rejected": -15.547945205479452,
      "step": 1640
    },
    {
      "epoch": 0.8083743842364532,
      "grad_norm": 0.5160512984663744,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -49974685.957446806,
      "logits/rejected": -50020958.81481481,
      "logps/chosen": -286.63829787234044,
      "logps/rejected": -626.5679012345679,
      "loss": 0.1867,
      "rewards/chosen": 3.083144816946476,
      "rewards/margins": 17.942650929504346,
      "rewards/rejected": -14.85950611255787,
      "step": 1641
    },
    {
      "epoch": 0.8088669950738916,
      "grad_norm": 0.48949625550564174,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -51894231.84313726,
      "logits/rejected": -59101556.36363637,
      "logps/chosen": -340.078431372549,
      "logps/rejected": -556.8831168831168,
      "loss": 0.1789,
      "rewards/chosen": 2.9007460650275734,
      "rewards/margins": 23.10853827281978,
      "rewards/rejected": -20.207792207792206,
      "step": 1642
    },
    {
      "epoch": 0.80935960591133,
      "grad_norm": 0.5870813806781386,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -66709406.47619048,
      "logits/rejected": -62231766.325581394,
      "logps/chosen": -425.5238095238095,
      "logps/rejected": -630.3255813953489,
      "loss": 0.1767,
      "rewards/chosen": -2.1785714285714284,
      "rewards/margins": 18.565614617940202,
      "rewards/rejected": -20.74418604651163,
      "step": 1643
    },
    {
      "epoch": 0.8098522167487685,
      "grad_norm": 0.4848393996678487,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -53901268.42553192,
      "logits/rejected": -53801010.56790123,
      "logps/chosen": -322.72340425531917,
      "logps/rejected": -592.5925925925926,
      "loss": 0.1841,
      "rewards/chosen": -0.9119015957446809,
      "rewards/margins": 20.841184824008405,
      "rewards/rejected": -21.753086419753085,
      "step": 1644
    },
    {
      "epoch": 0.8103448275862069,
      "grad_norm": 0.3848232248207419,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -30641720.888888888,
      "logits/rejected": -53212073.638554215,
      "logps/chosen": -312.8888888888889,
      "logps/rejected": -579.0843373493976,
      "loss": 0.1329,
      "rewards/chosen": 4.802108086480034,
      "rewards/margins": 25.284035797323405,
      "rewards/rejected": -20.481927710843372,
      "step": 1645
    },
    {
      "epoch": 0.8108374384236453,
      "grad_norm": 0.40790607260665873,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -69153587.2,
      "logits/rejected": -60054807.27272727,
      "logps/chosen": -233.0,
      "logps/rejected": -576.0,
      "loss": 0.1541,
      "rewards/chosen": -0.038671875,
      "rewards/margins": 19.37041903409091,
      "rewards/rejected": -19.40909090909091,
      "step": 1646
    },
    {
      "epoch": 0.8113300492610838,
      "grad_norm": 0.5271876238854319,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -48387016.14545455,
      "logits/rejected": -56364551.01369863,
      "logps/chosen": -295.8545454545455,
      "logps/rejected": -588.2739726027397,
      "loss": 0.1642,
      "rewards/chosen": 2.257103937322443,
      "rewards/margins": 22.147514896226554,
      "rewards/rejected": -19.89041095890411,
      "step": 1647
    },
    {
      "epoch": 0.8118226600985222,
      "grad_norm": 0.47733092117383535,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -46084915.2,
      "logits/rejected": -59578181.81818182,
      "logps/chosen": -289.2,
      "logps/rejected": -630.5454545454545,
      "loss": 0.1416,
      "rewards/chosen": 1.475,
      "rewards/margins": 21.63409090909091,
      "rewards/rejected": -20.15909090909091,
      "step": 1648
    },
    {
      "epoch": 0.8123152709359606,
      "grad_norm": 0.5240764113879074,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -51729749.333333336,
      "logits/rejected": -60874087.783783786,
      "logps/chosen": -341.3333333333333,
      "logps/rejected": -614.918918918919,
      "loss": 0.1785,
      "rewards/chosen": 1.8026883160626446,
      "rewards/margins": 21.39728291065724,
      "rewards/rejected": -19.594594594594593,
      "step": 1649
    },
    {
      "epoch": 0.812807881773399,
      "grad_norm": 0.4533521270904401,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -40894464.0,
      "logits/rejected": -54095766.974358976,
      "logps/chosen": -312.96,
      "logps/rejected": -558.7692307692307,
      "loss": 0.1416,
      "rewards/chosen": 3.585035400390625,
      "rewards/margins": 21.41836873372396,
      "rewards/rejected": -17.833333333333332,
      "step": 1650
    },
    {
      "epoch": 0.8133004926108375,
      "grad_norm": 0.5034756728337693,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -51030698.666666664,
      "logits/rejected": -55364812.8,
      "logps/chosen": -377.6666666666667,
      "logps/rejected": -606.4,
      "loss": 0.1412,
      "rewards/chosen": 1.0143229166666667,
      "rewards/margins": 20.98932291666667,
      "rewards/rejected": -19.975,
      "step": 1651
    },
    {
      "epoch": 0.8137931034482758,
      "grad_norm": 0.48135243241962233,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -58632874.666666664,
      "logits/rejected": -56675532.8,
      "logps/chosen": -236.83333333333334,
      "logps/rejected": -576.0,
      "loss": 0.1903,
      "rewards/chosen": 2.9003480275472007,
      "rewards/margins": 24.0253480275472,
      "rewards/rejected": -21.125,
      "step": 1652
    },
    {
      "epoch": 0.8142857142857143,
      "grad_norm": 0.44046673594333036,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -56910948.39215686,
      "logits/rejected": -47226773.61038961,
      "logps/chosen": -309.6470588235294,
      "logps/rejected": -568.5194805194806,
      "loss": 0.1486,
      "rewards/chosen": 0.5796568627450981,
      "rewards/margins": 21.488747771836007,
      "rewards/rejected": -20.90909090909091,
      "step": 1653
    },
    {
      "epoch": 0.8147783251231527,
      "grad_norm": 0.4507516956955133,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -55939250.08695652,
      "logits/rejected": -53042600.585365854,
      "logps/chosen": -303.30434782608694,
      "logps/rejected": -572.8780487804878,
      "loss": 0.1661,
      "rewards/chosen": 2.0774383544921875,
      "rewards/margins": 23.02865786668731,
      "rewards/rejected": -20.951219512195124,
      "step": 1654
    },
    {
      "epoch": 0.8152709359605911,
      "grad_norm": 0.6349320894997948,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -58062325.960784316,
      "logits/rejected": -44775556.98701299,
      "logps/chosen": -380.54901960784315,
      "logps/rejected": -550.2337662337662,
      "loss": 0.1988,
      "rewards/chosen": 0.14460784313725492,
      "rewards/margins": 18.014737713267124,
      "rewards/rejected": -17.87012987012987,
      "step": 1655
    },
    {
      "epoch": 0.8157635467980295,
      "grad_norm": 0.6726274428774147,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -45950098.28571428,
      "logits/rejected": -44098446.222222224,
      "logps/chosen": -271.14285714285717,
      "logps/rejected": -569.7777777777778,
      "loss": 0.1789,
      "rewards/chosen": 3.112187249319894,
      "rewards/margins": 22.167742804875452,
      "rewards/rejected": -19.055555555555557,
      "step": 1656
    },
    {
      "epoch": 0.816256157635468,
      "grad_norm": 0.44325651618900225,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -50194877.217391305,
      "logits/rejected": -61482358.63414634,
      "logps/chosen": -333.5652173913044,
      "logps/rejected": -589.2682926829268,
      "loss": 0.1693,
      "rewards/chosen": 3.132829749065897,
      "rewards/margins": 23.88892731004151,
      "rewards/rejected": -20.75609756097561,
      "step": 1657
    },
    {
      "epoch": 0.8167487684729065,
      "grad_norm": 0.4640872659726586,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -37721141.89473684,
      "logits/rejected": -53500677.68888889,
      "logps/chosen": -213.78947368421052,
      "logps/rejected": -572.4444444444445,
      "loss": 0.1377,
      "rewards/chosen": 3.6134651585629114,
      "rewards/margins": 21.369020714118466,
      "rewards/rejected": -17.755555555555556,
      "step": 1658
    },
    {
      "epoch": 0.8172413793103448,
      "grad_norm": 0.5315821258207298,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -55819939.404255316,
      "logits/rejected": -50797681.777777776,
      "logps/chosen": -335.6595744680851,
      "logps/rejected": -568.0987654320987,
      "loss": 0.165,
      "rewards/chosen": 1.2300531914893618,
      "rewards/margins": 17.341164302600472,
      "rewards/rejected": -16.11111111111111,
      "step": 1659
    },
    {
      "epoch": 0.8177339901477833,
      "grad_norm": 0.5336745976740332,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -49562692.266666666,
      "logits/rejected": -55636208.941176474,
      "logps/chosen": -236.53333333333333,
      "logps/rejected": -544.9411764705883,
      "loss": 0.162,
      "rewards/chosen": 1.6322916666666667,
      "rewards/margins": 19.823468137254903,
      "rewards/rejected": -18.191176470588236,
      "step": 1660
    },
    {
      "epoch": 0.8182266009852217,
      "grad_norm": 0.41009098536352434,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -49221391.058823526,
      "logits/rejected": -65529191.064935066,
      "logps/chosen": -332.54901960784315,
      "logps/rejected": -638.3376623376623,
      "loss": 0.1568,
      "rewards/chosen": 0.7221200980392157,
      "rewards/margins": 19.709133085052205,
      "rewards/rejected": -18.98701298701299,
      "step": 1661
    },
    {
      "epoch": 0.81871921182266,
      "grad_norm": 0.4761778906870931,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -51292842.666666664,
      "logits/rejected": -56151244.8,
      "logps/chosen": -267.3333333333333,
      "logps/rejected": -533.6,
      "loss": 0.1466,
      "rewards/chosen": 2.382201830546061,
      "rewards/margins": 20.03220183054606,
      "rewards/rejected": -17.65,
      "step": 1662
    },
    {
      "epoch": 0.8192118226600985,
      "grad_norm": 0.5918520050709198,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -45600268.487804875,
      "logits/rejected": -58286362.48275862,
      "logps/chosen": -373.0731707317073,
      "logps/rejected": -625.2873563218391,
      "loss": 0.1598,
      "rewards/chosen": 2.700448199016292,
      "rewards/margins": 19.964816015108248,
      "rewards/rejected": -17.264367816091955,
      "step": 1663
    },
    {
      "epoch": 0.819704433497537,
      "grad_norm": 0.4688818191599762,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -39663526.95652174,
      "logits/rejected": -70280167.02439025,
      "logps/chosen": -280.0,
      "logps/rejected": -560.390243902439,
      "loss": 0.1426,
      "rewards/chosen": 2.4334554257600205,
      "rewards/margins": 19.555406645272214,
      "rewards/rejected": -17.121951219512194,
      "step": 1664
    },
    {
      "epoch": 0.8201970443349754,
      "grad_norm": 0.4104608805717455,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -49233139.809523806,
      "logits/rejected": -60378469.20930233,
      "logps/chosen": -351.23809523809524,
      "logps/rejected": -614.6976744186046,
      "loss": 0.1339,
      "rewards/chosen": -0.2544642857142857,
      "rewards/margins": 19.350186877076414,
      "rewards/rejected": -19.6046511627907,
      "step": 1665
    },
    {
      "epoch": 0.8206896551724138,
      "grad_norm": 0.5910593176466991,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -47832913.70212766,
      "logits/rejected": -59238071.30864198,
      "logps/chosen": -297.1914893617021,
      "logps/rejected": -568.8888888888889,
      "loss": 0.17,
      "rewards/chosen": 0.6808510638297872,
      "rewards/margins": 14.565053055535033,
      "rewards/rejected": -13.884201991705247,
      "step": 1666
    },
    {
      "epoch": 0.8211822660098522,
      "grad_norm": 0.4743688988789372,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -72668755.34883721,
      "logits/rejected": -61088804.14117647,
      "logps/chosen": -419.72093023255815,
      "logps/rejected": -606.1176470588235,
      "loss": 0.142,
      "rewards/chosen": 1.629360465116279,
      "rewards/margins": 20.217595759233927,
      "rewards/rejected": -18.58823529411765,
      "step": 1667
    },
    {
      "epoch": 0.8216748768472907,
      "grad_norm": 0.4789531801910718,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -62641018.43478261,
      "logits/rejected": -51712699.317073174,
      "logps/chosen": -369.39130434782606,
      "logps/rejected": -461.2682926829268,
      "loss": 0.202,
      "rewards/chosen": -2.7948369565217392,
      "rewards/margins": 11.827114262990456,
      "rewards/rejected": -14.621951219512194,
      "step": 1668
    },
    {
      "epoch": 0.822167487684729,
      "grad_norm": 0.5261824580054156,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -70822570.66666667,
      "logits/rejected": -54421094.4,
      "logps/chosen": -303.8333333333333,
      "logps/rejected": -501.6,
      "loss": 0.1756,
      "rewards/chosen": 4.330112139383952,
      "rewards/margins": 19.35511213938395,
      "rewards/rejected": -15.025,
      "step": 1669
    },
    {
      "epoch": 0.8226600985221675,
      "grad_norm": 0.534994091383721,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -43644502.943396226,
      "logits/rejected": -56091825.49333333,
      "logps/chosen": -238.33962264150944,
      "logps/rejected": -531.2,
      "loss": 0.175,
      "rewards/chosen": 1.5,
      "rewards/margins": 18.713333333333335,
      "rewards/rejected": -17.213333333333335,
      "step": 1670
    },
    {
      "epoch": 0.8231527093596059,
      "grad_norm": 0.49694668244333734,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -46137344.0,
      "logits/rejected": -52925493.89473684,
      "logps/chosen": -220.76923076923077,
      "logps/rejected": -495.1578947368421,
      "loss": 0.1669,
      "rewards/chosen": 0.7932692307692307,
      "rewards/margins": 16.398532388663966,
      "rewards/rejected": -15.605263157894736,
      "step": 1671
    },
    {
      "epoch": 0.8236453201970443,
      "grad_norm": 0.4013236007388051,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -63882476.307692304,
      "logits/rejected": -60872596.21052632,
      "logps/chosen": -280.3076923076923,
      "logps/rejected": -567.1578947368421,
      "loss": 0.1848,
      "rewards/chosen": -1.2139423076923077,
      "rewards/margins": 14.549215587044534,
      "rewards/rejected": -15.763157894736842,
      "step": 1672
    },
    {
      "epoch": 0.8241379310344827,
      "grad_norm": 0.46879298577663076,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -50968283.428571425,
      "logits/rejected": -50448156.44444445,
      "logps/chosen": -301.7142857142857,
      "logps/rejected": -544.8888888888889,
      "loss": 0.1938,
      "rewards/chosen": 2.8839563642229353,
      "rewards/margins": 17.745067475334046,
      "rewards/rejected": -14.86111111111111,
      "step": 1673
    },
    {
      "epoch": 0.8246305418719212,
      "grad_norm": 0.48019239531302244,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -43556233.84615385,
      "logits/rejected": -60265525.89473684,
      "logps/chosen": -292.61538461538464,
      "logps/rejected": -653.4736842105264,
      "loss": 0.2247,
      "rewards/chosen": 0.07513281015249398,
      "rewards/margins": 17.838290704889335,
      "rewards/rejected": -17.763157894736842,
      "step": 1674
    },
    {
      "epoch": 0.8251231527093597,
      "grad_norm": 0.3763844368995507,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -51841597.44,
      "logits/rejected": -58021205.333333336,
      "logps/chosen": -305.44,
      "logps/rejected": -507.0769230769231,
      "loss": 0.1703,
      "rewards/chosen": -0.916875,
      "rewards/margins": 16.980560897435897,
      "rewards/rejected": -17.897435897435898,
      "step": 1675
    },
    {
      "epoch": 0.825615763546798,
      "grad_norm": 0.505543243020559,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -45744128.0,
      "logits/rejected": -66107950.54545455,
      "logps/chosen": -258.0,
      "logps/rejected": -560.7272727272727,
      "loss": 0.1885,
      "rewards/chosen": 3.751575469970703,
      "rewards/margins": 20.774302742697976,
      "rewards/rejected": -17.022727272727273,
      "step": 1676
    },
    {
      "epoch": 0.8261083743842365,
      "grad_norm": 0.45894268613108746,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -49439242.89361702,
      "logits/rejected": -57166810.074074075,
      "logps/chosen": -254.29787234042553,
      "logps/rejected": -563.358024691358,
      "loss": 0.1635,
      "rewards/chosen": 2.8470835584275265,
      "rewards/margins": 19.66189837324234,
      "rewards/rejected": -16.814814814814813,
      "step": 1677
    },
    {
      "epoch": 0.8266009852216749,
      "grad_norm": 0.5059225458531121,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -50550101.333333336,
      "logits/rejected": -56780390.4,
      "logps/chosen": -321.0,
      "logps/rejected": -596.8,
      "loss": 0.1549,
      "rewards/chosen": 1.0413411458333333,
      "rewards/margins": 16.641341145833334,
      "rewards/rejected": -15.6,
      "step": 1678
    },
    {
      "epoch": 0.8270935960591133,
      "grad_norm": 0.5706094075114807,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -48758784.0,
      "logits/rejected": -58487239.11111111,
      "logps/chosen": -221.14285714285714,
      "logps/rejected": -563.5555555555555,
      "loss": 0.1707,
      "rewards/chosen": 0.7719029017857143,
      "rewards/margins": 17.549680679563494,
      "rewards/rejected": -16.77777777777778,
      "step": 1679
    },
    {
      "epoch": 0.8275862068965517,
      "grad_norm": 0.5566573798514811,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -48467512.88888889,
      "logits/rejected": -70509651.02702703,
      "logps/chosen": -280.0,
      "logps/rejected": -555.6756756756756,
      "loss": 0.1946,
      "rewards/chosen": 4.16844403302228,
      "rewards/margins": 20.465741330319577,
      "rewards/rejected": -16.2972972972973,
      "step": 1680
    },
    {
      "epoch": 0.8280788177339902,
      "grad_norm": 0.6318898959572534,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -47011157.333333336,
      "logits/rejected": -57416620.972972974,
      "logps/chosen": -259.4074074074074,
      "logps/rejected": -556.972972972973,
      "loss": 0.1675,
      "rewards/chosen": 4.640057033962673,
      "rewards/margins": 2433751.667084061,
      "rewards/rejected": -2433747.027027027,
      "step": 1681
    },
    {
      "epoch": 0.8285714285714286,
      "grad_norm": 0.5334116554998454,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -46821197.91304348,
      "logits/rejected": -58515655.80487805,
      "logps/chosen": -257.04347826086956,
      "logps/rejected": -590.0487804878048,
      "loss": 0.2023,
      "rewards/chosen": 2.7605899313221807,
      "rewards/margins": 20.297175297175837,
      "rewards/rejected": -17.536585365853657,
      "step": 1682
    },
    {
      "epoch": 0.829064039408867,
      "grad_norm": 0.44895534376224494,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -63085756.08163265,
      "logits/rejected": -44969309.974683546,
      "logps/chosen": -271.0204081632653,
      "logps/rejected": -509.56962025316454,
      "loss": 0.1421,
      "rewards/chosen": 1.4005102040816326,
      "rewards/margins": 18.539750710410747,
      "rewards/rejected": -17.139240506329113,
      "step": 1683
    },
    {
      "epoch": 0.8295566502463054,
      "grad_norm": 0.45704724106109046,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -66007859.2,
      "logits/rejected": -52762437.81818182,
      "logps/chosen": -326.8,
      "logps/rejected": -595.6363636363636,
      "loss": 0.1486,
      "rewards/chosen": 4.309399032592774,
      "rewards/margins": 24.10485357804732,
      "rewards/rejected": -19.795454545454547,
      "step": 1684
    },
    {
      "epoch": 0.8300492610837439,
      "grad_norm": 0.6455254259160036,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -42249940.29268292,
      "logits/rejected": -57032892.32183908,
      "logps/chosen": -328.5853658536585,
      "logps/rejected": -561.2873563218391,
      "loss": 0.1448,
      "rewards/chosen": 4.407028570407775,
      "rewards/margins": 21.87829293822387,
      "rewards/rejected": -17.471264367816094,
      "step": 1685
    },
    {
      "epoch": 0.8305418719211822,
      "grad_norm": 0.5219068931719679,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -59885340.44444445,
      "logits/rejected": -47653357.4939759,
      "logps/chosen": -360.53333333333336,
      "logps/rejected": -495.8072289156627,
      "loss": 0.154,
      "rewards/chosen": 0.5833333333333334,
      "rewards/margins": 15.812248995983936,
      "rewards/rejected": -15.228915662650602,
      "step": 1686
    },
    {
      "epoch": 0.8310344827586207,
      "grad_norm": 0.5140858841881003,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -51647508.078431375,
      "logits/rejected": -61389358.54545455,
      "logps/chosen": -326.9019607843137,
      "logps/rejected": -569.3506493506494,
      "loss": 0.1585,
      "rewards/chosen": 4.237134746476715,
      "rewards/margins": 23.172199681541652,
      "rewards/rejected": -18.935064935064936,
      "step": 1687
    },
    {
      "epoch": 0.8315270935960591,
      "grad_norm": 0.4637955777830872,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -47892103.8367347,
      "logits/rejected": -50119278.17721519,
      "logps/chosen": -260.0816326530612,
      "logps/rejected": -510.37974683544303,
      "loss": 0.1807,
      "rewards/chosen": 3.60271033462213,
      "rewards/margins": 20.09638122069808,
      "rewards/rejected": -16.49367088607595,
      "step": 1688
    },
    {
      "epoch": 0.8320197044334975,
      "grad_norm": 0.5009213428825592,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -49121752.615384616,
      "logits/rejected": -45971779.368421055,
      "logps/chosen": -246.76923076923077,
      "logps/rejected": -474.94736842105266,
      "loss": 0.1903,
      "rewards/chosen": 1.893019749568059,
      "rewards/margins": 18.235125012725952,
      "rewards/rejected": -16.342105263157894,
      "step": 1689
    },
    {
      "epoch": 0.8325123152709359,
      "grad_norm": 0.6269636881669021,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -55897166.76923077,
      "logits/rejected": -66832922.94736842,
      "logps/chosen": -289.53846153846155,
      "logps/rejected": -576.0,
      "loss": 0.1909,
      "rewards/chosen": 1.09375,
      "rewards/margins": 18.054276315789473,
      "rewards/rejected": -16.960526315789473,
      "step": 1690
    },
    {
      "epoch": 0.8330049261083744,
      "grad_norm": 0.486178372807516,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -51447154.38297872,
      "logits/rejected": -72287017.08641975,
      "logps/chosen": -318.29787234042556,
      "logps/rejected": -562.5679012345679,
      "loss": 0.1596,
      "rewards/chosen": 3.2686287088597075,
      "rewards/margins": 19.910604017501683,
      "rewards/rejected": -16.641975308641975,
      "step": 1691
    },
    {
      "epoch": 0.8334975369458129,
      "grad_norm": 0.5824847153475049,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -36504946.38297872,
      "logits/rejected": -55975834.86419753,
      "logps/chosen": -328.51063829787233,
      "logps/rejected": -583.9012345679013,
      "loss": 0.1493,
      "rewards/chosen": 2.175531914893617,
      "rewards/margins": 21.829852902547938,
      "rewards/rejected": -19.65432098765432,
      "step": 1692
    },
    {
      "epoch": 0.8339901477832512,
      "grad_norm": 0.535227558614664,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -80597364.36363636,
      "logits/rejected": -50980766.47619048,
      "logps/chosen": -290.90909090909093,
      "logps/rejected": -608.7619047619048,
      "loss": 0.1472,
      "rewards/chosen": 0.11079545454545454,
      "rewards/margins": 17.872700216450216,
      "rewards/rejected": -17.761904761904763,
      "step": 1693
    },
    {
      "epoch": 0.8344827586206897,
      "grad_norm": 0.47520182973346264,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -44698122.039215684,
      "logits/rejected": -62206430.753246754,
      "logps/chosen": -348.54901960784315,
      "logps/rejected": -568.5194805194806,
      "loss": 0.1498,
      "rewards/chosen": 1.6311274509803921,
      "rewards/margins": 14.981776801629742,
      "rewards/rejected": -13.35064935064935,
      "step": 1694
    },
    {
      "epoch": 0.8349753694581281,
      "grad_norm": 0.5419254728485379,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -53127850.666666664,
      "logits/rejected": -43978511.058823526,
      "logps/chosen": -333.6,
      "logps/rejected": -553.8823529411765,
      "loss": 0.193,
      "rewards/chosen": 0.4270833333333333,
      "rewards/margins": 16.647671568627448,
      "rewards/rejected": -16.220588235294116,
      "step": 1695
    },
    {
      "epoch": 0.8354679802955665,
      "grad_norm": 0.4592314073076356,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -38577842.60465116,
      "logits/rejected": -46729481.035294116,
      "logps/chosen": -282.2325581395349,
      "logps/rejected": -539.8588235294118,
      "loss": 0.1721,
      "rewards/chosen": 3.860523312590843,
      "rewards/margins": 22.754640959649667,
      "rewards/rejected": -18.894117647058824,
      "step": 1696
    },
    {
      "epoch": 0.8359605911330049,
      "grad_norm": 0.4680736203536098,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -54916119.81395349,
      "logits/rejected": -53242988.42352941,
      "logps/chosen": -343.4418604651163,
      "logps/rejected": -544.3764705882353,
      "loss": 0.1506,
      "rewards/chosen": 6.224639182867006,
      "rewards/margins": 24.91875682992583,
      "rewards/rejected": -18.694117647058825,
      "step": 1697
    },
    {
      "epoch": 0.8364532019704434,
      "grad_norm": 0.5281847046625123,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -49673717.960784316,
      "logits/rejected": -59210499.32467532,
      "logps/chosen": -349.4901960784314,
      "logps/rejected": -633.3506493506494,
      "loss": 0.2276,
      "rewards/chosen": -4.294117647058823,
      "rewards/margins": 16.06951871657754,
      "rewards/rejected": -20.363636363636363,
      "step": 1698
    },
    {
      "epoch": 0.8369458128078818,
      "grad_norm": 0.4926643856009293,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -47395635.2,
      "logits/rejected": -59634399.179487176,
      "logps/chosen": -288.64,
      "logps/rejected": -536.6153846153846,
      "loss": 0.2005,
      "rewards/chosen": 4.1625128173828125,
      "rewards/margins": 20.52148717635717,
      "rewards/rejected": -16.358974358974358,
      "step": 1699
    },
    {
      "epoch": 0.8374384236453202,
      "grad_norm": 0.5226368411654536,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -50366600.53333333,
      "logits/rejected": -56438061.176470585,
      "logps/chosen": -277.06666666666666,
      "logps/rejected": -553.4117647058823,
      "loss": 0.2068,
      "rewards/chosen": -0.7177083333333333,
      "rewards/margins": 16.796997549019608,
      "rewards/rejected": -17.514705882352942,
      "step": 1700
    },
    {
      "epoch": 0.8379310344827586,
      "grad_norm": 0.45108538651116353,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -57988691.34883721,
      "logits/rejected": -59164358.77647059,
      "logps/chosen": -273.48837209302326,
      "logps/rejected": -613.6470588235294,
      "loss": 0.1503,
      "rewards/chosen": -0.31976744186046513,
      "rewards/margins": 18.880232558139536,
      "rewards/rejected": -19.2,
      "step": 1701
    },
    {
      "epoch": 0.8384236453201971,
      "grad_norm": 0.36062765925304896,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -54011944.15686274,
      "logits/rejected": -46164579.74025974,
      "logps/chosen": -330.5098039215686,
      "logps/rejected": -560.2077922077922,
      "loss": 0.1819,
      "rewards/chosen": -1.9178921568627452,
      "rewards/margins": 16.549640310669723,
      "rewards/rejected": -18.467532467532468,
      "step": 1702
    },
    {
      "epoch": 0.8389162561576354,
      "grad_norm": 0.7055842172058324,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -42820030.836363636,
      "logits/rejected": -58145693.80821918,
      "logps/chosen": -282.4727272727273,
      "logps/rejected": -661.917808219178,
      "loss": 0.1825,
      "rewards/chosen": 2.6368974165482952,
      "rewards/margins": 22.93826727956199,
      "rewards/rejected": -20.301369863013697,
      "step": 1703
    },
    {
      "epoch": 0.8394088669950739,
      "grad_norm": 0.4607584477944961,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -54406795.63636363,
      "logits/rejected": -54476019.809523806,
      "logps/chosen": -257.09090909090907,
      "logps/rejected": -560.7619047619048,
      "loss": 0.1311,
      "rewards/chosen": 3.5049785267223013,
      "rewards/margins": 20.37402614576992,
      "rewards/rejected": -16.86904761904762,
      "step": 1704
    },
    {
      "epoch": 0.8399014778325123,
      "grad_norm": 0.5618924783793534,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -44806459.07692308,
      "logits/rejected": -49890142.315789476,
      "logps/chosen": -236.30769230769232,
      "logps/rejected": -559.1578947368421,
      "loss": 0.196,
      "rewards/chosen": -0.9627403846153846,
      "rewards/margins": 19.195154352226723,
      "rewards/rejected": -20.157894736842106,
      "step": 1705
    },
    {
      "epoch": 0.8403940886699507,
      "grad_norm": 0.4926838589341022,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -46137344.0,
      "logits/rejected": -57305897.674418606,
      "logps/chosen": -280.1904761904762,
      "logps/rejected": -606.5116279069767,
      "loss": 0.1657,
      "rewards/chosen": 2.021543230329241,
      "rewards/margins": 21.27735718381761,
      "rewards/rejected": -19.25581395348837,
      "step": 1706
    },
    {
      "epoch": 0.8408866995073891,
      "grad_norm": 0.4369315732526755,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -43821738.666666664,
      "logits/rejected": -60712550.4,
      "logps/chosen": -376.6666666666667,
      "logps/rejected": -570.0,
      "loss": 0.1981,
      "rewards/chosen": 0.5052218437194824,
      "rewards/margins": 18.505221843719482,
      "rewards/rejected": -18.0,
      "step": 1707
    },
    {
      "epoch": 0.8413793103448276,
      "grad_norm": 0.44273034822123436,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -48115339.63636363,
      "logits/rejected": -57022561.52380952,
      "logps/chosen": -349.8181818181818,
      "logps/rejected": -577.5238095238095,
      "loss": 0.1315,
      "rewards/chosen": 8.198853926225143,
      "rewards/margins": 27.841711069082287,
      "rewards/rejected": -19.642857142857142,
      "step": 1708
    },
    {
      "epoch": 0.8418719211822661,
      "grad_norm": 0.5199157553979825,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -43684071.8490566,
      "logits/rejected": -54302255.78666667,
      "logps/chosen": -297.35849056603774,
      "logps/rejected": -579.4133333333333,
      "loss": 0.2122,
      "rewards/chosen": 0.2547617498433815,
      "rewards/margins": 17.934761749843382,
      "rewards/rejected": -17.68,
      "step": 1709
    },
    {
      "epoch": 0.8423645320197044,
      "grad_norm": 0.4135350970797819,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -45583379.320754714,
      "logits/rejected": -60733521.92,
      "logps/chosen": -305.50943396226415,
      "logps/rejected": -575.1466666666666,
      "loss": 0.1867,
      "rewards/chosen": -1.3791273584905661,
      "rewards/margins": 17.5275393081761,
      "rewards/rejected": -18.906666666666666,
      "step": 1710
    },
    {
      "epoch": 0.8428571428571429,
      "grad_norm": 0.6420829354494254,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -63864213.73584906,
      "logits/rejected": -50723116.373333335,
      "logps/chosen": -339.62264150943395,
      "logps/rejected": -532.0533333333333,
      "loss": 0.2552,
      "rewards/chosen": -3.1179245283018866,
      "rewards/margins": 13.162075471698115,
      "rewards/rejected": -16.28,
      "step": 1711
    },
    {
      "epoch": 0.8433497536945813,
      "grad_norm": 0.682920358031436,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -50573627.07692308,
      "logits/rejected": -50993906.526315786,
      "logps/chosen": -250.46153846153845,
      "logps/rejected": -593.6842105263158,
      "loss": 0.1735,
      "rewards/chosen": 2.0,
      "rewards/margins": 22.57894736842105,
      "rewards/rejected": -20.57894736842105,
      "step": 1712
    },
    {
      "epoch": 0.8438423645320197,
      "grad_norm": 0.38917840707389456,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -75406291.47826087,
      "logits/rejected": -50433948.09756097,
      "logps/chosen": -377.04347826086956,
      "logps/rejected": -592.390243902439,
      "loss": 0.1609,
      "rewards/chosen": -2.2364130434782608,
      "rewards/margins": 17.495294273594908,
      "rewards/rejected": -19.73170731707317,
      "step": 1713
    },
    {
      "epoch": 0.8443349753694581,
      "grad_norm": 0.4536191790916608,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -40218715.02222222,
      "logits/rejected": -54778620.915662654,
      "logps/chosen": -205.42222222222222,
      "logps/rejected": -615.710843373494,
      "loss": 0.1059,
      "rewards/chosen": 2.1083333333333334,
      "rewards/margins": 23.023995983935745,
      "rewards/rejected": -20.91566265060241,
      "step": 1714
    },
    {
      "epoch": 0.8448275862068966,
      "grad_norm": 0.43185846809894035,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -55356074.666666664,
      "logits/rejected": -58091110.4,
      "logps/chosen": -336.0,
      "logps/rejected": -590.4,
      "loss": 0.1534,
      "rewards/chosen": 0.8499348958333334,
      "rewards/margins": 19.54993489583333,
      "rewards/rejected": -18.7,
      "step": 1715
    },
    {
      "epoch": 0.8453201970443349,
      "grad_norm": 0.6897460690764221,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -40876067.92982456,
      "logits/rejected": -62028439.43661972,
      "logps/chosen": -245.05263157894737,
      "logps/rejected": -704.9014084507043,
      "loss": 0.1955,
      "rewards/chosen": 0.3580043859649123,
      "rewards/margins": 22.27349734371139,
      "rewards/rejected": -21.91549295774648,
      "step": 1716
    },
    {
      "epoch": 0.8458128078817734,
      "grad_norm": 0.5947919967477624,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -51692957.19298246,
      "logits/rejected": -52990009.69014084,
      "logps/chosen": -275.0877192982456,
      "logps/rejected": -612.9577464788732,
      "loss": 0.2076,
      "rewards/chosen": -0.5241228070175439,
      "rewards/margins": 19.53221522115147,
      "rewards/rejected": -20.056338028169016,
      "step": 1717
    },
    {
      "epoch": 0.8463054187192118,
      "grad_norm": 0.4993388483845808,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -44258645.333333336,
      "logits/rejected": -59559116.8,
      "logps/chosen": -248.0,
      "logps/rejected": -588.8,
      "loss": 0.1543,
      "rewards/chosen": 0.9479166666666666,
      "rewards/margins": 16.647916666666667,
      "rewards/rejected": -15.7,
      "step": 1718
    },
    {
      "epoch": 0.8467980295566503,
      "grad_norm": 0.6907653024444133,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -50833140.86956522,
      "logits/rejected": -59640956.87804878,
      "logps/chosen": -339.4782608695652,
      "logps/rejected": -611.9024390243902,
      "loss": 0.1576,
      "rewards/chosen": 0.2907608695652174,
      "rewards/margins": 19.778565747614,
      "rewards/rejected": -19.48780487804878,
      "step": 1719
    },
    {
      "epoch": 0.8472906403940886,
      "grad_norm": 0.49633546853007476,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -62750077.49019608,
      "logits/rejected": -53981237.1948052,
      "logps/chosen": -282.3529411764706,
      "logps/rejected": -578.4935064935065,
      "loss": 0.1853,
      "rewards/chosen": 1.5110294117647058,
      "rewards/margins": 18.030509931245227,
      "rewards/rejected": -16.51948051948052,
      "step": 1720
    },
    {
      "epoch": 0.8477832512315271,
      "grad_norm": 0.4952723322020361,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -46176912.905660376,
      "logits/rejected": -47982837.76,
      "logps/chosen": -344.1509433962264,
      "logps/rejected": -551.2533333333333,
      "loss": 0.1354,
      "rewards/chosen": 3.1014150943396226,
      "rewards/margins": 20.46141509433962,
      "rewards/rejected": -17.36,
      "step": 1721
    },
    {
      "epoch": 0.8482758620689655,
      "grad_norm": 0.5212010270180881,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -56575441.45454545,
      "logits/rejected": -65810627.047619045,
      "logps/chosen": -365.45454545454544,
      "logps/rejected": -598.0952380952381,
      "loss": 0.1774,
      "rewards/chosen": 2.132811806418679,
      "rewards/margins": 19.204240377847253,
      "rewards/rejected": -17.071428571428573,
      "step": 1722
    },
    {
      "epoch": 0.8487684729064039,
      "grad_norm": 0.42718711797181463,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -47866574.596491225,
      "logits/rejected": -67640536.33802816,
      "logps/chosen": -360.42105263157896,
      "logps/rejected": -556.169014084507,
      "loss": 0.1607,
      "rewards/chosen": 0.22697368421052633,
      "rewards/margins": 17.832607487027428,
      "rewards/rejected": -17.6056338028169,
      "step": 1723
    },
    {
      "epoch": 0.8492610837438423,
      "grad_norm": 0.6025002192685208,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -42627824.32653061,
      "logits/rejected": -66896494.17721519,
      "logps/chosen": -280.81632653061223,
      "logps/rejected": -601.9240506329114,
      "loss": 0.1675,
      "rewards/chosen": 2.9853556107501595,
      "rewards/margins": 20.656241686699524,
      "rewards/rejected": -17.670886075949365,
      "step": 1724
    },
    {
      "epoch": 0.8497536945812808,
      "grad_norm": 0.5314919380142131,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -55856836.92307692,
      "logits/rejected": -77484247.57894737,
      "logps/chosen": -304.9230769230769,
      "logps/rejected": -688.0,
      "loss": 0.1869,
      "rewards/chosen": 1.9711330120380108,
      "rewards/margins": 23.576396169932746,
      "rewards/rejected": -21.605263157894736,
      "step": 1725
    },
    {
      "epoch": 0.8502463054187193,
      "grad_norm": 0.6722736873756051,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -42114236.08163265,
      "logits/rejected": -54419767.088607594,
      "logps/chosen": -273.9591836734694,
      "logps/rejected": -576.4050632911392,
      "loss": 0.1677,
      "rewards/chosen": 2.4821453483737246,
      "rewards/margins": 22.12771496862689,
      "rewards/rejected": -19.645569620253166,
      "step": 1726
    },
    {
      "epoch": 0.8507389162561576,
      "grad_norm": 0.43450442094643427,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -37698803.809523806,
      "logits/rejected": -55403829.58139535,
      "logps/chosen": -269.7142857142857,
      "logps/rejected": -649.6744186046511,
      "loss": 0.1721,
      "rewards/chosen": 2.006712413969494,
      "rewards/margins": 22.35554962327182,
      "rewards/rejected": -20.348837209302324,
      "step": 1727
    },
    {
      "epoch": 0.8512315270935961,
      "grad_norm": 0.6373521639304787,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -62171818.666666664,
      "logits/rejected": -59454259.2,
      "logps/chosen": -346.8333333333333,
      "logps/rejected": -618.4,
      "loss": 0.1844,
      "rewards/chosen": 1.4222221374511719,
      "rewards/margins": 21.94722213745117,
      "rewards/rejected": -20.525,
      "step": 1728
    },
    {
      "epoch": 0.8517241379310345,
      "grad_norm": 0.48287753726538946,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -49283072.0,
      "logits/rejected": -48088183.069767445,
      "logps/chosen": -249.9047619047619,
      "logps/rejected": -526.8837209302326,
      "loss": 0.1724,
      "rewards/chosen": -0.7142857142857143,
      "rewards/margins": 14.471760797342194,
      "rewards/rejected": -15.186046511627907,
      "step": 1729
    },
    {
      "epoch": 0.8522167487684729,
      "grad_norm": 0.4574774226523035,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -52186820.92307692,
      "logits/rejected": -50938718.315789476,
      "logps/chosen": -280.0,
      "logps/rejected": -583.1578947368421,
      "loss": 0.1757,
      "rewards/chosen": 0.4170673076923077,
      "rewards/margins": 19.785488360323885,
      "rewards/rejected": -19.36842105263158,
      "step": 1730
    },
    {
      "epoch": 0.8527093596059113,
      "grad_norm": 0.5703261391794401,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -56489243.234042555,
      "logits/rejected": -57115028.54320987,
      "logps/chosen": -234.89361702127658,
      "logps/rejected": -574.4197530864197,
      "loss": 0.1753,
      "rewards/chosen": -0.013962765957446808,
      "rewards/margins": 17.986037234042552,
      "rewards/rejected": -18.0,
      "step": 1731
    },
    {
      "epoch": 0.8532019704433498,
      "grad_norm": 0.5505639598505019,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -49283072.0,
      "logits/rejected": -48801293.83783784,
      "logps/chosen": -284.44444444444446,
      "logps/rejected": -520.6486486486486,
      "loss": 0.182,
      "rewards/chosen": -0.5894097222222222,
      "rewards/margins": 19.46464433183183,
      "rewards/rejected": -20.054054054054053,
      "step": 1732
    },
    {
      "epoch": 0.8536945812807881,
      "grad_norm": 0.5416632972014833,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -54168989.957446806,
      "logits/rejected": -54111699.75308642,
      "logps/chosen": -390.8085106382979,
      "logps/rejected": -601.283950617284,
      "loss": 0.177,
      "rewards/chosen": -0.9501329787234043,
      "rewards/margins": 19.593076897819806,
      "rewards/rejected": -20.54320987654321,
      "step": 1733
    },
    {
      "epoch": 0.8541871921182266,
      "grad_norm": 0.47516957874011956,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -54861496.32,
      "logits/rejected": -52052388.102564104,
      "logps/chosen": -263.04,
      "logps/rejected": -580.9230769230769,
      "loss": 0.1657,
      "rewards/chosen": -0.290625,
      "rewards/margins": 21.06834935897436,
      "rewards/rejected": -21.358974358974358,
      "step": 1734
    },
    {
      "epoch": 0.854679802955665,
      "grad_norm": 0.5042748944551058,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -45976024.615384616,
      "logits/rejected": -60541466.94736842,
      "logps/chosen": -339.38461538461536,
      "logps/rejected": -537.6842105263158,
      "loss": 0.186,
      "rewards/chosen": 0.38221153846153844,
      "rewards/margins": 14.78948628661121,
      "rewards/rejected": -14.407274748149671,
      "step": 1735
    },
    {
      "epoch": 0.8551724137931035,
      "grad_norm": 0.4049668624699605,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -56623104.0,
      "logits/rejected": -65011712.0,
      "logps/chosen": -336.57142857142856,
      "logps/rejected": -653.3333333333334,
      "loss": 0.146,
      "rewards/chosen": 3.2017386300223216,
      "rewards/margins": 23.923960852244544,
      "rewards/rejected": -20.72222222222222,
      "step": 1736
    },
    {
      "epoch": 0.8556650246305418,
      "grad_norm": 0.5786322778062862,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -32615590.69767442,
      "logits/rejected": -62667836.23529412,
      "logps/chosen": -316.6511627906977,
      "logps/rejected": -597.8352941176471,
      "loss": 0.1884,
      "rewards/chosen": 1.1548390942950582,
      "rewards/margins": 19.248956741353883,
      "rewards/rejected": -18.094117647058823,
      "step": 1737
    },
    {
      "epoch": 0.8561576354679803,
      "grad_norm": 0.4464909561740337,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -59191045.22448979,
      "logits/rejected": -53570287.79746836,
      "logps/chosen": -274.9387755102041,
      "logps/rejected": -559.7974683544304,
      "loss": 0.1526,
      "rewards/chosen": 1.3822071308992347,
      "rewards/margins": 22.648029915709362,
      "rewards/rejected": -21.265822784810126,
      "step": 1738
    },
    {
      "epoch": 0.8566502463054187,
      "grad_norm": 0.5506419257495093,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -52149179.733333334,
      "logits/rejected": -66401391.03614458,
      "logps/chosen": -200.0,
      "logps/rejected": -610.6987951807229,
      "loss": 0.1773,
      "rewards/chosen": -1.5708333333333333,
      "rewards/margins": 16.140010040160643,
      "rewards/rejected": -17.710843373493976,
      "step": 1739
    },
    {
      "epoch": 0.8571428571428571,
      "grad_norm": 0.4866198004351141,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -51904512.0,
      "logits/rejected": -51049094.7368421,
      "logps/chosen": -361.53846153846155,
      "logps/rejected": -587.7894736842105,
      "loss": 0.2193,
      "rewards/chosen": -2.219951923076923,
      "rewards/margins": 16.727416497975707,
      "rewards/rejected": -18.94736842105263,
      "step": 1740
    },
    {
      "epoch": 0.8576354679802956,
      "grad_norm": 0.5795951917669028,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -57997100.137931034,
      "logits/rejected": -58959930.51428571,
      "logps/chosen": -364.41379310344826,
      "logps/rejected": -565.0285714285715,
      "loss": 0.2128,
      "rewards/chosen": -1.5150862068965518,
      "rewards/margins": 18.227770935960592,
      "rewards/rejected": -19.742857142857144,
      "step": 1741
    },
    {
      "epoch": 0.858128078817734,
      "grad_norm": 0.4585644153279719,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -52479950.048780486,
      "logits/rejected": -49945964.873563215,
      "logps/chosen": -328.1951219512195,
      "logps/rejected": -492.13793103448273,
      "loss": 0.167,
      "rewards/chosen": -1.6676829268292683,
      "rewards/margins": 16.700133165124754,
      "rewards/rejected": -18.367816091954023,
      "step": 1742
    },
    {
      "epoch": 0.8586206896551725,
      "grad_norm": 0.45997561859105707,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -58103446.5882353,
      "logits/rejected": -53926765.71428572,
      "logps/chosen": -377.0980392156863,
      "logps/rejected": -580.5714285714286,
      "loss": 0.1833,
      "rewards/chosen": 4.715078914866728,
      "rewards/margins": 25.93585813564595,
      "rewards/rejected": -21.22077922077922,
      "step": 1743
    },
    {
      "epoch": 0.8591133004926108,
      "grad_norm": 0.4793399599649742,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -40559812.08510638,
      "logits/rejected": -48622857.481481485,
      "logps/chosen": -273.36170212765956,
      "logps/rejected": -598.1234567901234,
      "loss": 0.1904,
      "rewards/chosen": 1.64029612439744,
      "rewards/margins": 21.455110939212254,
      "rewards/rejected": -19.814814814814813,
      "step": 1744
    },
    {
      "epoch": 0.8596059113300493,
      "grad_norm": 0.46452553133169877,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -61744056.55813953,
      "logits/rejected": -68983964.6117647,
      "logps/chosen": -375.8139534883721,
      "logps/rejected": -571.4823529411765,
      "loss": 0.1793,
      "rewards/chosen": 5.823490231536155,
      "rewards/margins": 25.988196113889096,
      "rewards/rejected": -20.16470588235294,
      "step": 1745
    },
    {
      "epoch": 0.8600985221674877,
      "grad_norm": 0.4831786198019052,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -44282171.07692308,
      "logits/rejected": -61865984.0,
      "logps/chosen": -224.6153846153846,
      "logps/rejected": -538.9473684210526,
      "loss": 0.1893,
      "rewards/chosen": 1.3377530024601862,
      "rewards/margins": 18.166700370881237,
      "rewards/rejected": -16.82894736842105,
      "step": 1746
    },
    {
      "epoch": 0.8605911330049261,
      "grad_norm": 0.4365661872507729,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -50674040.1632653,
      "logits/rejected": -55375431.29113924,
      "logps/chosen": -218.28571428571428,
      "logps/rejected": -612.4556962025316,
      "loss": 0.1398,
      "rewards/chosen": 4.916422941246811,
      "rewards/margins": 27.01768876403162,
      "rewards/rejected": -22.10126582278481,
      "step": 1747
    },
    {
      "epoch": 0.8610837438423645,
      "grad_norm": 0.646691300265292,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -45808378.98039216,
      "logits/rejected": -46682058.8051948,
      "logps/chosen": -316.3921568627451,
      "logps/rejected": -517.4025974025974,
      "loss": 0.2065,
      "rewards/chosen": -2.1348039215686274,
      "rewards/margins": 14.163897377132672,
      "rewards/rejected": -16.2987012987013,
      "step": 1748
    },
    {
      "epoch": 0.861576354679803,
      "grad_norm": 0.512093085707614,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -41037451.63636363,
      "logits/rejected": -47135987.809523806,
      "logps/chosen": -254.1818181818182,
      "logps/rejected": -622.4761904761905,
      "loss": 0.1643,
      "rewards/chosen": 0.3210227272727273,
      "rewards/margins": 24.249594155844154,
      "rewards/rejected": -23.928571428571427,
      "step": 1749
    },
    {
      "epoch": 0.8620689655172413,
      "grad_norm": 0.5818251754690211,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -67742886.69767442,
      "logits/rejected": -63950799.8117647,
      "logps/chosen": -360.1860465116279,
      "logps/rejected": -672.3764705882353,
      "loss": 0.1319,
      "rewards/chosen": 3.8888475285019983,
      "rewards/margins": 27.30061223438435,
      "rewards/rejected": -23.41176470588235,
      "step": 1750
    },
    {
      "epoch": 0.8625615763546798,
      "grad_norm": 0.6687856698887048,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -52511041.25490196,
      "logits/rejected": -81162505.97402598,
      "logps/chosen": -322.5098039215686,
      "logps/rejected": -690.7012987012987,
      "loss": 0.1822,
      "rewards/chosen": 2.5435988482306984,
      "rewards/margins": 24.98515728978914,
      "rewards/rejected": -22.441558441558442,
      "step": 1751
    },
    {
      "epoch": 0.8630541871921182,
      "grad_norm": 0.35561505113605346,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -46230550.755555555,
      "logits/rejected": -53717411.469879515,
      "logps/chosen": -290.4888888888889,
      "logps/rejected": -618.4096385542168,
      "loss": 0.1499,
      "rewards/chosen": -0.6920138888888889,
      "rewards/margins": 16.68894255407045,
      "rewards/rejected": -17.380956442959338,
      "step": 1752
    },
    {
      "epoch": 0.8635467980295567,
      "grad_norm": 0.4434571634319385,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -46322386.823529415,
      "logits/rejected": -54471480.51948052,
      "logps/chosen": -269.80392156862746,
      "logps/rejected": -615.0649350649351,
      "loss": 0.1598,
      "rewards/chosen": 5.387277640548407,
      "rewards/margins": 26.99766725093802,
      "rewards/rejected": -21.61038961038961,
      "step": 1753
    },
    {
      "epoch": 0.864039408866995,
      "grad_norm": 0.8375732001685482,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -56730650.256410256,
      "logits/rejected": -63103068.04494382,
      "logps/chosen": -329.43589743589746,
      "logps/rejected": -639.2808988764045,
      "loss": 0.1746,
      "rewards/chosen": -2.780448717948718,
      "rewards/margins": 19.489214203399595,
      "rewards/rejected": -22.269662921348313,
      "step": 1754
    },
    {
      "epoch": 0.8645320197044335,
      "grad_norm": 0.5943333123012543,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -57157672.15686274,
      "logits/rejected": -54961723.84415584,
      "logps/chosen": -307.7647058823529,
      "logps/rejected": -571.012987012987,
      "loss": 0.1364,
      "rewards/chosen": 1.8112745098039216,
      "rewards/margins": 19.811274509803923,
      "rewards/rejected": -18.0,
      "step": 1755
    },
    {
      "epoch": 0.865024630541872,
      "grad_norm": 0.539041302216467,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -58043755.35483871,
      "logits/rejected": -57639904.96969697,
      "logps/chosen": -445.4193548387097,
      "logps/rejected": -636.6060606060606,
      "loss": 0.1846,
      "rewards/chosen": 1.0211950732815651,
      "rewards/margins": 25.06664961873611,
      "rewards/rejected": -24.045454545454547,
      "step": 1756
    },
    {
      "epoch": 0.8655172413793103,
      "grad_norm": 0.44167647958045786,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -82302516.24489796,
      "logits/rejected": -65834645.06329114,
      "logps/chosen": -395.1020408163265,
      "logps/rejected": -589.7721518987341,
      "loss": 0.1708,
      "rewards/chosen": -1.086734693877551,
      "rewards/margins": 20.989214673211055,
      "rewards/rejected": -22.075949367088608,
      "step": 1757
    },
    {
      "epoch": 0.8660098522167488,
      "grad_norm": 0.47927294784527397,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -44820527.62790698,
      "logits/rejected": -50134268.988235295,
      "logps/chosen": -294.3255813953488,
      "logps/rejected": -586.5411764705882,
      "loss": 0.1796,
      "rewards/chosen": -0.626453488372093,
      "rewards/margins": 19.25589945280438,
      "rewards/rejected": -19.88235294117647,
      "step": 1758
    },
    {
      "epoch": 0.8665024630541872,
      "grad_norm": 0.5954727689225662,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -62346581.333333336,
      "logits/rejected": -62023270.4,
      "logps/chosen": -449.0,
      "logps/rejected": -651.2,
      "loss": 0.1511,
      "rewards/chosen": -0.017578125,
      "rewards/margins": 20.644921875,
      "rewards/rejected": -20.6625,
      "step": 1759
    },
    {
      "epoch": 0.8669950738916257,
      "grad_norm": 0.5106116129558252,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -59529683.0877193,
      "logits/rejected": -60906020.05633803,
      "logps/chosen": -364.35087719298247,
      "logps/rejected": -588.169014084507,
      "loss": 0.1827,
      "rewards/chosen": 2.9616281609786186,
      "rewards/margins": 23.215149287739184,
      "rewards/rejected": -20.253521126760564,
      "step": 1760
    },
    {
      "epoch": 0.867487684729064,
      "grad_norm": 0.5068134171478371,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -53114407.384615384,
      "logits/rejected": -52759929.2631579,
      "logps/chosen": -366.7692307692308,
      "logps/rejected": -561.6842105263158,
      "loss": 0.1961,
      "rewards/chosen": 0.7898960113525391,
      "rewards/margins": 19.947790748194645,
      "rewards/rejected": -19.157894736842106,
      "step": 1761
    },
    {
      "epoch": 0.8679802955665025,
      "grad_norm": 0.623745014143916,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -51790536.347826086,
      "logits/rejected": -54270201.75609756,
      "logps/chosen": -236.8695652173913,
      "logps/rejected": -589.2682926829268,
      "loss": 0.1369,
      "rewards/chosen": 3.08356177288553,
      "rewards/margins": 21.449415431422118,
      "rewards/rejected": -18.365853658536587,
      "step": 1762
    },
    {
      "epoch": 0.8684729064039409,
      "grad_norm": 0.48572244100300144,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -57278464.0,
      "logits/rejected": -63333990.4,
      "logps/chosen": -332.0,
      "logps/rejected": -625.6,
      "loss": 0.1796,
      "rewards/chosen": 2.175128618876139,
      "rewards/margins": 21.45012861887614,
      "rewards/rejected": -19.275,
      "step": 1763
    },
    {
      "epoch": 0.8689655172413793,
      "grad_norm": 0.5441304557331067,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -45336613.236363634,
      "logits/rejected": -49469804.71232877,
      "logps/chosen": -330.4727272727273,
      "logps/rejected": -533.917808219178,
      "loss": 0.193,
      "rewards/chosen": 2.7264509721235797,
      "rewards/margins": 21.27439617760303,
      "rewards/rejected": -18.54794520547945,
      "step": 1764
    },
    {
      "epoch": 0.8694581280788177,
      "grad_norm": 0.41061796707291043,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -63214153.14285714,
      "logits/rejected": -63233114.73417722,
      "logps/chosen": -241.14285714285714,
      "logps/rejected": -571.1392405063291,
      "loss": 0.12,
      "rewards/chosen": 1.5395408163265305,
      "rewards/margins": 13.680621648506119,
      "rewards/rejected": -12.141080832179588,
      "step": 1765
    },
    {
      "epoch": 0.8699507389162562,
      "grad_norm": 0.4750827184825605,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -52095162.18181818,
      "logits/rejected": -58021205.333333336,
      "logps/chosen": -327.27272727272725,
      "logps/rejected": -582.0952380952381,
      "loss": 0.1467,
      "rewards/chosen": 4.1533990339799365,
      "rewards/margins": 25.10577998636089,
      "rewards/rejected": -20.952380952380953,
      "step": 1766
    },
    {
      "epoch": 0.8704433497536945,
      "grad_norm": 0.4433191658468955,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -61027123.2,
      "logits/rejected": -61437021.09090909,
      "logps/chosen": -372.6,
      "logps/rejected": -557.8181818181819,
      "loss": 0.1113,
      "rewards/chosen": 4.589061737060547,
      "rewards/margins": 24.134516282515094,
      "rewards/rejected": -19.545454545454547,
      "step": 1767
    },
    {
      "epoch": 0.870935960591133,
      "grad_norm": 0.4685282843335273,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -53477376.0,
      "logits/rejected": -55824188.952380955,
      "logps/chosen": -349.45454545454544,
      "logps/rejected": -632.3809523809524,
      "loss": 0.1607,
      "rewards/chosen": -0.47798295454545453,
      "rewards/margins": 21.069636093073594,
      "rewards/rejected": -21.547619047619047,
      "step": 1768
    },
    {
      "epoch": 0.8714285714285714,
      "grad_norm": 0.5215078359748015,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -84957957.68888889,
      "logits/rejected": -56041965.4939759,
      "logps/chosen": -286.0444444444444,
      "logps/rejected": -633.0602409638554,
      "loss": 0.134,
      "rewards/chosen": 2.8299158732096354,
      "rewards/margins": 22.420277318992767,
      "rewards/rejected": -19.59036144578313,
      "step": 1769
    },
    {
      "epoch": 0.8719211822660099,
      "grad_norm": 0.5622388550757527,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -63072835.62264151,
      "logits/rejected": -57657698.986666664,
      "logps/chosen": -362.2641509433962,
      "logps/rejected": -589.6533333333333,
      "loss": 0.1573,
      "rewards/chosen": 0.1875,
      "rewards/margins": 19.574166666666667,
      "rewards/rejected": -19.386666666666667,
      "step": 1770
    },
    {
      "epoch": 0.8724137931034482,
      "grad_norm": 0.5312728655697243,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -53580177.568627454,
      "logits/rejected": -58720256.0,
      "logps/chosen": -300.70588235294116,
      "logps/rejected": -653.2987012987013,
      "loss": 0.14,
      "rewards/chosen": 3.8566445743336395,
      "rewards/margins": 26.63586535355442,
      "rewards/rejected": -22.77922077922078,
      "step": 1771
    },
    {
      "epoch": 0.8729064039408867,
      "grad_norm": 0.6178913453779967,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -53710392.88888889,
      "logits/rejected": -53562395.675675675,
      "logps/chosen": -298.962962962963,
      "logps/rejected": -588.5405405405405,
      "loss": 0.2058,
      "rewards/chosen": -0.059027777777777776,
      "rewards/margins": 20.292323573573572,
      "rewards/rejected": -20.35135135135135,
      "step": 1772
    },
    {
      "epoch": 0.8733990147783252,
      "grad_norm": 0.4664553417714862,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -67021482.666666664,
      "logits/rejected": -47657779.2,
      "logps/chosen": -383.3333333333333,
      "logps/rejected": -586.4,
      "loss": 0.1532,
      "rewards/chosen": 0.3385416666666667,
      "rewards/margins": 22.18854166666667,
      "rewards/rejected": -21.85,
      "step": 1773
    },
    {
      "epoch": 0.8738916256157635,
      "grad_norm": 0.4385601477325264,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -43480951.46666667,
      "logits/rejected": -51948729.06024096,
      "logps/chosen": -269.8666666666667,
      "logps/rejected": -626.1204819277109,
      "loss": 0.15,
      "rewards/chosen": -0.46458333333333335,
      "rewards/margins": 23.559513052208835,
      "rewards/rejected": -24.02409638554217,
      "step": 1774
    },
    {
      "epoch": 0.874384236453202,
      "grad_norm": 0.5454759023345584,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -50415534.08,
      "logits/rejected": -62860786.87179487,
      "logps/chosen": -313.92,
      "logps/rejected": -583.3846153846154,
      "loss": 0.1771,
      "rewards/chosen": 0.091875,
      "rewards/margins": 20.886746794871797,
      "rewards/rejected": -20.794871794871796,
      "step": 1775
    },
    {
      "epoch": 0.8748768472906404,
      "grad_norm": 0.5136296419292355,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -90811558.69767442,
      "logits/rejected": -49838200.47058824,
      "logps/chosen": -263.4418604651163,
      "logps/rejected": -585.7882352941176,
      "loss": 0.1313,
      "rewards/chosen": 3.5552417843840844,
      "rewards/margins": 24.472888843207613,
      "rewards/rejected": -20.91764705882353,
      "step": 1776
    },
    {
      "epoch": 0.8753694581280789,
      "grad_norm": 0.5858296016348054,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -76755763.2,
      "logits/rejected": -53611808.820512824,
      "logps/chosen": -298.24,
      "logps/rejected": -602.2564102564103,
      "loss": 0.1887,
      "rewards/chosen": 2.3337969970703125,
      "rewards/margins": 23.51328417655749,
      "rewards/rejected": -21.17948717948718,
      "step": 1777
    },
    {
      "epoch": 0.8758620689655172,
      "grad_norm": 0.4073054689341259,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -73352657.45454545,
      "logits/rejected": -52229071.23809524,
      "logps/chosen": -336.3636363636364,
      "logps/rejected": -622.4761904761905,
      "loss": 0.2057,
      "rewards/chosen": -1.868616971102628,
      "rewards/margins": 13.079229941099754,
      "rewards/rejected": -14.947846912202381,
      "step": 1778
    },
    {
      "epoch": 0.8763546798029557,
      "grad_norm": 0.47625888182928955,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -48799113.84615385,
      "logits/rejected": -56678292.21052632,
      "logps/chosen": -272.0,
      "logps/rejected": -633.2631578947369,
      "loss": 0.1842,
      "rewards/chosen": 4.503062321589543,
      "rewards/margins": 29.7399044268527,
      "rewards/rejected": -25.236842105263158,
      "step": 1779
    },
    {
      "epoch": 0.8768472906403941,
      "grad_norm": 0.527792295524905,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -73439888.90566038,
      "logits/rejected": -58664331.946666665,
      "logps/chosen": -353.20754716981133,
      "logps/rejected": -636.5866666666667,
      "loss": 0.1685,
      "rewards/chosen": 7.4311200987617925,
      "rewards/margins": 28.444453432095123,
      "rewards/rejected": -21.013333333333332,
      "step": 1780
    },
    {
      "epoch": 0.8773399014778325,
      "grad_norm": 0.47661818477792517,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -59787897.018181816,
      "logits/rejected": -59524643.06849315,
      "logps/chosen": -266.76363636363635,
      "logps/rejected": -651.8356164383562,
      "loss": 0.1808,
      "rewards/chosen": 1.6249855735085228,
      "rewards/margins": 25.556492422823588,
      "rewards/rejected": -23.931506849315067,
      "step": 1781
    },
    {
      "epoch": 0.8778325123152709,
      "grad_norm": 1.201880681055627,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -56885248.0,
      "logits/rejected": -45088768.0,
      "logps/chosen": -338.0,
      "logps/rejected": -604.8888888888889,
      "loss": 0.1645,
      "rewards/chosen": 0.9787946428571429,
      "rewards/margins": 21.201016865079364,
      "rewards/rejected": -20.22222222222222,
      "step": 1782
    },
    {
      "epoch": 0.8783251231527094,
      "grad_norm": 0.5666569898566655,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -51683145.95555556,
      "logits/rejected": -53363674.98795181,
      "logps/chosen": -273.06666666666666,
      "logps/rejected": -660.0481927710844,
      "loss": 0.1457,
      "rewards/chosen": 0.531923590766059,
      "rewards/margins": 24.70059828956124,
      "rewards/rejected": -24.16867469879518,
      "step": 1783
    },
    {
      "epoch": 0.8788177339901477,
      "grad_norm": 1.046178458998751,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -64927825.92,
      "logits/rejected": -66839998.35897436,
      "logps/chosen": -402.88,
      "logps/rejected": -692.5128205128206,
      "loss": 0.1977,
      "rewards/chosen": 0.36782150268554686,
      "rewards/margins": 23.265257400121445,
      "rewards/rejected": -22.897435897435898,
      "step": 1784
    },
    {
      "epoch": 0.8793103448275862,
      "grad_norm": 0.46975778486329967,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -57693079.510204084,
      "logits/rejected": -62808375.088607594,
      "logps/chosen": -346.1224489795918,
      "logps/rejected": -604.3544303797469,
      "loss": 0.1594,
      "rewards/chosen": 2.3667571398676657,
      "rewards/margins": 22.92371916518412,
      "rewards/rejected": -20.556962025316455,
      "step": 1785
    },
    {
      "epoch": 0.8798029556650246,
      "grad_norm": 0.6309848523532143,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -78570043.53488372,
      "logits/rejected": -55266123.294117644,
      "logps/chosen": -305.6744186046512,
      "logps/rejected": -589.5529411764705,
      "loss": 0.1779,
      "rewards/chosen": 2.0995575660882992,
      "rewards/margins": 22.193675213147124,
      "rewards/rejected": -20.094117647058823,
      "step": 1786
    },
    {
      "epoch": 0.8802955665024631,
      "grad_norm": 0.3564359530107374,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -57671680.0,
      "logits/rejected": -58941008.84210526,
      "logps/chosen": -320.3076923076923,
      "logps/rejected": -599.578947368421,
      "loss": 0.1577,
      "rewards/chosen": -1.9284855769230769,
      "rewards/margins": 18.518882844129553,
      "rewards/rejected": -20.44736842105263,
      "step": 1787
    },
    {
      "epoch": 0.8807881773399014,
      "grad_norm": 0.6092437709689504,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -50032054.85714286,
      "logits/rejected": -79109233.77777778,
      "logps/chosen": -312.2857142857143,
      "logps/rejected": -781.3333333333334,
      "loss": 0.2091,
      "rewards/chosen": 0.36607142857142855,
      "rewards/margins": 2100840.5882936507,
      "rewards/rejected": -2100840.222222222,
      "step": 1788
    },
    {
      "epoch": 0.8812807881773399,
      "grad_norm": 0.41609990829897014,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -41686245.87755102,
      "logits/rejected": -68064528.20253165,
      "logps/chosen": -360.16326530612247,
      "logps/rejected": -597.0632911392405,
      "loss": 0.1867,
      "rewards/chosen": -3.0153061224489797,
      "rewards/margins": 16.858111599070007,
      "rewards/rejected": -19.873417721518987,
      "step": 1789
    },
    {
      "epoch": 0.8817733990147784,
      "grad_norm": 0.4029958263544193,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -43019049.674418606,
      "logits/rejected": -56845155.38823529,
      "logps/chosen": -265.86046511627904,
      "logps/rejected": -633.2235294117647,
      "loss": 0.1347,
      "rewards/chosen": 2.6315655375635902,
      "rewards/margins": 24.608036125798883,
      "rewards/rejected": -21.976470588235294,
      "step": 1790
    },
    {
      "epoch": 0.8822660098522167,
      "grad_norm": 0.4372079120480066,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -52536346.256410256,
      "logits/rejected": -57212191.640449435,
      "logps/chosen": -273.64102564102564,
      "logps/rejected": -617.7078651685393,
      "loss": 0.1507,
      "rewards/chosen": 3.9407066932091346,
      "rewards/margins": 27.423852760624865,
      "rewards/rejected": -23.48314606741573,
      "step": 1791
    },
    {
      "epoch": 0.8827586206896552,
      "grad_norm": 0.5442580067983414,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -58296363.57446808,
      "logits/rejected": -58150659.16049383,
      "logps/chosen": -304.3404255319149,
      "logps/rejected": -613.925925925926,
      "loss": 0.1699,
      "rewards/chosen": 2.5345514175739696,
      "rewards/margins": 22.744427960783845,
      "rewards/rejected": -20.209876543209877,
      "step": 1792
    },
    {
      "epoch": 0.8832512315270936,
      "grad_norm": 0.40635091498006415,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -51694796.8,
      "logits/rejected": -59149218.90909091,
      "logps/chosen": -306.8,
      "logps/rejected": -597.8181818181819,
      "loss": 0.1217,
      "rewards/chosen": 5.419193267822266,
      "rewards/margins": 24.714647813276812,
      "rewards/rejected": -19.295454545454547,
      "step": 1793
    },
    {
      "epoch": 0.8837438423645321,
      "grad_norm": 0.5482433538698038,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -48272626.03636364,
      "logits/rejected": -53319371.39726027,
      "logps/chosen": -194.3272727272727,
      "logps/rejected": -589.5890410958904,
      "loss": 0.1723,
      "rewards/chosen": 0.9134943181818181,
      "rewards/margins": 19.65322034557908,
      "rewards/rejected": -18.73972602739726,
      "step": 1794
    },
    {
      "epoch": 0.8842364532019704,
      "grad_norm": 0.5915406349943187,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -89109175.5471698,
      "logits/rejected": -75329699.84,
      "logps/chosen": -343.54716981132077,
      "logps/rejected": -607.5733333333334,
      "loss": 0.2311,
      "rewards/chosen": -2.2066627358490565,
      "rewards/margins": 17.03333726415094,
      "rewards/rejected": -19.24,
      "step": 1795
    },
    {
      "epoch": 0.8847290640394089,
      "grad_norm": 0.47735397618823844,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -54811927.27272727,
      "logits/rejected": -67508321.52380952,
      "logps/chosen": -272.0,
      "logps/rejected": -612.5714285714286,
      "loss": 0.1501,
      "rewards/chosen": 3.5692440379749644,
      "rewards/margins": 21.66448213321306,
      "rewards/rejected": -18.095238095238095,
      "step": 1796
    },
    {
      "epoch": 0.8852216748768473,
      "grad_norm": 0.5006722045551156,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -70548193.28,
      "logits/rejected": -65173031.384615384,
      "logps/chosen": -335.04,
      "logps/rejected": -629.3333333333334,
      "loss": 0.1565,
      "rewards/chosen": 0.4175,
      "rewards/margins": 20.82775641025641,
      "rewards/rejected": -20.41025641025641,
      "step": 1797
    },
    {
      "epoch": 0.8857142857142857,
      "grad_norm": 1.629979958011043,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -44040192.0,
      "logits/rejected": -63300877.473684214,
      "logps/chosen": -345.53846153846155,
      "logps/rejected": -637.4736842105264,
      "loss": 0.1651,
      "rewards/chosen": 3.28009033203125,
      "rewards/margins": 22.09587980571546,
      "rewards/rejected": -18.81578947368421,
      "step": 1798
    },
    {
      "epoch": 0.8862068965517241,
      "grad_norm": 0.6224614395720679,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -67013538.90909091,
      "logits/rejected": -61965848.38095238,
      "logps/chosen": -326.1818181818182,
      "logps/rejected": -625.5238095238095,
      "loss": 0.1396,
      "rewards/chosen": 4.259958440607244,
      "rewards/margins": 24.617101297750104,
      "rewards/rejected": -20.357142857142858,
      "step": 1799
    },
    {
      "epoch": 0.8866995073891626,
      "grad_norm": 0.4765279674175741,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -51292842.666666664,
      "logits/rejected": -58405683.2,
      "logps/chosen": -220.33333333333334,
      "logps/rejected": -607.2,
      "loss": 0.1361,
      "rewards/chosen": 1.2421875,
      "rewards/margins": 21.4671875,
      "rewards/rejected": -20.225,
      "step": 1800
    },
    {
      "epoch": 0.8871921182266009,
      "grad_norm": 1.2343518992568845,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -50153166.97872341,
      "logits/rejected": -45464184.09876543,
      "logps/chosen": -392.8510638297872,
      "logps/rejected": -542.8148148148148,
      "loss": 0.2075,
      "rewards/chosen": 0.7832920804936835,
      "rewards/margins": 17.375884673086276,
      "rewards/rejected": -16.59259259259259,
      "step": 1801
    },
    {
      "epoch": 0.8876847290640394,
      "grad_norm": 0.5998692394778624,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -44296986.12244898,
      "logits/rejected": -48951244.151898734,
      "logps/chosen": -258.44897959183675,
      "logps/rejected": -571.9493670886076,
      "loss": 0.1935,
      "rewards/chosen": -0.7630739795918368,
      "rewards/margins": 16.021736146990442,
      "rewards/rejected": -16.78481012658228,
      "step": 1802
    },
    {
      "epoch": 0.8881773399014778,
      "grad_norm": 0.5986731559601783,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -39048970.24,
      "logits/rejected": -55332548.92307692,
      "logps/chosen": -241.44,
      "logps/rejected": -589.9487179487179,
      "loss": 0.1806,
      "rewards/chosen": -0.69359375,
      "rewards/margins": 17.4089703525641,
      "rewards/rejected": -18.102564102564102,
      "step": 1803
    },
    {
      "epoch": 0.8886699507389163,
      "grad_norm": 0.7580711959009379,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -53238227.0877193,
      "logits/rejected": -52635561.464788735,
      "logps/chosen": -321.4035087719298,
      "logps/rejected": -506.59154929577466,
      "loss": 0.2029,
      "rewards/chosen": 0.27110745614035087,
      "rewards/margins": 17.31336097726711,
      "rewards/rejected": -17.04225352112676,
      "step": 1804
    },
    {
      "epoch": 0.8891625615763546,
      "grad_norm": 0.8974662715347307,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -54720132.74074074,
      "logits/rejected": -51493583.567567565,
      "logps/chosen": -293.9259259259259,
      "logps/rejected": -507.2432432432432,
      "loss": 0.1728,
      "rewards/chosen": 0.3540943287037037,
      "rewards/margins": 15.016256490865866,
      "rewards/rejected": -14.662162162162161,
      "step": 1805
    },
    {
      "epoch": 0.8896551724137931,
      "grad_norm": 0.564961246107951,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -56694193.89830509,
      "logits/rejected": -48568824.57971015,
      "logps/chosen": -311.864406779661,
      "logps/rejected": -488.81159420289856,
      "loss": 0.2004,
      "rewards/chosen": 0.670021186440678,
      "rewards/margins": 15.800455969049374,
      "rewards/rejected": -15.130434782608695,
      "step": 1806
    },
    {
      "epoch": 0.8901477832512316,
      "grad_norm": 0.4801241137023098,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -50526731.906976745,
      "logits/rejected": -60496667.105882354,
      "logps/chosen": -320.74418604651163,
      "logps/rejected": -578.2588235294118,
      "loss": 0.1905,
      "rewards/chosen": 0.31976744186046513,
      "rewards/margins": 14.61388508891929,
      "rewards/rejected": -14.294117647058824,
      "step": 1807
    },
    {
      "epoch": 0.8906403940886699,
      "grad_norm": 0.6584494386939941,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -62875723.85185185,
      "logits/rejected": -53335676.54054054,
      "logps/chosen": -308.74074074074076,
      "logps/rejected": -518.0540540540541,
      "loss": 0.2036,
      "rewards/chosen": 2.024893866644965,
      "rewards/margins": 15.065434407185506,
      "rewards/rejected": -13.04054054054054,
      "step": 1808
    },
    {
      "epoch": 0.8911330049261084,
      "grad_norm": 0.4658414500061395,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -41371089.45454545,
      "logits/rejected": -46884274.84931507,
      "logps/chosen": -249.6,
      "logps/rejected": -415.56164383561645,
      "loss": 0.1608,
      "rewards/chosen": 3.1713170831853694,
      "rewards/margins": 13.253508864007287,
      "rewards/rejected": -10.082191780821917,
      "step": 1809
    },
    {
      "epoch": 0.8916256157635468,
      "grad_norm": 0.5087841862771718,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -51128565.76,
      "logits/rejected": -65872082.05128205,
      "logps/chosen": -341.12,
      "logps/rejected": -515.6923076923077,
      "loss": 0.1839,
      "rewards/chosen": 0.15328125,
      "rewards/margins": 10.56353766025641,
      "rewards/rejected": -10.41025641025641,
      "step": 1810
    },
    {
      "epoch": 0.8921182266009853,
      "grad_norm": 0.608824143564207,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -61465618.61818182,
      "logits/rejected": -66764126.68493151,
      "logps/chosen": -248.72727272727272,
      "logps/rejected": -490.52054794520546,
      "loss": 0.1396,
      "rewards/chosen": 2.409090909090909,
      "rewards/margins": 13.422789539227896,
      "rewards/rejected": -11.013698630136986,
      "step": 1811
    },
    {
      "epoch": 0.8926108374384236,
      "grad_norm": 0.6504573177942782,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -77441173.85365854,
      "logits/rejected": -62673508.04597701,
      "logps/chosen": -224.0,
      "logps/rejected": -471.5402298850575,
      "loss": 0.1496,
      "rewards/chosen": 3.7962959103467986,
      "rewards/margins": 13.922732691955993,
      "rewards/rejected": -10.126436781609195,
      "step": 1812
    },
    {
      "epoch": 0.8931034482758621,
      "grad_norm": 0.6489250516658763,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -35693527.04,
      "logits/rejected": -57644793.43589743,
      "logps/chosen": -265.6,
      "logps/rejected": -482.46153846153845,
      "loss": 0.1996,
      "rewards/chosen": 0.75,
      "rewards/margins": 9.967948717948717,
      "rewards/rejected": -9.217948717948717,
      "step": 1813
    },
    {
      "epoch": 0.8935960591133005,
      "grad_norm": 0.6749688212463996,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -52708420.266666666,
      "logits/rejected": -47299621.01204819,
      "logps/chosen": -310.93333333333334,
      "logps/rejected": -432.578313253012,
      "loss": 0.1583,
      "rewards/chosen": 3.5944678412543403,
      "rewards/margins": 12.980010009929039,
      "rewards/rejected": -9.385542168674698,
      "step": 1814
    },
    {
      "epoch": 0.8940886699507389,
      "grad_norm": 0.465910505659725,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -44657001.4117647,
      "logits/rejected": -67108864.0,
      "logps/chosen": -193.72549019607843,
      "logps/rejected": -421.8181818181818,
      "loss": 0.1974,
      "rewards/chosen": 0.4323299632352941,
      "rewards/margins": 9.666096197001528,
      "rewards/rejected": -9.233766233766234,
      "step": 1815
    },
    {
      "epoch": 0.8945812807881773,
      "grad_norm": 0.48554926541323634,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -57016320.0,
      "logits/rejected": -54001664.0,
      "logps/chosen": -260.5,
      "logps/rejected": -454.4,
      "loss": 0.1435,
      "rewards/chosen": 1.2194010416666667,
      "rewards/margins": 11.669401041666665,
      "rewards/rejected": -10.45,
      "step": 1816
    },
    {
      "epoch": 0.8950738916256158,
      "grad_norm": 0.4895282138488156,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -48949434.18181818,
      "logits/rejected": -62165577.14285714,
      "logps/chosen": -280.90909090909093,
      "logps/rejected": -503.6190476190476,
      "loss": 0.1628,
      "rewards/chosen": 3.329581520774148,
      "rewards/margins": 14.936724377917006,
      "rewards/rejected": -11.607142857142858,
      "step": 1817
    },
    {
      "epoch": 0.8955665024630541,
      "grad_norm": 0.5157600475326192,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -59186289.777777776,
      "logits/rejected": -56648370.89156626,
      "logps/chosen": -329.4222222222222,
      "logps/rejected": -524.3373493975904,
      "loss": 0.1801,
      "rewards/chosen": 2.5465162489149304,
      "rewards/margins": 14.209166851324568,
      "rewards/rejected": -11.662650602409638,
      "step": 1818
    },
    {
      "epoch": 0.8960591133004926,
      "grad_norm": 0.5120591417084392,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -38497718.85714286,
      "logits/rejected": -50782648.43010753,
      "logps/chosen": -242.74285714285713,
      "logps/rejected": -450.06451612903226,
      "loss": 0.1222,
      "rewards/chosen": 1.1357142857142857,
      "rewards/margins": 12.458294930875576,
      "rewards/rejected": -11.32258064516129,
      "step": 1819
    },
    {
      "epoch": 0.896551724137931,
      "grad_norm": 0.4549578041488837,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -64189299.450980395,
      "logits/rejected": -53872294.233766235,
      "logps/chosen": -259.45098039215685,
      "logps/rejected": -537.3506493506494,
      "loss": 0.1736,
      "rewards/chosen": 2.2248711679496016,
      "rewards/margins": 14.484611427689861,
      "rewards/rejected": -12.25974025974026,
      "step": 1820
    },
    {
      "epoch": 0.8970443349753695,
      "grad_norm": 0.4648367858526113,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -49378397.09090909,
      "logits/rejected": -54870689.31506849,
      "logps/chosen": -262.1090909090909,
      "logps/rejected": -560.2191780821918,
      "loss": 0.1756,
      "rewards/chosen": 0.6522727272727272,
      "rewards/margins": 13.953642590286426,
      "rewards/rejected": -13.301369863013699,
      "step": 1821
    },
    {
      "epoch": 0.8975369458128079,
      "grad_norm": 0.5374120887836898,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -65145572.765957445,
      "logits/rejected": -55483910.32098766,
      "logps/chosen": -298.72340425531917,
      "logps/rejected": -572.8395061728395,
      "loss": 0.1471,
      "rewards/chosen": 6.554523873836436,
      "rewards/margins": 19.344647330626557,
      "rewards/rejected": -12.790123456790123,
      "step": 1822
    },
    {
      "epoch": 0.8980295566502463,
      "grad_norm": 0.4771798894412942,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -46137344.0,
      "logits/rejected": -50694118.71604938,
      "logps/chosen": -304.6808510638298,
      "logps/rejected": -503.7037037037037,
      "loss": 0.1299,
      "rewards/chosen": 5.6156862948803195,
      "rewards/margins": 20.220624566485256,
      "rewards/rejected": -14.604938271604938,
      "step": 1823
    },
    {
      "epoch": 0.8985221674876848,
      "grad_norm": 0.40854280643612234,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -47185920.0,
      "logits/rejected": -55538370.20689655,
      "logps/chosen": -381.6585365853659,
      "logps/rejected": -528.183908045977,
      "loss": 0.1232,
      "rewards/chosen": 8.072416724228278,
      "rewards/margins": 22.2218420115846,
      "rewards/rejected": -14.149425287356323,
      "step": 1824
    },
    {
      "epoch": 0.8990147783251231,
      "grad_norm": 0.6484479590100161,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -50094234.56603774,
      "logits/rejected": -52960078.50666667,
      "logps/chosen": -230.33962264150944,
      "logps/rejected": -503.46666666666664,
      "loss": 0.186,
      "rewards/chosen": 0.5238797169811321,
      "rewards/margins": 14.737213050314464,
      "rewards/rejected": -14.213333333333333,
      "step": 1825
    },
    {
      "epoch": 0.8995073891625616,
      "grad_norm": 0.5255019114008268,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -49968679.384615384,
      "logits/rejected": -57726868.21052632,
      "logps/chosen": -280.9230769230769,
      "logps/rejected": -521.6842105263158,
      "loss": 0.1676,
      "rewards/chosen": 1.6204927884615385,
      "rewards/margins": 15.015229630566802,
      "rewards/rejected": -13.394736842105264,
      "step": 1826
    },
    {
      "epoch": 0.9,
      "grad_norm": 0.8090024642139444,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -61213097.056603774,
      "logits/rejected": -36818998.61333334,
      "logps/chosen": -278.9433962264151,
      "logps/rejected": -537.1733333333333,
      "loss": 0.1853,
      "rewards/chosen": 2.3510914928508253,
      "rewards/margins": 18.137758159517492,
      "rewards/rejected": -15.786666666666667,
      "step": 1827
    },
    {
      "epoch": 0.9004926108374385,
      "grad_norm": 0.46760940476454754,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -64050517.333333336,
      "logits/rejected": -47815065.6,
      "logps/chosen": -283.0,
      "logps/rejected": -482.4,
      "loss": 0.173,
      "rewards/chosen": -0.21875,
      "rewards/margins": 14.89375,
      "rewards/rejected": -15.1125,
      "step": 1828
    },
    {
      "epoch": 0.9009852216748768,
      "grad_norm": 0.5750764238367638,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -52307810.461538464,
      "logits/rejected": -49007130.94736842,
      "logps/chosen": -307.0769230769231,
      "logps/rejected": -573.0526315789474,
      "loss": 0.1683,
      "rewards/chosen": 2.8960389357346754,
      "rewards/margins": 16.211828409418885,
      "rewards/rejected": -13.31578947368421,
      "step": 1829
    },
    {
      "epoch": 0.9014778325123153,
      "grad_norm": 0.3723536404334675,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -49387929.6,
      "logits/rejected": -51713861.81818182,
      "logps/chosen": -306.8,
      "logps/rejected": -541.0909090909091,
      "loss": 0.1423,
      "rewards/chosen": 6.399203109741211,
      "rewards/margins": 23.53556674610485,
      "rewards/rejected": -17.136363636363637,
      "step": 1830
    },
    {
      "epoch": 0.9019704433497537,
      "grad_norm": 0.5163419304391055,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -35302058.666666664,
      "logits/rejected": -52315440.432432435,
      "logps/chosen": -175.7037037037037,
      "logps/rejected": -580.7567567567568,
      "loss": 0.1598,
      "rewards/chosen": 3.3755730523003473,
      "rewards/margins": 21.186383863111157,
      "rewards/rejected": -17.81081081081081,
      "step": 1831
    },
    {
      "epoch": 0.9024630541871921,
      "grad_norm": 0.5940011509976718,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -43967876.4137931,
      "logits/rejected": -54466033.37142857,
      "logps/chosen": -275.3103448275862,
      "logps/rejected": -581.4857142857143,
      "loss": 0.1725,
      "rewards/chosen": 3.4504486610149514,
      "rewards/margins": 21.621877232443524,
      "rewards/rejected": -18.17142857142857,
      "step": 1832
    },
    {
      "epoch": 0.9029556650246305,
      "grad_norm": 0.804667690104791,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -43315721.30909091,
      "logits/rejected": -66361933.15068493,
      "logps/chosen": -280.72727272727275,
      "logps/rejected": -614.5753424657535,
      "loss": 0.1973,
      "rewards/chosen": 0.9511363636363637,
      "rewards/margins": 12.028122598118385,
      "rewards/rejected": -11.076986234482021,
      "step": 1833
    },
    {
      "epoch": 0.903448275862069,
      "grad_norm": 0.5260526541616576,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -41175789.26829268,
      "logits/rejected": -69808645.88505746,
      "logps/chosen": -262.6341463414634,
      "logps/rejected": -581.8850574712644,
      "loss": 0.1438,
      "rewards/chosen": 3.7294036120903202,
      "rewards/margins": 20.855840393699516,
      "rewards/rejected": -17.126436781609197,
      "step": 1834
    },
    {
      "epoch": 0.9039408866995073,
      "grad_norm": 0.4433789423378763,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -50914190.222222224,
      "logits/rejected": -55432828.54054054,
      "logps/chosen": -271.7037037037037,
      "logps/rejected": -568.2162162162163,
      "loss": 0.1663,
      "rewards/chosen": -0.37037037037037035,
      "rewards/margins": 17.305305305305307,
      "rewards/rejected": -17.675675675675677,
      "step": 1835
    },
    {
      "epoch": 0.9044334975369458,
      "grad_norm": 0.4593249629771216,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -40087867.07692308,
      "logits/rejected": -57450927.15789474,
      "logps/chosen": -275.0769230769231,
      "logps/rejected": -532.2105263157895,
      "loss": 0.1409,
      "rewards/chosen": 1.6322115384615385,
      "rewards/margins": 15.842737854251013,
      "rewards/rejected": -14.210526315789474,
      "step": 1836
    },
    {
      "epoch": 0.9049261083743843,
      "grad_norm": 0.4482361055067557,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -43526603.75510204,
      "logits/rejected": -57764591.79746836,
      "logps/chosen": -306.9387755102041,
      "logps/rejected": -589.7721518987341,
      "loss": 0.1519,
      "rewards/chosen": -0.5,
      "rewards/margins": 18.537974683544302,
      "rewards/rejected": -19.037974683544302,
      "step": 1837
    },
    {
      "epoch": 0.9054187192118227,
      "grad_norm": 0.5120963019726775,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -50062782.35897436,
      "logits/rejected": -60511082.426966295,
      "logps/chosen": -316.71794871794873,
      "logps/rejected": -580.314606741573,
      "loss": 0.1471,
      "rewards/chosen": 3.0630045181665664,
      "rewards/margins": 18.501206765357576,
      "rewards/rejected": -15.438202247191011,
      "step": 1838
    },
    {
      "epoch": 0.905911330049261,
      "grad_norm": 0.44826342781292716,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -48144983.414634146,
      "logits/rejected": -49656702.52873563,
      "logps/chosen": -331.1219512195122,
      "logps/rejected": -521.5632183908046,
      "loss": 0.1683,
      "rewards/chosen": 2.924495790062881,
      "rewards/margins": 20.16587510040771,
      "rewards/rejected": -17.24137931034483,
      "step": 1839
    },
    {
      "epoch": 0.9064039408866995,
      "grad_norm": 0.5234852245283944,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -60436107.63636363,
      "logits/rejected": -52928121.9047619,
      "logps/chosen": -358.1818181818182,
      "logps/rejected": -525.7142857142857,
      "loss": 0.1641,
      "rewards/chosen": -0.6107954545454546,
      "rewards/margins": 15.389204545454545,
      "rewards/rejected": -16.0,
      "step": 1840
    },
    {
      "epoch": 0.906896551724138,
      "grad_norm": 0.40974221766740876,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -50688610.042553194,
      "logits/rejected": -72131672.49382716,
      "logps/chosen": -282.5531914893617,
      "logps/rejected": -544.395061728395,
      "loss": 0.1544,
      "rewards/chosen": 2.712800046230884,
      "rewards/margins": 14.629346903146393,
      "rewards/rejected": -11.91654685691551,
      "step": 1841
    },
    {
      "epoch": 0.9073891625615763,
      "grad_norm": 0.5284923994613089,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -64936813.71428572,
      "logits/rejected": -58137713.777777776,
      "logps/chosen": -273.42857142857144,
      "logps/rejected": -593.7777777777778,
      "loss": 0.181,
      "rewards/chosen": 0.8125,
      "rewards/margins": 18.71527777777778,
      "rewards/rejected": -17.90277777777778,
      "step": 1842
    },
    {
      "epoch": 0.9078817733990148,
      "grad_norm": 0.6044945246551386,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -46277154.13333333,
      "logits/rejected": -54323816.86746988,
      "logps/chosen": -259.9111111111111,
      "logps/rejected": -571.3734939759037,
      "loss": 0.179,
      "rewards/chosen": 4.02011481391059,
      "rewards/margins": 22.50204252475396,
      "rewards/rejected": -18.481927710843372,
      "step": 1843
    },
    {
      "epoch": 0.9083743842364532,
      "grad_norm": 0.4488692960522831,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -49623150.7027027,
      "logits/rejected": -55355594.54945055,
      "logps/chosen": -234.8108108108108,
      "logps/rejected": -543.6483516483516,
      "loss": 0.0993,
      "rewards/chosen": 5.548147665487753,
      "rewards/margins": 23.811883929224017,
      "rewards/rejected": -18.263736263736263,
      "step": 1844
    },
    {
      "epoch": 0.9088669950738916,
      "grad_norm": 0.5069576322081502,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -52718062.344827585,
      "logits/rejected": -64322647.77142857,
      "logps/chosen": -287.17241379310343,
      "logps/rejected": -492.8,
      "loss": 0.211,
      "rewards/chosen": 3.869145886651401,
      "rewards/margins": 17.912003029508544,
      "rewards/rejected": -14.042857142857143,
      "step": 1845
    },
    {
      "epoch": 0.90935960591133,
      "grad_norm": 0.4731319465302173,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -20621994.666666668,
      "logits/rejected": -63694895.62790698,
      "logps/chosen": -320.76190476190476,
      "logps/rejected": -601.3023255813954,
      "loss": 0.1175,
      "rewards/chosen": 5.244814191545759,
      "rewards/margins": 25.895976982243432,
      "rewards/rejected": -20.651162790697676,
      "step": 1846
    },
    {
      "epoch": 0.9098522167487685,
      "grad_norm": 0.5949864281331693,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -44766129.23076923,
      "logits/rejected": -71855050.10526316,
      "logps/chosen": -266.61538461538464,
      "logps/rejected": -618.1052631578947,
      "loss": 0.1674,
      "rewards/chosen": 3.472387460561899,
      "rewards/margins": 21.998703250035582,
      "rewards/rejected": -18.526315789473685,
      "step": 1847
    },
    {
      "epoch": 0.9103448275862069,
      "grad_norm": 0.4805248289109513,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -46561236.42553192,
      "logits/rejected": -65348291.95061728,
      "logps/chosen": -290.5531914893617,
      "logps/rejected": -592.5925925925926,
      "loss": 0.1525,
      "rewards/chosen": 1.257313829787234,
      "rewards/margins": 19.825215064355135,
      "rewards/rejected": -18.567901234567902,
      "step": 1848
    },
    {
      "epoch": 0.9108374384236453,
      "grad_norm": 0.5214840695538185,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -49347270.530612245,
      "logits/rejected": -55693986.025316454,
      "logps/chosen": -369.9591836734694,
      "logps/rejected": -541.5696202531645,
      "loss": 0.206,
      "rewards/chosen": 2.475159625617825,
      "rewards/margins": 15.71566595473175,
      "rewards/rejected": -13.240506329113924,
      "step": 1849
    },
    {
      "epoch": 0.9113300492610837,
      "grad_norm": 0.48565643263564984,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -51105596.952380955,
      "logits/rejected": -52916509.76744186,
      "logps/chosen": -369.5238095238095,
      "logps/rejected": -498.6046511627907,
      "loss": 0.1422,
      "rewards/chosen": 0.7380952380952381,
      "rewards/margins": 16.528792912513843,
      "rewards/rejected": -15.790697674418604,
      "step": 1850
    },
    {
      "epoch": 0.9118226600985222,
      "grad_norm": 0.6710993766620834,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -38284179.06382979,
      "logits/rejected": -48933546.666666664,
      "logps/chosen": -227.2340425531915,
      "logps/rejected": -538.074074074074,
      "loss": 0.126,
      "rewards/chosen": 6.7672956750748,
      "rewards/margins": 26.150011724457514,
      "rewards/rejected": -19.382716049382715,
      "step": 1851
    },
    {
      "epoch": 0.9123152709359605,
      "grad_norm": 0.4354277419895636,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -40227188.36363637,
      "logits/rejected": -55175070.47619048,
      "logps/chosen": -282.3636363636364,
      "logps/rejected": -608.0,
      "loss": 0.1453,
      "rewards/chosen": 0.4090909090909091,
      "rewards/margins": 19.21861471861472,
      "rewards/rejected": -18.80952380952381,
      "step": 1852
    },
    {
      "epoch": 0.912807881773399,
      "grad_norm": 0.4931597802050695,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -55855853.26829268,
      "logits/rejected": -51392276.59770115,
      "logps/chosen": -249.9512195121951,
      "logps/rejected": -610.5747126436781,
      "loss": 0.1793,
      "rewards/chosen": 2.928359241020389,
      "rewards/margins": 20.744451195043375,
      "rewards/rejected": -17.816091954022987,
      "step": 1853
    },
    {
      "epoch": 0.9133004926108375,
      "grad_norm": 0.5202539928782247,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -55713019.16981132,
      "logits/rejected": -49604635.306666665,
      "logps/chosen": -302.64150943396226,
      "logps/rejected": -539.7333333333333,
      "loss": 0.1829,
      "rewards/chosen": -0.08962264150943396,
      "rewards/margins": 19.297044025157234,
      "rewards/rejected": -19.386666666666667,
      "step": 1854
    },
    {
      "epoch": 0.9137931034482759,
      "grad_norm": 0.5423939298931227,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -54106521.6,
      "logits/rejected": -45231755.63636363,
      "logps/chosen": -398.0,
      "logps/rejected": -536.7272727272727,
      "loss": 0.1287,
      "rewards/chosen": 7.559459686279297,
      "rewards/margins": 26.01400514082475,
      "rewards/rejected": -18.454545454545453,
      "step": 1855
    },
    {
      "epoch": 0.9142857142857143,
      "grad_norm": 0.5419361940721461,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -48568133.81818182,
      "logits/rejected": -60717543.61904762,
      "logps/chosen": -360.72727272727275,
      "logps/rejected": -601.1428571428571,
      "loss": 0.1458,
      "rewards/chosen": 1.4822443181818181,
      "rewards/margins": 22.672720508658006,
      "rewards/rejected": -21.19047619047619,
      "step": 1856
    },
    {
      "epoch": 0.9147783251231527,
      "grad_norm": 0.45447627488958225,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -58505163.48717949,
      "logits/rejected": -62207654.83146068,
      "logps/chosen": -287.38461538461536,
      "logps/rejected": -591.8202247191011,
      "loss": 0.1344,
      "rewards/chosen": 7.7347881610576925,
      "rewards/margins": 24.611192655439716,
      "rewards/rejected": -16.876404494382022,
      "step": 1857
    },
    {
      "epoch": 0.9152709359605912,
      "grad_norm": 0.452104808970444,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -64662186.666666664,
      "logits/rejected": -63938750.511627905,
      "logps/chosen": -347.8095238095238,
      "logps/rejected": -599.0697674418604,
      "loss": 0.1631,
      "rewards/chosen": 2.346013387044271,
      "rewards/margins": 20.415780828904733,
      "rewards/rejected": -18.069767441860463,
      "step": 1858
    },
    {
      "epoch": 0.9157635467980295,
      "grad_norm": 0.4475599826533493,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -35817148.631578945,
      "logits/rejected": -52517412.05633803,
      "logps/chosen": -262.4561403508772,
      "logps/rejected": -537.2394366197183,
      "loss": 0.2195,
      "rewards/chosen": -1.462719298245614,
      "rewards/margins": 15.931646898937485,
      "rewards/rejected": -17.3943661971831,
      "step": 1859
    },
    {
      "epoch": 0.916256157635468,
      "grad_norm": 0.4658127283499692,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -32593237.333333332,
      "logits/rejected": -50541363.2,
      "logps/chosen": -267.6666666666667,
      "logps/rejected": -553.6,
      "loss": 0.1589,
      "rewards/chosen": 2.6262877782185874,
      "rewards/margins": 21.901287778218585,
      "rewards/rejected": -19.275,
      "step": 1860
    },
    {
      "epoch": 0.9167487684729064,
      "grad_norm": 0.572864978902408,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -39904644.4137931,
      "logits/rejected": -45777832.22857143,
      "logps/chosen": -229.79310344827587,
      "logps/rejected": -535.7714285714286,
      "loss": 0.1948,
      "rewards/chosen": 0.39870689655172414,
      "rewards/margins": 17.141564039408866,
      "rewards/rejected": -16.742857142857144,
      "step": 1861
    },
    {
      "epoch": 0.9172413793103448,
      "grad_norm": 0.47361347924071384,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -53101471.39622641,
      "logits/rejected": -52764344.32,
      "logps/chosen": -356.52830188679246,
      "logps/rejected": -617.3866666666667,
      "loss": 0.1464,
      "rewards/chosen": 1.5141509433962264,
      "rewards/margins": 22.047484276729563,
      "rewards/rejected": -20.533333333333335,
      "step": 1862
    },
    {
      "epoch": 0.9177339901477832,
      "grad_norm": 0.5844588892980664,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -51796325.58730159,
      "logits/rejected": -56203673.6,
      "logps/chosen": -299.1746031746032,
      "logps/rejected": -548.9230769230769,
      "loss": 0.2328,
      "rewards/chosen": 2.0823817177424355,
      "rewards/margins": 19.63622787158859,
      "rewards/rejected": -17.553846153846155,
      "step": 1863
    },
    {
      "epoch": 0.9182266009852217,
      "grad_norm": 0.5656480354986807,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -46065850.18181818,
      "logits/rejected": -46636665.9047619,
      "logps/chosen": -330.72727272727275,
      "logps/rejected": -541.7142857142857,
      "loss": 0.1624,
      "rewards/chosen": 0.19602272727272727,
      "rewards/margins": 20.053165584415584,
      "rewards/rejected": -19.857142857142858,
      "step": 1864
    },
    {
      "epoch": 0.9187192118226601,
      "grad_norm": 0.53769035660151,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -32056466.285714287,
      "logits/rejected": -63089322.666666664,
      "logps/chosen": -246.28571428571428,
      "logps/rejected": -588.4444444444445,
      "loss": 0.2178,
      "rewards/chosen": -1.6707589285714286,
      "rewards/margins": 18.60701884920635,
      "rewards/rejected": -20.27777777777778,
      "step": 1865
    },
    {
      "epoch": 0.9192118226600985,
      "grad_norm": 0.4768315591763385,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -53891929.302325584,
      "logits/rejected": -58818945.50588235,
      "logps/chosen": -334.13953488372096,
      "logps/rejected": -615.9058823529411,
      "loss": 0.1164,
      "rewards/chosen": 1.6962209302325582,
      "rewards/margins": 21.649162106703148,
      "rewards/rejected": -19.95294117647059,
      "step": 1866
    },
    {
      "epoch": 0.9197044334975369,
      "grad_norm": 0.5295438572233604,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -65011712.0,
      "logits/rejected": -43610006.974358976,
      "logps/chosen": -341.76,
      "logps/rejected": -553.025641025641,
      "loss": 0.1536,
      "rewards/chosen": 3.5937762451171875,
      "rewards/margins": 21.721981373322315,
      "rewards/rejected": -18.128205128205128,
      "step": 1867
    },
    {
      "epoch": 0.9201970443349754,
      "grad_norm": 0.5349401979067261,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -44163553.88235294,
      "logits/rejected": -49569047.27272727,
      "logps/chosen": -249.41176470588235,
      "logps/rejected": -575.1688311688312,
      "loss": 0.1751,
      "rewards/chosen": 0.35232843137254904,
      "rewards/margins": 18.910769989814106,
      "rewards/rejected": -18.558441558441558,
      "step": 1868
    },
    {
      "epoch": 0.9206896551724137,
      "grad_norm": 1.2266207543826906,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -54749053.27659574,
      "logits/rejected": -65814325.72839506,
      "logps/chosen": -326.8085106382979,
      "logps/rejected": -551.5061728395061,
      "loss": 0.18,
      "rewards/chosen": -1.2952127659574468,
      "rewards/margins": 17.24799711058576,
      "rewards/rejected": -18.54320987654321,
      "step": 1869
    },
    {
      "epoch": 0.9211822660098522,
      "grad_norm": 0.7518826382653439,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -56425259.47169811,
      "logits/rejected": -61013142.18666667,
      "logps/chosen": -414.64150943396226,
      "logps/rejected": -609.28,
      "loss": 0.1774,
      "rewards/chosen": 0.27387971698113206,
      "rewards/margins": 19.9805463836478,
      "rewards/rejected": -19.706666666666667,
      "step": 1870
    },
    {
      "epoch": 0.9216748768472907,
      "grad_norm": 0.3474673550227792,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -52803291.428571425,
      "logits/rejected": -59011527.11111111,
      "logps/chosen": -272.42857142857144,
      "logps/rejected": -691.5555555555555,
      "loss": 0.1362,
      "rewards/chosen": 3.8091812133789062,
      "rewards/margins": 26.920292324490017,
      "rewards/rejected": -23.11111111111111,
      "step": 1871
    },
    {
      "epoch": 0.9221674876847291,
      "grad_norm": 0.5703371462645939,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -50252510.18867925,
      "logits/rejected": -52848230.4,
      "logps/chosen": -300.07547169811323,
      "logps/rejected": -581.9733333333334,
      "loss": 0.2084,
      "rewards/chosen": 1.395963560860112,
      "rewards/margins": 19.70263022752678,
      "rewards/rejected": -18.30666666666667,
      "step": 1872
    },
    {
      "epoch": 0.9226600985221675,
      "grad_norm": 0.6713042064311805,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -45855035.07692308,
      "logits/rejected": -61038160.84210526,
      "logps/chosen": -306.9230769230769,
      "logps/rejected": -617.2631578947369,
      "loss": 0.1891,
      "rewards/chosen": -0.27696814903846156,
      "rewards/margins": 18.117768693066804,
      "rewards/rejected": -18.394736842105264,
      "step": 1873
    },
    {
      "epoch": 0.9231527093596059,
      "grad_norm": 0.5203368399433665,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -48013743.15789474,
      "logits/rejected": -49918125.07042254,
      "logps/chosen": -321.4035087719298,
      "logps/rejected": -630.9859154929577,
      "loss": 0.1878,
      "rewards/chosen": 0.3092105263157895,
      "rewards/margins": 19.87259080800593,
      "rewards/rejected": -19.56338028169014,
      "step": 1874
    },
    {
      "epoch": 0.9236453201970444,
      "grad_norm": 0.522001803899193,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -45950930.48888889,
      "logits/rejected": -65794985.638554215,
      "logps/chosen": -270.5777777777778,
      "logps/rejected": -592.9638554216867,
      "loss": 0.1161,
      "rewards/chosen": 5.350723605685764,
      "rewards/margins": 22.91698866592673,
      "rewards/rejected": -17.566265060240966,
      "step": 1875
    },
    {
      "epoch": 0.9241379310344827,
      "grad_norm": 0.48373926308941395,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -38285216.744186044,
      "logits/rejected": -59805840.564705886,
      "logps/chosen": -236.65116279069767,
      "logps/rejected": -624.9411764705883,
      "loss": 0.1279,
      "rewards/chosen": 4.7274524777434594,
      "rewards/margins": 26.303923065978754,
      "rewards/rejected": -21.576470588235296,
      "step": 1876
    },
    {
      "epoch": 0.9246305418719212,
      "grad_norm": 0.47117333599397615,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -58452534.4680851,
      "logits/rejected": -46448033.18518519,
      "logps/chosen": -257.02127659574467,
      "logps/rejected": -516.3456790123457,
      "loss": 0.1333,
      "rewards/chosen": 3.598436396172706,
      "rewards/margins": 13.827133619600561,
      "rewards/rejected": -10.228697223427854,
      "step": 1877
    },
    {
      "epoch": 0.9251231527093596,
      "grad_norm": 0.5262616685566271,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -40512170.666666664,
      "logits/rejected": -45848985.6,
      "logps/chosen": -225.83333333333334,
      "logps/rejected": -550.0,
      "loss": 0.197,
      "rewards/chosen": 5.469320933024089,
      "rewards/margins": 22.49432093302409,
      "rewards/rejected": -17.025,
      "step": 1878
    },
    {
      "epoch": 0.925615763546798,
      "grad_norm": 0.532123012395963,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -61630589.3877551,
      "logits/rejected": -62702190.17721519,
      "logps/chosen": -304.0,
      "logps/rejected": -586.5316455696203,
      "loss": 0.1445,
      "rewards/chosen": 2.0462372448979593,
      "rewards/margins": 18.286743574011883,
      "rewards/rejected": -16.240506329113924,
      "step": 1879
    },
    {
      "epoch": 0.9261083743842364,
      "grad_norm": 0.5708812729069275,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -37655529.244444445,
      "logits/rejected": -51039120.96385542,
      "logps/chosen": -288.0,
      "logps/rejected": -550.5542168674699,
      "loss": 0.127,
      "rewards/chosen": 4.5798760308159725,
      "rewards/margins": 20.79674350069549,
      "rewards/rejected": -16.216867469879517,
      "step": 1880
    },
    {
      "epoch": 0.9266009852216749,
      "grad_norm": 0.4424972349605966,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -58205166.03508772,
      "logits/rejected": -53344457.91549296,
      "logps/chosen": -266.10526315789474,
      "logps/rejected": -568.7887323943662,
      "loss": 0.1698,
      "rewards/chosen": 3.9183542351973686,
      "rewards/margins": 22.52398803801427,
      "rewards/rejected": -18.6056338028169,
      "step": 1881
    },
    {
      "epoch": 0.9270935960591133,
      "grad_norm": 0.4433100779371766,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -69968616.72727273,
      "logits/rejected": -49083343.23809524,
      "logps/chosen": -297.09090909090907,
      "logps/rejected": -509.7142857142857,
      "loss": 0.1674,
      "rewards/chosen": 1.9556076743386008,
      "rewards/margins": 18.550845769576696,
      "rewards/rejected": -16.595238095238095,
      "step": 1882
    },
    {
      "epoch": 0.9275862068965517,
      "grad_norm": 0.5101988101187446,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -42030421.333333336,
      "logits/rejected": -55574528.0,
      "logps/chosen": -256.6666666666667,
      "logps/rejected": -536.8,
      "loss": 0.1568,
      "rewards/chosen": 1.435546875,
      "rewards/margins": 20.835546875,
      "rewards/rejected": -19.4,
      "step": 1883
    },
    {
      "epoch": 0.9280788177339901,
      "grad_norm": 0.55450894242812,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -49988477.67272727,
      "logits/rejected": -56910385.09589041,
      "logps/chosen": -261.5272727272727,
      "logps/rejected": -552.3287671232877,
      "loss": 0.2044,
      "rewards/chosen": 0.28494318181818185,
      "rewards/margins": 16.723299346201745,
      "rewards/rejected": -16.438356164383563,
      "step": 1884
    },
    {
      "epoch": 0.9285714285714286,
      "grad_norm": 0.5163932486355676,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -51044679.68,
      "logits/rejected": -59365533.538461536,
      "logps/chosen": -325.76,
      "logps/rejected": -562.0512820512821,
      "loss": 0.152,
      "rewards/chosen": 2.115,
      "rewards/margins": 18.52525641025641,
      "rewards/rejected": -16.41025641025641,
      "step": 1885
    },
    {
      "epoch": 0.929064039408867,
      "grad_norm": 0.45233594580007347,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -43287952.69565217,
      "logits/rejected": -51866149.46341463,
      "logps/chosen": -245.2173913043478,
      "logps/rejected": -533.8536585365854,
      "loss": 0.1164,
      "rewards/chosen": 4.053007042926291,
      "rewards/margins": 22.662763140487264,
      "rewards/rejected": -18.609756097560975,
      "step": 1886
    },
    {
      "epoch": 0.9295566502463054,
      "grad_norm": 0.5162073435653659,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -42189763.76470588,
      "logits/rejected": -54253594.597402595,
      "logps/chosen": -317.4901960784314,
      "logps/rejected": -526.1298701298701,
      "loss": 0.1736,
      "rewards/chosen": 0.3235294117647059,
      "rewards/margins": 15.167685255920551,
      "rewards/rejected": -14.844155844155845,
      "step": 1887
    },
    {
      "epoch": 0.9300492610837439,
      "grad_norm": 0.604619026578607,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -30241885.09090909,
      "logits/rejected": -72002218.66666667,
      "logps/chosen": -245.8181818181818,
      "logps/rejected": -497.5238095238095,
      "loss": 0.1248,
      "rewards/chosen": 2.3607954545454546,
      "rewards/margins": 17.289366883116884,
      "rewards/rejected": -14.928571428571429,
      "step": 1888
    },
    {
      "epoch": 0.9305418719211823,
      "grad_norm": 0.4679706715619695,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -47102033.92,
      "logits/rejected": -58935348.51282051,
      "logps/chosen": -297.28,
      "logps/rejected": -589.9487179487179,
      "loss": 0.1761,
      "rewards/chosen": 0.3571875,
      "rewards/margins": 18.4597516025641,
      "rewards/rejected": -18.102564102564102,
      "step": 1889
    },
    {
      "epoch": 0.9310344827586207,
      "grad_norm": 0.5619531423968422,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -59139686.4,
      "logits/rejected": -56945742.76923077,
      "logps/chosen": -263.36,
      "logps/rejected": -550.974358974359,
      "loss": 0.1702,
      "rewards/chosen": 1.37625,
      "rewards/margins": 17.132660256410258,
      "rewards/rejected": -15.756410256410257,
      "step": 1890
    },
    {
      "epoch": 0.9315270935960591,
      "grad_norm": 0.47879235049218954,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -44692639.28888889,
      "logits/rejected": -55385026.313253015,
      "logps/chosen": -262.75555555555553,
      "logps/rejected": -608.3855421686746,
      "loss": 0.1414,
      "rewards/chosen": 4.453457302517361,
      "rewards/margins": 23.89924043504748,
      "rewards/rejected": -19.44578313253012,
      "step": 1891
    },
    {
      "epoch": 0.9320197044334976,
      "grad_norm": 0.48478108390125785,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -54777610.24,
      "logits/rejected": -60871181.12820513,
      "logps/chosen": -302.4,
      "logps/rejected": -556.3076923076923,
      "loss": 0.1468,
      "rewards/chosen": 2.849417419433594,
      "rewards/margins": 22.849417419433593,
      "rewards/rejected": -20.0,
      "step": 1892
    },
    {
      "epoch": 0.9325123152709359,
      "grad_norm": 0.6237926509549611,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -52387679.37254902,
      "logits/rejected": -57957655.27272727,
      "logps/chosen": -346.6666666666667,
      "logps/rejected": -643.3246753246754,
      "loss": 0.1598,
      "rewards/chosen": -0.0024509803921568627,
      "rewards/margins": 18.750795772854595,
      "rewards/rejected": -18.753246753246753,
      "step": 1893
    },
    {
      "epoch": 0.9330049261083744,
      "grad_norm": 0.4573879304924823,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -67276636.16,
      "logits/rejected": -51245791.179487176,
      "logps/chosen": -353.28,
      "logps/rejected": -555.4871794871794,
      "loss": 0.1304,
      "rewards/chosen": 1.175,
      "rewards/margins": 20.867307692307694,
      "rewards/rejected": -19.692307692307693,
      "step": 1894
    },
    {
      "epoch": 0.9334975369458128,
      "grad_norm": 0.43746104682671766,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -56930004.29268292,
      "logits/rejected": -53176061.05747127,
      "logps/chosen": -283.7073170731707,
      "logps/rejected": -547.3103448275862,
      "loss": 0.135,
      "rewards/chosen": 0.7044588414634146,
      "rewards/margins": 17.80790711732548,
      "rewards/rejected": -17.103448275862068,
      "step": 1895
    },
    {
      "epoch": 0.9339901477832512,
      "grad_norm": 0.5452694585816943,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -78929175.27272727,
      "logits/rejected": -70304524.1904762,
      "logps/chosen": -344.72727272727275,
      "logps/rejected": -616.3809523809524,
      "loss": 0.1837,
      "rewards/chosen": -2.028409090909091,
      "rewards/margins": 14.923971861471863,
      "rewards/rejected": -16.952380952380953,
      "step": 1896
    },
    {
      "epoch": 0.9344827586206896,
      "grad_norm": 0.561721555296582,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -62670705.11627907,
      "logits/rejected": -58522876.988235295,
      "logps/chosen": -303.6279069767442,
      "logps/rejected": -607.6235294117647,
      "loss": 0.1788,
      "rewards/chosen": -0.6315406976744186,
      "rewards/margins": 17.368459302325583,
      "rewards/rejected": -18.0,
      "step": 1897
    },
    {
      "epoch": 0.9349753694581281,
      "grad_norm": 0.47608359224211066,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -51170508.8,
      "logits/rejected": -66132108.2739726,
      "logps/chosen": -360.43636363636364,
      "logps/rejected": -593.5342465753424,
      "loss": 0.1761,
      "rewards/chosen": 0.8727272727272727,
      "rewards/margins": 19.119302615193025,
      "rewards/rejected": -18.246575342465754,
      "step": 1898
    },
    {
      "epoch": 0.9354679802955665,
      "grad_norm": 0.5628481366147359,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -50172082.08695652,
      "logits/rejected": -53298350.82926829,
      "logps/chosen": -312.60869565217394,
      "logps/rejected": -557.2682926829268,
      "loss": 0.1444,
      "rewards/chosen": 3.8852127738620923,
      "rewards/margins": 20.604724968984044,
      "rewards/rejected": -16.71951219512195,
      "step": 1899
    },
    {
      "epoch": 0.9359605911330049,
      "grad_norm": 0.4165652270473248,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -47797589.333333336,
      "logits/rejected": -51642368.0,
      "logps/chosen": -238.0,
      "logps/rejected": -517.6,
      "loss": 0.1188,
      "rewards/chosen": 6.103560129801433,
      "rewards/margins": 24.278560129801434,
      "rewards/rejected": -18.175,
      "step": 1900
    },
    {
      "epoch": 0.9364532019704433,
      "grad_norm": 0.5806344100950914,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -64914170.04651163,
      "logits/rejected": -58325497.97647059,
      "logps/chosen": -378.7906976744186,
      "logps/rejected": -577.5058823529412,
      "loss": 0.1582,
      "rewards/chosen": 4.0966541378997094,
      "rewards/margins": 20.86136002025265,
      "rewards/rejected": -16.764705882352942,
      "step": 1901
    },
    {
      "epoch": 0.9369458128078818,
      "grad_norm": 0.4863418937171279,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -68735226.7755102,
      "logits/rejected": -57446037.06329114,
      "logps/chosen": -298.1224489795918,
      "logps/rejected": -582.4810126582279,
      "loss": 0.132,
      "rewards/chosen": 2.239795918367347,
      "rewards/margins": 20.29042882975975,
      "rewards/rejected": -18.050632911392405,
      "step": 1902
    },
    {
      "epoch": 0.9374384236453202,
      "grad_norm": 0.5818898335208178,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -42547987.692307696,
      "logits/rejected": -49172695.578947365,
      "logps/chosen": -285.2307692307692,
      "logps/rejected": -544.8421052631579,
      "loss": 0.2005,
      "rewards/chosen": 0.09945913461538461,
      "rewards/margins": 16.546827555668013,
      "rewards/rejected": -16.44736842105263,
      "step": 1903
    },
    {
      "epoch": 0.9379310344827586,
      "grad_norm": 0.5475802150957078,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -54254098.96296296,
      "logits/rejected": -75214073.08108108,
      "logps/chosen": -313.48148148148147,
      "logps/rejected": -612.3243243243244,
      "loss": 0.1845,
      "rewards/chosen": 0.3587962962962963,
      "rewards/margins": 20.22366116116116,
      "rewards/rejected": -19.864864864864863,
      "step": 1904
    },
    {
      "epoch": 0.9384236453201971,
      "grad_norm": 0.48275474966907445,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -46967466.666666664,
      "logits/rejected": -54368665.6,
      "logps/chosen": -288.8333333333333,
      "logps/rejected": -544.8,
      "loss": 0.1696,
      "rewards/chosen": 1.5592684745788574,
      "rewards/margins": 20.009268474578857,
      "rewards/rejected": -18.45,
      "step": 1905
    },
    {
      "epoch": 0.9389162561576355,
      "grad_norm": 0.5682581780761358,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -55841812.078431375,
      "logits/rejected": -35025161.97402597,
      "logps/chosen": -433.5686274509804,
      "logps/rejected": -531.9480519480519,
      "loss": 0.1724,
      "rewards/chosen": 0.19362745098039216,
      "rewards/margins": 18.115705373058315,
      "rewards/rejected": -17.92207792207792,
      "step": 1906
    },
    {
      "epoch": 0.9394088669950739,
      "grad_norm": 0.5000685215671948,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -52582250.146341465,
      "logits/rejected": -55924053.333333336,
      "logps/chosen": -339.3170731707317,
      "logps/rejected": -581.1494252873563,
      "loss": 0.1441,
      "rewards/chosen": -0.1951219512195122,
      "rewards/margins": 20.080740117746004,
      "rewards/rejected": -20.275862068965516,
      "step": 1907
    },
    {
      "epoch": 0.9399014778325123,
      "grad_norm": 0.5645670122778423,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -53989471.255813956,
      "logits/rejected": -60003219.57647059,
      "logps/chosen": -219.90697674418604,
      "logps/rejected": -601.6,
      "loss": 0.1468,
      "rewards/chosen": 1.010174418604651,
      "rewards/margins": 17.457233242134063,
      "rewards/rejected": -16.44705882352941,
      "step": 1908
    },
    {
      "epoch": 0.9403940886699508,
      "grad_norm": 0.5290984990206257,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -58921047.14893617,
      "logits/rejected": -67108864.0,
      "logps/chosen": -285.78723404255317,
      "logps/rejected": -558.6172839506173,
      "loss": 0.1383,
      "rewards/chosen": 0.6888297872340425,
      "rewards/margins": 13.276730207999861,
      "rewards/rejected": -12.587900420765818,
      "step": 1909
    },
    {
      "epoch": 0.9408866995073891,
      "grad_norm": 0.472614005563421,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -50415534.08,
      "logits/rejected": -56784423.384615384,
      "logps/chosen": -278.08,
      "logps/rejected": -543.1794871794872,
      "loss": 0.1877,
      "rewards/chosen": 6.06567138671875,
      "rewards/margins": 23.60413292518029,
      "rewards/rejected": -17.53846153846154,
      "step": 1910
    },
    {
      "epoch": 0.9413793103448276,
      "grad_norm": 0.466148002651102,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -42710290.73170732,
      "logits/rejected": -60070146.94252873,
      "logps/chosen": -215.02439024390245,
      "logps/rejected": -618.6666666666666,
      "loss": 0.1122,
      "rewards/chosen": 2.997720299697504,
      "rewards/margins": 24.537950184754976,
      "rewards/rejected": -21.54022988505747,
      "step": 1911
    },
    {
      "epoch": 0.941871921182266,
      "grad_norm": 0.4080807681769372,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -51170508.8,
      "logits/rejected": -56749438.45783132,
      "logps/chosen": -254.22222222222223,
      "logps/rejected": -559.4216867469879,
      "loss": 0.1638,
      "rewards/chosen": 2.1437859429253474,
      "rewards/margins": 23.083544979069927,
      "rewards/rejected": -20.93975903614458,
      "step": 1912
    },
    {
      "epoch": 0.9423645320197044,
      "grad_norm": 0.4630369329894559,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -59292206.54545455,
      "logits/rejected": -52378867.809523806,
      "logps/chosen": -217.63636363636363,
      "logps/rejected": -580.5714285714286,
      "loss": 0.167,
      "rewards/chosen": 1.9062640450217507,
      "rewards/margins": 19.59674023549794,
      "rewards/rejected": -17.69047619047619,
      "step": 1913
    },
    {
      "epoch": 0.9428571428571428,
      "grad_norm": 0.5258808049626176,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -52756480.0,
      "logits/rejected": -60869836.8,
      "logps/chosen": -289.8333333333333,
      "logps/rejected": -642.4,
      "loss": 0.1437,
      "rewards/chosen": 3.7975165049235025,
      "rewards/margins": 25.422516504923504,
      "rewards/rejected": -21.625,
      "step": 1914
    },
    {
      "epoch": 0.9433497536945813,
      "grad_norm": 0.4678777397101576,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -42657978.18181818,
      "logits/rejected": -48084699.428571425,
      "logps/chosen": -308.72727272727275,
      "logps/rejected": -531.047619047619,
      "loss": 0.1557,
      "rewards/chosen": 2.832573110407049,
      "rewards/margins": 20.761144538978478,
      "rewards/rejected": -17.928571428571427,
      "step": 1915
    },
    {
      "epoch": 0.9438423645320198,
      "grad_norm": 0.5362445306709624,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -61031403.10204082,
      "logits/rejected": -51393497.11392405,
      "logps/chosen": -317.55102040816325,
      "logps/rejected": -569.5189873417721,
      "loss": 0.1916,
      "rewards/chosen": 0.29591836734693877,
      "rewards/margins": 19.992120898992507,
      "rewards/rejected": -19.696202531645568,
      "step": 1916
    },
    {
      "epoch": 0.9443349753694581,
      "grad_norm": 0.5064246492145367,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -46394888.98245614,
      "logits/rejected": -64686801.126760565,
      "logps/chosen": -282.94736842105266,
      "logps/rejected": -655.3239436619718,
      "loss": 0.1994,
      "rewards/chosen": -0.005482456140350877,
      "rewards/margins": 20.755080924141343,
      "rewards/rejected": -20.760563380281692,
      "step": 1917
    },
    {
      "epoch": 0.9448275862068966,
      "grad_norm": 0.4957494178428111,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -57218885.81818182,
      "logits/rejected": -52378867.809523806,
      "logps/chosen": -355.95454545454544,
      "logps/rejected": -553.1428571428571,
      "loss": 0.1665,
      "rewards/chosen": 4.987898046320135,
      "rewards/margins": 24.487898046320133,
      "rewards/rejected": -19.5,
      "step": 1918
    },
    {
      "epoch": 0.945320197044335,
      "grad_norm": 0.5532227115366949,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -59789392.313725494,
      "logits/rejected": -63949518.12987013,
      "logps/chosen": -233.5686274509804,
      "logps/rejected": -581.8181818181819,
      "loss": 0.1677,
      "rewards/chosen": -0.5392156862745098,
      "rewards/margins": 20.16208301502419,
      "rewards/rejected": -20.7012987012987,
      "step": 1919
    },
    {
      "epoch": 0.9458128078817734,
      "grad_norm": 0.5734591828788822,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -72142028.8,
      "logits/rejected": -60064584.20512821,
      "logps/chosen": -245.76,
      "logps/rejected": -558.7692307692307,
      "loss": 0.1755,
      "rewards/chosen": 0.215,
      "rewards/margins": 19.93294871794872,
      "rewards/rejected": -19.71794871794872,
      "step": 1920
    },
    {
      "epoch": 0.9463054187192118,
      "grad_norm": 0.6217612531498912,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -62836887.7037037,
      "logits/rejected": -59797171.89189189,
      "logps/chosen": -322.962962962963,
      "logps/rejected": -603.6756756756756,
      "loss": 0.1837,
      "rewards/chosen": 0.16203703703703703,
      "rewards/margins": 20.567442442442445,
      "rewards/rejected": -20.405405405405407,
      "step": 1921
    },
    {
      "epoch": 0.9467980295566503,
      "grad_norm": 0.7100899716087944,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -59308930.24561404,
      "logits/rejected": -62501037.07042254,
      "logps/chosen": -346.94736842105266,
      "logps/rejected": -589.5211267605633,
      "loss": 0.2235,
      "rewards/chosen": 4.576221131441886,
      "rewards/margins": 23.618474652568647,
      "rewards/rejected": -19.04225352112676,
      "step": 1922
    },
    {
      "epoch": 0.9472906403940887,
      "grad_norm": 0.4538247145010092,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -39895820.190476194,
      "logits/rejected": -54379639.069767445,
      "logps/chosen": -227.04761904761904,
      "logps/rejected": -569.3023255813954,
      "loss": 0.1131,
      "rewards/chosen": 1.7529761904761905,
      "rewards/margins": 19.845999446290143,
      "rewards/rejected": -18.093023255813954,
      "step": 1923
    },
    {
      "epoch": 0.9477832512315271,
      "grad_norm": 0.46111966473642196,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -46094544.97959184,
      "logits/rejected": -62861467.5443038,
      "logps/chosen": -342.2040816326531,
      "logps/rejected": -641.620253164557,
      "loss": 0.1338,
      "rewards/chosen": 3.0349451181839924,
      "rewards/margins": 20.337543516127027,
      "rewards/rejected": -17.302598397943036,
      "step": 1924
    },
    {
      "epoch": 0.9482758620689655,
      "grad_norm": 0.6186288974298495,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -47617686.5882353,
      "logits/rejected": -56377982.33766234,
      "logps/chosen": -282.03921568627453,
      "logps/rejected": -540.2597402597403,
      "loss": 0.1992,
      "rewards/chosen": -0.09681312710631128,
      "rewards/margins": 17.149940119646935,
      "rewards/rejected": -17.246753246753247,
      "step": 1925
    },
    {
      "epoch": 0.948768472906404,
      "grad_norm": 0.35868747108666743,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -76164747.63636364,
      "logits/rejected": -59319442.28571428,
      "logps/chosen": -336.0,
      "logps/rejected": -652.952380952381,
      "loss": 0.189,
      "rewards/chosen": -2.2049005681818183,
      "rewards/margins": 17.390337527056275,
      "rewards/rejected": -19.595238095238095,
      "step": 1926
    },
    {
      "epoch": 0.9492610837438423,
      "grad_norm": 0.5301288369622242,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -52894833.777777776,
      "logits/rejected": -63116695.13253012,
      "logps/chosen": -319.2888888888889,
      "logps/rejected": -594.5060240963855,
      "loss": 0.1703,
      "rewards/chosen": 1.970095486111111,
      "rewards/margins": 22.76527620900268,
      "rewards/rejected": -20.795180722891565,
      "step": 1927
    },
    {
      "epoch": 0.9497536945812808,
      "grad_norm": 0.5109217206616813,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -55665708.52173913,
      "logits/rejected": -59896707.12195122,
      "logps/chosen": -356.5217391304348,
      "logps/rejected": -573.6585365853658,
      "loss": 0.1384,
      "rewards/chosen": 4.559150364087976,
      "rewards/margins": 26.266467437258708,
      "rewards/rejected": -21.70731707317073,
      "step": 1928
    },
    {
      "epoch": 0.9502463054187192,
      "grad_norm": 0.6784364942673891,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -50129998.76923077,
      "logits/rejected": -62583430.7368421,
      "logps/chosen": -308.3076923076923,
      "logps/rejected": -539.7894736842105,
      "loss": 0.1991,
      "rewards/chosen": 0.2824519230769231,
      "rewards/margins": 18.650872975708502,
      "rewards/rejected": -18.36842105263158,
      "step": 1929
    },
    {
      "epoch": 0.9507389162561576,
      "grad_norm": 0.4491612160837105,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -48334360.38095238,
      "logits/rejected": -47210305.488372095,
      "logps/chosen": -364.0,
      "logps/rejected": -566.3255813953489,
      "loss": 0.1617,
      "rewards/chosen": 0.9523936680385044,
      "rewards/margins": 18.952393668038503,
      "rewards/rejected": -18.0,
      "step": 1930
    },
    {
      "epoch": 0.951231527093596,
      "grad_norm": 0.5298451954367319,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -41811968.0,
      "logits/rejected": -60526136.88888889,
      "logps/chosen": -263.7142857142857,
      "logps/rejected": -660.4444444444445,
      "loss": 0.1919,
      "rewards/chosen": 0.17857142857142858,
      "rewards/margins": 22.845238095238095,
      "rewards/rejected": -22.666666666666668,
      "step": 1931
    },
    {
      "epoch": 0.9517241379310345,
      "grad_norm": 0.495950105066435,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -50417246.04081633,
      "logits/rejected": -77249523.03797469,
      "logps/chosen": -297.14285714285717,
      "logps/rejected": -661.873417721519,
      "loss": 0.2026,
      "rewards/chosen": -1.1741071428571428,
      "rewards/margins": 19.686652350813745,
      "rewards/rejected": -20.860759493670887,
      "step": 1932
    },
    {
      "epoch": 0.952216748768473,
      "grad_norm": 0.5093656673886138,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -51224879.40740741,
      "logits/rejected": -56566424.216216214,
      "logps/chosen": -284.44444444444446,
      "logps/rejected": -573.4054054054054,
      "loss": 0.1906,
      "rewards/chosen": 1.2679420753761574,
      "rewards/margins": 20.26794207537616,
      "rewards/rejected": -19.0,
      "step": 1933
    },
    {
      "epoch": 0.9527093596059113,
      "grad_norm": 0.6791795423456273,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -58148305.45454545,
      "logits/rejected": -53477376.0,
      "logps/chosen": -394.54545454545456,
      "logps/rejected": -582.8571428571429,
      "loss": 0.1579,
      "rewards/chosen": 1.125,
      "rewards/margins": 18.74404761904762,
      "rewards/rejected": -17.61904761904762,
      "step": 1934
    },
    {
      "epoch": 0.9532019704433498,
      "grad_norm": 0.386508032469301,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -57404395.921568625,
      "logits/rejected": -66019434.38961039,
      "logps/chosen": -296.7843137254902,
      "logps/rejected": -654.1298701298701,
      "loss": 0.1319,
      "rewards/chosen": 1.767156862745098,
      "rewards/margins": 21.55936465495289,
      "rewards/rejected": -19.792207792207794,
      "step": 1935
    },
    {
      "epoch": 0.9536945812807882,
      "grad_norm": 0.4170845833947158,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -42991616.0,
      "logits/rejected": -54945382.4,
      "logps/chosen": -289.0,
      "logps/rejected": -572.8,
      "loss": 0.1435,
      "rewards/chosen": 0.17447916666666666,
      "rewards/margins": 20.16197916666667,
      "rewards/rejected": -19.9875,
      "step": 1936
    },
    {
      "epoch": 0.9541871921182266,
      "grad_norm": 0.6164682829121618,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -38507280.34042553,
      "logits/rejected": -61620021.72839506,
      "logps/chosen": -229.10638297872342,
      "logps/rejected": -561.7777777777778,
      "loss": 0.1975,
      "rewards/chosen": 0.5372340425531915,
      "rewards/margins": 11.583101585714843,
      "rewards/rejected": -11.045867543161652,
      "step": 1937
    },
    {
      "epoch": 0.954679802955665,
      "grad_norm": 0.4047473408669821,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -46165683.89189189,
      "logits/rejected": -53097123.16483516,
      "logps/chosen": -298.81081081081084,
      "logps/rejected": -603.4285714285714,
      "loss": 0.123,
      "rewards/chosen": 3.592914478198902,
      "rewards/margins": 24.01049689578132,
      "rewards/rejected": -20.417582417582416,
      "step": 1938
    },
    {
      "epoch": 0.9551724137931035,
      "grad_norm": 0.47268450973656156,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -47977701.87755102,
      "logits/rejected": -67533603.64556962,
      "logps/chosen": -283.7551020408163,
      "logps/rejected": -535.493670886076,
      "loss": 0.1817,
      "rewards/chosen": 4.354655363121811,
      "rewards/margins": 24.07617435046358,
      "rewards/rejected": -19.72151898734177,
      "step": 1939
    },
    {
      "epoch": 0.9556650246305419,
      "grad_norm": 0.5519260870466404,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -41418752.0,
      "logits/rejected": -65710762.666666664,
      "logps/chosen": -302.0,
      "logps/rejected": -544.0,
      "loss": 0.1568,
      "rewards/chosen": 0.12464488636363637,
      "rewards/margins": 11.310595755969292,
      "rewards/rejected": -11.185950869605655,
      "step": 1940
    },
    {
      "epoch": 0.9561576354679803,
      "grad_norm": 0.48733779478126976,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -47965630.35897436,
      "logits/rejected": -68899690.4269663,
      "logps/chosen": -316.1025641025641,
      "logps/rejected": -546.5168539325842,
      "loss": 0.1634,
      "rewards/chosen": 2.1763133513621793,
      "rewards/margins": 20.356088632261056,
      "rewards/rejected": -18.179775280898877,
      "step": 1941
    },
    {
      "epoch": 0.9566502463054187,
      "grad_norm": 0.5392868532771473,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -49775260.73469388,
      "logits/rejected": -75603656.9113924,
      "logps/chosen": -335.0204081632653,
      "logps/rejected": -587.3417721518987,
      "loss": 0.1662,
      "rewards/chosen": 1.4630102040816326,
      "rewards/margins": 18.070605140790494,
      "rewards/rejected": -16.60759493670886,
      "step": 1942
    },
    {
      "epoch": 0.9571428571428572,
      "grad_norm": 0.42703608688035166,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -61785324.307692304,
      "logits/rejected": -54305199.15789474,
      "logps/chosen": -214.92307692307693,
      "logps/rejected": -551.578947368421,
      "loss": 0.1736,
      "rewards/chosen": 0.18704927884615385,
      "rewards/margins": 18.318628226214575,
      "rewards/rejected": -18.13157894736842,
      "step": 1943
    },
    {
      "epoch": 0.9576354679802955,
      "grad_norm": 0.7481638824484873,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -58506986.30508474,
      "logits/rejected": -57808450.782608695,
      "logps/chosen": -312.135593220339,
      "logps/rejected": -559.304347826087,
      "loss": 0.1834,
      "rewards/chosen": 1.4316737288135593,
      "rewards/margins": 5.304708662152554,
      "rewards/rejected": -3.8730349333389946,
      "step": 1944
    },
    {
      "epoch": 0.958128078817734,
      "grad_norm": 0.48749071972345476,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -50160451.91836735,
      "logits/rejected": -56224910.58227848,
      "logps/chosen": -318.3673469387755,
      "logps/rejected": -593.8227848101266,
      "loss": 0.1874,
      "rewards/chosen": -0.4017857142857143,
      "rewards/margins": 17.7501130198915,
      "rewards/rejected": -18.151898734177216,
      "step": 1945
    },
    {
      "epoch": 0.9586206896551724,
      "grad_norm": 0.5233505868486777,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -55458019.55555555,
      "logits/rejected": -67564766.60869566,
      "logps/chosen": -340.0,
      "logps/rejected": -580.8695652173913,
      "loss": 0.1519,
      "rewards/chosen": 5.044698927137587,
      "rewards/margins": 23.414264144528893,
      "rewards/rejected": -18.369565217391305,
      "step": 1946
    },
    {
      "epoch": 0.9591133004926108,
      "grad_norm": 0.5423899733436414,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -55479202.90909091,
      "logits/rejected": -59984292.82191781,
      "logps/chosen": -287.7090909090909,
      "logps/rejected": -596.1643835616438,
      "loss": 0.1872,
      "rewards/chosen": -0.24886363636363637,
      "rewards/margins": 19.805930884184306,
      "rewards/rejected": -20.054794520547944,
      "step": 1947
    },
    {
      "epoch": 0.9596059113300492,
      "grad_norm": 0.5277935099954878,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -37711286.85714286,
      "logits/rejected": -51263715.55555555,
      "logps/chosen": -307.7142857142857,
      "logps/rejected": -516.4444444444445,
      "loss": 0.1483,
      "rewards/chosen": 2.4107142857142856,
      "rewards/margins": 19.369047619047617,
      "rewards/rejected": -16.958333333333332,
      "step": 1948
    },
    {
      "epoch": 0.9600985221674877,
      "grad_norm": 0.464074061591762,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -46247720.421052635,
      "logits/rejected": -57834135.43661972,
      "logps/chosen": -233.12280701754386,
      "logps/rejected": -569.2394366197183,
      "loss": 0.1535,
      "rewards/chosen": 1.995202850877193,
      "rewards/margins": 15.59414255845862,
      "rewards/rejected": -13.598939707581426,
      "step": 1949
    },
    {
      "epoch": 0.9605911330049262,
      "grad_norm": 0.4459947498499948,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -35045740.08888889,
      "logits/rejected": -54121681.73493976,
      "logps/chosen": -187.0222222222222,
      "logps/rejected": -551.3253012048193,
      "loss": 0.1453,
      "rewards/chosen": 3.9882266574435765,
      "rewards/margins": 20.590636295997793,
      "rewards/rejected": -16.602409638554217,
      "step": 1950
    },
    {
      "epoch": 0.9610837438423645,
      "grad_norm": 0.4818588149024643,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -57107062.15384615,
      "logits/rejected": -60872596.21052632,
      "logps/chosen": -328.9230769230769,
      "logps/rejected": -553.2631578947369,
      "loss": 0.1671,
      "rewards/chosen": 0.4170673076923077,
      "rewards/margins": 19.2591725708502,
      "rewards/rejected": -18.842105263157894,
      "step": 1951
    },
    {
      "epoch": 0.961576354679803,
      "grad_norm": 0.4878667220743204,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -45806214.7368421,
      "logits/rejected": -49209228.61971831,
      "logps/chosen": -309.89473684210526,
      "logps/rejected": -534.0845070422536,
      "loss": 0.1844,
      "rewards/chosen": -0.32785087719298245,
      "rewards/margins": 11.996092784778847,
      "rewards/rejected": -12.32394366197183,
      "step": 1952
    },
    {
      "epoch": 0.9620689655172414,
      "grad_norm": 0.36513384205341376,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -58573943.069767445,
      "logits/rejected": -53785780.705882356,
      "logps/chosen": -288.0,
      "logps/rejected": -565.4588235294118,
      "loss": 0.1594,
      "rewards/chosen": 3.161374114280523,
      "rewards/margins": 20.47902117310405,
      "rewards/rejected": -17.31764705882353,
      "step": 1953
    },
    {
      "epoch": 0.9625615763546798,
      "grad_norm": 0.39149614420420764,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -47915364.17391305,
      "logits/rejected": -55497802.92682927,
      "logps/chosen": -249.04347826086956,
      "logps/rejected": -554.9268292682926,
      "loss": 0.1444,
      "rewards/chosen": 0.39487092391304346,
      "rewards/margins": 17.577797753181336,
      "rewards/rejected": -17.182926829268293,
      "step": 1954
    },
    {
      "epoch": 0.9630541871921182,
      "grad_norm": 0.547950197561811,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -36951818.24,
      "logits/rejected": -56139145.84615385,
      "logps/chosen": -325.12,
      "logps/rejected": -604.7179487179487,
      "loss": 0.1684,
      "rewards/chosen": 1.96875,
      "rewards/margins": 19.43028846153846,
      "rewards/rejected": -17.46153846153846,
      "step": 1955
    },
    {
      "epoch": 0.9635467980295567,
      "grad_norm": 0.450819459652393,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -50043803.60784314,
      "logits/rejected": -68851951.37662338,
      "logps/chosen": -209.5686274509804,
      "logps/rejected": -549.4025974025974,
      "loss": 0.1368,
      "rewards/chosen": 0.8651960784313726,
      "rewards/margins": 18.034027247262543,
      "rewards/rejected": -17.16883116883117,
      "step": 1956
    },
    {
      "epoch": 0.9640394088669951,
      "grad_norm": 0.4855770758400046,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -48153836.307692304,
      "logits/rejected": -54525952.0,
      "logps/chosen": -274.15384615384613,
      "logps/rejected": -637.4736842105264,
      "loss": 0.1774,
      "rewards/chosen": -0.12477463942307693,
      "rewards/margins": 19.769962202682187,
      "rewards/rejected": -19.894736842105264,
      "step": 1957
    },
    {
      "epoch": 0.9645320197044335,
      "grad_norm": 0.5522194185417822,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -55938319.67346939,
      "logits/rejected": -50013093.26582278,
      "logps/chosen": -397.0612244897959,
      "logps/rejected": -539.5443037974684,
      "loss": 0.1729,
      "rewards/chosen": 0.5637755102040817,
      "rewards/margins": 19.120737535520536,
      "rewards/rejected": -18.556962025316455,
      "step": 1958
    },
    {
      "epoch": 0.9650246305418719,
      "grad_norm": 0.5263187033364866,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -47919923.2,
      "logits/rejected": -54764264.72727273,
      "logps/chosen": -340.4,
      "logps/rejected": -585.4545454545455,
      "loss": 0.1791,
      "rewards/chosen": -1.548046875,
      "rewards/margins": 17.247407670454546,
      "rewards/rejected": -18.795454545454547,
      "step": 1959
    },
    {
      "epoch": 0.9655172413793104,
      "grad_norm": 0.48691190984791194,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -50407908.07272727,
      "logits/rejected": -67453601.3150685,
      "logps/chosen": -214.98181818181817,
      "logps/rejected": -609.3150684931506,
      "loss": 0.1605,
      "rewards/chosen": 1.4102272727272727,
      "rewards/margins": 20.506117683686178,
      "rewards/rejected": -19.095890410958905,
      "step": 1960
    },
    {
      "epoch": 0.9660098522167487,
      "grad_norm": 0.5840667023490154,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -50184479.438596494,
      "logits/rejected": -61260468.28169014,
      "logps/chosen": -282.10526315789474,
      "logps/rejected": -573.2957746478874,
      "loss": 0.1799,
      "rewards/chosen": 0.39144736842105265,
      "rewards/margins": 19.34919384729429,
      "rewards/rejected": -18.95774647887324,
      "step": 1961
    },
    {
      "epoch": 0.9665024630541872,
      "grad_norm": 0.5337170773508606,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -59698926.93333333,
      "logits/rejected": -53212073.638554215,
      "logps/chosen": -358.93333333333334,
      "logps/rejected": -504.28915662650604,
      "loss": 0.1197,
      "rewards/chosen": 2.8680555555555554,
      "rewards/margins": 18.964441097724233,
      "rewards/rejected": -16.096385542168676,
      "step": 1962
    },
    {
      "epoch": 0.9669950738916256,
      "grad_norm": 0.4651860480795191,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -37539991.7037037,
      "logits/rejected": -77651303.78378378,
      "logps/chosen": -236.14814814814815,
      "logps/rejected": -637.4054054054054,
      "loss": 0.1371,
      "rewards/chosen": 2.127314814814815,
      "rewards/margins": 21.9110985985986,
      "rewards/rejected": -19.783783783783782,
      "step": 1963
    },
    {
      "epoch": 0.967487684729064,
      "grad_norm": 0.423095935060157,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -42710290.73170732,
      "logits/rejected": -75401051.21839081,
      "logps/chosen": -230.82926829268294,
      "logps/rejected": -604.6896551724138,
      "loss": 0.1504,
      "rewards/chosen": -1.0356326219512195,
      "rewards/margins": 18.113792665405104,
      "rewards/rejected": -19.149425287356323,
      "step": 1964
    },
    {
      "epoch": 0.9679802955665024,
      "grad_norm": 0.5231361033797145,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -46766489.6,
      "logits/rejected": -41336634.60240964,
      "logps/chosen": -291.73333333333335,
      "logps/rejected": -544.3855421686746,
      "loss": 0.1169,
      "rewards/chosen": 6.976427544487847,
      "rewards/margins": 23.530644411957727,
      "rewards/rejected": -16.55421686746988,
      "step": 1965
    },
    {
      "epoch": 0.9684729064039409,
      "grad_norm": 0.6062630031918235,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -53826901.333333336,
      "logits/rejected": -58890295.35135135,
      "logps/chosen": -350.22222222222223,
      "logps/rejected": -588.1081081081081,
      "loss": 0.206,
      "rewards/chosen": -0.14351851851851852,
      "rewards/margins": 19.66729229229229,
      "rewards/rejected": -19.81081081081081,
      "step": 1966
    },
    {
      "epoch": 0.9689655172413794,
      "grad_norm": 0.4865831546606758,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -45888529.355932206,
      "logits/rejected": -57322154.666666664,
      "logps/chosen": -364.20338983050846,
      "logps/rejected": -599.1884057971015,
      "loss": 0.21,
      "rewards/chosen": -0.8622881355932204,
      "rewards/margins": 18.38408867600098,
      "rewards/rejected": -19.246376811594203,
      "step": 1967
    },
    {
      "epoch": 0.9694581280788177,
      "grad_norm": 0.5038235753504121,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -49500018.75862069,
      "logits/rejected": -67108864.0,
      "logps/chosen": -295.17241379310343,
      "logps/rejected": -617.1428571428571,
      "loss": 0.1729,
      "rewards/chosen": 0.8631465517241379,
      "rewards/margins": 17.577432266009854,
      "rewards/rejected": -16.714285714285715,
      "step": 1968
    },
    {
      "epoch": 0.9699507389162562,
      "grad_norm": 0.4437300650950922,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -45671310.222222224,
      "logits/rejected": -41801340.54054054,
      "logps/chosen": -314.3703703703704,
      "logps/rejected": -590.7027027027027,
      "loss": 0.1551,
      "rewards/chosen": 1.3987268518518519,
      "rewards/margins": 20.66899712212212,
      "rewards/rejected": -19.27027027027027,
      "step": 1969
    },
    {
      "epoch": 0.9704433497536946,
      "grad_norm": 0.7308303559214571,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -48001479.11111111,
      "logits/rejected": -55886266.81081081,
      "logps/chosen": -239.25925925925927,
      "logps/rejected": -529.7297297297297,
      "loss": 0.1698,
      "rewards/chosen": 1.1163194444444444,
      "rewards/margins": 17.008211336336334,
      "rewards/rejected": -15.891891891891891,
      "step": 1970
    },
    {
      "epoch": 0.970935960591133,
      "grad_norm": 0.5224091028018542,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -47573437.217391305,
      "logits/rejected": -57492654.82926829,
      "logps/chosen": -285.9130434782609,
      "logps/rejected": -568.1951219512196,
      "loss": 0.1362,
      "rewards/chosen": 6.26219774329144,
      "rewards/margins": 23.896344084754855,
      "rewards/rejected": -17.634146341463413,
      "step": 1971
    },
    {
      "epoch": 0.9714285714285714,
      "grad_norm": 0.5242359032727045,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -52721425.86046512,
      "logits/rejected": -52946919.90588235,
      "logps/chosen": -269.95348837209303,
      "logps/rejected": -563.2,
      "loss": 0.1396,
      "rewards/chosen": 0.3488372093023256,
      "rewards/margins": 12.367896693447546,
      "rewards/rejected": -12.01905948414522,
      "step": 1972
    },
    {
      "epoch": 0.9719211822660099,
      "grad_norm": 0.528135011026444,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -58622714.04651163,
      "logits/rejected": -54032504.47058824,
      "logps/chosen": -273.1162790697674,
      "logps/rejected": -581.2705882352941,
      "loss": 0.1492,
      "rewards/chosen": 2.297980729923692,
      "rewards/margins": 18.274451318158988,
      "rewards/rejected": -15.976470588235294,
      "step": 1973
    },
    {
      "epoch": 0.9724137931034482,
      "grad_norm": 0.5326861620293508,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -49779765.89473684,
      "logits/rejected": -55884670.197183095,
      "logps/chosen": -295.0175438596491,
      "logps/rejected": -526.4225352112676,
      "loss": 0.1957,
      "rewards/chosen": 0.2850877192982456,
      "rewards/margins": 12.776713867790763,
      "rewards/rejected": -12.491626148492518,
      "step": 1974
    },
    {
      "epoch": 0.9729064039408867,
      "grad_norm": 0.49842104083383687,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -54243643.07692308,
      "logits/rejected": -48013743.15789474,
      "logps/chosen": -252.15384615384616,
      "logps/rejected": -545.6842105263158,
      "loss": 0.1744,
      "rewards/chosen": 0.08533653846153846,
      "rewards/margins": 18.374810222672068,
      "rewards/rejected": -18.289473684210527,
      "step": 1975
    },
    {
      "epoch": 0.9733990147783251,
      "grad_norm": 0.6073929722157799,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -58807637.333333336,
      "logits/rejected": -59611545.6,
      "logps/chosen": -335.6666666666667,
      "logps/rejected": -589.6,
      "loss": 0.1676,
      "rewards/chosen": -0.7702229817708334,
      "rewards/margins": 15.779777018229167,
      "rewards/rejected": -16.55,
      "step": 1976
    },
    {
      "epoch": 0.9738916256157636,
      "grad_norm": 0.40686683455070827,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -34854666.24,
      "logits/rejected": -51622203.07692308,
      "logps/chosen": -245.6,
      "logps/rejected": -534.5641025641025,
      "loss": 0.1394,
      "rewards/chosen": 4.381914672851562,
      "rewards/margins": 20.79217108310797,
      "rewards/rejected": -16.41025641025641,
      "step": 1977
    },
    {
      "epoch": 0.9743842364532019,
      "grad_norm": 0.6108740763274934,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -72947993.09803921,
      "logits/rejected": -60190985.97402597,
      "logps/chosen": -273.88235294117646,
      "logps/rejected": -531.1168831168832,
      "loss": 0.2113,
      "rewards/chosen": 3.0116388657513786,
      "rewards/margins": 16.72592458003709,
      "rewards/rejected": -13.714285714285714,
      "step": 1978
    },
    {
      "epoch": 0.9748768472906404,
      "grad_norm": 0.560736758165283,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -51780589.38181818,
      "logits/rejected": -52428800.0,
      "logps/chosen": -303.41818181818184,
      "logps/rejected": -513.7534246575342,
      "loss": 0.1881,
      "rewards/chosen": 1.4625,
      "rewards/margins": 15.284417808219178,
      "rewards/rejected": -13.821917808219178,
      "step": 1979
    },
    {
      "epoch": 0.9753694581280788,
      "grad_norm": 0.5144981941207214,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -44587275.13043478,
      "logits/rejected": -53400650.92682927,
      "logps/chosen": -236.0,
      "logps/rejected": -523.3170731707318,
      "loss": 0.1422,
      "rewards/chosen": 0.44633152173913043,
      "rewards/margins": 17.37316079003181,
      "rewards/rejected": -16.926829268292682,
      "step": 1980
    },
    {
      "epoch": 0.9758620689655172,
      "grad_norm": 0.4821927096175501,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -54575884.190476194,
      "logits/rejected": -81837698.97674419,
      "logps/chosen": -324.3809523809524,
      "logps/rejected": -530.6046511627907,
      "loss": 0.1461,
      "rewards/chosen": 2.7418728783017112,
      "rewards/margins": 19.64884962248776,
      "rewards/rejected": -16.906976744186046,
      "step": 1981
    },
    {
      "epoch": 0.9763546798029556,
      "grad_norm": 0.6999688075615871,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -50514009.04347826,
      "logits/rejected": -56776554.146341465,
      "logps/chosen": -270.7826086956522,
      "logps/rejected": -558.829268292683,
      "loss": 0.1421,
      "rewards/chosen": 0.9565217391304348,
      "rewards/margins": 15.456521739130435,
      "rewards/rejected": -14.5,
      "step": 1982
    },
    {
      "epoch": 0.9768472906403941,
      "grad_norm": 0.4712911672894992,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -52140955.60784314,
      "logits/rejected": -65038947.74025974,
      "logps/chosen": -308.3921568627451,
      "logps/rejected": -603.4285714285714,
      "loss": 0.1918,
      "rewards/chosen": -1.275735294117647,
      "rewards/margins": 15.737251718869365,
      "rewards/rejected": -17.01298701298701,
      "step": 1983
    },
    {
      "epoch": 0.9773399014778326,
      "grad_norm": 0.4805704559307744,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -51562585.04347826,
      "logits/rejected": -46699994.53658537,
      "logps/chosen": -231.82608695652175,
      "logps/rejected": -472.9756097560976,
      "loss": 0.1597,
      "rewards/chosen": 0.15557065217391305,
      "rewards/margins": 14.155570652173912,
      "rewards/rejected": -14.0,
      "step": 1984
    },
    {
      "epoch": 0.9778325123152709,
      "grad_norm": 0.5883985583640708,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -48043845.81818182,
      "logits/rejected": -61822891.83561644,
      "logps/chosen": -275.2,
      "logps/rejected": -583.8904109589041,
      "loss": 0.1972,
      "rewards/chosen": 1.1238636363636363,
      "rewards/margins": 10.515925052780588,
      "rewards/rejected": -9.392061416416952,
      "step": 1985
    },
    {
      "epoch": 0.9783251231527094,
      "grad_norm": 0.6386499832617036,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -57957655.27272727,
      "logits/rejected": -54755776.87671233,
      "logps/chosen": -290.90909090909093,
      "logps/rejected": -531.7260273972603,
      "loss": 0.1973,
      "rewards/chosen": 3.4545462868430397,
      "rewards/margins": 13.851806560815643,
      "rewards/rejected": -10.397260273972602,
      "step": 1986
    },
    {
      "epoch": 0.9788177339901478,
      "grad_norm": 0.483057583629947,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -56900086.33962264,
      "logits/rejected": -57322154.666666664,
      "logps/chosen": -348.07547169811323,
      "logps/rejected": -564.0533333333333,
      "loss": 0.1458,
      "rewards/chosen": 1.9693396226415094,
      "rewards/margins": 18.196006289308176,
      "rewards/rejected": -16.226666666666667,
      "step": 1987
    },
    {
      "epoch": 0.9793103448275862,
      "grad_norm": 0.6594867334824984,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -48278186.666666664,
      "logits/rejected": -52900659.2,
      "logps/chosen": -264.3333333333333,
      "logps/rejected": -520.4,
      "loss": 0.1884,
      "rewards/chosen": -0.6848958333333334,
      "rewards/margins": 15.140104166666665,
      "rewards/rejected": -15.825,
      "step": 1988
    },
    {
      "epoch": 0.9798029556650246,
      "grad_norm": 0.5411822047398224,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -44398915.368421055,
      "logits/rejected": -55504622.93333333,
      "logps/chosen": -213.1578947368421,
      "logps/rejected": -492.0888888888889,
      "loss": 0.149,
      "rewards/chosen": 0.8898026315789473,
      "rewards/margins": 14.156469298245614,
      "rewards/rejected": -13.266666666666667,
      "step": 1989
    },
    {
      "epoch": 0.9802955665024631,
      "grad_norm": 0.7958246068209599,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -37895536.64,
      "logits/rejected": -56408011.48717949,
      "logps/chosen": -303.04,
      "logps/rejected": -604.7179487179487,
      "loss": 0.1502,
      "rewards/chosen": 1.555,
      "rewards/margins": 16.04217948717949,
      "rewards/rejected": -14.487179487179487,
      "step": 1990
    },
    {
      "epoch": 0.9807881773399014,
      "grad_norm": 0.42543951901404237,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -48143315.47826087,
      "logits/rejected": -54372501.853658535,
      "logps/chosen": -310.95652173913044,
      "logps/rejected": -575.219512195122,
      "loss": 0.1689,
      "rewards/chosen": -1.377717391304348,
      "rewards/margins": 14.768624072110285,
      "rewards/rejected": -16.146341463414632,
      "step": 1991
    },
    {
      "epoch": 0.9812807881773399,
      "grad_norm": 0.49046948295622833,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -57715370.666666664,
      "logits/rejected": -60660121.6,
      "logps/chosen": -335.3333333333333,
      "logps/rejected": -565.6,
      "loss": 0.1517,
      "rewards/chosen": 0.13557942708333334,
      "rewards/margins": 17.83557942708333,
      "rewards/rejected": -17.7,
      "step": 1992
    },
    {
      "epoch": 0.9817733990147783,
      "grad_norm": 0.600111850521021,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -69010932.09302326,
      "logits/rejected": -68095759.05882353,
      "logps/chosen": -366.13953488372096,
      "logps/rejected": -594.8235294117648,
      "loss": 0.1586,
      "rewards/chosen": 2.97099109028661,
      "rewards/margins": 18.57099109028661,
      "rewards/rejected": -15.6,
      "step": 1993
    },
    {
      "epoch": 0.9822660098522168,
      "grad_norm": 0.5317432009613473,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -42751317.333333336,
      "logits/rejected": -54735667.2,
      "logps/chosen": -294.3333333333333,
      "logps/rejected": -563.2,
      "loss": 0.1272,
      "rewards/chosen": 4.307298024495442,
      "rewards/margins": 21.282298024495443,
      "rewards/rejected": -16.975,
      "step": 1994
    },
    {
      "epoch": 0.9827586206896551,
      "grad_norm": 0.6052774670173642,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -49847689.84615385,
      "logits/rejected": -48896754.526315786,
      "logps/chosen": -329.2307692307692,
      "logps/rejected": -544.0,
      "loss": 0.196,
      "rewards/chosen": -0.0889423076923077,
      "rewards/margins": 17.38474190283401,
      "rewards/rejected": -17.473684210526315,
      "step": 1995
    },
    {
      "epoch": 0.9832512315270936,
      "grad_norm": 0.612149006309371,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -64877851.234042555,
      "logits/rejected": -57425717.72839506,
      "logps/chosen": -374.97872340425533,
      "logps/rejected": -545.9753086419753,
      "loss": 0.1607,
      "rewards/chosen": 0.26105385638297873,
      "rewards/margins": 14.878337807000262,
      "rewards/rejected": -14.617283950617283,
      "step": 1996
    },
    {
      "epoch": 0.983743842364532,
      "grad_norm": 0.6375954868083543,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -79375224.75471698,
      "logits/rejected": -62019775.14666667,
      "logps/chosen": -345.0566037735849,
      "logps/rejected": -603.3066666666666,
      "loss": 0.1848,
      "rewards/chosen": 2.2709483020710497,
      "rewards/margins": 18.55094830207105,
      "rewards/rejected": -16.28,
      "step": 1997
    },
    {
      "epoch": 0.9842364532019704,
      "grad_norm": 0.5722120642936189,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -50766905.96226415,
      "logits/rejected": -66325927.25333333,
      "logps/chosen": -308.22641509433964,
      "logps/rejected": -563.2,
      "loss": 0.1621,
      "rewards/chosen": 1.490566037735849,
      "rewards/margins": 16.063899371069184,
      "rewards/rejected": -14.573333333333334,
      "step": 1998
    },
    {
      "epoch": 0.9847290640394089,
      "grad_norm": 0.4818225278489153,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -41686245.87755102,
      "logits/rejected": -56437280.405063294,
      "logps/chosen": -246.85714285714286,
      "logps/rejected": -513.620253164557,
      "loss": 0.1631,
      "rewards/chosen": 0.3840880102040816,
      "rewards/margins": 15.586619655773701,
      "rewards/rejected": -15.20253164556962,
      "step": 1999
    },
    {
      "epoch": 0.9852216748768473,
      "grad_norm": 0.6341859245369807,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -51982597.44680851,
      "logits/rejected": -54784859.654320985,
      "logps/chosen": -307.9148936170213,
      "logps/rejected": -566.5185185185185,
      "loss": 0.1939,
      "rewards/chosen": 1.8324270045503657,
      "rewards/margins": 16.202797374920735,
      "rewards/rejected": -14.37037037037037,
      "step": 2000
    },
    {
      "epoch": 0.9857142857142858,
      "grad_norm": 0.46525961831886276,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -30819016.347826086,
      "logits/rejected": -70177866.92682926,
      "logps/chosen": -218.43478260869566,
      "logps/rejected": -586.9268292682926,
      "loss": 0.1704,
      "rewards/chosen": -0.001358695652173913,
      "rewards/margins": 17.144982767762457,
      "rewards/rejected": -17.146341463414632,
      "step": 2001
    },
    {
      "epoch": 0.9862068965517241,
      "grad_norm": 0.46456061874587384,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -51218904.615384616,
      "logits/rejected": -53863693.473684214,
      "logps/chosen": -328.0,
      "logps/rejected": -546.5263157894736,
      "loss": 0.1468,
      "rewards/chosen": 1.9591346153846154,
      "rewards/margins": 18.695976720647774,
      "rewards/rejected": -16.736842105263158,
      "step": 2002
    },
    {
      "epoch": 0.9866995073891626,
      "grad_norm": 0.5418357566665795,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -55073035.13043478,
      "logits/rejected": -54781702.24390244,
      "logps/chosen": -389.5652173913044,
      "logps/rejected": -576.780487804878,
      "loss": 0.1544,
      "rewards/chosen": 0.3637058423913043,
      "rewards/margins": 18.631998525318135,
      "rewards/rejected": -18.26829268292683,
      "step": 2003
    },
    {
      "epoch": 0.987192118226601,
      "grad_norm": 0.43612631389137974,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -52270524.37735849,
      "logits/rejected": -70576155.30666667,
      "logps/chosen": -274.7169811320755,
      "logps/rejected": -484.26666666666665,
      "loss": 0.1688,
      "rewards/chosen": 0.19855542452830188,
      "rewards/margins": 15.131888757861635,
      "rewards/rejected": -14.933333333333334,
      "step": 2004
    },
    {
      "epoch": 0.9876847290640394,
      "grad_norm": 0.5310480955187807,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -46436937.14285714,
      "logits/rejected": -56490372.8607595,
      "logps/chosen": -268.734693877551,
      "logps/rejected": -593.0126582278481,
      "loss": 0.1614,
      "rewards/chosen": 2.427286264847736,
      "rewards/margins": 19.009564745860395,
      "rewards/rejected": -16.582278481012658,
      "step": 2005
    },
    {
      "epoch": 0.9881773399014778,
      "grad_norm": 0.5180233978147291,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -51860821.333333336,
      "logits/rejected": -60607692.8,
      "logps/chosen": -218.33333333333334,
      "logps/rejected": -584.8,
      "loss": 0.1248,
      "rewards/chosen": 3.3548380533854165,
      "rewards/margins": 21.304838053385417,
      "rewards/rejected": -17.95,
      "step": 2006
    },
    {
      "epoch": 0.9886699507389163,
      "grad_norm": 0.5925894629677116,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -68382134.85714285,
      "logits/rejected": -54991985.777777776,
      "logps/chosen": -233.42857142857142,
      "logps/rejected": -488.44444444444446,
      "loss": 0.1775,
      "rewards/chosen": 0.828125,
      "rewards/margins": 17.022569444444443,
      "rewards/rejected": -16.194444444444443,
      "step": 2007
    },
    {
      "epoch": 0.9891625615763546,
      "grad_norm": 0.46550127435682537,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -46250703.567567565,
      "logits/rejected": -57706248.43956044,
      "logps/chosen": -370.1621621621622,
      "logps/rejected": -507.0769230769231,
      "loss": 0.1401,
      "rewards/chosen": 3.6045235298775338,
      "rewards/margins": 19.648479573833576,
      "rewards/rejected": -16.043956043956044,
      "step": 2008
    },
    {
      "epoch": 0.9896551724137931,
      "grad_norm": 0.5533020645581923,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -44865300.983606555,
      "logits/rejected": -63164966.20895522,
      "logps/chosen": -250.2295081967213,
      "logps/rejected": -600.8358208955224,
      "loss": 0.1679,
      "rewards/chosen": 1.4360911885245902,
      "rewards/margins": 19.122658352703695,
      "rewards/rejected": -17.686567164179106,
      "step": 2009
    },
    {
      "epoch": 0.9901477832512315,
      "grad_norm": 0.5278552681102284,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -47140329.73913044,
      "logits/rejected": -60510507.70731708,
      "logps/chosen": -248.8695652173913,
      "logps/rejected": -588.4878048780488,
      "loss": 0.1485,
      "rewards/chosen": 5.752771128778872,
      "rewards/margins": 22.508868689754483,
      "rewards/rejected": -16.75609756097561,
      "step": 2010
    },
    {
      "epoch": 0.99064039408867,
      "grad_norm": 0.6734490430153751,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -52017593.7254902,
      "logits/rejected": -69069837.2987013,
      "logps/chosen": -333.80392156862746,
      "logps/rejected": -565.1948051948052,
      "loss": 0.1663,
      "rewards/chosen": 2.3480697332643996,
      "rewards/margins": 18.86755025274492,
      "rewards/rejected": -16.51948051948052,
      "step": 2011
    },
    {
      "epoch": 0.9911330049261083,
      "grad_norm": 0.5719752869407855,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -40608488.72727273,
      "logits/rejected": -59237361.97260274,
      "logps/chosen": -202.47272727272727,
      "logps/rejected": -533.917808219178,
      "loss": 0.1732,
      "rewards/chosen": 1.4051416570490056,
      "rewards/margins": 12.001386573572978,
      "rewards/rejected": -10.596244916523972,
      "step": 2012
    },
    {
      "epoch": 0.9916256157635468,
      "grad_norm": 0.44534121046536224,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -58537894.95652174,
      "logits/rejected": -56265053.65853658,
      "logps/chosen": -299.1304347826087,
      "logps/rejected": -521.3658536585366,
      "loss": 0.1594,
      "rewards/chosen": 3.2554248312245244,
      "rewards/margins": 19.59688824585867,
      "rewards/rejected": -16.341463414634145,
      "step": 2013
    },
    {
      "epoch": 0.9921182266009853,
      "grad_norm": 0.4154318405644332,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -47448064.0,
      "logits/rejected": -54263808.0,
      "logps/chosen": -307.0,
      "logps/rejected": -551.2,
      "loss": 0.1378,
      "rewards/chosen": 6.140041351318359,
      "rewards/margins": 22.46504135131836,
      "rewards/rejected": -16.325,
      "step": 2014
    },
    {
      "epoch": 0.9926108374384236,
      "grad_norm": 0.6780357506863465,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -52494336.0,
      "logits/rejected": -49597644.8,
      "logps/chosen": -305.6666666666667,
      "logps/rejected": -530.4,
      "loss": 0.1436,
      "rewards/chosen": 1.5188802083333333,
      "rewards/margins": 16.09388020833333,
      "rewards/rejected": -14.575,
      "step": 2015
    },
    {
      "epoch": 0.993103448275862,
      "grad_norm": 0.47893018103061963,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -52205698.72340426,
      "logits/rejected": -62293181.62962963,
      "logps/chosen": -321.70212765957444,
      "logps/rejected": -616.2962962962963,
      "loss": 0.1827,
      "rewards/chosen": -0.19847074468085107,
      "rewards/margins": 16.36943048988705,
      "rewards/rejected": -16.567901234567902,
      "step": 2016
    },
    {
      "epoch": 0.9935960591133005,
      "grad_norm": 0.513305881239918,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -52830382.29787234,
      "logits/rejected": -59496978.96296296,
      "logps/chosen": -318.29787234042556,
      "logps/rejected": -561.7777777777778,
      "loss": 0.1563,
      "rewards/chosen": 0.7938829787234043,
      "rewards/margins": 17.867957052797475,
      "rewards/rejected": -17.074074074074073,
      "step": 2017
    },
    {
      "epoch": 0.994088669950739,
      "grad_norm": 0.5770821631341249,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -52507937.81132075,
      "logits/rejected": -43683676.16,
      "logps/chosen": -254.18867924528303,
      "logps/rejected": -552.1066666666667,
      "loss": 0.169,
      "rewards/chosen": 0.40639740566037735,
      "rewards/margins": 16.593064072327046,
      "rewards/rejected": -16.186666666666667,
      "step": 2018
    },
    {
      "epoch": 0.9945812807881773,
      "grad_norm": 0.5062385879263089,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -56426496.0,
      "logits/rejected": -64836949.333333336,
      "logps/chosen": -409.25,
      "logps/rejected": -535.3333333333334,
      "loss": 0.0969,
      "rewards/chosen": 6.833079814910889,
      "rewards/margins": 23.749746481577557,
      "rewards/rejected": -16.916666666666668,
      "step": 2019
    },
    {
      "epoch": 0.9950738916256158,
      "grad_norm": 0.5537069192985868,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -59586470.95652174,
      "logits/rejected": -57441504.780487806,
      "logps/chosen": -348.69565217391306,
      "logps/rejected": -540.8780487804878,
      "loss": 0.1381,
      "rewards/chosen": 3.4646646250849185,
      "rewards/margins": 19.44027438118248,
      "rewards/rejected": -15.975609756097562,
      "step": 2020
    },
    {
      "epoch": 0.9955665024630542,
      "grad_norm": 0.5292664705380207,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -52149179.733333334,
      "logits/rejected": -53970080.38554217,
      "logps/chosen": -244.0888888888889,
      "logps/rejected": -532.8192771084338,
      "loss": 0.1516,
      "rewards/chosen": 0.7916666666666666,
      "rewards/margins": 17.008534136546185,
      "rewards/rejected": -16.216867469879517,
      "step": 2021
    },
    {
      "epoch": 0.9960591133004926,
      "grad_norm": 0.4415067046191091,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -54220117.333333336,
      "logits/rejected": -63071846.4,
      "logps/chosen": -258.3333333333333,
      "logps/rejected": -574.8,
      "loss": 0.1513,
      "rewards/chosen": 1.83984375,
      "rewards/margins": 15.58984375,
      "rewards/rejected": -13.75,
      "step": 2022
    },
    {
      "epoch": 0.996551724137931,
      "grad_norm": 0.4572426282965045,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -50964750.49056604,
      "logits/rejected": -64592281.6,
      "logps/chosen": -249.66037735849056,
      "logps/rejected": -574.2933333333333,
      "loss": 0.1955,
      "rewards/chosen": -1.6882370283018868,
      "rewards/margins": 14.111762971698115,
      "rewards/rejected": -15.8,
      "step": 2023
    },
    {
      "epoch": 0.9970443349753695,
      "grad_norm": 0.3974246501382297,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -58606896.432432435,
      "logits/rejected": -56692240.87912088,
      "logps/chosen": -305.18918918918916,
      "logps/rejected": -551.3846153846154,
      "loss": 0.1138,
      "rewards/chosen": 6.941736169763513,
      "rewards/margins": 21.513164741192085,
      "rewards/rejected": -14.571428571428571,
      "step": 2024
    },
    {
      "epoch": 0.9975369458128078,
      "grad_norm": 0.592285784742669,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -44792001.20754717,
      "logits/rejected": -57322154.666666664,
      "logps/chosen": -250.26415094339623,
      "logps/rejected": -532.0533333333333,
      "loss": 0.1629,
      "rewards/chosen": 2.1533018867924527,
      "rewards/margins": 17.166635220125787,
      "rewards/rejected": -15.013333333333334,
      "step": 2025
    },
    {
      "epoch": 0.9980295566502463,
      "grad_norm": 0.42614290534623966,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -53338884.83018868,
      "logits/rejected": -50052027.733333334,
      "logps/chosen": -284.37735849056605,
      "logps/rejected": -523.0933333333334,
      "loss": 0.1708,
      "rewards/chosen": 3.2028961181640625,
      "rewards/margins": 18.136229451497396,
      "rewards/rejected": -14.933333333333334,
      "step": 2026
    },
    {
      "epoch": 0.9985221674876847,
      "grad_norm": 0.4565901273316064,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -77969115.42857143,
      "logits/rejected": -60118357.333333336,
      "logps/chosen": -231.42857142857142,
      "logps/rejected": -571.5555555555555,
      "loss": 0.2095,
      "rewards/chosen": 2.0569629669189453,
      "rewards/margins": 17.056962966918945,
      "rewards/rejected": -15.0,
      "step": 2027
    },
    {
      "epoch": 0.9990147783251232,
      "grad_norm": 0.4993073041107752,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -67066064.97959184,
      "logits/rejected": -53304825.51898734,
      "logps/chosen": -359.18367346938777,
      "logps/rejected": -538.7341772151899,
      "loss": 0.1233,
      "rewards/chosen": 2.431122448979592,
      "rewards/margins": 17.139983208473264,
      "rewards/rejected": -14.708860759493671,
      "step": 2028
    },
    {
      "epoch": 0.9995073891625615,
      "grad_norm": 0.4421760445439319,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -56331832.88888889,
      "logits/rejected": -61638032.69565217,
      "logps/chosen": -355.8888888888889,
      "logps/rejected": -502.2608695652174,
      "loss": 0.1312,
      "rewards/chosen": 1.2013888888888888,
      "rewards/margins": 13.070954106280194,
      "rewards/rejected": -11.869565217391305,
      "step": 2029
    },
    {
      "epoch": 1.0,
      "grad_norm": 0.5244999511649109,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "logits/chosen": -63652446.81481481,
      "logits/rejected": -68695897.94594595,
      "logps/chosen": -347.25925925925924,
      "logps/rejected": -559.5675675675676,
      "loss": 0.1863,
      "rewards/chosen": 0.8634259259259259,
      "rewards/margins": 13.971534034034034,
      "rewards/rejected": -13.108108108108109,
      "step": 2030
    }
  ],
  "logging_steps": 1,
  "max_steps": 2030,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 508,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}