{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.012329886102677127,
  "eval_steps": 500,
  "global_step": 400,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 3.082471525669282e-05,
      "grad_norm": 0.4096410633951961,
      "kl": 0.0,
      "learning_rate": 1e-05,
      "loss": 0.0,
      "reward": 0.27293290942907333,
      "reward_std": 0.0449131247587502,
      "rewards/clip_reward": 0.27293290942907333,
      "step": 1
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 6.164943051338563e-05,
      "grad_norm": 2.7610875519018947,
      "kl": 0.048065185546875,
      "learning_rate": 1e-05,
      "loss": 0.0019,
      "reward": 0.27704446762800217,
      "reward_std": 0.04381310846656561,
      "rewards/clip_reward": 0.27704446762800217,
      "step": 2
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 9.247414577007844e-05,
      "grad_norm": 2.6854813853406663,
      "kl": 0.08209228515625,
      "learning_rate": 1e-05,
      "loss": 0.0033,
      "reward": 0.24470487236976624,
      "reward_std": 0.039878456853330135,
      "rewards/clip_reward": 0.24470487236976624,
      "step": 3
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.00012329886102677127,
      "grad_norm": 1.844139759481107,
      "kl": 0.094390869140625,
      "learning_rate": 1e-05,
      "loss": 0.0038,
      "reward": 0.2730746790766716,
      "reward_std": 0.03929354250431061,
      "rewards/clip_reward": 0.2730746790766716,
      "step": 4
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.00015412357628346408,
      "grad_norm": 2.648057613593892,
      "kl": 0.149658203125,
      "learning_rate": 1e-05,
      "loss": 0.006,
      "reward": 0.22251639142632484,
      "reward_std": 0.035196226090192795,
      "rewards/clip_reward": 0.22251639142632484,
      "step": 5
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.0001849482915401569,
      "grad_norm": 2320.3210476556847,
      "kl": 35.0732421875,
      "learning_rate": 1e-05,
      "loss": 1.408,
      "reward": 0.26333311200141907,
      "reward_std": 0.043474785052239895,
      "rewards/clip_reward": 0.26333311200141907,
      "step": 6
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.00021577300679684973,
      "grad_norm": 7.940267210806191,
      "kl": 0.358489990234375,
      "learning_rate": 1e-05,
      "loss": 0.0144,
      "reward": 0.26793035864830017,
      "reward_std": 0.03859096858650446,
      "rewards/clip_reward": 0.26793035864830017,
      "step": 7
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.00024659772205354254,
      "grad_norm": 1.2528619108757855,
      "kl": 0.093505859375,
      "learning_rate": 1e-05,
      "loss": 0.0037,
      "reward": 0.25467605516314507,
      "reward_std": 0.0469845924526453,
      "rewards/clip_reward": 0.25467605516314507,
      "step": 8
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.0002774224373102353,
      "grad_norm": 0.3609296984986853,
      "kl": 0.03167724609375,
      "learning_rate": 1e-05,
      "loss": 0.0013,
      "reward": 0.2549675516784191,
      "reward_std": 0.050234788097441196,
      "rewards/clip_reward": 0.2549675516784191,
      "step": 9
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.00030824715256692816,
      "grad_norm": 0.340167108350015,
      "kl": 0.026275634765625,
      "learning_rate": 1e-05,
      "loss": 0.0011,
      "reward": 0.2656533867120743,
      "reward_std": 0.041638208553195,
      "rewards/clip_reward": 0.2656533867120743,
      "step": 10
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.000339071867823621,
      "grad_norm": 1.2337998822225458,
      "kl": 0.038787841796875,
      "learning_rate": 1e-05,
      "loss": 0.0016,
      "reward": 0.23507770523428917,
      "reward_std": 0.03971440531313419,
      "rewards/clip_reward": 0.23507770523428917,
      "step": 11
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.0003698965830803138,
      "grad_norm": 0.4192393740192319,
      "kl": 0.027679443359375,
      "learning_rate": 1e-05,
      "loss": 0.0011,
      "reward": 0.28236085921525955,
      "reward_std": 0.047151658684015274,
      "rewards/clip_reward": 0.28236085921525955,
      "step": 12
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.0004007212983370066,
      "grad_norm": 0.44693100447868084,
      "kl": 0.04705810546875,
      "learning_rate": 1e-05,
      "loss": 0.0019,
      "reward": 0.2532140128314495,
      "reward_std": 0.045407955069094896,
      "rewards/clip_reward": 0.2532140128314495,
      "step": 13
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.00043154601359369945,
      "grad_norm": 0.4438582117717643,
      "kl": 0.031341552734375,
      "learning_rate": 1e-05,
      "loss": 0.0013,
      "reward": 0.2620070278644562,
      "reward_std": 0.044824035838246346,
      "rewards/clip_reward": 0.2620070278644562,
      "step": 14
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.00046237072885039224,
      "grad_norm": 0.4582616987800618,
      "kl": 0.029937744140625,
      "learning_rate": 1e-05,
      "loss": 0.0012,
      "reward": 0.2698071673512459,
      "reward_std": 0.044542488642036915,
      "rewards/clip_reward": 0.2698071673512459,
      "step": 15
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.0004931954441070851,
      "grad_norm": 1.7444707877124468,
      "kl": 0.171722412109375,
      "learning_rate": 1e-05,
      "loss": 0.0069,
      "reward": 0.25084393844008446,
      "reward_std": 0.03859854955226183,
      "rewards/clip_reward": 0.25084393844008446,
      "step": 16
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.0005240201593637779,
      "grad_norm": 0.9397492604252154,
      "kl": 0.03289794921875,
      "learning_rate": 1e-05,
      "loss": 0.0013,
      "reward": 0.2502448856830597,
      "reward_std": 0.046720145270228386,
      "rewards/clip_reward": 0.2502448856830597,
      "step": 17
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.0005548448746204706,
      "grad_norm": 0.2795480658769616,
      "kl": 0.02972412109375,
      "learning_rate": 1e-05,
      "loss": 0.0012,
      "reward": 0.25642987713217735,
      "reward_std": 0.040699112229049206,
      "rewards/clip_reward": 0.25642987713217735,
      "step": 18
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.0005856695898771635,
      "grad_norm": 0.42906609124114997,
      "kl": 0.030792236328125,
      "learning_rate": 1e-05,
      "loss": 0.0012,
      "reward": 0.26084326952695847,
      "reward_std": 0.046253882348537445,
      "rewards/clip_reward": 0.26084326952695847,
      "step": 19
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.0006164943051338563,
      "grad_norm": 0.27464154300178234,
      "kl": 0.034576416015625,
      "learning_rate": 1e-05,
      "loss": 0.0014,
      "reward": 0.2642279714345932,
      "reward_std": 0.04200063832104206,
      "rewards/clip_reward": 0.2642279714345932,
      "step": 20
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.0006473190203905491,
      "grad_norm": 0.28947614961512397,
      "kl": 0.030029296875,
      "learning_rate": 1e-05,
      "loss": 0.0012,
      "reward": 0.25506409257650375,
      "reward_std": 0.0439683748409152,
      "rewards/clip_reward": 0.25506409257650375,
      "step": 21
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.000678143735647242,
      "grad_norm": 0.3388326949584448,
      "kl": 0.0302734375,
      "learning_rate": 1e-05,
      "loss": 0.0012,
      "reward": 0.2455521523952484,
      "reward_std": 0.04773388337343931,
      "rewards/clip_reward": 0.2455521523952484,
      "step": 22
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.0007089684509039348,
      "grad_norm": 0.251274728050997,
      "kl": 0.031219482421875,
      "learning_rate": 1e-05,
      "loss": 0.0012,
      "reward": 0.26682381331920624,
      "reward_std": 0.045767911709845066,
      "rewards/clip_reward": 0.26682381331920624,
      "step": 23
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.0007397931661606276,
      "grad_norm": 0.3330664131200773,
      "kl": 0.03167724609375,
      "learning_rate": 1e-05,
      "loss": 0.0013,
      "reward": 0.27093350887298584,
      "reward_std": 0.04017470218241215,
      "rewards/clip_reward": 0.27093350887298584,
      "step": 24
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.0007706178814173204,
      "grad_norm": 0.2855775494556483,
      "kl": 0.030181884765625,
      "learning_rate": 1e-05,
      "loss": 0.0012,
      "reward": 0.2470133751630783,
      "reward_std": 0.045277868397533894,
      "rewards/clip_reward": 0.2470133751630783,
      "step": 25
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.0008014425966740132,
      "grad_norm": 0.23359940849678845,
      "kl": 0.03143310546875,
      "learning_rate": 1e-05,
      "loss": 0.0013,
      "reward": 0.2605663165450096,
      "reward_std": 0.036497367545962334,
      "rewards/clip_reward": 0.2605663165450096,
      "step": 26
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.000832267311930706,
      "grad_norm": 0.2565562945436733,
      "kl": 0.031494140625,
      "learning_rate": 1e-05,
      "loss": 0.0013,
      "reward": 0.25371479243040085,
      "reward_std": 0.04583714855834842,
      "rewards/clip_reward": 0.25371479243040085,
      "step": 27
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.0008630920271873989,
      "grad_norm": 0.37271054820762656,
      "kl": 0.0306396484375,
      "learning_rate": 1e-05,
      "loss": 0.0012,
      "reward": 0.2469511665403843,
      "reward_std": 0.035832284949719906,
      "rewards/clip_reward": 0.2469511665403843,
      "step": 28
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.0008939167424440917,
      "grad_norm": 0.24495908441078176,
      "kl": 0.03009033203125,
      "learning_rate": 1e-05,
      "loss": 0.0012,
      "reward": 0.2665953040122986,
      "reward_std": 0.0440530339255929,
      "rewards/clip_reward": 0.2665953040122986,
      "step": 29
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.0009247414577007845,
      "grad_norm": 0.27952316749339523,
      "kl": 0.029510498046875,
      "learning_rate": 1e-05,
      "loss": 0.0012,
      "reward": 0.2593040131032467,
      "reward_std": 0.04028139542788267,
      "rewards/clip_reward": 0.2593040131032467,
      "step": 30
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.0009555661729574773,
      "grad_norm": 0.29685109479945,
      "kl": 0.03131103515625,
      "learning_rate": 1e-05,
      "loss": 0.0013,
      "reward": 0.2597034126520157,
      "reward_std": 0.045073311775922775,
      "rewards/clip_reward": 0.2597034126520157,
      "step": 31
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.0009863908882141701,
      "grad_norm": 0.2788133739842884,
      "kl": 0.029815673828125,
      "learning_rate": 1e-05,
      "loss": 0.0012,
      "reward": 0.26862068474292755,
      "reward_std": 0.04331609793007374,
      "rewards/clip_reward": 0.26862068474292755,
      "step": 32
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.001017215603470863,
      "grad_norm": 0.2793158814370596,
      "kl": 0.032684326171875,
      "learning_rate": 1e-05,
      "loss": 0.0013,
      "reward": 0.2581031806766987,
      "reward_std": 0.04781654942780733,
      "rewards/clip_reward": 0.2581031806766987,
      "step": 33
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.0010480403187275557,
      "grad_norm": 0.2410712843439065,
      "kl": 0.031158447265625,
      "learning_rate": 1e-05,
      "loss": 0.0012,
      "reward": 0.25184522196650505,
      "reward_std": 0.04620496183633804,
      "rewards/clip_reward": 0.25184522196650505,
      "step": 34
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.0010788650339842486,
      "grad_norm": 0.6426585983726603,
      "kl": 0.034698486328125,
      "learning_rate": 1e-05,
      "loss": 0.0014,
      "reward": 0.26873790100216866,
      "reward_std": 0.03604905540123582,
      "rewards/clip_reward": 0.26873790100216866,
      "step": 35
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.0011096897492409413,
      "grad_norm": 0.30049643240824964,
      "kl": 0.028167724609375,
      "learning_rate": 1e-05,
      "loss": 0.0011,
      "reward": 0.28773902356624603,
      "reward_std": 0.04903798084706068,
      "rewards/clip_reward": 0.28773902356624603,
      "step": 36
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.0011405144644976342,
      "grad_norm": 0.286651911122606,
      "kl": 0.03125,
      "learning_rate": 1e-05,
      "loss": 0.0013,
      "reward": 0.2746337428689003,
      "reward_std": 0.039260104298591614,
      "rewards/clip_reward": 0.2746337428689003,
      "step": 37
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.001171339179754327,
      "grad_norm": 0.23526530529450262,
      "kl": 0.03240966796875,
      "learning_rate": 1e-05,
      "loss": 0.0013,
      "reward": 0.2732752785086632,
      "reward_std": 0.03741883207112551,
      "rewards/clip_reward": 0.2732752785086632,
      "step": 38
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.0012021638950110197,
      "grad_norm": 0.23423409222245298,
      "kl": 0.0341796875,
      "learning_rate": 1e-05,
      "loss": 0.0014,
      "reward": 0.24805432185530663,
      "reward_std": 0.03564950963482261,
      "rewards/clip_reward": 0.24805432185530663,
      "step": 39
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.0012329886102677126,
      "grad_norm": 0.24360760682907664,
      "kl": 0.03375244140625,
      "learning_rate": 1e-05,
      "loss": 0.0014,
      "reward": 0.2747276946902275,
      "reward_std": 0.04160565137863159,
      "rewards/clip_reward": 0.2747276946902275,
      "step": 40
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.0012638133255244055,
      "grad_norm": 0.24537882409018658,
      "kl": 0.0321044921875,
      "learning_rate": 1e-05,
      "loss": 0.0013,
      "reward": 0.2780498266220093,
      "reward_std": 0.036946577951312065,
      "rewards/clip_reward": 0.2780498266220093,
      "step": 41
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.0012946380407810982,
      "grad_norm": 0.40977788179371805,
      "kl": 0.03350830078125,
      "learning_rate": 1e-05,
      "loss": 0.0013,
      "reward": 0.2628883123397827,
      "reward_std": 0.04087233170866966,
      "rewards/clip_reward": 0.2628883123397827,
      "step": 42
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.001325462756037791,
      "grad_norm": 0.3840340108983825,
      "kl": 0.0338134765625,
      "learning_rate": 1e-05,
      "loss": 0.0014,
      "reward": 0.27363643795251846,
      "reward_std": 0.03607825469225645,
      "rewards/clip_reward": 0.27363643795251846,
      "step": 43
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.001356287471294484,
      "grad_norm": 0.2581672136879213,
      "kl": 0.03271484375,
      "learning_rate": 1e-05,
      "loss": 0.0013,
      "reward": 0.2834562137722969,
      "reward_std": 0.04080742411315441,
      "rewards/clip_reward": 0.2834562137722969,
      "step": 44
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.0013871121865511767,
      "grad_norm": 0.22063460755244194,
      "kl": 0.03253173828125,
      "learning_rate": 1e-05,
      "loss": 0.0013,
      "reward": 0.2772428020834923,
      "reward_std": 0.04003895306959748,
      "rewards/clip_reward": 0.2772428020834923,
      "step": 45
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.0014179369018078695,
      "grad_norm": 0.2565525371064913,
      "kl": 0.03271484375,
      "learning_rate": 1e-05,
      "loss": 0.0013,
      "reward": 0.2706209644675255,
      "reward_std": 0.04194391146302223,
      "rewards/clip_reward": 0.2706209644675255,
      "step": 46
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.0014487616170645624,
      "grad_norm": 0.2490150749271912,
      "kl": 0.03497314453125,
      "learning_rate": 1e-05,
      "loss": 0.0014,
      "reward": 0.2823899984359741,
      "reward_std": 0.04330089082941413,
      "rewards/clip_reward": 0.2823899984359741,
      "step": 47
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.0014795863323212551,
      "grad_norm": 0.31942389183248127,
      "kl": 0.033203125,
      "learning_rate": 1e-05,
      "loss": 0.0013,
      "reward": 0.27612780779600143,
      "reward_std": 0.03909243643283844,
      "rewards/clip_reward": 0.27612780779600143,
      "step": 48
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.001510411047577948,
      "grad_norm": 0.25176125337143185,
      "kl": 0.0323486328125,
      "learning_rate": 1e-05,
      "loss": 0.0013,
      "reward": 0.27114395797252655,
      "reward_std": 0.043553344905376434,
      "rewards/clip_reward": 0.27114395797252655,
      "step": 49
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.001541235762834641,
      "grad_norm": 0.25140342833503415,
      "kl": 0.03302001953125,
      "learning_rate": 1e-05,
      "loss": 0.0013,
      "reward": 0.2741701751947403,
      "reward_std": 0.04216139670461416,
      "rewards/clip_reward": 0.2741701751947403,
      "step": 50
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.0015720604780913336,
      "grad_norm": 0.2712169111225742,
      "kl": 0.0338134765625,
      "learning_rate": 1e-05,
      "loss": 0.0014,
      "reward": 0.2718612402677536,
      "reward_std": 0.0403730683028698,
      "rewards/clip_reward": 0.2718612402677536,
      "step": 51
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.0016028851933480265,
      "grad_norm": 0.24185377611726078,
      "kl": 0.03265380859375,
      "learning_rate": 1e-05,
      "loss": 0.0013,
      "reward": 0.2797749862074852,
      "reward_std": 0.035524442326277494,
      "rewards/clip_reward": 0.2797749862074852,
      "step": 52
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.0016337099086047194,
      "grad_norm": 0.2925738372367075,
      "kl": 0.031890869140625,
      "learning_rate": 1e-05,
      "loss": 0.0013,
      "reward": 0.27509263157844543,
      "reward_std": 0.03820755332708359,
      "rewards/clip_reward": 0.27509263157844543,
      "step": 53
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.001664534623861412,
      "grad_norm": 0.2243087092506723,
      "kl": 0.037109375,
      "learning_rate": 1e-05,
      "loss": 0.0015,
      "reward": 0.2797403186559677,
      "reward_std": 0.03433629125356674,
      "rewards/clip_reward": 0.2797403186559677,
      "step": 54
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.001695359339118105,
      "grad_norm": 0.5392048466375525,
      "kl": 0.034881591796875,
      "learning_rate": 1e-05,
      "loss": 0.0014,
      "reward": 0.2881343886256218,
      "reward_std": 0.034057734068483114,
      "rewards/clip_reward": 0.2881343886256218,
      "step": 55
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.0017261840543747978,
      "grad_norm": 0.3005202554746241,
      "kl": 0.03155517578125,
      "learning_rate": 1e-05,
      "loss": 0.0013,
      "reward": 0.28284603357315063,
      "reward_std": 0.035686352755874395,
      "rewards/clip_reward": 0.28284603357315063,
      "step": 56
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.0017570087696314905,
      "grad_norm": 0.4427830267310237,
      "kl": 0.03265380859375,
      "learning_rate": 1e-05,
      "loss": 0.0013,
      "reward": 0.2999447137117386,
      "reward_std": 0.037478470243513584,
      "rewards/clip_reward": 0.2999447137117386,
      "step": 57
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.0017878334848881834,
      "grad_norm": 0.3046115937307048,
      "kl": 0.03387451171875,
      "learning_rate": 1e-05,
      "loss": 0.0014,
      "reward": 0.2802000045776367,
      "reward_std": 0.042125691659748554,
      "rewards/clip_reward": 0.2802000045776367,
      "step": 58
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.001818658200144876,
      "grad_norm": 0.2405611837420283,
      "kl": 0.03338623046875,
      "learning_rate": 1e-05,
      "loss": 0.0013,
      "reward": 0.29059184342622757,
      "reward_std": 0.037164853885769844,
      "rewards/clip_reward": 0.29059184342622757,
      "step": 59
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.001849482915401569,
      "grad_norm": 0.24105929492457603,
      "kl": 0.03192138671875,
      "learning_rate": 1e-05,
      "loss": 0.0013,
      "reward": 0.2885151877999306,
      "reward_std": 0.03311763470992446,
      "rewards/clip_reward": 0.2885151877999306,
      "step": 60
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.0018803076306582618,
      "grad_norm": 0.3666040943996562,
      "kl": 0.035308837890625,
      "learning_rate": 1e-05,
      "loss": 0.0014,
      "reward": 0.2791602984070778,
      "reward_std": 0.04032097943127155,
      "rewards/clip_reward": 0.2791602984070778,
      "step": 61
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.0019111323459149545,
      "grad_norm": 0.30249506470602205,
      "kl": 0.0343017578125,
      "learning_rate": 1e-05,
      "loss": 0.0014,
      "reward": 0.2850872874259949,
      "reward_std": 0.04306080937385559,
      "rewards/clip_reward": 0.2850872874259949,
      "step": 62
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.0019419570611716474,
      "grad_norm": 0.2627870943706208,
      "kl": 0.0323486328125,
      "learning_rate": 1e-05,
      "loss": 0.0013,
      "reward": 0.28769853711128235,
      "reward_std": 0.03929570922628045,
      "rewards/clip_reward": 0.28769853711128235,
      "step": 63
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.0019727817764283403,
      "grad_norm": 0.302176588700791,
      "kl": 0.03369140625,
      "learning_rate": 1e-05,
      "loss": 0.0013,
      "reward": 0.27818652242422104,
      "reward_std": 0.03506180923432112,
      "rewards/clip_reward": 0.27818652242422104,
      "step": 64
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.002003606491685033,
      "grad_norm": 0.24687044072190295,
      "kl": 0.0352783203125,
      "learning_rate": 1e-05,
      "loss": 0.0014,
      "reward": 0.30166226625442505,
      "reward_std": 0.03524617711082101,
      "rewards/clip_reward": 0.30166226625442505,
      "step": 65
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.002034431206941726,
      "grad_norm": 0.30419214057266597,
      "kl": 0.0382080078125,
      "learning_rate": 1e-05,
      "loss": 0.0015,
      "reward": 0.2780046612024307,
      "reward_std": 0.039600692223757505,
      "rewards/clip_reward": 0.2780046612024307,
      "step": 66
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.0020652559221984185,
      "grad_norm": 0.27119766585139016,
      "kl": 0.03387451171875,
      "learning_rate": 1e-05,
      "loss": 0.0014,
      "reward": 0.294754721224308,
      "reward_std": 0.034248299431055784,
      "rewards/clip_reward": 0.294754721224308,
      "step": 67
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.0020960806374551114,
      "grad_norm": 0.24490582535286695,
      "kl": 0.0340576171875,
      "learning_rate": 1e-05,
      "loss": 0.0014,
      "reward": 0.29758618772029877,
      "reward_std": 0.03921351861208677,
      "rewards/clip_reward": 0.29758618772029877,
      "step": 68
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.0021269053527118043,
      "grad_norm": 0.2906006315520449,
      "kl": 0.0374755859375,
      "learning_rate": 1e-05,
      "loss": 0.0015,
      "reward": 0.28643497824668884,
      "reward_std": 0.03893239703029394,
      "rewards/clip_reward": 0.28643497824668884,
      "step": 69
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.002157730067968497,
      "grad_norm": 0.2861214644137581,
      "kl": 0.0352783203125,
      "learning_rate": 1e-05,
      "loss": 0.0014,
      "reward": 0.2825808748602867,
      "reward_std": 0.03427910804748535,
      "rewards/clip_reward": 0.2825808748602867,
      "step": 70
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.00218855478322519,
      "grad_norm": 0.37011498013054983,
      "kl": 0.03814697265625,
      "learning_rate": 1e-05,
      "loss": 0.0015,
      "reward": 0.2854940667748451,
      "reward_std": 0.034281593747437,
      "rewards/clip_reward": 0.2854940667748451,
      "step": 71
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.0022193794984818826,
      "grad_norm": 0.24442639828837878,
      "kl": 0.0372314453125,
      "learning_rate": 1e-05,
      "loss": 0.0015,
      "reward": 0.28134068101644516,
      "reward_std": 0.03528659883886576,
      "rewards/clip_reward": 0.28134068101644516,
      "step": 72
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.0022502042137385755,
      "grad_norm": 0.4685706452282796,
      "kl": 0.04559326171875,
      "learning_rate": 1e-05,
      "loss": 0.0018,
      "reward": 0.2905724048614502,
      "reward_std": 0.040434951428323984,
      "rewards/clip_reward": 0.2905724048614502,
      "step": 73
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.0022810289289952683,
      "grad_norm": 0.2978744286423797,
      "kl": 0.03948974609375,
      "learning_rate": 1e-05,
      "loss": 0.0016,
      "reward": 0.2837352082133293,
      "reward_std": 0.037585786543786526,
      "rewards/clip_reward": 0.2837352082133293,
      "step": 74
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.0023118536442519612,
      "grad_norm": 0.2642233216051261,
      "kl": 0.03631591796875,
      "learning_rate": 1e-05,
      "loss": 0.0015,
      "reward": 0.2846062183380127,
      "reward_std": 0.032358222641050816,
      "rewards/clip_reward": 0.2846062183380127,
      "step": 75
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.002342678359508654,
      "grad_norm": 0.38580930898434906,
      "kl": 0.03814697265625,
      "learning_rate": 1e-05,
      "loss": 0.0015,
      "reward": 0.2916061654686928,
      "reward_std": 0.03999630082398653,
      "rewards/clip_reward": 0.2916061654686928,
      "step": 76
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.002373503074765347,
      "grad_norm": 0.2926547008151926,
      "kl": 0.03857421875,
      "learning_rate": 1e-05,
      "loss": 0.0015,
      "reward": 0.295952245593071,
      "reward_std": 0.03780945483595133,
      "rewards/clip_reward": 0.295952245593071,
      "step": 77
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.0024043277900220395,
      "grad_norm": 0.25208665362075805,
      "kl": 0.0377197265625,
      "learning_rate": 1e-05,
      "loss": 0.0015,
      "reward": 0.2766958400607109,
      "reward_std": 0.034185357857495546,
      "rewards/clip_reward": 0.2766958400607109,
      "step": 78
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.0024351525052787324,
      "grad_norm": 0.24422171158940864,
      "kl": 0.0357666015625,
      "learning_rate": 1e-05,
      "loss": 0.0014,
      "reward": 0.2962762862443924,
      "reward_std": 0.037650241516530514,
      "rewards/clip_reward": 0.2962762862443924,
      "step": 79
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.0024659772205354253,
      "grad_norm": 0.22603888808512623,
      "kl": 0.03509521484375,
      "learning_rate": 1e-05,
      "loss": 0.0014,
      "reward": 0.30113714188337326,
      "reward_std": 0.03855917416512966,
      "rewards/clip_reward": 0.30113714188337326,
      "step": 80
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.002496801935792118,
      "grad_norm": 0.37733238977424605,
      "kl": 0.0335693359375,
      "learning_rate": 1e-05,
      "loss": 0.0013,
      "reward": 0.28931064158678055,
      "reward_std": 0.041936729568988085,
      "rewards/clip_reward": 0.28931064158678055,
      "step": 81
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.002527626651048811,
      "grad_norm": 2397.9127472616574,
      "kl": 2.27880859375,
      "learning_rate": 1e-05,
      "loss": 0.0912,
      "reward": 0.2881544306874275,
      "reward_std": 0.03951821103692055,
      "rewards/clip_reward": 0.2881544306874275,
      "step": 82
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.002558451366305504,
      "grad_norm": 0.2551335788444245,
      "kl": 0.035888671875,
      "learning_rate": 1e-05,
      "loss": 0.0014,
      "reward": 0.29982026666402817,
      "reward_std": 0.041388670448213816,
      "rewards/clip_reward": 0.29982026666402817,
      "step": 83
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.0025892760815621964,
      "grad_norm": 0.2368918105634009,
      "kl": 0.0340576171875,
      "learning_rate": 1e-05,
      "loss": 0.0014,
      "reward": 0.29101932793855667,
      "reward_std": 0.02901851013302803,
      "rewards/clip_reward": 0.29101932793855667,
      "step": 84
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.0026201007968188893,
      "grad_norm": 0.37688170465955695,
      "kl": 0.0369873046875,
      "learning_rate": 1e-05,
      "loss": 0.0015,
      "reward": 0.28874391317367554,
      "reward_std": 0.03887001145631075,
      "rewards/clip_reward": 0.28874391317367554,
      "step": 85
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.002650925512075582,
      "grad_norm": 0.2519378074219347,
      "kl": 0.033203125,
      "learning_rate": 1e-05,
      "loss": 0.0013,
      "reward": 0.30362261086702347,
      "reward_std": 0.03999508544802666,
      "rewards/clip_reward": 0.30362261086702347,
      "step": 86
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.002681750227332275,
      "grad_norm": 0.35429225967137656,
      "kl": 0.03558349609375,
      "learning_rate": 1e-05,
      "loss": 0.0014,
      "reward": 0.28408853709697723,
      "reward_std": 0.038170519284904,
      "rewards/clip_reward": 0.28408853709697723,
      "step": 87
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.002712574942588968,
      "grad_norm": 0.24826760619770735,
      "kl": 0.03509521484375,
      "learning_rate": 1e-05,
      "loss": 0.0014,
      "reward": 0.2828948646783829,
      "reward_std": 0.030402940697968006,
      "rewards/clip_reward": 0.2828948646783829,
      "step": 88
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.002743399657845661,
      "grad_norm": 0.24050960559813894,
      "kl": 0.0369873046875,
      "learning_rate": 1e-05,
      "loss": 0.0015,
      "reward": 0.28854893147945404,
      "reward_std": 0.031570473685860634,
      "rewards/clip_reward": 0.28854893147945404,
      "step": 89
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.0027742243731023533,
      "grad_norm": 0.3157405419841437,
      "kl": 0.03387451171875,
      "learning_rate": 1e-05,
      "loss": 0.0014,
      "reward": 0.3040951117873192,
      "reward_std": 0.03156033856794238,
      "rewards/clip_reward": 0.3040951117873192,
      "step": 90
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.002805049088359046,
      "grad_norm": 0.255465711049786,
      "kl": 0.036865234375,
      "learning_rate": 1e-05,
      "loss": 0.0015,
      "reward": 0.28177835047245026,
      "reward_std": 0.03183179069310427,
      "rewards/clip_reward": 0.28177835047245026,
      "step": 91
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.002835873803615739,
      "grad_norm": 0.2754084224378065,
      "kl": 0.0357666015625,
      "learning_rate": 1e-05,
      "loss": 0.0014,
      "reward": 0.28247682750225067,
      "reward_std": 0.04544441122561693,
      "rewards/clip_reward": 0.28247682750225067,
      "step": 92
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.002866698518872432,
      "grad_norm": 0.30917890933321285,
      "kl": 0.03411865234375,
      "learning_rate": 1e-05,
      "loss": 0.0014,
      "reward": 0.30449671298265457,
      "reward_std": 0.038043808192014694,
      "rewards/clip_reward": 0.30449671298265457,
      "step": 93
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.002897523234129125,
      "grad_norm": 0.2739772980532729,
      "kl": 0.0352783203125,
      "learning_rate": 1e-05,
      "loss": 0.0014,
      "reward": 0.2972172349691391,
      "reward_std": 0.04181887488812208,
      "rewards/clip_reward": 0.2972172349691391,
      "step": 94
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.0029283479493858173,
      "grad_norm": 0.4882887735659522,
      "kl": 0.0369873046875,
      "learning_rate": 1e-05,
      "loss": 0.0015,
      "reward": 0.30260052531957626,
      "reward_std": 0.03962193429470062,
      "rewards/clip_reward": 0.30260052531957626,
      "step": 95
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.0029591726646425102,
      "grad_norm": 0.2758869488382661,
      "kl": 0.03765869140625,
      "learning_rate": 1e-05,
      "loss": 0.0015,
      "reward": 0.273670993745327,
      "reward_std": 0.034096458461135626,
      "rewards/clip_reward": 0.273670993745327,
      "step": 96
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.002989997379899203,
      "grad_norm": 0.3218139742013513,
      "kl": 0.0357666015625,
      "learning_rate": 1e-05,
      "loss": 0.0014,
      "reward": 0.2953087314963341,
      "reward_std": 0.03857262898236513,
      "rewards/clip_reward": 0.2953087314963341,
      "step": 97
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.003020822095155896,
      "grad_norm": 0.27436643426318075,
      "kl": 0.03802490234375,
      "learning_rate": 1e-05,
      "loss": 0.0015,
      "reward": 0.2942623794078827,
      "reward_std": 0.04214351158589125,
      "rewards/clip_reward": 0.2942623794078827,
      "step": 98
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.003051646810412589,
      "grad_norm": 0.46160645575216935,
      "kl": 0.0382080078125,
      "learning_rate": 1e-05,
      "loss": 0.0015,
      "reward": 0.2718002200126648,
      "reward_std": 0.03470963425934315,
      "rewards/clip_reward": 0.2718002200126648,
      "step": 99
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.003082471525669282,
      "grad_norm": 0.2710130788062584,
      "kl": 0.0394287109375,
      "learning_rate": 1e-05,
      "loss": 0.0016,
      "reward": 0.30108004808425903,
      "reward_std": 0.03474955866113305,
      "rewards/clip_reward": 0.30108004808425903,
      "step": 100
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.0031132962409259743,
      "grad_norm": 0.49951830790663787,
      "kl": 0.0382080078125,
      "learning_rate": 1e-05,
      "loss": 0.0015,
      "reward": 0.2938985303044319,
      "reward_std": 0.03756955498829484,
      "rewards/clip_reward": 0.2938985303044319,
      "step": 101
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.003144120956182667,
      "grad_norm": 0.309853879858268,
      "kl": 0.03790283203125,
      "learning_rate": 1e-05,
      "loss": 0.0015,
      "reward": 0.2770190164446831,
      "reward_std": 0.04118606820702553,
      "rewards/clip_reward": 0.2770190164446831,
      "step": 102
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.00317494567143936,
      "grad_norm": 0.303540839737249,
      "kl": 0.04150390625,
      "learning_rate": 1e-05,
      "loss": 0.0017,
      "reward": 0.3010944053530693,
      "reward_std": 0.03948027174919844,
      "rewards/clip_reward": 0.3010944053530693,
      "step": 103
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.003205770386696053,
      "grad_norm": 0.4242493698059439,
      "kl": 0.04302978515625,
      "learning_rate": 1e-05,
      "loss": 0.0017,
      "reward": 0.30070605874061584,
      "reward_std": 0.03413457376882434,
      "rewards/clip_reward": 0.30070605874061584,
      "step": 104
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.003236595101952746,
      "grad_norm": 0.2893333008136438,
      "kl": 0.04876708984375,
      "learning_rate": 1e-05,
      "loss": 0.002,
      "reward": 0.30694443732500076,
      "reward_std": 0.03970211138948798,
      "rewards/clip_reward": 0.30694443732500076,
      "step": 105
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.0032674198172094387,
      "grad_norm": 0.9533572888059393,
      "kl": 0.05682373046875,
      "learning_rate": 1e-05,
      "loss": 0.0023,
      "reward": 0.3035873845219612,
      "reward_std": 0.03094083722680807,
      "rewards/clip_reward": 0.3035873845219612,
      "step": 106
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.003298244532466131,
      "grad_norm": 0.6987792496689551,
      "kl": 0.0474853515625,
      "learning_rate": 1e-05,
      "loss": 0.0019,
      "reward": 0.26798417791724205,
      "reward_std": 0.0339839537627995,
      "rewards/clip_reward": 0.26798417791724205,
      "step": 107
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.003329069247722824,
      "grad_norm": 0.2695872192954547,
      "kl": 0.03973388671875,
      "learning_rate": 1e-05,
      "loss": 0.0016,
      "reward": 0.2790006026625633,
      "reward_std": 0.040701782796531916,
      "rewards/clip_reward": 0.2790006026625633,
      "step": 108
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.003359893962979517,
      "grad_norm": 2.392119350950025,
      "kl": 0.04656982421875,
      "learning_rate": 1e-05,
      "loss": 0.0019,
      "reward": 0.29154668748378754,
      "reward_std": 0.03984204959124327,
      "rewards/clip_reward": 0.29154668748378754,
      "step": 109
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.00339071867823621,
      "grad_norm": 0.30051030904173764,
      "kl": 0.0389404296875,
      "learning_rate": 1e-05,
      "loss": 0.0016,
      "reward": 0.28251485526561737,
      "reward_std": 0.03768534865230322,
      "rewards/clip_reward": 0.28251485526561737,
      "step": 110
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.0034215433934929027,
      "grad_norm": 0.33966462863188296,
      "kl": 0.04290771484375,
      "learning_rate": 1e-05,
      "loss": 0.0017,
      "reward": 0.28882861137390137,
      "reward_std": 0.030829247552901506,
      "rewards/clip_reward": 0.28882861137390137,
      "step": 111
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.0034523681087495956,
      "grad_norm": 0.27353141186845165,
      "kl": 0.043212890625,
      "learning_rate": 1e-05,
      "loss": 0.0017,
      "reward": 0.2902967408299446,
      "reward_std": 0.03842968260869384,
      "rewards/clip_reward": 0.2902967408299446,
      "step": 112
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.003483192824006288,
      "grad_norm": 0.31909208905959036,
      "kl": 0.039794921875,
      "learning_rate": 1e-05,
      "loss": 0.0016,
      "reward": 0.30095263570547104,
      "reward_std": 0.03839526232331991,
      "rewards/clip_reward": 0.30095263570547104,
      "step": 113
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.003514017539262981,
      "grad_norm": 0.2628770760125084,
      "kl": 0.04241943359375,
      "learning_rate": 1e-05,
      "loss": 0.0017,
      "reward": 0.30361922085285187,
      "reward_std": 0.03204685868695378,
      "rewards/clip_reward": 0.30361922085285187,
      "step": 114
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.003544842254519674,
      "grad_norm": 0.5077902647628189,
      "kl": 0.04266357421875,
      "learning_rate": 1e-05,
      "loss": 0.0017,
      "reward": 0.28539516031742096,
      "reward_std": 0.03661313094198704,
      "rewards/clip_reward": 0.28539516031742096,
      "step": 115
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.0035756669697763668,
      "grad_norm": 0.2750571920299247,
      "kl": 0.04144287109375,
      "learning_rate": 1e-05,
      "loss": 0.0017,
      "reward": 0.28498194366693497,
      "reward_std": 0.033483162987977266,
      "rewards/clip_reward": 0.28498194366693497,
      "step": 116
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.0036064916850330597,
      "grad_norm": 0.4090948836642478,
      "kl": 0.05047607421875,
      "learning_rate": 1e-05,
      "loss": 0.002,
      "reward": 0.2884545177221298,
      "reward_std": 0.03879341948777437,
      "rewards/clip_reward": 0.2884545177221298,
      "step": 117
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.003637316400289752,
      "grad_norm": 0.2747726497219782,
      "kl": 0.0418701171875,
      "learning_rate": 1e-05,
      "loss": 0.0017,
      "reward": 0.2886419966816902,
      "reward_std": 0.03201331151649356,
      "rewards/clip_reward": 0.2886419966816902,
      "step": 118
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.003668141115546445,
      "grad_norm": 0.24335488340779768,
      "kl": 0.041015625,
      "learning_rate": 1e-05,
      "loss": 0.0016,
      "reward": 0.29286789894104004,
      "reward_std": 0.03851965814828873,
      "rewards/clip_reward": 0.29286789894104004,
      "step": 119
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.003698965830803138,
      "grad_norm": 0.3272604578026692,
      "kl": 0.0421142578125,
      "learning_rate": 1e-05,
      "loss": 0.0017,
      "reward": 0.3037740960717201,
      "reward_std": 0.04023708775639534,
      "rewards/clip_reward": 0.3037740960717201,
      "step": 120
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.003729790546059831,
      "grad_norm": 0.2573037344425057,
      "kl": 0.042724609375,
      "learning_rate": 1e-05,
      "loss": 0.0017,
      "reward": 0.3032229617238045,
      "reward_std": 0.03995621297508478,
      "rewards/clip_reward": 0.3032229617238045,
      "step": 121
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.0037606152613165237,
      "grad_norm": 0.23884237904332833,
      "kl": 0.040771484375,
      "learning_rate": 1e-05,
      "loss": 0.0016,
      "reward": 0.2992851510643959,
      "reward_std": 0.03295802231878042,
      "rewards/clip_reward": 0.2992851510643959,
      "step": 122
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.0037914399765732166,
      "grad_norm": 0.2563902015046439,
      "kl": 0.03961181640625,
      "learning_rate": 1e-05,
      "loss": 0.0016,
      "reward": 0.29188019037246704,
      "reward_std": 0.03674457548186183,
      "rewards/clip_reward": 0.29188019037246704,
      "step": 123
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.003822264691829909,
      "grad_norm": 0.2983101831944227,
      "kl": 0.0421142578125,
      "learning_rate": 1e-05,
      "loss": 0.0017,
      "reward": 0.2967969849705696,
      "reward_std": 0.035267810337245464,
      "rewards/clip_reward": 0.2967969849705696,
      "step": 124
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.003853089407086602,
      "grad_norm": 0.24729135355433982,
      "kl": 0.04266357421875,
      "learning_rate": 1e-05,
      "loss": 0.0017,
      "reward": 0.2964273914694786,
      "reward_std": 0.03762258403003216,
      "rewards/clip_reward": 0.2964273914694786,
      "step": 125
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.003883914122343295,
      "grad_norm": 0.36172960084474337,
      "kl": 0.0379638671875,
      "learning_rate": 1e-05,
      "loss": 0.0015,
      "reward": 0.2893693521618843,
      "reward_std": 0.030456844717264175,
      "rewards/clip_reward": 0.2893693521618843,
      "step": 126
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.003914738837599988,
      "grad_norm": 1.3235252455659776,
      "kl": 0.06842041015625,
      "learning_rate": 1e-05,
      "loss": 0.0027,
      "reward": 0.28936315327882767,
      "reward_std": 0.032889596186578274,
      "rewards/clip_reward": 0.28936315327882767,
      "step": 127
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.003945563552856681,
      "grad_norm": 0.33282790631669,
      "kl": 0.04095458984375,
      "learning_rate": 1e-05,
      "loss": 0.0016,
      "reward": 0.3032195121049881,
      "reward_std": 0.03782738745212555,
      "rewards/clip_reward": 0.3032195121049881,
      "step": 128
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.0039763882681133735,
      "grad_norm": 0.26941621679510896,
      "kl": 0.0401611328125,
      "learning_rate": 1e-05,
      "loss": 0.0016,
      "reward": 0.29521265625953674,
      "reward_std": 0.028109443373978138,
      "rewards/clip_reward": 0.29521265625953674,
      "step": 129
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.004007212983370066,
      "grad_norm": 0.6011583420828783,
      "kl": 0.042236328125,
      "learning_rate": 1e-05,
      "loss": 0.0017,
      "reward": 0.29040753841400146,
      "reward_std": 0.03746784944087267,
      "rewards/clip_reward": 0.29040753841400146,
      "step": 130
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.004038037698626759,
      "grad_norm": 0.28850122675094336,
      "kl": 0.03900146484375,
      "learning_rate": 1e-05,
      "loss": 0.0016,
      "reward": 0.298222653567791,
      "reward_std": 0.032059306744486094,
      "rewards/clip_reward": 0.298222653567791,
      "step": 131
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.004068862413883452,
      "grad_norm": 0.297623960565589,
      "kl": 0.04571533203125,
      "learning_rate": 1e-05,
      "loss": 0.0018,
      "reward": 0.28365931659936905,
      "reward_std": 0.03448425652459264,
      "rewards/clip_reward": 0.28365931659936905,
      "step": 132
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.004099687129140144,
      "grad_norm": 0.33237851102895244,
      "kl": 0.04266357421875,
      "learning_rate": 1e-05,
      "loss": 0.0017,
      "reward": 0.30908743292093277,
      "reward_std": 0.03715136833488941,
      "rewards/clip_reward": 0.30908743292093277,
      "step": 133
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.004130511844396837,
      "grad_norm": 0.28432833313640343,
      "kl": 0.04046630859375,
      "learning_rate": 1e-05,
      "loss": 0.0016,
      "reward": 0.28690189868211746,
      "reward_std": 0.036837459076195955,
      "rewards/clip_reward": 0.28690189868211746,
      "step": 134
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.00416133655965353,
      "grad_norm": 0.289418120507171,
      "kl": 0.042236328125,
      "learning_rate": 1e-05,
      "loss": 0.0017,
      "reward": 0.27922137826681137,
      "reward_std": 0.04220917448401451,
      "rewards/clip_reward": 0.27922137826681137,
      "step": 135
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.004192161274910223,
      "grad_norm": 0.443250812892132,
      "kl": 0.04840087890625,
      "learning_rate": 1e-05,
      "loss": 0.0019,
      "reward": 0.29268399626016617,
      "reward_std": 0.042575713247060776,
      "rewards/clip_reward": 0.29268399626016617,
      "step": 136
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.004222985990166916,
      "grad_norm": 0.2936447251353246,
      "kl": 0.0419921875,
      "learning_rate": 1e-05,
      "loss": 0.0017,
      "reward": 0.3000137060880661,
      "reward_std": 0.03598225861787796,
      "rewards/clip_reward": 0.3000137060880661,
      "step": 137
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.004253810705423609,
      "grad_norm": 0.2638651763481444,
      "kl": 0.044189453125,
      "learning_rate": 1e-05,
      "loss": 0.0018,
      "reward": 0.29852742701768875,
      "reward_std": 0.03497701371088624,
      "rewards/clip_reward": 0.29852742701768875,
      "step": 138
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.0042846354206803015,
      "grad_norm": 0.6490276376018281,
      "kl": 0.04449462890625,
      "learning_rate": 1e-05,
      "loss": 0.0018,
      "reward": 0.2969634085893631,
      "reward_std": 0.03716578893363476,
      "rewards/clip_reward": 0.2969634085893631,
      "step": 139
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.004315460135936994,
      "grad_norm": 0.4290585504001145,
      "kl": 0.048583984375,
      "learning_rate": 1e-05,
      "loss": 0.0019,
      "reward": 0.30021680146455765,
      "reward_std": 0.04055328294634819,
      "rewards/clip_reward": 0.30021680146455765,
      "step": 140
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.004346284851193687,
      "grad_norm": 0.2880701541000658,
      "kl": 0.046142578125,
      "learning_rate": 1e-05,
      "loss": 0.0018,
      "reward": 0.2995045334100723,
      "reward_std": 0.03819493483752012,
      "rewards/clip_reward": 0.2995045334100723,
      "step": 141
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.00437710956645038,
      "grad_norm": 0.8593874213184524,
      "kl": 0.0513916015625,
      "learning_rate": 1e-05,
      "loss": 0.0021,
      "reward": 0.30462899804115295,
      "reward_std": 0.03424446424469352,
      "rewards/clip_reward": 0.30462899804115295,
      "step": 142
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.004407934281707073,
      "grad_norm": 0.21911976066768077,
      "kl": 0.04931640625,
      "learning_rate": 1e-05,
      "loss": 0.002,
      "reward": 0.3081146106123924,
      "reward_std": 0.03195359604433179,
      "rewards/clip_reward": 0.3081146106123924,
      "step": 143
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.004438758996963765,
      "grad_norm": 0.34265255923839133,
      "kl": 0.0526123046875,
      "learning_rate": 1e-05,
      "loss": 0.0021,
      "reward": 0.2893141508102417,
      "reward_std": 0.03520893771201372,
      "rewards/clip_reward": 0.2893141508102417,
      "step": 144
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.004469583712220458,
      "grad_norm": 0.3221549984597468,
      "kl": 0.05474853515625,
      "learning_rate": 1e-05,
      "loss": 0.0022,
      "reward": 0.3002786561846733,
      "reward_std": 0.033627052791416645,
      "rewards/clip_reward": 0.3002786561846733,
      "step": 145
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.004500408427477151,
      "grad_norm": 0.35731934724740755,
      "kl": 0.0506591796875,
      "learning_rate": 1e-05,
      "loss": 0.002,
      "reward": 0.2964288890361786,
      "reward_std": 0.03710630023851991,
      "rewards/clip_reward": 0.2964288890361786,
      "step": 146
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.004531233142733844,
      "grad_norm": 0.498796371247507,
      "kl": 0.063232421875,
      "learning_rate": 1e-05,
      "loss": 0.0025,
      "reward": 0.2942727655172348,
      "reward_std": 0.034661782905459404,
      "rewards/clip_reward": 0.2942727655172348,
      "step": 147
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.004562057857990537,
      "grad_norm": 0.35040193973506645,
      "kl": 0.04986572265625,
      "learning_rate": 1e-05,
      "loss": 0.002,
      "reward": 0.3076755926012993,
      "reward_std": 0.038345606066286564,
      "rewards/clip_reward": 0.3076755926012993,
      "step": 148
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.00459288257324723,
      "grad_norm": 0.284116312840595,
      "kl": 0.0498046875,
      "learning_rate": 1e-05,
      "loss": 0.002,
      "reward": 0.2897453159093857,
      "reward_std": 0.03225108701735735,
      "rewards/clip_reward": 0.2897453159093857,
      "step": 149
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.0046237072885039225,
      "grad_norm": 2.510868241115512,
      "kl": 0.0657958984375,
      "learning_rate": 1e-05,
      "loss": 0.0026,
      "reward": 0.28433967381715775,
      "reward_std": 0.03440393693745136,
      "rewards/clip_reward": 0.28433967381715775,
      "step": 150
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.004654532003760615,
      "grad_norm": 0.2919131449174652,
      "kl": 0.0528564453125,
      "learning_rate": 1e-05,
      "loss": 0.0021,
      "reward": 0.2939991131424904,
      "reward_std": 0.040119947865605354,
      "rewards/clip_reward": 0.2939991131424904,
      "step": 151
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.004685356719017308,
      "grad_norm": 0.2728005479353481,
      "kl": 0.0621337890625,
      "learning_rate": 1e-05,
      "loss": 0.0025,
      "reward": 0.31283795088529587,
      "reward_std": 0.04251454817131162,
      "rewards/clip_reward": 0.31283795088529587,
      "step": 152
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.004716181434274001,
      "grad_norm": 0.3046425258813171,
      "kl": 0.056884765625,
      "learning_rate": 1e-05,
      "loss": 0.0023,
      "reward": 0.3088100552558899,
      "reward_std": 0.03973885904997587,
      "rewards/clip_reward": 0.3088100552558899,
      "step": 153
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.004747006149530694,
      "grad_norm": 0.36647907531053076,
      "kl": 0.06378173828125,
      "learning_rate": 1e-05,
      "loss": 0.0026,
      "reward": 0.2990795224905014,
      "reward_std": 0.03181237168610096,
      "rewards/clip_reward": 0.2990795224905014,
      "step": 154
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.004777830864787387,
      "grad_norm": 0.2874990294452977,
      "kl": 0.05413818359375,
      "learning_rate": 1e-05,
      "loss": 0.0022,
      "reward": 0.29652759432792664,
      "reward_std": 0.034958623349666595,
      "rewards/clip_reward": 0.29652759432792664,
      "step": 155
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.004808655580044079,
      "grad_norm": 0.2571961957720778,
      "kl": 0.04888916015625,
      "learning_rate": 1e-05,
      "loss": 0.002,
      "reward": 0.30016767233610153,
      "reward_std": 0.03672470944002271,
      "rewards/clip_reward": 0.30016767233610153,
      "step": 156
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.004839480295300772,
      "grad_norm": 0.2488580090168899,
      "kl": 0.05389404296875,
      "learning_rate": 1e-05,
      "loss": 0.0022,
      "reward": 0.3036653697490692,
      "reward_std": 0.03518738830462098,
      "rewards/clip_reward": 0.3036653697490692,
      "step": 157
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.004870305010557465,
      "grad_norm": 0.3740970067478605,
      "kl": 0.0546875,
      "learning_rate": 1e-05,
      "loss": 0.0022,
      "reward": 0.3022882342338562,
      "reward_std": 0.040427629835903645,
      "rewards/clip_reward": 0.3022882342338562,
      "step": 158
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.004901129725814158,
      "grad_norm": 0.27811303327109416,
      "kl": 0.0537109375,
      "learning_rate": 1e-05,
      "loss": 0.0021,
      "reward": 0.29319214075803757,
      "reward_std": 0.03445435827597976,
      "rewards/clip_reward": 0.29319214075803757,
      "step": 159
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.0049319544410708505,
      "grad_norm": 0.3118292624775214,
      "kl": 0.0531005859375,
      "learning_rate": 1e-05,
      "loss": 0.0021,
      "reward": 0.29692448675632477,
      "reward_std": 0.02798191551119089,
      "rewards/clip_reward": 0.29692448675632477,
      "step": 160
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.004962779156327543,
      "grad_norm": 0.24174045123252508,
      "kl": 0.05987548828125,
      "learning_rate": 1e-05,
      "loss": 0.0024,
      "reward": 0.2794330157339573,
      "reward_std": 0.033141561318188906,
      "rewards/clip_reward": 0.2794330157339573,
      "step": 161
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.004993603871584236,
      "grad_norm": 0.3249638089165546,
      "kl": 0.0574951171875,
      "learning_rate": 1e-05,
      "loss": 0.0023,
      "reward": 0.3030798211693764,
      "reward_std": 0.03338931826874614,
      "rewards/clip_reward": 0.3030798211693764,
      "step": 162
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.005024428586840929,
      "grad_norm": 0.2633110889874991,
      "kl": 0.05413818359375,
      "learning_rate": 1e-05,
      "loss": 0.0022,
      "reward": 0.29485588520765305,
      "reward_std": 0.03062314447015524,
      "rewards/clip_reward": 0.29485588520765305,
      "step": 163
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.005055253302097622,
      "grad_norm": 0.2803956547691715,
      "kl": 0.05316162109375,
      "learning_rate": 1e-05,
      "loss": 0.0021,
      "reward": 0.2970665916800499,
      "reward_std": 0.036208128556609154,
      "rewards/clip_reward": 0.2970665916800499,
      "step": 164
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.005086078017354315,
      "grad_norm": 0.30407819707590145,
      "kl": 0.0552978515625,
      "learning_rate": 1e-05,
      "loss": 0.0022,
      "reward": 0.28191495686769485,
      "reward_std": 0.035574122332036495,
      "rewards/clip_reward": 0.28191495686769485,
      "step": 165
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.005116902732611008,
      "grad_norm": 0.7069970857165138,
      "kl": 0.05682373046875,
      "learning_rate": 1e-05,
      "loss": 0.0023,
      "reward": 0.3111490234732628,
      "reward_std": 0.03303992748260498,
      "rewards/clip_reward": 0.3111490234732628,
      "step": 166
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.0051477274478677,
      "grad_norm": 0.5033892459063507,
      "kl": 0.09039306640625,
      "learning_rate": 1e-05,
      "loss": 0.0036,
      "reward": 0.2889687344431877,
      "reward_std": 0.030694663524627686,
      "rewards/clip_reward": 0.2889687344431877,
      "step": 167
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.005178552163124393,
      "grad_norm": 0.41528612307548435,
      "kl": 0.056884765625,
      "learning_rate": 1e-05,
      "loss": 0.0023,
      "reward": 0.3075498342514038,
      "reward_std": 0.03620789339765906,
      "rewards/clip_reward": 0.3075498342514038,
      "step": 168
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.005209376878381086,
      "grad_norm": 0.32166704594267,
      "kl": 0.0555419921875,
      "learning_rate": 1e-05,
      "loss": 0.0022,
      "reward": 0.29549096524715424,
      "reward_std": 0.037705546244978905,
      "rewards/clip_reward": 0.29549096524715424,
      "step": 169
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.005240201593637779,
      "grad_norm": 0.21631365869879202,
      "kl": 0.05328369140625,
      "learning_rate": 1e-05,
      "loss": 0.0021,
      "reward": 0.2998160049319267,
      "reward_std": 0.03276947420090437,
      "rewards/clip_reward": 0.2998160049319267,
      "step": 170
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.0052710263088944715,
      "grad_norm": 0.2516096372371072,
      "kl": 0.05267333984375,
      "learning_rate": 1e-05,
      "loss": 0.0021,
      "reward": 0.31134357303380966,
      "reward_std": 0.03470080904662609,
      "rewards/clip_reward": 0.31134357303380966,
      "step": 171
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.005301851024151164,
      "grad_norm": 0.29419080361317934,
      "kl": 0.0538330078125,
      "learning_rate": 1e-05,
      "loss": 0.0022,
      "reward": 0.3061481937766075,
      "reward_std": 0.03180012432858348,
      "rewards/clip_reward": 0.3061481937766075,
      "step": 172
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.005332675739407857,
      "grad_norm": 0.2498996223533549,
      "kl": 0.0516357421875,
      "learning_rate": 1e-05,
      "loss": 0.0021,
      "reward": 0.2943173050880432,
      "reward_std": 0.037327985279262066,
      "rewards/clip_reward": 0.2943173050880432,
      "step": 173
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.00536350045466455,
      "grad_norm": 0.2978051890631816,
      "kl": 0.05413818359375,
      "learning_rate": 1e-05,
      "loss": 0.0022,
      "reward": 0.3035401850938797,
      "reward_std": 0.0322981933131814,
      "rewards/clip_reward": 0.3035401850938797,
      "step": 174
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.005394325169921243,
      "grad_norm": 0.24015029890010034,
      "kl": 0.05206298828125,
      "learning_rate": 1e-05,
      "loss": 0.0021,
      "reward": 0.30325891077518463,
      "reward_std": 0.04060996416956186,
      "rewards/clip_reward": 0.30325891077518463,
      "step": 175
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.005425149885177936,
      "grad_norm": 0.246629237118055,
      "kl": 0.05426025390625,
      "learning_rate": 1e-05,
      "loss": 0.0022,
      "reward": 0.296851709485054,
      "reward_std": 0.03895132802426815,
      "rewards/clip_reward": 0.296851709485054,
      "step": 176
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.005455974600434629,
      "grad_norm": 0.26568277866945256,
      "kl": 0.05633544921875,
      "learning_rate": 1e-05,
      "loss": 0.0023,
      "reward": 0.3115716502070427,
      "reward_std": 0.037875589448958635,
      "rewards/clip_reward": 0.3115716502070427,
      "step": 177
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.005486799315691322,
      "grad_norm": 0.2868547511774212,
      "kl": 0.05279541015625,
      "learning_rate": 1e-05,
      "loss": 0.0021,
      "reward": 0.31741077452898026,
      "reward_std": 0.03210792690515518,
      "rewards/clip_reward": 0.31741077452898026,
      "step": 178
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.005517624030948014,
      "grad_norm": 0.2559585074400544,
      "kl": 0.05120849609375,
      "learning_rate": 1e-05,
      "loss": 0.002,
      "reward": 0.3083617463707924,
      "reward_std": 0.0392221063375473,
      "rewards/clip_reward": 0.3083617463707924,
      "step": 179
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.005548448746204707,
      "grad_norm": 0.24542077699632728,
      "kl": 0.05108642578125,
      "learning_rate": 1e-05,
      "loss": 0.002,
      "reward": 0.30339157581329346,
      "reward_std": 0.03507227078080177,
      "rewards/clip_reward": 0.30339157581329346,
      "step": 180
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.0055792734614613995,
      "grad_norm": 0.2540319224053049,
      "kl": 0.05084228515625,
      "learning_rate": 1e-05,
      "loss": 0.002,
      "reward": 0.30484145134687424,
      "reward_std": 0.036596270743757486,
      "rewards/clip_reward": 0.30484145134687424,
      "step": 181
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.005610098176718092,
      "grad_norm": 0.37332314299297165,
      "kl": 0.0526123046875,
      "learning_rate": 1e-05,
      "loss": 0.0021,
      "reward": 0.30233796685934067,
      "reward_std": 0.030493673402816057,
      "rewards/clip_reward": 0.30233796685934067,
      "step": 182
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.005640922891974785,
      "grad_norm": 0.2303993900918174,
      "kl": 0.05133056640625,
      "learning_rate": 1e-05,
      "loss": 0.0021,
      "reward": 0.3098205327987671,
      "reward_std": 0.041695406660437584,
      "rewards/clip_reward": 0.3098205327987671,
      "step": 183
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.005671747607231478,
      "grad_norm": 0.2435717876843093,
      "kl": 0.05206298828125,
      "learning_rate": 1e-05,
      "loss": 0.0021,
      "reward": 0.29874222725629807,
      "reward_std": 0.029267210513353348,
      "rewards/clip_reward": 0.29874222725629807,
      "step": 184
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.005702572322488171,
      "grad_norm": 0.31961072153806486,
      "kl": 0.05194091796875,
      "learning_rate": 1e-05,
      "loss": 0.0021,
      "reward": 0.29779237508773804,
      "reward_std": 0.035278864204883575,
      "rewards/clip_reward": 0.29779237508773804,
      "step": 185
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.005733397037744864,
      "grad_norm": 0.21282193347830292,
      "kl": 0.0513916015625,
      "learning_rate": 1e-05,
      "loss": 0.0021,
      "reward": 0.3018389344215393,
      "reward_std": 0.03779670037329197,
      "rewards/clip_reward": 0.3018389344215393,
      "step": 186
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.005764221753001557,
      "grad_norm": 0.23426135318521582,
      "kl": 0.052001953125,
      "learning_rate": 1e-05,
      "loss": 0.0021,
      "reward": 0.30514706671237946,
      "reward_std": 0.03661506250500679,
      "rewards/clip_reward": 0.30514706671237946,
      "step": 187
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.00579504646825825,
      "grad_norm": 0.35531046478807454,
      "kl": 0.0570068359375,
      "learning_rate": 1e-05,
      "loss": 0.0023,
      "reward": 0.2925054803490639,
      "reward_std": 0.03318297350779176,
      "rewards/clip_reward": 0.2925054803490639,
      "step": 188
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.005825871183514943,
      "grad_norm": 0.26126815879357707,
      "kl": 0.05340576171875,
      "learning_rate": 1e-05,
      "loss": 0.0021,
      "reward": 0.30776098370552063,
      "reward_std": 0.03303914796561003,
      "rewards/clip_reward": 0.30776098370552063,
      "step": 189
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.005856695898771635,
      "grad_norm": 0.3913467133913027,
      "kl": 0.05584716796875,
      "learning_rate": 1e-05,
      "loss": 0.0022,
      "reward": 0.3134502246975899,
      "reward_std": 0.033362182322889566,
      "rewards/clip_reward": 0.3134502246975899,
      "step": 190
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.005887520614028328,
      "grad_norm": 0.5481147865863106,
      "kl": 0.05731201171875,
      "learning_rate": 1e-05,
      "loss": 0.0023,
      "reward": 0.3062956631183624,
      "reward_std": 0.03714533941820264,
      "rewards/clip_reward": 0.3062956631183624,
      "step": 191
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.0059183453292850205,
      "grad_norm": 0.38319922904187664,
      "kl": 0.0501708984375,
      "learning_rate": 1e-05,
      "loss": 0.002,
      "reward": 0.29817763715982437,
      "reward_std": 0.031161442399024963,
      "rewards/clip_reward": 0.29817763715982437,
      "step": 192
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.005949170044541713,
      "grad_norm": 0.7525966257009459,
      "kl": 0.05181884765625,
      "learning_rate": 1e-05,
      "loss": 0.0021,
      "reward": 0.2918746843934059,
      "reward_std": 0.04155853856354952,
      "rewards/clip_reward": 0.2918746843934059,
      "step": 193
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.005979994759798406,
      "grad_norm": 0.28730243211205864,
      "kl": 0.0511474609375,
      "learning_rate": 1e-05,
      "loss": 0.002,
      "reward": 0.2828526049852371,
      "reward_std": 0.03443083353340626,
      "rewards/clip_reward": 0.2828526049852371,
      "step": 194
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.006010819475055099,
      "grad_norm": 0.25755366719945655,
      "kl": 0.05511474609375,
      "learning_rate": 1e-05,
      "loss": 0.0022,
      "reward": 0.30451397597789764,
      "reward_std": 0.030547112692147493,
      "rewards/clip_reward": 0.30451397597789764,
      "step": 195
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.006041644190311792,
      "grad_norm": 0.28409469826599004,
      "kl": 0.0528564453125,
      "learning_rate": 1e-05,
      "loss": 0.0021,
      "reward": 0.3040374740958214,
      "reward_std": 0.03506700927391648,
      "rewards/clip_reward": 0.3040374740958214,
      "step": 196
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.006072468905568485,
      "grad_norm": 0.24631301394880073,
      "kl": 0.05291748046875,
      "learning_rate": 1e-05,
      "loss": 0.0021,
      "reward": 0.3064323738217354,
      "reward_std": 0.03699143324047327,
      "rewards/clip_reward": 0.3064323738217354,
      "step": 197
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.006103293620825178,
      "grad_norm": 0.4776380300716616,
      "kl": 0.06048583984375,
      "learning_rate": 1e-05,
      "loss": 0.0024,
      "reward": 0.31653470546007156,
      "reward_std": 0.03443007543683052,
      "rewards/clip_reward": 0.31653470546007156,
      "step": 198
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.006134118336081871,
      "grad_norm": 0.2717998039709889,
      "kl": 0.05657958984375,
      "learning_rate": 1e-05,
      "loss": 0.0023,
      "reward": 0.2982766330242157,
      "reward_std": 0.030803233850747347,
      "rewards/clip_reward": 0.2982766330242157,
      "step": 199
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.006164943051338564,
      "grad_norm": 0.28573048152617825,
      "kl": 0.05755615234375,
      "learning_rate": 1e-05,
      "loss": 0.0023,
      "reward": 0.2895183190703392,
      "reward_std": 0.031674451660364866,
      "rewards/clip_reward": 0.2895183190703392,
      "step": 200
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.0061957677665952565,
      "grad_norm": 0.3012206807393158,
      "kl": 0.055908203125,
      "learning_rate": 1e-05,
      "loss": 0.0022,
      "reward": 0.3014395534992218,
      "reward_std": 0.03418161487206817,
      "rewards/clip_reward": 0.3014395534992218,
      "step": 201
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.0062265924818519485,
      "grad_norm": 0.24480682812310384,
      "kl": 0.057373046875,
      "learning_rate": 1e-05,
      "loss": 0.0023,
      "reward": 0.28224068135023117,
      "reward_std": 0.03003824595361948,
      "rewards/clip_reward": 0.28224068135023117,
      "step": 202
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.006257417197108641,
      "grad_norm": 0.2531535146591285,
      "kl": 0.05511474609375,
      "learning_rate": 1e-05,
      "loss": 0.0022,
      "reward": 0.30891023576259613,
      "reward_std": 0.03753681713715196,
      "rewards/clip_reward": 0.30891023576259613,
      "step": 203
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.006288241912365334,
      "grad_norm": 0.23120591809436197,
      "kl": 0.05841064453125,
      "learning_rate": 1e-05,
      "loss": 0.0023,
      "reward": 0.3163115456700325,
      "reward_std": 0.02788963308557868,
      "rewards/clip_reward": 0.3163115456700325,
      "step": 204
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.006319066627622027,
      "grad_norm": 0.23022883641873151,
      "kl": 0.056884765625,
      "learning_rate": 1e-05,
      "loss": 0.0023,
      "reward": 0.2952466085553169,
      "reward_std": 0.03618847485631704,
      "rewards/clip_reward": 0.2952466085553169,
      "step": 205
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.00634989134287872,
      "grad_norm": 0.2604310735559812,
      "kl": 0.05767822265625,
      "learning_rate": 1e-05,
      "loss": 0.0023,
      "reward": 0.30279867351055145,
      "reward_std": 0.03358080657199025,
      "rewards/clip_reward": 0.30279867351055145,
      "step": 206
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.006380716058135413,
      "grad_norm": 0.3065404532343023,
      "kl": 0.06097412109375,
      "learning_rate": 1e-05,
      "loss": 0.0024,
      "reward": 0.2905031070113182,
      "reward_std": 0.03163592051714659,
      "rewards/clip_reward": 0.2905031070113182,
      "step": 207
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.006411540773392106,
      "grad_norm": 0.2302581935455144,
      "kl": 0.06036376953125,
      "learning_rate": 1e-05,
      "loss": 0.0024,
      "reward": 0.3036247417330742,
      "reward_std": 0.04002736788243055,
      "rewards/clip_reward": 0.3036247417330742,
      "step": 208
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.006442365488648799,
      "grad_norm": 0.22939030410663466,
      "kl": 0.0615234375,
      "learning_rate": 1e-05,
      "loss": 0.0025,
      "reward": 0.3065522834658623,
      "reward_std": 0.035838291980326176,
      "rewards/clip_reward": 0.3065522834658623,
      "step": 209
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.006473190203905492,
      "grad_norm": 1.1535119363825117,
      "kl": 0.171630859375,
      "learning_rate": 1e-05,
      "loss": 0.0068,
      "reward": 0.3034718856215477,
      "reward_std": 0.028684359975159168,
      "rewards/clip_reward": 0.3034718856215477,
      "step": 210
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.0065040149191621845,
      "grad_norm": 0.475789174353479,
      "kl": 0.06304931640625,
      "learning_rate": 1e-05,
      "loss": 0.0025,
      "reward": 0.3188106268644333,
      "reward_std": 0.03359405370429158,
      "rewards/clip_reward": 0.3188106268644333,
      "step": 211
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.006534839634418877,
      "grad_norm": 0.30958280497728724,
      "kl": 0.06011962890625,
      "learning_rate": 1e-05,
      "loss": 0.0024,
      "reward": 0.31080956757068634,
      "reward_std": 0.03438552375882864,
      "rewards/clip_reward": 0.31080956757068634,
      "step": 212
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.0065656643496755695,
      "grad_norm": 0.2945478977489792,
      "kl": 0.06329345703125,
      "learning_rate": 1e-05,
      "loss": 0.0025,
      "reward": 0.29854634404182434,
      "reward_std": 0.03470074059441686,
      "rewards/clip_reward": 0.29854634404182434,
      "step": 213
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.006596489064932262,
      "grad_norm": 0.311983746378144,
      "kl": 0.061279296875,
      "learning_rate": 1e-05,
      "loss": 0.0025,
      "reward": 0.310188353061676,
      "reward_std": 0.04189980635419488,
      "rewards/clip_reward": 0.310188353061676,
      "step": 214
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.006627313780188955,
      "grad_norm": 0.26157966652434694,
      "kl": 0.06146240234375,
      "learning_rate": 1e-05,
      "loss": 0.0025,
      "reward": 0.31515590101480484,
      "reward_std": 0.0373789188452065,
      "rewards/clip_reward": 0.31515590101480484,
      "step": 215
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.006658138495445648,
      "grad_norm": 0.3427812618986616,
      "kl": 0.06329345703125,
      "learning_rate": 1e-05,
      "loss": 0.0025,
      "reward": 0.31115034967660904,
      "reward_std": 0.03764009568840265,
      "rewards/clip_reward": 0.31115034967660904,
      "step": 216
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.006688963210702341,
      "grad_norm": 0.31645931728689874,
      "kl": 0.05767822265625,
      "learning_rate": 1e-05,
      "loss": 0.0023,
      "reward": 0.28984929621219635,
      "reward_std": 0.03542311815544963,
      "rewards/clip_reward": 0.28984929621219635,
      "step": 217
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.006719787925959034,
      "grad_norm": 0.2228608743429633,
      "kl": 0.05902099609375,
      "learning_rate": 1e-05,
      "loss": 0.0024,
      "reward": 0.3080795630812645,
      "reward_std": 0.03324572555720806,
      "rewards/clip_reward": 0.3080795630812645,
      "step": 218
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.006750612641215727,
      "grad_norm": 0.6337063521534685,
      "kl": 0.06610107421875,
      "learning_rate": 1e-05,
      "loss": 0.0026,
      "reward": 0.3004284203052521,
      "reward_std": 0.030279683880507946,
      "rewards/clip_reward": 0.3004284203052521,
      "step": 219
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.00678143735647242,
      "grad_norm": 0.35877724546468126,
      "kl": 0.0599365234375,
      "learning_rate": 1e-05,
      "loss": 0.0024,
      "reward": 0.2890964075922966,
      "reward_std": 0.03512955084443092,
      "rewards/clip_reward": 0.2890964075922966,
      "step": 220
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.006812262071729113,
      "grad_norm": 0.28709775498334184,
      "kl": 0.06353759765625,
      "learning_rate": 1e-05,
      "loss": 0.0025,
      "reward": 0.3098203092813492,
      "reward_std": 0.03407229436561465,
      "rewards/clip_reward": 0.3098203092813492,
      "step": 221
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.0068430867869858055,
      "grad_norm": 0.2587966892355202,
      "kl": 0.0631103515625,
      "learning_rate": 1e-05,
      "loss": 0.0025,
      "reward": 0.2910105660557747,
      "reward_std": 0.03544299304485321,
      "rewards/clip_reward": 0.2910105660557747,
      "step": 222
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.006873911502242498,
      "grad_norm": 0.20375336506340044,
      "kl": 0.06231689453125,
      "learning_rate": 1e-05,
      "loss": 0.0025,
      "reward": 0.3012443706393242,
      "reward_std": 0.03373287199065089,
      "rewards/clip_reward": 0.3012443706393242,
      "step": 223
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.006904736217499191,
      "grad_norm": 0.22796858972291945,
      "kl": 0.058349609375,
      "learning_rate": 1e-05,
      "loss": 0.0023,
      "reward": 0.27249932289123535,
      "reward_std": 0.034992088098078966,
      "rewards/clip_reward": 0.27249932289123535,
      "step": 224
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.006935560932755883,
      "grad_norm": 0.23923560288044327,
      "kl": 0.05914306640625,
      "learning_rate": 1e-05,
      "loss": 0.0024,
      "reward": 0.30393005907535553,
      "reward_std": 0.04046585503965616,
      "rewards/clip_reward": 0.30393005907535553,
      "step": 225
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.006966385648012576,
      "grad_norm": 0.24781024716109332,
      "kl": 0.05828857421875,
      "learning_rate": 1e-05,
      "loss": 0.0023,
      "reward": 0.29162784665822983,
      "reward_std": 0.03501830715686083,
      "rewards/clip_reward": 0.29162784665822983,
      "step": 226
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.006997210363269269,
      "grad_norm": 0.2709603773452262,
      "kl": 0.0609130859375,
      "learning_rate": 1e-05,
      "loss": 0.0024,
      "reward": 0.2914000228047371,
      "reward_std": 0.030948857311159372,
      "rewards/clip_reward": 0.2914000228047371,
      "step": 227
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.007028035078525962,
      "grad_norm": 0.21567469316732074,
      "kl": 0.0567626953125,
      "learning_rate": 1e-05,
      "loss": 0.0023,
      "reward": 0.3019692078232765,
      "reward_std": 0.038113043643534184,
      "rewards/clip_reward": 0.3019692078232765,
      "step": 228
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.007058859793782655,
      "grad_norm": 0.223865539956382,
      "kl": 0.05474853515625,
      "learning_rate": 1e-05,
      "loss": 0.0022,
      "reward": 0.276552177965641,
      "reward_std": 0.03583432175219059,
      "rewards/clip_reward": 0.276552177965641,
      "step": 229
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.007089684509039348,
      "grad_norm": 0.23022368412719968,
      "kl": 0.05950927734375,
      "learning_rate": 1e-05,
      "loss": 0.0024,
      "reward": 0.30329927057027817,
      "reward_std": 0.035789184272289276,
      "rewards/clip_reward": 0.30329927057027817,
      "step": 230
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.007120509224296041,
      "grad_norm": 0.2872615168609158,
      "kl": 0.05938720703125,
      "learning_rate": 1e-05,
      "loss": 0.0024,
      "reward": 0.3115493804216385,
      "reward_std": 0.03708712290972471,
      "rewards/clip_reward": 0.3115493804216385,
      "step": 231
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.0071513339395527335,
      "grad_norm": 0.34806552807362773,
      "kl": 0.06298828125,
      "learning_rate": 1e-05,
      "loss": 0.0025,
      "reward": 0.3054001107811928,
      "reward_std": 0.034296643920242786,
      "rewards/clip_reward": 0.3054001107811928,
      "step": 232
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.007182158654809426,
      "grad_norm": 0.286918714200751,
      "kl": 0.06072998046875,
      "learning_rate": 1e-05,
      "loss": 0.0024,
      "reward": 0.299840584397316,
      "reward_std": 0.029763209633529186,
      "rewards/clip_reward": 0.299840584397316,
      "step": 233
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.007212983370066119,
      "grad_norm": 0.2246601984953947,
      "kl": 0.0570068359375,
      "learning_rate": 1e-05,
      "loss": 0.0023,
      "reward": 0.3116656318306923,
      "reward_std": 0.038989217951893806,
      "rewards/clip_reward": 0.3116656318306923,
      "step": 234
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.007243808085322812,
      "grad_norm": 0.2300241445857201,
      "kl": 0.06207275390625,
      "learning_rate": 1e-05,
      "loss": 0.0025,
      "reward": 0.3119165748357773,
      "reward_std": 0.030884731095284224,
      "rewards/clip_reward": 0.3119165748357773,
      "step": 235
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.007274632800579504,
      "grad_norm": 0.37606313252297296,
      "kl": 0.06292724609375,
      "learning_rate": 1e-05,
      "loss": 0.0025,
      "reward": 0.2938763126730919,
      "reward_std": 0.03854981064796448,
      "rewards/clip_reward": 0.2938763126730919,
      "step": 236
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.007305457515836197,
      "grad_norm": 0.27390469202393175,
      "kl": 0.06268310546875,
      "learning_rate": 1e-05,
      "loss": 0.0025,
      "reward": 0.30878350138664246,
      "reward_std": 0.0367919635027647,
      "rewards/clip_reward": 0.30878350138664246,
      "step": 237
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.00733628223109289,
      "grad_norm": 0.3923664539501469,
      "kl": 0.06011962890625,
      "learning_rate": 1e-05,
      "loss": 0.0024,
      "reward": 0.31253431737422943,
      "reward_std": 0.03306168969720602,
      "rewards/clip_reward": 0.31253431737422943,
      "step": 238
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.007367106946349583,
      "grad_norm": 19.999839999172977,
      "kl": 0.11572265625,
      "learning_rate": 1e-05,
      "loss": 0.0046,
      "reward": 0.2926497831940651,
      "reward_std": 0.03439700556918979,
      "rewards/clip_reward": 0.2926497831940651,
      "step": 239
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.007397931661606276,
      "grad_norm": 0.3274619779549125,
      "kl": 0.0623779296875,
      "learning_rate": 1e-05,
      "loss": 0.0025,
      "reward": 0.31169167160987854,
      "reward_std": 0.03383385669440031,
      "rewards/clip_reward": 0.31169167160987854,
      "step": 240
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.007428756376862969,
      "grad_norm": 0.3930715184021957,
      "kl": 0.064453125,
      "learning_rate": 1e-05,
      "loss": 0.0026,
      "reward": 0.3149360120296478,
      "reward_std": 0.036022236570715904,
      "rewards/clip_reward": 0.3149360120296478,
      "step": 241
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.007459581092119662,
      "grad_norm": 0.23471411051429383,
      "kl": 0.0631103515625,
      "learning_rate": 1e-05,
      "loss": 0.0025,
      "reward": 0.2915840148925781,
      "reward_std": 0.04799613729119301,
      "rewards/clip_reward": 0.2915840148925781,
      "step": 242
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.0074904058073763545,
      "grad_norm": 0.25074369126631646,
      "kl": 0.06268310546875,
      "learning_rate": 1e-05,
      "loss": 0.0025,
      "reward": 0.30563782155513763,
      "reward_std": 0.036503950599581,
      "rewards/clip_reward": 0.30563782155513763,
      "step": 243
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.007521230522633047,
      "grad_norm": 0.25106966232242506,
      "kl": 0.06280517578125,
      "learning_rate": 1e-05,
      "loss": 0.0025,
      "reward": 0.31641364842653275,
      "reward_std": 0.042671964969486,
      "rewards/clip_reward": 0.31641364842653275,
      "step": 244
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.00755205523788974,
      "grad_norm": 0.24863565809024027,
      "kl": 0.06585693359375,
      "learning_rate": 1e-05,
      "loss": 0.0026,
      "reward": 0.29705505073070526,
      "reward_std": 0.02980767609551549,
      "rewards/clip_reward": 0.29705505073070526,
      "step": 245
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.007582879953146433,
      "grad_norm": 0.3215906875694069,
      "kl": 0.0626220703125,
      "learning_rate": 1e-05,
      "loss": 0.0025,
      "reward": 0.2966236397624016,
      "reward_std": 0.032666172832250595,
      "rewards/clip_reward": 0.2966236397624016,
      "step": 246
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.007613704668403126,
      "grad_norm": 0.25802005781105997,
      "kl": 0.0655517578125,
      "learning_rate": 1e-05,
      "loss": 0.0026,
      "reward": 0.29460806399583817,
      "reward_std": 0.03424982167780399,
      "rewards/clip_reward": 0.29460806399583817,
      "step": 247
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.007644529383659818,
      "grad_norm": 0.41480007000983,
      "kl": 0.06292724609375,
      "learning_rate": 1e-05,
      "loss": 0.0025,
      "reward": 0.30160878598690033,
      "reward_std": 0.03246263647451997,
      "rewards/clip_reward": 0.30160878598690033,
      "step": 248
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.007675354098916511,
      "grad_norm": 0.38518441595490016,
      "kl": 0.07415771484375,
      "learning_rate": 1e-05,
      "loss": 0.003,
      "reward": 0.2882128059864044,
      "reward_std": 0.03637995757162571,
      "rewards/clip_reward": 0.2882128059864044,
      "step": 249
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.007706178814173204,
      "grad_norm": 0.2296837125573533,
      "kl": 0.06439208984375,
      "learning_rate": 1e-05,
      "loss": 0.0026,
      "reward": 0.31887152045965195,
      "reward_std": 0.031468767672777176,
      "rewards/clip_reward": 0.31887152045965195,
      "step": 250
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.007737003529429897,
      "grad_norm": 0.2771302981384482,
      "kl": 0.06256103515625,
      "learning_rate": 1e-05,
      "loss": 0.0025,
      "reward": 0.29794805496931076,
      "reward_std": 0.03718498535454273,
      "rewards/clip_reward": 0.29794805496931076,
      "step": 251
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.00776782824468659,
      "grad_norm": 0.26425224371180456,
      "kl": 0.0660400390625,
      "learning_rate": 1e-05,
      "loss": 0.0026,
      "reward": 0.31550634652376175,
      "reward_std": 0.03392920456826687,
      "rewards/clip_reward": 0.31550634652376175,
      "step": 252
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.0077986529599432825,
      "grad_norm": 0.3039933530405912,
      "kl": 0.06451416015625,
      "learning_rate": 1e-05,
      "loss": 0.0026,
      "reward": 0.31224576383829117,
      "reward_std": 0.03579189581796527,
      "rewards/clip_reward": 0.31224576383829117,
      "step": 253
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.007829477675199975,
      "grad_norm": 0.21264552398464287,
      "kl": 0.066650390625,
      "learning_rate": 1e-05,
      "loss": 0.0027,
      "reward": 0.30356816947460175,
      "reward_std": 0.03573842905461788,
      "rewards/clip_reward": 0.30356816947460175,
      "step": 254
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.007860302390456668,
      "grad_norm": 0.23510191594709257,
      "kl": 0.06463623046875,
      "learning_rate": 1e-05,
      "loss": 0.0026,
      "reward": 0.29867023229599,
      "reward_std": 0.03537313872948289,
      "rewards/clip_reward": 0.29867023229599,
      "step": 255
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.007891127105713361,
      "grad_norm": 0.8798380521542469,
      "kl": 0.0672607421875,
      "learning_rate": 1e-05,
      "loss": 0.0027,
      "reward": 0.2907034158706665,
      "reward_std": 0.03144656075164676,
      "rewards/clip_reward": 0.2907034158706665,
      "step": 256
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.007921951820970054,
      "grad_norm": 0.30489147633127095,
      "kl": 0.06390380859375,
      "learning_rate": 1e-05,
      "loss": 0.0026,
      "reward": 0.2994539961218834,
      "reward_std": 0.036147382110357285,
      "rewards/clip_reward": 0.2994539961218834,
      "step": 257
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.007952776536226747,
      "grad_norm": 0.2724891171334766,
      "kl": 0.07373046875,
      "learning_rate": 1e-05,
      "loss": 0.003,
      "reward": 0.29950564354658127,
      "reward_std": 0.031049554236233234,
      "rewards/clip_reward": 0.29950564354658127,
      "step": 258
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.00798360125148344,
      "grad_norm": 0.2257138587782837,
      "kl": 0.0606689453125,
      "learning_rate": 1e-05,
      "loss": 0.0024,
      "reward": 0.31028740108013153,
      "reward_std": 0.03820298006758094,
      "rewards/clip_reward": 0.31028740108013153,
      "step": 259
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.008014425966740133,
      "grad_norm": 0.23904222780230097,
      "kl": 0.06512451171875,
      "learning_rate": 1e-05,
      "loss": 0.0026,
      "reward": 0.30369560420513153,
      "reward_std": 0.034888788126409054,
      "rewards/clip_reward": 0.30369560420513153,
      "step": 260
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.008045250681996826,
      "grad_norm": 0.26551280919013587,
      "kl": 0.069091796875,
      "learning_rate": 1e-05,
      "loss": 0.0028,
      "reward": 0.3005470857024193,
      "reward_std": 0.03615998663008213,
      "rewards/clip_reward": 0.3005470857024193,
      "step": 261
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.008076075397253519,
      "grad_norm": 0.2641814708454766,
      "kl": 0.067138671875,
      "learning_rate": 1e-05,
      "loss": 0.0027,
      "reward": 0.30573634058237076,
      "reward_std": 0.03119312133640051,
      "rewards/clip_reward": 0.30573634058237076,
      "step": 262
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.008106900112510211,
      "grad_norm": 0.2538072823054899,
      "kl": 0.067626953125,
      "learning_rate": 1e-05,
      "loss": 0.0027,
      "reward": 0.28952275961637497,
      "reward_std": 0.02620957838371396,
      "rewards/clip_reward": 0.28952275961637497,
      "step": 263
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.008137724827766904,
      "grad_norm": 0.27840725784727466,
      "kl": 0.0692138671875,
      "learning_rate": 1e-05,
      "loss": 0.0028,
      "reward": 0.31542008370161057,
      "reward_std": 0.037005496211349964,
      "rewards/clip_reward": 0.31542008370161057,
      "step": 264
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.008168549543023595,
      "grad_norm": 0.3168534549683751,
      "kl": 0.069091796875,
      "learning_rate": 1e-05,
      "loss": 0.0028,
      "reward": 0.2891938388347626,
      "reward_std": 0.032484165858477354,
      "rewards/clip_reward": 0.2891938388347626,
      "step": 265
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.008199374258280288,
      "grad_norm": 0.34875471295741794,
      "kl": 0.071533203125,
      "learning_rate": 1e-05,
      "loss": 0.0029,
      "reward": 0.3205078914761543,
      "reward_std": 0.042449533008039,
      "rewards/clip_reward": 0.3205078914761543,
      "step": 266
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.008230198973536981,
      "grad_norm": 0.2913604348839751,
      "kl": 0.0697021484375,
      "learning_rate": 1e-05,
      "loss": 0.0028,
      "reward": 0.3089185729622841,
      "reward_std": 0.03769417991861701,
      "rewards/clip_reward": 0.3089185729622841,
      "step": 267
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.008261023688793674,
      "grad_norm": 0.2408505354217034,
      "kl": 0.06768798828125,
      "learning_rate": 1e-05,
      "loss": 0.0027,
      "reward": 0.30584635585546494,
      "reward_std": 0.03537123091518879,
      "rewards/clip_reward": 0.30584635585546494,
      "step": 268
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.008291848404050367,
      "grad_norm": 0.23962011506896905,
      "kl": 0.0709228515625,
      "learning_rate": 1e-05,
      "loss": 0.0028,
      "reward": 0.2843191996216774,
      "reward_std": 0.030163435731083155,
      "rewards/clip_reward": 0.2843191996216774,
      "step": 269
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.00832267311930706,
      "grad_norm": 0.267565227028639,
      "kl": 0.072021484375,
      "learning_rate": 1e-05,
      "loss": 0.0029,
      "reward": 0.30715377628803253,
      "reward_std": 0.03210371499881148,
      "rewards/clip_reward": 0.30715377628803253,
      "step": 270
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.008353497834563753,
      "grad_norm": 0.26478125349454257,
      "kl": 0.071533203125,
      "learning_rate": 1e-05,
      "loss": 0.0029,
      "reward": 0.3105710372328758,
      "reward_std": 0.03365481458604336,
      "rewards/clip_reward": 0.3105710372328758,
      "step": 271
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.008384322549820446,
      "grad_norm": 0.25310448605992364,
      "kl": 0.0711669921875,
      "learning_rate": 1e-05,
      "loss": 0.0028,
      "reward": 0.2988099977374077,
      "reward_std": 0.03356699598953128,
      "rewards/clip_reward": 0.2988099977374077,
      "step": 272
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.008415147265077139,
      "grad_norm": 0.6415597863522857,
      "kl": 0.0704345703125,
      "learning_rate": 1e-05,
      "loss": 0.0028,
      "reward": 0.3161829188466072,
      "reward_std": 0.03346774587407708,
      "rewards/clip_reward": 0.3161829188466072,
      "step": 273
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.008445971980333832,
      "grad_norm": 0.2963215633498198,
      "kl": 0.0755615234375,
      "learning_rate": 1e-05,
      "loss": 0.003,
      "reward": 0.29340942949056625,
      "reward_std": 0.029817141592502594,
      "rewards/clip_reward": 0.29340942949056625,
      "step": 274
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.008476796695590524,
      "grad_norm": 0.28624937293920455,
      "kl": 0.07177734375,
      "learning_rate": 1e-05,
      "loss": 0.0029,
      "reward": 0.29413383454084396,
      "reward_std": 0.03951750136911869,
      "rewards/clip_reward": 0.29413383454084396,
      "step": 275
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.008507621410847217,
      "grad_norm": 0.3584737945543695,
      "kl": 0.065185546875,
      "learning_rate": 1e-05,
      "loss": 0.0026,
      "reward": 0.30792876332998276,
      "reward_std": 0.03842892590910196,
      "rewards/clip_reward": 0.30792876332998276,
      "step": 276
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.00853844612610391,
      "grad_norm": 0.41014371409185196,
      "kl": 0.086669921875,
      "learning_rate": 1e-05,
      "loss": 0.0035,
      "reward": 0.30880793929100037,
      "reward_std": 0.03384514432400465,
      "rewards/clip_reward": 0.30880793929100037,
      "step": 277
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.008569270841360603,
      "grad_norm": 0.6923907529205436,
      "kl": 0.07275390625,
      "learning_rate": 1e-05,
      "loss": 0.0029,
      "reward": 0.30738433450460434,
      "reward_std": 0.030436881817877293,
      "rewards/clip_reward": 0.30738433450460434,
      "step": 278
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.008600095556617296,
      "grad_norm": 0.2508894379559603,
      "kl": 0.06707763671875,
      "learning_rate": 1e-05,
      "loss": 0.0027,
      "reward": 0.29732104390859604,
      "reward_std": 0.03281824570149183,
      "rewards/clip_reward": 0.29732104390859604,
      "step": 279
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.008630920271873989,
      "grad_norm": 0.21788541287282853,
      "kl": 0.066650390625,
      "learning_rate": 1e-05,
      "loss": 0.0027,
      "reward": 0.3087487816810608,
      "reward_std": 0.03466750681400299,
      "rewards/clip_reward": 0.3087487816810608,
      "step": 280
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.008661744987130682,
      "grad_norm": 0.3677827474941058,
      "kl": 0.077880859375,
      "learning_rate": 1e-05,
      "loss": 0.0031,
      "reward": 0.3030627593398094,
      "reward_std": 0.0359897562302649,
      "rewards/clip_reward": 0.3030627593398094,
      "step": 281
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.008692569702387375,
      "grad_norm": 0.314089431725954,
      "kl": 0.07061767578125,
      "learning_rate": 1e-05,
      "loss": 0.0028,
      "reward": 0.30142877995967865,
      "reward_std": 0.032457209192216396,
      "rewards/clip_reward": 0.30142877995967865,
      "step": 282
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.008723394417644068,
      "grad_norm": 0.2690848375206651,
      "kl": 0.06787109375,
      "learning_rate": 1e-05,
      "loss": 0.0027,
      "reward": 0.29926204681396484,
      "reward_std": 0.046972066164016724,
      "rewards/clip_reward": 0.29926204681396484,
      "step": 283
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.00875421913290076,
      "grad_norm": 0.5301348946984951,
      "kl": 0.06787109375,
      "learning_rate": 1e-05,
      "loss": 0.0027,
      "reward": 0.3038446083664894,
      "reward_std": 0.03355177864432335,
      "rewards/clip_reward": 0.3038446083664894,
      "step": 284
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.008785043848157453,
      "grad_norm": 0.3640215089911138,
      "kl": 0.0740966796875,
      "learning_rate": 1e-05,
      "loss": 0.003,
      "reward": 0.29393448680639267,
      "reward_std": 0.029715597163885832,
      "rewards/clip_reward": 0.29393448680639267,
      "step": 285
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.008815868563414146,
      "grad_norm": 0.25955791764443836,
      "kl": 0.0677490234375,
      "learning_rate": 1e-05,
      "loss": 0.0027,
      "reward": 0.29612670093774796,
      "reward_std": 0.03200107906013727,
      "rewards/clip_reward": 0.29612670093774796,
      "step": 286
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.008846693278670839,
      "grad_norm": 0.2840084484574733,
      "kl": 0.0736083984375,
      "learning_rate": 1e-05,
      "loss": 0.0029,
      "reward": 0.28986190259456635,
      "reward_std": 0.03536898456513882,
      "rewards/clip_reward": 0.28986190259456635,
      "step": 287
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.00887751799392753,
      "grad_norm": 0.2504845781925923,
      "kl": 0.0736083984375,
      "learning_rate": 1e-05,
      "loss": 0.0029,
      "reward": 0.28574925661087036,
      "reward_std": 0.029350985307246447,
      "rewards/clip_reward": 0.28574925661087036,
      "step": 288
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.008908342709184223,
      "grad_norm": 1.9538978684731851,
      "kl": 0.073486328125,
      "learning_rate": 1e-05,
      "loss": 0.0029,
      "reward": 0.2989484444260597,
      "reward_std": 0.037056506145745516,
      "rewards/clip_reward": 0.2989484444260597,
      "step": 289
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.008939167424440916,
      "grad_norm": 0.29595148973162033,
      "kl": 0.068359375,
      "learning_rate": 1e-05,
      "loss": 0.0027,
      "reward": 0.2984117269515991,
      "reward_std": 0.04048395995050669,
      "rewards/clip_reward": 0.2984117269515991,
      "step": 290
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.008969992139697609,
      "grad_norm": 0.46786991142607903,
      "kl": 0.0726318359375,
      "learning_rate": 1e-05,
      "loss": 0.0029,
      "reward": 0.29787522554397583,
      "reward_std": 0.03701747301965952,
      "rewards/clip_reward": 0.29787522554397583,
      "step": 291
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.009000816854954302,
      "grad_norm": 0.2415371991232022,
      "kl": 0.0673828125,
      "learning_rate": 1e-05,
      "loss": 0.0027,
      "reward": 0.2959031015634537,
      "reward_std": 0.03375337179750204,
      "rewards/clip_reward": 0.2959031015634537,
      "step": 292
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.009031641570210995,
      "grad_norm": 0.2509472132536602,
      "kl": 0.0736083984375,
      "learning_rate": 1e-05,
      "loss": 0.003,
      "reward": 0.3059312403202057,
      "reward_std": 0.029257553163915873,
      "rewards/clip_reward": 0.3059312403202057,
      "step": 293
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.009062466285467688,
      "grad_norm": 0.27142301376417255,
      "kl": 0.0682373046875,
      "learning_rate": 1e-05,
      "loss": 0.0027,
      "reward": 0.302873820066452,
      "reward_std": 0.03212234936654568,
      "rewards/clip_reward": 0.302873820066452,
      "step": 294
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.00909329100072438,
      "grad_norm": 0.6714767650621216,
      "kl": 0.0770263671875,
      "learning_rate": 1e-05,
      "loss": 0.0031,
      "reward": 0.302773617208004,
      "reward_std": 0.037844753824174404,
      "rewards/clip_reward": 0.302773617208004,
      "step": 295
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.009124115715981073,
      "grad_norm": 0.3313051439529105,
      "kl": 0.071533203125,
      "learning_rate": 1e-05,
      "loss": 0.0029,
      "reward": 0.3004399463534355,
      "reward_std": 0.03778542298823595,
      "rewards/clip_reward": 0.3004399463534355,
      "step": 296
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.009154940431237766,
      "grad_norm": 0.2861755625116893,
      "kl": 0.0765380859375,
      "learning_rate": 1e-05,
      "loss": 0.0031,
      "reward": 0.2957393676042557,
      "reward_std": 0.03911779401823878,
      "rewards/clip_reward": 0.2957393676042557,
      "step": 297
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.00918576514649446,
      "grad_norm": 0.3643143782064657,
      "kl": 0.0721435546875,
      "learning_rate": 1e-05,
      "loss": 0.0029,
      "reward": 0.3204945772886276,
      "reward_std": 0.04190400242805481,
      "rewards/clip_reward": 0.3204945772886276,
      "step": 298
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.009216589861751152,
      "grad_norm": 0.3506675665500401,
      "kl": 0.0740966796875,
      "learning_rate": 1e-05,
      "loss": 0.003,
      "reward": 0.30117396265268326,
      "reward_std": 0.03484671004116535,
      "rewards/clip_reward": 0.30117396265268326,
      "step": 299
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.009247414577007845,
      "grad_norm": 0.4873394391686336,
      "kl": 0.0721435546875,
      "learning_rate": 1e-05,
      "loss": 0.0029,
      "reward": 0.3100610002875328,
      "reward_std": 0.03472679574042559,
      "rewards/clip_reward": 0.3100610002875328,
      "step": 300
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.009278239292264538,
      "grad_norm": 0.4157803977878561,
      "kl": 0.072021484375,
      "learning_rate": 1e-05,
      "loss": 0.0029,
      "reward": 0.30167850106954575,
      "reward_std": 0.031223418191075325,
      "rewards/clip_reward": 0.30167850106954575,
      "step": 301
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.00930906400752123,
      "grad_norm": 0.3797545814324745,
      "kl": 0.0731201171875,
      "learning_rate": 1e-05,
      "loss": 0.0029,
      "reward": 0.31539446860551834,
      "reward_std": 0.041049075312912464,
      "rewards/clip_reward": 0.31539446860551834,
      "step": 302
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.009339888722777924,
      "grad_norm": 0.2743776739836363,
      "kl": 0.0791015625,
      "learning_rate": 1e-05,
      "loss": 0.0032,
      "reward": 0.30482209473848343,
      "reward_std": 0.03603475447744131,
      "rewards/clip_reward": 0.30482209473848343,
      "step": 303
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.009370713438034617,
      "grad_norm": 0.3102618921636113,
      "kl": 0.0723876953125,
      "learning_rate": 1e-05,
      "loss": 0.0029,
      "reward": 0.3093208372592926,
      "reward_std": 0.03316945396363735,
      "rewards/clip_reward": 0.3093208372592926,
      "step": 304
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.00940153815329131,
      "grad_norm": 0.22184784039477515,
      "kl": 0.0692138671875,
      "learning_rate": 1e-05,
      "loss": 0.0028,
      "reward": 0.3097628057003021,
      "reward_std": 0.034655320923775434,
      "rewards/clip_reward": 0.3097628057003021,
      "step": 305
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.009432362868548002,
      "grad_norm": 0.2679648313292432,
      "kl": 0.0689697265625,
      "learning_rate": 1e-05,
      "loss": 0.0028,
      "reward": 0.31062986701726913,
      "reward_std": 0.0346312508918345,
      "rewards/clip_reward": 0.31062986701726913,
      "step": 306
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.009463187583804695,
      "grad_norm": 0.2091622067188512,
      "kl": 0.068115234375,
      "learning_rate": 1e-05,
      "loss": 0.0027,
      "reward": 0.2891889810562134,
      "reward_std": 0.03427890222519636,
      "rewards/clip_reward": 0.2891889810562134,
      "step": 307
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.009494012299061388,
      "grad_norm": 0.380322887324008,
      "kl": 0.077392578125,
      "learning_rate": 1e-05,
      "loss": 0.0031,
      "reward": 0.2787666954100132,
      "reward_std": 0.030210648197680712,
      "rewards/clip_reward": 0.2787666954100132,
      "step": 308
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.009524837014318081,
      "grad_norm": 0.2639105229044439,
      "kl": 0.06744384765625,
      "learning_rate": 1e-05,
      "loss": 0.0027,
      "reward": 0.30674491077661514,
      "reward_std": 0.026672531850636005,
      "rewards/clip_reward": 0.30674491077661514,
      "step": 309
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.009555661729574774,
      "grad_norm": 0.24148098115752395,
      "kl": 0.0694580078125,
      "learning_rate": 1e-05,
      "loss": 0.0028,
      "reward": 0.2937857508659363,
      "reward_std": 0.03315737470984459,
      "rewards/clip_reward": 0.2937857508659363,
      "step": 310
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.009586486444831465,
      "grad_norm": 0.2751403655433819,
      "kl": 0.0654296875,
      "learning_rate": 1e-05,
      "loss": 0.0026,
      "reward": 0.3029419779777527,
      "reward_std": 0.037644670344889164,
      "rewards/clip_reward": 0.3029419779777527,
      "step": 311
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.009617311160088158,
      "grad_norm": 0.43265267199518154,
      "kl": 0.067626953125,
      "learning_rate": 1e-05,
      "loss": 0.0027,
      "reward": 0.3041282668709755,
      "reward_std": 0.036585441790521145,
      "rewards/clip_reward": 0.3041282668709755,
      "step": 312
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.00964813587534485,
      "grad_norm": 0.23342772151709762,
      "kl": 0.0709228515625,
      "learning_rate": 1e-05,
      "loss": 0.0028,
      "reward": 0.281108133494854,
      "reward_std": 0.032666866201907396,
      "rewards/clip_reward": 0.281108133494854,
      "step": 313
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.009678960590601544,
      "grad_norm": 0.2544379988905811,
      "kl": 0.071044921875,
      "learning_rate": 1e-05,
      "loss": 0.0028,
      "reward": 0.31978850811719894,
      "reward_std": 0.035083431750535965,
      "rewards/clip_reward": 0.31978850811719894,
      "step": 314
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.009709785305858237,
      "grad_norm": 0.2804560271375766,
      "kl": 0.068115234375,
      "learning_rate": 1e-05,
      "loss": 0.0027,
      "reward": 0.3023409992456436,
      "reward_std": 0.039583622477948666,
      "rewards/clip_reward": 0.3023409992456436,
      "step": 315
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.00974061002111493,
      "grad_norm": 0.3308718322184164,
      "kl": 0.065673828125,
      "learning_rate": 1e-05,
      "loss": 0.0026,
      "reward": 0.3051457703113556,
      "reward_std": 0.030992726795375347,
      "rewards/clip_reward": 0.3051457703113556,
      "step": 316
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.009771434736371622,
      "grad_norm": 0.28901404215876114,
      "kl": 0.0743408203125,
      "learning_rate": 1e-05,
      "loss": 0.003,
      "reward": 0.30402371287345886,
      "reward_std": 0.029232127591967583,
      "rewards/clip_reward": 0.30402371287345886,
      "step": 317
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.009802259451628315,
      "grad_norm": 0.35413328500218205,
      "kl": 0.072021484375,
      "learning_rate": 1e-05,
      "loss": 0.0029,
      "reward": 0.30789412558078766,
      "reward_std": 0.03223106591030955,
      "rewards/clip_reward": 0.30789412558078766,
      "step": 318
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.009833084166885008,
      "grad_norm": 0.2340787790478016,
      "kl": 0.0662841796875,
      "learning_rate": 1e-05,
      "loss": 0.0027,
      "reward": 0.3076978102326393,
      "reward_std": 0.026181872468441725,
      "rewards/clip_reward": 0.3076978102326393,
      "step": 319
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.009863908882141701,
      "grad_norm": 0.2160311104140781,
      "kl": 0.06451416015625,
      "learning_rate": 1e-05,
      "loss": 0.0026,
      "reward": 0.3071356937289238,
      "reward_std": 0.03830281179398298,
      "rewards/clip_reward": 0.3071356937289238,
      "step": 320
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.009894733597398394,
      "grad_norm": 0.2377449652657432,
      "kl": 0.069580078125,
      "learning_rate": 1e-05,
      "loss": 0.0028,
      "reward": 0.2980581670999527,
      "reward_std": 0.04127457272261381,
      "rewards/clip_reward": 0.2980581670999527,
      "step": 321
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.009925558312655087,
      "grad_norm": 0.24180517546337557,
      "kl": 0.06591796875,
      "learning_rate": 1e-05,
      "loss": 0.0026,
      "reward": 0.3178565129637718,
      "reward_std": 0.03440323146060109,
      "rewards/clip_reward": 0.3178565129637718,
      "step": 322
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.00995638302791178,
      "grad_norm": 0.2316875481005505,
      "kl": 0.0665283203125,
      "learning_rate": 1e-05,
      "loss": 0.0027,
      "reward": 0.30603795498609543,
      "reward_std": 0.03252642601728439,
      "rewards/clip_reward": 0.30603795498609543,
      "step": 323
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.009987207743168473,
      "grad_norm": 0.3112158582767738,
      "kl": 0.0694580078125,
      "learning_rate": 1e-05,
      "loss": 0.0028,
      "reward": 0.2933913469314575,
      "reward_std": 0.03322177054360509,
      "rewards/clip_reward": 0.2933913469314575,
      "step": 324
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.010018032458425166,
      "grad_norm": 0.2406318070865998,
      "kl": 0.06787109375,
      "learning_rate": 1e-05,
      "loss": 0.0027,
      "reward": 0.2959599047899246,
      "reward_std": 0.04275544732809067,
      "rewards/clip_reward": 0.2959599047899246,
      "step": 325
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.010048857173681858,
      "grad_norm": 0.25585002828007836,
      "kl": 0.06561279296875,
      "learning_rate": 1e-05,
      "loss": 0.0026,
      "reward": 0.306929275393486,
      "reward_std": 0.03498923219740391,
      "rewards/clip_reward": 0.306929275393486,
      "step": 326
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.010079681888938551,
      "grad_norm": 0.2860165243450835,
      "kl": 0.06787109375,
      "learning_rate": 1e-05,
      "loss": 0.0027,
      "reward": 0.3039538636803627,
      "reward_std": 0.0350054414011538,
      "rewards/clip_reward": 0.3039538636803627,
      "step": 327
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.010110506604195244,
      "grad_norm": 0.2915143244249602,
      "kl": 0.064697265625,
      "learning_rate": 1e-05,
      "loss": 0.0026,
      "reward": 0.30542658269405365,
      "reward_std": 0.03288308531045914,
      "rewards/clip_reward": 0.30542658269405365,
      "step": 328
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.010141331319451937,
      "grad_norm": 0.2132282152683129,
      "kl": 0.06341552734375,
      "learning_rate": 1e-05,
      "loss": 0.0025,
      "reward": 0.30350538343191147,
      "reward_std": 0.03533684695139527,
      "rewards/clip_reward": 0.30350538343191147,
      "step": 329
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.01017215603470863,
      "grad_norm": 0.2229730055471249,
      "kl": 0.0667724609375,
      "learning_rate": 1e-05,
      "loss": 0.0027,
      "reward": 0.3181814104318619,
      "reward_std": 0.04055699147284031,
      "rewards/clip_reward": 0.3181814104318619,
      "step": 330
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.010202980749965323,
      "grad_norm": 0.29467120899645105,
      "kl": 0.078369140625,
      "learning_rate": 1e-05,
      "loss": 0.0031,
      "reward": 0.3067881911993027,
      "reward_std": 0.03262139391154051,
      "rewards/clip_reward": 0.3067881911993027,
      "step": 331
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.010233805465222016,
      "grad_norm": 0.2626359604798955,
      "kl": 0.0689697265625,
      "learning_rate": 1e-05,
      "loss": 0.0028,
      "reward": 0.3152911067008972,
      "reward_std": 0.033262383192777634,
      "rewards/clip_reward": 0.3152911067008972,
      "step": 332
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.010264630180478709,
      "grad_norm": 0.2557414678170153,
      "kl": 0.065185546875,
      "learning_rate": 1e-05,
      "loss": 0.0026,
      "reward": 0.2922421917319298,
      "reward_std": 0.03196657868102193,
      "rewards/clip_reward": 0.2922421917319298,
      "step": 333
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.0102954548957354,
      "grad_norm": 0.46611276904322874,
      "kl": 0.0941162109375,
      "learning_rate": 1e-05,
      "loss": 0.0038,
      "reward": 0.3090960830450058,
      "reward_std": 0.03677598666399717,
      "rewards/clip_reward": 0.3090960830450058,
      "step": 334
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.010326279610992093,
      "grad_norm": 0.2309027643372804,
      "kl": 0.0677490234375,
      "learning_rate": 1e-05,
      "loss": 0.0027,
      "reward": 0.3062281683087349,
      "reward_std": 0.02914317138493061,
      "rewards/clip_reward": 0.3062281683087349,
      "step": 335
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.010357104326248786,
      "grad_norm": 0.24277503406886997,
      "kl": 0.064208984375,
      "learning_rate": 1e-05,
      "loss": 0.0026,
      "reward": 0.30345892161130905,
      "reward_std": 0.03988218680024147,
      "rewards/clip_reward": 0.30345892161130905,
      "step": 336
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.010387929041505478,
      "grad_norm": 0.24331709055222814,
      "kl": 0.06549072265625,
      "learning_rate": 1e-05,
      "loss": 0.0026,
      "reward": 0.28554844856262207,
      "reward_std": 0.03036739816889167,
      "rewards/clip_reward": 0.28554844856262207,
      "step": 337
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.010418753756762171,
      "grad_norm": 0.3529980048921207,
      "kl": 0.06640625,
      "learning_rate": 1e-05,
      "loss": 0.0027,
      "reward": 0.30094072222709656,
      "reward_std": 0.03639454022049904,
      "rewards/clip_reward": 0.30094072222709656,
      "step": 338
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.010449578472018864,
      "grad_norm": 0.4648925688267997,
      "kl": 0.070068359375,
      "learning_rate": 1e-05,
      "loss": 0.0028,
      "reward": 0.3127727806568146,
      "reward_std": 0.037409001495689154,
      "rewards/clip_reward": 0.3127727806568146,
      "step": 339
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.010480403187275557,
      "grad_norm": 0.21891573660402955,
      "kl": 0.0687255859375,
      "learning_rate": 1e-05,
      "loss": 0.0027,
      "reward": 0.2772018723189831,
      "reward_std": 0.02981334552168846,
      "rewards/clip_reward": 0.2772018723189831,
      "step": 340
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.01051122790253225,
      "grad_norm": 0.22903622425258144,
      "kl": 0.0693359375,
      "learning_rate": 1e-05,
      "loss": 0.0028,
      "reward": 0.30165959894657135,
      "reward_std": 0.03327458165585995,
      "rewards/clip_reward": 0.30165959894657135,
      "step": 341
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.010542052617788943,
      "grad_norm": 0.2563729286323683,
      "kl": 0.0703125,
      "learning_rate": 1e-05,
      "loss": 0.0028,
      "reward": 0.29880737513303757,
      "reward_std": 0.03046613559126854,
      "rewards/clip_reward": 0.29880737513303757,
      "step": 342
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.010572877333045636,
      "grad_norm": 0.23890758116942548,
      "kl": 0.07080078125,
      "learning_rate": 1e-05,
      "loss": 0.0028,
      "reward": 0.30336084961891174,
      "reward_std": 0.03497588029131293,
      "rewards/clip_reward": 0.30336084961891174,
      "step": 343
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.010603702048302329,
      "grad_norm": 0.3415925254166041,
      "kl": 0.0689697265625,
      "learning_rate": 1e-05,
      "loss": 0.0028,
      "reward": 0.31711456179618835,
      "reward_std": 0.03901077900081873,
      "rewards/clip_reward": 0.31711456179618835,
      "step": 344
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.010634526763559022,
      "grad_norm": 0.28715059266697696,
      "kl": 0.0751953125,
      "learning_rate": 1e-05,
      "loss": 0.003,
      "reward": 0.2894616648554802,
      "reward_std": 0.03519205283373594,
      "rewards/clip_reward": 0.2894616648554802,
      "step": 345
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.010665351478815715,
      "grad_norm": 0.21252915417396734,
      "kl": 0.072265625,
      "learning_rate": 1e-05,
      "loss": 0.0029,
      "reward": 0.3071034103631973,
      "reward_std": 0.03865744452923536,
      "rewards/clip_reward": 0.3071034103631973,
      "step": 346
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.010696176194072407,
      "grad_norm": 0.2213810998809183,
      "kl": 0.0711669921875,
      "learning_rate": 1e-05,
      "loss": 0.0028,
      "reward": 0.30335263907909393,
      "reward_std": 0.03155648289248347,
      "rewards/clip_reward": 0.30335263907909393,
      "step": 347
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.0107270009093291,
      "grad_norm": 0.22688649328130808,
      "kl": 0.0701904296875,
      "learning_rate": 1e-05,
      "loss": 0.0028,
      "reward": 0.30418138951063156,
      "reward_std": 0.03234159108251333,
      "rewards/clip_reward": 0.30418138951063156,
      "step": 348
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.010757825624585793,
      "grad_norm": 0.29663311793497593,
      "kl": 0.06951904296875,
      "learning_rate": 1e-05,
      "loss": 0.0028,
      "reward": 0.3119578883051872,
      "reward_std": 0.035748135298490524,
      "rewards/clip_reward": 0.3119578883051872,
      "step": 349
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.010788650339842486,
      "grad_norm": 0.21987020789836037,
      "kl": 0.0726318359375,
      "learning_rate": 1e-05,
      "loss": 0.0029,
      "reward": 0.2886430397629738,
      "reward_std": 0.028050173074007034,
      "rewards/clip_reward": 0.2886430397629738,
      "step": 350
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.010819475055099179,
      "grad_norm": 0.25690691289795353,
      "kl": 0.0743408203125,
      "learning_rate": 1e-05,
      "loss": 0.003,
      "reward": 0.3163754343986511,
      "reward_std": 0.03696467122063041,
      "rewards/clip_reward": 0.3163754343986511,
      "step": 351
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.010850299770355872,
      "grad_norm": 0.20820549385701986,
      "kl": 0.068603515625,
      "learning_rate": 1e-05,
      "loss": 0.0027,
      "reward": 0.2914758771657944,
      "reward_std": 0.036506949458271265,
      "rewards/clip_reward": 0.2914758771657944,
      "step": 352
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.010881124485612565,
      "grad_norm": 0.23881025300895642,
      "kl": 0.0703125,
      "learning_rate": 1e-05,
      "loss": 0.0028,
      "reward": 0.305471271276474,
      "reward_std": 0.03957742918282747,
      "rewards/clip_reward": 0.305471271276474,
      "step": 353
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.010911949200869258,
      "grad_norm": 0.23403139952741436,
      "kl": 0.073974609375,
      "learning_rate": 1e-05,
      "loss": 0.003,
      "reward": 0.29349175840616226,
      "reward_std": 0.029270636849105358,
      "rewards/clip_reward": 0.29349175840616226,
      "step": 354
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.01094277391612595,
      "grad_norm": 0.29623954478847825,
      "kl": 0.0679931640625,
      "learning_rate": 1e-05,
      "loss": 0.0027,
      "reward": 0.3115154132246971,
      "reward_std": 0.03451331192627549,
      "rewards/clip_reward": 0.3115154132246971,
      "step": 355
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.010973598631382643,
      "grad_norm": 0.22254248972166946,
      "kl": 0.07177734375,
      "learning_rate": 1e-05,
      "loss": 0.0029,
      "reward": 0.2945594787597656,
      "reward_std": 0.037838808726519346,
      "rewards/clip_reward": 0.2945594787597656,
      "step": 356
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.011004423346639335,
      "grad_norm": 0.31078887072255357,
      "kl": 0.069580078125,
      "learning_rate": 1e-05,
      "loss": 0.0028,
      "reward": 0.2983166500926018,
      "reward_std": 0.038911592215299606,
      "rewards/clip_reward": 0.2983166500926018,
      "step": 357
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.011035248061896027,
      "grad_norm": 0.21415415508214233,
      "kl": 0.0740966796875,
      "learning_rate": 1e-05,
      "loss": 0.003,
      "reward": 0.2967897802591324,
      "reward_std": 0.034322294406592846,
      "rewards/clip_reward": 0.2967897802591324,
      "step": 358
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.01106607277715272,
      "grad_norm": 0.22868327148604967,
      "kl": 0.073486328125,
      "learning_rate": 1e-05,
      "loss": 0.0029,
      "reward": 0.30835631489753723,
      "reward_std": 0.030591858085244894,
      "rewards/clip_reward": 0.30835631489753723,
      "step": 359
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.011096897492409413,
      "grad_norm": 0.2536865522371228,
      "kl": 0.0755615234375,
      "learning_rate": 1e-05,
      "loss": 0.003,
      "reward": 0.3081829249858856,
      "reward_std": 0.03304952848702669,
      "rewards/clip_reward": 0.3081829249858856,
      "step": 360
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.011127722207666106,
      "grad_norm": 0.28703592507172904,
      "kl": 0.0833740234375,
      "learning_rate": 1e-05,
      "loss": 0.0033,
      "reward": 0.31137142330408096,
      "reward_std": 0.036431248765438795,
      "rewards/clip_reward": 0.31137142330408096,
      "step": 361
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.011158546922922799,
      "grad_norm": 0.25966729830990576,
      "kl": 0.072265625,
      "learning_rate": 1e-05,
      "loss": 0.0029,
      "reward": 0.32117273658514023,
      "reward_std": 0.03095503943040967,
      "rewards/clip_reward": 0.32117273658514023,
      "step": 362
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.011189371638179492,
      "grad_norm": 0.25023628654317165,
      "kl": 0.072509765625,
      "learning_rate": 1e-05,
      "loss": 0.0029,
      "reward": 0.31916752457618713,
      "reward_std": 0.03393755853176117,
      "rewards/clip_reward": 0.31916752457618713,
      "step": 363
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.011220196353436185,
      "grad_norm": 0.23509232771567978,
      "kl": 0.0703125,
      "learning_rate": 1e-05,
      "loss": 0.0028,
      "reward": 0.3117026165127754,
      "reward_std": 0.02881152043119073,
      "rewards/clip_reward": 0.3117026165127754,
      "step": 364
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.011251021068692878,
      "grad_norm": 0.25193424249683816,
      "kl": 0.0743408203125,
      "learning_rate": 1e-05,
      "loss": 0.003,
      "reward": 0.3166244179010391,
      "reward_std": 0.03036302560940385,
      "rewards/clip_reward": 0.3166244179010391,
      "step": 365
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.01128184578394957,
      "grad_norm": 0.33920942950993677,
      "kl": 0.078369140625,
      "learning_rate": 1e-05,
      "loss": 0.0031,
      "reward": 0.30771925300359726,
      "reward_std": 0.03633692301809788,
      "rewards/clip_reward": 0.30771925300359726,
      "step": 366
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.011312670499206263,
      "grad_norm": 0.21836890161975292,
      "kl": 0.0755615234375,
      "learning_rate": 1e-05,
      "loss": 0.003,
      "reward": 0.2963472232222557,
      "reward_std": 0.039488581009209156,
      "rewards/clip_reward": 0.2963472232222557,
      "step": 367
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.011343495214462956,
      "grad_norm": 0.5069167126879947,
      "kl": 0.0767822265625,
      "learning_rate": 1e-05,
      "loss": 0.0031,
      "reward": 0.2941269427537918,
      "reward_std": 0.0324277700856328,
      "rewards/clip_reward": 0.2941269427537918,
      "step": 368
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.01137431992971965,
      "grad_norm": 0.23131621578167488,
      "kl": 0.072509765625,
      "learning_rate": 1e-05,
      "loss": 0.0029,
      "reward": 0.3116966411471367,
      "reward_std": 0.035368080250918865,
      "rewards/clip_reward": 0.3116966411471367,
      "step": 369
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.011405144644976342,
      "grad_norm": 0.2722113704455896,
      "kl": 0.071533203125,
      "learning_rate": 1e-05,
      "loss": 0.0029,
      "reward": 0.3093196973204613,
      "reward_std": 0.0321941040456295,
      "rewards/clip_reward": 0.3093196973204613,
      "step": 370
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.011435969360233035,
      "grad_norm": 0.28730230026079845,
      "kl": 0.0716552734375,
      "learning_rate": 1e-05,
      "loss": 0.0029,
      "reward": 0.30720267444849014,
      "reward_std": 0.03148469375446439,
      "rewards/clip_reward": 0.30720267444849014,
      "step": 371
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.011466794075489728,
      "grad_norm": 0.2195587993801826,
      "kl": 0.0721435546875,
      "learning_rate": 1e-05,
      "loss": 0.0029,
      "reward": 0.31224188208580017,
      "reward_std": 0.03507778514176607,
      "rewards/clip_reward": 0.31224188208580017,
      "step": 372
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.01149761879074642,
      "grad_norm": 0.22359313280470233,
      "kl": 0.07666015625,
      "learning_rate": 1e-05,
      "loss": 0.0031,
      "reward": 0.3193345367908478,
      "reward_std": 0.03153304476290941,
      "rewards/clip_reward": 0.3193345367908478,
      "step": 373
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.011528443506003114,
      "grad_norm": 0.24587352736332965,
      "kl": 0.07958984375,
      "learning_rate": 1e-05,
      "loss": 0.0032,
      "reward": 0.3059887960553169,
      "reward_std": 0.03858480043709278,
      "rewards/clip_reward": 0.3059887960553169,
      "step": 374
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.011559268221259807,
      "grad_norm": 0.20861176096611583,
      "kl": 0.0777587890625,
      "learning_rate": 1e-05,
      "loss": 0.0031,
      "reward": 0.31156501173973083,
      "reward_std": 0.03071795403957367,
      "rewards/clip_reward": 0.31156501173973083,
      "step": 375
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.0115900929365165,
      "grad_norm": 0.3384083028926716,
      "kl": 0.0843505859375,
      "learning_rate": 1e-05,
      "loss": 0.0034,
      "reward": 0.2961001992225647,
      "reward_std": 0.03549221996217966,
      "rewards/clip_reward": 0.2961001992225647,
      "step": 376
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.011620917651773192,
      "grad_norm": 0.25505451962723275,
      "kl": 0.0787353515625,
      "learning_rate": 1e-05,
      "loss": 0.0032,
      "reward": 0.3040069565176964,
      "reward_std": 0.030104033648967743,
      "rewards/clip_reward": 0.3040069565176964,
      "step": 377
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.011651742367029885,
      "grad_norm": 0.28066405899218805,
      "kl": 0.0821533203125,
      "learning_rate": 1e-05,
      "loss": 0.0033,
      "reward": 0.3125178590416908,
      "reward_std": 0.03651127126067877,
      "rewards/clip_reward": 0.3125178590416908,
      "step": 378
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.011682567082286578,
      "grad_norm": 0.27665197939767366,
      "kl": 0.079345703125,
      "learning_rate": 1e-05,
      "loss": 0.0032,
      "reward": 0.3094138279557228,
      "reward_std": 0.038227169774472713,
      "rewards/clip_reward": 0.3094138279557228,
      "step": 379
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.01171339179754327,
      "grad_norm": 0.2597327901842019,
      "kl": 0.080810546875,
      "learning_rate": 1e-05,
      "loss": 0.0032,
      "reward": 0.3057212755084038,
      "reward_std": 0.03503256104886532,
      "rewards/clip_reward": 0.3057212755084038,
      "step": 380
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.011744216512799962,
      "grad_norm": 0.32226220932101085,
      "kl": 0.0845947265625,
      "learning_rate": 1e-05,
      "loss": 0.0034,
      "reward": 0.3279699385166168,
      "reward_std": 0.0381966782733798,
      "rewards/clip_reward": 0.3279699385166168,
      "step": 381
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.011775041228056655,
      "grad_norm": 0.2956679464457214,
      "kl": 0.081787109375,
      "learning_rate": 1e-05,
      "loss": 0.0033,
      "reward": 0.30723875761032104,
      "reward_std": 0.03168717911466956,
      "rewards/clip_reward": 0.30723875761032104,
      "step": 382
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.011805865943313348,
      "grad_norm": 0.5191055044770609,
      "kl": 0.091552734375,
      "learning_rate": 1e-05,
      "loss": 0.0037,
      "reward": 0.31639552116394043,
      "reward_std": 0.03826092462986708,
      "rewards/clip_reward": 0.31639552116394043,
      "step": 383
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.011836690658570041,
      "grad_norm": 0.3776553446735074,
      "kl": 0.091552734375,
      "learning_rate": 1e-05,
      "loss": 0.0037,
      "reward": 0.3041198328137398,
      "reward_std": 0.026013361755758524,
      "rewards/clip_reward": 0.3041198328137398,
      "step": 384
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.011867515373826734,
      "grad_norm": 0.2572694780557513,
      "kl": 0.08837890625,
      "learning_rate": 1e-05,
      "loss": 0.0035,
      "reward": 0.2923019975423813,
      "reward_std": 0.03231561556458473,
      "rewards/clip_reward": 0.2923019975423813,
      "step": 385
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.011898340089083427,
      "grad_norm": 15.002263434692885,
      "kl": 0.1844482421875,
      "learning_rate": 1e-05,
      "loss": 0.0074,
      "reward": 0.31261809915304184,
      "reward_std": 0.033471152652055025,
      "rewards/clip_reward": 0.31261809915304184,
      "step": 386
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.01192916480434012,
      "grad_norm": 34.74334790635388,
      "kl": 0.585205078125,
      "learning_rate": 1e-05,
      "loss": 0.0234,
      "reward": 0.31164050102233887,
      "reward_std": 0.03602212620899081,
      "rewards/clip_reward": 0.31164050102233887,
      "step": 387
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.011959989519596812,
      "grad_norm": 0.2788251418304219,
      "kl": 0.082275390625,
      "learning_rate": 1e-05,
      "loss": 0.0033,
      "reward": 0.2998761534690857,
      "reward_std": 0.03641665726900101,
      "rewards/clip_reward": 0.2998761534690857,
      "step": 388
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.011990814234853505,
      "grad_norm": 0.23027728154942576,
      "kl": 0.086669921875,
      "learning_rate": 1e-05,
      "loss": 0.0035,
      "reward": 0.30939532816410065,
      "reward_std": 0.033424354158341885,
      "rewards/clip_reward": 0.30939532816410065,
      "step": 389
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.012021638950110198,
      "grad_norm": 0.25021193487381366,
      "kl": 0.091552734375,
      "learning_rate": 1e-05,
      "loss": 0.0037,
      "reward": 0.3133769854903221,
      "reward_std": 0.034206162206828594,
      "rewards/clip_reward": 0.3133769854903221,
      "step": 390
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.012052463665366891,
      "grad_norm": 0.23903302345332506,
      "kl": 0.0802001953125,
      "learning_rate": 1e-05,
      "loss": 0.0032,
      "reward": 0.3138531744480133,
      "reward_std": 0.03734522592276335,
      "rewards/clip_reward": 0.3138531744480133,
      "step": 391
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.012083288380623584,
      "grad_norm": 0.2598108600282988,
      "kl": 0.083251953125,
      "learning_rate": 1e-05,
      "loss": 0.0033,
      "reward": 0.32199686020612717,
      "reward_std": 0.03402840159833431,
      "rewards/clip_reward": 0.32199686020612717,
      "step": 392
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.012114113095880277,
      "grad_norm": 0.23642883730734804,
      "kl": 0.08056640625,
      "learning_rate": 1e-05,
      "loss": 0.0032,
      "reward": 0.3051422908902168,
      "reward_std": 0.03256607661023736,
      "rewards/clip_reward": 0.3051422908902168,
      "step": 393
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.01214493781113697,
      "grad_norm": 0.22592478949720543,
      "kl": 0.0802001953125,
      "learning_rate": 1e-05,
      "loss": 0.0032,
      "reward": 0.3141862004995346,
      "reward_std": 0.03128534881398082,
      "rewards/clip_reward": 0.3141862004995346,
      "step": 394
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.012175762526393663,
      "grad_norm": 0.27814956809020164,
      "kl": 0.080810546875,
      "learning_rate": 1e-05,
      "loss": 0.0032,
      "reward": 0.30130794644355774,
      "reward_std": 0.028409887570887804,
      "rewards/clip_reward": 0.30130794644355774,
      "step": 395
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.012206587241650356,
      "grad_norm": 0.21400571540908192,
      "kl": 0.086669921875,
      "learning_rate": 1e-05,
      "loss": 0.0035,
      "reward": 0.30789605528116226,
      "reward_std": 0.034056794829666615,
      "rewards/clip_reward": 0.30789605528116226,
      "step": 396
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.012237411956907049,
      "grad_norm": 0.21349466740102746,
      "kl": 0.080810546875,
      "learning_rate": 1e-05,
      "loss": 0.0032,
      "reward": 0.3021450638771057,
      "reward_std": 0.035551701206713915,
      "rewards/clip_reward": 0.3021450638771057,
      "step": 397
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.012268236672163741,
      "grad_norm": 0.21264064002989824,
      "kl": 0.080322265625,
      "learning_rate": 1e-05,
      "loss": 0.0032,
      "reward": 0.3078059256076813,
      "reward_std": 0.03562043095007539,
      "rewards/clip_reward": 0.3078059256076813,
      "step": 398
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.012299061387420434,
      "grad_norm": 0.2898591178898953,
      "kl": 0.08642578125,
      "learning_rate": 1e-05,
      "loss": 0.0035,
      "reward": 0.3178284019231796,
      "reward_std": 0.03499773098155856,
      "rewards/clip_reward": 0.3178284019231796,
      "step": 399
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4096.0,
      "epoch": 0.012329886102677127,
      "grad_norm": 0.5180005174036416,
      "kl": 0.08935546875,
      "learning_rate": 1e-05,
      "loss": 0.0036,
      "reward": 0.32223907113075256,
      "reward_std": 0.0368791688233614,
      "rewards/clip_reward": 0.32223907113075256,
      "step": 400
    }
  ],
  "logging_steps": 1,
  "max_steps": 32441,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 10,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 6,
  "trial_name": null,
  "trial_params": null
}