diff --git "a/realiad/log_train.txt" "b/realiad/log_train.txt" new file mode 100644--- /dev/null +++ "b/realiad/log_train.txt" @@ -0,0 +1,3218 @@ +2024-04-08 23:28:39,914 - ==> Logging on master GPU: 0 +2024-04-08 23:28:39,914 - ==> Running Trainer: MAMBAADTrainer +2024-04-08 23:28:39,914 - ==> Using GPU: [0] for Training +2024-04-08 23:28:39,915 - ==> Building model +2024-04-08 23:28:40,406 - Loaded from checkpoint 'model/pretrain/resnet34-43635321.pth' +2024-04-08 23:28:58,865 - +------------------------------------ MAMBAAD ------------------------------------ +| module | #parameters or shape | #flops | +|:------------------------------------------------------------|:-----------------------|:-------------| +| model | 25.667M | 8.3G | +| net_t | 8.17M | 3.955G | +| net_t.conv1 | 9.408K | 0.154G | +| net_t.conv1.weight | (64, 3, 7, 7) | | +| net_t.bn1 | 0.128K | 2.097M | +| net_t.bn1.weight | (64,) | | +| net_t.bn1.bias | (64,) | | +| net_t.layer1 | 0.222M | 0.909G | +| net_t.layer1.0 | 73.984K | 0.303G | +| net_t.layer1.0.conv1 | 36.864K | 0.151G | +| net_t.layer1.0.conv1.weight | (64, 64, 3, 3) | | +| net_t.layer1.0.bn1 | 0.128K | 0.524M | +| net_t.layer1.0.bn1.weight | (64,) | | +| net_t.layer1.0.bn1.bias | (64,) | | +| net_t.layer1.0.conv2 | 36.864K | 0.151G | +| net_t.layer1.0.conv2.weight | (64, 64, 3, 3) | | +| net_t.layer1.0.bn2 | 0.128K | 0.524M | +| net_t.layer1.0.bn2.weight | (64,) | | +| net_t.layer1.0.bn2.bias | (64,) | | +| net_t.layer1.1 | 73.984K | 0.303G | +| net_t.layer1.1.conv1 | 36.864K | 0.151G | +| net_t.layer1.1.conv1.weight | (64, 64, 3, 3) | | +| net_t.layer1.1.bn1 | 0.128K | 0.524M | +| net_t.layer1.1.bn1.weight | (64,) | | +| net_t.layer1.1.bn1.bias | (64,) | | +| net_t.layer1.1.conv2 | 36.864K | 0.151G | +| net_t.layer1.1.conv2.weight | (64, 64, 3, 3) | | +| net_t.layer1.1.bn2 | 0.128K | 0.524M | +| net_t.layer1.1.bn2.weight | (64,) | | +| net_t.layer1.1.bn2.bias | (64,) | | +| net_t.layer1.2 | 73.984K | 0.303G | +| net_t.layer1.2.conv1 | 36.864K | 0.151G | +| net_t.layer1.2.conv1.weight | (64, 64, 3, 3) | | +| net_t.layer1.2.bn1 | 0.128K | 0.524M | +| net_t.layer1.2.bn1.weight | (64,) | | +| net_t.layer1.2.bn1.bias | (64,) | | +| net_t.layer1.2.conv2 | 36.864K | 0.151G | +| net_t.layer1.2.conv2.weight | (64, 64, 3, 3) | | +| net_t.layer1.2.bn2 | 0.128K | 0.524M | +| net_t.layer1.2.bn2.weight | (64,) | | +| net_t.layer1.2.bn2.bias | (64,) | | +| net_t.layer2 | 1.116M | 1.143G | +| net_t.layer2.0 | 0.23M | 0.236G | +| net_t.layer2.0.conv1 | 73.728K | 75.497M | +| net_t.layer2.0.conv1.weight | (128, 64, 3, 3) | | +| net_t.layer2.0.bn1 | 0.256K | 0.262M | +| net_t.layer2.0.bn1.weight | (128,) | | +| net_t.layer2.0.bn1.bias | (128,) | | +| net_t.layer2.0.conv2 | 0.147M | 0.151G | +| net_t.layer2.0.conv2.weight | (128, 128, 3, 3) | | +| net_t.layer2.0.bn2 | 0.256K | 0.262M | +| net_t.layer2.0.bn2.weight | (128,) | | +| net_t.layer2.0.bn2.bias | (128,) | | +| net_t.layer2.0.downsample | 8.448K | 8.651M | +| net_t.layer2.0.downsample.0 | 8.192K | 8.389M | +| net_t.layer2.0.downsample.1 | 0.256K | 0.262M | +| net_t.layer2.1 | 0.295M | 0.303G | +| net_t.layer2.1.conv1 | 0.147M | 0.151G | +| net_t.layer2.1.conv1.weight | (128, 128, 3, 3) | | +| net_t.layer2.1.bn1 | 0.256K | 0.262M | +| net_t.layer2.1.bn1.weight | (128,) | | +| net_t.layer2.1.bn1.bias | (128,) | | +| net_t.layer2.1.conv2 | 0.147M | 0.151G | +| net_t.layer2.1.conv2.weight | (128, 128, 3, 3) | | +| net_t.layer2.1.bn2 | 0.256K | 0.262M | +| net_t.layer2.1.bn2.weight | (128,) | | +| net_t.layer2.1.bn2.bias | (128,) | | +| net_t.layer2.2 | 0.295M | 0.303G | +| net_t.layer2.2.conv1 | 0.147M | 0.151G | +| net_t.layer2.2.conv1.weight | (128, 128, 3, 3) | | +| net_t.layer2.2.bn1 | 0.256K | 0.262M | +| net_t.layer2.2.bn1.weight | (128,) | | +| net_t.layer2.2.bn1.bias | (128,) | | +| net_t.layer2.2.conv2 | 0.147M | 0.151G | +| net_t.layer2.2.conv2.weight | (128, 128, 3, 3) | | +| net_t.layer2.2.bn2 | 0.256K | 0.262M | +| net_t.layer2.2.bn2.weight | (128,) | | +| net_t.layer2.2.bn2.bias | (128,) | | +| net_t.layer2.3 | 0.295M | 0.303G | +| net_t.layer2.3.conv1 | 0.147M | 0.151G | +| net_t.layer2.3.conv1.weight | (128, 128, 3, 3) | | +| net_t.layer2.3.bn1 | 0.256K | 0.262M | +| net_t.layer2.3.bn1.weight | (128,) | | +| net_t.layer2.3.bn1.bias | (128,) | | +| net_t.layer2.3.conv2 | 0.147M | 0.151G | +| net_t.layer2.3.conv2.weight | (128, 128, 3, 3) | | +| net_t.layer2.3.bn2 | 0.256K | 0.262M | +| net_t.layer2.3.bn2.weight | (128,) | | +| net_t.layer2.3.bn2.bias | (128,) | | +| net_t.layer3 | 6.822M | 1.747G | +| net_t.layer3.0 | 0.919M | 0.235G | +| net_t.layer3.0.conv1 | 0.295M | 75.497M | +| net_t.layer3.0.conv1.weight | (256, 128, 3, 3) | | +| net_t.layer3.0.bn1 | 0.512K | 0.131M | +| net_t.layer3.0.bn1.weight | (256,) | | +| net_t.layer3.0.bn1.bias | (256,) | | +| net_t.layer3.0.conv2 | 0.59M | 0.151G | +| net_t.layer3.0.conv2.weight | (256, 256, 3, 3) | | +| net_t.layer3.0.bn2 | 0.512K | 0.131M | +| net_t.layer3.0.bn2.weight | (256,) | | +| net_t.layer3.0.bn2.bias | (256,) | | +| net_t.layer3.0.downsample | 33.28K | 8.52M | +| net_t.layer3.0.downsample.0 | 32.768K | 8.389M | +| net_t.layer3.0.downsample.1 | 0.512K | 0.131M | +| net_t.layer3.1 | 1.181M | 0.302G | +| net_t.layer3.1.conv1 | 0.59M | 0.151G | +| net_t.layer3.1.conv1.weight | (256, 256, 3, 3) | | +| net_t.layer3.1.bn1 | 0.512K | 0.131M | +| net_t.layer3.1.bn1.weight | (256,) | | +| net_t.layer3.1.bn1.bias | (256,) | | +| net_t.layer3.1.conv2 | 0.59M | 0.151G | +| net_t.layer3.1.conv2.weight | (256, 256, 3, 3) | | +| net_t.layer3.1.bn2 | 0.512K | 0.131M | +| net_t.layer3.1.bn2.weight | (256,) | | +| net_t.layer3.1.bn2.bias | (256,) | | +| net_t.layer3.2 | 1.181M | 0.302G | +| net_t.layer3.2.conv1 | 0.59M | 0.151G | +| net_t.layer3.2.conv1.weight | (256, 256, 3, 3) | | +| net_t.layer3.2.bn1 | 0.512K | 0.131M | +| net_t.layer3.2.bn1.weight | (256,) | | +| net_t.layer3.2.bn1.bias | (256,) | | +| net_t.layer3.2.conv2 | 0.59M | 0.151G | +| net_t.layer3.2.conv2.weight | (256, 256, 3, 3) | | +| net_t.layer3.2.bn2 | 0.512K | 0.131M | +| net_t.layer3.2.bn2.weight | (256,) | | +| net_t.layer3.2.bn2.bias | (256,) | | +| net_t.layer3.3 | 1.181M | 0.302G | +| net_t.layer3.3.conv1 | 0.59M | 0.151G | +| net_t.layer3.3.conv1.weight | (256, 256, 3, 3) | | +| net_t.layer3.3.bn1 | 0.512K | 0.131M | +| net_t.layer3.3.bn1.weight | (256,) | | +| net_t.layer3.3.bn1.bias | (256,) | | +| net_t.layer3.3.conv2 | 0.59M | 0.151G | +| net_t.layer3.3.conv2.weight | (256, 256, 3, 3) | | +| net_t.layer3.3.bn2 | 0.512K | 0.131M | +| net_t.layer3.3.bn2.weight | (256,) | | +| net_t.layer3.3.bn2.bias | (256,) | | +| net_t.layer3.4 | 1.181M | 0.302G | +| net_t.layer3.4.conv1 | 0.59M | 0.151G | +| net_t.layer3.4.conv1.weight | (256, 256, 3, 3) | | +| net_t.layer3.4.bn1 | 0.512K | 0.131M | +| net_t.layer3.4.bn1.weight | (256,) | | +| net_t.layer3.4.bn1.bias | (256,) | | +| net_t.layer3.4.conv2 | 0.59M | 0.151G | +| net_t.layer3.4.conv2.weight | (256, 256, 3, 3) | | +| net_t.layer3.4.bn2 | 0.512K | 0.131M | +| net_t.layer3.4.bn2.weight | (256,) | | +| net_t.layer3.4.bn2.bias | (256,) | | +| net_t.layer3.5 | 1.181M | 0.302G | +| net_t.layer3.5.conv1 | 0.59M | 0.151G | +| net_t.layer3.5.conv1.weight | (256, 256, 3, 3) | | +| net_t.layer3.5.bn1 | 0.512K | 0.131M | +| net_t.layer3.5.bn1.weight | (256,) | | +| net_t.layer3.5.bn1.bias | (256,) | | +| net_t.layer3.5.conv2 | 0.59M | 0.151G | +| net_t.layer3.5.conv2.weight | (256, 256, 3, 3) | | +| net_t.layer3.5.bn2 | 0.512K | 0.131M | +| net_t.layer3.5.bn2.weight | (256,) | | +| net_t.layer3.5.bn2.bias | (256,) | | +| mff_oce | 1.458M | 0.269G | +| mff_oce.bn_layer | 0.94M | 66.47M | +| mff_oce.bn_layer.0 | 0.379M | 30.622M | +| mff_oce.bn_layer.0.conv1 | 32.768K | 8.389M | +| mff_oce.bn_layer.0.conv1.weight | (128, 256, 1, 1) | | +| mff_oce.bn_layer.0.bn1 | 0.256K | 65.536K | +| mff_oce.bn_layer.0.bn1.weight | (128,) | | +| mff_oce.bn_layer.0.bn1.bias | (128,) | | +| mff_oce.bn_layer.0.conv2 | 0.147M | 9.437M | +| mff_oce.bn_layer.0.conv2.weight | (128, 128, 3, 3) | | +| mff_oce.bn_layer.0.bn2 | 0.256K | 16.384K | +| mff_oce.bn_layer.0.bn2.weight | (128,) | | +| mff_oce.bn_layer.0.bn2.bias | (128,) | | +| mff_oce.bn_layer.0.conv3 | 65.536K | 4.194M | +| mff_oce.bn_layer.0.conv3.weight | (512, 128, 1, 1) | | +| mff_oce.bn_layer.0.bn3 | 1.024K | 65.536K | +| mff_oce.bn_layer.0.bn3.weight | (512,) | | +| mff_oce.bn_layer.0.bn3.bias | (512,) | | +| mff_oce.bn_layer.0.downsample | 0.132M | 8.454M | +| mff_oce.bn_layer.0.downsample.0 | 0.131M | 8.389M | +| mff_oce.bn_layer.0.downsample.1 | 1.024K | 65.536K | +| mff_oce.bn_layer.1 | 0.28M | 17.924M | +| mff_oce.bn_layer.1.conv1 | 65.536K | 4.194M | +| mff_oce.bn_layer.1.conv1.weight | (128, 512, 1, 1) | | +| mff_oce.bn_layer.1.bn1 | 0.256K | 16.384K | +| mff_oce.bn_layer.1.bn1.weight | (128,) | | +| mff_oce.bn_layer.1.bn1.bias | (128,) | | +| mff_oce.bn_layer.1.conv2 | 0.147M | 9.437M | +| mff_oce.bn_layer.1.conv2.weight | (128, 128, 3, 3) | | +| mff_oce.bn_layer.1.bn2 | 0.256K | 16.384K | +| mff_oce.bn_layer.1.bn2.weight | (128,) | | +| mff_oce.bn_layer.1.bn2.bias | (128,) | | +| mff_oce.bn_layer.1.conv3 | 65.536K | 4.194M | +| mff_oce.bn_layer.1.conv3.weight | (512, 128, 1, 1) | | +| mff_oce.bn_layer.1.bn3 | 1.024K | 65.536K | +| mff_oce.bn_layer.1.bn3.weight | (512,) | | +| mff_oce.bn_layer.1.bn3.bias | (512,) | | +| mff_oce.bn_layer.2 | 0.28M | 17.924M | +| mff_oce.bn_layer.2.conv1 | 65.536K | 4.194M | +| mff_oce.bn_layer.2.conv1.weight | (128, 512, 1, 1) | | +| mff_oce.bn_layer.2.bn1 | 0.256K | 16.384K | +| mff_oce.bn_layer.2.bn1.weight | (128,) | | +| mff_oce.bn_layer.2.bn1.bias | (128,) | | +| mff_oce.bn_layer.2.conv2 | 0.147M | 9.437M | +| mff_oce.bn_layer.2.conv2.weight | (128, 128, 3, 3) | | +| mff_oce.bn_layer.2.bn2 | 0.256K | 16.384K | +| mff_oce.bn_layer.2.bn2.weight | (128,) | | +| mff_oce.bn_layer.2.bn2.bias | (128,) | | +| mff_oce.bn_layer.2.conv3 | 65.536K | 4.194M | +| mff_oce.bn_layer.2.conv3.weight | (512, 128, 1, 1) | | +| mff_oce.bn_layer.2.bn3 | 1.024K | 65.536K | +| mff_oce.bn_layer.2.bn3.weight | (512,) | | +| mff_oce.bn_layer.2.bn3.bias | (512,) | | +| mff_oce.conv1 | 73.728K | 75.497M | +| mff_oce.conv1.weight | (128, 64, 3, 3) | | +| mff_oce.bn1 | 0.256K | 0.262M | +| mff_oce.bn1.weight | (128,) | | +| mff_oce.bn1.bias | (128,) | | +| mff_oce.conv2 | 0.295M | 75.497M | +| mff_oce.conv2.weight | (256, 128, 3, 3) | | +| mff_oce.bn2 | 0.512K | 0.131M | +| mff_oce.bn2.weight | (256,) | | +| mff_oce.bn2.bias | (256,) | | +| mff_oce.conv21 | 16.512K | 16.777M | +| mff_oce.conv21.weight | (128, 128, 1, 1) | | +| mff_oce.conv21.bias | (128,) | | +| mff_oce.bn21 | 0.256K | 0.262M | +| mff_oce.bn21.weight | (128,) | | +| mff_oce.bn21.bias | (128,) | | +| mff_oce.conv31 | 65.792K | 16.777M | +| mff_oce.conv31.weight | (256, 256, 1, 1) | | +| mff_oce.conv31.bias | (256,) | | +| mff_oce.bn31 | 0.512K | 0.131M | +| mff_oce.bn31.weight | (256,) | | +| mff_oce.bn31.bias | (256,) | | +| mff_oce.convf | 65.792K | 16.777M | +| mff_oce.convf.weight | (256, 256, 1, 1) | | +| mff_oce.convf.bias | (256,) | | +| mff_oce.bnf | 0.512K | 0.131M | +| mff_oce.bnf.weight | (256,) | | +| mff_oce.bnf.bias | (256,) | | +| net_s.layers_up | 16.038M | 4.076G | +| net_s.layers_up.0.blocks.0 | 9.441M | 0.577G | +| net_s.layers_up.0.blocks.0.smm_blocks | 7.561M | 0.456G | +| net_s.layers_up.0.blocks.0.smm_blocks.0 | 2.52M | 0.152G | +| net_s.layers_up.0.blocks.0.smm_blocks.0.ln_1 | 1.024K | 0.164M | +| net_s.layers_up.0.blocks.0.smm_blocks.0.self_attention | 2.519M | 0.152G | +| net_s.layers_up.0.blocks.0.smm_blocks.1 | 2.52M | 0.152G | +| net_s.layers_up.0.blocks.0.smm_blocks.1.ln_1 | 1.024K | 0.164M | +| net_s.layers_up.0.blocks.0.smm_blocks.1.self_attention | 2.519M | 0.152G | +| net_s.layers_up.0.blocks.0.smm_blocks.2 | 2.52M | 0.152G | +| net_s.layers_up.0.blocks.0.smm_blocks.2.ln_1 | 1.024K | 0.164M | +| net_s.layers_up.0.blocks.0.smm_blocks.2.self_attention | 2.519M | 0.152G | +| net_s.layers_up.0.blocks.0.conv1b3 | 0.263M | 16.908M | +| net_s.layers_up.0.blocks.0.conv1b3.0 | 0.263M | 16.777M | +| net_s.layers_up.0.blocks.0.conv1b3.0.weight | (512, 512, 1, 1) | | +| net_s.layers_up.0.blocks.0.conv1b3.0.bias | (512,) | | +| net_s.layers_up.0.blocks.0.conv1b3.1 | | 0.131M | +| net_s.layers_up.0.blocks.0.conv1a3 | 0.263M | 16.908M | +| net_s.layers_up.0.blocks.0.conv1a3.0 | 0.263M | 16.777M | +| net_s.layers_up.0.blocks.0.conv1a3.0.weight | (512, 512, 1, 1) | | +| net_s.layers_up.0.blocks.0.conv1a3.0.bias | (512,) | | +| net_s.layers_up.0.blocks.0.conv1a3.1 | | 0.131M | +| net_s.layers_up.0.blocks.0.conv1b5 | 0.263M | 16.908M | +| net_s.layers_up.0.blocks.0.conv1b5.0 | 0.263M | 16.777M | +| net_s.layers_up.0.blocks.0.conv1b5.0.weight | (512, 512, 1, 1) | | +| net_s.layers_up.0.blocks.0.conv1b5.0.bias | (512,) | | +| net_s.layers_up.0.blocks.0.conv1b5.1 | | 0.131M | +| net_s.layers_up.0.blocks.0.conv1a5 | 0.263M | 16.908M | +| net_s.layers_up.0.blocks.0.conv1a5.0 | 0.263M | 16.777M | +| net_s.layers_up.0.blocks.0.conv1a5.0.weight | (512, 512, 1, 1) | | +| net_s.layers_up.0.blocks.0.conv1a5.0.bias | (512,) | | +| net_s.layers_up.0.blocks.0.conv1a5.1 | | 0.131M | +| net_s.layers_up.0.blocks.0.conv33.0 | 4.608K | | +| net_s.layers_up.0.blocks.0.conv33.0.weight | (512, 1, 3, 3) | | +| net_s.layers_up.0.blocks.0.conv55 | 12.8K | 0.95M | +| net_s.layers_up.0.blocks.0.conv55.0 | 12.8K | 0.819M | +| net_s.layers_up.0.blocks.0.conv55.0.weight | (512, 1, 5, 5) | | +| net_s.layers_up.0.blocks.0.conv55.1 | | 0.131M | +| net_s.layers_up.0.blocks.0.conv77 | 25.088K | 1.737M | +| net_s.layers_up.0.blocks.0.conv77.0 | 25.088K | 1.606M | +| net_s.layers_up.0.blocks.0.conv77.0.weight | (512, 1, 7, 7) | | +| net_s.layers_up.0.blocks.0.conv77.1 | | 0.131M | +| net_s.layers_up.0.blocks.0.finalconv11 | 0.787M | 50.332M | +| net_s.layers_up.0.blocks.0.finalconv11.weight | (512, 1536, 1, 1) | | +| net_s.layers_up.0.blocks.0.finalconv11.bias | (512,) | | +| net_s.layers_up.1 | 4.432M | 0.961G | +| net_s.layers_up.1.blocks | 3.908M | 0.928G | +| net_s.layers_up.1.blocks.0 | 1.954M | 0.464G | +| net_s.layers_up.1.blocks.0.smm_blocks | 1.473M | 0.34G | +| net_s.layers_up.1.blocks.0.conv1b3 | 65.792K | 17.039M | +| net_s.layers_up.1.blocks.0.conv1a3 | 65.792K | 17.039M | +| net_s.layers_up.1.blocks.0.conv1b5 | 65.792K | 17.039M | +| net_s.layers_up.1.blocks.0.conv1a5 | 65.792K | 17.039M | +| net_s.layers_up.1.blocks.0.conv33.0 | 2.304K | | +| net_s.layers_up.1.blocks.0.conv55 | 6.4K | 1.901M | +| net_s.layers_up.1.blocks.0.conv77 | 12.544K | 3.473M | +| net_s.layers_up.1.blocks.0.finalconv11 | 0.197M | 50.332M | +| net_s.layers_up.1.blocks.1 | 1.954M | 0.464G | +| net_s.layers_up.1.blocks.1.smm_blocks | 1.473M | 0.34G | +| net_s.layers_up.1.blocks.1.conv1b3 | 65.792K | 17.039M | +| net_s.layers_up.1.blocks.1.conv1a3 | 65.792K | 17.039M | +| net_s.layers_up.1.blocks.1.conv1b5 | 65.792K | 17.039M | +| net_s.layers_up.1.blocks.1.conv1a5 | 65.792K | 17.039M | +| net_s.layers_up.1.blocks.1.conv33.0 | 2.304K | | +| net_s.layers_up.1.blocks.1.conv55 | 6.4K | 1.901M | +| net_s.layers_up.1.blocks.1.conv77 | 12.544K | 3.473M | +| net_s.layers_up.1.blocks.1.finalconv11 | 0.197M | 50.332M | +| net_s.layers_up.1.upsample | 0.525M | 33.882M | +| net_s.layers_up.1.upsample.expand | 0.524M | 33.554M | +| net_s.layers_up.1.upsample.expand.weight | (1024, 512) | | +| net_s.layers_up.1.upsample.norm | 0.512K | 0.328M | +| net_s.layers_up.1.upsample.norm.weight | (256,) | | +| net_s.layers_up.1.upsample.norm.bias | (256,) | | +| net_s.layers_up.2 | 1.816M | 1.529G | +| net_s.layers_up.2.blocks | 1.685M | 1.495G | +| net_s.layers_up.2.blocks.0 | 0.842M | 0.747G | +| net_s.layers_up.2.blocks.0.smm_blocks | 0.717M | 0.617G | +| net_s.layers_up.2.blocks.0.conv1b3 | 16.512K | 17.302M | +| net_s.layers_up.2.blocks.0.conv1a3 | 16.512K | 17.302M | +| net_s.layers_up.2.blocks.0.conv1b5 | 16.512K | 17.302M | +| net_s.layers_up.2.blocks.0.conv1a5 | 16.512K | 17.302M | +| net_s.layers_up.2.blocks.0.conv33.0 | 1.152K | | +| net_s.layers_up.2.blocks.0.conv55 | 3.2K | 3.801M | +| net_s.layers_up.2.blocks.0.conv77 | 6.272K | 6.947M | +| net_s.layers_up.2.blocks.0.finalconv11 | 49.28K | 50.332M | +| net_s.layers_up.2.blocks.1 | 0.842M | 0.747G | +| net_s.layers_up.2.blocks.1.smm_blocks | 0.717M | 0.617G | +| net_s.layers_up.2.blocks.1.conv1b3 | 16.512K | 17.302M | +| net_s.layers_up.2.blocks.1.conv1a3 | 16.512K | 17.302M | +| net_s.layers_up.2.blocks.1.conv1b5 | 16.512K | 17.302M | +| net_s.layers_up.2.blocks.1.conv1a5 | 16.512K | 17.302M | +| net_s.layers_up.2.blocks.1.conv33.0 | 1.152K | | +| net_s.layers_up.2.blocks.1.conv55 | 3.2K | 3.801M | +| net_s.layers_up.2.blocks.1.conv77 | 6.272K | 6.947M | +| net_s.layers_up.2.blocks.1.finalconv11 | 49.28K | 50.332M | +| net_s.layers_up.2.upsample | 0.131M | 34.21M | +| net_s.layers_up.2.upsample.expand | 0.131M | 33.554M | +| net_s.layers_up.2.upsample.expand.weight | (512, 256) | | +| net_s.layers_up.2.upsample.norm | 0.256K | 0.655M | +| net_s.layers_up.2.upsample.norm.weight | (128,) | | +| net_s.layers_up.2.upsample.norm.bias | (128,) | | +| net_s.layers_up.3 | 0.349M | 1.009G | +| net_s.layers_up.3.blocks.0 | 0.316M | 0.974G | +| net_s.layers_up.3.blocks.0.smm_blocks | 0.281M | 0.831G | +| net_s.layers_up.3.blocks.0.smm_blocks.0 | 93.824K | 0.277G | +| net_s.layers_up.3.blocks.0.smm_blocks.1 | 93.824K | 0.277G | +| net_s.layers_up.3.blocks.0.smm_blocks.2 | 93.824K | 0.277G | +| net_s.layers_up.3.blocks.0.conv1b3 | 4.16K | 17.826M | +| net_s.layers_up.3.blocks.0.conv1b3.0 | 4.16K | 16.777M | +| net_s.layers_up.3.blocks.0.conv1b3.1 | | 1.049M | +| net_s.layers_up.3.blocks.0.conv1a3 | 4.16K | 17.826M | +| net_s.layers_up.3.blocks.0.conv1a3.0 | 4.16K | 16.777M | +| net_s.layers_up.3.blocks.0.conv1a3.1 | | 1.049M | +| net_s.layers_up.3.blocks.0.conv1b5 | 4.16K | 17.826M | +| net_s.layers_up.3.blocks.0.conv1b5.0 | 4.16K | 16.777M | +| net_s.layers_up.3.blocks.0.conv1b5.1 | | 1.049M | +| net_s.layers_up.3.blocks.0.conv1a5 | 4.16K | 17.826M | +| net_s.layers_up.3.blocks.0.conv1a5.0 | 4.16K | 16.777M | +| net_s.layers_up.3.blocks.0.conv1a5.1 | | 1.049M | +| net_s.layers_up.3.blocks.0.conv33.0 | 0.576K | | +| net_s.layers_up.3.blocks.0.conv33.0.weight | (64, 1, 3, 3) | | +| net_s.layers_up.3.blocks.0.conv55 | 1.6K | 7.602M | +| net_s.layers_up.3.blocks.0.conv55.0 | 1.6K | 6.554M | +| net_s.layers_up.3.blocks.0.conv55.1 | | 1.049M | +| net_s.layers_up.3.blocks.0.conv77 | 3.136K | 13.894M | +| net_s.layers_up.3.blocks.0.conv77.0 | 3.136K | 12.845M | +| net_s.layers_up.3.blocks.0.conv77.1 | | 1.049M | +| net_s.layers_up.3.blocks.0.finalconv11 | 12.352K | 50.332M | +| net_s.layers_up.3.blocks.0.finalconv11.weight | (64, 192, 1, 1) | | +| net_s.layers_up.3.blocks.0.finalconv11.bias | (64,) | | +| net_s.layers_up.3.upsample | 32.896K | 34.865M | +| net_s.layers_up.3.upsample.expand | 32.768K | 33.554M | +| net_s.layers_up.3.upsample.expand.weight | (256, 128) | | +| net_s.layers_up.3.upsample.norm | 0.128K | 1.311M | +| net_s.layers_up.3.upsample.norm.weight | (64,) | | +| net_s.layers_up.3.upsample.norm.bias | (64,) | | +--------------------------------------------------------------------------------- +2024-04-08 23:28:58,866 - ==> Creating optimizer +2024-04-08 23:28:58,881 - ==> Loading dataset: RealIADO +2024-04-08 23:29:00,003 - ==> ********** cfg ********** +epoch_full : 1000 +fvcore_b : 1 +fvcore_c : 3 +metrics : ['mAUROC_sp_max', 'mAP_sp_max', 'mF1_max_sp_max', 'mAUROC_px', 'mAP_px', 'mF1_max_px', 'mAUPRO_px', 'mF1_px_0.2_0.8_0.1', 'mAcc_px_0.2_0.8_0.1', 'mIoU_px_0.2_0.8_0.1', 'mIoU_max_px'] +evaluator.kwargs : {'metrics': ['mAUROC_sp_max', 'mAP_sp_max', 'mF1_max_sp_max', 'mAUROC_px', 'mAP_px', 'mF1_max_px', 'mAUPRO_px', 'mF1_px_0.2_0.8_0.1', 'mAcc_px_0.2_0.8_0.1', 'mIoU_px_0.2_0.8_0.1', 'mIoU_max_px'], 'pooling_ks': None, 'max_step_aupro': 100} +optim.lr : 0.005 +optim.kwargs : {'name': 'adamw', 'betas': (0.9, 0.999), 'eps': 1e-08, 'weight_decay': 0.0001, 'amsgrad': False} +trainer.name : MAMBAADTrainer +trainer.checkpoint : runs +trainer.logdir_sub : +trainer.resume_dir : +trainer.cuda_deterministic : False +trainer.epoch_full : 1000 +trainer.scheduler_kwargs : {'name': 'step', 'lr_noise': None, 'noise_pct': 0.67, 'noise_std': 1.0, 'noise_seed': 42, 'lr_min': 5e-05, 'warmup_lr': 5e-06, 'warmup_iters': -1, 'cooldown_iters': 0, 'warmup_epochs': 0, 'cooldown_epochs': 0, 'use_iters': True, 'patience_iters': 0, 'patience_epochs': 0, 'decay_iters': 0, 'decay_epochs': 800, 'cycle_decay': 0.1, 'decay_rate': 0.1} +trainer.mixup_kwargs : {'mixup_alpha': 0.8, 'cutmix_alpha': 1.0, 'cutmix_minmax': None, 'prob': 0.0, 'switch_prob': 0.5, 'mode': 'batch', 'correct_lam': True, 'label_smoothing': 0.1} +trainer.test_start_epoch : 1000 +trainer.test_per_epoch : 50 +trainer.find_unused_parameters : False +trainer.sync_BN : apex +trainer.dist_BN : +trainer.scaler : none +trainer.data.batch_size : 16 +trainer.data.batch_size_per_gpu : 16 +trainer.data.batch_size_test : 16 +trainer.data.batch_size_per_gpu_test : 16 +trainer.data.num_workers_per_gpu : 4 +trainer.data.drop_last : True +trainer.data.pin_memory : True +trainer.data.persistent_workers : False +trainer.data.num_workers : 4 +trainer.iter : 0 +trainer.epoch : 0 +trainer.iter_full : 2279000 +trainer.metric_recorder : {'mAUROC_sp_max_audiojack': [], 'mAP_sp_max_audiojack': [], 'mF1_max_sp_max_audiojack': [], 'mAUROC_px_audiojack': [], 'mAP_px_audiojack': [], 'mF1_max_px_audiojack': [], 'mAUPRO_px_audiojack': [], 'mF1_px_0.2_0.8_0.1_audiojack': [], 'mAcc_px_0.2_0.8_0.1_audiojack': [], 'mIoU_px_0.2_0.8_0.1_audiojack': [], 'mIoU_max_px_audiojack': [], 'mAUROC_sp_max_pcb': [], 'mAP_sp_max_pcb': [], 'mF1_max_sp_max_pcb': [], 'mAUROC_px_pcb': [], 'mAP_px_pcb': [], 'mF1_max_px_pcb': [], 'mAUPRO_px_pcb': [], 'mF1_px_0.2_0.8_0.1_pcb': [], 'mAcc_px_0.2_0.8_0.1_pcb': [], 'mIoU_px_0.2_0.8_0.1_pcb': [], 'mIoU_max_px_pcb': [], 'mAUROC_sp_max_phone_battery': [], 'mAP_sp_max_phone_battery': [], 'mF1_max_sp_max_phone_battery': [], 'mAUROC_px_phone_battery': [], 'mAP_px_phone_battery': [], 'mF1_max_px_phone_battery': [], 'mAUPRO_px_phone_battery': [], 'mF1_px_0.2_0.8_0.1_phone_battery': [], 'mAcc_px_0.2_0.8_0.1_phone_battery': [], 'mIoU_px_0.2_0.8_0.1_phone_battery': [], 'mIoU_max_px_phone_battery': [], 'mAUROC_sp_max_sim_card_set': [], 'mAP_sp_max_sim_card_set': [], 'mF1_max_sp_max_sim_card_set': [], 'mAUROC_px_sim_card_set': [], 'mAP_px_sim_card_set': [], 'mF1_max_px_sim_card_set': [], 'mAUPRO_px_sim_card_set': [], 'mF1_px_0.2_0.8_0.1_sim_card_set': [], 'mAcc_px_0.2_0.8_0.1_sim_card_set': [], 'mIoU_px_0.2_0.8_0.1_sim_card_set': [], 'mIoU_max_px_sim_card_set': [], 'mAUROC_sp_max_switch': [], 'mAP_sp_max_switch': [], 'mF1_max_sp_max_switch': [], 'mAUROC_px_switch': [], 'mAP_px_switch': [], 'mF1_max_px_switch': [], 'mAUPRO_px_switch': [], 'mF1_px_0.2_0.8_0.1_switch': [], 'mAcc_px_0.2_0.8_0.1_switch': [], 'mIoU_px_0.2_0.8_0.1_switch': [], 'mIoU_max_px_switch': [], 'mAUROC_sp_max_terminalblock': [], 'mAP_sp_max_terminalblock': [], 'mF1_max_sp_max_terminalblock': [], 'mAUROC_px_terminalblock': [], 'mAP_px_terminalblock': [], 'mF1_max_px_terminalblock': [], 'mAUPRO_px_terminalblock': [], 'mF1_px_0.2_0.8_0.1_terminalblock': [], 'mAcc_px_0.2_0.8_0.1_terminalblock': [], 'mIoU_px_0.2_0.8_0.1_terminalblock': [], 'mIoU_max_px_terminalblock': [], 'mAUROC_sp_max_toothbrush': [], 'mAP_sp_max_toothbrush': [], 'mF1_max_sp_max_toothbrush': [], 'mAUROC_px_toothbrush': [], 'mAP_px_toothbrush': [], 'mF1_max_px_toothbrush': [], 'mAUPRO_px_toothbrush': [], 'mF1_px_0.2_0.8_0.1_toothbrush': [], 'mAcc_px_0.2_0.8_0.1_toothbrush': [], 'mIoU_px_0.2_0.8_0.1_toothbrush': [], 'mIoU_max_px_toothbrush': [], 'mAUROC_sp_max_toy': [], 'mAP_sp_max_toy': [], 'mF1_max_sp_max_toy': [], 'mAUROC_px_toy': [], 'mAP_px_toy': [], 'mF1_max_px_toy': [], 'mAUPRO_px_toy': [], 'mF1_px_0.2_0.8_0.1_toy': [], 'mAcc_px_0.2_0.8_0.1_toy': [], 'mIoU_px_0.2_0.8_0.1_toy': [], 'mIoU_max_px_toy': [], 'mAUROC_sp_max_transistor1': [], 'mAP_sp_max_transistor1': [], 'mF1_max_sp_max_transistor1': [], 'mAUROC_px_transistor1': [], 'mAP_px_transistor1': [], 'mF1_max_px_transistor1': [], 'mAUPRO_px_transistor1': [], 'mF1_px_0.2_0.8_0.1_transistor1': [], 'mAcc_px_0.2_0.8_0.1_transistor1': [], 'mIoU_px_0.2_0.8_0.1_transistor1': [], 'mIoU_max_px_transistor1': [], 'mAUROC_sp_max_usb': [], 'mAP_sp_max_usb': [], 'mF1_max_sp_max_usb': [], 'mAUROC_px_usb': [], 'mAP_px_usb': [], 'mF1_max_px_usb': [], 'mAUPRO_px_usb': [], 'mF1_px_0.2_0.8_0.1_usb': [], 'mAcc_px_0.2_0.8_0.1_usb': [], 'mIoU_px_0.2_0.8_0.1_usb': [], 'mIoU_max_px_usb': [], 'mAUROC_sp_max_usb_adaptor': [], 'mAP_sp_max_usb_adaptor': [], 'mF1_max_sp_max_usb_adaptor': [], 'mAUROC_px_usb_adaptor': [], 'mAP_px_usb_adaptor': [], 'mF1_max_px_usb_adaptor': [], 'mAUPRO_px_usb_adaptor': [], 'mF1_px_0.2_0.8_0.1_usb_adaptor': [], 'mAcc_px_0.2_0.8_0.1_usb_adaptor': [], 'mIoU_px_0.2_0.8_0.1_usb_adaptor': [], 'mIoU_max_px_usb_adaptor': [], 'mAUROC_sp_max_zipper': [], 'mAP_sp_max_zipper': [], 'mF1_max_sp_max_zipper': [], 'mAUROC_px_zipper': [], 'mAP_px_zipper': [], 'mF1_max_px_zipper': [], 'mAUPRO_px_zipper': [], 'mF1_px_0.2_0.8_0.1_zipper': [], 'mAcc_px_0.2_0.8_0.1_zipper': [], 'mIoU_px_0.2_0.8_0.1_zipper': [], 'mIoU_max_px_zipper': [], 'mAUROC_sp_max_bottle_cap': [], 'mAP_sp_max_bottle_cap': [], 'mF1_max_sp_max_bottle_cap': [], 'mAUROC_px_bottle_cap': [], 'mAP_px_bottle_cap': [], 'mF1_max_px_bottle_cap': [], 'mAUPRO_px_bottle_cap': [], 'mF1_px_0.2_0.8_0.1_bottle_cap': [], 'mAcc_px_0.2_0.8_0.1_bottle_cap': [], 'mIoU_px_0.2_0.8_0.1_bottle_cap': [], 'mIoU_max_px_bottle_cap': [], 'mAUROC_sp_max_end_cap': [], 'mAP_sp_max_end_cap': [], 'mF1_max_sp_max_end_cap': [], 'mAUROC_px_end_cap': [], 'mAP_px_end_cap': [], 'mF1_max_px_end_cap': [], 'mAUPRO_px_end_cap': [], 'mF1_px_0.2_0.8_0.1_end_cap': [], 'mAcc_px_0.2_0.8_0.1_end_cap': [], 'mIoU_px_0.2_0.8_0.1_end_cap': [], 'mIoU_max_px_end_cap': [], 'mAUROC_sp_max_fire_hood': [], 'mAP_sp_max_fire_hood': [], 'mF1_max_sp_max_fire_hood': [], 'mAUROC_px_fire_hood': [], 'mAP_px_fire_hood': [], 'mF1_max_px_fire_hood': [], 'mAUPRO_px_fire_hood': [], 'mF1_px_0.2_0.8_0.1_fire_hood': [], 'mAcc_px_0.2_0.8_0.1_fire_hood': [], 'mIoU_px_0.2_0.8_0.1_fire_hood': [], 'mIoU_max_px_fire_hood': [], 'mAUROC_sp_max_mounts': [], 'mAP_sp_max_mounts': [], 'mF1_max_sp_max_mounts': [], 'mAUROC_px_mounts': [], 'mAP_px_mounts': [], 'mF1_max_px_mounts': [], 'mAUPRO_px_mounts': [], 'mF1_px_0.2_0.8_0.1_mounts': [], 'mAcc_px_0.2_0.8_0.1_mounts': [], 'mIoU_px_0.2_0.8_0.1_mounts': [], 'mIoU_max_px_mounts': [], 'mAUROC_sp_max_plastic_nut': [], 'mAP_sp_max_plastic_nut': [], 'mF1_max_sp_max_plastic_nut': [], 'mAUROC_px_plastic_nut': [], 'mAP_px_plastic_nut': [], 'mF1_max_px_plastic_nut': [], 'mAUPRO_px_plastic_nut': [], 'mF1_px_0.2_0.8_0.1_plastic_nut': [], 'mAcc_px_0.2_0.8_0.1_plastic_nut': [], 'mIoU_px_0.2_0.8_0.1_plastic_nut': [], 'mIoU_max_px_plastic_nut': [], 'mAUROC_sp_max_plastic_plug': [], 'mAP_sp_max_plastic_plug': [], 'mF1_max_sp_max_plastic_plug': [], 'mAUROC_px_plastic_plug': [], 'mAP_px_plastic_plug': [], 'mF1_max_px_plastic_plug': [], 'mAUPRO_px_plastic_plug': [], 'mF1_px_0.2_0.8_0.1_plastic_plug': [], 'mAcc_px_0.2_0.8_0.1_plastic_plug': [], 'mIoU_px_0.2_0.8_0.1_plastic_plug': [], 'mIoU_max_px_plastic_plug': [], 'mAUROC_sp_max_regulator': [], 'mAP_sp_max_regulator': [], 'mF1_max_sp_max_regulator': [], 'mAUROC_px_regulator': [], 'mAP_px_regulator': [], 'mF1_max_px_regulator': [], 'mAUPRO_px_regulator': [], 'mF1_px_0.2_0.8_0.1_regulator': [], 'mAcc_px_0.2_0.8_0.1_regulator': [], 'mIoU_px_0.2_0.8_0.1_regulator': [], 'mIoU_max_px_regulator': [], 'mAUROC_sp_max_rolled_strip_base': [], 'mAP_sp_max_rolled_strip_base': [], 'mF1_max_sp_max_rolled_strip_base': [], 'mAUROC_px_rolled_strip_base': [], 'mAP_px_rolled_strip_base': [], 'mF1_max_px_rolled_strip_base': [], 'mAUPRO_px_rolled_strip_base': [], 'mF1_px_0.2_0.8_0.1_rolled_strip_base': [], 'mAcc_px_0.2_0.8_0.1_rolled_strip_base': [], 'mIoU_px_0.2_0.8_0.1_rolled_strip_base': [], 'mIoU_max_px_rolled_strip_base': [], 'mAUROC_sp_max_toy_brick': [], 'mAP_sp_max_toy_brick': [], 'mF1_max_sp_max_toy_brick': [], 'mAUROC_px_toy_brick': [], 'mAP_px_toy_brick': [], 'mF1_max_px_toy_brick': [], 'mAUPRO_px_toy_brick': [], 'mF1_px_0.2_0.8_0.1_toy_brick': [], 'mAcc_px_0.2_0.8_0.1_toy_brick': [], 'mIoU_px_0.2_0.8_0.1_toy_brick': [], 'mIoU_max_px_toy_brick': [], 'mAUROC_sp_max_u_block': [], 'mAP_sp_max_u_block': [], 'mF1_max_sp_max_u_block': [], 'mAUROC_px_u_block': [], 'mAP_px_u_block': [], 'mF1_max_px_u_block': [], 'mAUPRO_px_u_block': [], 'mF1_px_0.2_0.8_0.1_u_block': [], 'mAcc_px_0.2_0.8_0.1_u_block': [], 'mIoU_px_0.2_0.8_0.1_u_block': [], 'mIoU_max_px_u_block': [], 'mAUROC_sp_max_vcpill': [], 'mAP_sp_max_vcpill': [], 'mF1_max_sp_max_vcpill': [], 'mAUROC_px_vcpill': [], 'mAP_px_vcpill': [], 'mF1_max_px_vcpill': [], 'mAUPRO_px_vcpill': [], 'mF1_px_0.2_0.8_0.1_vcpill': [], 'mAcc_px_0.2_0.8_0.1_vcpill': [], 'mIoU_px_0.2_0.8_0.1_vcpill': [], 'mIoU_max_px_vcpill': [], 'mAUROC_sp_max_wooden_beads': [], 'mAP_sp_max_wooden_beads': [], 'mF1_max_sp_max_wooden_beads': [], 'mAUROC_px_wooden_beads': [], 'mAP_px_wooden_beads': [], 'mF1_max_px_wooden_beads': [], 'mAUPRO_px_wooden_beads': [], 'mF1_px_0.2_0.8_0.1_wooden_beads': [], 'mAcc_px_0.2_0.8_0.1_wooden_beads': [], 'mIoU_px_0.2_0.8_0.1_wooden_beads': [], 'mIoU_max_px_wooden_beads': [], 'mAUROC_sp_max_woodstick': [], 'mAP_sp_max_woodstick': [], 'mF1_max_sp_max_woodstick': [], 'mAUROC_px_woodstick': [], 'mAP_px_woodstick': [], 'mF1_max_px_woodstick': [], 'mAUPRO_px_woodstick': [], 'mF1_px_0.2_0.8_0.1_woodstick': [], 'mAcc_px_0.2_0.8_0.1_woodstick': [], 'mIoU_px_0.2_0.8_0.1_woodstick': [], 'mIoU_max_px_woodstick': [], 'mAUROC_sp_max_tape': [], 'mAP_sp_max_tape': [], 'mF1_max_sp_max_tape': [], 'mAUROC_px_tape': [], 'mAP_px_tape': [], 'mF1_max_px_tape': [], 'mAUPRO_px_tape': [], 'mF1_px_0.2_0.8_0.1_tape': [], 'mAcc_px_0.2_0.8_0.1_tape': [], 'mIoU_px_0.2_0.8_0.1_tape': [], 'mIoU_max_px_tape': [], 'mAUROC_sp_max_porcelain_doll': [], 'mAP_sp_max_porcelain_doll': [], 'mF1_max_sp_max_porcelain_doll': [], 'mAUROC_px_porcelain_doll': [], 'mAP_px_porcelain_doll': [], 'mF1_max_px_porcelain_doll': [], 'mAUPRO_px_porcelain_doll': [], 'mF1_px_0.2_0.8_0.1_porcelain_doll': [], 'mAcc_px_0.2_0.8_0.1_porcelain_doll': [], 'mIoU_px_0.2_0.8_0.1_porcelain_doll': [], 'mIoU_max_px_porcelain_doll': [], 'mAUROC_sp_max_mint': [], 'mAP_sp_max_mint': [], 'mF1_max_sp_max_mint': [], 'mAUROC_px_mint': [], 'mAP_px_mint': [], 'mF1_max_px_mint': [], 'mAUPRO_px_mint': [], 'mF1_px_0.2_0.8_0.1_mint': [], 'mAcc_px_0.2_0.8_0.1_mint': [], 'mIoU_px_0.2_0.8_0.1_mint': [], 'mIoU_max_px_mint': [], 'mAUROC_sp_max_eraser': [], 'mAP_sp_max_eraser': [], 'mF1_max_sp_max_eraser': [], 'mAUROC_px_eraser': [], 'mAP_px_eraser': [], 'mF1_max_px_eraser': [], 'mAUPRO_px_eraser': [], 'mF1_px_0.2_0.8_0.1_eraser': [], 'mAcc_px_0.2_0.8_0.1_eraser': [], 'mIoU_px_0.2_0.8_0.1_eraser': [], 'mIoU_max_px_eraser': [], 'mAUROC_sp_max_button_battery': [], 'mAUROC_sp_max_Avg': [], 'mAP_sp_max_button_battery': [], 'mAP_sp_max_Avg': [], 'mF1_max_sp_max_button_battery': [], 'mF1_max_sp_max_Avg': [], 'mAUROC_px_button_battery': [], 'mAUROC_px_Avg': [], 'mAP_px_button_battery': [], 'mAP_px_Avg': [], 'mF1_max_px_button_battery': [], 'mF1_max_px_Avg': [], 'mAUPRO_px_button_battery': [], 'mAUPRO_px_Avg': [], 'mF1_px_0.2_0.8_0.1_button_battery': [], 'mF1_px_0.2_0.8_0.1_Avg': [], 'mAcc_px_0.2_0.8_0.1_button_battery': [], 'mAcc_px_0.2_0.8_0.1_Avg': [], 'mIoU_px_0.2_0.8_0.1_button_battery': [], 'mIoU_px_0.2_0.8_0.1_Avg': [], 'mIoU_max_px_button_battery': [], 'mIoU_max_px_Avg': []} +loss.loss_terms : [{'type': 'L2Loss', 'name': 'pixel', 'lam': 5.0}] +loss.clip_grad : 5.0 +loss.create_graph : False +loss.retain_graph : False +adv : False +logging.log_terms_train : [{'name': 'batch_t', 'fmt': ':>5.3f', 'add_name': 'avg'}, {'name': 'data_t', 'fmt': ':>5.3f'}, {'name': 'optim_t', 'fmt': ':>5.3f'}, {'name': 'lr', 'fmt': ':>7.6f'}, {'name': 'cos', 'suffixes': [''], 'fmt': ':>5.3f', 'add_name': 'avg'}] +logging.log_terms_test : [{'name': 'batch_t', 'fmt': ':>5.3f', 'add_name': 'avg'}, {'name': 'cos', 'suffixes': [''], 'fmt': ':>5.3f', 'add_name': 'avg'}] +logging.train_reset_log_per : 100 +logging.train_log_per : 100 +logging.test_log_per : 50 +data.sampler : naive +data.loader_type : pil +data.loader_type_target : pil_L +data.type : RealIADO +data.root : data/realiad +data.meta : meta.json +data.cls_names : ['audiojack', 'pcb', 'phone_battery', 'sim_card_set', 'switch', 'terminalblock', 'toothbrush', 'toy', 'transistor1', 'usb', 'usb_adaptor', 'zipper', 'bottle_cap', 'end_cap', 'fire_hood', 'mounts', 'plastic_nut', 'plastic_plug', 'regulator', 'rolled_strip_base', 'toy_brick', 'u_block', 'vcpill', 'wooden_beads', 'woodstick', 'tape', 'porcelain_doll', 'mint', 'eraser', 'button_battery'] +data.train_transforms : [{'type': 'Resize', 'size': (256, 256), 'interpolation': }, {'type': 'CenterCrop', 'size': (256, 256)}, {'type': 'ToTensor'}, {'type': 'Normalize', 'mean': (0.485, 0.456, 0.406), 'std': (0.229, 0.224, 0.225), 'inplace': True}] +data.test_transforms : [{'type': 'Resize', 'size': (256, 256), 'interpolation': }, {'type': 'CenterCrop', 'size': (256, 256)}, {'type': 'ToTensor'}, {'type': 'Normalize', 'mean': (0.485, 0.456, 0.406), 'std': (0.229, 0.224, 0.225), 'inplace': True}] +data.target_transforms : [{'type': 'Resize', 'size': (256, 256), 'interpolation': }, {'type': 'CenterCrop', 'size': (256, 256)}, {'type': 'ToTensor'}] +data.views : [] +data.train_size : 2279 +data.test_size : 7162 +data.train_length : 36465 +data.test_length : 114585 +model_t.name : timm_resnet34 +model_t.kwargs : {'pretrained': False, 'checkpoint_path': 'model/pretrain/resnet34-43635321.pth', 'strict': False, 'features_only': True, 'out_indices': [1, 2, 3]} +model_s : {'depths_decoder': [3, 4, 6, 3], 'scan_type': 'hilbert', 'num_direction': 8} +model.name : mambaadhcs57c1 +model.kwargs : {'pretrained': False, 'checkpoint_path': '', 'strict': True, 'model_t': Namespace(name='timm_resnet34', kwargs={'pretrained': False, 'checkpoint_path': 'model/pretrain/resnet34-43635321.pth', 'strict': False, 'features_only': True, 'out_indices': [1, 2, 3]}), 'model_s': {'depths_decoder': [3, 4, 6, 3], 'scan_type': 'hilbert', 'num_direction': 8}} +seed : 42 +size : 256 +warmup_epochs : 0 +test_start_epoch : 1000 +test_per_epoch : 50 +batch_train : 16 +batch_test_per : 16 +lr : 0.005 +weight_decay : 0.0001 +cfg_path : configs.mambaad.mambaad_realiad_nhcs57c1o +mode : train +sleep : -1 +memory : -1 +dist_url : env:// +logger_rank : 0 +opts : [] +command : python3 -m torch.distributed.launch --nproc_per_node=$nproc_per_node --nnodes=$nnodes --node_rank=$node_rank --master_addr=$master_addr --master_port=$master_port --use_env run.py -c configs.mambaad.mambaad_realiad_nhcs57c1o -m train --sleep -1 --memory -1 --dist_url env:// --logger_rank 0 +task_start_time : 4191351.235898727 +dist : False +world_size : 1 +rank : 0 +local_rank : 0 +ngpus_per_node : 1 +nnodes : 1 +master : True +logdir : runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839 +logger.filters : [] +logger.name : root +logger.level : 20 +logger.parent : None +logger.propagate : True +logger.disabled : False +logdir_train : runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839/show_train +logdir_test : runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839/show_test +2024-04-08 23:29:00,004 - ==> Starting training with 1 nodes x 1 GPUs +2024-04-08 23:30:08,823 - Train: 0.00% [100/2279000] [0.0/1000.0] [batch_t 0.605 (0.684)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-08 23:31:09,173 - Train: 0.01% [200/2279000] [0.1/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-08 23:32:09,528 - Train: 0.01% [300/2279000] [0.1/1000.0] [batch_t 0.601 (0.603)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-08 23:33:09,897 - Train: 0.02% [400/2279000] [0.2/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-08 23:34:10,349 - Train: 0.02% [500/2279000] [0.2/1000.0] [batch_t 0.608 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-08 23:35:10,900 - Train: 0.03% [600/2279000] [0.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-08 23:36:11,288 - Train: 0.03% [700/2279000] [0.3/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-08 23:37:11,744 - Train: 0.04% [800/2279000] [0.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-08 23:38:12,194 - Train: 0.04% [900/2279000] [0.4/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-08 23:39:12,649 - Train: 0.04% [1000/2279000] [0.4/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-08 23:40:13,015 - Train: 0.05% [1100/2279000] [0.5/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-08 23:41:13,377 - Train: 0.05% [1200/2279000] [0.5/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-08 23:42:13,878 - Train: 0.06% [1300/2279000] [0.6/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-08 23:43:14,305 - Train: 0.06% [1400/2279000] [0.6/1000.0] [batch_t 0.603 (0.604)] [data_t 0.003] [optim_t 0.600] [lr 0.005000] +2024-04-08 23:44:14,698 - Train: 0.07% [1500/2279000] [0.7/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-08 23:45:15,136 - Train: 0.07% [1600/2279000] [0.7/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-08 23:46:15,613 - Train: 0.07% [1700/2279000] [0.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-08 23:47:16,162 - Train: 0.08% [1800/2279000] [0.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-08 23:48:16,650 - Train: 0.08% [1900/2279000] [0.8/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-08 23:49:17,093 - Train: 0.09% [2000/2279000] [0.9/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-08 23:50:17,538 - Train: 0.09% [2100/2279000] [0.9/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-08 23:51:17,972 - Train: 0.10% [2200/2279000] [1.0/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-08 23:52:05,705 - ==> Total time: 0:23:25 Eta: 16 days, 6:06:40 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-08 23:52:20,353 - Train: 0.10% [2300/2279000] [1.0/1000.0] [batch_t 0.606 (0.621)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-08 23:53:20,915 - Train: 0.11% [2400/2279000] [1.1/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-08 23:54:21,354 - Train: 0.11% [2500/2279000] [1.1/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-08 23:55:21,810 - Train: 0.11% [2600/2279000] [1.1/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-08 23:56:22,251 - Train: 0.12% [2700/2279000] [1.2/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-08 23:57:22,693 - Train: 0.12% [2800/2279000] [1.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-08 23:58:23,138 - Train: 0.13% [2900/2279000] [1.3/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-08 23:59:23,657 - Train: 0.13% [3000/2279000] [1.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 00:00:24,088 - Train: 0.14% [3100/2279000] [1.4/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 00:01:24,518 - Train: 0.14% [3200/2279000] [1.4/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 00:02:24,934 - Train: 0.14% [3300/2279000] [1.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 00:03:25,360 - Train: 0.15% [3400/2279000] [1.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 00:04:25,862 - Train: 0.15% [3500/2279000] [1.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 00:05:26,289 - Train: 0.16% [3600/2279000] [1.6/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 00:06:26,707 - Train: 0.16% [3700/2279000] [1.6/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 00:07:27,127 - Train: 0.17% [3800/2279000] [1.7/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 00:08:27,549 - Train: 0.17% [3900/2279000] [1.7/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 00:09:27,998 - Train: 0.18% [4000/2279000] [1.8/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 00:10:28,422 - Train: 0.18% [4100/2279000] [1.8/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 00:11:28,859 - Train: 0.18% [4200/2279000] [1.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 00:12:29,349 - Train: 0.19% [4300/2279000] [1.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 00:13:29,796 - Train: 0.19% [4400/2279000] [1.9/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 00:14:30,253 - Train: 0.20% [4500/2279000] [2.0/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 00:15:05,316 - ==> Total time: 0:46:25 Eta: 16 days, 2:05:23 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-09 00:15:32,578 - Train: 0.20% [4600/2279000] [2.0/1000.0] [batch_t 0.606 (0.616)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 00:16:33,041 - Train: 0.21% [4700/2279000] [2.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 00:17:33,622 - Train: 0.21% [4800/2279000] [2.1/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 00:18:34,089 - Train: 0.22% [4900/2279000] [2.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 00:19:34,546 - Train: 0.22% [5000/2279000] [2.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 00:20:34,978 - Train: 0.22% [5100/2279000] [2.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 00:21:35,411 - Train: 0.23% [5200/2279000] [2.3/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 00:22:35,858 - Train: 0.23% [5300/2279000] [2.3/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 00:23:36,299 - Train: 0.24% [5400/2279000] [2.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 00:24:36,872 - Train: 0.24% [5500/2279000] [2.4/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 00:25:37,324 - Train: 0.25% [5600/2279000] [2.5/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 00:26:37,777 - Train: 0.25% [5700/2279000] [2.5/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 00:27:38,208 - Train: 0.25% [5800/2279000] [2.5/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 00:28:38,632 - Train: 0.26% [5900/2279000] [2.6/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 00:29:39,065 - Train: 0.26% [6000/2279000] [2.6/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 00:30:39,606 - Train: 0.27% [6100/2279000] [2.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 00:31:40,045 - Train: 0.27% [6200/2279000] [2.7/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 00:32:40,496 - Train: 0.28% [6300/2279000] [2.8/1000.0] [batch_t 0.608 (0.604)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-09 00:33:40,932 - Train: 0.28% [6400/2279000] [2.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 00:34:41,367 - Train: 0.29% [6500/2279000] [2.9/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 00:35:41,821 - Train: 0.29% [6600/2279000] [2.9/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 00:36:42,246 - Train: 0.29% [6700/2279000] [2.9/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 00:37:42,768 - Train: 0.30% [6800/2279000] [3.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 00:38:05,129 - ==> Total time: 1:09:25 Eta: 16 days, 0:30:45 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-09 00:38:45,101 - Train: 0.30% [6900/2279000] [3.0/1000.0] [batch_t 0.606 (0.613)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 00:39:45,543 - Train: 0.31% [7000/2279000] [3.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 00:40:45,982 - Train: 0.31% [7100/2279000] [3.1/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 00:41:46,403 - Train: 0.32% [7200/2279000] [3.2/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 00:42:46,828 - Train: 0.32% [7300/2279000] [3.2/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 00:43:47,254 - Train: 0.32% [7400/2279000] [3.2/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 00:44:47,806 - Train: 0.33% [7500/2279000] [3.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 00:45:48,252 - Train: 0.33% [7600/2279000] [3.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 00:46:48,683 - Train: 0.34% [7700/2279000] [3.4/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 00:47:49,124 - Train: 0.34% [7800/2279000] [3.4/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 00:48:49,566 - Train: 0.35% [7900/2279000] [3.5/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 00:49:50,189 - Train: 0.35% [8000/2279000] [3.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 00:50:50,640 - Train: 0.36% [8100/2279000] [3.6/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 00:51:51,071 - Train: 0.36% [8200/2279000] [3.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 00:52:51,509 - Train: 0.36% [8300/2279000] [3.6/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 00:53:51,951 - Train: 0.37% [8400/2279000] [3.7/1000.0] [batch_t 0.608 (0.604)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-09 00:54:52,422 - Train: 0.37% [8500/2279000] [3.7/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 00:55:52,881 - Train: 0.38% [8600/2279000] [3.8/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 00:56:53,495 - Train: 0.38% [8700/2279000] [3.8/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 00:57:53,943 - Train: 0.39% [8800/2279000] [3.9/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 00:58:54,392 - Train: 0.39% [8900/2279000] [3.9/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 00:59:54,850 - Train: 0.39% [9000/2279000] [3.9/1000.0] [batch_t 0.607 (0.604)] [data_t 0.003] [optim_t 0.605] [lr 0.005000] +2024-04-09 01:00:55,332 - Train: 0.40% [9100/2279000] [4.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 01:01:05,008 - ==> Total time: 1:32:25 Eta: 15 days, 23:32:12 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-09 01:01:57,918 - Train: 0.40% [9200/2279000] [4.0/1000.0] [batch_t 0.606 (0.613)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 01:02:58,368 - Train: 0.41% [9300/2279000] [4.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 01:03:58,803 - Train: 0.41% [9400/2279000] [4.1/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 01:04:59,218 - Train: 0.42% [9500/2279000] [4.2/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 01:05:59,666 - Train: 0.42% [9600/2279000] [4.2/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 01:07:00,097 - Train: 0.43% [9700/2279000] [4.3/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 01:08:00,541 - Train: 0.43% [9800/2279000] [4.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 01:09:01,069 - Train: 0.43% [9900/2279000] [4.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 01:10:01,512 - Train: 0.44% [10000/2279000] [4.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 01:11:01,966 - Train: 0.44% [10100/2279000] [4.4/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 01:12:02,392 - Train: 0.45% [10200/2279000] [4.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 01:13:02,835 - Train: 0.45% [10300/2279000] [4.5/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 01:14:03,289 - Train: 0.46% [10400/2279000] [4.6/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 01:15:03,822 - Train: 0.46% [10500/2279000] [4.6/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 01:16:04,261 - Train: 0.47% [10600/2279000] [4.7/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 01:17:04,718 - Train: 0.47% [10700/2279000] [4.7/1000.0] [batch_t 0.606 (0.604)] [data_t 0.003] [optim_t 0.603] [lr 0.005000] +2024-04-09 01:18:05,162 - Train: 0.47% [10800/2279000] [4.7/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 01:19:05,618 - Train: 0.48% [10900/2279000] [4.8/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 01:20:06,058 - Train: 0.48% [11000/2279000] [4.8/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 01:21:06,494 - Train: 0.49% [11100/2279000] [4.9/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 01:22:07,033 - Train: 0.49% [11200/2279000] [4.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 01:23:07,482 - Train: 0.50% [11300/2279000] [5.0/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 01:24:04,931 - ==> Total time: 1:55:25 Eta: 15 days, 22:48:01 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-09 01:24:09,867 - Train: 0.50% [11400/2279000] [5.0/1000.0] [batch_t 0.606 (0.713)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 01:25:10,335 - Train: 0.50% [11500/2279000] [5.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 01:26:10,782 - Train: 0.51% [11600/2279000] [5.1/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 01:27:11,366 - Train: 0.51% [11700/2279000] [5.1/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 01:28:11,803 - Train: 0.52% [11800/2279000] [5.2/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 01:29:12,267 - Train: 0.52% [11900/2279000] [5.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 01:30:12,748 - Train: 0.53% [12000/2279000] [5.3/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 01:31:13,193 - Train: 0.53% [12100/2279000] [5.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 01:32:13,654 - Train: 0.54% [12200/2279000] [5.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 01:33:14,125 - Train: 0.54% [12300/2279000] [5.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-09 01:34:14,667 - Train: 0.54% [12400/2279000] [5.4/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 01:35:15,135 - Train: 0.55% [12500/2279000] [5.5/1000.0] [batch_t 0.607 (0.605)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-09 01:36:15,583 - Train: 0.55% [12600/2279000] [5.5/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 01:37:16,047 - Train: 0.56% [12700/2279000] [5.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 01:38:16,483 - Train: 0.56% [12800/2279000] [5.6/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 01:39:16,916 - Train: 0.57% [12900/2279000] [5.7/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 01:40:17,377 - Train: 0.57% [13000/2279000] [5.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 01:41:17,936 - Train: 0.57% [13100/2279000] [5.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 01:42:18,394 - Train: 0.58% [13200/2279000] [5.8/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 01:43:18,842 - Train: 0.58% [13300/2279000] [5.8/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 01:44:19,298 - Train: 0.59% [13400/2279000] [5.9/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 01:45:19,765 - Train: 0.59% [13500/2279000] [5.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 01:46:20,207 - Train: 0.60% [13600/2279000] [6.0/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 01:47:05,043 - ==> Total time: 2:18:25 Eta: 15 days, 22:11:25 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-09 01:47:22,591 - Train: 0.60% [13700/2279000] [6.0/1000.0] [batch_t 0.606 (0.619)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 01:48:23,052 - Train: 0.61% [13800/2279000] [6.1/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 01:49:23,538 - Train: 0.61% [13900/2279000] [6.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 01:50:24,016 - Train: 0.61% [14000/2279000] [6.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 01:51:24,449 - Train: 0.62% [14100/2279000] [6.2/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 01:52:24,856 - Train: 0.62% [14200/2279000] [6.2/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 01:53:25,301 - Train: 0.63% [14300/2279000] [6.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 01:54:25,876 - Train: 0.63% [14400/2279000] [6.3/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 01:55:26,314 - Train: 0.64% [14500/2279000] [6.4/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 01:56:26,755 - Train: 0.64% [14600/2279000] [6.4/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 01:57:27,225 - Train: 0.65% [14700/2279000] [6.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 01:58:27,663 - Train: 0.65% [14800/2279000] [6.5/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 01:59:28,224 - Train: 0.65% [14900/2279000] [6.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 02:00:28,772 - Train: 0.66% [15000/2279000] [6.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 02:01:29,216 - Train: 0.66% [15100/2279000] [6.6/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 02:02:29,684 - Train: 0.67% [15200/2279000] [6.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 02:03:30,117 - Train: 0.67% [15300/2279000] [6.7/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 02:04:30,561 - Train: 0.68% [15400/2279000] [6.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 02:05:31,118 - Train: 0.68% [15500/2279000] [6.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 02:06:31,578 - Train: 0.68% [15600/2279000] [6.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 02:07:32,087 - Train: 0.69% [15700/2279000] [6.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 02:08:32,522 - Train: 0.69% [15800/2279000] [6.9/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 02:09:32,983 - Train: 0.70% [15900/2279000] [7.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 02:10:05,016 - ==> Total time: 2:41:25 Eta: 15 days, 21:38:23 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-09 02:10:35,343 - Train: 0.70% [16000/2279000] [7.0/1000.0] [batch_t 0.606 (0.612)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 02:11:35,930 - Train: 0.71% [16100/2279000] [7.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 02:12:36,370 - Train: 0.71% [16200/2279000] [7.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 02:13:36,800 - Train: 0.72% [16300/2279000] [7.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 02:14:37,246 - Train: 0.72% [16400/2279000] [7.2/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 02:15:37,712 - Train: 0.72% [16500/2279000] [7.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 02:16:38,202 - Train: 0.73% [16600/2279000] [7.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 02:17:38,658 - Train: 0.73% [16700/2279000] [7.3/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 02:18:39,113 - Train: 0.74% [16800/2279000] [7.4/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 02:19:39,628 - Train: 0.74% [16900/2279000] [7.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 02:20:40,064 - Train: 0.75% [17000/2279000] [7.5/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 02:21:40,521 - Train: 0.75% [17100/2279000] [7.5/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 02:22:40,972 - Train: 0.75% [17200/2279000] [7.5/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 02:23:41,418 - Train: 0.76% [17300/2279000] [7.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 02:24:41,957 - Train: 0.76% [17400/2279000] [7.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 02:25:42,388 - Train: 0.77% [17500/2279000] [7.7/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 02:26:42,827 - Train: 0.77% [17600/2279000] [7.7/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 02:27:43,262 - Train: 0.78% [17700/2279000] [7.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 02:28:43,698 - Train: 0.78% [17800/2279000] [7.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 02:29:44,147 - Train: 0.79% [17900/2279000] [7.9/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 02:30:44,772 - Train: 0.79% [18000/2279000] [7.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 02:31:45,206 - Train: 0.79% [18100/2279000] [7.9/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 02:32:45,695 - Train: 0.80% [18200/2279000] [8.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.003] [optim_t 0.603] [lr 0.005000] +2024-04-09 02:33:05,043 - ==> Total time: 3:04:25 Eta: 15 days, 21:07:58 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-09 02:33:48,126 - Train: 0.80% [18300/2279000] [8.0/1000.0] [batch_t 0.606 (0.610)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 02:34:48,604 - Train: 0.81% [18400/2279000] [8.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 02:35:49,025 - Train: 0.81% [18500/2279000] [8.1/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 02:36:49,457 - Train: 0.82% [18600/2279000] [8.2/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 02:37:49,992 - Train: 0.82% [18700/2279000] [8.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 02:38:50,425 - Train: 0.82% [18800/2279000] [8.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 02:39:50,849 - Train: 0.83% [18900/2279000] [8.3/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 02:40:51,283 - Train: 0.83% [19000/2279000] [8.3/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 02:41:51,727 - Train: 0.84% [19100/2279000] [8.4/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 02:42:52,246 - Train: 0.84% [19200/2279000] [8.4/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 02:43:52,680 - Train: 0.85% [19300/2279000] [8.5/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 02:44:53,124 - Train: 0.85% [19400/2279000] [8.5/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 02:45:53,584 - Train: 0.86% [19500/2279000] [8.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 02:46:54,030 - Train: 0.86% [19600/2279000] [8.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 02:47:54,455 - Train: 0.86% [19700/2279000] [8.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 02:48:54,896 - Train: 0.87% [19800/2279000] [8.7/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 02:49:55,426 - Train: 0.87% [19900/2279000] [8.7/1000.0] [batch_t 0.678 (0.605)] [data_t 0.002] [optim_t 0.676] [lr 0.005000] +2024-04-09 02:50:55,862 - Train: 0.88% [20000/2279000] [8.8/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 02:51:56,291 - Train: 0.88% [20100/2279000] [8.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.003] [optim_t 0.603] [lr 0.005000] +2024-04-09 02:52:56,699 - Train: 0.89% [20200/2279000] [8.9/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 02:53:57,160 - Train: 0.89% [20300/2279000] [8.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 02:54:57,585 - Train: 0.90% [20400/2279000] [9.0/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 02:55:58,035 - Train: 0.90% [20500/2279000] [9.0/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 02:56:04,687 - ==> Total time: 3:27:24 Eta: 15 days, 20:38:29 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-09 02:57:00,827 - Train: 0.90% [20600/2279000] [9.0/1000.0] [batch_t 0.606 (0.613)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 02:58:01,273 - Train: 0.91% [20700/2279000] [9.1/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 02:59:01,733 - Train: 0.91% [20800/2279000] [9.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 03:00:02,182 - Train: 0.92% [20900/2279000] [9.2/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 03:01:02,631 - Train: 0.92% [21000/2279000] [9.2/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 03:02:03,074 - Train: 0.93% [21100/2279000] [9.3/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 03:03:03,602 - Train: 0.93% [21200/2279000] [9.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 03:04:04,048 - Train: 0.93% [21300/2279000] [9.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 03:05:04,482 - Train: 0.94% [21400/2279000] [9.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 03:06:04,932 - Train: 0.94% [21500/2279000] [9.4/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 03:07:05,378 - Train: 0.95% [21600/2279000] [9.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 03:08:05,815 - Train: 0.95% [21700/2279000] [9.5/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 03:09:06,342 - Train: 0.96% [21800/2279000] [9.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 03:10:06,789 - Train: 0.96% [21900/2279000] [9.6/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 03:11:07,225 - Train: 0.97% [22000/2279000] [9.7/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 03:12:07,684 - Train: 0.97% [22100/2279000] [9.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 03:13:08,130 - Train: 0.97% [22200/2279000] [9.7/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 03:14:08,576 - Train: 0.98% [22300/2279000] [9.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 03:15:09,010 - Train: 0.98% [22400/2279000] [9.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 03:16:09,454 - Train: 0.99% [22500/2279000] [9.9/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 03:17:09,968 - Train: 0.99% [22600/2279000] [9.9/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 03:18:10,411 - Train: 1.00% [22700/2279000] [10.0/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 03:19:04,823 - ==> Total time: 3:50:24 Eta: 15 days, 20:11:07 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-09 03:19:12,742 - Train: 1.00% [22800/2279000] [10.0/1000.0] [batch_t 0.606 (0.638)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 03:20:13,173 - Train: 1.00% [22900/2279000] [10.0/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 03:21:13,616 - Train: 1.01% [23000/2279000] [10.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 03:22:14,095 - Train: 1.01% [23100/2279000] [10.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 03:23:14,633 - Train: 1.02% [23200/2279000] [10.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 03:24:15,096 - Train: 1.02% [23300/2279000] [10.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 03:25:15,559 - Train: 1.03% [23400/2279000] [10.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 03:26:16,070 - Train: 1.03% [23500/2279000] [10.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 03:27:16,547 - Train: 1.04% [23600/2279000] [10.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 03:28:17,063 - Train: 1.04% [23700/2279000] [10.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 03:29:17,673 - Train: 1.04% [23800/2279000] [10.4/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 03:30:18,165 - Train: 1.05% [23900/2279000] [10.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 03:31:18,611 - Train: 1.05% [24000/2279000] [10.5/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 03:32:19,084 - Train: 1.06% [24100/2279000] [10.6/1000.0] [batch_t 0.610 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-09 03:33:19,561 - Train: 1.06% [24200/2279000] [10.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 03:34:20,092 - Train: 1.07% [24300/2279000] [10.7/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 03:35:20,666 - Train: 1.07% [24400/2279000] [10.7/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 03:36:21,086 - Train: 1.08% [24500/2279000] [10.8/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 03:37:21,531 - Train: 1.08% [24600/2279000] [10.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 03:38:22,009 - Train: 1.08% [24700/2279000] [10.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 03:39:22,487 - Train: 1.09% [24800/2279000] [10.9/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 03:40:23,073 - Train: 1.09% [24900/2279000] [10.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 03:41:23,659 - Train: 1.10% [25000/2279000] [11.0/1000.0] [batch_t 0.693 (0.606)] [data_t 0.002] [optim_t 0.691] [lr 0.005000] +2024-04-09 03:42:05,370 - ==> Total time: 4:13:25 Eta: 15 days, 19:45:10 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-09 03:42:25,997 - Train: 1.10% [25100/2279000] [11.0/1000.0] [batch_t 0.604 (0.618)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 03:43:26,477 - Train: 1.11% [25200/2279000] [11.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 03:44:26,946 - Train: 1.11% [25300/2279000] [11.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 03:45:27,434 - Train: 1.11% [25400/2279000] [11.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 03:46:27,899 - Train: 1.12% [25500/2279000] [11.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 03:47:28,506 - Train: 1.12% [25600/2279000] [11.2/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 03:48:28,948 - Train: 1.13% [25700/2279000] [11.3/1000.0] [batch_t 0.606 (0.604)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-09 03:49:29,431 - Train: 1.13% [25800/2279000] [11.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 03:50:29,912 - Train: 1.14% [25900/2279000] [11.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 03:51:30,390 - Train: 1.14% [26000/2279000] [11.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 03:52:30,895 - Train: 1.15% [26100/2279000] [11.5/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 03:53:31,508 - Train: 1.15% [26200/2279000] [11.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 03:54:31,997 - Train: 1.15% [26300/2279000] [11.5/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-09 03:55:32,494 - Train: 1.16% [26400/2279000] [11.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 03:56:33,010 - Train: 1.16% [26500/2279000] [11.6/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 03:57:33,504 - Train: 1.17% [26600/2279000] [11.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 03:58:34,072 - Train: 1.17% [26700/2279000] [11.7/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 03:59:34,565 - Train: 1.18% [26800/2279000] [11.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 04:00:35,114 - Train: 1.18% [26900/2279000] [11.8/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 04:01:35,789 - Train: 1.18% [27000/2279000] [11.8/1000.0] [batch_t 0.610 (0.607)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-09 04:02:36,309 - Train: 1.19% [27100/2279000] [11.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 04:03:36,893 - Train: 1.19% [27200/2279000] [11.9/1000.0] [batch_t 0.617 (0.606)] [data_t 0.003] [optim_t 0.614] [lr 0.005000] +2024-04-09 04:04:37,435 - Train: 1.20% [27300/2279000] [12.0/1000.0] [batch_t 0.608 (0.605)] [data_t 0.003] [optim_t 0.605] [lr 0.005000] +2024-04-09 04:05:06,473 - ==> Total time: 4:36:26 Eta: 15 days, 19:20:28 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-09 04:05:39,815 - Train: 1.20% [27400/2279000] [12.0/1000.0] [batch_t 0.607 (0.613)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-09 04:06:40,477 - Train: 1.21% [27500/2279000] [12.1/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 04:07:40,955 - Train: 1.21% [27600/2279000] [12.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 04:08:41,418 - Train: 1.22% [27700/2279000] [12.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 04:09:41,923 - Train: 1.22% [27800/2279000] [12.2/1000.0] [batch_t 0.609 (0.605)] [data_t 0.003] [optim_t 0.607] [lr 0.005000] +2024-04-09 04:10:42,442 - Train: 1.22% [27900/2279000] [12.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 04:11:42,904 - Train: 1.23% [28000/2279000] [12.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 04:12:43,413 - Train: 1.23% [28100/2279000] [12.3/1000.0] [batch_t 0.610 (0.605)] [data_t 0.003] [optim_t 0.607] [lr 0.005000] +2024-04-09 04:13:44,116 - Train: 1.24% [28200/2279000] [12.4/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 04:14:44,647 - Train: 1.24% [28300/2279000] [12.4/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-09 04:15:45,185 - Train: 1.25% [28400/2279000] [12.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 04:16:45,748 - Train: 1.25% [28500/2279000] [12.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.003] [optim_t 0.603] [lr 0.005000] +2024-04-09 04:17:46,232 - Train: 1.25% [28600/2279000] [12.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 04:18:46,750 - Train: 1.26% [28700/2279000] [12.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 04:19:47,332 - Train: 1.26% [28800/2279000] [12.6/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 04:20:47,934 - Train: 1.27% [28900/2279000] [12.7/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 04:21:48,430 - Train: 1.27% [29000/2279000] [12.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 04:22:48,897 - Train: 1.28% [29100/2279000] [12.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 04:23:49,399 - Train: 1.28% [29200/2279000] [12.8/1000.0] [batch_t 0.610 (0.605)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-09 04:24:49,892 - Train: 1.29% [29300/2279000] [12.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 04:25:50,379 - Train: 1.29% [29400/2279000] [12.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 04:26:50,961 - Train: 1.29% [29500/2279000] [12.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 04:27:51,492 - Train: 1.30% [29600/2279000] [13.0/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 04:28:07,809 - ==> Total time: 4:59:27 Eta: 15 days, 18:56:19 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-09 04:28:53,627 - Train: 1.30% [29700/2279000] [13.0/1000.0] [batch_t 0.606 (0.609)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 04:29:54,076 - Train: 1.31% [29800/2279000] [13.1/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 04:30:54,549 - Train: 1.31% [29900/2279000] [13.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 04:31:55,056 - Train: 1.32% [30000/2279000] [13.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 04:32:55,547 - Train: 1.32% [30100/2279000] [13.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 04:33:56,101 - Train: 1.33% [30200/2279000] [13.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 04:34:56,567 - Train: 1.33% [30300/2279000] [13.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 04:35:57,055 - Train: 1.33% [30400/2279000] [13.3/1000.0] [batch_t 0.610 (0.605)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-09 04:36:57,522 - Train: 1.34% [30500/2279000] [13.4/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-09 04:37:58,000 - Train: 1.34% [30600/2279000] [13.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 04:38:58,603 - Train: 1.35% [30700/2279000] [13.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 04:39:59,065 - Train: 1.35% [30800/2279000] [13.5/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 04:40:59,523 - Train: 1.36% [30900/2279000] [13.6/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 04:41:59,999 - Train: 1.36% [31000/2279000] [13.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 04:43:00,523 - Train: 1.36% [31100/2279000] [13.6/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-09 04:44:00,984 - Train: 1.37% [31200/2279000] [13.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 04:45:01,453 - Train: 1.37% [31300/2279000] [13.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.003] [optim_t 0.603] [lr 0.005000] +2024-04-09 04:46:01,917 - Train: 1.38% [31400/2279000] [13.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 04:47:02,465 - Train: 1.38% [31500/2279000] [13.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 04:48:02,935 - Train: 1.39% [31600/2279000] [13.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-09 04:49:03,427 - Train: 1.39% [31700/2279000] [13.9/1000.0] [batch_t 0.610 (0.605)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-09 04:50:03,986 - Train: 1.40% [31800/2279000] [14.0/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 04:51:04,429 - Train: 1.40% [31900/2279000] [14.0/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 04:51:08,065 - ==> Total time: 5:22:28 Eta: 15 days, 18:31:03 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-09 04:52:06,835 - Train: 1.40% [32000/2279000] [14.0/1000.0] [batch_t 0.605 (0.609)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 04:53:07,389 - Train: 1.41% [32100/2279000] [14.1/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-09 04:54:07,869 - Train: 1.41% [32200/2279000] [14.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 04:55:08,362 - Train: 1.42% [32300/2279000] [14.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 04:56:08,857 - Train: 1.42% [32400/2279000] [14.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 04:57:09,292 - Train: 1.43% [32500/2279000] [14.3/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 04:58:09,738 - Train: 1.43% [32600/2279000] [14.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 04:59:10,270 - Train: 1.43% [32700/2279000] [14.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 05:00:10,737 - Train: 1.44% [32800/2279000] [14.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 05:01:11,189 - Train: 1.44% [32900/2279000] [14.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 05:02:11,632 - Train: 1.45% [33000/2279000] [14.5/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 05:03:12,087 - Train: 1.45% [33100/2279000] [14.5/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 05:04:12,545 - Train: 1.46% [33200/2279000] [14.6/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 05:05:13,136 - Train: 1.46% [33300/2279000] [14.6/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 05:06:13,611 - Train: 1.47% [33400/2279000] [14.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 05:07:14,110 - Train: 1.47% [33500/2279000] [14.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 05:08:14,582 - Train: 1.47% [33600/2279000] [14.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 05:09:15,077 - Train: 1.48% [33700/2279000] [14.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 05:10:15,542 - Train: 1.48% [33800/2279000] [14.8/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 05:11:16,061 - Train: 1.49% [33900/2279000] [14.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 05:12:16,526 - Train: 1.49% [34000/2279000] [14.9/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-09 05:13:16,988 - Train: 1.50% [34100/2279000] [15.0/1000.0] [batch_t 0.611 (0.605)] [data_t 0.002] [optim_t 0.609] [lr 0.005000] +2024-04-09 05:14:08,416 - ==> Total time: 5:45:28 Eta: 15 days, 18:06:12 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-09 05:14:19,306 - Train: 1.50% [34200/2279000] [15.0/1000.0] [batch_t 0.607 (0.630)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 05:15:19,854 - Train: 1.51% [34300/2279000] [15.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 05:16:20,398 - Train: 1.51% [34400/2279000] [15.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 05:17:21,042 - Train: 1.51% [34500/2279000] [15.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 05:18:21,528 - Train: 1.52% [34600/2279000] [15.2/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 05:19:22,013 - Train: 1.52% [34700/2279000] [15.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 05:20:22,508 - Train: 1.53% [34800/2279000] [15.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 05:21:23,004 - Train: 1.53% [34900/2279000] [15.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 05:22:23,548 - Train: 1.54% [35000/2279000] [15.4/1000.0] [batch_t 0.610 (0.605)] [data_t 0.003] [optim_t 0.608] [lr 0.005000] +2024-04-09 05:23:24,083 - Train: 1.54% [35100/2279000] [15.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 05:24:24,666 - Train: 1.54% [35200/2279000] [15.4/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 05:25:25,124 - Train: 1.55% [35300/2279000] [15.5/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 05:26:25,578 - Train: 1.55% [35400/2279000] [15.5/1000.0] [batch_t 0.608 (0.604)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-09 05:27:26,105 - Train: 1.56% [35500/2279000] [15.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 05:28:26,596 - Train: 1.56% [35600/2279000] [15.6/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 05:29:27,207 - Train: 1.57% [35700/2279000] [15.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 05:30:27,730 - Train: 1.57% [35800/2279000] [15.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 05:31:28,206 - Train: 1.58% [35900/2279000] [15.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 05:32:28,660 - Train: 1.58% [36000/2279000] [15.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 05:33:29,098 - Train: 1.58% [36100/2279000] [15.8/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 05:34:29,612 - Train: 1.59% [36200/2279000] [15.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 05:35:30,143 - Train: 1.59% [36300/2279000] [15.9/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 05:36:30,792 - Train: 1.60% [36400/2279000] [16.0/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 05:37:09,542 - ==> Total time: 6:08:29 Eta: 15 days, 17:42:23 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-09 05:37:33,179 - Train: 1.60% [36500/2279000] [16.0/1000.0] [batch_t 0.607 (0.616)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-09 05:38:33,760 - Train: 1.61% [36600/2279000] [16.1/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-09 05:39:34,240 - Train: 1.61% [36700/2279000] [16.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 05:40:34,794 - Train: 1.61% [36800/2279000] [16.1/1000.0] [batch_t 0.610 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-09 05:41:35,300 - Train: 1.62% [36900/2279000] [16.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 05:42:35,769 - Train: 1.62% [37000/2279000] [16.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 05:43:36,418 - Train: 1.63% [37100/2279000] [16.3/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 05:44:36,873 - Train: 1.63% [37200/2279000] [16.3/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 05:45:37,347 - Train: 1.64% [37300/2279000] [16.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 05:46:37,820 - Train: 1.64% [37400/2279000] [16.4/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 05:47:38,352 - Train: 1.65% [37500/2279000] [16.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 05:48:38,929 - Train: 1.65% [37600/2279000] [16.5/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 05:49:39,381 - Train: 1.65% [37700/2279000] [16.5/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 05:50:39,896 - Train: 1.66% [37800/2279000] [16.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 05:51:40,360 - Train: 1.66% [37900/2279000] [16.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 05:52:40,796 - Train: 1.67% [38000/2279000] [16.7/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 05:53:41,261 - Train: 1.67% [38100/2279000] [16.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 05:54:41,738 - Train: 1.68% [38200/2279000] [16.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 05:55:42,231 - Train: 1.68% [38300/2279000] [16.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 05:56:42,980 - Train: 1.68% [38400/2279000] [16.8/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 05:57:43,478 - Train: 1.69% [38500/2279000] [16.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.003] [optim_t 0.603] [lr 0.005000] +2024-04-09 05:58:43,992 - Train: 1.69% [38600/2279000] [16.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 05:59:44,489 - Train: 1.70% [38700/2279000] [17.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 06:00:10,499 - ==> Total time: 6:31:30 Eta: 15 days, 17:18:29 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-09 06:00:46,680 - Train: 1.70% [38800/2279000] [17.0/1000.0] [batch_t 0.606 (0.611)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 06:01:47,306 - Train: 1.71% [38900/2279000] [17.1/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 06:02:47,769 - Train: 1.71% [39000/2279000] [17.1/1000.0] [batch_t 0.607 (0.605)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-09 06:03:48,259 - Train: 1.72% [39100/2279000] [17.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 06:04:48,747 - Train: 1.72% [39200/2279000] [17.2/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 06:05:49,230 - Train: 1.72% [39300/2279000] [17.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 06:06:49,709 - Train: 1.73% [39400/2279000] [17.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 06:07:50,315 - Train: 1.73% [39500/2279000] [17.3/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 06:08:50,816 - Train: 1.74% [39600/2279000] [17.4/1000.0] [batch_t 0.610 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-09 06:09:51,349 - Train: 1.74% [39700/2279000] [17.4/1000.0] [batch_t 0.610 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-09 06:10:51,936 - Train: 1.75% [39800/2279000] [17.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 06:11:52,411 - Train: 1.75% [39900/2279000] [17.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 06:12:52,899 - Train: 1.76% [40000/2279000] [17.6/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 06:13:53,520 - Train: 1.76% [40100/2279000] [17.6/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 06:14:54,034 - Train: 1.76% [40200/2279000] [17.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 06:15:54,606 - Train: 1.77% [40300/2279000] [17.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 06:16:55,098 - Train: 1.77% [40400/2279000] [17.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 06:17:55,573 - Train: 1.78% [40500/2279000] [17.8/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 06:18:56,043 - Train: 1.78% [40600/2279000] [17.8/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 06:19:56,567 - Train: 1.79% [40700/2279000] [17.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 06:20:57,068 - Train: 1.79% [40800/2279000] [17.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 06:21:57,652 - Train: 1.79% [40900/2279000] [17.9/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 06:22:58,136 - Train: 1.80% [41000/2279000] [18.0/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 06:23:11,455 - ==> Total time: 6:54:31 Eta: 15 days, 16:54:41 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-09 06:24:00,368 - Train: 1.80% [41100/2279000] [18.0/1000.0] [batch_t 0.605 (0.610)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 06:25:00,868 - Train: 1.81% [41200/2279000] [18.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 06:26:01,341 - Train: 1.81% [41300/2279000] [18.1/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 06:27:01,995 - Train: 1.82% [41400/2279000] [18.2/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 06:28:02,517 - Train: 1.82% [41500/2279000] [18.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 06:29:02,966 - Train: 1.83% [41600/2279000] [18.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 06:30:03,409 - Train: 1.83% [41700/2279000] [18.3/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 06:31:03,876 - Train: 1.83% [41800/2279000] [18.3/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-09 06:32:04,363 - Train: 1.84% [41900/2279000] [18.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 06:33:04,914 - Train: 1.84% [42000/2279000] [18.4/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-09 06:34:05,584 - Train: 1.85% [42100/2279000] [18.5/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 06:35:06,110 - Train: 1.85% [42200/2279000] [18.5/1000.0] [batch_t 0.610 (0.605)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-09 06:36:06,594 - Train: 1.86% [42300/2279000] [18.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 06:37:07,042 - Train: 1.86% [42400/2279000] [18.6/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 06:38:07,520 - Train: 1.86% [42500/2279000] [18.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.003] [optim_t 0.602] [lr 0.005000] +2024-04-09 06:39:08,033 - Train: 1.87% [42600/2279000] [18.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 06:40:08,701 - Train: 1.87% [42700/2279000] [18.7/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 06:41:09,212 - Train: 1.88% [42800/2279000] [18.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 06:42:09,672 - Train: 1.88% [42900/2279000] [18.8/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 06:43:10,158 - Train: 1.89% [43000/2279000] [18.9/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 06:44:10,608 - Train: 1.89% [43100/2279000] [18.9/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 06:45:11,096 - Train: 1.90% [43200/2279000] [19.0/1000.0] [batch_t 0.607 (0.605)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-09 06:46:11,600 - Train: 1.90% [43300/2279000] [19.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 06:46:12,209 - ==> Total time: 7:17:32 Eta: 15 days, 16:30:48 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-09 06:47:14,088 - Train: 1.90% [43400/2279000] [19.0/1000.0] [batch_t 0.606 (0.608)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 06:48:14,580 - Train: 1.91% [43500/2279000] [19.1/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-09 06:49:15,103 - Train: 1.91% [43600/2279000] [19.1/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 06:50:15,585 - Train: 1.92% [43700/2279000] [19.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 06:51:16,077 - Train: 1.92% [43800/2279000] [19.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 06:52:16,536 - Train: 1.93% [43900/2279000] [19.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 06:53:16,986 - Train: 1.93% [44000/2279000] [19.3/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 06:54:17,603 - Train: 1.94% [44100/2279000] [19.4/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 06:55:18,091 - Train: 1.94% [44200/2279000] [19.4/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 06:56:18,555 - Train: 1.94% [44300/2279000] [19.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 06:57:19,001 - Train: 1.95% [44400/2279000] [19.5/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 06:58:19,494 - Train: 1.95% [44500/2279000] [19.5/1000.0] [batch_t 0.604 (0.605)] [data_t 0.003] [optim_t 0.602] [lr 0.005000] +2024-04-09 06:59:19,989 - Train: 1.96% [44600/2279000] [19.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 07:00:20,490 - Train: 1.96% [44700/2279000] [19.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 07:01:21,077 - Train: 1.97% [44800/2279000] [19.7/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 07:02:21,585 - Train: 1.97% [44900/2279000] [19.7/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 07:03:22,061 - Train: 1.97% [45000/2279000] [19.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 07:04:22,537 - Train: 1.98% [45100/2279000] [19.8/1000.0] [batch_t 0.611 (0.605)] [data_t 0.002] [optim_t 0.609] [lr 0.005000] +2024-04-09 07:05:23,066 - Train: 1.98% [45200/2279000] [19.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 07:06:23,552 - Train: 1.99% [45300/2279000] [19.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 07:07:24,044 - Train: 1.99% [45400/2279000] [19.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 07:08:24,586 - Train: 2.00% [45500/2279000] [20.0/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 07:09:12,946 - ==> Total time: 7:40:33 Eta: 15 days, 16:06:59 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-09 07:09:26,830 - Train: 2.00% [45600/2279000] [20.0/1000.0] [batch_t 0.606 (0.623)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 07:10:27,306 - Train: 2.01% [45700/2279000] [20.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 07:11:27,798 - Train: 2.01% [45800/2279000] [20.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 07:12:28,287 - Train: 2.01% [45900/2279000] [20.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 07:13:28,917 - Train: 2.02% [46000/2279000] [20.2/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 07:14:29,349 - Train: 2.02% [46100/2279000] [20.2/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 07:15:29,809 - Train: 2.03% [46200/2279000] [20.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 07:16:30,274 - Train: 2.03% [46300/2279000] [20.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 07:17:30,769 - Train: 2.04% [46400/2279000] [20.4/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-09 07:18:31,269 - Train: 2.04% [46500/2279000] [20.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 07:19:31,752 - Train: 2.04% [46600/2279000] [20.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 07:20:32,310 - Train: 2.05% [46700/2279000] [20.5/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 07:21:32,839 - Train: 2.05% [46800/2279000] [20.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 07:22:33,339 - Train: 2.06% [46900/2279000] [20.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 07:23:33,798 - Train: 2.06% [47000/2279000] [20.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 07:24:34,278 - Train: 2.07% [47100/2279000] [20.7/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 07:25:34,738 - Train: 2.07% [47200/2279000] [20.7/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 07:26:35,196 - Train: 2.08% [47300/2279000] [20.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 07:27:35,760 - Train: 2.08% [47400/2279000] [20.8/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 07:28:36,216 - Train: 2.08% [47500/2279000] [20.8/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 07:29:36,801 - Train: 2.09% [47600/2279000] [20.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 07:30:37,275 - Train: 2.09% [47700/2279000] [20.9/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 07:31:37,795 - Train: 2.10% [47800/2279000] [21.0/1000.0] [batch_t 0.607 (0.605)] [data_t 0.003] [optim_t 0.605] [lr 0.005000] +2024-04-09 07:32:13,483 - ==> Total time: 8:03:33 Eta: 15 days, 15:43:05 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-09 07:32:40,266 - Train: 2.10% [47900/2279000] [21.0/1000.0] [batch_t 0.611 (0.620)] [data_t 0.002] [optim_t 0.609] [lr 0.005000] +2024-04-09 07:33:40,750 - Train: 2.11% [48000/2279000] [21.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 07:34:41,253 - Train: 2.11% [48100/2279000] [21.1/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 07:35:41,749 - Train: 2.11% [48200/2279000] [21.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 07:36:42,238 - Train: 2.12% [48300/2279000] [21.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 07:37:42,731 - Train: 2.12% [48400/2279000] [21.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 07:38:43,214 - Train: 2.13% [48500/2279000] [21.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 07:39:43,680 - Train: 2.13% [48600/2279000] [21.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 07:40:44,300 - Train: 2.14% [48700/2279000] [21.4/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 07:41:44,784 - Train: 2.14% [48800/2279000] [21.4/1000.0] [batch_t 0.610 (0.605)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-09 07:42:45,262 - Train: 2.15% [48900/2279000] [21.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.003] [optim_t 0.603] [lr 0.005000] +2024-04-09 07:43:45,822 - Train: 2.15% [49000/2279000] [21.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 07:44:46,290 - Train: 2.15% [49100/2279000] [21.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 07:45:47,007 - Train: 2.16% [49200/2279000] [21.6/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 07:46:47,507 - Train: 2.16% [49300/2279000] [21.6/1000.0] [batch_t 0.611 (0.605)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-09 07:47:47,980 - Train: 2.17% [49400/2279000] [21.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 07:48:48,447 - Train: 2.17% [49500/2279000] [21.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 07:49:48,985 - Train: 2.18% [49600/2279000] [21.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 07:50:49,460 - Train: 2.18% [49700/2279000] [21.8/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-09 07:51:49,939 - Train: 2.19% [49800/2279000] [21.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 07:52:50,513 - Train: 2.19% [49900/2279000] [21.9/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 07:53:51,022 - Train: 2.19% [50000/2279000] [21.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-09 07:54:51,469 - Train: 2.20% [50100/2279000] [22.0/1000.0] [batch_t 0.606 (0.604)] [data_t 0.003] [optim_t 0.603] [lr 0.005000] +2024-04-09 07:55:14,451 - ==> Total time: 8:26:34 Eta: 15 days, 15:19:36 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-09 07:55:53,635 - Train: 2.20% [50200/2279000] [22.0/1000.0] [batch_t 0.605 (0.610)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 07:56:54,091 - Train: 2.21% [50300/2279000] [22.1/1000.0] [batch_t 0.606 (0.604)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-09 07:57:54,751 - Train: 2.21% [50400/2279000] [22.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 07:58:55,228 - Train: 2.22% [50500/2279000] [22.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 07:59:55,735 - Train: 2.22% [50600/2279000] [22.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 08:00:56,230 - Train: 2.22% [50700/2279000] [22.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.003] [optim_t 0.603] [lr 0.005000] +2024-04-09 08:01:56,711 - Train: 2.23% [50800/2279000] [22.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 08:02:57,174 - Train: 2.23% [50900/2279000] [22.3/1000.0] [batch_t 0.610 (0.605)] [data_t 0.003] [optim_t 0.607] [lr 0.005000] +2024-04-09 08:03:57,676 - Train: 2.24% [51000/2279000] [22.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 08:04:58,151 - Train: 2.24% [51100/2279000] [22.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 08:05:58,740 - Train: 2.25% [51200/2279000] [22.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 08:06:59,198 - Train: 2.25% [51300/2279000] [22.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 08:07:59,657 - Train: 2.26% [51400/2279000] [22.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 08:09:00,149 - Train: 2.26% [51500/2279000] [22.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 08:10:00,646 - Train: 2.26% [51600/2279000] [22.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 08:11:01,292 - Train: 2.27% [51700/2279000] [22.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 08:12:01,764 - Train: 2.27% [51800/2279000] [22.7/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 08:13:02,219 - Train: 2.28% [51900/2279000] [22.8/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 08:14:02,704 - Train: 2.28% [52000/2279000] [22.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 08:15:03,168 - Train: 2.29% [52100/2279000] [22.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 08:16:03,670 - Train: 2.29% [52200/2279000] [22.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 08:17:04,085 - Train: 2.29% [52300/2279000] [22.9/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 08:18:04,546 - Train: 2.30% [52400/2279000] [23.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 08:18:14,823 - ==> Total time: 8:49:34 Eta: 15 days, 14:55:43 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-09 08:19:07,013 - Train: 2.30% [52500/2279000] [23.0/1000.0] [batch_t 0.606 (0.609)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 08:20:07,423 - Train: 2.31% [52600/2279000] [23.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 08:21:07,868 - Train: 2.31% [52700/2279000] [23.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 08:22:08,328 - Train: 2.32% [52800/2279000] [23.2/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-09 08:23:08,742 - Train: 2.32% [52900/2279000] [23.2/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 08:24:09,162 - Train: 2.33% [53000/2279000] [23.3/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 08:25:09,656 - Train: 2.33% [53100/2279000] [23.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 08:26:10,031 - Train: 2.33% [53200/2279000] [23.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 08:27:10,414 - Train: 2.34% [53300/2279000] [23.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 08:28:10,913 - Train: 2.34% [53400/2279000] [23.4/1000.0] [batch_t 0.610 (0.605)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-09 08:29:11,304 - Train: 2.35% [53500/2279000] [23.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 08:30:11,733 - Train: 2.35% [53600/2279000] [23.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 08:31:12,148 - Train: 2.36% [53700/2279000] [23.6/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 08:32:12,613 - Train: 2.36% [53800/2279000] [23.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 08:33:13,080 - Train: 2.37% [53900/2279000] [23.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 08:34:13,515 - Train: 2.37% [54000/2279000] [23.7/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 08:35:13,957 - Train: 2.37% [54100/2279000] [23.7/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 08:36:14,397 - Train: 2.38% [54200/2279000] [23.8/1000.0] [batch_t 0.601 (0.604)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-09 08:37:14,817 - Train: 2.38% [54300/2279000] [23.8/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 08:38:15,424 - Train: 2.39% [54400/2279000] [23.9/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 08:39:15,892 - Train: 2.39% [54500/2279000] [23.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 08:40:16,359 - Train: 2.40% [54600/2279000] [24.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 08:41:14,391 - ==> Total time: 9:12:34 Eta: 15 days, 14:31:22 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-09 08:41:18,758 - Train: 2.40% [54700/2279000] [24.0/1000.0] [batch_t 0.604 (0.727)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 08:42:19,194 - Train: 2.40% [54800/2279000] [24.0/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 08:43:19,602 - Train: 2.41% [54900/2279000] [24.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 08:44:20,229 - Train: 2.41% [55000/2279000] [24.1/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 08:45:20,694 - Train: 2.42% [55100/2279000] [24.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 08:46:21,123 - Train: 2.42% [55200/2279000] [24.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 08:47:21,565 - Train: 2.43% [55300/2279000] [24.3/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 08:48:22,012 - Train: 2.43% [55400/2279000] [24.3/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 08:49:22,441 - Train: 2.44% [55500/2279000] [24.4/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 08:50:22,870 - Train: 2.44% [55600/2279000] [24.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 08:51:23,305 - Train: 2.44% [55700/2279000] [24.4/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 08:52:23,758 - Train: 2.45% [55800/2279000] [24.5/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 08:53:24,168 - Train: 2.45% [55900/2279000] [24.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 08:54:24,568 - Train: 2.46% [56000/2279000] [24.6/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 08:55:24,989 - Train: 2.46% [56100/2279000] [24.6/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 08:56:25,469 - Train: 2.47% [56200/2279000] [24.7/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 08:57:25,946 - Train: 2.47% [56300/2279000] [24.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 08:58:26,445 - Train: 2.47% [56400/2279000] [24.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 08:59:26,912 - Train: 2.48% [56500/2279000] [24.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 09:00:27,364 - Train: 2.48% [56600/2279000] [24.8/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 09:01:27,856 - Train: 2.49% [56700/2279000] [24.9/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 09:02:28,335 - Train: 2.49% [56800/2279000] [24.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 09:03:28,752 - Train: 2.50% [56900/2279000] [25.0/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 09:04:14,253 - ==> Total time: 9:35:34 Eta: 15 days, 14:07:19 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-09 09:04:31,136 - Train: 2.50% [57000/2279000] [25.0/1000.0] [batch_t 0.605 (0.619)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 09:05:31,568 - Train: 2.51% [57100/2279000] [25.1/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 09:06:31,986 - Train: 2.51% [57200/2279000] [25.1/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 09:07:32,410 - Train: 2.51% [57300/2279000] [25.1/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 09:08:32,841 - Train: 2.52% [57400/2279000] [25.2/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 09:09:33,440 - Train: 2.52% [57500/2279000] [25.2/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 09:10:33,860 - Train: 2.53% [57600/2279000] [25.3/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 09:11:34,331 - Train: 2.53% [57700/2279000] [25.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 09:12:34,775 - Train: 2.54% [57800/2279000] [25.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 09:13:35,209 - Train: 2.54% [57900/2279000] [25.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 09:14:35,711 - Train: 2.54% [58000/2279000] [25.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 09:15:36,112 - Train: 2.55% [58100/2279000] [25.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 09:16:36,547 - Train: 2.55% [58200/2279000] [25.5/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 09:17:37,007 - Train: 2.56% [58300/2279000] [25.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 09:18:37,439 - Train: 2.56% [58400/2279000] [25.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 09:19:37,849 - Train: 2.57% [58500/2279000] [25.7/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 09:20:38,222 - Train: 2.57% [58600/2279000] [25.7/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 09:21:38,741 - Train: 2.58% [58700/2279000] [25.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 09:22:39,206 - Train: 2.58% [58800/2279000] [25.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 09:23:39,620 - Train: 2.58% [58900/2279000] [25.8/1000.0] [batch_t 0.608 (0.604)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-09 09:24:40,069 - Train: 2.59% [59000/2279000] [25.9/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 09:25:40,499 - Train: 2.59% [59100/2279000] [25.9/1000.0] [batch_t 0.608 (0.604)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-09 09:26:40,908 - Train: 2.60% [59200/2279000] [26.0/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 09:27:13,570 - ==> Total time: 9:58:33 Eta: 15 days, 13:43:01 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-09 09:27:43,566 - Train: 2.60% [59300/2279000] [26.0/1000.0] [batch_t 0.605 (0.618)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 09:28:43,986 - Train: 2.61% [59400/2279000] [26.1/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 09:29:44,391 - Train: 2.61% [59500/2279000] [26.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 09:30:44,799 - Train: 2.62% [59600/2279000] [26.2/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 09:31:45,280 - Train: 2.62% [59700/2279000] [26.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 09:32:45,855 - Train: 2.62% [59800/2279000] [26.2/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 09:33:46,318 - Train: 2.63% [59900/2279000] [26.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 09:34:46,726 - Train: 2.63% [60000/2279000] [26.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 09:35:47,166 - Train: 2.64% [60100/2279000] [26.4/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 09:36:47,592 - Train: 2.64% [60200/2279000] [26.4/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 09:37:48,018 - Train: 2.65% [60300/2279000] [26.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 09:38:48,458 - Train: 2.65% [60400/2279000] [26.5/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 09:39:48,873 - Train: 2.65% [60500/2279000] [26.5/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 09:40:49,367 - Train: 2.66% [60600/2279000] [26.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 09:41:49,785 - Train: 2.66% [60700/2279000] [26.6/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 09:42:50,193 - Train: 2.67% [60800/2279000] [26.7/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 09:43:50,650 - Train: 2.67% [60900/2279000] [26.7/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 09:44:51,115 - Train: 2.68% [61000/2279000] [26.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 09:45:51,771 - Train: 2.68% [61100/2279000] [26.8/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 09:46:52,203 - Train: 2.69% [61200/2279000] [26.9/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 09:47:52,621 - Train: 2.69% [61300/2279000] [26.9/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 09:48:53,057 - Train: 2.69% [61400/2279000] [26.9/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 09:49:53,486 - Train: 2.70% [61500/2279000] [27.0/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 09:50:13,443 - ==> Total time: 10:21:33 Eta: 15 days, 13:19:08 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-09 09:50:55,567 - Train: 2.70% [61600/2279000] [27.0/1000.0] [batch_t 0.608 (0.609)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-09 09:51:55,958 - Train: 2.71% [61700/2279000] [27.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 09:52:56,530 - Train: 2.71% [61800/2279000] [27.1/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 09:53:56,970 - Train: 2.72% [61900/2279000] [27.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 09:54:57,437 - Train: 2.72% [62000/2279000] [27.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 09:55:57,966 - Train: 2.72% [62100/2279000] [27.2/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-09 09:56:58,401 - Train: 2.73% [62200/2279000] [27.3/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 09:57:58,827 - Train: 2.73% [62300/2279000] [27.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 09:58:59,221 - Train: 2.74% [62400/2279000] [27.4/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 09:59:59,770 - Train: 2.74% [62500/2279000] [27.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 10:01:00,168 - Train: 2.75% [62600/2279000] [27.5/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 10:02:00,576 - Train: 2.75% [62700/2279000] [27.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 10:03:00,979 - Train: 2.76% [62800/2279000] [27.6/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 10:04:01,411 - Train: 2.76% [62900/2279000] [27.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 10:05:01,821 - Train: 2.76% [63000/2279000] [27.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 10:06:02,456 - Train: 2.77% [63100/2279000] [27.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 10:07:02,909 - Train: 2.77% [63200/2279000] [27.7/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 10:08:03,331 - Train: 2.78% [63300/2279000] [27.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 10:09:03,731 - Train: 2.78% [63400/2279000] [27.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 10:10:04,146 - Train: 2.79% [63500/2279000] [27.9/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 10:11:04,635 - Train: 2.79% [63600/2279000] [27.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 10:12:05,074 - Train: 2.80% [63700/2279000] [28.0/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 10:13:05,673 - Train: 2.80% [63800/2279000] [28.0/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 10:13:12,934 - ==> Total time: 10:44:33 Eta: 15 days, 12:55:06 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-09 10:14:08,282 - Train: 2.80% [63900/2279000] [28.0/1000.0] [batch_t 0.604 (0.613)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 10:15:08,685 - Train: 2.81% [64000/2279000] [28.1/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 10:16:09,094 - Train: 2.81% [64100/2279000] [28.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 10:17:09,552 - Train: 2.82% [64200/2279000] [28.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 10:18:09,982 - Train: 2.82% [64300/2279000] [28.2/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 10:19:10,540 - Train: 2.83% [64400/2279000] [28.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 10:20:10,944 - Train: 2.83% [64500/2279000] [28.3/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 10:21:11,359 - Train: 2.83% [64600/2279000] [28.3/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 10:22:11,806 - Train: 2.84% [64700/2279000] [28.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 10:23:12,259 - Train: 2.84% [64800/2279000] [28.4/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 10:24:12,762 - Train: 2.85% [64900/2279000] [28.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 10:25:13,260 - Train: 2.85% [65000/2279000] [28.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 10:26:13,675 - Train: 2.86% [65100/2279000] [28.6/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 10:27:14,086 - Train: 2.86% [65200/2279000] [28.6/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 10:28:14,515 - Train: 2.87% [65300/2279000] [28.7/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 10:29:14,994 - Train: 2.87% [65400/2279000] [28.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 10:30:15,437 - Train: 2.87% [65500/2279000] [28.7/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 10:31:15,987 - Train: 2.88% [65600/2279000] [28.8/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 10:32:16,398 - Train: 2.88% [65700/2279000] [28.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 10:33:16,816 - Train: 2.89% [65800/2279000] [28.9/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 10:34:17,265 - Train: 2.89% [65900/2279000] [28.9/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 10:35:17,772 - Train: 2.90% [66000/2279000] [29.0/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-09 10:36:12,955 - ==> Total time: 11:07:33 Eta: 15 days, 12:31:26 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-09 10:36:20,519 - Train: 2.90% [66100/2279000] [29.0/1000.0] [batch_t 0.604 (0.686)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 10:37:20,933 - Train: 2.90% [66200/2279000] [29.0/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 10:38:21,350 - Train: 2.91% [66300/2279000] [29.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 10:39:21,755 - Train: 2.91% [66400/2279000] [29.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 10:40:22,224 - Train: 2.92% [66500/2279000] [29.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 10:41:22,717 - Train: 2.92% [66600/2279000] [29.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 10:42:23,180 - Train: 2.93% [66700/2279000] [29.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 10:43:23,800 - Train: 2.93% [66800/2279000] [29.3/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 10:44:24,237 - Train: 2.94% [66900/2279000] [29.4/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 10:45:24,694 - Train: 2.94% [67000/2279000] [29.4/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 10:46:25,136 - Train: 2.94% [67100/2279000] [29.4/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 10:47:25,585 - Train: 2.95% [67200/2279000] [29.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 10:48:26,035 - Train: 2.95% [67300/2279000] [29.5/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 10:49:26,603 - Train: 2.96% [67400/2279000] [29.6/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 10:50:27,027 - Train: 2.96% [67500/2279000] [29.6/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 10:51:27,455 - Train: 2.97% [67600/2279000] [29.7/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 10:52:27,849 - Train: 2.97% [67700/2279000] [29.7/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 10:53:28,280 - Train: 2.97% [67800/2279000] [29.7/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 10:54:28,694 - Train: 2.98% [67900/2279000] [29.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 10:55:29,181 - Train: 2.98% [68000/2279000] [29.8/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-09 10:56:29,817 - Train: 2.99% [68100/2279000] [29.9/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 10:57:30,270 - Train: 2.99% [68200/2279000] [29.9/1000.0] [batch_t 0.608 (0.604)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-09 10:58:30,730 - Train: 3.00% [68300/2279000] [30.0/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 10:59:13,079 - ==> Total time: 11:30:33 Eta: 15 days, 12:07:52 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-09 10:59:33,234 - Train: 3.00% [68400/2279000] [30.0/1000.0] [batch_t 0.606 (0.622)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 11:00:33,662 - Train: 3.01% [68500/2279000] [30.1/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 11:01:34,078 - Train: 3.01% [68600/2279000] [30.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 11:02:34,662 - Train: 3.01% [68700/2279000] [30.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 11:03:35,114 - Train: 3.02% [68800/2279000] [30.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 11:04:35,517 - Train: 3.02% [68900/2279000] [30.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 11:05:36,003 - Train: 3.03% [69000/2279000] [30.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 11:06:36,405 - Train: 3.03% [69100/2279000] [30.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 11:07:36,837 - Train: 3.04% [69200/2279000] [30.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 11:08:37,276 - Train: 3.04% [69300/2279000] [30.4/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 11:09:37,791 - Train: 3.05% [69400/2279000] [30.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 11:10:38,238 - Train: 3.05% [69500/2279000] [30.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 11:11:38,656 - Train: 3.05% [69600/2279000] [30.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 11:12:39,092 - Train: 3.06% [69700/2279000] [30.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 11:13:39,537 - Train: 3.06% [69800/2279000] [30.6/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 11:14:39,962 - Train: 3.07% [69900/2279000] [30.7/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 11:15:40,468 - Train: 3.07% [70000/2279000] [30.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 11:16:40,900 - Train: 3.08% [70100/2279000] [30.8/1000.0] [batch_t 0.608 (0.604)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-09 11:17:41,315 - Train: 3.08% [70200/2279000] [30.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 11:18:41,747 - Train: 3.08% [70300/2279000] [30.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 11:19:42,191 - Train: 3.09% [70400/2279000] [30.9/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 11:20:42,626 - Train: 3.09% [70500/2279000] [30.9/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 11:21:43,137 - Train: 3.10% [70600/2279000] [31.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 11:22:12,727 - ==> Total time: 11:53:32 Eta: 15 days, 11:44:06 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-09 11:22:45,434 - Train: 3.10% [70700/2279000] [31.0/1000.0] [batch_t 0.606 (0.613)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 11:23:45,861 - Train: 3.11% [70800/2279000] [31.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 11:24:46,315 - Train: 3.11% [70900/2279000] [31.1/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 11:25:46,748 - Train: 3.12% [71000/2279000] [31.2/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-09 11:26:47,347 - Train: 3.12% [71100/2279000] [31.2/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 11:27:47,842 - Train: 3.12% [71200/2279000] [31.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 11:28:48,247 - Train: 3.13% [71300/2279000] [31.3/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 11:29:48,643 - Train: 3.13% [71400/2279000] [31.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 11:30:49,078 - Train: 3.14% [71500/2279000] [31.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 11:31:49,545 - Train: 3.14% [71600/2279000] [31.4/1000.0] [batch_t 0.610 (0.605)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-09 11:32:50,099 - Train: 3.15% [71700/2279000] [31.5/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 11:33:50,531 - Train: 3.15% [71800/2279000] [31.5/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 11:34:50,972 - Train: 3.15% [71900/2279000] [31.5/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 11:35:51,444 - Train: 3.16% [72000/2279000] [31.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 11:36:51,859 - Train: 3.16% [72100/2279000] [31.6/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 11:37:52,252 - Train: 3.17% [72200/2279000] [31.7/1000.0] [batch_t 0.609 (0.604)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-09 11:38:52,719 - Train: 3.17% [72300/2279000] [31.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 11:39:53,245 - Train: 3.18% [72400/2279000] [31.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 11:40:53,635 - Train: 3.18% [72500/2279000] [31.8/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 11:41:54,049 - Train: 3.19% [72600/2279000] [31.9/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 11:42:54,479 - Train: 3.19% [72700/2279000] [31.9/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 11:43:54,919 - Train: 3.19% [72800/2279000] [31.9/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 11:44:55,378 - Train: 3.20% [72900/2279000] [32.0/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 11:45:12,319 - ==> Total time: 12:16:32 Eta: 15 days, 11:20:20 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-09 11:45:58,017 - Train: 3.20% [73000/2279000] [32.0/1000.0] [batch_t 0.605 (0.614)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 11:46:58,442 - Train: 3.21% [73100/2279000] [32.1/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 11:47:58,859 - Train: 3.21% [73200/2279000] [32.1/1000.0] [batch_t 0.607 (0.604)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-09 11:49:00,091 - Train: 3.22% [73300/2279000] [32.2/1000.0] [batch_t 0.605 (0.612)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 11:50:00,558 - Train: 3.22% [73400/2279000] [32.2/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 11:51:01,125 - Train: 3.23% [73500/2279000] [32.3/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 11:52:01,537 - Train: 3.23% [73600/2279000] [32.3/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 11:53:01,930 - Train: 3.23% [73700/2279000] [32.3/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 11:54:02,351 - Train: 3.24% [73800/2279000] [32.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 11:55:02,770 - Train: 3.24% [73900/2279000] [32.4/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 11:56:04,177 - Train: 3.25% [74000/2279000] [32.5/1000.0] [batch_t 0.606 (0.614)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 11:57:04,572 - Train: 3.25% [74100/2279000] [32.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 11:58:04,989 - Train: 3.26% [74200/2279000] [32.6/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 11:59:05,488 - Train: 3.26% [74300/2279000] [32.6/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 12:00:05,954 - Train: 3.26% [74400/2279000] [32.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 12:01:11,852 - Train: 3.27% [74500/2279000] [32.7/1000.0] [batch_t 0.607 (0.659)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-09 12:02:12,296 - Train: 3.27% [74600/2279000] [32.7/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 12:03:12,776 - Train: 3.28% [74700/2279000] [32.8/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 12:04:13,187 - Train: 3.28% [74800/2279000] [32.8/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 12:05:13,723 - Train: 3.29% [74900/2279000] [32.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 12:06:14,195 - Train: 3.29% [75000/2279000] [32.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 12:07:14,658 - Train: 3.30% [75100/2279000] [33.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 12:08:15,101 - Train: 3.30% [75200/2279000] [33.0/1000.0] [batch_t 0.607 (0.604)] [data_t 0.004] [optim_t 0.603] [lr 0.005000] +2024-04-09 12:08:19,335 - ==> Total time: 12:39:39 Eta: 15 days, 11:00:15 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-09 12:09:17,266 - Train: 3.30% [75300/2279000] [33.0/1000.0] [batch_t 0.605 (0.608)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 12:10:17,826 - Train: 3.31% [75400/2279000] [33.1/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 12:11:18,236 - Train: 3.31% [75500/2279000] [33.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 12:12:18,697 - Train: 3.32% [75600/2279000] [33.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 12:13:19,145 - Train: 3.32% [75700/2279000] [33.2/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 12:14:19,594 - Train: 3.33% [75800/2279000] [33.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 12:15:20,046 - Train: 3.33% [75900/2279000] [33.3/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 12:16:20,508 - Train: 3.33% [76000/2279000] [33.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 12:17:21,184 - Train: 3.34% [76100/2279000] [33.4/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 12:18:21,583 - Train: 3.34% [76200/2279000] [33.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 12:19:22,045 - Train: 3.35% [76300/2279000] [33.5/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 12:20:22,460 - Train: 3.35% [76400/2279000] [33.5/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 12:21:22,866 - Train: 3.36% [76500/2279000] [33.6/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 12:22:23,299 - Train: 3.36% [76600/2279000] [33.6/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 12:23:23,869 - Train: 3.37% [76700/2279000] [33.7/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 12:24:24,295 - Train: 3.37% [76800/2279000] [33.7/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 12:25:24,681 - Train: 3.37% [76900/2279000] [33.7/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 12:26:25,101 - Train: 3.38% [77000/2279000] [33.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 12:27:25,505 - Train: 3.38% [77100/2279000] [33.8/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 12:28:25,927 - Train: 3.39% [77200/2279000] [33.9/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 12:29:26,366 - Train: 3.39% [77300/2279000] [33.9/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 12:30:26,939 - Train: 3.40% [77400/2279000] [34.0/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 12:31:18,918 - ==> Total time: 13:02:39 Eta: 15 days, 10:36:28 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-09 12:31:29,250 - Train: 3.40% [77500/2279000] [34.0/1000.0] [batch_t 0.605 (0.633)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 12:32:29,657 - Train: 3.41% [77600/2279000] [34.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 12:33:30,090 - Train: 3.41% [77700/2279000] [34.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 12:34:30,582 - Train: 3.41% [77800/2279000] [34.1/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 12:35:31,123 - Train: 3.42% [77900/2279000] [34.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 12:36:31,555 - Train: 3.42% [78000/2279000] [34.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 12:37:31,962 - Train: 3.43% [78100/2279000] [34.3/1000.0] [batch_t 0.608 (0.604)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-09 12:38:32,412 - Train: 3.43% [78200/2279000] [34.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 12:39:32,824 - Train: 3.44% [78300/2279000] [34.4/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 12:40:33,240 - Train: 3.44% [78400/2279000] [34.4/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 12:41:33,657 - Train: 3.44% [78500/2279000] [34.4/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 12:42:34,183 - Train: 3.45% [78600/2279000] [34.5/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-09 12:43:34,636 - Train: 3.45% [78700/2279000] [34.5/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 12:44:35,077 - Train: 3.46% [78800/2279000] [34.6/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 12:45:35,484 - Train: 3.46% [78900/2279000] [34.6/1000.0] [batch_t 0.608 (0.604)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-09 12:46:35,925 - Train: 3.47% [79000/2279000] [34.7/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 12:47:36,391 - Train: 3.47% [79100/2279000] [34.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 12:48:36,862 - Train: 3.48% [79200/2279000] [34.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 12:49:37,314 - Train: 3.48% [79300/2279000] [34.8/1000.0] [batch_t 0.610 (0.604)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-09 12:50:37,776 - Train: 3.48% [79400/2279000] [34.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 12:51:38,220 - Train: 3.49% [79500/2279000] [34.9/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 12:52:38,612 - Train: 3.49% [79600/2279000] [34.9/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 12:53:39,049 - Train: 3.50% [79700/2279000] [35.0/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 12:54:18,306 - ==> Total time: 13:25:38 Eta: 15 days, 10:12:38 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-09 12:54:41,188 - Train: 3.50% [79800/2279000] [35.0/1000.0] [batch_t 0.603 (0.615)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 12:55:41,744 - Train: 3.51% [79900/2279000] [35.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 12:56:42,176 - Train: 3.51% [80000/2279000] [35.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 12:57:42,631 - Train: 3.51% [80100/2279000] [35.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 12:58:43,073 - Train: 3.52% [80200/2279000] [35.2/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 12:59:43,501 - Train: 3.52% [80300/2279000] [35.2/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 13:00:43,935 - Train: 3.53% [80400/2279000] [35.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 13:01:44,399 - Train: 3.53% [80500/2279000] [35.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 13:02:44,861 - Train: 3.54% [80600/2279000] [35.4/1000.0] [batch_t 0.610 (0.605)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-09 13:03:45,447 - Train: 3.54% [80700/2279000] [35.4/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 13:04:45,873 - Train: 3.55% [80800/2279000] [35.5/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 13:05:46,318 - Train: 3.55% [80900/2279000] [35.5/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 13:06:46,820 - Train: 3.55% [81000/2279000] [35.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 13:07:47,256 - Train: 3.56% [81100/2279000] [35.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 13:08:47,824 - Train: 3.56% [81200/2279000] [35.6/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-09 13:09:48,273 - Train: 3.57% [81300/2279000] [35.7/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 13:10:48,678 - Train: 3.57% [81400/2279000] [35.7/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 13:11:49,104 - Train: 3.58% [81500/2279000] [35.8/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 13:12:49,537 - Train: 3.58% [81600/2279000] [35.8/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 13:13:49,944 - Train: 3.58% [81700/2279000] [35.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 13:14:50,321 - Train: 3.59% [81800/2279000] [35.9/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 13:15:50,706 - Train: 3.59% [81900/2279000] [35.9/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 13:16:51,225 - Train: 3.60% [82000/2279000] [36.0/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-09 13:17:17,835 - ==> Total time: 13:48:37 Eta: 15 days, 9:48:55 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-09 13:17:53,546 - Train: 3.60% [82100/2279000] [36.0/1000.0] [batch_t 0.606 (0.611)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 13:18:53,931 - Train: 3.61% [82200/2279000] [36.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 13:19:54,395 - Train: 3.61% [82300/2279000] [36.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 13:20:54,823 - Train: 3.62% [82400/2279000] [36.2/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 13:21:55,251 - Train: 3.62% [82500/2279000] [36.2/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 13:22:55,835 - Train: 3.62% [82600/2279000] [36.2/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 13:23:56,275 - Train: 3.63% [82700/2279000] [36.3/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 13:24:56,730 - Train: 3.63% [82800/2279000] [36.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 13:25:57,181 - Train: 3.64% [82900/2279000] [36.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 13:26:57,606 - Train: 3.64% [83000/2279000] [36.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 13:27:58,060 - Train: 3.65% [83100/2279000] [36.5/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 13:28:58,626 - Train: 3.65% [83200/2279000] [36.5/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 13:29:59,064 - Train: 3.66% [83300/2279000] [36.6/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 13:30:59,488 - Train: 3.66% [83400/2279000] [36.6/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 13:31:59,928 - Train: 3.66% [83500/2279000] [36.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 13:33:00,333 - Train: 3.67% [83600/2279000] [36.7/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 13:34:00,812 - Train: 3.67% [83700/2279000] [36.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 13:35:01,330 - Train: 3.68% [83800/2279000] [36.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 13:36:01,726 - Train: 3.68% [83900/2279000] [36.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 13:37:02,143 - Train: 3.69% [84000/2279000] [36.9/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 13:38:02,555 - Train: 3.69% [84100/2279000] [36.9/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 13:39:02,973 - Train: 3.69% [84200/2279000] [36.9/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 13:40:03,518 - Train: 3.70% [84300/2279000] [37.0/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-09 13:40:17,449 - ==> Total time: 14:11:37 Eta: 15 days, 9:25:17 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-09 13:41:05,794 - Train: 3.70% [84400/2279000] [37.0/1000.0] [batch_t 0.604 (0.609)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 13:42:06,208 - Train: 3.71% [84500/2279000] [37.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 13:43:06,695 - Train: 3.71% [84600/2279000] [37.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 13:44:07,163 - Train: 3.72% [84700/2279000] [37.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 13:45:07,618 - Train: 3.72% [84800/2279000] [37.2/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 13:46:08,179 - Train: 3.73% [84900/2279000] [37.3/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 13:47:08,605 - Train: 3.73% [85000/2279000] [37.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 13:48:09,020 - Train: 3.73% [85100/2279000] [37.3/1000.0] [batch_t 0.608 (0.604)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-09 13:49:09,425 - Train: 3.74% [85200/2279000] [37.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 13:50:09,846 - Train: 3.74% [85300/2279000] [37.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 13:51:10,284 - Train: 3.75% [85400/2279000] [37.5/1000.0] [batch_t 0.607 (0.604)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-09 13:52:10,656 - Train: 3.75% [85500/2279000] [37.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 13:53:11,271 - Train: 3.76% [85600/2279000] [37.6/1000.0] [batch_t 0.606 (0.606)] [data_t 0.003] [optim_t 0.603] [lr 0.005000] +2024-04-09 13:54:11,714 - Train: 3.76% [85700/2279000] [37.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 13:55:12,163 - Train: 3.76% [85800/2279000] [37.6/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 13:56:12,601 - Train: 3.77% [85900/2279000] [37.7/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 13:57:13,080 - Train: 3.77% [86000/2279000] [37.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 13:58:13,505 - Train: 3.78% [86100/2279000] [37.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 13:59:14,098 - Train: 3.78% [86200/2279000] [37.8/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 14:00:14,489 - Train: 3.79% [86300/2279000] [37.9/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 14:01:14,935 - Train: 3.79% [86400/2279000] [37.9/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 14:02:15,362 - Train: 3.80% [86500/2279000] [38.0/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 14:03:15,836 - Train: 3.80% [86600/2279000] [38.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 14:03:17,048 - ==> Total time: 14:34:37 Eta: 15 days, 9:01:40 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-09 14:04:18,092 - Train: 3.80% [86700/2279000] [38.0/1000.0] [batch_t 0.604 (0.608)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 14:05:18,720 - Train: 3.81% [86800/2279000] [38.1/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 14:06:19,157 - Train: 3.81% [86900/2279000] [38.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 14:07:19,570 - Train: 3.82% [87000/2279000] [38.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 14:08:19,975 - Train: 3.82% [87100/2279000] [38.2/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 14:09:20,371 - Train: 3.83% [87200/2279000] [38.3/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 14:10:20,781 - Train: 3.83% [87300/2279000] [38.3/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 14:11:21,205 - Train: 3.84% [87400/2279000] [38.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 14:12:21,793 - Train: 3.84% [87500/2279000] [38.4/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 14:13:22,243 - Train: 3.84% [87600/2279000] [38.4/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 14:14:22,675 - Train: 3.85% [87700/2279000] [38.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 14:15:23,098 - Train: 3.85% [87800/2279000] [38.5/1000.0] [batch_t 0.609 (0.604)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-09 14:16:23,544 - Train: 3.86% [87900/2279000] [38.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 14:17:24,056 - Train: 3.86% [88000/2279000] [38.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 14:18:24,683 - Train: 3.87% [88100/2279000] [38.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 14:19:25,133 - Train: 3.87% [88200/2279000] [38.7/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 14:20:25,579 - Train: 3.87% [88300/2279000] [38.7/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 14:21:26,001 - Train: 3.88% [88400/2279000] [38.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 14:22:26,481 - Train: 3.88% [88500/2279000] [38.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 14:23:26,983 - Train: 3.89% [88600/2279000] [38.9/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 14:24:27,452 - Train: 3.89% [88700/2279000] [38.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 14:25:28,019 - Train: 3.90% [88800/2279000] [39.0/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 14:26:16,985 - ==> Total time: 14:57:37 Eta: 15 days, 8:38:13 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-09 14:26:30,388 - Train: 3.90% [88900/2279000] [39.0/1000.0] [batch_t 0.605 (0.632)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 14:27:30,808 - Train: 3.91% [89000/2279000] [39.1/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 14:28:31,274 - Train: 3.91% [89100/2279000] [39.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 14:29:31,693 - Train: 3.91% [89200/2279000] [39.1/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 14:30:32,300 - Train: 3.92% [89300/2279000] [39.2/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 14:31:32,720 - Train: 3.92% [89400/2279000] [39.2/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 14:32:33,190 - Train: 3.93% [89500/2279000] [39.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 14:33:33,698 - Train: 3.93% [89600/2279000] [39.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 14:34:34,170 - Train: 3.94% [89700/2279000] [39.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 14:35:34,555 - Train: 3.94% [89800/2279000] [39.4/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 14:36:34,956 - Train: 3.94% [89900/2279000] [39.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 14:37:35,369 - Train: 3.95% [90000/2279000] [39.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 14:38:35,938 - Train: 3.95% [90100/2279000] [39.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 14:39:36,372 - Train: 3.96% [90200/2279000] [39.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 14:40:36,847 - Train: 3.96% [90300/2279000] [39.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 14:41:37,294 - Train: 3.97% [90400/2279000] [39.7/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 14:42:37,747 - Train: 3.97% [90500/2279000] [39.7/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 14:43:38,354 - Train: 3.98% [90600/2279000] [39.8/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 14:44:38,809 - Train: 3.98% [90700/2279000] [39.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 14:45:39,235 - Train: 3.98% [90800/2279000] [39.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 14:46:39,691 - Train: 3.99% [90900/2279000] [39.9/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 14:47:40,119 - Train: 3.99% [91000/2279000] [39.9/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 14:48:40,601 - Train: 4.00% [91100/2279000] [40.0/1000.0] [batch_t 0.610 (0.605)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-09 14:49:16,864 - ==> Total time: 15:20:36 Eta: 15 days, 8:14:47 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-09 14:49:43,046 - Train: 4.00% [91200/2279000] [40.0/1000.0] [batch_t 0.605 (0.617)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 14:50:43,499 - Train: 4.01% [91300/2279000] [40.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 14:51:43,942 - Train: 4.01% [91400/2279000] [40.1/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 14:52:44,344 - Train: 4.01% [91500/2279000] [40.1/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 14:53:44,788 - Train: 4.02% [91600/2279000] [40.2/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 14:54:45,194 - Train: 4.02% [91700/2279000] [40.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 14:55:45,845 - Train: 4.03% [91800/2279000] [40.3/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 14:56:46,291 - Train: 4.03% [91900/2279000] [40.3/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 14:57:46,742 - Train: 4.04% [92000/2279000] [40.4/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 14:58:47,192 - Train: 4.04% [92100/2279000] [40.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 14:59:47,611 - Train: 4.05% [92200/2279000] [40.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 15:00:48,036 - Train: 4.05% [92300/2279000] [40.5/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 15:01:48,628 - Train: 4.05% [92400/2279000] [40.5/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 15:02:49,065 - Train: 4.06% [92500/2279000] [40.6/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 15:03:49,505 - Train: 4.06% [92600/2279000] [40.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 15:04:49,932 - Train: 4.07% [92700/2279000] [40.7/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 15:05:50,349 - Train: 4.07% [92800/2279000] [40.7/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 15:06:50,815 - Train: 4.08% [92900/2279000] [40.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 15:07:51,267 - Train: 4.08% [93000/2279000] [40.8/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 15:08:51,878 - Train: 4.09% [93100/2279000] [40.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 15:09:52,286 - Train: 4.09% [93200/2279000] [40.9/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 15:10:52,739 - Train: 4.09% [93300/2279000] [40.9/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 15:11:53,214 - Train: 4.10% [93400/2279000] [41.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 15:12:16,782 - ==> Total time: 15:43:36 Eta: 15 days, 7:51:22 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-09 15:12:55,518 - Train: 4.10% [93500/2279000] [41.0/1000.0] [batch_t 0.604 (0.612)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 15:13:56,106 - Train: 4.11% [93600/2279000] [41.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 15:14:56,549 - Train: 4.11% [93700/2279000] [41.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 15:15:56,985 - Train: 4.12% [93800/2279000] [41.2/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 15:16:57,428 - Train: 4.12% [93900/2279000] [41.2/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 15:17:57,896 - Train: 4.12% [94000/2279000] [41.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 15:18:58,323 - Train: 4.13% [94100/2279000] [41.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 15:19:58,749 - Train: 4.13% [94200/2279000] [41.3/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 15:20:59,376 - Train: 4.14% [94300/2279000] [41.4/1000.0] [batch_t 0.609 (0.606)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-09 15:21:59,896 - Train: 4.14% [94400/2279000] [41.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 15:23:00,350 - Train: 4.15% [94500/2279000] [41.5/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 15:24:00,814 - Train: 4.15% [94600/2279000] [41.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 15:25:01,317 - Train: 4.16% [94700/2279000] [41.6/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 15:26:01,730 - Train: 4.16% [94800/2279000] [41.6/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 15:27:02,150 - Train: 4.16% [94900/2279000] [41.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 15:28:02,857 - Train: 4.17% [95000/2279000] [41.7/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 15:29:03,399 - Train: 4.17% [95100/2279000] [41.7/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-09 15:30:03,839 - Train: 4.18% [95200/2279000] [41.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 15:31:04,229 - Train: 4.18% [95300/2279000] [41.8/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 15:32:04,644 - Train: 4.19% [95400/2279000] [41.9/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 15:33:05,070 - Train: 4.19% [95500/2279000] [41.9/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 15:34:05,505 - Train: 4.19% [95600/2279000] [41.9/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 15:35:06,093 - Train: 4.20% [95700/2279000] [42.0/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 15:35:16,975 - ==> Total time: 16:06:37 Eta: 15 days, 7:28:05 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-09 15:36:08,846 - Train: 4.20% [95800/2279000] [42.0/1000.0] [batch_t 0.605 (0.612)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 15:37:20,892 - Train: 4.21% [95900/2279000] [42.1/1000.0] [batch_t 0.605 (0.720)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 15:38:21,266 - Train: 4.21% [96000/2279000] [42.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 15:39:21,675 - Train: 4.22% [96100/2279000] [42.2/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 15:40:22,248 - Train: 4.22% [96200/2279000] [42.2/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 15:41:22,672 - Train: 4.23% [96300/2279000] [42.3/1000.0] [batch_t 0.610 (0.604)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-09 15:42:23,136 - Train: 4.23% [96400/2279000] [42.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 15:43:23,558 - Train: 4.23% [96500/2279000] [42.3/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 15:44:23,999 - Train: 4.24% [96600/2279000] [42.4/1000.0] [batch_t 0.609 (0.604)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-09 15:45:24,411 - Train: 4.24% [96700/2279000] [42.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 15:46:24,949 - Train: 4.25% [96800/2279000] [42.5/1000.0] [batch_t 0.617 (0.605)] [data_t 0.002] [optim_t 0.615] [lr 0.005000] +2024-04-09 15:47:25,396 - Train: 4.25% [96900/2279000] [42.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 15:48:25,822 - Train: 4.26% [97000/2279000] [42.6/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 15:49:26,215 - Train: 4.26% [97100/2279000] [42.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 15:50:26,613 - Train: 4.27% [97200/2279000] [42.7/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 15:51:27,093 - Train: 4.27% [97300/2279000] [42.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 15:52:27,508 - Train: 4.27% [97400/2279000] [42.7/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 15:53:27,930 - Train: 4.28% [97500/2279000] [42.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 15:54:28,578 - Train: 4.28% [97600/2279000] [42.8/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 15:55:29,002 - Train: 4.29% [97700/2279000] [42.9/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 15:56:29,381 - Train: 4.29% [97800/2279000] [42.9/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 15:57:29,777 - Train: 4.30% [97900/2279000] [43.0/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 15:58:28,414 - ==> Total time: 16:29:48 Eta: 15 days, 7:08:59 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-09 15:58:32,082 - Train: 4.30% [98000/2279000] [43.0/1000.0] [batch_t 0.605 (0.751)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 15:59:32,661 - Train: 4.30% [98100/2279000] [43.0/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 16:00:33,077 - Train: 4.31% [98200/2279000] [43.1/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 16:01:33,597 - Train: 4.31% [98300/2279000] [43.1/1000.0] [batch_t 0.610 (0.605)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-09 16:02:34,020 - Train: 4.32% [98400/2279000] [43.2/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 16:03:34,448 - Train: 4.32% [98500/2279000] [43.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 16:04:34,854 - Train: 4.33% [98600/2279000] [43.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 16:05:35,400 - Train: 4.33% [98700/2279000] [43.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 16:06:35,861 - Train: 4.34% [98800/2279000] [43.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 16:07:42,024 - Train: 4.34% [98900/2279000] [43.4/1000.0] [batch_t 0.605 (0.662)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 16:08:42,416 - Train: 4.34% [99000/2279000] [43.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 16:09:42,838 - Train: 4.35% [99100/2279000] [43.5/1000.0] [batch_t 0.609 (0.604)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-09 16:10:43,277 - Train: 4.35% [99200/2279000] [43.5/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 16:11:43,746 - Train: 4.36% [99300/2279000] [43.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 16:12:44,240 - Train: 4.36% [99400/2279000] [43.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 16:13:44,646 - Train: 4.37% [99500/2279000] [43.7/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 16:14:45,089 - Train: 4.37% [99600/2279000] [43.7/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 16:15:45,489 - Train: 4.37% [99700/2279000] [43.7/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 16:16:45,927 - Train: 4.38% [99800/2279000] [43.8/1000.0] [batch_t 0.609 (0.604)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-09 16:17:46,341 - Train: 4.38% [99900/2279000] [43.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 16:18:46,746 - Train: 4.39% [100000/2279000] [43.9/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 16:19:47,239 - Train: 4.39% [100100/2279000] [43.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 16:20:47,655 - Train: 4.40% [100200/2279000] [44.0/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 16:21:33,572 - ==> Total time: 16:52:53 Eta: 15 days, 6:47:26 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-09 16:21:50,308 - Train: 4.40% [100300/2279000] [44.0/1000.0] [batch_t 0.605 (0.628)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 16:22:50,718 - Train: 4.41% [100400/2279000] [44.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 16:23:51,260 - Train: 4.41% [100500/2279000] [44.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 16:24:51,665 - Train: 4.41% [100600/2279000] [44.1/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 16:25:52,065 - Train: 4.42% [100700/2279000] [44.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 16:26:52,521 - Train: 4.42% [100800/2279000] [44.2/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 16:27:52,935 - Train: 4.43% [100900/2279000] [44.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 16:28:53,373 - Train: 4.43% [101000/2279000] [44.3/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 16:29:53,903 - Train: 4.44% [101100/2279000] [44.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 16:30:54,293 - Train: 4.44% [101200/2279000] [44.4/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 16:31:54,694 - Train: 4.44% [101300/2279000] [44.4/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 16:32:55,127 - Train: 4.45% [101400/2279000] [44.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 16:33:55,594 - Train: 4.45% [101500/2279000] [44.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 16:35:01,873 - Train: 4.46% [101600/2279000] [44.6/1000.0] [batch_t 0.603 (0.663)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 16:36:32,340 - Train: 4.46% [101700/2279000] [44.6/1000.0] [batch_t 0.974 (0.905)] [data_t 0.370] [optim_t 0.604] [lr 0.005000] +2024-04-09 16:38:11,423 - Train: 4.47% [101800/2279000] [44.7/1000.0] [batch_t 0.605 (0.991)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 16:39:11,821 - Train: 4.47% [101900/2279000] [44.7/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 16:40:12,257 - Train: 4.48% [102000/2279000] [44.8/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 16:41:12,690 - Train: 4.48% [102100/2279000] [44.8/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 16:42:13,122 - Train: 4.48% [102200/2279000] [44.8/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 16:43:13,518 - Train: 4.49% [102300/2279000] [44.9/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 16:44:13,973 - Train: 4.49% [102400/2279000] [44.9/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 16:45:14,473 - Train: 4.50% [102500/2279000] [45.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 16:45:47,683 - ==> Total time: 17:17:07 Eta: 15 days, 6:50:11 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-09 16:46:16,626 - Train: 4.50% [102600/2279000] [45.0/1000.0] [batch_t 0.605 (0.611)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 16:47:25,812 - Train: 4.51% [102700/2279000] [45.1/1000.0] [batch_t 0.605 (0.692)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 16:48:26,216 - Train: 4.51% [102800/2279000] [45.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 16:49:26,621 - Train: 4.52% [102900/2279000] [45.2/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 16:50:27,168 - Train: 4.52% [103000/2279000] [45.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 16:51:27,596 - Train: 4.52% [103100/2279000] [45.2/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 16:52:27,980 - Train: 4.53% [103200/2279000] [45.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 16:53:28,444 - Train: 4.53% [103300/2279000] [45.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 16:54:28,912 - Train: 4.54% [103400/2279000] [45.4/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-09 16:55:29,662 - Train: 4.54% [103500/2279000] [45.4/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 16:56:30,169 - Train: 4.55% [103600/2279000] [45.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 16:57:30,665 - Train: 4.55% [103700/2279000] [45.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 16:58:31,143 - Train: 4.55% [103800/2279000] [45.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 16:59:31,624 - Train: 4.56% [103900/2279000] [45.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 17:00:32,135 - Train: 4.56% [104000/2279000] [45.6/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 17:01:32,563 - Train: 4.57% [104100/2279000] [45.7/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 17:02:33,009 - Train: 4.57% [104200/2279000] [45.7/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 17:03:33,738 - Train: 4.58% [104300/2279000] [45.8/1000.0] [batch_t 0.610 (0.607)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-09 17:04:34,293 - Train: 4.58% [104400/2279000] [45.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 17:05:34,750 - Train: 4.59% [104500/2279000] [45.9/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 17:06:35,215 - Train: 4.59% [104600/2279000] [45.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 17:07:35,631 - Train: 4.59% [104700/2279000] [45.9/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 17:08:36,039 - Train: 4.60% [104800/2279000] [46.0/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 17:08:56,582 - ==> Total time: 17:40:16 Eta: 15 days, 6:29:14 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-09 17:09:38,972 - Train: 4.60% [104900/2279000] [46.0/1000.0] [batch_t 0.605 (0.618)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 17:10:39,472 - Train: 4.61% [105000/2279000] [46.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 17:11:39,981 - Train: 4.61% [105100/2279000] [46.1/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-09 17:12:40,462 - Train: 4.62% [105200/2279000] [46.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 17:13:40,885 - Train: 4.62% [105300/2279000] [46.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 17:14:41,336 - Train: 4.62% [105400/2279000] [46.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 17:15:41,777 - Train: 4.63% [105500/2279000] [46.3/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 17:16:42,352 - Train: 4.63% [105600/2279000] [46.3/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 17:17:42,831 - Train: 4.64% [105700/2279000] [46.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 17:18:43,298 - Train: 4.64% [105800/2279000] [46.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 17:19:43,763 - Train: 4.65% [105900/2279000] [46.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 17:20:44,253 - Train: 4.65% [106000/2279000] [46.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 17:21:44,678 - Train: 4.66% [106100/2279000] [46.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 17:22:57,393 - Train: 4.66% [106200/2279000] [46.6/1000.0] [batch_t 0.605 (0.727)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 17:23:57,816 - Train: 4.66% [106300/2279000] [46.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 17:24:58,635 - Train: 4.67% [106400/2279000] [46.7/1000.0] [batch_t 0.605 (0.608)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 17:25:59,129 - Train: 4.67% [106500/2279000] [46.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 17:26:59,583 - Train: 4.68% [106600/2279000] [46.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 17:28:00,082 - Train: 4.68% [106700/2279000] [46.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 17:29:00,543 - Train: 4.69% [106800/2279000] [46.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 17:30:01,067 - Train: 4.69% [106900/2279000] [46.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 17:31:01,451 - Train: 4.70% [107000/2279000] [47.0/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 17:32:01,833 - Train: 4.70% [107100/2279000] [47.0/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 17:32:09,687 - ==> Total time: 18:03:29 Eta: 15 days, 6:09:37 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-09 17:33:04,592 - Train: 4.70% [107200/2279000] [47.0/1000.0] [batch_t 0.605 (0.613)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 17:34:05,022 - Train: 4.71% [107300/2279000] [47.1/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 17:35:14,593 - Train: 4.71% [107400/2279000] [47.1/1000.0] [batch_t 0.607 (0.696)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 17:36:15,043 - Train: 4.72% [107500/2279000] [47.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 17:37:15,471 - Train: 4.72% [107600/2279000] [47.2/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 17:38:15,913 - Train: 4.73% [107700/2279000] [47.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 17:39:16,356 - Train: 4.73% [107800/2279000] [47.3/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 17:40:16,812 - Train: 4.73% [107900/2279000] [47.3/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 17:41:17,455 - Train: 4.74% [108000/2279000] [47.4/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 17:42:17,905 - Train: 4.74% [108100/2279000] [47.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 17:43:18,367 - Train: 4.75% [108200/2279000] [47.5/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 17:44:18,820 - Train: 4.75% [108300/2279000] [47.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 17:45:19,264 - Train: 4.76% [108400/2279000] [47.6/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 17:46:19,778 - Train: 4.76% [108500/2279000] [47.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 17:47:20,239 - Train: 4.77% [108600/2279000] [47.7/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 17:48:20,870 - Train: 4.77% [108700/2279000] [47.7/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 17:49:21,327 - Train: 4.77% [108800/2279000] [47.7/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 17:50:21,741 - Train: 4.78% [108900/2279000] [47.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 17:51:22,236 - Train: 4.78% [109000/2279000] [47.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 17:52:22,668 - Train: 4.79% [109100/2279000] [47.9/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 17:53:23,112 - Train: 4.79% [109200/2279000] [47.9/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 17:54:33,121 - Train: 4.80% [109300/2279000] [48.0/1000.0] [batch_t 0.605 (0.700)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 17:55:28,977 - ==> Total time: 18:26:49 Eta: 15 days, 5:51:53 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-09 17:55:35,847 - Train: 4.80% [109400/2279000] [48.0/1000.0] [batch_t 0.606 (0.658)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 17:56:36,326 - Train: 4.80% [109500/2279000] [48.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 17:57:36,774 - Train: 4.81% [109600/2279000] [48.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 17:58:37,202 - Train: 4.81% [109700/2279000] [48.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 17:59:37,615 - Train: 4.82% [109800/2279000] [48.2/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 18:00:38,195 - Train: 4.82% [109900/2279000] [48.2/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 18:01:38,650 - Train: 4.83% [110000/2279000] [48.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 18:02:39,069 - Train: 4.83% [110100/2279000] [48.3/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 18:03:39,487 - Train: 4.84% [110200/2279000] [48.4/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 18:04:39,930 - Train: 4.84% [110300/2279000] [48.4/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 18:05:40,377 - Train: 4.84% [110400/2279000] [48.4/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 18:06:40,980 - Train: 4.85% [110500/2279000] [48.5/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 18:07:41,446 - Train: 4.85% [110600/2279000] [48.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 18:08:41,831 - Train: 4.86% [110700/2279000] [48.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 18:09:42,297 - Train: 4.86% [110800/2279000] [48.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 18:10:42,731 - Train: 4.87% [110900/2279000] [48.7/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 18:11:43,188 - Train: 4.87% [111000/2279000] [48.7/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 18:12:43,617 - Train: 4.87% [111100/2279000] [48.7/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 18:13:44,261 - Train: 4.88% [111200/2279000] [48.8/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 18:14:44,709 - Train: 4.88% [111300/2279000] [48.8/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 18:15:45,169 - Train: 4.89% [111400/2279000] [48.9/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 18:16:45,644 - Train: 4.89% [111500/2279000] [48.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 18:17:46,081 - Train: 4.90% [111600/2279000] [49.0/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 18:18:28,981 - ==> Total time: 18:49:49 Eta: 15 days, 5:27:41 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-09 18:18:48,503 - Train: 4.90% [111700/2279000] [49.0/1000.0] [batch_t 0.606 (0.616)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 18:19:49,024 - Train: 4.91% [111800/2279000] [49.1/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-09 18:20:49,491 - Train: 4.91% [111900/2279000] [49.1/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 18:21:49,926 - Train: 4.91% [112000/2279000] [49.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 18:22:50,319 - Train: 4.92% [112100/2279000] [49.2/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 18:23:50,701 - Train: 4.92% [112200/2279000] [49.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 18:24:51,089 - Train: 4.93% [112300/2279000] [49.3/1000.0] [batch_t 0.610 (0.604)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-09 18:25:51,640 - Train: 4.93% [112400/2279000] [49.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 18:26:52,054 - Train: 4.94% [112500/2279000] [49.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 18:27:52,476 - Train: 4.94% [112600/2279000] [49.4/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 18:28:52,895 - Train: 4.95% [112700/2279000] [49.5/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 18:29:53,309 - Train: 4.95% [112800/2279000] [49.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 18:30:53,701 - Train: 4.95% [112900/2279000] [49.5/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 18:31:54,140 - Train: 4.96% [113000/2279000] [49.6/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 18:32:54,672 - Train: 4.96% [113100/2279000] [49.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 18:33:55,085 - Train: 4.97% [113200/2279000] [49.7/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 18:34:55,500 - Train: 4.97% [113300/2279000] [49.7/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 18:35:55,920 - Train: 4.98% [113400/2279000] [49.8/1000.0] [batch_t 0.608 (0.604)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-09 18:36:56,410 - Train: 4.98% [113500/2279000] [49.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 18:37:56,821 - Train: 4.98% [113600/2279000] [49.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 18:38:57,242 - Train: 4.99% [113700/2279000] [49.9/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 18:39:57,723 - Train: 4.99% [113800/2279000] [49.9/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 18:40:58,128 - Train: 5.00% [113900/2279000] [50.0/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 18:41:40,375 - Test: 0.70% [50/7162] [batch_t 0.213 (0.233)] +2024-04-09 18:41:51,099 - Test: 1.40% [100/7162] [batch_t 0.211 (0.224)] +2024-04-09 18:42:01,715 - Test: 2.09% [150/7162] [batch_t 0.212 (0.220)] +2024-04-09 18:42:13,490 - Test: 2.79% [200/7162] [batch_t 0.211 (0.224)] +2024-04-09 18:42:24,348 - Test: 3.49% [250/7162] [batch_t 0.212 (0.222)] +2024-04-09 18:42:35,745 - Test: 4.19% [300/7162] [batch_t 0.211 (0.223)] +2024-04-09 18:42:47,348 - Test: 4.89% [350/7162] [batch_t 0.211 (0.225)] +2024-04-09 18:42:59,621 - Test: 5.59% [400/7162] [batch_t 0.289 (0.227)] +2024-04-09 18:43:12,032 - Test: 6.28% [450/7162] [batch_t 0.210 (0.230)] +2024-04-09 18:43:22,604 - Test: 6.98% [500/7162] [batch_t 0.225 (0.228)] +2024-04-09 18:43:34,617 - Test: 7.68% [550/7162] [batch_t 0.211 (0.229)] +2024-04-09 18:43:46,709 - Test: 8.38% [600/7162] [batch_t 0.211 (0.230)] +2024-04-09 18:43:58,326 - Test: 9.08% [650/7162] [batch_t 0.213 (0.230)] +2024-04-09 18:44:10,722 - Test: 9.77% [700/7162] [batch_t 0.211 (0.231)] +2024-04-09 18:44:22,297 - Test: 10.47% [750/7162] [batch_t 0.211 (0.231)] +2024-04-09 18:44:34,356 - Test: 11.17% [800/7162] [batch_t 0.348 (0.232)] +2024-04-09 18:44:46,709 - Test: 11.87% [850/7162] [batch_t 0.211 (0.233)] +2024-04-09 18:44:58,825 - Test: 12.57% [900/7162] [batch_t 0.212 (0.233)] +2024-04-09 18:45:11,548 - Test: 13.26% [950/7162] [batch_t 0.211 (0.235)] +2024-04-09 18:45:22,147 - Test: 13.96% [1000/7162] [batch_t 0.212 (0.233)] +2024-04-09 18:45:32,777 - Test: 14.66% [1050/7162] [batch_t 0.212 (0.232)] +2024-04-09 18:45:44,017 - Test: 15.36% [1100/7162] [batch_t 0.215 (0.232)] +2024-04-09 18:45:54,583 - Test: 16.06% [1150/7162] [batch_t 0.211 (0.231)] +2024-04-09 18:46:06,550 - Test: 16.76% [1200/7162] [batch_t 0.219 (0.231)] +2024-04-09 18:46:17,964 - Test: 17.45% [1250/7162] [batch_t 0.213 (0.231)] +2024-04-09 18:46:29,998 - Test: 18.15% [1300/7162] [batch_t 0.234 (0.232)] +2024-04-09 18:46:41,462 - Test: 18.85% [1350/7162] [batch_t 0.347 (0.232)] +2024-04-09 18:46:52,581 - Test: 19.55% [1400/7162] [batch_t 0.215 (0.231)] +2024-04-09 18:47:05,177 - Test: 20.25% [1450/7162] [batch_t 0.530 (0.232)] +2024-04-09 18:47:16,379 - Test: 20.94% [1500/7162] [batch_t 0.364 (0.232)] +2024-04-09 18:47:27,633 - Test: 21.64% [1550/7162] [batch_t 0.214 (0.232)] +2024-04-09 18:47:38,670 - Test: 22.34% [1600/7162] [batch_t 0.217 (0.231)] +2024-04-09 18:47:49,678 - Test: 23.04% [1650/7162] [batch_t 0.228 (0.231)] +2024-04-09 18:48:01,316 - Test: 23.74% [1700/7162] [batch_t 0.217 (0.231)] +2024-04-09 18:48:15,168 - Test: 24.43% [1750/7162] [batch_t 0.213 (0.232)] +2024-04-09 18:48:27,093 - Test: 25.13% [1800/7162] [batch_t 0.211 (0.232)] +2024-04-09 18:48:38,330 - Test: 25.83% [1850/7162] [batch_t 0.214 (0.232)] +2024-04-09 18:48:48,985 - Test: 26.53% [1900/7162] [batch_t 0.212 (0.232)] +2024-04-09 18:48:59,670 - Test: 27.23% [1950/7162] [batch_t 0.213 (0.231)] +2024-04-09 18:49:12,865 - Test: 27.93% [2000/7162] [batch_t 0.214 (0.232)] +2024-04-09 18:49:23,549 - Test: 28.62% [2050/7162] [batch_t 0.217 (0.232)] +2024-04-09 18:49:34,274 - Test: 29.32% [2100/7162] [batch_t 0.212 (0.231)] +2024-04-09 18:49:44,881 - Test: 30.02% [2150/7162] [batch_t 0.213 (0.231)] +2024-04-09 18:49:55,491 - Test: 30.72% [2200/7162] [batch_t 0.212 (0.230)] +2024-04-09 18:50:07,835 - Test: 31.42% [2250/7162] [batch_t 0.214 (0.231)] +2024-04-09 18:50:18,515 - Test: 32.11% [2300/7162] [batch_t 0.212 (0.230)] +2024-04-09 18:50:29,157 - Test: 32.81% [2350/7162] [batch_t 0.211 (0.230)] +2024-04-09 18:50:39,902 - Test: 33.51% [2400/7162] [batch_t 0.219 (0.230)] +2024-04-09 18:50:51,067 - Test: 34.21% [2450/7162] [batch_t 0.212 (0.229)] +2024-04-09 18:51:04,447 - Test: 34.91% [2500/7162] [batch_t 0.212 (0.230)] +2024-04-09 18:51:17,098 - Test: 35.60% [2550/7162] [batch_t 0.212 (0.231)] +2024-04-09 18:51:27,923 - Test: 36.30% [2600/7162] [batch_t 0.212 (0.230)] +2024-04-09 18:51:38,748 - Test: 37.00% [2650/7162] [batch_t 0.212 (0.230)] +2024-04-09 18:51:49,433 - Test: 37.70% [2700/7162] [batch_t 0.211 (0.230)] +2024-04-09 18:52:00,175 - Test: 38.40% [2750/7162] [batch_t 0.214 (0.230)] +2024-04-09 18:52:12,104 - Test: 39.10% [2800/7162] [batch_t 0.214 (0.230)] +2024-04-09 18:52:23,050 - Test: 39.79% [2850/7162] [batch_t 0.212 (0.230)] +2024-04-09 18:52:33,713 - Test: 40.49% [2900/7162] [batch_t 0.212 (0.229)] +2024-04-09 18:52:44,354 - Test: 41.19% [2950/7162] [batch_t 0.211 (0.229)] +2024-04-09 18:52:54,939 - Test: 41.89% [3000/7162] [batch_t 0.212 (0.229)] +2024-04-09 18:53:06,442 - Test: 42.59% [3050/7162] [batch_t 0.213 (0.229)] +2024-04-09 18:53:17,086 - Test: 43.28% [3100/7162] [batch_t 0.226 (0.228)] +2024-04-09 18:53:27,735 - Test: 43.98% [3150/7162] [batch_t 0.210 (0.228)] +2024-04-09 18:53:38,324 - Test: 44.68% [3200/7162] [batch_t 0.211 (0.228)] +2024-04-09 18:53:48,886 - Test: 45.38% [3250/7162] [batch_t 0.211 (0.228)] +2024-04-09 18:53:59,559 - Test: 46.08% [3300/7162] [batch_t 0.213 (0.227)] +2024-04-09 18:54:11,588 - Test: 46.77% [3350/7162] [batch_t 0.219 (0.228)] +2024-04-09 18:54:22,298 - Test: 47.47% [3400/7162] [batch_t 0.212 (0.227)] +2024-04-09 18:54:32,987 - Test: 48.17% [3450/7162] [batch_t 0.213 (0.227)] +2024-04-09 18:54:43,723 - Test: 48.87% [3500/7162] [batch_t 0.217 (0.227)] +2024-04-09 18:54:54,601 - Test: 49.57% [3550/7162] [batch_t 0.212 (0.227)] +2024-04-09 18:55:07,066 - Test: 50.27% [3600/7162] [batch_t 0.445 (0.227)] +2024-04-09 18:55:17,756 - Test: 50.96% [3650/7162] [batch_t 0.211 (0.227)] +2024-04-09 18:55:28,320 - Test: 51.66% [3700/7162] [batch_t 0.212 (0.227)] +2024-04-09 18:55:38,941 - Test: 52.36% [3750/7162] [batch_t 0.213 (0.227)] +2024-04-09 18:55:49,512 - Test: 53.06% [3800/7162] [batch_t 0.211 (0.226)] +2024-04-09 18:56:00,120 - Test: 53.76% [3850/7162] [batch_t 0.211 (0.226)] +2024-04-09 18:56:12,262 - Test: 54.45% [3900/7162] [batch_t 0.211 (0.227)] +2024-04-09 18:56:22,875 - Test: 55.15% [3950/7162] [batch_t 0.211 (0.226)] +2024-04-09 18:56:33,426 - Test: 55.85% [4000/7162] [batch_t 0.210 (0.226)] +2024-04-09 18:56:44,009 - Test: 56.55% [4050/7162] [batch_t 0.213 (0.226)] +2024-04-09 18:56:54,598 - Test: 57.25% [4100/7162] [batch_t 0.214 (0.226)] +2024-04-09 18:57:06,587 - Test: 57.94% [4150/7162] [batch_t 0.212 (0.226)] +2024-04-09 18:57:17,156 - Test: 58.64% [4200/7162] [batch_t 0.213 (0.226)] +2024-04-09 18:57:27,727 - Test: 59.34% [4250/7162] [batch_t 0.211 (0.226)] +2024-04-09 18:57:38,295 - Test: 60.04% [4300/7162] [batch_t 0.211 (0.225)] +2024-04-09 18:57:49,141 - Test: 60.74% [4350/7162] [batch_t 0.228 (0.225)] +2024-04-09 18:58:00,285 - Test: 61.44% [4400/7162] [batch_t 0.212 (0.225)] +2024-04-09 18:58:12,441 - Test: 62.13% [4450/7162] [batch_t 0.212 (0.226)] +2024-04-09 18:58:23,076 - Test: 62.83% [4500/7162] [batch_t 0.211 (0.225)] +2024-04-09 18:58:33,609 - Test: 63.53% [4550/7162] [batch_t 0.210 (0.225)] +2024-04-09 18:58:44,191 - Test: 64.23% [4600/7162] [batch_t 0.213 (0.225)] +2024-04-09 18:58:54,900 - Test: 64.93% [4650/7162] [batch_t 0.211 (0.225)] +2024-04-09 18:59:07,169 - Test: 65.62% [4700/7162] [batch_t 0.212 (0.225)] +2024-04-09 18:59:17,775 - Test: 66.32% [4750/7162] [batch_t 0.212 (0.225)] +2024-04-09 18:59:28,370 - Test: 67.02% [4800/7162] [batch_t 0.212 (0.225)] +2024-04-09 18:59:39,012 - Test: 67.72% [4850/7162] [batch_t 0.213 (0.225)] +2024-04-09 18:59:49,639 - Test: 68.42% [4900/7162] [batch_t 0.211 (0.225)] +2024-04-09 19:00:00,222 - Test: 69.11% [4950/7162] [batch_t 0.212 (0.225)] +2024-04-09 19:00:12,077 - Test: 69.81% [5000/7162] [batch_t 0.210 (0.225)] +2024-04-09 19:00:22,620 - Test: 70.51% [5050/7162] [batch_t 0.210 (0.224)] +2024-04-09 19:00:33,154 - Test: 71.21% [5100/7162] [batch_t 0.213 (0.224)] +2024-04-09 19:00:43,721 - Test: 71.91% [5150/7162] [batch_t 0.210 (0.224)] +2024-04-09 19:00:54,276 - Test: 72.61% [5200/7162] [batch_t 0.211 (0.224)] +2024-04-09 19:01:06,288 - Test: 73.30% [5250/7162] [batch_t 0.211 (0.224)] +2024-04-09 19:01:16,888 - Test: 74.00% [5300/7162] [batch_t 0.211 (0.224)] +2024-04-09 19:01:27,441 - Test: 74.70% [5350/7162] [batch_t 0.210 (0.224)] +2024-04-09 19:01:38,019 - Test: 75.40% [5400/7162] [batch_t 0.211 (0.224)] +2024-04-09 19:01:48,596 - Test: 76.10% [5450/7162] [batch_t 0.210 (0.224)] +2024-04-09 19:01:59,200 - Test: 76.79% [5500/7162] [batch_t 0.210 (0.224)] +2024-04-09 19:02:10,841 - Test: 77.49% [5550/7162] [batch_t 0.211 (0.224)] +2024-04-09 19:02:21,400 - Test: 78.19% [5600/7162] [batch_t 0.213 (0.224)] +2024-04-09 19:02:31,945 - Test: 78.89% [5650/7162] [batch_t 0.210 (0.224)] +2024-04-09 19:02:42,493 - Test: 79.59% [5700/7162] [batch_t 0.213 (0.223)] +2024-04-09 19:02:53,053 - Test: 80.28% [5750/7162] [batch_t 0.210 (0.223)] +2024-04-09 19:03:04,285 - Test: 80.98% [5800/7162] [batch_t 0.216 (0.223)] +2024-04-09 19:03:15,694 - Test: 81.68% [5850/7162] [batch_t 0.210 (0.223)] +2024-04-09 19:03:26,568 - Test: 82.38% [5900/7162] [batch_t 0.210 (0.223)] +2024-04-09 19:03:37,255 - Test: 83.08% [5950/7162] [batch_t 0.213 (0.223)] +2024-04-09 19:03:47,845 - Test: 83.78% [6000/7162] [batch_t 0.213 (0.223)] +2024-04-09 19:03:58,429 - Test: 84.47% [6050/7162] [batch_t 0.211 (0.223)] +2024-04-09 19:04:09,939 - Test: 85.17% [6100/7162] [batch_t 0.210 (0.223)] +2024-04-09 19:04:20,475 - Test: 85.87% [6150/7162] [batch_t 0.210 (0.223)] +2024-04-09 19:04:31,247 - Test: 86.57% [6200/7162] [batch_t 0.221 (0.223)] +2024-04-09 19:04:41,907 - Test: 87.27% [6250/7162] [batch_t 0.212 (0.223)] +2024-04-09 19:04:52,559 - Test: 87.96% [6300/7162] [batch_t 0.211 (0.223)] +2024-04-09 19:05:03,158 - Test: 88.66% [6350/7162] [batch_t 0.210 (0.223)] +2024-04-09 19:05:15,019 - Test: 89.36% [6400/7162] [batch_t 0.221 (0.223)] +2024-04-09 19:05:25,674 - Test: 90.06% [6450/7162] [batch_t 0.211 (0.223)] +2024-04-09 19:05:36,192 - Test: 90.76% [6500/7162] [batch_t 0.210 (0.223)] +2024-04-09 19:05:46,714 - Test: 91.45% [6550/7162] [batch_t 0.209 (0.223)] +2024-04-09 19:05:57,224 - Test: 92.15% [6600/7162] [batch_t 0.210 (0.222)] +2024-04-09 19:06:08,676 - Test: 92.85% [6650/7162] [batch_t 0.214 (0.223)] +2024-04-09 19:06:19,193 - Test: 93.55% [6700/7162] [batch_t 0.209 (0.222)] +2024-04-09 19:06:41,373 - Test: 94.25% [6750/7162] [batch_t 0.216 (0.224)] +2024-04-09 19:06:53,251 - Test: 94.95% [6800/7162] [batch_t 0.211 (0.224)] +2024-04-09 19:07:04,368 - Test: 95.64% [6850/7162] [batch_t 0.311 (0.224)] +2024-04-09 19:07:15,016 - Test: 96.34% [6900/7162] [batch_t 0.210 (0.224)] +2024-04-09 19:07:25,544 - Test: 97.04% [6950/7162] [batch_t 0.210 (0.224)] +2024-04-09 19:07:36,083 - Test: 97.74% [7000/7162] [batch_t 0.212 (0.224)] +2024-04-09 19:07:46,608 - Test: 98.44% [7050/7162] [batch_t 0.211 (0.224)] +2024-04-09 19:07:57,163 - Test: 99.13% [7100/7162] [batch_t 0.209 (0.224)] +2024-04-09 19:08:09,089 - Test: 99.83% [7150/7162] [batch_t 0.212 (0.224)] +2024-04-09 19:08:12,082 - Test: 100.00% [7162/7162] [batch_t 0.682 (0.224)] +2024-04-09 19:29:06,164 - ==> Metric Time for audiojack : 0.004 (mAUROC_sp_max) 0.001 (mAP_sp_max) 0.001 (mF1_max_sp_max) 299.863 (mAUROC_px) 209.622 (mAP_px) 27.090 (mF1_max_px) 548.427 (mAUPRO_px) 9.271 (mF1_px_0.2_0.8_0.1) 9.247 (mAcc_px_0.2_0.8_0.1) 9.306 (mIoU_px_0.2_0.8_0.1) 26.798 (mIoU_max_px) +2024-04-09 19:47:26,101 - ==> Metric Time for pcb : 0.002 (mAUROC_sp_max) 0.001 (mAP_sp_max) 0.001 (mF1_max_sp_max) 275.267 (mAUROC_px) 196.127 (mAP_px) 26.581 (mF1_max_px) 542.456 (mAUPRO_px) 9.109 (mF1_px_0.2_0.8_0.1) 9.048 (mAcc_px_0.2_0.8_0.1) 9.109 (mIoU_px_0.2_0.8_0.1) 26.557 (mIoU_max_px) +2024-04-09 20:07:08,030 - ==> Metric Time for phone_battery : 0.002 (mAUROC_sp_max) 0.001 (mAP_sp_max) 0.001 (mF1_max_sp_max) 315.442 (mAUROC_px) 229.811 (mAP_px) 25.236 (mF1_max_px) 549.659 (mAUPRO_px) 9.004 (mF1_px_0.2_0.8_0.1) 9.667 (mAcc_px_0.2_0.8_0.1) 9.341 (mIoU_px_0.2_0.8_0.1) 28.557 (mIoU_max_px) +2024-04-09 20:26:55,377 - ==> Metric Time for sim_card_set : 0.002 (mAUROC_sp_max) 0.001 (mAP_sp_max) 0.001 (mF1_max_sp_max) 312.145 (mAUROC_px) 216.084 (mAP_px) 25.809 (mF1_max_px) 570.472 (mAUPRO_px) 9.506 (mF1_px_0.2_0.8_0.1) 9.596 (mAcc_px_0.2_0.8_0.1) 9.767 (mIoU_px_0.2_0.8_0.1) 28.097 (mIoU_max_px) +2024-04-09 20:46:23,947 - ==> Metric Time for switch : 0.002 (mAUROC_sp_max) 0.001 (mAP_sp_max) 0.001 (mF1_max_sp_max) 262.637 (mAUROC_px) 222.251 (mAP_px) 27.400 (mF1_max_px) 593.278 (mAUPRO_px) 9.363 (mF1_px_0.2_0.8_0.1) 9.297 (mAcc_px_0.2_0.8_0.1) 9.782 (mIoU_px_0.2_0.8_0.1) 29.097 (mIoU_max_px) +2024-04-09 21:06:12,508 - ==> Metric Time for terminalblock : 0.002 (mAUROC_sp_max) 0.001 (mAP_sp_max) 0.001 (mF1_max_sp_max) 293.065 (mAUROC_px) 222.078 (mAP_px) 25.428 (mF1_max_px) 586.396 (mAUPRO_px) 9.288 (mF1_px_0.2_0.8_0.1) 8.961 (mAcc_px_0.2_0.8_0.1) 8.753 (mIoU_px_0.2_0.8_0.1) 27.219 (mIoU_max_px) +2024-04-09 21:24:42,465 - ==> Metric Time for toothbrush : 0.002 (mAUROC_sp_max) 0.001 (mAP_sp_max) 0.001 (mF1_max_sp_max) 260.475 (mAUROC_px) 198.294 (mAP_px) 25.698 (mF1_max_px) 567.325 (mAUPRO_px) 8.677 (mF1_px_0.2_0.8_0.1) 8.896 (mAcc_px_0.2_0.8_0.1) 8.783 (mIoU_px_0.2_0.8_0.1) 25.546 (mIoU_max_px) +2024-04-09 21:43:54,757 - ==> Metric Time for toy : 0.002 (mAUROC_sp_max) 0.001 (mAP_sp_max) 0.001 (mF1_max_sp_max) 281.291 (mAUROC_px) 231.226 (mAP_px) 26.960 (mF1_max_px) 552.506 (mAUPRO_px) 9.429 (mF1_px_0.2_0.8_0.1) 9.226 (mAcc_px_0.2_0.8_0.1) 9.278 (mIoU_px_0.2_0.8_0.1) 26.683 (mIoU_max_px) +2024-04-09 22:03:35,233 - ==> Metric Time for transistor1 : 0.002 (mAUROC_sp_max) 0.001 (mAP_sp_max) 0.001 (mF1_max_sp_max) 306.638 (mAUROC_px) 212.513 (mAP_px) 26.894 (mF1_max_px) 569.519 (mAUPRO_px) 9.292 (mF1_px_0.2_0.8_0.1) 9.698 (mAcc_px_0.2_0.8_0.1) 10.032 (mIoU_px_0.2_0.8_0.1) 29.823 (mIoU_max_px) +2024-04-09 22:23:29,042 - ==> Metric Time for usb : 0.002 (mAUROC_sp_max) 0.001 (mAP_sp_max) 0.001 (mF1_max_sp_max) 311.977 (mAUROC_px) 215.354 (mAP_px) 27.286 (mF1_max_px) 577.451 (mAUPRO_px) 9.648 (mF1_px_0.2_0.8_0.1) 9.773 (mAcc_px_0.2_0.8_0.1) 9.603 (mIoU_px_0.2_0.8_0.1) 26.787 (mIoU_max_px) +2024-04-09 22:43:25,355 - ==> Metric Time for usb_adaptor : 0.002 (mAUROC_sp_max) 0.001 (mAP_sp_max) 0.001 (mF1_max_sp_max) 300.271 (mAUROC_px) 223.618 (mAP_px) 25.190 (mF1_max_px) 587.235 (mAUPRO_px) 8.701 (mF1_px_0.2_0.8_0.1) 8.936 (mAcc_px_0.2_0.8_0.1) 9.189 (mIoU_px_0.2_0.8_0.1) 27.678 (mIoU_max_px) +2024-04-09 23:02:49,627 - ==> Metric Time for zipper : 0.002 (mAUROC_sp_max) 0.001 (mAP_sp_max) 0.001 (mF1_max_sp_max) 307.323 (mAUROC_px) 203.262 (mAP_px) 25.275 (mF1_max_px) 570.815 (mAUPRO_px) 8.494 (mF1_px_0.2_0.8_0.1) 8.529 (mAcc_px_0.2_0.8_0.1) 8.463 (mIoU_px_0.2_0.8_0.1) 26.712 (mIoU_max_px) +2024-04-09 23:20:58,285 - ==> Metric Time for bottle_cap : 0.002 (mAUROC_sp_max) 0.001 (mAP_sp_max) 0.001 (mF1_max_sp_max) 289.738 (mAUROC_px) 197.984 (mAP_px) 24.325 (mF1_max_px) 520.905 (mAUPRO_px) 8.496 (mF1_px_0.2_0.8_0.1) 8.432 (mAcc_px_0.2_0.8_0.1) 8.518 (mIoU_px_0.2_0.8_0.1) 24.867 (mIoU_max_px) +2024-04-09 23:39:10,532 - ==> Metric Time for end_cap : 0.002 (mAUROC_sp_max) 0.001 (mAP_sp_max) 0.001 (mF1_max_sp_max) 268.799 (mAUROC_px) 203.403 (mAP_px) 25.098 (mF1_max_px) 535.058 (mAUPRO_px) 8.658 (mF1_px_0.2_0.8_0.1) 9.248 (mAcc_px_0.2_0.8_0.1) 9.040 (mIoU_px_0.2_0.8_0.1) 27.535 (mIoU_max_px) +2024-04-09 23:57:47,339 - ==> Metric Time for fire_hood : 0.003 (mAUROC_sp_max) 0.001 (mAP_sp_max) 0.001 (mF1_max_sp_max) 268.318 (mAUROC_px) 208.897 (mAP_px) 25.625 (mF1_max_px) 555.638 (mAUPRO_px) 9.271 (mF1_px_0.2_0.8_0.1) 8.886 (mAcc_px_0.2_0.8_0.1) 8.740 (mIoU_px_0.2_0.8_0.1) 25.222 (mIoU_max_px) +2024-04-10 00:16:06,854 - ==> Metric Time for mounts : 0.002 (mAUROC_sp_max) 0.001 (mAP_sp_max) 0.001 (mF1_max_sp_max) 275.097 (mAUROC_px) 213.950 (mAP_px) 26.631 (mF1_max_px) 524.491 (mAUPRO_px) 9.046 (mF1_px_0.2_0.8_0.1) 9.023 (mAcc_px_0.2_0.8_0.1) 9.356 (mIoU_px_0.2_0.8_0.1) 26.328 (mIoU_max_px) +2024-04-10 00:35:13,692 - ==> Metric Time for plastic_nut : 0.002 (mAUROC_sp_max) 0.001 (mAP_sp_max) 0.001 (mF1_max_sp_max) 301.554 (mAUROC_px) 210.339 (mAP_px) 26.254 (mF1_max_px) 548.675 (mAUPRO_px) 9.070 (mF1_px_0.2_0.8_0.1) 9.154 (mAcc_px_0.2_0.8_0.1) 9.158 (mIoU_px_0.2_0.8_0.1) 27.139 (mIoU_max_px) +2024-04-10 00:52:53,984 - ==> Metric Time for plastic_plug : 0.002 (mAUROC_sp_max) 0.001 (mAP_sp_max) 0.001 (mF1_max_sp_max) 251.580 (mAUROC_px) 198.679 (mAP_px) 24.717 (mF1_max_px) 525.124 (mAUPRO_px) 9.360 (mF1_px_0.2_0.8_0.1) 9.467 (mAcc_px_0.2_0.8_0.1) 9.243 (mIoU_px_0.2_0.8_0.1) 26.710 (mIoU_max_px) +2024-04-10 01:11:19,189 - ==> Metric Time for regulator : 0.002 (mAUROC_sp_max) 0.001 (mAP_sp_max) 0.001 (mF1_max_sp_max) 280.048 (mAUROC_px) 209.254 (mAP_px) 25.332 (mF1_max_px) 533.579 (mAUPRO_px) 9.016 (mF1_px_0.2_0.8_0.1) 8.610 (mAcc_px_0.2_0.8_0.1) 8.641 (mIoU_px_0.2_0.8_0.1) 25.128 (mIoU_max_px) +2024-04-10 01:29:55,118 - ==> Metric Time for rolled_strip_base: 0.002 (mAUROC_sp_max) 0.001 (mAP_sp_max) 0.001 (mF1_max_sp_max) 278.639 (mAUROC_px) 205.452 (mAP_px) 25.758 (mF1_max_px) 549.833 (mAUPRO_px) 8.586 (mF1_px_0.2_0.8_0.1) 8.659 (mAcc_px_0.2_0.8_0.1) 8.531 (mIoU_px_0.2_0.8_0.1) 25.040 (mIoU_max_px) +2024-04-10 01:48:21,074 - ==> Metric Time for toy_brick : 0.002 (mAUROC_sp_max) 0.001 (mAP_sp_max) 0.001 (mF1_max_sp_max) 279.495 (mAUROC_px) 212.553 (mAP_px) 24.873 (mF1_max_px) 532.930 (mAUPRO_px) 8.486 (mF1_px_0.2_0.8_0.1) 8.548 (mAcc_px_0.2_0.8_0.1) 8.499 (mIoU_px_0.2_0.8_0.1) 25.117 (mIoU_max_px) +2024-04-10 02:06:16,340 - ==> Metric Time for u_block : 0.002 (mAUROC_sp_max) 0.001 (mAP_sp_max) 0.001 (mF1_max_sp_max) 269.504 (mAUROC_px) 199.605 (mAP_px) 24.539 (mF1_max_px) 525.968 (mAUPRO_px) 8.448 (mF1_px_0.2_0.8_0.1) 8.519 (mAcc_px_0.2_0.8_0.1) 8.470 (mIoU_px_0.2_0.8_0.1) 24.792 (mIoU_max_px) +2024-04-10 02:23:41,338 - ==> Metric Time for vcpill : 0.002 (mAUROC_sp_max) 0.001 (mAP_sp_max) 0.001 (mF1_max_sp_max) 268.346 (mAUROC_px) 192.698 (mAP_px) 24.436 (mF1_max_px) 503.856 (mAUPRO_px) 8.436 (mF1_px_0.2_0.8_0.1) 8.473 (mAcc_px_0.2_0.8_0.1) 8.466 (mIoU_px_0.2_0.8_0.1) 24.673 (mIoU_max_px) +2024-04-10 02:42:46,213 - ==> Metric Time for wooden_beads : 0.002 (mAUROC_sp_max) 0.001 (mAP_sp_max) 0.001 (mF1_max_sp_max) 280.369 (mAUROC_px) 205.888 (mAP_px) 25.129 (mF1_max_px) 569.512 (mAUPRO_px) 9.775 (mF1_px_0.2_0.8_0.1) 9.983 (mAcc_px_0.2_0.8_0.1) 9.830 (mIoU_px_0.2_0.8_0.1) 28.690 (mIoU_max_px) +2024-04-10 03:00:22,696 - ==> Metric Time for woodstick : 0.002 (mAUROC_sp_max) 0.001 (mAP_sp_max) 0.001 (mF1_max_sp_max) 250.323 (mAUROC_px) 216.037 (mAP_px) 26.415 (mF1_max_px) 508.185 (mAUPRO_px) 8.713 (mF1_px_0.2_0.8_0.1) 8.494 (mAcc_px_0.2_0.8_0.1) 8.252 (mIoU_px_0.2_0.8_0.1) 24.724 (mIoU_max_px) +2024-04-10 03:18:04,487 - ==> Metric Time for tape : 0.002 (mAUROC_sp_max) 0.001 (mAP_sp_max) 0.001 (mF1_max_sp_max) 257.286 (mAUROC_px) 202.603 (mAP_px) 24.914 (mF1_max_px) 516.680 (mAUPRO_px) 9.292 (mF1_px_0.2_0.8_0.1) 9.369 (mAcc_px_0.2_0.8_0.1) 9.276 (mIoU_px_0.2_0.8_0.1) 27.079 (mIoU_max_px) +2024-04-10 03:37:08,384 - ==> Metric Time for porcelain_doll : 0.002 (mAUROC_sp_max) 0.001 (mAP_sp_max) 0.001 (mF1_max_sp_max) 299.886 (mAUROC_px) 201.525 (mAP_px) 27.409 (mF1_max_px) 558.612 (mAUPRO_px) 8.731 (mF1_px_0.2_0.8_0.1) 8.572 (mAcc_px_0.2_0.8_0.1) 8.568 (mIoU_px_0.2_0.8_0.1) 25.037 (mIoU_max_px) +2024-04-10 04:00:03,925 - ==> Metric Time for mint : 0.002 (mAUROC_sp_max) 0.001 (mAP_sp_max) 0.001 (mF1_max_sp_max) 362.578 (mAUROC_px) 253.885 (mAP_px) 29.280 (mF1_max_px) 664.349 (mAUPRO_px) 9.991 (mF1_px_0.2_0.8_0.1) 9.971 (mAcc_px_0.2_0.8_0.1) 9.983 (mIoU_px_0.2_0.8_0.1) 29.284 (mIoU_max_px) +2024-04-10 04:18:44,558 - ==> Metric Time for eraser : 0.002 (mAUROC_sp_max) 0.001 (mAP_sp_max) 0.001 (mF1_max_sp_max) 278.252 (mAUROC_px) 207.721 (mAP_px) 25.055 (mF1_max_px) 552.268 (mAUPRO_px) 8.586 (mF1_px_0.2_0.8_0.1) 8.666 (mAcc_px_0.2_0.8_0.1) 9.455 (mIoU_px_0.2_0.8_0.1) 24.962 (mIoU_max_px) +2024-04-10 04:37:25,734 - ==> Metric Time for button_battery : 0.002 (mAUROC_sp_max) 0.001 (mAP_sp_max) 0.001 (mF1_max_sp_max) 279.904 (mAUROC_px) 200.752 (mAP_px) 24.696 (mF1_max_px) 560.189 (mAUPRO_px) 8.488 (mF1_px_0.2_0.8_0.1) 8.501 (mAcc_px_0.2_0.8_0.1) 8.456 (mIoU_px_0.2_0.8_0.1) 24.750 (mIoU_max_px) +2024-04-10 04:37:26,164 - +| Name | mAUROC_sp_max | mAUROC_sp_max (Max) | mAP_sp_max | mAP_sp_max (Max) | mF1_max_sp_max | mF1_max_sp_max (Max) | mAUROC_px | mAUROC_px (Max) | mAP_px | mAP_px (Max) | mF1_max_px | mF1_max_px (Max) | mAUPRO_px | mAUPRO_px (Max) | mF1_px_0.2_0.8_0.1 | mF1_px_0.2_0.8_0.1 (Max) | mAcc_px_0.2_0.8_0.1 | mAcc_px_0.2_0.8_0.1 (Max) | mIoU_px_0.2_0.8_0.1 | mIoU_px_0.2_0.8_0.1 (Max) | mIoU_max_px | mIoU_max_px (Max) | +|:-----------------:|:---------------:|:---------------------:|:------------:|:------------------:|:----------------:|:----------------------:|:-----------:|:------------------:|:--------:|:------------------:|:------------:|:------------------:|:-----------:|:------------------:|:--------------------:|:--------------------------:|:---------------------:|:---------------------------:|:---------------------:|:---------------------------:|:-------------:|:-------------------:| +| audiojack | 84.178 | 84.178 (50 epoch) | 76.463 | 76.463 (50 epoch) | 67.373 | 67.373 (50 epoch) | 97.732 | 97.732 (50 epoch) | 21.611 | 21.611 (50 epoch) | 29.491 | 29.491 (50 epoch) | 83.892 | 83.892 (50 epoch) | 12.571 | 12.571 (50 epoch) | 58.233 | 58.233 (50 epoch) | 7.041 | 7.041 (50 epoch) | 17.296 | 17.296 (50 epoch) | +| pcb | 89.093 | 89.093 (50 epoch) | 93.721 | 93.721 (50 epoch) | 84.007 | 84.007 (50 epoch) | 99.177 | 99.177 (50 epoch) | 46.315 | 46.315 (50 epoch) | 50.405 | 50.405 (50 epoch) | 93.073 | 93.073 (50 epoch) | 20.510 | 20.510 (50 epoch) | 46.961 | 46.961 (50 epoch) | 12.628 | 12.628 (50 epoch) | 33.694 | 33.694 (50 epoch) | +| phone_battery | 90.245 | 90.245 (50 epoch) | 88.884 | 88.884 (50 epoch) | 80.476 | 80.476 (50 epoch) | 99.402 | 99.402 (50 epoch) | 36.261 | 36.261 (50 epoch) | 41.315 | 41.315 (50 epoch) | 95.265 | 95.265 (50 epoch) | 20.705 | 20.705 (50 epoch) | 46.420 | 46.420 (50 epoch) | 12.243 | 12.243 (50 epoch) | 26.036 | 26.036 (50 epoch) | +| sim_card_set | 94.355 | 94.355 (50 epoch) | 95.085 | 95.085 (50 epoch) | 87.197 | 87.197 (50 epoch) | 98.793 | 98.793 (50 epoch) | 51.147 | 51.147 (50 epoch) | 50.554 | 50.554 (50 epoch) | 89.444 | 89.444 (50 epoch) | 26.200 | 26.200 (50 epoch) | 57.129 | 57.129 (50 epoch) | 16.267 | 16.267 (50 epoch) | 33.827 | 33.827 (50 epoch) | +| switch | 91.711 | 91.711 (50 epoch) | 94.008 | 94.008 (50 epoch) | 85.425 | 85.425 (50 epoch) | 98.183 | 98.183 (50 epoch) | 39.881 | 39.881 (50 epoch) | 45.408 | 45.408 (50 epoch) | 92.940 | 92.940 (50 epoch) | 22.674 | 22.674 (50 epoch) | 42.666 | 42.666 (50 epoch) | 13.751 | 13.751 (50 epoch) | 29.372 | 29.372 (50 epoch) | +| terminalblock | 96.061 | 96.061 (50 epoch) | 96.829 | 96.829 (50 epoch) | 90.023 | 90.023 (50 epoch) | 99.760 | 99.760 (50 epoch) | 35.285 | 35.285 (50 epoch) | 39.651 | 39.651 (50 epoch) | 98.211 | 98.211 (50 epoch) | 18.440 | 18.440 (50 epoch) | 37.131 | 37.131 (50 epoch) | 10.847 | 10.847 (50 epoch) | 24.728 | 24.728 (50 epoch) | +| toothbrush | 85.135 | 85.135 (50 epoch) | 86.230 | 86.230 (50 epoch) | 80.290 | 80.290 (50 epoch) | 97.480 | 97.480 (50 epoch) | 27.759 | 27.759 (50 epoch) | 36.739 | 36.739 (50 epoch) | 91.354 | 91.354 (50 epoch) | 17.485 | 17.485 (50 epoch) | 43.270 | 43.270 (50 epoch) | 10.124 | 10.124 (50 epoch) | 22.504 | 22.504 (50 epoch) | +| toy | 82.976 | 82.976 (50 epoch) | 87.535 | 87.535 (50 epoch) | 79.567 | 79.567 (50 epoch) | 95.996 | 95.996 (50 epoch) | 16.410 | 16.410 (50 epoch) | 25.771 | 25.771 (50 epoch) | 86.300 | 86.300 (50 epoch) | 11.342 | 11.342 (50 epoch) | 36.116 | 36.116 (50 epoch) | 6.247 | 6.247 (50 epoch) | 14.792 | 14.792 (50 epoch) | +| transistor1 | 94.361 | 94.361 (50 epoch) | 95.980 | 95.980 (50 epoch) | 89.048 | 89.048 (50 epoch) | 99.373 | 99.373 (50 epoch) | 39.417 | 39.417 (50 epoch) | 39.992 | 39.992 (50 epoch) | 96.516 | 96.516 (50 epoch) | 21.009 | 21.009 (50 epoch) | 47.389 | 47.389 (50 epoch) | 12.420 | 12.420 (50 epoch) | 24.993 | 24.993 (50 epoch) | +| usb | 91.953 | 91.953 (50 epoch) | 92.186 | 92.186 (50 epoch) | 84.544 | 84.544 (50 epoch) | 99.217 | 99.217 (50 epoch) | 39.065 | 39.065 (50 epoch) | 44.388 | 44.388 (50 epoch) | 95.198 | 95.198 (50 epoch) | 23.737 | 23.737 (50 epoch) | 56.911 | 56.911 (50 epoch) | 14.332 | 14.332 (50 epoch) | 28.525 | 28.525 (50 epoch) | +| usb_adaptor | 79.370 | 79.370 (50 epoch) | 76.003 | 76.003 (50 epoch) | 66.293 | 66.293 (50 epoch) | 97.304 | 97.304 (50 epoch) | 15.285 | 15.285 (50 epoch) | 22.633 | 22.633 (50 epoch) | 82.490 | 82.490 (50 epoch) | 12.200 | 12.200 (50 epoch) | 43.052 | 43.052 (50 epoch) | 6.699 | 6.699 (50 epoch) | 12.760 | 12.760 (50 epoch) | +| zipper | 99.225 | 99.225 (50 epoch) | 99.571 | 99.571 (50 epoch) | 96.856 | 96.856 (50 epoch) | 99.280 | 99.280 (50 epoch) | 58.244 | 58.244 (50 epoch) | 61.337 | 61.337 (50 epoch) | 97.574 | 97.574 (50 epoch) | 29.868 | 29.868 (50 epoch) | 52.417 | 52.417 (50 epoch) | 19.340 | 19.340 (50 epoch) | 44.235 | 44.235 (50 epoch) | +| bottle_cap | 92.825 | 92.825 (50 epoch) | 91.978 | 91.978 (50 epoch) | 82.118 | 82.118 (50 epoch) | 99.681 | 99.681 (50 epoch) | 30.638 | 30.638 (50 epoch) | 34.606 | 34.606 (50 epoch) | 97.199 | 97.199 (50 epoch) | 20.535 | 20.535 (50 epoch) | 46.698 | 46.698 (50 epoch) | 11.866 | 11.866 (50 epoch) | 20.924 | 20.924 (50 epoch) | +| end_cap | 77.967 | 77.967 (50 epoch) | 82.827 | 82.827 (50 epoch) | 77.201 | 77.201 (50 epoch) | 96.977 | 96.977 (50 epoch) | 11.961 | 11.961 (50 epoch) | 19.647 | 19.647 (50 epoch) | 89.371 | 89.371 (50 epoch) | 9.056 | 9.056 (50 epoch) | 37.864 | 37.864 (50 epoch) | 4.875 | 4.875 (50 epoch) | 10.894 | 10.894 (50 epoch) | +| fire_hood | 79.329 | 79.329 (50 epoch) | 72.516 | 72.516 (50 epoch) | 64.756 | 64.756 (50 epoch) | 98.713 | 98.713 (50 epoch) | 25.055 | 25.055 (50 epoch) | 31.289 | 31.289 (50 epoch) | 86.343 | 86.343 (50 epoch) | 13.044 | 13.044 (50 epoch) | 52.618 | 52.618 (50 epoch) | 7.351 | 7.351 (50 epoch) | 18.546 | 18.546 (50 epoch) | +| mounts | 86.752 | 86.752 (50 epoch) | 77.964 | 77.964 (50 epoch) | 73.460 | 73.460 (50 epoch) | 99.212 | 99.212 (50 epoch) | 31.357 | 31.357 (50 epoch) | 35.381 | 35.381 (50 epoch) | 93.499 | 93.499 (50 epoch) | 19.381 | 19.381 (50 epoch) | 40.995 | 40.995 (50 epoch) | 11.179 | 11.179 (50 epoch) | 21.492 | 21.492 (50 epoch) | +| plastic_nut | 87.055 | 87.055 (50 epoch) | 80.708 | 80.708 (50 epoch) | 70.749 | 70.749 (50 epoch) | 99.415 | 99.415 (50 epoch) | 33.106 | 33.106 (50 epoch) | 37.321 | 37.321 (50 epoch) | 96.128 | 96.128 (50 epoch) | 18.548 | 18.548 (50 epoch) | 43.271 | 43.271 (50 epoch) | 10.788 | 10.788 (50 epoch) | 22.941 | 22.941 (50 epoch) | +| plastic_plug | 85.671 | 85.671 (50 epoch) | 82.243 | 82.243 (50 epoch) | 72.625 | 72.625 (50 epoch) | 98.984 | 98.984 (50 epoch) | 24.241 | 24.241 (50 epoch) | 31.666 | 31.666 (50 epoch) | 91.501 | 91.501 (50 epoch) | 13.935 | 13.935 (50 epoch) | 38.271 | 38.271 (50 epoch) | 7.873 | 7.873 (50 epoch) | 18.811 | 18.811 (50 epoch) | +| regulator | 69.723 | 69.723 (50 epoch) | 58.747 | 58.747 (50 epoch) | 50.433 | 50.433 (50 epoch) | 97.560 | 97.560 (50 epoch) | 20.636 | 20.636 (50 epoch) | 29.796 | 29.796 (50 epoch) | 87.032 | 87.032 (50 epoch) | 13.654 | 13.654 (50 epoch) | 40.399 | 40.399 (50 epoch) | 7.684 | 7.684 (50 epoch) | 17.506 | 17.506 (50 epoch) | +| rolled_strip_base | 97.982 | 97.982 (50 epoch) | 98.999 | 98.999 (50 epoch) | 94.999 | 94.999 (50 epoch) | 99.743 | 99.743 (50 epoch) | 37.449 | 37.449 (50 epoch) | 42.499 | 42.499 (50 epoch) | 98.775 | 98.775 (50 epoch) | 22.845 | 22.845 (50 epoch) | 50.780 | 50.780 (50 epoch) | 13.589 | 13.589 (50 epoch) | 26.983 | 26.983 (50 epoch) | +| toy_brick | 70.499 | 70.499 (50 epoch) | 63.716 | 63.716 (50 epoch) | 61.611 | 61.611 (50 epoch) | 96.598 | 96.598 (50 epoch) | 18.026 | 18.026 (50 epoch) | 25.847 | 25.847 (50 epoch) | 74.712 | 74.712 (50 epoch) | 9.014 | 9.014 (50 epoch) | 51.032 | 51.032 (50 epoch) | 4.960 | 4.960 (50 epoch) | 14.841 | 14.841 (50 epoch) | +| u_block | 89.729 | 89.729 (50 epoch) | 85.680 | 85.680 (50 epoch) | 75.319 | 75.319 (50 epoch) | 99.542 | 99.542 (50 epoch) | 37.785 | 37.785 (50 epoch) | 46.148 | 46.148 (50 epoch) | 95.390 | 95.390 (50 epoch) | 21.952 | 21.952 (50 epoch) | 46.295 | 46.295 (50 epoch) | 13.166 | 13.166 (50 epoch) | 29.995 | 29.995 (50 epoch) | +| vcpill | 88.273 | 88.273 (50 epoch) | 87.718 | 87.718 (50 epoch) | 77.379 | 77.379 (50 epoch) | 98.702 | 98.702 (50 epoch) | 50.185 | 50.185 (50 epoch) | 54.531 | 54.531 (50 epoch) | 89.311 | 89.311 (50 epoch) | 22.552 | 22.552 (50 epoch) | 46.655 | 46.655 (50 epoch) | 14.052 | 14.052 (50 epoch) | 37.487 | 37.487 (50 epoch) | +| wooden_beads | 82.531 | 82.531 (50 epoch) | 81.737 | 81.737 (50 epoch) | 71.833 | 71.833 (50 epoch) | 98.044 | 98.044 (50 epoch) | 32.578 | 32.578 (50 epoch) | 39.787 | 39.787 (50 epoch) | 84.503 | 84.503 (50 epoch) | 17.098 | 17.098 (50 epoch) | 48.094 | 48.094 (50 epoch) | 9.975 | 9.975 (50 epoch) | 24.834 | 24.834 (50 epoch) | +| woodstick | 80.408 | 80.408 (50 epoch) | 69.004 | 69.004 (50 epoch) | 63.383 | 63.383 (50 epoch) | 97.695 | 97.695 (50 epoch) | 40.107 | 40.107 (50 epoch) | 44.914 | 44.914 (50 epoch) | 82.733 | 82.733 (50 epoch) | 21.050 | 21.050 (50 epoch) | 61.122 | 61.122 (50 epoch) | 12.712 | 12.712 (50 epoch) | 28.960 | 28.960 (50 epoch) | +| tape | 96.800 | 96.800 (50 epoch) | 95.859 | 95.859 (50 epoch) | 89.278 | 89.278 (50 epoch) | 99.759 | 99.759 (50 epoch) | 47.054 | 47.054 (50 epoch) | 48.156 | 48.156 (50 epoch) | 97.952 | 97.952 (50 epoch) | 26.690 | 26.690 (50 epoch) | 50.682 | 50.682 (50 epoch) | 16.354 | 16.354 (50 epoch) | 31.714 | 31.714 (50 epoch) | +| porcelain_doll | 87.982 | 87.982 (50 epoch) | 82.225 | 82.225 (50 epoch) | 74.060 | 74.060 (50 epoch) | 99.187 | 99.187 (50 epoch) | 31.320 | 31.320 (50 epoch) | 36.605 | 36.605 (50 epoch) | 95.381 | 95.381 (50 epoch) | 19.626 | 19.626 (50 epoch) | 38.073 | 38.073 (50 epoch) | 11.397 | 11.397 (50 epoch) | 22.403 | 22.403 (50 epoch) | +| mint | 70.128 | 70.128 (50 epoch) | 70.806 | 70.806 (50 epoch) | 65.520 | 65.520 (50 epoch) | 96.452 | 96.452 (50 epoch) | 15.941 | 15.941 (50 epoch) | 27.044 | 27.044 (50 epoch) | 72.586 | 72.586 (50 epoch) | 12.362 | 12.362 (50 epoch) | 39.475 | 39.475 (50 epoch) | 6.872 | 6.872 (50 epoch) | 15.637 | 15.637 (50 epoch) | +| eraser | 87.528 | 87.528 (50 epoch) | 86.221 | 86.221 (50 epoch) | 76.146 | 76.146 (50 epoch) | 99.235 | 99.235 (50 epoch) | 30.238 | 30.238 (50 epoch) | 38.344 | 38.344 (50 epoch) | 93.734 | 93.734 (50 epoch) | 20.646 | 20.646 (50 epoch) | 57.530 | 57.530 (50 epoch) | 12.081 | 12.081 (50 epoch) | 23.719 | 23.719 (50 epoch) | +| button_battery | 79.832 | 79.832 (50 epoch) | 85.262 | 85.262 (50 epoch) | 77.757 | 77.757 (50 epoch) | 98.093 | 98.093 (50 epoch) | 46.712 | 46.712 (50 epoch) | 49.495 | 49.495 (50 epoch) | 86.186 | 86.186 (50 epoch) | 23.143 | 23.143 (50 epoch) | 44.337 | 44.337 (50 epoch) | 14.140 | 14.140 (50 epoch) | 32.886 | 32.886 (50 epoch) | +| Avg | 86.323 | 86.323 (50 epoch) | 84.557 | 84.557 (50 epoch) | 76.991 | 76.991 (50 epoch) | 98.510 | 98.510 (50 epoch) | 33.036 | 33.036 (50 epoch) | 38.692 | 38.692 (50 epoch) | 90.487 | 90.487 (50 epoch) | 18.729 | 18.729 (50 epoch) | 46.730 | 46.730 (50 epoch) | 11.095 | 11.095 (50 epoch) | 24.445 | 24.445 (50 epoch) | +2024-04-10 04:37:33,215 - ==> Total time: 1 day, 5:08:53 Eta: 23 days, 1:48:53 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-10 04:38:20,919 - Train: 5.00% [114000/2279000] [50.0/1000.0] [batch_t 0.605 (0.640)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 04:39:21,377 - Train: 5.01% [114100/2279000] [50.1/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 04:40:21,739 - Train: 5.01% [114200/2279000] [50.1/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 04:41:22,193 - Train: 5.02% [114300/2279000] [50.2/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 04:42:22,561 - Train: 5.02% [114400/2279000] [50.2/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 04:43:22,875 - Train: 5.02% [114500/2279000] [50.2/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 04:44:23,203 - Train: 5.03% [114600/2279000] [50.3/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 04:45:23,553 - Train: 5.03% [114700/2279000] [50.3/1000.0] [batch_t 0.601 (0.603)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 04:46:23,906 - Train: 5.04% [114800/2279000] [50.4/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 04:47:24,270 - Train: 5.04% [114900/2279000] [50.4/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 04:48:24,691 - Train: 5.05% [115000/2279000] [50.5/1000.0] [batch_t 0.600 (0.604)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-10 04:49:25,054 - Train: 5.05% [115100/2279000] [50.5/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 04:50:25,398 - Train: 5.05% [115200/2279000] [50.5/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 04:51:25,688 - Train: 5.06% [115300/2279000] [50.6/1000.0] [batch_t 0.601 (0.603)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 04:52:26,026 - Train: 5.06% [115400/2279000] [50.6/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 04:53:26,406 - Train: 5.07% [115500/2279000] [50.7/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 04:54:26,737 - Train: 5.07% [115600/2279000] [50.7/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 04:55:27,058 - Train: 5.08% [115700/2279000] [50.8/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 04:56:27,452 - Train: 5.08% [115800/2279000] [50.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 04:57:27,751 - Train: 5.09% [115900/2279000] [50.9/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 04:58:28,079 - Train: 5.09% [116000/2279000] [50.9/1000.0] [batch_t 0.607 (0.603)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 04:59:28,394 - Train: 5.09% [116100/2279000] [50.9/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 05:00:28,728 - Train: 5.10% [116200/2279000] [51.0/1000.0] [batch_t 0.601 (0.603)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 05:00:46,251 - ==> Total time: 1 day, 5:32:06 Eta: 22 days, 21:35:04 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-10 05:01:44,606 - Train: 5.10% [116300/2279000] [51.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 05:02:44,959 - Train: 5.11% [116400/2279000] [51.1/1000.0] [batch_t 0.606 (0.603)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 05:03:45,256 - Train: 5.11% [116500/2279000] [51.1/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 05:04:45,572 - Train: 5.12% [116600/2279000] [51.2/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 05:05:45,848 - Train: 5.12% [116700/2279000] [51.2/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 05:06:46,151 - Train: 5.13% [116800/2279000] [51.3/1000.0] [batch_t 0.601 (0.603)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 05:07:46,456 - Train: 5.13% [116900/2279000] [51.3/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 05:08:46,837 - Train: 5.13% [117000/2279000] [51.3/1000.0] [batch_t 0.609 (0.604)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-10 05:09:47,133 - Train: 5.14% [117100/2279000] [51.4/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 05:10:47,445 - Train: 5.14% [117200/2279000] [51.4/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 05:11:47,776 - Train: 5.15% [117300/2279000] [51.5/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 05:12:48,108 - Train: 5.15% [117400/2279000] [51.5/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 05:13:48,491 - Train: 5.16% [117500/2279000] [51.6/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 05:14:49,063 - Train: 5.16% [117600/2279000] [51.6/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 05:15:49,445 - Train: 5.16% [117700/2279000] [51.6/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 05:16:49,781 - Train: 5.17% [117800/2279000] [51.7/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 05:17:50,160 - Train: 5.17% [117900/2279000] [51.7/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 05:18:50,496 - Train: 5.18% [118000/2279000] [51.8/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 05:19:50,825 - Train: 5.18% [118100/2279000] [51.8/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 05:20:51,288 - Train: 5.19% [118200/2279000] [51.9/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 05:21:51,595 - Train: 5.19% [118300/2279000] [51.9/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 05:22:51,858 - Train: 5.20% [118400/2279000] [52.0/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 05:23:52,160 - Train: 5.20% [118500/2279000] [52.0/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 05:23:57,003 - ==> Total time: 1 day, 5:55:17 Eta: 22 days, 17:29:25 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-10 05:25:07,870 - Train: 5.20% [118600/2279000] [52.0/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 05:26:08,227 - Train: 5.21% [118700/2279000] [52.1/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 05:27:08,703 - Train: 5.21% [118800/2279000] [52.1/1000.0] [batch_t 0.759 (0.605)] [data_t 0.002] [optim_t 0.757] [lr 0.005000] +2024-04-10 05:28:09,027 - Train: 5.22% [118900/2279000] [52.2/1000.0] [batch_t 0.600 (0.603)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-10 05:29:09,358 - Train: 5.22% [119000/2279000] [52.2/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 05:30:09,647 - Train: 5.23% [119100/2279000] [52.3/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 05:31:09,973 - Train: 5.23% [119200/2279000] [52.3/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 05:32:10,294 - Train: 5.23% [119300/2279000] [52.3/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 05:33:10,589 - Train: 5.24% [119400/2279000] [52.4/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 05:34:10,861 - Train: 5.24% [119500/2279000] [52.4/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 05:35:11,267 - Train: 5.25% [119600/2279000] [52.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 05:36:11,616 - Train: 5.25% [119700/2279000] [52.5/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 05:37:11,964 - Train: 5.26% [119800/2279000] [52.6/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 05:38:12,281 - Train: 5.26% [119900/2279000] [52.6/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 05:39:12,568 - Train: 5.27% [120000/2279000] [52.7/1000.0] [batch_t 0.606 (0.603)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 05:40:12,910 - Train: 5.27% [120100/2279000] [52.7/1000.0] [batch_t 0.600 (0.603)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-10 05:41:13,307 - Train: 5.27% [120200/2279000] [52.7/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 05:42:13,633 - Train: 5.28% [120300/2279000] [52.8/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 05:43:13,924 - Train: 5.28% [120400/2279000] [52.8/1000.0] [batch_t 0.600 (0.603)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-10 05:44:14,221 - Train: 5.29% [120500/2279000] [52.9/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 05:45:14,573 - Train: 5.29% [120600/2279000] [52.9/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 05:46:14,998 - Train: 5.30% [120700/2279000] [53.0/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 05:47:07,503 - ==> Total time: 1 day, 6:18:27 Eta: 22 days, 13:32:06 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-10 05:47:32,042 - Train: 5.30% [120800/2279000] [53.0/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 05:48:32,410 - Train: 5.30% [120900/2279000] [53.0/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 05:49:32,717 - Train: 5.31% [121000/2279000] [53.1/1000.0] [batch_t 0.599 (0.603)] [data_t 0.002] [optim_t 0.597] [lr 0.005000] +2024-04-10 05:50:33,041 - Train: 5.31% [121100/2279000] [53.1/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 05:51:33,374 - Train: 5.32% [121200/2279000] [53.2/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 05:52:33,877 - Train: 5.32% [121300/2279000] [53.2/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 05:53:34,203 - Train: 5.33% [121400/2279000] [53.3/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 05:54:34,528 - Train: 5.33% [121500/2279000] [53.3/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 05:55:34,912 - Train: 5.34% [121600/2279000] [53.4/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 05:56:35,252 - Train: 5.34% [121700/2279000] [53.4/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 05:57:35,559 - Train: 5.34% [121800/2279000] [53.4/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 05:58:35,905 - Train: 5.35% [121900/2279000] [53.5/1000.0] [batch_t 0.606 (0.603)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 05:59:36,327 - Train: 5.35% [122000/2279000] [53.5/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 06:00:36,620 - Train: 5.36% [122100/2279000] [53.6/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 06:01:36,911 - Train: 5.36% [122200/2279000] [53.6/1000.0] [batch_t 0.601 (0.603)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 06:02:37,239 - Train: 5.37% [122300/2279000] [53.7/1000.0] [batch_t 0.607 (0.603)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 06:03:37,574 - Train: 5.37% [122400/2279000] [53.7/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 06:04:38,039 - Train: 5.38% [122500/2279000] [53.8/1000.0] [batch_t 0.599 (0.605)] [data_t 0.002] [optim_t 0.597] [lr 0.005000] +2024-04-10 06:05:38,313 - Train: 5.38% [122600/2279000] [53.8/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 06:06:38,664 - Train: 5.38% [122700/2279000] [53.8/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 06:07:38,964 - Train: 5.39% [122800/2279000] [53.9/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 06:08:39,281 - Train: 5.39% [122900/2279000] [53.9/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 06:09:39,583 - Train: 5.40% [123000/2279000] [54.0/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 06:10:19,381 - ==> Total time: 1 day, 6:41:39 Eta: 22 days, 9:43:07 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-10 06:10:57,349 - Train: 5.40% [123100/2279000] [54.0/1000.0] [batch_t 0.601 (0.603)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 06:11:57,717 - Train: 5.41% [123200/2279000] [54.1/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 06:12:58,051 - Train: 5.41% [123300/2279000] [54.1/1000.0] [batch_t 0.606 (0.603)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 06:13:58,395 - Train: 5.41% [123400/2279000] [54.1/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 06:14:58,728 - Train: 5.42% [123500/2279000] [54.2/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 06:15:59,061 - Train: 5.42% [123600/2279000] [54.2/1000.0] [batch_t 0.606 (0.603)] [data_t 0.003] [optim_t 0.603] [lr 0.005000] +2024-04-10 06:16:59,573 - Train: 5.43% [123700/2279000] [54.3/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 06:17:59,924 - Train: 5.43% [123800/2279000] [54.3/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 06:19:00,277 - Train: 5.44% [123900/2279000] [54.4/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 06:20:00,609 - Train: 5.44% [124000/2279000] [54.4/1000.0] [batch_t 0.599 (0.603)] [data_t 0.002] [optim_t 0.597] [lr 0.005000] +2024-04-10 06:21:00,944 - Train: 5.45% [124100/2279000] [54.5/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 06:22:01,241 - Train: 5.45% [124200/2279000] [54.5/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 06:23:01,585 - Train: 5.45% [124300/2279000] [54.5/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 06:24:02,085 - Train: 5.46% [124400/2279000] [54.6/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 06:25:02,392 - Train: 5.46% [124500/2279000] [54.6/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 06:26:02,751 - Train: 5.47% [124600/2279000] [54.7/1000.0] [batch_t 0.597 (0.603)] [data_t 0.002] [optim_t 0.595] [lr 0.005000] +2024-04-10 06:27:03,087 - Train: 5.47% [124700/2279000] [54.7/1000.0] [batch_t 0.608 (0.603)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-10 06:28:03,401 - Train: 5.48% [124800/2279000] [54.8/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 06:29:03,670 - Train: 5.48% [124900/2279000] [54.8/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 06:30:04,226 - Train: 5.48% [125000/2279000] [54.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 06:31:04,575 - Train: 5.49% [125100/2279000] [54.9/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 06:32:04,901 - Train: 5.49% [125200/2279000] [54.9/1000.0] [batch_t 0.608 (0.603)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-10 06:33:05,302 - Train: 5.50% [125300/2279000] [55.0/1000.0] [batch_t 0.601 (0.604)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 06:33:32,450 - ==> Total time: 1 day, 7:04:52 Eta: 22 days, 6:01:57 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-10 06:34:23,113 - Train: 5.50% [125400/2279000] [55.0/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 06:35:23,396 - Train: 5.51% [125500/2279000] [55.1/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 06:36:23,773 - Train: 5.51% [125600/2279000] [55.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 06:37:24,255 - Train: 5.52% [125700/2279000] [55.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 06:38:24,608 - Train: 5.52% [125800/2279000] [55.2/1000.0] [batch_t 0.608 (0.603)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-10 06:39:24,952 - Train: 5.52% [125900/2279000] [55.2/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 06:40:25,311 - Train: 5.53% [126000/2279000] [55.3/1000.0] [batch_t 0.606 (0.603)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 06:41:25,662 - Train: 5.53% [126100/2279000] [55.3/1000.0] [batch_t 0.601 (0.603)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 06:42:26,112 - Train: 5.54% [126200/2279000] [55.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 06:43:26,489 - Train: 5.54% [126300/2279000] [55.4/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 06:44:26,784 - Train: 5.55% [126400/2279000] [55.5/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 06:45:27,131 - Train: 5.55% [126500/2279000] [55.5/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 06:46:27,455 - Train: 5.56% [126600/2279000] [55.6/1000.0] [batch_t 0.606 (0.603)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 06:47:27,808 - Train: 5.56% [126700/2279000] [55.6/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 06:48:28,133 - Train: 5.56% [126800/2279000] [55.6/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 06:49:28,447 - Train: 5.57% [126900/2279000] [55.7/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 06:50:28,859 - Train: 5.57% [127000/2279000] [55.7/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 06:51:29,149 - Train: 5.58% [127100/2279000] [55.8/1000.0] [batch_t 0.600 (0.603)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-10 06:52:29,427 - Train: 5.58% [127200/2279000] [55.8/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 06:53:29,764 - Train: 5.59% [127300/2279000] [55.9/1000.0] [batch_t 0.599 (0.603)] [data_t 0.002] [optim_t 0.597] [lr 0.005000] +2024-04-10 06:54:30,061 - Train: 5.59% [127400/2279000] [55.9/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 06:55:30,480 - Train: 5.59% [127500/2279000] [55.9/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 06:56:30,792 - Train: 5.60% [127600/2279000] [56.0/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 06:56:45,268 - ==> Total time: 1 day, 7:28:05 Eta: 22 days, 2:27:47 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-10 06:57:48,845 - Train: 5.60% [127700/2279000] [56.0/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 06:58:49,130 - Train: 5.61% [127800/2279000] [56.1/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 06:59:49,418 - Train: 5.61% [127900/2279000] [56.1/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 07:00:49,744 - Train: 5.62% [128000/2279000] [56.2/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 07:01:50,209 - Train: 5.62% [128100/2279000] [56.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 07:02:50,488 - Train: 5.63% [128200/2279000] [56.3/1000.0] [batch_t 0.592 (0.603)] [data_t 0.002] [optim_t 0.590] [lr 0.005000] +2024-04-10 07:03:50,793 - Train: 5.63% [128300/2279000] [56.3/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 07:04:51,111 - Train: 5.63% [128400/2279000] [56.3/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 07:05:51,424 - Train: 5.64% [128500/2279000] [56.4/1000.0] [batch_t 0.601 (0.603)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 07:06:51,727 - Train: 5.64% [128600/2279000] [56.4/1000.0] [batch_t 0.609 (0.603)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-10 07:07:52,033 - Train: 5.65% [128700/2279000] [56.5/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 07:08:52,422 - Train: 5.65% [128800/2279000] [56.5/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 07:09:52,730 - Train: 5.66% [128900/2279000] [56.6/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 07:10:53,040 - Train: 5.66% [129000/2279000] [56.6/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 07:11:53,367 - Train: 5.66% [129100/2279000] [56.6/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 07:12:53,653 - Train: 5.67% [129200/2279000] [56.7/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 07:13:53,945 - Train: 5.67% [129300/2279000] [56.7/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 07:14:54,350 - Train: 5.68% [129400/2279000] [56.8/1000.0] [batch_t 0.601 (0.604)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 07:15:54,676 - Train: 5.68% [129500/2279000] [56.8/1000.0] [batch_t 0.601 (0.603)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 07:16:54,959 - Train: 5.69% [129600/2279000] [56.9/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 07:17:55,269 - Train: 5.69% [129700/2279000] [56.9/1000.0] [batch_t 0.601 (0.603)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 07:18:55,579 - Train: 5.70% [129800/2279000] [57.0/1000.0] [batch_t 0.599 (0.603)] [data_t 0.002] [optim_t 0.597] [lr 0.005000] +2024-04-10 07:19:55,919 - Train: 5.70% [129900/2279000] [57.0/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 07:19:57,733 - ==> Total time: 1 day, 7:51:17 Eta: 21 days, 23:00:14 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-10 07:21:12,179 - Train: 5.70% [130000/2279000] [57.0/1000.0] [batch_t 0.610 (0.605)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-10 07:22:12,532 - Train: 5.71% [130100/2279000] [57.1/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 07:23:12,839 - Train: 5.71% [130200/2279000] [57.1/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 07:24:13,216 - Train: 5.72% [130300/2279000] [57.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 07:25:13,536 - Train: 5.72% [130400/2279000] [57.2/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 07:26:14,096 - Train: 5.73% [130500/2279000] [57.3/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 07:27:14,432 - Train: 5.73% [130600/2279000] [57.3/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 07:28:14,772 - Train: 5.73% [130700/2279000] [57.3/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 07:29:15,096 - Train: 5.74% [130800/2279000] [57.4/1000.0] [batch_t 0.606 (0.603)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 07:30:15,433 - Train: 5.74% [130900/2279000] [57.4/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 07:31:15,758 - Train: 5.75% [131000/2279000] [57.5/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 07:32:16,105 - Train: 5.75% [131100/2279000] [57.5/1000.0] [batch_t 0.607 (0.603)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 07:33:16,561 - Train: 5.76% [131200/2279000] [57.6/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 07:34:16,943 - Train: 5.76% [131300/2279000] [57.6/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 07:35:17,226 - Train: 5.77% [131400/2279000] [57.7/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 07:36:17,597 - Train: 5.77% [131500/2279000] [57.7/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 07:37:17,941 - Train: 5.77% [131600/2279000] [57.7/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 07:38:18,375 - Train: 5.78% [131700/2279000] [57.8/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 07:39:18,845 - Train: 5.78% [131800/2279000] [57.8/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 07:40:19,177 - Train: 5.79% [131900/2279000] [57.9/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 07:41:19,481 - Train: 5.79% [132000/2279000] [57.9/1000.0] [batch_t 0.608 (0.603)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-10 07:42:19,781 - Train: 5.80% [132100/2279000] [58.0/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 07:43:09,247 - ==> Total time: 1 day, 8:14:29 Eta: 21 days, 19:38:46 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-10 07:43:35,134 - Train: 5.80% [132200/2279000] [58.0/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 07:44:35,640 - Train: 5.81% [132300/2279000] [58.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 07:45:35,987 - Train: 5.81% [132400/2279000] [58.1/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 07:46:36,306 - Train: 5.81% [132500/2279000] [58.1/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 07:47:36,630 - Train: 5.82% [132600/2279000] [58.2/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 07:48:36,981 - Train: 5.82% [132700/2279000] [58.2/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 07:49:37,301 - Train: 5.83% [132800/2279000] [58.3/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 07:50:37,731 - Train: 5.83% [132900/2279000] [58.3/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 07:51:38,059 - Train: 5.84% [133000/2279000] [58.4/1000.0] [batch_t 0.597 (0.603)] [data_t 0.002] [optim_t 0.596] [lr 0.005000] +2024-04-10 07:52:38,385 - Train: 5.84% [133100/2279000] [58.4/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 07:53:38,688 - Train: 5.84% [133200/2279000] [58.4/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 07:54:39,001 - Train: 5.85% [133300/2279000] [58.5/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 07:55:39,332 - Train: 5.85% [133400/2279000] [58.5/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 07:56:39,710 - Train: 5.86% [133500/2279000] [58.6/1000.0] [batch_t 0.600 (0.604)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-10 07:57:40,074 - Train: 5.86% [133600/2279000] [58.6/1000.0] [batch_t 0.600 (0.604)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-10 07:58:40,420 - Train: 5.87% [133700/2279000] [58.7/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 07:59:40,808 - Train: 5.87% [133800/2279000] [58.7/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 08:00:41,178 - Train: 5.88% [133900/2279000] [58.8/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 08:01:41,544 - Train: 5.88% [134000/2279000] [58.8/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 08:02:42,041 - Train: 5.88% [134100/2279000] [58.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 08:03:42,386 - Train: 5.89% [134200/2279000] [58.9/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 08:04:42,710 - Train: 5.89% [134300/2279000] [58.9/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 08:05:43,050 - Train: 5.90% [134400/2279000] [59.0/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 08:06:19,853 - ==> Total time: 1 day, 8:37:39 Eta: 21 days, 16:23:06 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-10 08:07:01,118 - Train: 5.90% [134500/2279000] [59.0/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 08:08:01,418 - Train: 5.91% [134600/2279000] [59.1/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 08:09:01,865 - Train: 5.91% [134700/2279000] [59.1/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 08:10:02,191 - Train: 5.91% [134800/2279000] [59.1/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 08:11:02,482 - Train: 5.92% [134900/2279000] [59.2/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 08:12:02,806 - Train: 5.92% [135000/2279000] [59.2/1000.0] [batch_t 0.608 (0.603)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-10 08:13:03,171 - Train: 5.93% [135100/2279000] [59.3/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 08:14:03,451 - Train: 5.93% [135200/2279000] [59.3/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 08:15:03,817 - Train: 5.94% [135300/2279000] [59.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 08:16:04,305 - Train: 5.94% [135400/2279000] [59.4/1000.0] [batch_t 0.599 (0.605)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-10 08:17:04,660 - Train: 5.95% [135500/2279000] [59.5/1000.0] [batch_t 0.606 (0.603)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 08:18:05,020 - Train: 5.95% [135600/2279000] [59.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 08:19:05,359 - Train: 5.95% [135700/2279000] [59.5/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 08:20:05,667 - Train: 5.96% [135800/2279000] [59.6/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 08:21:06,002 - Train: 5.96% [135900/2279000] [59.6/1000.0] [batch_t 0.607 (0.603)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 08:22:06,549 - Train: 5.97% [136000/2279000] [59.7/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 08:23:06,880 - Train: 5.97% [136100/2279000] [59.7/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 08:24:07,202 - Train: 5.98% [136200/2279000] [59.8/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 08:25:07,489 - Train: 5.98% [136300/2279000] [59.8/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 08:26:07,839 - Train: 5.99% [136400/2279000] [59.9/1000.0] [batch_t 0.601 (0.603)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 08:27:08,196 - Train: 5.99% [136500/2279000] [59.9/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 08:28:08,702 - Train: 5.99% [136600/2279000] [59.9/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 08:29:09,025 - Train: 6.00% [136700/2279000] [60.0/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 08:29:33,142 - ==> Total time: 1 day, 9:00:53 Eta: 21 days, 13:13:54 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-10 08:30:28,125 - Train: 6.00% [136800/2279000] [60.0/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 08:31:28,396 - Train: 6.01% [136900/2279000] [60.1/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 08:32:28,704 - Train: 6.01% [137000/2279000] [60.1/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 08:33:29,019 - Train: 6.02% [137100/2279000] [60.2/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 08:34:29,452 - Train: 6.02% [137200/2279000] [60.2/1000.0] [batch_t 0.600 (0.604)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 08:35:29,745 - Train: 6.02% [137300/2279000] [60.2/1000.0] [batch_t 0.601 (0.603)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 08:36:30,022 - Train: 6.03% [137400/2279000] [60.3/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 08:37:30,315 - Train: 6.03% [137500/2279000] [60.3/1000.0] [batch_t 0.600 (0.603)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-10 08:38:30,671 - Train: 6.04% [137600/2279000] [60.4/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 08:39:31,006 - Train: 6.04% [137700/2279000] [60.4/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 08:40:31,359 - Train: 6.05% [137800/2279000] [60.5/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 08:41:31,814 - Train: 6.05% [137900/2279000] [60.5/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 08:42:32,134 - Train: 6.06% [138000/2279000] [60.6/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 08:43:32,412 - Train: 6.06% [138100/2279000] [60.6/1000.0] [batch_t 0.595 (0.603)] [data_t 0.002] [optim_t 0.594] [lr 0.005000] +2024-04-10 08:44:32,717 - Train: 6.06% [138200/2279000] [60.6/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 08:45:33,001 - Train: 6.07% [138300/2279000] [60.7/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 08:46:33,423 - Train: 6.07% [138400/2279000] [60.7/1000.0] [batch_t 0.609 (0.604)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-10 08:47:33,712 - Train: 6.08% [138500/2279000] [60.8/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 08:48:34,067 - Train: 6.08% [138600/2279000] [60.8/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 08:49:34,410 - Train: 6.09% [138700/2279000] [60.9/1000.0] [batch_t 0.600 (0.603)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 08:50:34,698 - Train: 6.09% [138800/2279000] [60.9/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 08:51:35,036 - Train: 6.09% [138900/2279000] [60.9/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 08:52:35,298 - Train: 6.10% [139000/2279000] [61.0/1000.0] [batch_t 0.600 (0.603)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 08:52:46,758 - ==> Total time: 1 day, 9:24:06 Eta: 21 days, 10:10:13 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-10 08:53:52,497 - Train: 6.10% [139100/2279000] [61.0/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 08:54:52,884 - Train: 6.11% [139200/2279000] [61.1/1000.0] [batch_t 0.612 (0.604)] [data_t 0.002] [optim_t 0.610] [lr 0.005000] +2024-04-10 08:55:53,173 - Train: 6.11% [139300/2279000] [61.1/1000.0] [batch_t 0.599 (0.603)] [data_t 0.002] [optim_t 0.597] [lr 0.005000] +2024-04-10 08:56:53,507 - Train: 6.12% [139400/2279000] [61.2/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 08:57:53,795 - Train: 6.12% [139500/2279000] [61.2/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 08:58:54,273 - Train: 6.13% [139600/2279000] [61.3/1000.0] [batch_t 0.596 (0.605)] [data_t 0.002] [optim_t 0.594] [lr 0.005000] +2024-04-10 08:59:54,585 - Train: 6.13% [139700/2279000] [61.3/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 09:00:54,937 - Train: 6.13% [139800/2279000] [61.3/1000.0] [batch_t 0.608 (0.603)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-10 09:01:55,239 - Train: 6.14% [139900/2279000] [61.4/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 09:02:55,585 - Train: 6.14% [140000/2279000] [61.4/1000.0] [batch_t 0.608 (0.603)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 09:03:55,897 - Train: 6.15% [140100/2279000] [61.5/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 09:04:56,329 - Train: 6.15% [140200/2279000] [61.5/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 09:05:56,720 - Train: 6.16% [140300/2279000] [61.6/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 09:06:57,068 - Train: 6.16% [140400/2279000] [61.6/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 09:07:57,391 - Train: 6.16% [140500/2279000] [61.6/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 09:08:57,690 - Train: 6.17% [140600/2279000] [61.7/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 09:09:57,985 - Train: 6.17% [140700/2279000] [61.7/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 09:10:58,303 - Train: 6.18% [140800/2279000] [61.8/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 09:11:58,652 - Train: 6.18% [140900/2279000] [61.8/1000.0] [batch_t 0.606 (0.603)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 09:12:59,090 - Train: 6.19% [141000/2279000] [61.9/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 09:13:59,417 - Train: 6.19% [141100/2279000] [61.9/1000.0] [batch_t 0.600 (0.603)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-10 09:14:59,731 - Train: 6.20% [141200/2279000] [62.0/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 09:15:58,829 - ==> Total time: 1 day, 9:47:18 Eta: 21 days, 7:11:19 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-10 09:16:15,546 - Train: 6.20% [141300/2279000] [62.0/1000.0] [batch_t 0.606 (0.608)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 09:17:15,814 - Train: 6.20% [141400/2279000] [62.0/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 09:18:16,305 - Train: 6.21% [141500/2279000] [62.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 09:19:16,643 - Train: 6.21% [141600/2279000] [62.1/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 09:20:17,012 - Train: 6.22% [141700/2279000] [62.2/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 09:21:17,368 - Train: 6.22% [141800/2279000] [62.2/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 09:22:17,701 - Train: 6.23% [141900/2279000] [62.3/1000.0] [batch_t 0.606 (0.603)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 09:23:18,118 - Train: 6.23% [142000/2279000] [62.3/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 09:24:18,608 - Train: 6.24% [142100/2279000] [62.4/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 09:25:18,973 - Train: 6.24% [142200/2279000] [62.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 09:26:19,329 - Train: 6.24% [142300/2279000] [62.4/1000.0] [batch_t 0.601 (0.603)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 09:27:19,641 - Train: 6.25% [142400/2279000] [62.5/1000.0] [batch_t 0.601 (0.603)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 09:28:19,971 - Train: 6.25% [142500/2279000] [62.5/1000.0] [batch_t 0.608 (0.603)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-10 09:29:20,277 - Train: 6.26% [142600/2279000] [62.6/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 09:30:20,589 - Train: 6.26% [142700/2279000] [62.6/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 09:31:21,001 - Train: 6.27% [142800/2279000] [62.7/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 09:32:21,308 - Train: 6.27% [142900/2279000] [62.7/1000.0] [batch_t 0.599 (0.603)] [data_t 0.002] [optim_t 0.597] [lr 0.005000] +2024-04-10 09:33:21,650 - Train: 6.27% [143000/2279000] [62.7/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 09:34:22,017 - Train: 6.28% [143100/2279000] [62.8/1000.0] [batch_t 0.599 (0.604)] [data_t 0.002] [optim_t 0.597] [lr 0.005000] +2024-04-10 09:35:22,327 - Train: 6.28% [143200/2279000] [62.8/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 09:36:22,767 - Train: 6.29% [143300/2279000] [62.9/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 09:37:23,105 - Train: 6.29% [143400/2279000] [62.9/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 09:38:23,390 - Train: 6.30% [143500/2279000] [63.0/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 09:39:09,813 - ==> Total time: 1 day, 10:10:29 Eta: 21 days, 4:17:05 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-10 09:39:41,873 - Train: 6.30% [143600/2279000] [63.0/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 09:40:42,195 - Train: 6.31% [143700/2279000] [63.1/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 09:41:42,556 - Train: 6.31% [143800/2279000] [63.1/1000.0] [batch_t 0.601 (0.604)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 09:42:43,006 - Train: 6.31% [143900/2279000] [63.1/1000.0] [batch_t 0.601 (0.604)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 09:43:43,321 - Train: 6.32% [144000/2279000] [63.2/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 09:44:43,637 - Train: 6.32% [144100/2279000] [63.2/1000.0] [batch_t 0.599 (0.603)] [data_t 0.002] [optim_t 0.597] [lr 0.005000] +2024-04-10 09:45:43,961 - Train: 6.33% [144200/2279000] [63.3/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 09:46:44,224 - Train: 6.33% [144300/2279000] [63.3/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 09:47:44,556 - Train: 6.34% [144400/2279000] [63.4/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 09:48:44,900 - Train: 6.34% [144500/2279000] [63.4/1000.0] [batch_t 0.600 (0.603)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-10 09:49:45,231 - Train: 6.34% [144600/2279000] [63.4/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 09:50:45,629 - Train: 6.35% [144700/2279000] [63.5/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 09:51:45,960 - Train: 6.35% [144800/2279000] [63.5/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 09:52:46,280 - Train: 6.36% [144900/2279000] [63.6/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 09:53:46,643 - Train: 6.36% [145000/2279000] [63.6/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 09:54:46,961 - Train: 6.37% [145100/2279000] [63.7/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 09:55:47,441 - Train: 6.37% [145200/2279000] [63.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 09:56:47,753 - Train: 6.38% [145300/2279000] [63.8/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 09:57:48,120 - Train: 6.38% [145400/2279000] [63.8/1000.0] [batch_t 0.610 (0.604)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-10 09:58:48,455 - Train: 6.38% [145500/2279000] [63.8/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 09:59:48,766 - Train: 6.39% [145600/2279000] [63.9/1000.0] [batch_t 0.600 (0.603)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-10 10:00:49,153 - Train: 6.39% [145700/2279000] [63.9/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 10:01:49,507 - Train: 6.40% [145800/2279000] [64.0/1000.0] [batch_t 0.606 (0.603)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 10:02:23,306 - ==> Total time: 1 day, 10:33:43 Eta: 21 days, 1:28:12 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-10 10:03:07,515 - Train: 6.40% [145900/2279000] [64.0/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 10:04:07,915 - Train: 6.41% [146000/2279000] [64.1/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 10:05:08,246 - Train: 6.41% [146100/2279000] [64.1/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 10:06:08,532 - Train: 6.42% [146200/2279000] [64.2/1000.0] [batch_t 0.600 (0.603)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-10 10:07:08,835 - Train: 6.42% [146300/2279000] [64.2/1000.0] [batch_t 0.601 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 10:08:09,151 - Train: 6.42% [146400/2279000] [64.2/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 10:09:09,429 - Train: 6.43% [146500/2279000] [64.3/1000.0] [batch_t 0.600 (0.603)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-10 10:10:09,751 - Train: 6.43% [146600/2279000] [64.3/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 10:11:10,231 - Train: 6.44% [146700/2279000] [64.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 10:12:10,541 - Train: 6.44% [146800/2279000] [64.4/1000.0] [batch_t 0.601 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 10:13:10,866 - Train: 6.45% [146900/2279000] [64.5/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 10:14:11,165 - Train: 6.45% [147000/2279000] [64.5/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 10:15:11,524 - Train: 6.45% [147100/2279000] [64.5/1000.0] [batch_t 0.600 (0.603)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-10 10:16:13,141 - Train: 6.46% [147200/2279000] [64.6/1000.0] [batch_t 1.049 (0.616)] [data_t 0.444] [optim_t 0.605] [lr 0.005000] +2024-04-10 10:17:13,468 - Train: 6.46% [147300/2279000] [64.6/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 10:18:14,760 - Train: 6.47% [147400/2279000] [64.7/1000.0] [batch_t 0.602 (0.613)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 10:19:15,891 - Train: 6.47% [147500/2279000] [64.7/1000.0] [batch_t 0.604 (0.611)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 10:20:20,788 - Train: 6.48% [147600/2279000] [64.8/1000.0] [batch_t 3.479 (0.649)] [data_t 2.876] [optim_t 0.603] [lr 0.005000] +2024-04-10 10:21:37,690 - Train: 6.48% [147700/2279000] [64.8/1000.0] [batch_t 0.602 (0.769)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 10:22:37,985 - Train: 6.49% [147800/2279000] [64.9/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 10:23:38,490 - Train: 6.49% [147900/2279000] [64.9/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 10:24:38,832 - Train: 6.49% [148000/2279000] [64.9/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 10:25:39,185 - Train: 6.50% [148100/2279000] [65.0/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 10:26:00,295 - ==> Total time: 1 day, 10:57:20 Eta: 20 days, 22:49:25 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-10 10:27:00,131 - Train: 6.50% [148200/2279000] [65.0/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 10:28:00,416 - Train: 6.51% [148300/2279000] [65.1/1000.0] [batch_t 0.593 (0.603)] [data_t 0.002] [optim_t 0.592] [lr 0.005000] +2024-04-10 10:29:00,800 - Train: 6.51% [148400/2279000] [65.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 10:30:01,213 - Train: 6.52% [148500/2279000] [65.2/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 10:31:01,525 - Train: 6.52% [148600/2279000] [65.2/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 10:32:01,849 - Train: 6.52% [148700/2279000] [65.2/1000.0] [batch_t 0.610 (0.603)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-10 10:33:02,167 - Train: 6.53% [148800/2279000] [65.3/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 10:34:02,492 - Train: 6.53% [148900/2279000] [65.3/1000.0] [batch_t 0.601 (0.603)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 10:35:02,809 - Train: 6.54% [149000/2279000] [65.4/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 10:36:03,265 - Train: 6.54% [149100/2279000] [65.4/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 10:37:03,566 - Train: 6.55% [149200/2279000] [65.5/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 10:38:03,900 - Train: 6.55% [149300/2279000] [65.5/1000.0] [batch_t 0.601 (0.603)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 10:39:04,240 - Train: 6.56% [149400/2279000] [65.6/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 10:40:04,540 - Train: 6.56% [149500/2279000] [65.6/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 10:41:04,923 - Train: 6.56% [149600/2279000] [65.6/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 10:42:05,243 - Train: 6.57% [149700/2279000] [65.7/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 10:43:05,712 - Train: 6.57% [149800/2279000] [65.7/1000.0] [batch_t 0.598 (0.605)] [data_t 0.002] [optim_t 0.596] [lr 0.005000] +2024-04-10 10:44:06,041 - Train: 6.58% [149900/2279000] [65.8/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 10:45:06,372 - Train: 6.58% [150000/2279000] [65.8/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 10:46:06,672 - Train: 6.59% [150100/2279000] [65.9/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 10:47:07,006 - Train: 6.59% [150200/2279000] [65.9/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 10:48:07,364 - Train: 6.59% [150300/2279000] [65.9/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 10:49:07,680 - Train: 6.60% [150400/2279000] [66.0/1000.0] [batch_t 0.601 (0.603)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 10:49:16,132 - ==> Total time: 1 day, 11:20:36 Eta: 20 days, 20:09:45 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-10 10:50:26,408 - Train: 6.60% [150500/2279000] [66.0/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 10:51:26,779 - Train: 6.61% [150600/2279000] [66.1/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 10:52:27,132 - Train: 6.61% [150700/2279000] [66.1/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 10:53:27,463 - Train: 6.62% [150800/2279000] [66.2/1000.0] [batch_t 0.601 (0.603)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 10:54:27,770 - Train: 6.62% [150900/2279000] [66.2/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 10:55:28,083 - Train: 6.63% [151000/2279000] [66.3/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 10:56:28,517 - Train: 6.63% [151100/2279000] [66.3/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 10:57:28,852 - Train: 6.63% [151200/2279000] [66.3/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 10:58:29,191 - Train: 6.64% [151300/2279000] [66.4/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 10:59:29,513 - Train: 6.64% [151400/2279000] [66.4/1000.0] [batch_t 0.601 (0.603)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 11:00:29,846 - Train: 6.65% [151500/2279000] [66.5/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 11:01:30,154 - Train: 6.65% [151600/2279000] [66.5/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 11:02:30,486 - Train: 6.66% [151700/2279000] [66.6/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 11:03:30,924 - Train: 6.66% [151800/2279000] [66.6/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 11:04:31,228 - Train: 6.67% [151900/2279000] [66.7/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 11:05:31,609 - Train: 6.67% [152000/2279000] [66.7/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 11:06:31,956 - Train: 6.67% [152100/2279000] [66.7/1000.0] [batch_t 0.606 (0.603)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 11:07:32,236 - Train: 6.68% [152200/2279000] [66.8/1000.0] [batch_t 0.601 (0.603)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 11:08:32,546 - Train: 6.68% [152300/2279000] [66.8/1000.0] [batch_t 0.601 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 11:09:32,961 - Train: 6.69% [152400/2279000] [66.9/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 11:10:33,295 - Train: 6.69% [152500/2279000] [66.9/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 11:11:33,633 - Train: 6.70% [152600/2279000] [67.0/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 11:12:29,766 - ==> Total time: 1 day, 11:43:49 Eta: 20 days, 17:33:38 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-10 11:12:49,028 - Train: 6.70% [152700/2279000] [67.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 11:13:49,400 - Train: 6.70% [152800/2279000] [67.0/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 11:14:49,702 - Train: 6.71% [152900/2279000] [67.1/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 11:15:50,207 - Train: 6.71% [153000/2279000] [67.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 11:16:50,548 - Train: 6.72% [153100/2279000] [67.2/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 11:17:50,906 - Train: 6.72% [153200/2279000] [67.2/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 11:18:51,316 - Train: 6.73% [153300/2279000] [67.3/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 11:19:51,671 - Train: 6.73% [153400/2279000] [67.3/1000.0] [batch_t 0.600 (0.603)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-10 11:20:52,003 - Train: 6.74% [153500/2279000] [67.4/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 11:21:52,368 - Train: 6.74% [153600/2279000] [67.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 11:22:52,827 - Train: 6.74% [153700/2279000] [67.4/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 11:23:53,176 - Train: 6.75% [153800/2279000] [67.5/1000.0] [batch_t 0.608 (0.603)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-10 11:24:53,492 - Train: 6.75% [153900/2279000] [67.5/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 11:25:53,881 - Train: 6.76% [154000/2279000] [67.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 11:26:54,192 - Train: 6.76% [154100/2279000] [67.6/1000.0] [batch_t 0.601 (0.603)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 11:27:54,513 - Train: 6.77% [154200/2279000] [67.7/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 11:28:54,899 - Train: 6.77% [154300/2279000] [67.7/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 11:29:55,187 - Train: 6.77% [154400/2279000] [67.7/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 11:30:55,483 - Train: 6.78% [154500/2279000] [67.8/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 11:31:55,838 - Train: 6.78% [154600/2279000] [67.8/1000.0] [batch_t 0.600 (0.603)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-10 11:32:56,176 - Train: 6.79% [154700/2279000] [67.9/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 11:33:56,524 - Train: 6.79% [154800/2279000] [67.9/1000.0] [batch_t 0.606 (0.603)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 11:34:56,852 - Train: 6.80% [154900/2279000] [68.0/1000.0] [batch_t 0.601 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 11:35:40,393 - ==> Total time: 1 day, 12:07:00 Eta: 20 days, 15:00:45 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-10 11:36:15,356 - Train: 6.80% [155000/2279000] [68.0/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 11:37:15,659 - Train: 6.81% [155100/2279000] [68.1/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 11:38:15,982 - Train: 6.81% [155200/2279000] [68.1/1000.0] [batch_t 0.601 (0.603)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 11:39:16,322 - Train: 6.81% [155300/2279000] [68.1/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 11:40:16,678 - Train: 6.82% [155400/2279000] [68.2/1000.0] [batch_t 0.610 (0.603)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-10 11:41:17,012 - Train: 6.82% [155500/2279000] [68.2/1000.0] [batch_t 0.600 (0.603)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-10 11:42:17,512 - Train: 6.83% [155600/2279000] [68.3/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 11:43:17,818 - Train: 6.83% [155700/2279000] [68.3/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 11:44:18,115 - Train: 6.84% [155800/2279000] [68.4/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 11:45:18,485 - Train: 6.84% [155900/2279000] [68.4/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 11:46:18,829 - Train: 6.85% [156000/2279000] [68.5/1000.0] [batch_t 0.611 (0.603)] [data_t 0.002] [optim_t 0.609] [lr 0.005000] +2024-04-10 11:47:19,276 - Train: 6.85% [156100/2279000] [68.5/1000.0] [batch_t 0.600 (0.604)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 11:48:19,581 - Train: 6.85% [156200/2279000] [68.5/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 11:49:19,878 - Train: 6.86% [156300/2279000] [68.6/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 11:50:20,218 - Train: 6.86% [156400/2279000] [68.6/1000.0] [batch_t 0.601 (0.603)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 11:51:20,559 - Train: 6.87% [156500/2279000] [68.7/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 11:52:21,084 - Train: 6.87% [156600/2279000] [68.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 11:53:21,394 - Train: 6.88% [156700/2279000] [68.8/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 11:54:21,810 - Train: 6.88% [156800/2279000] [68.8/1000.0] [batch_t 0.601 (0.604)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 11:55:22,170 - Train: 6.88% [156900/2279000] [68.8/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 11:56:22,503 - Train: 6.89% [157000/2279000] [68.9/1000.0] [batch_t 0.600 (0.603)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-10 11:57:22,821 - Train: 6.89% [157100/2279000] [68.9/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 11:58:23,161 - Train: 6.90% [157200/2279000] [69.0/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 11:58:53,946 - ==> Total time: 1 day, 12:30:14 Eta: 20 days, 12:32:17 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-10 11:59:39,271 - Train: 6.90% [157300/2279000] [69.0/1000.0] [batch_t 0.601 (0.603)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 12:00:39,627 - Train: 6.91% [157400/2279000] [69.1/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 12:01:39,942 - Train: 6.91% [157500/2279000] [69.1/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 12:02:40,334 - Train: 6.92% [157600/2279000] [69.2/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 12:03:40,654 - Train: 6.92% [157700/2279000] [69.2/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 12:04:40,965 - Train: 6.92% [157800/2279000] [69.2/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 12:05:41,268 - Train: 6.93% [157900/2279000] [69.3/1000.0] [batch_t 0.598 (0.603)] [data_t 0.002] [optim_t 0.596] [lr 0.005000] +2024-04-10 12:06:41,722 - Train: 6.93% [158000/2279000] [69.3/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 12:07:42,084 - Train: 6.94% [158100/2279000] [69.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 12:08:42,408 - Train: 6.94% [158200/2279000] [69.4/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 12:09:42,736 - Train: 6.95% [158300/2279000] [69.5/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 12:10:42,988 - Train: 6.95% [158400/2279000] [69.5/1000.0] [batch_t 0.602 (0.602)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 12:11:43,324 - Train: 6.95% [158500/2279000] [69.5/1000.0] [batch_t 0.601 (0.603)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 12:12:43,826 - Train: 6.96% [158600/2279000] [69.6/1000.0] [batch_t 0.595 (0.605)] [data_t 0.002] [optim_t 0.593] [lr 0.005000] +2024-04-10 12:13:44,158 - Train: 6.96% [158700/2279000] [69.6/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 12:14:44,560 - Train: 6.97% [158800/2279000] [69.7/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 12:15:44,843 - Train: 6.97% [158900/2279000] [69.7/1000.0] [batch_t 0.600 (0.603)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-10 12:16:45,178 - Train: 6.98% [159000/2279000] [69.8/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 12:17:45,484 - Train: 6.98% [159100/2279000] [69.8/1000.0] [batch_t 0.600 (0.603)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-10 12:18:45,969 - Train: 6.99% [159200/2279000] [69.9/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 12:19:46,295 - Train: 6.99% [159300/2279000] [69.9/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 12:20:46,639 - Train: 6.99% [159400/2279000] [69.9/1000.0] [batch_t 0.600 (0.603)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-10 12:21:46,975 - Train: 7.00% [159500/2279000] [70.0/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 12:22:05,081 - ==> Total time: 1 day, 12:53:25 Eta: 20 days, 10:06:51 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-10 12:23:04,191 - Train: 7.00% [159600/2279000] [70.0/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 12:24:04,614 - Train: 7.01% [159700/2279000] [70.1/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 12:25:04,917 - Train: 7.01% [159800/2279000] [70.1/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 12:26:05,229 - Train: 7.02% [159900/2279000] [70.2/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 12:27:05,519 - Train: 7.02% [160000/2279000] [70.2/1000.0] [batch_t 0.601 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 12:28:05,851 - Train: 7.03% [160100/2279000] [70.3/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 12:29:06,168 - Train: 7.03% [160200/2279000] [70.3/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 12:30:06,630 - Train: 7.03% [160300/2279000] [70.3/1000.0] [batch_t 0.600 (0.605)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-10 12:31:06,933 - Train: 7.04% [160400/2279000] [70.4/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 12:32:07,246 - Train: 7.04% [160500/2279000] [70.4/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 12:33:07,532 - Train: 7.05% [160600/2279000] [70.5/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 12:34:07,803 - Train: 7.05% [160700/2279000] [70.5/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 12:35:08,118 - Train: 7.06% [160800/2279000] [70.6/1000.0] [batch_t 0.608 (0.603)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-10 12:36:08,467 - Train: 7.06% [160900/2279000] [70.6/1000.0] [batch_t 0.601 (0.603)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 12:37:08,748 - Train: 7.06% [161000/2279000] [70.6/1000.0] [batch_t 0.607 (0.603)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 12:38:09,151 - Train: 7.07% [161100/2279000] [70.7/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 12:39:09,484 - Train: 7.07% [161200/2279000] [70.7/1000.0] [batch_t 0.601 (0.603)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 12:40:09,846 - Train: 7.08% [161300/2279000] [70.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 12:41:10,223 - Train: 7.08% [161400/2279000] [70.8/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 12:42:10,514 - Train: 7.09% [161500/2279000] [70.9/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 12:43:10,887 - Train: 7.09% [161600/2279000] [70.9/1000.0] [batch_t 0.601 (0.604)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 12:44:11,162 - Train: 7.10% [161700/2279000] [71.0/1000.0] [batch_t 0.606 (0.603)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 12:45:11,487 - Train: 7.10% [161800/2279000] [71.0/1000.0] [batch_t 0.601 (0.603)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 12:45:16,928 - ==> Total time: 1 day, 13:16:37 Eta: 20 days, 7:45:01 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-10 12:46:26,310 - Train: 7.10% [161900/2279000] [71.0/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 12:47:26,611 - Train: 7.11% [162000/2279000] [71.1/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 12:48:26,931 - Train: 7.11% [162100/2279000] [71.1/1000.0] [batch_t 0.598 (0.603)] [data_t 0.002] [optim_t 0.596] [lr 0.005000] +2024-04-10 12:49:27,358 - Train: 7.12% [162200/2279000] [71.2/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 12:50:27,744 - Train: 7.12% [162300/2279000] [71.2/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 12:51:28,083 - Train: 7.13% [162400/2279000] [71.3/1000.0] [batch_t 0.600 (0.603)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-10 12:52:28,348 - Train: 7.13% [162500/2279000] [71.3/1000.0] [batch_t 0.601 (0.603)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 12:53:28,699 - Train: 7.13% [162600/2279000] [71.3/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 12:54:29,074 - Train: 7.14% [162700/2279000] [71.4/1000.0] [batch_t 0.601 (0.604)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 12:55:29,376 - Train: 7.14% [162800/2279000] [71.4/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 12:56:29,808 - Train: 7.15% [162900/2279000] [71.5/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 12:57:30,131 - Train: 7.15% [163000/2279000] [71.5/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 12:58:30,451 - Train: 7.16% [163100/2279000] [71.6/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 12:59:30,822 - Train: 7.16% [163200/2279000] [71.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 13:00:31,160 - Train: 7.17% [163300/2279000] [71.7/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 13:01:31,443 - Train: 7.17% [163400/2279000] [71.7/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 13:02:31,766 - Train: 7.17% [163500/2279000] [71.7/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 13:03:32,149 - Train: 7.18% [163600/2279000] [71.8/1000.0] [batch_t 0.601 (0.604)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 13:04:32,478 - Train: 7.18% [163700/2279000] [71.8/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 13:05:32,783 - Train: 7.19% [163800/2279000] [71.9/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 13:06:33,110 - Train: 7.19% [163900/2279000] [71.9/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 13:07:33,361 - Train: 7.20% [164000/2279000] [72.0/1000.0] [batch_t 0.602 (0.602)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 13:08:26,479 - ==> Total time: 1 day, 13:39:46 Eta: 20 days, 5:26:00 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-10 13:08:50,840 - Train: 7.20% [164100/2279000] [72.0/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 13:09:51,290 - Train: 7.20% [164200/2279000] [72.0/1000.0] [batch_t 0.601 (0.604)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 13:10:51,617 - Train: 7.21% [164300/2279000] [72.1/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 13:11:51,968 - Train: 7.21% [164400/2279000] [72.1/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 13:12:52,302 - Train: 7.22% [164500/2279000] [72.2/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 13:13:52,619 - Train: 7.22% [164600/2279000] [72.2/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 13:14:52,909 - Train: 7.23% [164700/2279000] [72.3/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 13:15:53,388 - Train: 7.23% [164800/2279000] [72.3/1000.0] [batch_t 0.600 (0.605)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-10 13:16:53,732 - Train: 7.24% [164900/2279000] [72.4/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 13:17:54,079 - Train: 7.24% [165000/2279000] [72.4/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 13:18:54,360 - Train: 7.24% [165100/2279000] [72.4/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 13:19:54,688 - Train: 7.25% [165200/2279000] [72.5/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 13:20:55,039 - Train: 7.25% [165300/2279000] [72.5/1000.0] [batch_t 0.601 (0.603)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 13:21:55,359 - Train: 7.26% [165400/2279000] [72.6/1000.0] [batch_t 0.598 (0.603)] [data_t 0.002] [optim_t 0.596] [lr 0.005000] +2024-04-10 13:22:55,753 - Train: 7.26% [165500/2279000] [72.6/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 13:23:56,047 - Train: 7.27% [165600/2279000] [72.7/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 13:24:56,351 - Train: 7.27% [165700/2279000] [72.7/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 13:25:56,680 - Train: 7.28% [165800/2279000] [72.8/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 13:26:56,993 - Train: 7.28% [165900/2279000] [72.8/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 13:27:57,295 - Train: 7.28% [166000/2279000] [72.8/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 13:28:57,809 - Train: 7.29% [166100/2279000] [72.9/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 13:29:58,107 - Train: 7.29% [166200/2279000] [72.9/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 13:30:58,447 - Train: 7.30% [166300/2279000] [73.0/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 13:31:38,891 - ==> Total time: 1 day, 14:02:58 Eta: 20 days, 3:10:45 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-10 13:32:17,951 - Train: 7.30% [166400/2279000] [73.0/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 13:33:18,251 - Train: 7.31% [166500/2279000] [73.1/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 13:34:18,605 - Train: 7.31% [166600/2279000] [73.1/1000.0] [batch_t 0.601 (0.603)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 13:35:19,093 - Train: 7.31% [166700/2279000] [73.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 13:36:19,422 - Train: 7.32% [166800/2279000] [73.2/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 13:37:19,698 - Train: 7.32% [166900/2279000] [73.2/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 13:38:20,042 - Train: 7.33% [167000/2279000] [73.3/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 13:39:20,358 - Train: 7.33% [167100/2279000] [73.3/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 13:40:20,652 - Train: 7.34% [167200/2279000] [73.4/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 13:41:21,019 - Train: 7.34% [167300/2279000] [73.4/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 13:42:21,436 - Train: 7.35% [167400/2279000] [73.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 13:43:21,801 - Train: 7.35% [167500/2279000] [73.5/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 13:44:22,146 - Train: 7.35% [167600/2279000] [73.5/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 13:45:22,449 - Train: 7.36% [167700/2279000] [73.6/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 13:46:22,753 - Train: 7.36% [167800/2279000] [73.6/1000.0] [batch_t 0.600 (0.603)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 13:47:23,118 - Train: 7.37% [167900/2279000] [73.7/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 13:48:23,552 - Train: 7.37% [168000/2279000] [73.7/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 13:49:23,891 - Train: 7.38% [168100/2279000] [73.8/1000.0] [batch_t 0.601 (0.603)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 13:50:24,209 - Train: 7.38% [168200/2279000] [73.8/1000.0] [batch_t 0.606 (0.603)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 13:51:24,559 - Train: 7.38% [168300/2279000] [73.8/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 13:52:24,855 - Train: 7.39% [168400/2279000] [73.9/1000.0] [batch_t 0.601 (0.603)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 13:53:25,162 - Train: 7.39% [168500/2279000] [73.9/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 13:54:25,473 - Train: 7.40% [168600/2279000] [74.0/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 13:54:53,206 - ==> Total time: 1 day, 14:26:13 Eta: 20 days, 0:58:56 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-10 13:55:43,974 - Train: 7.40% [168700/2279000] [74.0/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 13:56:44,289 - Train: 7.41% [168800/2279000] [74.1/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 13:57:44,583 - Train: 7.41% [168900/2279000] [74.1/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 13:58:44,867 - Train: 7.42% [169000/2279000] [74.2/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 13:59:45,204 - Train: 7.42% [169100/2279000] [74.2/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 14:00:45,726 - Train: 7.42% [169200/2279000] [74.2/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 14:01:46,079 - Train: 7.43% [169300/2279000] [74.3/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 14:02:46,373 - Train: 7.43% [169400/2279000] [74.3/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 14:03:46,738 - Train: 7.44% [169500/2279000] [74.4/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 14:04:47,088 - Train: 7.44% [169600/2279000] [74.4/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 14:05:47,462 - Train: 7.45% [169700/2279000] [74.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 14:06:47,818 - Train: 7.45% [169800/2279000] [74.5/1000.0] [batch_t 0.610 (0.603)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-10 14:07:48,258 - Train: 7.46% [169900/2279000] [74.6/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 14:08:48,611 - Train: 7.46% [170000/2279000] [74.6/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 14:09:48,998 - Train: 7.46% [170100/2279000] [74.6/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 14:10:49,283 - Train: 7.47% [170200/2279000] [74.7/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 14:11:49,607 - Train: 7.47% [170300/2279000] [74.7/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 14:12:50,119 - Train: 7.48% [170400/2279000] [74.8/1000.0] [batch_t 0.607 (0.605)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-10 14:13:50,446 - Train: 7.48% [170500/2279000] [74.8/1000.0] [batch_t 0.601 (0.603)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 14:14:50,773 - Train: 7.49% [170600/2279000] [74.9/1000.0] [batch_t 0.606 (0.603)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 14:15:51,114 - Train: 7.49% [170700/2279000] [74.9/1000.0] [batch_t 0.601 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 14:16:51,487 - Train: 7.49% [170800/2279000] [74.9/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 14:17:51,847 - Train: 7.50% [170900/2279000] [75.0/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 14:18:06,956 - ==> Total time: 1 day, 14:49:27 Eta: 19 days, 22:49:53 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-10 14:19:10,650 - Train: 7.50% [171000/2279000] [75.0/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 14:20:11,125 - Train: 7.51% [171100/2279000] [75.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 14:21:11,491 - Train: 7.51% [171200/2279000] [75.1/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 14:22:11,803 - Train: 7.52% [171300/2279000] [75.2/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 14:23:12,116 - Train: 7.52% [171400/2279000] [75.2/1000.0] [batch_t 0.599 (0.603)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-10 14:24:12,424 - Train: 7.53% [171500/2279000] [75.3/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 14:25:12,705 - Train: 7.53% [171600/2279000] [75.3/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 14:26:13,017 - Train: 7.53% [171700/2279000] [75.3/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 14:27:13,445 - Train: 7.54% [171800/2279000] [75.4/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 14:28:13,761 - Train: 7.54% [171900/2279000] [75.4/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 14:29:14,093 - Train: 7.55% [172000/2279000] [75.5/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 14:30:14,407 - Train: 7.55% [172100/2279000] [75.5/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 14:31:14,727 - Train: 7.56% [172200/2279000] [75.6/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 14:32:15,145 - Train: 7.56% [172300/2279000] [75.6/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 14:33:15,512 - Train: 7.56% [172400/2279000] [75.6/1000.0] [batch_t 0.601 (0.604)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 14:34:15,878 - Train: 7.57% [172500/2279000] [75.7/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 14:35:16,199 - Train: 7.57% [172600/2279000] [75.7/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 14:36:16,496 - Train: 7.58% [172700/2279000] [75.8/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 14:37:16,799 - Train: 7.58% [172800/2279000] [75.8/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 14:38:17,273 - Train: 7.59% [172900/2279000] [75.9/1000.0] [batch_t 0.599 (0.605)] [data_t 0.002] [optim_t 0.597] [lr 0.005000] +2024-04-10 14:39:17,594 - Train: 7.59% [173000/2279000] [75.9/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 14:40:17,927 - Train: 7.60% [173100/2279000] [76.0/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 14:41:18,271 - Train: 7.60% [173200/2279000] [76.0/1000.0] [batch_t 0.601 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 14:41:20,680 - ==> Total time: 1 day, 15:12:40 Eta: 19 days, 20:43:37 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-10 14:42:35,026 - Train: 7.60% [173300/2279000] [76.0/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 14:43:35,375 - Train: 7.61% [173400/2279000] [76.1/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 14:44:35,842 - Train: 7.61% [173500/2279000] [76.1/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 14:45:36,149 - Train: 7.62% [173600/2279000] [76.2/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 14:46:36,515 - Train: 7.62% [173700/2279000] [76.2/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 14:47:36,816 - Train: 7.63% [173800/2279000] [76.3/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 14:48:37,169 - Train: 7.63% [173900/2279000] [76.3/1000.0] [batch_t 0.599 (0.603)] [data_t 0.002] [optim_t 0.597] [lr 0.005000] +2024-04-10 14:49:37,516 - Train: 7.63% [174000/2279000] [76.3/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 14:50:37,837 - Train: 7.64% [174100/2279000] [76.4/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 14:51:38,133 - Train: 7.64% [174200/2279000] [76.4/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 14:52:38,698 - Train: 7.65% [174300/2279000] [76.5/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 14:53:39,023 - Train: 7.65% [174400/2279000] [76.5/1000.0] [batch_t 0.599 (0.603)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-10 14:54:39,340 - Train: 7.66% [174500/2279000] [76.6/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 14:55:39,672 - Train: 7.66% [174600/2279000] [76.6/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 14:56:40,034 - Train: 7.67% [174700/2279000] [76.7/1000.0] [batch_t 0.601 (0.604)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 14:57:40,587 - Train: 7.67% [174800/2279000] [76.7/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 14:58:40,930 - Train: 7.67% [174900/2279000] [76.7/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 14:59:41,287 - Train: 7.68% [175000/2279000] [76.8/1000.0] [batch_t 0.601 (0.603)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 15:00:41,623 - Train: 7.68% [175100/2279000] [76.8/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 15:01:41,963 - Train: 7.69% [175200/2279000] [76.9/1000.0] [batch_t 0.606 (0.603)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 15:02:42,263 - Train: 7.69% [175300/2279000] [76.9/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 15:03:42,625 - Train: 7.70% [175400/2279000] [77.0/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 15:04:32,740 - ==> Total time: 1 day, 15:35:52 Eta: 19 days, 18:39:42 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-10 15:04:59,975 - Train: 7.70% [175500/2279000] [77.0/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 15:06:00,301 - Train: 7.71% [175600/2279000] [77.1/1000.0] [batch_t 0.600 (0.603)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 15:07:00,602 - Train: 7.71% [175700/2279000] [77.1/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 15:08:00,956 - Train: 7.71% [175800/2279000] [77.1/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 15:09:01,246 - Train: 7.72% [175900/2279000] [77.2/1000.0] [batch_t 0.601 (0.603)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 15:10:01,574 - Train: 7.72% [176000/2279000] [77.2/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 15:11:01,876 - Train: 7.73% [176100/2279000] [77.3/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 15:12:02,252 - Train: 7.73% [176200/2279000] [77.3/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 15:13:02,757 - Train: 7.74% [176300/2279000] [77.4/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 15:14:03,082 - Train: 7.74% [176400/2279000] [77.4/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 15:15:03,403 - Train: 7.74% [176500/2279000] [77.4/1000.0] [batch_t 0.600 (0.603)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-10 15:16:03,744 - Train: 7.75% [176600/2279000] [77.5/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 15:17:04,077 - Train: 7.75% [176700/2279000] [77.5/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 15:18:04,464 - Train: 7.76% [176800/2279000] [77.6/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 15:19:04,808 - Train: 7.76% [176900/2279000] [77.6/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 15:20:05,312 - Train: 7.77% [177000/2279000] [77.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 15:21:05,631 - Train: 7.77% [177100/2279000] [77.7/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 15:22:05,902 - Train: 7.78% [177200/2279000] [77.8/1000.0] [batch_t 0.609 (0.603)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-10 15:23:06,226 - Train: 7.78% [177300/2279000] [77.8/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 15:24:06,587 - Train: 7.78% [177400/2279000] [77.8/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 15:25:06,925 - Train: 7.79% [177500/2279000] [77.9/1000.0] [batch_t 0.606 (0.603)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 15:26:07,246 - Train: 7.79% [177600/2279000] [77.9/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 15:27:07,680 - Train: 7.80% [177700/2279000] [78.0/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 15:27:45,071 - ==> Total time: 1 day, 15:59:05 Eta: 19 days, 16:38:25 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-10 15:28:25,377 - Train: 7.80% [177800/2279000] [78.0/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 15:29:25,667 - Train: 7.81% [177900/2279000] [78.1/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 15:30:26,000 - Train: 7.81% [178000/2279000] [78.1/1000.0] [batch_t 0.600 (0.603)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-10 15:31:31,381 - Train: 7.81% [178100/2279000] [78.1/1000.0] [batch_t 0.601 (0.654)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 15:32:31,713 - Train: 7.82% [178200/2279000] [78.2/1000.0] [batch_t 0.607 (0.603)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 15:33:32,023 - Train: 7.82% [178300/2279000] [78.2/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 15:34:32,345 - Train: 7.83% [178400/2279000] [78.3/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 15:35:32,629 - Train: 7.83% [178500/2279000] [78.3/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 15:36:33,007 - Train: 7.84% [178600/2279000] [78.4/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 15:37:33,399 - Train: 7.84% [178700/2279000] [78.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 15:38:33,769 - Train: 7.85% [178800/2279000] [78.5/1000.0] [batch_t 0.601 (0.604)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 15:39:34,273 - Train: 7.85% [178900/2279000] [78.5/1000.0] [batch_t 0.598 (0.605)] [data_t 0.002] [optim_t 0.597] [lr 0.005000] +2024-04-10 15:40:34,625 - Train: 7.85% [179000/2279000] [78.5/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 15:41:35,008 - Train: 7.86% [179100/2279000] [78.6/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 15:42:35,306 - Train: 7.86% [179200/2279000] [78.6/1000.0] [batch_t 0.597 (0.603)] [data_t 0.002] [optim_t 0.595] [lr 0.005000] +2024-04-10 15:43:35,651 - Train: 7.87% [179300/2279000] [78.7/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 15:44:36,018 - Train: 7.87% [179400/2279000] [78.7/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 15:45:36,538 - Train: 7.88% [179500/2279000] [78.8/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 15:46:36,920 - Train: 7.88% [179600/2279000] [78.8/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 15:47:37,277 - Train: 7.89% [179700/2279000] [78.9/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 15:48:37,563 - Train: 7.89% [179800/2279000] [78.9/1000.0] [batch_t 0.600 (0.603)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-10 15:49:37,895 - Train: 7.89% [179900/2279000] [78.9/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 15:50:38,230 - Train: 7.90% [180000/2279000] [79.0/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 15:51:03,128 - ==> Total time: 1 day, 16:22:23 Eta: 19 days, 14:40:44 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-10 15:51:58,283 - Train: 7.90% [180100/2279000] [79.0/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 15:52:58,563 - Train: 7.91% [180200/2279000] [79.1/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 15:53:58,907 - Train: 7.91% [180300/2279000] [79.1/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 15:54:59,238 - Train: 7.92% [180400/2279000] [79.2/1000.0] [batch_t 0.601 (0.603)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 15:55:59,566 - Train: 7.92% [180500/2279000] [79.2/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 15:57:00,076 - Train: 7.92% [180600/2279000] [79.2/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 15:58:00,391 - Train: 7.93% [180700/2279000] [79.3/1000.0] [batch_t 0.600 (0.603)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 15:59:00,682 - Train: 7.93% [180800/2279000] [79.3/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 16:00:01,034 - Train: 7.94% [180900/2279000] [79.4/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 16:01:01,402 - Train: 7.94% [181000/2279000] [79.4/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 16:02:01,910 - Train: 7.95% [181100/2279000] [79.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 16:03:02,283 - Train: 7.95% [181200/2279000] [79.5/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 16:04:02,614 - Train: 7.96% [181300/2279000] [79.6/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 16:05:03,933 - Train: 7.96% [181400/2279000] [79.6/1000.0] [batch_t 0.607 (0.613)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 16:06:04,412 - Train: 7.96% [181500/2279000] [79.6/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 16:07:04,759 - Train: 7.97% [181600/2279000] [79.7/1000.0] [batch_t 0.599 (0.603)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-10 16:08:07,491 - Train: 7.97% [181700/2279000] [79.7/1000.0] [batch_t 0.604 (0.627)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 16:09:07,998 - Train: 7.98% [181800/2279000] [79.8/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 16:10:27,004 - Train: 7.98% [181900/2279000] [79.8/1000.0] [batch_t 0.606 (0.790)] [data_t 0.003] [optim_t 0.603] [lr 0.005000] +2024-04-10 16:11:43,551 - Train: 7.99% [182000/2279000] [79.9/1000.0] [batch_t 0.602 (0.765)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 16:12:43,862 - Train: 7.99% [182100/2279000] [79.9/1000.0] [batch_t 0.607 (0.603)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 16:14:08,016 - Train: 7.99% [182200/2279000] [79.9/1000.0] [batch_t 0.604 (0.841)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 16:15:09,124 - Train: 8.00% [182300/2279000] [80.0/1000.0] [batch_t 0.603 (0.611)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 16:15:21,186 - ==> Total time: 1 day, 16:46:41 Eta: 19 days, 12:56:54 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-10 16:16:29,574 - Train: 8.00% [182400/2279000] [80.0/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 16:17:29,862 - Train: 8.01% [182500/2279000] [80.1/1000.0] [batch_t 0.600 (0.603)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-10 16:18:30,223 - Train: 8.01% [182600/2279000] [80.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 16:19:30,615 - Train: 8.02% [182700/2279000] [80.2/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 16:20:32,953 - Train: 8.02% [182800/2279000] [80.2/1000.0] [batch_t 0.602 (0.623)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 16:21:34,643 - Train: 8.03% [182900/2279000] [80.3/1000.0] [batch_t 0.604 (0.617)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 16:22:42,091 - Train: 8.03% [183000/2279000] [80.3/1000.0] [batch_t 0.606 (0.674)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 16:23:42,588 - Train: 8.03% [183100/2279000] [80.3/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 16:24:50,402 - Train: 8.04% [183200/2279000] [80.4/1000.0] [batch_t 0.603 (0.678)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 16:25:51,579 - Train: 8.04% [183300/2279000] [80.4/1000.0] [batch_t 0.607 (0.612)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 16:26:53,473 - Train: 8.05% [183400/2279000] [80.5/1000.0] [batch_t 0.599 (0.619)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-10 16:27:53,862 - Train: 8.05% [183500/2279000] [80.5/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 16:28:55,045 - Train: 8.06% [183600/2279000] [80.6/1000.0] [batch_t 0.601 (0.612)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 16:29:55,338 - Train: 8.06% [183700/2279000] [80.6/1000.0] [batch_t 0.601 (0.603)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 16:30:55,785 - Train: 8.06% [183800/2279000] [80.6/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 16:31:56,102 - Train: 8.07% [183900/2279000] [80.7/1000.0] [batch_t 0.606 (0.603)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 16:32:57,628 - Train: 8.07% [184000/2279000] [80.7/1000.0] [batch_t 0.603 (0.615)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 16:33:57,931 - Train: 8.08% [184100/2279000] [80.8/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 16:34:58,900 - Train: 8.08% [184200/2279000] [80.8/1000.0] [batch_t 0.603 (0.610)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 16:35:59,287 - Train: 8.09% [184300/2279000] [80.9/1000.0] [batch_t 0.601 (0.604)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 16:37:00,017 - Train: 8.09% [184400/2279000] [80.9/1000.0] [batch_t 0.602 (0.607)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 16:38:00,358 - Train: 8.10% [184500/2279000] [81.0/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 16:39:00,109 - ==> Total time: 1 day, 17:10:20 Eta: 19 days, 11:07:38 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-10 16:39:18,599 - Train: 8.10% [184600/2279000] [81.0/1000.0] [batch_t 0.608 (0.608)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-10 16:40:18,912 - Train: 8.10% [184700/2279000] [81.0/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 16:41:19,193 - Train: 8.11% [184800/2279000] [81.1/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 16:42:20,247 - Train: 8.11% [184900/2279000] [81.1/1000.0] [batch_t 0.600 (0.610)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-10 16:43:20,668 - Train: 8.12% [185000/2279000] [81.2/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 16:44:21,032 - Train: 8.12% [185100/2279000] [81.2/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 16:45:21,387 - Train: 8.13% [185200/2279000] [81.3/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 16:46:21,691 - Train: 8.13% [185300/2279000] [81.3/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 16:47:22,040 - Train: 8.14% [185400/2279000] [81.4/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 16:48:22,350 - Train: 8.14% [185500/2279000] [81.4/1000.0] [batch_t 0.601 (0.603)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 16:49:22,752 - Train: 8.14% [185600/2279000] [81.4/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 16:50:23,102 - Train: 8.15% [185700/2279000] [81.5/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 16:51:23,433 - Train: 8.15% [185800/2279000] [81.5/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 16:52:23,744 - Train: 8.16% [185900/2279000] [81.6/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 16:53:24,054 - Train: 8.16% [186000/2279000] [81.6/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 16:54:24,388 - Train: 8.17% [186100/2279000] [81.7/1000.0] [batch_t 0.601 (0.603)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 16:55:24,714 - Train: 8.17% [186200/2279000] [81.7/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 16:56:25,099 - Train: 8.17% [186300/2279000] [81.7/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 16:57:25,416 - Train: 8.18% [186400/2279000] [81.8/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 16:58:25,769 - Train: 8.18% [186500/2279000] [81.8/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 16:59:26,080 - Train: 8.19% [186600/2279000] [81.9/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 17:00:26,410 - Train: 8.19% [186700/2279000] [81.9/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 17:01:26,740 - Train: 8.20% [186800/2279000] [82.0/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 17:02:13,814 - ==> Total time: 1 day, 17:33:33 Eta: 19 days, 9:15:46 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-10 17:02:43,046 - Train: 8.20% [186900/2279000] [82.0/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 17:03:43,595 - Train: 8.21% [187000/2279000] [82.1/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 17:04:43,949 - Train: 8.21% [187100/2279000] [82.1/1000.0] [batch_t 0.607 (0.603)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 17:05:44,273 - Train: 8.21% [187200/2279000] [82.1/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 17:06:44,583 - Train: 8.22% [187300/2279000] [82.2/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 17:07:44,934 - Train: 8.22% [187400/2279000] [82.2/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 17:08:45,374 - Train: 8.23% [187500/2279000] [82.3/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 17:09:45,702 - Train: 8.23% [187600/2279000] [82.3/1000.0] [batch_t 0.599 (0.603)] [data_t 0.002] [optim_t 0.597] [lr 0.005000] +2024-04-10 17:10:45,995 - Train: 8.24% [187700/2279000] [82.4/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 17:11:46,270 - Train: 8.24% [187800/2279000] [82.4/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 17:12:46,623 - Train: 8.24% [187900/2279000] [82.4/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 17:13:46,958 - Train: 8.25% [188000/2279000] [82.5/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 17:14:47,257 - Train: 8.25% [188100/2279000] [82.5/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 17:15:47,605 - Train: 8.26% [188200/2279000] [82.6/1000.0] [batch_t 0.607 (0.603)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 17:16:48,034 - Train: 8.26% [188300/2279000] [82.6/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 17:17:48,391 - Train: 8.27% [188400/2279000] [82.7/1000.0] [batch_t 0.601 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 17:18:48,733 - Train: 8.27% [188500/2279000] [82.7/1000.0] [batch_t 0.601 (0.603)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 17:19:49,089 - Train: 8.28% [188600/2279000] [82.8/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 17:20:49,458 - Train: 8.28% [188700/2279000] [82.8/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 17:21:51,949 - Train: 8.28% [188800/2279000] [82.8/1000.0] [batch_t 0.598 (0.625)] [data_t 0.002] [optim_t 0.596] [lr 0.005000] +2024-04-10 17:22:55,134 - Train: 8.29% [188900/2279000] [82.9/1000.0] [batch_t 0.605 (0.632)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 17:23:55,431 - Train: 8.29% [189000/2279000] [82.9/1000.0] [batch_t 0.600 (0.603)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-10 17:24:55,767 - Train: 8.30% [189100/2279000] [83.0/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 17:25:30,166 - ==> Total time: 1 day, 17:56:50 Eta: 19 days, 7:26:30 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-10 17:26:13,858 - Train: 8.30% [189200/2279000] [83.0/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 17:27:14,374 - Train: 8.31% [189300/2279000] [83.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 17:28:14,804 - Train: 8.31% [189400/2279000] [83.1/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 17:29:15,120 - Train: 8.32% [189500/2279000] [83.2/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 17:30:15,449 - Train: 8.32% [189600/2279000] [83.2/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 17:31:15,756 - Train: 8.32% [189700/2279000] [83.2/1000.0] [batch_t 0.600 (0.603)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-10 17:32:16,072 - Train: 8.33% [189800/2279000] [83.3/1000.0] [batch_t 0.599 (0.603)] [data_t 0.002] [optim_t 0.597] [lr 0.005000] +2024-04-10 17:33:16,554 - Train: 8.33% [189900/2279000] [83.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 17:34:16,915 - Train: 8.34% [190000/2279000] [83.4/1000.0] [batch_t 0.601 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 17:35:17,233 - Train: 8.34% [190100/2279000] [83.4/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 17:36:17,564 - Train: 8.35% [190200/2279000] [83.5/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 17:37:17,894 - Train: 8.35% [190300/2279000] [83.5/1000.0] [batch_t 0.601 (0.603)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 17:38:18,233 - Train: 8.35% [190400/2279000] [83.5/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 17:39:18,543 - Train: 8.36% [190500/2279000] [83.6/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 17:40:18,902 - Train: 8.36% [190600/2279000] [83.6/1000.0] [batch_t 0.601 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 17:41:19,247 - Train: 8.37% [190700/2279000] [83.7/1000.0] [batch_t 0.608 (0.603)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-10 17:42:19,619 - Train: 8.37% [190800/2279000] [83.7/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-10 17:43:19,978 - Train: 8.38% [190900/2279000] [83.8/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 17:44:20,277 - Train: 8.38% [191000/2279000] [83.8/1000.0] [batch_t 0.601 (0.603)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 17:45:20,611 - Train: 8.39% [191100/2279000] [83.9/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 17:46:21,096 - Train: 8.39% [191200/2279000] [83.9/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 17:47:21,414 - Train: 8.39% [191300/2279000] [83.9/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 17:48:21,773 - Train: 8.40% [191400/2279000] [84.0/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 17:48:43,488 - ==> Total time: 1 day, 18:20:03 Eta: 19 days, 5:38:44 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-10 17:49:38,417 - Train: 8.40% [191500/2279000] [84.0/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 17:50:38,740 - Train: 8.41% [191600/2279000] [84.1/1000.0] [batch_t 0.609 (0.603)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-10 17:51:39,078 - Train: 8.41% [191700/2279000] [84.1/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 17:52:39,509 - Train: 8.42% [191800/2279000] [84.2/1000.0] [batch_t 0.600 (0.604)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-10 17:53:39,884 - Train: 8.42% [191900/2279000] [84.2/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 17:54:40,205 - Train: 8.42% [192000/2279000] [84.2/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 17:55:40,545 - Train: 8.43% [192100/2279000] [84.3/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 17:56:40,853 - Train: 8.43% [192200/2279000] [84.3/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 17:57:41,185 - Train: 8.44% [192300/2279000] [84.4/1000.0] [batch_t 0.609 (0.603)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-10 17:58:41,554 - Train: 8.44% [192400/2279000] [84.4/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 17:59:41,910 - Train: 8.45% [192500/2279000] [84.5/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 18:00:42,205 - Train: 8.45% [192600/2279000] [84.5/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 18:01:42,507 - Train: 8.46% [192700/2279000] [84.6/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 18:02:42,808 - Train: 8.46% [192800/2279000] [84.6/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 18:03:43,100 - Train: 8.46% [192900/2279000] [84.6/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 18:04:43,395 - Train: 8.47% [193000/2279000] [84.7/1000.0] [batch_t 0.601 (0.603)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 18:05:43,736 - Train: 8.47% [193100/2279000] [84.7/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 18:06:44,174 - Train: 8.48% [193200/2279000] [84.8/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 18:07:44,466 - Train: 8.48% [193300/2279000] [84.8/1000.0] [batch_t 0.601 (0.603)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 18:08:44,787 - Train: 8.49% [193400/2279000] [84.9/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 18:09:45,140 - Train: 8.49% [193500/2279000] [84.9/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 18:10:45,435 - Train: 8.49% [193600/2279000] [84.9/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 18:11:45,888 - Train: 8.50% [193700/2279000] [85.0/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 18:11:54,937 - ==> Total time: 1 day, 18:43:15 Eta: 19 days, 3:52:38 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-10 18:13:04,454 - Train: 8.50% [193800/2279000] [85.0/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 18:14:04,772 - Train: 8.51% [193900/2279000] [85.1/1000.0] [batch_t 0.600 (0.603)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-10 18:15:05,071 - Train: 8.51% [194000/2279000] [85.1/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 18:16:05,407 - Train: 8.52% [194100/2279000] [85.2/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 18:17:05,827 - Train: 8.52% [194200/2279000] [85.2/1000.0] [batch_t 0.597 (0.604)] [data_t 0.002] [optim_t 0.595] [lr 0.005000] +2024-04-10 18:18:06,141 - Train: 8.53% [194300/2279000] [85.3/1000.0] [batch_t 0.600 (0.603)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-10 18:19:06,453 - Train: 8.53% [194400/2279000] [85.3/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 18:20:06,767 - Train: 8.53% [194500/2279000] [85.3/1000.0] [batch_t 0.601 (0.603)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 18:21:07,081 - Train: 8.54% [194600/2279000] [85.4/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 18:22:07,375 - Train: 8.54% [194700/2279000] [85.4/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 18:23:07,681 - Train: 8.55% [194800/2279000] [85.5/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 18:24:08,112 - Train: 8.55% [194900/2279000] [85.5/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 18:25:08,457 - Train: 8.56% [195000/2279000] [85.6/1000.0] [batch_t 0.606 (0.603)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 18:26:08,755 - Train: 8.56% [195100/2279000] [85.6/1000.0] [batch_t 0.601 (0.603)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 18:27:09,070 - Train: 8.57% [195200/2279000] [85.7/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 18:28:09,380 - Train: 8.57% [195300/2279000] [85.7/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 18:29:09,666 - Train: 8.57% [195400/2279000] [85.7/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 18:30:10,051 - Train: 8.58% [195500/2279000] [85.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 18:31:10,548 - Train: 8.58% [195600/2279000] [85.8/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 18:32:10,857 - Train: 8.59% [195700/2279000] [85.9/1000.0] [batch_t 0.601 (0.603)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 18:33:11,184 - Train: 8.59% [195800/2279000] [85.9/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 18:34:11,529 - Train: 8.60% [195900/2279000] [86.0/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 18:35:08,199 - ==> Total time: 1 day, 19:06:28 Eta: 19 days, 2:08:46 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-10 18:35:28,002 - Train: 8.60% [196000/2279000] [86.0/1000.0] [batch_t 0.601 (0.603)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 18:36:28,280 - Train: 8.60% [196100/2279000] [86.0/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 18:37:28,678 - Train: 8.61% [196200/2279000] [86.1/1000.0] [batch_t 0.601 (0.604)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 18:38:29,024 - Train: 8.61% [196300/2279000] [86.1/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 18:39:29,312 - Train: 8.62% [196400/2279000] [86.2/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 18:40:29,593 - Train: 8.62% [196500/2279000] [86.2/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 18:41:29,937 - Train: 8.63% [196600/2279000] [86.3/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 18:42:30,250 - Train: 8.63% [196700/2279000] [86.3/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 18:43:30,571 - Train: 8.64% [196800/2279000] [86.4/1000.0] [batch_t 0.606 (0.603)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 18:44:30,967 - Train: 8.64% [196900/2279000] [86.4/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 18:45:31,399 - Train: 8.64% [197000/2279000] [86.4/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 18:46:31,802 - Train: 8.65% [197100/2279000] [86.5/1000.0] [batch_t 0.609 (0.604)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-10 18:47:32,066 - Train: 8.65% [197200/2279000] [86.5/1000.0] [batch_t 0.600 (0.603)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 18:48:32,375 - Train: 8.66% [197300/2279000] [86.6/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 18:49:32,701 - Train: 8.66% [197400/2279000] [86.6/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 18:50:33,042 - Train: 8.67% [197500/2279000] [86.7/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 18:51:33,436 - Train: 8.67% [197600/2279000] [86.7/1000.0] [batch_t 0.609 (0.604)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-10 18:52:33,755 - Train: 8.67% [197700/2279000] [86.7/1000.0] [batch_t 0.593 (0.603)] [data_t 0.002] [optim_t 0.591] [lr 0.005000] +2024-04-10 18:53:34,103 - Train: 8.68% [197800/2279000] [86.8/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 18:54:34,435 - Train: 8.68% [197900/2279000] [86.8/1000.0] [batch_t 0.606 (0.603)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 18:55:34,727 - Train: 8.69% [198000/2279000] [86.9/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 18:56:35,087 - Train: 8.69% [198100/2279000] [86.9/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 18:57:35,404 - Train: 8.70% [198200/2279000] [87.0/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 18:58:19,590 - ==> Total time: 1 day, 19:29:39 Eta: 19 days, 0:26:26 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-10 18:58:52,817 - Train: 8.70% [198300/2279000] [87.0/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 18:59:53,100 - Train: 8.71% [198400/2279000] [87.1/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 19:00:53,422 - Train: 8.71% [198500/2279000] [87.1/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 19:01:53,731 - Train: 8.71% [198600/2279000] [87.1/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 19:02:54,080 - Train: 8.72% [198700/2279000] [87.2/1000.0] [batch_t 0.601 (0.603)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 19:03:54,565 - Train: 8.72% [198800/2279000] [87.2/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 19:04:54,854 - Train: 8.73% [198900/2279000] [87.3/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 19:05:55,192 - Train: 8.73% [199000/2279000] [87.3/1000.0] [batch_t 0.599 (0.603)] [data_t 0.002] [optim_t 0.597] [lr 0.005000] +2024-04-10 19:06:55,589 - Train: 8.74% [199100/2279000] [87.4/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 19:07:55,879 - Train: 8.74% [199200/2279000] [87.4/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 19:08:56,201 - Train: 8.75% [199300/2279000] [87.5/1000.0] [batch_t 0.599 (0.603)] [data_t 0.002] [optim_t 0.597] [lr 0.005000] +2024-04-10 19:09:56,510 - Train: 8.75% [199400/2279000] [87.5/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 19:10:56,882 - Train: 8.75% [199500/2279000] [87.5/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 19:11:57,220 - Train: 8.76% [199600/2279000] [87.6/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 19:12:57,511 - Train: 8.76% [199700/2279000] [87.6/1000.0] [batch_t 0.607 (0.603)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 19:13:57,814 - Train: 8.77% [199800/2279000] [87.7/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 19:14:58,154 - Train: 8.77% [199900/2279000] [87.7/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 19:15:58,579 - Train: 8.78% [200000/2279000] [87.8/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-10 19:16:58,896 - Train: 8.78% [200100/2279000] [87.8/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 19:17:59,285 - Train: 8.78% [200200/2279000] [87.8/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 19:18:59,615 - Train: 8.79% [200300/2279000] [87.9/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 19:19:59,927 - Train: 8.79% [200400/2279000] [87.9/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 19:21:00,199 - Train: 8.80% [200500/2279000] [88.0/1000.0] [batch_t 0.598 (0.603)] [data_t 0.002] [optim_t 0.596] [lr 0.005000] +2024-04-10 19:21:31,630 - ==> Total time: 1 day, 19:52:51 Eta: 18 days, 22:46:01 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-10 19:22:16,585 - Train: 8.80% [200600/2279000] [88.0/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 19:23:16,954 - Train: 8.81% [200700/2279000] [88.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 19:24:17,290 - Train: 8.81% [200800/2279000] [88.1/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 19:25:17,575 - Train: 8.82% [200900/2279000] [88.2/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 19:26:17,939 - Train: 8.82% [201000/2279000] [88.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 19:27:18,387 - Train: 8.82% [201100/2279000] [88.2/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 19:28:18,713 - Train: 8.83% [201200/2279000] [88.3/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 19:29:19,028 - Train: 8.83% [201300/2279000] [88.3/1000.0] [batch_t 0.606 (0.603)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 19:30:19,351 - Train: 8.84% [201400/2279000] [88.4/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 19:31:19,752 - Train: 8.84% [201500/2279000] [88.4/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 19:32:20,094 - Train: 8.85% [201600/2279000] [88.5/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 19:33:20,529 - Train: 8.85% [201700/2279000] [88.5/1000.0] [batch_t 0.601 (0.604)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 19:34:20,876 - Train: 8.85% [201800/2279000] [88.5/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 19:35:21,199 - Train: 8.86% [201900/2279000] [88.6/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 19:36:21,501 - Train: 8.86% [202000/2279000] [88.6/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 19:37:21,807 - Train: 8.87% [202100/2279000] [88.7/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 19:38:22,144 - Train: 8.87% [202200/2279000] [88.7/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 19:39:22,633 - Train: 8.88% [202300/2279000] [88.8/1000.0] [batch_t 0.599 (0.605)] [data_t 0.002] [optim_t 0.597] [lr 0.005000] +2024-04-10 19:40:22,960 - Train: 8.88% [202400/2279000] [88.8/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 19:41:23,292 - Train: 8.89% [202500/2279000] [88.9/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 19:42:23,606 - Train: 8.89% [202600/2279000] [88.9/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 19:43:23,926 - Train: 8.89% [202700/2279000] [88.9/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 19:44:24,253 - Train: 8.90% [202800/2279000] [89.0/1000.0] [batch_t 0.606 (0.603)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 19:44:42,937 - ==> Total time: 1 day, 20:16:03 Eta: 18 days, 21:07:12 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-10 19:45:40,228 - Train: 8.90% [202900/2279000] [89.0/1000.0] [batch_t 0.607 (0.603)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 19:46:40,532 - Train: 8.91% [203000/2279000] [89.1/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 19:47:40,884 - Train: 8.91% [203100/2279000] [89.1/1000.0] [batch_t 0.601 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 19:48:41,199 - Train: 8.92% [203200/2279000] [89.2/1000.0] [batch_t 0.590 (0.603)] [data_t 0.002] [optim_t 0.588] [lr 0.005000] +2024-04-10 19:49:41,527 - Train: 8.92% [203300/2279000] [89.2/1000.0] [batch_t 0.607 (0.603)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 19:50:41,823 - Train: 8.92% [203400/2279000] [89.2/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 19:51:42,147 - Train: 8.93% [203500/2279000] [89.3/1000.0] [batch_t 0.607 (0.603)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 19:52:42,595 - Train: 8.93% [203600/2279000] [89.3/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 19:53:42,888 - Train: 8.94% [203700/2279000] [89.4/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 19:54:43,250 - Train: 8.94% [203800/2279000] [89.4/1000.0] [batch_t 0.600 (0.604)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-10 19:55:43,550 - Train: 8.95% [203900/2279000] [89.5/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 19:56:43,887 - Train: 8.95% [204000/2279000] [89.5/1000.0] [batch_t 0.606 (0.603)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 19:57:44,232 - Train: 8.96% [204100/2279000] [89.6/1000.0] [batch_t 0.599 (0.603)] [data_t 0.002] [optim_t 0.597] [lr 0.005000] +2024-04-10 19:58:44,525 - Train: 8.96% [204200/2279000] [89.6/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 19:59:44,955 - Train: 8.96% [204300/2279000] [89.6/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 20:00:45,277 - Train: 8.97% [204400/2279000] [89.7/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 20:01:45,609 - Train: 8.97% [204500/2279000] [89.7/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 20:02:45,936 - Train: 8.98% [204600/2279000] [89.8/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 20:03:46,241 - Train: 8.98% [204700/2279000] [89.8/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 20:04:46,682 - Train: 8.99% [204800/2279000] [89.9/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 20:05:46,996 - Train: 8.99% [204900/2279000] [89.9/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 20:06:47,366 - Train: 9.00% [205000/2279000] [90.0/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 20:07:47,683 - Train: 9.00% [205100/2279000] [90.0/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 20:07:53,724 - ==> Total time: 1 day, 20:39:13 Eta: 18 days, 19:29:59 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-10 20:09:05,075 - Train: 9.00% [205200/2279000] [90.0/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 20:10:05,383 - Train: 9.01% [205300/2279000] [90.1/1000.0] [batch_t 0.601 (0.603)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 20:11:05,917 - Train: 9.01% [205400/2279000] [90.1/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 20:12:06,259 - Train: 9.02% [205500/2279000] [90.2/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 20:13:06,624 - Train: 9.02% [205600/2279000] [90.2/1000.0] [batch_t 0.601 (0.604)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 20:14:06,968 - Train: 9.03% [205700/2279000] [90.3/1000.0] [batch_t 0.606 (0.603)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 20:15:07,261 - Train: 9.03% [205800/2279000] [90.3/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 20:16:07,546 - Train: 9.03% [205900/2279000] [90.3/1000.0] [batch_t 0.601 (0.603)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 20:17:07,821 - Train: 9.04% [206000/2279000] [90.4/1000.0] [batch_t 0.606 (0.603)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 20:18:08,311 - Train: 9.04% [206100/2279000] [90.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 20:19:08,625 - Train: 9.05% [206200/2279000] [90.5/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 20:20:08,943 - Train: 9.05% [206300/2279000] [90.5/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 20:21:09,270 - Train: 9.06% [206400/2279000] [90.6/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 20:22:09,546 - Train: 9.06% [206500/2279000] [90.6/1000.0] [batch_t 0.599 (0.603)] [data_t 0.002] [optim_t 0.597] [lr 0.005000] +2024-04-10 20:23:09,815 - Train: 9.07% [206600/2279000] [90.7/1000.0] [batch_t 0.601 (0.603)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 20:24:10,152 - Train: 9.07% [206700/2279000] [90.7/1000.0] [batch_t 0.606 (0.603)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 20:25:10,599 - Train: 9.07% [206800/2279000] [90.7/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 20:26:10,932 - Train: 9.08% [206900/2279000] [90.8/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 20:27:11,260 - Train: 9.08% [207000/2279000] [90.8/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 20:28:11,575 - Train: 9.09% [207100/2279000] [90.9/1000.0] [batch_t 0.607 (0.603)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 20:29:11,887 - Train: 9.09% [207200/2279000] [90.9/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 20:30:12,423 - Train: 9.10% [207300/2279000] [91.0/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 20:31:06,098 - ==> Total time: 1 day, 21:02:26 Eta: 18 days, 17:54:40 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-10 20:31:29,889 - Train: 9.10% [207400/2279000] [91.0/1000.0] [batch_t 0.600 (0.604)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-10 20:32:30,158 - Train: 9.10% [207500/2279000] [91.0/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 20:33:30,467 - Train: 9.11% [207600/2279000] [91.1/1000.0] [batch_t 0.607 (0.603)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 20:34:30,856 - Train: 9.11% [207700/2279000] [91.1/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 20:35:31,349 - Train: 9.12% [207800/2279000] [91.2/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-10 20:36:31,627 - Train: 9.12% [207900/2279000] [91.2/1000.0] [batch_t 0.610 (0.603)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-10 20:37:31,960 - Train: 9.13% [208000/2279000] [91.3/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 20:38:32,273 - Train: 9.13% [208100/2279000] [91.3/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 20:39:32,594 - Train: 9.14% [208200/2279000] [91.4/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 20:40:32,944 - Train: 9.14% [208300/2279000] [91.4/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 20:41:33,297 - Train: 9.14% [208400/2279000] [91.4/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 20:42:33,712 - Train: 9.15% [208500/2279000] [91.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 20:43:34,019 - Train: 9.15% [208600/2279000] [91.5/1000.0] [batch_t 0.601 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 20:44:34,338 - Train: 9.16% [208700/2279000] [91.6/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 20:45:34,696 - Train: 9.16% [208800/2279000] [91.6/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 20:46:35,035 - Train: 9.17% [208900/2279000] [91.7/1000.0] [batch_t 0.599 (0.603)] [data_t 0.002] [optim_t 0.597] [lr 0.005000] +2024-04-10 20:47:35,521 - Train: 9.17% [209000/2279000] [91.7/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 20:48:35,813 - Train: 9.18% [209100/2279000] [91.8/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 20:49:36,120 - Train: 9.18% [209200/2279000] [91.8/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 20:50:36,458 - Train: 9.18% [209300/2279000] [91.8/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 20:51:36,762 - Train: 9.19% [209400/2279000] [91.9/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 20:52:37,054 - Train: 9.19% [209500/2279000] [91.9/1000.0] [batch_t 0.601 (0.603)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 20:53:37,502 - Train: 9.20% [209600/2279000] [92.0/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 20:54:18,521 - ==> Total time: 1 day, 21:25:38 Eta: 18 days, 16:20:55 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-10 20:54:52,723 - Train: 9.20% [209700/2279000] [92.0/1000.0] [batch_t 0.606 (0.603)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 20:55:53,049 - Train: 9.21% [209800/2279000] [92.1/1000.0] [batch_t 0.601 (0.603)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 20:56:53,355 - Train: 9.21% [209900/2279000] [92.1/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 20:57:53,768 - Train: 9.21% [210000/2279000] [92.1/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 20:58:54,226 - Train: 9.22% [210100/2279000] [92.2/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 20:59:54,767 - Train: 9.22% [210200/2279000] [92.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 21:00:55,126 - Train: 9.23% [210300/2279000] [92.3/1000.0] [batch_t 0.609 (0.604)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-10 21:01:55,423 - Train: 9.23% [210400/2279000] [92.3/1000.0] [batch_t 0.600 (0.603)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-10 21:02:55,802 - Train: 9.24% [210500/2279000] [92.4/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 21:03:56,209 - Train: 9.24% [210600/2279000] [92.4/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 21:04:56,512 - Train: 9.25% [210700/2279000] [92.5/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 21:05:56,814 - Train: 9.25% [210800/2279000] [92.5/1000.0] [batch_t 0.600 (0.603)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 21:06:57,358 - Train: 9.25% [210900/2279000] [92.5/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 21:07:57,679 - Train: 9.26% [211000/2279000] [92.6/1000.0] [batch_t 0.601 (0.603)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 21:08:58,021 - Train: 9.26% [211100/2279000] [92.6/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 21:09:58,369 - Train: 9.27% [211200/2279000] [92.7/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 21:10:58,677 - Train: 9.27% [211300/2279000] [92.7/1000.0] [batch_t 0.598 (0.603)] [data_t 0.002] [optim_t 0.596] [lr 0.005000] +2024-04-10 21:11:58,984 - Train: 9.28% [211400/2279000] [92.8/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 21:12:59,288 - Train: 9.28% [211500/2279000] [92.8/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 21:13:59,580 - Train: 9.28% [211600/2279000] [92.8/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 21:15:00,026 - Train: 9.29% [211700/2279000] [92.9/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 21:16:00,378 - Train: 9.29% [211800/2279000] [92.9/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 21:17:00,729 - Train: 9.30% [211900/2279000] [93.0/1000.0] [batch_t 0.601 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 21:17:29,161 - ==> Total time: 1 day, 21:48:49 Eta: 18 days, 14:48:23 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-10 21:18:17,389 - Train: 9.30% [212000/2279000] [93.0/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 21:19:17,742 - Train: 9.31% [212100/2279000] [93.1/1000.0] [batch_t 0.606 (0.603)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 21:20:18,102 - Train: 9.31% [212200/2279000] [93.1/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 21:21:18,560 - Train: 9.32% [212300/2279000] [93.2/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 21:22:18,910 - Train: 9.32% [212400/2279000] [93.2/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 21:23:19,209 - Train: 9.32% [212500/2279000] [93.2/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 21:24:19,524 - Train: 9.33% [212600/2279000] [93.3/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 21:25:19,805 - Train: 9.33% [212700/2279000] [93.3/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 21:26:20,124 - Train: 9.34% [212800/2279000] [93.4/1000.0] [batch_t 0.601 (0.603)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 21:27:20,642 - Train: 9.34% [212900/2279000] [93.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 21:28:20,963 - Train: 9.35% [213000/2279000] [93.5/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 21:29:21,315 - Train: 9.35% [213100/2279000] [93.5/1000.0] [batch_t 0.601 (0.603)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 21:30:21,613 - Train: 9.35% [213200/2279000] [93.5/1000.0] [batch_t 0.607 (0.603)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 21:31:21,940 - Train: 9.36% [213300/2279000] [93.6/1000.0] [batch_t 0.606 (0.603)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 21:32:22,284 - Train: 9.36% [213400/2279000] [93.6/1000.0] [batch_t 0.600 (0.603)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-10 21:33:22,591 - Train: 9.37% [213500/2279000] [93.7/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 21:34:23,082 - Train: 9.37% [213600/2279000] [93.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 21:35:23,406 - Train: 9.38% [213700/2279000] [93.8/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 21:36:23,712 - Train: 9.38% [213800/2279000] [93.8/1000.0] [batch_t 0.601 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 21:37:24,046 - Train: 9.39% [213900/2279000] [93.9/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 21:38:24,389 - Train: 9.39% [214000/2279000] [93.9/1000.0] [batch_t 0.601 (0.603)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 21:39:24,661 - Train: 9.39% [214100/2279000] [93.9/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 21:40:25,117 - Train: 9.40% [214200/2279000] [94.0/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 21:40:40,816 - ==> Total time: 1 day, 22:12:00 Eta: 18 days, 13:17:30 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-10 21:41:43,353 - Train: 9.40% [214300/2279000] [94.0/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 21:42:43,687 - Train: 9.41% [214400/2279000] [94.1/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 21:43:44,007 - Train: 9.41% [214500/2279000] [94.1/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 21:44:44,284 - Train: 9.42% [214600/2279000] [94.2/1000.0] [batch_t 0.601 (0.603)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 21:45:44,768 - Train: 9.42% [214700/2279000] [94.2/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 21:46:45,129 - Train: 9.43% [214800/2279000] [94.3/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 21:47:45,440 - Train: 9.43% [214900/2279000] [94.3/1000.0] [batch_t 0.600 (0.603)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-10 21:48:45,764 - Train: 9.43% [215000/2279000] [94.3/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 21:49:46,092 - Train: 9.44% [215100/2279000] [94.4/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 21:50:46,390 - Train: 9.44% [215200/2279000] [94.4/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 21:51:46,844 - Train: 9.45% [215300/2279000] [94.5/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 21:52:47,177 - Train: 9.45% [215400/2279000] [94.5/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 21:53:47,564 - Train: 9.46% [215500/2279000] [94.6/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 21:54:47,851 - Train: 9.46% [215600/2279000] [94.6/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 21:55:48,159 - Train: 9.46% [215700/2279000] [94.6/1000.0] [batch_t 0.607 (0.603)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 21:56:48,496 - Train: 9.47% [215800/2279000] [94.7/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 21:57:49,012 - Train: 9.47% [215900/2279000] [94.7/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 21:58:49,373 - Train: 9.48% [216000/2279000] [94.8/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 21:59:49,696 - Train: 9.48% [216100/2279000] [94.8/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 22:00:50,072 - Train: 9.49% [216200/2279000] [94.9/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 22:01:50,400 - Train: 9.49% [216300/2279000] [94.9/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 22:02:50,747 - Train: 9.50% [216400/2279000] [95.0/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 22:03:51,197 - Train: 9.50% [216500/2279000] [95.0/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 22:03:54,210 - ==> Total time: 1 day, 22:35:14 Eta: 18 days, 11:48:19 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-10 22:05:07,736 - Train: 9.50% [216600/2279000] [95.0/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 22:06:08,106 - Train: 9.51% [216700/2279000] [95.1/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 22:07:08,397 - Train: 9.51% [216800/2279000] [95.1/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 22:08:08,795 - Train: 9.52% [216900/2279000] [95.2/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 22:09:09,342 - Train: 9.52% [217000/2279000] [95.2/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 22:10:09,726 - Train: 9.53% [217100/2279000] [95.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 22:11:10,185 - Train: 9.53% [217200/2279000] [95.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 22:12:10,489 - Train: 9.53% [217300/2279000] [95.3/1000.0] [batch_t 0.600 (0.603)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-10 22:13:10,821 - Train: 9.54% [217400/2279000] [95.4/1000.0] [batch_t 0.608 (0.603)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-10 22:14:11,214 - Train: 9.54% [217500/2279000] [95.4/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 22:15:11,552 - Train: 9.55% [217600/2279000] [95.5/1000.0] [batch_t 0.599 (0.603)] [data_t 0.002] [optim_t 0.597] [lr 0.005000] +2024-04-10 22:16:11,977 - Train: 9.55% [217700/2279000] [95.5/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 22:17:12,292 - Train: 9.56% [217800/2279000] [95.6/1000.0] [batch_t 0.609 (0.603)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-10 22:18:12,595 - Train: 9.56% [217900/2279000] [95.6/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 22:19:12,928 - Train: 9.57% [218000/2279000] [95.7/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 22:20:13,272 - Train: 9.57% [218100/2279000] [95.7/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 22:21:13,643 - Train: 9.57% [218200/2279000] [95.7/1000.0] [batch_t 0.601 (0.604)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 22:22:13,986 - Train: 9.58% [218300/2279000] [95.8/1000.0] [batch_t 0.609 (0.603)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-10 22:23:14,416 - Train: 9.58% [218400/2279000] [95.8/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 22:24:14,737 - Train: 9.59% [218500/2279000] [95.9/1000.0] [batch_t 0.601 (0.603)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 22:25:15,139 - Train: 9.59% [218600/2279000] [95.9/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 22:26:15,456 - Train: 9.60% [218700/2279000] [96.0/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 22:27:06,126 - ==> Total time: 1 day, 22:58:26 Eta: 18 days, 10:20:16 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-10 22:27:33,623 - Train: 9.60% [218800/2279000] [96.0/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 22:28:33,952 - Train: 9.61% [218900/2279000] [96.1/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 22:29:34,404 - Train: 9.61% [219000/2279000] [96.1/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 22:30:34,779 - Train: 9.61% [219100/2279000] [96.1/1000.0] [batch_t 0.604 (0.604)] [data_t 0.003] [optim_t 0.602] [lr 0.005000] +2024-04-10 22:31:35,222 - Train: 9.62% [219200/2279000] [96.2/1000.0] [batch_t 0.608 (0.604)] [data_t 0.003] [optim_t 0.605] [lr 0.005000] +2024-04-10 22:32:35,640 - Train: 9.62% [219300/2279000] [96.2/1000.0] [batch_t 0.606 (0.604)] [data_t 0.003] [optim_t 0.603] [lr 0.005000] +2024-04-10 22:33:36,181 - Train: 9.63% [219400/2279000] [96.3/1000.0] [batch_t 0.602 (0.605)] [data_t 0.003] [optim_t 0.599] [lr 0.005000] +2024-04-10 22:34:36,712 - Train: 9.63% [219500/2279000] [96.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.003] [optim_t 0.603] [lr 0.005000] +2024-04-10 22:35:37,195 - Train: 9.64% [219600/2279000] [96.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 22:36:37,641 - Train: 9.64% [219700/2279000] [96.4/1000.0] [batch_t 0.604 (0.604)] [data_t 0.003] [optim_t 0.601] [lr 0.005000] +2024-04-10 22:37:38,079 - Train: 9.64% [219800/2279000] [96.4/1000.0] [batch_t 0.603 (0.604)] [data_t 0.003] [optim_t 0.600] [lr 0.005000] +2024-04-10 22:38:38,534 - Train: 9.65% [219900/2279000] [96.5/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 22:39:38,972 - Train: 9.65% [220000/2279000] [96.5/1000.0] [batch_t 0.602 (0.604)] [data_t 0.003] [optim_t 0.600] [lr 0.005000] +2024-04-10 22:40:39,605 - Train: 9.66% [220100/2279000] [96.6/1000.0] [batch_t 0.605 (0.606)] [data_t 0.003] [optim_t 0.603] [lr 0.005000] +2024-04-10 22:41:40,082 - Train: 9.66% [220200/2279000] [96.6/1000.0] [batch_t 0.603 (0.605)] [data_t 0.003] [optim_t 0.600] [lr 0.005000] +2024-04-10 22:42:40,484 - Train: 9.67% [220300/2279000] [96.7/1000.0] [batch_t 0.602 (0.604)] [data_t 0.003] [optim_t 0.600] [lr 0.005000] +2024-04-10 22:43:40,911 - Train: 9.67% [220400/2279000] [96.7/1000.0] [batch_t 0.604 (0.604)] [data_t 0.003] [optim_t 0.601] [lr 0.005000] +2024-04-10 22:44:41,362 - Train: 9.68% [220500/2279000] [96.8/1000.0] [batch_t 0.607 (0.604)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-10 22:45:41,838 - Train: 9.68% [220600/2279000] [96.8/1000.0] [batch_t 0.609 (0.605)] [data_t 0.003] [optim_t 0.606] [lr 0.005000] +2024-04-10 22:46:42,301 - Train: 9.68% [220700/2279000] [96.8/1000.0] [batch_t 0.603 (0.605)] [data_t 0.003] [optim_t 0.600] [lr 0.005000] +2024-04-10 22:47:42,803 - Train: 9.69% [220800/2279000] [96.9/1000.0] [batch_t 0.603 (0.605)] [data_t 0.003] [optim_t 0.601] [lr 0.005000] +2024-04-10 22:48:43,375 - Train: 9.69% [220900/2279000] [96.9/1000.0] [batch_t 0.602 (0.606)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 22:49:43,852 - Train: 9.70% [221000/2279000] [97.0/1000.0] [batch_t 0.604 (0.605)] [data_t 0.003] [optim_t 0.602] [lr 0.005000] +2024-04-10 22:50:21,941 - ==> Total time: 1 day, 23:21:42 Eta: 18 days, 8:54:10 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-10 22:50:59,469 - Train: 9.70% [221100/2279000] [97.0/1000.0] [batch_t 0.607 (0.604)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-10 22:51:59,901 - Train: 9.71% [221200/2279000] [97.1/1000.0] [batch_t 0.603 (0.604)] [data_t 0.003] [optim_t 0.600] [lr 0.005000] +2024-04-10 22:53:00,288 - Train: 9.71% [221300/2279000] [97.1/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 22:54:00,797 - Train: 9.71% [221400/2279000] [97.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 22:55:01,096 - Train: 9.72% [221500/2279000] [97.2/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 22:56:01,395 - Train: 9.72% [221600/2279000] [97.2/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 22:57:01,696 - Train: 9.73% [221700/2279000] [97.3/1000.0] [batch_t 0.609 (0.603)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-10 22:58:02,013 - Train: 9.73% [221800/2279000] [97.3/1000.0] [batch_t 0.601 (0.603)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 22:59:02,347 - Train: 9.74% [221900/2279000] [97.4/1000.0] [batch_t 0.608 (0.603)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-10 23:00:02,833 - Train: 9.74% [222000/2279000] [97.4/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 23:01:03,318 - Train: 9.75% [222100/2279000] [97.5/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 23:02:03,643 - Train: 9.75% [222200/2279000] [97.5/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 23:03:03,991 - Train: 9.75% [222300/2279000] [97.5/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 23:04:04,297 - Train: 9.76% [222400/2279000] [97.6/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 23:05:04,592 - Train: 9.76% [222500/2279000] [97.6/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 23:06:04,899 - Train: 9.77% [222600/2279000] [97.7/1000.0] [batch_t 0.601 (0.603)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 23:07:05,187 - Train: 9.77% [222700/2279000] [97.7/1000.0] [batch_t 0.600 (0.603)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-10 23:08:05,497 - Train: 9.78% [222800/2279000] [97.8/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 23:09:05,858 - Train: 9.78% [222900/2279000] [97.8/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 23:10:06,139 - Train: 9.78% [223000/2279000] [97.8/1000.0] [batch_t 0.601 (0.603)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 23:11:06,442 - Train: 9.79% [223100/2279000] [97.9/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 23:12:06,786 - Train: 9.79% [223200/2279000] [97.9/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 23:13:07,112 - Train: 9.80% [223300/2279000] [98.0/1000.0] [batch_t 0.601 (0.603)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 23:13:32,429 - ==> Total time: 1 day, 23:44:52 Eta: 18 days, 7:28:32 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-10 23:14:23,025 - Train: 9.80% [223400/2279000] [98.0/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 23:15:23,307 - Train: 9.81% [223500/2279000] [98.1/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 23:16:23,639 - Train: 9.81% [223600/2279000] [98.1/1000.0] [batch_t 0.601 (0.603)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 23:17:23,991 - Train: 9.82% [223700/2279000] [98.2/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 23:18:24,362 - Train: 9.82% [223800/2279000] [98.2/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 23:19:24,697 - Train: 9.82% [223900/2279000] [98.2/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 23:20:25,099 - Train: 9.83% [224000/2279000] [98.3/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 23:21:25,583 - Train: 9.83% [224100/2279000] [98.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 23:22:25,934 - Train: 9.84% [224200/2279000] [98.4/1000.0] [batch_t 0.602 (0.603)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 23:23:26,276 - Train: 9.84% [224300/2279000] [98.4/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 23:24:26,612 - Train: 9.85% [224400/2279000] [98.5/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 23:25:26,916 - Train: 9.85% [224500/2279000] [98.5/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 23:26:27,267 - Train: 9.86% [224600/2279000] [98.6/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 23:27:27,651 - Train: 9.86% [224700/2279000] [98.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 23:28:28,113 - Train: 9.86% [224800/2279000] [98.6/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 23:29:28,419 - Train: 9.87% [224900/2279000] [98.7/1000.0] [batch_t 0.606 (0.603)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 23:30:28,763 - Train: 9.87% [225000/2279000] [98.7/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 23:31:29,058 - Train: 9.88% [225100/2279000] [98.8/1000.0] [batch_t 0.599 (0.603)] [data_t 0.002] [optim_t 0.597] [lr 0.005000] +2024-04-10 23:32:29,479 - Train: 9.88% [225200/2279000] [98.8/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 23:33:29,826 - Train: 9.89% [225300/2279000] [98.9/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 23:34:30,121 - Train: 9.89% [225400/2279000] [98.9/1000.0] [batch_t 0.598 (0.603)] [data_t 0.002] [optim_t 0.596] [lr 0.005000] +2024-04-10 23:35:30,578 - Train: 9.89% [225500/2279000] [98.9/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 23:36:30,867 - Train: 9.90% [225600/2279000] [99.0/1000.0] [batch_t 0.606 (0.603)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 23:36:43,538 - ==> Total time: 2 days, 0:08:03 Eta: 18 days, 6:04:16 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_realiad_nhcs57c1o_20240408-232839' +2024-04-10 23:37:48,354 - Train: 9.90% [225700/2279000] [99.0/1000.0] [batch_t 0.599 (0.603)] [data_t 0.002] [optim_t 0.597] [lr 0.005000] +2024-04-10 23:38:48,716 - Train: 9.91% [225800/2279000] [99.1/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 23:39:49,055 - Train: 9.91% [225900/2279000] [99.1/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 23:40:49,383 - Train: 9.92% [226000/2279000] [99.2/1000.0] [batch_t 0.600 (0.603)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 23:41:49,812 - Train: 9.92% [226100/2279000] [99.2/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 23:42:50,241 - Train: 9.93% [226200/2279000] [99.3/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 23:43:50,539 - Train: 9.93% [226300/2279000] [99.3/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 23:44:50,852 - Train: 9.93% [226400/2279000] [99.3/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 23:45:51,167 - Train: 9.94% [226500/2279000] [99.4/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 23:46:51,459 - Train: 9.94% [226600/2279000] [99.4/1000.0] [batch_t 0.599 (0.603)] [data_t 0.002] [optim_t 0.597] [lr 0.005000] +2024-04-10 23:47:51,856 - Train: 9.95% [226700/2279000] [99.5/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 23:48:52,123 - Train: 9.95% [226800/2279000] [99.5/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 23:49:52,462 - Train: 9.96% [226900/2279000] [99.6/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 23:50:52,844 - Train: 9.96% [227000/2279000] [99.6/1000.0] [batch_t 0.600 (0.604)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-10 23:51:53,146 - Train: 9.96% [227100/2279000] [99.6/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 23:52:53,495 - Train: 9.97% [227200/2279000] [99.7/1000.0] [batch_t 0.599 (0.603)] [data_t 0.002] [optim_t 0.597] [lr 0.005000] +2024-04-10 23:53:53,782 - Train: 9.97% [227300/2279000] [99.7/1000.0] [batch_t 0.601 (0.603)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 23:54:54,181 - Train: 9.98% [227400/2279000] [99.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 23:55:54,502 - Train: 9.98% [227500/2279000] [99.8/1000.0] [batch_t 0.600 (0.603)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-10 23:56:54,828 - Train: 9.99% [227600/2279000] [99.9/1000.0] [batch_t 0.601 (0.603)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 23:57:55,115 - Train: 9.99% [227700/2279000] [99.9/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 23:58:55,438 - Train: 10.00% [227800/2279000] [100.0/1000.0] [batch_t 0.601 (0.603)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 23:59:55,742 - Train: 10.00% [227900/2279000] [100.0/1000.0] [batch_t 0.600 (0.603)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-11 00:00:17,644 - Test: 0.70% [50/7162] [batch_t 0.221 (0.228)] +2024-04-11 00:00:28,680 - Test: 1.40% [100/7162] [batch_t 0.214 (0.224)] +2024-04-11 00:00:39,727 - Test: 2.09% [150/7162] [batch_t 0.219 (0.223)] +2024-04-11 00:00:50,365 - Test: 2.79% [200/7162] [batch_t 0.224 (0.221)] +2024-04-11 00:01:01,032 - Test: 3.49% [250/7162] [batch_t 0.217 (0.219)] +2024-04-11 00:01:14,336 - Test: 4.19% [300/7162] [batch_t 0.220 (0.227)] +2024-04-11 00:01:24,951 - Test: 4.89% [350/7162] [batch_t 0.210 (0.225)] +2024-04-11 00:01:35,451 - Test: 5.59% [400/7162] [batch_t 0.211 (0.223)] +2024-04-11 00:01:45,955 - Test: 6.28% [450/7162] [batch_t 0.215 (0.222)] +2024-04-11 00:01:56,463 - Test: 6.98% [500/7162] [batch_t 0.214 (0.220)] +2024-04-11 00:02:10,050 - Test: 7.68% [550/7162] [batch_t 0.204 (0.225)] +2024-04-11 00:02:20,665 - Test: 8.38% [600/7162] [batch_t 0.214 (0.224)] +2024-04-11 00:02:31,301 - Test: 9.08% [650/7162] [batch_t 0.207 (0.223)] +2024-04-11 00:02:41,879 - Test: 9.77% [700/7162] [batch_t 0.208 (0.222)] +2024-04-11 00:02:52,637 - Test: 10.47% [750/7162] [batch_t 0.206 (0.222)] +2024-04-11 00:03:06,814 - Test: 11.17% [800/7162] [batch_t 0.209 (0.226)] +2024-04-11 00:03:17,535 - Test: 11.87% [850/7162] [batch_t 0.205 (0.225)] +2024-04-11 00:03:28,271 - Test: 12.57% [900/7162] [batch_t 0.206 (0.224)] +2024-04-11 00:03:38,938 - Test: 13.26% [950/7162] [batch_t 0.205 (0.224)] +2024-04-11 00:03:49,624 - Test: 13.96% [1000/7162] [batch_t 0.205 (0.223)] +2024-04-11 00:04:00,248 - Test: 14.66% [1050/7162] [batch_t 0.204 (0.223)] +2024-04-11 00:04:12,229 - Test: 15.36% [1100/7162] [batch_t 0.206 (0.224)] +2024-04-11 00:04:22,893 - Test: 16.06% [1150/7162] [batch_t 0.205 (0.223)] +2024-04-11 00:04:33,569 - Test: 16.76% [1200/7162] [batch_t 0.205 (0.223)] +2024-04-11 00:04:44,230 - Test: 17.45% [1250/7162] [batch_t 0.205 (0.222)] +2024-04-11 00:04:54,908 - Test: 18.15% [1300/7162] [batch_t 0.204 (0.222)] +2024-04-11 00:05:07,204 - Test: 18.85% [1350/7162] [batch_t 0.212 (0.223)] +2024-04-11 00:05:17,776 - Test: 19.55% [1400/7162] [batch_t 0.209 (0.222)] +2024-04-11 00:05:28,358 - Test: 20.25% [1450/7162] [batch_t 0.210 (0.222)] +2024-04-11 00:05:38,924 - Test: 20.94% [1500/7162] [batch_t 0.212 (0.222)] +2024-04-11 00:05:49,519 - Test: 21.64% [1550/7162] [batch_t 0.212 (0.221)] +2024-04-11 00:06:00,221 - Test: 22.34% [1600/7162] [batch_t 0.235 (0.221)] +2024-04-11 00:06:11,749 - Test: 23.04% [1650/7162] [batch_t 0.216 (0.221)] +2024-04-11 00:06:22,318 - Test: 23.74% [1700/7162] [batch_t 0.208 (0.221)] +2024-04-11 00:06:32,863 - Test: 24.43% [1750/7162] [batch_t 0.210 (0.221)] +2024-04-11 00:06:43,391 - Test: 25.13% [1800/7162] [batch_t 0.209 (0.221)] +2024-04-11 00:06:53,961 - Test: 25.83% [1850/7162] [batch_t 0.210 (0.220)] +2024-04-11 00:07:06,344 - Test: 26.53% [1900/7162] [batch_t 0.737 (0.221)] +2024-04-11 00:07:16,919 - Test: 27.23% [1950/7162] [batch_t 0.211 (0.221)] +2024-04-11 00:07:27,535 - Test: 27.93% [2000/7162] [batch_t 0.238 (0.221)] +2024-04-11 00:07:38,233 - Test: 28.62% [2050/7162] [batch_t 0.210 (0.220)] +2024-04-11 00:07:48,790 - Test: 29.32% [2100/7162] [batch_t 0.210 (0.220)] +2024-04-11 00:07:59,291 - Test: 30.02% [2150/7162] [batch_t 0.209 (0.220)] +2024-04-11 00:08:11,513 - Test: 30.72% [2200/7162] [batch_t 0.211 (0.221)] +2024-04-11 00:08:22,018 - Test: 31.42% [2250/7162] [batch_t 0.210 (0.220)] +2024-04-11 00:08:32,626 - Test: 32.11% [2300/7162] [batch_t 0.207 (0.220)] +2024-04-11 00:08:43,048 - Test: 32.81% [2350/7162] [batch_t 0.211 (0.220)] +2024-04-11 00:08:53,473 - Test: 33.51% [2400/7162] [batch_t 0.206 (0.220)] +2024-04-11 00:09:05,467 - Test: 34.21% [2450/7162] [batch_t 0.207 (0.220)] +2024-04-11 00:09:16,305 - Test: 34.91% [2500/7162] [batch_t 0.207 (0.220)] +2024-04-11 00:09:26,785 - Test: 35.60% [2550/7162] [batch_t 0.210 (0.220)] +2024-04-11 00:09:37,250 - Test: 36.30% [2600/7162] [batch_t 0.211 (0.220)] +2024-04-11 00:09:47,646 - Test: 37.00% [2650/7162] [batch_t 0.207 (0.219)] +2024-04-11 00:09:58,008 - Test: 37.70% [2700/7162] [batch_t 0.209 (0.219)] +2024-04-11 00:10:10,065 - Test: 38.40% [2750/7162] [batch_t 0.212 (0.220)] +2024-04-11 00:10:20,524 - Test: 39.10% [2800/7162] [batch_t 0.208 (0.219)] +2024-04-11 00:10:30,971 - Test: 39.79% [2850/7162] [batch_t 0.211 (0.219)] +2024-04-11 00:10:41,406 - Test: 40.49% [2900/7162] [batch_t 0.212 (0.219)] +2024-04-11 00:10:51,869 - Test: 41.19% [2950/7162] [batch_t 0.210 (0.219)] +2024-04-11 00:11:02,363 - Test: 41.89% [3000/7162] [batch_t 0.211 (0.219)] +2024-04-11 00:11:15,113 - Test: 42.59% [3050/7162] [batch_t 0.208 (0.219)] +2024-04-11 00:11:25,582 - Test: 43.28% [3100/7162] [batch_t 0.208 (0.219)] +2024-04-11 00:11:36,081 - Test: 43.98% [3150/7162] [batch_t 0.212 (0.219)] +2024-04-11 00:11:46,524 - Test: 44.68% [3200/7162] [batch_t 0.208 (0.219)] +2024-04-11 00:11:56,991 - Test: 45.38% [3250/7162] [batch_t 0.210 (0.219)] +2024-04-11 00:12:08,087 - Test: 46.08% [3300/7162] [batch_t 0.210 (0.219)] +2024-04-11 00:12:18,557 - Test: 46.77% [3350/7162] [batch_t 0.208 (0.219)] +2024-04-11 00:12:29,040 - Test: 47.47% [3400/7162] [batch_t 0.219 (0.218)] +2024-04-11 00:12:39,441 - Test: 48.17% [3450/7162] [batch_t 0.207 (0.218)] +2024-04-11 00:12:49,894 - Test: 48.87% [3500/7162] [batch_t 0.206 (0.218)] +2024-04-11 00:13:00,315 - Test: 49.57% [3550/7162] [batch_t 0.206 (0.218)] +2024-04-11 00:13:12,427 - Test: 50.27% [3600/7162] [batch_t 0.207 (0.218)] +2024-04-11 00:13:22,932 - Test: 50.96% [3650/7162] [batch_t 0.210 (0.218)] +2024-04-11 00:13:33,502 - Test: 51.66% [3700/7162] [batch_t 0.208 (0.218)] +2024-04-11 00:13:44,017 - Test: 52.36% [3750/7162] [batch_t 0.208 (0.218)] +2024-04-11 00:13:54,514 - Test: 53.06% [3800/7162] [batch_t 0.209 (0.218)] +2024-04-11 00:14:07,140 - Test: 53.76% [3850/7162] [batch_t 0.219 (0.218)] +2024-04-11 00:14:17,750 - Test: 54.45% [3900/7162] [batch_t 0.211 (0.218)] +2024-04-11 00:14:28,371 - Test: 55.15% [3950/7162] [batch_t 0.211 (0.218)] +2024-04-11 00:14:38,943 - Test: 55.85% [4000/7162] [batch_t 0.213 (0.218)] +2024-04-11 00:14:49,594 - Test: 56.55% [4050/7162] [batch_t 0.210 (0.218)] +2024-04-11 00:15:00,235 - Test: 57.25% [4100/7162] [batch_t 0.212 (0.218)] +2024-04-11 00:15:13,151 - Test: 57.94% [4150/7162] [batch_t 0.210 (0.218)] +2024-04-11 00:15:23,731 - Test: 58.64% [4200/7162] [batch_t 0.214 (0.218)] +2024-04-11 00:15:34,395 - Test: 59.34% [4250/7162] [batch_t 0.212 (0.218)] +2024-04-11 00:15:45,220 - Test: 60.04% [4300/7162] [batch_t 0.215 (0.218)] +2024-04-11 00:15:55,893 - Test: 60.74% [4350/7162] [batch_t 0.209 (0.218)] +2024-04-11 00:16:07,826 - Test: 61.44% [4400/7162] [batch_t 0.214 (0.218)] +2024-04-11 00:16:18,491 - Test: 62.13% [4450/7162] [batch_t 0.212 (0.218)] +2024-04-11 00:16:29,105 - Test: 62.83% [4500/7162] [batch_t 0.213 (0.218)] +2024-04-11 00:16:39,746 - Test: 63.53% [4550/7162] [batch_t 0.210 (0.218)] +2024-04-11 00:16:50,453 - Test: 64.23% [4600/7162] [batch_t 0.216 (0.218)] +2024-04-11 00:17:01,094 - Test: 64.93% [4650/7162] [batch_t 0.212 (0.218)] +2024-04-11 00:17:14,408 - Test: 65.62% [4700/7162] [batch_t 0.212 (0.219)] +2024-04-11 00:17:25,010 - Test: 66.32% [4750/7162] [batch_t 0.212 (0.219)] +2024-04-11 00:17:35,679 - Test: 67.02% [4800/7162] [batch_t 0.212 (0.219)] +2024-04-11 00:17:46,373 - Test: 67.72% [4850/7162] [batch_t 0.217 (0.219)] +2024-04-11 00:17:57,027 - Test: 68.42% [4900/7162] [batch_t 0.214 (0.218)] +2024-04-11 00:18:10,279 - Test: 69.11% [4950/7162] [batch_t 0.213 (0.219)] +2024-04-11 00:18:20,929 - Test: 69.81% [5000/7162] [batch_t 0.211 (0.219)] +2024-04-11 00:18:32,041 - Test: 70.51% [5050/7162] [batch_t 0.210 (0.219)] +2024-04-11 00:18:42,709 - Test: 71.21% [5100/7162] [batch_t 0.214 (0.219)] +2024-04-11 00:18:53,500 - Test: 71.91% [5150/7162] [batch_t 0.211 (0.219)] +2024-04-11 00:19:05,149 - Test: 72.61% [5200/7162] [batch_t 0.210 (0.219)] +2024-04-11 00:19:16,276 - Test: 73.30% [5250/7162] [batch_t 0.212 (0.219)] +2024-04-11 00:19:27,070 - Test: 74.00% [5300/7162] [batch_t 0.212 (0.219)] +2024-04-11 00:19:37,919 - Test: 74.70% [5350/7162] [batch_t 0.234 (0.219)] +2024-04-11 00:19:48,865 - Test: 75.40% [5400/7162] [batch_t 0.222 (0.219)] +2024-04-11 00:19:59,491 - Test: 76.10% [5450/7162] [batch_t 0.213 (0.219)] +2024-04-11 00:20:12,845 - Test: 76.79% [5500/7162] [batch_t 0.209 (0.219)] +2024-04-11 00:20:23,556 - Test: 77.49% [5550/7162] [batch_t 0.216 (0.219)] +2024-04-11 00:20:34,221 - Test: 78.19% [5600/7162] [batch_t 0.213 (0.219)] +2024-04-11 00:20:44,984 - Test: 78.89% [5650/7162] [batch_t 0.226 (0.219)] +2024-04-11 00:20:55,563 - Test: 79.59% [5700/7162] [batch_t 0.211 (0.219)] +2024-04-11 00:21:08,050 - Test: 80.28% [5750/7162] [batch_t 0.212 (0.219)] +2024-04-11 00:21:18,617 - Test: 80.98% [5800/7162] [batch_t 0.210 (0.219)] +2024-04-11 00:21:29,174 - Test: 81.68% [5850/7162] [batch_t 0.212 (0.219)] +2024-04-11 00:21:39,704 - Test: 82.38% [5900/7162] [batch_t 0.209 (0.219)] +2024-04-11 00:21:50,276 - Test: 83.08% [5950/7162] [batch_t 0.210 (0.219)] +2024-04-11 00:22:00,845 - Test: 83.78% [6000/7162] [batch_t 0.211 (0.219)] +2024-04-11 00:22:12,859 - Test: 84.47% [6050/7162] [batch_t 0.210 (0.219)] +2024-04-11 00:22:23,405 - Test: 85.17% [6100/7162] [batch_t 0.210 (0.219)] +2024-04-11 00:22:33,967 - Test: 85.87% [6150/7162] [batch_t 0.210 (0.219)] +2024-04-11 00:22:44,498 - Test: 86.57% [6200/7162] [batch_t 0.210 (0.219)] +2024-04-11 00:22:55,085 - Test: 87.27% [6250/7162] [batch_t 0.213 (0.219)] +2024-04-11 00:23:06,951 - Test: 87.96% [6300/7162] [batch_t 0.208 (0.219)] +2024-04-11 00:23:17,727 - Test: 88.66% [6350/7162] [batch_t 0.210 (0.219)] +2024-04-11 00:23:28,421 - Test: 89.36% [6400/7162] [batch_t 0.215 (0.219)] +2024-04-11 00:23:39,126 - Test: 90.06% [6450/7162] [batch_t 0.211 (0.219)] +2024-04-11 00:23:49,891 - Test: 90.76% [6500/7162] [batch_t 0.217 (0.219)] +2024-04-11 00:24:00,631 - Test: 91.45% [6550/7162] [batch_t 0.213 (0.219)] +2024-04-11 00:24:13,153 - Test: 92.15% [6600/7162] [batch_t 0.212 (0.219)] +2024-04-11 00:24:23,834 - Test: 92.85% [6650/7162] [batch_t 0.213 (0.219)] +2024-04-11 00:24:34,545 - Test: 93.55% [6700/7162] [batch_t 0.215 (0.219)] +2024-04-11 00:24:45,131 - Test: 94.25% [6750/7162] [batch_t 0.212 (0.219)] +2024-04-11 00:24:55,857 - Test: 94.95% [6800/7162] [batch_t 0.217 (0.219)] +2024-04-11 00:25:07,778 - Test: 95.64% [6850/7162] [batch_t 0.215 (0.219)] +2024-04-11 00:25:18,508 - Test: 96.34% [6900/7162] [batch_t 0.211 (0.219)] +2024-04-11 00:25:29,257 - Test: 97.04% [6950/7162] [batch_t 0.213 (0.219)] +2024-04-11 00:25:40,007 - Test: 97.74% [7000/7162] [batch_t 0.207 (0.219)] +2024-04-11 00:25:50,791 - Test: 98.44% [7050/7162] [batch_t 0.223 (0.219)] +2024-04-11 00:26:01,606 - Test: 99.13% [7100/7162] [batch_t 0.220 (0.219)] +2024-04-11 00:26:13,734 - Test: 99.83% [7150/7162] [batch_t 0.221 (0.219)] +2024-04-11 00:26:16,224 - Test: 100.00% [7162/7162] [batch_t 0.123 (0.219)]