diff --git "a/cocoad/coco1/mambaad_coco1_log_train.txt" "b/cocoad/coco1/mambaad_coco1_log_train.txt" new file mode 100644--- /dev/null +++ "b/cocoad/coco1/mambaad_coco1_log_train.txt" @@ -0,0 +1,10468 @@ +2024-04-02 01:00:29,393 - ==> Logging on master GPU: 0 +2024-04-02 01:00:29,393 - ==> Running Trainer: MAMBAADTrainer +2024-04-02 01:00:29,394 - ==> Using GPU: [0] for Training +2024-04-02 01:00:29,394 - ==> Building model +2024-04-02 01:00:29,824 - Loaded from checkpoint 'model/pretrain/resnet34-43635321.pth' +2024-04-02 01:00:49,852 - +------------------------------------ MAMBAAD ------------------------------------ +| module | #parameters or shape | #flops | +|:------------------------------------------------------------|:-----------------------|:-------------| +| model | 25.667M | 8.3G | +| net_t | 8.17M | 3.955G | +| net_t.conv1 | 9.408K | 0.154G | +| net_t.conv1.weight | (64, 3, 7, 7) | | +| net_t.bn1 | 0.128K | 2.097M | +| net_t.bn1.weight | (64,) | | +| net_t.bn1.bias | (64,) | | +| net_t.layer1 | 0.222M | 0.909G | +| net_t.layer1.0 | 73.984K | 0.303G | +| net_t.layer1.0.conv1 | 36.864K | 0.151G | +| net_t.layer1.0.conv1.weight | (64, 64, 3, 3) | | +| net_t.layer1.0.bn1 | 0.128K | 0.524M | +| net_t.layer1.0.bn1.weight | (64,) | | +| net_t.layer1.0.bn1.bias | (64,) | | +| net_t.layer1.0.conv2 | 36.864K | 0.151G | +| net_t.layer1.0.conv2.weight | (64, 64, 3, 3) | | +| net_t.layer1.0.bn2 | 0.128K | 0.524M | +| net_t.layer1.0.bn2.weight | (64,) | | +| net_t.layer1.0.bn2.bias | (64,) | | +| net_t.layer1.1 | 73.984K | 0.303G | +| net_t.layer1.1.conv1 | 36.864K | 0.151G | +| net_t.layer1.1.conv1.weight | (64, 64, 3, 3) | | +| net_t.layer1.1.bn1 | 0.128K | 0.524M | +| net_t.layer1.1.bn1.weight | (64,) | | +| net_t.layer1.1.bn1.bias | (64,) | | +| net_t.layer1.1.conv2 | 36.864K | 0.151G | +| net_t.layer1.1.conv2.weight | (64, 64, 3, 3) | | +| net_t.layer1.1.bn2 | 0.128K | 0.524M | +| net_t.layer1.1.bn2.weight | (64,) | | +| net_t.layer1.1.bn2.bias | (64,) | | +| net_t.layer1.2 | 73.984K | 0.303G | +| net_t.layer1.2.conv1 | 36.864K | 0.151G | +| net_t.layer1.2.conv1.weight | (64, 64, 3, 3) | | +| net_t.layer1.2.bn1 | 0.128K | 0.524M | +| net_t.layer1.2.bn1.weight | (64,) | | +| net_t.layer1.2.bn1.bias | (64,) | | +| net_t.layer1.2.conv2 | 36.864K | 0.151G | +| net_t.layer1.2.conv2.weight | (64, 64, 3, 3) | | +| net_t.layer1.2.bn2 | 0.128K | 0.524M | +| net_t.layer1.2.bn2.weight | (64,) | | +| net_t.layer1.2.bn2.bias | (64,) | | +| net_t.layer2 | 1.116M | 1.143G | +| net_t.layer2.0 | 0.23M | 0.236G | +| net_t.layer2.0.conv1 | 73.728K | 75.497M | +| net_t.layer2.0.conv1.weight | (128, 64, 3, 3) | | +| net_t.layer2.0.bn1 | 0.256K | 0.262M | +| net_t.layer2.0.bn1.weight | (128,) | | +| net_t.layer2.0.bn1.bias | (128,) | | +| net_t.layer2.0.conv2 | 0.147M | 0.151G | +| net_t.layer2.0.conv2.weight | (128, 128, 3, 3) | | +| net_t.layer2.0.bn2 | 0.256K | 0.262M | +| net_t.layer2.0.bn2.weight | (128,) | | +| net_t.layer2.0.bn2.bias | (128,) | | +| net_t.layer2.0.downsample | 8.448K | 8.651M | +| net_t.layer2.0.downsample.0 | 8.192K | 8.389M | +| net_t.layer2.0.downsample.1 | 0.256K | 0.262M | +| net_t.layer2.1 | 0.295M | 0.303G | +| net_t.layer2.1.conv1 | 0.147M | 0.151G | +| net_t.layer2.1.conv1.weight | (128, 128, 3, 3) | | +| net_t.layer2.1.bn1 | 0.256K | 0.262M | +| net_t.layer2.1.bn1.weight | (128,) | | +| net_t.layer2.1.bn1.bias | (128,) | | +| net_t.layer2.1.conv2 | 0.147M | 0.151G | +| net_t.layer2.1.conv2.weight | (128, 128, 3, 3) | | +| net_t.layer2.1.bn2 | 0.256K | 0.262M | +| net_t.layer2.1.bn2.weight | (128,) | | +| net_t.layer2.1.bn2.bias | (128,) | | +| net_t.layer2.2 | 0.295M | 0.303G | +| net_t.layer2.2.conv1 | 0.147M | 0.151G | +| net_t.layer2.2.conv1.weight | (128, 128, 3, 3) | | +| net_t.layer2.2.bn1 | 0.256K | 0.262M | +| net_t.layer2.2.bn1.weight | (128,) | | +| net_t.layer2.2.bn1.bias | (128,) | | +| net_t.layer2.2.conv2 | 0.147M | 0.151G | +| net_t.layer2.2.conv2.weight | (128, 128, 3, 3) | | +| net_t.layer2.2.bn2 | 0.256K | 0.262M | +| net_t.layer2.2.bn2.weight | (128,) | | +| net_t.layer2.2.bn2.bias | (128,) | | +| net_t.layer2.3 | 0.295M | 0.303G | +| net_t.layer2.3.conv1 | 0.147M | 0.151G | +| net_t.layer2.3.conv1.weight | (128, 128, 3, 3) | | +| net_t.layer2.3.bn1 | 0.256K | 0.262M | +| net_t.layer2.3.bn1.weight | (128,) | | +| net_t.layer2.3.bn1.bias | (128,) | | +| net_t.layer2.3.conv2 | 0.147M | 0.151G | +| net_t.layer2.3.conv2.weight | (128, 128, 3, 3) | | +| net_t.layer2.3.bn2 | 0.256K | 0.262M | +| net_t.layer2.3.bn2.weight | (128,) | | +| net_t.layer2.3.bn2.bias | (128,) | | +| net_t.layer3 | 6.822M | 1.747G | +| net_t.layer3.0 | 0.919M | 0.235G | +| net_t.layer3.0.conv1 | 0.295M | 75.497M | +| net_t.layer3.0.conv1.weight | (256, 128, 3, 3) | | +| net_t.layer3.0.bn1 | 0.512K | 0.131M | +| net_t.layer3.0.bn1.weight | (256,) | | +| net_t.layer3.0.bn1.bias | (256,) | | +| net_t.layer3.0.conv2 | 0.59M | 0.151G | +| net_t.layer3.0.conv2.weight | (256, 256, 3, 3) | | +| net_t.layer3.0.bn2 | 0.512K | 0.131M | +| net_t.layer3.0.bn2.weight | (256,) | | +| net_t.layer3.0.bn2.bias | (256,) | | +| net_t.layer3.0.downsample | 33.28K | 8.52M | +| net_t.layer3.0.downsample.0 | 32.768K | 8.389M | +| net_t.layer3.0.downsample.1 | 0.512K | 0.131M | +| net_t.layer3.1 | 1.181M | 0.302G | +| net_t.layer3.1.conv1 | 0.59M | 0.151G | +| net_t.layer3.1.conv1.weight | (256, 256, 3, 3) | | +| net_t.layer3.1.bn1 | 0.512K | 0.131M | +| net_t.layer3.1.bn1.weight | (256,) | | +| net_t.layer3.1.bn1.bias | (256,) | | +| net_t.layer3.1.conv2 | 0.59M | 0.151G | +| net_t.layer3.1.conv2.weight | (256, 256, 3, 3) | | +| net_t.layer3.1.bn2 | 0.512K | 0.131M | +| net_t.layer3.1.bn2.weight | (256,) | | +| net_t.layer3.1.bn2.bias | (256,) | | +| net_t.layer3.2 | 1.181M | 0.302G | +| net_t.layer3.2.conv1 | 0.59M | 0.151G | +| net_t.layer3.2.conv1.weight | (256, 256, 3, 3) | | +| net_t.layer3.2.bn1 | 0.512K | 0.131M | +| net_t.layer3.2.bn1.weight | (256,) | | +| net_t.layer3.2.bn1.bias | (256,) | | +| net_t.layer3.2.conv2 | 0.59M | 0.151G | +| net_t.layer3.2.conv2.weight | (256, 256, 3, 3) | | +| net_t.layer3.2.bn2 | 0.512K | 0.131M | +| net_t.layer3.2.bn2.weight | (256,) | | +| net_t.layer3.2.bn2.bias | (256,) | | +| net_t.layer3.3 | 1.181M | 0.302G | +| net_t.layer3.3.conv1 | 0.59M | 0.151G | +| net_t.layer3.3.conv1.weight | (256, 256, 3, 3) | | +| net_t.layer3.3.bn1 | 0.512K | 0.131M | +| net_t.layer3.3.bn1.weight | (256,) | | +| net_t.layer3.3.bn1.bias | (256,) | | +| net_t.layer3.3.conv2 | 0.59M | 0.151G | +| net_t.layer3.3.conv2.weight | (256, 256, 3, 3) | | +| net_t.layer3.3.bn2 | 0.512K | 0.131M | +| net_t.layer3.3.bn2.weight | (256,) | | +| net_t.layer3.3.bn2.bias | (256,) | | +| net_t.layer3.4 | 1.181M | 0.302G | +| net_t.layer3.4.conv1 | 0.59M | 0.151G | +| net_t.layer3.4.conv1.weight | (256, 256, 3, 3) | | +| net_t.layer3.4.bn1 | 0.512K | 0.131M | +| net_t.layer3.4.bn1.weight | (256,) | | +| net_t.layer3.4.bn1.bias | (256,) | | +| net_t.layer3.4.conv2 | 0.59M | 0.151G | +| net_t.layer3.4.conv2.weight | (256, 256, 3, 3) | | +| net_t.layer3.4.bn2 | 0.512K | 0.131M | +| net_t.layer3.4.bn2.weight | (256,) | | +| net_t.layer3.4.bn2.bias | (256,) | | +| net_t.layer3.5 | 1.181M | 0.302G | +| net_t.layer3.5.conv1 | 0.59M | 0.151G | +| net_t.layer3.5.conv1.weight | (256, 256, 3, 3) | | +| net_t.layer3.5.bn1 | 0.512K | 0.131M | +| net_t.layer3.5.bn1.weight | (256,) | | +| net_t.layer3.5.bn1.bias | (256,) | | +| net_t.layer3.5.conv2 | 0.59M | 0.151G | +| net_t.layer3.5.conv2.weight | (256, 256, 3, 3) | | +| net_t.layer3.5.bn2 | 0.512K | 0.131M | +| net_t.layer3.5.bn2.weight | (256,) | | +| net_t.layer3.5.bn2.bias | (256,) | | +| mff_oce | 1.458M | 0.269G | +| mff_oce.bn_layer | 0.94M | 66.47M | +| mff_oce.bn_layer.0 | 0.379M | 30.622M | +| mff_oce.bn_layer.0.conv1 | 32.768K | 8.389M | +| mff_oce.bn_layer.0.conv1.weight | (128, 256, 1, 1) | | +| mff_oce.bn_layer.0.bn1 | 0.256K | 65.536K | +| mff_oce.bn_layer.0.bn1.weight | (128,) | | +| mff_oce.bn_layer.0.bn1.bias | (128,) | | +| mff_oce.bn_layer.0.conv2 | 0.147M | 9.437M | +| mff_oce.bn_layer.0.conv2.weight | (128, 128, 3, 3) | | +| mff_oce.bn_layer.0.bn2 | 0.256K | 16.384K | +| mff_oce.bn_layer.0.bn2.weight | (128,) | | +| mff_oce.bn_layer.0.bn2.bias | (128,) | | +| mff_oce.bn_layer.0.conv3 | 65.536K | 4.194M | +| mff_oce.bn_layer.0.conv3.weight | (512, 128, 1, 1) | | +| mff_oce.bn_layer.0.bn3 | 1.024K | 65.536K | +| mff_oce.bn_layer.0.bn3.weight | (512,) | | +| mff_oce.bn_layer.0.bn3.bias | (512,) | | +| mff_oce.bn_layer.0.downsample | 0.132M | 8.454M | +| mff_oce.bn_layer.0.downsample.0 | 0.131M | 8.389M | +| mff_oce.bn_layer.0.downsample.1 | 1.024K | 65.536K | +| mff_oce.bn_layer.1 | 0.28M | 17.924M | +| mff_oce.bn_layer.1.conv1 | 65.536K | 4.194M | +| mff_oce.bn_layer.1.conv1.weight | (128, 512, 1, 1) | | +| mff_oce.bn_layer.1.bn1 | 0.256K | 16.384K | +| mff_oce.bn_layer.1.bn1.weight | (128,) | | +| mff_oce.bn_layer.1.bn1.bias | (128,) | | +| mff_oce.bn_layer.1.conv2 | 0.147M | 9.437M | +| mff_oce.bn_layer.1.conv2.weight | (128, 128, 3, 3) | | +| mff_oce.bn_layer.1.bn2 | 0.256K | 16.384K | +| mff_oce.bn_layer.1.bn2.weight | (128,) | | +| mff_oce.bn_layer.1.bn2.bias | (128,) | | +| mff_oce.bn_layer.1.conv3 | 65.536K | 4.194M | +| mff_oce.bn_layer.1.conv3.weight | (512, 128, 1, 1) | | +| mff_oce.bn_layer.1.bn3 | 1.024K | 65.536K | +| mff_oce.bn_layer.1.bn3.weight | (512,) | | +| mff_oce.bn_layer.1.bn3.bias | (512,) | | +| mff_oce.bn_layer.2 | 0.28M | 17.924M | +| mff_oce.bn_layer.2.conv1 | 65.536K | 4.194M | +| mff_oce.bn_layer.2.conv1.weight | (128, 512, 1, 1) | | +| mff_oce.bn_layer.2.bn1 | 0.256K | 16.384K | +| mff_oce.bn_layer.2.bn1.weight | (128,) | | +| mff_oce.bn_layer.2.bn1.bias | (128,) | | +| mff_oce.bn_layer.2.conv2 | 0.147M | 9.437M | +| mff_oce.bn_layer.2.conv2.weight | (128, 128, 3, 3) | | +| mff_oce.bn_layer.2.bn2 | 0.256K | 16.384K | +| mff_oce.bn_layer.2.bn2.weight | (128,) | | +| mff_oce.bn_layer.2.bn2.bias | (128,) | | +| mff_oce.bn_layer.2.conv3 | 65.536K | 4.194M | +| mff_oce.bn_layer.2.conv3.weight | (512, 128, 1, 1) | | +| mff_oce.bn_layer.2.bn3 | 1.024K | 65.536K | +| mff_oce.bn_layer.2.bn3.weight | (512,) | | +| mff_oce.bn_layer.2.bn3.bias | (512,) | | +| mff_oce.conv1 | 73.728K | 75.497M | +| mff_oce.conv1.weight | (128, 64, 3, 3) | | +| mff_oce.bn1 | 0.256K | 0.262M | +| mff_oce.bn1.weight | (128,) | | +| mff_oce.bn1.bias | (128,) | | +| mff_oce.conv2 | 0.295M | 75.497M | +| mff_oce.conv2.weight | (256, 128, 3, 3) | | +| mff_oce.bn2 | 0.512K | 0.131M | +| mff_oce.bn2.weight | (256,) | | +| mff_oce.bn2.bias | (256,) | | +| mff_oce.conv21 | 16.512K | 16.777M | +| mff_oce.conv21.weight | (128, 128, 1, 1) | | +| mff_oce.conv21.bias | (128,) | | +| mff_oce.bn21 | 0.256K | 0.262M | +| mff_oce.bn21.weight | (128,) | | +| mff_oce.bn21.bias | (128,) | | +| mff_oce.conv31 | 65.792K | 16.777M | +| mff_oce.conv31.weight | (256, 256, 1, 1) | | +| mff_oce.conv31.bias | (256,) | | +| mff_oce.bn31 | 0.512K | 0.131M | +| mff_oce.bn31.weight | (256,) | | +| mff_oce.bn31.bias | (256,) | | +| mff_oce.convf | 65.792K | 16.777M | +| mff_oce.convf.weight | (256, 256, 1, 1) | | +| mff_oce.convf.bias | (256,) | | +| mff_oce.bnf | 0.512K | 0.131M | +| mff_oce.bnf.weight | (256,) | | +| mff_oce.bnf.bias | (256,) | | +| net_s.layers_up | 16.038M | 4.076G | +| net_s.layers_up.0.blocks.0 | 9.441M | 0.577G | +| net_s.layers_up.0.blocks.0.smm_blocks | 7.561M | 0.456G | +| net_s.layers_up.0.blocks.0.smm_blocks.0 | 2.52M | 0.152G | +| net_s.layers_up.0.blocks.0.smm_blocks.0.ln_1 | 1.024K | 0.164M | +| net_s.layers_up.0.blocks.0.smm_blocks.0.self_attention | 2.519M | 0.152G | +| net_s.layers_up.0.blocks.0.smm_blocks.1 | 2.52M | 0.152G | +| net_s.layers_up.0.blocks.0.smm_blocks.1.ln_1 | 1.024K | 0.164M | +| net_s.layers_up.0.blocks.0.smm_blocks.1.self_attention | 2.519M | 0.152G | +| net_s.layers_up.0.blocks.0.smm_blocks.2 | 2.52M | 0.152G | +| net_s.layers_up.0.blocks.0.smm_blocks.2.ln_1 | 1.024K | 0.164M | +| net_s.layers_up.0.blocks.0.smm_blocks.2.self_attention | 2.519M | 0.152G | +| net_s.layers_up.0.blocks.0.conv1b3 | 0.263M | 16.908M | +| net_s.layers_up.0.blocks.0.conv1b3.0 | 0.263M | 16.777M | +| net_s.layers_up.0.blocks.0.conv1b3.0.weight | (512, 512, 1, 1) | | +| net_s.layers_up.0.blocks.0.conv1b3.0.bias | (512,) | | +| net_s.layers_up.0.blocks.0.conv1b3.1 | | 0.131M | +| net_s.layers_up.0.blocks.0.conv1a3 | 0.263M | 16.908M | +| net_s.layers_up.0.blocks.0.conv1a3.0 | 0.263M | 16.777M | +| net_s.layers_up.0.blocks.0.conv1a3.0.weight | (512, 512, 1, 1) | | +| net_s.layers_up.0.blocks.0.conv1a3.0.bias | (512,) | | +| net_s.layers_up.0.blocks.0.conv1a3.1 | | 0.131M | +| net_s.layers_up.0.blocks.0.conv1b5 | 0.263M | 16.908M | +| net_s.layers_up.0.blocks.0.conv1b5.0 | 0.263M | 16.777M | +| net_s.layers_up.0.blocks.0.conv1b5.0.weight | (512, 512, 1, 1) | | +| net_s.layers_up.0.blocks.0.conv1b5.0.bias | (512,) | | +| net_s.layers_up.0.blocks.0.conv1b5.1 | | 0.131M | +| net_s.layers_up.0.blocks.0.conv1a5 | 0.263M | 16.908M | +| net_s.layers_up.0.blocks.0.conv1a5.0 | 0.263M | 16.777M | +| net_s.layers_up.0.blocks.0.conv1a5.0.weight | (512, 512, 1, 1) | | +| net_s.layers_up.0.blocks.0.conv1a5.0.bias | (512,) | | +| net_s.layers_up.0.blocks.0.conv1a5.1 | | 0.131M | +| net_s.layers_up.0.blocks.0.conv33.0 | 4.608K | | +| net_s.layers_up.0.blocks.0.conv33.0.weight | (512, 1, 3, 3) | | +| net_s.layers_up.0.blocks.0.conv55 | 12.8K | 0.95M | +| net_s.layers_up.0.blocks.0.conv55.0 | 12.8K | 0.819M | +| net_s.layers_up.0.blocks.0.conv55.0.weight | (512, 1, 5, 5) | | +| net_s.layers_up.0.blocks.0.conv55.1 | | 0.131M | +| net_s.layers_up.0.blocks.0.conv77 | 25.088K | 1.737M | +| net_s.layers_up.0.blocks.0.conv77.0 | 25.088K | 1.606M | +| net_s.layers_up.0.blocks.0.conv77.0.weight | (512, 1, 7, 7) | | +| net_s.layers_up.0.blocks.0.conv77.1 | | 0.131M | +| net_s.layers_up.0.blocks.0.finalconv11 | 0.787M | 50.332M | +| net_s.layers_up.0.blocks.0.finalconv11.weight | (512, 1536, 1, 1) | | +| net_s.layers_up.0.blocks.0.finalconv11.bias | (512,) | | +| net_s.layers_up.1 | 4.432M | 0.961G | +| net_s.layers_up.1.blocks | 3.908M | 0.928G | +| net_s.layers_up.1.blocks.0 | 1.954M | 0.464G | +| net_s.layers_up.1.blocks.0.smm_blocks | 1.473M | 0.34G | +| net_s.layers_up.1.blocks.0.conv1b3 | 65.792K | 17.039M | +| net_s.layers_up.1.blocks.0.conv1a3 | 65.792K | 17.039M | +| net_s.layers_up.1.blocks.0.conv1b5 | 65.792K | 17.039M | +| net_s.layers_up.1.blocks.0.conv1a5 | 65.792K | 17.039M | +| net_s.layers_up.1.blocks.0.conv33.0 | 2.304K | | +| net_s.layers_up.1.blocks.0.conv55 | 6.4K | 1.901M | +| net_s.layers_up.1.blocks.0.conv77 | 12.544K | 3.473M | +| net_s.layers_up.1.blocks.0.finalconv11 | 0.197M | 50.332M | +| net_s.layers_up.1.blocks.1 | 1.954M | 0.464G | +| net_s.layers_up.1.blocks.1.smm_blocks | 1.473M | 0.34G | +| net_s.layers_up.1.blocks.1.conv1b3 | 65.792K | 17.039M | +| net_s.layers_up.1.blocks.1.conv1a3 | 65.792K | 17.039M | +| net_s.layers_up.1.blocks.1.conv1b5 | 65.792K | 17.039M | +| net_s.layers_up.1.blocks.1.conv1a5 | 65.792K | 17.039M | +| net_s.layers_up.1.blocks.1.conv33.0 | 2.304K | | +| net_s.layers_up.1.blocks.1.conv55 | 6.4K | 1.901M | +| net_s.layers_up.1.blocks.1.conv77 | 12.544K | 3.473M | +| net_s.layers_up.1.blocks.1.finalconv11 | 0.197M | 50.332M | +| net_s.layers_up.1.upsample | 0.525M | 33.882M | +| net_s.layers_up.1.upsample.expand | 0.524M | 33.554M | +| net_s.layers_up.1.upsample.expand.weight | (1024, 512) | | +| net_s.layers_up.1.upsample.norm | 0.512K | 0.328M | +| net_s.layers_up.1.upsample.norm.weight | (256,) | | +| net_s.layers_up.1.upsample.norm.bias | (256,) | | +| net_s.layers_up.2 | 1.816M | 1.529G | +| net_s.layers_up.2.blocks | 1.685M | 1.495G | +| net_s.layers_up.2.blocks.0 | 0.842M | 0.747G | +| net_s.layers_up.2.blocks.0.smm_blocks | 0.717M | 0.617G | +| net_s.layers_up.2.blocks.0.conv1b3 | 16.512K | 17.302M | +| net_s.layers_up.2.blocks.0.conv1a3 | 16.512K | 17.302M | +| net_s.layers_up.2.blocks.0.conv1b5 | 16.512K | 17.302M | +| net_s.layers_up.2.blocks.0.conv1a5 | 16.512K | 17.302M | +| net_s.layers_up.2.blocks.0.conv33.0 | 1.152K | | +| net_s.layers_up.2.blocks.0.conv55 | 3.2K | 3.801M | +| net_s.layers_up.2.blocks.0.conv77 | 6.272K | 6.947M | +| net_s.layers_up.2.blocks.0.finalconv11 | 49.28K | 50.332M | +| net_s.layers_up.2.blocks.1 | 0.842M | 0.747G | +| net_s.layers_up.2.blocks.1.smm_blocks | 0.717M | 0.617G | +| net_s.layers_up.2.blocks.1.conv1b3 | 16.512K | 17.302M | +| net_s.layers_up.2.blocks.1.conv1a3 | 16.512K | 17.302M | +| net_s.layers_up.2.blocks.1.conv1b5 | 16.512K | 17.302M | +| net_s.layers_up.2.blocks.1.conv1a5 | 16.512K | 17.302M | +| net_s.layers_up.2.blocks.1.conv33.0 | 1.152K | | +| net_s.layers_up.2.blocks.1.conv55 | 3.2K | 3.801M | +| net_s.layers_up.2.blocks.1.conv77 | 6.272K | 6.947M | +| net_s.layers_up.2.blocks.1.finalconv11 | 49.28K | 50.332M | +| net_s.layers_up.2.upsample | 0.131M | 34.21M | +| net_s.layers_up.2.upsample.expand | 0.131M | 33.554M | +| net_s.layers_up.2.upsample.expand.weight | (512, 256) | | +| net_s.layers_up.2.upsample.norm | 0.256K | 0.655M | +| net_s.layers_up.2.upsample.norm.weight | (128,) | | +| net_s.layers_up.2.upsample.norm.bias | (128,) | | +| net_s.layers_up.3 | 0.349M | 1.009G | +| net_s.layers_up.3.blocks.0 | 0.316M | 0.974G | +| net_s.layers_up.3.blocks.0.smm_blocks | 0.281M | 0.831G | +| net_s.layers_up.3.blocks.0.smm_blocks.0 | 93.824K | 0.277G | +| net_s.layers_up.3.blocks.0.smm_blocks.1 | 93.824K | 0.277G | +| net_s.layers_up.3.blocks.0.smm_blocks.2 | 93.824K | 0.277G | +| net_s.layers_up.3.blocks.0.conv1b3 | 4.16K | 17.826M | +| net_s.layers_up.3.blocks.0.conv1b3.0 | 4.16K | 16.777M | +| net_s.layers_up.3.blocks.0.conv1b3.1 | | 1.049M | +| net_s.layers_up.3.blocks.0.conv1a3 | 4.16K | 17.826M | +| net_s.layers_up.3.blocks.0.conv1a3.0 | 4.16K | 16.777M | +| net_s.layers_up.3.blocks.0.conv1a3.1 | | 1.049M | +| net_s.layers_up.3.blocks.0.conv1b5 | 4.16K | 17.826M | +| net_s.layers_up.3.blocks.0.conv1b5.0 | 4.16K | 16.777M | +| net_s.layers_up.3.blocks.0.conv1b5.1 | | 1.049M | +| net_s.layers_up.3.blocks.0.conv1a5 | 4.16K | 17.826M | +| net_s.layers_up.3.blocks.0.conv1a5.0 | 4.16K | 16.777M | +| net_s.layers_up.3.blocks.0.conv1a5.1 | | 1.049M | +| net_s.layers_up.3.blocks.0.conv33.0 | 0.576K | | +| net_s.layers_up.3.blocks.0.conv33.0.weight | (64, 1, 3, 3) | | +| net_s.layers_up.3.blocks.0.conv55 | 1.6K | 7.602M | +| net_s.layers_up.3.blocks.0.conv55.0 | 1.6K | 6.554M | +| net_s.layers_up.3.blocks.0.conv55.1 | | 1.049M | +| net_s.layers_up.3.blocks.0.conv77 | 3.136K | 13.894M | +| net_s.layers_up.3.blocks.0.conv77.0 | 3.136K | 12.845M | +| net_s.layers_up.3.blocks.0.conv77.1 | | 1.049M | +| net_s.layers_up.3.blocks.0.finalconv11 | 12.352K | 50.332M | +| net_s.layers_up.3.blocks.0.finalconv11.weight | (64, 192, 1, 1) | | +| net_s.layers_up.3.blocks.0.finalconv11.bias | (64,) | | +| net_s.layers_up.3.upsample | 32.896K | 34.865M | +| net_s.layers_up.3.upsample.expand | 32.768K | 33.554M | +| net_s.layers_up.3.upsample.expand.weight | (256, 128) | | +| net_s.layers_up.3.upsample.norm | 0.128K | 1.311M | +| net_s.layers_up.3.upsample.norm.weight | (64,) | | +| net_s.layers_up.3.upsample.norm.bias | (64,) | | +--------------------------------------------------------------------------------- +2024-04-02 01:00:49,853 - ==> Creating optimizer +2024-04-02 01:00:49,950 - ==> Loading dataset: DefaultAD +2024-04-02 01:00:50,357 - ==> ********** cfg ********** +epoch_full : 1000 +fvcore_b : 1 +fvcore_c : 3 +metrics : ['mAUROC_sp_max', 'mAP_sp_max', 'mF1_max_sp_max', 'mAUROC_px', 'mAP_px', 'mF1_max_px', 'mAUPRO_px', 'mF1_px_0.2_0.8_0.1', 'mAcc_px_0.2_0.8_0.1', 'mIoU_px_0.2_0.8_0.1', 'mIoU_max_px'] +evaluator.kwargs : {'metrics': ['mAUROC_sp_max', 'mAP_sp_max', 'mF1_max_sp_max', 'mAUROC_px', 'mAP_px', 'mF1_max_px', 'mAUPRO_px', 'mF1_px_0.2_0.8_0.1', 'mAcc_px_0.2_0.8_0.1', 'mIoU_px_0.2_0.8_0.1', 'mIoU_max_px'], 'pooling_ks': None, 'max_step_aupro': 100} +optim.lr : 0.005 +optim.kwargs : {'name': 'adamw', 'betas': (0.9, 0.999), 'eps': 1e-08, 'weight_decay': 0.0001, 'amsgrad': False} +trainer.name : MAMBAADTrainer +trainer.checkpoint : runs +trainer.logdir_sub : +trainer.resume_dir : +trainer.cuda_deterministic : False +trainer.epoch_full : 1000 +trainer.scheduler_kwargs : {'name': 'step', 'lr_noise': None, 'noise_pct': 0.67, 'noise_std': 1.0, 'noise_seed': 42, 'lr_min': 5e-05, 'warmup_lr': 5e-06, 'warmup_iters': -1, 'cooldown_iters': 0, 'warmup_epochs': 0, 'cooldown_epochs': 0, 'use_iters': True, 'patience_iters': 0, 'patience_epochs': 0, 'decay_iters': 0, 'decay_epochs': 800, 'cycle_decay': 0.1, 'decay_rate': 0.1} +trainer.mixup_kwargs : {'mixup_alpha': 0.8, 'cutmix_alpha': 1.0, 'cutmix_minmax': None, 'prob': 0.0, 'switch_prob': 0.5, 'mode': 'batch', 'correct_lam': True, 'label_smoothing': 0.1} +trainer.test_start_epoch : 1000 +trainer.test_per_epoch : 200 +trainer.find_unused_parameters : False +trainer.sync_BN : apex +trainer.dist_BN : +trainer.scaler : none +trainer.data.batch_size : 16 +trainer.data.batch_size_per_gpu : 16 +trainer.data.batch_size_test : 16 +trainer.data.batch_size_per_gpu_test : 16 +trainer.data.num_workers_per_gpu : 4 +trainer.data.drop_last : True +trainer.data.pin_memory : True +trainer.data.persistent_workers : False +trainer.data.num_workers : 4 +trainer.iter : 0 +trainer.epoch : 0 +trainer.iter_full : 4070000 +trainer.metric_recorder : {'mAUROC_sp_max_coco': [], 'mAP_sp_max_coco': [], 'mF1_max_sp_max_coco': [], 'mAUROC_px_coco': [], 'mAP_px_coco': [], 'mF1_max_px_coco': [], 'mAUPRO_px_coco': [], 'mF1_px_0.2_0.8_0.1_coco': [], 'mAcc_px_0.2_0.8_0.1_coco': [], 'mIoU_px_0.2_0.8_0.1_coco': [], 'mIoU_max_px_coco': []} +loss.loss_terms : [{'type': 'L2Loss', 'name': 'pixel', 'lam': 5.0}] +loss.clip_grad : 5.0 +loss.create_graph : False +loss.retain_graph : False +adv : False +logging.log_terms_train : [{'name': 'batch_t', 'fmt': ':>5.3f', 'add_name': 'avg'}, {'name': 'data_t', 'fmt': ':>5.3f'}, {'name': 'optim_t', 'fmt': ':>5.3f'}, {'name': 'lr', 'fmt': ':>7.6f'}, {'name': 'cos', 'suffixes': [''], 'fmt': ':>5.3f', 'add_name': 'avg'}] +logging.log_terms_test : [{'name': 'batch_t', 'fmt': ':>5.3f', 'add_name': 'avg'}, {'name': 'cos', 'suffixes': [''], 'fmt': ':>5.3f', 'add_name': 'avg'}] +logging.train_reset_log_per : 100 +logging.train_log_per : 100 +logging.test_log_per : 50 +data.sampler : naive +data.loader_type : pil +data.loader_type_target : pil_L +data.type : DefaultAD +data.root : data/coco +data.meta : meta_20_1.json +data.cls_names : ['coco'] +data.train_transforms : [{'type': 'Resize', 'size': (256, 256), 'interpolation': }, {'type': 'CenterCrop', 'size': (256, 256)}, {'type': 'ToTensor'}, {'type': 'Normalize', 'mean': (0.485, 0.456, 0.406), 'std': (0.229, 0.224, 0.225), 'inplace': True}] +data.test_transforms : [{'type': 'Resize', 'size': (256, 256), 'interpolation': }, {'type': 'CenterCrop', 'size': (256, 256)}, {'type': 'ToTensor'}, {'type': 'Normalize', 'mean': (0.485, 0.456, 0.406), 'std': (0.229, 0.224, 0.225), 'inplace': True}] +data.target_transforms : [{'type': 'Resize', 'size': (256, 256), 'interpolation': }, {'type': 'CenterCrop', 'size': (256, 256)}, {'type': 'ToTensor'}] +data.train_size : 4070 +data.test_size : 310 +data.train_length : 65133 +data.test_length : 4952 +model_t.name : timm_resnet34 +model_t.kwargs : {'pretrained': False, 'checkpoint_path': 'model/pretrain/resnet34-43635321.pth', 'strict': False, 'features_only': True, 'out_indices': [1, 2, 3]} +model_s : {'depths_decoder': [3, 4, 6, 3], 'scan_type': 'hilbert', 'num_direction': 8} +model.name : mambaadhcs57c1 +model.kwargs : {'pretrained': False, 'checkpoint_path': '', 'strict': True, 'model_t': Namespace(name='timm_resnet34', kwargs={'pretrained': False, 'checkpoint_path': 'model/pretrain/resnet34-43635321.pth', 'strict': False, 'features_only': True, 'out_indices': [1, 2, 3]}), 'model_s': {'depths_decoder': [3, 4, 6, 3], 'scan_type': 'hilbert', 'num_direction': 8}} +seed : 42 +size : 256 +warmup_epochs : 0 +test_start_epoch : 1000 +test_per_epoch : 200 +batch_train : 16 +batch_test_per : 16 +lr : 0.005 +weight_decay : 0.0001 +cfg_path : configs.mambaad.mambaad_coco1_nhcs57c1 +mode : train +sleep : -1 +memory : -1 +dist_url : env:// +logger_rank : 0 +opts : [] +command : python3 -m torch.distributed.launch --nproc_per_node=$nproc_per_node --nnodes=$nnodes --node_rank=$node_rank --master_addr=$master_addr --master_port=$master_port --use_env run.py -c configs.mambaad.mambaad_coco1_nhcs57c1 -m train --sleep -1 --memory -1 --dist_url env:// --logger_rank 0 +task_start_time : 3592060.481483774 +dist : False +world_size : 1 +rank : 0 +local_rank : 0 +ngpus_per_node : 1 +nnodes : 1 +master : True +logdir : runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029 +logger.filters : [] +logger.name : root +logger.level : 20 +logger.parent : None +logger.propagate : True +logger.disabled : False +logdir_train : runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029/show_train +logdir_test : runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029/show_test +2024-04-02 01:00:50,357 - ==> Starting training with 1 nodes x 1 GPUs +2024-04-02 01:03:02,276 - Train: 0.00% [100/4070000] [0.0/1000.0] [batch_t 0.605 (1.316)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 01:04:14,278 - Train: 0.00% [200/4070000] [0.0/1000.0] [batch_t 0.611 (0.720)] [data_t 0.002] [optim_t 0.609] [lr 0.005000] +2024-04-02 01:05:22,406 - Train: 0.01% [300/4070000] [0.1/1000.0] [batch_t 0.606 (0.681)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 01:06:23,832 - Train: 0.01% [400/4070000] [0.1/1000.0] [batch_t 0.606 (0.614)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 01:11:08,853 - Train: 0.01% [500/4070000] [0.1/1000.0] [batch_t 16.140 (2.849)] [data_t 15.538] [optim_t 0.602] [lr 0.005000] +2024-04-02 01:17:52,232 - Train: 0.01% [600/4070000] [0.1/1000.0] [batch_t 1.424 (4.034)] [data_t 0.003] [optim_t 1.422] [lr 0.005000] +2024-04-02 01:20:15,453 - Train: 0.02% [700/4070000] [0.2/1000.0] [batch_t 1.440 (1.432)] [data_t 0.002] [optim_t 1.438] [lr 0.005000] +2024-04-02 01:22:38,547 - Train: 0.02% [800/4070000] [0.2/1000.0] [batch_t 1.429 (1.431)] [data_t 0.003] [optim_t 1.426] [lr 0.005000] +2024-04-02 01:25:01,659 - Train: 0.02% [900/4070000] [0.2/1000.0] [batch_t 1.425 (1.431)] [data_t 0.003] [optim_t 1.422] [lr 0.005000] +2024-04-02 01:27:24,705 - Train: 0.02% [1000/4070000] [0.2/1000.0] [batch_t 1.435 (1.430)] [data_t 0.003] [optim_t 1.432] [lr 0.005000] +2024-04-02 01:29:47,806 - Train: 0.03% [1100/4070000] [0.3/1000.0] [batch_t 1.432 (1.431)] [data_t 0.003] [optim_t 1.429] [lr 0.005000] +2024-04-02 01:32:10,897 - Train: 0.03% [1200/4070000] [0.3/1000.0] [batch_t 1.435 (1.431)] [data_t 0.003] [optim_t 1.432] [lr 0.005000] +2024-04-02 01:34:34,021 - Train: 0.03% [1300/4070000] [0.3/1000.0] [batch_t 1.425 (1.431)] [data_t 0.003] [optim_t 1.422] [lr 0.005000] +2024-04-02 01:36:57,207 - Train: 0.03% [1400/4070000] [0.3/1000.0] [batch_t 1.431 (1.432)] [data_t 0.002] [optim_t 1.428] [lr 0.005000] +2024-04-02 01:39:20,960 - Train: 0.04% [1500/4070000] [0.4/1000.0] [batch_t 1.433 (1.437)] [data_t 0.003] [optim_t 1.430] [lr 0.005000] +2024-04-02 01:41:43,985 - Train: 0.04% [1600/4070000] [0.4/1000.0] [batch_t 1.427 (1.430)] [data_t 0.002] [optim_t 1.425] [lr 0.005000] +2024-04-02 01:44:07,204 - Train: 0.04% [1700/4070000] [0.4/1000.0] [batch_t 1.437 (1.432)] [data_t 0.003] [optim_t 1.434] [lr 0.005000] +2024-04-02 01:46:30,277 - Train: 0.04% [1800/4070000] [0.4/1000.0] [batch_t 1.431 (1.431)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-02 01:48:53,451 - Train: 0.05% [1900/4070000] [0.5/1000.0] [batch_t 1.439 (1.432)] [data_t 0.002] [optim_t 1.437] [lr 0.005000] +2024-04-02 01:51:16,637 - Train: 0.05% [2000/4070000] [0.5/1000.0] [batch_t 1.434 (1.432)] [data_t 0.003] [optim_t 1.431] [lr 0.005000] +2024-04-02 01:53:39,599 - Train: 0.05% [2100/4070000] [0.5/1000.0] [batch_t 1.443 (1.430)] [data_t 0.003] [optim_t 1.440] [lr 0.005000] +2024-04-02 01:56:02,953 - Train: 0.05% [2200/4070000] [0.5/1000.0] [batch_t 1.437 (1.433)] [data_t 0.003] [optim_t 1.434] [lr 0.005000] +2024-04-02 01:58:25,950 - Train: 0.06% [2300/4070000] [0.6/1000.0] [batch_t 1.446 (1.430)] [data_t 0.002] [optim_t 1.444] [lr 0.005000] +2024-04-02 02:00:49,037 - Train: 0.06% [2400/4070000] [0.6/1000.0] [batch_t 1.429 (1.431)] [data_t 0.002] [optim_t 1.427] [lr 0.005000] +2024-04-02 02:03:12,188 - Train: 0.06% [2500/4070000] [0.6/1000.0] [batch_t 1.433 (1.431)] [data_t 0.003] [optim_t 1.430] [lr 0.005000] +2024-04-02 02:05:35,380 - Train: 0.06% [2600/4070000] [0.6/1000.0] [batch_t 1.434 (1.432)] [data_t 0.003] [optim_t 1.431] [lr 0.005000] +2024-04-02 02:07:36,864 - Train: 0.07% [2700/4070000] [0.7/1000.0] [batch_t 0.604 (1.215)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 02:08:47,829 - Train: 0.07% [2800/4070000] [0.7/1000.0] [batch_t 0.604 (0.710)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 02:09:51,201 - Train: 0.07% [2900/4070000] [0.7/1000.0] [batch_t 0.608 (0.634)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-02 02:10:53,002 - Train: 0.07% [3000/4070000] [0.7/1000.0] [batch_t 0.605 (0.618)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 02:11:53,490 - Train: 0.08% [3100/4070000] [0.8/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 02:12:57,200 - Train: 0.08% [3200/4070000] [0.8/1000.0] [batch_t 0.605 (0.637)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 02:14:13,408 - Train: 0.08% [3300/4070000] [0.8/1000.0] [batch_t 0.604 (0.762)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 02:15:16,952 - Train: 0.08% [3400/4070000] [0.8/1000.0] [batch_t 0.606 (0.635)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 02:16:20,263 - Train: 0.09% [3500/4070000] [0.9/1000.0] [batch_t 0.604 (0.633)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 02:17:22,416 - Train: 0.09% [3600/4070000] [0.9/1000.0] [batch_t 0.603 (0.621)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-02 02:18:35,329 - Train: 0.09% [3700/4070000] [0.9/1000.0] [batch_t 0.605 (0.729)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 02:19:35,927 - Train: 0.09% [3800/4070000] [0.9/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 02:20:36,430 - Train: 0.10% [3900/4070000] [1.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 02:21:39,309 - Train: 0.10% [4000/4070000] [1.0/1000.0] [batch_t 0.605 (0.629)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 02:22:24,035 - ==> Total time: 1:21:54 Eta: 56 days, 19:52:57 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-02 02:22:44,254 - Train: 0.10% [4100/4070000] [1.0/1000.0] [batch_t 0.605 (0.617)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 02:23:48,538 - Train: 0.10% [4200/4070000] [1.0/1000.0] [batch_t 0.605 (0.643)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 02:24:51,958 - Train: 0.11% [4300/4070000] [1.1/1000.0] [batch_t 0.608 (0.634)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-02 02:25:52,447 - Train: 0.11% [4400/4070000] [1.1/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-02 02:26:53,032 - Train: 0.11% [4500/4070000] [1.1/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 02:27:57,829 - Train: 0.11% [4600/4070000] [1.1/1000.0] [batch_t 0.606 (0.648)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 02:28:59,974 - Train: 0.12% [4700/4070000] [1.2/1000.0] [batch_t 0.604 (0.621)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 02:30:01,146 - Train: 0.12% [4800/4070000] [1.2/1000.0] [batch_t 0.606 (0.612)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 02:31:03,837 - Train: 0.12% [4900/4070000] [1.2/1000.0] [batch_t 0.606 (0.627)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 02:32:06,666 - Train: 0.12% [5000/4070000] [1.2/1000.0] [batch_t 0.605 (0.628)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 02:33:10,031 - Train: 0.13% [5100/4070000] [1.3/1000.0] [batch_t 0.604 (0.634)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 02:34:15,876 - Train: 0.13% [5200/4070000] [1.3/1000.0] [batch_t 0.606 (0.658)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 02:35:16,388 - Train: 0.13% [5300/4070000] [1.3/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 02:36:19,769 - Train: 0.13% [5400/4070000] [1.3/1000.0] [batch_t 0.603 (0.634)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-02 02:37:20,267 - Train: 0.14% [5500/4070000] [1.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 02:38:22,296 - Train: 0.14% [5600/4070000] [1.4/1000.0] [batch_t 0.607 (0.620)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 02:39:22,862 - Train: 0.14% [5700/4070000] [1.4/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-02 02:40:23,688 - Train: 0.14% [5800/4070000] [1.4/1000.0] [batch_t 0.609 (0.608)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-02 02:41:25,379 - Train: 0.14% [5900/4070000] [1.4/1000.0] [batch_t 0.605 (0.617)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 02:42:25,903 - Train: 0.15% [6000/4070000] [1.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 02:43:26,583 - Train: 0.15% [6100/4070000] [1.5/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 02:44:27,923 - Train: 0.15% [6200/4070000] [1.5/1000.0] [batch_t 0.605 (0.613)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 02:45:31,518 - Train: 0.15% [6300/4070000] [1.5/1000.0] [batch_t 0.608 (0.636)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-02 02:46:36,534 - Train: 0.16% [6400/4070000] [1.6/1000.0] [batch_t 0.601 (0.650)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-02 02:47:40,071 - Train: 0.16% [6500/4070000] [1.6/1000.0] [batch_t 0.604 (0.635)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 02:48:40,984 - Train: 0.16% [6600/4070000] [1.6/1000.0] [batch_t 0.606 (0.609)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 02:49:41,553 - Train: 0.16% [6700/4070000] [1.6/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 02:50:46,676 - Train: 0.17% [6800/4070000] [1.7/1000.0] [batch_t 0.606 (0.651)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 02:51:48,796 - Train: 0.17% [6900/4070000] [1.7/1000.0] [batch_t 0.605 (0.621)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 02:52:49,457 - Train: 0.17% [7000/4070000] [1.7/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 02:53:52,655 - Train: 0.17% [7100/4070000] [1.7/1000.0] [batch_t 0.602 (0.632)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-02 02:54:55,180 - Train: 0.18% [7200/4070000] [1.8/1000.0] [batch_t 0.606 (0.625)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 02:55:55,847 - Train: 0.18% [7300/4070000] [1.8/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 02:56:57,582 - Train: 0.18% [7400/4070000] [1.8/1000.0] [batch_t 0.604 (0.617)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 02:58:20,003 - Train: 0.18% [7500/4070000] [1.8/1000.0] [batch_t 2.272 (0.824)] [data_t 1.671] [optim_t 0.601] [lr 0.005000] +2024-04-02 02:59:21,477 - Train: 0.19% [7600/4070000] [1.9/1000.0] [batch_t 0.605 (0.615)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 03:00:22,016 - Train: 0.19% [7700/4070000] [1.9/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 03:01:25,120 - Train: 0.19% [7800/4070000] [1.9/1000.0] [batch_t 0.604 (0.631)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 03:02:26,387 - Train: 0.19% [7900/4070000] [1.9/1000.0] [batch_t 0.606 (0.613)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 03:03:33,705 - Train: 0.20% [8000/4070000] [2.0/1000.0] [batch_t 0.606 (0.673)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 03:04:41,296 - Train: 0.20% [8100/4070000] [2.0/1000.0] [batch_t 0.607 (0.676)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 03:05:05,495 - ==> Total time: 2:04:36 Eta: 43 days, 4:18:19 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-02 03:05:45,265 - Train: 0.20% [8200/4070000] [2.0/1000.0] [batch_t 0.606 (0.627)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 03:06:49,195 - Train: 0.20% [8300/4070000] [2.0/1000.0] [batch_t 0.605 (0.639)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 03:07:55,942 - Train: 0.21% [8400/4070000] [2.1/1000.0] [batch_t 0.606 (0.667)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 03:09:08,682 - Train: 0.21% [8500/4070000] [2.1/1000.0] [batch_t 0.605 (0.727)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 03:10:09,661 - Train: 0.21% [8600/4070000] [2.1/1000.0] [batch_t 0.604 (0.610)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 03:11:12,633 - Train: 0.21% [8700/4070000] [2.1/1000.0] [batch_t 0.606 (0.630)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 03:12:20,143 - Train: 0.22% [8800/4070000] [2.2/1000.0] [batch_t 0.605 (0.675)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 03:13:27,602 - Train: 0.22% [8900/4070000] [2.2/1000.0] [batch_t 0.605 (0.675)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 03:14:51,376 - Train: 0.22% [9000/4070000] [2.2/1000.0] [batch_t 0.597 (0.838)] [data_t 0.002] [optim_t 0.595] [lr 0.005000] +2024-04-02 03:16:12,361 - Train: 0.22% [9100/4070000] [2.2/1000.0] [batch_t 0.605 (0.810)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 03:17:28,965 - Train: 0.23% [9200/4070000] [2.3/1000.0] [batch_t 0.610 (0.766)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-02 03:19:12,994 - Train: 0.23% [9300/4070000] [2.3/1000.0] [batch_t 0.605 (1.040)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 03:20:15,998 - Train: 0.23% [9400/4070000] [2.3/1000.0] [batch_t 0.606 (0.630)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 03:21:37,130 - Train: 0.23% [9500/4070000] [2.3/1000.0] [batch_t 0.620 (0.811)] [data_t 0.020] [optim_t 0.600] [lr 0.005000] +2024-04-02 03:24:23,839 - Train: 0.24% [9600/4070000] [2.4/1000.0] [batch_t 0.606 (1.667)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 03:25:31,462 - Train: 0.24% [9700/4070000] [2.4/1000.0] [batch_t 0.605 (0.676)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 03:26:53,838 - Train: 0.24% [9800/4070000] [2.4/1000.0] [batch_t 0.605 (0.824)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 03:27:59,856 - Train: 0.24% [9900/4070000] [2.4/1000.0] [batch_t 0.600 (0.660)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-02 03:29:15,602 - Train: 0.25% [10000/4070000] [2.5/1000.0] [batch_t 0.605 (0.757)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 03:30:35,937 - Train: 0.25% [10100/4070000] [2.5/1000.0] [batch_t 0.609 (0.803)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-02 03:31:52,074 - Train: 0.25% [10200/4070000] [2.5/1000.0] [batch_t 0.607 (0.761)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 03:33:18,577 - Train: 0.25% [10300/4070000] [2.5/1000.0] [batch_t 0.609 (0.865)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-02 03:34:20,380 - Train: 0.26% [10400/4070000] [2.6/1000.0] [batch_t 0.605 (0.618)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 03:36:46,262 - Train: 0.26% [10500/4070000] [2.6/1000.0] [batch_t 0.605 (1.459)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 03:38:12,893 - Train: 0.26% [10600/4070000] [2.6/1000.0] [batch_t 0.607 (0.866)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 03:39:15,840 - Train: 0.26% [10700/4070000] [2.6/1000.0] [batch_t 0.604 (0.629)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 03:40:36,916 - Train: 0.27% [10800/4070000] [2.7/1000.0] [batch_t 0.941 (0.811)] [data_t 0.340] [optim_t 0.602] [lr 0.005000] +2024-04-02 03:41:50,336 - Train: 0.27% [10900/4070000] [2.7/1000.0] [batch_t 0.602 (0.734)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-02 03:43:22,509 - Train: 0.27% [11000/4070000] [2.7/1000.0] [batch_t 0.606 (0.922)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 03:44:26,381 - Train: 0.27% [11100/4070000] [2.7/1000.0] [batch_t 0.605 (0.639)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 03:45:34,036 - Train: 0.28% [11200/4070000] [2.8/1000.0] [batch_t 0.605 (0.676)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 03:46:53,864 - Train: 0.28% [11300/4070000] [2.8/1000.0] [batch_t 0.605 (0.798)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 03:47:58,855 - Train: 0.28% [11400/4070000] [2.8/1000.0] [batch_t 0.604 (0.650)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 03:49:00,135 - Train: 0.28% [11500/4070000] [2.8/1000.0] [batch_t 0.887 (0.613)] [data_t 0.284] [optim_t 0.603] [lr 0.005000] +2024-04-02 03:50:12,045 - Train: 0.29% [11600/4070000] [2.9/1000.0] [batch_t 0.607 (0.719)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 03:51:18,971 - Train: 0.29% [11700/4070000] [2.9/1000.0] [batch_t 0.606 (0.669)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 03:52:27,375 - Train: 0.29% [11800/4070000] [2.9/1000.0] [batch_t 1.310 (0.684)] [data_t 0.707] [optim_t 0.603] [lr 0.005000] +2024-04-02 03:53:33,308 - Train: 0.29% [11900/4070000] [2.9/1000.0] [batch_t 0.603 (0.659)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-02 03:54:36,916 - Train: 0.29% [12000/4070000] [2.9/1000.0] [batch_t 0.604 (0.636)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 03:56:49,490 - Train: 0.30% [12100/4070000] [3.0/1000.0] [batch_t 0.605 (1.326)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 03:57:50,335 - Train: 0.30% [12200/4070000] [3.0/1000.0] [batch_t 0.605 (0.608)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 03:57:56,394 - ==> Total time: 2:57:27 Eta: 40 days, 22:53:56 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-02 03:59:14,966 - Train: 0.30% [12300/4070000] [3.0/1000.0] [batch_t 0.604 (0.848)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 04:01:36,823 - Train: 0.30% [12400/4070000] [3.0/1000.0] [batch_t 0.604 (1.418)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 04:04:18,910 - Train: 0.31% [12500/4070000] [3.1/1000.0] [batch_t 0.606 (1.621)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 04:05:50,798 - Train: 0.31% [12600/4070000] [3.1/1000.0] [batch_t 0.608 (0.919)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-02 04:08:06,019 - Train: 0.31% [12700/4070000] [3.1/1000.0] [batch_t 0.605 (1.352)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 04:09:45,482 - Train: 0.31% [12800/4070000] [3.1/1000.0] [batch_t 0.607 (0.995)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 04:11:23,446 - Train: 0.32% [12900/4070000] [3.2/1000.0] [batch_t 0.604 (0.980)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 04:12:43,111 - Train: 0.32% [13000/4070000] [3.2/1000.0] [batch_t 0.603 (0.797)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-02 04:14:17,083 - Train: 0.32% [13100/4070000] [3.2/1000.0] [batch_t 1.987 (0.940)] [data_t 1.387] [optim_t 0.599] [lr 0.005000] +2024-04-02 04:15:27,402 - Train: 0.32% [13200/4070000] [3.2/1000.0] [batch_t 0.600 (0.703)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-02 04:16:32,127 - Train: 0.33% [13300/4070000] [3.3/1000.0] [batch_t 0.607 (0.647)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 04:18:18,747 - Train: 0.33% [13400/4070000] [3.3/1000.0] [batch_t 0.606 (1.066)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 04:19:25,783 - Train: 0.33% [13500/4070000] [3.3/1000.0] [batch_t 0.605 (0.670)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 04:20:31,313 - Train: 0.33% [13600/4070000] [3.3/1000.0] [batch_t 0.604 (0.655)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 04:21:46,105 - Train: 0.34% [13700/4070000] [3.4/1000.0] [batch_t 0.605 (0.748)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 04:22:55,387 - Train: 0.34% [13800/4070000] [3.4/1000.0] [batch_t 0.607 (0.693)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 04:24:51,641 - Train: 0.34% [13900/4070000] [3.4/1000.0] [batch_t 0.606 (1.162)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 04:26:11,578 - Train: 0.34% [14000/4070000] [3.4/1000.0] [batch_t 0.662 (0.799)] [data_t 0.058] [optim_t 0.604] [lr 0.005000] +2024-04-02 04:27:22,176 - Train: 0.35% [14100/4070000] [3.5/1000.0] [batch_t 0.606 (0.706)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 04:28:25,698 - Train: 0.35% [14200/4070000] [3.5/1000.0] [batch_t 0.605 (0.635)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 04:29:30,183 - Train: 0.35% [14300/4070000] [3.5/1000.0] [batch_t 0.604 (0.644)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 04:30:44,690 - Train: 0.35% [14400/4070000] [3.5/1000.0] [batch_t 0.606 (0.745)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 04:31:54,946 - Train: 0.36% [14500/4070000] [3.6/1000.0] [batch_t 0.606 (0.702)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 04:33:04,715 - Train: 0.36% [14600/4070000] [3.6/1000.0] [batch_t 0.602 (0.698)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-02 04:34:15,203 - Train: 0.36% [14700/4070000] [3.6/1000.0] [batch_t 0.604 (0.695)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 04:35:38,769 - Train: 0.36% [14800/4070000] [3.6/1000.0] [batch_t 1.441 (0.836)] [data_t 0.837] [optim_t 0.604] [lr 0.005000] +2024-04-02 04:36:40,457 - Train: 0.37% [14900/4070000] [3.7/1000.0] [batch_t 0.605 (0.617)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 04:37:40,978 - Train: 0.37% [15000/4070000] [3.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 04:38:42,617 - Train: 0.37% [15100/4070000] [3.7/1000.0] [batch_t 0.603 (0.616)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-02 04:39:54,801 - Train: 0.37% [15200/4070000] [3.7/1000.0] [batch_t 0.599 (0.722)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-02 04:41:14,860 - Train: 0.38% [15300/4070000] [3.8/1000.0] [batch_t 0.607 (0.801)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 04:42:27,963 - Train: 0.38% [15400/4070000] [3.8/1000.0] [batch_t 0.606 (0.731)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 04:43:28,820 - Train: 0.38% [15500/4070000] [3.8/1000.0] [batch_t 0.606 (0.608)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 04:44:55,658 - Train: 0.38% [15600/4070000] [3.8/1000.0] [batch_t 0.604 (0.868)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 04:46:01,337 - Train: 0.39% [15700/4070000] [3.9/1000.0] [batch_t 0.607 (0.657)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 04:47:05,140 - Train: 0.39% [15800/4070000] [3.9/1000.0] [batch_t 0.606 (0.638)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 04:48:07,793 - Train: 0.39% [15900/4070000] [3.9/1000.0] [batch_t 0.606 (0.626)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 04:49:12,531 - Train: 0.39% [16000/4070000] [3.9/1000.0] [batch_t 0.604 (0.647)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 04:50:15,311 - Train: 0.40% [16100/4070000] [4.0/1000.0] [batch_t 0.603 (0.628)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 04:51:27,861 - Train: 0.40% [16200/4070000] [4.0/1000.0] [batch_t 0.606 (0.725)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 04:52:18,311 - ==> Total time: 3:51:49 Eta: 40 days, 2:03:02 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-02 04:52:35,016 - Train: 0.40% [16300/4070000] [4.0/1000.0] [batch_t 0.781 (0.764)] [data_t 0.178] [optim_t 0.603] [lr 0.005000] +2024-04-02 04:53:42,564 - Train: 0.40% [16400/4070000] [4.0/1000.0] [batch_t 0.606 (0.675)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 04:54:46,060 - Train: 0.41% [16500/4070000] [4.1/1000.0] [batch_t 0.604 (0.635)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 04:55:50,787 - Train: 0.41% [16600/4070000] [4.1/1000.0] [batch_t 0.605 (0.647)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 04:56:58,321 - Train: 0.41% [16700/4070000] [4.1/1000.0] [batch_t 0.606 (0.675)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 04:57:59,195 - Train: 0.41% [16800/4070000] [4.1/1000.0] [batch_t 0.605 (0.609)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 04:59:02,532 - Train: 0.42% [16900/4070000] [4.2/1000.0] [batch_t 0.603 (0.633)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-02 05:00:04,749 - Train: 0.42% [17000/4070000] [4.2/1000.0] [batch_t 0.603 (0.622)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-02 05:01:10,675 - Train: 0.42% [17100/4070000] [4.2/1000.0] [batch_t 0.609 (0.659)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-02 05:02:13,216 - Train: 0.42% [17200/4070000] [4.2/1000.0] [batch_t 0.605 (0.625)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 05:03:15,113 - Train: 0.43% [17300/4070000] [4.3/1000.0] [batch_t 0.603 (0.619)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-02 05:04:20,647 - Train: 0.43% [17400/4070000] [4.3/1000.0] [batch_t 0.602 (0.655)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-02 05:05:25,060 - Train: 0.43% [17500/4070000] [4.3/1000.0] [batch_t 0.606 (0.644)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 05:06:25,574 - Train: 0.43% [17600/4070000] [4.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 05:07:27,900 - Train: 0.43% [17700/4070000] [4.3/1000.0] [batch_t 0.604 (0.623)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 05:08:31,745 - Train: 0.44% [17800/4070000] [4.4/1000.0] [batch_t 0.606 (0.638)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 05:09:33,302 - Train: 0.44% [17900/4070000] [4.4/1000.0] [batch_t 0.607 (0.615)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 05:10:34,750 - Train: 0.44% [18000/4070000] [4.4/1000.0] [batch_t 0.606 (0.614)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 05:11:41,276 - Train: 0.44% [18100/4070000] [4.4/1000.0] [batch_t 0.605 (0.665)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 05:12:42,930 - Train: 0.45% [18200/4070000] [4.5/1000.0] [batch_t 0.606 (0.616)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 05:13:46,472 - Train: 0.45% [18300/4070000] [4.5/1000.0] [batch_t 0.605 (0.635)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 05:14:52,491 - Train: 0.45% [18400/4070000] [4.5/1000.0] [batch_t 0.605 (0.660)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 05:15:58,053 - Train: 0.45% [18500/4070000] [4.5/1000.0] [batch_t 0.606 (0.656)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 05:17:00,038 - Train: 0.46% [18600/4070000] [4.6/1000.0] [batch_t 0.609 (0.620)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-02 05:18:01,860 - Train: 0.46% [18700/4070000] [4.6/1000.0] [batch_t 0.603 (0.618)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-02 05:19:05,401 - Train: 0.46% [18800/4070000] [4.6/1000.0] [batch_t 0.601 (0.615)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-02 05:20:11,080 - Train: 0.46% [18900/4070000] [4.6/1000.0] [batch_t 0.604 (0.657)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 05:21:12,745 - Train: 0.47% [19000/4070000] [4.7/1000.0] [batch_t 0.604 (0.617)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 05:22:16,855 - Train: 0.47% [19100/4070000] [4.7/1000.0] [batch_t 0.604 (0.641)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 05:23:17,611 - Train: 0.47% [19200/4070000] [4.7/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 05:24:19,775 - Train: 0.47% [19300/4070000] [4.7/1000.0] [batch_t 0.603 (0.622)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-02 05:25:21,021 - Train: 0.48% [19400/4070000] [4.8/1000.0] [batch_t 0.605 (0.612)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 05:26:24,252 - Train: 0.48% [19500/4070000] [4.8/1000.0] [batch_t 0.604 (0.632)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 05:27:25,694 - Train: 0.48% [19600/4070000] [4.8/1000.0] [batch_t 0.604 (0.614)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 05:28:32,311 - Train: 0.48% [19700/4070000] [4.8/1000.0] [batch_t 0.605 (0.666)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 05:29:34,900 - Train: 0.49% [19800/4070000] [4.9/1000.0] [batch_t 0.606 (0.626)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 05:30:38,443 - Train: 0.49% [19900/4070000] [4.9/1000.0] [batch_t 0.606 (0.635)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 05:31:38,961 - Train: 0.49% [20000/4070000] [4.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 05:32:47,220 - Train: 0.49% [20100/4070000] [4.9/1000.0] [batch_t 0.604 (0.683)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 05:33:48,019 - Train: 0.50% [20200/4070000] [5.0/1000.0] [batch_t 0.604 (0.608)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 05:34:50,480 - Train: 0.50% [20300/4070000] [5.0/1000.0] [batch_t 0.606 (0.625)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 05:35:21,919 - ==> Total time: 4:34:52 Eta: 37 days, 23:41:02 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-02 05:35:54,560 - Train: 0.50% [20400/4070000] [5.0/1000.0] [batch_t 0.605 (0.617)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 05:36:55,684 - Train: 0.50% [20500/4070000] [5.0/1000.0] [batch_t 0.602 (0.611)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-02 05:37:57,583 - Train: 0.51% [20600/4070000] [5.1/1000.0] [batch_t 0.602 (0.619)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-02 05:38:59,237 - Train: 0.51% [20700/4070000] [5.1/1000.0] [batch_t 0.606 (0.616)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 05:40:02,698 - Train: 0.51% [20800/4070000] [5.1/1000.0] [batch_t 0.606 (0.635)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 05:41:09,405 - Train: 0.51% [20900/4070000] [5.1/1000.0] [batch_t 0.602 (0.667)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-02 05:42:09,931 - Train: 0.52% [21000/4070000] [5.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 05:43:11,767 - Train: 0.52% [21100/4070000] [5.2/1000.0] [batch_t 0.608 (0.618)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-02 05:44:14,451 - Train: 0.52% [21200/4070000] [5.2/1000.0] [batch_t 0.605 (0.626)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 05:45:15,687 - Train: 0.52% [21300/4070000] [5.2/1000.0] [batch_t 0.606 (0.612)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 05:46:22,008 - Train: 0.53% [21400/4070000] [5.3/1000.0] [batch_t 1.175 (0.663)] [data_t 0.579] [optim_t 0.596] [lr 0.005000] +2024-04-02 05:47:23,166 - Train: 0.53% [21500/4070000] [5.3/1000.0] [batch_t 0.603 (0.611)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-02 05:48:23,743 - Train: 0.53% [21600/4070000] [5.3/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 05:49:28,250 - Train: 0.53% [21700/4070000] [5.3/1000.0] [batch_t 0.605 (0.645)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 05:50:28,805 - Train: 0.54% [21800/4070000] [5.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 05:51:30,223 - Train: 0.54% [21900/4070000] [5.4/1000.0] [batch_t 0.603 (0.614)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-02 05:52:31,486 - Train: 0.54% [22000/4070000] [5.4/1000.0] [batch_t 0.605 (0.613)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 05:53:46,078 - Train: 0.54% [22100/4070000] [5.4/1000.0] [batch_t 1.218 (0.746)] [data_t 0.613] [optim_t 0.605] [lr 0.005000] +2024-04-02 05:55:12,905 - Train: 0.55% [22200/4070000] [5.5/1000.0] [batch_t 0.604 (0.868)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 05:56:16,415 - Train: 0.55% [22300/4070000] [5.5/1000.0] [batch_t 0.606 (0.635)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 05:57:28,399 - Train: 0.55% [22400/4070000] [5.5/1000.0] [batch_t 2.878 (0.720)] [data_t 2.274] [optim_t 0.604] [lr 0.005000] +2024-04-02 05:58:33,352 - Train: 0.55% [22500/4070000] [5.5/1000.0] [batch_t 0.606 (0.649)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 05:59:35,052 - Train: 0.56% [22600/4070000] [5.6/1000.0] [batch_t 0.606 (0.617)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 06:00:37,220 - Train: 0.56% [22700/4070000] [5.6/1000.0] [batch_t 0.608 (0.622)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-02 06:01:40,378 - Train: 0.56% [22800/4070000] [5.6/1000.0] [batch_t 0.605 (0.631)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 06:02:43,028 - Train: 0.56% [22900/4070000] [5.6/1000.0] [batch_t 0.605 (0.626)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 06:03:44,948 - Train: 0.57% [23000/4070000] [5.7/1000.0] [batch_t 0.605 (0.619)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 06:04:50,753 - Train: 0.57% [23100/4070000] [5.7/1000.0] [batch_t 0.603 (0.658)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-02 06:05:51,250 - Train: 0.57% [23200/4070000] [5.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 06:06:54,699 - Train: 0.57% [23300/4070000] [5.7/1000.0] [batch_t 0.604 (0.634)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 06:07:58,480 - Train: 0.57% [23400/4070000] [5.7/1000.0] [batch_t 0.606 (0.638)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 06:09:01,390 - Train: 0.58% [23500/4070000] [5.8/1000.0] [batch_t 0.607 (0.629)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 06:10:02,358 - Train: 0.58% [23600/4070000] [5.8/1000.0] [batch_t 0.604 (0.610)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 06:11:03,620 - Train: 0.58% [23700/4070000] [5.8/1000.0] [batch_t 0.605 (0.613)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 06:12:06,053 - Train: 0.58% [23800/4070000] [5.8/1000.0] [batch_t 0.604 (0.624)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 06:13:11,368 - Train: 0.59% [23900/4070000] [5.9/1000.0] [batch_t 0.604 (0.653)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 06:14:11,920 - Train: 0.59% [24000/4070000] [5.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 06:15:15,607 - Train: 0.59% [24100/4070000] [5.9/1000.0] [batch_t 0.604 (0.637)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 06:16:16,473 - Train: 0.59% [24200/4070000] [5.9/1000.0] [batch_t 0.604 (0.609)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 06:17:22,178 - Train: 0.60% [24300/4070000] [6.0/1000.0] [batch_t 0.604 (0.657)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 06:18:22,996 - Train: 0.60% [24400/4070000] [6.0/1000.0] [batch_t 0.607 (0.608)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-02 06:18:35,105 - ==> Total time: 5:18:05 Eta: 36 days, 14:18:27 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-02 06:19:40,250 - Train: 0.60% [24500/4070000] [6.0/1000.0] [batch_t 1.007 (0.791)] [data_t 0.407] [optim_t 0.600] [lr 0.005000] +2024-04-02 06:20:40,942 - Train: 0.60% [24600/4070000] [6.0/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 06:21:42,908 - Train: 0.61% [24700/4070000] [6.1/1000.0] [batch_t 0.607 (0.620)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 06:22:44,860 - Train: 0.61% [24800/4070000] [6.1/1000.0] [batch_t 0.603 (0.619)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-02 06:23:46,387 - Train: 0.61% [24900/4070000] [6.1/1000.0] [batch_t 0.604 (0.615)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 06:24:46,921 - Train: 0.61% [25000/4070000] [6.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 06:25:49,188 - Train: 0.62% [25100/4070000] [6.2/1000.0] [batch_t 0.602 (0.623)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-02 06:26:50,953 - Train: 0.62% [25200/4070000] [6.2/1000.0] [batch_t 0.604 (0.618)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 06:27:55,066 - Train: 0.62% [25300/4070000] [6.2/1000.0] [batch_t 0.606 (0.641)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 06:28:56,978 - Train: 0.62% [25400/4070000] [6.2/1000.0] [batch_t 0.607 (0.619)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-02 06:29:59,729 - Train: 0.63% [25500/4070000] [6.3/1000.0] [batch_t 0.601 (0.627)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-02 06:31:02,507 - Train: 0.63% [25600/4070000] [6.3/1000.0] [batch_t 0.606 (0.628)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 06:32:04,979 - Train: 0.63% [25700/4070000] [6.3/1000.0] [batch_t 0.607 (0.625)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 06:33:06,433 - Train: 0.63% [25800/4070000] [6.3/1000.0] [batch_t 0.605 (0.614)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 06:34:08,211 - Train: 0.64% [25900/4070000] [6.4/1000.0] [batch_t 0.606 (0.618)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 06:35:11,219 - Train: 0.64% [26000/4070000] [6.4/1000.0] [batch_t 0.607 (0.630)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 06:36:14,373 - Train: 0.64% [26100/4070000] [6.4/1000.0] [batch_t 0.606 (0.631)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 06:37:16,667 - Train: 0.64% [26200/4070000] [6.4/1000.0] [batch_t 0.603 (0.623)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 06:38:19,856 - Train: 0.65% [26300/4070000] [6.5/1000.0] [batch_t 0.609 (0.632)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-02 06:39:22,477 - Train: 0.65% [26400/4070000] [6.5/1000.0] [batch_t 0.604 (0.626)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 06:40:23,909 - Train: 0.65% [26500/4070000] [6.5/1000.0] [batch_t 0.605 (0.614)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 06:41:25,793 - Train: 0.65% [26600/4070000] [6.5/1000.0] [batch_t 0.604 (0.619)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 06:42:29,447 - Train: 0.66% [26700/4070000] [6.6/1000.0] [batch_t 0.606 (0.636)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 06:43:30,015 - Train: 0.66% [26800/4070000] [6.6/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 06:44:32,618 - Train: 0.66% [26900/4070000] [6.6/1000.0] [batch_t 0.605 (0.626)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 06:45:33,217 - Train: 0.66% [27000/4070000] [6.6/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 06:46:35,720 - Train: 0.67% [27100/4070000] [6.7/1000.0] [batch_t 0.604 (0.625)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 06:47:37,646 - Train: 0.67% [27200/4070000] [6.7/1000.0] [batch_t 0.604 (0.619)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 06:48:40,384 - Train: 0.67% [27300/4070000] [6.7/1000.0] [batch_t 0.606 (0.627)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 06:49:43,061 - Train: 0.67% [27400/4070000] [6.7/1000.0] [batch_t 0.604 (0.627)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 06:50:46,190 - Train: 0.68% [27500/4070000] [6.8/1000.0] [batch_t 0.602 (0.631)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-02 06:51:47,549 - Train: 0.68% [27600/4070000] [6.8/1000.0] [batch_t 0.605 (0.613)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 06:52:51,123 - Train: 0.68% [27700/4070000] [6.8/1000.0] [batch_t 0.606 (0.636)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 06:53:53,013 - Train: 0.68% [27800/4070000] [6.8/1000.0] [batch_t 0.606 (0.619)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 06:54:56,084 - Train: 0.69% [27900/4070000] [6.9/1000.0] [batch_t 0.603 (0.631)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-02 06:55:57,782 - Train: 0.69% [28000/4070000] [6.9/1000.0] [batch_t 0.604 (0.617)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 06:56:59,002 - Train: 0.69% [28100/4070000] [6.9/1000.0] [batch_t 0.606 (0.612)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 06:58:02,416 - Train: 0.69% [28200/4070000] [6.9/1000.0] [batch_t 0.605 (0.634)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 06:59:03,863 - Train: 0.70% [28300/4070000] [7.0/1000.0] [batch_t 0.602 (0.614)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-02 07:00:07,579 - Train: 0.70% [28400/4070000] [7.0/1000.0] [batch_t 0.606 (0.637)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 07:01:02,037 - ==> Total time: 6:00:32 Eta: 35 days, 12:26:20 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-02 07:01:14,214 - Train: 0.70% [28500/4070000] [7.0/1000.0] [batch_t 0.603 (0.793)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-02 07:02:20,281 - Train: 0.70% [28600/4070000] [7.0/1000.0] [batch_t 0.604 (0.661)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 07:03:20,946 - Train: 0.71% [28700/4070000] [7.1/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 07:04:24,585 - Train: 0.71% [28800/4070000] [7.1/1000.0] [batch_t 0.605 (0.636)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 07:05:25,922 - Train: 0.71% [28900/4070000] [7.1/1000.0] [batch_t 0.607 (0.613)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 07:06:28,452 - Train: 0.71% [29000/4070000] [7.1/1000.0] [batch_t 0.604 (0.625)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 07:07:32,797 - Train: 0.71% [29100/4070000] [7.1/1000.0] [batch_t 0.601 (0.643)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-02 07:08:40,378 - Train: 0.72% [29200/4070000] [7.2/1000.0] [batch_t 0.606 (0.676)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 07:09:42,944 - Train: 0.72% [29300/4070000] [7.2/1000.0] [batch_t 0.607 (0.626)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 07:10:45,804 - Train: 0.72% [29400/4070000] [7.2/1000.0] [batch_t 0.605 (0.629)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 07:11:47,752 - Train: 0.72% [29500/4070000] [7.2/1000.0] [batch_t 0.607 (0.619)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 07:12:52,687 - Train: 0.73% [29600/4070000] [7.3/1000.0] [batch_t 0.603 (0.649)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-02 07:14:03,724 - Train: 0.73% [29700/4070000] [7.3/1000.0] [batch_t 0.606 (0.710)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 07:15:06,982 - Train: 0.73% [29800/4070000] [7.3/1000.0] [batch_t 0.605 (0.632)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 07:16:12,011 - Train: 0.73% [29900/4070000] [7.3/1000.0] [batch_t 0.609 (0.650)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-02 07:17:18,109 - Train: 0.74% [30000/4070000] [7.4/1000.0] [batch_t 0.604 (0.661)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 07:18:22,343 - Train: 0.74% [30100/4070000] [7.4/1000.0] [batch_t 0.604 (0.642)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 07:19:28,717 - Train: 0.74% [30200/4070000] [7.4/1000.0] [batch_t 0.608 (0.664)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-02 07:20:30,017 - Train: 0.74% [30300/4070000] [7.4/1000.0] [batch_t 0.606 (0.613)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 07:21:56,001 - Train: 0.75% [30400/4070000] [7.5/1000.0] [batch_t 0.605 (0.860)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 07:23:18,143 - Train: 0.75% [30500/4070000] [7.5/1000.0] [batch_t 0.605 (0.821)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 07:24:22,691 - Train: 0.75% [30600/4070000] [7.5/1000.0] [batch_t 0.603 (0.645)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-02 07:25:23,502 - Train: 0.75% [30700/4070000] [7.5/1000.0] [batch_t 0.607 (0.608)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 07:26:26,072 - Train: 0.76% [30800/4070000] [7.6/1000.0] [batch_t 0.604 (0.626)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 07:27:30,016 - Train: 0.76% [30900/4070000] [7.6/1000.0] [batch_t 0.603 (0.639)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 07:28:30,536 - Train: 0.76% [31000/4070000] [7.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 07:29:33,332 - Train: 0.76% [31100/4070000] [7.6/1000.0] [batch_t 0.604 (0.628)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 07:30:36,029 - Train: 0.77% [31200/4070000] [7.7/1000.0] [batch_t 0.606 (0.627)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 07:31:36,565 - Train: 0.77% [31300/4070000] [7.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 07:32:37,184 - Train: 0.77% [31400/4070000] [7.7/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 07:33:40,827 - Train: 0.77% [31500/4070000] [7.7/1000.0] [batch_t 0.602 (0.636)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-02 07:34:42,584 - Train: 0.78% [31600/4070000] [7.8/1000.0] [batch_t 0.607 (0.617)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 07:35:43,478 - Train: 0.78% [31700/4070000] [7.8/1000.0] [batch_t 0.606 (0.609)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 07:36:45,806 - Train: 0.78% [31800/4070000] [7.8/1000.0] [batch_t 0.606 (0.623)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 07:37:47,084 - Train: 0.78% [31900/4070000] [7.8/1000.0] [batch_t 0.606 (0.613)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 07:39:00,417 - Train: 0.79% [32000/4070000] [7.9/1000.0] [batch_t 0.603 (0.733)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-02 07:40:02,070 - Train: 0.79% [32100/4070000] [7.9/1000.0] [batch_t 0.606 (0.616)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 07:41:02,764 - Train: 0.79% [32200/4070000] [7.9/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 07:42:03,248 - Train: 0.79% [32300/4070000] [7.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 07:43:03,741 - Train: 0.80% [32400/4070000] [8.0/1000.0] [batch_t 0.600 (0.605)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-02 07:44:05,865 - Train: 0.80% [32500/4070000] [8.0/1000.0] [batch_t 0.603 (0.621)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 07:44:47,321 - ==> Total time: 6:44:18 Eta: 34 days, 19:33:34 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-02 07:45:19,084 - Train: 0.80% [32600/4070000] [8.0/1000.0] [batch_t 0.608 (0.743)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-02 07:46:20,222 - Train: 0.80% [32700/4070000] [8.0/1000.0] [batch_t 0.606 (0.611)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 07:47:21,134 - Train: 0.81% [32800/4070000] [8.1/1000.0] [batch_t 0.605 (0.609)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 07:48:22,946 - Train: 0.81% [32900/4070000] [8.1/1000.0] [batch_t 0.605 (0.618)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 07:49:23,452 - Train: 0.81% [33000/4070000] [8.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 07:50:24,978 - Train: 0.81% [33100/4070000] [8.1/1000.0] [batch_t 0.605 (0.615)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 07:51:25,632 - Train: 0.82% [33200/4070000] [8.2/1000.0] [batch_t 0.602 (0.606)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-02 07:52:27,034 - Train: 0.82% [33300/4070000] [8.2/1000.0] [batch_t 0.606 (0.614)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 07:53:30,098 - Train: 0.82% [33400/4070000] [8.2/1000.0] [batch_t 0.606 (0.631)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 07:54:34,891 - Train: 0.82% [33500/4070000] [8.2/1000.0] [batch_t 0.607 (0.648)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 07:55:37,319 - Train: 0.83% [33600/4070000] [8.3/1000.0] [batch_t 0.600 (0.624)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-02 07:56:39,335 - Train: 0.83% [33700/4070000] [8.3/1000.0] [batch_t 0.605 (0.620)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 07:57:42,131 - Train: 0.83% [33800/4070000] [8.3/1000.0] [batch_t 0.605 (0.628)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 07:58:53,097 - Train: 0.83% [33900/4070000] [8.3/1000.0] [batch_t 0.606 (0.710)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 07:59:53,843 - Train: 0.84% [34000/4070000] [8.4/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 08:00:54,433 - Train: 0.84% [34100/4070000] [8.4/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 08:01:55,454 - Train: 0.84% [34200/4070000] [8.4/1000.0] [batch_t 0.607 (0.610)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 08:02:58,107 - Train: 0.84% [34300/4070000] [8.4/1000.0] [batch_t 0.604 (0.626)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 08:04:02,614 - Train: 0.85% [34400/4070000] [8.5/1000.0] [batch_t 0.605 (0.645)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 08:05:04,157 - Train: 0.85% [34500/4070000] [8.5/1000.0] [batch_t 0.605 (0.615)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 08:06:06,427 - Train: 0.85% [34600/4070000] [8.5/1000.0] [batch_t 0.607 (0.623)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 08:07:08,107 - Train: 0.85% [34700/4070000] [8.5/1000.0] [batch_t 0.602 (0.617)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-02 08:08:11,882 - Train: 0.86% [34800/4070000] [8.6/1000.0] [batch_t 0.610 (0.638)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-02 08:09:12,929 - Train: 0.86% [34900/4070000] [8.6/1000.0] [batch_t 0.606 (0.610)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 08:10:13,418 - Train: 0.86% [35000/4070000] [8.6/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-02 08:11:14,555 - Train: 0.86% [35100/4070000] [8.6/1000.0] [batch_t 0.600 (0.611)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-02 08:12:19,179 - Train: 0.86% [35200/4070000] [8.6/1000.0] [batch_t 0.605 (0.646)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 08:13:22,112 - Train: 0.87% [35300/4070000] [8.7/1000.0] [batch_t 0.607 (0.629)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-02 08:14:25,548 - Train: 0.87% [35400/4070000] [8.7/1000.0] [batch_t 0.606 (0.634)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 08:15:26,960 - Train: 0.87% [35500/4070000] [8.7/1000.0] [batch_t 0.606 (0.614)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 08:16:31,595 - Train: 0.87% [35600/4070000] [8.7/1000.0] [batch_t 0.603 (0.646)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-02 08:17:33,392 - Train: 0.88% [35700/4070000] [8.8/1000.0] [batch_t 0.606 (0.618)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 08:18:36,615 - Train: 0.88% [35800/4070000] [8.8/1000.0] [batch_t 0.606 (0.632)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 08:19:37,207 - Train: 0.88% [35900/4070000] [8.8/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 08:20:37,702 - Train: 0.88% [36000/4070000] [8.8/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 08:21:40,350 - Train: 0.89% [36100/4070000] [8.9/1000.0] [batch_t 0.604 (0.626)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 08:22:41,260 - Train: 0.89% [36200/4070000] [8.9/1000.0] [batch_t 0.608 (0.609)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-02 08:23:41,796 - Train: 0.89% [36300/4070000] [8.9/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 08:24:43,211 - Train: 0.89% [36400/4070000] [8.9/1000.0] [batch_t 0.605 (0.614)] [data_t 0.003] [optim_t 0.602] [lr 0.005000] +2024-04-02 08:25:44,419 - Train: 0.90% [36500/4070000] [9.0/1000.0] [batch_t 0.607 (0.612)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 08:26:46,040 - Train: 0.90% [36600/4070000] [9.0/1000.0] [batch_t 0.602 (0.616)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-02 08:27:04,194 - ==> Total time: 7:26:35 Eta: 34 days, 3:33:52 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-02 08:27:50,186 - Train: 0.90% [36700/4070000] [9.0/1000.0] [batch_t 0.605 (0.621)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 08:28:57,301 - Train: 0.90% [36800/4070000] [9.0/1000.0] [batch_t 0.608 (0.671)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-02 08:29:58,663 - Train: 0.91% [36900/4070000] [9.1/1000.0] [batch_t 0.605 (0.614)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 08:30:59,147 - Train: 0.91% [37000/4070000] [9.1/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-02 08:31:59,810 - Train: 0.91% [37100/4070000] [9.1/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-02 08:33:02,233 - Train: 0.91% [37200/4070000] [9.1/1000.0] [batch_t 0.604 (0.624)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 08:34:04,418 - Train: 0.92% [37300/4070000] [9.2/1000.0] [batch_t 0.606 (0.622)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 08:35:04,898 - Train: 0.92% [37400/4070000] [9.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 08:36:05,731 - Train: 0.92% [37500/4070000] [9.2/1000.0] [batch_t 0.603 (0.608)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-02 08:37:07,264 - Train: 0.92% [37600/4070000] [9.2/1000.0] [batch_t 0.605 (0.615)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 08:38:07,770 - Train: 0.93% [37700/4070000] [9.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 08:39:13,438 - Train: 0.93% [37800/4070000] [9.3/1000.0] [batch_t 0.604 (0.657)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 08:40:17,483 - Train: 0.93% [37900/4070000] [9.3/1000.0] [batch_t 0.611 (0.640)] [data_t 0.002] [optim_t 0.609] [lr 0.005000] +2024-04-02 08:41:19,792 - Train: 0.93% [38000/4070000] [9.3/1000.0] [batch_t 0.610 (0.623)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-02 08:42:21,458 - Train: 0.94% [38100/4070000] [9.4/1000.0] [batch_t 0.606 (0.617)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 08:43:21,950 - Train: 0.94% [38200/4070000] [9.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 08:44:23,229 - Train: 0.94% [38300/4070000] [9.4/1000.0] [batch_t 0.612 (0.613)] [data_t 0.002] [optim_t 0.610] [lr 0.005000] +2024-04-02 08:45:23,713 - Train: 0.94% [38400/4070000] [9.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 08:46:24,374 - Train: 0.95% [38500/4070000] [9.5/1000.0] [batch_t 0.603 (0.607)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-02 08:47:24,922 - Train: 0.95% [38600/4070000] [9.5/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 08:48:26,698 - Train: 0.95% [38700/4070000] [9.5/1000.0] [batch_t 0.606 (0.618)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 08:49:28,265 - Train: 0.95% [38800/4070000] [9.5/1000.0] [batch_t 0.610 (0.616)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-02 08:50:29,977 - Train: 0.96% [38900/4070000] [9.6/1000.0] [batch_t 0.604 (0.617)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 08:51:30,999 - Train: 0.96% [39000/4070000] [9.6/1000.0] [batch_t 0.605 (0.610)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 08:52:31,670 - Train: 0.96% [39100/4070000] [9.6/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 08:54:03,676 - Train: 0.96% [39200/4070000] [9.6/1000.0] [batch_t 0.601 (0.920)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-02 08:55:04,160 - Train: 0.97% [39300/4070000] [9.7/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 08:56:04,626 - Train: 0.97% [39400/4070000] [9.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 08:57:08,964 - Train: 0.97% [39500/4070000] [9.7/1000.0] [batch_t 0.606 (0.643)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 08:58:10,314 - Train: 0.97% [39600/4070000] [9.7/1000.0] [batch_t 0.600 (0.613)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-02 08:59:12,029 - Train: 0.98% [39700/4070000] [9.8/1000.0] [batch_t 0.602 (0.617)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-02 09:00:15,429 - Train: 0.98% [39800/4070000] [9.8/1000.0] [batch_t 0.605 (0.634)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 09:01:15,918 - Train: 0.98% [39900/4070000] [9.8/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 09:02:18,686 - Train: 0.98% [40000/4070000] [9.8/1000.0] [batch_t 0.606 (0.628)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 09:03:23,621 - Train: 0.99% [40100/4070000] [9.9/1000.0] [batch_t 0.604 (0.647)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 09:04:39,601 - Train: 0.99% [40200/4070000] [9.9/1000.0] [batch_t 0.683 (0.760)] [data_t 0.082] [optim_t 0.601] [lr 0.005000] +2024-04-02 09:05:43,145 - Train: 0.99% [40300/4070000] [9.9/1000.0] [batch_t 0.605 (0.635)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 09:06:43,773 - Train: 0.99% [40400/4070000] [9.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 09:07:46,190 - Train: 1.00% [40500/4070000] [10.0/1000.0] [batch_t 0.605 (0.624)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 09:08:46,640 - Train: 1.00% [40600/4070000] [10.0/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 09:09:49,061 - Train: 1.00% [40700/4070000] [10.0/1000.0] [batch_t 0.605 (0.624)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 09:09:49,065 - ==> Total time: 8:09:19 Eta: 33 days, 15:23:52 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-02 09:10:54,078 - Train: 1.00% [40800/4070000] [10.0/1000.0] [batch_t 0.606 (0.632)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 09:11:57,643 - Train: 1.00% [40900/4070000] [10.0/1000.0] [batch_t 0.604 (0.636)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 09:12:59,391 - Train: 1.01% [41000/4070000] [10.1/1000.0] [batch_t 0.603 (0.617)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-02 09:14:01,568 - Train: 1.01% [41100/4070000] [10.1/1000.0] [batch_t 0.602 (0.622)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-02 09:15:03,473 - Train: 1.01% [41200/4070000] [10.1/1000.0] [batch_t 0.608 (0.619)] [data_t 0.003] [optim_t 0.606] [lr 0.005000] +2024-04-02 09:16:04,952 - Train: 1.01% [41300/4070000] [10.1/1000.0] [batch_t 0.606 (0.615)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 09:17:05,431 - Train: 1.02% [41400/4070000] [10.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 09:18:05,904 - Train: 1.02% [41500/4070000] [10.2/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-02 09:19:06,800 - Train: 1.02% [41600/4070000] [10.2/1000.0] [batch_t 0.604 (0.609)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 09:20:24,524 - Train: 1.02% [41700/4070000] [10.2/1000.0] [batch_t 1.339 (0.777)] [data_t 0.736] [optim_t 0.602] [lr 0.005000] +2024-04-02 09:21:35,425 - Train: 1.03% [41800/4070000] [10.3/1000.0] [batch_t 0.606 (0.709)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 09:22:37,388 - Train: 1.03% [41900/4070000] [10.3/1000.0] [batch_t 0.606 (0.620)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 09:23:39,119 - Train: 1.03% [42000/4070000] [10.3/1000.0] [batch_t 0.602 (0.617)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-02 09:24:41,481 - Train: 1.03% [42100/4070000] [10.3/1000.0] [batch_t 0.605 (0.624)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 09:25:42,172 - Train: 1.04% [42200/4070000] [10.4/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 09:26:43,353 - Train: 1.04% [42300/4070000] [10.4/1000.0] [batch_t 0.604 (0.612)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 09:27:44,090 - Train: 1.04% [42400/4070000] [10.4/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 09:28:44,578 - Train: 1.04% [42500/4070000] [10.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 09:29:45,133 - Train: 1.05% [42600/4070000] [10.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 09:30:45,825 - Train: 1.05% [42700/4070000] [10.5/1000.0] [batch_t 0.608 (0.607)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-02 09:31:46,376 - Train: 1.05% [42800/4070000] [10.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 09:32:46,892 - Train: 1.05% [42900/4070000] [10.5/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-02 09:33:47,412 - Train: 1.06% [43000/4070000] [10.6/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 09:34:47,914 - Train: 1.06% [43100/4070000] [10.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 09:35:49,910 - Train: 1.06% [43200/4070000] [10.6/1000.0] [batch_t 0.605 (0.620)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 09:36:54,356 - Train: 1.06% [43300/4070000] [10.6/1000.0] [batch_t 0.606 (0.644)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 09:38:21,211 - Train: 1.07% [43400/4070000] [10.7/1000.0] [batch_t 0.605 (0.868)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 09:39:22,325 - Train: 1.07% [43500/4070000] [10.7/1000.0] [batch_t 0.606 (0.611)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 09:40:23,864 - Train: 1.07% [43600/4070000] [10.7/1000.0] [batch_t 0.606 (0.615)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 09:41:26,162 - Train: 1.07% [43700/4070000] [10.7/1000.0] [batch_t 0.605 (0.623)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 09:42:29,538 - Train: 1.08% [43800/4070000] [10.8/1000.0] [batch_t 0.607 (0.634)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 09:43:30,036 - Train: 1.08% [43900/4070000] [10.8/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-02 09:44:30,801 - Train: 1.08% [44000/4070000] [10.8/1000.0] [batch_t 0.607 (0.608)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 09:45:33,543 - Train: 1.08% [44100/4070000] [10.8/1000.0] [batch_t 0.605 (0.627)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 09:46:34,020 - Train: 1.09% [44200/4070000] [10.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 09:47:35,417 - Train: 1.09% [44300/4070000] [10.9/1000.0] [batch_t 0.601 (0.614)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-02 09:48:35,924 - Train: 1.09% [44400/4070000] [10.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 09:49:36,517 - Train: 1.09% [44500/4070000] [10.9/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 09:50:39,160 - Train: 1.10% [44600/4070000] [11.0/1000.0] [batch_t 0.601 (0.626)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-02 09:51:40,959 - Train: 1.10% [44700/4070000] [11.0/1000.0] [batch_t 0.605 (0.618)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 09:52:25,233 - ==> Total time: 8:51:56 Eta: 33 days, 5:05:46 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-02 09:52:45,582 - Train: 1.10% [44800/4070000] [11.0/1000.0] [batch_t 0.608 (0.622)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-02 09:53:55,325 - Train: 1.10% [44900/4070000] [11.0/1000.0] [batch_t 0.605 (0.697)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 09:55:11,538 - Train: 1.11% [45000/4070000] [11.1/1000.0] [batch_t 0.606 (0.762)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 09:58:19,857 - Train: 1.11% [45100/4070000] [11.1/1000.0] [batch_t 0.605 (1.883)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 10:01:57,916 - Train: 1.11% [45200/4070000] [11.1/1000.0] [batch_t 0.605 (2.181)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 10:09:26,530 - Train: 1.11% [45300/4070000] [11.1/1000.0] [batch_t 2.931 (4.486)] [data_t 1.492] [optim_t 1.439] [lr 0.005000] +2024-04-02 10:18:33,570 - Train: 1.12% [45400/4070000] [11.2/1000.0] [batch_t 1.445 (5.470)] [data_t 0.003] [optim_t 1.442] [lr 0.005000] +2024-04-02 10:32:02,491 - Train: 1.12% [45500/4070000] [11.2/1000.0] [batch_t 7.144 (8.089)] [data_t 5.692] [optim_t 1.453] [lr 0.005000] +2024-04-02 10:36:38,088 - Train: 1.12% [45600/4070000] [11.2/1000.0] [batch_t 1.447 (2.756)] [data_t 0.003] [optim_t 1.445] [lr 0.005000] +2024-04-02 10:40:48,075 - Train: 1.12% [45700/4070000] [11.2/1000.0] [batch_t 1.444 (2.500)] [data_t 0.002] [optim_t 1.442] [lr 0.005000] +2024-04-02 10:43:11,101 - Train: 1.13% [45800/4070000] [11.3/1000.0] [batch_t 1.438 (1.430)] [data_t 0.003] [optim_t 1.435] [lr 0.005000] +2024-04-02 10:45:34,142 - Train: 1.13% [45900/4070000] [11.3/1000.0] [batch_t 1.447 (1.430)] [data_t 0.003] [optim_t 1.444] [lr 0.005000] +2024-04-02 10:47:57,286 - Train: 1.13% [46000/4070000] [11.3/1000.0] [batch_t 1.425 (1.431)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-02 10:50:23,935 - Train: 1.13% [46100/4070000] [11.3/1000.0] [batch_t 1.439 (1.466)] [data_t 0.002] [optim_t 1.436] [lr 0.005000] +2024-04-02 10:53:06,844 - Train: 1.14% [46200/4070000] [11.4/1000.0] [batch_t 1.420 (1.629)] [data_t 0.003] [optim_t 1.417] [lr 0.005000] +2024-04-02 10:55:30,085 - Train: 1.14% [46300/4070000] [11.4/1000.0] [batch_t 1.431 (1.432)] [data_t 0.002] [optim_t 1.429] [lr 0.005000] +2024-04-02 10:57:53,392 - Train: 1.14% [46400/4070000] [11.4/1000.0] [batch_t 1.440 (1.433)] [data_t 0.003] [optim_t 1.438] [lr 0.005000] +2024-04-02 11:01:11,340 - Train: 1.14% [46500/4070000] [11.4/1000.0] [batch_t 1.429 (1.979)] [data_t 0.003] [optim_t 1.426] [lr 0.005000] +2024-04-02 11:04:30,132 - Train: 1.14% [46600/4070000] [11.4/1000.0] [batch_t 1.421 (1.988)] [data_t 0.003] [optim_t 1.418] [lr 0.005000] +2024-04-02 11:08:00,326 - Train: 1.15% [46700/4070000] [11.5/1000.0] [batch_t 0.605 (2.102)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 11:09:18,196 - Train: 1.15% [46800/4070000] [11.5/1000.0] [batch_t 0.595 (0.779)] [data_t 0.002] [optim_t 0.593] [lr 0.005000] +2024-04-02 11:10:42,296 - Train: 1.15% [46900/4070000] [11.5/1000.0] [batch_t 0.966 (0.841)] [data_t 0.362] [optim_t 0.604] [lr 0.005000] +2024-04-02 11:11:52,800 - Train: 1.15% [47000/4070000] [11.5/1000.0] [batch_t 0.602 (0.705)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-02 11:12:55,829 - Train: 1.16% [47100/4070000] [11.6/1000.0] [batch_t 0.605 (0.630)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 11:15:10,411 - Train: 1.16% [47200/4070000] [11.6/1000.0] [batch_t 1.083 (1.346)] [data_t 0.480] [optim_t 0.603] [lr 0.005000] +2024-04-02 11:20:16,688 - Train: 1.16% [47300/4070000] [11.6/1000.0] [batch_t 0.605 (3.063)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 11:21:51,426 - Train: 1.16% [47400/4070000] [11.6/1000.0] [batch_t 0.606 (0.947)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 11:22:54,656 - Train: 1.17% [47500/4070000] [11.7/1000.0] [batch_t 0.606 (0.632)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 11:23:55,684 - Train: 1.17% [47600/4070000] [11.7/1000.0] [batch_t 0.604 (0.610)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 11:24:59,404 - Train: 1.17% [47700/4070000] [11.7/1000.0] [batch_t 0.603 (0.637)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-02 11:26:01,087 - Train: 1.17% [47800/4070000] [11.7/1000.0] [batch_t 0.604 (0.617)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 11:27:01,746 - Train: 1.18% [47900/4070000] [11.8/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 11:29:12,469 - Train: 1.18% [48000/4070000] [11.8/1000.0] [batch_t 0.605 (1.307)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 11:30:28,630 - Train: 1.18% [48100/4070000] [11.8/1000.0] [batch_t 0.605 (0.762)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 11:33:22,683 - Train: 1.18% [48200/4070000] [11.8/1000.0] [batch_t 0.606 (1.740)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 11:34:24,929 - Train: 1.19% [48300/4070000] [11.9/1000.0] [batch_t 0.607 (0.622)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-02 11:35:26,450 - Train: 1.19% [48400/4070000] [11.9/1000.0] [batch_t 0.602 (0.615)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-02 11:36:28,318 - Train: 1.19% [48500/4070000] [11.9/1000.0] [batch_t 0.604 (0.619)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 11:37:34,953 - Train: 1.19% [48600/4070000] [11.9/1000.0] [batch_t 0.606 (0.666)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 11:38:38,042 - Train: 1.20% [48700/4070000] [12.0/1000.0] [batch_t 0.605 (0.631)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 11:39:38,933 - Train: 1.20% [48800/4070000] [12.0/1000.0] [batch_t 0.603 (0.609)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-02 11:40:03,236 - ==> Total time: 10:39:34 Eta: 36 days, 13:37:46 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-02 11:40:43,245 - Train: 1.20% [48900/4070000] [12.0/1000.0] [batch_t 0.606 (0.621)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 11:41:47,268 - Train: 1.20% [49000/4070000] [12.0/1000.0] [batch_t 0.606 (0.640)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 11:42:54,001 - Train: 1.21% [49100/4070000] [12.1/1000.0] [batch_t 0.605 (0.667)] [data_t 0.003] [optim_t 0.602] [lr 0.005000] +2024-04-02 11:43:58,976 - Train: 1.21% [49200/4070000] [12.1/1000.0] [batch_t 0.601 (0.650)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-02 11:45:00,228 - Train: 1.21% [49300/4070000] [12.1/1000.0] [batch_t 0.608 (0.612)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-02 11:46:05,784 - Train: 1.21% [49400/4070000] [12.1/1000.0] [batch_t 0.605 (0.655)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 11:47:11,013 - Train: 1.22% [49500/4070000] [12.2/1000.0] [batch_t 1.779 (0.652)] [data_t 1.178] [optim_t 0.601] [lr 0.005000] +2024-04-02 11:48:12,786 - Train: 1.22% [49600/4070000] [12.2/1000.0] [batch_t 0.611 (0.618)] [data_t 0.002] [optim_t 0.609] [lr 0.005000] +2024-04-02 11:49:13,275 - Train: 1.22% [49700/4070000] [12.2/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-02 11:50:13,980 - Train: 1.22% [49800/4070000] [12.2/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 11:51:16,999 - Train: 1.23% [49900/4070000] [12.3/1000.0] [batch_t 0.605 (0.630)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 11:52:18,253 - Train: 1.23% [50000/4070000] [12.3/1000.0] [batch_t 0.605 (0.612)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 11:53:19,084 - Train: 1.23% [50100/4070000] [12.3/1000.0] [batch_t 0.605 (0.608)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 11:54:19,718 - Train: 1.23% [50200/4070000] [12.3/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 11:55:20,873 - Train: 1.24% [50300/4070000] [12.4/1000.0] [batch_t 0.606 (0.611)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 11:57:58,312 - Train: 1.24% [50400/4070000] [12.4/1000.0] [batch_t 0.609 (1.574)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-02 11:59:02,070 - Train: 1.24% [50500/4070000] [12.4/1000.0] [batch_t 0.607 (0.637)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 12:00:03,812 - Train: 1.24% [50600/4070000] [12.4/1000.0] [batch_t 0.604 (0.617)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 12:01:04,323 - Train: 1.25% [50700/4070000] [12.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 12:02:04,923 - Train: 1.25% [50800/4070000] [12.5/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-02 12:03:13,300 - Train: 1.25% [50900/4070000] [12.5/1000.0] [batch_t 0.944 (0.684)] [data_t 0.341] [optim_t 0.603] [lr 0.005000] +2024-04-02 12:04:17,261 - Train: 1.25% [51000/4070000] [12.5/1000.0] [batch_t 0.606 (0.640)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 12:05:18,319 - Train: 1.26% [51100/4070000] [12.6/1000.0] [batch_t 0.605 (0.610)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 12:06:18,806 - Train: 1.26% [51200/4070000] [12.6/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-02 12:07:33,046 - Train: 1.26% [51300/4070000] [12.6/1000.0] [batch_t 0.606 (0.742)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 12:09:47,527 - Train: 1.26% [51400/4070000] [12.6/1000.0] [batch_t 0.956 (1.345)] [data_t 0.352] [optim_t 0.604] [lr 0.005000] +2024-04-02 12:11:38,680 - Train: 1.27% [51500/4070000] [12.7/1000.0] [batch_t 0.604 (1.111)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 12:12:40,363 - Train: 1.27% [51600/4070000] [12.7/1000.0] [batch_t 0.604 (0.617)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 12:13:51,651 - Train: 1.27% [51700/4070000] [12.7/1000.0] [batch_t 0.606 (0.713)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 12:14:52,140 - Train: 1.27% [51800/4070000] [12.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 12:15:53,737 - Train: 1.28% [51900/4070000] [12.8/1000.0] [batch_t 0.607 (0.616)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 12:16:55,102 - Train: 1.28% [52000/4070000] [12.8/1000.0] [batch_t 0.606 (0.614)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 12:17:55,562 - Train: 1.28% [52100/4070000] [12.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 12:18:56,063 - Train: 1.28% [52200/4070000] [12.8/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 12:19:57,035 - Train: 1.29% [52300/4070000] [12.9/1000.0] [batch_t 0.605 (0.610)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 12:20:58,264 - Train: 1.29% [52400/4070000] [12.9/1000.0] [batch_t 0.606 (0.612)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 12:21:58,778 - Train: 1.29% [52500/4070000] [12.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 12:22:59,295 - Train: 1.29% [52600/4070000] [12.9/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 12:24:00,830 - Train: 1.29% [52700/4070000] [12.9/1000.0] [batch_t 0.606 (0.615)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 12:25:01,348 - Train: 1.30% [52800/4070000] [13.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 12:26:01,844 - Train: 1.30% [52900/4070000] [13.0/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 12:26:08,119 - ==> Total time: 11:25:38 Eta: 36 days, 3:36:37 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-02 12:27:04,727 - Train: 1.30% [53000/4070000] [13.0/1000.0] [batch_t 0.604 (0.611)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 12:28:05,253 - Train: 1.30% [53100/4070000] [13.0/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-02 12:29:06,298 - Train: 1.31% [53200/4070000] [13.1/1000.0] [batch_t 0.607 (0.610)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 12:30:06,778 - Train: 1.31% [53300/4070000] [13.1/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-02 12:31:07,463 - Train: 1.31% [53400/4070000] [13.1/1000.0] [batch_t 0.775 (0.607)] [data_t 0.174] [optim_t 0.602] [lr 0.005000] +2024-04-02 12:32:07,983 - Train: 1.31% [53500/4070000] [13.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 12:33:10,359 - Train: 1.32% [53600/4070000] [13.2/1000.0] [batch_t 0.605 (0.624)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 12:34:11,914 - Train: 1.32% [53700/4070000] [13.2/1000.0] [batch_t 0.602 (0.615)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-02 12:35:21,831 - Train: 1.32% [53800/4070000] [13.2/1000.0] [batch_t 0.601 (0.699)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-02 12:36:27,043 - Train: 1.32% [53900/4070000] [13.2/1000.0] [batch_t 0.606 (0.652)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 12:37:29,566 - Train: 1.33% [54000/4070000] [13.3/1000.0] [batch_t 0.604 (0.625)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 12:38:30,928 - Train: 1.33% [54100/4070000] [13.3/1000.0] [batch_t 0.607 (0.614)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 12:39:31,408 - Train: 1.33% [54200/4070000] [13.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 12:40:34,521 - Train: 1.33% [54300/4070000] [13.3/1000.0] [batch_t 0.604 (0.631)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 12:41:35,035 - Train: 1.34% [54400/4070000] [13.4/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-02 12:42:35,575 - Train: 1.34% [54500/4070000] [13.4/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-02 12:43:36,425 - Train: 1.34% [54600/4070000] [13.4/1000.0] [batch_t 0.605 (0.608)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 12:44:36,896 - Train: 1.34% [54700/4070000] [13.4/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 12:45:37,786 - Train: 1.35% [54800/4070000] [13.5/1000.0] [batch_t 0.605 (0.609)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 12:46:39,466 - Train: 1.35% [54900/4070000] [13.5/1000.0] [batch_t 0.606 (0.617)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 12:47:41,850 - Train: 1.35% [55000/4070000] [13.5/1000.0] [batch_t 0.606 (0.624)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 12:48:42,773 - Train: 1.35% [55100/4070000] [13.5/1000.0] [batch_t 0.604 (0.609)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 12:49:43,266 - Train: 1.36% [55200/4070000] [13.6/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-02 12:50:46,198 - Train: 1.36% [55300/4070000] [13.6/1000.0] [batch_t 0.602 (0.629)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-02 12:52:06,297 - Train: 1.36% [55400/4070000] [13.6/1000.0] [batch_t 0.602 (0.801)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-02 12:53:08,210 - Train: 1.36% [55500/4070000] [13.6/1000.0] [batch_t 2.022 (0.619)] [data_t 1.419] [optim_t 0.603] [lr 0.005000] +2024-04-02 12:54:14,293 - Train: 1.37% [55600/4070000] [13.7/1000.0] [batch_t 0.605 (0.661)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 12:55:16,383 - Train: 1.37% [55700/4070000] [13.7/1000.0] [batch_t 0.603 (0.621)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-02 12:56:22,782 - Train: 1.37% [55800/4070000] [13.7/1000.0] [batch_t 0.608 (0.664)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-02 12:57:34,996 - Train: 1.37% [55900/4070000] [13.7/1000.0] [batch_t 0.607 (0.722)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 12:58:39,740 - Train: 1.38% [56000/4070000] [13.8/1000.0] [batch_t 0.606 (0.647)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 13:02:00,767 - Train: 1.38% [56100/4070000] [13.8/1000.0] [batch_t 2.962 (2.009)] [data_t 2.357] [optim_t 0.604] [lr 0.005000] +2024-04-02 13:06:54,028 - Train: 1.38% [56200/4070000] [13.8/1000.0] [batch_t 1.398 (2.933)] [data_t 0.002] [optim_t 1.396] [lr 0.005000] +2024-04-02 13:09:15,174 - Train: 1.38% [56300/4070000] [13.8/1000.0] [batch_t 1.409 (1.411)] [data_t 0.002] [optim_t 1.407] [lr 0.005000] +2024-04-02 13:11:49,357 - Train: 1.39% [56400/4070000] [13.9/1000.0] [batch_t 1.419 (1.542)] [data_t 0.002] [optim_t 1.417] [lr 0.005000] +2024-04-02 13:14:10,520 - Train: 1.39% [56500/4070000] [13.9/1000.0] [batch_t 1.403 (1.412)] [data_t 0.002] [optim_t 1.401] [lr 0.005000] +2024-04-02 13:16:55,322 - Train: 1.39% [56600/4070000] [13.9/1000.0] [batch_t 1.417 (1.648)] [data_t 0.002] [optim_t 1.415] [lr 0.005000] +2024-04-02 13:19:16,483 - Train: 1.39% [56700/4070000] [13.9/1000.0] [batch_t 1.407 (1.412)] [data_t 0.002] [optim_t 1.405] [lr 0.005000] +2024-04-02 13:21:37,598 - Train: 1.40% [56800/4070000] [14.0/1000.0] [batch_t 1.412 (1.411)] [data_t 0.002] [optim_t 1.410] [lr 0.005000] +2024-04-02 13:23:58,779 - Train: 1.40% [56900/4070000] [14.0/1000.0] [batch_t 1.416 (1.412)] [data_t 0.002] [optim_t 1.414] [lr 0.005000] +2024-04-02 13:25:51,655 - ==> Total time: 12:25:22 Eta: 36 days, 10:55:42 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-02 13:26:23,143 - Train: 1.40% [57000/4070000] [14.0/1000.0] [batch_t 1.412 (1.475)] [data_t 0.002] [optim_t 1.410] [lr 0.005000] +2024-04-02 13:28:44,350 - Train: 1.40% [57100/4070000] [14.0/1000.0] [batch_t 1.403 (1.412)] [data_t 0.002] [optim_t 1.401] [lr 0.005000] +2024-04-02 13:31:05,556 - Train: 1.41% [57200/4070000] [14.1/1000.0] [batch_t 1.407 (1.412)] [data_t 0.002] [optim_t 1.405] [lr 0.005000] +2024-04-02 13:33:26,577 - Train: 1.41% [57300/4070000] [14.1/1000.0] [batch_t 1.404 (1.410)] [data_t 0.002] [optim_t 1.401] [lr 0.005000] +2024-04-02 13:35:47,786 - Train: 1.41% [57400/4070000] [14.1/1000.0] [batch_t 1.411 (1.412)] [data_t 0.002] [optim_t 1.409] [lr 0.005000] +2024-04-02 13:38:08,934 - Train: 1.41% [57500/4070000] [14.1/1000.0] [batch_t 1.419 (1.411)] [data_t 0.002] [optim_t 1.416] [lr 0.005000] +2024-04-02 13:40:30,166 - Train: 1.42% [57600/4070000] [14.2/1000.0] [batch_t 1.404 (1.412)] [data_t 0.002] [optim_t 1.402] [lr 0.005000] +2024-04-02 13:42:51,399 - Train: 1.42% [57700/4070000] [14.2/1000.0] [batch_t 1.408 (1.412)] [data_t 0.002] [optim_t 1.406] [lr 0.005000] +2024-04-02 13:45:12,384 - Train: 1.42% [57800/4070000] [14.2/1000.0] [batch_t 1.416 (1.410)] [data_t 0.002] [optim_t 1.414] [lr 0.005000] +2024-04-02 13:47:33,372 - Train: 1.42% [57900/4070000] [14.2/1000.0] [batch_t 1.391 (1.410)] [data_t 0.002] [optim_t 1.389] [lr 0.005000] +2024-04-02 13:49:54,467 - Train: 1.43% [58000/4070000] [14.3/1000.0] [batch_t 1.416 (1.411)] [data_t 0.002] [optim_t 1.414] [lr 0.005000] +2024-04-02 13:52:15,530 - Train: 1.43% [58100/4070000] [14.3/1000.0] [batch_t 1.395 (1.411)] [data_t 0.002] [optim_t 1.393] [lr 0.005000] +2024-04-02 13:54:36,502 - Train: 1.43% [58200/4070000] [14.3/1000.0] [batch_t 1.411 (1.410)] [data_t 0.002] [optim_t 1.409] [lr 0.005000] +2024-04-02 13:56:57,607 - Train: 1.43% [58300/4070000] [14.3/1000.0] [batch_t 1.415 (1.411)] [data_t 0.002] [optim_t 1.413] [lr 0.005000] +2024-04-02 13:59:18,732 - Train: 1.43% [58400/4070000] [14.3/1000.0] [batch_t 1.411 (1.411)] [data_t 0.002] [optim_t 1.409] [lr 0.005000] +2024-04-02 14:01:39,865 - Train: 1.44% [58500/4070000] [14.4/1000.0] [batch_t 1.417 (1.411)] [data_t 0.002] [optim_t 1.415] [lr 0.005000] +2024-04-02 14:04:00,934 - Train: 1.44% [58600/4070000] [14.4/1000.0] [batch_t 1.411 (1.411)] [data_t 0.002] [optim_t 1.409] [lr 0.005000] +2024-04-02 14:06:21,984 - Train: 1.44% [58700/4070000] [14.4/1000.0] [batch_t 1.409 (1.410)] [data_t 0.002] [optim_t 1.406] [lr 0.005000] +2024-04-02 14:08:05,370 - Train: 1.44% [58800/4070000] [14.4/1000.0] [batch_t 0.606 (1.034)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 14:09:05,868 - Train: 1.45% [58900/4070000] [14.5/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 14:10:08,269 - Train: 1.45% [59000/4070000] [14.5/1000.0] [batch_t 1.284 (0.624)] [data_t 0.682] [optim_t 0.602] [lr 0.005000] +2024-04-02 14:11:09,829 - Train: 1.45% [59100/4070000] [14.5/1000.0] [batch_t 0.605 (0.616)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 14:12:11,126 - Train: 1.45% [59200/4070000] [14.5/1000.0] [batch_t 0.605 (0.613)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 14:13:12,036 - Train: 1.46% [59300/4070000] [14.6/1000.0] [batch_t 0.603 (0.609)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 14:14:12,822 - Train: 1.46% [59400/4070000] [14.6/1000.0] [batch_t 0.603 (0.608)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-02 14:15:14,523 - Train: 1.46% [59500/4070000] [14.6/1000.0] [batch_t 0.606 (0.617)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 14:16:15,026 - Train: 1.46% [59600/4070000] [14.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 14:17:18,591 - Train: 1.47% [59700/4070000] [14.7/1000.0] [batch_t 0.603 (0.636)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 14:18:21,624 - Train: 1.47% [59800/4070000] [14.7/1000.0] [batch_t 0.603 (0.630)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 14:19:22,596 - Train: 1.47% [59900/4070000] [14.7/1000.0] [batch_t 0.607 (0.610)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 14:20:24,591 - Train: 1.47% [60000/4070000] [14.7/1000.0] [batch_t 0.606 (0.620)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 14:21:25,573 - Train: 1.48% [60100/4070000] [14.8/1000.0] [batch_t 0.606 (0.610)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 14:22:28,992 - Train: 1.48% [60200/4070000] [14.8/1000.0] [batch_t 0.606 (0.634)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 14:23:30,237 - Train: 1.48% [60300/4070000] [14.8/1000.0] [batch_t 0.603 (0.612)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 14:24:32,348 - Train: 1.48% [60400/4070000] [14.8/1000.0] [batch_t 0.605 (0.621)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 14:25:34,375 - Train: 1.49% [60500/4070000] [14.9/1000.0] [batch_t 0.604 (0.620)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 14:26:36,666 - Train: 1.49% [60600/4070000] [14.9/1000.0] [batch_t 0.605 (0.623)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 14:27:41,533 - Train: 1.49% [60700/4070000] [14.9/1000.0] [batch_t 0.603 (0.649)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-02 14:28:42,210 - Train: 1.49% [60800/4070000] [14.9/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 14:29:47,454 - Train: 1.50% [60900/4070000] [15.0/1000.0] [batch_t 0.609 (0.652)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-02 14:30:51,143 - Train: 1.50% [61000/4070000] [15.0/1000.0] [batch_t 0.823 (0.637)] [data_t 0.215] [optim_t 0.608] [lr 0.005000] +2024-04-02 14:31:28,065 - ==> Total time: 13:30:58 Eta: 36 days, 23:34:29 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-02 14:32:10,750 - Train: 1.50% [61100/4070000] [15.0/1000.0] [batch_t 0.604 (0.804)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 14:33:15,497 - Train: 1.50% [61200/4070000] [15.0/1000.0] [batch_t 0.603 (0.647)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-02 14:34:18,026 - Train: 1.51% [61300/4070000] [15.1/1000.0] [batch_t 0.603 (0.625)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 14:35:20,995 - Train: 1.51% [61400/4070000] [15.1/1000.0] [batch_t 0.606 (0.630)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 14:36:22,532 - Train: 1.51% [61500/4070000] [15.1/1000.0] [batch_t 0.606 (0.615)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 14:37:29,137 - Train: 1.51% [61600/4070000] [15.1/1000.0] [batch_t 0.603 (0.666)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-02 14:38:41,673 - Train: 1.52% [61700/4070000] [15.2/1000.0] [batch_t 0.605 (0.725)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 14:39:45,503 - Train: 1.52% [61800/4070000] [15.2/1000.0] [batch_t 0.606 (0.638)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 14:40:49,842 - Train: 1.52% [61900/4070000] [15.2/1000.0] [batch_t 0.607 (0.643)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 14:42:00,865 - Train: 1.52% [62000/4070000] [15.2/1000.0] [batch_t 0.606 (0.710)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 14:43:05,299 - Train: 1.53% [62100/4070000] [15.3/1000.0] [batch_t 0.605 (0.644)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 14:44:05,755 - Train: 1.53% [62200/4070000] [15.3/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 14:45:12,651 - Train: 1.53% [62300/4070000] [15.3/1000.0] [batch_t 0.607 (0.669)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-02 14:46:17,210 - Train: 1.53% [62400/4070000] [15.3/1000.0] [batch_t 0.603 (0.645)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-02 14:47:18,282 - Train: 1.54% [62500/4070000] [15.4/1000.0] [batch_t 0.606 (0.611)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 14:48:19,421 - Train: 1.54% [62600/4070000] [15.4/1000.0] [batch_t 0.605 (0.611)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 14:49:23,884 - Train: 1.54% [62700/4070000] [15.4/1000.0] [batch_t 0.604 (0.645)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 14:50:25,938 - Train: 1.54% [62800/4070000] [15.4/1000.0] [batch_t 0.606 (0.620)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 14:51:28,261 - Train: 1.55% [62900/4070000] [15.5/1000.0] [batch_t 0.606 (0.623)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 14:52:37,928 - Train: 1.55% [63000/4070000] [15.5/1000.0] [batch_t 0.602 (0.697)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-02 14:53:41,548 - Train: 1.55% [63100/4070000] [15.5/1000.0] [batch_t 0.606 (0.636)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 14:54:44,696 - Train: 1.55% [63200/4070000] [15.5/1000.0] [batch_t 0.606 (0.631)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 14:56:00,012 - Train: 1.56% [63300/4070000] [15.6/1000.0] [batch_t 0.604 (0.753)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 15:00:15,276 - Train: 1.56% [63400/4070000] [15.6/1000.0] [batch_t 1.417 (2.553)] [data_t 0.002] [optim_t 1.415] [lr 0.005000] +2024-04-02 15:03:01,794 - Train: 1.56% [63500/4070000] [15.6/1000.0] [batch_t 1.403 (1.665)] [data_t 0.002] [optim_t 1.401] [lr 0.005000] +2024-04-02 15:06:27,673 - Train: 1.56% [63600/4070000] [15.6/1000.0] [batch_t 1.408 (2.059)] [data_t 0.002] [optim_t 1.405] [lr 0.005000] +2024-04-02 15:08:51,873 - Train: 1.57% [63700/4070000] [15.7/1000.0] [batch_t 1.417 (1.442)] [data_t 0.002] [optim_t 1.415] [lr 0.005000] +2024-04-02 15:11:31,933 - Train: 1.57% [63800/4070000] [15.7/1000.0] [batch_t 1.422 (1.601)] [data_t 0.002] [optim_t 1.419] [lr 0.005000] +2024-04-02 15:15:25,443 - Train: 1.57% [63900/4070000] [15.7/1000.0] [batch_t 1.409 (2.335)] [data_t 0.002] [optim_t 1.407] [lr 0.005000] +2024-04-02 15:17:48,839 - Train: 1.57% [64000/4070000] [15.7/1000.0] [batch_t 1.413 (1.434)] [data_t 0.002] [optim_t 1.411] [lr 0.005000] +2024-04-02 15:20:10,013 - Train: 1.57% [64100/4070000] [15.7/1000.0] [batch_t 1.407 (1.412)] [data_t 0.002] [optim_t 1.405] [lr 0.005000] +2024-04-02 15:22:40,762 - Train: 1.58% [64200/4070000] [15.8/1000.0] [batch_t 1.412 (1.507)] [data_t 0.002] [optim_t 1.410] [lr 0.005000] +2024-04-02 15:25:01,937 - Train: 1.58% [64300/4070000] [15.8/1000.0] [batch_t 1.406 (1.412)] [data_t 0.002] [optim_t 1.404] [lr 0.005000] +2024-04-02 15:27:23,071 - Train: 1.58% [64400/4070000] [15.8/1000.0] [batch_t 1.411 (1.411)] [data_t 0.002] [optim_t 1.409] [lr 0.005000] +2024-04-02 15:29:44,279 - Train: 1.58% [64500/4070000] [15.8/1000.0] [batch_t 1.412 (1.412)] [data_t 0.002] [optim_t 1.409] [lr 0.005000] +2024-04-02 15:32:05,434 - Train: 1.59% [64600/4070000] [15.9/1000.0] [batch_t 1.411 (1.411)] [data_t 0.002] [optim_t 1.409] [lr 0.005000] +2024-04-02 15:34:26,528 - Train: 1.59% [64700/4070000] [15.9/1000.0] [batch_t 1.389 (1.411)] [data_t 0.002] [optim_t 1.387] [lr 0.005000] +2024-04-02 15:36:47,594 - Train: 1.59% [64800/4070000] [15.9/1000.0] [batch_t 1.413 (1.411)] [data_t 0.002] [optim_t 1.410] [lr 0.005000] +2024-04-02 15:39:08,828 - Train: 1.59% [64900/4070000] [15.9/1000.0] [batch_t 1.411 (1.412)] [data_t 0.002] [optim_t 1.409] [lr 0.005000] +2024-04-02 15:41:30,077 - Train: 1.60% [65000/4070000] [16.0/1000.0] [batch_t 1.413 (1.412)] [data_t 0.002] [optim_t 1.411] [lr 0.005000] +2024-04-02 15:43:51,314 - Train: 1.60% [65100/4070000] [16.0/1000.0] [batch_t 1.412 (1.412)] [data_t 0.002] [optim_t 1.410] [lr 0.005000] +2024-04-02 15:44:19,547 - ==> Total time: 14:43:50 Eta: 37 days, 17:56:09 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-02 15:46:14,386 - Train: 1.60% [65200/4070000] [16.0/1000.0] [batch_t 1.409 (1.416)] [data_t 0.002] [optim_t 1.407] [lr 0.005000] +2024-04-02 15:48:35,612 - Train: 1.60% [65300/4070000] [16.0/1000.0] [batch_t 1.413 (1.412)] [data_t 0.002] [optim_t 1.411] [lr 0.005000] +2024-04-02 15:50:56,869 - Train: 1.61% [65400/4070000] [16.1/1000.0] [batch_t 1.409 (1.412)] [data_t 0.002] [optim_t 1.407] [lr 0.005000] +2024-04-02 15:53:17,990 - Train: 1.61% [65500/4070000] [16.1/1000.0] [batch_t 1.416 (1.411)] [data_t 0.002] [optim_t 1.414] [lr 0.005000] +2024-04-02 15:55:39,147 - Train: 1.61% [65600/4070000] [16.1/1000.0] [batch_t 1.411 (1.411)] [data_t 0.002] [optim_t 1.409] [lr 0.005000] +2024-04-02 15:58:00,341 - Train: 1.61% [65700/4070000] [16.1/1000.0] [batch_t 1.412 (1.412)] [data_t 0.002] [optim_t 1.409] [lr 0.005000] +2024-04-02 16:00:21,479 - Train: 1.62% [65800/4070000] [16.2/1000.0] [batch_t 1.412 (1.411)] [data_t 0.002] [optim_t 1.410] [lr 0.005000] +2024-04-02 16:02:42,488 - Train: 1.62% [65900/4070000] [16.2/1000.0] [batch_t 1.418 (1.410)] [data_t 0.002] [optim_t 1.415] [lr 0.005000] +2024-04-02 16:05:03,577 - Train: 1.62% [66000/4070000] [16.2/1000.0] [batch_t 1.413 (1.411)] [data_t 0.002] [optim_t 1.410] [lr 0.005000] +2024-04-02 16:07:24,828 - Train: 1.62% [66100/4070000] [16.2/1000.0] [batch_t 1.416 (1.412)] [data_t 0.002] [optim_t 1.414] [lr 0.005000] +2024-04-02 16:09:45,939 - Train: 1.63% [66200/4070000] [16.3/1000.0] [batch_t 1.414 (1.411)] [data_t 0.002] [optim_t 1.411] [lr 0.005000] +2024-04-02 16:12:09,682 - Train: 1.63% [66300/4070000] [16.3/1000.0] [batch_t 1.412 (1.437)] [data_t 0.002] [optim_t 1.410] [lr 0.005000] +2024-04-02 16:14:30,741 - Train: 1.63% [66400/4070000] [16.3/1000.0] [batch_t 1.408 (1.411)] [data_t 0.002] [optim_t 1.406] [lr 0.005000] +2024-04-02 16:17:01,841 - Train: 1.63% [66500/4070000] [16.3/1000.0] [batch_t 1.418 (1.511)] [data_t 0.002] [optim_t 1.415] [lr 0.005000] +2024-04-02 16:19:23,936 - Train: 1.64% [66600/4070000] [16.4/1000.0] [batch_t 1.420 (1.421)] [data_t 0.002] [optim_t 1.418] [lr 0.005000] +2024-04-02 16:21:45,579 - Train: 1.64% [66700/4070000] [16.4/1000.0] [batch_t 1.412 (1.416)] [data_t 0.002] [optim_t 1.409] [lr 0.005000] +2024-04-02 16:24:48,988 - Train: 1.64% [66800/4070000] [16.4/1000.0] [batch_t 1.412 (1.834)] [data_t 0.002] [optim_t 1.410] [lr 0.005000] +2024-04-02 16:29:09,434 - Train: 1.64% [66900/4070000] [16.4/1000.0] [batch_t 6.512 (2.604)] [data_t 5.108] [optim_t 1.404] [lr 0.005000] +2024-04-02 16:31:48,338 - Train: 1.65% [67000/4070000] [16.5/1000.0] [batch_t 1.418 (1.589)] [data_t 0.002] [optim_t 1.416] [lr 0.005000] +2024-04-02 16:35:37,835 - Train: 1.65% [67100/4070000] [16.5/1000.0] [batch_t 1.416 (2.295)] [data_t 0.002] [optim_t 1.414] [lr 0.005000] +2024-04-02 16:41:24,886 - Train: 1.65% [67200/4070000] [16.5/1000.0] [batch_t 7.673 (3.470)] [data_t 6.240] [optim_t 1.432] [lr 0.005000] +2024-04-02 16:45:31,525 - Train: 1.65% [67300/4070000] [16.5/1000.0] [batch_t 1.439 (2.466)] [data_t 0.003] [optim_t 1.436] [lr 0.005000] +2024-04-02 16:47:54,537 - Train: 1.66% [67400/4070000] [16.6/1000.0] [batch_t 1.415 (1.430)] [data_t 0.003] [optim_t 1.412] [lr 0.005000] +2024-04-02 16:50:17,887 - Train: 1.66% [67500/4070000] [16.6/1000.0] [batch_t 1.439 (1.433)] [data_t 0.002] [optim_t 1.436] [lr 0.005000] +2024-04-02 16:52:40,850 - Train: 1.66% [67600/4070000] [16.6/1000.0] [batch_t 1.435 (1.430)] [data_t 0.003] [optim_t 1.432] [lr 0.005000] +2024-04-02 16:56:02,647 - Train: 1.66% [67700/4070000] [16.6/1000.0] [batch_t 1.427 (2.018)] [data_t 0.003] [optim_t 1.424] [lr 0.005000] +2024-04-02 16:58:31,487 - Train: 1.67% [67800/4070000] [16.7/1000.0] [batch_t 1.427 (1.488)] [data_t 0.003] [optim_t 1.424] [lr 0.005000] +2024-04-02 17:00:54,553 - Train: 1.67% [67900/4070000] [16.7/1000.0] [batch_t 1.420 (1.431)] [data_t 0.003] [optim_t 1.417] [lr 0.005000] +2024-04-02 17:03:22,818 - Train: 1.67% [68000/4070000] [16.7/1000.0] [batch_t 1.430 (1.483)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-02 17:05:45,836 - Train: 1.67% [68100/4070000] [16.7/1000.0] [batch_t 1.416 (1.430)] [data_t 0.003] [optim_t 1.413] [lr 0.005000] +2024-04-02 17:07:54,420 - Train: 1.68% [68200/4070000] [16.8/1000.0] [batch_t 0.603 (1.286)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 17:10:54,464 - Train: 1.68% [68300/4070000] [16.8/1000.0] [batch_t 0.606 (1.800)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 17:13:10,406 - Train: 1.68% [68400/4070000] [16.8/1000.0] [batch_t 0.608 (1.359)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-02 17:14:13,689 - Train: 1.68% [68500/4070000] [16.8/1000.0] [batch_t 0.605 (0.633)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 17:15:18,687 - Train: 1.69% [68600/4070000] [16.9/1000.0] [batch_t 0.606 (0.650)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 17:16:23,670 - Train: 1.69% [68700/4070000] [16.9/1000.0] [batch_t 0.604 (0.650)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 17:17:28,567 - Train: 1.69% [68800/4070000] [16.9/1000.0] [batch_t 0.605 (0.649)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 17:18:38,338 - Train: 1.69% [68900/4070000] [16.9/1000.0] [batch_t 0.603 (0.698)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-02 17:19:41,055 - Train: 1.70% [69000/4070000] [17.0/1000.0] [batch_t 0.605 (0.627)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 17:20:45,523 - Train: 1.70% [69100/4070000] [17.0/1000.0] [batch_t 0.606 (0.645)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 17:21:42,405 - ==> Total time: 16:21:13 Eta: 39 days, 9:37:39 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-02 17:21:50,700 - Train: 1.70% [69200/4070000] [17.0/1000.0] [batch_t 0.602 (0.647)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-02 17:22:55,175 - Train: 1.70% [69300/4070000] [17.0/1000.0] [batch_t 0.605 (0.645)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 17:24:47,275 - Train: 1.71% [69400/4070000] [17.1/1000.0] [batch_t 0.605 (1.120)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 17:27:50,299 - Train: 1.71% [69500/4070000] [17.1/1000.0] [batch_t 2.192 (1.830)] [data_t 1.587] [optim_t 0.605] [lr 0.005000] +2024-04-02 17:29:52,693 - Train: 1.71% [69600/4070000] [17.1/1000.0] [batch_t 0.605 (1.224)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 17:30:57,853 - Train: 1.71% [69700/4070000] [17.1/1000.0] [batch_t 0.606 (0.652)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 17:33:01,478 - Train: 1.71% [69800/4070000] [17.1/1000.0] [batch_t 0.605 (1.236)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 17:34:06,583 - Train: 1.72% [69900/4070000] [17.2/1000.0] [batch_t 0.605 (0.651)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 17:35:07,794 - Train: 1.72% [70000/4070000] [17.2/1000.0] [batch_t 1.088 (0.612)] [data_t 0.484] [optim_t 0.604] [lr 0.005000] +2024-04-02 17:36:09,788 - Train: 1.72% [70100/4070000] [17.2/1000.0] [batch_t 0.605 (0.620)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 17:37:12,974 - Train: 1.72% [70200/4070000] [17.2/1000.0] [batch_t 0.963 (0.632)] [data_t 0.364] [optim_t 0.599] [lr 0.005000] +2024-04-02 17:38:27,220 - Train: 1.73% [70300/4070000] [17.3/1000.0] [batch_t 0.607 (0.742)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 17:39:30,288 - Train: 1.73% [70400/4070000] [17.3/1000.0] [batch_t 0.606 (0.631)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 17:40:31,646 - Train: 1.73% [70500/4070000] [17.3/1000.0] [batch_t 0.605 (0.613)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 17:41:33,508 - Train: 1.73% [70600/4070000] [17.3/1000.0] [batch_t 0.608 (0.619)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-02 17:42:38,381 - Train: 1.74% [70700/4070000] [17.4/1000.0] [batch_t 0.604 (0.649)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 17:43:40,716 - Train: 1.74% [70800/4070000] [17.4/1000.0] [batch_t 0.609 (0.623)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-02 17:45:01,511 - Train: 1.74% [70900/4070000] [17.4/1000.0] [batch_t 0.608 (0.808)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-02 17:46:03,776 - Train: 1.74% [71000/4070000] [17.4/1000.0] [batch_t 0.602 (0.623)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-02 17:47:07,568 - Train: 1.75% [71100/4070000] [17.5/1000.0] [batch_t 0.607 (0.638)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 17:48:08,086 - Train: 1.75% [71200/4070000] [17.5/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-02 17:49:09,872 - Train: 1.75% [71300/4070000] [17.5/1000.0] [batch_t 0.605 (0.618)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 17:50:20,274 - Train: 1.75% [71400/4070000] [17.5/1000.0] [batch_t 0.605 (0.704)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 17:51:22,834 - Train: 1.76% [71500/4070000] [17.6/1000.0] [batch_t 0.604 (0.626)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 17:52:26,251 - Train: 1.76% [71600/4070000] [17.6/1000.0] [batch_t 0.605 (0.634)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 17:53:28,914 - Train: 1.76% [71700/4070000] [17.6/1000.0] [batch_t 0.606 (0.627)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 17:54:31,491 - Train: 1.76% [71800/4070000] [17.6/1000.0] [batch_t 0.607 (0.626)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 17:55:33,084 - Train: 1.77% [71900/4070000] [17.7/1000.0] [batch_t 0.606 (0.616)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 17:56:36,185 - Train: 1.77% [72000/4070000] [17.7/1000.0] [batch_t 0.612 (0.631)] [data_t 0.002] [optim_t 0.610] [lr 0.005000] +2024-04-02 17:58:04,834 - Train: 1.77% [72100/4070000] [17.7/1000.0] [batch_t 2.815 (0.886)] [data_t 2.211] [optim_t 0.604] [lr 0.005000] +2024-04-02 17:59:05,700 - Train: 1.77% [72200/4070000] [17.7/1000.0] [batch_t 0.609 (0.609)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-02 18:00:11,607 - Train: 1.78% [72300/4070000] [17.8/1000.0] [batch_t 4.355 (0.659)] [data_t 3.752] [optim_t 0.603] [lr 0.005000] +2024-04-02 18:01:16,785 - Train: 1.78% [72400/4070000] [17.8/1000.0] [batch_t 0.601 (0.652)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-02 18:02:35,249 - Train: 1.78% [72500/4070000] [17.8/1000.0] [batch_t 0.786 (0.785)] [data_t 0.185] [optim_t 0.602] [lr 0.005000] +2024-04-02 18:03:38,410 - Train: 1.78% [72600/4070000] [17.8/1000.0] [batch_t 0.603 (0.632)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-02 18:04:41,289 - Train: 1.79% [72700/4070000] [17.9/1000.0] [batch_t 0.603 (0.629)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 18:05:46,375 - Train: 1.79% [72800/4070000] [17.9/1000.0] [batch_t 0.605 (0.651)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 18:06:48,030 - Train: 1.79% [72900/4070000] [17.9/1000.0] [batch_t 0.606 (0.616)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 18:07:50,527 - Train: 1.79% [73000/4070000] [17.9/1000.0] [batch_t 0.605 (0.625)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 18:09:03,775 - Train: 1.80% [73100/4070000] [18.0/1000.0] [batch_t 0.605 (0.732)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 18:10:08,522 - Train: 1.80% [73200/4070000] [18.0/1000.0] [batch_t 0.605 (0.647)] [data_t 0.003] [optim_t 0.603] [lr 0.005000] +2024-04-02 18:10:44,819 - ==> Total time: 17:10:15 Eta: 39 days, 0:46:28 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-02 18:11:13,195 - Train: 1.80% [73300/4070000] [18.0/1000.0] [batch_t 0.605 (0.660)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 18:12:20,212 - Train: 1.80% [73400/4070000] [18.0/1000.0] [batch_t 0.604 (0.670)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 18:13:21,291 - Train: 1.81% [73500/4070000] [18.1/1000.0] [batch_t 0.604 (0.611)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 18:14:24,483 - Train: 1.81% [73600/4070000] [18.1/1000.0] [batch_t 0.605 (0.632)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 18:15:33,798 - Train: 1.81% [73700/4070000] [18.1/1000.0] [batch_t 0.604 (0.693)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 18:16:40,588 - Train: 1.81% [73800/4070000] [18.1/1000.0] [batch_t 0.603 (0.668)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 18:17:47,598 - Train: 1.82% [73900/4070000] [18.2/1000.0] [batch_t 0.604 (0.670)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 18:18:51,720 - Train: 1.82% [74000/4070000] [18.2/1000.0] [batch_t 0.606 (0.641)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 18:20:00,865 - Train: 1.82% [74100/4070000] [18.2/1000.0] [batch_t 0.606 (0.691)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 18:21:01,370 - Train: 1.82% [74200/4070000] [18.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 18:22:02,172 - Train: 1.83% [74300/4070000] [18.3/1000.0] [batch_t 0.608 (0.608)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-02 18:23:02,764 - Train: 1.83% [74400/4070000] [18.3/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 18:24:03,263 - Train: 1.83% [74500/4070000] [18.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 18:25:03,791 - Train: 1.83% [74600/4070000] [18.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 18:26:04,272 - Train: 1.84% [74700/4070000] [18.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 18:27:04,885 - Train: 1.84% [74800/4070000] [18.4/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 18:28:05,387 - Train: 1.84% [74900/4070000] [18.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 18:29:05,901 - Train: 1.84% [75000/4070000] [18.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 18:30:06,386 - Train: 1.85% [75100/4070000] [18.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 18:31:06,924 - Train: 1.85% [75200/4070000] [18.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 18:32:07,558 - Train: 1.85% [75300/4070000] [18.5/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-02 18:33:08,078 - Train: 1.85% [75400/4070000] [18.5/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-02 18:34:09,570 - Train: 1.86% [75500/4070000] [18.6/1000.0] [batch_t 0.606 (0.615)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 18:35:11,389 - Train: 1.86% [75600/4070000] [18.6/1000.0] [batch_t 0.783 (0.618)] [data_t 0.180] [optim_t 0.603] [lr 0.005000] +2024-04-02 18:36:13,301 - Train: 1.86% [75700/4070000] [18.6/1000.0] [batch_t 0.606 (0.619)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 18:37:13,808 - Train: 1.86% [75800/4070000] [18.6/1000.0] [batch_t 0.597 (0.605)] [data_t 0.002] [optim_t 0.596] [lr 0.005000] +2024-04-02 18:38:31,561 - Train: 1.86% [75900/4070000] [18.6/1000.0] [batch_t 0.605 (0.777)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 18:39:32,077 - Train: 1.87% [76000/4070000] [18.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 18:40:33,972 - Train: 1.87% [76100/4070000] [18.7/1000.0] [batch_t 0.609 (0.619)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-02 18:41:34,516 - Train: 1.87% [76200/4070000] [18.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 18:42:35,013 - Train: 1.87% [76300/4070000] [18.7/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-02 18:43:46,244 - Train: 1.88% [76400/4070000] [18.8/1000.0] [batch_t 0.606 (0.712)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 18:44:46,750 - Train: 1.88% [76500/4070000] [18.8/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-02 18:45:48,011 - Train: 1.88% [76600/4070000] [18.8/1000.0] [batch_t 0.605 (0.613)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 18:46:49,542 - Train: 1.88% [76700/4070000] [18.8/1000.0] [batch_t 0.608 (0.615)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-02 18:47:53,232 - Train: 1.89% [76800/4070000] [18.9/1000.0] [batch_t 0.603 (0.637)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-02 18:48:54,963 - Train: 1.89% [76900/4070000] [18.9/1000.0] [batch_t 0.604 (0.617)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 18:49:55,630 - Train: 1.89% [77000/4070000] [18.9/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 18:50:57,022 - Train: 1.89% [77100/4070000] [18.9/1000.0] [batch_t 0.606 (0.614)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 18:51:57,948 - Train: 1.90% [77200/4070000] [19.0/1000.0] [batch_t 0.602 (0.609)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-02 18:53:38,413 - Train: 1.90% [77300/4070000] [19.0/1000.0] [batch_t 1.092 (1.005)] [data_t 0.486] [optim_t 0.606] [lr 0.005000] +2024-04-02 18:53:59,389 - ==> Total time: 17:53:30 Eta: 38 days, 11:46:42 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-02 18:55:05,524 - Train: 1.90% [77400/4070000] [19.0/1000.0] [batch_t 0.603 (0.916)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-02 18:57:36,821 - Train: 1.90% [77500/4070000] [19.0/1000.0] [batch_t 1.279 (1.513)] [data_t 0.675] [optim_t 0.603] [lr 0.005000] +2024-04-02 19:16:21,520 - Train: 1.91% [77600/4070000] [19.1/1000.0] [batch_t 1.427 (11.247)] [data_t 0.002] [optim_t 1.425] [lr 0.005000] +2024-04-02 19:19:24,771 - Train: 1.91% [77700/4070000] [19.1/1000.0] [batch_t 1.427 (1.832)] [data_t 0.003] [optim_t 1.425] [lr 0.005000] +2024-04-02 19:22:43,654 - Train: 1.91% [77800/4070000] [19.1/1000.0] [batch_t 5.038 (1.989)] [data_t 3.599] [optim_t 1.439] [lr 0.005000] +2024-04-02 19:27:16,236 - Train: 1.91% [77900/4070000] [19.1/1000.0] [batch_t 2.164 (2.726)] [data_t 0.723] [optim_t 1.441] [lr 0.005000] +2024-04-02 19:31:35,598 - Train: 1.92% [78000/4070000] [19.2/1000.0] [batch_t 2.663 (2.594)] [data_t 1.221] [optim_t 1.442] [lr 0.005000] +2024-04-02 19:34:41,044 - Train: 1.92% [78100/4070000] [19.2/1000.0] [batch_t 1.439 (1.854)] [data_t 0.002] [optim_t 1.437] [lr 0.005000] +2024-04-02 19:37:04,881 - Train: 1.92% [78200/4070000] [19.2/1000.0] [batch_t 1.441 (1.438)] [data_t 0.003] [optim_t 1.438] [lr 0.005000] +2024-04-02 19:39:28,357 - Train: 1.92% [78300/4070000] [19.2/1000.0] [batch_t 1.437 (1.435)] [data_t 0.002] [optim_t 1.435] [lr 0.005000] +2024-04-02 19:41:51,608 - Train: 1.93% [78400/4070000] [19.3/1000.0] [batch_t 1.435 (1.432)] [data_t 0.002] [optim_t 1.432] [lr 0.005000] +2024-04-02 19:44:14,807 - Train: 1.93% [78500/4070000] [19.3/1000.0] [batch_t 1.437 (1.432)] [data_t 0.002] [optim_t 1.435] [lr 0.005000] +2024-04-02 19:46:38,008 - Train: 1.93% [78600/4070000] [19.3/1000.0] [batch_t 1.431 (1.432)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-02 19:49:01,391 - Train: 1.93% [78700/4070000] [19.3/1000.0] [batch_t 1.441 (1.434)] [data_t 0.002] [optim_t 1.438] [lr 0.005000] +2024-04-02 19:51:24,545 - Train: 1.94% [78800/4070000] [19.4/1000.0] [batch_t 1.434 (1.431)] [data_t 0.003] [optim_t 1.432] [lr 0.005000] +2024-04-02 19:53:58,265 - Train: 1.94% [78900/4070000] [19.4/1000.0] [batch_t 1.448 (1.537)] [data_t 0.002] [optim_t 1.445] [lr 0.005000] +2024-04-02 19:56:21,445 - Train: 1.94% [79000/4070000] [19.4/1000.0] [batch_t 1.429 (1.432)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-02 19:58:44,690 - Train: 1.94% [79100/4070000] [19.4/1000.0] [batch_t 1.419 (1.432)] [data_t 0.003] [optim_t 1.416] [lr 0.005000] +2024-04-02 20:01:08,016 - Train: 1.95% [79200/4070000] [19.5/1000.0] [batch_t 1.441 (1.433)] [data_t 0.003] [optim_t 1.439] [lr 0.005000] +2024-04-02 20:04:56,193 - Train: 1.95% [79300/4070000] [19.5/1000.0] [batch_t 1.438 (2.282)] [data_t 0.002] [optim_t 1.436] [lr 0.005000] +2024-04-02 20:07:34,521 - Train: 1.95% [79400/4070000] [19.5/1000.0] [batch_t 1.423 (1.583)] [data_t 0.003] [optim_t 1.420] [lr 0.005000] +2024-04-02 20:10:27,954 - Train: 1.95% [79500/4070000] [19.5/1000.0] [batch_t 3.205 (1.734)] [data_t 1.776] [optim_t 1.429] [lr 0.005000] +2024-04-02 20:13:13,533 - Train: 1.96% [79600/4070000] [19.6/1000.0] [batch_t 1.436 (1.656)] [data_t 0.002] [optim_t 1.434] [lr 0.005000] +2024-04-02 20:16:43,050 - Train: 1.96% [79700/4070000] [19.6/1000.0] [batch_t 1.421 (2.095)] [data_t 0.002] [optim_t 1.419] [lr 0.005000] +2024-04-02 20:19:06,202 - Train: 1.96% [79800/4070000] [19.6/1000.0] [batch_t 1.428 (1.431)] [data_t 0.002] [optim_t 1.426] [lr 0.005000] +2024-04-02 20:21:29,197 - Train: 1.96% [79900/4070000] [19.6/1000.0] [batch_t 1.426 (1.430)] [data_t 0.002] [optim_t 1.424] [lr 0.005000] +2024-04-02 20:25:04,291 - Train: 1.97% [80000/4070000] [19.7/1000.0] [batch_t 1.807 (2.151)] [data_t 0.355] [optim_t 1.452] [lr 0.005000] +2024-04-02 20:30:12,503 - Train: 1.97% [80100/4070000] [19.7/1000.0] [batch_t 1.417 (3.082)] [data_t 0.003] [optim_t 1.414] [lr 0.005000] +2024-04-02 20:32:35,508 - Train: 1.97% [80200/4070000] [19.7/1000.0] [batch_t 1.430 (1.430)] [data_t 0.002] [optim_t 1.427] [lr 0.005000] +2024-04-02 20:34:58,673 - Train: 1.97% [80300/4070000] [19.7/1000.0] [batch_t 1.441 (1.432)] [data_t 0.003] [optim_t 1.438] [lr 0.005000] +2024-04-02 20:37:21,756 - Train: 1.98% [80400/4070000] [19.8/1000.0] [batch_t 1.423 (1.431)] [data_t 0.002] [optim_t 1.421] [lr 0.005000] +2024-04-02 20:39:44,687 - Train: 1.98% [80500/4070000] [19.8/1000.0] [batch_t 1.421 (1.429)] [data_t 0.003] [optim_t 1.418] [lr 0.005000] +2024-04-02 20:42:27,520 - Train: 1.98% [80600/4070000] [19.8/1000.0] [batch_t 1.431 (1.628)] [data_t 0.002] [optim_t 1.428] [lr 0.005000] +2024-04-02 20:44:50,509 - Train: 1.98% [80700/4070000] [19.8/1000.0] [batch_t 1.435 (1.430)] [data_t 0.002] [optim_t 1.432] [lr 0.005000] +2024-04-02 20:47:30,632 - Train: 1.99% [80800/4070000] [19.9/1000.0] [batch_t 1.426 (1.601)] [data_t 0.002] [optim_t 1.424] [lr 0.005000] +2024-04-02 20:50:12,623 - Train: 1.99% [80900/4070000] [19.9/1000.0] [batch_t 1.425 (1.568)] [data_t 0.003] [optim_t 1.422] [lr 0.005000] +2024-04-02 20:52:35,706 - Train: 1.99% [81000/4070000] [19.9/1000.0] [batch_t 1.430 (1.431)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-02 20:55:02,252 - Train: 1.99% [81100/4070000] [19.9/1000.0] [batch_t 1.439 (1.465)] [data_t 0.003] [optim_t 1.436] [lr 0.005000] +2024-04-02 20:57:26,953 - Train: 2.00% [81200/4070000] [20.0/1000.0] [batch_t 1.427 (1.447)] [data_t 0.003] [optim_t 1.424] [lr 0.005000] +2024-04-02 20:59:54,198 - Train: 2.00% [81300/4070000] [20.0/1000.0] [batch_t 1.433 (1.472)] [data_t 0.002] [optim_t 1.430] [lr 0.005000] +2024-04-02 21:02:37,839 - Train: 2.00% [81400/4070000] [20.0/1000.0] [batch_t 1.434 (1.636)] [data_t 0.002] [optim_t 1.432] [lr 0.005000] +2024-04-02 21:02:37,842 - ==> Total time: 20:02:08 Eta: 40 days, 21:45:06 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-02 21:06:12,038 - Train: 2.00% [81500/4070000] [20.0/1000.0] [batch_t 1.435 (2.118)] [data_t 0.002] [optim_t 1.432] [lr 0.005000] +2024-04-02 21:08:38,247 - Train: 2.00% [81600/4070000] [20.0/1000.0] [batch_t 1.426 (1.462)] [data_t 0.003] [optim_t 1.424] [lr 0.005000] +2024-04-02 21:11:01,628 - Train: 2.01% [81700/4070000] [20.1/1000.0] [batch_t 1.430 (1.434)] [data_t 0.002] [optim_t 1.427] [lr 0.005000] +2024-04-02 21:15:57,332 - Train: 2.01% [81800/4070000] [20.1/1000.0] [batch_t 3.355 (2.957)] [data_t 1.927] [optim_t 1.428] [lr 0.005000] +2024-04-02 21:19:53,845 - Train: 2.01% [81900/4070000] [20.1/1000.0] [batch_t 1.450 (2.365)] [data_t 0.002] [optim_t 1.447] [lr 0.005000] +2024-04-02 21:22:56,207 - Train: 2.01% [82000/4070000] [20.1/1000.0] [batch_t 2.398 (1.824)] [data_t 0.964] [optim_t 1.434] [lr 0.005000] +2024-04-02 21:25:20,227 - Train: 2.02% [82100/4070000] [20.2/1000.0] [batch_t 1.446 (1.440)] [data_t 0.003] [optim_t 1.443] [lr 0.005000] +2024-04-02 21:27:55,889 - Train: 2.02% [82200/4070000] [20.2/1000.0] [batch_t 1.430 (1.555)] [data_t 0.002] [optim_t 1.428] [lr 0.005000] +2024-04-02 21:30:21,420 - Train: 2.02% [82300/4070000] [20.2/1000.0] [batch_t 1.430 (1.455)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-02 21:32:53,159 - Train: 2.02% [82400/4070000] [20.2/1000.0] [batch_t 1.434 (1.517)] [data_t 0.003] [optim_t 1.431] [lr 0.005000] +2024-04-02 21:40:01,981 - Train: 2.03% [82500/4070000] [20.3/1000.0] [batch_t 18.465 (4.288)] [data_t 17.038] [optim_t 1.427] [lr 0.005000] +2024-04-02 21:43:43,983 - Train: 2.03% [82600/4070000] [20.3/1000.0] [batch_t 1.433 (2.220)] [data_t 0.003] [optim_t 1.430] [lr 0.005000] +2024-04-02 21:46:53,630 - Train: 2.03% [82700/4070000] [20.3/1000.0] [batch_t 1.435 (1.896)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-02 21:49:19,167 - Train: 2.03% [82800/4070000] [20.3/1000.0] [batch_t 1.443 (1.455)] [data_t 0.003] [optim_t 1.440] [lr 0.005000] +2024-04-02 21:52:28,270 - Train: 2.04% [82900/4070000] [20.4/1000.0] [batch_t 16.165 (1.891)] [data_t 14.706] [optim_t 1.460] [lr 0.005000] +2024-04-02 21:56:24,472 - Train: 2.04% [83000/4070000] [20.4/1000.0] [batch_t 1.436 (2.362)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-02 21:58:47,727 - Train: 2.04% [83100/4070000] [20.4/1000.0] [batch_t 1.442 (1.432)] [data_t 0.003] [optim_t 1.439] [lr 0.005000] +2024-04-02 22:01:10,889 - Train: 2.04% [83200/4070000] [20.4/1000.0] [batch_t 1.436 (1.432)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-02 22:03:38,867 - Train: 2.05% [83300/4070000] [20.5/1000.0] [batch_t 1.438 (1.480)] [data_t 0.003] [optim_t 1.435] [lr 0.005000] +2024-04-02 22:06:20,805 - Train: 2.05% [83400/4070000] [20.5/1000.0] [batch_t 1.433 (1.619)] [data_t 0.002] [optim_t 1.430] [lr 0.005000] +2024-04-02 22:08:43,876 - Train: 2.05% [83500/4070000] [20.5/1000.0] [batch_t 1.433 (1.431)] [data_t 0.002] [optim_t 1.430] [lr 0.005000] +2024-04-02 22:11:07,023 - Train: 2.05% [83600/4070000] [20.5/1000.0] [batch_t 1.411 (1.431)] [data_t 0.003] [optim_t 1.408] [lr 0.005000] +2024-04-02 22:13:30,043 - Train: 2.06% [83700/4070000] [20.6/1000.0] [batch_t 1.421 (1.430)] [data_t 0.003] [optim_t 1.419] [lr 0.005000] +2024-04-02 22:15:53,007 - Train: 2.06% [83800/4070000] [20.6/1000.0] [batch_t 1.432 (1.430)] [data_t 0.002] [optim_t 1.430] [lr 0.005000] +2024-04-02 22:18:15,956 - Train: 2.06% [83900/4070000] [20.6/1000.0] [batch_t 1.435 (1.429)] [data_t 0.003] [optim_t 1.432] [lr 0.005000] +2024-04-02 22:20:39,208 - Train: 2.06% [84000/4070000] [20.6/1000.0] [batch_t 1.421 (1.432)] [data_t 0.002] [optim_t 1.418] [lr 0.005000] +2024-04-02 22:23:02,073 - Train: 2.07% [84100/4070000] [20.7/1000.0] [batch_t 1.432 (1.429)] [data_t 0.003] [optim_t 1.429] [lr 0.005000] +2024-04-02 22:25:36,010 - Train: 2.07% [84200/4070000] [20.7/1000.0] [batch_t 1.427 (1.539)] [data_t 0.002] [optim_t 1.425] [lr 0.005000] +2024-04-02 22:27:59,298 - Train: 2.07% [84300/4070000] [20.7/1000.0] [batch_t 1.442 (1.433)] [data_t 0.002] [optim_t 1.440] [lr 0.005000] +2024-04-02 22:30:25,881 - Train: 2.07% [84400/4070000] [20.7/1000.0] [batch_t 1.413 (1.466)] [data_t 0.002] [optim_t 1.411] [lr 0.005000] +2024-04-02 22:32:49,132 - Train: 2.08% [84500/4070000] [20.8/1000.0] [batch_t 1.433 (1.432)] [data_t 0.003] [optim_t 1.430] [lr 0.005000] +2024-04-02 22:35:15,008 - Train: 2.08% [84600/4070000] [20.8/1000.0] [batch_t 1.437 (1.459)] [data_t 0.003] [optim_t 1.435] [lr 0.005000] +2024-04-02 22:37:38,053 - Train: 2.08% [84700/4070000] [20.8/1000.0] [batch_t 1.431 (1.430)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-02 22:40:01,273 - Train: 2.08% [84800/4070000] [20.8/1000.0] [batch_t 1.407 (1.432)] [data_t 0.003] [optim_t 1.404] [lr 0.005000] +2024-04-02 22:42:24,451 - Train: 2.09% [84900/4070000] [20.9/1000.0] [batch_t 1.433 (1.432)] [data_t 0.003] [optim_t 1.430] [lr 0.005000] +2024-04-02 22:44:47,608 - Train: 2.09% [85000/4070000] [20.9/1000.0] [batch_t 1.430 (1.431)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-02 22:47:10,687 - Train: 2.09% [85100/4070000] [20.9/1000.0] [batch_t 1.426 (1.431)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-02 22:49:33,800 - Train: 2.09% [85200/4070000] [20.9/1000.0] [batch_t 1.430 (1.431)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-02 22:52:01,327 - Train: 2.10% [85300/4070000] [21.0/1000.0] [batch_t 1.444 (1.475)] [data_t 0.002] [optim_t 1.442] [lr 0.005000] +2024-04-02 22:54:24,640 - Train: 2.10% [85400/4070000] [21.0/1000.0] [batch_t 1.431 (1.433)] [data_t 0.002] [optim_t 1.429] [lr 0.005000] +2024-04-02 22:56:04,821 - ==> Total time: 21:55:35 Eta: 42 days, 14:11:46 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-02 22:56:52,330 - Train: 2.10% [85500/4070000] [21.0/1000.0] [batch_t 1.432 (1.504)] [data_t 0.002] [optim_t 1.430] [lr 0.005000] +2024-04-02 22:59:15,585 - Train: 2.10% [85600/4070000] [21.0/1000.0] [batch_t 1.432 (1.432)] [data_t 0.002] [optim_t 1.430] [lr 0.005000] +2024-04-02 23:01:38,575 - Train: 2.11% [85700/4070000] [21.1/1000.0] [batch_t 1.434 (1.430)] [data_t 0.002] [optim_t 1.432] [lr 0.005000] +2024-04-02 23:04:01,631 - Train: 2.11% [85800/4070000] [21.1/1000.0] [batch_t 1.417 (1.430)] [data_t 0.003] [optim_t 1.415] [lr 0.005000] +2024-04-02 23:06:24,655 - Train: 2.11% [85900/4070000] [21.1/1000.0] [batch_t 1.425 (1.430)] [data_t 0.003] [optim_t 1.422] [lr 0.005000] +2024-04-02 23:08:47,574 - Train: 2.11% [86000/4070000] [21.1/1000.0] [batch_t 1.440 (1.429)] [data_t 0.002] [optim_t 1.438] [lr 0.005000] +2024-04-02 23:11:10,404 - Train: 2.12% [86100/4070000] [21.2/1000.0] [batch_t 1.422 (1.428)] [data_t 0.002] [optim_t 1.420] [lr 0.005000] +2024-04-02 23:13:54,142 - Train: 2.12% [86200/4070000] [21.2/1000.0] [batch_t 1.424 (1.637)] [data_t 0.003] [optim_t 1.421] [lr 0.005000] +2024-04-02 23:16:25,314 - Train: 2.12% [86300/4070000] [21.2/1000.0] [batch_t 1.421 (1.512)] [data_t 0.003] [optim_t 1.418] [lr 0.005000] +2024-04-02 23:18:48,109 - Train: 2.12% [86400/4070000] [21.2/1000.0] [batch_t 1.417 (1.428)] [data_t 0.003] [optim_t 1.414] [lr 0.005000] +2024-04-02 23:21:11,025 - Train: 2.13% [86500/4070000] [21.3/1000.0] [batch_t 1.426 (1.429)] [data_t 0.002] [optim_t 1.424] [lr 0.005000] +2024-04-02 23:23:38,020 - Train: 2.13% [86600/4070000] [21.3/1000.0] [batch_t 1.423 (1.470)] [data_t 0.003] [optim_t 1.420] [lr 0.005000] +2024-04-02 23:26:17,090 - Train: 2.13% [86700/4070000] [21.3/1000.0] [batch_t 1.443 (1.591)] [data_t 0.002] [optim_t 1.440] [lr 0.005000] +2024-04-02 23:28:40,235 - Train: 2.13% [86800/4070000] [21.3/1000.0] [batch_t 1.421 (1.431)] [data_t 0.002] [optim_t 1.418] [lr 0.005000] +2024-04-02 23:31:03,102 - Train: 2.14% [86900/4070000] [21.4/1000.0] [batch_t 1.438 (1.429)] [data_t 0.003] [optim_t 1.436] [lr 0.005000] +2024-04-02 23:33:26,123 - Train: 2.14% [87000/4070000] [21.4/1000.0] [batch_t 1.430 (1.430)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-02 23:35:49,236 - Train: 2.14% [87100/4070000] [21.4/1000.0] [batch_t 1.428 (1.431)] [data_t 0.003] [optim_t 1.425] [lr 0.005000] +2024-04-02 23:38:12,417 - Train: 2.14% [87200/4070000] [21.4/1000.0] [batch_t 1.431 (1.432)] [data_t 0.002] [optim_t 1.429] [lr 0.005000] +2024-04-02 23:40:39,572 - Train: 2.14% [87300/4070000] [21.4/1000.0] [batch_t 1.422 (1.471)] [data_t 0.003] [optim_t 1.419] [lr 0.005000] +2024-04-02 23:43:02,633 - Train: 2.15% [87400/4070000] [21.5/1000.0] [batch_t 1.430 (1.430)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-02 23:45:25,857 - Train: 2.15% [87500/4070000] [21.5/1000.0] [batch_t 1.440 (1.432)] [data_t 0.002] [optim_t 1.438] [lr 0.005000] +2024-04-02 23:47:48,957 - Train: 2.15% [87600/4070000] [21.5/1000.0] [batch_t 1.421 (1.431)] [data_t 0.003] [optim_t 1.419] [lr 0.005000] +2024-04-02 23:50:12,044 - Train: 2.15% [87700/4070000] [21.5/1000.0] [batch_t 1.446 (1.431)] [data_t 0.002] [optim_t 1.444] [lr 0.005000] +2024-04-02 23:52:35,094 - Train: 2.16% [87800/4070000] [21.6/1000.0] [batch_t 1.420 (1.430)] [data_t 0.003] [optim_t 1.417] [lr 0.005000] +2024-04-02 23:54:58,188 - Train: 2.16% [87900/4070000] [21.6/1000.0] [batch_t 1.391 (1.431)] [data_t 0.003] [optim_t 1.388] [lr 0.005000] +2024-04-02 23:57:21,372 - Train: 2.16% [88000/4070000] [21.6/1000.0] [batch_t 1.419 (1.432)] [data_t 0.002] [optim_t 1.417] [lr 0.005000] +2024-04-02 23:59:44,561 - Train: 2.16% [88100/4070000] [21.6/1000.0] [batch_t 1.434 (1.432)] [data_t 0.002] [optim_t 1.432] [lr 0.005000] +2024-04-03 00:02:07,733 - Train: 2.17% [88200/4070000] [21.7/1000.0] [batch_t 1.412 (1.432)] [data_t 0.002] [optim_t 1.410] [lr 0.005000] +2024-04-03 00:04:31,089 - Train: 2.17% [88300/4070000] [21.7/1000.0] [batch_t 1.433 (1.433)] [data_t 0.003] [optim_t 1.430] [lr 0.005000] +2024-04-03 00:06:54,425 - Train: 2.17% [88400/4070000] [21.7/1000.0] [batch_t 1.431 (1.433)] [data_t 0.002] [optim_t 1.429] [lr 0.005000] +2024-04-03 00:09:17,652 - Train: 2.17% [88500/4070000] [21.7/1000.0] [batch_t 1.422 (1.432)] [data_t 0.003] [optim_t 1.419] [lr 0.005000] +2024-04-03 00:11:40,837 - Train: 2.18% [88600/4070000] [21.8/1000.0] [batch_t 1.423 (1.432)] [data_t 0.002] [optim_t 1.421] [lr 0.005000] +2024-04-03 00:14:04,096 - Train: 2.18% [88700/4070000] [21.8/1000.0] [batch_t 1.446 (1.432)] [data_t 0.003] [optim_t 1.443] [lr 0.005000] +2024-04-03 00:16:27,363 - Train: 2.18% [88800/4070000] [21.8/1000.0] [batch_t 1.441 (1.433)] [data_t 0.002] [optim_t 1.439] [lr 0.005000] +2024-04-03 00:18:50,687 - Train: 2.18% [88900/4070000] [21.8/1000.0] [batch_t 1.436 (1.433)] [data_t 0.002] [optim_t 1.433] [lr 0.005000] +2024-04-03 00:21:14,071 - Train: 2.19% [89000/4070000] [21.9/1000.0] [batch_t 1.418 (1.434)] [data_t 0.003] [optim_t 1.415] [lr 0.005000] +2024-04-03 00:23:37,398 - Train: 2.19% [89100/4070000] [21.9/1000.0] [batch_t 1.439 (1.433)] [data_t 0.003] [optim_t 1.436] [lr 0.005000] +2024-04-03 00:26:00,746 - Train: 2.19% [89200/4070000] [21.9/1000.0] [batch_t 1.436 (1.433)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-03 00:28:24,080 - Train: 2.19% [89300/4070000] [21.9/1000.0] [batch_t 1.431 (1.433)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-03 00:30:47,408 - Train: 2.20% [89400/4070000] [22.0/1000.0] [batch_t 1.423 (1.433)] [data_t 0.003] [optim_t 1.420] [lr 0.005000] +2024-04-03 00:33:10,621 - Train: 2.20% [89500/4070000] [22.0/1000.0] [batch_t 1.433 (1.432)] [data_t 0.002] [optim_t 1.431] [lr 0.005000] +2024-04-03 00:34:07,883 - ==> Total time: 23:33:38 Eta: 43 days, 15:22:58 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-03 00:35:36,252 - Train: 2.20% [89600/4070000] [22.0/1000.0] [batch_t 1.436 (1.446)] [data_t 0.002] [optim_t 1.434] [lr 0.005000] +2024-04-03 00:37:59,512 - Train: 2.20% [89700/4070000] [22.0/1000.0] [batch_t 1.435 (1.433)] [data_t 0.004] [optim_t 1.431] [lr 0.005000] +2024-04-03 00:40:22,779 - Train: 2.21% [89800/4070000] [22.1/1000.0] [batch_t 1.435 (1.433)] [data_t 0.002] [optim_t 1.433] [lr 0.005000] +2024-04-03 00:42:45,851 - Train: 2.21% [89900/4070000] [22.1/1000.0] [batch_t 1.434 (1.431)] [data_t 0.003] [optim_t 1.431] [lr 0.005000] +2024-04-03 00:45:09,101 - Train: 2.21% [90000/4070000] [22.1/1000.0] [batch_t 1.422 (1.432)] [data_t 0.003] [optim_t 1.419] [lr 0.005000] +2024-04-03 00:47:32,256 - Train: 2.21% [90100/4070000] [22.1/1000.0] [batch_t 1.430 (1.431)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-03 00:49:55,437 - Train: 2.22% [90200/4070000] [22.2/1000.0] [batch_t 1.431 (1.432)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-03 00:52:18,549 - Train: 2.22% [90300/4070000] [22.2/1000.0] [batch_t 1.425 (1.431)] [data_t 0.002] [optim_t 1.423] [lr 0.005000] +2024-04-03 00:54:41,642 - Train: 2.22% [90400/4070000] [22.2/1000.0] [batch_t 1.417 (1.431)] [data_t 0.003] [optim_t 1.414] [lr 0.005000] +2024-04-03 00:57:04,672 - Train: 2.22% [90500/4070000] [22.2/1000.0] [batch_t 1.451 (1.430)] [data_t 0.003] [optim_t 1.448] [lr 0.005000] +2024-04-03 00:59:28,023 - Train: 2.23% [90600/4070000] [22.3/1000.0] [batch_t 1.432 (1.433)] [data_t 0.003] [optim_t 1.429] [lr 0.005000] +2024-04-03 01:01:51,286 - Train: 2.23% [90700/4070000] [22.3/1000.0] [batch_t 1.436 (1.433)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-03 01:04:14,479 - Train: 2.23% [90800/4070000] [22.3/1000.0] [batch_t 1.426 (1.432)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-03 01:06:37,559 - Train: 2.23% [90900/4070000] [22.3/1000.0] [batch_t 1.428 (1.431)] [data_t 0.003] [optim_t 1.425] [lr 0.005000] +2024-04-03 01:09:00,792 - Train: 2.24% [91000/4070000] [22.4/1000.0] [batch_t 1.423 (1.432)] [data_t 0.003] [optim_t 1.420] [lr 0.005000] +2024-04-03 01:11:23,989 - Train: 2.24% [91100/4070000] [22.4/1000.0] [batch_t 1.437 (1.432)] [data_t 0.003] [optim_t 1.434] [lr 0.005000] +2024-04-03 01:13:47,319 - Train: 2.24% [91200/4070000] [22.4/1000.0] [batch_t 1.441 (1.433)] [data_t 0.003] [optim_t 1.438] [lr 0.005000] +2024-04-03 01:16:10,630 - Train: 2.24% [91300/4070000] [22.4/1000.0] [batch_t 1.427 (1.433)] [data_t 0.003] [optim_t 1.425] [lr 0.005000] +2024-04-03 01:18:33,750 - Train: 2.25% [91400/4070000] [22.5/1000.0] [batch_t 1.436 (1.431)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-03 01:20:56,967 - Train: 2.25% [91500/4070000] [22.5/1000.0] [batch_t 1.442 (1.432)] [data_t 0.002] [optim_t 1.439] [lr 0.005000] +2024-04-03 01:23:20,381 - Train: 2.25% [91600/4070000] [22.5/1000.0] [batch_t 1.433 (1.434)] [data_t 0.002] [optim_t 1.430] [lr 0.005000] +2024-04-03 01:25:43,658 - Train: 2.25% [91700/4070000] [22.5/1000.0] [batch_t 1.442 (1.433)] [data_t 0.002] [optim_t 1.439] [lr 0.005000] +2024-04-03 01:28:06,777 - Train: 2.26% [91800/4070000] [22.6/1000.0] [batch_t 1.376 (1.431)] [data_t 0.002] [optim_t 1.373] [lr 0.005000] +2024-04-03 01:30:30,116 - Train: 2.26% [91900/4070000] [22.6/1000.0] [batch_t 1.433 (1.433)] [data_t 0.002] [optim_t 1.430] [lr 0.005000] +2024-04-03 01:32:53,379 - Train: 2.26% [92000/4070000] [22.6/1000.0] [batch_t 1.441 (1.433)] [data_t 0.003] [optim_t 1.438] [lr 0.005000] +2024-04-03 01:35:16,694 - Train: 2.26% [92100/4070000] [22.6/1000.0] [batch_t 1.444 (1.433)] [data_t 0.002] [optim_t 1.442] [lr 0.005000] +2024-04-03 01:37:39,980 - Train: 2.27% [92200/4070000] [22.7/1000.0] [batch_t 1.446 (1.433)] [data_t 0.003] [optim_t 1.444] [lr 0.005000] +2024-04-03 01:40:03,195 - Train: 2.27% [92300/4070000] [22.7/1000.0] [batch_t 1.427 (1.432)] [data_t 0.002] [optim_t 1.425] [lr 0.005000] +2024-04-03 01:42:26,653 - Train: 2.27% [92400/4070000] [22.7/1000.0] [batch_t 1.444 (1.434)] [data_t 0.003] [optim_t 1.441] [lr 0.005000] +2024-04-03 01:44:49,877 - Train: 2.27% [92500/4070000] [22.7/1000.0] [batch_t 1.427 (1.432)] [data_t 0.002] [optim_t 1.425] [lr 0.005000] +2024-04-03 01:47:13,251 - Train: 2.28% [92600/4070000] [22.8/1000.0] [batch_t 1.431 (1.434)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-03 01:49:36,474 - Train: 2.28% [92700/4070000] [22.8/1000.0] [batch_t 1.446 (1.432)] [data_t 0.003] [optim_t 1.444] [lr 0.005000] +2024-04-03 01:51:59,689 - Train: 2.28% [92800/4070000] [22.8/1000.0] [batch_t 1.436 (1.432)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-03 01:54:22,945 - Train: 2.28% [92900/4070000] [22.8/1000.0] [batch_t 1.433 (1.432)] [data_t 0.002] [optim_t 1.430] [lr 0.005000] +2024-04-03 01:56:46,297 - Train: 2.29% [93000/4070000] [22.9/1000.0] [batch_t 1.440 (1.433)] [data_t 0.002] [optim_t 1.437] [lr 0.005000] +2024-04-03 01:59:09,442 - Train: 2.29% [93100/4070000] [22.9/1000.0] [batch_t 1.432 (1.431)] [data_t 0.003] [optim_t 1.429] [lr 0.005000] +2024-04-03 02:01:32,641 - Train: 2.29% [93200/4070000] [22.9/1000.0] [batch_t 1.424 (1.432)] [data_t 0.003] [optim_t 1.421] [lr 0.005000] +2024-04-03 02:03:55,913 - Train: 2.29% [93300/4070000] [22.9/1000.0] [batch_t 1.423 (1.433)] [data_t 0.002] [optim_t 1.421] [lr 0.005000] +2024-04-03 02:06:18,987 - Train: 2.29% [93400/4070000] [22.9/1000.0] [batch_t 1.432 (1.431)] [data_t 0.002] [optim_t 1.430] [lr 0.005000] +2024-04-03 02:08:42,228 - Train: 2.30% [93500/4070000] [23.0/1000.0] [batch_t 1.423 (1.432)] [data_t 0.002] [optim_t 1.421] [lr 0.005000] +2024-04-03 02:11:05,565 - Train: 2.30% [93600/4070000] [23.0/1000.0] [batch_t 1.418 (1.433)] [data_t 0.002] [optim_t 1.416] [lr 0.005000] +2024-04-03 02:11:19,895 - ==> Total time: 1 day, 1:10:50 Eta: 44 days, 13:38:06 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-03 02:13:30,761 - Train: 2.30% [93700/4070000] [23.0/1000.0] [batch_t 1.450 (1.436)] [data_t 0.003] [optim_t 1.447] [lr 0.005000] +2024-04-03 02:15:53,831 - Train: 2.30% [93800/4070000] [23.0/1000.0] [batch_t 1.432 (1.431)] [data_t 0.003] [optim_t 1.429] [lr 0.005000] +2024-04-03 02:18:17,061 - Train: 2.31% [93900/4070000] [23.1/1000.0] [batch_t 1.421 (1.432)] [data_t 0.002] [optim_t 1.419] [lr 0.005000] +2024-04-03 02:20:40,434 - Train: 2.31% [94000/4070000] [23.1/1000.0] [batch_t 1.437 (1.434)] [data_t 0.002] [optim_t 1.435] [lr 0.005000] +2024-04-03 02:23:03,735 - Train: 2.31% [94100/4070000] [23.1/1000.0] [batch_t 1.445 (1.433)] [data_t 0.003] [optim_t 1.442] [lr 0.005000] +2024-04-03 02:25:26,927 - Train: 2.31% [94200/4070000] [23.1/1000.0] [batch_t 1.435 (1.432)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-03 02:27:50,197 - Train: 2.32% [94300/4070000] [23.2/1000.0] [batch_t 1.433 (1.433)] [data_t 0.002] [optim_t 1.431] [lr 0.005000] +2024-04-03 02:30:13,557 - Train: 2.32% [94400/4070000] [23.2/1000.0] [batch_t 1.436 (1.434)] [data_t 0.003] [optim_t 1.434] [lr 0.005000] +2024-04-03 02:32:36,771 - Train: 2.32% [94500/4070000] [23.2/1000.0] [batch_t 1.431 (1.432)] [data_t 0.002] [optim_t 1.429] [lr 0.005000] +2024-04-03 02:34:59,972 - Train: 2.32% [94600/4070000] [23.2/1000.0] [batch_t 1.436 (1.432)] [data_t 0.002] [optim_t 1.433] [lr 0.005000] +2024-04-03 02:37:23,291 - Train: 2.33% [94700/4070000] [23.3/1000.0] [batch_t 1.427 (1.433)] [data_t 0.003] [optim_t 1.424] [lr 0.005000] +2024-04-03 02:39:46,559 - Train: 2.33% [94800/4070000] [23.3/1000.0] [batch_t 1.435 (1.433)] [data_t 0.003] [optim_t 1.432] [lr 0.005000] +2024-04-03 02:42:09,562 - Train: 2.33% [94900/4070000] [23.3/1000.0] [batch_t 1.442 (1.430)] [data_t 0.003] [optim_t 1.439] [lr 0.005000] +2024-04-03 02:44:32,820 - Train: 2.33% [95000/4070000] [23.3/1000.0] [batch_t 1.422 (1.432)] [data_t 0.003] [optim_t 1.420] [lr 0.005000] +2024-04-03 02:46:55,964 - Train: 2.34% [95100/4070000] [23.4/1000.0] [batch_t 1.438 (1.431)] [data_t 0.003] [optim_t 1.436] [lr 0.005000] +2024-04-03 02:49:19,296 - Train: 2.34% [95200/4070000] [23.4/1000.0] [batch_t 1.435 (1.433)] [data_t 0.002] [optim_t 1.433] [lr 0.005000] +2024-04-03 02:51:42,531 - Train: 2.34% [95300/4070000] [23.4/1000.0] [batch_t 1.422 (1.432)] [data_t 0.003] [optim_t 1.420] [lr 0.005000] +2024-04-03 02:54:05,680 - Train: 2.34% [95400/4070000] [23.4/1000.0] [batch_t 1.428 (1.431)] [data_t 0.003] [optim_t 1.425] [lr 0.005000] +2024-04-03 02:56:28,983 - Train: 2.35% [95500/4070000] [23.5/1000.0] [batch_t 1.432 (1.433)] [data_t 0.003] [optim_t 1.429] [lr 0.005000] +2024-04-03 02:58:52,252 - Train: 2.35% [95600/4070000] [23.5/1000.0] [batch_t 1.437 (1.433)] [data_t 0.002] [optim_t 1.435] [lr 0.005000] +2024-04-03 03:01:15,369 - Train: 2.35% [95700/4070000] [23.5/1000.0] [batch_t 1.431 (1.431)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-03 03:03:38,627 - Train: 2.35% [95800/4070000] [23.5/1000.0] [batch_t 1.433 (1.432)] [data_t 0.003] [optim_t 1.430] [lr 0.005000] +2024-04-03 03:06:02,019 - Train: 2.36% [95900/4070000] [23.6/1000.0] [batch_t 1.433 (1.434)] [data_t 0.002] [optim_t 1.431] [lr 0.005000] +2024-04-03 03:08:25,307 - Train: 2.36% [96000/4070000] [23.6/1000.0] [batch_t 1.434 (1.433)] [data_t 0.003] [optim_t 1.432] [lr 0.005000] +2024-04-03 03:10:48,416 - Train: 2.36% [96100/4070000] [23.6/1000.0] [batch_t 1.436 (1.431)] [data_t 0.002] [optim_t 1.434] [lr 0.005000] +2024-04-03 03:13:11,591 - Train: 2.36% [96200/4070000] [23.6/1000.0] [batch_t 1.434 (1.432)] [data_t 0.003] [optim_t 1.431] [lr 0.005000] +2024-04-03 03:15:34,944 - Train: 2.37% [96300/4070000] [23.7/1000.0] [batch_t 1.437 (1.433)] [data_t 0.003] [optim_t 1.434] [lr 0.005000] +2024-04-03 03:17:58,162 - Train: 2.37% [96400/4070000] [23.7/1000.0] [batch_t 1.440 (1.432)] [data_t 0.003] [optim_t 1.438] [lr 0.005000] +2024-04-03 03:20:21,329 - Train: 2.37% [96500/4070000] [23.7/1000.0] [batch_t 1.436 (1.432)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-03 03:22:44,641 - Train: 2.37% [96600/4070000] [23.7/1000.0] [batch_t 1.426 (1.433)] [data_t 0.003] [optim_t 1.424] [lr 0.005000] +2024-04-03 03:25:07,915 - Train: 2.38% [96700/4070000] [23.8/1000.0] [batch_t 1.438 (1.433)] [data_t 0.002] [optim_t 1.435] [lr 0.005000] +2024-04-03 03:27:31,242 - Train: 2.38% [96800/4070000] [23.8/1000.0] [batch_t 1.430 (1.433)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-03 03:29:54,334 - Train: 2.38% [96900/4070000] [23.8/1000.0] [batch_t 1.432 (1.431)] [data_t 0.003] [optim_t 1.429] [lr 0.005000] +2024-04-03 03:32:17,623 - Train: 2.38% [97000/4070000] [23.8/1000.0] [batch_t 1.428 (1.433)] [data_t 0.003] [optim_t 1.425] [lr 0.005000] +2024-04-03 03:34:40,839 - Train: 2.39% [97100/4070000] [23.9/1000.0] [batch_t 1.438 (1.432)] [data_t 0.002] [optim_t 1.436] [lr 0.005000] +2024-04-03 03:37:03,973 - Train: 2.39% [97200/4070000] [23.9/1000.0] [batch_t 1.424 (1.431)] [data_t 0.002] [optim_t 1.422] [lr 0.005000] +2024-04-03 03:39:26,967 - Train: 2.39% [97300/4070000] [23.9/1000.0] [batch_t 1.417 (1.430)] [data_t 0.003] [optim_t 1.415] [lr 0.005000] +2024-04-03 03:41:50,261 - Train: 2.39% [97400/4070000] [23.9/1000.0] [batch_t 1.435 (1.433)] [data_t 0.002] [optim_t 1.433] [lr 0.005000] +2024-04-03 03:44:13,498 - Train: 2.40% [97500/4070000] [24.0/1000.0] [batch_t 1.442 (1.432)] [data_t 0.003] [optim_t 1.439] [lr 0.005000] +2024-04-03 03:46:36,762 - Train: 2.40% [97600/4070000] [24.0/1000.0] [batch_t 1.429 (1.433)] [data_t 0.003] [optim_t 1.426] [lr 0.005000] +2024-04-03 03:48:31,274 - ==> Total time: 1 day, 2:48:02 Eta: 45 days, 9:53:26 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-03 03:49:02,256 - Train: 2.40% [97700/4070000] [24.0/1000.0] [batch_t 1.435 (1.461)] [data_t 0.002] [optim_t 1.432] [lr 0.005000] +2024-04-03 03:51:25,586 - Train: 2.40% [97800/4070000] [24.0/1000.0] [batch_t 1.424 (1.433)] [data_t 0.003] [optim_t 1.422] [lr 0.005000] +2024-04-03 03:53:48,831 - Train: 2.41% [97900/4070000] [24.1/1000.0] [batch_t 1.431 (1.432)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-03 03:56:12,168 - Train: 2.41% [98000/4070000] [24.1/1000.0] [batch_t 1.418 (1.433)] [data_t 0.003] [optim_t 1.415] [lr 0.005000] +2024-04-03 03:58:35,527 - Train: 2.41% [98100/4070000] [24.1/1000.0] [batch_t 1.435 (1.434)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-03 04:00:58,935 - Train: 2.41% [98200/4070000] [24.1/1000.0] [batch_t 1.444 (1.434)] [data_t 0.002] [optim_t 1.441] [lr 0.005000] +2024-04-03 04:03:22,291 - Train: 2.42% [98300/4070000] [24.2/1000.0] [batch_t 1.442 (1.433)] [data_t 0.003] [optim_t 1.439] [lr 0.005000] +2024-04-03 04:05:45,265 - Train: 2.42% [98400/4070000] [24.2/1000.0] [batch_t 1.433 (1.430)] [data_t 0.002] [optim_t 1.431] [lr 0.005000] +2024-04-03 04:08:08,401 - Train: 2.42% [98500/4070000] [24.2/1000.0] [batch_t 1.424 (1.431)] [data_t 0.003] [optim_t 1.421] [lr 0.005000] +2024-04-03 04:10:31,620 - Train: 2.42% [98600/4070000] [24.2/1000.0] [batch_t 1.443 (1.432)] [data_t 0.002] [optim_t 1.440] [lr 0.005000] +2024-04-03 04:12:54,869 - Train: 2.43% [98700/4070000] [24.3/1000.0] [batch_t 1.438 (1.432)] [data_t 0.002] [optim_t 1.436] [lr 0.005000] +2024-04-03 04:15:18,166 - Train: 2.43% [98800/4070000] [24.3/1000.0] [batch_t 1.427 (1.433)] [data_t 0.003] [optim_t 1.424] [lr 0.005000] +2024-04-03 04:17:41,452 - Train: 2.43% [98900/4070000] [24.3/1000.0] [batch_t 1.433 (1.433)] [data_t 0.003] [optim_t 1.430] [lr 0.005000] +2024-04-03 04:20:04,649 - Train: 2.43% [99000/4070000] [24.3/1000.0] [batch_t 1.436 (1.432)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-03 04:22:27,902 - Train: 2.43% [99100/4070000] [24.3/1000.0] [batch_t 1.429 (1.432)] [data_t 0.003] [optim_t 1.426] [lr 0.005000] +2024-04-03 04:24:51,147 - Train: 2.44% [99200/4070000] [24.4/1000.0] [batch_t 1.425 (1.432)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-03 04:27:14,518 - Train: 2.44% [99300/4070000] [24.4/1000.0] [batch_t 1.432 (1.434)] [data_t 0.002] [optim_t 1.430] [lr 0.005000] +2024-04-03 04:29:37,964 - Train: 2.44% [99400/4070000] [24.4/1000.0] [batch_t 1.427 (1.434)] [data_t 0.003] [optim_t 1.424] [lr 0.005000] +2024-04-03 04:32:01,349 - Train: 2.44% [99500/4070000] [24.4/1000.0] [batch_t 1.423 (1.434)] [data_t 0.002] [optim_t 1.420] [lr 0.005000] +2024-04-03 04:34:24,559 - Train: 2.45% [99600/4070000] [24.5/1000.0] [batch_t 1.436 (1.432)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-03 04:36:47,642 - Train: 2.45% [99700/4070000] [24.5/1000.0] [batch_t 1.431 (1.431)] [data_t 0.002] [optim_t 1.428] [lr 0.005000] +2024-04-03 04:39:10,909 - Train: 2.45% [99800/4070000] [24.5/1000.0] [batch_t 1.428 (1.433)] [data_t 0.003] [optim_t 1.426] [lr 0.005000] +2024-04-03 04:41:34,299 - Train: 2.45% [99900/4070000] [24.5/1000.0] [batch_t 1.434 (1.434)] [data_t 0.003] [optim_t 1.431] [lr 0.005000] +2024-04-03 04:43:57,516 - Train: 2.46% [100000/4070000] [24.6/1000.0] [batch_t 1.432 (1.432)] [data_t 0.003] [optim_t 1.429] [lr 0.005000] +2024-04-03 04:46:20,792 - Train: 2.46% [100100/4070000] [24.6/1000.0] [batch_t 1.430 (1.433)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-03 04:48:44,118 - Train: 2.46% [100200/4070000] [24.6/1000.0] [batch_t 1.442 (1.433)] [data_t 0.002] [optim_t 1.440] [lr 0.005000] +2024-04-03 04:51:07,477 - Train: 2.46% [100300/4070000] [24.6/1000.0] [batch_t 1.437 (1.433)] [data_t 0.003] [optim_t 1.434] [lr 0.005000] +2024-04-03 04:53:30,499 - Train: 2.47% [100400/4070000] [24.7/1000.0] [batch_t 1.432 (1.430)] [data_t 0.003] [optim_t 1.429] [lr 0.005000] +2024-04-03 04:55:53,804 - Train: 2.47% [100500/4070000] [24.7/1000.0] [batch_t 1.430 (1.433)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-03 04:58:17,019 - Train: 2.47% [100600/4070000] [24.7/1000.0] [batch_t 1.426 (1.432)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-03 05:00:40,112 - Train: 2.47% [100700/4070000] [24.7/1000.0] [batch_t 1.435 (1.431)] [data_t 0.002] [optim_t 1.433] [lr 0.005000] +2024-04-03 05:03:03,537 - Train: 2.48% [100800/4070000] [24.8/1000.0] [batch_t 1.433 (1.434)] [data_t 0.002] [optim_t 1.430] [lr 0.005000] +2024-04-03 05:05:26,761 - Train: 2.48% [100900/4070000] [24.8/1000.0] [batch_t 1.391 (1.432)] [data_t 0.003] [optim_t 1.388] [lr 0.005000] +2024-04-03 05:07:50,114 - Train: 2.48% [101000/4070000] [24.8/1000.0] [batch_t 1.409 (1.433)] [data_t 0.002] [optim_t 1.407] [lr 0.005000] +2024-04-03 05:10:13,331 - Train: 2.48% [101100/4070000] [24.8/1000.0] [batch_t 1.432 (1.432)] [data_t 0.003] [optim_t 1.429] [lr 0.005000] +2024-04-03 05:12:36,557 - Train: 2.49% [101200/4070000] [24.9/1000.0] [batch_t 1.437 (1.432)] [data_t 0.003] [optim_t 1.434] [lr 0.005000] +2024-04-03 05:14:59,851 - Train: 2.49% [101300/4070000] [24.9/1000.0] [batch_t 1.374 (1.433)] [data_t 0.003] [optim_t 1.372] [lr 0.005000] +2024-04-03 05:17:23,227 - Train: 2.49% [101400/4070000] [24.9/1000.0] [batch_t 1.413 (1.434)] [data_t 0.003] [optim_t 1.411] [lr 0.005000] +2024-04-03 05:19:46,501 - Train: 2.49% [101500/4070000] [24.9/1000.0] [batch_t 1.453 (1.433)] [data_t 0.003] [optim_t 1.450] [lr 0.005000] +2024-04-03 05:22:09,583 - Train: 2.50% [101600/4070000] [25.0/1000.0] [batch_t 1.433 (1.431)] [data_t 0.002] [optim_t 1.431] [lr 0.005000] +2024-04-03 05:24:33,027 - Train: 2.50% [101700/4070000] [25.0/1000.0] [batch_t 1.443 (1.434)] [data_t 0.003] [optim_t 1.440] [lr 0.005000] +2024-04-03 05:25:44,557 - ==> Total time: 1 day, 4:25:15 Eta: 46 days, 4:25:01 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-03 05:26:59,272 - Train: 2.50% [101800/4070000] [25.0/1000.0] [batch_t 1.432 (1.441)] [data_t 0.002] [optim_t 1.430] [lr 0.005000] +2024-04-03 05:29:22,637 - Train: 2.50% [101900/4070000] [25.0/1000.0] [batch_t 1.437 (1.434)] [data_t 0.003] [optim_t 1.435] [lr 0.005000] +2024-04-03 05:31:45,799 - Train: 2.51% [102000/4070000] [25.1/1000.0] [batch_t 1.443 (1.432)] [data_t 0.003] [optim_t 1.440] [lr 0.005000] +2024-04-03 05:34:08,964 - Train: 2.51% [102100/4070000] [25.1/1000.0] [batch_t 1.441 (1.432)] [data_t 0.002] [optim_t 1.439] [lr 0.005000] +2024-04-03 05:36:32,212 - Train: 2.51% [102200/4070000] [25.1/1000.0] [batch_t 1.450 (1.432)] [data_t 0.003] [optim_t 1.447] [lr 0.005000] +2024-04-03 05:38:55,290 - Train: 2.51% [102300/4070000] [25.1/1000.0] [batch_t 1.378 (1.431)] [data_t 0.003] [optim_t 1.375] [lr 0.005000] +2024-04-03 05:41:18,640 - Train: 2.52% [102400/4070000] [25.2/1000.0] [batch_t 1.425 (1.433)] [data_t 0.003] [optim_t 1.422] [lr 0.005000] +2024-04-03 05:43:41,884 - Train: 2.52% [102500/4070000] [25.2/1000.0] [batch_t 1.433 (1.432)] [data_t 0.003] [optim_t 1.430] [lr 0.005000] +2024-04-03 05:46:04,985 - Train: 2.52% [102600/4070000] [25.2/1000.0] [batch_t 1.436 (1.431)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-03 05:48:28,363 - Train: 2.52% [102700/4070000] [25.2/1000.0] [batch_t 1.432 (1.434)] [data_t 0.003] [optim_t 1.429] [lr 0.005000] +2024-04-03 05:50:51,784 - Train: 2.53% [102800/4070000] [25.3/1000.0] [batch_t 1.429 (1.434)] [data_t 0.002] [optim_t 1.427] [lr 0.005000] +2024-04-03 05:53:14,845 - Train: 2.53% [102900/4070000] [25.3/1000.0] [batch_t 1.428 (1.431)] [data_t 0.003] [optim_t 1.425] [lr 0.005000] +2024-04-03 05:55:38,134 - Train: 2.53% [103000/4070000] [25.3/1000.0] [batch_t 1.434 (1.433)] [data_t 0.002] [optim_t 1.432] [lr 0.005000] +2024-04-03 05:58:01,476 - Train: 2.53% [103100/4070000] [25.3/1000.0] [batch_t 1.432 (1.433)] [data_t 0.003] [optim_t 1.430] [lr 0.005000] +2024-04-03 06:00:24,551 - Train: 2.54% [103200/4070000] [25.4/1000.0] [batch_t 1.430 (1.431)] [data_t 0.002] [optim_t 1.427] [lr 0.005000] +2024-04-03 06:02:47,751 - Train: 2.54% [103300/4070000] [25.4/1000.0] [batch_t 1.431 (1.432)] [data_t 0.002] [optim_t 1.429] [lr 0.005000] +2024-04-03 06:05:11,100 - Train: 2.54% [103400/4070000] [25.4/1000.0] [batch_t 1.436 (1.433)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-03 06:07:34,520 - Train: 2.54% [103500/4070000] [25.4/1000.0] [batch_t 1.442 (1.434)] [data_t 0.002] [optim_t 1.440] [lr 0.005000] +2024-04-03 06:09:57,616 - Train: 2.55% [103600/4070000] [25.5/1000.0] [batch_t 1.426 (1.431)] [data_t 0.002] [optim_t 1.424] [lr 0.005000] +2024-04-03 06:12:20,911 - Train: 2.55% [103700/4070000] [25.5/1000.0] [batch_t 1.442 (1.433)] [data_t 0.002] [optim_t 1.439] [lr 0.005000] +2024-04-03 06:14:44,231 - Train: 2.55% [103800/4070000] [25.5/1000.0] [batch_t 1.431 (1.433)] [data_t 0.002] [optim_t 1.429] [lr 0.005000] +2024-04-03 06:17:07,421 - Train: 2.55% [103900/4070000] [25.5/1000.0] [batch_t 1.431 (1.432)] [data_t 0.003] [optim_t 1.429] [lr 0.005000] +2024-04-03 06:19:30,676 - Train: 2.56% [104000/4070000] [25.6/1000.0] [batch_t 1.421 (1.432)] [data_t 0.003] [optim_t 1.418] [lr 0.005000] +2024-04-03 06:21:54,011 - Train: 2.56% [104100/4070000] [25.6/1000.0] [batch_t 1.432 (1.433)] [data_t 0.003] [optim_t 1.429] [lr 0.005000] +2024-04-03 06:24:17,247 - Train: 2.56% [104200/4070000] [25.6/1000.0] [batch_t 1.418 (1.432)] [data_t 0.003] [optim_t 1.415] [lr 0.005000] +2024-04-03 06:26:40,504 - Train: 2.56% [104300/4070000] [25.6/1000.0] [batch_t 1.442 (1.432)] [data_t 0.003] [optim_t 1.439] [lr 0.005000] +2024-04-03 06:29:03,778 - Train: 2.57% [104400/4070000] [25.7/1000.0] [batch_t 1.422 (1.433)] [data_t 0.003] [optim_t 1.419] [lr 0.005000] +2024-04-03 06:31:26,927 - Train: 2.57% [104500/4070000] [25.7/1000.0] [batch_t 1.427 (1.431)] [data_t 0.003] [optim_t 1.425] [lr 0.005000] +2024-04-03 06:33:50,231 - Train: 2.57% [104600/4070000] [25.7/1000.0] [batch_t 1.428 (1.433)] [data_t 0.003] [optim_t 1.425] [lr 0.005000] +2024-04-03 06:36:13,446 - Train: 2.57% [104700/4070000] [25.7/1000.0] [batch_t 1.435 (1.432)] [data_t 0.003] [optim_t 1.432] [lr 0.005000] +2024-04-03 06:38:36,767 - Train: 2.57% [104800/4070000] [25.7/1000.0] [batch_t 1.435 (1.433)] [data_t 0.003] [optim_t 1.432] [lr 0.005000] +2024-04-03 06:41:00,082 - Train: 2.58% [104900/4070000] [25.8/1000.0] [batch_t 1.406 (1.433)] [data_t 0.003] [optim_t 1.404] [lr 0.005000] +2024-04-03 06:43:23,347 - Train: 2.58% [105000/4070000] [25.8/1000.0] [batch_t 1.430 (1.433)] [data_t 0.002] [optim_t 1.427] [lr 0.005000] +2024-04-03 06:45:46,552 - Train: 2.58% [105100/4070000] [25.8/1000.0] [batch_t 1.428 (1.432)] [data_t 0.003] [optim_t 1.425] [lr 0.005000] +2024-04-03 06:48:09,906 - Train: 2.58% [105200/4070000] [25.8/1000.0] [batch_t 1.433 (1.433)] [data_t 0.003] [optim_t 1.430] [lr 0.005000] +2024-04-03 06:50:32,861 - Train: 2.59% [105300/4070000] [25.9/1000.0] [batch_t 1.442 (1.429)] [data_t 0.002] [optim_t 1.440] [lr 0.005000] +2024-04-03 06:52:56,130 - Train: 2.59% [105400/4070000] [25.9/1000.0] [batch_t 1.430 (1.433)] [data_t 0.002] [optim_t 1.428] [lr 0.005000] +2024-04-03 06:55:19,411 - Train: 2.59% [105500/4070000] [25.9/1000.0] [batch_t 1.434 (1.433)] [data_t 0.003] [optim_t 1.431] [lr 0.005000] +2024-04-03 06:57:42,737 - Train: 2.59% [105600/4070000] [25.9/1000.0] [batch_t 1.436 (1.433)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-03 07:00:06,063 - Train: 2.60% [105700/4070000] [26.0/1000.0] [batch_t 1.445 (1.433)] [data_t 0.003] [optim_t 1.443] [lr 0.005000] +2024-04-03 07:02:29,346 - Train: 2.60% [105800/4070000] [26.0/1000.0] [batch_t 1.444 (1.433)] [data_t 0.003] [optim_t 1.441] [lr 0.005000] +2024-04-03 07:02:58,016 - ==> Total time: 1 day, 6:02:28 Eta: 46 days, 21:23:43 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-03 07:04:54,555 - Train: 2.60% [105900/4070000] [26.0/1000.0] [batch_t 1.413 (1.436)] [data_t 0.003] [optim_t 1.410] [lr 0.005000] +2024-04-03 07:07:18,004 - Train: 2.60% [106000/4070000] [26.0/1000.0] [batch_t 1.437 (1.434)] [data_t 0.003] [optim_t 1.434] [lr 0.005000] +2024-04-03 07:09:41,359 - Train: 2.61% [106100/4070000] [26.1/1000.0] [batch_t 1.430 (1.433)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-03 07:12:04,448 - Train: 2.61% [106200/4070000] [26.1/1000.0] [batch_t 1.418 (1.431)] [data_t 0.002] [optim_t 1.416] [lr 0.005000] +2024-04-03 07:14:27,902 - Train: 2.61% [106300/4070000] [26.1/1000.0] [batch_t 1.441 (1.434)] [data_t 0.002] [optim_t 1.439] [lr 0.005000] +2024-04-03 07:16:51,007 - Train: 2.61% [106400/4070000] [26.1/1000.0] [batch_t 1.435 (1.431)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-03 07:19:14,112 - Train: 2.62% [106500/4070000] [26.2/1000.0] [batch_t 1.436 (1.431)] [data_t 0.002] [optim_t 1.434] [lr 0.005000] +2024-04-03 07:21:37,496 - Train: 2.62% [106600/4070000] [26.2/1000.0] [batch_t 1.421 (1.434)] [data_t 0.003] [optim_t 1.419] [lr 0.005000] +2024-04-03 07:24:00,606 - Train: 2.62% [106700/4070000] [26.2/1000.0] [batch_t 1.431 (1.431)] [data_t 0.003] [optim_t 1.429] [lr 0.005000] +2024-04-03 07:26:23,853 - Train: 2.62% [106800/4070000] [26.2/1000.0] [batch_t 1.437 (1.432)] [data_t 0.002] [optim_t 1.435] [lr 0.005000] +2024-04-03 07:28:46,980 - Train: 2.63% [106900/4070000] [26.3/1000.0] [batch_t 1.427 (1.431)] [data_t 0.003] [optim_t 1.425] [lr 0.005000] +2024-04-03 07:31:10,252 - Train: 2.63% [107000/4070000] [26.3/1000.0] [batch_t 1.428 (1.433)] [data_t 0.002] [optim_t 1.426] [lr 0.005000] +2024-04-03 07:33:33,672 - Train: 2.63% [107100/4070000] [26.3/1000.0] [batch_t 1.434 (1.434)] [data_t 0.002] [optim_t 1.432] [lr 0.005000] +2024-04-03 07:35:56,930 - Train: 2.63% [107200/4070000] [26.3/1000.0] [batch_t 1.444 (1.432)] [data_t 0.003] [optim_t 1.441] [lr 0.005000] +2024-04-03 07:38:20,191 - Train: 2.64% [107300/4070000] [26.4/1000.0] [batch_t 1.433 (1.433)] [data_t 0.003] [optim_t 1.430] [lr 0.005000] +2024-04-03 07:40:43,593 - Train: 2.64% [107400/4070000] [26.4/1000.0] [batch_t 1.436 (1.434)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-03 07:43:06,815 - Train: 2.64% [107500/4070000] [26.4/1000.0] [batch_t 1.435 (1.432)] [data_t 0.002] [optim_t 1.433] [lr 0.005000] +2024-04-03 07:45:30,060 - Train: 2.64% [107600/4070000] [26.4/1000.0] [batch_t 1.428 (1.432)] [data_t 0.002] [optim_t 1.426] [lr 0.005000] +2024-04-03 07:47:53,275 - Train: 2.65% [107700/4070000] [26.5/1000.0] [batch_t 1.437 (1.432)] [data_t 0.003] [optim_t 1.434] [lr 0.005000] +2024-04-03 07:50:16,610 - Train: 2.65% [107800/4070000] [26.5/1000.0] [batch_t 1.444 (1.433)] [data_t 0.002] [optim_t 1.442] [lr 0.005000] +2024-04-03 07:52:39,945 - Train: 2.65% [107900/4070000] [26.5/1000.0] [batch_t 1.422 (1.433)] [data_t 0.003] [optim_t 1.419] [lr 0.005000] +2024-04-03 07:55:03,167 - Train: 2.65% [108000/4070000] [26.5/1000.0] [batch_t 1.440 (1.432)] [data_t 0.002] [optim_t 1.438] [lr 0.005000] +2024-04-03 07:57:26,300 - Train: 2.66% [108100/4070000] [26.6/1000.0] [batch_t 1.441 (1.431)] [data_t 0.002] [optim_t 1.439] [lr 0.005000] +2024-04-03 07:59:49,512 - Train: 2.66% [108200/4070000] [26.6/1000.0] [batch_t 1.446 (1.432)] [data_t 0.002] [optim_t 1.444] [lr 0.005000] +2024-04-03 08:02:12,737 - Train: 2.66% [108300/4070000] [26.6/1000.0] [batch_t 1.431 (1.432)] [data_t 0.003] [optim_t 1.429] [lr 0.005000] +2024-04-03 08:04:35,837 - Train: 2.66% [108400/4070000] [26.6/1000.0] [batch_t 1.430 (1.431)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-03 08:06:58,962 - Train: 2.67% [108500/4070000] [26.7/1000.0] [batch_t 1.429 (1.431)] [data_t 0.002] [optim_t 1.426] [lr 0.005000] +2024-04-03 08:09:22,221 - Train: 2.67% [108600/4070000] [26.7/1000.0] [batch_t 1.450 (1.432)] [data_t 0.002] [optim_t 1.448] [lr 0.005000] +2024-04-03 08:11:45,396 - Train: 2.67% [108700/4070000] [26.7/1000.0] [batch_t 1.437 (1.432)] [data_t 0.003] [optim_t 1.434] [lr 0.005000] +2024-04-03 08:14:08,443 - Train: 2.67% [108800/4070000] [26.7/1000.0] [batch_t 1.422 (1.430)] [data_t 0.003] [optim_t 1.419] [lr 0.005000] +2024-04-03 08:16:31,657 - Train: 2.68% [108900/4070000] [26.8/1000.0] [batch_t 1.426 (1.432)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-03 08:18:54,846 - Train: 2.68% [109000/4070000] [26.8/1000.0] [batch_t 1.424 (1.432)] [data_t 0.003] [optim_t 1.420] [lr 0.005000] +2024-04-03 08:21:18,086 - Train: 2.68% [109100/4070000] [26.8/1000.0] [batch_t 1.417 (1.432)] [data_t 0.003] [optim_t 1.414] [lr 0.005000] +2024-04-03 08:23:41,214 - Train: 2.68% [109200/4070000] [26.8/1000.0] [batch_t 1.441 (1.431)] [data_t 0.002] [optim_t 1.439] [lr 0.005000] +2024-04-03 08:26:04,154 - Train: 2.69% [109300/4070000] [26.9/1000.0] [batch_t 1.432 (1.429)] [data_t 0.003] [optim_t 1.429] [lr 0.005000] +2024-04-03 08:28:27,296 - Train: 2.69% [109400/4070000] [26.9/1000.0] [batch_t 1.419 (1.431)] [data_t 0.003] [optim_t 1.416] [lr 0.005000] +2024-04-03 08:30:50,373 - Train: 2.69% [109500/4070000] [26.9/1000.0] [batch_t 1.418 (1.431)] [data_t 0.002] [optim_t 1.415] [lr 0.005000] +2024-04-03 08:33:13,547 - Train: 2.69% [109600/4070000] [26.9/1000.0] [batch_t 1.421 (1.432)] [data_t 0.002] [optim_t 1.419] [lr 0.005000] +2024-04-03 08:35:36,720 - Train: 2.70% [109700/4070000] [27.0/1000.0] [batch_t 1.444 (1.432)] [data_t 0.002] [optim_t 1.442] [lr 0.005000] +2024-04-03 08:38:00,056 - Train: 2.70% [109800/4070000] [27.0/1000.0] [batch_t 1.431 (1.433)] [data_t 0.002] [optim_t 1.428] [lr 0.005000] +2024-04-03 08:40:08,956 - ==> Total time: 1 day, 7:39:39 Eta: 47 days, 12:58:14 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-03 08:40:24,986 - Train: 2.70% [109900/4070000] [27.0/1000.0] [batch_t 1.433 (1.452)] [data_t 0.002] [optim_t 1.430] [lr 0.005000] +2024-04-03 08:42:48,081 - Train: 2.70% [110000/4070000] [27.0/1000.0] [batch_t 1.428 (1.431)] [data_t 0.003] [optim_t 1.425] [lr 0.005000] +2024-04-03 08:45:11,201 - Train: 2.71% [110100/4070000] [27.1/1000.0] [batch_t 1.434 (1.431)] [data_t 0.003] [optim_t 1.432] [lr 0.005000] +2024-04-03 08:47:34,311 - Train: 2.71% [110200/4070000] [27.1/1000.0] [batch_t 1.432 (1.431)] [data_t 0.003] [optim_t 1.429] [lr 0.005000] +2024-04-03 08:49:57,433 - Train: 2.71% [110300/4070000] [27.1/1000.0] [batch_t 1.435 (1.431)] [data_t 0.002] [optim_t 1.433] [lr 0.005000] +2024-04-03 08:52:20,587 - Train: 2.71% [110400/4070000] [27.1/1000.0] [batch_t 1.433 (1.431)] [data_t 0.002] [optim_t 1.431] [lr 0.005000] +2024-04-03 08:54:43,658 - Train: 2.71% [110500/4070000] [27.1/1000.0] [batch_t 1.435 (1.431)] [data_t 0.002] [optim_t 1.433] [lr 0.005000] +2024-04-03 08:57:06,785 - Train: 2.72% [110600/4070000] [27.2/1000.0] [batch_t 1.437 (1.431)] [data_t 0.002] [optim_t 1.435] [lr 0.005000] +2024-04-03 08:59:30,015 - Train: 2.72% [110700/4070000] [27.2/1000.0] [batch_t 1.432 (1.432)] [data_t 0.002] [optim_t 1.429] [lr 0.005000] +2024-04-03 09:01:53,085 - Train: 2.72% [110800/4070000] [27.2/1000.0] [batch_t 1.432 (1.431)] [data_t 0.002] [optim_t 1.430] [lr 0.005000] +2024-04-03 09:04:16,079 - Train: 2.72% [110900/4070000] [27.2/1000.0] [batch_t 1.413 (1.430)] [data_t 0.003] [optim_t 1.410] [lr 0.005000] +2024-04-03 09:06:39,186 - Train: 2.73% [111000/4070000] [27.3/1000.0] [batch_t 1.425 (1.431)] [data_t 0.002] [optim_t 1.422] [lr 0.005000] +2024-04-03 09:09:02,272 - Train: 2.73% [111100/4070000] [27.3/1000.0] [batch_t 1.432 (1.431)] [data_t 0.003] [optim_t 1.429] [lr 0.005000] +2024-04-03 09:11:25,402 - Train: 2.73% [111200/4070000] [27.3/1000.0] [batch_t 1.423 (1.431)] [data_t 0.003] [optim_t 1.421] [lr 0.005000] +2024-04-03 09:13:48,420 - Train: 2.73% [111300/4070000] [27.3/1000.0] [batch_t 1.442 (1.430)] [data_t 0.002] [optim_t 1.440] [lr 0.005000] +2024-04-03 09:16:11,613 - Train: 2.74% [111400/4070000] [27.4/1000.0] [batch_t 1.424 (1.432)] [data_t 0.003] [optim_t 1.421] [lr 0.005000] +2024-04-03 09:18:34,831 - Train: 2.74% [111500/4070000] [27.4/1000.0] [batch_t 1.443 (1.432)] [data_t 0.002] [optim_t 1.441] [lr 0.005000] +2024-04-03 09:20:57,833 - Train: 2.74% [111600/4070000] [27.4/1000.0] [batch_t 1.437 (1.430)] [data_t 0.002] [optim_t 1.435] [lr 0.005000] +2024-04-03 09:23:20,883 - Train: 2.74% [111700/4070000] [27.4/1000.0] [batch_t 1.424 (1.430)] [data_t 0.002] [optim_t 1.421] [lr 0.005000] +2024-04-03 09:25:43,677 - Train: 2.75% [111800/4070000] [27.5/1000.0] [batch_t 1.425 (1.428)] [data_t 0.002] [optim_t 1.423] [lr 0.005000] +2024-04-03 09:28:06,622 - Train: 2.75% [111900/4070000] [27.5/1000.0] [batch_t 1.446 (1.429)] [data_t 0.003] [optim_t 1.444] [lr 0.005000] +2024-04-03 09:30:29,633 - Train: 2.75% [112000/4070000] [27.5/1000.0] [batch_t 1.431 (1.430)] [data_t 0.002] [optim_t 1.429] [lr 0.005000] +2024-04-03 09:32:52,556 - Train: 2.75% [112100/4070000] [27.5/1000.0] [batch_t 1.438 (1.429)] [data_t 0.002] [optim_t 1.436] [lr 0.005000] +2024-04-03 09:35:15,463 - Train: 2.76% [112200/4070000] [27.6/1000.0] [batch_t 1.446 (1.429)] [data_t 0.002] [optim_t 1.444] [lr 0.005000] +2024-04-03 09:37:38,429 - Train: 2.76% [112300/4070000] [27.6/1000.0] [batch_t 1.434 (1.430)] [data_t 0.003] [optim_t 1.431] [lr 0.005000] +2024-04-03 09:40:01,297 - Train: 2.76% [112400/4070000] [27.6/1000.0] [batch_t 1.431 (1.429)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-03 09:42:24,169 - Train: 2.76% [112500/4070000] [27.6/1000.0] [batch_t 1.444 (1.429)] [data_t 0.003] [optim_t 1.441] [lr 0.005000] +2024-04-03 09:44:47,070 - Train: 2.77% [112600/4070000] [27.7/1000.0] [batch_t 1.437 (1.429)] [data_t 0.003] [optim_t 1.434] [lr 0.005000] +2024-04-03 09:47:10,025 - Train: 2.77% [112700/4070000] [27.7/1000.0] [batch_t 1.440 (1.429)] [data_t 0.003] [optim_t 1.437] [lr 0.005000] +2024-04-03 09:49:32,804 - Train: 2.77% [112800/4070000] [27.7/1000.0] [batch_t 1.424 (1.428)] [data_t 0.002] [optim_t 1.422] [lr 0.005000] +2024-04-03 09:51:55,798 - Train: 2.77% [112900/4070000] [27.7/1000.0] [batch_t 1.414 (1.430)] [data_t 0.002] [optim_t 1.412] [lr 0.005000] +2024-04-03 09:54:18,688 - Train: 2.78% [113000/4070000] [27.8/1000.0] [batch_t 1.429 (1.429)] [data_t 0.002] [optim_t 1.427] [lr 0.005000] +2024-04-03 09:56:41,560 - Train: 2.78% [113100/4070000] [27.8/1000.0] [batch_t 1.430 (1.429)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-03 09:59:04,639 - Train: 2.78% [113200/4070000] [27.8/1000.0] [batch_t 1.424 (1.431)] [data_t 0.002] [optim_t 1.422] [lr 0.005000] +2024-04-03 10:01:27,646 - Train: 2.78% [113300/4070000] [27.8/1000.0] [batch_t 1.430 (1.430)] [data_t 0.002] [optim_t 1.428] [lr 0.005000] +2024-04-03 10:03:50,535 - Train: 2.79% [113400/4070000] [27.9/1000.0] [batch_t 1.431 (1.429)] [data_t 0.002] [optim_t 1.428] [lr 0.005000] +2024-04-03 10:06:13,356 - Train: 2.79% [113500/4070000] [27.9/1000.0] [batch_t 1.426 (1.428)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-03 10:08:36,333 - Train: 2.79% [113600/4070000] [27.9/1000.0] [batch_t 1.426 (1.430)] [data_t 0.002] [optim_t 1.423] [lr 0.005000] +2024-04-03 10:10:59,112 - Train: 2.79% [113700/4070000] [27.9/1000.0] [batch_t 1.429 (1.428)] [data_t 0.002] [optim_t 1.427] [lr 0.005000] +2024-04-03 10:13:22,002 - Train: 2.80% [113800/4070000] [28.0/1000.0] [batch_t 1.429 (1.429)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-03 10:15:48,722 - Train: 2.80% [113900/4070000] [28.0/1000.0] [batch_t 1.441 (1.467)] [data_t 0.002] [optim_t 1.439] [lr 0.005000] +2024-04-03 10:17:14,480 - ==> Total time: 1 day, 9:16:45 Eta: 48 days, 3:15:56 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-03 10:18:17,905 - Train: 2.80% [114000/4070000] [28.0/1000.0] [batch_t 1.424 (1.532)] [data_t 0.002] [optim_t 1.422] [lr 0.005000] +2024-04-03 10:21:01,377 - Train: 2.80% [114100/4070000] [28.0/1000.0] [batch_t 1.421 (1.635)] [data_t 0.002] [optim_t 1.419] [lr 0.005000] +2024-04-03 10:23:24,458 - Train: 2.81% [114200/4070000] [28.1/1000.0] [batch_t 1.442 (1.431)] [data_t 0.003] [optim_t 1.439] [lr 0.005000] +2024-04-03 10:26:16,228 - Train: 2.81% [114300/4070000] [28.1/1000.0] [batch_t 5.116 (1.718)] [data_t 3.665] [optim_t 1.452] [lr 0.005000] +2024-04-03 10:30:21,040 - Train: 2.81% [114400/4070000] [28.1/1000.0] [batch_t 1.441 (2.448)] [data_t 0.003] [optim_t 1.439] [lr 0.005000] +2024-04-03 10:32:48,019 - Train: 2.81% [114500/4070000] [28.1/1000.0] [batch_t 1.441 (1.470)] [data_t 0.002] [optim_t 1.439] [lr 0.005000] +2024-04-03 10:35:21,933 - Train: 2.82% [114600/4070000] [28.2/1000.0] [batch_t 1.433 (1.539)] [data_t 0.003] [optim_t 1.430] [lr 0.005000] +2024-04-03 10:37:44,925 - Train: 2.82% [114700/4070000] [28.2/1000.0] [batch_t 1.429 (1.430)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-03 10:40:08,008 - Train: 2.82% [114800/4070000] [28.2/1000.0] [batch_t 1.432 (1.431)] [data_t 0.003] [optim_t 1.430] [lr 0.005000] +2024-04-03 10:42:30,981 - Train: 2.82% [114900/4070000] [28.2/1000.0] [batch_t 1.421 (1.430)] [data_t 0.003] [optim_t 1.418] [lr 0.005000] +2024-04-03 10:44:54,128 - Train: 2.83% [115000/4070000] [28.3/1000.0] [batch_t 1.440 (1.431)] [data_t 0.002] [optim_t 1.437] [lr 0.005000] +2024-04-03 10:47:17,167 - Train: 2.83% [115100/4070000] [28.3/1000.0] [batch_t 1.403 (1.430)] [data_t 0.003] [optim_t 1.400] [lr 0.005000] +2024-04-03 10:49:43,341 - Train: 2.83% [115200/4070000] [28.3/1000.0] [batch_t 1.430 (1.462)] [data_t 0.002] [optim_t 1.428] [lr 0.005000] +2024-04-03 10:52:06,237 - Train: 2.83% [115300/4070000] [28.3/1000.0] [batch_t 1.435 (1.429)] [data_t 0.003] [optim_t 1.432] [lr 0.005000] +2024-04-03 10:54:29,109 - Train: 2.84% [115400/4070000] [28.4/1000.0] [batch_t 1.430 (1.429)] [data_t 0.002] [optim_t 1.428] [lr 0.005000] +2024-04-03 10:56:52,014 - Train: 2.84% [115500/4070000] [28.4/1000.0] [batch_t 1.429 (1.429)] [data_t 0.002] [optim_t 1.427] [lr 0.005000] +2024-04-03 10:59:14,839 - Train: 2.84% [115600/4070000] [28.4/1000.0] [batch_t 1.428 (1.428)] [data_t 0.002] [optim_t 1.425] [lr 0.005000] +2024-04-03 11:01:37,656 - Train: 2.84% [115700/4070000] [28.4/1000.0] [batch_t 1.435 (1.428)] [data_t 0.003] [optim_t 1.432] [lr 0.005000] +2024-04-03 11:04:00,587 - Train: 2.85% [115800/4070000] [28.5/1000.0] [batch_t 1.427 (1.429)] [data_t 0.002] [optim_t 1.425] [lr 0.005000] +2024-04-03 11:06:23,342 - Train: 2.85% [115900/4070000] [28.5/1000.0] [batch_t 1.442 (1.427)] [data_t 0.003] [optim_t 1.440] [lr 0.005000] +2024-04-03 11:08:46,280 - Train: 2.85% [116000/4070000] [28.5/1000.0] [batch_t 1.440 (1.429)] [data_t 0.002] [optim_t 1.438] [lr 0.005000] +2024-04-03 11:11:09,077 - Train: 2.85% [116100/4070000] [28.5/1000.0] [batch_t 1.424 (1.428)] [data_t 0.002] [optim_t 1.422] [lr 0.005000] +2024-04-03 11:13:32,054 - Train: 2.86% [116200/4070000] [28.6/1000.0] [batch_t 1.441 (1.430)] [data_t 0.003] [optim_t 1.438] [lr 0.005000] +2024-04-03 11:15:54,924 - Train: 2.86% [116300/4070000] [28.6/1000.0] [batch_t 1.445 (1.429)] [data_t 0.002] [optim_t 1.443] [lr 0.005000] +2024-04-03 11:18:17,709 - Train: 2.86% [116400/4070000] [28.6/1000.0] [batch_t 1.445 (1.428)] [data_t 0.002] [optim_t 1.443] [lr 0.005000] +2024-04-03 11:20:40,478 - Train: 2.86% [116500/4070000] [28.6/1000.0] [batch_t 1.419 (1.428)] [data_t 0.003] [optim_t 1.416] [lr 0.005000] +2024-04-03 11:23:03,428 - Train: 2.86% [116600/4070000] [28.6/1000.0] [batch_t 1.417 (1.429)] [data_t 0.003] [optim_t 1.414] [lr 0.005000] +2024-04-03 11:25:26,325 - Train: 2.87% [116700/4070000] [28.7/1000.0] [batch_t 1.440 (1.429)] [data_t 0.002] [optim_t 1.438] [lr 0.005000] +2024-04-03 11:27:49,138 - Train: 2.87% [116800/4070000] [28.7/1000.0] [batch_t 1.417 (1.428)] [data_t 0.003] [optim_t 1.414] [lr 0.005000] +2024-04-03 11:30:12,103 - Train: 2.87% [116900/4070000] [28.7/1000.0] [batch_t 1.429 (1.430)] [data_t 0.003] [optim_t 1.426] [lr 0.005000] +2024-04-03 11:32:34,855 - Train: 2.87% [117000/4070000] [28.7/1000.0] [batch_t 1.422 (1.427)] [data_t 0.002] [optim_t 1.420] [lr 0.005000] +2024-04-03 11:34:57,783 - Train: 2.88% [117100/4070000] [28.8/1000.0] [batch_t 1.436 (1.429)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-03 11:37:20,699 - Train: 2.88% [117200/4070000] [28.8/1000.0] [batch_t 1.439 (1.429)] [data_t 0.003] [optim_t 1.436] [lr 0.005000] +2024-04-03 11:39:43,494 - Train: 2.88% [117300/4070000] [28.8/1000.0] [batch_t 1.419 (1.428)] [data_t 0.003] [optim_t 1.416] [lr 0.005000] +2024-04-03 11:54:00,781 - Train: 2.88% [117400/4070000] [28.8/1000.0] [batch_t 41.556 (8.573)] [data_t 40.117] [optim_t 1.440] [lr 0.005000] +2024-04-03 11:58:51,675 - Train: 2.89% [117500/4070000] [28.9/1000.0] [batch_t 1.436 (2.909)] [data_t 0.002] [optim_t 1.433] [lr 0.005000] +2024-04-03 12:01:14,575 - Train: 2.89% [117600/4070000] [28.9/1000.0] [batch_t 1.413 (1.429)] [data_t 0.003] [optim_t 1.411] [lr 0.005000] +2024-04-03 12:03:37,367 - Train: 2.89% [117700/4070000] [28.9/1000.0] [batch_t 1.422 (1.428)] [data_t 0.002] [optim_t 1.419] [lr 0.005000] +2024-04-03 12:06:00,210 - Train: 2.89% [117800/4070000] [28.9/1000.0] [batch_t 1.428 (1.428)] [data_t 0.003] [optim_t 1.426] [lr 0.005000] +2024-04-03 12:08:23,090 - Train: 2.90% [117900/4070000] [29.0/1000.0] [batch_t 1.430 (1.429)] [data_t 0.002] [optim_t 1.428] [lr 0.005000] +2024-04-03 12:10:46,059 - Train: 2.90% [118000/4070000] [29.0/1000.0] [batch_t 1.422 (1.430)] [data_t 0.003] [optim_t 1.419] [lr 0.005000] +2024-04-03 12:11:28,890 - ==> Total time: 1 day, 11:10:59 Eta: 49 days, 2:01:57 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-03 12:13:10,742 - Train: 2.90% [118100/4070000] [29.0/1000.0] [batch_t 1.427 (1.431)] [data_t 0.002] [optim_t 1.424] [lr 0.005000] +2024-04-03 12:15:33,432 - Train: 2.90% [118200/4070000] [29.0/1000.0] [batch_t 1.423 (1.427)] [data_t 0.003] [optim_t 1.420] [lr 0.005000] +2024-04-03 12:17:56,329 - Train: 2.91% [118300/4070000] [29.1/1000.0] [batch_t 1.431 (1.429)] [data_t 0.002] [optim_t 1.429] [lr 0.005000] +2024-04-03 12:20:19,331 - Train: 2.91% [118400/4070000] [29.1/1000.0] [batch_t 1.430 (1.430)] [data_t 0.002] [optim_t 1.428] [lr 0.005000] +2024-04-03 12:22:41,976 - Train: 2.91% [118500/4070000] [29.1/1000.0] [batch_t 1.435 (1.426)] [data_t 0.002] [optim_t 1.433] [lr 0.005000] +2024-04-03 12:25:04,801 - Train: 2.91% [118600/4070000] [29.1/1000.0] [batch_t 1.426 (1.428)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-03 12:27:27,761 - Train: 2.92% [118700/4070000] [29.2/1000.0] [batch_t 1.426 (1.430)] [data_t 0.002] [optim_t 1.424] [lr 0.005000] +2024-04-03 12:29:50,509 - Train: 2.92% [118800/4070000] [29.2/1000.0] [batch_t 1.444 (1.427)] [data_t 0.002] [optim_t 1.442] [lr 0.005000] +2024-04-03 12:32:13,502 - Train: 2.92% [118900/4070000] [29.2/1000.0] [batch_t 1.437 (1.430)] [data_t 0.002] [optim_t 1.435] [lr 0.005000] +2024-04-03 12:34:36,447 - Train: 2.92% [119000/4070000] [29.2/1000.0] [batch_t 1.420 (1.429)] [data_t 0.003] [optim_t 1.417] [lr 0.005000] +2024-04-03 12:36:59,194 - Train: 2.93% [119100/4070000] [29.3/1000.0] [batch_t 1.430 (1.427)] [data_t 0.002] [optim_t 1.428] [lr 0.005000] +2024-04-03 12:39:22,080 - Train: 2.93% [119200/4070000] [29.3/1000.0] [batch_t 1.419 (1.429)] [data_t 0.003] [optim_t 1.417] [lr 0.005000] +2024-04-03 12:41:45,129 - Train: 2.93% [119300/4070000] [29.3/1000.0] [batch_t 1.426 (1.430)] [data_t 0.002] [optim_t 1.423] [lr 0.005000] +2024-04-03 12:44:07,998 - Train: 2.93% [119400/4070000] [29.3/1000.0] [batch_t 1.434 (1.429)] [data_t 0.002] [optim_t 1.432] [lr 0.005000] +2024-04-03 12:46:30,817 - Train: 2.94% [119500/4070000] [29.4/1000.0] [batch_t 1.430 (1.428)] [data_t 0.002] [optim_t 1.427] [lr 0.005000] +2024-04-03 12:48:53,657 - Train: 2.94% [119600/4070000] [29.4/1000.0] [batch_t 1.428 (1.428)] [data_t 0.002] [optim_t 1.426] [lr 0.005000] +2024-04-03 12:51:16,426 - Train: 2.94% [119700/4070000] [29.4/1000.0] [batch_t 1.441 (1.428)] [data_t 0.002] [optim_t 1.438] [lr 0.005000] +2024-04-03 12:53:39,201 - Train: 2.94% [119800/4070000] [29.4/1000.0] [batch_t 1.444 (1.428)] [data_t 0.002] [optim_t 1.442] [lr 0.005000] +2024-04-03 12:56:02,235 - Train: 2.95% [119900/4070000] [29.5/1000.0] [batch_t 1.441 (1.430)] [data_t 0.003] [optim_t 1.438] [lr 0.005000] +2024-04-03 12:58:25,092 - Train: 2.95% [120000/4070000] [29.5/1000.0] [batch_t 1.436 (1.428)] [data_t 0.002] [optim_t 1.433] [lr 0.005000] +2024-04-03 13:00:47,864 - Train: 2.95% [120100/4070000] [29.5/1000.0] [batch_t 1.439 (1.428)] [data_t 0.002] [optim_t 1.437] [lr 0.005000] +2024-04-03 13:03:10,688 - Train: 2.95% [120200/4070000] [29.5/1000.0] [batch_t 1.417 (1.428)] [data_t 0.002] [optim_t 1.414] [lr 0.005000] +2024-04-03 13:05:33,501 - Train: 2.96% [120300/4070000] [29.6/1000.0] [batch_t 1.420 (1.428)] [data_t 0.003] [optim_t 1.417] [lr 0.005000] +2024-04-03 13:07:56,432 - Train: 2.96% [120400/4070000] [29.6/1000.0] [batch_t 1.427 (1.429)] [data_t 0.003] [optim_t 1.425] [lr 0.005000] +2024-04-03 13:10:19,185 - Train: 2.96% [120500/4070000] [29.6/1000.0] [batch_t 1.433 (1.427)] [data_t 0.002] [optim_t 1.430] [lr 0.005000] +2024-04-03 13:12:42,115 - Train: 2.96% [120600/4070000] [29.6/1000.0] [batch_t 1.424 (1.429)] [data_t 0.003] [optim_t 1.422] [lr 0.005000] +2024-04-03 13:15:04,927 - Train: 2.97% [120700/4070000] [29.7/1000.0] [batch_t 1.420 (1.428)] [data_t 0.003] [optim_t 1.417] [lr 0.005000] +2024-04-03 13:17:27,816 - Train: 2.97% [120800/4070000] [29.7/1000.0] [batch_t 1.427 (1.429)] [data_t 0.003] [optim_t 1.425] [lr 0.005000] +2024-04-03 13:19:50,751 - Train: 2.97% [120900/4070000] [29.7/1000.0] [batch_t 1.425 (1.429)] [data_t 0.003] [optim_t 1.422] [lr 0.005000] +2024-04-03 13:22:13,540 - Train: 2.97% [121000/4070000] [29.7/1000.0] [batch_t 1.437 (1.428)] [data_t 0.003] [optim_t 1.434] [lr 0.005000] +2024-04-03 13:24:36,379 - Train: 2.98% [121100/4070000] [29.8/1000.0] [batch_t 1.412 (1.428)] [data_t 0.002] [optim_t 1.410] [lr 0.005000] +2024-04-03 13:26:59,239 - Train: 2.98% [121200/4070000] [29.8/1000.0] [batch_t 1.439 (1.429)] [data_t 0.002] [optim_t 1.436] [lr 0.005000] +2024-04-03 13:29:22,029 - Train: 2.98% [121300/4070000] [29.8/1000.0] [batch_t 1.448 (1.428)] [data_t 0.003] [optim_t 1.445] [lr 0.005000] +2024-04-03 13:31:45,069 - Train: 2.98% [121400/4070000] [29.8/1000.0] [batch_t 1.426 (1.430)] [data_t 0.002] [optim_t 1.424] [lr 0.005000] +2024-04-03 13:34:08,057 - Train: 2.99% [121500/4070000] [29.9/1000.0] [batch_t 1.426 (1.430)] [data_t 0.002] [optim_t 1.424] [lr 0.005000] +2024-04-03 13:36:30,997 - Train: 2.99% [121600/4070000] [29.9/1000.0] [batch_t 1.431 (1.429)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-03 13:38:53,722 - Train: 2.99% [121700/4070000] [29.9/1000.0] [batch_t 1.425 (1.427)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-03 13:41:16,641 - Train: 2.99% [121800/4070000] [29.9/1000.0] [batch_t 1.439 (1.429)] [data_t 0.003] [optim_t 1.436] [lr 0.005000] +2024-04-03 13:43:39,477 - Train: 3.00% [121900/4070000] [30.0/1000.0] [batch_t 1.425 (1.428)] [data_t 0.002] [optim_t 1.423] [lr 0.005000] +2024-04-03 13:46:02,300 - Train: 3.00% [122000/4070000] [30.0/1000.0] [batch_t 1.428 (1.428)] [data_t 0.003] [optim_t 1.425] [lr 0.005000] +2024-04-03 13:48:25,207 - Train: 3.00% [122100/4070000] [30.0/1000.0] [batch_t 1.450 (1.429)] [data_t 0.003] [optim_t 1.447] [lr 0.005000] +2024-04-03 13:48:25,210 - ==> Total time: 1 day, 12:47:56 Eta: 49 days, 13:49:52 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-03 13:50:49,734 - Train: 3.00% [122200/4070000] [30.0/1000.0] [batch_t 1.434 (1.432)] [data_t 0.002] [optim_t 1.432] [lr 0.005000] +2024-04-03 13:53:12,679 - Train: 3.00% [122300/4070000] [30.0/1000.0] [batch_t 1.430 (1.429)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-03 13:55:35,616 - Train: 3.01% [122400/4070000] [30.1/1000.0] [batch_t 1.430 (1.429)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-03 13:57:58,583 - Train: 3.01% [122500/4070000] [30.1/1000.0] [batch_t 1.420 (1.430)] [data_t 0.003] [optim_t 1.417] [lr 0.005000] +2024-04-03 14:00:21,455 - Train: 3.01% [122600/4070000] [30.1/1000.0] [batch_t 1.432 (1.429)] [data_t 0.002] [optim_t 1.429] [lr 0.005000] +2024-04-03 14:02:44,319 - Train: 3.01% [122700/4070000] [30.1/1000.0] [batch_t 1.425 (1.429)] [data_t 0.003] [optim_t 1.422] [lr 0.005000] +2024-04-03 14:05:07,084 - Train: 3.02% [122800/4070000] [30.2/1000.0] [batch_t 1.433 (1.428)] [data_t 0.003] [optim_t 1.430] [lr 0.005000] +2024-04-03 14:07:30,036 - Train: 3.02% [122900/4070000] [30.2/1000.0] [batch_t 1.423 (1.429)] [data_t 0.002] [optim_t 1.421] [lr 0.005000] +2024-04-03 14:09:52,839 - Train: 3.02% [123000/4070000] [30.2/1000.0] [batch_t 1.422 (1.428)] [data_t 0.002] [optim_t 1.420] [lr 0.005000] +2024-04-03 14:12:15,679 - Train: 3.02% [123100/4070000] [30.2/1000.0] [batch_t 1.435 (1.428)] [data_t 0.002] [optim_t 1.433] [lr 0.005000] +2024-04-03 14:14:38,620 - Train: 3.03% [123200/4070000] [30.3/1000.0] [batch_t 1.427 (1.429)] [data_t 0.002] [optim_t 1.425] [lr 0.005000] +2024-04-03 14:21:08,533 - Train: 3.03% [123300/4070000] [30.3/1000.0] [batch_t 1.411 (3.899)] [data_t 0.003] [optim_t 1.409] [lr 0.005000] +2024-04-03 14:23:31,483 - Train: 3.03% [123400/4070000] [30.3/1000.0] [batch_t 1.420 (1.429)] [data_t 0.002] [optim_t 1.418] [lr 0.005000] +2024-04-03 14:25:54,551 - Train: 3.03% [123500/4070000] [30.3/1000.0] [batch_t 1.434 (1.431)] [data_t 0.003] [optim_t 1.431] [lr 0.005000] +2024-04-03 14:28:17,457 - Train: 3.04% [123600/4070000] [30.4/1000.0] [batch_t 1.437 (1.429)] [data_t 0.002] [optim_t 1.434] [lr 0.005000] +2024-04-03 14:30:40,556 - Train: 3.04% [123700/4070000] [30.4/1000.0] [batch_t 1.430 (1.431)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-03 14:41:52,112 - Train: 3.04% [123800/4070000] [30.4/1000.0] [batch_t 1.409 (6.715)] [data_t 0.003] [optim_t 1.407] [lr 0.005000] +2024-04-03 14:44:15,040 - Train: 3.04% [123900/4070000] [30.4/1000.0] [batch_t 1.425 (1.429)] [data_t 0.002] [optim_t 1.423] [lr 0.005000] +2024-04-03 14:46:37,863 - Train: 3.05% [124000/4070000] [30.5/1000.0] [batch_t 1.429 (1.428)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-03 14:49:00,844 - Train: 3.05% [124100/4070000] [30.5/1000.0] [batch_t 1.439 (1.430)] [data_t 0.002] [optim_t 1.437] [lr 0.005000] +2024-04-03 14:51:23,719 - Train: 3.05% [124200/4070000] [30.5/1000.0] [batch_t 1.423 (1.429)] [data_t 0.003] [optim_t 1.420] [lr 0.005000] +2024-04-03 14:53:46,559 - Train: 3.05% [124300/4070000] [30.5/1000.0] [batch_t 1.425 (1.428)] [data_t 0.002] [optim_t 1.423] [lr 0.005000] +2024-04-03 14:56:09,518 - Train: 3.06% [124400/4070000] [30.6/1000.0] [batch_t 1.424 (1.430)] [data_t 0.003] [optim_t 1.421] [lr 0.005000] +2024-04-03 14:58:32,390 - Train: 3.06% [124500/4070000] [30.6/1000.0] [batch_t 1.424 (1.429)] [data_t 0.003] [optim_t 1.421] [lr 0.005000] +2024-04-03 15:00:55,363 - Train: 3.06% [124600/4070000] [30.6/1000.0] [batch_t 1.427 (1.430)] [data_t 0.002] [optim_t 1.425] [lr 0.005000] +2024-04-03 15:03:18,272 - Train: 3.06% [124700/4070000] [30.6/1000.0] [batch_t 1.392 (1.429)] [data_t 0.003] [optim_t 1.389] [lr 0.005000] +2024-04-03 15:05:41,270 - Train: 3.07% [124800/4070000] [30.7/1000.0] [batch_t 1.431 (1.430)] [data_t 0.002] [optim_t 1.429] [lr 0.005000] +2024-04-03 15:08:04,138 - Train: 3.07% [124900/4070000] [30.7/1000.0] [batch_t 1.416 (1.429)] [data_t 0.002] [optim_t 1.413] [lr 0.005000] +2024-04-03 15:10:27,077 - Train: 3.07% [125000/4070000] [30.7/1000.0] [batch_t 1.412 (1.429)] [data_t 0.002] [optim_t 1.410] [lr 0.005000] +2024-04-03 15:12:50,111 - Train: 3.07% [125100/4070000] [30.7/1000.0] [batch_t 1.428 (1.430)] [data_t 0.003] [optim_t 1.425] [lr 0.005000] +2024-04-03 15:15:12,964 - Train: 3.08% [125200/4070000] [30.8/1000.0] [batch_t 1.423 (1.428)] [data_t 0.002] [optim_t 1.421] [lr 0.005000] +2024-04-03 15:17:35,911 - Train: 3.08% [125300/4070000] [30.8/1000.0] [batch_t 1.425 (1.429)] [data_t 0.002] [optim_t 1.423] [lr 0.005000] +2024-04-03 15:19:58,914 - Train: 3.08% [125400/4070000] [30.8/1000.0] [batch_t 1.416 (1.430)] [data_t 0.003] [optim_t 1.414] [lr 0.005000] +2024-04-03 15:22:21,649 - Train: 3.08% [125500/4070000] [30.8/1000.0] [batch_t 1.425 (1.427)] [data_t 0.003] [optim_t 1.422] [lr 0.005000] +2024-04-03 15:24:44,543 - Train: 3.09% [125600/4070000] [30.9/1000.0] [batch_t 1.439 (1.429)] [data_t 0.002] [optim_t 1.436] [lr 0.005000] +2024-04-03 15:27:07,463 - Train: 3.09% [125700/4070000] [30.9/1000.0] [batch_t 1.420 (1.429)] [data_t 0.002] [optim_t 1.417] [lr 0.005000] +2024-04-03 15:29:30,388 - Train: 3.09% [125800/4070000] [30.9/1000.0] [batch_t 1.431 (1.429)] [data_t 0.002] [optim_t 1.428] [lr 0.005000] +2024-04-03 15:31:53,303 - Train: 3.09% [125900/4070000] [30.9/1000.0] [batch_t 1.436 (1.429)] [data_t 0.002] [optim_t 1.433] [lr 0.005000] +2024-04-03 15:34:16,346 - Train: 3.10% [126000/4070000] [31.0/1000.0] [batch_t 1.427 (1.430)] [data_t 0.003] [optim_t 1.424] [lr 0.005000] +2024-04-03 15:36:39,307 - Train: 3.10% [126100/4070000] [31.0/1000.0] [batch_t 1.447 (1.430)] [data_t 0.003] [optim_t 1.444] [lr 0.005000] +2024-04-03 15:38:19,372 - ==> Total time: 1 day, 14:37:50 Eta: 50 days, 7:31:06 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-03 15:39:03,874 - Train: 3.10% [126200/4070000] [31.0/1000.0] [batch_t 1.435 (1.442)] [data_t 0.002] [optim_t 1.432] [lr 0.005000] +2024-04-03 15:41:26,844 - Train: 3.10% [126300/4070000] [31.0/1000.0] [batch_t 1.414 (1.430)] [data_t 0.002] [optim_t 1.412] [lr 0.005000] +2024-04-03 15:43:49,735 - Train: 3.11% [126400/4070000] [31.1/1000.0] [batch_t 1.417 (1.429)] [data_t 0.003] [optim_t 1.415] [lr 0.005000] +2024-04-03 15:46:12,554 - Train: 3.11% [126500/4070000] [31.1/1000.0] [batch_t 1.434 (1.428)] [data_t 0.002] [optim_t 1.431] [lr 0.005000] +2024-04-03 15:48:35,593 - Train: 3.11% [126600/4070000] [31.1/1000.0] [batch_t 1.427 (1.430)] [data_t 0.003] [optim_t 1.424] [lr 0.005000] +2024-04-03 15:50:58,457 - Train: 3.11% [126700/4070000] [31.1/1000.0] [batch_t 1.415 (1.429)] [data_t 0.003] [optim_t 1.412] [lr 0.005000] +2024-04-03 15:53:21,181 - Train: 3.12% [126800/4070000] [31.2/1000.0] [batch_t 1.431 (1.427)] [data_t 0.002] [optim_t 1.429] [lr 0.005000] +2024-04-03 15:55:44,034 - Train: 3.12% [126900/4070000] [31.2/1000.0] [batch_t 1.438 (1.428)] [data_t 0.003] [optim_t 1.435] [lr 0.005000] +2024-04-03 15:58:06,990 - Train: 3.12% [127000/4070000] [31.2/1000.0] [batch_t 1.427 (1.429)] [data_t 0.003] [optim_t 1.425] [lr 0.005000] +2024-04-03 16:00:29,943 - Train: 3.12% [127100/4070000] [31.2/1000.0] [batch_t 1.434 (1.429)] [data_t 0.003] [optim_t 1.431] [lr 0.005000] +2024-04-03 16:02:52,778 - Train: 3.13% [127200/4070000] [31.3/1000.0] [batch_t 1.429 (1.428)] [data_t 0.002] [optim_t 1.427] [lr 0.005000] +2024-04-03 16:05:15,588 - Train: 3.13% [127300/4070000] [31.3/1000.0] [batch_t 1.423 (1.428)] [data_t 0.003] [optim_t 1.421] [lr 0.005000] +2024-04-03 16:07:38,711 - Train: 3.13% [127400/4070000] [31.3/1000.0] [batch_t 1.439 (1.431)] [data_t 0.002] [optim_t 1.437] [lr 0.005000] +2024-04-03 16:10:01,699 - Train: 3.13% [127500/4070000] [31.3/1000.0] [batch_t 1.425 (1.430)] [data_t 0.002] [optim_t 1.423] [lr 0.005000] +2024-04-03 16:12:24,489 - Train: 3.14% [127600/4070000] [31.4/1000.0] [batch_t 1.422 (1.428)] [data_t 0.002] [optim_t 1.420] [lr 0.005000] +2024-04-03 16:14:47,255 - Train: 3.14% [127700/4070000] [31.4/1000.0] [batch_t 1.378 (1.428)] [data_t 0.002] [optim_t 1.376] [lr 0.005000] +2024-04-03 16:17:10,224 - Train: 3.14% [127800/4070000] [31.4/1000.0] [batch_t 1.439 (1.430)] [data_t 0.002] [optim_t 1.437] [lr 0.005000] +2024-04-03 16:19:33,249 - Train: 3.14% [127900/4070000] [31.4/1000.0] [batch_t 1.428 (1.430)] [data_t 0.002] [optim_t 1.426] [lr 0.005000] +2024-04-03 16:21:56,212 - Train: 3.14% [128000/4070000] [31.4/1000.0] [batch_t 1.437 (1.430)] [data_t 0.003] [optim_t 1.435] [lr 0.005000] +2024-04-03 16:24:19,179 - Train: 3.15% [128100/4070000] [31.5/1000.0] [batch_t 1.429 (1.430)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-03 16:26:42,159 - Train: 3.15% [128200/4070000] [31.5/1000.0] [batch_t 1.420 (1.430)] [data_t 0.002] [optim_t 1.417] [lr 0.005000] +2024-04-03 16:29:04,942 - Train: 3.15% [128300/4070000] [31.5/1000.0] [batch_t 1.436 (1.428)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-03 16:31:27,912 - Train: 3.15% [128400/4070000] [31.5/1000.0] [batch_t 1.419 (1.430)] [data_t 0.002] [optim_t 1.417] [lr 0.005000] +2024-04-03 16:33:50,887 - Train: 3.16% [128500/4070000] [31.6/1000.0] [batch_t 1.415 (1.430)] [data_t 0.002] [optim_t 1.413] [lr 0.005000] +2024-04-03 16:36:13,721 - Train: 3.16% [128600/4070000] [31.6/1000.0] [batch_t 1.440 (1.428)] [data_t 0.002] [optim_t 1.438] [lr 0.005000] +2024-04-03 16:38:36,560 - Train: 3.16% [128700/4070000] [31.6/1000.0] [batch_t 1.434 (1.428)] [data_t 0.003] [optim_t 1.432] [lr 0.005000] +2024-04-03 16:40:59,347 - Train: 3.16% [128800/4070000] [31.6/1000.0] [batch_t 1.440 (1.428)] [data_t 0.002] [optim_t 1.437] [lr 0.005000] +2024-04-03 16:43:22,100 - Train: 3.17% [128900/4070000] [31.7/1000.0] [batch_t 1.425 (1.427)] [data_t 0.002] [optim_t 1.423] [lr 0.005000] +2024-04-03 16:45:45,162 - Train: 3.17% [129000/4070000] [31.7/1000.0] [batch_t 1.442 (1.431)] [data_t 0.002] [optim_t 1.439] [lr 0.005000] +2024-04-03 16:48:07,972 - Train: 3.17% [129100/4070000] [31.7/1000.0] [batch_t 1.437 (1.428)] [data_t 0.002] [optim_t 1.435] [lr 0.005000] +2024-04-03 16:50:30,742 - Train: 3.17% [129200/4070000] [31.7/1000.0] [batch_t 1.421 (1.428)] [data_t 0.002] [optim_t 1.418] [lr 0.005000] +2024-04-03 16:52:53,598 - Train: 3.18% [129300/4070000] [31.8/1000.0] [batch_t 1.437 (1.428)] [data_t 0.002] [optim_t 1.435] [lr 0.005000] +2024-04-03 16:55:16,294 - Train: 3.18% [129400/4070000] [31.8/1000.0] [batch_t 1.421 (1.427)] [data_t 0.002] [optim_t 1.419] [lr 0.005000] +2024-04-03 16:57:39,143 - Train: 3.18% [129500/4070000] [31.8/1000.0] [batch_t 1.424 (1.428)] [data_t 0.003] [optim_t 1.421] [lr 0.005000] +2024-04-03 17:00:02,088 - Train: 3.18% [129600/4070000] [31.8/1000.0] [batch_t 1.442 (1.429)] [data_t 0.002] [optim_t 1.440] [lr 0.005000] +2024-04-03 17:02:25,008 - Train: 3.19% [129700/4070000] [31.9/1000.0] [batch_t 1.431 (1.429)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-03 17:04:47,917 - Train: 3.19% [129800/4070000] [31.9/1000.0] [batch_t 1.426 (1.429)] [data_t 0.002] [optim_t 1.424] [lr 0.005000] +2024-04-03 17:07:10,769 - Train: 3.19% [129900/4070000] [31.9/1000.0] [batch_t 1.425 (1.428)] [data_t 0.002] [optim_t 1.423] [lr 0.005000] +2024-04-03 17:09:33,727 - Train: 3.19% [130000/4070000] [31.9/1000.0] [batch_t 1.432 (1.430)] [data_t 0.003] [optim_t 1.429] [lr 0.005000] +2024-04-03 17:11:56,573 - Train: 3.20% [130100/4070000] [32.0/1000.0] [batch_t 1.430 (1.428)] [data_t 0.002] [optim_t 1.428] [lr 0.005000] +2024-04-03 17:14:19,519 - Train: 3.20% [130200/4070000] [32.0/1000.0] [batch_t 1.423 (1.429)] [data_t 0.003] [optim_t 1.420] [lr 0.005000] +2024-04-03 17:15:16,729 - ==> Total time: 1 day, 16:14:47 Eta: 50 days, 17:27:29 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-03 17:16:44,992 - Train: 3.20% [130300/4070000] [32.0/1000.0] [batch_t 1.431 (1.443)] [data_t 0.002] [optim_t 1.428] [lr 0.005000] +2024-04-03 17:19:08,036 - Train: 3.20% [130400/4070000] [32.0/1000.0] [batch_t 1.436 (1.430)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-03 17:21:30,951 - Train: 3.21% [130500/4070000] [32.1/1000.0] [batch_t 1.425 (1.429)] [data_t 0.002] [optim_t 1.423] [lr 0.005000] +2024-04-03 17:23:53,874 - Train: 3.21% [130600/4070000] [32.1/1000.0] [batch_t 1.426 (1.429)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-03 17:26:26,555 - Train: 3.21% [130700/4070000] [32.1/1000.0] [batch_t 1.427 (1.527)] [data_t 0.002] [optim_t 1.425] [lr 0.005000] +2024-04-03 17:28:49,599 - Train: 3.21% [130800/4070000] [32.1/1000.0] [batch_t 1.431 (1.430)] [data_t 0.002] [optim_t 1.428] [lr 0.005000] +2024-04-03 17:31:12,538 - Train: 3.22% [130900/4070000] [32.2/1000.0] [batch_t 1.431 (1.429)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-03 17:33:35,493 - Train: 3.22% [131000/4070000] [32.2/1000.0] [batch_t 1.427 (1.429)] [data_t 0.002] [optim_t 1.425] [lr 0.005000] +2024-04-03 17:35:58,383 - Train: 3.22% [131100/4070000] [32.2/1000.0] [batch_t 1.425 (1.429)] [data_t 0.002] [optim_t 1.422] [lr 0.005000] +2024-04-03 17:38:21,349 - Train: 3.22% [131200/4070000] [32.2/1000.0] [batch_t 1.425 (1.430)] [data_t 0.003] [optim_t 1.422] [lr 0.005000] +2024-04-03 17:40:44,320 - Train: 3.23% [131300/4070000] [32.3/1000.0] [batch_t 1.431 (1.430)] [data_t 0.002] [optim_t 1.429] [lr 0.005000] +2024-04-03 17:43:07,221 - Train: 3.23% [131400/4070000] [32.3/1000.0] [batch_t 1.430 (1.429)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-03 17:45:30,224 - Train: 3.23% [131500/4070000] [32.3/1000.0] [batch_t 1.428 (1.430)] [data_t 0.003] [optim_t 1.425] [lr 0.005000] +2024-04-03 17:47:53,188 - Train: 3.23% [131600/4070000] [32.3/1000.0] [batch_t 1.432 (1.430)] [data_t 0.003] [optim_t 1.430] [lr 0.005000] +2024-04-03 17:50:16,035 - Train: 3.24% [131700/4070000] [32.4/1000.0] [batch_t 1.431 (1.428)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-03 17:53:03,224 - Train: 3.24% [131800/4070000] [32.4/1000.0] [batch_t 1.440 (1.672)] [data_t 0.002] [optim_t 1.438] [lr 0.005000] +2024-04-03 17:55:25,959 - Train: 3.24% [131900/4070000] [32.4/1000.0] [batch_t 1.379 (1.427)] [data_t 0.002] [optim_t 1.377] [lr 0.005000] +2024-04-03 17:57:49,045 - Train: 3.24% [132000/4070000] [32.4/1000.0] [batch_t 1.440 (1.431)] [data_t 0.002] [optim_t 1.437] [lr 0.005000] +2024-04-03 18:00:11,942 - Train: 3.25% [132100/4070000] [32.5/1000.0] [batch_t 1.427 (1.429)] [data_t 0.002] [optim_t 1.424] [lr 0.005000] +2024-04-03 18:02:34,910 - Train: 3.25% [132200/4070000] [32.5/1000.0] [batch_t 1.432 (1.430)] [data_t 0.002] [optim_t 1.430] [lr 0.005000] +2024-04-03 18:04:57,780 - Train: 3.25% [132300/4070000] [32.5/1000.0] [batch_t 1.434 (1.429)] [data_t 0.002] [optim_t 1.432] [lr 0.005000] +2024-04-03 18:07:20,644 - Train: 3.25% [132400/4070000] [32.5/1000.0] [batch_t 1.430 (1.429)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-03 18:09:43,492 - Train: 3.26% [132500/4070000] [32.6/1000.0] [batch_t 1.431 (1.428)] [data_t 0.002] [optim_t 1.428] [lr 0.005000] +2024-04-03 18:12:06,450 - Train: 3.26% [132600/4070000] [32.6/1000.0] [batch_t 1.425 (1.429)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-03 18:14:29,194 - Train: 3.26% [132700/4070000] [32.6/1000.0] [batch_t 1.408 (1.427)] [data_t 0.003] [optim_t 1.405] [lr 0.005000] +2024-04-03 18:16:51,918 - Train: 3.26% [132800/4070000] [32.6/1000.0] [batch_t 1.444 (1.427)] [data_t 0.002] [optim_t 1.442] [lr 0.005000] +2024-04-03 18:19:14,778 - Train: 3.27% [132900/4070000] [32.7/1000.0] [batch_t 1.430 (1.429)] [data_t 0.002] [optim_t 1.428] [lr 0.005000] +2024-04-03 18:21:37,503 - Train: 3.27% [133000/4070000] [32.7/1000.0] [batch_t 1.436 (1.427)] [data_t 0.003] [optim_t 1.434] [lr 0.005000] +2024-04-03 18:24:00,397 - Train: 3.27% [133100/4070000] [32.7/1000.0] [batch_t 1.425 (1.429)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-03 18:26:23,459 - Train: 3.27% [133200/4070000] [32.7/1000.0] [batch_t 1.417 (1.431)] [data_t 0.003] [optim_t 1.414] [lr 0.005000] +2024-04-03 18:28:46,672 - Train: 3.28% [133300/4070000] [32.8/1000.0] [batch_t 1.446 (1.432)] [data_t 0.003] [optim_t 1.444] [lr 0.005000] +2024-04-03 18:31:09,662 - Train: 3.28% [133400/4070000] [32.8/1000.0] [batch_t 1.415 (1.430)] [data_t 0.003] [optim_t 1.412] [lr 0.005000] +2024-04-03 18:33:32,507 - Train: 3.28% [133500/4070000] [32.8/1000.0] [batch_t 1.422 (1.428)] [data_t 0.002] [optim_t 1.420] [lr 0.005000] +2024-04-03 18:35:55,430 - Train: 3.28% [133600/4070000] [32.8/1000.0] [batch_t 1.434 (1.429)] [data_t 0.002] [optim_t 1.431] [lr 0.005000] +2024-04-03 18:38:18,289 - Train: 3.29% [133700/4070000] [32.9/1000.0] [batch_t 1.432 (1.429)] [data_t 0.002] [optim_t 1.429] [lr 0.005000] +2024-04-03 18:40:41,166 - Train: 3.29% [133800/4070000] [32.9/1000.0] [batch_t 1.417 (1.429)] [data_t 0.003] [optim_t 1.414] [lr 0.005000] +2024-04-03 18:43:04,154 - Train: 3.29% [133900/4070000] [32.9/1000.0] [batch_t 1.430 (1.430)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-03 18:45:27,134 - Train: 3.29% [134000/4070000] [32.9/1000.0] [batch_t 1.441 (1.430)] [data_t 0.003] [optim_t 1.438] [lr 0.005000] +2024-04-03 18:47:50,052 - Train: 3.29% [134100/4070000] [32.9/1000.0] [batch_t 1.457 (1.429)] [data_t 0.002] [optim_t 1.455] [lr 0.005000] +2024-04-03 18:50:12,945 - Train: 3.30% [134200/4070000] [33.0/1000.0] [batch_t 1.418 (1.429)] [data_t 0.002] [optim_t 1.416] [lr 0.005000] +2024-04-03 18:52:35,781 - Train: 3.30% [134300/4070000] [33.0/1000.0] [batch_t 1.437 (1.428)] [data_t 0.003] [optim_t 1.435] [lr 0.005000] +2024-04-03 18:52:50,067 - ==> Total time: 1 day, 17:52:20 Eta: 51 days, 2:59:25 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-03 18:55:00,218 - Train: 3.30% [134400/4070000] [33.0/1000.0] [batch_t 1.430 (1.431)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-03 18:57:23,105 - Train: 3.30% [134500/4070000] [33.0/1000.0] [batch_t 1.425 (1.429)] [data_t 0.002] [optim_t 1.423] [lr 0.005000] +2024-04-03 18:59:45,956 - Train: 3.31% [134600/4070000] [33.1/1000.0] [batch_t 1.426 (1.428)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-03 19:02:09,003 - Train: 3.31% [134700/4070000] [33.1/1000.0] [batch_t 1.436 (1.430)] [data_t 0.003] [optim_t 1.434] [lr 0.005000] +2024-04-03 19:04:31,840 - Train: 3.31% [134800/4070000] [33.1/1000.0] [batch_t 1.422 (1.428)] [data_t 0.003] [optim_t 1.420] [lr 0.005000] +2024-04-03 19:06:54,771 - Train: 3.31% [134900/4070000] [33.1/1000.0] [batch_t 1.435 (1.429)] [data_t 0.002] [optim_t 1.433] [lr 0.005000] +2024-04-03 19:09:17,630 - Train: 3.32% [135000/4070000] [33.2/1000.0] [batch_t 1.426 (1.429)] [data_t 0.002] [optim_t 1.423] [lr 0.005000] +2024-04-03 19:11:40,430 - Train: 3.32% [135100/4070000] [33.2/1000.0] [batch_t 1.429 (1.428)] [data_t 0.002] [optim_t 1.426] [lr 0.005000] +2024-04-03 19:14:03,244 - Train: 3.32% [135200/4070000] [33.2/1000.0] [batch_t 1.422 (1.428)] [data_t 0.003] [optim_t 1.420] [lr 0.005000] +2024-04-03 19:16:26,075 - Train: 3.32% [135300/4070000] [33.2/1000.0] [batch_t 1.437 (1.428)] [data_t 0.003] [optim_t 1.434] [lr 0.005000] +2024-04-03 19:18:48,808 - Train: 3.33% [135400/4070000] [33.3/1000.0] [batch_t 1.429 (1.427)] [data_t 0.002] [optim_t 1.427] [lr 0.005000] +2024-04-03 19:21:11,646 - Train: 3.33% [135500/4070000] [33.3/1000.0] [batch_t 1.434 (1.428)] [data_t 0.002] [optim_t 1.431] [lr 0.005000] +2024-04-03 19:23:34,360 - Train: 3.33% [135600/4070000] [33.3/1000.0] [batch_t 1.419 (1.427)] [data_t 0.003] [optim_t 1.416] [lr 0.005000] +2024-04-03 19:25:57,288 - Train: 3.33% [135700/4070000] [33.3/1000.0] [batch_t 1.422 (1.429)] [data_t 0.003] [optim_t 1.419] [lr 0.005000] +2024-04-03 19:28:20,381 - Train: 3.34% [135800/4070000] [33.4/1000.0] [batch_t 1.434 (1.431)] [data_t 0.003] [optim_t 1.431] [lr 0.005000] +2024-04-03 19:30:43,474 - Train: 3.34% [135900/4070000] [33.4/1000.0] [batch_t 1.446 (1.431)] [data_t 0.002] [optim_t 1.444] [lr 0.005000] +2024-04-03 19:33:06,546 - Train: 3.34% [136000/4070000] [33.4/1000.0] [batch_t 1.434 (1.431)] [data_t 0.003] [optim_t 1.432] [lr 0.005000] +2024-04-03 19:35:29,582 - Train: 3.34% [136100/4070000] [33.4/1000.0] [batch_t 1.438 (1.430)] [data_t 0.002] [optim_t 1.435] [lr 0.005000] +2024-04-03 19:37:52,699 - Train: 3.35% [136200/4070000] [33.5/1000.0] [batch_t 1.422 (1.431)] [data_t 0.003] [optim_t 1.419] [lr 0.005000] +2024-04-03 19:40:15,522 - Train: 3.35% [136300/4070000] [33.5/1000.0] [batch_t 1.430 (1.428)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-03 19:42:38,589 - Train: 3.35% [136400/4070000] [33.5/1000.0] [batch_t 1.432 (1.431)] [data_t 0.002] [optim_t 1.430] [lr 0.005000] +2024-04-03 19:45:01,732 - Train: 3.35% [136500/4070000] [33.5/1000.0] [batch_t 1.430 (1.431)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-03 19:47:24,821 - Train: 3.36% [136600/4070000] [33.6/1000.0] [batch_t 1.398 (1.431)] [data_t 0.003] [optim_t 1.395] [lr 0.005000] +2024-04-03 19:49:47,918 - Train: 3.36% [136700/4070000] [33.6/1000.0] [batch_t 1.431 (1.431)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-03 19:52:11,049 - Train: 3.36% [136800/4070000] [33.6/1000.0] [batch_t 1.440 (1.431)] [data_t 0.003] [optim_t 1.438] [lr 0.005000] +2024-04-03 19:54:34,200 - Train: 3.36% [136900/4070000] [33.6/1000.0] [batch_t 1.433 (1.431)] [data_t 0.002] [optim_t 1.430] [lr 0.005000] +2024-04-03 19:56:57,297 - Train: 3.37% [137000/4070000] [33.7/1000.0] [batch_t 1.434 (1.431)] [data_t 0.003] [optim_t 1.432] [lr 0.005000] +2024-04-03 19:59:20,532 - Train: 3.37% [137100/4070000] [33.7/1000.0] [batch_t 1.426 (1.432)] [data_t 0.002] [optim_t 1.424] [lr 0.005000] +2024-04-03 20:01:43,631 - Train: 3.37% [137200/4070000] [33.7/1000.0] [batch_t 1.438 (1.431)] [data_t 0.002] [optim_t 1.436] [lr 0.005000] +2024-04-03 20:04:06,825 - Train: 3.37% [137300/4070000] [33.7/1000.0] [batch_t 1.451 (1.432)] [data_t 0.002] [optim_t 1.449] [lr 0.005000] +2024-04-03 20:06:30,066 - Train: 3.38% [137400/4070000] [33.8/1000.0] [batch_t 1.424 (1.432)] [data_t 0.003] [optim_t 1.422] [lr 0.005000] +2024-04-03 20:08:53,202 - Train: 3.38% [137500/4070000] [33.8/1000.0] [batch_t 1.446 (1.431)] [data_t 0.003] [optim_t 1.444] [lr 0.005000] +2024-04-03 20:11:16,278 - Train: 3.38% [137600/4070000] [33.8/1000.0] [batch_t 1.429 (1.431)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-03 20:13:39,299 - Train: 3.38% [137700/4070000] [33.8/1000.0] [batch_t 1.441 (1.430)] [data_t 0.003] [optim_t 1.438] [lr 0.005000] +2024-04-03 20:16:02,446 - Train: 3.39% [137800/4070000] [33.9/1000.0] [batch_t 1.423 (1.431)] [data_t 0.003] [optim_t 1.420] [lr 0.005000] +2024-04-03 20:18:25,379 - Train: 3.39% [137900/4070000] [33.9/1000.0] [batch_t 1.405 (1.429)] [data_t 0.003] [optim_t 1.402] [lr 0.005000] +2024-04-03 20:20:48,416 - Train: 3.39% [138000/4070000] [33.9/1000.0] [batch_t 1.424 (1.430)] [data_t 0.002] [optim_t 1.421] [lr 0.005000] +2024-04-03 20:23:11,316 - Train: 3.39% [138100/4070000] [33.9/1000.0] [batch_t 1.430 (1.429)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-03 20:25:34,275 - Train: 3.40% [138200/4070000] [34.0/1000.0] [batch_t 1.429 (1.430)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-03 20:27:57,326 - Train: 3.40% [138300/4070000] [34.0/1000.0] [batch_t 1.417 (1.430)] [data_t 0.003] [optim_t 1.414] [lr 0.005000] +2024-04-03 20:29:51,722 - ==> Total time: 1 day, 19:29:22 Eta: 51 days, 11:36:59 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-03 20:30:21,945 - Train: 3.40% [138400/4070000] [34.0/1000.0] [batch_t 1.435 (1.443)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-03 20:32:45,102 - Train: 3.40% [138500/4070000] [34.0/1000.0] [batch_t 1.423 (1.431)] [data_t 0.002] [optim_t 1.421] [lr 0.005000] +2024-04-03 20:35:08,243 - Train: 3.41% [138600/4070000] [34.1/1000.0] [batch_t 1.440 (1.431)] [data_t 0.003] [optim_t 1.437] [lr 0.005000] +2024-04-03 20:37:31,494 - Train: 3.41% [138700/4070000] [34.1/1000.0] [batch_t 1.435 (1.432)] [data_t 0.002] [optim_t 1.433] [lr 0.005000] +2024-04-03 20:39:54,664 - Train: 3.41% [138800/4070000] [34.1/1000.0] [batch_t 1.425 (1.432)] [data_t 0.003] [optim_t 1.422] [lr 0.005000] +2024-04-03 20:42:17,781 - Train: 3.41% [138900/4070000] [34.1/1000.0] [batch_t 1.431 (1.431)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-03 20:44:40,901 - Train: 3.42% [139000/4070000] [34.2/1000.0] [batch_t 1.435 (1.431)] [data_t 0.003] [optim_t 1.432] [lr 0.005000] +2024-04-03 20:47:04,013 - Train: 3.42% [139100/4070000] [34.2/1000.0] [batch_t 1.442 (1.431)] [data_t 0.002] [optim_t 1.440] [lr 0.005000] +2024-04-03 20:49:27,154 - Train: 3.42% [139200/4070000] [34.2/1000.0] [batch_t 1.442 (1.431)] [data_t 0.002] [optim_t 1.440] [lr 0.005000] +2024-04-03 20:51:50,102 - Train: 3.42% [139300/4070000] [34.2/1000.0] [batch_t 1.419 (1.429)] [data_t 0.002] [optim_t 1.416] [lr 0.005000] +2024-04-03 20:54:13,070 - Train: 3.43% [139400/4070000] [34.3/1000.0] [batch_t 1.426 (1.430)] [data_t 0.002] [optim_t 1.424] [lr 0.005000] +2024-04-03 20:56:36,313 - Train: 3.43% [139500/4070000] [34.3/1000.0] [batch_t 1.424 (1.432)] [data_t 0.002] [optim_t 1.422] [lr 0.005000] +2024-04-03 20:58:59,349 - Train: 3.43% [139600/4070000] [34.3/1000.0] [batch_t 1.441 (1.430)] [data_t 0.002] [optim_t 1.438] [lr 0.005000] +2024-04-03 21:01:22,483 - Train: 3.43% [139700/4070000] [34.3/1000.0] [batch_t 1.426 (1.431)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-03 21:03:45,627 - Train: 3.43% [139800/4070000] [34.3/1000.0] [batch_t 1.449 (1.431)] [data_t 0.003] [optim_t 1.446] [lr 0.005000] +2024-04-03 21:06:08,619 - Train: 3.44% [139900/4070000] [34.4/1000.0] [batch_t 1.437 (1.430)] [data_t 0.003] [optim_t 1.435] [lr 0.005000] +2024-04-03 21:08:31,773 - Train: 3.44% [140000/4070000] [34.4/1000.0] [batch_t 1.421 (1.431)] [data_t 0.002] [optim_t 1.419] [lr 0.005000] +2024-04-03 21:10:54,885 - Train: 3.44% [140100/4070000] [34.4/1000.0] [batch_t 1.405 (1.431)] [data_t 0.002] [optim_t 1.403] [lr 0.005000] +2024-04-03 21:13:18,024 - Train: 3.44% [140200/4070000] [34.4/1000.0] [batch_t 1.416 (1.431)] [data_t 0.002] [optim_t 1.414] [lr 0.005000] +2024-04-03 21:15:41,082 - Train: 3.45% [140300/4070000] [34.5/1000.0] [batch_t 1.425 (1.430)] [data_t 0.003] [optim_t 1.422] [lr 0.005000] +2024-04-03 21:18:04,109 - Train: 3.45% [140400/4070000] [34.5/1000.0] [batch_t 1.406 (1.430)] [data_t 0.002] [optim_t 1.403] [lr 0.005000] +2024-04-03 21:20:27,228 - Train: 3.45% [140500/4070000] [34.5/1000.0] [batch_t 1.432 (1.431)] [data_t 0.003] [optim_t 1.429] [lr 0.005000] +2024-04-03 21:22:50,120 - Train: 3.45% [140600/4070000] [34.5/1000.0] [batch_t 1.412 (1.429)] [data_t 0.003] [optim_t 1.409] [lr 0.005000] +2024-04-03 21:25:13,163 - Train: 3.46% [140700/4070000] [34.6/1000.0] [batch_t 1.414 (1.430)] [data_t 0.002] [optim_t 1.412] [lr 0.005000] +2024-04-03 21:27:36,397 - Train: 3.46% [140800/4070000] [34.6/1000.0] [batch_t 1.447 (1.432)] [data_t 0.003] [optim_t 1.444] [lr 0.005000] +2024-04-03 21:29:59,652 - Train: 3.46% [140900/4070000] [34.6/1000.0] [batch_t 1.448 (1.432)] [data_t 0.003] [optim_t 1.444] [lr 0.005000] +2024-04-03 21:32:22,925 - Train: 3.46% [141000/4070000] [34.6/1000.0] [batch_t 1.426 (1.433)] [data_t 0.002] [optim_t 1.424] [lr 0.005000] +2024-04-03 21:34:46,131 - Train: 3.47% [141100/4070000] [34.7/1000.0] [batch_t 1.422 (1.432)] [data_t 0.002] [optim_t 1.420] [lr 0.005000] +2024-04-03 21:37:09,260 - Train: 3.47% [141200/4070000] [34.7/1000.0] [batch_t 1.426 (1.431)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-03 21:39:32,345 - Train: 3.47% [141300/4070000] [34.7/1000.0] [batch_t 1.432 (1.431)] [data_t 0.003] [optim_t 1.429] [lr 0.005000] +2024-04-03 21:41:55,324 - Train: 3.47% [141400/4070000] [34.7/1000.0] [batch_t 1.429 (1.430)] [data_t 0.002] [optim_t 1.426] [lr 0.005000] +2024-04-03 21:44:18,317 - Train: 3.48% [141500/4070000] [34.8/1000.0] [batch_t 1.433 (1.430)] [data_t 0.002] [optim_t 1.430] [lr 0.005000] +2024-04-03 21:46:41,371 - Train: 3.48% [141600/4070000] [34.8/1000.0] [batch_t 1.422 (1.430)] [data_t 0.002] [optim_t 1.419] [lr 0.005000] +2024-04-03 21:49:04,447 - Train: 3.48% [141700/4070000] [34.8/1000.0] [batch_t 1.441 (1.431)] [data_t 0.003] [optim_t 1.438] [lr 0.005000] +2024-04-03 21:51:27,507 - Train: 3.48% [141800/4070000] [34.8/1000.0] [batch_t 1.432 (1.430)] [data_t 0.003] [optim_t 1.430] [lr 0.005000] +2024-04-03 21:53:50,522 - Train: 3.49% [141900/4070000] [34.9/1000.0] [batch_t 1.426 (1.430)] [data_t 0.002] [optim_t 1.424] [lr 0.005000] +2024-04-03 21:56:13,565 - Train: 3.49% [142000/4070000] [34.9/1000.0] [batch_t 1.412 (1.430)] [data_t 0.002] [optim_t 1.409] [lr 0.005000] +2024-04-03 21:58:36,720 - Train: 3.49% [142100/4070000] [34.9/1000.0] [batch_t 1.439 (1.431)] [data_t 0.002] [optim_t 1.437] [lr 0.005000] +2024-04-03 22:00:59,868 - Train: 3.49% [142200/4070000] [34.9/1000.0] [batch_t 1.428 (1.431)] [data_t 0.003] [optim_t 1.425] [lr 0.005000] +2024-04-03 22:03:22,930 - Train: 3.50% [142300/4070000] [35.0/1000.0] [batch_t 1.433 (1.431)] [data_t 0.003] [optim_t 1.430] [lr 0.005000] +2024-04-03 22:05:46,072 - Train: 3.50% [142400/4070000] [35.0/1000.0] [batch_t 1.431 (1.431)] [data_t 0.003] [optim_t 1.429] [lr 0.005000] +2024-04-03 22:06:57,624 - ==> Total time: 1 day, 21:06:28 Eta: 51 days, 19:41:22 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-03 22:08:10,945 - Train: 3.50% [142500/4070000] [35.0/1000.0] [batch_t 1.421 (1.438)] [data_t 0.002] [optim_t 1.419] [lr 0.005000] +2024-04-03 22:10:34,114 - Train: 3.50% [142600/4070000] [35.0/1000.0] [batch_t 1.436 (1.432)] [data_t 0.003] [optim_t 1.434] [lr 0.005000] +2024-04-03 22:12:57,147 - Train: 3.51% [142700/4070000] [35.1/1000.0] [batch_t 1.430 (1.430)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-03 22:15:20,469 - Train: 3.51% [142800/4070000] [35.1/1000.0] [batch_t 1.441 (1.433)] [data_t 0.003] [optim_t 1.438] [lr 0.005000] +2024-04-03 22:17:43,494 - Train: 3.51% [142900/4070000] [35.1/1000.0] [batch_t 1.441 (1.430)] [data_t 0.003] [optim_t 1.438] [lr 0.005000] +2024-04-03 22:20:06,765 - Train: 3.51% [143000/4070000] [35.1/1000.0] [batch_t 1.432 (1.433)] [data_t 0.003] [optim_t 1.430] [lr 0.005000] +2024-04-03 22:22:30,052 - Train: 3.52% [143100/4070000] [35.2/1000.0] [batch_t 1.435 (1.433)] [data_t 0.002] [optim_t 1.433] [lr 0.005000] +2024-04-03 22:24:53,072 - Train: 3.52% [143200/4070000] [35.2/1000.0] [batch_t 1.426 (1.430)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-03 22:27:15,869 - Train: 3.52% [143300/4070000] [35.2/1000.0] [batch_t 1.436 (1.428)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-03 22:29:38,914 - Train: 3.52% [143400/4070000] [35.2/1000.0] [batch_t 1.425 (1.430)] [data_t 0.003] [optim_t 1.422] [lr 0.005000] +2024-04-03 22:32:01,995 - Train: 3.53% [143500/4070000] [35.3/1000.0] [batch_t 1.421 (1.431)] [data_t 0.003] [optim_t 1.418] [lr 0.005000] +2024-04-03 22:34:25,239 - Train: 3.53% [143600/4070000] [35.3/1000.0] [batch_t 1.435 (1.432)] [data_t 0.002] [optim_t 1.433] [lr 0.005000] +2024-04-03 22:36:48,462 - Train: 3.53% [143700/4070000] [35.3/1000.0] [batch_t 1.432 (1.432)] [data_t 0.003] [optim_t 1.429] [lr 0.005000] +2024-04-03 22:39:11,612 - Train: 3.53% [143800/4070000] [35.3/1000.0] [batch_t 1.426 (1.431)] [data_t 0.003] [optim_t 1.424] [lr 0.005000] +2024-04-03 22:41:34,748 - Train: 3.54% [143900/4070000] [35.4/1000.0] [batch_t 1.436 (1.431)] [data_t 0.002] [optim_t 1.434] [lr 0.005000] +2024-04-03 22:43:57,977 - Train: 3.54% [144000/4070000] [35.4/1000.0] [batch_t 1.426 (1.432)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-03 22:46:21,056 - Train: 3.54% [144100/4070000] [35.4/1000.0] [batch_t 1.439 (1.431)] [data_t 0.002] [optim_t 1.437] [lr 0.005000] +2024-04-03 22:48:44,329 - Train: 3.54% [144200/4070000] [35.4/1000.0] [batch_t 1.435 (1.433)] [data_t 0.002] [optim_t 1.433] [lr 0.005000] +2024-04-03 22:51:07,645 - Train: 3.55% [144300/4070000] [35.5/1000.0] [batch_t 1.438 (1.433)] [data_t 0.002] [optim_t 1.435] [lr 0.005000] +2024-04-03 22:53:30,879 - Train: 3.55% [144400/4070000] [35.5/1000.0] [batch_t 1.428 (1.432)] [data_t 0.003] [optim_t 1.425] [lr 0.005000] +2024-04-03 22:55:54,074 - Train: 3.55% [144500/4070000] [35.5/1000.0] [batch_t 1.425 (1.432)] [data_t 0.003] [optim_t 1.422] [lr 0.005000] +2024-04-03 22:58:17,447 - Train: 3.55% [144600/4070000] [35.5/1000.0] [batch_t 1.432 (1.434)] [data_t 0.003] [optim_t 1.429] [lr 0.005000] +2024-04-03 23:00:40,628 - Train: 3.56% [144700/4070000] [35.6/1000.0] [batch_t 1.428 (1.432)] [data_t 0.003] [optim_t 1.425] [lr 0.005000] +2024-04-03 23:03:03,810 - Train: 3.56% [144800/4070000] [35.6/1000.0] [batch_t 1.446 (1.432)] [data_t 0.002] [optim_t 1.444] [lr 0.005000] +2024-04-03 23:05:27,038 - Train: 3.56% [144900/4070000] [35.6/1000.0] [batch_t 1.436 (1.432)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-03 23:07:50,156 - Train: 3.56% [145000/4070000] [35.6/1000.0] [batch_t 1.439 (1.431)] [data_t 0.003] [optim_t 1.436] [lr 0.005000] +2024-04-03 23:10:13,382 - Train: 3.57% [145100/4070000] [35.7/1000.0] [batch_t 1.431 (1.432)] [data_t 0.002] [optim_t 1.428] [lr 0.005000] +2024-04-03 23:12:36,467 - Train: 3.57% [145200/4070000] [35.7/1000.0] [batch_t 1.435 (1.431)] [data_t 0.003] [optim_t 1.432] [lr 0.005000] +2024-04-03 23:14:59,527 - Train: 3.57% [145300/4070000] [35.7/1000.0] [batch_t 1.448 (1.431)] [data_t 0.002] [optim_t 1.445] [lr 0.005000] +2024-04-03 23:17:22,696 - Train: 3.57% [145400/4070000] [35.7/1000.0] [batch_t 1.427 (1.432)] [data_t 0.002] [optim_t 1.425] [lr 0.005000] +2024-04-03 23:19:45,603 - Train: 3.57% [145500/4070000] [35.7/1000.0] [batch_t 1.422 (1.429)] [data_t 0.003] [optim_t 1.419] [lr 0.005000] +2024-04-03 23:22:08,614 - Train: 3.58% [145600/4070000] [35.8/1000.0] [batch_t 1.435 (1.430)] [data_t 0.002] [optim_t 1.433] [lr 0.005000] +2024-04-03 23:24:31,927 - Train: 3.58% [145700/4070000] [35.8/1000.0] [batch_t 1.447 (1.433)] [data_t 0.002] [optim_t 1.445] [lr 0.005000] +2024-04-03 23:26:55,028 - Train: 3.58% [145800/4070000] [35.8/1000.0] [batch_t 1.432 (1.431)] [data_t 0.003] [optim_t 1.429] [lr 0.005000] +2024-04-03 23:29:18,196 - Train: 3.58% [145900/4070000] [35.8/1000.0] [batch_t 1.436 (1.432)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-03 23:31:41,418 - Train: 3.59% [146000/4070000] [35.9/1000.0] [batch_t 1.432 (1.432)] [data_t 0.003] [optim_t 1.429] [lr 0.005000] +2024-04-03 23:34:04,528 - Train: 3.59% [146100/4070000] [35.9/1000.0] [batch_t 1.435 (1.431)] [data_t 0.002] [optim_t 1.433] [lr 0.005000] +2024-04-03 23:36:27,650 - Train: 3.59% [146200/4070000] [35.9/1000.0] [batch_t 1.426 (1.431)] [data_t 0.003] [optim_t 1.424] [lr 0.005000] +2024-04-03 23:38:50,771 - Train: 3.59% [146300/4070000] [35.9/1000.0] [batch_t 1.429 (1.431)] [data_t 0.002] [optim_t 1.427] [lr 0.005000] +2024-04-03 23:41:13,796 - Train: 3.60% [146400/4070000] [36.0/1000.0] [batch_t 1.451 (1.430)] [data_t 0.002] [optim_t 1.449] [lr 0.005000] +2024-04-03 23:43:36,919 - Train: 3.60% [146500/4070000] [36.0/1000.0] [batch_t 1.427 (1.431)] [data_t 0.002] [optim_t 1.425] [lr 0.005000] +2024-04-03 23:44:05,541 - ==> Total time: 1 day, 22:43:36 Eta: 52 days, 3:14:21 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-03 23:46:01,945 - Train: 3.60% [146600/4070000] [36.0/1000.0] [batch_t 1.437 (1.434)] [data_t 0.002] [optim_t 1.434] [lr 0.005000] +2024-04-03 23:48:24,960 - Train: 3.60% [146700/4070000] [36.0/1000.0] [batch_t 1.437 (1.430)] [data_t 0.002] [optim_t 1.434] [lr 0.005000] +2024-04-03 23:50:48,091 - Train: 3.61% [146800/4070000] [36.1/1000.0] [batch_t 1.404 (1.431)] [data_t 0.003] [optim_t 1.401] [lr 0.005000] +2024-04-03 23:53:11,164 - Train: 3.61% [146900/4070000] [36.1/1000.0] [batch_t 1.436 (1.431)] [data_t 0.002] [optim_t 1.433] [lr 0.005000] +2024-04-03 23:55:34,408 - Train: 3.61% [147000/4070000] [36.1/1000.0] [batch_t 1.441 (1.432)] [data_t 0.003] [optim_t 1.439] [lr 0.005000] +2024-04-03 23:57:57,569 - Train: 3.61% [147100/4070000] [36.1/1000.0] [batch_t 1.440 (1.432)] [data_t 0.002] [optim_t 1.438] [lr 0.005000] +2024-04-04 00:00:20,585 - Train: 3.62% [147200/4070000] [36.2/1000.0] [batch_t 1.432 (1.430)] [data_t 0.003] [optim_t 1.429] [lr 0.005000] +2024-04-04 00:02:43,640 - Train: 3.62% [147300/4070000] [36.2/1000.0] [batch_t 1.436 (1.430)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-04 00:05:06,605 - Train: 3.62% [147400/4070000] [36.2/1000.0] [batch_t 1.430 (1.430)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-04 00:07:29,557 - Train: 3.62% [147500/4070000] [36.2/1000.0] [batch_t 1.440 (1.429)] [data_t 0.003] [optim_t 1.437] [lr 0.005000] +2024-04-04 00:09:52,538 - Train: 3.63% [147600/4070000] [36.3/1000.0] [batch_t 1.427 (1.430)] [data_t 0.003] [optim_t 1.424] [lr 0.005000] +2024-04-04 00:12:15,510 - Train: 3.63% [147700/4070000] [36.3/1000.0] [batch_t 1.426 (1.430)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-04 00:14:38,625 - Train: 3.63% [147800/4070000] [36.3/1000.0] [batch_t 1.422 (1.431)] [data_t 0.003] [optim_t 1.419] [lr 0.005000] +2024-04-04 00:17:01,695 - Train: 3.63% [147900/4070000] [36.3/1000.0] [batch_t 1.426 (1.431)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-04 00:19:24,781 - Train: 3.64% [148000/4070000] [36.4/1000.0] [batch_t 1.425 (1.431)] [data_t 0.002] [optim_t 1.422] [lr 0.005000] +2024-04-04 00:21:47,848 - Train: 3.64% [148100/4070000] [36.4/1000.0] [batch_t 1.429 (1.431)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-04 00:24:10,929 - Train: 3.64% [148200/4070000] [36.4/1000.0] [batch_t 1.438 (1.431)] [data_t 0.002] [optim_t 1.436] [lr 0.005000] +2024-04-04 00:26:33,930 - Train: 3.64% [148300/4070000] [36.4/1000.0] [batch_t 1.432 (1.430)] [data_t 0.002] [optim_t 1.430] [lr 0.005000] +2024-04-04 00:28:57,061 - Train: 3.65% [148400/4070000] [36.5/1000.0] [batch_t 1.429 (1.431)] [data_t 0.003] [optim_t 1.426] [lr 0.005000] +2024-04-04 00:31:20,375 - Train: 3.65% [148500/4070000] [36.5/1000.0] [batch_t 1.433 (1.433)] [data_t 0.002] [optim_t 1.430] [lr 0.005000] +2024-04-04 00:33:43,450 - Train: 3.65% [148600/4070000] [36.5/1000.0] [batch_t 1.428 (1.431)] [data_t 0.003] [optim_t 1.425] [lr 0.005000] +2024-04-04 00:36:06,442 - Train: 3.65% [148700/4070000] [36.5/1000.0] [batch_t 1.450 (1.430)] [data_t 0.002] [optim_t 1.448] [lr 0.005000] +2024-04-04 00:38:29,447 - Train: 3.66% [148800/4070000] [36.6/1000.0] [batch_t 1.437 (1.430)] [data_t 0.002] [optim_t 1.435] [lr 0.005000] +2024-04-04 00:40:52,527 - Train: 3.66% [148900/4070000] [36.6/1000.0] [batch_t 1.445 (1.431)] [data_t 0.003] [optim_t 1.442] [lr 0.005000] +2024-04-04 00:43:15,421 - Train: 3.66% [149000/4070000] [36.6/1000.0] [batch_t 1.429 (1.429)] [data_t 0.003] [optim_t 1.426] [lr 0.005000] +2024-04-04 00:45:38,596 - Train: 3.66% [149100/4070000] [36.6/1000.0] [batch_t 1.427 (1.432)] [data_t 0.003] [optim_t 1.424] [lr 0.005000] +2024-04-04 00:48:01,757 - Train: 3.67% [149200/4070000] [36.7/1000.0] [batch_t 1.441 (1.432)] [data_t 0.002] [optim_t 1.439] [lr 0.005000] +2024-04-04 00:50:24,745 - Train: 3.67% [149300/4070000] [36.7/1000.0] [batch_t 1.427 (1.430)] [data_t 0.002] [optim_t 1.424] [lr 0.005000] +2024-04-04 00:52:47,750 - Train: 3.67% [149400/4070000] [36.7/1000.0] [batch_t 1.433 (1.430)] [data_t 0.003] [optim_t 1.430] [lr 0.005000] +2024-04-04 00:55:11,029 - Train: 3.67% [149500/4070000] [36.7/1000.0] [batch_t 1.431 (1.433)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-04 00:57:34,143 - Train: 3.68% [149600/4070000] [36.8/1000.0] [batch_t 1.425 (1.431)] [data_t 0.003] [optim_t 1.422] [lr 0.005000] +2024-04-04 00:59:57,462 - Train: 3.68% [149700/4070000] [36.8/1000.0] [batch_t 1.445 (1.433)] [data_t 0.002] [optim_t 1.443] [lr 0.005000] +2024-04-04 01:02:20,584 - Train: 3.68% [149800/4070000] [36.8/1000.0] [batch_t 1.451 (1.431)] [data_t 0.002] [optim_t 1.449] [lr 0.005000] +2024-04-04 01:04:43,669 - Train: 3.68% [149900/4070000] [36.8/1000.0] [batch_t 1.431 (1.431)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-04 01:07:06,623 - Train: 3.69% [150000/4070000] [36.9/1000.0] [batch_t 1.423 (1.429)] [data_t 0.003] [optim_t 1.420] [lr 0.005000] +2024-04-04 01:09:29,592 - Train: 3.69% [150100/4070000] [36.9/1000.0] [batch_t 1.433 (1.430)] [data_t 0.002] [optim_t 1.430] [lr 0.005000] +2024-04-04 01:11:52,730 - Train: 3.69% [150200/4070000] [36.9/1000.0] [batch_t 1.440 (1.431)] [data_t 0.003] [optim_t 1.438] [lr 0.005000] +2024-04-04 01:14:15,856 - Train: 3.69% [150300/4070000] [36.9/1000.0] [batch_t 1.422 (1.431)] [data_t 0.003] [optim_t 1.419] [lr 0.005000] +2024-04-04 01:16:38,868 - Train: 3.70% [150400/4070000] [37.0/1000.0] [batch_t 1.437 (1.430)] [data_t 0.002] [optim_t 1.434] [lr 0.005000] +2024-04-04 01:19:02,052 - Train: 3.70% [150500/4070000] [37.0/1000.0] [batch_t 1.430 (1.432)] [data_t 0.002] [optim_t 1.427] [lr 0.005000] +2024-04-04 01:21:11,118 - ==> Total time: 2 days, 0:20:41 Eta: 52 days, 10:16:35 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-04 01:21:27,240 - Train: 3.70% [150600/4070000] [37.0/1000.0] [batch_t 1.425 (1.447)] [data_t 0.003] [optim_t 1.422] [lr 0.005000] +2024-04-04 01:23:50,429 - Train: 3.70% [150700/4070000] [37.0/1000.0] [batch_t 1.445 (1.432)] [data_t 0.003] [optim_t 1.442] [lr 0.005000] +2024-04-04 01:26:13,482 - Train: 3.71% [150800/4070000] [37.1/1000.0] [batch_t 1.449 (1.430)] [data_t 0.002] [optim_t 1.446] [lr 0.005000] +2024-04-04 01:28:36,765 - Train: 3.71% [150900/4070000] [37.1/1000.0] [batch_t 1.444 (1.433)] [data_t 0.002] [optim_t 1.442] [lr 0.005000] +2024-04-04 01:30:59,765 - Train: 3.71% [151000/4070000] [37.1/1000.0] [batch_t 1.446 (1.430)] [data_t 0.003] [optim_t 1.443] [lr 0.005000] +2024-04-04 01:33:22,790 - Train: 3.71% [151100/4070000] [37.1/1000.0] [batch_t 1.408 (1.430)] [data_t 0.002] [optim_t 1.406] [lr 0.005000] +2024-04-04 01:35:45,918 - Train: 3.71% [151200/4070000] [37.1/1000.0] [batch_t 1.432 (1.431)] [data_t 0.003] [optim_t 1.429] [lr 0.005000] +2024-04-04 01:38:08,781 - Train: 3.72% [151300/4070000] [37.2/1000.0] [batch_t 1.439 (1.429)] [data_t 0.002] [optim_t 1.436] [lr 0.005000] +2024-04-04 01:40:31,780 - Train: 3.72% [151400/4070000] [37.2/1000.0] [batch_t 1.429 (1.430)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-04 01:42:55,044 - Train: 3.72% [151500/4070000] [37.2/1000.0] [batch_t 1.419 (1.433)] [data_t 0.003] [optim_t 1.416] [lr 0.005000] +2024-04-04 01:45:17,972 - Train: 3.72% [151600/4070000] [37.2/1000.0] [batch_t 1.430 (1.429)] [data_t 0.002] [optim_t 1.428] [lr 0.005000] +2024-04-04 01:47:41,123 - Train: 3.73% [151700/4070000] [37.3/1000.0] [batch_t 1.433 (1.431)] [data_t 0.002] [optim_t 1.431] [lr 0.005000] +2024-04-04 01:50:04,358 - Train: 3.73% [151800/4070000] [37.3/1000.0] [batch_t 1.430 (1.432)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-04 01:52:27,604 - Train: 3.73% [151900/4070000] [37.3/1000.0] [batch_t 1.446 (1.432)] [data_t 0.003] [optim_t 1.444] [lr 0.005000] +2024-04-04 01:54:50,802 - Train: 3.73% [152000/4070000] [37.3/1000.0] [batch_t 1.428 (1.432)] [data_t 0.002] [optim_t 1.426] [lr 0.005000] +2024-04-04 01:57:13,968 - Train: 3.74% [152100/4070000] [37.4/1000.0] [batch_t 1.424 (1.432)] [data_t 0.003] [optim_t 1.422] [lr 0.005000] +2024-04-04 01:59:37,389 - Train: 3.74% [152200/4070000] [37.4/1000.0] [batch_t 1.429 (1.434)] [data_t 0.002] [optim_t 1.427] [lr 0.005000] +2024-04-04 02:02:00,582 - Train: 3.74% [152300/4070000] [37.4/1000.0] [batch_t 1.425 (1.432)] [data_t 0.003] [optim_t 1.422] [lr 0.005000] +2024-04-04 02:04:23,749 - Train: 3.74% [152400/4070000] [37.4/1000.0] [batch_t 1.431 (1.432)] [data_t 0.002] [optim_t 1.429] [lr 0.005000] +2024-04-04 02:06:46,875 - Train: 3.75% [152500/4070000] [37.5/1000.0] [batch_t 1.436 (1.431)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-04 02:08:54,222 - Train: 3.75% [152600/4070000] [37.5/1000.0] [batch_t 0.606 (1.273)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 02:09:54,713 - Train: 3.75% [152700/4070000] [37.5/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-04 02:10:55,231 - Train: 3.75% [152800/4070000] [37.5/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 02:11:55,719 - Train: 3.76% [152900/4070000] [37.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 02:12:56,195 - Train: 3.76% [153000/4070000] [37.6/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 02:13:56,759 - Train: 3.76% [153100/4070000] [37.6/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 02:14:57,256 - Train: 3.76% [153200/4070000] [37.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 02:15:57,754 - Train: 3.77% [153300/4070000] [37.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 02:16:58,262 - Train: 3.77% [153400/4070000] [37.7/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 02:17:58,696 - Train: 3.77% [153500/4070000] [37.7/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 02:18:59,175 - Train: 3.77% [153600/4070000] [37.7/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 02:19:59,692 - Train: 3.78% [153700/4070000] [37.8/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 02:21:00,187 - Train: 3.78% [153800/4070000] [37.8/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 02:22:00,739 - Train: 3.78% [153900/4070000] [37.8/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 02:23:01,207 - Train: 3.78% [154000/4070000] [37.8/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 02:24:01,669 - Train: 3.79% [154100/4070000] [37.9/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 02:25:02,160 - Train: 3.79% [154200/4070000] [37.9/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 02:26:02,650 - Train: 3.79% [154300/4070000] [37.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 02:27:03,253 - Train: 3.79% [154400/4070000] [37.9/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-04 02:28:03,769 - Train: 3.80% [154500/4070000] [38.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 02:29:04,270 - Train: 3.80% [154600/4070000] [38.0/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-04 02:29:40,551 - ==> Total time: 2 days, 1:29:11 Eta: 52 days, 4:47:23 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-04 02:30:06,612 - Train: 3.80% [154700/4070000] [38.0/1000.0] [batch_t 0.606 (0.613)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 02:31:07,121 - Train: 3.80% [154800/4070000] [38.0/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 02:32:07,631 - Train: 3.81% [154900/4070000] [38.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 02:33:08,154 - Train: 3.81% [155000/4070000] [38.1/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 02:34:08,766 - Train: 3.81% [155100/4070000] [38.1/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 02:35:09,245 - Train: 3.81% [155200/4070000] [38.1/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 02:36:09,739 - Train: 3.82% [155300/4070000] [38.2/1000.0] [batch_t 0.600 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-04 02:37:10,262 - Train: 3.82% [155400/4070000] [38.2/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 02:38:10,779 - Train: 3.82% [155500/4070000] [38.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 02:39:11,284 - Train: 3.82% [155600/4070000] [38.2/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 02:40:11,959 - Train: 3.83% [155700/4070000] [38.3/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 02:41:12,480 - Train: 3.83% [155800/4070000] [38.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 02:42:12,986 - Train: 3.83% [155900/4070000] [38.3/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 02:43:13,454 - Train: 3.83% [156000/4070000] [38.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 02:44:13,927 - Train: 3.84% [156100/4070000] [38.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 02:45:14,450 - Train: 3.84% [156200/4070000] [38.4/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 02:46:14,948 - Train: 3.84% [156300/4070000] [38.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 02:47:15,575 - Train: 3.84% [156400/4070000] [38.4/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-04 02:48:16,085 - Train: 3.85% [156500/4070000] [38.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 02:49:16,579 - Train: 3.85% [156600/4070000] [38.5/1000.0] [batch_t 0.611 (0.605)] [data_t 0.002] [optim_t 0.609] [lr 0.005000] +2024-04-04 02:50:17,117 - Train: 3.85% [156700/4070000] [38.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 02:51:17,630 - Train: 3.85% [156800/4070000] [38.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 02:52:18,291 - Train: 3.86% [156900/4070000] [38.6/1000.0] [batch_t 0.603 (0.607)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 02:53:18,842 - Train: 3.86% [157000/4070000] [38.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 02:54:19,375 - Train: 3.86% [157100/4070000] [38.6/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 02:55:19,846 - Train: 3.86% [157200/4070000] [38.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 02:56:20,343 - Train: 3.86% [157300/4070000] [38.6/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 02:57:20,851 - Train: 3.87% [157400/4070000] [38.7/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 02:58:21,352 - Train: 3.87% [157500/4070000] [38.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 02:59:21,831 - Train: 3.87% [157600/4070000] [38.7/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 03:00:22,491 - Train: 3.87% [157700/4070000] [38.7/1000.0] [batch_t 0.601 (0.607)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 03:01:22,990 - Train: 3.88% [157800/4070000] [38.8/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 03:02:23,493 - Train: 3.88% [157900/4070000] [38.8/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 03:03:24,067 - Train: 3.88% [158000/4070000] [38.8/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 03:04:24,603 - Train: 3.88% [158100/4070000] [38.8/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 03:05:25,223 - Train: 3.89% [158200/4070000] [38.9/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 03:06:25,756 - Train: 3.89% [158300/4070000] [38.9/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 03:07:26,235 - Train: 3.89% [158400/4070000] [38.9/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 03:08:26,714 - Train: 3.89% [158500/4070000] [38.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 03:09:27,216 - Train: 3.90% [158600/4070000] [39.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 03:10:27,689 - Train: 3.90% [158700/4070000] [39.0/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-04 03:10:45,846 - ==> Total time: 2 days, 2:10:16 Eta: 51 days, 12:16:20 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-04 03:11:29,766 - Train: 3.90% [158800/4070000] [39.0/1000.0] [batch_t 0.606 (0.608)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 03:12:30,383 - Train: 3.90% [158900/4070000] [39.0/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 03:13:30,869 - Train: 3.91% [159000/4070000] [39.1/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 03:14:31,348 - Train: 3.91% [159100/4070000] [39.1/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 03:15:31,837 - Train: 3.91% [159200/4070000] [39.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 03:16:32,292 - Train: 3.91% [159300/4070000] [39.1/1000.0] [batch_t 0.608 (0.604)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-04 03:17:32,760 - Train: 3.92% [159400/4070000] [39.2/1000.0] [batch_t 0.599 (0.605)] [data_t 0.002] [optim_t 0.597] [lr 0.005000] +2024-04-04 03:18:33,373 - Train: 3.92% [159500/4070000] [39.2/1000.0] [batch_t 0.602 (0.606)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 03:19:33,870 - Train: 3.92% [159600/4070000] [39.2/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-04 03:20:34,299 - Train: 3.92% [159700/4070000] [39.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 03:21:34,774 - Train: 3.93% [159800/4070000] [39.3/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 03:22:35,273 - Train: 3.93% [159900/4070000] [39.3/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 03:23:35,769 - Train: 3.93% [160000/4070000] [39.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 03:24:36,237 - Train: 3.93% [160100/4070000] [39.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 03:25:36,784 - Train: 3.94% [160200/4070000] [39.4/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 03:26:37,237 - Train: 3.94% [160300/4070000] [39.4/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 03:27:37,698 - Train: 3.94% [160400/4070000] [39.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 03:28:38,153 - Train: 3.94% [160500/4070000] [39.4/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 03:29:38,650 - Train: 3.95% [160600/4070000] [39.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 03:30:39,250 - Train: 3.95% [160700/4070000] [39.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 03:31:39,760 - Train: 3.95% [160800/4070000] [39.5/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 03:32:40,257 - Train: 3.95% [160900/4070000] [39.5/1000.0] [batch_t 0.610 (0.605)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-04 03:33:40,751 - Train: 3.96% [161000/4070000] [39.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 03:34:41,257 - Train: 3.96% [161100/4070000] [39.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 03:35:41,754 - Train: 3.96% [161200/4070000] [39.6/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 03:36:42,249 - Train: 3.96% [161300/4070000] [39.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 03:37:42,744 - Train: 3.97% [161400/4070000] [39.7/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-04 03:38:43,295 - Train: 3.97% [161500/4070000] [39.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 03:39:43,818 - Train: 3.97% [161600/4070000] [39.7/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 03:40:44,310 - Train: 3.97% [161700/4070000] [39.7/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 03:41:44,750 - Train: 3.98% [161800/4070000] [39.8/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-04 03:42:45,211 - Train: 3.98% [161900/4070000] [39.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 03:43:45,759 - Train: 3.98% [162000/4070000] [39.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 03:44:46,273 - Train: 3.98% [162100/4070000] [39.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 03:45:46,812 - Train: 3.99% [162200/4070000] [39.9/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-04 03:46:47,304 - Train: 3.99% [162300/4070000] [39.9/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 03:47:47,820 - Train: 3.99% [162400/4070000] [39.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 03:48:48,311 - Train: 3.99% [162500/4070000] [39.9/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 03:49:48,809 - Train: 4.00% [162600/4070000] [40.0/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 03:50:49,351 - Train: 4.00% [162700/4070000] [40.0/1000.0] [batch_t 0.610 (0.605)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-04 03:51:49,971 - Train: 4.00% [162800/4070000] [40.0/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 03:51:49,977 - ==> Total time: 2 days, 2:51:20 Eta: 50 days, 20:32:20 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-04 03:52:52,118 - Train: 4.00% [162900/4070000] [40.0/1000.0] [batch_t 0.603 (0.607)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 03:53:52,566 - Train: 4.00% [163000/4070000] [40.0/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 03:54:53,037 - Train: 4.01% [163100/4070000] [40.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 03:55:53,536 - Train: 4.01% [163200/4070000] [40.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 03:56:54,176 - Train: 4.01% [163300/4070000] [40.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 03:57:54,677 - Train: 4.01% [163400/4070000] [40.1/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 03:58:55,231 - Train: 4.02% [163500/4070000] [40.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 03:59:55,690 - Train: 4.02% [163600/4070000] [40.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 04:00:56,178 - Train: 4.02% [163700/4070000] [40.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 04:01:56,653 - Train: 4.02% [163800/4070000] [40.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 04:02:57,126 - Train: 4.03% [163900/4070000] [40.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 04:03:57,683 - Train: 4.03% [164000/4070000] [40.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 04:04:58,206 - Train: 4.03% [164100/4070000] [40.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 04:05:58,692 - Train: 4.03% [164200/4070000] [40.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 04:06:59,165 - Train: 4.04% [164300/4070000] [40.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 04:07:59,632 - Train: 4.04% [164400/4070000] [40.4/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 04:09:00,085 - Train: 4.04% [164500/4070000] [40.4/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 04:10:00,674 - Train: 4.04% [164600/4070000] [40.4/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 04:11:01,142 - Train: 4.05% [164700/4070000] [40.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 04:12:01,602 - Train: 4.05% [164800/4070000] [40.5/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 04:13:02,118 - Train: 4.05% [164900/4070000] [40.5/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 04:14:02,572 - Train: 4.05% [165000/4070000] [40.5/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 04:15:03,043 - Train: 4.06% [165100/4070000] [40.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 04:16:03,497 - Train: 4.06% [165200/4070000] [40.6/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 04:17:04,021 - Train: 4.06% [165300/4070000] [40.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 04:18:04,482 - Train: 4.06% [165400/4070000] [40.6/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-04 04:19:04,959 - Train: 4.07% [165500/4070000] [40.7/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 04:20:05,430 - Train: 4.07% [165600/4070000] [40.7/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 04:21:05,887 - Train: 4.07% [165700/4070000] [40.7/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 04:22:06,366 - Train: 4.07% [165800/4070000] [40.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 04:23:06,930 - Train: 4.08% [165900/4070000] [40.8/1000.0] [batch_t 0.602 (0.606)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 04:24:07,381 - Train: 4.08% [166000/4070000] [40.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 04:25:07,911 - Train: 4.08% [166100/4070000] [40.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 04:26:08,390 - Train: 4.08% [166200/4070000] [40.8/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 04:27:08,864 - Train: 4.09% [166300/4070000] [40.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 04:28:09,337 - Train: 4.09% [166400/4070000] [40.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 04:29:09,814 - Train: 4.09% [166500/4070000] [40.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 04:30:10,363 - Train: 4.09% [166600/4070000] [40.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 04:31:10,830 - Train: 4.10% [166700/4070000] [41.0/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 04:32:11,325 - Train: 4.10% [166800/4070000] [41.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 04:32:53,671 - ==> Total time: 2 days, 3:32:24 Eta: 50 days, 5:32:11 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-04 04:33:13,269 - Train: 4.10% [166900/4070000] [41.0/1000.0] [batch_t 0.603 (0.609)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 04:34:13,715 - Train: 4.10% [167000/4070000] [41.0/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 04:35:14,201 - Train: 4.11% [167100/4070000] [41.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 04:36:14,700 - Train: 4.11% [167200/4070000] [41.1/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 04:37:15,256 - Train: 4.11% [167300/4070000] [41.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 04:38:15,710 - Train: 4.11% [167400/4070000] [41.1/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 04:39:16,189 - Train: 4.12% [167500/4070000] [41.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 04:40:16,650 - Train: 4.12% [167600/4070000] [41.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 04:41:17,102 - Train: 4.12% [167700/4070000] [41.2/1000.0] [batch_t 0.608 (0.604)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-04 04:42:17,660 - Train: 4.12% [167800/4070000] [41.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 04:43:18,115 - Train: 4.13% [167900/4070000] [41.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 04:44:18,575 - Train: 4.13% [168000/4070000] [41.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 04:45:19,024 - Train: 4.13% [168100/4070000] [41.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 04:46:19,532 - Train: 4.13% [168200/4070000] [41.3/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 04:47:20,039 - Train: 4.14% [168300/4070000] [41.4/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 04:48:20,609 - Train: 4.14% [168400/4070000] [41.4/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 04:49:21,082 - Train: 4.14% [168500/4070000] [41.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 04:50:21,565 - Train: 4.14% [168600/4070000] [41.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 04:51:22,017 - Train: 4.14% [168700/4070000] [41.4/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 04:52:22,448 - Train: 4.15% [168800/4070000] [41.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 04:53:23,607 - Train: 4.15% [168900/4070000] [41.5/1000.0] [batch_t 0.603 (0.612)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 04:54:24,097 - Train: 4.15% [169000/4070000] [41.5/1000.0] [batch_t 0.600 (0.605)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-04 04:55:24,579 - Train: 4.15% [169100/4070000] [41.5/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-04 04:56:25,124 - Train: 4.16% [169200/4070000] [41.6/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 04:57:25,597 - Train: 4.16% [169300/4070000] [41.6/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 04:58:26,065 - Train: 4.16% [169400/4070000] [41.6/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 04:59:26,554 - Train: 4.16% [169500/4070000] [41.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 05:00:27,003 - Train: 4.17% [169600/4070000] [41.7/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 05:01:27,521 - Train: 4.17% [169700/4070000] [41.7/1000.0] [batch_t 0.591 (0.605)] [data_t 0.002] [optim_t 0.589] [lr 0.005000] +2024-04-04 05:02:27,971 - Train: 4.17% [169800/4070000] [41.7/1000.0] [batch_t 0.600 (0.604)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-04 05:03:28,432 - Train: 4.17% [169900/4070000] [41.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 05:04:28,896 - Train: 4.18% [170000/4070000] [41.8/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 05:05:29,366 - Train: 4.18% [170100/4070000] [41.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 05:06:29,847 - Train: 4.18% [170200/4070000] [41.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 05:07:30,312 - Train: 4.18% [170300/4070000] [41.8/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 05:08:30,778 - Train: 4.19% [170400/4070000] [41.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 05:09:31,371 - Train: 4.19% [170500/4070000] [41.9/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 05:10:31,828 - Train: 4.19% [170600/4070000] [41.9/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 05:11:32,301 - Train: 4.19% [170700/4070000] [41.9/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 05:12:32,801 - Train: 4.20% [170800/4070000] [42.0/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 05:13:33,282 - Train: 4.20% [170900/4070000] [42.0/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 05:13:57,476 - ==> Total time: 2 days, 4:13:28 Eta: 49 days, 15:13:00 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-04 05:14:35,398 - Train: 4.20% [171000/4070000] [42.0/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 05:15:35,866 - Train: 4.20% [171100/4070000] [42.0/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 05:16:36,345 - Train: 4.21% [171200/4070000] [42.1/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 05:17:36,776 - Train: 4.21% [171300/4070000] [42.1/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 05:18:37,226 - Train: 4.21% [171400/4070000] [42.1/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 05:19:37,901 - Train: 4.21% [171500/4070000] [42.1/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 05:20:38,349 - Train: 4.22% [171600/4070000] [42.2/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 05:21:38,822 - Train: 4.22% [171700/4070000] [42.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 05:22:39,320 - Train: 4.22% [171800/4070000] [42.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 05:23:39,789 - Train: 4.22% [171900/4070000] [42.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 05:24:40,257 - Train: 4.23% [172000/4070000] [42.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 05:25:40,747 - Train: 4.23% [172100/4070000] [42.3/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 05:26:41,243 - Train: 4.23% [172200/4070000] [42.3/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 05:27:41,840 - Train: 4.23% [172300/4070000] [42.3/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 05:28:42,331 - Train: 4.24% [172400/4070000] [42.4/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 05:29:42,854 - Train: 4.24% [172500/4070000] [42.4/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-04 05:30:43,341 - Train: 4.24% [172600/4070000] [42.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 05:31:43,811 - Train: 4.24% [172700/4070000] [42.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 05:32:44,459 - Train: 4.25% [172800/4070000] [42.5/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-04 05:33:44,945 - Train: 4.25% [172900/4070000] [42.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 05:34:45,406 - Train: 4.25% [173000/4070000] [42.5/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 05:35:45,879 - Train: 4.25% [173100/4070000] [42.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 05:36:46,337 - Train: 4.26% [173200/4070000] [42.6/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 05:37:46,813 - Train: 4.26% [173300/4070000] [42.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 05:38:47,287 - Train: 4.26% [173400/4070000] [42.6/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 05:39:47,750 - Train: 4.26% [173500/4070000] [42.6/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 05:40:48,372 - Train: 4.27% [173600/4070000] [42.7/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 05:41:48,838 - Train: 4.27% [173700/4070000] [42.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 05:42:49,300 - Train: 4.27% [173800/4070000] [42.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 05:43:49,765 - Train: 4.27% [173900/4070000] [42.7/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 05:44:50,219 - Train: 4.28% [174000/4070000] [42.8/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 05:45:50,836 - Train: 4.28% [174100/4070000] [42.8/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 05:46:51,292 - Train: 4.28% [174200/4070000] [42.8/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 05:47:51,726 - Train: 4.28% [174300/4070000] [42.8/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 05:48:52,250 - Train: 4.29% [174400/4070000] [42.9/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 05:49:52,738 - Train: 4.29% [174500/4070000] [42.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 05:50:53,225 - Train: 4.29% [174600/4070000] [42.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 05:51:53,727 - Train: 4.29% [174700/4070000] [42.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 05:52:54,224 - Train: 4.29% [174800/4070000] [42.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 05:53:54,823 - Train: 4.30% [174900/4070000] [43.0/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 05:54:55,327 - Train: 4.30% [175000/4070000] [43.0/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 05:55:01,377 - ==> Total time: 2 days, 4:54:32 Eta: 49 days, 1:31:54 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-04 05:55:59,576 - Train: 4.30% [175100/4070000] [43.0/1000.0] [batch_t 0.604 (0.615)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 05:57:00,055 - Train: 4.30% [175200/4070000] [43.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 05:58:00,520 - Train: 4.31% [175300/4070000] [43.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 05:59:00,994 - Train: 4.31% [175400/4070000] [43.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 06:00:01,620 - Train: 4.31% [175500/4070000] [43.1/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 06:01:02,080 - Train: 4.31% [175600/4070000] [43.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 06:02:02,551 - Train: 4.32% [175700/4070000] [43.2/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-04 06:03:03,019 - Train: 4.32% [175800/4070000] [43.2/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 06:04:03,486 - Train: 4.32% [175900/4070000] [43.2/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 06:05:04,047 - Train: 4.32% [176000/4070000] [43.2/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 06:06:04,497 - Train: 4.33% [176100/4070000] [43.3/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 06:07:04,974 - Train: 4.33% [176200/4070000] [43.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 06:08:05,430 - Train: 4.33% [176300/4070000] [43.3/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 06:09:05,893 - Train: 4.33% [176400/4070000] [43.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 06:10:06,364 - Train: 4.34% [176500/4070000] [43.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 06:11:06,818 - Train: 4.34% [176600/4070000] [43.4/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 06:12:07,313 - Train: 4.34% [176700/4070000] [43.4/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 06:13:07,886 - Train: 4.34% [176800/4070000] [43.4/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 06:14:08,336 - Train: 4.35% [176900/4070000] [43.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 06:15:08,809 - Train: 4.35% [177000/4070000] [43.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 06:16:09,318 - Train: 4.35% [177100/4070000] [43.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 06:17:09,795 - Train: 4.35% [177200/4070000] [43.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 06:18:10,342 - Train: 4.36% [177300/4070000] [43.6/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 06:19:10,801 - Train: 4.36% [177400/4070000] [43.6/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 06:20:11,290 - Train: 4.36% [177500/4070000] [43.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 06:21:11,766 - Train: 4.36% [177600/4070000] [43.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 06:22:12,231 - Train: 4.37% [177700/4070000] [43.7/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 06:23:12,761 - Train: 4.37% [177800/4070000] [43.7/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 06:24:13,291 - Train: 4.37% [177900/4070000] [43.7/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 06:25:13,750 - Train: 4.37% [178000/4070000] [43.7/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 06:26:14,420 - Train: 4.38% [178100/4070000] [43.8/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 06:27:14,916 - Train: 4.38% [178200/4070000] [43.8/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 06:28:15,397 - Train: 4.38% [178300/4070000] [43.8/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 06:29:15,866 - Train: 4.38% [178400/4070000] [43.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 06:30:16,353 - Train: 4.39% [178500/4070000] [43.9/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 06:31:16,955 - Train: 4.39% [178600/4070000] [43.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 06:32:17,493 - Train: 4.39% [178700/4070000] [43.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 06:33:17,970 - Train: 4.39% [178800/4070000] [43.9/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-04 06:34:18,426 - Train: 4.40% [178900/4070000] [44.0/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 06:35:18,931 - Train: 4.40% [179000/4070000] [44.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 06:36:07,339 - ==> Total time: 2 days, 5:35:38 Eta: 48 days, 12:27:00 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-04 06:36:21,079 - Train: 4.40% [179100/4070000] [44.0/1000.0] [batch_t 0.606 (0.615)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 06:37:21,718 - Train: 4.40% [179200/4070000] [44.0/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 06:38:22,168 - Train: 4.41% [179300/4070000] [44.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 06:39:22,663 - Train: 4.41% [179400/4070000] [44.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 06:40:23,194 - Train: 4.41% [179500/4070000] [44.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 06:41:23,664 - Train: 4.41% [179600/4070000] [44.1/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-04 06:42:24,135 - Train: 4.42% [179700/4070000] [44.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 06:43:24,601 - Train: 4.42% [179800/4070000] [44.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 06:44:25,061 - Train: 4.42% [179900/4070000] [44.2/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 06:45:25,629 - Train: 4.42% [180000/4070000] [44.2/1000.0] [batch_t 0.602 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 06:46:26,101 - Train: 4.43% [180100/4070000] [44.3/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 06:47:26,566 - Train: 4.43% [180200/4070000] [44.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 06:48:27,038 - Train: 4.43% [180300/4070000] [44.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 06:49:27,547 - Train: 4.43% [180400/4070000] [44.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 06:50:28,135 - Train: 4.43% [180500/4070000] [44.3/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 06:51:28,614 - Train: 4.44% [180600/4070000] [44.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 06:52:29,094 - Train: 4.44% [180700/4070000] [44.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 06:53:29,592 - Train: 4.44% [180800/4070000] [44.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 06:54:30,050 - Train: 4.44% [180900/4070000] [44.4/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 06:55:30,476 - Train: 4.45% [181000/4070000] [44.5/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 06:56:30,957 - Train: 4.45% [181100/4070000] [44.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 06:57:31,458 - Train: 4.45% [181200/4070000] [44.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 06:58:31,993 - Train: 4.45% [181300/4070000] [44.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 06:59:32,486 - Train: 4.46% [181400/4070000] [44.6/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-04 07:00:32,971 - Train: 4.46% [181500/4070000] [44.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 07:01:33,471 - Train: 4.46% [181600/4070000] [44.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 07:02:33,962 - Train: 4.46% [181700/4070000] [44.6/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 07:03:34,649 - Train: 4.47% [181800/4070000] [44.7/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 07:04:35,142 - Train: 4.47% [181900/4070000] [44.7/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 07:05:35,616 - Train: 4.47% [182000/4070000] [44.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 07:06:36,111 - Train: 4.47% [182100/4070000] [44.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 07:07:36,572 - Train: 4.48% [182200/4070000] [44.8/1000.0] [batch_t 0.599 (0.605)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-04 07:08:37,045 - Train: 4.48% [182300/4070000] [44.8/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 07:09:37,563 - Train: 4.48% [182400/4070000] [44.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 07:10:38,083 - Train: 4.48% [182500/4070000] [44.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 07:11:38,693 - Train: 4.49% [182600/4070000] [44.9/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 07:12:39,172 - Train: 4.49% [182700/4070000] [44.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 07:13:39,657 - Train: 4.49% [182800/4070000] [44.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 07:14:40,150 - Train: 4.49% [182900/4070000] [44.9/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 07:15:40,646 - Train: 4.50% [183000/4070000] [45.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 07:16:41,355 - Train: 4.50% [183100/4070000] [45.0/1000.0] [batch_t 0.609 (0.607)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-04 07:17:11,624 - ==> Total time: 2 days, 6:16:42 Eta: 47 days, 23:54:34 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-04 07:17:43,302 - Train: 4.50% [183200/4070000] [45.0/1000.0] [batch_t 0.605 (0.608)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 07:18:43,740 - Train: 4.50% [183300/4070000] [45.0/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 07:19:44,250 - Train: 4.51% [183400/4070000] [45.1/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-04 07:20:44,773 - Train: 4.51% [183500/4070000] [45.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 07:21:45,226 - Train: 4.51% [183600/4070000] [45.1/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 07:22:45,644 - Train: 4.51% [183700/4070000] [45.1/1000.0] [batch_t 0.600 (0.604)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-04 07:23:46,237 - Train: 4.52% [183800/4070000] [45.2/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 07:24:46,705 - Train: 4.52% [183900/4070000] [45.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 07:25:47,169 - Train: 4.52% [184000/4070000] [45.2/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 07:26:47,634 - Train: 4.52% [184100/4070000] [45.2/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 07:27:48,092 - Train: 4.53% [184200/4070000] [45.3/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 07:28:48,716 - Train: 4.53% [184300/4070000] [45.3/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 07:29:49,219 - Train: 4.53% [184400/4070000] [45.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 07:30:49,726 - Train: 4.53% [184500/4070000] [45.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 07:31:50,226 - Train: 4.54% [184600/4070000] [45.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 07:32:50,693 - Train: 4.54% [184700/4070000] [45.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 07:33:51,211 - Train: 4.54% [184800/4070000] [45.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 07:34:51,690 - Train: 4.54% [184900/4070000] [45.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 07:35:52,193 - Train: 4.55% [185000/4070000] [45.5/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-04 07:36:52,854 - Train: 4.55% [185100/4070000] [45.5/1000.0] [batch_t 0.609 (0.607)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-04 07:37:53,340 - Train: 4.55% [185200/4070000] [45.5/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 07:38:53,833 - Train: 4.55% [185300/4070000] [45.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 07:39:54,378 - Train: 4.56% [185400/4070000] [45.6/1000.0] [batch_t 0.610 (0.605)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-04 07:40:54,868 - Train: 4.56% [185500/4070000] [45.6/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-04 07:41:55,479 - Train: 4.56% [185600/4070000] [45.6/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 07:42:55,937 - Train: 4.56% [185700/4070000] [45.6/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 07:43:56,466 - Train: 4.57% [185800/4070000] [45.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 07:44:56,984 - Train: 4.57% [185900/4070000] [45.7/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 07:45:57,457 - Train: 4.57% [186000/4070000] [45.7/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 07:46:57,972 - Train: 4.57% [186100/4070000] [45.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 07:47:58,500 - Train: 4.57% [186200/4070000] [45.7/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 07:48:58,994 - Train: 4.58% [186300/4070000] [45.8/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 07:49:59,606 - Train: 4.58% [186400/4070000] [45.8/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 07:51:00,102 - Train: 4.58% [186500/4070000] [45.8/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 07:52:00,548 - Train: 4.58% [186600/4070000] [45.8/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 07:53:01,043 - Train: 4.59% [186700/4070000] [45.9/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-04 07:54:01,599 - Train: 4.59% [186800/4070000] [45.9/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 07:55:02,210 - Train: 4.59% [186900/4070000] [45.9/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 07:56:02,709 - Train: 4.59% [187000/4070000] [45.9/1000.0] [batch_t 0.600 (0.605)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-04 07:57:03,246 - Train: 4.60% [187100/4070000] [46.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 07:58:03,706 - Train: 4.60% [187200/4070000] [46.0/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 07:58:15,800 - ==> Total time: 2 days, 6:57:46 Eta: 47 days, 11:53:02 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-04 07:59:05,605 - Train: 4.60% [187300/4070000] [46.0/1000.0] [batch_t 0.602 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 08:00:06,092 - Train: 4.60% [187400/4070000] [46.0/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 08:01:06,558 - Train: 4.61% [187500/4070000] [46.1/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 08:02:07,131 - Train: 4.61% [187600/4070000] [46.1/1000.0] [batch_t 0.602 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 08:03:07,599 - Train: 4.61% [187700/4070000] [46.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 08:04:08,081 - Train: 4.61% [187800/4070000] [46.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 08:05:08,548 - Train: 4.62% [187900/4070000] [46.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 08:06:09,023 - Train: 4.62% [188000/4070000] [46.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 08:07:09,491 - Train: 4.62% [188100/4070000] [46.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 08:08:10,076 - Train: 4.62% [188200/4070000] [46.2/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 08:09:10,549 - Train: 4.63% [188300/4070000] [46.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 08:10:11,038 - Train: 4.63% [188400/4070000] [46.3/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 08:11:11,554 - Train: 4.63% [188500/4070000] [46.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 08:12:12,014 - Train: 4.63% [188600/4070000] [46.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 08:13:12,483 - Train: 4.64% [188700/4070000] [46.4/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 08:14:12,962 - Train: 4.64% [188800/4070000] [46.4/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-04 08:15:13,578 - Train: 4.64% [188900/4070000] [46.4/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 08:16:14,066 - Train: 4.64% [189000/4070000] [46.4/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 08:17:14,634 - Train: 4.65% [189100/4070000] [46.5/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 08:18:15,132 - Train: 4.65% [189200/4070000] [46.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 08:19:15,614 - Train: 4.65% [189300/4070000] [46.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 08:20:16,275 - Train: 4.65% [189400/4070000] [46.5/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 08:21:16,780 - Train: 4.66% [189500/4070000] [46.6/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 08:22:18,038 - Train: 4.66% [189600/4070000] [46.6/1000.0] [batch_t 0.606 (0.612)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 08:23:18,530 - Train: 4.66% [189700/4070000] [46.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 08:24:19,067 - Train: 4.66% [189800/4070000] [46.6/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-04 08:25:19,537 - Train: 4.67% [189900/4070000] [46.7/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 08:26:20,021 - Train: 4.67% [190000/4070000] [46.7/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 08:27:20,516 - Train: 4.67% [190100/4070000] [46.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 08:28:21,110 - Train: 4.67% [190200/4070000] [46.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 08:29:21,594 - Train: 4.68% [190300/4070000] [46.8/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 08:30:22,089 - Train: 4.68% [190400/4070000] [46.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 08:31:22,584 - Train: 4.68% [190500/4070000] [46.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 08:32:23,070 - Train: 4.68% [190600/4070000] [46.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 08:33:23,777 - Train: 4.69% [190700/4070000] [46.9/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 08:34:24,244 - Train: 4.69% [190800/4070000] [46.9/1000.0] [batch_t 0.597 (0.605)] [data_t 0.002] [optim_t 0.595] [lr 0.005000] +2024-04-04 08:35:24,701 - Train: 4.69% [190900/4070000] [46.9/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 08:36:25,193 - Train: 4.69% [191000/4070000] [46.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 08:37:25,703 - Train: 4.70% [191100/4070000] [47.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 08:38:26,189 - Train: 4.70% [191200/4070000] [47.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 08:39:20,633 - ==> Total time: 2 days, 7:38:51 Eta: 47 days, 0:20:40 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-04 08:39:28,185 - Train: 4.70% [191300/4070000] [47.0/1000.0] [batch_t 0.604 (0.624)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 08:40:28,808 - Train: 4.70% [191400/4070000] [47.0/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 08:41:29,357 - Train: 4.71% [191500/4070000] [47.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 08:42:29,825 - Train: 4.71% [191600/4070000] [47.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 08:43:30,320 - Train: 4.71% [191700/4070000] [47.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 08:44:30,796 - Train: 4.71% [191800/4070000] [47.1/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 08:45:31,270 - Train: 4.71% [191900/4070000] [47.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 08:46:31,883 - Train: 4.72% [192000/4070000] [47.2/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 08:47:32,389 - Train: 4.72% [192100/4070000] [47.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 08:48:32,840 - Train: 4.72% [192200/4070000] [47.2/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 08:49:33,301 - Train: 4.72% [192300/4070000] [47.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 08:50:33,836 - Train: 4.73% [192400/4070000] [47.3/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 08:51:34,329 - Train: 4.73% [192500/4070000] [47.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 08:52:34,775 - Train: 4.73% [192600/4070000] [47.3/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 08:53:35,324 - Train: 4.73% [192700/4070000] [47.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 08:54:35,852 - Train: 4.74% [192800/4070000] [47.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 08:55:36,339 - Train: 4.74% [192900/4070000] [47.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 08:56:36,840 - Train: 4.74% [193000/4070000] [47.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 08:57:37,360 - Train: 4.74% [193100/4070000] [47.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 08:58:37,932 - Train: 4.75% [193200/4070000] [47.5/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 08:59:38,391 - Train: 4.75% [193300/4070000] [47.5/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 09:00:38,896 - Train: 4.75% [193400/4070000] [47.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 09:01:39,361 - Train: 4.75% [193500/4070000] [47.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 09:02:39,844 - Train: 4.76% [193600/4070000] [47.6/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 09:03:40,310 - Train: 4.76% [193700/4070000] [47.6/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 09:04:40,795 - Train: 4.76% [193800/4070000] [47.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 09:05:41,259 - Train: 4.76% [193900/4070000] [47.6/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 09:06:41,783 - Train: 4.77% [194000/4070000] [47.7/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 09:07:42,269 - Train: 4.77% [194100/4070000] [47.7/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 09:08:42,753 - Train: 4.77% [194200/4070000] [47.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 09:09:43,263 - Train: 4.77% [194300/4070000] [47.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 09:10:43,753 - Train: 4.78% [194400/4070000] [47.8/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 09:11:44,206 - Train: 4.78% [194500/4070000] [47.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 09:12:44,758 - Train: 4.78% [194600/4070000] [47.8/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 09:13:45,231 - Train: 4.78% [194700/4070000] [47.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 09:14:45,733 - Train: 4.79% [194800/4070000] [47.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 09:15:46,185 - Train: 4.79% [194900/4070000] [47.9/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 09:16:46,663 - Train: 4.79% [195000/4070000] [47.9/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 09:17:47,259 - Train: 4.79% [195100/4070000] [47.9/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 09:18:47,733 - Train: 4.80% [195200/4070000] [48.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 09:19:48,227 - Train: 4.80% [195300/4070000] [48.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 09:20:24,526 - ==> Total time: 2 days, 8:19:55 Eta: 46 days, 13:15:08 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-04 09:20:50,486 - Train: 4.80% [195400/4070000] [48.0/1000.0] [batch_t 0.607 (0.612)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 09:21:50,996 - Train: 4.80% [195500/4070000] [48.0/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-04 09:22:51,466 - Train: 4.81% [195600/4070000] [48.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 09:23:51,955 - Train: 4.81% [195700/4070000] [48.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 09:24:52,548 - Train: 4.81% [195800/4070000] [48.1/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 09:25:53,016 - Train: 4.81% [195900/4070000] [48.1/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 09:26:53,502 - Train: 4.82% [196000/4070000] [48.2/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 09:27:53,998 - Train: 4.82% [196100/4070000] [48.2/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 09:28:54,489 - Train: 4.82% [196200/4070000] [48.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 09:29:54,955 - Train: 4.82% [196300/4070000] [48.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 09:30:55,500 - Train: 4.83% [196400/4070000] [48.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 09:31:56,001 - Train: 4.83% [196500/4070000] [48.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 09:32:56,458 - Train: 4.83% [196600/4070000] [48.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 09:33:56,930 - Train: 4.83% [196700/4070000] [48.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 09:34:57,378 - Train: 4.84% [196800/4070000] [48.4/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 09:35:57,884 - Train: 4.84% [196900/4070000] [48.4/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 09:36:58,451 - Train: 4.84% [197000/4070000] [48.4/1000.0] [batch_t 0.602 (0.606)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 09:37:58,943 - Train: 4.84% [197100/4070000] [48.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 09:38:59,407 - Train: 4.85% [197200/4070000] [48.5/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 09:39:59,863 - Train: 4.85% [197300/4070000] [48.5/1000.0] [batch_t 0.608 (0.604)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-04 09:41:00,343 - Train: 4.85% [197400/4070000] [48.5/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 09:42:00,841 - Train: 4.85% [197500/4070000] [48.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 09:43:01,347 - Train: 4.86% [197600/4070000] [48.6/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 09:44:01,970 - Train: 4.86% [197700/4070000] [48.6/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 09:45:02,462 - Train: 4.86% [197800/4070000] [48.6/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 09:46:02,940 - Train: 4.86% [197900/4070000] [48.6/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 09:47:03,428 - Train: 4.86% [198000/4070000] [48.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 09:48:03,904 - Train: 4.87% [198100/4070000] [48.7/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-04 09:49:04,500 - Train: 4.87% [198200/4070000] [48.7/1000.0] [batch_t 0.695 (0.606)] [data_t 0.002] [optim_t 0.693] [lr 0.005000] +2024-04-04 09:50:04,980 - Train: 4.87% [198300/4070000] [48.7/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-04 09:51:05,481 - Train: 4.87% [198400/4070000] [48.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 09:52:05,929 - Train: 4.88% [198500/4070000] [48.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 09:53:06,370 - Train: 4.88% [198600/4070000] [48.8/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 09:54:06,883 - Train: 4.88% [198700/4070000] [48.8/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 09:55:07,364 - Train: 4.88% [198800/4070000] [48.8/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-04 09:56:07,868 - Train: 4.89% [198900/4070000] [48.9/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 09:57:08,445 - Train: 4.89% [199000/4070000] [48.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 09:58:08,934 - Train: 4.89% [199100/4070000] [48.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 09:59:09,414 - Train: 4.89% [199200/4070000] [48.9/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 10:00:09,874 - Train: 4.90% [199300/4070000] [49.0/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 10:01:10,350 - Train: 4.90% [199400/4070000] [49.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 10:01:28,490 - ==> Total time: 2 days, 9:00:59 Eta: 46 days, 2:35:06 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-04 10:02:12,474 - Train: 4.90% [199500/4070000] [49.0/1000.0] [batch_t 0.606 (0.609)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 10:03:12,943 - Train: 4.90% [199600/4070000] [49.0/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 10:04:13,426 - Train: 4.91% [199700/4070000] [49.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 10:05:13,912 - Train: 4.91% [199800/4070000] [49.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 10:06:14,393 - Train: 4.91% [199900/4070000] [49.1/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 10:07:14,846 - Train: 4.91% [200000/4070000] [49.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 10:08:15,324 - Train: 4.92% [200100/4070000] [49.2/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 10:09:15,810 - Train: 4.92% [200200/4070000] [49.2/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 10:10:16,371 - Train: 4.92% [200300/4070000] [49.2/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 10:11:16,843 - Train: 4.92% [200400/4070000] [49.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 10:12:17,310 - Train: 4.93% [200500/4070000] [49.3/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 10:13:17,783 - Train: 4.93% [200600/4070000] [49.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 10:14:18,258 - Train: 4.93% [200700/4070000] [49.3/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-04 10:15:18,860 - Train: 4.93% [200800/4070000] [49.3/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 10:16:19,313 - Train: 4.94% [200900/4070000] [49.4/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 10:17:19,792 - Train: 4.94% [201000/4070000] [49.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 10:18:20,273 - Train: 4.94% [201100/4070000] [49.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 10:19:20,747 - Train: 4.94% [201200/4070000] [49.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 10:20:21,246 - Train: 4.95% [201300/4070000] [49.5/1000.0] [batch_t 0.610 (0.605)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-04 10:21:21,697 - Train: 4.95% [201400/4070000] [49.5/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 10:22:22,160 - Train: 4.95% [201500/4070000] [49.5/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 10:23:22,701 - Train: 4.95% [201600/4070000] [49.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 10:24:23,214 - Train: 4.96% [201700/4070000] [49.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 10:25:23,668 - Train: 4.96% [201800/4070000] [49.6/1000.0] [batch_t 0.601 (0.604)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 10:26:24,139 - Train: 4.96% [201900/4070000] [49.6/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 10:27:24,602 - Train: 4.96% [202000/4070000] [49.6/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 10:28:25,163 - Train: 4.97% [202100/4070000] [49.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 10:29:25,642 - Train: 4.97% [202200/4070000] [49.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 10:30:26,114 - Train: 4.97% [202300/4070000] [49.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 10:31:26,587 - Train: 4.97% [202400/4070000] [49.7/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 10:32:27,074 - Train: 4.98% [202500/4070000] [49.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 10:33:27,543 - Train: 4.98% [202600/4070000] [49.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 10:34:28,018 - Train: 4.98% [202700/4070000] [49.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 10:35:28,590 - Train: 4.98% [202800/4070000] [49.8/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 10:36:29,068 - Train: 4.99% [202900/4070000] [49.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 10:37:29,530 - Train: 4.99% [203000/4070000] [49.9/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-04 10:38:30,000 - Train: 4.99% [203100/4070000] [49.9/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 10:39:30,458 - Train: 4.99% [203200/4070000] [49.9/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 10:40:30,932 - Train: 5.00% [203300/4070000] [50.0/1000.0] [batch_t 0.598 (0.605)] [data_t 0.002] [optim_t 0.596] [lr 0.005000] +2024-04-04 10:41:31,510 - Train: 5.00% [203400/4070000] [50.0/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 10:42:32,008 - Train: 5.00% [203500/4070000] [50.0/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 10:42:32,012 - ==> Total time: 2 days, 9:42:02 Eta: 45 days, 16:18:54 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-04 10:43:34,719 - Train: 5.00% [203600/4070000] [50.0/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 10:44:35,183 - Train: 5.00% [203700/4070000] [50.0/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 10:45:35,644 - Train: 5.01% [203800/4070000] [50.1/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 10:46:36,116 - Train: 5.01% [203900/4070000] [50.1/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 10:47:36,580 - Train: 5.01% [204000/4070000] [50.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 10:48:37,184 - Train: 5.01% [204100/4070000] [50.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 10:49:37,670 - Train: 5.02% [204200/4070000] [50.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 10:50:38,157 - Train: 5.02% [204300/4070000] [50.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 10:51:38,615 - Train: 5.02% [204400/4070000] [50.2/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 10:52:39,020 - Train: 5.02% [204500/4070000] [50.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 10:53:39,651 - Train: 5.03% [204600/4070000] [50.3/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 10:54:40,132 - Train: 5.03% [204700/4070000] [50.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 10:55:40,623 - Train: 5.03% [204800/4070000] [50.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 10:56:41,122 - Train: 5.03% [204900/4070000] [50.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 10:57:41,601 - Train: 5.04% [205000/4070000] [50.4/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 10:58:42,105 - Train: 5.04% [205100/4070000] [50.4/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-04 10:59:42,595 - Train: 5.04% [205200/4070000] [50.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 11:00:43,105 - Train: 5.04% [205300/4070000] [50.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 11:01:43,746 - Train: 5.05% [205400/4070000] [50.5/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 11:02:44,282 - Train: 5.05% [205500/4070000] [50.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 11:03:44,775 - Train: 5.05% [205600/4070000] [50.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 11:04:45,302 - Train: 5.05% [205700/4070000] [50.5/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 11:05:45,776 - Train: 5.06% [205800/4070000] [50.6/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 11:06:46,376 - Train: 5.06% [205900/4070000] [50.6/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 11:07:46,870 - Train: 5.06% [206000/4070000] [50.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 11:08:47,387 - Train: 5.06% [206100/4070000] [50.6/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-04 11:09:47,923 - Train: 5.07% [206200/4070000] [50.7/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 11:10:48,416 - Train: 5.07% [206300/4070000] [50.7/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-04 11:11:48,895 - Train: 5.07% [206400/4070000] [50.7/1000.0] [batch_t 0.600 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-04 11:12:49,343 - Train: 5.07% [206500/4070000] [50.7/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 11:13:49,800 - Train: 5.08% [206600/4070000] [50.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 11:14:50,328 - Train: 5.08% [206700/4070000] [50.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 11:15:50,780 - Train: 5.08% [206800/4070000] [50.8/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 11:16:51,250 - Train: 5.08% [206900/4070000] [50.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 11:17:51,797 - Train: 5.09% [207000/4070000] [50.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 11:18:52,338 - Train: 5.09% [207100/4070000] [50.9/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 11:19:52,974 - Train: 5.09% [207200/4070000] [50.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 11:20:53,463 - Train: 5.09% [207300/4070000] [50.9/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-04 11:21:53,969 - Train: 5.10% [207400/4070000] [51.0/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-04 11:22:54,448 - Train: 5.10% [207500/4070000] [51.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 11:23:36,788 - ==> Total time: 2 days, 10:23:07 Eta: 45 days, 6:25:38 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-04 11:23:56,637 - Train: 5.10% [207600/4070000] [51.0/1000.0] [batch_t 0.605 (0.611)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 11:24:57,161 - Train: 5.10% [207700/4070000] [51.0/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 11:25:57,643 - Train: 5.11% [207800/4070000] [51.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 11:26:58,261 - Train: 5.11% [207900/4070000] [51.1/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 11:27:58,752 - Train: 5.11% [208000/4070000] [51.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 11:28:59,221 - Train: 5.11% [208100/4070000] [51.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 11:29:59,721 - Train: 5.12% [208200/4070000] [51.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 11:31:00,255 - Train: 5.12% [208300/4070000] [51.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 11:32:00,843 - Train: 5.12% [208400/4070000] [51.2/1000.0] [batch_t 0.602 (0.606)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 11:33:01,354 - Train: 5.12% [208500/4070000] [51.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 11:34:01,827 - Train: 5.13% [208600/4070000] [51.3/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-04 11:35:02,335 - Train: 5.13% [208700/4070000] [51.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 11:36:02,836 - Train: 5.13% [208800/4070000] [51.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 11:37:03,317 - Train: 5.13% [208900/4070000] [51.3/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 11:38:03,803 - Train: 5.14% [209000/4070000] [51.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 11:39:04,366 - Train: 5.14% [209100/4070000] [51.4/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 11:40:04,835 - Train: 5.14% [209200/4070000] [51.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 11:41:05,383 - Train: 5.14% [209300/4070000] [51.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 11:42:05,862 - Train: 5.14% [209400/4070000] [51.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 11:43:06,359 - Train: 5.15% [209500/4070000] [51.5/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 11:44:06,863 - Train: 5.15% [209600/4070000] [51.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 11:45:07,355 - Train: 5.15% [209700/4070000] [51.5/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 11:46:07,832 - Train: 5.15% [209800/4070000] [51.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 11:47:08,351 - Train: 5.16% [209900/4070000] [51.6/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 11:48:08,809 - Train: 5.16% [210000/4070000] [51.6/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 11:49:09,277 - Train: 5.16% [210100/4070000] [51.6/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-04 11:50:09,751 - Train: 5.16% [210200/4070000] [51.6/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 11:51:10,214 - Train: 5.17% [210300/4070000] [51.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 11:52:10,775 - Train: 5.17% [210400/4070000] [51.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 11:53:11,247 - Train: 5.17% [210500/4070000] [51.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 11:54:11,762 - Train: 5.17% [210600/4070000] [51.7/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 11:55:12,237 - Train: 5.18% [210700/4070000] [51.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 11:56:12,712 - Train: 5.18% [210800/4070000] [51.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 11:57:13,168 - Train: 5.18% [210900/4070000] [51.8/1000.0] [batch_t 0.601 (0.604)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-04 11:58:13,645 - Train: 5.18% [211000/4070000] [51.8/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 11:59:14,203 - Train: 5.19% [211100/4070000] [51.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 12:00:14,674 - Train: 5.19% [211200/4070000] [51.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 12:01:15,143 - Train: 5.19% [211300/4070000] [51.9/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 12:02:15,601 - Train: 5.19% [211400/4070000] [51.9/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 12:03:16,080 - Train: 5.20% [211500/4070000] [52.0/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 12:04:16,567 - Train: 5.20% [211600/4070000] [52.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 12:04:40,858 - ==> Total time: 2 days, 11:04:11 Eta: 44 days, 20:53:24 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-04 12:05:18,690 - Train: 5.20% [211700/4070000] [52.0/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 12:06:19,162 - Train: 5.20% [211800/4070000] [52.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 12:07:19,612 - Train: 5.21% [211900/4070000] [52.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 12:08:20,096 - Train: 5.21% [212000/4070000] [52.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 12:09:20,572 - Train: 5.21% [212100/4070000] [52.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 12:10:21,219 - Train: 5.21% [212200/4070000] [52.1/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 12:11:21,712 - Train: 5.22% [212300/4070000] [52.2/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 12:12:22,185 - Train: 5.22% [212400/4070000] [52.2/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 12:13:22,681 - Train: 5.22% [212500/4070000] [52.2/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 12:14:23,188 - Train: 5.22% [212600/4070000] [52.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 12:15:23,650 - Train: 5.23% [212700/4070000] [52.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 12:16:24,124 - Train: 5.23% [212800/4070000] [52.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 12:17:24,682 - Train: 5.23% [212900/4070000] [52.3/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-04 12:18:25,176 - Train: 5.23% [213000/4070000] [52.3/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 12:19:25,685 - Train: 5.24% [213100/4070000] [52.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 12:20:26,206 - Train: 5.24% [213200/4070000] [52.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 12:21:26,688 - Train: 5.24% [213300/4070000] [52.4/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-04 12:22:27,208 - Train: 5.24% [213400/4070000] [52.4/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 12:23:27,789 - Train: 5.25% [213500/4070000] [52.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 12:24:28,276 - Train: 5.25% [213600/4070000] [52.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 12:25:28,746 - Train: 5.25% [213700/4070000] [52.5/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 12:26:29,221 - Train: 5.25% [213800/4070000] [52.5/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 12:27:29,704 - Train: 5.26% [213900/4070000] [52.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 12:28:30,202 - Train: 5.26% [214000/4070000] [52.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 12:29:30,661 - Train: 5.26% [214100/4070000] [52.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 12:30:31,250 - Train: 5.26% [214200/4070000] [52.6/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 12:31:31,740 - Train: 5.27% [214300/4070000] [52.7/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 12:32:32,214 - Train: 5.27% [214400/4070000] [52.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 12:33:32,681 - Train: 5.27% [214500/4070000] [52.7/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 12:34:33,140 - Train: 5.27% [214600/4070000] [52.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 12:35:33,623 - Train: 5.28% [214700/4070000] [52.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 12:36:34,247 - Train: 5.28% [214800/4070000] [52.8/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 12:37:34,679 - Train: 5.28% [214900/4070000] [52.8/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 12:38:35,137 - Train: 5.28% [215000/4070000] [52.8/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 12:39:35,617 - Train: 5.29% [215100/4070000] [52.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 12:40:36,095 - Train: 5.29% [215200/4070000] [52.9/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 12:41:36,702 - Train: 5.29% [215300/4070000] [52.9/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 12:42:37,208 - Train: 5.29% [215400/4070000] [52.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 12:43:37,705 - Train: 5.29% [215500/4070000] [52.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 12:44:38,209 - Train: 5.30% [215600/4070000] [53.0/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 12:45:38,705 - Train: 5.30% [215700/4070000] [53.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 12:45:44,781 - ==> Total time: 2 days, 11:45:15 Eta: 44 days, 11:41:09 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-04 12:46:40,778 - Train: 5.30% [215800/4070000] [53.0/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 12:47:41,393 - Train: 5.30% [215900/4070000] [53.0/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 12:48:41,868 - Train: 5.31% [216000/4070000] [53.1/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 12:49:42,328 - Train: 5.31% [216100/4070000] [53.1/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 12:50:42,784 - Train: 5.31% [216200/4070000] [53.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 12:51:43,218 - Train: 5.31% [216300/4070000] [53.1/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 12:52:43,701 - Train: 5.32% [216400/4070000] [53.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 12:53:44,215 - Train: 5.32% [216500/4070000] [53.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 12:54:44,684 - Train: 5.32% [216600/4070000] [53.2/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 12:55:45,178 - Train: 5.32% [216700/4070000] [53.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 12:56:45,641 - Train: 5.33% [216800/4070000] [53.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 12:57:46,091 - Train: 5.33% [216900/4070000] [53.3/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 12:58:46,544 - Train: 5.33% [217000/4070000] [53.3/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 12:59:46,989 - Train: 5.33% [217100/4070000] [53.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 13:00:47,561 - Train: 5.34% [217200/4070000] [53.4/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 13:01:48,016 - Train: 5.34% [217300/4070000] [53.4/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 13:02:48,465 - Train: 5.34% [217400/4070000] [53.4/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 13:03:48,907 - Train: 5.34% [217500/4070000] [53.4/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 13:04:49,356 - Train: 5.35% [217600/4070000] [53.5/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 13:05:49,778 - Train: 5.35% [217700/4070000] [53.5/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 13:06:50,273 - Train: 5.35% [217800/4070000] [53.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 13:07:50,736 - Train: 5.35% [217900/4070000] [53.5/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 13:08:51,266 - Train: 5.36% [218000/4070000] [53.6/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 13:09:51,749 - Train: 5.36% [218100/4070000] [53.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 13:10:52,234 - Train: 5.36% [218200/4070000] [53.6/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 13:11:52,723 - Train: 5.36% [218300/4070000] [53.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 13:12:53,218 - Train: 5.37% [218400/4070000] [53.7/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 13:13:53,770 - Train: 5.37% [218500/4070000] [53.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 13:14:54,212 - Train: 5.37% [218600/4070000] [53.7/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 13:15:54,678 - Train: 5.37% [218700/4070000] [53.7/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 13:16:55,178 - Train: 5.38% [218800/4070000] [53.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 13:17:55,638 - Train: 5.38% [218900/4070000] [53.8/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 13:18:56,126 - Train: 5.38% [219000/4070000] [53.8/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 13:19:56,608 - Train: 5.38% [219100/4070000] [53.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 13:20:57,075 - Train: 5.39% [219200/4070000] [53.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 13:21:57,702 - Train: 5.39% [219300/4070000] [53.9/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 13:22:58,149 - Train: 5.39% [219400/4070000] [53.9/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 13:23:58,627 - Train: 5.39% [219500/4070000] [53.9/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 13:24:59,123 - Train: 5.40% [219600/4070000] [54.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 13:25:59,605 - Train: 5.40% [219700/4070000] [54.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 13:26:48,139 - ==> Total time: 2 days, 12:26:18 Eta: 44 days, 2:47:41 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-04 13:27:01,727 - Train: 5.40% [219800/4070000] [54.0/1000.0] [batch_t 0.605 (0.613)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 13:28:02,159 - Train: 5.40% [219900/4070000] [54.0/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 13:29:02,604 - Train: 5.41% [220000/4070000] [54.1/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 13:30:03,049 - Train: 5.41% [220100/4070000] [54.1/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 13:31:03,510 - Train: 5.41% [220200/4070000] [54.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 13:32:04,144 - Train: 5.41% [220300/4070000] [54.1/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 13:33:04,614 - Train: 5.42% [220400/4070000] [54.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 13:34:05,116 - Train: 5.42% [220500/4070000] [54.2/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-04 13:35:05,632 - Train: 5.42% [220600/4070000] [54.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 13:36:06,115 - Train: 5.42% [220700/4070000] [54.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 13:37:06,576 - Train: 5.43% [220800/4070000] [54.3/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 13:38:07,047 - Train: 5.43% [220900/4070000] [54.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 13:39:07,518 - Train: 5.43% [221000/4070000] [54.3/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 13:40:08,071 - Train: 5.43% [221100/4070000] [54.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 13:41:08,573 - Train: 5.43% [221200/4070000] [54.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 13:42:09,017 - Train: 5.44% [221300/4070000] [54.4/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 13:43:09,502 - Train: 5.44% [221400/4070000] [54.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 13:44:09,938 - Train: 5.44% [221500/4070000] [54.4/1000.0] [batch_t 0.601 (0.604)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-04 13:45:10,501 - Train: 5.44% [221600/4070000] [54.4/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 13:46:10,959 - Train: 5.45% [221700/4070000] [54.5/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 13:47:11,425 - Train: 5.45% [221800/4070000] [54.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 13:48:11,915 - Train: 5.45% [221900/4070000] [54.5/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 13:49:12,370 - Train: 5.45% [222000/4070000] [54.5/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 13:50:12,819 - Train: 5.46% [222100/4070000] [54.6/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 13:51:13,273 - Train: 5.46% [222200/4070000] [54.6/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 13:52:13,723 - Train: 5.46% [222300/4070000] [54.6/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 13:53:14,289 - Train: 5.46% [222400/4070000] [54.6/1000.0] [batch_t 0.602 (0.606)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 13:54:14,755 - Train: 5.47% [222500/4070000] [54.7/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-04 13:55:15,223 - Train: 5.47% [222600/4070000] [54.7/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 13:56:15,673 - Train: 5.47% [222700/4070000] [54.7/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 13:57:16,126 - Train: 5.47% [222800/4070000] [54.7/1000.0] [batch_t 0.609 (0.604)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-04 13:58:16,675 - Train: 5.48% [222900/4070000] [54.8/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 13:59:17,119 - Train: 5.48% [223000/4070000] [54.8/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 14:00:17,841 - Train: 5.48% [223100/4070000] [54.8/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-04 14:01:18,323 - Train: 5.48% [223200/4070000] [54.8/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-04 14:02:18,788 - Train: 5.49% [223300/4070000] [54.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 14:03:19,239 - Train: 5.49% [223400/4070000] [54.9/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 14:04:19,717 - Train: 5.49% [223500/4070000] [54.9/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 14:05:20,197 - Train: 5.49% [223600/4070000] [54.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 14:06:20,702 - Train: 5.50% [223700/4070000] [55.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 14:07:21,196 - Train: 5.50% [223800/4070000] [55.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 14:07:51,441 - ==> Total time: 2 days, 13:07:22 Eta: 43 days, 18:12:06 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-04 14:08:23,412 - Train: 5.50% [223900/4070000] [55.0/1000.0] [batch_t 0.602 (0.611)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 14:09:23,891 - Train: 5.50% [224000/4070000] [55.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 14:10:24,388 - Train: 5.51% [224100/4070000] [55.1/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 14:11:25,426 - Train: 5.51% [224200/4070000] [55.1/1000.0] [batch_t 0.605 (0.610)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 14:12:25,994 - Train: 5.51% [224300/4070000] [55.1/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 14:13:26,439 - Train: 5.51% [224400/4070000] [55.1/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 14:14:26,930 - Train: 5.52% [224500/4070000] [55.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 14:15:27,396 - Train: 5.52% [224600/4070000] [55.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 14:16:27,865 - Train: 5.52% [224700/4070000] [55.2/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 14:17:28,450 - Train: 5.52% [224800/4070000] [55.2/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 14:18:28,905 - Train: 5.53% [224900/4070000] [55.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 14:19:29,424 - Train: 5.53% [225000/4070000] [55.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 14:20:29,918 - Train: 5.53% [225100/4070000] [55.3/1000.0] [batch_t 0.600 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-04 14:21:30,391 - Train: 5.53% [225200/4070000] [55.3/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 14:22:30,834 - Train: 5.54% [225300/4070000] [55.4/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 14:23:31,264 - Train: 5.54% [225400/4070000] [55.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 14:24:31,831 - Train: 5.54% [225500/4070000] [55.4/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 14:25:32,279 - Train: 5.54% [225600/4070000] [55.4/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 14:26:32,763 - Train: 5.55% [225700/4070000] [55.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 14:27:33,223 - Train: 5.55% [225800/4070000] [55.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 14:28:33,686 - Train: 5.55% [225900/4070000] [55.5/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 14:29:34,168 - Train: 5.55% [226000/4070000] [55.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 14:30:34,739 - Train: 5.56% [226100/4070000] [55.6/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 14:31:35,237 - Train: 5.56% [226200/4070000] [55.6/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-04 14:32:35,700 - Train: 5.56% [226300/4070000] [55.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 14:33:36,184 - Train: 5.56% [226400/4070000] [55.6/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 14:34:36,654 - Train: 5.57% [226500/4070000] [55.7/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 14:35:37,112 - Train: 5.57% [226600/4070000] [55.7/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-04 14:36:37,585 - Train: 5.57% [226700/4070000] [55.7/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-04 14:37:38,156 - Train: 5.57% [226800/4070000] [55.7/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 14:38:38,630 - Train: 5.57% [226900/4070000] [55.7/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 14:39:39,121 - Train: 5.58% [227000/4070000] [55.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 14:40:39,611 - Train: 5.58% [227100/4070000] [55.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 14:41:40,107 - Train: 5.58% [227200/4070000] [55.8/1000.0] [batch_t 0.600 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-04 14:42:40,552 - Train: 5.58% [227300/4070000] [55.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 14:43:41,116 - Train: 5.59% [227400/4070000] [55.9/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 14:44:41,580 - Train: 5.59% [227500/4070000] [55.9/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-04 14:45:42,039 - Train: 5.59% [227600/4070000] [55.9/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 14:46:42,513 - Train: 5.59% [227700/4070000] [55.9/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 14:47:42,991 - Train: 5.60% [227800/4070000] [56.0/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 14:48:43,468 - Train: 5.60% [227900/4070000] [56.0/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 14:48:55,571 - ==> Total time: 2 days, 13:48:26 Eta: 43 days, 9:53:42 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-04 14:49:45,487 - Train: 5.60% [228000/4070000] [56.0/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 14:50:46,086 - Train: 5.60% [228100/4070000] [56.0/1000.0] [batch_t 0.602 (0.606)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 14:51:46,526 - Train: 5.61% [228200/4070000] [56.1/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 14:52:46,981 - Train: 5.61% [228300/4070000] [56.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 14:53:47,443 - Train: 5.61% [228400/4070000] [56.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 14:54:47,909 - Train: 5.61% [228500/4070000] [56.1/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 14:55:48,479 - Train: 5.62% [228600/4070000] [56.2/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 14:56:48,945 - Train: 5.62% [228700/4070000] [56.2/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 14:57:49,402 - Train: 5.62% [228800/4070000] [56.2/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 14:58:49,854 - Train: 5.62% [228900/4070000] [56.2/1000.0] [batch_t 0.601 (0.604)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-04 14:59:50,341 - Train: 5.63% [229000/4070000] [56.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 15:00:50,834 - Train: 5.63% [229100/4070000] [56.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 15:01:51,401 - Train: 5.63% [229200/4070000] [56.3/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 15:02:51,881 - Train: 5.63% [229300/4070000] [56.3/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 15:03:52,351 - Train: 5.64% [229400/4070000] [56.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 15:04:52,832 - Train: 5.64% [229500/4070000] [56.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 15:05:53,292 - Train: 5.64% [229600/4070000] [56.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 15:06:53,780 - Train: 5.64% [229700/4070000] [56.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 15:07:54,219 - Train: 5.65% [229800/4070000] [56.5/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 15:08:54,761 - Train: 5.65% [229900/4070000] [56.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 15:09:55,227 - Train: 5.65% [230000/4070000] [56.5/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-04 15:10:55,708 - Train: 5.65% [230100/4070000] [56.5/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 15:11:56,178 - Train: 5.66% [230200/4070000] [56.6/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 15:12:56,635 - Train: 5.66% [230300/4070000] [56.6/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 15:13:57,089 - Train: 5.66% [230400/4070000] [56.6/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 15:14:57,664 - Train: 5.66% [230500/4070000] [56.6/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 15:15:58,160 - Train: 5.67% [230600/4070000] [56.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 15:16:58,643 - Train: 5.67% [230700/4070000] [56.7/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 15:17:59,083 - Train: 5.67% [230800/4070000] [56.7/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 15:18:59,522 - Train: 5.67% [230900/4070000] [56.7/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 15:19:59,998 - Train: 5.68% [231000/4070000] [56.8/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 15:21:00,489 - Train: 5.68% [231100/4070000] [56.8/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 15:22:01,052 - Train: 5.68% [231200/4070000] [56.8/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 15:23:01,610 - Train: 5.68% [231300/4070000] [56.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 15:24:02,101 - Train: 5.69% [231400/4070000] [56.9/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 15:25:02,576 - Train: 5.69% [231500/4070000] [56.9/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 15:26:03,017 - Train: 5.69% [231600/4070000] [56.9/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 15:27:03,473 - Train: 5.69% [231700/4070000] [56.9/1000.0] [batch_t 0.608 (0.604)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-04 15:28:04,042 - Train: 5.70% [231800/4070000] [57.0/1000.0] [batch_t 0.602 (0.606)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 15:29:04,499 - Train: 5.70% [231900/4070000] [57.0/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 15:29:58,937 - ==> Total time: 2 days, 14:29:29 Eta: 43 days, 1:51:08 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-04 15:30:06,491 - Train: 5.70% [232000/4070000] [57.0/1000.0] [batch_t 0.603 (0.619)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 15:31:06,963 - Train: 5.70% [232100/4070000] [57.0/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 15:32:07,457 - Train: 5.71% [232200/4070000] [57.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 15:33:07,931 - Train: 5.71% [232300/4070000] [57.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 15:34:08,519 - Train: 5.71% [232400/4070000] [57.1/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 15:35:08,996 - Train: 5.71% [232500/4070000] [57.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 15:36:09,484 - Train: 5.71% [232600/4070000] [57.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 15:37:09,963 - Train: 5.72% [232700/4070000] [57.2/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 15:38:10,439 - Train: 5.72% [232800/4070000] [57.2/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-04 15:39:10,955 - Train: 5.72% [232900/4070000] [57.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 15:40:11,524 - Train: 5.72% [233000/4070000] [57.2/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 15:41:11,997 - Train: 5.73% [233100/4070000] [57.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 15:42:12,516 - Train: 5.73% [233200/4070000] [57.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 15:43:12,962 - Train: 5.73% [233300/4070000] [57.3/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 15:44:13,491 - Train: 5.73% [233400/4070000] [57.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 15:45:13,996 - Train: 5.74% [233500/4070000] [57.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 15:46:14,477 - Train: 5.74% [233600/4070000] [57.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 15:47:15,019 - Train: 5.74% [233700/4070000] [57.4/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 15:48:15,525 - Train: 5.74% [233800/4070000] [57.4/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 15:49:15,994 - Train: 5.75% [233900/4070000] [57.5/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 15:50:16,466 - Train: 5.75% [234000/4070000] [57.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 15:51:16,922 - Train: 5.75% [234100/4070000] [57.5/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 15:52:17,352 - Train: 5.75% [234200/4070000] [57.5/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 15:53:17,901 - Train: 5.76% [234300/4070000] [57.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 15:54:18,383 - Train: 5.76% [234400/4070000] [57.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 15:55:18,871 - Train: 5.76% [234500/4070000] [57.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 15:56:19,343 - Train: 5.76% [234600/4070000] [57.6/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 15:57:19,824 - Train: 5.77% [234700/4070000] [57.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 15:58:20,303 - Train: 5.77% [234800/4070000] [57.7/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-04 15:59:20,780 - Train: 5.77% [234900/4070000] [57.7/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 16:00:21,442 - Train: 5.77% [235000/4070000] [57.7/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 16:01:21,940 - Train: 5.78% [235100/4070000] [57.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 16:02:22,434 - Train: 5.78% [235200/4070000] [57.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 16:03:22,909 - Train: 5.78% [235300/4070000] [57.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 16:04:23,352 - Train: 5.78% [235400/4070000] [57.8/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 16:05:23,823 - Train: 5.79% [235500/4070000] [57.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 16:06:24,327 - Train: 5.79% [235600/4070000] [57.9/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 16:07:24,923 - Train: 5.79% [235700/4070000] [57.9/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 16:08:25,400 - Train: 5.79% [235800/4070000] [57.9/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 16:09:25,898 - Train: 5.80% [235900/4070000] [58.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 16:10:26,367 - Train: 5.80% [236000/4070000] [58.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 16:11:02,666 - ==> Total time: 2 days, 15:10:33 Eta: 42 days, 18:03:54 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-04 16:11:29,498 - Train: 5.80% [236100/4070000] [58.0/1000.0] [batch_t 0.603 (0.609)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 16:12:29,958 - Train: 5.80% [236200/4070000] [58.0/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 16:13:30,603 - Train: 5.81% [236300/4070000] [58.1/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 16:14:31,063 - Train: 5.81% [236400/4070000] [58.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 16:15:31,519 - Train: 5.81% [236500/4070000] [58.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 16:16:31,952 - Train: 5.81% [236600/4070000] [58.1/1000.0] [batch_t 0.601 (0.604)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 16:17:38,898 - Train: 5.82% [236700/4070000] [58.2/1000.0] [batch_t 0.604 (0.669)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 16:18:39,382 - Train: 5.82% [236800/4070000] [58.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 16:19:39,948 - Train: 5.82% [236900/4070000] [58.2/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 16:20:40,415 - Train: 5.82% [237000/4070000] [58.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 16:21:40,887 - Train: 5.83% [237100/4070000] [58.3/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 16:22:41,370 - Train: 5.83% [237200/4070000] [58.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 16:23:41,820 - Train: 5.83% [237300/4070000] [58.3/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 16:24:42,325 - Train: 5.83% [237400/4070000] [58.3/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-04 16:25:42,792 - Train: 5.84% [237500/4070000] [58.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 16:26:43,372 - Train: 5.84% [237600/4070000] [58.4/1000.0] [batch_t 0.609 (0.606)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-04 16:27:43,856 - Train: 5.84% [237700/4070000] [58.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 16:28:44,319 - Train: 5.84% [237800/4070000] [58.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 16:29:46,862 - Train: 5.85% [237900/4070000] [58.5/1000.0] [batch_t 0.609 (0.625)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-04 16:30:47,322 - Train: 5.85% [238000/4070000] [58.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 16:31:47,812 - Train: 5.85% [238100/4070000] [58.5/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 16:32:48,275 - Train: 5.85% [238200/4070000] [58.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 16:33:48,900 - Train: 5.86% [238300/4070000] [58.6/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 16:34:49,416 - Train: 5.86% [238400/4070000] [58.6/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 16:35:49,931 - Train: 5.86% [238500/4070000] [58.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 16:36:50,379 - Train: 5.86% [238600/4070000] [58.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 16:37:50,848 - Train: 5.86% [238700/4070000] [58.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 16:38:51,328 - Train: 5.87% [238800/4070000] [58.7/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 16:39:51,893 - Train: 5.87% [238900/4070000] [58.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 16:40:52,383 - Train: 5.87% [239000/4070000] [58.7/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-04 16:41:52,852 - Train: 5.87% [239100/4070000] [58.7/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 16:42:53,314 - Train: 5.88% [239200/4070000] [58.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 16:43:53,765 - Train: 5.88% [239300/4070000] [58.8/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 16:44:54,264 - Train: 5.88% [239400/4070000] [58.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 16:45:54,728 - Train: 5.88% [239500/4070000] [58.8/1000.0] [batch_t 0.598 (0.605)] [data_t 0.002] [optim_t 0.596] [lr 0.005000] +2024-04-04 16:46:55,285 - Train: 5.89% [239600/4070000] [58.9/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 16:47:55,747 - Train: 5.89% [239700/4070000] [58.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 16:48:56,196 - Train: 5.89% [239800/4070000] [58.9/1000.0] [batch_t 0.598 (0.604)] [data_t 0.002] [optim_t 0.596] [lr 0.005000] +2024-04-04 16:49:56,676 - Train: 5.89% [239900/4070000] [58.9/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 16:50:57,154 - Train: 5.90% [240000/4070000] [59.0/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 16:51:57,600 - Train: 5.90% [240100/4070000] [59.0/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 16:52:15,858 - ==> Total time: 2 days, 15:51:46 Eta: 42 days, 10:33:37 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-04 16:52:59,919 - Train: 5.90% [240200/4070000] [59.0/1000.0] [batch_t 0.604 (0.608)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 16:54:00,404 - Train: 5.90% [240300/4070000] [59.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 16:55:00,975 - Train: 5.91% [240400/4070000] [59.1/1000.0] [batch_t 0.609 (0.606)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-04 16:56:01,491 - Train: 5.91% [240500/4070000] [59.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 16:57:01,958 - Train: 5.91% [240600/4070000] [59.1/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 16:58:02,629 - Train: 5.91% [240700/4070000] [59.1/1000.0] [batch_t 0.602 (0.607)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 16:59:03,115 - Train: 5.92% [240800/4070000] [59.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 17:00:03,618 - Train: 5.92% [240900/4070000] [59.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 17:01:04,098 - Train: 5.92% [241000/4070000] [59.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 17:02:04,590 - Train: 5.92% [241100/4070000] [59.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 17:03:05,084 - Train: 5.93% [241200/4070000] [59.3/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 17:04:05,596 - Train: 5.93% [241300/4070000] [59.3/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 17:05:06,141 - Train: 5.93% [241400/4070000] [59.3/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 17:06:07,177 - Train: 5.93% [241500/4070000] [59.3/1000.0] [batch_t 0.604 (0.610)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 17:07:07,685 - Train: 5.94% [241600/4070000] [59.4/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 17:08:08,168 - Train: 5.94% [241700/4070000] [59.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 17:09:08,787 - Train: 5.94% [241800/4070000] [59.4/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 17:10:09,370 - Train: 5.94% [241900/4070000] [59.4/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 17:11:09,847 - Train: 5.95% [242000/4070000] [59.5/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 17:12:10,327 - Train: 5.95% [242100/4070000] [59.5/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 17:13:10,822 - Train: 5.95% [242200/4070000] [59.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 17:14:11,291 - Train: 5.95% [242300/4070000] [59.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 17:15:11,738 - Train: 5.96% [242400/4070000] [59.6/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 17:16:12,595 - Train: 5.96% [242500/4070000] [59.6/1000.0] [batch_t 0.602 (0.608)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 17:17:13,079 - Train: 5.96% [242600/4070000] [59.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 17:18:13,680 - Train: 5.96% [242700/4070000] [59.6/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 17:19:14,158 - Train: 5.97% [242800/4070000] [59.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 17:20:14,619 - Train: 5.97% [242900/4070000] [59.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 17:21:15,106 - Train: 5.97% [243000/4070000] [59.7/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 17:22:15,578 - Train: 5.97% [243100/4070000] [59.7/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 17:23:16,197 - Train: 5.98% [243200/4070000] [59.8/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 17:24:16,664 - Train: 5.98% [243300/4070000] [59.8/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 17:25:17,155 - Train: 5.98% [243400/4070000] [59.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 17:26:17,783 - Train: 5.98% [243500/4070000] [59.8/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 17:27:18,241 - Train: 5.99% [243600/4070000] [59.9/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 17:28:18,731 - Train: 5.99% [243700/4070000] [59.9/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 17:29:19,190 - Train: 5.99% [243800/4070000] [59.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 17:30:19,682 - Train: 5.99% [243900/4070000] [59.9/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 17:31:20,303 - Train: 6.00% [244000/4070000] [60.0/1000.0] [batch_t 0.602 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 17:32:20,786 - Train: 6.00% [244100/4070000] [60.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 17:33:21,240 - Train: 6.00% [244200/4070000] [60.0/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 17:33:21,244 - ==> Total time: 2 days, 16:32:52 Eta: 42 days, 3:14:56 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-04 17:34:23,358 - Train: 6.00% [244300/4070000] [60.0/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 17:35:23,818 - Train: 6.00% [244400/4070000] [60.0/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 17:36:24,462 - Train: 6.01% [244500/4070000] [60.1/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 17:37:24,944 - Train: 6.01% [244600/4070000] [60.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 17:38:25,443 - Train: 6.01% [244700/4070000] [60.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 17:39:25,947 - Train: 6.01% [244800/4070000] [60.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 17:40:26,431 - Train: 6.02% [244900/4070000] [60.2/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 17:41:26,887 - Train: 6.02% [245000/4070000] [60.2/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 17:42:30,961 - Train: 6.02% [245100/4070000] [60.2/1000.0] [batch_t 0.605 (0.641)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 17:43:31,453 - Train: 6.02% [245200/4070000] [60.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 17:44:31,928 - Train: 6.03% [245300/4070000] [60.3/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 17:45:33,016 - Train: 6.03% [245400/4070000] [60.3/1000.0] [batch_t 0.605 (0.611)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 17:46:34,391 - Train: 6.03% [245500/4070000] [60.3/1000.0] [batch_t 0.606 (0.614)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 17:47:34,858 - Train: 6.03% [245600/4070000] [60.3/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-04 17:48:35,709 - Train: 6.04% [245700/4070000] [60.4/1000.0] [batch_t 0.604 (0.608)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 17:49:36,176 - Train: 6.04% [245800/4070000] [60.4/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-04 17:50:37,143 - Train: 6.04% [245900/4070000] [60.4/1000.0] [batch_t 0.604 (0.610)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 17:51:37,597 - Train: 6.04% [246000/4070000] [60.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 17:52:38,039 - Train: 6.05% [246100/4070000] [60.5/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 17:53:38,493 - Train: 6.05% [246200/4070000] [60.5/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 17:54:39,604 - Train: 6.05% [246300/4070000] [60.5/1000.0] [batch_t 0.601 (0.611)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-04 17:55:40,079 - Train: 6.05% [246400/4070000] [60.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 17:56:40,751 - Train: 6.06% [246500/4070000] [60.6/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 17:57:41,230 - Train: 6.06% [246600/4070000] [60.6/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 17:58:41,755 - Train: 6.06% [246700/4070000] [60.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 17:59:42,272 - Train: 6.06% [246800/4070000] [60.6/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 18:00:42,760 - Train: 6.07% [246900/4070000] [60.7/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 18:01:43,257 - Train: 6.07% [247000/4070000] [60.7/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 18:02:43,872 - Train: 6.07% [247100/4070000] [60.7/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 18:03:44,337 - Train: 6.07% [247200/4070000] [60.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 18:04:44,824 - Train: 6.08% [247300/4070000] [60.8/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-04 18:05:45,383 - Train: 6.08% [247400/4070000] [60.8/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 18:06:45,893 - Train: 6.08% [247500/4070000] [60.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 18:07:46,569 - Train: 6.08% [247600/4070000] [60.8/1000.0] [batch_t 0.602 (0.607)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 18:08:47,046 - Train: 6.09% [247700/4070000] [60.9/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 18:09:47,512 - Train: 6.09% [247800/4070000] [60.9/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 18:10:47,994 - Train: 6.09% [247900/4070000] [60.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 18:11:48,474 - Train: 6.09% [248000/4070000] [60.9/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 18:12:48,962 - Train: 6.10% [248100/4070000] [61.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 18:13:49,492 - Train: 6.10% [248200/4070000] [61.0/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 18:14:31,842 - ==> Total time: 2 days, 17:14:02 Eta: 41 days, 20:10:37 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-04 18:14:51,977 - Train: 6.10% [248300/4070000] [61.0/1000.0] [batch_t 0.604 (0.625)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 18:15:52,424 - Train: 6.10% [248400/4070000] [61.0/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 18:16:52,859 - Train: 6.11% [248500/4070000] [61.1/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 18:17:53,300 - Train: 6.11% [248600/4070000] [61.1/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 18:18:53,731 - Train: 6.11% [248700/4070000] [61.1/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 18:19:54,188 - Train: 6.11% [248800/4070000] [61.1/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 18:20:54,722 - Train: 6.12% [248900/4070000] [61.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 18:21:55,196 - Train: 6.12% [249000/4070000] [61.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 18:22:55,662 - Train: 6.12% [249100/4070000] [61.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 18:23:56,117 - Train: 6.12% [249200/4070000] [61.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 18:24:56,582 - Train: 6.13% [249300/4070000] [61.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 18:25:57,086 - Train: 6.13% [249400/4070000] [61.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 18:26:57,621 - Train: 6.13% [249500/4070000] [61.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 18:27:58,156 - Train: 6.13% [249600/4070000] [61.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 18:28:58,645 - Train: 6.14% [249700/4070000] [61.4/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 18:29:59,157 - Train: 6.14% [249800/4070000] [61.4/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 18:30:59,621 - Train: 6.14% [249900/4070000] [61.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 18:32:00,104 - Train: 6.14% [250000/4070000] [61.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 18:33:00,570 - Train: 6.14% [250100/4070000] [61.4/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 18:34:01,102 - Train: 6.15% [250200/4070000] [61.5/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 18:35:01,559 - Train: 6.15% [250300/4070000] [61.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 18:36:02,024 - Train: 6.15% [250400/4070000] [61.5/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 18:37:02,481 - Train: 6.15% [250500/4070000] [61.5/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 18:38:02,945 - Train: 6.16% [250600/4070000] [61.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 18:39:03,419 - Train: 6.16% [250700/4070000] [61.6/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 18:40:03,894 - Train: 6.16% [250800/4070000] [61.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 18:41:04,466 - Train: 6.16% [250900/4070000] [61.6/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 18:42:04,922 - Train: 6.17% [251000/4070000] [61.7/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 18:43:05,393 - Train: 6.17% [251100/4070000] [61.7/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 18:44:05,873 - Train: 6.17% [251200/4070000] [61.7/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 18:45:06,326 - Train: 6.17% [251300/4070000] [61.7/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 18:46:06,803 - Train: 6.18% [251400/4070000] [61.8/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 18:47:07,340 - Train: 6.18% [251500/4070000] [61.8/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 18:48:07,809 - Train: 6.18% [251600/4070000] [61.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 18:49:08,261 - Train: 6.18% [251700/4070000] [61.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 18:50:08,741 - Train: 6.19% [251800/4070000] [61.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 18:51:09,206 - Train: 6.19% [251900/4070000] [61.9/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 18:52:09,710 - Train: 6.19% [252000/4070000] [61.9/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 18:53:10,182 - Train: 6.19% [252100/4070000] [61.9/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 18:54:10,649 - Train: 6.20% [252200/4070000] [62.0/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-04 18:55:11,104 - Train: 6.20% [252300/4070000] [62.0/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 18:55:35,283 - ==> Total time: 2 days, 17:55:06 Eta: 41 days, 13:16:52 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-04 18:56:13,211 - Train: 6.20% [252400/4070000] [62.0/1000.0] [batch_t 0.607 (0.609)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 18:57:13,707 - Train: 6.20% [252500/4070000] [62.0/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 18:58:14,195 - Train: 6.21% [252600/4070000] [62.1/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 18:59:14,779 - Train: 6.21% [252700/4070000] [62.1/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 19:00:15,227 - Train: 6.21% [252800/4070000] [62.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 19:01:15,702 - Train: 6.21% [252900/4070000] [62.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 19:02:16,172 - Train: 6.22% [253000/4070000] [62.2/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-04 19:03:16,635 - Train: 6.22% [253100/4070000] [62.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 19:04:17,136 - Train: 6.22% [253200/4070000] [62.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 19:05:17,675 - Train: 6.22% [253300/4070000] [62.2/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 19:06:18,116 - Train: 6.23% [253400/4070000] [62.3/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 19:07:18,608 - Train: 6.23% [253500/4070000] [62.3/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 19:08:19,077 - Train: 6.23% [253600/4070000] [62.3/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 19:09:19,532 - Train: 6.23% [253700/4070000] [62.3/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 19:10:19,977 - Train: 6.24% [253800/4070000] [62.4/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 19:11:20,443 - Train: 6.24% [253900/4070000] [62.4/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-04 19:12:20,962 - Train: 6.24% [254000/4070000] [62.4/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 19:13:21,416 - Train: 6.24% [254100/4070000] [62.4/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 19:14:21,887 - Train: 6.25% [254200/4070000] [62.5/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 19:15:22,352 - Train: 6.25% [254300/4070000] [62.5/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 19:16:22,801 - Train: 6.25% [254400/4070000] [62.5/1000.0] [batch_t 0.601 (0.604)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-04 19:17:23,294 - Train: 6.25% [254500/4070000] [62.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 19:18:23,885 - Train: 6.26% [254600/4070000] [62.6/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 19:19:24,367 - Train: 6.26% [254700/4070000] [62.6/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 19:20:24,846 - Train: 6.26% [254800/4070000] [62.6/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 19:21:25,306 - Train: 6.26% [254900/4070000] [62.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 19:22:25,781 - Train: 6.27% [255000/4070000] [62.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 19:23:26,341 - Train: 6.27% [255100/4070000] [62.7/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 19:24:26,802 - Train: 6.27% [255200/4070000] [62.7/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 19:25:27,257 - Train: 6.27% [255300/4070000] [62.7/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 19:26:27,730 - Train: 6.28% [255400/4070000] [62.8/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 19:27:28,160 - Train: 6.28% [255500/4070000] [62.8/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 19:28:28,621 - Train: 6.28% [255600/4070000] [62.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 19:29:29,070 - Train: 6.28% [255700/4070000] [62.8/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 19:30:29,557 - Train: 6.29% [255800/4070000] [62.9/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 19:31:30,103 - Train: 6.29% [255900/4070000] [62.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 19:32:30,559 - Train: 6.29% [256000/4070000] [62.9/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 19:33:31,027 - Train: 6.29% [256100/4070000] [62.9/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 19:34:31,530 - Train: 6.29% [256200/4070000] [62.9/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 19:35:31,984 - Train: 6.30% [256300/4070000] [63.0/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 19:36:32,612 - Train: 6.30% [256400/4070000] [63.0/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 19:36:38,672 - ==> Total time: 2 days, 18:36:09 Eta: 41 days, 6:34:56 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-04 19:37:34,623 - Train: 6.30% [256500/4070000] [63.0/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 19:38:35,104 - Train: 6.30% [256600/4070000] [63.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 19:39:35,569 - Train: 6.31% [256700/4070000] [63.1/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 19:40:36,015 - Train: 6.31% [256800/4070000] [63.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 19:41:36,606 - Train: 6.31% [256900/4070000] [63.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 19:42:37,071 - Train: 6.31% [257000/4070000] [63.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 19:43:37,535 - Train: 6.32% [257100/4070000] [63.2/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-04 19:44:38,026 - Train: 6.32% [257200/4070000] [63.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 19:45:38,508 - Train: 6.32% [257300/4070000] [63.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 19:46:38,975 - Train: 6.32% [257400/4070000] [63.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 19:47:39,454 - Train: 6.33% [257500/4070000] [63.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 19:48:39,952 - Train: 6.33% [257600/4070000] [63.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 19:49:40,509 - Train: 6.33% [257700/4070000] [63.3/1000.0] [batch_t 0.599 (0.605)] [data_t 0.002] [optim_t 0.597] [lr 0.005000] +2024-04-04 19:50:40,957 - Train: 6.33% [257800/4070000] [63.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 19:51:41,419 - Train: 6.34% [257900/4070000] [63.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 19:52:41,833 - Train: 6.34% [258000/4070000] [63.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 19:53:42,299 - Train: 6.34% [258100/4070000] [63.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 19:54:42,865 - Train: 6.34% [258200/4070000] [63.4/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 19:55:43,354 - Train: 6.35% [258300/4070000] [63.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 19:56:43,792 - Train: 6.35% [258400/4070000] [63.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 19:57:44,247 - Train: 6.35% [258500/4070000] [63.5/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 19:58:44,705 - Train: 6.35% [258600/4070000] [63.5/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 19:59:45,172 - Train: 6.36% [258700/4070000] [63.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 20:00:45,658 - Train: 6.36% [258800/4070000] [63.6/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 20:01:46,137 - Train: 6.36% [258900/4070000] [63.6/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 20:02:46,681 - Train: 6.36% [259000/4070000] [63.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 20:03:47,145 - Train: 6.37% [259100/4070000] [63.7/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-04 20:04:47,587 - Train: 6.37% [259200/4070000] [63.7/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 20:05:48,042 - Train: 6.37% [259300/4070000] [63.7/1000.0] [batch_t 0.608 (0.604)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-04 20:06:48,496 - Train: 6.37% [259400/4070000] [63.7/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 20:07:51,415 - Train: 6.38% [259500/4070000] [63.8/1000.0] [batch_t 0.605 (0.629)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 20:08:51,869 - Train: 6.38% [259600/4070000] [63.8/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 20:09:52,351 - Train: 6.38% [259700/4070000] [63.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 20:10:52,830 - Train: 6.38% [259800/4070000] [63.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 20:11:54,551 - Train: 6.39% [259900/4070000] [63.9/1000.0] [batch_t 0.604 (0.617)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 20:12:55,036 - Train: 6.39% [260000/4070000] [63.9/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-04 20:13:55,515 - Train: 6.39% [260100/4070000] [63.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 20:14:55,985 - Train: 6.39% [260200/4070000] [63.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 20:15:56,550 - Train: 6.40% [260300/4070000] [64.0/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 20:16:57,044 - Train: 6.40% [260400/4070000] [64.0/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 20:17:45,430 - ==> Total time: 2 days, 19:17:16 Eta: 41 days, 0:05:05 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-04 20:17:59,313 - Train: 6.40% [260500/4070000] [64.0/1000.0] [batch_t 0.606 (0.627)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 20:18:59,785 - Train: 6.40% [260600/4070000] [64.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 20:20:00,248 - Train: 6.41% [260700/4070000] [64.1/1000.0] [batch_t 0.598 (0.605)] [data_t 0.002] [optim_t 0.596] [lr 0.005000] +2024-04-04 20:21:00,834 - Train: 6.41% [260800/4070000] [64.1/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 20:22:01,300 - Train: 6.41% [260900/4070000] [64.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 20:23:01,803 - Train: 6.41% [261000/4070000] [64.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 20:24:02,318 - Train: 6.42% [261100/4070000] [64.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 20:25:02,813 - Train: 6.42% [261200/4070000] [64.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 20:26:03,351 - Train: 6.42% [261300/4070000] [64.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 20:27:04,000 - Train: 6.42% [261400/4070000] [64.2/1000.0] [batch_t 0.602 (0.606)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 20:28:04,480 - Train: 6.43% [261500/4070000] [64.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 20:29:04,964 - Train: 6.43% [261600/4070000] [64.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 20:30:05,474 - Train: 6.43% [261700/4070000] [64.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 20:31:05,983 - Train: 6.43% [261800/4070000] [64.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 20:32:06,479 - Train: 6.43% [261900/4070000] [64.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 20:33:07,084 - Train: 6.44% [262000/4070000] [64.4/1000.0] [batch_t 0.609 (0.606)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-04 20:34:07,533 - Train: 6.44% [262100/4070000] [64.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 20:35:08,024 - Train: 6.44% [262200/4070000] [64.4/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 20:36:08,542 - Train: 6.44% [262300/4070000] [64.4/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-04 20:37:09,025 - Train: 6.45% [262400/4070000] [64.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 20:38:09,504 - Train: 6.45% [262500/4070000] [64.5/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 20:39:09,983 - Train: 6.45% [262600/4070000] [64.5/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 20:40:10,438 - Train: 6.45% [262700/4070000] [64.5/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 20:41:11,066 - Train: 6.46% [262800/4070000] [64.6/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 20:42:11,571 - Train: 6.46% [262900/4070000] [64.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 20:43:12,078 - Train: 6.46% [263000/4070000] [64.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 20:44:12,529 - Train: 6.46% [263100/4070000] [64.6/1000.0] [batch_t 0.592 (0.604)] [data_t 0.002] [optim_t 0.590] [lr 0.005000] +2024-04-04 20:45:12,965 - Train: 6.47% [263200/4070000] [64.7/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 20:46:13,480 - Train: 6.47% [263300/4070000] [64.7/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-04 20:47:14,104 - Train: 6.47% [263400/4070000] [64.7/1000.0] [batch_t 0.609 (0.606)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-04 20:48:14,594 - Train: 6.47% [263500/4070000] [64.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 20:49:15,031 - Train: 6.48% [263600/4070000] [64.8/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 20:50:15,519 - Train: 6.48% [263700/4070000] [64.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 20:51:15,991 - Train: 6.48% [263800/4070000] [64.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 20:52:16,550 - Train: 6.48% [263900/4070000] [64.8/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 20:53:17,047 - Train: 6.49% [264000/4070000] [64.9/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 20:54:17,520 - Train: 6.49% [264100/4070000] [64.9/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 20:55:17,980 - Train: 6.49% [264200/4070000] [64.9/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 20:56:18,468 - Train: 6.49% [264300/4070000] [64.9/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 20:57:18,964 - Train: 6.50% [264400/4070000] [65.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 20:58:19,443 - Train: 6.50% [264500/4070000] [65.0/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 20:58:49,681 - ==> Total time: 2 days, 19:58:20 Eta: 40 days, 17:45:23 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-04 20:59:21,949 - Train: 6.50% [264600/4070000] [65.0/1000.0] [batch_t 0.603 (0.617)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 21:00:22,459 - Train: 6.50% [264700/4070000] [65.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 21:01:22,962 - Train: 6.51% [264800/4070000] [65.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 21:02:23,462 - Train: 6.51% [264900/4070000] [65.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 21:03:23,944 - Train: 6.51% [265000/4070000] [65.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 21:04:24,420 - Train: 6.51% [265100/4070000] [65.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 21:05:24,890 - Train: 6.52% [265200/4070000] [65.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 21:06:25,396 - Train: 6.52% [265300/4070000] [65.2/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 21:07:25,970 - Train: 6.52% [265400/4070000] [65.2/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 21:08:26,438 - Train: 6.52% [265500/4070000] [65.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 21:09:26,914 - Train: 6.53% [265600/4070000] [65.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 21:10:27,409 - Train: 6.53% [265700/4070000] [65.3/1000.0] [batch_t 0.600 (0.605)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-04 21:11:27,898 - Train: 6.53% [265800/4070000] [65.3/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 21:12:28,476 - Train: 6.53% [265900/4070000] [65.3/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 21:13:28,961 - Train: 6.54% [266000/4070000] [65.4/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 21:14:30,414 - Train: 6.54% [266100/4070000] [65.4/1000.0] [batch_t 0.606 (0.614)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 21:15:30,894 - Train: 6.54% [266200/4070000] [65.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 21:16:31,389 - Train: 6.54% [266300/4070000] [65.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 21:17:32,849 - Train: 6.55% [266400/4070000] [65.5/1000.0] [batch_t 0.602 (0.615)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 21:18:33,299 - Train: 6.55% [266500/4070000] [65.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 21:19:47,573 - Train: 6.55% [266600/4070000] [65.5/1000.0] [batch_t 0.606 (0.743)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 21:20:48,110 - Train: 6.55% [266700/4070000] [65.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 21:21:48,630 - Train: 6.56% [266800/4070000] [65.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 21:22:49,180 - Train: 6.56% [266900/4070000] [65.6/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-04 21:23:49,711 - Train: 6.56% [267000/4070000] [65.6/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-04 21:25:01,653 - Train: 6.56% [267100/4070000] [65.6/1000.0] [batch_t 0.605 (0.719)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 21:26:02,264 - Train: 6.57% [267200/4070000] [65.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 21:27:02,785 - Train: 6.57% [267300/4070000] [65.7/1000.0] [batch_t 0.607 (0.605)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-04 21:28:03,334 - Train: 6.57% [267400/4070000] [65.7/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 21:29:03,851 - Train: 6.57% [267500/4070000] [65.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 21:30:04,358 - Train: 6.57% [267600/4070000] [65.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 21:31:04,885 - Train: 6.58% [267700/4070000] [65.8/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 21:32:05,803 - Train: 6.58% [267800/4070000] [65.8/1000.0] [batch_t 0.605 (0.609)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 21:33:06,316 - Train: 6.58% [267900/4070000] [65.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 21:34:06,854 - Train: 6.58% [268000/4070000] [65.8/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 21:35:07,829 - Train: 6.59% [268100/4070000] [65.9/1000.0] [batch_t 0.605 (0.610)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 21:36:08,350 - Train: 6.59% [268200/4070000] [65.9/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-04 21:37:09,013 - Train: 6.59% [268300/4070000] [65.9/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 21:38:09,554 - Train: 6.59% [268400/4070000] [65.9/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 21:39:10,079 - Train: 6.60% [268500/4070000] [66.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 21:40:10,615 - Train: 6.60% [268600/4070000] [66.0/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 21:40:22,730 - ==> Total time: 2 days, 20:39:53 Eta: 40 days, 11:42:43 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-04 21:41:13,459 - Train: 6.60% [268700/4070000] [66.0/1000.0] [batch_t 0.603 (0.616)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 21:42:13,961 - Train: 6.60% [268800/4070000] [66.0/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 21:43:14,442 - Train: 6.61% [268900/4070000] [66.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 21:44:21,832 - Train: 6.61% [269000/4070000] [66.1/1000.0] [batch_t 0.735 (0.674)] [data_t 0.002] [optim_t 0.733] [lr 0.005000] +2024-04-04 21:45:22,335 - Train: 6.61% [269100/4070000] [66.1/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 21:46:22,844 - Train: 6.61% [269200/4070000] [66.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 21:47:23,370 - Train: 6.62% [269300/4070000] [66.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 21:48:23,870 - Train: 6.62% [269400/4070000] [66.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 21:49:24,365 - Train: 6.62% [269500/4070000] [66.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 21:50:25,891 - Train: 6.62% [269600/4070000] [66.2/1000.0] [batch_t 0.605 (0.615)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 21:51:26,387 - Train: 6.63% [269700/4070000] [66.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 21:52:26,863 - Train: 6.63% [269800/4070000] [66.3/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 21:53:27,362 - Train: 6.63% [269900/4070000] [66.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 21:54:27,874 - Train: 6.63% [270000/4070000] [66.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 21:55:28,369 - Train: 6.64% [270100/4070000] [66.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 21:56:28,882 - Train: 6.64% [270200/4070000] [66.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 21:57:29,445 - Train: 6.64% [270300/4070000] [66.4/1000.0] [batch_t 0.601 (0.606)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-04 21:58:29,969 - Train: 6.64% [270400/4070000] [66.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 21:59:30,476 - Train: 6.65% [270500/4070000] [66.5/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 22:00:30,958 - Train: 6.65% [270600/4070000] [66.5/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 22:01:31,456 - Train: 6.65% [270700/4070000] [66.5/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 22:02:31,960 - Train: 6.65% [270800/4070000] [66.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 22:03:32,558 - Train: 6.66% [270900/4070000] [66.6/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 22:04:33,076 - Train: 6.66% [271000/4070000] [66.6/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 22:05:33,804 - Train: 6.66% [271100/4070000] [66.6/1000.0] [batch_t 0.603 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 22:06:34,321 - Train: 6.66% [271200/4070000] [66.6/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-04 22:07:34,813 - Train: 6.67% [271300/4070000] [66.7/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 22:08:35,326 - Train: 6.67% [271400/4070000] [66.7/1000.0] [batch_t 0.599 (0.605)] [data_t 0.002] [optim_t 0.597] [lr 0.005000] +2024-04-04 22:09:35,837 - Train: 6.67% [271500/4070000] [66.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 22:10:36,516 - Train: 6.67% [271600/4070000] [66.7/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 22:11:37,016 - Train: 6.68% [271700/4070000] [66.8/1000.0] [batch_t 0.600 (0.605)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-04 22:12:37,537 - Train: 6.68% [271800/4070000] [66.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 22:13:38,097 - Train: 6.68% [271900/4070000] [66.8/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 22:14:38,630 - Train: 6.68% [272000/4070000] [66.8/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 22:15:39,174 - Train: 6.69% [272100/4070000] [66.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 22:16:39,813 - Train: 6.69% [272200/4070000] [66.9/1000.0] [batch_t 0.605 (0.606)] [data_t 0.003] [optim_t 0.602] [lr 0.005000] +2024-04-04 22:17:40,683 - Train: 6.69% [272300/4070000] [66.9/1000.0] [batch_t 0.605 (0.609)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 22:18:41,216 - Train: 6.69% [272400/4070000] [66.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 22:19:41,730 - Train: 6.70% [272500/4070000] [67.0/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 22:20:42,223 - Train: 6.70% [272600/4070000] [67.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 22:21:36,656 - ==> Total time: 2 days, 21:21:07 Eta: 40 days, 5:45:13 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-04 22:21:44,338 - Train: 6.70% [272700/4070000] [67.0/1000.0] [batch_t 0.604 (0.628)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 22:22:44,958 - Train: 6.70% [272800/4070000] [67.0/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 22:23:45,461 - Train: 6.71% [272900/4070000] [67.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 22:24:45,953 - Train: 6.71% [273000/4070000] [67.1/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 22:25:46,423 - Train: 6.71% [273100/4070000] [67.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 22:26:46,921 - Train: 6.71% [273200/4070000] [67.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 22:27:47,424 - Train: 6.71% [273300/4070000] [67.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 22:28:47,917 - Train: 6.72% [273400/4070000] [67.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 22:29:48,500 - Train: 6.72% [273500/4070000] [67.2/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 22:30:48,984 - Train: 6.72% [273600/4070000] [67.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 22:31:49,505 - Train: 6.72% [273700/4070000] [67.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 22:32:50,540 - Train: 6.73% [273800/4070000] [67.3/1000.0] [batch_t 0.609 (0.610)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-04 22:33:51,021 - Train: 6.73% [273900/4070000] [67.3/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 22:34:51,471 - Train: 6.73% [274000/4070000] [67.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 22:35:51,945 - Train: 6.73% [274100/4070000] [67.3/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-04 22:36:52,420 - Train: 6.74% [274200/4070000] [67.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 22:37:52,961 - Train: 6.74% [274300/4070000] [67.4/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 22:38:53,408 - Train: 6.74% [274400/4070000] [67.4/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 22:39:53,913 - Train: 6.74% [274500/4070000] [67.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 22:40:54,442 - Train: 6.75% [274600/4070000] [67.5/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 22:41:54,983 - Train: 6.75% [274700/4070000] [67.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 22:42:55,747 - Train: 6.75% [274800/4070000] [67.5/1000.0] [batch_t 0.603 (0.608)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 22:43:56,313 - Train: 6.75% [274900/4070000] [67.5/1000.0] [batch_t 0.599 (0.606)] [data_t 0.002] [optim_t 0.597] [lr 0.005000] +2024-04-04 22:44:56,836 - Train: 6.76% [275000/4070000] [67.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 22:45:57,436 - Train: 6.76% [275100/4070000] [67.6/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 22:46:58,029 - Train: 6.76% [275200/4070000] [67.6/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 22:47:58,557 - Train: 6.76% [275300/4070000] [67.6/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 22:48:59,228 - Train: 6.77% [275400/4070000] [67.7/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 22:49:59,768 - Train: 6.77% [275500/4070000] [67.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 22:51:00,256 - Train: 6.77% [275600/4070000] [67.7/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 22:52:00,736 - Train: 6.77% [275700/4070000] [67.7/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 22:53:01,305 - Train: 6.78% [275800/4070000] [67.8/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 22:54:01,804 - Train: 6.78% [275900/4070000] [67.8/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 22:55:02,355 - Train: 6.78% [276000/4070000] [67.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 22:56:02,862 - Train: 6.78% [276100/4070000] [67.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 22:57:03,484 - Train: 6.79% [276200/4070000] [67.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 22:58:03,996 - Train: 6.79% [276300/4070000] [67.9/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 22:59:04,581 - Train: 6.79% [276400/4070000] [67.9/1000.0] [batch_t 0.602 (0.606)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 23:00:05,110 - Train: 6.79% [276500/4070000] [67.9/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 23:01:05,609 - Train: 6.80% [276600/4070000] [68.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 23:02:08,279 - Train: 6.80% [276700/4070000] [68.0/1000.0] [batch_t 0.605 (0.627)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 23:02:44,575 - ==> Total time: 2 days, 22:02:15 Eta: 39 days, 23:55:38 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-04 23:03:12,059 - Train: 6.80% [276800/4070000] [68.0/1000.0] [batch_t 0.599 (0.649)] [data_t 0.002] [optim_t 0.597] [lr 0.005000] +2024-04-04 23:04:12,551 - Train: 6.80% [276900/4070000] [68.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 23:05:14,380 - Train: 6.81% [277000/4070000] [68.1/1000.0] [batch_t 0.607 (0.618)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 23:06:34,227 - Train: 6.81% [277100/4070000] [68.1/1000.0] [batch_t 0.607 (0.798)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 23:07:35,883 - Train: 6.81% [277200/4070000] [68.1/1000.0] [batch_t 0.606 (0.616)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 23:08:48,595 - Train: 6.81% [277300/4070000] [68.1/1000.0] [batch_t 0.602 (0.727)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 23:09:52,487 - Train: 6.82% [277400/4070000] [68.2/1000.0] [batch_t 0.605 (0.639)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 23:11:33,870 - Train: 6.82% [277500/4070000] [68.2/1000.0] [batch_t 0.604 (1.014)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 23:13:25,642 - Train: 6.82% [277600/4070000] [68.2/1000.0] [batch_t 0.606 (1.118)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 23:14:42,669 - Train: 6.82% [277700/4070000] [68.2/1000.0] [batch_t 0.605 (0.770)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 23:15:44,056 - Train: 6.83% [277800/4070000] [68.3/1000.0] [batch_t 0.604 (0.614)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 23:16:45,785 - Train: 6.83% [277900/4070000] [68.3/1000.0] [batch_t 0.604 (0.617)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 23:17:47,938 - Train: 6.83% [278000/4070000] [68.3/1000.0] [batch_t 0.606 (0.621)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 23:18:49,284 - Train: 6.83% [278100/4070000] [68.3/1000.0] [batch_t 0.603 (0.613)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 23:19:49,909 - Train: 6.84% [278200/4070000] [68.4/1000.0] [batch_t 0.602 (0.606)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 23:20:50,443 - Train: 6.84% [278300/4070000] [68.4/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 23:21:51,182 - Train: 6.84% [278400/4070000] [68.4/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 23:22:51,673 - Train: 6.84% [278500/4070000] [68.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 23:23:53,274 - Train: 6.85% [278600/4070000] [68.5/1000.0] [batch_t 0.604 (0.616)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 23:24:53,749 - Train: 6.85% [278700/4070000] [68.5/1000.0] [batch_t 0.586 (0.605)] [data_t 0.002] [optim_t 0.585] [lr 0.005000] +2024-04-04 23:25:54,322 - Train: 6.85% [278800/4070000] [68.5/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 23:26:54,821 - Train: 6.85% [278900/4070000] [68.5/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 23:27:55,325 - Train: 6.86% [279000/4070000] [68.6/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 23:28:56,612 - Train: 6.86% [279100/4070000] [68.6/1000.0] [batch_t 0.604 (0.613)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 23:29:57,281 - Train: 6.86% [279200/4070000] [68.6/1000.0] [batch_t 0.608 (0.607)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-04 23:30:57,745 - Train: 6.86% [279300/4070000] [68.6/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 23:31:58,223 - Train: 6.86% [279400/4070000] [68.6/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 23:32:59,529 - Train: 6.87% [279500/4070000] [68.7/1000.0] [batch_t 0.606 (0.613)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 23:34:00,023 - Train: 6.87% [279600/4070000] [68.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 23:35:00,521 - Train: 6.87% [279700/4070000] [68.7/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 23:36:01,004 - Train: 6.87% [279800/4070000] [68.7/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 23:37:01,596 - Train: 6.88% [279900/4070000] [68.8/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 23:38:02,070 - Train: 6.88% [280000/4070000] [68.8/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 23:39:02,597 - Train: 6.88% [280100/4070000] [68.8/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 23:40:03,097 - Train: 6.88% [280200/4070000] [68.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 23:41:03,626 - Train: 6.89% [280300/4070000] [68.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 23:42:04,139 - Train: 6.89% [280400/4070000] [68.9/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 23:43:04,662 - Train: 6.89% [280500/4070000] [68.9/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 23:44:05,234 - Train: 6.89% [280600/4070000] [68.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 23:45:05,732 - Train: 6.90% [280700/4070000] [69.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 23:46:06,853 - Train: 6.90% [280800/4070000] [69.0/1000.0] [batch_t 0.601 (0.611)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-04 23:46:24,998 - ==> Total time: 2 days, 22:45:55 Eta: 39 days, 18:49:17 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-04 23:47:08,948 - Train: 6.90% [280900/4070000] [69.0/1000.0] [batch_t 0.592 (0.608)] [data_t 0.002] [optim_t 0.590] [lr 0.005000] +2024-04-04 23:48:09,469 - Train: 6.90% [281000/4070000] [69.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 23:49:10,009 - Train: 6.91% [281100/4070000] [69.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 23:50:10,650 - Train: 6.91% [281200/4070000] [69.1/1000.0] [batch_t 0.609 (0.606)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-04 23:51:11,146 - Train: 6.91% [281300/4070000] [69.1/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 23:52:11,612 - Train: 6.91% [281400/4070000] [69.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 23:53:12,094 - Train: 6.92% [281500/4070000] [69.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 23:54:12,594 - Train: 6.92% [281600/4070000] [69.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 23:55:13,109 - Train: 6.92% [281700/4070000] [69.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 23:56:13,759 - Train: 6.92% [281800/4070000] [69.2/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 23:57:14,281 - Train: 6.93% [281900/4070000] [69.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 23:58:14,806 - Train: 6.93% [282000/4070000] [69.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 23:59:15,269 - Train: 6.93% [282100/4070000] [69.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 00:00:15,789 - Train: 6.93% [282200/4070000] [69.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 00:01:16,328 - Train: 6.94% [282300/4070000] [69.4/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 00:02:16,909 - Train: 6.94% [282400/4070000] [69.4/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 00:03:17,583 - Train: 6.94% [282500/4070000] [69.4/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 00:04:18,075 - Train: 6.94% [282600/4070000] [69.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 00:05:19,308 - Train: 6.95% [282700/4070000] [69.5/1000.0] [batch_t 0.604 (0.612)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 00:06:19,800 - Train: 6.95% [282800/4070000] [69.5/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 00:07:20,342 - Train: 6.95% [282900/4070000] [69.5/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 00:08:20,874 - Train: 6.95% [283000/4070000] [69.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 00:09:21,567 - Train: 6.96% [283100/4070000] [69.6/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 00:10:22,071 - Train: 6.96% [283200/4070000] [69.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 00:11:22,576 - Train: 6.96% [283300/4070000] [69.6/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 00:12:23,104 - Train: 6.96% [283400/4070000] [69.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 00:13:23,564 - Train: 6.97% [283500/4070000] [69.7/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 00:14:24,063 - Train: 6.97% [283600/4070000] [69.7/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 00:15:24,563 - Train: 6.97% [283700/4070000] [69.7/1000.0] [batch_t 0.600 (0.605)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-05 00:16:25,113 - Train: 6.97% [283800/4070000] [69.7/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 00:17:25,643 - Train: 6.98% [283900/4070000] [69.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 00:18:26,177 - Train: 6.98% [284000/4070000] [69.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 00:19:26,699 - Train: 6.98% [284100/4070000] [69.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 00:20:27,176 - Train: 6.98% [284200/4070000] [69.8/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 00:21:27,696 - Train: 6.99% [284300/4070000] [69.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 00:22:28,427 - Train: 6.99% [284400/4070000] [69.9/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 00:23:28,938 - Train: 6.99% [284500/4070000] [69.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 00:24:29,485 - Train: 6.99% [284600/4070000] [69.9/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 00:25:29,978 - Train: 7.00% [284700/4070000] [70.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 00:26:30,495 - Train: 7.00% [284800/4070000] [70.0/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 00:27:30,994 - Train: 7.00% [284900/4070000] [70.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 00:27:30,998 - ==> Total time: 2 days, 23:27:01 Eta: 39 days, 13:16:16 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-05 00:28:33,164 - Train: 7.00% [285000/4070000] [70.0/1000.0] [batch_t 0.601 (0.608)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-05 00:29:33,674 - Train: 7.00% [285100/4070000] [70.0/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 00:30:34,187 - Train: 7.01% [285200/4070000] [70.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 00:31:34,703 - Train: 7.01% [285300/4070000] [70.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 00:32:35,225 - Train: 7.01% [285400/4070000] [70.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 00:33:35,693 - Train: 7.01% [285500/4070000] [70.1/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-05 00:34:36,330 - Train: 7.02% [285600/4070000] [70.2/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 00:35:36,825 - Train: 7.02% [285700/4070000] [70.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 00:36:37,299 - Train: 7.02% [285800/4070000] [70.2/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-05 00:37:37,796 - Train: 7.02% [285900/4070000] [70.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 00:38:38,280 - Train: 7.03% [286000/4070000] [70.3/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 00:39:38,791 - Train: 7.03% [286100/4070000] [70.3/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 00:40:39,317 - Train: 7.03% [286200/4070000] [70.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 00:41:39,903 - Train: 7.03% [286300/4070000] [70.3/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 00:42:40,384 - Train: 7.04% [286400/4070000] [70.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 00:43:40,884 - Train: 7.04% [286500/4070000] [70.4/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 00:44:41,360 - Train: 7.04% [286600/4070000] [70.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 00:45:41,879 - Train: 7.04% [286700/4070000] [70.4/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-05 00:46:42,437 - Train: 7.05% [286800/4070000] [70.5/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 00:47:43,138 - Train: 7.05% [286900/4070000] [70.5/1000.0] [batch_t 0.609 (0.607)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-05 00:48:43,632 - Train: 7.05% [287000/4070000] [70.5/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 00:49:44,133 - Train: 7.05% [287100/4070000] [70.5/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-05 00:50:44,612 - Train: 7.06% [287200/4070000] [70.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 00:51:45,113 - Train: 7.06% [287300/4070000] [70.6/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 00:52:45,562 - Train: 7.06% [287400/4070000] [70.6/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 00:53:46,080 - Train: 7.06% [287500/4070000] [70.6/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 00:54:46,735 - Train: 7.07% [287600/4070000] [70.7/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 00:55:47,250 - Train: 7.07% [287700/4070000] [70.7/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 00:56:47,750 - Train: 7.07% [287800/4070000] [70.7/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 00:57:48,260 - Train: 7.07% [287900/4070000] [70.7/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 00:58:48,775 - Train: 7.08% [288000/4070000] [70.8/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 00:59:49,261 - Train: 7.08% [288100/4070000] [70.8/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 01:00:49,937 - Train: 7.08% [288200/4070000] [70.8/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 01:01:50,413 - Train: 7.08% [288300/4070000] [70.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 01:02:50,924 - Train: 7.09% [288400/4070000] [70.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 01:03:51,477 - Train: 7.09% [288500/4070000] [70.9/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 01:04:51,997 - Train: 7.09% [288600/4070000] [70.9/1000.0] [batch_t 0.611 (0.605)] [data_t 0.002] [optim_t 0.609] [lr 0.005000] +2024-04-05 01:05:52,496 - Train: 7.09% [288700/4070000] [70.9/1000.0] [batch_t 0.610 (0.605)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-05 01:06:53,008 - Train: 7.10% [288800/4070000] [71.0/1000.0] [batch_t 0.597 (0.605)] [data_t 0.002] [optim_t 0.595] [lr 0.005000] +2024-04-05 01:07:53,698 - Train: 7.10% [288900/4070000] [71.0/1000.0] [batch_t 0.602 (0.607)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-05 01:08:36,057 - ==> Total time: 3 days, 0:08:06 Eta: 39 days, 7:51:15 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-05 01:08:56,166 - Train: 7.10% [289000/4070000] [71.0/1000.0] [batch_t 0.609 (0.613)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-05 01:09:56,628 - Train: 7.10% [289100/4070000] [71.0/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 01:10:57,126 - Train: 7.11% [289200/4070000] [71.1/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 01:11:57,574 - Train: 7.11% [289300/4070000] [71.1/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 01:12:58,245 - Train: 7.11% [289400/4070000] [71.1/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 01:13:58,737 - Train: 7.11% [289500/4070000] [71.1/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 01:14:59,202 - Train: 7.12% [289600/4070000] [71.2/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 01:15:59,672 - Train: 7.12% [289700/4070000] [71.2/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 01:17:00,161 - Train: 7.12% [289800/4070000] [71.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 01:18:00,666 - Train: 7.12% [289900/4070000] [71.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 01:19:01,137 - Train: 7.13% [290000/4070000] [71.3/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 01:20:01,933 - Train: 7.13% [290100/4070000] [71.3/1000.0] [batch_t 0.607 (0.608)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 01:21:02,476 - Train: 7.13% [290200/4070000] [71.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 01:22:02,961 - Train: 7.13% [290300/4070000] [71.3/1000.0] [batch_t 0.597 (0.605)] [data_t 0.002] [optim_t 0.595] [lr 0.005000] +2024-04-05 01:23:03,437 - Train: 7.14% [290400/4070000] [71.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 01:24:03,895 - Train: 7.14% [290500/4070000] [71.4/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 01:25:04,397 - Train: 7.14% [290600/4070000] [71.4/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 01:26:04,981 - Train: 7.14% [290700/4070000] [71.4/1000.0] [batch_t 0.611 (0.606)] [data_t 0.002] [optim_t 0.609] [lr 0.005000] +2024-04-05 01:27:05,474 - Train: 7.14% [290800/4070000] [71.4/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 01:28:06,006 - Train: 7.15% [290900/4070000] [71.5/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-05 01:29:06,447 - Train: 7.15% [291000/4070000] [71.5/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-05 01:30:06,937 - Train: 7.15% [291100/4070000] [71.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 01:31:07,819 - Train: 7.15% [291200/4070000] [71.5/1000.0] [batch_t 0.947 (0.609)] [data_t 0.343] [optim_t 0.605] [lr 0.005000] +2024-04-05 01:32:08,406 - Train: 7.16% [291300/4070000] [71.6/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 01:33:08,949 - Train: 7.16% [291400/4070000] [71.6/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 01:34:09,595 - Train: 7.16% [291500/4070000] [71.6/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 01:35:10,150 - Train: 7.16% [291600/4070000] [71.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 01:36:10,681 - Train: 7.17% [291700/4070000] [71.7/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-05 01:37:11,528 - Train: 7.17% [291800/4070000] [71.7/1000.0] [batch_t 0.604 (0.608)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 01:38:12,063 - Train: 7.17% [291900/4070000] [71.7/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 01:39:12,696 - Train: 7.17% [292000/4070000] [71.7/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 01:40:13,177 - Train: 7.18% [292100/4070000] [71.8/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 01:41:13,714 - Train: 7.18% [292200/4070000] [71.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 01:42:14,202 - Train: 7.18% [292300/4070000] [71.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 01:43:14,682 - Train: 7.18% [292400/4070000] [71.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 01:44:15,185 - Train: 7.19% [292500/4070000] [71.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 01:45:15,657 - Train: 7.19% [292600/4070000] [71.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 01:46:16,252 - Train: 7.19% [292700/4070000] [71.9/1000.0] [batch_t 0.602 (0.606)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-05 01:47:16,764 - Train: 7.19% [292800/4070000] [71.9/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 01:48:17,269 - Train: 7.20% [292900/4070000] [72.0/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 01:49:17,787 - Train: 7.20% [293000/4070000] [72.0/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 01:49:41,988 - ==> Total time: 3 days, 0:49:12 Eta: 39 days, 2:34:18 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-05 01:50:19,962 - Train: 7.20% [293100/4070000] [72.0/1000.0] [batch_t 0.606 (0.608)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 01:51:20,458 - Train: 7.20% [293200/4070000] [72.0/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 01:52:21,133 - Train: 7.21% [293300/4070000] [72.1/1000.0] [batch_t 0.603 (0.607)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 01:53:21,663 - Train: 7.21% [293400/4070000] [72.1/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 01:54:22,165 - Train: 7.21% [293500/4070000] [72.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 01:55:22,627 - Train: 7.21% [293600/4070000] [72.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 01:56:23,120 - Train: 7.22% [293700/4070000] [72.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 01:57:23,863 - Train: 7.22% [293800/4070000] [72.2/1000.0] [batch_t 0.603 (0.607)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 01:58:24,413 - Train: 7.22% [293900/4070000] [72.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 01:59:24,877 - Train: 7.22% [294000/4070000] [72.2/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 02:00:25,402 - Train: 7.23% [294100/4070000] [72.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 02:01:25,931 - Train: 7.23% [294200/4070000] [72.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 02:02:26,466 - Train: 7.23% [294300/4070000] [72.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 02:03:27,710 - Train: 7.23% [294400/4070000] [72.3/1000.0] [batch_t 0.606 (0.612)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 02:04:28,183 - Train: 7.24% [294500/4070000] [72.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 02:05:28,634 - Train: 7.24% [294600/4070000] [72.4/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 02:06:29,140 - Train: 7.24% [294700/4070000] [72.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 02:07:29,643 - Train: 7.24% [294800/4070000] [72.4/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 02:08:30,133 - Train: 7.25% [294900/4070000] [72.5/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 02:09:31,755 - Train: 7.25% [295000/4070000] [72.5/1000.0] [batch_t 0.607 (0.616)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 02:10:32,438 - Train: 7.25% [295100/4070000] [72.5/1000.0] [batch_t 0.609 (0.607)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-05 02:11:32,939 - Train: 7.25% [295200/4070000] [72.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 02:12:33,448 - Train: 7.26% [295300/4070000] [72.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 02:13:33,996 - Train: 7.26% [295400/4070000] [72.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 02:14:34,531 - Train: 7.26% [295500/4070000] [72.6/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 02:15:35,012 - Train: 7.26% [295600/4070000] [72.6/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 02:16:35,527 - Train: 7.27% [295700/4070000] [72.7/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 02:17:36,048 - Train: 7.27% [295800/4070000] [72.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 02:18:36,703 - Train: 7.27% [295900/4070000] [72.7/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 02:19:37,222 - Train: 7.27% [296000/4070000] [72.7/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 02:20:37,730 - Train: 7.28% [296100/4070000] [72.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 02:21:38,222 - Train: 7.28% [296200/4070000] [72.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 02:22:38,714 - Train: 7.28% [296300/4070000] [72.8/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 02:23:39,417 - Train: 7.28% [296400/4070000] [72.8/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 02:24:39,932 - Train: 7.29% [296500/4070000] [72.9/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 02:25:40,407 - Train: 7.29% [296600/4070000] [72.9/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 02:26:40,938 - Train: 7.29% [296700/4070000] [72.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 02:27:41,460 - Train: 7.29% [296800/4070000] [72.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 02:28:41,929 - Train: 7.29% [296900/4070000] [72.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 02:29:42,526 - Train: 7.30% [297000/4070000] [73.0/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 02:30:43,012 - Train: 7.30% [297100/4070000] [73.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 02:30:49,073 - ==> Total time: 3 days, 1:30:19 Eta: 38 days, 21:25:10 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-05 02:31:45,306 - Train: 7.30% [297200/4070000] [73.0/1000.0] [batch_t 0.611 (0.607)] [data_t 0.002] [optim_t 0.609] [lr 0.005000] +2024-04-05 02:32:45,789 - Train: 7.30% [297300/4070000] [73.0/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 02:33:46,310 - Train: 7.31% [297400/4070000] [73.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 02:34:46,813 - Train: 7.31% [297500/4070000] [73.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 02:35:47,262 - Train: 7.31% [297600/4070000] [73.1/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 02:36:47,871 - Train: 7.31% [297700/4070000] [73.1/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 02:37:48,349 - Train: 7.32% [297800/4070000] [73.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 02:38:48,831 - Train: 7.32% [297900/4070000] [73.2/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-05 02:39:49,331 - Train: 7.32% [298000/4070000] [73.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 02:40:49,826 - Train: 7.32% [298100/4070000] [73.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 02:41:50,410 - Train: 7.33% [298200/4070000] [73.3/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 02:42:50,926 - Train: 7.33% [298300/4070000] [73.3/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 02:43:51,413 - Train: 7.33% [298400/4070000] [73.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 02:44:51,922 - Train: 7.33% [298500/4070000] [73.3/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 02:45:52,415 - Train: 7.34% [298600/4070000] [73.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 02:46:52,904 - Train: 7.34% [298700/4070000] [73.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 02:47:53,374 - Train: 7.34% [298800/4070000] [73.4/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 02:48:53,834 - Train: 7.34% [298900/4070000] [73.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 02:49:54,365 - Train: 7.35% [299000/4070000] [73.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 02:50:54,884 - Train: 7.35% [299100/4070000] [73.5/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-05 02:51:55,289 - Train: 7.35% [299200/4070000] [73.5/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 02:52:55,794 - Train: 7.35% [299300/4070000] [73.5/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 02:53:56,286 - Train: 7.36% [299400/4070000] [73.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 02:54:56,990 - Train: 7.36% [299500/4070000] [73.6/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 02:55:57,486 - Train: 7.36% [299600/4070000] [73.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 02:56:58,002 - Train: 7.36% [299700/4070000] [73.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 02:57:58,499 - Train: 7.37% [299800/4070000] [73.7/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 02:58:59,003 - Train: 7.37% [299900/4070000] [73.7/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 02:59:59,517 - Train: 7.37% [300000/4070000] [73.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 03:01:00,001 - Train: 7.37% [300100/4070000] [73.7/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 03:02:00,466 - Train: 7.38% [300200/4070000] [73.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 03:03:01,164 - Train: 7.38% [300300/4070000] [73.8/1000.0] [batch_t 0.609 (0.607)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-05 03:04:01,644 - Train: 7.38% [300400/4070000] [73.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 03:05:02,199 - Train: 7.38% [300500/4070000] [73.8/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 03:06:02,680 - Train: 7.39% [300600/4070000] [73.9/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 03:07:03,173 - Train: 7.39% [300700/4070000] [73.9/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 03:08:03,809 - Train: 7.39% [300800/4070000] [73.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 03:09:04,331 - Train: 7.39% [300900/4070000] [73.9/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 03:10:04,865 - Train: 7.40% [301000/4070000] [74.0/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 03:11:05,383 - Train: 7.40% [301100/4070000] [74.0/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-05 03:11:53,769 - ==> Total time: 3 days, 2:11:24 Eta: 38 days, 16:22:47 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-05 03:12:07,916 - Train: 7.40% [301200/4070000] [74.0/1000.0] [batch_t 0.607 (0.615)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 03:13:08,402 - Train: 7.40% [301300/4070000] [74.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 03:14:09,027 - Train: 7.41% [301400/4070000] [74.1/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 03:15:09,515 - Train: 7.41% [301500/4070000] [74.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 03:16:09,986 - Train: 7.41% [301600/4070000] [74.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 03:17:10,468 - Train: 7.41% [301700/4070000] [74.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 03:18:11,002 - Train: 7.42% [301800/4070000] [74.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 03:19:11,535 - Train: 7.42% [301900/4070000] [74.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 03:20:12,046 - Train: 7.42% [302000/4070000] [74.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 03:21:12,554 - Train: 7.42% [302100/4070000] [74.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 03:22:13,212 - Train: 7.43% [302200/4070000] [74.3/1000.0] [batch_t 0.610 (0.606)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-05 03:23:13,735 - Train: 7.43% [302300/4070000] [74.3/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 03:24:14,258 - Train: 7.43% [302400/4070000] [74.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 03:25:14,747 - Train: 7.43% [302500/4070000] [74.3/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 03:26:15,267 - Train: 7.43% [302600/4070000] [74.3/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-05 03:27:15,942 - Train: 7.44% [302700/4070000] [74.4/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 03:28:16,449 - Train: 7.44% [302800/4070000] [74.4/1000.0] [batch_t 0.599 (0.605)] [data_t 0.002] [optim_t 0.597] [lr 0.005000] +2024-04-05 03:29:16,976 - Train: 7.44% [302900/4070000] [74.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 03:30:17,511 - Train: 7.44% [303000/4070000] [74.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 03:31:18,028 - Train: 7.45% [303100/4070000] [74.5/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 03:32:18,580 - Train: 7.45% [303200/4070000] [74.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 03:33:19,088 - Train: 7.45% [303300/4070000] [74.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 03:34:19,768 - Train: 7.45% [303400/4070000] [74.5/1000.0] [batch_t 0.609 (0.607)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-05 03:35:20,255 - Train: 7.46% [303500/4070000] [74.6/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 03:36:20,710 - Train: 7.46% [303600/4070000] [74.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 03:37:21,188 - Train: 7.46% [303700/4070000] [74.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 03:38:21,722 - Train: 7.46% [303800/4070000] [74.6/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 03:39:22,227 - Train: 7.47% [303900/4070000] [74.7/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 03:40:22,707 - Train: 7.47% [304000/4070000] [74.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 03:41:23,244 - Train: 7.47% [304100/4070000] [74.7/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 03:42:23,761 - Train: 7.47% [304200/4070000] [74.7/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 03:43:24,248 - Train: 7.48% [304300/4070000] [74.8/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 03:44:24,705 - Train: 7.48% [304400/4070000] [74.8/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 03:45:25,169 - Train: 7.48% [304500/4070000] [74.8/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 03:46:25,732 - Train: 7.48% [304600/4070000] [74.8/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 03:47:26,190 - Train: 7.49% [304700/4070000] [74.9/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 03:48:26,637 - Train: 7.49% [304800/4070000] [74.9/1000.0] [batch_t 0.601 (0.604)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-05 03:49:27,106 - Train: 7.49% [304900/4070000] [74.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 03:50:27,584 - Train: 7.49% [305000/4070000] [74.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 03:51:28,100 - Train: 7.50% [305100/4070000] [75.0/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 03:52:28,699 - Train: 7.50% [305200/4070000] [75.0/1000.0] [batch_t 0.602 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 03:52:58,962 - ==> Total time: 3 days, 2:52:29 Eta: 38 days, 11:27:27 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-05 03:53:30,939 - Train: 7.50% [305300/4070000] [75.0/1000.0] [batch_t 0.604 (0.611)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 03:54:31,418 - Train: 7.50% [305400/4070000] [75.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 03:55:31,925 - Train: 7.51% [305500/4070000] [75.1/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-05 03:56:32,377 - Train: 7.51% [305600/4070000] [75.1/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 03:57:33,000 - Train: 7.51% [305700/4070000] [75.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 03:58:33,466 - Train: 7.51% [305800/4070000] [75.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 03:59:34,005 - Train: 7.52% [305900/4070000] [75.2/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-05 04:00:34,486 - Train: 7.52% [306000/4070000] [75.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 04:01:35,007 - Train: 7.52% [306100/4070000] [75.2/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 04:02:35,556 - Train: 7.52% [306200/4070000] [75.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 04:03:36,089 - Train: 7.53% [306300/4070000] [75.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 04:04:36,580 - Train: 7.53% [306400/4070000] [75.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 04:05:37,226 - Train: 7.53% [306500/4070000] [75.3/1000.0] [batch_t 0.610 (0.606)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-05 04:06:37,773 - Train: 7.53% [306600/4070000] [75.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 04:07:38,266 - Train: 7.54% [306700/4070000] [75.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 04:08:38,756 - Train: 7.54% [306800/4070000] [75.4/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 04:09:39,281 - Train: 7.54% [306900/4070000] [75.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 04:10:39,918 - Train: 7.54% [307000/4070000] [75.4/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 04:11:40,874 - Train: 7.55% [307100/4070000] [75.5/1000.0] [batch_t 0.607 (0.609)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 04:12:41,305 - Train: 7.55% [307200/4070000] [75.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 04:13:41,752 - Train: 7.55% [307300/4070000] [75.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 04:14:42,271 - Train: 7.55% [307400/4070000] [75.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 04:15:42,826 - Train: 7.56% [307500/4070000] [75.6/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-05 04:16:43,333 - Train: 7.56% [307600/4070000] [75.6/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 04:17:43,818 - Train: 7.56% [307700/4070000] [75.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 04:18:44,388 - Train: 7.56% [307800/4070000] [75.6/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 04:19:44,867 - Train: 7.57% [307900/4070000] [75.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 04:20:45,342 - Train: 7.57% [308000/4070000] [75.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 04:21:45,826 - Train: 7.57% [308100/4070000] [75.7/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-05 04:22:46,310 - Train: 7.57% [308200/4070000] [75.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 04:23:46,863 - Train: 7.57% [308300/4070000] [75.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 04:24:47,335 - Train: 7.58% [308400/4070000] [75.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 04:25:47,827 - Train: 7.58% [308500/4070000] [75.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 04:26:48,322 - Train: 7.58% [308600/4070000] [75.8/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-05 04:27:48,869 - Train: 7.58% [308700/4070000] [75.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 04:28:49,369 - Train: 7.59% [308800/4070000] [75.9/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 04:29:49,878 - Train: 7.59% [308900/4070000] [75.9/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 04:30:50,359 - Train: 7.59% [309000/4070000] [75.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 04:31:50,929 - Train: 7.59% [309100/4070000] [75.9/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 04:32:51,395 - Train: 7.60% [309200/4070000] [76.0/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 04:33:51,880 - Train: 7.60% [309300/4070000] [76.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 04:34:03,969 - ==> Total time: 3 days, 3:33:34 Eta: 38 days, 6:38:47 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-05 04:34:54,669 - Train: 7.60% [309400/4070000] [76.0/1000.0] [batch_t 0.610 (0.614)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-05 04:35:55,171 - Train: 7.60% [309500/4070000] [76.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 04:36:55,763 - Train: 7.61% [309600/4070000] [76.1/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 04:37:56,250 - Train: 7.61% [309700/4070000] [76.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 04:38:56,739 - Train: 7.61% [309800/4070000] [76.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 04:39:57,211 - Train: 7.61% [309900/4070000] [76.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 04:40:57,757 - Train: 7.62% [310000/4070000] [76.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 04:41:58,248 - Train: 7.62% [310100/4070000] [76.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 04:42:58,853 - Train: 7.62% [310200/4070000] [76.2/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 04:43:59,330 - Train: 7.62% [310300/4070000] [76.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 04:44:59,828 - Train: 7.63% [310400/4070000] [76.3/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-05 04:46:00,308 - Train: 7.63% [310500/4070000] [76.3/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 04:47:00,808 - Train: 7.63% [310600/4070000] [76.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 04:48:01,279 - Train: 7.63% [310700/4070000] [76.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 04:49:01,799 - Train: 7.64% [310800/4070000] [76.4/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 04:50:02,402 - Train: 7.64% [310900/4070000] [76.4/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 04:51:02,951 - Train: 7.64% [311000/4070000] [76.4/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 04:52:03,432 - Train: 7.64% [311100/4070000] [76.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 04:53:03,936 - Train: 7.65% [311200/4070000] [76.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 04:54:04,458 - Train: 7.65% [311300/4070000] [76.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 04:55:04,930 - Train: 7.65% [311400/4070000] [76.5/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 04:56:05,503 - Train: 7.65% [311500/4070000] [76.5/1000.0] [batch_t 0.598 (0.606)] [data_t 0.002] [optim_t 0.596] [lr 0.005000] +2024-04-05 04:57:05,993 - Train: 7.66% [311600/4070000] [76.6/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 04:58:06,510 - Train: 7.66% [311700/4070000] [76.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 04:59:06,971 - Train: 7.66% [311800/4070000] [76.6/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 05:00:07,482 - Train: 7.66% [311900/4070000] [76.6/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 05:01:08,075 - Train: 7.67% [312000/4070000] [76.7/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 05:02:08,577 - Train: 7.67% [312100/4070000] [76.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 05:03:09,052 - Train: 7.67% [312200/4070000] [76.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 05:04:10,144 - Train: 7.67% [312300/4070000] [76.7/1000.0] [batch_t 0.605 (0.611)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 05:05:10,662 - Train: 7.68% [312400/4070000] [76.8/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 05:06:11,135 - Train: 7.68% [312500/4070000] [76.8/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-05 05:07:11,641 - Train: 7.68% [312600/4070000] [76.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 05:08:12,125 - Train: 7.68% [312700/4070000] [76.8/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 05:09:12,765 - Train: 7.69% [312800/4070000] [76.9/1000.0] [batch_t 0.602 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 05:10:13,214 - Train: 7.69% [312900/4070000] [76.9/1000.0] [batch_t 0.594 (0.604)] [data_t 0.002] [optim_t 0.592] [lr 0.005000] +2024-04-05 05:11:13,681 - Train: 7.69% [313000/4070000] [76.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 05:12:14,185 - Train: 7.69% [313100/4070000] [76.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 05:13:14,672 - Train: 7.70% [313200/4070000] [77.0/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 05:14:15,352 - Train: 7.70% [313300/4070000] [77.0/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 05:15:09,796 - ==> Total time: 3 days, 4:14:40 Eta: 38 days, 1:56:43 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-05 05:15:17,786 - Train: 7.70% [313400/4070000] [77.0/1000.0] [batch_t 0.603 (0.654)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 05:16:18,300 - Train: 7.70% [313500/4070000] [77.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 05:17:18,796 - Train: 7.71% [313600/4070000] [77.1/1000.0] [batch_t 0.600 (0.605)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-05 05:18:19,242 - Train: 7.71% [313700/4070000] [77.1/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 05:19:19,727 - Train: 7.71% [313800/4070000] [77.1/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 05:20:20,180 - Train: 7.71% [313900/4070000] [77.1/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 05:21:20,791 - Train: 7.71% [314000/4070000] [77.1/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 05:22:21,257 - Train: 7.72% [314100/4070000] [77.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 05:23:21,723 - Train: 7.72% [314200/4070000] [77.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 05:24:22,203 - Train: 7.72% [314300/4070000] [77.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 05:25:22,695 - Train: 7.72% [314400/4070000] [77.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 05:26:23,261 - Train: 7.73% [314500/4070000] [77.3/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 05:27:23,776 - Train: 7.73% [314600/4070000] [77.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 05:28:24,273 - Train: 7.73% [314700/4070000] [77.3/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 05:29:24,768 - Train: 7.73% [314800/4070000] [77.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 05:30:25,283 - Train: 7.74% [314900/4070000] [77.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 05:31:25,767 - Train: 7.74% [315000/4070000] [77.4/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-05 05:32:26,247 - Train: 7.74% [315100/4070000] [77.4/1000.0] [batch_t 0.599 (0.605)] [data_t 0.002] [optim_t 0.597] [lr 0.005000] +2024-04-05 05:33:26,890 - Train: 7.74% [315200/4070000] [77.4/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 05:34:27,372 - Train: 7.75% [315300/4070000] [77.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 05:35:27,860 - Train: 7.75% [315400/4070000] [77.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 05:36:28,307 - Train: 7.75% [315500/4070000] [77.5/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 05:37:28,767 - Train: 7.75% [315600/4070000] [77.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 05:38:29,352 - Train: 7.76% [315700/4070000] [77.6/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 05:39:29,853 - Train: 7.76% [315800/4070000] [77.6/1000.0] [batch_t 0.611 (0.605)] [data_t 0.002] [optim_t 0.609] [lr 0.005000] +2024-04-05 05:40:30,348 - Train: 7.76% [315900/4070000] [77.6/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 05:41:30,863 - Train: 7.76% [316000/4070000] [77.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 05:42:31,340 - Train: 7.77% [316100/4070000] [77.7/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 05:43:31,835 - Train: 7.77% [316200/4070000] [77.7/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 05:44:32,318 - Train: 7.77% [316300/4070000] [77.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 05:45:32,823 - Train: 7.77% [316400/4070000] [77.7/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 05:46:33,394 - Train: 7.78% [316500/4070000] [77.8/1000.0] [batch_t 0.600 (0.606)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-05 05:47:33,891 - Train: 7.78% [316600/4070000] [77.8/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-05 05:48:34,395 - Train: 7.78% [316700/4070000] [77.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 05:49:34,935 - Train: 7.78% [316800/4070000] [77.8/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 05:50:35,391 - Train: 7.79% [316900/4070000] [77.9/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 05:51:35,947 - Train: 7.79% [317000/4070000] [77.9/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 05:52:36,418 - Train: 7.79% [317100/4070000] [77.9/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 05:53:36,895 - Train: 7.79% [317200/4070000] [77.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 05:54:37,388 - Train: 7.80% [317300/4070000] [78.0/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 05:55:37,871 - Train: 7.80% [317400/4070000] [78.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 05:56:14,148 - ==> Total time: 3 days, 4:55:45 Eta: 37 days, 21:20:31 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-05 05:56:39,873 - Train: 7.80% [317500/4070000] [78.0/1000.0] [batch_t 0.598 (0.610)] [data_t 0.002] [optim_t 0.596] [lr 0.005000] +2024-04-05 05:57:40,416 - Train: 7.80% [317600/4070000] [78.0/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 05:58:41,051 - Train: 7.81% [317700/4070000] [78.1/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 05:59:41,561 - Train: 7.81% [317800/4070000] [78.1/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 06:00:42,060 - Train: 7.81% [317900/4070000] [78.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 06:01:42,547 - Train: 7.81% [318000/4070000] [78.1/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 06:02:43,033 - Train: 7.82% [318100/4070000] [78.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 06:03:43,592 - Train: 7.82% [318200/4070000] [78.2/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 06:04:44,062 - Train: 7.82% [318300/4070000] [78.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 06:05:44,540 - Train: 7.82% [318400/4070000] [78.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 06:06:44,984 - Train: 7.83% [318500/4070000] [78.3/1000.0] [batch_t 0.598 (0.604)] [data_t 0.002] [optim_t 0.597] [lr 0.005000] +2024-04-05 06:07:45,497 - Train: 7.83% [318600/4070000] [78.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 06:08:46,025 - Train: 7.83% [318700/4070000] [78.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 06:09:46,509 - Train: 7.83% [318800/4070000] [78.3/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 06:10:46,962 - Train: 7.84% [318900/4070000] [78.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 06:11:47,522 - Train: 7.84% [319000/4070000] [78.4/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 06:12:47,999 - Train: 7.84% [319100/4070000] [78.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 06:13:48,506 - Train: 7.84% [319200/4070000] [78.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 06:14:49,003 - Train: 7.85% [319300/4070000] [78.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 06:15:49,492 - Train: 7.85% [319400/4070000] [78.5/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 06:16:50,075 - Train: 7.85% [319500/4070000] [78.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 06:17:50,583 - Train: 7.85% [319600/4070000] [78.5/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 06:18:51,064 - Train: 7.86% [319700/4070000] [78.6/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 06:19:51,514 - Train: 7.86% [319800/4070000] [78.6/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 06:20:51,972 - Train: 7.86% [319900/4070000] [78.6/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-05 06:21:52,440 - Train: 7.86% [320000/4070000] [78.6/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 06:22:52,926 - Train: 7.86% [320100/4070000] [78.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 06:23:53,403 - Train: 7.87% [320200/4070000] [78.7/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 06:24:53,937 - Train: 7.87% [320300/4070000] [78.7/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 06:25:54,430 - Train: 7.87% [320400/4070000] [78.7/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 06:26:54,948 - Train: 7.87% [320500/4070000] [78.7/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 06:27:55,456 - Train: 7.88% [320600/4070000] [78.8/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 06:28:55,998 - Train: 7.88% [320700/4070000] [78.8/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-05 06:29:56,685 - Train: 7.88% [320800/4070000] [78.8/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 06:30:57,207 - Train: 7.88% [320900/4070000] [78.8/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-05 06:31:57,721 - Train: 7.89% [321000/4070000] [78.9/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 06:32:58,200 - Train: 7.89% [321100/4070000] [78.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 06:33:58,687 - Train: 7.89% [321200/4070000] [78.9/1000.0] [batch_t 0.611 (0.605)] [data_t 0.002] [optim_t 0.609] [lr 0.005000] +2024-04-05 06:34:59,194 - Train: 7.89% [321300/4070000] [78.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 06:35:59,682 - Train: 7.90% [321400/4070000] [79.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 06:37:00,155 - Train: 7.90% [321500/4070000] [79.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 06:37:18,416 - ==> Total time: 3 days, 5:36:49 Eta: 37 days, 16:50:16 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-05 06:38:02,471 - Train: 7.90% [321600/4070000] [79.0/1000.0] [batch_t 0.605 (0.608)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 06:39:02,992 - Train: 7.90% [321700/4070000] [79.0/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 06:40:03,505 - Train: 7.91% [321800/4070000] [79.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 06:41:04,017 - Train: 7.91% [321900/4070000] [79.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 06:42:04,503 - Train: 7.91% [322000/4070000] [79.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 06:43:05,018 - Train: 7.91% [322100/4070000] [79.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 06:44:05,542 - Train: 7.92% [322200/4070000] [79.2/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 06:45:06,246 - Train: 7.92% [322300/4070000] [79.2/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 06:46:06,733 - Train: 7.92% [322400/4070000] [79.2/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 06:47:07,181 - Train: 7.92% [322500/4070000] [79.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 06:48:07,647 - Train: 7.93% [322600/4070000] [79.3/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 06:49:08,158 - Train: 7.93% [322700/4070000] [79.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 06:50:08,780 - Train: 7.93% [322800/4070000] [79.3/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 06:51:09,286 - Train: 7.93% [322900/4070000] [79.3/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-05 06:52:09,761 - Train: 7.94% [323000/4070000] [79.4/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 06:53:10,255 - Train: 7.94% [323100/4070000] [79.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 06:54:10,754 - Train: 7.94% [323200/4070000] [79.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 06:55:11,311 - Train: 7.94% [323300/4070000] [79.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 06:56:11,813 - Train: 7.95% [323400/4070000] [79.5/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 06:57:12,415 - Train: 7.95% [323500/4070000] [79.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 06:58:12,930 - Train: 7.95% [323600/4070000] [79.5/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 06:59:13,388 - Train: 7.95% [323700/4070000] [79.5/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 07:00:13,895 - Train: 7.96% [323800/4070000] [79.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 07:01:14,436 - Train: 7.96% [323900/4070000] [79.6/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 07:02:14,911 - Train: 7.96% [324000/4070000] [79.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 07:03:15,612 - Train: 7.96% [324100/4070000] [79.6/1000.0] [batch_t 0.609 (0.607)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-05 07:04:16,127 - Train: 7.97% [324200/4070000] [79.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 07:05:16,616 - Train: 7.97% [324300/4070000] [79.7/1000.0] [batch_t 0.597 (0.605)] [data_t 0.002] [optim_t 0.595] [lr 0.005000] +2024-04-05 07:06:17,115 - Train: 7.97% [324400/4070000] [79.7/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 07:07:17,620 - Train: 7.97% [324500/4070000] [79.7/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 07:08:18,183 - Train: 7.98% [324600/4070000] [79.8/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 07:09:18,717 - Train: 7.98% [324700/4070000] [79.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 07:10:19,192 - Train: 7.98% [324800/4070000] [79.8/1000.0] [batch_t 0.600 (0.605)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-05 07:11:19,698 - Train: 7.98% [324900/4070000] [79.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 07:12:20,195 - Train: 7.99% [325000/4070000] [79.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 07:13:20,680 - Train: 7.99% [325100/4070000] [79.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 07:14:21,194 - Train: 7.99% [325200/4070000] [79.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 07:15:21,687 - Train: 7.99% [325300/4070000] [79.9/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 07:16:22,262 - Train: 8.00% [325400/4070000] [80.0/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 07:17:22,735 - Train: 8.00% [325500/4070000] [80.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 07:18:23,230 - Train: 8.00% [325600/4070000] [80.0/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-05 07:18:23,234 - ==> Total time: 3 days, 6:17:54 Eta: 37 days, 12:25:52 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-05 07:19:25,442 - Train: 8.00% [325700/4070000] [80.0/1000.0] [batch_t 0.606 (0.608)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 07:20:25,958 - Train: 8.00% [325800/4070000] [80.0/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 07:21:26,598 - Train: 8.01% [325900/4070000] [80.1/1000.0] [batch_t 0.609 (0.606)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-05 07:22:27,102 - Train: 8.01% [326000/4070000] [80.1/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 07:23:27,619 - Train: 8.01% [326100/4070000] [80.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 07:24:28,146 - Train: 8.01% [326200/4070000] [80.1/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 07:25:28,715 - Train: 8.02% [326300/4070000] [80.2/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 07:26:29,180 - Train: 8.02% [326400/4070000] [80.2/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 07:27:29,684 - Train: 8.02% [326500/4070000] [80.2/1000.0] [batch_t 0.613 (0.605)] [data_t 0.002] [optim_t 0.611] [lr 0.005000] +2024-04-05 07:28:30,134 - Train: 8.02% [326600/4070000] [80.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 07:29:30,695 - Train: 8.03% [326700/4070000] [80.3/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 07:30:31,200 - Train: 8.03% [326800/4070000] [80.3/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-05 07:31:31,722 - Train: 8.03% [326900/4070000] [80.3/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 07:32:32,253 - Train: 8.03% [327000/4070000] [80.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 07:33:32,765 - Train: 8.04% [327100/4070000] [80.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 07:34:33,402 - Train: 8.04% [327200/4070000] [80.4/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 07:35:33,891 - Train: 8.04% [327300/4070000] [80.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 07:36:34,432 - Train: 8.04% [327400/4070000] [80.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 07:37:34,895 - Train: 8.05% [327500/4070000] [80.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 07:38:35,433 - Train: 8.05% [327600/4070000] [80.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 07:39:35,954 - Train: 8.05% [327700/4070000] [80.5/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 07:40:36,538 - Train: 8.05% [327800/4070000] [80.5/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 07:41:37,055 - Train: 8.06% [327900/4070000] [80.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 07:42:37,616 - Train: 8.06% [328000/4070000] [80.6/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 07:43:38,136 - Train: 8.06% [328100/4070000] [80.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 07:44:38,642 - Train: 8.06% [328200/4070000] [80.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 07:45:39,141 - Train: 8.07% [328300/4070000] [80.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 07:46:39,639 - Train: 8.07% [328400/4070000] [80.7/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 07:47:40,292 - Train: 8.07% [328500/4070000] [80.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 07:48:40,771 - Train: 8.07% [328600/4070000] [80.7/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 07:49:41,298 - Train: 8.08% [328700/4070000] [80.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 07:50:41,786 - Train: 8.08% [328800/4070000] [80.8/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 07:51:42,300 - Train: 8.08% [328900/4070000] [80.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 07:52:42,801 - Train: 8.08% [329000/4070000] [80.8/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 07:53:43,255 - Train: 8.09% [329100/4070000] [80.9/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 07:54:43,877 - Train: 8.09% [329200/4070000] [80.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 07:55:44,420 - Train: 8.09% [329300/4070000] [80.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 07:56:44,922 - Train: 8.09% [329400/4070000] [80.9/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 07:57:45,414 - Train: 8.10% [329500/4070000] [81.0/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-05 07:58:45,893 - Train: 8.10% [329600/4070000] [81.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 07:59:28,229 - ==> Total time: 3 days, 6:58:59 Eta: 37 days, 8:07:00 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-05 07:59:48,091 - Train: 8.10% [329700/4070000] [81.0/1000.0] [batch_t 0.605 (0.618)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 08:00:48,613 - Train: 8.10% [329800/4070000] [81.0/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 08:01:49,104 - Train: 8.11% [329900/4070000] [81.1/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-05 08:02:49,561 - Train: 8.11% [330000/4070000] [81.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 08:03:50,045 - Train: 8.11% [330100/4070000] [81.1/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 08:04:50,526 - Train: 8.11% [330200/4070000] [81.1/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-05 08:05:50,982 - Train: 8.12% [330300/4070000] [81.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 08:06:51,484 - Train: 8.12% [330400/4070000] [81.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 08:07:52,042 - Train: 8.12% [330500/4070000] [81.2/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 08:08:52,521 - Train: 8.12% [330600/4070000] [81.2/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-05 08:09:53,015 - Train: 8.13% [330700/4070000] [81.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 08:10:53,496 - Train: 8.13% [330800/4070000] [81.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 08:11:53,997 - Train: 8.13% [330900/4070000] [81.3/1000.0] [batch_t 0.610 (0.605)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-05 08:12:54,681 - Train: 8.13% [331000/4070000] [81.3/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 08:13:55,170 - Train: 8.14% [331100/4070000] [81.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 08:14:55,653 - Train: 8.14% [331200/4070000] [81.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 08:15:56,173 - Train: 8.14% [331300/4070000] [81.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 08:16:56,698 - Train: 8.14% [331400/4070000] [81.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 08:17:57,248 - Train: 8.14% [331500/4070000] [81.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 08:18:57,764 - Train: 8.15% [331600/4070000] [81.5/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 08:19:58,398 - Train: 8.15% [331700/4070000] [81.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 08:20:58,922 - Train: 8.15% [331800/4070000] [81.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 08:21:59,442 - Train: 8.15% [331900/4070000] [81.5/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 08:22:59,934 - Train: 8.16% [332000/4070000] [81.6/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 08:24:00,433 - Train: 8.16% [332100/4070000] [81.6/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 08:25:00,930 - Train: 8.16% [332200/4070000] [81.6/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 08:26:01,430 - Train: 8.16% [332300/4070000] [81.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 08:27:01,987 - Train: 8.17% [332400/4070000] [81.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 08:28:02,485 - Train: 8.17% [332500/4070000] [81.7/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 08:29:02,964 - Train: 8.17% [332600/4070000] [81.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 08:30:03,445 - Train: 8.17% [332700/4070000] [81.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 08:31:03,940 - Train: 8.18% [332800/4070000] [81.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 08:32:04,421 - Train: 8.18% [332900/4070000] [81.8/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 08:33:05,036 - Train: 8.18% [333000/4070000] [81.8/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 08:34:05,535 - Train: 8.18% [333100/4070000] [81.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 08:35:06,024 - Train: 8.19% [333200/4070000] [81.9/1000.0] [batch_t 0.613 (0.605)] [data_t 0.002] [optim_t 0.611] [lr 0.005000] +2024-04-05 08:36:06,469 - Train: 8.19% [333300/4070000] [81.9/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 08:37:06,954 - Train: 8.19% [333400/4070000] [81.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 08:38:07,474 - Train: 8.19% [333500/4070000] [81.9/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 08:39:07,983 - Train: 8.20% [333600/4070000] [82.0/1000.0] [batch_t 0.600 (0.605)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-05 08:40:08,573 - Train: 8.20% [333700/4070000] [82.0/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 08:40:32,772 - ==> Total time: 3 days, 7:40:03 Eta: 37 days, 3:53:21 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-05 08:41:10,672 - Train: 8.20% [333800/4070000] [82.0/1000.0] [batch_t 0.605 (0.608)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 08:42:11,219 - Train: 8.20% [333900/4070000] [82.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 08:43:11,695 - Train: 8.21% [334000/4070000] [82.1/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-05 08:44:12,156 - Train: 8.21% [334100/4070000] [82.1/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-05 08:45:12,842 - Train: 8.21% [334200/4070000] [82.1/1000.0] [batch_t 0.612 (0.607)] [data_t 0.002] [optim_t 0.610] [lr 0.005000] +2024-04-05 08:46:13,358 - Train: 8.21% [334300/4070000] [82.1/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 08:47:13,889 - Train: 8.22% [334400/4070000] [82.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 08:48:14,400 - Train: 8.22% [334500/4070000] [82.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 08:49:14,902 - Train: 8.22% [334600/4070000] [82.2/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 08:50:15,407 - Train: 8.22% [334700/4070000] [82.2/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 08:51:15,899 - Train: 8.23% [334800/4070000] [82.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 08:52:16,376 - Train: 8.23% [334900/4070000] [82.3/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-05 08:53:16,941 - Train: 8.23% [335000/4070000] [82.3/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 08:54:17,445 - Train: 8.23% [335100/4070000] [82.3/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 08:55:17,945 - Train: 8.24% [335200/4070000] [82.4/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 08:56:18,442 - Train: 8.24% [335300/4070000] [82.4/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-05 08:57:18,948 - Train: 8.24% [335400/4070000] [82.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 08:58:19,511 - Train: 8.24% [335500/4070000] [82.4/1000.0] [batch_t 0.610 (0.606)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-05 08:59:19,991 - Train: 8.25% [335600/4070000] [82.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 09:00:20,488 - Train: 8.25% [335700/4070000] [82.5/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 09:01:21,000 - Train: 8.25% [335800/4070000] [82.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 09:02:21,486 - Train: 8.25% [335900/4070000] [82.5/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 09:03:21,974 - Train: 8.26% [336000/4070000] [82.6/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 09:04:22,468 - Train: 8.26% [336100/4070000] [82.6/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 09:05:22,959 - Train: 8.26% [336200/4070000] [82.6/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 09:06:23,526 - Train: 8.26% [336300/4070000] [82.6/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 09:07:24,002 - Train: 8.27% [336400/4070000] [82.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 09:08:24,508 - Train: 8.27% [336500/4070000] [82.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 09:09:24,965 - Train: 8.27% [336600/4070000] [82.7/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 09:10:25,465 - Train: 8.27% [336700/4070000] [82.7/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-05 09:11:26,059 - Train: 8.28% [336800/4070000] [82.8/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 09:12:26,833 - Train: 8.28% [336900/4070000] [82.8/1000.0] [batch_t 0.606 (0.608)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 09:13:27,336 - Train: 8.28% [337000/4070000] [82.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 09:14:27,818 - Train: 8.28% [337100/4070000] [82.8/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 09:15:28,345 - Train: 8.29% [337200/4070000] [82.9/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 09:16:28,831 - Train: 8.29% [337300/4070000] [82.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 09:17:29,336 - Train: 8.29% [337400/4070000] [82.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 09:18:29,987 - Train: 8.29% [337500/4070000] [82.9/1000.0] [batch_t 0.609 (0.606)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-05 09:19:30,507 - Train: 8.29% [337600/4070000] [82.9/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 09:20:30,991 - Train: 8.30% [337700/4070000] [83.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 09:21:31,429 - Train: 8.30% [337800/4070000] [83.0/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 09:21:37,468 - ==> Total time: 3 days, 8:21:08 Eta: 36 days, 23:44:52 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-05 09:22:33,843 - Train: 8.30% [337900/4070000] [83.0/1000.0] [batch_t 0.605 (0.611)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 09:23:34,369 - Train: 8.30% [338000/4070000] [83.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 09:24:34,867 - Train: 8.31% [338100/4070000] [83.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 09:25:35,499 - Train: 8.31% [338200/4070000] [83.1/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 09:26:35,996 - Train: 8.31% [338300/4070000] [83.1/1000.0] [batch_t 0.610 (0.605)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-05 09:27:36,505 - Train: 8.31% [338400/4070000] [83.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 09:28:37,005 - Train: 8.32% [338500/4070000] [83.2/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 09:29:37,481 - Train: 8.32% [338600/4070000] [83.2/1000.0] [batch_t 0.600 (0.605)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-05 09:30:38,000 - Train: 8.32% [338700/4070000] [83.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 09:31:38,600 - Train: 8.32% [338800/4070000] [83.2/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 09:32:39,076 - Train: 8.33% [338900/4070000] [83.3/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-05 09:33:39,539 - Train: 8.33% [339000/4070000] [83.3/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-05 09:34:39,991 - Train: 8.33% [339100/4070000] [83.3/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 09:35:40,503 - Train: 8.33% [339200/4070000] [83.3/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 09:36:41,050 - Train: 8.34% [339300/4070000] [83.4/1000.0] [batch_t 0.599 (0.605)] [data_t 0.002] [optim_t 0.597] [lr 0.005000] +2024-04-05 09:37:41,548 - Train: 8.34% [339400/4070000] [83.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 09:38:42,152 - Train: 8.34% [339500/4070000] [83.4/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 09:39:42,635 - Train: 8.34% [339600/4070000] [83.4/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-05 09:40:43,145 - Train: 8.35% [339700/4070000] [83.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 09:41:43,635 - Train: 8.35% [339800/4070000] [83.5/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 09:42:44,139 - Train: 8.35% [339900/4070000] [83.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 09:43:44,689 - Train: 8.35% [340000/4070000] [83.5/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 09:44:45,425 - Train: 8.36% [340100/4070000] [83.6/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 09:45:45,926 - Train: 8.36% [340200/4070000] [83.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 09:46:46,394 - Train: 8.36% [340300/4070000] [83.6/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-05 09:47:46,893 - Train: 8.36% [340400/4070000] [83.6/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 09:48:47,396 - Train: 8.37% [340500/4070000] [83.7/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 09:49:48,084 - Train: 8.37% [340600/4070000] [83.7/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 09:50:48,684 - Train: 8.37% [340700/4070000] [83.7/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 09:51:49,161 - Train: 8.37% [340800/4070000] [83.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 09:52:49,681 - Train: 8.38% [340900/4070000] [83.8/1000.0] [batch_t 0.610 (0.605)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-05 09:53:50,202 - Train: 8.38% [341000/4070000] [83.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 09:54:51,450 - Train: 8.38% [341100/4070000] [83.8/1000.0] [batch_t 0.603 (0.612)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 09:55:54,430 - Train: 8.38% [341200/4070000] [83.8/1000.0] [batch_t 0.605 (0.630)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 09:56:57,981 - Train: 8.39% [341300/4070000] [83.9/1000.0] [batch_t 0.605 (0.635)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 09:57:58,674 - Train: 8.39% [341400/4070000] [83.9/1000.0] [batch_t 0.608 (0.607)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 09:59:09,733 - Train: 8.39% [341500/4070000] [83.9/1000.0] [batch_t 0.607 (0.710)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 10:00:49,378 - Train: 8.39% [341600/4070000] [83.9/1000.0] [batch_t 0.601 (0.996)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-05 10:01:49,893 - Train: 8.40% [341700/4070000] [84.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 10:02:50,578 - Train: 8.40% [341800/4070000] [84.0/1000.0] [batch_t 0.603 (0.607)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 10:03:39,045 - ==> Total time: 3 days, 9:03:09 Eta: 36 days, 19:51:39 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-05 10:03:53,018 - Train: 8.40% [341900/4070000] [84.0/1000.0] [batch_t 0.606 (0.614)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 10:04:53,775 - Train: 8.40% [342000/4070000] [84.0/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 10:05:54,273 - Train: 8.41% [342100/4070000] [84.1/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 10:06:54,789 - Train: 8.41% [342200/4070000] [84.1/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 10:07:58,695 - Train: 8.41% [342300/4070000] [84.1/1000.0] [batch_t 0.605 (0.639)] [data_t 0.003] [optim_t 0.602] [lr 0.005000] +2024-04-05 10:08:59,200 - Train: 8.41% [342400/4070000] [84.1/1000.0] [batch_t 0.599 (0.605)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-05 10:09:59,824 - Train: 8.42% [342500/4070000] [84.2/1000.0] [batch_t 0.595 (0.606)] [data_t 0.002] [optim_t 0.593] [lr 0.005000] +2024-04-05 10:11:10,582 - Train: 8.42% [342600/4070000] [84.2/1000.0] [batch_t 0.602 (0.707)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 10:12:11,696 - Train: 8.42% [342700/4070000] [84.2/1000.0] [batch_t 0.606 (0.611)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 10:13:12,171 - Train: 8.42% [342800/4070000] [84.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 10:14:12,665 - Train: 8.43% [342900/4070000] [84.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 10:15:13,139 - Train: 8.43% [343000/4070000] [84.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 10:16:13,619 - Train: 8.43% [343100/4070000] [84.3/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 10:17:14,256 - Train: 8.43% [343200/4070000] [84.3/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 10:18:14,757 - Train: 8.43% [343300/4070000] [84.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 10:19:15,237 - Train: 8.44% [343400/4070000] [84.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 10:20:15,730 - Train: 8.44% [343500/4070000] [84.4/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 10:21:16,246 - Train: 8.44% [343600/4070000] [84.4/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-05 10:22:16,697 - Train: 8.44% [343700/4070000] [84.4/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 10:23:17,249 - Train: 8.45% [343800/4070000] [84.5/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-05 10:24:17,725 - Train: 8.45% [343900/4070000] [84.5/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 10:25:18,182 - Train: 8.45% [344000/4070000] [84.5/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 10:26:18,674 - Train: 8.45% [344100/4070000] [84.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 10:27:19,178 - Train: 8.46% [344200/4070000] [84.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 10:28:19,831 - Train: 8.46% [344300/4070000] [84.6/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 10:29:20,355 - Train: 8.46% [344400/4070000] [84.6/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-05 10:30:20,859 - Train: 8.46% [344500/4070000] [84.6/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 10:31:21,358 - Train: 8.47% [344600/4070000] [84.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 10:32:21,846 - Train: 8.47% [344700/4070000] [84.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 10:33:22,352 - Train: 8.47% [344800/4070000] [84.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 10:34:22,839 - Train: 8.47% [344900/4070000] [84.7/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 10:35:23,308 - Train: 8.48% [345000/4070000] [84.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 10:36:23,940 - Train: 8.48% [345100/4070000] [84.8/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 10:37:24,426 - Train: 8.48% [345200/4070000] [84.8/1000.0] [batch_t 0.612 (0.605)] [data_t 0.002] [optim_t 0.610] [lr 0.005000] +2024-04-05 10:38:24,923 - Train: 8.48% [345300/4070000] [84.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 10:39:25,433 - Train: 8.49% [345400/4070000] [84.9/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 10:40:25,939 - Train: 8.49% [345500/4070000] [84.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 10:41:26,424 - Train: 8.49% [345600/4070000] [84.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 10:42:26,911 - Train: 8.49% [345700/4070000] [84.9/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 10:43:27,552 - Train: 8.50% [345800/4070000] [85.0/1000.0] [batch_t 0.601 (0.606)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-05 10:44:31,650 - Train: 8.50% [345900/4070000] [85.0/1000.0] [batch_t 0.609 (0.641)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-05 10:45:01,914 - ==> Total time: 3 days, 9:44:32 Eta: 36 days, 15:55:59 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-05 10:45:37,549 - Train: 8.50% [346000/4070000] [85.0/1000.0] [batch_t 0.607 (0.625)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 10:46:38,022 - Train: 8.50% [346100/4070000] [85.0/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 10:47:38,538 - Train: 8.51% [346200/4070000] [85.1/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 10:48:39,214 - Train: 8.51% [346300/4070000] [85.1/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 10:49:39,701 - Train: 8.51% [346400/4070000] [85.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 10:50:40,216 - Train: 8.51% [346500/4070000] [85.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 10:51:40,729 - Train: 8.52% [346600/4070000] [85.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 10:52:41,219 - Train: 8.52% [346700/4070000] [85.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 10:53:41,724 - Train: 8.52% [346800/4070000] [85.2/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 10:54:42,275 - Train: 8.52% [346900/4070000] [85.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 10:55:42,772 - Train: 8.53% [347000/4070000] [85.3/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 10:56:43,395 - Train: 8.53% [347100/4070000] [85.3/1000.0] [batch_t 0.600 (0.606)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-05 10:57:43,880 - Train: 8.53% [347200/4070000] [85.3/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 10:58:44,779 - Train: 8.53% [347300/4070000] [85.3/1000.0] [batch_t 0.607 (0.609)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 10:59:45,316 - Train: 8.54% [347400/4070000] [85.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 11:00:45,859 - Train: 8.54% [347500/4070000] [85.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 11:01:46,408 - Train: 8.54% [347600/4070000] [85.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 11:02:46,971 - Train: 8.54% [347700/4070000] [85.4/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 11:03:47,479 - Train: 8.55% [347800/4070000] [85.5/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 11:04:48,011 - Train: 8.55% [347900/4070000] [85.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 11:05:48,600 - Train: 8.55% [348000/4070000] [85.5/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 11:06:49,097 - Train: 8.55% [348100/4070000] [85.5/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 11:07:49,580 - Train: 8.56% [348200/4070000] [85.6/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-05 11:08:50,057 - Train: 8.56% [348300/4070000] [85.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 11:09:50,608 - Train: 8.56% [348400/4070000] [85.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 11:10:51,186 - Train: 8.56% [348500/4070000] [85.6/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 11:11:51,709 - Train: 8.57% [348600/4070000] [85.7/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 11:12:52,251 - Train: 8.57% [348700/4070000] [85.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 11:13:52,787 - Train: 8.57% [348800/4070000] [85.7/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 11:14:53,288 - Train: 8.57% [348900/4070000] [85.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 11:15:53,906 - Train: 8.57% [349000/4070000] [85.7/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 11:16:55,044 - Train: 8.58% [349100/4070000] [85.8/1000.0] [batch_t 0.602 (0.611)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-05 11:17:56,637 - Train: 8.58% [349200/4070000] [85.8/1000.0] [batch_t 0.604 (0.616)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 11:18:57,149 - Train: 8.58% [349300/4070000] [85.8/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 11:19:57,657 - Train: 8.58% [349400/4070000] [85.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 11:20:58,350 - Train: 8.59% [349500/4070000] [85.9/1000.0] [batch_t 0.602 (0.607)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-05 11:21:58,819 - Train: 8.59% [349600/4070000] [85.9/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 11:22:59,308 - Train: 8.59% [349700/4070000] [85.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 11:23:59,930 - Train: 8.59% [349800/4070000] [85.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 11:25:00,437 - Train: 8.60% [349900/4070000] [86.0/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 11:26:00,937 - Train: 8.60% [350000/4070000] [86.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 11:26:13,041 - ==> Total time: 3 days, 10:25:43 Eta: 36 days, 12:02:46 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-05 11:27:03,074 - Train: 8.60% [350100/4070000] [86.0/1000.0] [batch_t 0.606 (0.608)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 11:28:03,570 - Train: 8.60% [350200/4070000] [86.0/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 11:29:04,041 - Train: 8.61% [350300/4070000] [86.1/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 11:30:04,680 - Train: 8.61% [350400/4070000] [86.1/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 11:31:05,198 - Train: 8.61% [350500/4070000] [86.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 11:32:05,717 - Train: 8.61% [350600/4070000] [86.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 11:33:06,208 - Train: 8.62% [350700/4070000] [86.2/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 11:34:06,674 - Train: 8.62% [350800/4070000] [86.2/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 11:35:07,282 - Train: 8.62% [350900/4070000] [86.2/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 11:36:07,762 - Train: 8.62% [351000/4070000] [86.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 11:37:08,255 - Train: 8.63% [351100/4070000] [86.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 11:38:08,749 - Train: 8.63% [351200/4070000] [86.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 11:39:09,258 - Train: 8.63% [351300/4070000] [86.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 11:40:09,804 - Train: 8.63% [351400/4070000] [86.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 11:41:10,326 - Train: 8.64% [351500/4070000] [86.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 11:42:10,901 - Train: 8.64% [351600/4070000] [86.4/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 11:43:11,390 - Train: 8.64% [351700/4070000] [86.4/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-05 11:44:11,863 - Train: 8.64% [351800/4070000] [86.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 11:45:12,333 - Train: 8.65% [351900/4070000] [86.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 11:46:12,828 - Train: 8.65% [352000/4070000] [86.5/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 11:47:13,337 - Train: 8.65% [352100/4070000] [86.5/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 11:48:13,901 - Train: 8.65% [352200/4070000] [86.5/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 11:49:14,381 - Train: 8.66% [352300/4070000] [86.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 11:50:14,854 - Train: 8.66% [352400/4070000] [86.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 11:51:15,283 - Train: 8.66% [352500/4070000] [86.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 11:52:15,908 - Train: 8.66% [352600/4070000] [86.6/1000.0] [batch_t 0.601 (0.606)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-05 11:53:16,381 - Train: 8.67% [352700/4070000] [86.7/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-05 11:54:16,991 - Train: 8.67% [352800/4070000] [86.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 11:55:17,500 - Train: 8.67% [352900/4070000] [86.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 11:56:18,018 - Train: 8.67% [353000/4070000] [86.7/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 11:57:18,560 - Train: 8.68% [353100/4070000] [86.8/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 11:58:19,025 - Train: 8.68% [353200/4070000] [86.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 11:59:19,522 - Train: 8.68% [353300/4070000] [86.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 12:00:20,103 - Train: 8.68% [353400/4070000] [86.8/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 12:01:20,783 - Train: 8.69% [353500/4070000] [86.9/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 12:02:21,281 - Train: 8.69% [353600/4070000] [86.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 12:03:21,811 - Train: 8.69% [353700/4070000] [86.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 12:04:22,324 - Train: 8.69% [353800/4070000] [86.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 12:05:22,851 - Train: 8.70% [353900/4070000] [87.0/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 12:06:23,399 - Train: 8.70% [354000/4070000] [87.0/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 12:07:18,508 - ==> Total time: 3 days, 11:06:49 Eta: 36 days, 8:12:58 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-05 12:07:26,432 - Train: 8.70% [354100/4070000] [87.0/1000.0] [batch_t 0.604 (0.630)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 12:08:26,917 - Train: 8.70% [354200/4070000] [87.0/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 12:09:27,758 - Train: 8.71% [354300/4070000] [87.1/1000.0] [batch_t 0.604 (0.608)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 12:10:28,274 - Train: 8.71% [354400/4070000] [87.1/1000.0] [batch_t 0.600 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-05 12:11:28,840 - Train: 8.71% [354500/4070000] [87.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 12:12:29,308 - Train: 8.71% [354600/4070000] [87.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 12:13:29,757 - Train: 8.71% [354700/4070000] [87.1/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 12:14:30,376 - Train: 8.72% [354800/4070000] [87.2/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 12:15:30,881 - Train: 8.72% [354900/4070000] [87.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 12:16:31,395 - Train: 8.72% [355000/4070000] [87.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 12:17:32,670 - Train: 8.72% [355100/4070000] [87.2/1000.0] [batch_t 0.606 (0.613)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 12:18:33,189 - Train: 8.73% [355200/4070000] [87.3/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 12:19:33,887 - Train: 8.73% [355300/4070000] [87.3/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 12:20:34,401 - Train: 8.73% [355400/4070000] [87.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 12:21:34,926 - Train: 8.73% [355500/4070000] [87.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 12:22:35,439 - Train: 8.74% [355600/4070000] [87.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 12:23:35,964 - Train: 8.74% [355700/4070000] [87.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 12:24:36,490 - Train: 8.74% [355800/4070000] [87.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 12:25:36,997 - Train: 8.74% [355900/4070000] [87.4/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 12:26:37,643 - Train: 8.75% [356000/4070000] [87.5/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 12:27:38,171 - Train: 8.75% [356100/4070000] [87.5/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 12:28:38,689 - Train: 8.75% [356200/4070000] [87.5/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 12:29:39,176 - Train: 8.75% [356300/4070000] [87.5/1000.0] [batch_t 0.598 (0.605)] [data_t 0.002] [optim_t 0.596] [lr 0.005000] +2024-04-05 12:30:39,661 - Train: 8.76% [356400/4070000] [87.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 12:31:40,171 - Train: 8.76% [356500/4070000] [87.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 12:32:40,672 - Train: 8.76% [356600/4070000] [87.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 12:33:41,366 - Train: 8.76% [356700/4070000] [87.6/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 12:34:42,025 - Train: 8.77% [356800/4070000] [87.7/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 12:35:42,523 - Train: 8.77% [356900/4070000] [87.7/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 12:36:43,005 - Train: 8.77% [357000/4070000] [87.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 12:37:43,499 - Train: 8.77% [357100/4070000] [87.7/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-05 12:38:44,001 - Train: 8.78% [357200/4070000] [87.8/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-05 12:39:44,685 - Train: 8.78% [357300/4070000] [87.8/1000.0] [batch_t 0.610 (0.607)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-05 12:40:45,209 - Train: 8.78% [357400/4070000] [87.8/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 12:41:45,685 - Train: 8.78% [357500/4070000] [87.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 12:42:46,201 - Train: 8.79% [357600/4070000] [87.9/1000.0] [batch_t 0.610 (0.605)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-05 12:43:48,111 - Train: 8.79% [357700/4070000] [87.9/1000.0] [batch_t 0.604 (0.619)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 12:44:48,637 - Train: 8.79% [357800/4070000] [87.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 12:45:49,149 - Train: 8.79% [357900/4070000] [87.9/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 12:46:49,795 - Train: 8.80% [358000/4070000] [88.0/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 12:47:50,336 - Train: 8.80% [358100/4070000] [88.0/1000.0] [batch_t 0.610 (0.605)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-05 12:48:26,658 - ==> Total time: 3 days, 11:47:57 Eta: 36 days, 4:27:56 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-05 12:48:52,588 - Train: 8.80% [358200/4070000] [88.0/1000.0] [batch_t 0.604 (0.610)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 12:49:53,078 - Train: 8.80% [358300/4070000] [88.0/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 12:50:53,617 - Train: 8.81% [358400/4070000] [88.1/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 12:51:54,282 - Train: 8.81% [358500/4070000] [88.1/1000.0] [batch_t 0.602 (0.607)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-05 12:52:54,784 - Train: 8.81% [358600/4070000] [88.1/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-05 12:53:55,337 - Train: 8.81% [358700/4070000] [88.1/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-05 12:54:55,850 - Train: 8.82% [358800/4070000] [88.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 12:55:56,360 - Train: 8.82% [358900/4070000] [88.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 12:56:56,857 - Train: 8.82% [359000/4070000] [88.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 12:57:57,406 - Train: 8.82% [359100/4070000] [88.2/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 12:58:58,154 - Train: 8.83% [359200/4070000] [88.3/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 12:59:58,689 - Train: 8.83% [359300/4070000] [88.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 13:00:59,209 - Train: 8.83% [359400/4070000] [88.3/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 13:02:01,537 - Train: 8.83% [359500/4070000] [88.3/1000.0] [batch_t 0.604 (0.623)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 13:03:02,071 - Train: 8.84% [359600/4070000] [88.4/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 13:04:02,571 - Train: 8.84% [359700/4070000] [88.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 13:05:03,086 - Train: 8.84% [359800/4070000] [88.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 13:06:03,779 - Train: 8.84% [359900/4070000] [88.4/1000.0] [batch_t 0.601 (0.607)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-05 13:07:04,306 - Train: 8.85% [360000/4070000] [88.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 13:08:04,842 - Train: 8.85% [360100/4070000] [88.5/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-05 13:09:05,352 - Train: 8.85% [360200/4070000] [88.5/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-05 13:10:05,886 - Train: 8.85% [360300/4070000] [88.5/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 13:11:06,418 - Train: 8.86% [360400/4070000] [88.6/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 13:12:07,053 - Train: 8.86% [360500/4070000] [88.6/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 13:13:07,620 - Train: 8.86% [360600/4070000] [88.6/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 13:14:08,116 - Train: 8.86% [360700/4070000] [88.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 13:15:08,620 - Train: 8.86% [360800/4070000] [88.6/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 13:16:09,177 - Train: 8.87% [360900/4070000] [88.7/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 13:17:09,750 - Train: 8.87% [361000/4070000] [88.7/1000.0] [batch_t 0.611 (0.606)] [data_t 0.002] [optim_t 0.609] [lr 0.005000] +2024-04-05 13:18:10,291 - Train: 8.87% [361100/4070000] [88.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 13:19:10,815 - Train: 8.87% [361200/4070000] [88.7/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-05 13:20:11,469 - Train: 8.88% [361300/4070000] [88.8/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 13:21:12,013 - Train: 8.88% [361400/4070000] [88.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 13:22:12,546 - Train: 8.88% [361500/4070000] [88.8/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 13:23:13,047 - Train: 8.88% [361600/4070000] [88.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 13:24:13,550 - Train: 8.89% [361700/4070000] [88.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 13:25:14,396 - Train: 8.89% [361800/4070000] [88.9/1000.0] [batch_t 0.607 (0.608)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 13:26:15,001 - Train: 8.89% [361900/4070000] [88.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 13:27:15,485 - Train: 8.89% [362000/4070000] [88.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 13:28:15,969 - Train: 8.90% [362100/4070000] [89.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 13:29:16,467 - Train: 8.90% [362200/4070000] [89.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 13:29:34,628 - ==> Total time: 3 days, 12:29:05 Eta: 36 days, 0:46:59 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-05 13:30:20,950 - Train: 8.90% [362300/4070000] [89.0/1000.0] [batch_t 0.603 (0.640)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 13:31:30,158 - Train: 8.90% [362400/4070000] [89.0/1000.0] [batch_t 0.605 (0.692)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 13:32:30,665 - Train: 8.91% [362500/4070000] [89.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 13:33:31,259 - Train: 8.91% [362600/4070000] [89.1/1000.0] [batch_t 0.602 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 13:34:31,776 - Train: 8.91% [362700/4070000] [89.1/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-05 13:35:37,552 - Train: 8.91% [362800/4070000] [89.1/1000.0] [batch_t 0.606 (0.658)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 13:36:38,054 - Train: 8.92% [362900/4070000] [89.2/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 13:37:38,528 - Train: 8.92% [363000/4070000] [89.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 13:38:39,005 - Train: 8.92% [363100/4070000] [89.2/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 13:39:42,181 - Train: 8.92% [363200/4070000] [89.2/1000.0] [batch_t 0.607 (0.632)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 13:40:42,840 - Train: 8.93% [363300/4070000] [89.3/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 13:41:44,803 - Train: 8.93% [363400/4070000] [89.3/1000.0] [batch_t 0.609 (0.620)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-05 13:42:45,298 - Train: 8.93% [363500/4070000] [89.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 13:43:46,112 - Train: 8.93% [363600/4070000] [89.3/1000.0] [batch_t 0.599 (0.608)] [data_t 0.002] [optim_t 0.597] [lr 0.005000] +2024-04-05 13:44:46,639 - Train: 8.94% [363700/4070000] [89.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 13:45:47,142 - Train: 8.94% [363800/4070000] [89.4/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 13:46:47,830 - Train: 8.94% [363900/4070000] [89.4/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 13:47:48,363 - Train: 8.94% [364000/4070000] [89.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 13:48:48,874 - Train: 8.95% [364100/4070000] [89.5/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 13:49:49,395 - Train: 8.95% [364200/4070000] [89.5/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 13:50:49,940 - Train: 8.95% [364300/4070000] [89.5/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 13:51:50,598 - Train: 8.95% [364400/4070000] [89.5/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 13:52:51,090 - Train: 8.96% [364500/4070000] [89.6/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 13:53:51,620 - Train: 8.96% [364600/4070000] [89.6/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-05 13:54:52,147 - Train: 8.96% [364700/4070000] [89.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 13:55:52,670 - Train: 8.96% [364800/4070000] [89.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 13:56:53,166 - Train: 8.97% [364900/4070000] [89.7/1000.0] [batch_t 0.610 (0.605)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-05 13:57:53,676 - Train: 8.97% [365000/4070000] [89.7/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 13:58:54,211 - Train: 8.97% [365100/4070000] [89.7/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 13:59:54,870 - Train: 8.97% [365200/4070000] [89.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 14:00:55,399 - Train: 8.98% [365300/4070000] [89.8/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-05 14:01:55,904 - Train: 8.98% [365400/4070000] [89.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 14:02:56,615 - Train: 8.98% [365500/4070000] [89.8/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 14:03:58,203 - Train: 8.98% [365600/4070000] [89.8/1000.0] [batch_t 0.609 (0.616)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-05 14:04:58,864 - Train: 8.99% [365700/4070000] [89.9/1000.0] [batch_t 0.612 (0.607)] [data_t 0.002] [optim_t 0.610] [lr 0.005000] +2024-04-05 14:05:59,387 - Train: 8.99% [365800/4070000] [89.9/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 14:06:59,914 - Train: 8.99% [365900/4070000] [89.9/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 14:08:00,502 - Train: 8.99% [366000/4070000] [89.9/1000.0] [batch_t 0.611 (0.606)] [data_t 0.002] [optim_t 0.609] [lr 0.005000] +2024-04-05 14:09:00,998 - Train: 9.00% [366100/4070000] [90.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 14:10:01,500 - Train: 9.00% [366200/4070000] [90.0/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 14:11:02,012 - Train: 9.00% [366300/4070000] [90.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 14:11:02,016 - ==> Total time: 3 days, 13:10:32 Eta: 35 days, 21:13:19 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-05 14:12:07,816 - Train: 9.00% [366400/4070000] [90.0/1000.0] [batch_t 0.606 (0.620)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 14:13:08,369 - Train: 9.00% [366500/4070000] [90.0/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-05 14:14:08,867 - Train: 9.01% [366600/4070000] [90.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 14:15:10,025 - Train: 9.01% [366700/4070000] [90.1/1000.0] [batch_t 0.606 (0.611)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 14:16:11,085 - Train: 9.01% [366800/4070000] [90.1/1000.0] [batch_t 0.604 (0.610)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 14:17:11,617 - Train: 9.01% [366900/4070000] [90.1/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 14:18:12,218 - Train: 9.02% [367000/4070000] [90.2/1000.0] [batch_t 0.611 (0.606)] [data_t 0.002] [optim_t 0.609] [lr 0.005000] +2024-04-05 14:19:12,774 - Train: 9.02% [367100/4070000] [90.2/1000.0] [batch_t 0.611 (0.605)] [data_t 0.002] [optim_t 0.609] [lr 0.005000] +2024-04-05 14:20:13,272 - Train: 9.02% [367200/4070000] [90.2/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-05 14:21:13,782 - Train: 9.02% [367300/4070000] [90.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 14:22:14,264 - Train: 9.03% [367400/4070000] [90.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 14:23:14,793 - Train: 9.03% [367500/4070000] [90.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 14:24:15,310 - Train: 9.03% [367600/4070000] [90.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 14:25:15,879 - Train: 9.03% [367700/4070000] [90.3/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 14:26:16,746 - Train: 9.04% [367800/4070000] [90.4/1000.0] [batch_t 0.605 (0.609)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 14:27:17,232 - Train: 9.04% [367900/4070000] [90.4/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-05 14:28:17,714 - Train: 9.04% [368000/4070000] [90.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 14:29:18,252 - Train: 9.04% [368100/4070000] [90.4/1000.0] [batch_t 0.617 (0.605)] [data_t 0.002] [optim_t 0.615] [lr 0.005000] +2024-04-05 14:30:18,790 - Train: 9.05% [368200/4070000] [90.5/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 14:31:19,431 - Train: 9.05% [368300/4070000] [90.5/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 14:32:19,896 - Train: 9.05% [368400/4070000] [90.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 14:33:20,386 - Train: 9.05% [368500/4070000] [90.5/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-05 14:34:20,887 - Train: 9.06% [368600/4070000] [90.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 14:35:21,377 - Train: 9.06% [368700/4070000] [90.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 14:36:21,929 - Train: 9.06% [368800/4070000] [90.6/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 14:37:22,457 - Train: 9.06% [368900/4070000] [90.6/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-05 14:38:23,056 - Train: 9.07% [369000/4070000] [90.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 14:39:23,589 - Train: 9.07% [369100/4070000] [90.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 14:40:24,128 - Train: 9.07% [369200/4070000] [90.7/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 14:41:24,638 - Train: 9.07% [369300/4070000] [90.7/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 14:42:25,168 - Train: 9.08% [369400/4070000] [90.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 14:43:25,819 - Train: 9.08% [369500/4070000] [90.8/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 14:44:26,310 - Train: 9.08% [369600/4070000] [90.8/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 14:45:26,827 - Train: 9.08% [369700/4070000] [90.8/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 14:46:27,307 - Train: 9.09% [369800/4070000] [90.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 14:47:27,754 - Train: 9.09% [369900/4070000] [90.9/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 14:48:28,213 - Train: 9.09% [370000/4070000] [90.9/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 14:49:28,710 - Train: 9.09% [370100/4070000] [90.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 14:50:29,196 - Train: 9.10% [370200/4070000] [91.0/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 14:51:29,760 - Train: 9.10% [370300/4070000] [91.0/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 14:52:12,097 - ==> Total time: 3 days, 13:51:42 Eta: 35 days, 17:40:32 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-05 14:52:32,019 - Train: 9.10% [370400/4070000] [91.0/1000.0] [batch_t 0.605 (0.614)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 14:53:32,487 - Train: 9.10% [370500/4070000] [91.0/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 14:54:32,991 - Train: 9.11% [370600/4070000] [91.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 14:55:33,474 - Train: 9.11% [370700/4070000] [91.1/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-05 14:56:34,128 - Train: 9.11% [370800/4070000] [91.1/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 14:57:34,628 - Train: 9.11% [370900/4070000] [91.1/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 14:58:35,137 - Train: 9.12% [371000/4070000] [91.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 14:59:35,634 - Train: 9.12% [371100/4070000] [91.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 15:00:36,186 - Train: 9.12% [371200/4070000] [91.2/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 15:01:36,654 - Train: 9.12% [371300/4070000] [91.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 15:02:37,160 - Train: 9.13% [371400/4070000] [91.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 15:03:37,730 - Train: 9.13% [371500/4070000] [91.3/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 15:04:38,225 - Train: 9.13% [371600/4070000] [91.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 15:05:38,700 - Train: 9.13% [371700/4070000] [91.3/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-05 15:06:39,185 - Train: 9.14% [371800/4070000] [91.4/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 15:07:39,685 - Train: 9.14% [371900/4070000] [91.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 15:08:40,194 - Train: 9.14% [372000/4070000] [91.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 15:09:40,848 - Train: 9.14% [372100/4070000] [91.4/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 15:10:41,361 - Train: 9.14% [372200/4070000] [91.4/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 15:11:41,867 - Train: 9.15% [372300/4070000] [91.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 15:12:42,421 - Train: 9.15% [372400/4070000] [91.5/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 15:13:42,915 - Train: 9.15% [372500/4070000] [91.5/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 15:14:43,433 - Train: 9.15% [372600/4070000] [91.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 15:15:43,938 - Train: 9.16% [372700/4070000] [91.6/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 15:16:44,609 - Train: 9.16% [372800/4070000] [91.6/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 15:17:45,155 - Train: 9.16% [372900/4070000] [91.6/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 15:18:45,672 - Train: 9.16% [373000/4070000] [91.6/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 15:19:46,660 - Train: 9.17% [373100/4070000] [91.7/1000.0] [batch_t 0.604 (0.610)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 15:20:47,158 - Train: 9.17% [373200/4070000] [91.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 15:21:47,718 - Train: 9.17% [373300/4070000] [91.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 15:22:48,382 - Train: 9.17% [373400/4070000] [91.7/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 15:23:48,877 - Train: 9.18% [373500/4070000] [91.8/1000.0] [batch_t 0.610 (0.605)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-05 15:24:49,417 - Train: 9.18% [373600/4070000] [91.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 15:25:49,924 - Train: 9.18% [373700/4070000] [91.8/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 15:26:50,484 - Train: 9.18% [373800/4070000] [91.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 15:27:51,009 - Train: 9.19% [373900/4070000] [91.9/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 15:28:51,497 - Train: 9.19% [374000/4070000] [91.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 15:29:52,134 - Train: 9.19% [374100/4070000] [91.9/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 15:30:52,636 - Train: 9.19% [374200/4070000] [91.9/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 15:31:53,553 - Train: 9.20% [374300/4070000] [92.0/1000.0] [batch_t 0.607 (0.609)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-05 15:32:54,117 - Train: 9.20% [374400/4070000] [92.0/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 15:33:18,308 - ==> Total time: 3 days, 14:32:49 Eta: 35 days, 14:10:52 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-05 15:33:56,346 - Train: 9.20% [374500/4070000] [92.0/1000.0] [batch_t 0.603 (0.608)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 15:34:57,029 - Train: 9.20% [374600/4070000] [92.0/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 15:35:57,564 - Train: 9.21% [374700/4070000] [92.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 15:36:58,077 - Train: 9.21% [374800/4070000] [92.1/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 15:37:58,569 - Train: 9.21% [374900/4070000] [92.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 15:38:59,068 - Train: 9.21% [375000/4070000] [92.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 15:39:59,571 - Train: 9.22% [375100/4070000] [92.2/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 15:41:00,081 - Train: 9.22% [375200/4070000] [92.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 15:42:00,591 - Train: 9.22% [375300/4070000] [92.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 15:43:01,372 - Train: 9.22% [375400/4070000] [92.2/1000.0] [batch_t 0.601 (0.608)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-05 15:44:02,408 - Train: 9.23% [375500/4070000] [92.3/1000.0] [batch_t 0.605 (0.610)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 15:45:02,897 - Train: 9.23% [375600/4070000] [92.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 15:46:03,798 - Train: 9.23% [375700/4070000] [92.3/1000.0] [batch_t 0.606 (0.609)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 15:47:04,336 - Train: 9.23% [375800/4070000] [92.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 15:48:04,835 - Train: 9.24% [375900/4070000] [92.4/1000.0] [batch_t 0.601 (0.605)] [data_t 0.003] [optim_t 0.598] [lr 0.005000] +2024-04-05 15:49:05,305 - Train: 9.24% [376000/4070000] [92.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 15:50:05,905 - Train: 9.24% [376100/4070000] [92.4/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 15:51:06,370 - Train: 9.24% [376200/4070000] [92.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 15:52:06,875 - Train: 9.25% [376300/4070000] [92.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 15:53:07,731 - Train: 9.25% [376400/4070000] [92.5/1000.0] [batch_t 0.604 (0.608)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 15:54:08,225 - Train: 9.25% [376500/4070000] [92.5/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-05 15:55:08,740 - Train: 9.25% [376600/4070000] [92.5/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 15:56:09,490 - Train: 9.26% [376700/4070000] [92.6/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 15:57:10,169 - Train: 9.26% [376800/4070000] [92.6/1000.0] [batch_t 0.602 (0.607)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-05 15:58:10,713 - Train: 9.26% [376900/4070000] [92.6/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-05 15:59:11,269 - Train: 9.26% [377000/4070000] [92.6/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 16:00:11,794 - Train: 9.27% [377100/4070000] [92.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 16:01:12,345 - Train: 9.27% [377200/4070000] [92.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 16:02:12,853 - Train: 9.27% [377300/4070000] [92.7/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-05 16:03:13,377 - Train: 9.27% [377400/4070000] [92.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 16:04:13,993 - Train: 9.28% [377500/4070000] [92.8/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 16:05:14,529 - Train: 9.28% [377600/4070000] [92.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 16:06:15,038 - Train: 9.28% [377700/4070000] [92.8/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-05 16:07:15,512 - Train: 9.28% [377800/4070000] [92.8/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 16:08:16,038 - Train: 9.29% [377900/4070000] [92.9/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 16:09:16,629 - Train: 9.29% [378000/4070000] [92.9/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 16:10:17,165 - Train: 9.29% [378100/4070000] [92.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 16:11:17,631 - Train: 9.29% [378200/4070000] [92.9/1000.0] [batch_t 0.611 (0.605)] [data_t 0.002] [optim_t 0.609] [lr 0.005000] +2024-04-05 16:12:18,144 - Train: 9.29% [378300/4070000] [92.9/1000.0] [batch_t 0.613 (0.605)] [data_t 0.004] [optim_t 0.609] [lr 0.005000] +2024-04-05 16:13:18,654 - Train: 9.30% [378400/4070000] [93.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 16:14:19,164 - Train: 9.30% [378500/4070000] [93.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 16:14:25,208 - ==> Total time: 3 days, 15:13:56 Eta: 35 days, 10:44:55 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-05 16:15:21,982 - Train: 9.30% [378600/4070000] [93.0/1000.0] [batch_t 0.605 (0.610)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 16:16:22,480 - Train: 9.30% [378700/4070000] [93.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 16:17:23,068 - Train: 9.31% [378800/4070000] [93.1/1000.0] [batch_t 0.596 (0.606)] [data_t 0.002] [optim_t 0.594] [lr 0.005000] +2024-04-05 16:18:23,558 - Train: 9.31% [378900/4070000] [93.1/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 16:19:24,044 - Train: 9.31% [379000/4070000] [93.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 16:20:24,682 - Train: 9.31% [379100/4070000] [93.1/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 16:21:25,157 - Train: 9.32% [379200/4070000] [93.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 16:22:25,667 - Train: 9.32% [379300/4070000] [93.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 16:23:26,221 - Train: 9.32% [379400/4070000] [93.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 16:24:26,710 - Train: 9.32% [379500/4070000] [93.2/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 16:25:27,193 - Train: 9.33% [379600/4070000] [93.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 16:26:27,687 - Train: 9.33% [379700/4070000] [93.3/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 16:27:28,279 - Train: 9.33% [379800/4070000] [93.3/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 16:28:28,759 - Train: 9.33% [379900/4070000] [93.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 16:29:29,257 - Train: 9.34% [380000/4070000] [93.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 16:30:29,731 - Train: 9.34% [380100/4070000] [93.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 16:31:30,224 - Train: 9.34% [380200/4070000] [93.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 16:32:30,707 - Train: 9.34% [380300/4070000] [93.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 16:33:31,316 - Train: 9.35% [380400/4070000] [93.5/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 16:34:31,812 - Train: 9.35% [380500/4070000] [93.5/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 16:35:32,306 - Train: 9.35% [380600/4070000] [93.5/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-05 16:36:32,796 - Train: 9.35% [380700/4070000] [93.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 16:37:33,259 - Train: 9.36% [380800/4070000] [93.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 16:38:33,783 - Train: 9.36% [380900/4070000] [93.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 16:39:34,295 - Train: 9.36% [381000/4070000] [93.6/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 16:40:34,851 - Train: 9.36% [381100/4070000] [93.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 16:41:35,336 - Train: 9.37% [381200/4070000] [93.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 16:42:35,843 - Train: 9.37% [381300/4070000] [93.7/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-05 16:43:36,333 - Train: 9.37% [381400/4070000] [93.7/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 16:44:36,822 - Train: 9.37% [381500/4070000] [93.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 16:45:37,418 - Train: 9.38% [381600/4070000] [93.8/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 16:46:37,893 - Train: 9.38% [381700/4070000] [93.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 16:47:38,400 - Train: 9.38% [381800/4070000] [93.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 16:48:38,854 - Train: 9.38% [381900/4070000] [93.8/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 16:49:39,355 - Train: 9.39% [382000/4070000] [93.9/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 16:50:39,881 - Train: 9.39% [382100/4070000] [93.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 16:51:40,360 - Train: 9.39% [382200/4070000] [93.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 16:52:40,866 - Train: 9.39% [382300/4070000] [93.9/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-05 16:53:41,421 - Train: 9.40% [382400/4070000] [94.0/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 16:54:41,894 - Train: 9.40% [382500/4070000] [94.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 16:55:30,252 - ==> Total time: 3 days, 15:55:01 Eta: 35 days, 7:22:11 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-05 16:55:44,316 - Train: 9.40% [382600/4070000] [94.0/1000.0] [batch_t 0.607 (0.616)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 16:56:44,817 - Train: 9.40% [382700/4070000] [94.0/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-05 16:57:45,310 - Train: 9.41% [382800/4070000] [94.1/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 16:58:45,940 - Train: 9.41% [382900/4070000] [94.1/1000.0] [batch_t 0.609 (0.606)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-05 16:59:46,384 - Train: 9.41% [383000/4070000] [94.1/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 17:00:46,891 - Train: 9.41% [383100/4070000] [94.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 17:01:47,376 - Train: 9.42% [383200/4070000] [94.2/1000.0] [batch_t 0.600 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-05 17:02:47,883 - Train: 9.42% [383300/4070000] [94.2/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-05 17:03:48,381 - Train: 9.42% [383400/4070000] [94.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 17:04:48,864 - Train: 9.42% [383500/4070000] [94.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 17:05:49,426 - Train: 9.43% [383600/4070000] [94.3/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 17:06:49,976 - Train: 9.43% [383700/4070000] [94.3/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 17:07:50,500 - Train: 9.43% [383800/4070000] [94.3/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 17:08:50,976 - Train: 9.43% [383900/4070000] [94.3/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 17:09:51,515 - Train: 9.43% [384000/4070000] [94.3/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 17:10:51,989 - Train: 9.44% [384100/4070000] [94.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 17:11:52,646 - Train: 9.44% [384200/4070000] [94.4/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 17:12:53,203 - Train: 9.44% [384300/4070000] [94.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 17:13:53,723 - Train: 9.44% [384400/4070000] [94.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 17:14:54,242 - Train: 9.45% [384500/4070000] [94.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 17:15:54,735 - Train: 9.45% [384600/4070000] [94.5/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 17:16:55,761 - Train: 9.45% [384700/4070000] [94.5/1000.0] [batch_t 0.605 (0.610)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 17:17:56,301 - Train: 9.45% [384800/4070000] [94.5/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 17:18:57,545 - Train: 9.46% [384900/4070000] [94.6/1000.0] [batch_t 0.605 (0.612)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 17:19:58,046 - Train: 9.46% [385000/4070000] [94.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 17:20:58,587 - Train: 9.46% [385100/4070000] [94.6/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 17:21:59,095 - Train: 9.46% [385200/4070000] [94.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 17:22:59,626 - Train: 9.47% [385300/4070000] [94.7/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-05 17:24:00,279 - Train: 9.47% [385400/4070000] [94.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 17:25:00,785 - Train: 9.47% [385500/4070000] [94.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 17:26:01,475 - Train: 9.47% [385600/4070000] [94.7/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 17:27:01,983 - Train: 9.48% [385700/4070000] [94.8/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 17:28:02,562 - Train: 9.48% [385800/4070000] [94.8/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 17:29:03,129 - Train: 9.48% [385900/4070000] [94.8/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 17:30:03,737 - Train: 9.48% [386000/4070000] [94.8/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 17:31:04,285 - Train: 9.49% [386100/4070000] [94.9/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 17:32:04,921 - Train: 9.49% [386200/4070000] [94.9/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 17:33:05,508 - Train: 9.49% [386300/4070000] [94.9/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 17:34:06,002 - Train: 9.49% [386400/4070000] [94.9/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 17:35:06,517 - Train: 9.50% [386500/4070000] [95.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 17:36:07,090 - Train: 9.50% [386600/4070000] [95.0/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 17:36:37,339 - ==> Total time: 3 days, 16:36:08 Eta: 35 days, 4:03:11 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-05 17:37:09,389 - Train: 9.50% [386700/4070000] [95.0/1000.0] [batch_t 0.605 (0.613)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 17:38:10,015 - Train: 9.50% [386800/4070000] [95.0/1000.0] [batch_t 0.608 (0.606)] [data_t 0.003] [optim_t 0.606] [lr 0.005000] +2024-04-05 17:39:10,545 - Train: 9.51% [386900/4070000] [95.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 17:40:11,199 - Train: 9.51% [387000/4070000] [95.1/1000.0] [batch_t 0.613 (0.606)] [data_t 0.002] [optim_t 0.611] [lr 0.005000] +2024-04-05 17:41:11,751 - Train: 9.51% [387100/4070000] [95.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 17:42:12,265 - Train: 9.51% [387200/4070000] [95.1/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 17:43:12,760 - Train: 9.52% [387300/4070000] [95.2/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-05 17:44:13,240 - Train: 9.52% [387400/4070000] [95.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 17:45:13,744 - Train: 9.52% [387500/4070000] [95.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 17:46:14,333 - Train: 9.52% [387600/4070000] [95.2/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 17:47:14,814 - Train: 9.53% [387700/4070000] [95.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 17:48:15,371 - Train: 9.53% [387800/4070000] [95.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 17:49:15,845 - Train: 9.53% [387900/4070000] [95.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 17:50:16,362 - Train: 9.53% [388000/4070000] [95.3/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 17:51:16,996 - Train: 9.54% [388100/4070000] [95.4/1000.0] [batch_t 0.610 (0.606)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-05 17:52:17,500 - Train: 9.54% [388200/4070000] [95.4/1000.0] [batch_t 0.612 (0.605)] [data_t 0.002] [optim_t 0.610] [lr 0.005000] +2024-04-05 17:53:18,006 - Train: 9.54% [388300/4070000] [95.4/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 17:54:18,512 - Train: 9.54% [388400/4070000] [95.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 17:55:19,027 - Train: 9.55% [388500/4070000] [95.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 17:56:19,507 - Train: 9.55% [388600/4070000] [95.5/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 17:57:20,579 - Train: 9.55% [388700/4070000] [95.5/1000.0] [batch_t 0.606 (0.611)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 17:58:21,085 - Train: 9.55% [388800/4070000] [95.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 17:59:21,710 - Train: 9.56% [388900/4070000] [95.6/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 18:00:22,248 - Train: 9.56% [389000/4070000] [95.6/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 18:01:22,734 - Train: 9.56% [389100/4070000] [95.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 18:02:23,265 - Train: 9.56% [389200/4070000] [95.6/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 18:03:23,773 - Train: 9.57% [389300/4070000] [95.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 18:04:24,392 - Train: 9.57% [389400/4070000] [95.7/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 18:05:24,913 - Train: 9.57% [389500/4070000] [95.7/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 18:06:25,480 - Train: 9.57% [389600/4070000] [95.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 18:07:25,974 - Train: 9.57% [389700/4070000] [95.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 18:08:26,532 - Train: 9.58% [389800/4070000] [95.8/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 18:09:27,007 - Train: 9.58% [389900/4070000] [95.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 18:10:27,545 - Train: 9.58% [390000/4070000] [95.8/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 18:11:28,046 - Train: 9.58% [390100/4070000] [95.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 18:12:28,690 - Train: 9.59% [390200/4070000] [95.9/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 18:13:29,216 - Train: 9.59% [390300/4070000] [95.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 18:14:29,773 - Train: 9.59% [390400/4070000] [95.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 18:15:30,351 - Train: 9.59% [390500/4070000] [95.9/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 18:16:30,887 - Train: 9.60% [390600/4070000] [96.0/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 18:17:31,758 - Train: 9.60% [390700/4070000] [96.0/1000.0] [batch_t 0.604 (0.609)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 18:17:43,863 - ==> Total time: 3 days, 17:17:14 Eta: 35 days, 0:47:23 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-05 18:18:34,022 - Train: 9.60% [390800/4070000] [96.0/1000.0] [batch_t 0.599 (0.607)] [data_t 0.002] [optim_t 0.597] [lr 0.005000] +2024-04-05 18:19:34,533 - Train: 9.60% [390900/4070000] [96.0/1000.0] [batch_t 0.598 (0.605)] [data_t 0.002] [optim_t 0.596] [lr 0.005000] +2024-04-05 18:20:35,029 - Train: 9.61% [391000/4070000] [96.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 18:21:35,543 - Train: 9.61% [391100/4070000] [96.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 18:22:49,665 - Train: 9.61% [391200/4070000] [96.1/1000.0] [batch_t 0.604 (0.741)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 18:24:02,269 - Train: 9.61% [391300/4070000] [96.1/1000.0] [batch_t 0.608 (0.726)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 18:25:12,351 - Train: 9.62% [391400/4070000] [96.2/1000.0] [batch_t 2.588 (0.701)] [data_t 1.984] [optim_t 0.604] [lr 0.005000] +2024-04-05 18:26:33,161 - Train: 9.62% [391500/4070000] [96.2/1000.0] [batch_t 0.604 (0.808)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 18:27:41,460 - Train: 9.62% [391600/4070000] [96.2/1000.0] [batch_t 0.606 (0.683)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 18:28:50,652 - Train: 9.62% [391700/4070000] [96.2/1000.0] [batch_t 0.606 (0.692)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 18:29:58,306 - Train: 9.63% [391800/4070000] [96.3/1000.0] [batch_t 0.603 (0.676)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 18:30:58,990 - Train: 9.63% [391900/4070000] [96.3/1000.0] [batch_t 0.603 (0.607)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 18:31:59,519 - Train: 9.63% [392000/4070000] [96.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 18:33:00,025 - Train: 9.63% [392100/4070000] [96.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 18:34:00,546 - Train: 9.64% [392200/4070000] [96.4/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-05 18:35:01,060 - Train: 9.64% [392300/4070000] [96.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 18:36:01,586 - Train: 9.64% [392400/4070000] [96.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 18:37:02,205 - Train: 9.64% [392500/4070000] [96.4/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 18:38:02,717 - Train: 9.65% [392600/4070000] [96.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 18:39:03,487 - Train: 9.65% [392700/4070000] [96.5/1000.0] [batch_t 0.606 (0.608)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 18:40:04,081 - Train: 9.65% [392800/4070000] [96.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.003] [optim_t 0.603] [lr 0.005000] +2024-04-05 18:41:04,599 - Train: 9.65% [392900/4070000] [96.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 18:42:05,214 - Train: 9.66% [393000/4070000] [96.6/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 18:43:05,761 - Train: 9.66% [393100/4070000] [96.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 18:44:08,122 - Train: 9.66% [393200/4070000] [96.6/1000.0] [batch_t 1.398 (0.624)] [data_t 0.791] [optim_t 0.606] [lr 0.005000] +2024-04-05 18:45:09,088 - Train: 9.66% [393300/4070000] [96.6/1000.0] [batch_t 0.603 (0.610)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 18:46:09,587 - Train: 9.67% [393400/4070000] [96.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 18:47:10,075 - Train: 9.67% [393500/4070000] [96.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 18:48:10,581 - Train: 9.67% [393600/4070000] [96.7/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 18:49:11,292 - Train: 9.67% [393700/4070000] [96.7/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 18:50:11,823 - Train: 9.68% [393800/4070000] [96.8/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 18:51:12,313 - Train: 9.68% [393900/4070000] [96.8/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 18:52:12,884 - Train: 9.68% [394000/4070000] [96.8/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 18:53:13,348 - Train: 9.68% [394100/4070000] [96.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 18:54:13,856 - Train: 9.69% [394200/4070000] [96.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 18:55:16,460 - Train: 9.69% [394300/4070000] [96.9/1000.0] [batch_t 0.605 (0.626)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 18:56:17,002 - Train: 9.69% [394400/4070000] [96.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 18:57:17,686 - Train: 9.69% [394500/4070000] [96.9/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 18:58:18,222 - Train: 9.70% [394600/4070000] [97.0/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 18:59:18,764 - Train: 9.70% [394700/4070000] [97.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 19:00:13,250 - ==> Total time: 3 days, 17:59:44 Eta: 34 days, 21:47:38 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-05 19:00:21,926 - Train: 9.70% [394800/4070000] [97.0/1000.0] [batch_t 0.604 (0.681)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 19:01:22,475 - Train: 9.70% [394900/4070000] [97.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 19:02:24,665 - Train: 9.71% [395000/4070000] [97.1/1000.0] [batch_t 0.604 (0.622)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 19:03:34,682 - Train: 9.71% [395100/4070000] [97.1/1000.0] [batch_t 0.603 (0.700)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 19:04:47,390 - Train: 9.71% [395200/4070000] [97.1/1000.0] [batch_t 0.606 (0.727)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 19:05:54,677 - Train: 9.71% [395300/4070000] [97.1/1000.0] [batch_t 0.606 (0.673)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 19:07:02,594 - Train: 9.71% [395400/4070000] [97.1/1000.0] [batch_t 0.893 (0.679)] [data_t 0.291] [optim_t 0.602] [lr 0.005000] +2024-04-05 19:08:08,638 - Train: 9.72% [395500/4070000] [97.2/1000.0] [batch_t 0.603 (0.660)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 19:09:11,466 - Train: 9.72% [395600/4070000] [97.2/1000.0] [batch_t 0.606 (0.628)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 19:10:18,024 - Train: 9.72% [395700/4070000] [97.2/1000.0] [batch_t 0.603 (0.665)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 19:11:56,696 - Train: 9.72% [395800/4070000] [97.2/1000.0] [batch_t 1.541 (0.987)] [data_t 0.934] [optim_t 0.607] [lr 0.005000] +2024-04-05 19:13:40,515 - Train: 9.73% [395900/4070000] [97.3/1000.0] [batch_t 1.204 (1.038)] [data_t 0.601] [optim_t 0.603] [lr 0.005000] +2024-04-05 19:15:14,611 - Train: 9.73% [396000/4070000] [97.3/1000.0] [batch_t 1.937 (0.941)] [data_t 1.336] [optim_t 0.600] [lr 0.005000] +2024-04-05 19:16:26,580 - Train: 9.73% [396100/4070000] [97.3/1000.0] [batch_t 0.605 (0.720)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 19:17:27,048 - Train: 9.73% [396200/4070000] [97.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 19:18:27,941 - Train: 9.74% [396300/4070000] [97.4/1000.0] [batch_t 0.602 (0.609)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-05 19:19:29,748 - Train: 9.74% [396400/4070000] [97.4/1000.0] [batch_t 0.607 (0.618)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 19:20:31,182 - Train: 9.74% [396500/4070000] [97.4/1000.0] [batch_t 0.607 (0.614)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 19:21:31,676 - Train: 9.74% [396600/4070000] [97.4/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-05 19:22:32,884 - Train: 9.75% [396700/4070000] [97.5/1000.0] [batch_t 0.606 (0.612)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 19:23:37,153 - Train: 9.75% [396800/4070000] [97.5/1000.0] [batch_t 0.606 (0.643)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 19:24:38,835 - Train: 9.75% [396900/4070000] [97.5/1000.0] [batch_t 0.604 (0.617)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 19:26:10,087 - Train: 9.75% [397000/4070000] [97.5/1000.0] [batch_t 0.834 (0.912)] [data_t 0.233] [optim_t 0.601] [lr 0.005000] +2024-04-05 19:27:38,520 - Train: 9.76% [397100/4070000] [97.6/1000.0] [batch_t 0.605 (0.884)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 19:29:31,795 - Train: 9.76% [397200/4070000] [97.6/1000.0] [batch_t 0.606 (1.133)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 19:32:07,845 - Train: 9.76% [397300/4070000] [97.6/1000.0] [batch_t 0.605 (1.560)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 19:33:41,871 - Train: 9.76% [397400/4070000] [97.6/1000.0] [batch_t 0.604 (0.940)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 19:36:03,184 - Train: 9.77% [397500/4070000] [97.7/1000.0] [batch_t 0.606 (1.413)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 19:38:12,439 - Train: 9.77% [397600/4070000] [97.7/1000.0] [batch_t 0.606 (1.292)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 19:39:33,957 - Train: 9.77% [397700/4070000] [97.7/1000.0] [batch_t 0.606 (0.815)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 19:41:06,507 - Train: 9.77% [397800/4070000] [97.7/1000.0] [batch_t 0.606 (0.925)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 19:42:06,993 - Train: 9.78% [397900/4070000] [97.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 19:43:07,507 - Train: 9.78% [398000/4070000] [97.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 19:44:08,009 - Train: 9.78% [398100/4070000] [97.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 19:45:08,498 - Train: 9.78% [398200/4070000] [97.8/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 19:46:09,185 - Train: 9.79% [398300/4070000] [97.9/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 19:47:09,694 - Train: 9.79% [398400/4070000] [97.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 19:48:10,211 - Train: 9.79% [398500/4070000] [97.9/1000.0] [batch_t 0.611 (0.605)] [data_t 0.002] [optim_t 0.609] [lr 0.005000] +2024-04-05 19:49:10,752 - Train: 9.79% [398600/4070000] [97.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 19:50:11,234 - Train: 9.80% [398700/4070000] [98.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 19:51:11,744 - Train: 9.80% [398800/4070000] [98.0/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 19:51:48,041 - ==> Total time: 3 days, 18:51:18 Eta: 34 days, 20:14:20 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-05 19:52:15,287 - Train: 9.80% [398900/4070000] [98.0/1000.0] [batch_t 0.603 (0.635)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 19:53:15,953 - Train: 9.80% [399000/4070000] [98.0/1000.0] [batch_t 0.608 (0.607)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 19:54:16,437 - Train: 9.81% [399100/4070000] [98.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 19:55:16,954 - Train: 9.81% [399200/4070000] [98.1/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-05 19:56:17,447 - Train: 9.81% [399300/4070000] [98.1/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 19:57:17,981 - Train: 9.81% [399400/4070000] [98.1/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 19:58:18,504 - Train: 9.82% [399500/4070000] [98.2/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 19:59:19,011 - Train: 9.82% [399600/4070000] [98.2/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 20:00:19,571 - Train: 9.82% [399700/4070000] [98.2/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 20:01:20,046 - Train: 9.82% [399800/4070000] [98.2/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 20:02:20,587 - Train: 9.83% [399900/4070000] [98.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 20:03:21,166 - Train: 9.83% [400000/4070000] [98.3/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 20:04:22,793 - Train: 9.83% [400100/4070000] [98.3/1000.0] [batch_t 0.609 (0.616)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-05 20:05:23,578 - Train: 9.83% [400200/4070000] [98.3/1000.0] [batch_t 0.609 (0.608)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-05 20:06:26,166 - Train: 9.84% [400300/4070000] [98.4/1000.0] [batch_t 0.603 (0.626)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 20:07:26,656 - Train: 9.84% [400400/4070000] [98.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 20:08:27,147 - Train: 9.84% [400500/4070000] [98.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 20:09:27,633 - Train: 9.84% [400600/4070000] [98.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 20:10:28,155 - Train: 9.85% [400700/4070000] [98.5/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 20:11:28,698 - Train: 9.85% [400800/4070000] [98.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 20:12:29,216 - Train: 9.85% [400900/4070000] [98.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 20:13:29,852 - Train: 9.85% [401000/4070000] [98.5/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 20:14:30,346 - Train: 9.86% [401100/4070000] [98.6/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-05 20:15:30,832 - Train: 9.86% [401200/4070000] [98.6/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 20:16:31,341 - Train: 9.86% [401300/4070000] [98.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 20:17:31,834 - Train: 9.86% [401400/4070000] [98.6/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 20:18:32,885 - Train: 9.86% [401500/4070000] [98.6/1000.0] [batch_t 0.601 (0.610)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-05 20:19:33,802 - Train: 9.87% [401600/4070000] [98.7/1000.0] [batch_t 0.606 (0.609)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 20:20:35,257 - Train: 9.87% [401700/4070000] [98.7/1000.0] [batch_t 0.607 (0.614)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 20:21:35,762 - Train: 9.87% [401800/4070000] [98.7/1000.0] [batch_t 0.610 (0.605)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-05 20:22:36,282 - Train: 9.87% [401900/4070000] [98.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 20:23:36,756 - Train: 9.88% [402000/4070000] [98.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 20:24:37,454 - Train: 9.88% [402100/4070000] [98.8/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 20:25:37,992 - Train: 9.88% [402200/4070000] [98.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 20:26:38,497 - Train: 9.88% [402300/4070000] [98.8/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 20:27:38,976 - Train: 9.89% [402400/4070000] [98.9/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 20:28:39,644 - Train: 9.89% [402500/4070000] [98.9/1000.0] [batch_t 0.603 (0.607)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 20:29:40,181 - Train: 9.89% [402600/4070000] [98.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 20:30:40,795 - Train: 9.89% [402700/4070000] [98.9/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 20:31:43,168 - Train: 9.90% [402800/4070000] [99.0/1000.0] [batch_t 0.603 (0.624)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 20:32:43,679 - Train: 9.90% [402900/4070000] [99.0/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 20:33:01,827 - ==> Total time: 3 days, 19:32:32 Eta: 34 days, 17:07:42 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-05 20:33:48,530 - Train: 9.90% [403000/4070000] [99.0/1000.0] [batch_t 0.607 (0.613)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 20:34:48,998 - Train: 9.90% [403100/4070000] [99.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 20:35:49,452 - Train: 9.91% [403200/4070000] [99.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 20:36:49,949 - Train: 9.91% [403300/4070000] [99.1/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 20:37:50,545 - Train: 9.91% [403400/4070000] [99.1/1000.0] [batch_t 0.602 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 20:38:51,034 - Train: 9.91% [403500/4070000] [99.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 20:39:52,548 - Train: 9.92% [403600/4070000] [99.2/1000.0] [batch_t 0.604 (0.615)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 20:40:53,040 - Train: 9.92% [403700/4070000] [99.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 20:41:53,558 - Train: 9.92% [403800/4070000] [99.2/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 20:42:54,199 - Train: 9.92% [403900/4070000] [99.2/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 20:43:54,708 - Train: 9.93% [404000/4070000] [99.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 20:44:55,270 - Train: 9.93% [404100/4070000] [99.3/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 20:45:55,794 - Train: 9.93% [404200/4070000] [99.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 20:46:56,310 - Train: 9.93% [404300/4070000] [99.3/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-05 20:47:56,800 - Train: 9.94% [404400/4070000] [99.4/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 20:48:57,273 - Train: 9.94% [404500/4070000] [99.4/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-05 20:49:57,775 - Train: 9.94% [404600/4070000] [99.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 20:50:58,459 - Train: 9.94% [404700/4070000] [99.4/1000.0] [batch_t 0.603 (0.607)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 20:51:59,006 - Train: 9.95% [404800/4070000] [99.5/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 20:52:59,549 - Train: 9.95% [404900/4070000] [99.5/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 20:54:00,047 - Train: 9.95% [405000/4070000] [99.5/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 20:55:00,579 - Train: 9.95% [405100/4070000] [99.5/1000.0] [batch_t 0.610 (0.605)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-05 20:56:01,252 - Train: 9.96% [405200/4070000] [99.6/1000.0] [batch_t 0.609 (0.607)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-05 20:57:01,775 - Train: 9.96% [405300/4070000] [99.6/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-05 20:58:02,278 - Train: 9.96% [405400/4070000] [99.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 20:59:02,767 - Train: 9.96% [405500/4070000] [99.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 21:00:07,544 - Train: 9.97% [405600/4070000] [99.7/1000.0] [batch_t 0.607 (0.648)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 21:01:08,333 - Train: 9.97% [405700/4070000] [99.7/1000.0] [batch_t 0.602 (0.608)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 21:02:08,840 - Train: 9.97% [405800/4070000] [99.7/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 21:03:09,531 - Train: 9.97% [405900/4070000] [99.7/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 21:04:10,360 - Train: 9.98% [406000/4070000] [99.8/1000.0] [batch_t 0.605 (0.608)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 21:05:10,961 - Train: 9.98% [406100/4070000] [99.8/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 21:06:11,492 - Train: 9.98% [406200/4070000] [99.8/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 21:07:12,407 - Train: 9.98% [406300/4070000] [99.8/1000.0] [batch_t 0.605 (0.609)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 21:08:12,924 - Train: 9.99% [406400/4070000] [99.9/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-05 21:09:13,873 - Train: 9.99% [406500/4070000] [99.9/1000.0] [batch_t 0.606 (0.609)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 21:10:14,524 - Train: 9.99% [406600/4070000] [99.9/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 21:11:16,414 - Train: 9.99% [406700/4070000] [99.9/1000.0] [batch_t 0.606 (0.619)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 21:12:16,891 - Train: 10.00% [406800/4070000] [100.0/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-05 21:13:17,384 - Train: 10.00% [406900/4070000] [100.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 21:14:17,884 - Train: 10.00% [407000/4070000] [100.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 21:14:17,888 - ==> Total time: 3 days, 20:13:48 Eta: 34 days, 14:04:18 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-05 21:15:20,267 - Train: 10.00% [407100/4070000] [100.0/1000.0] [batch_t 0.603 (0.608)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 21:16:20,795 - Train: 10.00% [407200/4070000] [100.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 21:17:21,320 - Train: 10.01% [407300/4070000] [100.1/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 21:18:21,928 - Train: 10.01% [407400/4070000] [100.1/1000.0] [batch_t 0.602 (0.606)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-05 21:19:22,443 - Train: 10.01% [407500/4070000] [100.1/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 21:20:22,974 - Train: 10.01% [407600/4070000] [100.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 21:21:23,491 - Train: 10.02% [407700/4070000] [100.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 21:22:24,167 - Train: 10.02% [407800/4070000] [100.2/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 21:23:24,679 - Train: 10.02% [407900/4070000] [100.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 21:24:25,186 - Train: 10.02% [408000/4070000] [100.2/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 21:25:25,999 - Train: 10.03% [408100/4070000] [100.3/1000.0] [batch_t 0.603 (0.608)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 21:26:26,640 - Train: 10.03% [408200/4070000] [100.3/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 21:27:27,130 - Train: 10.03% [408300/4070000] [100.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 21:28:27,611 - Train: 10.03% [408400/4070000] [100.3/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-05 21:29:28,322 - Train: 10.04% [408500/4070000] [100.4/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 21:30:28,791 - Train: 10.04% [408600/4070000] [100.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 21:31:29,330 - Train: 10.04% [408700/4070000] [100.4/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 21:32:30,758 - Train: 10.04% [408800/4070000] [100.4/1000.0] [batch_t 0.605 (0.614)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 21:33:31,283 - Train: 10.05% [408900/4070000] [100.5/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 21:34:31,798 - Train: 10.05% [409000/4070000] [100.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 21:35:32,458 - Train: 10.05% [409100/4070000] [100.5/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 21:36:32,950 - Train: 10.05% [409200/4070000] [100.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 21:37:33,412 - Train: 10.06% [409300/4070000] [100.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 21:38:34,604 - Train: 10.06% [409400/4070000] [100.6/1000.0] [batch_t 0.602 (0.612)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-05 21:39:35,117 - Train: 10.06% [409500/4070000] [100.6/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-05 21:40:35,753 - Train: 10.06% [409600/4070000] [100.6/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 21:41:36,292 - Train: 10.07% [409700/4070000] [100.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 21:42:36,764 - Train: 10.07% [409800/4070000] [100.7/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 21:43:38,454 - Train: 10.07% [409900/4070000] [100.7/1000.0] [batch_t 0.606 (0.617)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 21:44:39,012 - Train: 10.07% [410000/4070000] [100.7/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 21:45:39,480 - Train: 10.08% [410100/4070000] [100.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 21:46:40,008 - Train: 10.08% [410200/4070000] [100.8/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 21:47:40,497 - Train: 10.08% [410300/4070000] [100.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 21:48:41,116 - Train: 10.08% [410400/4070000] [100.8/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 21:49:41,581 - Train: 10.09% [410500/4070000] [100.9/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 21:50:42,079 - Train: 10.09% [410600/4070000] [100.9/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 21:51:42,556 - Train: 10.09% [410700/4070000] [100.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 21:52:43,092 - Train: 10.09% [410800/4070000] [100.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 21:53:43,675 - Train: 10.10% [410900/4070000] [101.0/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 21:54:44,196 - Train: 10.10% [411000/4070000] [101.0/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 21:55:26,560 - ==> Total time: 3 days, 20:54:57 Eta: 34 days, 11:02:38 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-05 21:55:46,450 - Train: 10.10% [411100/4070000] [101.0/1000.0] [batch_t 0.606 (0.613)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 21:56:46,930 - Train: 10.10% [411200/4070000] [101.0/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 21:57:47,426 - Train: 10.11% [411300/4070000] [101.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 21:58:47,938 - Train: 10.11% [411400/4070000] [101.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 21:59:48,397 - Train: 10.11% [411500/4070000] [101.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 22:00:49,873 - Train: 10.11% [411600/4070000] [101.1/1000.0] [batch_t 0.604 (0.615)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 22:01:50,390 - Train: 10.12% [411700/4070000] [101.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 22:02:50,899 - Train: 10.12% [411800/4070000] [101.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 22:03:51,798 - Train: 10.12% [411900/4070000] [101.2/1000.0] [batch_t 0.599 (0.609)] [data_t 0.002] [optim_t 0.597] [lr 0.005000] +2024-04-05 22:04:53,336 - Train: 10.12% [412000/4070000] [101.2/1000.0] [batch_t 0.602 (0.615)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 22:05:54,020 - Train: 10.13% [412100/4070000] [101.3/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 22:06:54,550 - Train: 10.13% [412200/4070000] [101.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 22:07:55,809 - Train: 10.13% [412300/4070000] [101.3/1000.0] [batch_t 0.607 (0.613)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 22:08:56,332 - Train: 10.13% [412400/4070000] [101.3/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-05 22:09:56,835 - Train: 10.14% [412500/4070000] [101.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 22:10:57,363 - Train: 10.14% [412600/4070000] [101.4/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 22:11:58,786 - Train: 10.14% [412700/4070000] [101.4/1000.0] [batch_t 0.605 (0.614)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 22:12:59,294 - Train: 10.14% [412800/4070000] [101.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 22:13:59,980 - Train: 10.14% [412900/4070000] [101.4/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 22:15:00,497 - Train: 10.15% [413000/4070000] [101.5/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 22:16:01,094 - Train: 10.15% [413100/4070000] [101.5/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 22:17:01,628 - Train: 10.15% [413200/4070000] [101.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 22:18:02,362 - Train: 10.15% [413300/4070000] [101.5/1000.0] [batch_t 0.609 (0.607)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-05 22:19:02,996 - Train: 10.16% [413400/4070000] [101.6/1000.0] [batch_t 0.599 (0.606)] [data_t 0.002] [optim_t 0.597] [lr 0.005000] +2024-04-05 22:20:03,509 - Train: 10.16% [413500/4070000] [101.6/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 22:21:04,018 - Train: 10.16% [413600/4070000] [101.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 22:22:04,523 - Train: 10.16% [413700/4070000] [101.6/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 22:23:05,006 - Train: 10.17% [413800/4070000] [101.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 22:24:05,509 - Train: 10.17% [413900/4070000] [101.7/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 22:25:05,996 - Train: 10.17% [414000/4070000] [101.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 22:26:06,482 - Train: 10.17% [414100/4070000] [101.7/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 22:27:07,009 - Train: 10.18% [414200/4070000] [101.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 22:28:07,509 - Train: 10.18% [414300/4070000] [101.8/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 22:29:08,010 - Train: 10.18% [414400/4070000] [101.8/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 22:30:08,484 - Train: 10.18% [414500/4070000] [101.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 22:31:08,961 - Train: 10.19% [414600/4070000] [101.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 22:32:09,425 - Train: 10.19% [414700/4070000] [101.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 22:33:09,955 - Train: 10.19% [414800/4070000] [101.9/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-05 22:34:10,426 - Train: 10.19% [414900/4070000] [101.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 22:35:10,929 - Train: 10.20% [415000/4070000] [102.0/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 22:36:11,420 - Train: 10.20% [415100/4070000] [102.0/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-05 22:36:35,617 - ==> Total time: 3 days, 21:36:06 Eta: 34 days, 8:03:46 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-05 22:37:13,718 - Train: 10.20% [415200/4070000] [102.0/1000.0] [batch_t 0.602 (0.609)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-05 22:38:14,367 - Train: 10.20% [415300/4070000] [102.0/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 22:39:15,225 - Train: 10.21% [415400/4070000] [102.1/1000.0] [batch_t 0.606 (0.608)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 22:40:15,939 - Train: 10.21% [415500/4070000] [102.1/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 22:41:16,418 - Train: 10.21% [415600/4070000] [102.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 22:42:17,511 - Train: 10.21% [415700/4070000] [102.1/1000.0] [batch_t 0.607 (0.611)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 22:43:17,987 - Train: 10.22% [415800/4070000] [102.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 22:44:18,624 - Train: 10.22% [415900/4070000] [102.2/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 22:45:19,141 - Train: 10.22% [416000/4070000] [102.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 22:46:19,632 - Train: 10.22% [416100/4070000] [102.2/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 22:47:20,161 - Train: 10.23% [416200/4070000] [102.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 22:48:20,621 - Train: 10.23% [416300/4070000] [102.3/1000.0] [batch_t 0.594 (0.605)] [data_t 0.002] [optim_t 0.592] [lr 0.005000] +2024-04-05 22:49:21,092 - Train: 10.23% [416400/4070000] [102.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 22:50:21,598 - Train: 10.23% [416500/4070000] [102.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 22:51:22,078 - Train: 10.24% [416600/4070000] [102.4/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 22:52:22,639 - Train: 10.24% [416700/4070000] [102.4/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 22:53:23,158 - Train: 10.24% [416800/4070000] [102.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 22:54:24,602 - Train: 10.24% [416900/4070000] [102.4/1000.0] [batch_t 0.605 (0.614)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 22:55:25,126 - Train: 10.25% [417000/4070000] [102.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 22:56:25,615 - Train: 10.25% [417100/4070000] [102.5/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-05 22:57:26,339 - Train: 10.25% [417200/4070000] [102.5/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 22:58:26,850 - Train: 10.25% [417300/4070000] [102.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 22:59:27,356 - Train: 10.26% [417400/4070000] [102.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 23:00:27,874 - Train: 10.26% [417500/4070000] [102.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 23:01:28,383 - Train: 10.26% [417600/4070000] [102.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 23:02:29,001 - Train: 10.26% [417700/4070000] [102.6/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 23:03:29,501 - Train: 10.27% [417800/4070000] [102.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 23:04:30,253 - Train: 10.27% [417900/4070000] [102.7/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 23:05:30,832 - Train: 10.27% [418000/4070000] [102.7/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 23:06:31,349 - Train: 10.27% [418100/4070000] [102.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 23:07:31,938 - Train: 10.28% [418200/4070000] [102.8/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 23:08:32,470 - Train: 10.28% [418300/4070000] [102.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 23:09:32,974 - Train: 10.28% [418400/4070000] [102.8/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 23:10:33,492 - Train: 10.28% [418500/4070000] [102.8/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 23:11:34,465 - Train: 10.29% [418600/4070000] [102.9/1000.0] [batch_t 0.606 (0.610)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 23:12:35,558 - Train: 10.29% [418700/4070000] [102.9/1000.0] [batch_t 0.606 (0.611)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 23:13:36,090 - Train: 10.29% [418800/4070000] [102.9/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-05 23:14:36,627 - Train: 10.29% [418900/4070000] [102.9/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 23:15:37,141 - Train: 10.29% [419000/4070000] [102.9/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 23:16:37,693 - Train: 10.30% [419100/4070000] [103.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 23:17:38,369 - Train: 10.30% [419200/4070000] [103.0/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 23:17:44,439 - ==> Total time: 3 days, 22:17:15 Eta: 34 days, 5:07:33 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-05 23:18:40,577 - Train: 10.30% [419300/4070000] [103.0/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 23:19:41,138 - Train: 10.30% [419400/4070000] [103.0/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 23:20:42,666 - Train: 10.31% [419500/4070000] [103.1/1000.0] [batch_t 0.607 (0.615)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-05 23:21:43,211 - Train: 10.31% [419600/4070000] [103.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 23:22:43,748 - Train: 10.31% [419700/4070000] [103.1/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-05 23:23:44,347 - Train: 10.31% [419800/4070000] [103.1/1000.0] [batch_t 0.611 (0.606)] [data_t 0.002] [optim_t 0.609] [lr 0.005000] +2024-04-05 23:24:44,874 - Train: 10.32% [419900/4070000] [103.2/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 23:25:45,405 - Train: 10.32% [420000/4070000] [103.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 23:26:45,971 - Train: 10.32% [420100/4070000] [103.2/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 23:27:46,524 - Train: 10.32% [420200/4070000] [103.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 23:28:47,027 - Train: 10.33% [420300/4070000] [103.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 23:29:47,719 - Train: 10.33% [420400/4070000] [103.3/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 23:30:48,258 - Train: 10.33% [420500/4070000] [103.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 23:31:48,839 - Train: 10.33% [420600/4070000] [103.3/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 23:32:49,371 - Train: 10.34% [420700/4070000] [103.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 23:33:49,913 - Train: 10.34% [420800/4070000] [103.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 23:34:50,721 - Train: 10.34% [420900/4070000] [103.4/1000.0] [batch_t 0.606 (0.608)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 23:35:51,251 - Train: 10.34% [421000/4070000] [103.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 23:36:52,223 - Train: 10.35% [421100/4070000] [103.5/1000.0] [batch_t 0.601 (0.610)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-05 23:37:52,725 - Train: 10.35% [421200/4070000] [103.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 23:38:53,244 - Train: 10.35% [421300/4070000] [103.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 23:39:53,836 - Train: 10.35% [421400/4070000] [103.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 23:40:54,842 - Train: 10.36% [421500/4070000] [103.6/1000.0] [batch_t 0.602 (0.610)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-05 23:41:56,304 - Train: 10.36% [421600/4070000] [103.6/1000.0] [batch_t 0.609 (0.615)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-05 23:42:56,803 - Train: 10.36% [421700/4070000] [103.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 23:43:57,434 - Train: 10.36% [421800/4070000] [103.6/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 23:44:57,931 - Train: 10.37% [421900/4070000] [103.7/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 23:45:58,438 - Train: 10.37% [422000/4070000] [103.7/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 23:46:59,222 - Train: 10.37% [422100/4070000] [103.7/1000.0] [batch_t 0.608 (0.608)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-05 23:48:00,132 - Train: 10.37% [422200/4070000] [103.7/1000.0] [batch_t 0.607 (0.609)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 23:49:00,649 - Train: 10.38% [422300/4070000] [103.8/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 23:50:01,252 - Train: 10.38% [422400/4070000] [103.8/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 23:51:01,767 - Train: 10.38% [422500/4070000] [103.8/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 23:52:02,381 - Train: 10.38% [422600/4070000] [103.8/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 23:53:03,014 - Train: 10.39% [422700/4070000] [103.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 23:54:03,511 - Train: 10.39% [422800/4070000] [103.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 23:55:04,085 - Train: 10.39% [422900/4070000] [103.9/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 23:56:04,576 - Train: 10.39% [423000/4070000] [103.9/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 23:57:05,570 - Train: 10.40% [423100/4070000] [104.0/1000.0] [batch_t 0.606 (0.610)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 23:58:06,048 - Train: 10.40% [423200/4070000] [104.0/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 23:58:55,018 - ==> Total time: 3 days, 22:58:25 Eta: 34 days, 2:14:10 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-05 23:59:09,962 - Train: 10.40% [423300/4070000] [104.0/1000.0] [batch_t 0.604 (0.669)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 00:00:10,446 - Train: 10.40% [423400/4070000] [104.0/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-06 00:01:10,956 - Train: 10.41% [423500/4070000] [104.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 00:02:11,601 - Train: 10.41% [423600/4070000] [104.1/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 00:03:12,103 - Train: 10.41% [423700/4070000] [104.1/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 00:04:12,587 - Train: 10.41% [423800/4070000] [104.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 00:05:13,087 - Train: 10.42% [423900/4070000] [104.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 00:06:13,591 - Train: 10.42% [424000/4070000] [104.2/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 00:07:14,417 - Train: 10.42% [424100/4070000] [104.2/1000.0] [batch_t 0.608 (0.608)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 00:08:14,912 - Train: 10.42% [424200/4070000] [104.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 00:09:15,519 - Train: 10.43% [424300/4070000] [104.3/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 00:10:16,101 - Train: 10.43% [424400/4070000] [104.3/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 00:11:18,081 - Train: 10.43% [424500/4070000] [104.3/1000.0] [batch_t 0.606 (0.620)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 00:12:18,620 - Train: 10.43% [424600/4070000] [104.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 00:13:19,114 - Train: 10.43% [424700/4070000] [104.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 00:14:19,650 - Train: 10.44% [424800/4070000] [104.4/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-06 00:15:20,258 - Train: 10.44% [424900/4070000] [104.4/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 00:16:20,803 - Train: 10.44% [425000/4070000] [104.4/1000.0] [batch_t 0.612 (0.605)] [data_t 0.002] [optim_t 0.610] [lr 0.005000] +2024-04-06 00:17:21,305 - Train: 10.44% [425100/4070000] [104.4/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 00:18:21,815 - Train: 10.45% [425200/4070000] [104.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 00:19:23,116 - Train: 10.45% [425300/4070000] [104.5/1000.0] [batch_t 0.603 (0.613)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 00:20:23,611 - Train: 10.45% [425400/4070000] [104.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 00:21:24,336 - Train: 10.45% [425500/4070000] [104.5/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 00:22:24,800 - Train: 10.46% [425600/4070000] [104.6/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 00:23:25,372 - Train: 10.46% [425700/4070000] [104.6/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 00:24:25,869 - Train: 10.46% [425800/4070000] [104.6/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 00:25:26,372 - Train: 10.46% [425900/4070000] [104.6/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 00:26:26,868 - Train: 10.47% [426000/4070000] [104.7/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 00:27:27,408 - Train: 10.47% [426100/4070000] [104.7/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-06 00:28:28,004 - Train: 10.47% [426200/4070000] [104.7/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 00:29:28,509 - Train: 10.47% [426300/4070000] [104.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 00:30:28,988 - Train: 10.48% [426400/4070000] [104.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 00:31:29,498 - Train: 10.48% [426500/4070000] [104.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 00:32:29,991 - Train: 10.48% [426600/4070000] [104.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 00:33:30,584 - Train: 10.48% [426700/4070000] [104.8/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 00:34:31,089 - Train: 10.49% [426800/4070000] [104.9/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 00:35:31,529 - Train: 10.49% [426900/4070000] [104.9/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 00:36:32,079 - Train: 10.49% [427000/4070000] [104.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 00:37:32,895 - Train: 10.49% [427100/4070000] [104.9/1000.0] [batch_t 0.606 (0.608)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 00:38:33,340 - Train: 10.50% [427200/4070000] [105.0/1000.0] [batch_t 0.600 (0.604)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-06 00:39:33,823 - Train: 10.50% [427300/4070000] [105.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 00:40:04,068 - ==> Total time: 3 days, 23:39:34 Eta: 33 days, 23:23:06 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-06 00:40:37,579 - Train: 10.50% [427400/4070000] [105.0/1000.0] [batch_t 0.607 (0.621)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 00:41:38,199 - Train: 10.50% [427500/4070000] [105.0/1000.0] [batch_t 0.586 (0.606)] [data_t 0.002] [optim_t 0.585] [lr 0.005000] +2024-04-06 00:42:38,733 - Train: 10.51% [427600/4070000] [105.1/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 00:43:39,224 - Train: 10.51% [427700/4070000] [105.1/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 00:44:39,717 - Train: 10.51% [427800/4070000] [105.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 00:45:40,195 - Train: 10.51% [427900/4070000] [105.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 00:46:40,800 - Train: 10.52% [428000/4070000] [105.2/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 00:47:41,443 - Train: 10.52% [428100/4070000] [105.2/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 00:48:42,388 - Train: 10.52% [428200/4070000] [105.2/1000.0] [batch_t 0.603 (0.609)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 00:49:42,882 - Train: 10.52% [428300/4070000] [105.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 00:50:43,357 - Train: 10.53% [428400/4070000] [105.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 00:51:43,840 - Train: 10.53% [428500/4070000] [105.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 00:52:44,624 - Train: 10.53% [428600/4070000] [105.3/1000.0] [batch_t 0.606 (0.608)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 00:53:45,208 - Train: 10.53% [428700/4070000] [105.3/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 00:54:45,682 - Train: 10.54% [428800/4070000] [105.4/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 00:55:46,301 - Train: 10.54% [428900/4070000] [105.4/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 00:56:46,772 - Train: 10.54% [429000/4070000] [105.4/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 00:57:47,264 - Train: 10.54% [429100/4070000] [105.4/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 00:58:47,781 - Train: 10.55% [429200/4070000] [105.5/1000.0] [batch_t 0.615 (0.605)] [data_t 0.002] [optim_t 0.613] [lr 0.005000] +2024-04-06 00:59:48,355 - Train: 10.55% [429300/4070000] [105.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 01:00:48,989 - Train: 10.55% [429400/4070000] [105.5/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 01:01:49,444 - Train: 10.55% [429500/4070000] [105.5/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 01:02:49,930 - Train: 10.56% [429600/4070000] [105.6/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-06 01:03:50,479 - Train: 10.56% [429700/4070000] [105.6/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-06 01:04:52,097 - Train: 10.56% [429800/4070000] [105.6/1000.0] [batch_t 0.606 (0.616)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 01:05:52,620 - Train: 10.56% [429900/4070000] [105.6/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 01:06:53,227 - Train: 10.57% [430000/4070000] [105.7/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 01:07:54,085 - Train: 10.57% [430100/4070000] [105.7/1000.0] [batch_t 0.605 (0.608)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 01:08:54,628 - Train: 10.57% [430200/4070000] [105.7/1000.0] [batch_t 0.610 (0.605)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-06 01:09:55,140 - Train: 10.57% [430300/4070000] [105.7/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 01:10:55,733 - Train: 10.57% [430400/4070000] [105.7/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 01:11:56,501 - Train: 10.58% [430500/4070000] [105.8/1000.0] [batch_t 0.605 (0.608)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 01:12:57,341 - Train: 10.58% [430600/4070000] [105.8/1000.0] [batch_t 0.603 (0.608)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 01:13:58,070 - Train: 10.58% [430700/4070000] [105.8/1000.0] [batch_t 0.603 (0.607)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 01:14:58,880 - Train: 10.58% [430800/4070000] [105.8/1000.0] [batch_t 0.605 (0.608)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 01:15:59,373 - Train: 10.59% [430900/4070000] [105.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 01:17:00,122 - Train: 10.59% [431000/4070000] [105.9/1000.0] [batch_t 0.598 (0.607)] [data_t 0.002] [optim_t 0.597] [lr 0.005000] +2024-04-06 01:18:00,615 - Train: 10.59% [431100/4070000] [105.9/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 01:19:01,122 - Train: 10.59% [431200/4070000] [105.9/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 01:20:01,733 - Train: 10.60% [431300/4070000] [106.0/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 01:21:02,239 - Train: 10.60% [431400/4070000] [106.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 01:21:14,342 - ==> Total time: 4 days, 0:20:45 Eta: 33 days, 20:34:39 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-06 01:22:04,396 - Train: 10.60% [431500/4070000] [106.0/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 01:23:04,854 - Train: 10.60% [431600/4070000] [106.0/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-06 01:24:05,993 - Train: 10.61% [431700/4070000] [106.1/1000.0] [batch_t 0.603 (0.611)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 01:25:06,602 - Train: 10.61% [431800/4070000] [106.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 01:26:07,101 - Train: 10.61% [431900/4070000] [106.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 01:27:07,631 - Train: 10.61% [432000/4070000] [106.1/1000.0] [batch_t 0.599 (0.605)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-06 01:28:08,915 - Train: 10.62% [432100/4070000] [106.2/1000.0] [batch_t 0.605 (0.613)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 01:29:10,170 - Train: 10.62% [432200/4070000] [106.2/1000.0] [batch_t 0.604 (0.612)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 01:30:11,487 - Train: 10.62% [432300/4070000] [106.2/1000.0] [batch_t 0.606 (0.613)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 01:31:12,169 - Train: 10.62% [432400/4070000] [106.2/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 01:32:12,785 - Train: 10.63% [432500/4070000] [106.3/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 01:33:13,309 - Train: 10.63% [432600/4070000] [106.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 01:34:13,824 - Train: 10.63% [432700/4070000] [106.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 01:35:14,304 - Train: 10.63% [432800/4070000] [106.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 01:36:14,798 - Train: 10.64% [432900/4070000] [106.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 01:37:15,374 - Train: 10.64% [433000/4070000] [106.4/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 01:38:16,066 - Train: 10.64% [433100/4070000] [106.4/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 01:39:16,537 - Train: 10.64% [433200/4070000] [106.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 01:40:17,074 - Train: 10.65% [433300/4070000] [106.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 01:41:17,886 - Train: 10.65% [433400/4070000] [106.5/1000.0] [batch_t 0.607 (0.608)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 01:42:18,408 - Train: 10.65% [433500/4070000] [106.5/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 01:43:19,077 - Train: 10.65% [433600/4070000] [106.5/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 01:44:19,559 - Train: 10.66% [433700/4070000] [106.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 01:45:20,095 - Train: 10.66% [433800/4070000] [106.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 01:46:20,578 - Train: 10.66% [433900/4070000] [106.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 01:47:21,075 - Train: 10.66% [434000/4070000] [106.6/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 01:48:21,624 - Train: 10.67% [434100/4070000] [106.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 01:49:22,178 - Train: 10.67% [434200/4070000] [106.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 01:50:22,725 - Train: 10.67% [434300/4070000] [106.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 01:51:23,418 - Train: 10.67% [434400/4070000] [106.7/1000.0] [batch_t 0.603 (0.607)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 01:52:23,897 - Train: 10.68% [434500/4070000] [106.8/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 01:53:24,419 - Train: 10.68% [434600/4070000] [106.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 01:54:24,926 - Train: 10.68% [434700/4070000] [106.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 01:55:25,419 - Train: 10.68% [434800/4070000] [106.8/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 01:56:26,131 - Train: 10.69% [434900/4070000] [106.9/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 01:57:26,608 - Train: 10.69% [435000/4070000] [106.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 01:58:27,120 - Train: 10.69% [435100/4070000] [106.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 01:59:27,662 - Train: 10.69% [435200/4070000] [106.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 02:00:28,174 - Train: 10.70% [435300/4070000] [107.0/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 02:01:28,675 - Train: 10.70% [435400/4070000] [107.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 02:02:23,110 - ==> Total time: 4 days, 1:01:53 Eta: 33 days, 17:48:22 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-06 02:02:30,810 - Train: 10.70% [435500/4070000] [107.0/1000.0] [batch_t 0.605 (0.623)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 02:03:31,472 - Train: 10.70% [435600/4070000] [107.0/1000.0] [batch_t 0.603 (0.607)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 02:04:32,010 - Train: 10.71% [435700/4070000] [107.1/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 02:05:32,490 - Train: 10.71% [435800/4070000] [107.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 02:06:33,469 - Train: 10.71% [435900/4070000] [107.1/1000.0] [batch_t 0.606 (0.610)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 02:07:33,953 - Train: 10.71% [436000/4070000] [107.1/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 02:08:34,431 - Train: 10.71% [436100/4070000] [107.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 02:09:35,116 - Train: 10.72% [436200/4070000] [107.2/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 02:10:35,648 - Train: 10.72% [436300/4070000] [107.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 02:11:36,128 - Train: 10.72% [436400/4070000] [107.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 02:12:36,611 - Train: 10.72% [436500/4070000] [107.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 02:13:37,798 - Train: 10.73% [436600/4070000] [107.3/1000.0] [batch_t 0.605 (0.612)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 02:14:38,300 - Train: 10.73% [436700/4070000] [107.3/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 02:15:38,806 - Train: 10.73% [436800/4070000] [107.3/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 02:16:39,398 - Train: 10.73% [436900/4070000] [107.3/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 02:17:39,872 - Train: 10.74% [437000/4070000] [107.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 02:18:40,380 - Train: 10.74% [437100/4070000] [107.4/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 02:19:40,872 - Train: 10.74% [437200/4070000] [107.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 02:20:41,399 - Train: 10.74% [437300/4070000] [107.4/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 02:21:41,908 - Train: 10.75% [437400/4070000] [107.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 02:22:42,519 - Train: 10.75% [437500/4070000] [107.5/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 02:23:43,028 - Train: 10.75% [437600/4070000] [107.5/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 02:24:43,536 - Train: 10.75% [437700/4070000] [107.5/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 02:25:44,067 - Train: 10.76% [437800/4070000] [107.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 02:26:44,543 - Train: 10.76% [437900/4070000] [107.6/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 02:27:45,052 - Train: 10.76% [438000/4070000] [107.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 02:28:45,549 - Train: 10.76% [438100/4070000] [107.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 02:29:46,124 - Train: 10.77% [438200/4070000] [107.7/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 02:30:46,611 - Train: 10.77% [438300/4070000] [107.7/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 02:31:47,100 - Train: 10.77% [438400/4070000] [107.7/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 02:32:47,653 - Train: 10.77% [438500/4070000] [107.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 02:33:48,118 - Train: 10.78% [438600/4070000] [107.8/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-06 02:34:48,597 - Train: 10.78% [438700/4070000] [107.8/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 02:35:49,293 - Train: 10.78% [438800/4070000] [107.8/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 02:36:49,870 - Train: 10.78% [438900/4070000] [107.8/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 02:37:50,674 - Train: 10.79% [439000/4070000] [107.9/1000.0] [batch_t 0.605 (0.608)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 02:38:51,164 - Train: 10.79% [439100/4070000] [107.9/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 02:39:51,672 - Train: 10.79% [439200/4070000] [107.9/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 02:40:52,216 - Train: 10.79% [439300/4070000] [107.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 02:41:53,227 - Train: 10.80% [439400/4070000] [108.0/1000.0] [batch_t 0.609 (0.610)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-06 02:42:53,691 - Train: 10.80% [439500/4070000] [108.0/1000.0] [batch_t 0.592 (0.605)] [data_t 0.002] [optim_t 0.591] [lr 0.005000] +2024-04-06 02:43:29,965 - ==> Total time: 4 days, 1:43:00 Eta: 33 days, 15:04:09 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-06 02:43:55,833 - Train: 10.80% [439600/4070000] [108.0/1000.0] [batch_t 0.606 (0.609)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 02:44:56,324 - Train: 10.80% [439700/4070000] [108.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 02:45:56,819 - Train: 10.81% [439800/4070000] [108.1/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 02:46:57,365 - Train: 10.81% [439900/4070000] [108.1/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 02:47:57,880 - Train: 10.81% [440000/4070000] [108.1/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-06 02:48:58,562 - Train: 10.81% [440100/4070000] [108.1/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 02:49:59,093 - Train: 10.82% [440200/4070000] [108.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 02:50:59,617 - Train: 10.82% [440300/4070000] [108.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 02:52:00,095 - Train: 10.82% [440400/4070000] [108.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 02:53:00,551 - Train: 10.82% [440500/4070000] [108.2/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-06 02:54:01,019 - Train: 10.83% [440600/4070000] [108.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 02:55:01,558 - Train: 10.83% [440700/4070000] [108.3/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 02:56:02,042 - Train: 10.83% [440800/4070000] [108.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 02:57:02,555 - Train: 10.83% [440900/4070000] [108.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 02:58:03,070 - Train: 10.84% [441000/4070000] [108.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 02:59:03,532 - Train: 10.84% [441100/4070000] [108.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 03:00:04,113 - Train: 10.84% [441200/4070000] [108.4/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 03:01:04,631 - Train: 10.84% [441300/4070000] [108.4/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-06 03:02:05,145 - Train: 10.85% [441400/4070000] [108.5/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 03:03:05,623 - Train: 10.85% [441500/4070000] [108.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 03:04:06,130 - Train: 10.85% [441600/4070000] [108.5/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-06 03:05:06,647 - Train: 10.85% [441700/4070000] [108.5/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 03:06:07,106 - Train: 10.86% [441800/4070000] [108.6/1000.0] [batch_t 0.599 (0.604)] [data_t 0.002] [optim_t 0.597] [lr 0.005000] +2024-04-06 03:07:07,616 - Train: 10.86% [441900/4070000] [108.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 03:08:08,167 - Train: 10.86% [442000/4070000] [108.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 03:09:08,679 - Train: 10.86% [442100/4070000] [108.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 03:10:09,179 - Train: 10.86% [442200/4070000] [108.6/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 03:11:09,706 - Train: 10.87% [442300/4070000] [108.7/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 03:12:10,194 - Train: 10.87% [442400/4070000] [108.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 03:13:10,802 - Train: 10.87% [442500/4070000] [108.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 03:14:11,296 - Train: 10.87% [442600/4070000] [108.7/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-06 03:15:11,758 - Train: 10.88% [442700/4070000] [108.8/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 03:16:12,256 - Train: 10.88% [442800/4070000] [108.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 03:17:12,742 - Train: 10.88% [442900/4070000] [108.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 03:18:13,221 - Train: 10.88% [443000/4070000] [108.8/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 03:19:13,684 - Train: 10.89% [443100/4070000] [108.9/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-06 03:20:14,176 - Train: 10.89% [443200/4070000] [108.9/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-06 03:21:15,843 - Train: 10.89% [443300/4070000] [108.9/1000.0] [batch_t 0.608 (0.617)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 03:22:16,586 - Train: 10.89% [443400/4070000] [108.9/1000.0] [batch_t 0.609 (0.607)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-06 03:23:17,104 - Train: 10.90% [443500/4070000] [109.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 03:24:17,629 - Train: 10.90% [443600/4070000] [109.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 03:24:35,764 - ==> Total time: 4 days, 2:24:06 Eta: 33 days, 12:22:02 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-06 03:25:19,739 - Train: 10.90% [443700/4070000] [109.0/1000.0] [batch_t 0.605 (0.608)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 03:26:20,356 - Train: 10.90% [443800/4070000] [109.0/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 03:27:20,852 - Train: 10.91% [443900/4070000] [109.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 03:28:21,316 - Train: 10.91% [444000/4070000] [109.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 03:29:21,993 - Train: 10.91% [444100/4070000] [109.1/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 03:30:22,488 - Train: 10.91% [444200/4070000] [109.1/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 03:31:22,977 - Train: 10.92% [444300/4070000] [109.2/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 03:32:23,544 - Train: 10.92% [444400/4070000] [109.2/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 03:33:24,020 - Train: 10.92% [444500/4070000] [109.2/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 03:34:24,611 - Train: 10.92% [444600/4070000] [109.2/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 03:35:25,133 - Train: 10.93% [444700/4070000] [109.3/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 03:36:25,626 - Train: 10.93% [444800/4070000] [109.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 03:37:26,116 - Train: 10.93% [444900/4070000] [109.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 03:38:26,612 - Train: 10.93% [445000/4070000] [109.3/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 03:39:27,191 - Train: 10.94% [445100/4070000] [109.4/1000.0] [batch_t 0.611 (0.606)] [data_t 0.002] [optim_t 0.609] [lr 0.005000] +2024-04-06 03:40:27,696 - Train: 10.94% [445200/4070000] [109.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 03:41:28,190 - Train: 10.94% [445300/4070000] [109.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 03:42:29,458 - Train: 10.94% [445400/4070000] [109.4/1000.0] [batch_t 0.603 (0.613)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 03:43:30,591 - Train: 10.95% [445500/4070000] [109.5/1000.0] [batch_t 0.606 (0.611)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 03:44:31,082 - Train: 10.95% [445600/4070000] [109.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 03:45:31,996 - Train: 10.95% [445700/4070000] [109.5/1000.0] [batch_t 0.603 (0.609)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 03:46:34,003 - Train: 10.95% [445800/4070000] [109.5/1000.0] [batch_t 0.605 (0.620)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 03:47:34,475 - Train: 10.96% [445900/4070000] [109.6/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 03:48:35,050 - Train: 10.96% [446000/4070000] [109.6/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 03:49:35,512 - Train: 10.96% [446100/4070000] [109.6/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 03:50:36,417 - Train: 10.96% [446200/4070000] [109.6/1000.0] [batch_t 0.603 (0.609)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 03:51:36,890 - Train: 10.97% [446300/4070000] [109.7/1000.0] [batch_t 0.600 (0.605)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-06 03:52:38,165 - Train: 10.97% [446400/4070000] [109.7/1000.0] [batch_t 0.606 (0.613)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 03:53:38,689 - Train: 10.97% [446500/4070000] [109.7/1000.0] [batch_t 0.596 (0.605)] [data_t 0.002] [optim_t 0.594] [lr 0.005000] +2024-04-06 03:54:39,780 - Train: 10.97% [446600/4070000] [109.7/1000.0] [batch_t 0.592 (0.611)] [data_t 0.002] [optim_t 0.590] [lr 0.005000] +2024-04-06 03:55:40,270 - Train: 10.98% [446700/4070000] [109.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 03:56:40,761 - Train: 10.98% [446800/4070000] [109.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 03:57:41,197 - Train: 10.98% [446900/4070000] [109.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 03:58:41,762 - Train: 10.98% [447000/4070000] [109.8/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 03:59:42,555 - Train: 10.99% [447100/4070000] [109.9/1000.0] [batch_t 0.606 (0.608)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 04:00:43,096 - Train: 10.99% [447200/4070000] [109.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 04:01:43,572 - Train: 10.99% [447300/4070000] [109.9/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-06 04:02:44,085 - Train: 10.99% [447400/4070000] [109.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 04:03:45,048 - Train: 11.00% [447500/4070000] [110.0/1000.0] [batch_t 0.605 (0.610)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 04:04:45,504 - Train: 11.00% [447600/4070000] [110.0/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 04:05:46,078 - Train: 11.00% [447700/4070000] [110.0/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 04:05:46,082 - ==> Total time: 4 days, 3:05:16 Eta: 33 days, 9:42:44 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-06 04:06:48,159 - Train: 11.00% [447800/4070000] [110.0/1000.0] [batch_t 0.599 (0.607)] [data_t 0.002] [optim_t 0.597] [lr 0.005000] +2024-04-06 04:07:48,651 - Train: 11.00% [447900/4070000] [110.0/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 04:08:49,149 - Train: 11.01% [448000/4070000] [110.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 04:09:49,667 - Train: 11.01% [448100/4070000] [110.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 04:10:50,168 - Train: 11.01% [448200/4070000] [110.1/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 04:11:50,684 - Train: 11.01% [448300/4070000] [110.1/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 04:12:51,959 - Train: 11.02% [448400/4070000] [110.2/1000.0] [batch_t 0.612 (0.613)] [data_t 0.002] [optim_t 0.610] [lr 0.005000] +2024-04-06 04:13:52,503 - Train: 11.02% [448500/4070000] [110.2/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 04:14:53,445 - Train: 11.02% [448600/4070000] [110.2/1000.0] [batch_t 0.603 (0.609)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 04:15:53,958 - Train: 11.02% [448700/4070000] [110.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 04:16:54,479 - Train: 11.03% [448800/4070000] [110.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 04:17:55,143 - Train: 11.03% [448900/4070000] [110.3/1000.0] [batch_t 0.608 (0.607)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 04:18:55,687 - Train: 11.03% [449000/4070000] [110.3/1000.0] [batch_t 0.610 (0.605)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-06 04:19:56,243 - Train: 11.03% [449100/4070000] [110.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 04:20:57,313 - Train: 11.04% [449200/4070000] [110.4/1000.0] [batch_t 0.609 (0.611)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-06 04:21:57,860 - Train: 11.04% [449300/4070000] [110.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 04:22:59,283 - Train: 11.04% [449400/4070000] [110.4/1000.0] [batch_t 0.606 (0.614)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 04:23:59,833 - Train: 11.04% [449500/4070000] [110.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 04:25:00,355 - Train: 11.05% [449600/4070000] [110.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 04:26:00,919 - Train: 11.05% [449700/4070000] [110.5/1000.0] [batch_t 0.602 (0.606)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-06 04:27:01,438 - Train: 11.05% [449800/4070000] [110.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 04:28:01,974 - Train: 11.05% [449900/4070000] [110.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 04:29:02,474 - Train: 11.06% [450000/4070000] [110.6/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 04:30:02,973 - Train: 11.06% [450100/4070000] [110.6/1000.0] [batch_t 0.600 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-06 04:31:03,644 - Train: 11.06% [450200/4070000] [110.6/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 04:32:04,211 - Train: 11.06% [450300/4070000] [110.6/1000.0] [batch_t 0.617 (0.606)] [data_t 0.002] [optim_t 0.615] [lr 0.005000] +2024-04-06 04:33:04,798 - Train: 11.07% [450400/4070000] [110.7/1000.0] [batch_t 0.601 (0.606)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-06 04:34:05,346 - Train: 11.07% [450500/4070000] [110.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 04:35:05,865 - Train: 11.07% [450600/4070000] [110.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 04:36:06,350 - Train: 11.07% [450700/4070000] [110.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 04:37:06,834 - Train: 11.08% [450800/4070000] [110.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 04:38:07,418 - Train: 11.08% [450900/4070000] [110.8/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 04:39:07,956 - Train: 11.08% [451000/4070000] [110.8/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 04:40:08,443 - Train: 11.08% [451100/4070000] [110.8/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 04:41:08,938 - Train: 11.09% [451200/4070000] [110.9/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 04:42:09,440 - Train: 11.09% [451300/4070000] [110.9/1000.0] [batch_t 0.611 (0.605)] [data_t 0.002] [optim_t 0.609] [lr 0.005000] +2024-04-06 04:43:09,954 - Train: 11.09% [451400/4070000] [110.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 04:44:10,418 - Train: 11.09% [451500/4070000] [110.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 04:45:10,928 - Train: 11.10% [451600/4070000] [111.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 04:46:11,525 - Train: 11.10% [451700/4070000] [111.0/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 04:46:53,926 - ==> Total time: 4 days, 3:46:24 Eta: 33 days, 7:05:14 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-06 04:47:13,937 - Train: 11.10% [451800/4070000] [111.0/1000.0] [batch_t 0.604 (0.611)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 04:48:14,453 - Train: 11.10% [451900/4070000] [111.0/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 04:49:14,961 - Train: 11.11% [452000/4070000] [111.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 04:50:15,450 - Train: 11.11% [452100/4070000] [111.1/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-06 04:51:16,099 - Train: 11.11% [452200/4070000] [111.1/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 04:52:16,519 - Train: 11.11% [452300/4070000] [111.1/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 04:53:17,018 - Train: 11.12% [452400/4070000] [111.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 04:54:17,845 - Train: 11.12% [452500/4070000] [111.2/1000.0] [batch_t 0.607 (0.608)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 04:55:18,340 - Train: 11.12% [452600/4070000] [111.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 04:56:18,803 - Train: 11.12% [452700/4070000] [111.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 04:57:19,294 - Train: 11.13% [452800/4070000] [111.3/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 04:58:20,104 - Train: 11.13% [452900/4070000] [111.3/1000.0] [batch_t 0.605 (0.608)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 04:59:20,584 - Train: 11.13% [453000/4070000] [111.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 05:00:21,055 - Train: 11.13% [453100/4070000] [111.3/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-06 05:01:21,549 - Train: 11.14% [453200/4070000] [111.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 05:02:22,062 - Train: 11.14% [453300/4070000] [111.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 05:03:22,712 - Train: 11.14% [453400/4070000] [111.4/1000.0] [batch_t 0.755 (0.606)] [data_t 0.002] [optim_t 0.754] [lr 0.005000] +2024-04-06 05:04:23,211 - Train: 11.14% [453500/4070000] [111.4/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 05:05:23,694 - Train: 11.14% [453600/4070000] [111.4/1000.0] [batch_t 0.600 (0.605)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-06 05:06:24,174 - Train: 11.15% [453700/4070000] [111.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 05:07:24,654 - Train: 11.15% [453800/4070000] [111.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 05:08:25,162 - Train: 11.15% [453900/4070000] [111.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 05:09:25,644 - Train: 11.15% [454000/4070000] [111.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 05:10:26,119 - Train: 11.16% [454100/4070000] [111.6/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 05:11:26,697 - Train: 11.16% [454200/4070000] [111.6/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 05:12:27,193 - Train: 11.16% [454300/4070000] [111.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 05:13:27,661 - Train: 11.16% [454400/4070000] [111.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 05:14:28,155 - Train: 11.17% [454500/4070000] [111.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.003] [optim_t 0.603] [lr 0.005000] +2024-04-06 05:15:28,606 - Train: 11.17% [454600/4070000] [111.7/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 05:16:29,086 - Train: 11.17% [454700/4070000] [111.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 05:17:29,649 - Train: 11.17% [454800/4070000] [111.7/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 05:18:30,130 - Train: 11.18% [454900/4070000] [111.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 05:19:30,697 - Train: 11.18% [455000/4070000] [111.8/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 05:20:31,207 - Train: 11.18% [455100/4070000] [111.8/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 05:21:31,756 - Train: 11.18% [455200/4070000] [111.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 05:22:33,022 - Train: 11.19% [455300/4070000] [111.9/1000.0] [batch_t 0.603 (0.613)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 05:23:33,509 - Train: 11.19% [455400/4070000] [111.9/1000.0] [batch_t 0.600 (0.605)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-06 05:24:34,023 - Train: 11.19% [455500/4070000] [111.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 05:25:34,498 - Train: 11.19% [455600/4070000] [111.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 05:26:34,951 - Train: 11.20% [455700/4070000] [112.0/1000.0] [batch_t 0.601 (0.604)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-06 05:27:35,428 - Train: 11.20% [455800/4070000] [112.0/1000.0] [batch_t 0.588 (0.605)] [data_t 0.002] [optim_t 0.586] [lr 0.005000] +2024-04-06 05:27:59,628 - ==> Total time: 4 days, 4:27:30 Eta: 33 days, 4:29:31 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-06 05:28:37,474 - Train: 11.20% [455900/4070000] [112.0/1000.0] [batch_t 0.609 (0.608)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-06 05:29:38,013 - Train: 11.20% [456000/4070000] [112.0/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 05:30:38,507 - Train: 11.21% [456100/4070000] [112.1/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 05:31:39,524 - Train: 11.21% [456200/4070000] [112.1/1000.0] [batch_t 0.605 (0.610)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 05:32:40,002 - Train: 11.21% [456300/4070000] [112.1/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 05:33:40,490 - Train: 11.21% [456400/4070000] [112.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 05:34:41,827 - Train: 11.22% [456500/4070000] [112.2/1000.0] [batch_t 0.606 (0.613)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 05:35:42,681 - Train: 11.22% [456600/4070000] [112.2/1000.0] [batch_t 0.606 (0.608)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 05:36:43,182 - Train: 11.22% [456700/4070000] [112.2/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 05:37:43,685 - Train: 11.22% [456800/4070000] [112.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 05:38:44,183 - Train: 11.23% [456900/4070000] [112.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 05:39:44,655 - Train: 11.23% [457000/4070000] [112.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 05:40:45,232 - Train: 11.23% [457100/4070000] [112.3/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 05:41:46,082 - Train: 11.23% [457200/4070000] [112.3/1000.0] [batch_t 0.604 (0.608)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 05:42:46,535 - Train: 11.24% [457300/4070000] [112.4/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 05:43:47,033 - Train: 11.24% [457400/4070000] [112.4/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 05:44:47,527 - Train: 11.24% [457500/4070000] [112.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 05:45:48,046 - Train: 11.24% [457600/4070000] [112.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 05:46:48,926 - Train: 11.25% [457700/4070000] [112.5/1000.0] [batch_t 0.604 (0.609)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 05:47:49,804 - Train: 11.25% [457800/4070000] [112.5/1000.0] [batch_t 0.606 (0.609)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 05:48:50,427 - Train: 11.25% [457900/4070000] [112.5/1000.0] [batch_t 0.615 (0.606)] [data_t 0.002] [optim_t 0.613] [lr 0.005000] +2024-04-06 05:49:51,949 - Train: 11.25% [458000/4070000] [112.5/1000.0] [batch_t 0.606 (0.615)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 05:50:52,421 - Train: 11.26% [458100/4070000] [112.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 05:51:52,894 - Train: 11.26% [458200/4070000] [112.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 05:52:53,362 - Train: 11.26% [458300/4070000] [112.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 05:53:53,944 - Train: 11.26% [458400/4070000] [112.6/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 05:54:54,487 - Train: 11.27% [458500/4070000] [112.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 05:55:55,033 - Train: 11.27% [458600/4070000] [112.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 05:56:55,926 - Train: 11.27% [458700/4070000] [112.7/1000.0] [batch_t 0.603 (0.609)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 05:57:56,432 - Train: 11.27% [458800/4070000] [112.7/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 05:58:58,103 - Train: 11.28% [458900/4070000] [112.8/1000.0] [batch_t 0.606 (0.617)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 05:59:58,600 - Train: 11.28% [459000/4070000] [112.8/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 06:00:59,100 - Train: 11.28% [459100/4070000] [112.8/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 06:01:59,696 - Train: 11.28% [459200/4070000] [112.8/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 06:03:00,236 - Train: 11.29% [459300/4070000] [112.9/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 06:04:00,717 - Train: 11.29% [459400/4070000] [112.9/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 06:05:01,286 - Train: 11.29% [459500/4070000] [112.9/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 06:06:01,800 - Train: 11.29% [459600/4070000] [112.9/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 06:07:02,419 - Train: 11.29% [459700/4070000] [112.9/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 06:08:02,929 - Train: 11.30% [459800/4070000] [113.0/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 06:09:03,423 - Train: 11.30% [459900/4070000] [113.0/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 06:09:09,493 - ==> Total time: 4 days, 5:08:40 Eta: 33 days, 1:56:23 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-06 06:10:05,533 - Train: 11.30% [460000/4070000] [113.0/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 06:11:06,076 - Train: 11.30% [460100/4070000] [113.0/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-06 06:12:06,596 - Train: 11.31% [460200/4070000] [113.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 06:13:07,229 - Train: 11.31% [460300/4070000] [113.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 06:14:07,798 - Train: 11.31% [460400/4070000] [113.1/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 06:15:08,344 - Train: 11.31% [460500/4070000] [113.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 06:16:08,877 - Train: 11.32% [460600/4070000] [113.2/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-06 06:17:09,394 - Train: 11.32% [460700/4070000] [113.2/1000.0] [batch_t 0.610 (0.605)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-06 06:18:09,919 - Train: 11.32% [460800/4070000] [113.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 06:19:11,089 - Train: 11.32% [460900/4070000] [113.2/1000.0] [batch_t 0.601 (0.612)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-06 06:20:11,596 - Train: 11.33% [461000/4070000] [113.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 06:21:12,090 - Train: 11.33% [461100/4070000] [113.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 06:22:12,615 - Train: 11.33% [461200/4070000] [113.3/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 06:23:13,181 - Train: 11.33% [461300/4070000] [113.3/1000.0] [batch_t 0.610 (0.606)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-06 06:24:13,723 - Train: 11.34% [461400/4070000] [113.4/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 06:25:14,267 - Train: 11.34% [461500/4070000] [113.4/1000.0] [batch_t 0.610 (0.605)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-06 06:26:14,763 - Train: 11.34% [461600/4070000] [113.4/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 06:27:15,425 - Train: 11.34% [461700/4070000] [113.4/1000.0] [batch_t 0.603 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 06:28:15,994 - Train: 11.35% [461800/4070000] [113.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 06:29:16,500 - Train: 11.35% [461900/4070000] [113.5/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 06:30:17,016 - Train: 11.35% [462000/4070000] [113.5/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 06:31:17,479 - Train: 11.35% [462100/4070000] [113.5/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 06:32:18,076 - Train: 11.36% [462200/4070000] [113.6/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 06:33:18,557 - Train: 11.36% [462300/4070000] [113.6/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 06:34:19,047 - Train: 11.36% [462400/4070000] [113.6/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 06:35:19,524 - Train: 11.36% [462500/4070000] [113.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 06:36:20,027 - Train: 11.37% [462600/4070000] [113.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 06:37:20,517 - Train: 11.37% [462700/4070000] [113.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 06:38:21,029 - Train: 11.37% [462800/4070000] [113.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 06:39:21,564 - Train: 11.37% [462900/4070000] [113.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 06:40:22,140 - Train: 11.38% [463000/4070000] [113.8/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 06:41:22,615 - Train: 11.38% [463100/4070000] [113.8/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-06 06:42:24,294 - Train: 11.38% [463200/4070000] [113.8/1000.0] [batch_t 0.605 (0.617)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 06:43:24,791 - Train: 11.38% [463300/4070000] [113.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 06:44:26,035 - Train: 11.39% [463400/4070000] [113.9/1000.0] [batch_t 0.607 (0.612)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 06:45:26,588 - Train: 11.39% [463500/4070000] [113.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 06:46:27,113 - Train: 11.39% [463600/4070000] [113.9/1000.0] [batch_t 0.598 (0.605)] [data_t 0.002] [optim_t 0.596] [lr 0.005000] +2024-04-06 06:47:27,600 - Train: 11.39% [463700/4070000] [113.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 06:48:28,103 - Train: 11.40% [463800/4070000] [114.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 06:49:28,619 - Train: 11.40% [463900/4070000] [114.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 06:50:17,385 - ==> Total time: 4 days, 5:49:48 Eta: 32 days, 23:24:58 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-06 06:50:31,302 - Train: 11.40% [464000/4070000] [114.0/1000.0] [batch_t 0.605 (0.623)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 06:51:31,783 - Train: 11.40% [464100/4070000] [114.0/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 06:52:32,748 - Train: 11.41% [464200/4070000] [114.1/1000.0] [batch_t 0.605 (0.610)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 06:53:33,787 - Train: 11.41% [464300/4070000] [114.1/1000.0] [batch_t 0.607 (0.610)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 06:54:34,261 - Train: 11.41% [464400/4070000] [114.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 06:55:34,721 - Train: 11.41% [464500/4070000] [114.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 06:56:35,193 - Train: 11.42% [464600/4070000] [114.2/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 06:57:36,130 - Train: 11.42% [464700/4070000] [114.2/1000.0] [batch_t 0.606 (0.609)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 06:58:37,097 - Train: 11.42% [464800/4070000] [114.2/1000.0] [batch_t 0.607 (0.610)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 06:59:37,576 - Train: 11.42% [464900/4070000] [114.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 07:00:38,052 - Train: 11.43% [465000/4070000] [114.3/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 07:01:38,528 - Train: 11.43% [465100/4070000] [114.3/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-06 07:02:40,320 - Train: 11.43% [465200/4070000] [114.3/1000.0] [batch_t 0.606 (0.618)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 07:03:40,808 - Train: 11.43% [465300/4070000] [114.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 07:04:41,332 - Train: 11.43% [465400/4070000] [114.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 07:05:41,885 - Train: 11.44% [465500/4070000] [114.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 07:06:42,902 - Train: 11.44% [465600/4070000] [114.4/1000.0] [batch_t 0.605 (0.610)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 07:07:43,361 - Train: 11.44% [465700/4070000] [114.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 07:08:43,814 - Train: 11.44% [465800/4070000] [114.4/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 07:09:44,323 - Train: 11.45% [465900/4070000] [114.5/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 07:10:44,794 - Train: 11.45% [466000/4070000] [114.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 07:11:45,996 - Train: 11.45% [466100/4070000] [114.5/1000.0] [batch_t 0.605 (0.612)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 07:12:46,983 - Train: 11.45% [466200/4070000] [114.5/1000.0] [batch_t 0.605 (0.610)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 07:13:47,473 - Train: 11.46% [466300/4070000] [114.6/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-06 07:14:47,946 - Train: 11.46% [466400/4070000] [114.6/1000.0] [batch_t 0.599 (0.605)] [data_t 0.002] [optim_t 0.597] [lr 0.005000] +2024-04-06 07:15:48,477 - Train: 11.46% [466500/4070000] [114.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 07:16:49,248 - Train: 11.46% [466600/4070000] [114.6/1000.0] [batch_t 0.607 (0.608)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 07:17:49,726 - Train: 11.47% [466700/4070000] [114.7/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 07:18:50,269 - Train: 11.47% [466800/4070000] [114.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 07:19:50,753 - Train: 11.47% [466900/4070000] [114.7/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 07:20:51,361 - Train: 11.47% [467000/4070000] [114.7/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 07:21:51,855 - Train: 11.48% [467100/4070000] [114.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 07:22:52,918 - Train: 11.48% [467200/4070000] [114.8/1000.0] [batch_t 0.605 (0.611)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 07:23:53,402 - Train: 11.48% [467300/4070000] [114.8/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-06 07:24:53,822 - Train: 11.48% [467400/4070000] [114.8/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 07:25:54,366 - Train: 11.49% [467500/4070000] [114.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-06 07:26:54,848 - Train: 11.49% [467600/4070000] [114.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 07:27:55,305 - Train: 11.49% [467700/4070000] [114.9/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 07:28:55,811 - Train: 11.49% [467800/4070000] [114.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 07:29:56,309 - Train: 11.50% [467900/4070000] [115.0/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 07:30:56,814 - Train: 11.50% [468000/4070000] [115.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 07:31:27,196 - ==> Total time: 4 days, 6:30:58 Eta: 32 days, 20:55:42 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-06 07:31:59,229 - Train: 11.50% [468100/4070000] [115.0/1000.0] [batch_t 0.606 (0.611)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 07:32:59,756 - Train: 11.50% [468200/4070000] [115.0/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 07:34:00,258 - Train: 11.51% [468300/4070000] [115.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 07:35:00,738 - Train: 11.51% [468400/4070000] [115.1/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 07:36:01,249 - Train: 11.51% [468500/4070000] [115.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 07:37:01,746 - Train: 11.51% [468600/4070000] [115.1/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 07:38:02,283 - Train: 11.52% [468700/4070000] [115.2/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 07:39:03,330 - Train: 11.52% [468800/4070000] [115.2/1000.0] [batch_t 0.605 (0.610)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 07:40:03,818 - Train: 11.52% [468900/4070000] [115.2/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 07:41:04,332 - Train: 11.52% [469000/4070000] [115.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 07:42:04,827 - Train: 11.53% [469100/4070000] [115.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 07:43:05,328 - Train: 11.53% [469200/4070000] [115.3/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 07:44:05,934 - Train: 11.53% [469300/4070000] [115.3/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 07:45:06,439 - Train: 11.53% [469400/4070000] [115.3/1000.0] [batch_t 0.610 (0.605)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-06 07:46:06,973 - Train: 11.54% [469500/4070000] [115.4/1000.0] [batch_t 0.612 (0.605)] [data_t 0.002] [optim_t 0.610] [lr 0.005000] +2024-04-06 07:47:07,969 - Train: 11.54% [469600/4070000] [115.4/1000.0] [batch_t 0.606 (0.610)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 07:48:08,479 - Train: 11.54% [469700/4070000] [115.4/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-06 07:49:08,970 - Train: 11.54% [469800/4070000] [115.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 07:50:09,452 - Train: 11.55% [469900/4070000] [115.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 07:51:09,943 - Train: 11.55% [470000/4070000] [115.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 07:52:10,454 - Train: 11.55% [470100/4070000] [115.5/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 07:53:10,917 - Train: 11.55% [470200/4070000] [115.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 07:54:11,419 - Train: 11.56% [470300/4070000] [115.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 07:55:11,907 - Train: 11.56% [470400/4070000] [115.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 07:56:12,388 - Train: 11.56% [470500/4070000] [115.6/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-06 07:57:12,970 - Train: 11.56% [470600/4070000] [115.6/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 07:58:13,463 - Train: 11.57% [470700/4070000] [115.7/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 07:59:13,967 - Train: 11.57% [470800/4070000] [115.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 08:00:14,469 - Train: 11.57% [470900/4070000] [115.7/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 08:01:14,969 - Train: 11.57% [471000/4070000] [115.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 08:02:15,464 - Train: 11.57% [471100/4070000] [115.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 08:03:15,974 - Train: 11.58% [471200/4070000] [115.8/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 08:04:17,013 - Train: 11.58% [471300/4070000] [115.8/1000.0] [batch_t 0.600 (0.610)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-06 08:05:17,485 - Train: 11.58% [471400/4070000] [115.8/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 08:06:18,018 - Train: 11.58% [471500/4070000] [115.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 08:07:18,477 - Train: 11.59% [471600/4070000] [115.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 08:08:18,981 - Train: 11.59% [471700/4070000] [115.9/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 08:09:19,481 - Train: 11.59% [471800/4070000] [115.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 08:10:20,088 - Train: 11.59% [471900/4070000] [115.9/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 08:11:20,583 - Train: 11.60% [472000/4070000] [116.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 08:12:21,068 - Train: 11.60% [472100/4070000] [116.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 08:12:33,171 - ==> Total time: 4 days, 7:12:04 Eta: 32 days, 18:27:49 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-06 08:13:23,057 - Train: 11.60% [472200/4070000] [116.0/1000.0] [batch_t 0.600 (0.607)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-06 08:14:23,522 - Train: 11.60% [472300/4070000] [116.0/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 08:15:24,131 - Train: 11.61% [472400/4070000] [116.1/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 08:16:24,642 - Train: 11.61% [472500/4070000] [116.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 08:17:25,169 - Train: 11.61% [472600/4070000] [116.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 08:18:25,680 - Train: 11.61% [472700/4070000] [116.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 08:19:26,140 - Train: 11.62% [472800/4070000] [116.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 08:20:26,629 - Train: 11.62% [472900/4070000] [116.2/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 08:21:27,171 - Train: 11.62% [473000/4070000] [116.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 08:22:27,948 - Train: 11.62% [473100/4070000] [116.2/1000.0] [batch_t 0.606 (0.608)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 08:23:28,573 - Train: 11.63% [473200/4070000] [116.3/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 08:24:29,072 - Train: 11.63% [473300/4070000] [116.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 08:25:29,558 - Train: 11.63% [473400/4070000] [116.3/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 08:26:30,063 - Train: 11.63% [473500/4070000] [116.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 08:27:30,530 - Train: 11.64% [473600/4070000] [116.4/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 08:28:31,108 - Train: 11.64% [473700/4070000] [116.4/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 08:29:31,606 - Train: 11.64% [473800/4070000] [116.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 08:30:32,084 - Train: 11.64% [473900/4070000] [116.4/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 08:31:32,614 - Train: 11.65% [474000/4070000] [116.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 08:32:33,110 - Train: 11.65% [474100/4070000] [116.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 08:33:33,595 - Train: 11.65% [474200/4070000] [116.5/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 08:34:34,122 - Train: 11.65% [474300/4070000] [116.5/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 08:35:34,637 - Train: 11.66% [474400/4070000] [116.6/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 08:36:35,230 - Train: 11.66% [474500/4070000] [116.6/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 08:37:35,704 - Train: 11.66% [474600/4070000] [116.6/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 08:38:36,231 - Train: 11.66% [474700/4070000] [116.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 08:39:36,726 - Train: 11.67% [474800/4070000] [116.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 08:40:37,168 - Train: 11.67% [474900/4070000] [116.7/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 08:41:37,665 - Train: 11.67% [475000/4070000] [116.7/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 08:42:38,199 - Train: 11.67% [475100/4070000] [116.7/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 08:43:38,714 - Train: 11.68% [475200/4070000] [116.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 08:44:39,181 - Train: 11.68% [475300/4070000] [116.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 08:45:39,668 - Train: 11.68% [475400/4070000] [116.8/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 08:46:40,126 - Train: 11.68% [475500/4070000] [116.8/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 08:47:40,685 - Train: 11.69% [475600/4070000] [116.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 08:48:41,153 - Train: 11.69% [475700/4070000] [116.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 08:49:41,614 - Train: 11.69% [475800/4070000] [116.9/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 08:50:43,173 - Train: 11.69% [475900/4070000] [116.9/1000.0] [batch_t 0.604 (0.615)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 08:51:43,877 - Train: 11.70% [476000/4070000] [117.0/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 08:52:44,328 - Train: 11.70% [476100/4070000] [117.0/1000.0] [batch_t 0.608 (0.604)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 08:53:38,744 - ==> Total time: 4 days, 7:53:09 Eta: 32 days, 16:01:42 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-06 08:53:46,592 - Train: 11.70% [476200/4070000] [117.0/1000.0] [batch_t 0.610 (0.623)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-06 08:54:47,255 - Train: 11.70% [476300/4070000] [117.0/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 08:55:47,706 - Train: 11.71% [476400/4070000] [117.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 08:56:48,162 - Train: 11.71% [476500/4070000] [117.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 08:57:48,652 - Train: 11.71% [476600/4070000] [117.1/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 08:58:49,136 - Train: 11.71% [476700/4070000] [117.1/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-06 08:59:49,613 - Train: 11.71% [476800/4070000] [117.1/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 09:00:50,805 - Train: 11.72% [476900/4070000] [117.2/1000.0] [batch_t 0.603 (0.612)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 09:01:51,420 - Train: 11.72% [477000/4070000] [117.2/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 09:02:51,909 - Train: 11.72% [477100/4070000] [117.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 09:03:52,432 - Train: 11.72% [477200/4070000] [117.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 09:04:52,924 - Train: 11.73% [477300/4070000] [117.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 09:05:53,407 - Train: 11.73% [477400/4070000] [117.3/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-06 09:06:53,947 - Train: 11.73% [477500/4070000] [117.3/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 09:07:56,004 - Train: 11.73% [477600/4070000] [117.3/1000.0] [batch_t 0.608 (0.620)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 09:08:56,496 - Train: 11.74% [477700/4070000] [117.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 09:09:57,039 - Train: 11.74% [477800/4070000] [117.4/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-06 09:10:57,521 - Train: 11.74% [477900/4070000] [117.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 09:11:58,026 - Train: 11.74% [478000/4070000] [117.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 09:12:59,998 - Train: 11.75% [478100/4070000] [117.5/1000.0] [batch_t 0.606 (0.620)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 09:14:00,474 - Train: 11.75% [478200/4070000] [117.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 09:15:01,027 - Train: 11.75% [478300/4070000] [117.5/1000.0] [batch_t 0.595 (0.605)] [data_t 0.002] [optim_t 0.593] [lr 0.005000] +2024-04-06 09:16:01,483 - Train: 11.75% [478400/4070000] [117.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 09:17:01,972 - Train: 11.76% [478500/4070000] [117.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 09:18:03,382 - Train: 11.76% [478600/4070000] [117.6/1000.0] [batch_t 0.605 (0.614)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 09:19:03,870 - Train: 11.76% [478700/4070000] [117.6/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 09:20:04,341 - Train: 11.76% [478800/4070000] [117.6/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 09:21:04,809 - Train: 11.77% [478900/4070000] [117.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 09:22:05,356 - Train: 11.77% [479000/4070000] [117.7/1000.0] [batch_t 0.677 (0.605)] [data_t 0.002] [optim_t 0.675] [lr 0.005000] +2024-04-06 09:23:05,860 - Train: 11.77% [479100/4070000] [117.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 09:24:06,358 - Train: 11.77% [479200/4070000] [117.7/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 09:25:06,847 - Train: 11.78% [479300/4070000] [117.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 09:26:07,320 - Train: 11.78% [479400/4070000] [117.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 09:27:07,830 - Train: 11.78% [479500/4070000] [117.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 09:28:08,475 - Train: 11.78% [479600/4070000] [117.8/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 09:29:08,993 - Train: 11.79% [479700/4070000] [117.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 09:30:09,544 - Train: 11.79% [479800/4070000] [117.9/1000.0] [batch_t 0.610 (0.605)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-06 09:31:10,047 - Train: 11.79% [479900/4070000] [117.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 09:32:10,543 - Train: 11.79% [480000/4070000] [117.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 09:33:11,013 - Train: 11.80% [480100/4070000] [118.0/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 09:34:11,879 - Train: 11.80% [480200/4070000] [118.0/1000.0] [batch_t 0.605 (0.609)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 09:34:48,188 - ==> Total time: 4 days, 8:34:19 Eta: 32 days, 13:37:51 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-06 09:35:14,068 - Train: 11.80% [480300/4070000] [118.0/1000.0] [batch_t 0.606 (0.609)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 09:36:14,547 - Train: 11.80% [480400/4070000] [118.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 09:37:15,006 - Train: 11.81% [480500/4070000] [118.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 09:38:15,542 - Train: 11.81% [480600/4070000] [118.1/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 09:39:16,021 - Train: 11.81% [480700/4070000] [118.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 09:40:16,504 - Train: 11.81% [480800/4070000] [118.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 09:41:17,053 - Train: 11.82% [480900/4070000] [118.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 09:42:17,659 - Train: 11.82% [481000/4070000] [118.2/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 09:43:18,140 - Train: 11.82% [481100/4070000] [118.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 09:44:18,634 - Train: 11.82% [481200/4070000] [118.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 09:45:19,188 - Train: 11.83% [481300/4070000] [118.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 09:46:19,755 - Train: 11.83% [481400/4070000] [118.3/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 09:47:20,823 - Train: 11.83% [481500/4070000] [118.3/1000.0] [batch_t 0.603 (0.611)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 09:48:21,521 - Train: 11.83% [481600/4070000] [118.3/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 09:49:21,989 - Train: 11.84% [481700/4070000] [118.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 09:50:22,602 - Train: 11.84% [481800/4070000] [118.4/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 09:51:23,059 - Train: 11.84% [481900/4070000] [118.4/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 09:52:23,549 - Train: 11.84% [482000/4070000] [118.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 09:53:24,125 - Train: 11.85% [482100/4070000] [118.5/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 09:54:24,623 - Train: 11.85% [482200/4070000] [118.5/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 09:55:25,270 - Train: 11.85% [482300/4070000] [118.5/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 09:56:25,752 - Train: 11.85% [482400/4070000] [118.5/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-06 09:57:26,202 - Train: 11.86% [482500/4070000] [118.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 09:58:26,733 - Train: 11.86% [482600/4070000] [118.6/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 09:59:27,283 - Train: 11.86% [482700/4070000] [118.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 10:00:27,798 - Train: 11.86% [482800/4070000] [118.6/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 10:01:28,585 - Train: 11.86% [482900/4070000] [118.6/1000.0] [batch_t 0.603 (0.608)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 10:02:29,083 - Train: 11.87% [483000/4070000] [118.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 10:03:30,060 - Train: 11.87% [483100/4070000] [118.7/1000.0] [batch_t 0.605 (0.610)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 10:04:30,531 - Train: 11.87% [483200/4070000] [118.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 10:05:31,020 - Train: 11.87% [483300/4070000] [118.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 10:06:31,527 - Train: 11.88% [483400/4070000] [118.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 10:07:32,280 - Train: 11.88% [483500/4070000] [118.8/1000.0] [batch_t 0.603 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 10:08:32,863 - Train: 11.88% [483600/4070000] [118.8/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 10:09:33,308 - Train: 11.88% [483700/4070000] [118.8/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-06 10:10:33,761 - Train: 11.89% [483800/4070000] [118.9/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 10:11:34,187 - Train: 11.89% [483900/4070000] [118.9/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 10:12:35,018 - Train: 11.89% [484000/4070000] [118.9/1000.0] [batch_t 0.606 (0.608)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 10:13:35,488 - Train: 11.89% [484100/4070000] [118.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 10:14:36,055 - Train: 11.90% [484200/4070000] [119.0/1000.0] [batch_t 0.600 (0.606)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-06 10:15:36,550 - Train: 11.90% [484300/4070000] [119.0/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 10:15:54,706 - ==> Total time: 4 days, 9:15:25 Eta: 32 days, 11:15:22 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-06 10:16:39,177 - Train: 11.90% [484400/4070000] [119.0/1000.0] [batch_t 0.607 (0.615)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 10:17:40,141 - Train: 11.90% [484500/4070000] [119.0/1000.0] [batch_t 0.606 (0.610)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 10:18:40,629 - Train: 11.91% [484600/4070000] [119.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 10:19:41,478 - Train: 11.91% [484700/4070000] [119.1/1000.0] [batch_t 0.601 (0.608)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-06 10:20:42,159 - Train: 11.91% [484800/4070000] [119.1/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 10:21:43,161 - Train: 11.91% [484900/4070000] [119.1/1000.0] [batch_t 0.605 (0.610)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 10:22:43,632 - Train: 11.92% [485000/4070000] [119.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 10:23:44,098 - Train: 11.92% [485100/4070000] [119.2/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-06 10:24:44,554 - Train: 11.92% [485200/4070000] [119.2/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 10:25:45,494 - Train: 11.92% [485300/4070000] [119.2/1000.0] [batch_t 0.605 (0.609)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 10:26:46,054 - Train: 11.93% [485400/4070000] [119.3/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 10:27:46,527 - Train: 11.93% [485500/4070000] [119.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 10:28:47,930 - Train: 11.93% [485600/4070000] [119.3/1000.0] [batch_t 0.603 (0.614)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 10:29:48,788 - Train: 11.93% [485700/4070000] [119.3/1000.0] [batch_t 0.606 (0.608)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 10:30:49,279 - Train: 11.94% [485800/4070000] [119.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 10:31:49,768 - Train: 11.94% [485900/4070000] [119.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 10:32:50,254 - Train: 11.94% [486000/4070000] [119.4/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-06 10:33:50,748 - Train: 11.94% [486100/4070000] [119.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 10:34:51,307 - Train: 11.95% [486200/4070000] [119.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 10:35:51,793 - Train: 11.95% [486300/4070000] [119.5/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 10:36:52,299 - Train: 11.95% [486400/4070000] [119.5/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 10:37:52,781 - Train: 11.95% [486500/4070000] [119.5/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 10:38:53,268 - Train: 11.96% [486600/4070000] [119.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 10:39:53,848 - Train: 11.96% [486700/4070000] [119.6/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 10:40:54,313 - Train: 11.96% [486800/4070000] [119.6/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 10:41:54,828 - Train: 11.96% [486900/4070000] [119.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 10:42:55,360 - Train: 11.97% [487000/4070000] [119.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 10:43:55,842 - Train: 11.97% [487100/4070000] [119.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 10:44:56,341 - Train: 11.97% [487200/4070000] [119.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 10:45:56,817 - Train: 11.97% [487300/4070000] [119.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 10:46:57,350 - Train: 11.98% [487400/4070000] [119.8/1000.0] [batch_t 0.678 (0.605)] [data_t 0.002] [optim_t 0.676] [lr 0.005000] +2024-04-06 10:47:57,815 - Train: 11.98% [487500/4070000] [119.8/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 10:48:58,282 - Train: 11.98% [487600/4070000] [119.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 10:49:58,732 - Train: 11.98% [487700/4070000] [119.8/1000.0] [batch_t 0.608 (0.604)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 10:50:59,239 - Train: 11.99% [487800/4070000] [119.9/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 10:51:59,676 - Train: 11.99% [487900/4070000] [119.9/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 10:53:00,234 - Train: 11.99% [488000/4070000] [119.9/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 10:54:00,733 - Train: 11.99% [488100/4070000] [119.9/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 10:55:01,232 - Train: 12.00% [488200/4070000] [120.0/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 10:56:01,707 - Train: 12.00% [488300/4070000] [120.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 10:57:02,229 - Train: 12.00% [488400/4070000] [120.0/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 10:57:02,233 - ==> Total time: 4 days, 9:56:33 Eta: 32 days, 8:54:42 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-06 10:58:07,029 - Train: 12.00% [488500/4070000] [120.0/1000.0] [batch_t 0.602 (0.610)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-06 10:59:07,532 - Train: 12.00% [488600/4070000] [120.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 11:00:08,153 - Train: 12.01% [488700/4070000] [120.1/1000.0] [batch_t 0.610 (0.606)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-06 11:01:08,639 - Train: 12.01% [488800/4070000] [120.1/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 11:02:09,163 - Train: 12.01% [488900/4070000] [120.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 11:03:09,690 - Train: 12.01% [489000/4070000] [120.1/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 11:04:11,039 - Train: 12.02% [489100/4070000] [120.2/1000.0] [batch_t 0.605 (0.613)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 11:05:11,844 - Train: 12.02% [489200/4070000] [120.2/1000.0] [batch_t 0.605 (0.608)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 11:06:12,348 - Train: 12.02% [489300/4070000] [120.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 11:07:13,091 - Train: 12.02% [489400/4070000] [120.2/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 11:08:13,597 - Train: 12.03% [489500/4070000] [120.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 11:09:14,383 - Train: 12.03% [489600/4070000] [120.3/1000.0] [batch_t 0.607 (0.608)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 11:10:14,878 - Train: 12.03% [489700/4070000] [120.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 11:11:16,377 - Train: 12.03% [489800/4070000] [120.3/1000.0] [batch_t 0.604 (0.615)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 11:12:16,883 - Train: 12.04% [489900/4070000] [120.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 11:13:18,077 - Train: 12.04% [490000/4070000] [120.4/1000.0] [batch_t 0.601 (0.612)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-06 11:14:18,578 - Train: 12.04% [490100/4070000] [120.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 11:15:19,094 - Train: 12.04% [490200/4070000] [120.4/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 11:16:19,612 - Train: 12.05% [490300/4070000] [120.5/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-06 11:17:20,118 - Train: 12.05% [490400/4070000] [120.5/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 11:18:20,781 - Train: 12.05% [490500/4070000] [120.5/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 11:19:21,322 - Train: 12.05% [490600/4070000] [120.5/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 11:20:21,810 - Train: 12.06% [490700/4070000] [120.6/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 11:21:22,315 - Train: 12.06% [490800/4070000] [120.6/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 11:22:23,406 - Train: 12.06% [490900/4070000] [120.6/1000.0] [batch_t 0.603 (0.611)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 11:23:24,301 - Train: 12.06% [491000/4070000] [120.6/1000.0] [batch_t 0.603 (0.609)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 11:24:24,800 - Train: 12.07% [491100/4070000] [120.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 11:25:25,286 - Train: 12.07% [491200/4070000] [120.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 11:26:26,706 - Train: 12.07% [491300/4070000] [120.7/1000.0] [batch_t 0.601 (0.614)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-06 11:27:27,220 - Train: 12.07% [491400/4070000] [120.7/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 11:28:27,719 - Train: 12.08% [491500/4070000] [120.8/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 11:29:28,238 - Train: 12.08% [491600/4070000] [120.8/1000.0] [batch_t 0.613 (0.605)] [data_t 0.002] [optim_t 0.611] [lr 0.005000] +2024-04-06 11:30:28,802 - Train: 12.08% [491700/4070000] [120.8/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 11:31:29,488 - Train: 12.08% [491800/4070000] [120.8/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 11:32:30,025 - Train: 12.09% [491900/4070000] [120.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 11:33:30,512 - Train: 12.09% [492000/4070000] [120.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 11:34:31,038 - Train: 12.09% [492100/4070000] [120.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 11:35:31,557 - Train: 12.09% [492200/4070000] [120.9/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 11:36:32,051 - Train: 12.10% [492300/4070000] [121.0/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 11:37:32,533 - Train: 12.10% [492400/4070000] [121.0/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 11:38:14,871 - ==> Total time: 4 days, 10:37:45 Eta: 32 days, 6:36:18 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-06 11:38:34,847 - Train: 12.10% [492500/4070000] [121.0/1000.0] [batch_t 0.604 (0.612)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 11:39:35,389 - Train: 12.10% [492600/4070000] [121.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 11:40:35,901 - Train: 12.11% [492700/4070000] [121.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 11:41:36,394 - Train: 12.11% [492800/4070000] [121.1/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-06 11:42:36,888 - Train: 12.11% [492900/4070000] [121.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 11:43:37,356 - Train: 12.11% [493000/4070000] [121.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 11:44:38,050 - Train: 12.12% [493100/4070000] [121.2/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 11:45:38,856 - Train: 12.12% [493200/4070000] [121.2/1000.0] [batch_t 0.608 (0.608)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 11:46:45,540 - Train: 12.12% [493300/4070000] [121.2/1000.0] [batch_t 0.601 (0.667)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-06 11:47:52,049 - Train: 12.12% [493400/4070000] [121.2/1000.0] [batch_t 0.605 (0.665)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 11:49:30,099 - Train: 12.13% [493500/4070000] [121.3/1000.0] [batch_t 0.606 (0.980)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 11:50:48,165 - Train: 12.13% [493600/4070000] [121.3/1000.0] [batch_t 0.606 (0.781)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 11:51:51,872 - Train: 12.13% [493700/4070000] [121.3/1000.0] [batch_t 0.600 (0.637)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-06 11:52:52,522 - Train: 12.13% [493800/4070000] [121.3/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 11:53:53,041 - Train: 12.14% [493900/4070000] [121.4/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 11:54:53,544 - Train: 12.14% [494000/4070000] [121.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 11:55:54,118 - Train: 12.14% [494100/4070000] [121.4/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 11:56:54,639 - Train: 12.14% [494200/4070000] [121.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 11:57:55,166 - Train: 12.14% [494300/4070000] [121.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 11:58:55,850 - Train: 12.15% [494400/4070000] [121.5/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 11:59:56,350 - Train: 12.15% [494500/4070000] [121.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 12:00:56,856 - Train: 12.15% [494600/4070000] [121.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 12:01:57,387 - Train: 12.15% [494700/4070000] [121.5/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 12:02:57,947 - Train: 12.16% [494800/4070000] [121.6/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-06 12:03:58,454 - Train: 12.16% [494900/4070000] [121.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 12:04:59,140 - Train: 12.16% [495000/4070000] [121.6/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 12:05:59,607 - Train: 12.16% [495100/4070000] [121.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 12:07:00,138 - Train: 12.17% [495200/4070000] [121.7/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 12:08:00,624 - Train: 12.17% [495300/4070000] [121.7/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-06 12:09:01,155 - Train: 12.17% [495400/4070000] [121.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 12:10:01,638 - Train: 12.17% [495500/4070000] [121.7/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 12:11:02,151 - Train: 12.18% [495600/4070000] [121.8/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 12:12:02,789 - Train: 12.18% [495700/4070000] [121.8/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 12:13:03,288 - Train: 12.18% [495800/4070000] [121.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 12:14:03,769 - Train: 12.18% [495900/4070000] [121.8/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-06 12:15:04,257 - Train: 12.19% [496000/4070000] [121.9/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 12:16:04,758 - Train: 12.19% [496100/4070000] [121.9/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 12:17:05,297 - Train: 12.19% [496200/4070000] [121.9/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 12:18:05,936 - Train: 12.19% [496300/4070000] [121.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 12:19:06,445 - Train: 12.20% [496400/4070000] [122.0/1000.0] [batch_t 0.610 (0.605)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-06 12:20:06,937 - Train: 12.20% [496500/4070000] [122.0/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 12:20:31,134 - ==> Total time: 4 days, 11:20:01 Eta: 32 days, 4:27:07 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-06 12:21:10,709 - Train: 12.20% [496600/4070000] [122.0/1000.0] [batch_t 0.604 (0.618)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 12:22:11,198 - Train: 12.20% [496700/4070000] [122.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 12:23:11,686 - Train: 12.21% [496800/4070000] [122.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 12:24:12,207 - Train: 12.21% [496900/4070000] [122.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 12:25:12,815 - Train: 12.21% [497000/4070000] [122.1/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 12:26:13,288 - Train: 12.21% [497100/4070000] [122.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 12:27:13,799 - Train: 12.22% [497200/4070000] [122.2/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 12:28:14,292 - Train: 12.22% [497300/4070000] [122.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 12:29:14,795 - Train: 12.22% [497400/4070000] [122.2/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 12:30:15,384 - Train: 12.22% [497500/4070000] [122.2/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 12:31:16,200 - Train: 12.23% [497600/4070000] [122.3/1000.0] [batch_t 0.605 (0.608)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 12:32:16,673 - Train: 12.23% [497700/4070000] [122.3/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 12:33:17,186 - Train: 12.23% [497800/4070000] [122.3/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 12:34:17,670 - Train: 12.23% [497900/4070000] [122.3/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 12:35:18,147 - Train: 12.24% [498000/4070000] [122.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 12:36:18,630 - Train: 12.24% [498100/4070000] [122.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 12:37:19,116 - Train: 12.24% [498200/4070000] [122.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 12:38:19,682 - Train: 12.24% [498300/4070000] [122.4/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 12:39:20,187 - Train: 12.25% [498400/4070000] [122.5/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-06 12:40:20,657 - Train: 12.25% [498500/4070000] [122.5/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 12:41:21,170 - Train: 12.25% [498600/4070000] [122.5/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-06 12:42:21,652 - Train: 12.25% [498700/4070000] [122.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 12:43:22,118 - Train: 12.26% [498800/4070000] [122.6/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 12:44:23,174 - Train: 12.26% [498900/4070000] [122.6/1000.0] [batch_t 0.602 (0.610)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-06 12:45:23,669 - Train: 12.26% [499000/4070000] [122.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 12:46:24,183 - Train: 12.26% [499100/4070000] [122.6/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 12:47:24,682 - Train: 12.27% [499200/4070000] [122.7/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 12:48:25,148 - Train: 12.27% [499300/4070000] [122.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 12:49:25,642 - Train: 12.27% [499400/4070000] [122.7/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 12:50:26,145 - Train: 12.27% [499500/4070000] [122.7/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-06 12:51:26,721 - Train: 12.28% [499600/4070000] [122.8/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 12:52:27,191 - Train: 12.28% [499700/4070000] [122.8/1000.0] [batch_t 0.600 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-06 12:53:27,646 - Train: 12.28% [499800/4070000] [122.8/1000.0] [batch_t 0.589 (0.604)] [data_t 0.002] [optim_t 0.587] [lr 0.005000] +2024-04-06 12:54:28,145 - Train: 12.28% [499900/4070000] [122.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 12:55:28,650 - Train: 12.29% [500000/4070000] [122.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 12:56:29,142 - Train: 12.29% [500100/4070000] [122.9/1000.0] [batch_t 0.600 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-06 12:57:29,743 - Train: 12.29% [500200/4070000] [122.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 12:58:30,206 - Train: 12.29% [500300/4070000] [122.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 12:59:30,713 - Train: 12.29% [500400/4070000] [122.9/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 13:00:31,205 - Train: 12.30% [500500/4070000] [123.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 13:01:31,717 - Train: 12.30% [500600/4070000] [123.0/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 13:01:37,783 - ==> Total time: 4 days, 12:01:08 Eta: 32 days, 2:11:05 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-06 13:02:33,951 - Train: 12.30% [500700/4070000] [123.0/1000.0] [batch_t 0.604 (0.609)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 13:03:34,416 - Train: 12.30% [500800/4070000] [123.0/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 13:04:34,905 - Train: 12.31% [500900/4070000] [123.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 13:05:35,356 - Train: 12.31% [501000/4070000] [123.1/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 13:06:35,831 - Train: 12.31% [501100/4070000] [123.1/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 13:07:36,311 - Train: 12.31% [501200/4070000] [123.1/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 13:08:36,757 - Train: 12.32% [501300/4070000] [123.2/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 13:09:37,276 - Train: 12.32% [501400/4070000] [123.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 13:10:38,442 - Train: 12.32% [501500/4070000] [123.2/1000.0] [batch_t 0.604 (0.612)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 13:11:38,918 - Train: 12.32% [501600/4070000] [123.2/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 13:12:39,425 - Train: 12.33% [501700/4070000] [123.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 13:13:40,157 - Train: 12.33% [501800/4070000] [123.3/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 13:14:40,625 - Train: 12.33% [501900/4070000] [123.3/1000.0] [batch_t 0.597 (0.605)] [data_t 0.002] [optim_t 0.596] [lr 0.005000] +2024-04-06 13:15:41,219 - Train: 12.33% [502000/4070000] [123.3/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 13:16:42,361 - Train: 12.34% [502100/4070000] [123.4/1000.0] [batch_t 0.604 (0.611)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 13:17:43,039 - Train: 12.34% [502200/4070000] [123.4/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 13:18:43,758 - Train: 12.34% [502300/4070000] [123.4/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 13:19:44,257 - Train: 12.34% [502400/4070000] [123.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 13:20:44,729 - Train: 12.35% [502500/4070000] [123.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 13:21:45,199 - Train: 12.35% [502600/4070000] [123.5/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 13:22:45,675 - Train: 12.35% [502700/4070000] [123.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 13:23:46,245 - Train: 12.35% [502800/4070000] [123.5/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 13:24:46,776 - Train: 12.36% [502900/4070000] [123.6/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-06 13:25:47,272 - Train: 12.36% [503000/4070000] [123.6/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 13:26:47,740 - Train: 12.36% [503100/4070000] [123.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 13:27:48,223 - Train: 12.36% [503200/4070000] [123.6/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 13:28:48,869 - Train: 12.37% [503300/4070000] [123.7/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 13:29:49,409 - Train: 12.37% [503400/4070000] [123.7/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-06 13:30:49,914 - Train: 12.37% [503500/4070000] [123.7/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 13:31:50,705 - Train: 12.37% [503600/4070000] [123.7/1000.0] [batch_t 0.606 (0.608)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 13:32:51,253 - Train: 12.38% [503700/4070000] [123.8/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 13:33:51,751 - Train: 12.38% [503800/4070000] [123.8/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-06 13:34:52,319 - Train: 12.38% [503900/4070000] [123.8/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 13:35:52,900 - Train: 12.38% [504000/4070000] [123.8/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 13:36:53,536 - Train: 12.39% [504100/4070000] [123.9/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 13:37:54,080 - Train: 12.39% [504200/4070000] [123.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 13:38:54,587 - Train: 12.39% [504300/4070000] [123.9/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 13:39:55,093 - Train: 12.39% [504400/4070000] [123.9/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 13:40:55,628 - Train: 12.40% [504500/4070000] [124.0/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 13:41:56,350 - Train: 12.40% [504600/4070000] [124.0/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 13:42:44,781 - ==> Total time: 4 days, 12:42:15 Eta: 31 days, 23:56:36 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-06 13:42:58,545 - Train: 12.40% [504700/4070000] [124.0/1000.0] [batch_t 0.605 (0.618)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 13:43:59,067 - Train: 12.40% [504800/4070000] [124.0/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 13:44:59,553 - Train: 12.41% [504900/4070000] [124.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 13:46:00,078 - Train: 12.41% [505000/4070000] [124.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 13:47:00,628 - Train: 12.41% [505100/4070000] [124.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 13:48:01,095 - Train: 12.41% [505200/4070000] [124.1/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-06 13:49:01,827 - Train: 12.42% [505300/4070000] [124.2/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 13:50:02,328 - Train: 12.42% [505400/4070000] [124.2/1000.0] [batch_t 0.611 (0.605)] [data_t 0.002] [optim_t 0.609] [lr 0.005000] +2024-04-06 13:51:02,820 - Train: 12.42% [505500/4070000] [124.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 13:52:03,331 - Train: 12.42% [505600/4070000] [124.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 13:53:03,824 - Train: 12.43% [505700/4070000] [124.3/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 13:54:04,417 - Train: 12.43% [505800/4070000] [124.3/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 13:55:04,936 - Train: 12.43% [505900/4070000] [124.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 13:56:05,432 - Train: 12.43% [506000/4070000] [124.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 13:57:05,946 - Train: 12.43% [506100/4070000] [124.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 13:58:06,452 - Train: 12.44% [506200/4070000] [124.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 13:59:06,980 - Train: 12.44% [506300/4070000] [124.4/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 14:00:07,500 - Train: 12.44% [506400/4070000] [124.4/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 14:01:08,015 - Train: 12.44% [506500/4070000] [124.4/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 14:02:08,548 - Train: 12.45% [506600/4070000] [124.5/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-06 14:03:09,035 - Train: 12.45% [506700/4070000] [124.5/1000.0] [batch_t 0.612 (0.605)] [data_t 0.002] [optim_t 0.610] [lr 0.005000] +2024-04-06 14:04:09,515 - Train: 12.45% [506800/4070000] [124.5/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 14:05:10,045 - Train: 12.45% [506900/4070000] [124.5/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 14:06:10,507 - Train: 12.46% [507000/4070000] [124.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 14:07:11,094 - Train: 12.46% [507100/4070000] [124.6/1000.0] [batch_t 0.610 (0.606)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-06 14:08:11,597 - Train: 12.46% [507200/4070000] [124.6/1000.0] [batch_t 0.600 (0.605)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-06 14:09:12,132 - Train: 12.46% [507300/4070000] [124.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 14:10:12,661 - Train: 12.47% [507400/4070000] [124.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 14:11:13,120 - Train: 12.47% [507500/4070000] [124.7/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 14:12:13,655 - Train: 12.47% [507600/4070000] [124.7/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 14:13:14,118 - Train: 12.47% [507700/4070000] [124.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 14:14:14,650 - Train: 12.48% [507800/4070000] [124.8/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 14:15:15,252 - Train: 12.48% [507900/4070000] [124.8/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 14:16:15,726 - Train: 12.48% [508000/4070000] [124.8/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 14:17:16,729 - Train: 12.48% [508100/4070000] [124.8/1000.0] [batch_t 0.606 (0.610)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 14:18:17,947 - Train: 12.49% [508200/4070000] [124.9/1000.0] [batch_t 0.605 (0.612)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 14:19:18,423 - Train: 12.49% [508300/4070000] [124.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 14:20:18,972 - Train: 12.49% [508400/4070000] [124.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 14:21:19,511 - Train: 12.49% [508500/4070000] [124.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 14:22:20,014 - Train: 12.50% [508600/4070000] [125.0/1000.0] [batch_t 0.600 (0.605)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-06 14:23:20,544 - Train: 12.50% [508700/4070000] [125.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 14:23:50,791 - ==> Total time: 4 days, 13:23:21 Eta: 31 days, 21:43:31 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-06 14:24:22,623 - Train: 12.50% [508800/4070000] [125.0/1000.0] [batch_t 0.602 (0.608)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 14:25:23,586 - Train: 12.50% [508900/4070000] [125.0/1000.0] [batch_t 0.603 (0.610)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 14:26:24,051 - Train: 12.51% [509000/4070000] [125.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 14:27:24,657 - Train: 12.51% [509100/4070000] [125.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 14:28:25,119 - Train: 12.51% [509200/4070000] [125.1/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-06 14:29:25,575 - Train: 12.51% [509300/4070000] [125.1/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 14:30:26,926 - Train: 12.52% [509400/4070000] [125.2/1000.0] [batch_t 0.604 (0.613)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 14:31:27,479 - Train: 12.52% [509500/4070000] [125.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 14:32:28,050 - Train: 12.52% [509600/4070000] [125.2/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 14:33:28,562 - Train: 12.52% [509700/4070000] [125.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 14:34:29,053 - Train: 12.53% [509800/4070000] [125.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 14:35:29,549 - Train: 12.53% [509900/4070000] [125.3/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-06 14:36:30,008 - Train: 12.53% [510000/4070000] [125.3/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 14:37:30,531 - Train: 12.53% [510100/4070000] [125.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 14:38:31,159 - Train: 12.54% [510200/4070000] [125.4/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 14:39:31,631 - Train: 12.54% [510300/4070000] [125.4/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 14:40:32,103 - Train: 12.54% [510400/4070000] [125.4/1000.0] [batch_t 0.598 (0.605)] [data_t 0.002] [optim_t 0.596] [lr 0.005000] +2024-04-06 14:41:32,559 - Train: 12.54% [510500/4070000] [125.4/1000.0] [batch_t 0.609 (0.604)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-06 14:42:33,035 - Train: 12.55% [510600/4070000] [125.5/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 14:43:33,459 - Train: 12.55% [510700/4070000] [125.5/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 14:44:33,956 - Train: 12.55% [510800/4070000] [125.5/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 14:45:34,472 - Train: 12.55% [510900/4070000] [125.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 14:46:35,093 - Train: 12.56% [511000/4070000] [125.6/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 14:47:35,571 - Train: 12.56% [511100/4070000] [125.6/1000.0] [batch_t 0.600 (0.605)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-06 14:48:36,086 - Train: 12.56% [511200/4070000] [125.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 14:49:36,574 - Train: 12.56% [511300/4070000] [125.6/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-06 14:50:37,119 - Train: 12.57% [511400/4070000] [125.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 14:51:37,753 - Train: 12.57% [511500/4070000] [125.7/1000.0] [batch_t 0.599 (0.606)] [data_t 0.002] [optim_t 0.597] [lr 0.005000] +2024-04-06 14:52:38,239 - Train: 12.57% [511600/4070000] [125.7/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 14:53:38,724 - Train: 12.57% [511700/4070000] [125.7/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 14:54:39,477 - Train: 12.57% [511800/4070000] [125.7/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 14:55:39,997 - Train: 12.58% [511900/4070000] [125.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 14:56:40,554 - Train: 12.58% [512000/4070000] [125.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 14:57:41,062 - Train: 12.58% [512100/4070000] [125.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 14:58:42,025 - Train: 12.58% [512200/4070000] [125.8/1000.0] [batch_t 0.605 (0.610)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 14:59:42,529 - Train: 12.59% [512300/4070000] [125.9/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 15:00:43,146 - Train: 12.59% [512400/4070000] [125.9/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 15:01:43,608 - Train: 12.59% [512500/4070000] [125.9/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-06 15:02:44,120 - Train: 12.59% [512600/4070000] [125.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 15:03:44,661 - Train: 12.60% [512700/4070000] [126.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 15:04:45,280 - Train: 12.60% [512800/4070000] [126.0/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 15:04:57,378 - ==> Total time: 4 days, 14:04:28 Eta: 31 days, 19:31:57 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-06 15:05:47,386 - Train: 12.60% [512900/4070000] [126.0/1000.0] [batch_t 0.601 (0.607)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-06 15:06:47,888 - Train: 12.60% [513000/4070000] [126.0/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 15:07:48,365 - Train: 12.61% [513100/4070000] [126.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 15:08:48,902 - Train: 12.61% [513200/4070000] [126.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 15:09:49,404 - Train: 12.61% [513300/4070000] [126.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 15:10:55,276 - Train: 12.61% [513400/4070000] [126.1/1000.0] [batch_t 0.601 (0.659)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-06 15:12:15,981 - Train: 12.62% [513500/4070000] [126.2/1000.0] [batch_t 0.607 (0.807)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 15:13:26,859 - Train: 12.62% [513600/4070000] [126.2/1000.0] [batch_t 0.606 (0.709)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 15:14:36,735 - Train: 12.62% [513700/4070000] [126.2/1000.0] [batch_t 0.607 (0.699)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 15:16:19,646 - Train: 12.62% [513800/4070000] [126.2/1000.0] [batch_t 1.236 (1.029)] [data_t 0.637] [optim_t 0.599] [lr 0.005000] +2024-04-06 15:17:55,592 - Train: 12.63% [513900/4070000] [126.3/1000.0] [batch_t 0.605 (0.959)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 15:19:05,858 - Train: 12.63% [514000/4070000] [126.3/1000.0] [batch_t 3.697 (0.703)] [data_t 3.095] [optim_t 0.603] [lr 0.005000] +2024-04-06 15:20:20,012 - Train: 12.63% [514100/4070000] [126.3/1000.0] [batch_t 0.604 (0.741)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 15:21:29,410 - Train: 12.63% [514200/4070000] [126.3/1000.0] [batch_t 0.606 (0.694)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 15:22:37,116 - Train: 12.64% [514300/4070000] [126.4/1000.0] [batch_t 0.609 (0.677)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-06 15:23:41,528 - Train: 12.64% [514400/4070000] [126.4/1000.0] [batch_t 0.609 (0.644)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 15:24:52,786 - Train: 12.64% [514500/4070000] [126.4/1000.0] [batch_t 0.603 (0.712)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 15:25:54,335 - Train: 12.64% [514600/4070000] [126.4/1000.0] [batch_t 0.607 (0.615)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 15:26:55,763 - Train: 12.65% [514700/4070000] [126.5/1000.0] [batch_t 0.606 (0.614)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 15:28:01,508 - Train: 12.65% [514800/4070000] [126.5/1000.0] [batch_t 0.605 (0.657)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 15:29:09,777 - Train: 12.65% [514900/4070000] [126.5/1000.0] [batch_t 0.603 (0.683)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 15:30:10,471 - Train: 12.65% [515000/4070000] [126.5/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 15:31:10,948 - Train: 12.66% [515100/4070000] [126.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 15:32:11,442 - Train: 12.66% [515200/4070000] [126.6/1000.0] [batch_t 0.599 (0.605)] [data_t 0.002] [optim_t 0.597] [lr 0.005000] +2024-04-06 15:33:11,919 - Train: 12.66% [515300/4070000] [126.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 15:34:12,391 - Train: 12.66% [515400/4070000] [126.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 15:35:12,999 - Train: 12.67% [515500/4070000] [126.7/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 15:36:13,473 - Train: 12.67% [515600/4070000] [126.7/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-06 15:37:13,959 - Train: 12.67% [515700/4070000] [126.7/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 15:38:14,468 - Train: 12.67% [515800/4070000] [126.7/1000.0] [batch_t 0.611 (0.605)] [data_t 0.002] [optim_t 0.609] [lr 0.005000] +2024-04-06 15:39:14,954 - Train: 12.68% [515900/4070000] [126.8/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 15:40:15,505 - Train: 12.68% [516000/4070000] [126.8/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 15:41:15,986 - Train: 12.68% [516100/4070000] [126.8/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 15:42:30,299 - Train: 12.68% [516200/4070000] [126.8/1000.0] [batch_t 1.008 (0.743)] [data_t 0.403] [optim_t 0.605] [lr 0.005000] +2024-04-06 15:43:41,338 - Train: 12.69% [516300/4070000] [126.9/1000.0] [batch_t 0.604 (0.710)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 15:44:51,087 - Train: 12.69% [516400/4070000] [126.9/1000.0] [batch_t 0.606 (0.697)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 15:45:51,723 - Train: 12.69% [516500/4070000] [126.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 15:46:52,198 - Train: 12.69% [516600/4070000] [126.9/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-06 15:47:52,737 - Train: 12.70% [516700/4070000] [127.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 15:48:53,309 - Train: 12.70% [516800/4070000] [127.0/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 15:49:48,195 - ==> Total time: 4 days, 14:49:19 Eta: 31 days, 17:47:30 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-06 15:49:58,977 - Train: 12.70% [516900/4070000] [127.0/1000.0] [batch_t 0.606 (0.871)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 15:51:01,702 - Train: 12.70% [517000/4070000] [127.0/1000.0] [batch_t 0.606 (0.627)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 15:52:03,783 - Train: 12.71% [517100/4070000] [127.1/1000.0] [batch_t 0.606 (0.621)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 15:53:04,430 - Train: 12.71% [517200/4070000] [127.1/1000.0] [batch_t 0.602 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 15:54:04,968 - Train: 12.71% [517300/4070000] [127.1/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 15:55:12,976 - Train: 12.71% [517400/4070000] [127.1/1000.0] [batch_t 0.605 (0.680)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 15:56:15,348 - Train: 12.71% [517500/4070000] [127.1/1000.0] [batch_t 0.605 (0.624)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 15:57:17,511 - Train: 12.72% [517600/4070000] [127.2/1000.0] [batch_t 0.603 (0.622)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 15:58:22,903 - Train: 12.72% [517700/4070000] [127.2/1000.0] [batch_t 1.828 (0.654)] [data_t 1.223] [optim_t 0.604] [lr 0.005000] +2024-04-06 16:01:51,187 - Train: 12.72% [517800/4070000] [127.2/1000.0] [batch_t 0.815 (2.083)] [data_t 0.211] [optim_t 0.604] [lr 0.005000] +2024-04-06 16:02:56,249 - Train: 12.72% [517900/4070000] [127.2/1000.0] [batch_t 0.605 (0.651)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 16:03:56,703 - Train: 12.73% [518000/4070000] [127.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 16:04:57,194 - Train: 12.73% [518100/4070000] [127.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 16:05:57,689 - Train: 12.73% [518200/4070000] [127.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 16:06:58,177 - Train: 12.73% [518300/4070000] [127.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 16:07:59,220 - Train: 12.74% [518400/4070000] [127.4/1000.0] [batch_t 0.606 (0.610)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 16:08:59,707 - Train: 12.74% [518500/4070000] [127.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 16:10:00,280 - Train: 12.74% [518600/4070000] [127.4/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 16:11:00,774 - Train: 12.74% [518700/4070000] [127.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 16:12:01,254 - Train: 12.75% [518800/4070000] [127.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 16:13:01,733 - Train: 12.75% [518900/4070000] [127.5/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 16:14:02,263 - Train: 12.75% [519000/4070000] [127.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 16:15:02,917 - Train: 12.75% [519100/4070000] [127.5/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 16:16:03,380 - Train: 12.76% [519200/4070000] [127.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 16:17:03,848 - Train: 12.76% [519300/4070000] [127.6/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 16:18:04,316 - Train: 12.76% [519400/4070000] [127.6/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 16:19:04,781 - Train: 12.76% [519500/4070000] [127.6/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 16:20:05,248 - Train: 12.77% [519600/4070000] [127.7/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 16:21:05,781 - Train: 12.77% [519700/4070000] [127.7/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 16:22:06,692 - Train: 12.77% [519800/4070000] [127.7/1000.0] [batch_t 0.604 (0.609)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 16:23:08,258 - Train: 12.77% [519900/4070000] [127.7/1000.0] [batch_t 0.602 (0.616)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-06 16:24:08,768 - Train: 12.78% [520000/4070000] [127.8/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 16:25:09,266 - Train: 12.78% [520100/4070000] [127.8/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 16:26:09,767 - Train: 12.78% [520200/4070000] [127.8/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 16:27:10,269 - Train: 12.78% [520300/4070000] [127.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 16:28:10,793 - Train: 12.79% [520400/4070000] [127.9/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 16:29:11,319 - Train: 12.79% [520500/4070000] [127.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 16:30:11,910 - Train: 12.79% [520600/4070000] [127.9/1000.0] [batch_t 0.610 (0.606)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-06 16:31:12,394 - Train: 12.79% [520700/4070000] [127.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 16:32:12,853 - Train: 12.80% [520800/4070000] [128.0/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 16:33:13,646 - Train: 12.80% [520900/4070000] [128.0/1000.0] [batch_t 0.605 (0.608)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 16:33:49,948 - ==> Total time: 4 days, 15:33:20 Eta: 31 days, 15:58:25 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-06 16:34:15,895 - Train: 12.80% [521000/4070000] [128.0/1000.0] [batch_t 0.605 (0.610)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 16:35:16,553 - Train: 12.80% [521100/4070000] [128.0/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 16:36:17,037 - Train: 12.81% [521200/4070000] [128.1/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 16:37:17,571 - Train: 12.81% [521300/4070000] [128.1/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 16:38:18,069 - Train: 12.81% [521400/4070000] [128.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 16:39:18,584 - Train: 12.81% [521500/4070000] [128.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 16:40:19,084 - Train: 12.82% [521600/4070000] [128.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 16:41:19,632 - Train: 12.82% [521700/4070000] [128.2/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-06 16:42:20,111 - Train: 12.82% [521800/4070000] [128.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 16:43:20,680 - Train: 12.82% [521900/4070000] [128.2/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 16:44:21,175 - Train: 12.83% [522000/4070000] [128.3/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 16:45:21,676 - Train: 12.83% [522100/4070000] [128.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 16:46:22,135 - Train: 12.83% [522200/4070000] [128.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 16:47:22,614 - Train: 12.83% [522300/4070000] [128.3/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 16:48:23,214 - Train: 12.84% [522400/4070000] [128.4/1000.0] [batch_t 0.611 (0.606)] [data_t 0.002] [optim_t 0.609] [lr 0.005000] +2024-04-06 16:49:23,727 - Train: 12.84% [522500/4070000] [128.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 16:50:24,207 - Train: 12.84% [522600/4070000] [128.4/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 16:51:24,723 - Train: 12.84% [522700/4070000] [128.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 16:52:25,185 - Train: 12.85% [522800/4070000] [128.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 16:53:25,696 - Train: 12.85% [522900/4070000] [128.5/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 16:54:26,193 - Train: 12.85% [523000/4070000] [128.5/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 16:55:26,717 - Train: 12.85% [523100/4070000] [128.5/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 16:56:27,285 - Train: 12.86% [523200/4070000] [128.6/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 16:57:27,819 - Train: 12.86% [523300/4070000] [128.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 16:58:28,287 - Train: 12.86% [523400/4070000] [128.6/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 16:59:28,798 - Train: 12.86% [523500/4070000] [128.6/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-06 17:00:29,512 - Train: 12.86% [523600/4070000] [128.6/1000.0] [batch_t 0.608 (0.607)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 17:01:30,072 - Train: 12.87% [523700/4070000] [128.7/1000.0] [batch_t 0.610 (0.606)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-06 17:02:30,596 - Train: 12.87% [523800/4070000] [128.7/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 17:03:31,232 - Train: 12.87% [523900/4070000] [128.7/1000.0] [batch_t 0.601 (0.606)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-06 17:04:31,750 - Train: 12.87% [524000/4070000] [128.7/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 17:05:32,222 - Train: 12.88% [524100/4070000] [128.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 17:06:32,744 - Train: 12.88% [524200/4070000] [128.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 17:07:33,236 - Train: 12.88% [524300/4070000] [128.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 17:08:33,714 - Train: 12.88% [524400/4070000] [128.8/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 17:09:34,301 - Train: 12.89% [524500/4070000] [128.9/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 17:10:36,010 - Train: 12.89% [524600/4070000] [128.9/1000.0] [batch_t 0.607 (0.617)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 17:11:37,417 - Train: 12.89% [524700/4070000] [128.9/1000.0] [batch_t 0.604 (0.614)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 17:12:37,914 - Train: 12.89% [524800/4070000] [128.9/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 17:13:38,377 - Train: 12.90% [524900/4070000] [129.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 17:14:38,963 - Train: 12.90% [525000/4070000] [129.0/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 17:14:57,116 - ==> Total time: 4 days, 16:14:27 Eta: 31 days, 13:50:41 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-06 17:15:41,272 - Train: 12.90% [525100/4070000] [129.0/1000.0] [batch_t 0.608 (0.610)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 17:16:41,799 - Train: 12.90% [525200/4070000] [129.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 17:17:42,290 - Train: 12.91% [525300/4070000] [129.1/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 17:18:42,790 - Train: 12.91% [525400/4070000] [129.1/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 17:19:43,739 - Train: 12.91% [525500/4070000] [129.1/1000.0] [batch_t 0.605 (0.609)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 17:20:44,228 - Train: 12.91% [525600/4070000] [129.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 17:21:44,876 - Train: 12.92% [525700/4070000] [129.2/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 17:22:45,420 - Train: 12.92% [525800/4070000] [129.2/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-06 17:23:45,902 - Train: 12.92% [525900/4070000] [129.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 17:24:46,513 - Train: 12.92% [526000/4070000] [129.2/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 17:25:47,728 - Train: 12.93% [526100/4070000] [129.3/1000.0] [batch_t 0.605 (0.612)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 17:26:48,359 - Train: 12.93% [526200/4070000] [129.3/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 17:27:48,833 - Train: 12.93% [526300/4070000] [129.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 17:28:50,662 - Train: 12.93% [526400/4070000] [129.3/1000.0] [batch_t 0.605 (0.618)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 17:29:54,806 - Train: 12.94% [526500/4070000] [129.4/1000.0] [batch_t 0.606 (0.641)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 17:30:55,344 - Train: 12.94% [526600/4070000] [129.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 17:31:55,899 - Train: 12.94% [526700/4070000] [129.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 17:32:56,902 - Train: 12.94% [526800/4070000] [129.4/1000.0] [batch_t 0.607 (0.610)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 17:33:57,564 - Train: 12.95% [526900/4070000] [129.5/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 17:34:58,061 - Train: 12.95% [527000/4070000] [129.5/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 17:35:58,595 - Train: 12.95% [527100/4070000] [129.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 17:36:59,150 - Train: 12.95% [527200/4070000] [129.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 17:37:59,677 - Train: 12.96% [527300/4070000] [129.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 17:39:00,164 - Train: 12.96% [527400/4070000] [129.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 17:40:00,671 - Train: 12.96% [527500/4070000] [129.6/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 17:41:01,303 - Train: 12.96% [527600/4070000] [129.6/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 17:42:01,808 - Train: 12.97% [527700/4070000] [129.7/1000.0] [batch_t 0.599 (0.605)] [data_t 0.002] [optim_t 0.597] [lr 0.005000] +2024-04-06 17:43:02,326 - Train: 12.97% [527800/4070000] [129.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 17:44:02,832 - Train: 12.97% [527900/4070000] [129.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 17:45:03,361 - Train: 12.97% [528000/4070000] [129.7/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 17:46:03,853 - Train: 12.98% [528100/4070000] [129.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 17:47:04,955 - Train: 12.98% [528200/4070000] [129.8/1000.0] [batch_t 0.607 (0.611)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 17:48:06,145 - Train: 12.98% [528300/4070000] [129.8/1000.0] [batch_t 0.605 (0.612)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 17:49:06,679 - Train: 12.98% [528400/4070000] [129.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 17:50:07,178 - Train: 12.99% [528500/4070000] [129.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 17:51:08,865 - Train: 12.99% [528600/4070000] [129.9/1000.0] [batch_t 1.773 (0.617)] [data_t 1.171] [optim_t 0.603] [lr 0.005000] +2024-04-06 17:52:09,305 - Train: 12.99% [528700/4070000] [129.9/1000.0] [batch_t 0.599 (0.604)] [data_t 0.002] [optim_t 0.597] [lr 0.005000] +2024-04-06 17:53:09,797 - Train: 12.99% [528800/4070000] [129.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 17:54:11,087 - Train: 13.00% [528900/4070000] [130.0/1000.0] [batch_t 0.606 (0.613)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 17:55:12,708 - Train: 13.00% [529000/4070000] [130.0/1000.0] [batch_t 0.604 (0.616)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 17:56:13,203 - Train: 13.00% [529100/4070000] [130.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 17:56:13,207 - ==> Total time: 4 days, 16:55:44 Eta: 31 days, 11:45:17 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-06 17:57:17,233 - Train: 13.00% [529200/4070000] [130.0/1000.0] [batch_t 0.605 (0.615)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 17:58:17,712 - Train: 13.00% [529300/4070000] [130.0/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-06 17:59:18,352 - Train: 13.01% [529400/4070000] [130.1/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 18:00:18,814 - Train: 13.01% [529500/4070000] [130.1/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 18:01:19,320 - Train: 13.01% [529600/4070000] [130.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 18:02:19,831 - Train: 13.01% [529700/4070000] [130.1/1000.0] [batch_t 0.598 (0.605)] [data_t 0.002] [optim_t 0.596] [lr 0.005000] +2024-04-06 18:03:20,281 - Train: 13.02% [529800/4070000] [130.2/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 18:04:20,792 - Train: 13.02% [529900/4070000] [130.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 18:05:21,302 - Train: 13.02% [530000/4070000] [130.2/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 18:06:21,966 - Train: 13.02% [530100/4070000] [130.2/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 18:07:22,472 - Train: 13.03% [530200/4070000] [130.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 18:08:22,953 - Train: 13.03% [530300/4070000] [130.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 18:09:23,445 - Train: 13.03% [530400/4070000] [130.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 18:10:23,924 - Train: 13.03% [530500/4070000] [130.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 18:11:24,420 - Train: 13.04% [530600/4070000] [130.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 18:12:24,922 - Train: 13.04% [530700/4070000] [130.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 18:13:25,521 - Train: 13.04% [530800/4070000] [130.4/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 18:14:26,044 - Train: 13.04% [530900/4070000] [130.4/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-06 18:15:26,507 - Train: 13.05% [531000/4070000] [130.5/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 18:16:27,011 - Train: 13.05% [531100/4070000] [130.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 18:17:27,524 - Train: 13.05% [531200/4070000] [130.5/1000.0] [batch_t 0.599 (0.605)] [data_t 0.002] [optim_t 0.597] [lr 0.005000] +2024-04-06 18:18:28,134 - Train: 13.05% [531300/4070000] [130.5/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 18:19:28,624 - Train: 13.06% [531400/4070000] [130.6/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-06 18:20:29,094 - Train: 13.06% [531500/4070000] [130.6/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 18:21:29,549 - Train: 13.06% [531600/4070000] [130.6/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 18:22:30,054 - Train: 13.06% [531700/4070000] [130.6/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 18:23:30,545 - Train: 13.07% [531800/4070000] [130.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 18:24:31,070 - Train: 13.07% [531900/4070000] [130.7/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 18:25:31,573 - Train: 13.07% [532000/4070000] [130.7/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 18:26:32,118 - Train: 13.07% [532100/4070000] [130.7/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-06 18:27:33,031 - Train: 13.08% [532200/4070000] [130.8/1000.0] [batch_t 0.605 (0.609)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 18:28:33,516 - Train: 13.08% [532300/4070000] [130.8/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 18:29:33,990 - Train: 13.08% [532400/4070000] [130.8/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 18:30:34,497 - Train: 13.08% [532500/4070000] [130.8/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 18:31:35,126 - Train: 13.09% [532600/4070000] [130.9/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 18:32:35,619 - Train: 13.09% [532700/4070000] [130.9/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 18:33:36,088 - Train: 13.09% [532800/4070000] [130.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 18:34:36,577 - Train: 13.09% [532900/4070000] [130.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 18:35:37,087 - Train: 13.10% [533000/4070000] [131.0/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 18:36:37,590 - Train: 13.10% [533100/4070000] [131.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 18:37:19,936 - ==> Total time: 4 days, 17:36:50 Eta: 31 days, 9:40:08 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-06 18:37:40,786 - Train: 13.10% [533200/4070000] [131.0/1000.0] [batch_t 0.606 (0.632)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 18:38:41,416 - Train: 13.10% [533300/4070000] [131.0/1000.0] [batch_t 0.609 (0.606)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-06 18:39:41,931 - Train: 13.11% [533400/4070000] [131.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 18:40:42,408 - Train: 13.11% [533500/4070000] [131.1/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 18:41:42,924 - Train: 13.11% [533600/4070000] [131.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 18:42:43,433 - Train: 13.11% [533700/4070000] [131.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 18:43:43,967 - Train: 13.12% [533800/4070000] [131.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 18:44:44,451 - Train: 13.12% [533900/4070000] [131.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 18:45:45,011 - Train: 13.12% [534000/4070000] [131.2/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 18:46:45,526 - Train: 13.12% [534100/4070000] [131.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 18:47:46,030 - Train: 13.13% [534200/4070000] [131.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 18:48:46,502 - Train: 13.13% [534300/4070000] [131.3/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 18:49:55,156 - Train: 13.13% [534400/4070000] [131.3/1000.0] [batch_t 0.606 (0.686)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 18:50:59,530 - Train: 13.13% [534500/4070000] [131.3/1000.0] [batch_t 0.604 (0.644)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 18:54:08,519 - Train: 13.14% [534600/4070000] [131.4/1000.0] [batch_t 0.603 (1.890)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 18:56:14,703 - Train: 13.14% [534700/4070000] [131.4/1000.0] [batch_t 0.605 (1.262)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 18:58:48,968 - Train: 13.14% [534800/4070000] [131.4/1000.0] [batch_t 0.605 (1.543)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 19:01:33,220 - Train: 13.14% [534900/4070000] [131.4/1000.0] [batch_t 0.604 (1.642)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 19:05:22,199 - Train: 13.14% [535000/4070000] [131.4/1000.0] [batch_t 3.583 (2.290)] [data_t 2.983] [optim_t 0.600] [lr 0.005000] +2024-04-06 19:09:29,557 - Train: 13.15% [535100/4070000] [131.5/1000.0] [batch_t 5.671 (2.473)] [data_t 5.068] [optim_t 0.602] [lr 0.005000] +2024-04-06 19:10:54,621 - Train: 13.15% [535200/4070000] [131.5/1000.0] [batch_t 0.608 (0.851)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 19:12:02,938 - Train: 13.15% [535300/4070000] [131.5/1000.0] [batch_t 0.605 (0.683)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 19:13:08,237 - Train: 13.15% [535400/4070000] [131.5/1000.0] [batch_t 0.603 (0.653)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 19:14:09,017 - Train: 13.16% [535500/4070000] [131.6/1000.0] [batch_t 0.605 (0.608)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 19:15:09,494 - Train: 13.16% [535600/4070000] [131.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 19:16:12,310 - Train: 13.16% [535700/4070000] [131.6/1000.0] [batch_t 0.604 (0.628)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 19:17:26,654 - Train: 13.16% [535800/4070000] [131.6/1000.0] [batch_t 1.975 (0.743)] [data_t 1.375] [optim_t 0.600] [lr 0.005000] +2024-04-06 19:18:53,708 - Train: 13.17% [535900/4070000] [131.7/1000.0] [batch_t 0.606 (0.870)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 19:19:55,830 - Train: 13.17% [536000/4070000] [131.7/1000.0] [batch_t 0.606 (0.621)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 19:20:56,294 - Train: 13.17% [536100/4070000] [131.7/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 19:21:56,755 - Train: 13.17% [536200/4070000] [131.7/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 19:22:57,395 - Train: 13.18% [536300/4070000] [131.8/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 19:23:57,884 - Train: 13.18% [536400/4070000] [131.8/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 19:24:58,369 - Train: 13.18% [536500/4070000] [131.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 19:25:58,843 - Train: 13.18% [536600/4070000] [131.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 19:26:59,367 - Train: 13.19% [536700/4070000] [131.9/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 19:28:05,415 - Train: 13.19% [536800/4070000] [131.9/1000.0] [batch_t 0.605 (0.660)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 19:29:05,929 - Train: 13.19% [536900/4070000] [131.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 19:30:06,544 - Train: 13.19% [537000/4070000] [131.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 19:31:07,033 - Train: 13.20% [537100/4070000] [132.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 19:32:07,525 - Train: 13.20% [537200/4070000] [132.0/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-06 19:32:31,736 - ==> Total time: 4 days, 18:32:02 Eta: 31 days, 9:08:53 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-06 19:33:09,654 - Train: 13.20% [537300/4070000] [132.0/1000.0] [batch_t 0.606 (0.608)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 19:34:10,209 - Train: 13.20% [537400/4070000] [132.0/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 19:35:10,847 - Train: 13.21% [537500/4070000] [132.1/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 19:36:11,339 - Train: 13.21% [537600/4070000] [132.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 19:37:11,826 - Train: 13.21% [537700/4070000] [132.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 19:38:12,739 - Train: 13.21% [537800/4070000] [132.1/1000.0] [batch_t 0.605 (0.609)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 19:39:13,228 - Train: 13.22% [537900/4070000] [132.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 19:40:13,727 - Train: 13.22% [538000/4070000] [132.2/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 19:41:14,193 - Train: 13.22% [538100/4070000] [132.2/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-06 19:42:14,750 - Train: 13.22% [538200/4070000] [132.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 19:43:15,238 - Train: 13.23% [538300/4070000] [132.3/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 19:44:15,733 - Train: 13.23% [538400/4070000] [132.3/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 19:45:16,263 - Train: 13.23% [538500/4070000] [132.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 19:46:30,105 - Train: 13.23% [538600/4070000] [132.3/1000.0] [batch_t 0.604 (0.738)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 19:47:43,023 - Train: 13.24% [538700/4070000] [132.4/1000.0] [batch_t 0.604 (0.729)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 19:48:43,570 - Train: 13.24% [538800/4070000] [132.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 19:49:44,050 - Train: 13.24% [538900/4070000] [132.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 19:50:44,965 - Train: 13.24% [539000/4070000] [132.4/1000.0] [batch_t 0.606 (0.609)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 19:51:45,731 - Train: 13.25% [539100/4070000] [132.5/1000.0] [batch_t 0.604 (0.608)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 19:52:46,189 - Train: 13.25% [539200/4070000] [132.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 19:53:47,250 - Train: 13.25% [539300/4070000] [132.5/1000.0] [batch_t 0.605 (0.611)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 19:54:47,751 - Train: 13.25% [539400/4070000] [132.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 19:55:48,220 - Train: 13.26% [539500/4070000] [132.6/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-06 19:56:48,759 - Train: 13.26% [539600/4070000] [132.6/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 19:57:49,231 - Train: 13.26% [539700/4070000] [132.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 19:58:49,683 - Train: 13.26% [539800/4070000] [132.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 19:59:50,143 - Train: 13.27% [539900/4070000] [132.7/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 20:00:50,664 - Train: 13.27% [540000/4070000] [132.7/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 20:01:51,252 - Train: 13.27% [540100/4070000] [132.7/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 20:02:51,746 - Train: 13.27% [540200/4070000] [132.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 20:03:52,275 - Train: 13.28% [540300/4070000] [132.8/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 20:04:52,769 - Train: 13.28% [540400/4070000] [132.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 20:05:53,235 - Train: 13.28% [540500/4070000] [132.8/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 20:06:53,707 - Train: 13.28% [540600/4070000] [132.8/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 20:07:54,246 - Train: 13.29% [540700/4070000] [132.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 20:08:54,813 - Train: 13.29% [540800/4070000] [132.9/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 20:09:55,306 - Train: 13.29% [540900/4070000] [132.9/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 20:10:55,765 - Train: 13.29% [541000/4070000] [132.9/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 20:11:56,253 - Train: 13.29% [541100/4070000] [132.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 20:12:56,773 - Train: 13.30% [541200/4070000] [133.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 20:13:57,267 - Train: 13.30% [541300/4070000] [133.0/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 20:14:03,321 - ==> Total time: 4 days, 19:13:34 Eta: 31 days, 7:08:09 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-06 20:15:00,715 - Train: 13.30% [541400/4070000] [133.0/1000.0] [batch_t 0.603 (0.610)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 20:16:01,350 - Train: 13.30% [541500/4070000] [133.0/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 20:17:01,869 - Train: 13.31% [541600/4070000] [133.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 20:18:02,366 - Train: 13.31% [541700/4070000] [133.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 20:19:02,841 - Train: 13.31% [541800/4070000] [133.1/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 20:20:03,314 - Train: 13.31% [541900/4070000] [133.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 20:21:03,786 - Train: 13.32% [542000/4070000] [133.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 20:22:04,367 - Train: 13.32% [542100/4070000] [133.2/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 20:23:04,859 - Train: 13.32% [542200/4070000] [133.2/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-06 20:24:05,362 - Train: 13.32% [542300/4070000] [133.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 20:25:05,847 - Train: 13.33% [542400/4070000] [133.3/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 20:26:06,336 - Train: 13.33% [542500/4070000] [133.3/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 20:27:06,837 - Train: 13.33% [542600/4070000] [133.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 20:28:07,318 - Train: 13.33% [542700/4070000] [133.3/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 20:29:07,896 - Train: 13.34% [542800/4070000] [133.4/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 20:30:08,397 - Train: 13.34% [542900/4070000] [133.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 20:31:08,935 - Train: 13.34% [543000/4070000] [133.4/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-06 20:32:09,454 - Train: 13.34% [543100/4070000] [133.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 20:33:11,790 - Train: 13.35% [543200/4070000] [133.5/1000.0] [batch_t 0.603 (0.623)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 20:34:13,362 - Train: 13.35% [543300/4070000] [133.5/1000.0] [batch_t 0.606 (0.616)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 20:35:13,989 - Train: 13.35% [543400/4070000] [133.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 20:36:14,470 - Train: 13.35% [543500/4070000] [133.5/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 20:37:14,937 - Train: 13.36% [543600/4070000] [133.6/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-06 20:38:15,440 - Train: 13.36% [543700/4070000] [133.6/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 20:39:15,935 - Train: 13.36% [543800/4070000] [133.6/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 20:40:16,442 - Train: 13.36% [543900/4070000] [133.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 20:41:17,026 - Train: 13.37% [544000/4070000] [133.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 20:42:17,516 - Train: 13.37% [544100/4070000] [133.7/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 20:43:18,003 - Train: 13.37% [544200/4070000] [133.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 20:44:19,146 - Train: 13.37% [544300/4070000] [133.7/1000.0] [batch_t 0.606 (0.611)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 20:45:19,659 - Train: 13.38% [544400/4070000] [133.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 20:46:20,171 - Train: 13.38% [544500/4070000] [133.8/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 20:47:20,659 - Train: 13.38% [544600/4070000] [133.8/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 20:48:21,236 - Train: 13.38% [544700/4070000] [133.8/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 20:49:21,725 - Train: 13.39% [544800/4070000] [133.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 20:50:22,204 - Train: 13.39% [544900/4070000] [133.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 20:51:22,639 - Train: 13.39% [545000/4070000] [133.9/1000.0] [batch_t 0.601 (0.604)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-06 20:52:23,089 - Train: 13.39% [545100/4070000] [133.9/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 20:53:23,577 - Train: 13.40% [545200/4070000] [134.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 20:54:24,130 - Train: 13.40% [545300/4070000] [134.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 20:55:12,529 - ==> Total time: 4 days, 19:54:43 Eta: 31 days, 5:06:11 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-06 20:55:26,331 - Train: 13.40% [545400/4070000] [134.0/1000.0] [batch_t 0.602 (0.614)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-06 20:56:26,776 - Train: 13.40% [545500/4070000] [134.0/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 20:57:27,260 - Train: 13.41% [545600/4070000] [134.1/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 20:58:30,145 - Train: 13.41% [545700/4070000] [134.1/1000.0] [batch_t 0.605 (0.629)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 20:59:30,908 - Train: 13.41% [545800/4070000] [134.1/1000.0] [batch_t 0.606 (0.608)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 21:00:31,409 - Train: 13.41% [545900/4070000] [134.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 21:01:32,704 - Train: 13.42% [546000/4070000] [134.2/1000.0] [batch_t 0.604 (0.613)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 21:02:33,182 - Train: 13.42% [546100/4070000] [134.2/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 21:03:33,657 - Train: 13.42% [546200/4070000] [134.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 21:04:34,474 - Train: 13.42% [546300/4070000] [134.2/1000.0] [batch_t 0.605 (0.608)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 21:05:36,594 - Train: 13.43% [546400/4070000] [134.3/1000.0] [batch_t 0.605 (0.621)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 21:06:37,262 - Train: 13.43% [546500/4070000] [134.3/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 21:07:37,767 - Train: 13.43% [546600/4070000] [134.3/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 21:08:38,241 - Train: 13.43% [546700/4070000] [134.3/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-06 21:09:38,730 - Train: 13.43% [546800/4070000] [134.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 21:10:39,294 - Train: 13.44% [546900/4070000] [134.4/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 21:11:39,846 - Train: 13.44% [547000/4070000] [134.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 21:12:40,362 - Train: 13.44% [547100/4070000] [134.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 21:13:40,984 - Train: 13.44% [547200/4070000] [134.4/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 21:14:41,498 - Train: 13.45% [547300/4070000] [134.5/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 21:15:42,043 - Train: 13.45% [547400/4070000] [134.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 21:16:42,511 - Train: 13.45% [547500/4070000] [134.5/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 21:17:43,017 - Train: 13.45% [547600/4070000] [134.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 21:18:43,524 - Train: 13.46% [547700/4070000] [134.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 21:19:44,181 - Train: 13.46% [547800/4070000] [134.6/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 21:20:44,679 - Train: 13.46% [547900/4070000] [134.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 21:21:45,177 - Train: 13.46% [548000/4070000] [134.6/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 21:22:45,644 - Train: 13.47% [548100/4070000] [134.7/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-06 21:23:46,099 - Train: 13.47% [548200/4070000] [134.7/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 21:24:46,630 - Train: 13.47% [548300/4070000] [134.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 21:25:47,120 - Train: 13.47% [548400/4070000] [134.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 21:26:47,751 - Train: 13.48% [548500/4070000] [134.8/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 21:27:48,253 - Train: 13.48% [548600/4070000] [134.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 21:28:48,780 - Train: 13.48% [548700/4070000] [134.8/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 21:29:49,271 - Train: 13.48% [548800/4070000] [134.8/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-06 21:30:49,776 - Train: 13.49% [548900/4070000] [134.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 21:31:50,299 - Train: 13.49% [549000/4070000] [134.9/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 21:32:50,949 - Train: 13.49% [549100/4070000] [134.9/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 21:33:51,428 - Train: 13.49% [549200/4070000] [134.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 21:34:51,963 - Train: 13.50% [549300/4070000] [135.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 21:35:52,475 - Train: 13.50% [549400/4070000] [135.0/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 21:36:22,736 - ==> Total time: 4 days, 20:35:53 Eta: 31 days, 3:05:32 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-06 21:36:54,602 - Train: 13.50% [549500/4070000] [135.0/1000.0] [batch_t 0.603 (0.609)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 21:37:55,118 - Train: 13.50% [549600/4070000] [135.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 21:38:55,593 - Train: 13.51% [549700/4070000] [135.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 21:39:56,249 - Train: 13.51% [549800/4070000] [135.1/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 21:40:56,743 - Train: 13.51% [549900/4070000] [135.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 21:41:57,231 - Train: 13.51% [550000/4070000] [135.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.003] [optim_t 0.601] [lr 0.005000] +2024-04-06 21:42:57,701 - Train: 13.52% [550100/4070000] [135.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 21:43:58,193 - Train: 13.52% [550200/4070000] [135.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 21:44:58,806 - Train: 13.52% [550300/4070000] [135.2/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 21:45:59,289 - Train: 13.52% [550400/4070000] [135.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 21:46:59,744 - Train: 13.53% [550500/4070000] [135.3/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 21:48:00,217 - Train: 13.53% [550600/4070000] [135.3/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-06 21:49:00,722 - Train: 13.53% [550700/4070000] [135.3/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 21:50:01,290 - Train: 13.53% [550800/4070000] [135.3/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 21:51:01,787 - Train: 13.54% [550900/4070000] [135.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 21:52:02,319 - Train: 13.54% [551000/4070000] [135.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 21:53:02,782 - Train: 13.54% [551100/4070000] [135.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 21:54:03,273 - Train: 13.54% [551200/4070000] [135.4/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 21:55:03,759 - Train: 13.55% [551300/4070000] [135.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 21:56:04,240 - Train: 13.55% [551400/4070000] [135.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 21:57:04,743 - Train: 13.55% [551500/4070000] [135.5/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 21:58:05,324 - Train: 13.55% [551600/4070000] [135.5/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 21:59:05,882 - Train: 13.56% [551700/4070000] [135.6/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 22:00:06,379 - Train: 13.56% [551800/4070000] [135.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 22:01:06,864 - Train: 13.56% [551900/4070000] [135.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 22:02:07,336 - Train: 13.56% [552000/4070000] [135.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 22:03:07,826 - Train: 13.57% [552100/4070000] [135.7/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 22:04:08,287 - Train: 13.57% [552200/4070000] [135.7/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 22:05:08,831 - Train: 13.57% [552300/4070000] [135.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 22:06:09,344 - Train: 13.57% [552400/4070000] [135.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 22:07:09,816 - Train: 13.57% [552500/4070000] [135.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 22:08:10,312 - Train: 13.58% [552600/4070000] [135.8/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-06 22:09:10,825 - Train: 13.58% [552700/4070000] [135.8/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 22:10:11,314 - Train: 13.58% [552800/4070000] [135.8/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 22:11:11,856 - Train: 13.58% [552900/4070000] [135.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 22:12:12,367 - Train: 13.59% [553000/4070000] [135.9/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 22:13:12,856 - Train: 13.59% [553100/4070000] [135.9/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 22:14:13,396 - Train: 13.59% [553200/4070000] [135.9/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 22:15:13,882 - Train: 13.59% [553300/4070000] [135.9/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 22:16:14,333 - Train: 13.60% [553400/4070000] [136.0/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 22:17:14,800 - Train: 13.60% [553500/4070000] [136.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 22:17:26,985 - ==> Total time: 4 days, 21:16:57 Eta: 31 days, 1:05:25 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-06 22:18:17,024 - Train: 13.60% [553600/4070000] [136.0/1000.0] [batch_t 0.605 (0.609)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 22:19:17,481 - Train: 13.60% [553700/4070000] [136.0/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 22:20:17,981 - Train: 13.61% [553800/4070000] [136.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 22:21:18,481 - Train: 13.61% [553900/4070000] [136.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 22:22:18,977 - Train: 13.61% [554000/4070000] [136.1/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 22:23:19,625 - Train: 13.61% [554100/4070000] [136.1/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 22:24:20,104 - Train: 13.62% [554200/4070000] [136.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 22:25:20,645 - Train: 13.62% [554300/4070000] [136.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 22:26:21,141 - Train: 13.62% [554400/4070000] [136.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 22:27:21,656 - Train: 13.62% [554500/4070000] [136.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 22:28:22,163 - Train: 13.63% [554600/4070000] [136.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 22:29:22,655 - Train: 13.63% [554700/4070000] [136.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 22:30:23,322 - Train: 13.63% [554800/4070000] [136.3/1000.0] [batch_t 0.609 (0.607)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-06 22:31:23,806 - Train: 13.63% [554900/4070000] [136.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 22:32:24,360 - Train: 13.64% [555000/4070000] [136.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 22:33:24,865 - Train: 13.64% [555100/4070000] [136.4/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-06 22:34:25,362 - Train: 13.64% [555200/4070000] [136.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 22:35:25,856 - Train: 13.64% [555300/4070000] [136.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 22:36:26,470 - Train: 13.65% [555400/4070000] [136.5/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 22:37:26,958 - Train: 13.65% [555500/4070000] [136.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 22:38:27,425 - Train: 13.65% [555600/4070000] [136.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 22:39:27,917 - Train: 13.65% [555700/4070000] [136.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 22:40:28,390 - Train: 13.66% [555800/4070000] [136.6/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-06 22:41:28,883 - Train: 13.66% [555900/4070000] [136.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 22:42:29,432 - Train: 13.66% [556000/4070000] [136.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 22:43:30,056 - Train: 13.66% [556100/4070000] [136.6/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 22:44:30,525 - Train: 13.67% [556200/4070000] [136.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 22:45:31,021 - Train: 13.67% [556300/4070000] [136.7/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 22:46:31,556 - Train: 13.67% [556400/4070000] [136.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 22:47:32,027 - Train: 13.67% [556500/4070000] [136.7/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 22:48:32,631 - Train: 13.68% [556600/4070000] [136.8/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 22:49:33,138 - Train: 13.68% [556700/4070000] [136.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 22:50:33,618 - Train: 13.68% [556800/4070000] [136.8/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 22:51:34,176 - Train: 13.68% [556900/4070000] [136.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 22:52:34,659 - Train: 13.69% [557000/4070000] [136.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 22:53:35,171 - Train: 13.69% [557100/4070000] [136.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 22:54:35,683 - Train: 13.69% [557200/4070000] [136.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 22:55:36,231 - Train: 13.69% [557300/4070000] [136.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 22:56:36,826 - Train: 13.70% [557400/4070000] [137.0/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 22:57:37,277 - Train: 13.70% [557500/4070000] [137.0/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 22:58:31,699 - ==> Total time: 4 days, 21:58:02 Eta: 30 days, 23:06:30 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-06 22:58:39,416 - Train: 13.70% [557600/4070000] [137.0/1000.0] [batch_t 0.606 (0.622)] [data_t 0.003] [optim_t 0.603] [lr 0.005000] +2024-04-06 22:59:39,928 - Train: 13.70% [557700/4070000] [137.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 23:00:40,461 - Train: 13.71% [557800/4070000] [137.1/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 23:01:40,931 - Train: 13.71% [557900/4070000] [137.1/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 23:02:41,540 - Train: 13.71% [558000/4070000] [137.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 23:03:42,025 - Train: 13.71% [558100/4070000] [137.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 23:04:42,504 - Train: 13.71% [558200/4070000] [137.1/1000.0] [batch_t 0.599 (0.605)] [data_t 0.002] [optim_t 0.597] [lr 0.005000] +2024-04-06 23:05:42,989 - Train: 13.72% [558300/4070000] [137.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 23:06:43,449 - Train: 13.72% [558400/4070000] [137.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 23:07:44,030 - Train: 13.72% [558500/4070000] [137.2/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 23:08:44,518 - Train: 13.72% [558600/4070000] [137.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 23:09:44,976 - Train: 13.73% [558700/4070000] [137.3/1000.0] [batch_t 0.596 (0.604)] [data_t 0.002] [optim_t 0.595] [lr 0.005000] +2024-04-06 23:10:45,460 - Train: 13.73% [558800/4070000] [137.3/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 23:11:45,966 - Train: 13.73% [558900/4070000] [137.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 23:12:46,453 - Train: 13.73% [559000/4070000] [137.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 23:13:46,959 - Train: 13.74% [559100/4070000] [137.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 23:14:47,488 - Train: 13.74% [559200/4070000] [137.4/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-06 23:15:48,096 - Train: 13.74% [559300/4070000] [137.4/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 23:16:48,603 - Train: 13.74% [559400/4070000] [137.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 23:17:49,077 - Train: 13.75% [559500/4070000] [137.5/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 23:18:49,562 - Train: 13.75% [559600/4070000] [137.5/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-06 23:19:50,021 - Train: 13.75% [559700/4070000] [137.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 23:20:50,656 - Train: 13.75% [559800/4070000] [137.5/1000.0] [batch_t 0.610 (0.606)] [data_t 0.002] [optim_t 0.609] [lr 0.005000] +2024-04-06 23:21:51,154 - Train: 13.76% [559900/4070000] [137.6/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 23:22:51,667 - Train: 13.76% [560000/4070000] [137.6/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 23:23:52,165 - Train: 13.76% [560100/4070000] [137.6/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 23:24:52,684 - Train: 13.76% [560200/4070000] [137.6/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 23:25:53,227 - Train: 13.77% [560300/4070000] [137.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 23:26:53,719 - Train: 13.77% [560400/4070000] [137.7/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 23:27:54,223 - Train: 13.77% [560500/4070000] [137.7/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 23:28:54,823 - Train: 13.77% [560600/4070000] [137.7/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 23:29:55,260 - Train: 13.78% [560700/4070000] [137.8/1000.0] [batch_t 0.608 (0.604)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 23:30:55,749 - Train: 13.78% [560800/4070000] [137.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 23:31:56,227 - Train: 13.78% [560900/4070000] [137.8/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 23:32:56,740 - Train: 13.78% [561000/4070000] [137.8/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 23:33:57,331 - Train: 13.79% [561100/4070000] [137.9/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 23:34:57,856 - Train: 13.79% [561200/4070000] [137.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 23:35:58,343 - Train: 13.79% [561300/4070000] [137.9/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 23:36:58,855 - Train: 13.79% [561400/4070000] [137.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 23:37:59,328 - Train: 13.80% [561500/4070000] [138.0/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 23:38:59,807 - Train: 13.80% [561600/4070000] [138.0/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 23:39:36,099 - ==> Total time: 4 days, 22:39:06 Eta: 30 days, 21:08:40 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-06 23:40:01,904 - Train: 13.80% [561700/4070000] [138.0/1000.0] [batch_t 0.605 (0.610)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 23:41:02,502 - Train: 13.80% [561800/4070000] [138.0/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 23:42:02,983 - Train: 13.81% [561900/4070000] [138.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 23:43:03,509 - Train: 13.81% [562000/4070000] [138.1/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 23:44:03,980 - Train: 13.81% [562100/4070000] [138.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 23:45:04,490 - Train: 13.81% [562200/4070000] [138.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 23:46:04,960 - Train: 13.82% [562300/4070000] [138.2/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 23:47:05,456 - Train: 13.82% [562400/4070000] [138.2/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 23:48:06,017 - Train: 13.82% [562500/4070000] [138.2/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 23:49:06,503 - Train: 13.82% [562600/4070000] [138.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 23:50:06,965 - Train: 13.83% [562700/4070000] [138.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 23:51:07,418 - Train: 13.83% [562800/4070000] [138.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 23:52:07,866 - Train: 13.83% [562900/4070000] [138.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 23:53:08,330 - Train: 13.83% [563000/4070000] [138.3/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 23:54:08,822 - Train: 13.84% [563100/4070000] [138.4/1000.0] [batch_t 0.611 (0.605)] [data_t 0.002] [optim_t 0.609] [lr 0.005000] +2024-04-06 23:55:09,468 - Train: 13.84% [563200/4070000] [138.4/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 23:56:10,035 - Train: 13.84% [563300/4070000] [138.4/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 23:57:10,552 - Train: 13.84% [563400/4070000] [138.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 23:58:11,025 - Train: 13.85% [563500/4070000] [138.5/1000.0] [batch_t 0.607 (0.605)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-06 23:59:11,515 - Train: 13.85% [563600/4070000] [138.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 00:00:12,096 - Train: 13.85% [563700/4070000] [138.5/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 00:01:12,599 - Train: 13.85% [563800/4070000] [138.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 00:02:13,088 - Train: 13.86% [563900/4070000] [138.6/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 00:03:13,572 - Train: 13.86% [564000/4070000] [138.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 00:04:14,070 - Train: 13.86% [564100/4070000] [138.6/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 00:05:14,554 - Train: 13.86% [564200/4070000] [138.6/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-07 00:06:14,989 - Train: 13.86% [564300/4070000] [138.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 00:07:15,551 - Train: 13.87% [564400/4070000] [138.7/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 00:08:16,013 - Train: 13.87% [564500/4070000] [138.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 00:09:16,537 - Train: 13.87% [564600/4070000] [138.7/1000.0] [batch_t 0.599 (0.605)] [data_t 0.002] [optim_t 0.597] [lr 0.005000] +2024-04-07 00:10:17,056 - Train: 13.87% [564700/4070000] [138.7/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 00:11:17,530 - Train: 13.88% [564800/4070000] [138.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 00:12:17,995 - Train: 13.88% [564900/4070000] [138.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 00:13:18,555 - Train: 13.88% [565000/4070000] [138.8/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 00:14:19,042 - Train: 13.88% [565100/4070000] [138.8/1000.0] [batch_t 0.600 (0.605)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-07 00:15:19,540 - Train: 13.89% [565200/4070000] [138.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 00:16:20,050 - Train: 13.89% [565300/4070000] [138.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 00:17:20,525 - Train: 13.89% [565400/4070000] [138.9/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-07 00:18:21,028 - Train: 13.89% [565500/4070000] [138.9/1000.0] [batch_t 0.598 (0.605)] [data_t 0.002] [optim_t 0.596] [lr 0.005000] +2024-04-07 00:19:21,486 - Train: 13.90% [565600/4070000] [139.0/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 00:20:22,022 - Train: 13.90% [565700/4070000] [139.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 00:20:40,180 - ==> Total time: 4 days, 23:20:11 Eta: 30 days, 19:11:55 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-07 00:21:24,193 - Train: 13.90% [565800/4070000] [139.0/1000.0] [batch_t 0.605 (0.608)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 00:22:24,676 - Train: 13.90% [565900/4070000] [139.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 00:23:25,145 - Train: 13.91% [566000/4070000] [139.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 00:24:25,642 - Train: 13.91% [566100/4070000] [139.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 00:25:26,140 - Train: 13.91% [566200/4070000] [139.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 00:26:26,617 - Train: 13.91% [566300/4070000] [139.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 00:27:27,260 - Train: 13.92% [566400/4070000] [139.2/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 00:28:27,748 - Train: 13.92% [566500/4070000] [139.2/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-07 00:29:28,212 - Train: 13.92% [566600/4070000] [139.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 00:30:28,720 - Train: 13.92% [566700/4070000] [139.2/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 00:31:29,220 - Train: 13.93% [566800/4070000] [139.3/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-07 00:32:29,730 - Train: 13.93% [566900/4070000] [139.3/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 00:33:30,235 - Train: 13.93% [567000/4070000] [139.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 00:34:30,728 - Train: 13.93% [567100/4070000] [139.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 00:35:31,271 - Train: 13.94% [567200/4070000] [139.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 00:36:31,761 - Train: 13.94% [567300/4070000] [139.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 00:37:32,226 - Train: 13.94% [567400/4070000] [139.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 00:38:32,706 - Train: 13.94% [567500/4070000] [139.4/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 00:39:33,201 - Train: 13.95% [567600/4070000] [139.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 00:40:33,784 - Train: 13.95% [567700/4070000] [139.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 00:41:34,250 - Train: 13.95% [567800/4070000] [139.5/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 00:42:34,717 - Train: 13.95% [567900/4070000] [139.5/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-07 00:43:35,196 - Train: 13.96% [568000/4070000] [139.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 00:44:35,661 - Train: 13.96% [568100/4070000] [139.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 00:45:36,153 - Train: 13.96% [568200/4070000] [139.6/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 00:46:36,613 - Train: 13.96% [568300/4070000] [139.6/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 00:47:37,192 - Train: 13.97% [568400/4070000] [139.7/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-07 00:48:37,696 - Train: 13.97% [568500/4070000] [139.7/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-07 00:49:38,172 - Train: 13.97% [568600/4070000] [139.7/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 00:50:38,636 - Train: 13.97% [568700/4070000] [139.7/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 00:51:39,111 - Train: 13.98% [568800/4070000] [139.8/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 00:52:39,601 - Train: 13.98% [568900/4070000] [139.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 00:53:40,217 - Train: 13.98% [569000/4070000] [139.8/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 00:54:40,747 - Train: 13.98% [569100/4070000] [139.8/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 00:55:41,233 - Train: 13.99% [569200/4070000] [139.9/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 00:56:41,743 - Train: 13.99% [569300/4070000] [139.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 00:57:42,229 - Train: 13.99% [569400/4070000] [139.9/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 00:58:42,698 - Train: 13.99% [569500/4070000] [139.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 00:59:43,170 - Train: 14.00% [569600/4070000] [140.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 01:00:43,757 - Train: 14.00% [569700/4070000] [140.0/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 01:01:44,234 - Train: 14.00% [569800/4070000] [140.0/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 01:01:44,240 - ==> Total time: 5 days, 0:01:15 Eta: 30 days, 17:16:15 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-07 01:02:46,398 - Train: 14.00% [569900/4070000] [140.0/1000.0] [batch_t 0.603 (0.607)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 01:03:46,867 - Train: 14.00% [570000/4070000] [140.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 01:04:47,345 - Train: 14.01% [570100/4070000] [140.1/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 01:05:47,989 - Train: 14.01% [570200/4070000] [140.1/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 01:06:48,497 - Train: 14.01% [570300/4070000] [140.1/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-07 01:07:48,995 - Train: 14.01% [570400/4070000] [140.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 01:08:49,483 - Train: 14.02% [570500/4070000] [140.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 01:09:49,970 - Train: 14.02% [570600/4070000] [140.2/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 01:10:50,477 - Train: 14.02% [570700/4070000] [140.2/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-07 01:11:51,002 - Train: 14.02% [570800/4070000] [140.2/1000.0] [batch_t 0.610 (0.605)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-07 01:12:51,517 - Train: 14.03% [570900/4070000] [140.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 01:13:52,086 - Train: 14.03% [571000/4070000] [140.3/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-07 01:14:52,570 - Train: 14.03% [571100/4070000] [140.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 01:15:53,014 - Train: 14.03% [571200/4070000] [140.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 01:16:53,468 - Train: 14.04% [571300/4070000] [140.4/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 01:17:53,957 - Train: 14.04% [571400/4070000] [140.4/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 01:18:54,558 - Train: 14.04% [571500/4070000] [140.4/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-07 01:19:55,055 - Train: 14.04% [571600/4070000] [140.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 01:20:55,542 - Train: 14.05% [571700/4070000] [140.5/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-07 01:21:56,053 - Train: 14.05% [571800/4070000] [140.5/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 01:22:56,533 - Train: 14.05% [571900/4070000] [140.5/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 01:23:57,053 - Train: 14.05% [572000/4070000] [140.5/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-07 01:24:57,570 - Train: 14.06% [572100/4070000] [140.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 01:25:58,022 - Train: 14.06% [572200/4070000] [140.6/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 01:26:58,646 - Train: 14.06% [572300/4070000] [140.6/1000.0] [batch_t 0.595 (0.606)] [data_t 0.002] [optim_t 0.593] [lr 0.005000] +2024-04-07 01:27:59,147 - Train: 14.06% [572400/4070000] [140.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 01:28:59,648 - Train: 14.07% [572500/4070000] [140.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 01:30:00,166 - Train: 14.07% [572600/4070000] [140.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 01:31:00,669 - Train: 14.07% [572700/4070000] [140.7/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-07 01:32:01,238 - Train: 14.07% [572800/4070000] [140.7/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 01:33:01,726 - Train: 14.08% [572900/4070000] [140.8/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-07 01:34:02,190 - Train: 14.08% [573000/4070000] [140.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 01:35:02,661 - Train: 14.08% [573100/4070000] [140.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 01:36:03,145 - Train: 14.08% [573200/4070000] [140.8/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 01:37:03,659 - Train: 14.09% [573300/4070000] [140.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 01:38:04,158 - Train: 14.09% [573400/4070000] [140.9/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 01:39:04,634 - Train: 14.09% [573500/4070000] [140.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 01:40:05,240 - Train: 14.09% [573600/4070000] [140.9/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 01:41:05,727 - Train: 14.10% [573700/4070000] [141.0/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-07 01:42:06,221 - Train: 14.10% [573800/4070000] [141.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 01:42:48,603 - ==> Total time: 5 days, 0:42:19 Eta: 30 days, 15:21:40 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-07 01:43:08,302 - Train: 14.10% [573900/4070000] [141.0/1000.0] [batch_t 0.606 (0.610)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 01:44:08,781 - Train: 14.10% [574000/4070000] [141.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 01:45:09,410 - Train: 14.11% [574100/4070000] [141.1/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 01:46:09,925 - Train: 14.11% [574200/4070000] [141.1/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 01:47:10,410 - Train: 14.11% [574300/4070000] [141.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 01:48:10,918 - Train: 14.11% [574400/4070000] [141.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.003] [optim_t 0.602] [lr 0.005000] +2024-04-07 01:49:11,414 - Train: 14.12% [574500/4070000] [141.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 01:50:11,898 - Train: 14.12% [574600/4070000] [141.2/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 01:51:12,406 - Train: 14.12% [574700/4070000] [141.2/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-07 01:52:12,924 - Train: 14.12% [574800/4070000] [141.2/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-07 01:53:13,391 - Train: 14.13% [574900/4070000] [141.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 01:54:13,856 - Train: 14.13% [575000/4070000] [141.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 01:55:14,351 - Train: 14.13% [575100/4070000] [141.3/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 01:56:14,823 - Train: 14.13% [575200/4070000] [141.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 01:57:15,428 - Train: 14.14% [575300/4070000] [141.4/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 01:58:15,878 - Train: 14.14% [575400/4070000] [141.4/1000.0] [batch_t 0.610 (0.604)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-07 01:59:16,337 - Train: 14.14% [575500/4070000] [141.4/1000.0] [batch_t 0.600 (0.604)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-07 02:00:16,828 - Train: 14.14% [575600/4070000] [141.4/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-07 02:01:17,346 - Train: 14.14% [575700/4070000] [141.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 02:02:17,826 - Train: 14.15% [575800/4070000] [141.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 02:03:18,297 - Train: 14.15% [575900/4070000] [141.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 02:04:18,766 - Train: 14.15% [576000/4070000] [141.5/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 02:05:19,333 - Train: 14.15% [576100/4070000] [141.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 02:06:19,820 - Train: 14.16% [576200/4070000] [141.6/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 02:07:20,333 - Train: 14.16% [576300/4070000] [141.6/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 02:08:20,839 - Train: 14.16% [576400/4070000] [141.6/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 02:09:21,321 - Train: 14.16% [576500/4070000] [141.6/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 02:10:22,043 - Train: 14.17% [576600/4070000] [141.7/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 02:11:22,574 - Train: 14.17% [576700/4070000] [141.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 02:12:23,109 - Train: 14.17% [576800/4070000] [141.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 02:13:23,623 - Train: 14.17% [576900/4070000] [141.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 02:14:24,116 - Train: 14.18% [577000/4070000] [141.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 02:15:24,658 - Train: 14.18% [577100/4070000] [141.8/1000.0] [batch_t 0.608 (0.605)] [data_t 0.003] [optim_t 0.605] [lr 0.005000] +2024-04-07 02:16:25,150 - Train: 14.18% [577200/4070000] [141.8/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 02:17:25,677 - Train: 14.18% [577300/4070000] [141.8/1000.0] [batch_t 0.611 (0.605)] [data_t 0.002] [optim_t 0.609] [lr 0.005000] +2024-04-07 02:18:26,321 - Train: 14.19% [577400/4070000] [141.9/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 02:19:26,847 - Train: 14.19% [577500/4070000] [141.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 02:20:27,336 - Train: 14.19% [577600/4070000] [141.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 02:21:27,802 - Train: 14.19% [577700/4070000] [141.9/1000.0] [batch_t 0.610 (0.605)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-07 02:22:28,297 - Train: 14.20% [577800/4070000] [142.0/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-07 02:23:28,896 - Train: 14.20% [577900/4070000] [142.0/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 02:23:53,110 - ==> Total time: 5 days, 1:23:23 Eta: 30 days, 13:28:08 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-07 02:24:30,948 - Train: 14.20% [578000/4070000] [142.0/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 02:25:31,451 - Train: 14.20% [578100/4070000] [142.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 02:26:31,922 - Train: 14.21% [578200/4070000] [142.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 02:27:32,429 - Train: 14.21% [578300/4070000] [142.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 02:28:32,958 - Train: 14.21% [578400/4070000] [142.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 02:29:33,458 - Train: 14.21% [578500/4070000] [142.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 02:30:34,043 - Train: 14.22% [578600/4070000] [142.2/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 02:31:34,526 - Train: 14.22% [578700/4070000] [142.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 02:32:35,045 - Train: 14.22% [578800/4070000] [142.2/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 02:33:35,525 - Train: 14.22% [578900/4070000] [142.2/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-07 02:34:36,004 - Train: 14.23% [579000/4070000] [142.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 02:35:36,640 - Train: 14.23% [579100/4070000] [142.3/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 02:36:37,184 - Train: 14.23% [579200/4070000] [142.3/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 02:37:37,659 - Train: 14.23% [579300/4070000] [142.3/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-07 02:38:38,151 - Train: 14.24% [579400/4070000] [142.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 02:39:38,662 - Train: 14.24% [579500/4070000] [142.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 02:40:39,171 - Train: 14.24% [579600/4070000] [142.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 02:41:39,690 - Train: 14.24% [579700/4070000] [142.4/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 02:42:40,194 - Train: 14.25% [579800/4070000] [142.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 02:43:40,830 - Train: 14.25% [579900/4070000] [142.5/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 02:44:41,324 - Train: 14.25% [580000/4070000] [142.5/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-07 02:45:41,817 - Train: 14.25% [580100/4070000] [142.5/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 02:46:42,422 - Train: 14.26% [580200/4070000] [142.6/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 02:47:42,936 - Train: 14.26% [580300/4070000] [142.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 02:48:43,602 - Train: 14.26% [580400/4070000] [142.6/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 02:49:44,090 - Train: 14.26% [580500/4070000] [142.6/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 02:50:44,588 - Train: 14.27% [580600/4070000] [142.7/1000.0] [batch_t 0.610 (0.605)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-07 02:51:45,084 - Train: 14.27% [580700/4070000] [142.7/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-07 02:52:45,574 - Train: 14.27% [580800/4070000] [142.7/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 02:53:46,066 - Train: 14.27% [580900/4070000] [142.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 02:54:46,592 - Train: 14.28% [581000/4070000] [142.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 02:55:47,098 - Train: 14.28% [581100/4070000] [142.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 02:56:47,746 - Train: 14.28% [581200/4070000] [142.8/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 02:57:48,257 - Train: 14.28% [581300/4070000] [142.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 02:58:48,775 - Train: 14.29% [581400/4070000] [142.9/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-07 02:59:49,294 - Train: 14.29% [581500/4070000] [142.9/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 03:00:49,824 - Train: 14.29% [581600/4070000] [142.9/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 03:01:50,482 - Train: 14.29% [581700/4070000] [142.9/1000.0] [batch_t 0.610 (0.606)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-07 03:02:50,964 - Train: 14.29% [581800/4070000] [142.9/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-07 03:03:51,447 - Train: 14.30% [581900/4070000] [143.0/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-07 03:04:51,936 - Train: 14.30% [582000/4070000] [143.0/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 03:04:57,992 - ==> Total time: 5 days, 2:04:28 Eta: 30 days, 11:35:39 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-07 03:05:54,031 - Train: 14.30% [582100/4070000] [143.0/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 03:06:54,548 - Train: 14.30% [582200/4070000] [143.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 03:07:55,040 - Train: 14.31% [582300/4070000] [143.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 03:08:55,632 - Train: 14.31% [582400/4070000] [143.1/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 03:09:56,123 - Train: 14.31% [582500/4070000] [143.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 03:10:56,604 - Train: 14.31% [582600/4070000] [143.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 03:11:57,119 - Train: 14.32% [582700/4070000] [143.2/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 03:12:57,618 - Train: 14.32% [582800/4070000] [143.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 03:13:58,100 - Train: 14.32% [582900/4070000] [143.2/1000.0] [batch_t 0.610 (0.605)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-07 03:14:58,696 - Train: 14.32% [583000/4070000] [143.2/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 03:15:59,154 - Train: 14.33% [583100/4070000] [143.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 03:16:59,675 - Train: 14.33% [583200/4070000] [143.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 03:18:00,203 - Train: 14.33% [583300/4070000] [143.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 03:19:00,699 - Train: 14.33% [583400/4070000] [143.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 03:20:01,231 - Train: 14.34% [583500/4070000] [143.4/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-07 03:21:01,703 - Train: 14.34% [583600/4070000] [143.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 03:22:02,340 - Train: 14.34% [583700/4070000] [143.4/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 03:23:02,850 - Train: 14.34% [583800/4070000] [143.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 03:24:03,406 - Train: 14.35% [583900/4070000] [143.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 03:25:03,944 - Train: 14.35% [584000/4070000] [143.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 03:26:04,456 - Train: 14.35% [584100/4070000] [143.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 03:27:04,944 - Train: 14.35% [584200/4070000] [143.5/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 03:28:05,543 - Train: 14.36% [584300/4070000] [143.6/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 03:29:06,011 - Train: 14.36% [584400/4070000] [143.6/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 03:30:06,509 - Train: 14.36% [584500/4070000] [143.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 03:31:06,976 - Train: 14.36% [584600/4070000] [143.6/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 03:32:07,454 - Train: 14.37% [584700/4070000] [143.7/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 03:33:07,961 - Train: 14.37% [584800/4070000] [143.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 03:34:08,455 - Train: 14.37% [584900/4070000] [143.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 03:35:09,059 - Train: 14.37% [585000/4070000] [143.7/1000.0] [batch_t 0.600 (0.606)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-07 03:36:09,549 - Train: 14.38% [585100/4070000] [143.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 03:37:10,020 - Train: 14.38% [585200/4070000] [143.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 03:38:10,485 - Train: 14.38% [585300/4070000] [143.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 03:39:10,988 - Train: 14.38% [585400/4070000] [143.8/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-07 03:40:11,600 - Train: 14.39% [585500/4070000] [143.9/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 03:41:12,048 - Train: 14.39% [585600/4070000] [143.9/1000.0] [batch_t 0.592 (0.604)] [data_t 0.002] [optim_t 0.591] [lr 0.005000] +2024-04-07 03:42:12,533 - Train: 14.39% [585700/4070000] [143.9/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-07 03:43:13,003 - Train: 14.39% [585800/4070000] [143.9/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-07 03:44:13,480 - Train: 14.40% [585900/4070000] [144.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 03:45:14,007 - Train: 14.40% [586000/4070000] [144.0/1000.0] [batch_t 0.610 (0.605)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-07 03:46:02,412 - ==> Total time: 5 days, 2:45:33 Eta: 30 days, 9:44:07 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-07 03:46:18,019 - Train: 14.40% [586100/4070000] [144.0/1000.0] [batch_t 0.602 (0.615)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-07 03:47:18,640 - Train: 14.40% [586200/4070000] [144.0/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 03:48:19,159 - Train: 14.41% [586300/4070000] [144.1/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 03:49:19,628 - Train: 14.41% [586400/4070000] [144.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 03:50:20,113 - Train: 14.41% [586500/4070000] [144.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 03:51:20,563 - Train: 14.41% [586600/4070000] [144.1/1000.0] [batch_t 0.601 (0.604)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-07 03:52:21,049 - Train: 14.42% [586700/4070000] [144.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 03:53:21,612 - Train: 14.42% [586800/4070000] [144.2/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-07 03:54:22,092 - Train: 14.42% [586900/4070000] [144.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 03:55:22,570 - Train: 14.42% [587000/4070000] [144.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 03:56:23,058 - Train: 14.43% [587100/4070000] [144.3/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 03:57:23,563 - Train: 14.43% [587200/4070000] [144.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 03:58:24,035 - Train: 14.43% [587300/4070000] [144.3/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 03:59:24,497 - Train: 14.43% [587400/4070000] [144.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 04:00:25,084 - Train: 14.43% [587500/4070000] [144.3/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 04:01:25,606 - Train: 14.44% [587600/4070000] [144.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 04:02:26,103 - Train: 14.44% [587700/4070000] [144.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 04:03:26,613 - Train: 14.44% [587800/4070000] [144.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 04:04:27,112 - Train: 14.44% [587900/4070000] [144.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 04:05:27,580 - Train: 14.45% [588000/4070000] [144.5/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 04:06:28,177 - Train: 14.45% [588100/4070000] [144.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 04:07:28,658 - Train: 14.45% [588200/4070000] [144.5/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 04:08:29,132 - Train: 14.45% [588300/4070000] [144.5/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 04:09:29,617 - Train: 14.46% [588400/4070000] [144.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 04:10:30,097 - Train: 14.46% [588500/4070000] [144.6/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 04:11:30,563 - Train: 14.46% [588600/4070000] [144.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 04:12:31,101 - Train: 14.46% [588700/4070000] [144.6/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 04:13:31,704 - Train: 14.47% [588800/4070000] [144.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 04:14:32,172 - Train: 14.47% [588900/4070000] [144.7/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 04:15:32,655 - Train: 14.47% [589000/4070000] [144.7/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 04:16:33,127 - Train: 14.47% [589100/4070000] [144.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 04:17:33,613 - Train: 14.48% [589200/4070000] [144.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 04:18:34,082 - Train: 14.48% [589300/4070000] [144.8/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-07 04:19:34,659 - Train: 14.48% [589400/4070000] [144.8/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 04:20:35,145 - Train: 14.48% [589500/4070000] [144.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 04:21:35,629 - Train: 14.49% [589600/4070000] [144.9/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 04:22:36,110 - Train: 14.49% [589700/4070000] [144.9/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 04:23:36,592 - Train: 14.49% [589800/4070000] [144.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 04:24:37,058 - Train: 14.49% [589900/4070000] [144.9/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 04:25:37,645 - Train: 14.50% [590000/4070000] [145.0/1000.0] [batch_t 0.602 (0.606)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-07 04:26:38,089 - Train: 14.50% [590100/4070000] [145.0/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 04:27:08,330 - ==> Total time: 5 days, 3:26:39 Eta: 30 days, 7:53:42 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-07 04:27:40,114 - Train: 14.50% [590200/4070000] [145.0/1000.0] [batch_t 0.608 (0.609)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-07 04:28:40,577 - Train: 14.50% [590300/4070000] [145.0/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 04:29:41,039 - Train: 14.51% [590400/4070000] [145.1/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-07 04:30:41,488 - Train: 14.51% [590500/4070000] [145.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 04:31:42,053 - Train: 14.51% [590600/4070000] [145.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 04:32:42,493 - Train: 14.51% [590700/4070000] [145.1/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 04:33:42,956 - Train: 14.52% [590800/4070000] [145.2/1000.0] [batch_t 0.596 (0.605)] [data_t 0.002] [optim_t 0.594] [lr 0.005000] +2024-04-07 04:34:43,427 - Train: 14.52% [590900/4070000] [145.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 04:35:43,892 - Train: 14.52% [591000/4070000] [145.2/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 04:36:44,345 - Train: 14.52% [591100/4070000] [145.2/1000.0] [batch_t 0.601 (0.604)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-07 04:37:44,988 - Train: 14.53% [591200/4070000] [145.3/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 04:38:45,477 - Train: 14.53% [591300/4070000] [145.3/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-07 04:39:45,936 - Train: 14.53% [591400/4070000] [145.3/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 04:40:46,427 - Train: 14.53% [591500/4070000] [145.3/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-07 04:41:46,904 - Train: 14.54% [591600/4070000] [145.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 04:42:47,373 - Train: 14.54% [591700/4070000] [145.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 04:43:47,862 - Train: 14.54% [591800/4070000] [145.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 04:44:48,402 - Train: 14.54% [591900/4070000] [145.4/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-07 04:45:48,894 - Train: 14.55% [592000/4070000] [145.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 04:46:49,382 - Train: 14.55% [592100/4070000] [145.5/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 04:47:49,845 - Train: 14.55% [592200/4070000] [145.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 04:48:50,317 - Train: 14.55% [592300/4070000] [145.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 04:49:50,808 - Train: 14.56% [592400/4070000] [145.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 04:50:51,429 - Train: 14.56% [592500/4070000] [145.6/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 04:51:51,893 - Train: 14.56% [592600/4070000] [145.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 04:52:52,390 - Train: 14.56% [592700/4070000] [145.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 04:53:52,850 - Train: 14.57% [592800/4070000] [145.7/1000.0] [batch_t 0.600 (0.605)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-07 04:54:53,358 - Train: 14.57% [592900/4070000] [145.7/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 04:55:53,841 - Train: 14.57% [593000/4070000] [145.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 04:56:54,301 - Train: 14.57% [593100/4070000] [145.7/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 04:57:54,852 - Train: 14.57% [593200/4070000] [145.7/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 04:58:55,323 - Train: 14.58% [593300/4070000] [145.8/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-07 04:59:55,810 - Train: 14.58% [593400/4070000] [145.8/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 05:00:56,300 - Train: 14.58% [593500/4070000] [145.8/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-07 05:01:56,797 - Train: 14.58% [593600/4070000] [145.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 05:02:57,260 - Train: 14.59% [593700/4070000] [145.9/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 05:03:57,833 - Train: 14.59% [593800/4070000] [145.9/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-07 05:04:58,321 - Train: 14.59% [593900/4070000] [145.9/1000.0] [batch_t 0.607 (0.605)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-07 05:05:58,786 - Train: 14.59% [594000/4070000] [145.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 05:06:59,256 - Train: 14.60% [594100/4070000] [146.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 05:07:59,705 - Train: 14.60% [594200/4070000] [146.0/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 05:08:11,822 - ==> Total time: 5 days, 4:07:42 Eta: 30 days, 6:04:00 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-07 05:09:01,838 - Train: 14.60% [594300/4070000] [146.0/1000.0] [batch_t 0.603 (0.607)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 05:10:02,354 - Train: 14.60% [594400/4070000] [146.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 05:11:02,993 - Train: 14.61% [594500/4070000] [146.1/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-07 05:12:03,486 - Train: 14.61% [594600/4070000] [146.1/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 05:13:03,994 - Train: 14.61% [594700/4070000] [146.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 05:14:04,454 - Train: 14.61% [594800/4070000] [146.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 05:15:04,968 - Train: 14.62% [594900/4070000] [146.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 05:16:05,550 - Train: 14.62% [595000/4070000] [146.2/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 05:17:06,037 - Train: 14.62% [595100/4070000] [146.2/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 05:18:06,539 - Train: 14.62% [595200/4070000] [146.2/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 05:19:07,043 - Train: 14.63% [595300/4070000] [146.3/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 05:20:07,507 - Train: 14.63% [595400/4070000] [146.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 05:21:08,000 - Train: 14.63% [595500/4070000] [146.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 05:22:08,497 - Train: 14.63% [595600/4070000] [146.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 05:23:09,033 - Train: 14.64% [595700/4070000] [146.4/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-07 05:24:09,498 - Train: 14.64% [595800/4070000] [146.4/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 05:25:09,960 - Train: 14.64% [595900/4070000] [146.4/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 05:26:10,429 - Train: 14.64% [596000/4070000] [146.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 05:27:10,915 - Train: 14.65% [596100/4070000] [146.5/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 05:28:11,399 - Train: 14.65% [596200/4070000] [146.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 05:29:12,056 - Train: 14.65% [596300/4070000] [146.5/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 05:30:12,612 - Train: 14.65% [596400/4070000] [146.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 05:31:13,151 - Train: 14.66% [596500/4070000] [146.6/1000.0] [batch_t 0.599 (0.605)] [data_t 0.002] [optim_t 0.597] [lr 0.005000] +2024-04-07 05:32:13,700 - Train: 14.66% [596600/4070000] [146.6/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 05:33:14,260 - Train: 14.66% [596700/4070000] [146.6/1000.0] [batch_t 0.607 (0.605)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-07 05:34:14,806 - Train: 14.66% [596800/4070000] [146.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 05:35:15,308 - Train: 14.67% [596900/4070000] [146.7/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 05:36:16,024 - Train: 14.67% [597000/4070000] [146.7/1000.0] [batch_t 0.603 (0.607)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 05:37:16,543 - Train: 14.67% [597100/4070000] [146.7/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 05:38:17,101 - Train: 14.67% [597200/4070000] [146.7/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-07 05:39:17,592 - Train: 14.68% [597300/4070000] [146.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 05:40:18,122 - Train: 14.68% [597400/4070000] [146.8/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 05:41:18,714 - Train: 14.68% [597500/4070000] [146.8/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 05:42:19,350 - Train: 14.68% [597600/4070000] [146.8/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 05:43:19,897 - Train: 14.69% [597700/4070000] [146.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 05:44:20,407 - Train: 14.69% [597800/4070000] [146.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 05:45:20,936 - Train: 14.69% [597900/4070000] [146.9/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 05:46:21,476 - Train: 14.69% [598000/4070000] [146.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 05:47:21,984 - Train: 14.70% [598100/4070000] [147.0/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 05:48:22,536 - Train: 14.70% [598200/4070000] [147.0/1000.0] [batch_t 0.610 (0.605)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-07 05:49:17,146 - ==> Total time: 5 days, 4:48:48 Eta: 30 days, 4:15:25 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-07 05:49:24,748 - Train: 14.70% [598300/4070000] [147.0/1000.0] [batch_t 0.604 (0.620)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 05:50:25,247 - Train: 14.70% [598400/4070000] [147.0/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-07 05:51:25,744 - Train: 14.71% [598500/4070000] [147.1/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-07 05:52:26,234 - Train: 14.71% [598600/4070000] [147.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 05:53:26,742 - Train: 14.71% [598700/4070000] [147.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 05:54:27,241 - Train: 14.71% [598800/4070000] [147.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 05:55:27,755 - Train: 14.71% [598900/4070000] [147.1/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-07 05:56:28,357 - Train: 14.72% [599000/4070000] [147.2/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 05:57:28,871 - Train: 14.72% [599100/4070000] [147.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 05:58:29,330 - Train: 14.72% [599200/4070000] [147.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 05:59:29,813 - Train: 14.72% [599300/4070000] [147.2/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 06:00:30,299 - Train: 14.73% [599400/4070000] [147.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 06:01:30,973 - Train: 14.73% [599500/4070000] [147.3/1000.0] [batch_t 0.603 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 06:02:31,476 - Train: 14.73% [599600/4070000] [147.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 06:03:31,994 - Train: 14.73% [599700/4070000] [147.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 06:04:32,452 - Train: 14.74% [599800/4070000] [147.4/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 06:05:32,928 - Train: 14.74% [599900/4070000] [147.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 06:06:33,445 - Train: 14.74% [600000/4070000] [147.4/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 06:07:33,940 - Train: 14.74% [600100/4070000] [147.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 06:08:34,439 - Train: 14.75% [600200/4070000] [147.5/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 06:09:35,010 - Train: 14.75% [600300/4070000] [147.5/1000.0] [batch_t 0.602 (0.606)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-07 06:10:35,557 - Train: 14.75% [600400/4070000] [147.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 06:11:36,104 - Train: 14.75% [600500/4070000] [147.5/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 06:12:36,580 - Train: 14.76% [600600/4070000] [147.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 06:13:37,084 - Train: 14.76% [600700/4070000] [147.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 06:14:37,798 - Train: 14.76% [600800/4070000] [147.6/1000.0] [batch_t 0.602 (0.607)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-07 06:15:38,296 - Train: 14.76% [600900/4070000] [147.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 06:16:38,785 - Train: 14.77% [601000/4070000] [147.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 06:17:39,262 - Train: 14.77% [601100/4070000] [147.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 06:18:39,773 - Train: 14.77% [601200/4070000] [147.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 06:19:40,269 - Train: 14.77% [601300/4070000] [147.7/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-07 06:20:40,781 - Train: 14.78% [601400/4070000] [147.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 06:21:41,311 - Train: 14.78% [601500/4070000] [147.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 06:22:41,963 - Train: 14.78% [601600/4070000] [147.8/1000.0] [batch_t 0.602 (0.606)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-07 06:23:42,442 - Train: 14.78% [601700/4070000] [147.8/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 06:24:42,972 - Train: 14.79% [601800/4070000] [147.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 06:25:43,496 - Train: 14.79% [601900/4070000] [147.9/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 06:26:44,041 - Train: 14.79% [602000/4070000] [147.9/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 06:27:44,701 - Train: 14.79% [602100/4070000] [147.9/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 06:28:45,197 - Train: 14.80% [602200/4070000] [148.0/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 06:29:45,718 - Train: 14.80% [602300/4070000] [148.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 06:30:22,021 - ==> Total time: 5 days, 5:29:52 Eta: 30 days, 2:27:41 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-07 06:30:47,833 - Train: 14.80% [602400/4070000] [148.0/1000.0] [batch_t 0.603 (0.610)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 06:31:48,320 - Train: 14.80% [602500/4070000] [148.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 06:32:48,816 - Train: 14.81% [602600/4070000] [148.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 06:33:49,308 - Train: 14.81% [602700/4070000] [148.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 06:34:49,921 - Train: 14.81% [602800/4070000] [148.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 06:35:50,435 - Train: 14.81% [602900/4070000] [148.1/1000.0] [batch_t 0.612 (0.605)] [data_t 0.002] [optim_t 0.610] [lr 0.005000] +2024-04-07 06:36:50,935 - Train: 14.82% [603000/4070000] [148.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 06:37:51,398 - Train: 14.82% [603100/4070000] [148.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 06:38:51,893 - Train: 14.82% [603200/4070000] [148.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 06:39:52,499 - Train: 14.82% [603300/4070000] [148.2/1000.0] [batch_t 0.609 (0.606)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-07 06:40:53,003 - Train: 14.83% [603400/4070000] [148.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 06:41:53,519 - Train: 14.83% [603500/4070000] [148.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 06:42:54,032 - Train: 14.83% [603600/4070000] [148.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 06:43:54,567 - Train: 14.83% [603700/4070000] [148.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 06:44:55,052 - Train: 14.84% [603800/4070000] [148.4/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 06:45:55,534 - Train: 14.84% [603900/4070000] [148.4/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 06:46:56,073 - Train: 14.84% [604000/4070000] [148.4/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-07 06:47:56,605 - Train: 14.84% [604100/4070000] [148.4/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 06:48:57,095 - Train: 14.85% [604200/4070000] [148.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 06:49:57,598 - Train: 14.85% [604300/4070000] [148.5/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 06:50:58,061 - Train: 14.85% [604400/4070000] [148.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 06:51:58,495 - Train: 14.85% [604500/4070000] [148.5/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 06:52:59,115 - Train: 14.86% [604600/4070000] [148.6/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 06:53:59,623 - Train: 14.86% [604700/4070000] [148.6/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 06:55:00,157 - Train: 14.86% [604800/4070000] [148.6/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-07 06:56:00,661 - Train: 14.86% [604900/4070000] [148.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 06:57:01,176 - Train: 14.86% [605000/4070000] [148.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 06:58:01,653 - Train: 14.87% [605100/4070000] [148.7/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 06:59:02,178 - Train: 14.87% [605200/4070000] [148.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 07:00:02,803 - Train: 14.87% [605300/4070000] [148.7/1000.0] [batch_t 0.601 (0.606)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-07 07:01:03,357 - Train: 14.87% [605400/4070000] [148.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 07:02:03,819 - Train: 14.88% [605500/4070000] [148.8/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 07:03:04,340 - Train: 14.88% [605600/4070000] [148.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 07:04:04,842 - Train: 14.88% [605700/4070000] [148.8/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-07 07:05:05,375 - Train: 14.88% [605800/4070000] [148.8/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-07 07:06:06,031 - Train: 14.89% [605900/4070000] [148.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 07:07:06,504 - Train: 14.89% [606000/4070000] [148.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 07:08:06,992 - Train: 14.89% [606100/4070000] [148.9/1000.0] [batch_t 0.610 (0.605)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-07 07:09:07,494 - Train: 14.89% [606200/4070000] [148.9/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 07:10:07,996 - Train: 14.90% [606300/4070000] [149.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 07:11:08,499 - Train: 14.90% [606400/4070000] [149.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 07:11:26,657 - ==> Total time: 5 days, 6:10:57 Eta: 30 days, 0:40:50 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-07 07:12:10,792 - Train: 14.90% [606500/4070000] [149.0/1000.0] [batch_t 0.605 (0.609)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 07:13:11,302 - Train: 14.90% [606600/4070000] [149.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 07:14:11,816 - Train: 14.91% [606700/4070000] [149.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 07:15:12,303 - Train: 14.91% [606800/4070000] [149.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 07:16:12,820 - Train: 14.91% [606900/4070000] [149.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 07:17:13,330 - Train: 14.91% [607000/4070000] [149.1/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 07:18:13,812 - Train: 14.92% [607100/4070000] [149.2/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-07 07:19:14,313 - Train: 14.92% [607200/4070000] [149.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 07:20:14,854 - Train: 14.92% [607300/4070000] [149.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 07:21:15,405 - Train: 14.92% [607400/4070000] [149.2/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 07:22:15,912 - Train: 14.93% [607500/4070000] [149.3/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 07:23:16,432 - Train: 14.93% [607600/4070000] [149.3/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 07:24:16,926 - Train: 14.93% [607700/4070000] [149.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 07:25:17,548 - Train: 14.93% [607800/4070000] [149.3/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 07:26:18,117 - Train: 14.94% [607900/4070000] [149.4/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 07:27:18,649 - Train: 14.94% [608000/4070000] [149.4/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 07:28:19,173 - Train: 14.94% [608100/4070000] [149.4/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-07 07:29:19,685 - Train: 14.94% [608200/4070000] [149.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 07:30:20,164 - Train: 14.95% [608300/4070000] [149.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 07:31:20,639 - Train: 14.95% [608400/4070000] [149.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 07:32:21,138 - Train: 14.95% [608500/4070000] [149.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 07:33:21,747 - Train: 14.95% [608600/4070000] [149.5/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 07:34:22,241 - Train: 14.96% [608700/4070000] [149.6/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 07:35:22,772 - Train: 14.96% [608800/4070000] [149.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 07:36:23,261 - Train: 14.96% [608900/4070000] [149.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 07:37:23,711 - Train: 14.96% [609000/4070000] [149.6/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 07:38:24,294 - Train: 14.97% [609100/4070000] [149.7/1000.0] [batch_t 0.602 (0.606)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-07 07:39:24,775 - Train: 14.97% [609200/4070000] [149.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 07:40:25,314 - Train: 14.97% [609300/4070000] [149.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 07:41:25,846 - Train: 14.97% [609400/4070000] [149.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 07:42:26,344 - Train: 14.98% [609500/4070000] [149.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 07:43:26,870 - Train: 14.98% [609600/4070000] [149.8/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-07 07:44:27,377 - Train: 14.98% [609700/4070000] [149.8/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 07:45:27,870 - Train: 14.98% [609800/4070000] [149.8/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 07:46:28,466 - Train: 14.99% [609900/4070000] [149.9/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-07 07:47:28,992 - Train: 14.99% [610000/4070000] [149.9/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-07 07:48:29,507 - Train: 14.99% [610100/4070000] [149.9/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 07:49:30,042 - Train: 14.99% [610200/4070000] [149.9/1000.0] [batch_t 0.610 (0.605)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-07 07:50:30,587 - Train: 15.00% [610300/4070000] [150.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 07:51:31,213 - Train: 15.00% [610400/4070000] [150.0/1000.0] [batch_t 0.600 (0.606)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-07 07:52:31,739 - Train: 15.00% [610500/4070000] [150.0/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 07:52:31,743 - ==> Total time: 5 days, 6:52:02 Eta: 29 days, 22:54:54 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-07 07:53:33,803 - Train: 15.00% [610600/4070000] [150.0/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 07:54:34,305 - Train: 15.00% [610700/4070000] [150.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 07:55:34,786 - Train: 15.01% [610800/4070000] [150.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 07:56:35,450 - Train: 15.01% [610900/4070000] [150.1/1000.0] [batch_t 0.748 (0.607)] [data_t 0.002] [optim_t 0.746] [lr 0.005000] +2024-04-07 07:57:35,953 - Train: 15.01% [611000/4070000] [150.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 07:58:36,467 - Train: 15.01% [611100/4070000] [150.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 07:59:36,961 - Train: 15.02% [611200/4070000] [150.2/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 08:00:37,464 - Train: 15.02% [611300/4070000] [150.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 08:01:37,920 - Train: 15.02% [611400/4070000] [150.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 08:02:38,409 - Train: 15.02% [611500/4070000] [150.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 08:03:38,904 - Train: 15.03% [611600/4070000] [150.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 08:04:39,497 - Train: 15.03% [611700/4070000] [150.3/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 08:05:40,034 - Train: 15.03% [611800/4070000] [150.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 08:06:40,510 - Train: 15.03% [611900/4070000] [150.3/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 08:07:41,002 - Train: 15.04% [612000/4070000] [150.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 08:08:41,508 - Train: 15.04% [612100/4070000] [150.4/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 08:09:42,103 - Train: 15.04% [612200/4070000] [150.4/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 08:10:42,609 - Train: 15.04% [612300/4070000] [150.4/1000.0] [batch_t 0.610 (0.605)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-07 08:11:43,129 - Train: 15.05% [612400/4070000] [150.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 08:12:43,644 - Train: 15.05% [612500/4070000] [150.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 08:13:51,447 - Train: 15.05% [612600/4070000] [150.5/1000.0] [batch_t 0.606 (0.678)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 08:14:51,936 - Train: 15.05% [612700/4070000] [150.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 08:15:52,400 - Train: 15.06% [612800/4070000] [150.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 08:16:52,921 - Train: 15.06% [612900/4070000] [150.6/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 08:17:53,482 - Train: 15.06% [613000/4070000] [150.6/1000.0] [batch_t 0.601 (0.606)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-07 08:18:53,973 - Train: 15.06% [613100/4070000] [150.6/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 08:19:54,479 - Train: 15.07% [613200/4070000] [150.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 08:20:54,967 - Train: 15.07% [613300/4070000] [150.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 08:21:55,492 - Train: 15.07% [613400/4070000] [150.7/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-07 08:22:56,102 - Train: 15.07% [613500/4070000] [150.7/1000.0] [batch_t 0.602 (0.606)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-07 08:23:56,603 - Train: 15.08% [613600/4070000] [150.8/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-07 08:24:57,069 - Train: 15.08% [613700/4070000] [150.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 08:25:57,528 - Train: 15.08% [613800/4070000] [150.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 08:26:58,035 - Train: 15.08% [613900/4070000] [150.8/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-07 08:27:58,519 - Train: 15.09% [614000/4070000] [150.9/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 08:28:59,013 - Train: 15.09% [614100/4070000] [150.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 08:29:59,591 - Train: 15.09% [614200/4070000] [150.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 08:31:00,095 - Train: 15.09% [614300/4070000] [150.9/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 08:32:00,571 - Train: 15.10% [614400/4070000] [151.0/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-07 08:33:01,067 - Train: 15.10% [614500/4070000] [151.0/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-07 08:33:43,401 - ==> Total time: 5 days, 7:33:14 Eta: 29 days, 21:10:27 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-07 08:34:03,204 - Train: 15.10% [614600/4070000] [151.0/1000.0] [batch_t 0.605 (0.613)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 08:35:03,684 - Train: 15.10% [614700/4070000] [151.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 08:36:04,164 - Train: 15.11% [614800/4070000] [151.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 08:37:04,757 - Train: 15.11% [614900/4070000] [151.1/1000.0] [batch_t 0.600 (0.606)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-07 08:38:05,230 - Train: 15.11% [615000/4070000] [151.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 08:39:05,779 - Train: 15.11% [615100/4070000] [151.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 08:40:06,305 - Train: 15.12% [615200/4070000] [151.2/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 08:41:06,785 - Train: 15.12% [615300/4070000] [151.2/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-07 08:42:07,425 - Train: 15.12% [615400/4070000] [151.2/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 08:43:07,908 - Train: 15.12% [615500/4070000] [151.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 08:44:08,402 - Train: 15.13% [615600/4070000] [151.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 08:45:08,947 - Train: 15.13% [615700/4070000] [151.3/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 08:46:09,447 - Train: 15.13% [615800/4070000] [151.3/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-07 08:47:09,958 - Train: 15.13% [615900/4070000] [151.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 08:48:10,485 - Train: 15.14% [616000/4070000] [151.4/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 08:49:11,020 - Train: 15.14% [616100/4070000] [151.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 08:50:11,709 - Train: 15.14% [616200/4070000] [151.4/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 08:51:12,224 - Train: 15.14% [616300/4070000] [151.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 08:52:12,722 - Train: 15.14% [616400/4070000] [151.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 08:53:13,216 - Train: 15.15% [616500/4070000] [151.5/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 08:54:13,731 - Train: 15.15% [616600/4070000] [151.5/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 08:55:14,436 - Train: 15.15% [616700/4070000] [151.5/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 08:56:14,946 - Train: 15.15% [616800/4070000] [151.5/1000.0] [batch_t 0.613 (0.605)] [data_t 0.002] [optim_t 0.610] [lr 0.005000] +2024-04-07 08:57:15,468 - Train: 15.16% [616900/4070000] [151.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 08:58:15,971 - Train: 15.16% [617000/4070000] [151.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 08:59:16,487 - Train: 15.16% [617100/4070000] [151.6/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 09:00:17,014 - Train: 15.16% [617200/4070000] [151.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 09:01:17,550 - Train: 15.17% [617300/4070000] [151.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 09:02:18,074 - Train: 15.17% [617400/4070000] [151.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 09:03:18,738 - Train: 15.17% [617500/4070000] [151.7/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 09:04:19,274 - Train: 15.17% [617600/4070000] [151.7/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-07 09:05:19,765 - Train: 15.18% [617700/4070000] [151.8/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-07 09:06:20,307 - Train: 15.18% [617800/4070000] [151.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 09:07:20,812 - Train: 15.18% [617900/4070000] [151.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 09:08:21,454 - Train: 15.18% [618000/4070000] [151.8/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 09:09:21,936 - Train: 15.19% [618100/4070000] [151.9/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 09:10:22,483 - Train: 15.19% [618200/4070000] [151.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 09:11:22,955 - Train: 15.19% [618300/4070000] [151.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 09:12:23,468 - Train: 15.19% [618400/4070000] [151.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 09:13:23,961 - Train: 15.20% [618500/4070000] [152.0/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 09:14:24,423 - Train: 15.20% [618600/4070000] [152.0/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 09:14:48,621 - ==> Total time: 5 days, 8:14:19 Eta: 29 days, 19:26:13 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-07 09:15:26,699 - Train: 15.20% [618700/4070000] [152.0/1000.0] [batch_t 0.606 (0.610)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 09:16:27,253 - Train: 15.20% [618800/4070000] [152.0/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 09:17:27,719 - Train: 15.21% [618900/4070000] [152.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 09:18:28,210 - Train: 15.21% [619000/4070000] [152.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 09:19:28,686 - Train: 15.21% [619100/4070000] [152.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 09:20:29,293 - Train: 15.21% [619200/4070000] [152.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 09:21:29,791 - Train: 15.22% [619300/4070000] [152.2/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 09:22:30,283 - Train: 15.22% [619400/4070000] [152.2/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-07 09:23:30,791 - Train: 15.22% [619500/4070000] [152.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 09:24:31,306 - Train: 15.22% [619600/4070000] [152.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 09:25:31,853 - Train: 15.23% [619700/4070000] [152.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 09:26:32,363 - Train: 15.23% [619800/4070000] [152.3/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-07 09:27:32,889 - Train: 15.23% [619900/4070000] [152.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 09:28:33,454 - Train: 15.23% [620000/4070000] [152.3/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 09:29:33,996 - Train: 15.24% [620100/4070000] [152.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 09:30:34,502 - Train: 15.24% [620200/4070000] [152.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 09:31:34,993 - Train: 15.24% [620300/4070000] [152.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 09:32:35,493 - Train: 15.24% [620400/4070000] [152.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 09:33:36,190 - Train: 15.25% [620500/4070000] [152.5/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 09:34:36,684 - Train: 15.25% [620600/4070000] [152.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 09:35:37,184 - Train: 15.25% [620700/4070000] [152.5/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 09:36:37,702 - Train: 15.25% [620800/4070000] [152.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 09:37:38,176 - Train: 15.26% [620900/4070000] [152.6/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-07 09:38:38,696 - Train: 15.26% [621000/4070000] [152.6/1000.0] [batch_t 0.611 (0.605)] [data_t 0.002] [optim_t 0.609] [lr 0.005000] +2024-04-07 09:39:39,249 - Train: 15.26% [621100/4070000] [152.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 09:40:39,790 - Train: 15.26% [621200/4070000] [152.6/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 09:41:40,454 - Train: 15.27% [621300/4070000] [152.7/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 09:42:40,955 - Train: 15.27% [621400/4070000] [152.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 09:43:41,442 - Train: 15.27% [621500/4070000] [152.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 09:44:41,940 - Train: 15.27% [621600/4070000] [152.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 09:45:42,419 - Train: 15.28% [621700/4070000] [152.8/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 09:46:42,985 - Train: 15.28% [621800/4070000] [152.8/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 09:47:43,463 - Train: 15.28% [621900/4070000] [152.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 09:48:43,963 - Train: 15.28% [622000/4070000] [152.8/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-07 09:49:44,491 - Train: 15.29% [622100/4070000] [152.9/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 09:50:45,007 - Train: 15.29% [622200/4070000] [152.9/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-07 09:51:45,512 - Train: 15.29% [622300/4070000] [152.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 09:52:46,058 - Train: 15.29% [622400/4070000] [152.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 09:53:46,493 - Train: 15.29% [622500/4070000] [152.9/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 09:54:47,033 - Train: 15.30% [622600/4070000] [153.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 09:55:47,559 - Train: 15.30% [622700/4070000] [153.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 09:55:53,622 - ==> Total time: 5 days, 8:55:24 Eta: 29 days, 17:42:48 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-07 09:56:49,745 - Train: 15.30% [622800/4070000] [153.0/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 09:57:50,239 - Train: 15.30% [622900/4070000] [153.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 09:58:50,687 - Train: 15.31% [623000/4070000] [153.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 09:59:51,318 - Train: 15.31% [623100/4070000] [153.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 10:00:51,791 - Train: 15.31% [623200/4070000] [153.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 10:01:52,270 - Train: 15.31% [623300/4070000] [153.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 10:02:52,751 - Train: 15.32% [623400/4070000] [153.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 10:03:53,187 - Train: 15.32% [623500/4070000] [153.2/1000.0] [batch_t 0.601 (0.604)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-07 10:04:53,659 - Train: 15.32% [623600/4070000] [153.2/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-07 10:05:54,132 - Train: 15.32% [623700/4070000] [153.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 10:06:54,691 - Train: 15.33% [623800/4070000] [153.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 10:07:55,168 - Train: 15.33% [623900/4070000] [153.3/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 10:08:55,660 - Train: 15.33% [624000/4070000] [153.3/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 10:09:56,153 - Train: 15.33% [624100/4070000] [153.3/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-07 10:10:56,634 - Train: 15.34% [624200/4070000] [153.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 10:11:57,114 - Train: 15.34% [624300/4070000] [153.4/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-07 10:12:57,742 - Train: 15.34% [624400/4070000] [153.4/1000.0] [batch_t 0.601 (0.606)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-07 10:13:58,175 - Train: 15.34% [624500/4070000] [153.4/1000.0] [batch_t 0.600 (0.604)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-07 10:14:58,670 - Train: 15.35% [624600/4070000] [153.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 10:15:59,146 - Train: 15.35% [624700/4070000] [153.5/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 10:16:59,611 - Train: 15.35% [624800/4070000] [153.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 10:18:00,082 - Train: 15.35% [624900/4070000] [153.5/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 10:19:00,573 - Train: 15.36% [625000/4070000] [153.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 10:20:01,209 - Train: 15.36% [625100/4070000] [153.6/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-07 10:21:01,696 - Train: 15.36% [625200/4070000] [153.6/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 10:22:02,136 - Train: 15.36% [625300/4070000] [153.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 10:23:02,627 - Train: 15.37% [625400/4070000] [153.7/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 10:24:35,416 - Train: 15.37% [625500/4070000] [153.7/1000.0] [batch_t 6.925 (0.928)] [data_t 6.324] [optim_t 0.602] [lr 0.005000] +2024-04-07 10:40:09,642 - Train: 15.37% [625600/4070000] [153.7/1000.0] [batch_t 15.884 (9.342)] [data_t 14.369] [optim_t 1.515] [lr 0.005000] +2024-04-07 10:54:31,844 - Train: 15.37% [625700/4070000] [153.7/1000.0] [batch_t 1.435 (8.622)] [data_t 0.003] [optim_t 1.432] [lr 0.005000] +2024-04-07 10:56:54,990 - Train: 15.38% [625800/4070000] [153.8/1000.0] [batch_t 1.424 (1.431)] [data_t 0.003] [optim_t 1.422] [lr 0.005000] +2024-04-07 10:59:18,200 - Train: 15.38% [625900/4070000] [153.8/1000.0] [batch_t 1.426 (1.432)] [data_t 0.002] [optim_t 1.424] [lr 0.005000] +2024-04-07 11:01:41,274 - Train: 15.38% [626000/4070000] [153.8/1000.0] [batch_t 1.421 (1.431)] [data_t 0.003] [optim_t 1.418] [lr 0.005000] +2024-04-07 11:04:04,462 - Train: 15.38% [626100/4070000] [153.8/1000.0] [batch_t 1.451 (1.432)] [data_t 0.003] [optim_t 1.449] [lr 0.005000] +2024-04-07 11:06:27,623 - Train: 15.39% [626200/4070000] [153.9/1000.0] [batch_t 1.436 (1.432)] [data_t 0.002] [optim_t 1.434] [lr 0.005000] +2024-04-07 11:08:50,743 - Train: 15.39% [626300/4070000] [153.9/1000.0] [batch_t 1.437 (1.431)] [data_t 0.002] [optim_t 1.434] [lr 0.005000] +2024-04-07 11:11:13,863 - Train: 15.39% [626400/4070000] [153.9/1000.0] [batch_t 1.433 (1.431)] [data_t 0.002] [optim_t 1.431] [lr 0.005000] +2024-04-07 11:12:52,939 - Train: 15.39% [626500/4070000] [153.9/1000.0] [batch_t 0.603 (0.991)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 11:13:55,751 - Train: 15.40% [626600/4070000] [154.0/1000.0] [batch_t 0.604 (0.628)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 11:14:56,228 - Train: 15.40% [626700/4070000] [154.0/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 11:15:47,997 - ==> Total time: 5 days, 10:15:18 Eta: 29 days, 19:33:28 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-07 11:16:05,621 - Train: 15.40% [626800/4070000] [154.0/1000.0] [batch_t 0.604 (0.777)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 11:17:09,540 - Train: 15.40% [626900/4070000] [154.0/1000.0] [batch_t 0.607 (0.639)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 11:18:13,250 - Train: 15.41% [627000/4070000] [154.1/1000.0] [batch_t 0.605 (0.637)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 11:19:23,208 - Train: 15.41% [627100/4070000] [154.1/1000.0] [batch_t 0.600 (0.699)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-07 11:20:23,672 - Train: 15.41% [627200/4070000] [154.1/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-07 11:21:24,169 - Train: 15.41% [627300/4070000] [154.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 11:22:24,695 - Train: 15.42% [627400/4070000] [154.2/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 11:23:25,311 - Train: 15.42% [627500/4070000] [154.2/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 11:24:25,807 - Train: 15.42% [627600/4070000] [154.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 11:25:26,302 - Train: 15.42% [627700/4070000] [154.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 11:26:26,767 - Train: 15.43% [627800/4070000] [154.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 11:27:27,218 - Train: 15.43% [627900/4070000] [154.3/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 11:28:27,679 - Train: 15.43% [628000/4070000] [154.3/1000.0] [batch_t 0.613 (0.605)] [data_t 0.002] [optim_t 0.611] [lr 0.005000] +2024-04-07 11:29:28,162 - Train: 15.43% [628100/4070000] [154.3/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 11:30:28,689 - Train: 15.43% [628200/4070000] [154.3/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 11:31:29,289 - Train: 15.44% [628300/4070000] [154.4/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 11:32:29,787 - Train: 15.44% [628400/4070000] [154.4/1000.0] [batch_t 0.610 (0.605)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-07 11:33:30,275 - Train: 15.44% [628500/4070000] [154.4/1000.0] [batch_t 0.599 (0.605)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-07 11:34:30,785 - Train: 15.44% [628600/4070000] [154.4/1000.0] [batch_t 0.610 (0.605)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-07 11:35:31,766 - Train: 15.45% [628700/4070000] [154.5/1000.0] [batch_t 0.607 (0.610)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 11:36:33,363 - Train: 15.45% [628800/4070000] [154.5/1000.0] [batch_t 0.602 (0.616)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-07 11:37:33,896 - Train: 15.45% [628900/4070000] [154.5/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 11:38:37,920 - Train: 15.45% [629000/4070000] [154.5/1000.0] [batch_t 0.606 (0.640)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 11:39:38,418 - Train: 15.46% [629100/4070000] [154.6/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 11:41:07,500 - Train: 15.46% [629200/4070000] [154.6/1000.0] [batch_t 4.292 (0.891)] [data_t 3.689] [optim_t 0.603] [lr 0.005000] +2024-04-07 11:51:26,214 - Train: 15.46% [629300/4070000] [154.6/1000.0] [batch_t 1.453 (6.187)] [data_t 0.003] [optim_t 1.449] [lr 0.005000] +2024-04-07 12:00:23,323 - Train: 15.46% [629400/4070000] [154.6/1000.0] [batch_t 7.795 (5.371)] [data_t 6.332] [optim_t 1.463] [lr 0.005000] +2024-04-07 12:14:51,645 - Train: 15.47% [629500/4070000] [154.7/1000.0] [batch_t 8.872 (8.683)] [data_t 7.465] [optim_t 1.407] [lr 0.005000] +2024-04-07 12:20:00,692 - Train: 15.47% [629600/4070000] [154.7/1000.0] [batch_t 1.429 (3.090)] [data_t 0.003] [optim_t 1.426] [lr 0.005000] +2024-04-07 12:22:23,883 - Train: 15.47% [629700/4070000] [154.7/1000.0] [batch_t 1.426 (1.432)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-07 12:24:47,027 - Train: 15.47% [629800/4070000] [154.7/1000.0] [batch_t 1.416 (1.431)] [data_t 0.003] [optim_t 1.413] [lr 0.005000] +2024-04-07 12:27:10,202 - Train: 15.48% [629900/4070000] [154.8/1000.0] [batch_t 1.439 (1.432)] [data_t 0.002] [optim_t 1.436] [lr 0.005000] +2024-04-07 12:29:33,481 - Train: 15.48% [630000/4070000] [154.8/1000.0] [batch_t 1.424 (1.433)] [data_t 0.003] [optim_t 1.421] [lr 0.005000] +2024-04-07 12:31:56,725 - Train: 15.48% [630100/4070000] [154.8/1000.0] [batch_t 1.426 (1.432)] [data_t 0.003] [optim_t 1.424] [lr 0.005000] +2024-04-07 12:34:19,848 - Train: 15.48% [630200/4070000] [154.8/1000.0] [batch_t 1.442 (1.431)] [data_t 0.002] [optim_t 1.440] [lr 0.005000] +2024-04-07 12:36:43,140 - Train: 15.49% [630300/4070000] [154.9/1000.0] [batch_t 1.438 (1.433)] [data_t 0.003] [optim_t 1.436] [lr 0.005000] +2024-04-07 12:39:06,190 - Train: 15.49% [630400/4070000] [154.9/1000.0] [batch_t 1.440 (1.430)] [data_t 0.002] [optim_t 1.438] [lr 0.005000] +2024-04-07 12:41:29,414 - Train: 15.49% [630500/4070000] [154.9/1000.0] [batch_t 1.426 (1.432)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-07 12:43:52,611 - Train: 15.49% [630600/4070000] [154.9/1000.0] [batch_t 1.408 (1.432)] [data_t 0.002] [optim_t 1.406] [lr 0.005000] +2024-04-07 12:46:15,919 - Train: 15.50% [630700/4070000] [155.0/1000.0] [batch_t 1.441 (1.433)] [data_t 0.002] [optim_t 1.439] [lr 0.005000] +2024-04-07 12:48:38,998 - Train: 15.50% [630800/4070000] [155.0/1000.0] [batch_t 1.440 (1.431)] [data_t 0.003] [optim_t 1.437] [lr 0.005000] +2024-04-07 12:49:50,606 - ==> Total time: 5 days, 11:49:21 Eta: 29 days, 22:38:45 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-07 12:51:04,223 - Train: 15.50% [630900/4070000] [155.0/1000.0] [batch_t 1.422 (1.434)] [data_t 0.003] [optim_t 1.419] [lr 0.005000] +2024-04-07 12:53:27,305 - Train: 15.50% [631000/4070000] [155.0/1000.0] [batch_t 1.424 (1.431)] [data_t 0.003] [optim_t 1.421] [lr 0.005000] +2024-04-07 12:55:50,509 - Train: 15.51% [631100/4070000] [155.1/1000.0] [batch_t 1.440 (1.432)] [data_t 0.003] [optim_t 1.437] [lr 0.005000] +2024-04-07 12:58:13,815 - Train: 15.51% [631200/4070000] [155.1/1000.0] [batch_t 1.437 (1.433)] [data_t 0.003] [optim_t 1.434] [lr 0.005000] +2024-04-07 13:00:36,958 - Train: 15.51% [631300/4070000] [155.1/1000.0] [batch_t 1.445 (1.431)] [data_t 0.002] [optim_t 1.442] [lr 0.005000] +2024-04-07 13:02:59,982 - Train: 15.51% [631400/4070000] [155.1/1000.0] [batch_t 1.431 (1.430)] [data_t 0.002] [optim_t 1.429] [lr 0.005000] +2024-04-07 13:05:23,182 - Train: 15.52% [631500/4070000] [155.2/1000.0] [batch_t 1.432 (1.432)] [data_t 0.003] [optim_t 1.430] [lr 0.005000] +2024-04-07 13:07:46,353 - Train: 15.52% [631600/4070000] [155.2/1000.0] [batch_t 1.427 (1.432)] [data_t 0.003] [optim_t 1.424] [lr 0.005000] +2024-04-07 13:10:09,576 - Train: 15.52% [631700/4070000] [155.2/1000.0] [batch_t 1.426 (1.432)] [data_t 0.002] [optim_t 1.424] [lr 0.005000] +2024-04-07 13:12:32,827 - Train: 15.52% [631800/4070000] [155.2/1000.0] [batch_t 1.441 (1.432)] [data_t 0.003] [optim_t 1.438] [lr 0.005000] +2024-04-07 13:14:55,920 - Train: 15.53% [631900/4070000] [155.3/1000.0] [batch_t 1.435 (1.431)] [data_t 0.003] [optim_t 1.432] [lr 0.005000] +2024-04-07 13:17:19,034 - Train: 15.53% [632000/4070000] [155.3/1000.0] [batch_t 1.422 (1.431)] [data_t 0.002] [optim_t 1.420] [lr 0.005000] +2024-04-07 13:19:42,015 - Train: 15.53% [632100/4070000] [155.3/1000.0] [batch_t 1.426 (1.430)] [data_t 0.003] [optim_t 1.424] [lr 0.005000] +2024-04-07 13:22:05,113 - Train: 15.53% [632200/4070000] [155.3/1000.0] [batch_t 1.434 (1.431)] [data_t 0.002] [optim_t 1.432] [lr 0.005000] +2024-04-07 13:24:28,156 - Train: 15.54% [632300/4070000] [155.4/1000.0] [batch_t 1.439 (1.430)] [data_t 0.003] [optim_t 1.436] [lr 0.005000] +2024-04-07 13:26:51,263 - Train: 15.54% [632400/4070000] [155.4/1000.0] [batch_t 1.430 (1.431)] [data_t 0.002] [optim_t 1.427] [lr 0.005000] +2024-04-07 13:29:14,399 - Train: 15.54% [632500/4070000] [155.4/1000.0] [batch_t 1.436 (1.431)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-07 13:31:37,631 - Train: 15.54% [632600/4070000] [155.4/1000.0] [batch_t 1.445 (1.432)] [data_t 0.002] [optim_t 1.443] [lr 0.005000] +2024-04-07 13:34:00,782 - Train: 15.55% [632700/4070000] [155.5/1000.0] [batch_t 1.433 (1.431)] [data_t 0.002] [optim_t 1.431] [lr 0.005000] +2024-04-07 13:36:23,931 - Train: 15.55% [632800/4070000] [155.5/1000.0] [batch_t 1.435 (1.431)] [data_t 0.002] [optim_t 1.433] [lr 0.005000] +2024-04-07 13:38:47,093 - Train: 15.55% [632900/4070000] [155.5/1000.0] [batch_t 1.424 (1.432)] [data_t 0.002] [optim_t 1.422] [lr 0.005000] +2024-04-07 13:41:10,238 - Train: 15.55% [633000/4070000] [155.5/1000.0] [batch_t 1.417 (1.431)] [data_t 0.003] [optim_t 1.414] [lr 0.005000] +2024-04-07 13:43:33,247 - Train: 15.56% [633100/4070000] [155.6/1000.0] [batch_t 1.428 (1.430)] [data_t 0.002] [optim_t 1.426] [lr 0.005000] +2024-04-07 13:45:56,419 - Train: 15.56% [633200/4070000] [155.6/1000.0] [batch_t 1.426 (1.432)] [data_t 0.003] [optim_t 1.424] [lr 0.005000] +2024-04-07 13:48:19,558 - Train: 15.56% [633300/4070000] [155.6/1000.0] [batch_t 1.430 (1.431)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-07 13:50:42,563 - Train: 15.56% [633400/4070000] [155.6/1000.0] [batch_t 1.429 (1.430)] [data_t 0.003] [optim_t 1.426] [lr 0.005000] +2024-04-07 13:53:05,555 - Train: 15.57% [633500/4070000] [155.7/1000.0] [batch_t 1.431 (1.430)] [data_t 0.002] [optim_t 1.429] [lr 0.005000] +2024-04-07 13:55:28,643 - Train: 15.57% [633600/4070000] [155.7/1000.0] [batch_t 1.424 (1.431)] [data_t 0.003] [optim_t 1.421] [lr 0.005000] +2024-04-07 13:57:51,747 - Train: 15.57% [633700/4070000] [155.7/1000.0] [batch_t 1.439 (1.431)] [data_t 0.002] [optim_t 1.437] [lr 0.005000] +2024-04-07 14:00:14,790 - Train: 15.57% [633800/4070000] [155.7/1000.0] [batch_t 1.437 (1.430)] [data_t 0.002] [optim_t 1.434] [lr 0.005000] +2024-04-07 14:02:37,960 - Train: 15.57% [633900/4070000] [155.7/1000.0] [batch_t 1.439 (1.432)] [data_t 0.002] [optim_t 1.437] [lr 0.005000] +2024-04-07 14:05:00,992 - Train: 15.58% [634000/4070000] [155.8/1000.0] [batch_t 1.407 (1.430)] [data_t 0.003] [optim_t 1.404] [lr 0.005000] +2024-04-07 14:07:24,032 - Train: 15.58% [634100/4070000] [155.8/1000.0] [batch_t 1.430 (1.430)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-07 14:09:47,138 - Train: 15.58% [634200/4070000] [155.8/1000.0] [batch_t 1.429 (1.431)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-07 14:12:35,994 - Train: 15.58% [634300/4070000] [155.8/1000.0] [batch_t 2.265 (1.688)] [data_t 0.815] [optim_t 1.450] [lr 0.005000] +2024-04-07 14:26:27,733 - Train: 15.59% [634400/4070000] [155.9/1000.0] [batch_t 4.193 (8.317)] [data_t 2.740] [optim_t 1.453] [lr 0.005000] +2024-04-07 14:29:16,315 - Train: 15.59% [634500/4070000] [155.9/1000.0] [batch_t 1.431 (1.686)] [data_t 0.002] [optim_t 1.428] [lr 0.005000] +2024-04-07 14:31:39,620 - Train: 15.59% [634600/4070000] [155.9/1000.0] [batch_t 1.427 (1.433)] [data_t 0.002] [optim_t 1.424] [lr 0.005000] +2024-04-07 14:34:02,767 - Train: 15.59% [634700/4070000] [155.9/1000.0] [batch_t 1.406 (1.431)] [data_t 0.002] [optim_t 1.403] [lr 0.005000] +2024-04-07 14:36:25,884 - Train: 15.60% [634800/4070000] [156.0/1000.0] [batch_t 1.436 (1.431)] [data_t 0.003] [optim_t 1.434] [lr 0.005000] +2024-04-07 14:38:48,754 - Train: 15.60% [634900/4070000] [156.0/1000.0] [batch_t 1.442 (1.429)] [data_t 0.002] [optim_t 1.440] [lr 0.005000] +2024-04-07 14:39:17,368 - ==> Total time: 5 days, 13:38:48 Eta: 30 days, 3:03:47 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-07 14:41:14,594 - Train: 15.60% [635000/4070000] [156.0/1000.0] [batch_t 1.429 (1.439)] [data_t 0.003] [optim_t 1.426] [lr 0.005000] +2024-04-07 14:43:37,685 - Train: 15.60% [635100/4070000] [156.0/1000.0] [batch_t 1.435 (1.431)] [data_t 0.002] [optim_t 1.433] [lr 0.005000] +2024-04-07 14:46:00,657 - Train: 15.61% [635200/4070000] [156.1/1000.0] [batch_t 1.434 (1.430)] [data_t 0.003] [optim_t 1.431] [lr 0.005000] +2024-04-07 14:48:23,845 - Train: 15.61% [635300/4070000] [156.1/1000.0] [batch_t 1.424 (1.432)] [data_t 0.003] [optim_t 1.421] [lr 0.005000] +2024-04-07 14:50:46,906 - Train: 15.61% [635400/4070000] [156.1/1000.0] [batch_t 1.430 (1.431)] [data_t 0.002] [optim_t 1.427] [lr 0.005000] +2024-04-07 14:53:10,029 - Train: 15.61% [635500/4070000] [156.1/1000.0] [batch_t 1.431 (1.431)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-07 14:55:33,127 - Train: 15.62% [635600/4070000] [156.2/1000.0] [batch_t 1.430 (1.431)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-07 14:57:56,199 - Train: 15.62% [635700/4070000] [156.2/1000.0] [batch_t 1.444 (1.431)] [data_t 0.002] [optim_t 1.441] [lr 0.005000] +2024-04-07 15:00:19,132 - Train: 15.62% [635800/4070000] [156.2/1000.0] [batch_t 1.431 (1.429)] [data_t 0.002] [optim_t 1.429] [lr 0.005000] +2024-04-07 15:02:42,320 - Train: 15.62% [635900/4070000] [156.2/1000.0] [batch_t 1.420 (1.432)] [data_t 0.003] [optim_t 1.417] [lr 0.005000] +2024-04-07 15:05:05,327 - Train: 15.63% [636000/4070000] [156.3/1000.0] [batch_t 1.421 (1.430)] [data_t 0.003] [optim_t 1.419] [lr 0.005000] +2024-04-07 15:07:28,442 - Train: 15.63% [636100/4070000] [156.3/1000.0] [batch_t 1.434 (1.431)] [data_t 0.003] [optim_t 1.431] [lr 0.005000] +2024-04-07 15:09:51,431 - Train: 15.63% [636200/4070000] [156.3/1000.0] [batch_t 1.415 (1.430)] [data_t 0.003] [optim_t 1.412] [lr 0.005000] +2024-04-07 15:12:14,564 - Train: 15.63% [636300/4070000] [156.3/1000.0] [batch_t 1.417 (1.431)] [data_t 0.003] [optim_t 1.414] [lr 0.005000] +2024-04-07 15:14:37,644 - Train: 15.64% [636400/4070000] [156.4/1000.0] [batch_t 1.440 (1.431)] [data_t 0.003] [optim_t 1.437] [lr 0.005000] +2024-04-07 15:17:00,754 - Train: 15.64% [636500/4070000] [156.4/1000.0] [batch_t 1.434 (1.431)] [data_t 0.003] [optim_t 1.431] [lr 0.005000] +2024-04-07 15:19:23,709 - Train: 15.64% [636600/4070000] [156.4/1000.0] [batch_t 1.424 (1.429)] [data_t 0.002] [optim_t 1.422] [lr 0.005000] +2024-04-07 15:21:46,897 - Train: 15.64% [636700/4070000] [156.4/1000.0] [batch_t 1.427 (1.432)] [data_t 0.002] [optim_t 1.425] [lr 0.005000] +2024-04-07 15:24:09,923 - Train: 15.65% [636800/4070000] [156.5/1000.0] [batch_t 1.429 (1.430)] [data_t 0.002] [optim_t 1.426] [lr 0.005000] +2024-04-07 15:26:33,029 - Train: 15.65% [636900/4070000] [156.5/1000.0] [batch_t 1.435 (1.431)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-07 15:28:56,080 - Train: 15.65% [637000/4070000] [156.5/1000.0] [batch_t 1.433 (1.430)] [data_t 0.002] [optim_t 1.431] [lr 0.005000] +2024-04-07 15:31:19,051 - Train: 15.65% [637100/4070000] [156.5/1000.0] [batch_t 1.425 (1.430)] [data_t 0.002] [optim_t 1.423] [lr 0.005000] +2024-04-07 15:33:42,136 - Train: 15.66% [637200/4070000] [156.6/1000.0] [batch_t 1.440 (1.431)] [data_t 0.003] [optim_t 1.438] [lr 0.005000] +2024-04-07 15:36:05,166 - Train: 15.66% [637300/4070000] [156.6/1000.0] [batch_t 1.431 (1.430)] [data_t 0.002] [optim_t 1.429] [lr 0.005000] +2024-04-07 15:38:28,112 - Train: 15.66% [637400/4070000] [156.6/1000.0] [batch_t 1.429 (1.429)] [data_t 0.002] [optim_t 1.427] [lr 0.005000] +2024-04-07 15:40:58,923 - Train: 15.66% [637500/4070000] [156.6/1000.0] [batch_t 1.427 (1.508)] [data_t 0.003] [optim_t 1.424] [lr 0.005000] +2024-04-07 15:43:53,124 - Train: 15.67% [637600/4070000] [156.7/1000.0] [batch_t 1.451 (1.742)] [data_t 0.002] [optim_t 1.448] [lr 0.005000] +2024-04-07 15:46:51,857 - Train: 15.67% [637700/4070000] [156.7/1000.0] [batch_t 1.431 (1.787)] [data_t 0.002] [optim_t 1.428] [lr 0.005000] +2024-04-07 15:49:17,364 - Train: 15.67% [637800/4070000] [156.7/1000.0] [batch_t 1.430 (1.455)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-07 15:52:02,079 - Train: 15.67% [637900/4070000] [156.7/1000.0] [batch_t 1.435 (1.647)] [data_t 0.002] [optim_t 1.432] [lr 0.005000] +2024-04-07 15:54:25,131 - Train: 15.68% [638000/4070000] [156.8/1000.0] [batch_t 1.436 (1.430)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-07 15:56:48,292 - Train: 15.68% [638100/4070000] [156.8/1000.0] [batch_t 1.416 (1.432)] [data_t 0.002] [optim_t 1.414] [lr 0.005000] +2024-04-07 15:59:11,284 - Train: 15.68% [638200/4070000] [156.8/1000.0] [batch_t 1.430 (1.430)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-07 16:01:34,328 - Train: 15.68% [638300/4070000] [156.8/1000.0] [batch_t 1.417 (1.430)] [data_t 0.002] [optim_t 1.414] [lr 0.005000] +2024-04-07 16:03:57,502 - Train: 15.69% [638400/4070000] [156.9/1000.0] [batch_t 1.427 (1.432)] [data_t 0.003] [optim_t 1.424] [lr 0.005000] +2024-04-07 16:06:20,676 - Train: 15.69% [638500/4070000] [156.9/1000.0] [batch_t 1.445 (1.432)] [data_t 0.003] [optim_t 1.442] [lr 0.005000] +2024-04-07 16:08:43,780 - Train: 15.69% [638600/4070000] [156.9/1000.0] [batch_t 1.431 (1.431)] [data_t 0.003] [optim_t 1.429] [lr 0.005000] +2024-04-07 16:11:06,834 - Train: 15.69% [638700/4070000] [156.9/1000.0] [batch_t 1.434 (1.430)] [data_t 0.003] [optim_t 1.431] [lr 0.005000] +2024-04-07 16:13:29,982 - Train: 15.70% [638800/4070000] [157.0/1000.0] [batch_t 1.417 (1.431)] [data_t 0.002] [optim_t 1.414] [lr 0.005000] +2024-04-07 16:15:53,188 - Train: 15.70% [638900/4070000] [157.0/1000.0] [batch_t 1.431 (1.432)] [data_t 0.002] [optim_t 1.428] [lr 0.005000] +2024-04-07 16:18:02,041 - ==> Total time: 5 days, 15:17:32 Eta: 30 days, 6:26:34 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-07 16:18:21,056 - Train: 15.70% [639000/4070000] [157.0/1000.0] [batch_t 1.427 (1.538)] [data_t 0.003] [optim_t 1.424] [lr 0.005000] +2024-04-07 16:20:44,095 - Train: 15.70% [639100/4070000] [157.0/1000.0] [batch_t 1.430 (1.430)] [data_t 0.002] [optim_t 1.428] [lr 0.005000] +2024-04-07 16:23:07,409 - Train: 15.71% [639200/4070000] [157.1/1000.0] [batch_t 1.416 (1.433)] [data_t 0.003] [optim_t 1.413] [lr 0.005000] +2024-04-07 16:25:30,405 - Train: 15.71% [639300/4070000] [157.1/1000.0] [batch_t 1.429 (1.430)] [data_t 0.003] [optim_t 1.426] [lr 0.005000] +2024-04-07 16:27:53,581 - Train: 15.71% [639400/4070000] [157.1/1000.0] [batch_t 1.435 (1.432)] [data_t 0.003] [optim_t 1.432] [lr 0.005000] +2024-04-07 16:30:16,967 - Train: 15.71% [639500/4070000] [157.1/1000.0] [batch_t 1.435 (1.434)] [data_t 0.003] [optim_t 1.432] [lr 0.005000] +2024-04-07 16:32:40,219 - Train: 15.71% [639600/4070000] [157.1/1000.0] [batch_t 1.422 (1.432)] [data_t 0.003] [optim_t 1.420] [lr 0.005000] +2024-04-07 16:35:03,512 - Train: 15.72% [639700/4070000] [157.2/1000.0] [batch_t 1.429 (1.433)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-07 16:37:26,860 - Train: 15.72% [639800/4070000] [157.2/1000.0] [batch_t 1.429 (1.433)] [data_t 0.003] [optim_t 1.426] [lr 0.005000] +2024-04-07 16:39:50,102 - Train: 15.72% [639900/4070000] [157.2/1000.0] [batch_t 1.429 (1.432)] [data_t 0.002] [optim_t 1.427] [lr 0.005000] +2024-04-07 16:42:13,233 - Train: 15.72% [640000/4070000] [157.2/1000.0] [batch_t 1.426 (1.431)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-07 16:44:36,330 - Train: 15.73% [640100/4070000] [157.3/1000.0] [batch_t 1.441 (1.431)] [data_t 0.002] [optim_t 1.439] [lr 0.005000] +2024-04-07 16:46:59,552 - Train: 15.73% [640200/4070000] [157.3/1000.0] [batch_t 1.431 (1.432)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-07 16:49:22,500 - Train: 15.73% [640300/4070000] [157.3/1000.0] [batch_t 1.434 (1.429)] [data_t 0.003] [optim_t 1.431] [lr 0.005000] +2024-04-07 16:51:45,747 - Train: 15.73% [640400/4070000] [157.3/1000.0] [batch_t 1.434 (1.432)] [data_t 0.002] [optim_t 1.432] [lr 0.005000] +2024-04-07 16:54:08,977 - Train: 15.74% [640500/4070000] [157.4/1000.0] [batch_t 1.433 (1.432)] [data_t 0.002] [optim_t 1.430] [lr 0.005000] +2024-04-07 16:56:31,895 - Train: 15.74% [640600/4070000] [157.4/1000.0] [batch_t 1.427 (1.429)] [data_t 0.003] [optim_t 1.425] [lr 0.005000] +2024-04-07 16:58:54,877 - Train: 15.74% [640700/4070000] [157.4/1000.0] [batch_t 1.426 (1.430)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-07 17:01:17,888 - Train: 15.74% [640800/4070000] [157.4/1000.0] [batch_t 1.424 (1.430)] [data_t 0.002] [optim_t 1.422] [lr 0.005000] +2024-04-07 17:03:41,019 - Train: 15.75% [640900/4070000] [157.5/1000.0] [batch_t 1.438 (1.431)] [data_t 0.003] [optim_t 1.435] [lr 0.005000] +2024-04-07 17:06:04,061 - Train: 15.75% [641000/4070000] [157.5/1000.0] [batch_t 1.430 (1.430)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-07 17:08:26,965 - Train: 15.75% [641100/4070000] [157.5/1000.0] [batch_t 1.430 (1.429)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-07 17:10:49,900 - Train: 15.75% [641200/4070000] [157.5/1000.0] [batch_t 1.424 (1.429)] [data_t 0.003] [optim_t 1.421] [lr 0.005000] +2024-04-07 17:13:13,059 - Train: 15.76% [641300/4070000] [157.6/1000.0] [batch_t 1.430 (1.431)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-07 17:15:36,277 - Train: 15.76% [641400/4070000] [157.6/1000.0] [batch_t 1.425 (1.432)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-07 17:17:59,188 - Train: 15.76% [641500/4070000] [157.6/1000.0] [batch_t 1.439 (1.429)] [data_t 0.003] [optim_t 1.436] [lr 0.005000] +2024-04-07 17:20:22,057 - Train: 15.76% [641600/4070000] [157.6/1000.0] [batch_t 1.430 (1.429)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-07 17:22:45,128 - Train: 15.77% [641700/4070000] [157.7/1000.0] [batch_t 1.451 (1.431)] [data_t 0.002] [optim_t 1.449] [lr 0.005000] +2024-04-07 17:25:08,211 - Train: 15.77% [641800/4070000] [157.7/1000.0] [batch_t 1.421 (1.431)] [data_t 0.003] [optim_t 1.418] [lr 0.005000] +2024-04-07 17:27:31,186 - Train: 15.77% [641900/4070000] [157.7/1000.0] [batch_t 1.429 (1.430)] [data_t 0.003] [optim_t 1.426] [lr 0.005000] +2024-04-07 17:29:54,322 - Train: 15.77% [642000/4070000] [157.7/1000.0] [batch_t 1.434 (1.431)] [data_t 0.003] [optim_t 1.432] [lr 0.005000] +2024-04-07 17:32:17,400 - Train: 15.78% [642100/4070000] [157.8/1000.0] [batch_t 1.442 (1.431)] [data_t 0.002] [optim_t 1.440] [lr 0.005000] +2024-04-07 17:34:40,396 - Train: 15.78% [642200/4070000] [157.8/1000.0] [batch_t 1.424 (1.430)] [data_t 0.003] [optim_t 1.422] [lr 0.005000] +2024-04-07 17:37:03,647 - Train: 15.78% [642300/4070000] [157.8/1000.0] [batch_t 1.419 (1.432)] [data_t 0.002] [optim_t 1.417] [lr 0.005000] +2024-04-07 17:39:26,645 - Train: 15.78% [642400/4070000] [157.8/1000.0] [batch_t 1.440 (1.430)] [data_t 0.003] [optim_t 1.437] [lr 0.005000] +2024-04-07 17:41:49,746 - Train: 15.79% [642500/4070000] [157.9/1000.0] [batch_t 1.438 (1.431)] [data_t 0.003] [optim_t 1.435] [lr 0.005000] +2024-04-07 17:44:12,871 - Train: 15.79% [642600/4070000] [157.9/1000.0] [batch_t 1.424 (1.431)] [data_t 0.002] [optim_t 1.422] [lr 0.005000] +2024-04-07 17:46:35,988 - Train: 15.79% [642700/4070000] [157.9/1000.0] [batch_t 1.426 (1.431)] [data_t 0.002] [optim_t 1.423] [lr 0.005000] +2024-04-07 17:48:59,070 - Train: 15.79% [642800/4070000] [157.9/1000.0] [batch_t 1.438 (1.431)] [data_t 0.003] [optim_t 1.436] [lr 0.005000] +2024-04-07 17:51:22,208 - Train: 15.80% [642900/4070000] [158.0/1000.0] [batch_t 1.434 (1.431)] [data_t 0.003] [optim_t 1.431] [lr 0.005000] +2024-04-07 17:53:45,320 - Train: 15.80% [643000/4070000] [158.0/1000.0] [batch_t 1.446 (1.431)] [data_t 0.002] [optim_t 1.443] [lr 0.005000] +2024-04-07 17:55:11,130 - ==> Total time: 5 days, 16:54:41 Eta: 30 days, 9:37:04 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-07 17:56:10,021 - Train: 15.80% [643100/4070000] [158.0/1000.0] [batch_t 1.430 (1.435)] [data_t 0.002] [optim_t 1.428] [lr 0.005000] +2024-04-07 17:58:33,129 - Train: 15.80% [643200/4070000] [158.0/1000.0] [batch_t 1.426 (1.431)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-07 18:00:56,075 - Train: 15.81% [643300/4070000] [158.1/1000.0] [batch_t 1.430 (1.429)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-07 18:03:19,190 - Train: 15.81% [643400/4070000] [158.1/1000.0] [batch_t 1.427 (1.431)] [data_t 0.003] [optim_t 1.424] [lr 0.005000] +2024-04-07 18:05:42,372 - Train: 15.81% [643500/4070000] [158.1/1000.0] [batch_t 1.420 (1.432)] [data_t 0.003] [optim_t 1.417] [lr 0.005000] +2024-04-07 18:08:05,471 - Train: 15.81% [643600/4070000] [158.1/1000.0] [batch_t 1.434 (1.431)] [data_t 0.003] [optim_t 1.431] [lr 0.005000] +2024-04-07 18:10:28,536 - Train: 15.82% [643700/4070000] [158.2/1000.0] [batch_t 1.429 (1.431)] [data_t 0.002] [optim_t 1.427] [lr 0.005000] +2024-04-07 18:12:51,880 - Train: 15.82% [643800/4070000] [158.2/1000.0] [batch_t 1.425 (1.433)] [data_t 0.003] [optim_t 1.422] [lr 0.005000] +2024-04-07 18:15:15,026 - Train: 15.82% [643900/4070000] [158.2/1000.0] [batch_t 1.432 (1.431)] [data_t 0.002] [optim_t 1.429] [lr 0.005000] +2024-04-07 18:17:38,117 - Train: 15.82% [644000/4070000] [158.2/1000.0] [batch_t 1.394 (1.431)] [data_t 0.003] [optim_t 1.391] [lr 0.005000] +2024-04-07 18:20:01,275 - Train: 15.83% [644100/4070000] [158.3/1000.0] [batch_t 1.436 (1.431)] [data_t 0.002] [optim_t 1.434] [lr 0.005000] +2024-04-07 18:22:24,430 - Train: 15.83% [644200/4070000] [158.3/1000.0] [batch_t 1.427 (1.431)] [data_t 0.003] [optim_t 1.424] [lr 0.005000] +2024-04-07 18:24:47,584 - Train: 15.83% [644300/4070000] [158.3/1000.0] [batch_t 1.430 (1.431)] [data_t 0.002] [optim_t 1.428] [lr 0.005000] +2024-04-07 18:27:10,673 - Train: 15.83% [644400/4070000] [158.3/1000.0] [batch_t 1.436 (1.431)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-07 18:29:33,928 - Train: 15.84% [644500/4070000] [158.4/1000.0] [batch_t 1.418 (1.432)] [data_t 0.003] [optim_t 1.415] [lr 0.005000] +2024-04-07 18:31:57,100 - Train: 15.84% [644600/4070000] [158.4/1000.0] [batch_t 1.431 (1.432)] [data_t 0.002] [optim_t 1.429] [lr 0.005000] +2024-04-07 18:34:20,114 - Train: 15.84% [644700/4070000] [158.4/1000.0] [batch_t 1.435 (1.430)] [data_t 0.003] [optim_t 1.432] [lr 0.005000] +2024-04-07 18:36:43,182 - Train: 15.84% [644800/4070000] [158.4/1000.0] [batch_t 1.445 (1.431)] [data_t 0.002] [optim_t 1.443] [lr 0.005000] +2024-04-07 18:39:06,113 - Train: 15.85% [644900/4070000] [158.5/1000.0] [batch_t 1.439 (1.429)] [data_t 0.002] [optim_t 1.437] [lr 0.005000] +2024-04-07 18:41:28,983 - Train: 15.85% [645000/4070000] [158.5/1000.0] [batch_t 1.430 (1.429)] [data_t 0.002] [optim_t 1.427] [lr 0.005000] +2024-04-07 18:43:51,962 - Train: 15.85% [645100/4070000] [158.5/1000.0] [batch_t 1.417 (1.430)] [data_t 0.002] [optim_t 1.415] [lr 0.005000] +2024-04-07 18:46:14,872 - Train: 15.85% [645200/4070000] [158.5/1000.0] [batch_t 1.438 (1.429)] [data_t 0.002] [optim_t 1.436] [lr 0.005000] +2024-04-07 18:48:37,659 - Train: 15.86% [645300/4070000] [158.6/1000.0] [batch_t 1.444 (1.428)] [data_t 0.002] [optim_t 1.441] [lr 0.005000] +2024-04-07 18:51:00,489 - Train: 15.86% [645400/4070000] [158.6/1000.0] [batch_t 1.420 (1.428)] [data_t 0.003] [optim_t 1.417] [lr 0.005000] +2024-04-07 18:53:23,298 - Train: 15.86% [645500/4070000] [158.6/1000.0] [batch_t 1.417 (1.428)] [data_t 0.003] [optim_t 1.414] [lr 0.005000] +2024-04-07 18:55:46,009 - Train: 15.86% [645600/4070000] [158.6/1000.0] [batch_t 1.430 (1.427)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-07 18:58:08,954 - Train: 15.86% [645700/4070000] [158.6/1000.0] [batch_t 1.425 (1.429)] [data_t 0.003] [optim_t 1.422] [lr 0.005000] +2024-04-07 19:00:31,892 - Train: 15.87% [645800/4070000] [158.7/1000.0] [batch_t 1.424 (1.429)] [data_t 0.002] [optim_t 1.421] [lr 0.005000] +2024-04-07 19:02:54,689 - Train: 15.87% [645900/4070000] [158.7/1000.0] [batch_t 1.415 (1.428)] [data_t 0.003] [optim_t 1.412] [lr 0.005000] +2024-04-07 19:05:17,643 - Train: 15.87% [646000/4070000] [158.7/1000.0] [batch_t 1.422 (1.429)] [data_t 0.003] [optim_t 1.420] [lr 0.005000] +2024-04-07 19:07:40,646 - Train: 15.87% [646100/4070000] [158.7/1000.0] [batch_t 1.440 (1.430)] [data_t 0.002] [optim_t 1.438] [lr 0.005000] +2024-04-07 19:10:03,513 - Train: 15.88% [646200/4070000] [158.8/1000.0] [batch_t 1.435 (1.429)] [data_t 0.002] [optim_t 1.433] [lr 0.005000] +2024-04-07 19:12:26,266 - Train: 15.88% [646300/4070000] [158.8/1000.0] [batch_t 1.435 (1.427)] [data_t 0.003] [optim_t 1.432] [lr 0.005000] +2024-04-07 19:14:49,086 - Train: 15.88% [646400/4070000] [158.8/1000.0] [batch_t 1.425 (1.428)] [data_t 0.002] [optim_t 1.423] [lr 0.005000] +2024-04-07 19:17:11,843 - Train: 15.88% [646500/4070000] [158.8/1000.0] [batch_t 1.426 (1.427)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-07 19:19:34,865 - Train: 15.89% [646600/4070000] [158.9/1000.0] [batch_t 1.428 (1.430)] [data_t 0.002] [optim_t 1.426] [lr 0.005000] +2024-04-07 19:21:57,964 - Train: 15.89% [646700/4070000] [158.9/1000.0] [batch_t 1.445 (1.431)] [data_t 0.002] [optim_t 1.442] [lr 0.005000] +2024-04-07 19:24:20,950 - Train: 15.89% [646800/4070000] [158.9/1000.0] [batch_t 1.430 (1.430)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-07 19:26:43,901 - Train: 15.89% [646900/4070000] [158.9/1000.0] [batch_t 1.418 (1.429)] [data_t 0.003] [optim_t 1.416] [lr 0.005000] +2024-04-07 19:29:06,784 - Train: 15.90% [647000/4070000] [159.0/1000.0] [batch_t 1.422 (1.429)] [data_t 0.003] [optim_t 1.419] [lr 0.005000] +2024-04-07 19:31:29,812 - Train: 15.90% [647100/4070000] [159.0/1000.0] [batch_t 1.445 (1.430)] [data_t 0.002] [optim_t 1.442] [lr 0.005000] +2024-04-07 19:32:12,732 - ==> Total time: 5 days, 18:31:43 Eta: 30 days, 12:43:16 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-07 19:33:54,438 - Train: 15.90% [647200/4070000] [159.0/1000.0] [batch_t 1.425 (1.432)] [data_t 0.003] [optim_t 1.422] [lr 0.005000] +2024-04-07 19:36:17,459 - Train: 15.90% [647300/4070000] [159.0/1000.0] [batch_t 1.397 (1.430)] [data_t 0.002] [optim_t 1.394] [lr 0.005000] +2024-04-07 19:38:40,704 - Train: 15.91% [647400/4070000] [159.1/1000.0] [batch_t 1.438 (1.432)] [data_t 0.003] [optim_t 1.435] [lr 0.005000] +2024-04-07 19:41:03,867 - Train: 15.91% [647500/4070000] [159.1/1000.0] [batch_t 1.446 (1.432)] [data_t 0.003] [optim_t 1.444] [lr 0.005000] +2024-04-07 19:43:26,881 - Train: 15.91% [647600/4070000] [159.1/1000.0] [batch_t 1.426 (1.430)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-07 19:45:49,987 - Train: 15.91% [647700/4070000] [159.1/1000.0] [batch_t 1.433 (1.431)] [data_t 0.002] [optim_t 1.431] [lr 0.005000] +2024-04-07 19:48:13,178 - Train: 15.92% [647800/4070000] [159.2/1000.0] [batch_t 1.439 (1.432)] [data_t 0.002] [optim_t 1.436] [lr 0.005000] +2024-04-07 19:50:36,335 - Train: 15.92% [647900/4070000] [159.2/1000.0] [batch_t 1.433 (1.431)] [data_t 0.002] [optim_t 1.430] [lr 0.005000] +2024-04-07 19:52:59,508 - Train: 15.92% [648000/4070000] [159.2/1000.0] [batch_t 1.416 (1.432)] [data_t 0.003] [optim_t 1.413] [lr 0.005000] +2024-04-07 19:55:22,717 - Train: 15.92% [648100/4070000] [159.2/1000.0] [batch_t 1.437 (1.432)] [data_t 0.003] [optim_t 1.434] [lr 0.005000] +2024-04-07 19:57:46,024 - Train: 15.93% [648200/4070000] [159.3/1000.0] [batch_t 1.436 (1.433)] [data_t 0.002] [optim_t 1.434] [lr 0.005000] +2024-04-07 20:00:09,245 - Train: 15.93% [648300/4070000] [159.3/1000.0] [batch_t 1.426 (1.432)] [data_t 0.002] [optim_t 1.424] [lr 0.005000] +2024-04-07 20:02:32,471 - Train: 15.93% [648400/4070000] [159.3/1000.0] [batch_t 1.432 (1.432)] [data_t 0.003] [optim_t 1.429] [lr 0.005000] +2024-04-07 20:04:55,811 - Train: 15.93% [648500/4070000] [159.3/1000.0] [batch_t 1.438 (1.433)] [data_t 0.002] [optim_t 1.435] [lr 0.005000] +2024-04-07 20:07:18,756 - Train: 15.94% [648600/4070000] [159.4/1000.0] [batch_t 1.425 (1.429)] [data_t 0.002] [optim_t 1.423] [lr 0.005000] +2024-04-07 20:09:41,811 - Train: 15.94% [648700/4070000] [159.4/1000.0] [batch_t 1.439 (1.430)] [data_t 0.003] [optim_t 1.436] [lr 0.005000] +2024-04-07 20:12:04,906 - Train: 15.94% [648800/4070000] [159.4/1000.0] [batch_t 1.435 (1.431)] [data_t 0.003] [optim_t 1.432] [lr 0.005000] +2024-04-07 20:14:28,002 - Train: 15.94% [648900/4070000] [159.4/1000.0] [batch_t 1.424 (1.431)] [data_t 0.003] [optim_t 1.421] [lr 0.005000] +2024-04-07 20:16:51,019 - Train: 15.95% [649000/4070000] [159.5/1000.0] [batch_t 1.428 (1.430)] [data_t 0.002] [optim_t 1.426] [lr 0.005000] +2024-04-07 20:19:14,142 - Train: 15.95% [649100/4070000] [159.5/1000.0] [batch_t 1.430 (1.431)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-07 20:21:37,270 - Train: 15.95% [649200/4070000] [159.5/1000.0] [batch_t 1.420 (1.431)] [data_t 0.003] [optim_t 1.418] [lr 0.005000] +2024-04-07 20:24:00,472 - Train: 15.95% [649300/4070000] [159.5/1000.0] [batch_t 1.418 (1.432)] [data_t 0.002] [optim_t 1.415] [lr 0.005000] +2024-04-07 20:26:23,535 - Train: 15.96% [649400/4070000] [159.6/1000.0] [batch_t 1.424 (1.431)] [data_t 0.003] [optim_t 1.421] [lr 0.005000] +2024-04-07 20:28:46,656 - Train: 15.96% [649500/4070000] [159.6/1000.0] [batch_t 1.434 (1.431)] [data_t 0.003] [optim_t 1.432] [lr 0.005000] +2024-04-07 20:31:09,720 - Train: 15.96% [649600/4070000] [159.6/1000.0] [batch_t 1.415 (1.431)] [data_t 0.003] [optim_t 1.412] [lr 0.005000] +2024-04-07 20:33:32,915 - Train: 15.96% [649700/4070000] [159.6/1000.0] [batch_t 1.441 (1.432)] [data_t 0.003] [optim_t 1.439] [lr 0.005000] +2024-04-07 20:35:56,002 - Train: 15.97% [649800/4070000] [159.7/1000.0] [batch_t 1.431 (1.431)] [data_t 0.002] [optim_t 1.428] [lr 0.005000] +2024-04-07 20:38:19,132 - Train: 15.97% [649900/4070000] [159.7/1000.0] [batch_t 1.440 (1.431)] [data_t 0.003] [optim_t 1.438] [lr 0.005000] +2024-04-07 20:40:42,124 - Train: 15.97% [650000/4070000] [159.7/1000.0] [batch_t 1.435 (1.430)] [data_t 0.003] [optim_t 1.432] [lr 0.005000] +2024-04-07 20:43:05,124 - Train: 15.97% [650100/4070000] [159.7/1000.0] [batch_t 1.430 (1.430)] [data_t 0.002] [optim_t 1.428] [lr 0.005000] +2024-04-07 20:45:28,273 - Train: 15.98% [650200/4070000] [159.8/1000.0] [batch_t 1.435 (1.431)] [data_t 0.002] [optim_t 1.432] [lr 0.005000] +2024-04-07 20:47:51,485 - Train: 15.98% [650300/4070000] [159.8/1000.0] [batch_t 1.421 (1.432)] [data_t 0.003] [optim_t 1.418] [lr 0.005000] +2024-04-07 20:50:14,702 - Train: 15.98% [650400/4070000] [159.8/1000.0] [batch_t 1.430 (1.432)] [data_t 0.002] [optim_t 1.428] [lr 0.005000] +2024-04-07 20:52:37,600 - Train: 15.98% [650500/4070000] [159.8/1000.0] [batch_t 1.426 (1.429)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-07 20:55:00,625 - Train: 15.99% [650600/4070000] [159.9/1000.0] [batch_t 1.435 (1.430)] [data_t 0.002] [optim_t 1.433] [lr 0.005000] +2024-04-07 20:57:23,718 - Train: 15.99% [650700/4070000] [159.9/1000.0] [batch_t 1.440 (1.431)] [data_t 0.002] [optim_t 1.438] [lr 0.005000] +2024-04-07 20:59:46,710 - Train: 15.99% [650800/4070000] [159.9/1000.0] [batch_t 1.441 (1.430)] [data_t 0.002] [optim_t 1.438] [lr 0.005000] +2024-04-07 21:02:09,736 - Train: 15.99% [650900/4070000] [159.9/1000.0] [batch_t 1.430 (1.430)] [data_t 0.002] [optim_t 1.428] [lr 0.005000] +2024-04-07 21:04:32,807 - Train: 16.00% [651000/4070000] [160.0/1000.0] [batch_t 1.426 (1.431)] [data_t 0.002] [optim_t 1.423] [lr 0.005000] +2024-04-07 21:06:55,751 - Train: 16.00% [651100/4070000] [160.0/1000.0] [batch_t 1.421 (1.429)] [data_t 0.003] [optim_t 1.418] [lr 0.005000] +2024-04-07 21:09:18,906 - Train: 16.00% [651200/4070000] [160.0/1000.0] [batch_t 1.429 (1.431)] [data_t 0.003] [optim_t 1.426] [lr 0.005000] +2024-04-07 21:09:18,910 - ==> Total time: 5 days, 20:08:49 Eta: 30 days, 15:46:21 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-07 21:11:43,717 - Train: 16.00% [651300/4070000] [160.0/1000.0] [batch_t 1.439 (1.432)] [data_t 0.003] [optim_t 1.436] [lr 0.005000] +2024-04-07 21:14:06,777 - Train: 16.00% [651400/4070000] [160.0/1000.0] [batch_t 1.445 (1.430)] [data_t 0.003] [optim_t 1.442] [lr 0.005000] +2024-04-07 21:16:29,848 - Train: 16.01% [651500/4070000] [160.1/1000.0] [batch_t 1.428 (1.431)] [data_t 0.002] [optim_t 1.425] [lr 0.005000] +2024-04-07 21:18:52,914 - Train: 16.01% [651600/4070000] [160.1/1000.0] [batch_t 1.424 (1.431)] [data_t 0.003] [optim_t 1.421] [lr 0.005000] +2024-04-07 21:21:15,902 - Train: 16.01% [651700/4070000] [160.1/1000.0] [batch_t 1.420 (1.430)] [data_t 0.002] [optim_t 1.417] [lr 0.005000] +2024-04-07 21:23:38,892 - Train: 16.01% [651800/4070000] [160.1/1000.0] [batch_t 1.420 (1.430)] [data_t 0.002] [optim_t 1.418] [lr 0.005000] +2024-04-07 21:26:01,809 - Train: 16.02% [651900/4070000] [160.2/1000.0] [batch_t 1.441 (1.429)] [data_t 0.003] [optim_t 1.438] [lr 0.005000] +2024-04-07 21:28:24,860 - Train: 16.02% [652000/4070000] [160.2/1000.0] [batch_t 1.436 (1.430)] [data_t 0.002] [optim_t 1.433] [lr 0.005000] +2024-04-07 21:30:47,661 - Train: 16.02% [652100/4070000] [160.2/1000.0] [batch_t 1.402 (1.428)] [data_t 0.003] [optim_t 1.400] [lr 0.005000] +2024-04-07 21:33:10,622 - Train: 16.02% [652200/4070000] [160.2/1000.0] [batch_t 1.430 (1.430)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-07 21:35:33,528 - Train: 16.03% [652300/4070000] [160.3/1000.0] [batch_t 1.431 (1.429)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-07 21:37:56,471 - Train: 16.03% [652400/4070000] [160.3/1000.0] [batch_t 1.430 (1.429)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-07 21:40:19,300 - Train: 16.03% [652500/4070000] [160.3/1000.0] [batch_t 1.437 (1.428)] [data_t 0.002] [optim_t 1.435] [lr 0.005000] +2024-04-07 21:42:42,261 - Train: 16.03% [652600/4070000] [160.3/1000.0] [batch_t 1.445 (1.430)] [data_t 0.003] [optim_t 1.442] [lr 0.005000] +2024-04-07 21:45:05,387 - Train: 16.04% [652700/4070000] [160.4/1000.0] [batch_t 1.429 (1.431)] [data_t 0.002] [optim_t 1.427] [lr 0.005000] +2024-04-07 21:47:28,514 - Train: 16.04% [652800/4070000] [160.4/1000.0] [batch_t 1.420 (1.431)] [data_t 0.003] [optim_t 1.417] [lr 0.005000] +2024-04-07 21:49:51,664 - Train: 16.04% [652900/4070000] [160.4/1000.0] [batch_t 1.431 (1.431)] [data_t 0.003] [optim_t 1.429] [lr 0.005000] +2024-04-07 21:52:14,666 - Train: 16.04% [653000/4070000] [160.4/1000.0] [batch_t 1.428 (1.430)] [data_t 0.002] [optim_t 1.426] [lr 0.005000] +2024-04-07 21:54:37,784 - Train: 16.05% [653100/4070000] [160.5/1000.0] [batch_t 1.430 (1.431)] [data_t 0.002] [optim_t 1.428] [lr 0.005000] +2024-04-07 21:57:00,880 - Train: 16.05% [653200/4070000] [160.5/1000.0] [batch_t 1.424 (1.431)] [data_t 0.003] [optim_t 1.421] [lr 0.005000] +2024-04-07 21:59:24,057 - Train: 16.05% [653300/4070000] [160.5/1000.0] [batch_t 1.434 (1.432)] [data_t 0.003] [optim_t 1.432] [lr 0.005000] +2024-04-07 22:01:47,183 - Train: 16.05% [653400/4070000] [160.5/1000.0] [batch_t 1.435 (1.431)] [data_t 0.003] [optim_t 1.432] [lr 0.005000] +2024-04-07 22:04:10,397 - Train: 16.06% [653500/4070000] [160.6/1000.0] [batch_t 1.441 (1.432)] [data_t 0.002] [optim_t 1.439] [lr 0.005000] +2024-04-07 22:06:33,480 - Train: 16.06% [653600/4070000] [160.6/1000.0] [batch_t 1.428 (1.431)] [data_t 0.003] [optim_t 1.425] [lr 0.005000] +2024-04-07 22:08:56,661 - Train: 16.06% [653700/4070000] [160.6/1000.0] [batch_t 1.430 (1.432)] [data_t 0.002] [optim_t 1.428] [lr 0.005000] +2024-04-07 22:11:19,773 - Train: 16.06% [653800/4070000] [160.6/1000.0] [batch_t 1.442 (1.431)] [data_t 0.002] [optim_t 1.440] [lr 0.005000] +2024-04-07 22:13:42,850 - Train: 16.07% [653900/4070000] [160.7/1000.0] [batch_t 1.421 (1.431)] [data_t 0.003] [optim_t 1.418] [lr 0.005000] +2024-04-07 22:16:06,068 - Train: 16.07% [654000/4070000] [160.7/1000.0] [batch_t 1.425 (1.432)] [data_t 0.003] [optim_t 1.422] [lr 0.005000] +2024-04-07 22:18:29,179 - Train: 16.07% [654100/4070000] [160.7/1000.0] [batch_t 1.425 (1.431)] [data_t 0.003] [optim_t 1.422] [lr 0.005000] +2024-04-07 22:20:52,372 - Train: 16.07% [654200/4070000] [160.7/1000.0] [batch_t 1.440 (1.432)] [data_t 0.002] [optim_t 1.437] [lr 0.005000] +2024-04-07 22:23:15,730 - Train: 16.08% [654300/4070000] [160.8/1000.0] [batch_t 1.440 (1.433)] [data_t 0.002] [optim_t 1.438] [lr 0.005000] +2024-04-07 22:25:39,044 - Train: 16.08% [654400/4070000] [160.8/1000.0] [batch_t 1.438 (1.433)] [data_t 0.003] [optim_t 1.436] [lr 0.005000] +2024-04-07 22:28:02,218 - Train: 16.08% [654500/4070000] [160.8/1000.0] [batch_t 1.429 (1.432)] [data_t 0.002] [optim_t 1.427] [lr 0.005000] +2024-04-07 22:30:25,366 - Train: 16.08% [654600/4070000] [160.8/1000.0] [batch_t 1.420 (1.431)] [data_t 0.002] [optim_t 1.417] [lr 0.005000] +2024-04-07 22:32:48,627 - Train: 16.09% [654700/4070000] [160.9/1000.0] [batch_t 1.441 (1.433)] [data_t 0.003] [optim_t 1.438] [lr 0.005000] +2024-04-07 22:35:11,810 - Train: 16.09% [654800/4070000] [160.9/1000.0] [batch_t 1.416 (1.432)] [data_t 0.002] [optim_t 1.414] [lr 0.005000] +2024-04-07 22:37:35,053 - Train: 16.09% [654900/4070000] [160.9/1000.0] [batch_t 1.436 (1.432)] [data_t 0.002] [optim_t 1.433] [lr 0.005000] +2024-04-07 22:39:58,212 - Train: 16.09% [655000/4070000] [160.9/1000.0] [batch_t 1.436 (1.431)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-07 22:42:21,420 - Train: 16.10% [655100/4070000] [161.0/1000.0] [batch_t 1.420 (1.432)] [data_t 0.002] [optim_t 1.418] [lr 0.005000] +2024-04-07 22:44:44,559 - Train: 16.10% [655200/4070000] [161.0/1000.0] [batch_t 1.435 (1.431)] [data_t 0.003] [optim_t 1.432] [lr 0.005000] +2024-04-07 22:46:24,784 - ==> Total time: 5 days, 21:45:55 Eta: 30 days, 18:45:55 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-07 22:47:09,371 - Train: 16.10% [655300/4070000] [161.0/1000.0] [batch_t 1.424 (1.437)] [data_t 0.002] [optim_t 1.421] [lr 0.005000] +2024-04-07 22:49:32,592 - Train: 16.10% [655400/4070000] [161.0/1000.0] [batch_t 1.435 (1.432)] [data_t 0.003] [optim_t 1.432] [lr 0.005000] +2024-04-07 22:51:55,748 - Train: 16.11% [655500/4070000] [161.1/1000.0] [batch_t 1.426 (1.431)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-07 22:54:18,828 - Train: 16.11% [655600/4070000] [161.1/1000.0] [batch_t 1.410 (1.431)] [data_t 0.002] [optim_t 1.408] [lr 0.005000] +2024-04-07 22:56:41,974 - Train: 16.11% [655700/4070000] [161.1/1000.0] [batch_t 1.430 (1.431)] [data_t 0.002] [optim_t 1.428] [lr 0.005000] +2024-04-07 22:59:05,207 - Train: 16.11% [655800/4070000] [161.1/1000.0] [batch_t 1.419 (1.432)] [data_t 0.003] [optim_t 1.416] [lr 0.005000] +2024-04-07 23:01:28,498 - Train: 16.12% [655900/4070000] [161.2/1000.0] [batch_t 1.421 (1.433)] [data_t 0.002] [optim_t 1.418] [lr 0.005000] +2024-04-07 23:03:51,687 - Train: 16.12% [656000/4070000] [161.2/1000.0] [batch_t 1.437 (1.432)] [data_t 0.002] [optim_t 1.434] [lr 0.005000] +2024-04-07 23:06:14,786 - Train: 16.12% [656100/4070000] [161.2/1000.0] [batch_t 1.431 (1.431)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-07 23:08:37,928 - Train: 16.12% [656200/4070000] [161.2/1000.0] [batch_t 1.424 (1.431)] [data_t 0.002] [optim_t 1.422] [lr 0.005000] +2024-04-07 23:11:01,142 - Train: 16.13% [656300/4070000] [161.3/1000.0] [batch_t 1.421 (1.432)] [data_t 0.003] [optim_t 1.418] [lr 0.005000] +2024-04-07 23:13:24,357 - Train: 16.13% [656400/4070000] [161.3/1000.0] [batch_t 1.432 (1.432)] [data_t 0.003] [optim_t 1.430] [lr 0.005000] +2024-04-07 23:15:47,540 - Train: 16.13% [656500/4070000] [161.3/1000.0] [batch_t 1.435 (1.432)] [data_t 0.002] [optim_t 1.433] [lr 0.005000] +2024-04-07 23:18:10,752 - Train: 16.13% [656600/4070000] [161.3/1000.0] [batch_t 1.429 (1.432)] [data_t 0.003] [optim_t 1.426] [lr 0.005000] +2024-04-07 23:20:33,962 - Train: 16.14% [656700/4070000] [161.4/1000.0] [batch_t 1.427 (1.432)] [data_t 0.003] [optim_t 1.424] [lr 0.005000] +2024-04-07 23:22:57,068 - Train: 16.14% [656800/4070000] [161.4/1000.0] [batch_t 1.433 (1.431)] [data_t 0.002] [optim_t 1.431] [lr 0.005000] +2024-04-07 23:25:20,303 - Train: 16.14% [656900/4070000] [161.4/1000.0] [batch_t 1.434 (1.432)] [data_t 0.003] [optim_t 1.431] [lr 0.005000] +2024-04-07 23:27:43,409 - Train: 16.14% [657000/4070000] [161.4/1000.0] [batch_t 1.439 (1.431)] [data_t 0.003] [optim_t 1.437] [lr 0.005000] +2024-04-07 23:30:06,477 - Train: 16.14% [657100/4070000] [161.4/1000.0] [batch_t 1.437 (1.431)] [data_t 0.003] [optim_t 1.434] [lr 0.005000] +2024-04-07 23:32:29,540 - Train: 16.15% [657200/4070000] [161.5/1000.0] [batch_t 1.440 (1.431)] [data_t 0.002] [optim_t 1.438] [lr 0.005000] +2024-04-07 23:34:52,699 - Train: 16.15% [657300/4070000] [161.5/1000.0] [batch_t 1.424 (1.431)] [data_t 0.003] [optim_t 1.422] [lr 0.005000] +2024-04-07 23:37:15,840 - Train: 16.15% [657400/4070000] [161.5/1000.0] [batch_t 1.425 (1.431)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-07 23:39:39,023 - Train: 16.15% [657500/4070000] [161.5/1000.0] [batch_t 1.430 (1.432)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-07 23:42:02,063 - Train: 16.16% [657600/4070000] [161.6/1000.0] [batch_t 1.430 (1.430)] [data_t 0.002] [optim_t 1.427] [lr 0.005000] +2024-04-07 23:44:25,149 - Train: 16.16% [657700/4070000] [161.6/1000.0] [batch_t 1.437 (1.431)] [data_t 0.004] [optim_t 1.433] [lr 0.005000] +2024-04-07 23:46:48,118 - Train: 16.16% [657800/4070000] [161.6/1000.0] [batch_t 1.430 (1.430)] [data_t 0.002] [optim_t 1.427] [lr 0.005000] +2024-04-07 23:49:11,298 - Train: 16.16% [657900/4070000] [161.6/1000.0] [batch_t 1.426 (1.432)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-07 23:51:34,375 - Train: 16.17% [658000/4070000] [161.7/1000.0] [batch_t 1.425 (1.431)] [data_t 0.002] [optim_t 1.422] [lr 0.005000] +2024-04-07 23:53:57,467 - Train: 16.17% [658100/4070000] [161.7/1000.0] [batch_t 1.432 (1.431)] [data_t 0.003] [optim_t 1.429] [lr 0.005000] +2024-04-07 23:56:20,675 - Train: 16.17% [658200/4070000] [161.7/1000.0] [batch_t 1.424 (1.432)] [data_t 0.003] [optim_t 1.421] [lr 0.005000] +2024-04-07 23:58:43,706 - Train: 16.17% [658300/4070000] [161.7/1000.0] [batch_t 1.426 (1.430)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-08 00:01:06,834 - Train: 16.18% [658400/4070000] [161.8/1000.0] [batch_t 1.431 (1.431)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-08 00:03:30,044 - Train: 16.18% [658500/4070000] [161.8/1000.0] [batch_t 1.440 (1.432)] [data_t 0.003] [optim_t 1.437] [lr 0.005000] +2024-04-08 00:05:53,040 - Train: 16.18% [658600/4070000] [161.8/1000.0] [batch_t 1.439 (1.430)] [data_t 0.002] [optim_t 1.437] [lr 0.005000] +2024-04-08 00:08:16,076 - Train: 16.18% [658700/4070000] [161.8/1000.0] [batch_t 1.431 (1.430)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-08 00:10:39,249 - Train: 16.19% [658800/4070000] [161.9/1000.0] [batch_t 1.422 (1.432)] [data_t 0.003] [optim_t 1.420] [lr 0.005000] +2024-04-08 00:13:02,140 - Train: 16.19% [658900/4070000] [161.9/1000.0] [batch_t 1.433 (1.429)] [data_t 0.002] [optim_t 1.431] [lr 0.005000] +2024-04-08 00:15:25,209 - Train: 16.19% [659000/4070000] [161.9/1000.0] [batch_t 1.438 (1.431)] [data_t 0.002] [optim_t 1.435] [lr 0.005000] +2024-04-08 00:17:48,155 - Train: 16.19% [659100/4070000] [161.9/1000.0] [batch_t 1.438 (1.429)] [data_t 0.002] [optim_t 1.436] [lr 0.005000] +2024-04-08 00:20:11,161 - Train: 16.20% [659200/4070000] [162.0/1000.0] [batch_t 1.426 (1.430)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-08 00:22:34,134 - Train: 16.20% [659300/4070000] [162.0/1000.0] [batch_t 1.419 (1.430)] [data_t 0.003] [optim_t 1.416] [lr 0.005000] +2024-04-08 00:23:31,362 - ==> Total time: 5 days, 23:23:02 Eta: 30 days, 21:42:07 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-08 00:24:58,806 - Train: 16.20% [659400/4070000] [162.0/1000.0] [batch_t 1.425 (1.434)] [data_t 0.003] [optim_t 1.422] [lr 0.005000] +2024-04-08 00:27:21,874 - Train: 16.20% [659500/4070000] [162.0/1000.0] [batch_t 1.436 (1.431)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-08 00:29:45,121 - Train: 16.21% [659600/4070000] [162.1/1000.0] [batch_t 1.436 (1.432)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-08 00:32:08,284 - Train: 16.21% [659700/4070000] [162.1/1000.0] [batch_t 1.434 (1.432)] [data_t 0.002] [optim_t 1.432] [lr 0.005000] +2024-04-08 00:34:31,392 - Train: 16.21% [659800/4070000] [162.1/1000.0] [batch_t 1.433 (1.431)] [data_t 0.003] [optim_t 1.430] [lr 0.005000] +2024-04-08 00:36:54,640 - Train: 16.21% [659900/4070000] [162.1/1000.0] [batch_t 1.429 (1.432)] [data_t 0.003] [optim_t 1.426] [lr 0.005000] +2024-04-08 00:39:17,700 - Train: 16.22% [660000/4070000] [162.2/1000.0] [batch_t 1.431 (1.431)] [data_t 0.002] [optim_t 1.429] [lr 0.005000] +2024-04-08 00:41:40,728 - Train: 16.22% [660100/4070000] [162.2/1000.0] [batch_t 1.421 (1.430)] [data_t 0.002] [optim_t 1.419] [lr 0.005000] +2024-04-08 00:44:03,781 - Train: 16.22% [660200/4070000] [162.2/1000.0] [batch_t 1.419 (1.430)] [data_t 0.003] [optim_t 1.417] [lr 0.005000] +2024-04-08 00:46:26,973 - Train: 16.22% [660300/4070000] [162.2/1000.0] [batch_t 1.432 (1.432)] [data_t 0.003] [optim_t 1.429] [lr 0.005000] +2024-04-08 00:48:50,021 - Train: 16.23% [660400/4070000] [162.3/1000.0] [batch_t 1.432 (1.430)] [data_t 0.002] [optim_t 1.430] [lr 0.005000] +2024-04-08 00:51:12,997 - Train: 16.23% [660500/4070000] [162.3/1000.0] [batch_t 1.439 (1.430)] [data_t 0.002] [optim_t 1.437] [lr 0.005000] +2024-04-08 00:53:36,010 - Train: 16.23% [660600/4070000] [162.3/1000.0] [batch_t 1.434 (1.430)] [data_t 0.003] [optim_t 1.431] [lr 0.005000] +2024-04-08 00:55:59,028 - Train: 16.23% [660700/4070000] [162.3/1000.0] [batch_t 1.425 (1.430)] [data_t 0.003] [optim_t 1.422] [lr 0.005000] +2024-04-08 00:58:22,188 - Train: 16.24% [660800/4070000] [162.4/1000.0] [batch_t 1.435 (1.432)] [data_t 0.003] [optim_t 1.432] [lr 0.005000] +2024-04-08 01:00:45,312 - Train: 16.24% [660900/4070000] [162.4/1000.0] [batch_t 1.425 (1.431)] [data_t 0.002] [optim_t 1.423] [lr 0.005000] +2024-04-08 01:03:08,469 - Train: 16.24% [661000/4070000] [162.4/1000.0] [batch_t 1.435 (1.431)] [data_t 0.002] [optim_t 1.432] [lr 0.005000] +2024-04-08 01:05:31,498 - Train: 16.24% [661100/4070000] [162.4/1000.0] [batch_t 1.435 (1.430)] [data_t 0.002] [optim_t 1.433] [lr 0.005000] +2024-04-08 01:07:54,542 - Train: 16.25% [661200/4070000] [162.5/1000.0] [batch_t 1.440 (1.430)] [data_t 0.002] [optim_t 1.437] [lr 0.005000] +2024-04-08 01:10:17,889 - Train: 16.25% [661300/4070000] [162.5/1000.0] [batch_t 1.452 (1.433)] [data_t 0.002] [optim_t 1.449] [lr 0.005000] +2024-04-08 01:12:40,896 - Train: 16.25% [661400/4070000] [162.5/1000.0] [batch_t 1.440 (1.430)] [data_t 0.003] [optim_t 1.437] [lr 0.005000] +2024-04-08 01:15:03,924 - Train: 16.25% [661500/4070000] [162.5/1000.0] [batch_t 1.431 (1.430)] [data_t 0.002] [optim_t 1.428] [lr 0.005000] +2024-04-08 01:17:26,909 - Train: 16.26% [661600/4070000] [162.6/1000.0] [batch_t 1.425 (1.430)] [data_t 0.003] [optim_t 1.422] [lr 0.005000] +2024-04-08 01:19:49,952 - Train: 16.26% [661700/4070000] [162.6/1000.0] [batch_t 1.441 (1.430)] [data_t 0.002] [optim_t 1.439] [lr 0.005000] +2024-04-08 01:22:13,008 - Train: 16.26% [661800/4070000] [162.6/1000.0] [batch_t 1.430 (1.430)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-08 01:24:36,015 - Train: 16.26% [661900/4070000] [162.6/1000.0] [batch_t 1.426 (1.430)] [data_t 0.003] [optim_t 1.424] [lr 0.005000] +2024-04-08 01:26:59,090 - Train: 16.27% [662000/4070000] [162.7/1000.0] [batch_t 1.434 (1.431)] [data_t 0.003] [optim_t 1.432] [lr 0.005000] +2024-04-08 01:29:22,113 - Train: 16.27% [662100/4070000] [162.7/1000.0] [batch_t 1.441 (1.430)] [data_t 0.003] [optim_t 1.438] [lr 0.005000] +2024-04-08 01:31:45,142 - Train: 16.27% [662200/4070000] [162.7/1000.0] [batch_t 1.431 (1.430)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-08 01:34:08,241 - Train: 16.27% [662300/4070000] [162.7/1000.0] [batch_t 1.432 (1.431)] [data_t 0.003] [optim_t 1.429] [lr 0.005000] +2024-04-08 01:36:31,236 - Train: 16.28% [662400/4070000] [162.8/1000.0] [batch_t 1.433 (1.430)] [data_t 0.002] [optim_t 1.431] [lr 0.005000] +2024-04-08 01:38:54,336 - Train: 16.28% [662500/4070000] [162.8/1000.0] [batch_t 1.440 (1.431)] [data_t 0.002] [optim_t 1.438] [lr 0.005000] +2024-04-08 01:41:17,455 - Train: 16.28% [662600/4070000] [162.8/1000.0] [batch_t 1.439 (1.431)] [data_t 0.002] [optim_t 1.436] [lr 0.005000] +2024-04-08 01:43:40,588 - Train: 16.28% [662700/4070000] [162.8/1000.0] [batch_t 1.420 (1.431)] [data_t 0.002] [optim_t 1.418] [lr 0.005000] +2024-04-08 01:46:03,611 - Train: 16.29% [662800/4070000] [162.9/1000.0] [batch_t 1.421 (1.430)] [data_t 0.003] [optim_t 1.418] [lr 0.005000] +2024-04-08 01:48:26,801 - Train: 16.29% [662900/4070000] [162.9/1000.0] [batch_t 1.423 (1.432)] [data_t 0.003] [optim_t 1.420] [lr 0.005000] +2024-04-08 01:50:49,901 - Train: 16.29% [663000/4070000] [162.9/1000.0] [batch_t 1.430 (1.431)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-08 01:53:12,940 - Train: 16.29% [663100/4070000] [162.9/1000.0] [batch_t 1.437 (1.430)] [data_t 0.002] [optim_t 1.434] [lr 0.005000] +2024-04-08 01:55:36,048 - Train: 16.29% [663200/4070000] [162.9/1000.0] [batch_t 1.428 (1.431)] [data_t 0.003] [optim_t 1.425] [lr 0.005000] +2024-04-08 01:57:59,170 - Train: 16.30% [663300/4070000] [163.0/1000.0] [batch_t 1.428 (1.431)] [data_t 0.002] [optim_t 1.426] [lr 0.005000] +2024-04-08 02:00:22,212 - Train: 16.30% [663400/4070000] [163.0/1000.0] [batch_t 1.432 (1.430)] [data_t 0.002] [optim_t 1.429] [lr 0.005000] +2024-04-08 02:00:36,552 - ==> Total time: 6 days, 1:00:07 Eta: 31 days, 0:34:52 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-08 02:02:46,946 - Train: 16.30% [663500/4070000] [163.0/1000.0] [batch_t 1.430 (1.433)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-08 02:05:10,082 - Train: 16.30% [663600/4070000] [163.0/1000.0] [batch_t 1.426 (1.431)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-08 02:07:33,161 - Train: 16.31% [663700/4070000] [163.1/1000.0] [batch_t 1.412 (1.431)] [data_t 0.003] [optim_t 1.409] [lr 0.005000] +2024-04-08 02:09:56,137 - Train: 16.31% [663800/4070000] [163.1/1000.0] [batch_t 1.406 (1.430)] [data_t 0.003] [optim_t 1.404] [lr 0.005000] +2024-04-08 02:12:19,095 - Train: 16.31% [663900/4070000] [163.1/1000.0] [batch_t 1.434 (1.429)] [data_t 0.002] [optim_t 1.432] [lr 0.005000] +2024-04-08 02:14:42,162 - Train: 16.31% [664000/4070000] [163.1/1000.0] [batch_t 1.438 (1.431)] [data_t 0.003] [optim_t 1.436] [lr 0.005000] +2024-04-08 02:17:05,233 - Train: 16.32% [664100/4070000] [163.2/1000.0] [batch_t 1.421 (1.431)] [data_t 0.002] [optim_t 1.418] [lr 0.005000] +2024-04-08 02:19:28,275 - Train: 16.32% [664200/4070000] [163.2/1000.0] [batch_t 1.424 (1.430)] [data_t 0.002] [optim_t 1.421] [lr 0.005000] +2024-04-08 02:21:51,376 - Train: 16.32% [664300/4070000] [163.2/1000.0] [batch_t 1.444 (1.431)] [data_t 0.003] [optim_t 1.441] [lr 0.005000] +2024-04-08 02:24:14,426 - Train: 16.32% [664400/4070000] [163.2/1000.0] [batch_t 1.441 (1.430)] [data_t 0.002] [optim_t 1.439] [lr 0.005000] +2024-04-08 02:26:37,521 - Train: 16.33% [664500/4070000] [163.3/1000.0] [batch_t 1.431 (1.431)] [data_t 0.003] [optim_t 1.429] [lr 0.005000] +2024-04-08 02:29:00,496 - Train: 16.33% [664600/4070000] [163.3/1000.0] [batch_t 1.437 (1.430)] [data_t 0.002] [optim_t 1.435] [lr 0.005000] +2024-04-08 02:31:23,629 - Train: 16.33% [664700/4070000] [163.3/1000.0] [batch_t 1.441 (1.431)] [data_t 0.002] [optim_t 1.439] [lr 0.005000] +2024-04-08 02:33:46,649 - Train: 16.33% [664800/4070000] [163.3/1000.0] [batch_t 1.456 (1.430)] [data_t 0.002] [optim_t 1.453] [lr 0.005000] +2024-04-08 02:36:09,733 - Train: 16.34% [664900/4070000] [163.4/1000.0] [batch_t 1.429 (1.431)] [data_t 0.002] [optim_t 1.427] [lr 0.005000] +2024-04-08 02:38:32,816 - Train: 16.34% [665000/4070000] [163.4/1000.0] [batch_t 1.436 (1.431)] [data_t 0.002] [optim_t 1.433] [lr 0.005000] +2024-04-08 02:40:55,948 - Train: 16.34% [665100/4070000] [163.4/1000.0] [batch_t 1.421 (1.431)] [data_t 0.002] [optim_t 1.418] [lr 0.005000] +2024-04-08 02:43:19,007 - Train: 16.34% [665200/4070000] [163.4/1000.0] [batch_t 1.432 (1.430)] [data_t 0.003] [optim_t 1.429] [lr 0.005000] +2024-04-08 02:45:41,992 - Train: 16.35% [665300/4070000] [163.5/1000.0] [batch_t 1.422 (1.430)] [data_t 0.003] [optim_t 1.419] [lr 0.005000] +2024-04-08 02:48:05,040 - Train: 16.35% [665400/4070000] [163.5/1000.0] [batch_t 1.429 (1.430)] [data_t 0.002] [optim_t 1.427] [lr 0.005000] +2024-04-08 02:50:28,257 - Train: 16.35% [665500/4070000] [163.5/1000.0] [batch_t 1.433 (1.432)] [data_t 0.003] [optim_t 1.430] [lr 0.005000] +2024-04-08 02:52:51,341 - Train: 16.35% [665600/4070000] [163.5/1000.0] [batch_t 1.434 (1.431)] [data_t 0.002] [optim_t 1.431] [lr 0.005000] +2024-04-08 02:55:14,413 - Train: 16.36% [665700/4070000] [163.6/1000.0] [batch_t 1.434 (1.431)] [data_t 0.002] [optim_t 1.432] [lr 0.005000] +2024-04-08 02:57:37,507 - Train: 16.36% [665800/4070000] [163.6/1000.0] [batch_t 1.438 (1.431)] [data_t 0.002] [optim_t 1.436] [lr 0.005000] +2024-04-08 03:00:00,659 - Train: 16.36% [665900/4070000] [163.6/1000.0] [batch_t 1.433 (1.431)] [data_t 0.002] [optim_t 1.431] [lr 0.005000] +2024-04-08 03:02:23,735 - Train: 16.36% [666000/4070000] [163.6/1000.0] [batch_t 1.426 (1.431)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-08 03:04:46,798 - Train: 16.37% [666100/4070000] [163.7/1000.0] [batch_t 1.414 (1.431)] [data_t 0.003] [optim_t 1.411] [lr 0.005000] +2024-04-08 03:07:09,757 - Train: 16.37% [666200/4070000] [163.7/1000.0] [batch_t 1.447 (1.429)] [data_t 0.004] [optim_t 1.443] [lr 0.005000] +2024-04-08 03:09:32,884 - Train: 16.37% [666300/4070000] [163.7/1000.0] [batch_t 1.424 (1.431)] [data_t 0.002] [optim_t 1.422] [lr 0.005000] +2024-04-08 03:11:55,884 - Train: 16.37% [666400/4070000] [163.7/1000.0] [batch_t 1.429 (1.430)] [data_t 0.003] [optim_t 1.426] [lr 0.005000] +2024-04-08 03:14:18,950 - Train: 16.38% [666500/4070000] [163.8/1000.0] [batch_t 1.427 (1.431)] [data_t 0.003] [optim_t 1.424] [lr 0.005000] +2024-04-08 03:16:42,081 - Train: 16.38% [666600/4070000] [163.8/1000.0] [batch_t 1.430 (1.431)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-08 03:19:05,124 - Train: 16.38% [666700/4070000] [163.8/1000.0] [batch_t 1.440 (1.430)] [data_t 0.002] [optim_t 1.438] [lr 0.005000] +2024-04-08 03:21:28,344 - Train: 16.38% [666800/4070000] [163.8/1000.0] [batch_t 1.434 (1.432)] [data_t 0.003] [optim_t 1.431] [lr 0.005000] +2024-04-08 03:23:51,467 - Train: 16.39% [666900/4070000] [163.9/1000.0] [batch_t 1.435 (1.431)] [data_t 0.003] [optim_t 1.432] [lr 0.005000] +2024-04-08 03:26:14,581 - Train: 16.39% [667000/4070000] [163.9/1000.0] [batch_t 1.431 (1.431)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-08 03:28:37,717 - Train: 16.39% [667100/4070000] [163.9/1000.0] [batch_t 1.434 (1.431)] [data_t 0.002] [optim_t 1.431] [lr 0.005000] +2024-04-08 03:31:00,817 - Train: 16.39% [667200/4070000] [163.9/1000.0] [batch_t 1.439 (1.431)] [data_t 0.003] [optim_t 1.436] [lr 0.005000] +2024-04-08 03:33:23,920 - Train: 16.40% [667300/4070000] [164.0/1000.0] [batch_t 1.429 (1.431)] [data_t 0.002] [optim_t 1.427] [lr 0.005000] +2024-04-08 03:35:47,072 - Train: 16.40% [667400/4070000] [164.0/1000.0] [batch_t 1.426 (1.431)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-08 03:37:41,617 - ==> Total time: 6 days, 2:37:12 Eta: 31 days, 3:24:18 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-08 03:38:11,865 - Train: 16.40% [667500/4070000] [164.0/1000.0] [batch_t 1.431 (1.442)] [data_t 0.002] [optim_t 1.428] [lr 0.005000] +2024-04-08 03:40:34,888 - Train: 16.40% [667600/4070000] [164.0/1000.0] [batch_t 1.436 (1.430)] [data_t 0.003] [optim_t 1.434] [lr 0.005000] +2024-04-08 03:42:57,869 - Train: 16.41% [667700/4070000] [164.1/1000.0] [batch_t 1.425 (1.430)] [data_t 0.002] [optim_t 1.423] [lr 0.005000] +2024-04-08 03:45:21,059 - Train: 16.41% [667800/4070000] [164.1/1000.0] [batch_t 1.430 (1.432)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-08 03:47:44,106 - Train: 16.41% [667900/4070000] [164.1/1000.0] [batch_t 1.436 (1.430)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-08 03:50:07,380 - Train: 16.41% [668000/4070000] [164.1/1000.0] [batch_t 1.421 (1.433)] [data_t 0.002] [optim_t 1.419] [lr 0.005000] +2024-04-08 03:52:30,335 - Train: 16.42% [668100/4070000] [164.2/1000.0] [batch_t 1.422 (1.429)] [data_t 0.003] [optim_t 1.419] [lr 0.005000] +2024-04-08 03:54:53,330 - Train: 16.42% [668200/4070000] [164.2/1000.0] [batch_t 1.430 (1.430)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-08 03:57:16,311 - Train: 16.42% [668300/4070000] [164.2/1000.0] [batch_t 1.431 (1.430)] [data_t 0.003] [optim_t 1.429] [lr 0.005000] +2024-04-08 03:59:39,405 - Train: 16.42% [668400/4070000] [164.2/1000.0] [batch_t 1.422 (1.431)] [data_t 0.003] [optim_t 1.420] [lr 0.005000] +2024-04-08 04:02:02,480 - Train: 16.43% [668500/4070000] [164.3/1000.0] [batch_t 1.424 (1.431)] [data_t 0.003] [optim_t 1.422] [lr 0.005000] +2024-04-08 04:04:25,507 - Train: 16.43% [668600/4070000] [164.3/1000.0] [batch_t 1.443 (1.430)] [data_t 0.003] [optim_t 1.441] [lr 0.005000] +2024-04-08 04:06:48,631 - Train: 16.43% [668700/4070000] [164.3/1000.0] [batch_t 1.421 (1.431)] [data_t 0.003] [optim_t 1.418] [lr 0.005000] +2024-04-08 04:09:11,798 - Train: 16.43% [668800/4070000] [164.3/1000.0] [batch_t 1.436 (1.432)] [data_t 0.002] [optim_t 1.433] [lr 0.005000] +2024-04-08 04:11:34,802 - Train: 16.43% [668900/4070000] [164.3/1000.0] [batch_t 1.440 (1.430)] [data_t 0.003] [optim_t 1.438] [lr 0.005000] +2024-04-08 04:13:57,894 - Train: 16.44% [669000/4070000] [164.4/1000.0] [batch_t 1.424 (1.431)] [data_t 0.003] [optim_t 1.422] [lr 0.005000] +2024-04-08 04:16:20,987 - Train: 16.44% [669100/4070000] [164.4/1000.0] [batch_t 1.423 (1.431)] [data_t 0.003] [optim_t 1.421] [lr 0.005000] +2024-04-08 04:18:44,091 - Train: 16.44% [669200/4070000] [164.4/1000.0] [batch_t 1.436 (1.431)] [data_t 0.002] [optim_t 1.434] [lr 0.005000] +2024-04-08 04:21:07,062 - Train: 16.44% [669300/4070000] [164.4/1000.0] [batch_t 1.426 (1.430)] [data_t 0.003] [optim_t 1.424] [lr 0.005000] +2024-04-08 04:23:30,140 - Train: 16.45% [669400/4070000] [164.5/1000.0] [batch_t 1.418 (1.431)] [data_t 0.002] [optim_t 1.416] [lr 0.005000] +2024-04-08 04:25:53,235 - Train: 16.45% [669500/4070000] [164.5/1000.0] [batch_t 1.433 (1.431)] [data_t 0.003] [optim_t 1.431] [lr 0.005000] +2024-04-08 04:28:16,421 - Train: 16.45% [669600/4070000] [164.5/1000.0] [batch_t 1.432 (1.432)] [data_t 0.003] [optim_t 1.429] [lr 0.005000] +2024-04-08 04:30:39,457 - Train: 16.45% [669700/4070000] [164.5/1000.0] [batch_t 1.420 (1.430)] [data_t 0.002] [optim_t 1.417] [lr 0.005000] +2024-04-08 04:33:02,505 - Train: 16.46% [669800/4070000] [164.6/1000.0] [batch_t 1.430 (1.430)] [data_t 0.002] [optim_t 1.427] [lr 0.005000] +2024-04-08 04:35:25,655 - Train: 16.46% [669900/4070000] [164.6/1000.0] [batch_t 1.436 (1.431)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-08 04:37:48,692 - Train: 16.46% [670000/4070000] [164.6/1000.0] [batch_t 1.434 (1.430)] [data_t 0.003] [optim_t 1.431] [lr 0.005000] +2024-04-08 04:40:11,576 - Train: 16.46% [670100/4070000] [164.6/1000.0] [batch_t 1.435 (1.429)] [data_t 0.003] [optim_t 1.432] [lr 0.005000] +2024-04-08 04:42:34,582 - Train: 16.47% [670200/4070000] [164.7/1000.0] [batch_t 1.431 (1.430)] [data_t 0.002] [optim_t 1.429] [lr 0.005000] +2024-04-08 04:44:57,521 - Train: 16.47% [670300/4070000] [164.7/1000.0] [batch_t 1.420 (1.429)] [data_t 0.002] [optim_t 1.418] [lr 0.005000] +2024-04-08 04:47:20,724 - Train: 16.47% [670400/4070000] [164.7/1000.0] [batch_t 1.416 (1.432)] [data_t 0.002] [optim_t 1.413] [lr 0.005000] +2024-04-08 04:49:43,811 - Train: 16.47% [670500/4070000] [164.7/1000.0] [batch_t 1.434 (1.431)] [data_t 0.003] [optim_t 1.431] [lr 0.005000] +2024-04-08 04:52:06,788 - Train: 16.48% [670600/4070000] [164.8/1000.0] [batch_t 1.425 (1.430)] [data_t 0.002] [optim_t 1.423] [lr 0.005000] +2024-04-08 04:54:29,972 - Train: 16.48% [670700/4070000] [164.8/1000.0] [batch_t 1.440 (1.432)] [data_t 0.002] [optim_t 1.438] [lr 0.005000] +2024-04-08 04:56:53,164 - Train: 16.48% [670800/4070000] [164.8/1000.0] [batch_t 1.439 (1.432)] [data_t 0.002] [optim_t 1.437] [lr 0.005000] +2024-04-08 04:59:16,325 - Train: 16.48% [670900/4070000] [164.8/1000.0] [batch_t 1.437 (1.432)] [data_t 0.002] [optim_t 1.435] [lr 0.005000] +2024-04-08 05:01:39,423 - Train: 16.49% [671000/4070000] [164.9/1000.0] [batch_t 1.425 (1.431)] [data_t 0.002] [optim_t 1.423] [lr 0.005000] +2024-04-08 05:04:02,579 - Train: 16.49% [671100/4070000] [164.9/1000.0] [batch_t 1.433 (1.431)] [data_t 0.002] [optim_t 1.431] [lr 0.005000] +2024-04-08 05:06:25,668 - Train: 16.49% [671200/4070000] [164.9/1000.0] [batch_t 1.428 (1.431)] [data_t 0.003] [optim_t 1.425] [lr 0.005000] +2024-04-08 05:08:48,814 - Train: 16.49% [671300/4070000] [164.9/1000.0] [batch_t 1.432 (1.431)] [data_t 0.002] [optim_t 1.430] [lr 0.005000] +2024-04-08 05:11:11,952 - Train: 16.50% [671400/4070000] [165.0/1000.0] [batch_t 1.424 (1.431)] [data_t 0.002] [optim_t 1.422] [lr 0.005000] +2024-04-08 05:13:35,167 - Train: 16.50% [671500/4070000] [165.0/1000.0] [batch_t 1.436 (1.432)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-08 05:14:46,643 - ==> Total time: 6 days, 4:14:17 Eta: 31 days, 6:10:30 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-08 05:16:00,086 - Train: 16.50% [671600/4070000] [165.0/1000.0] [batch_t 1.420 (1.438)] [data_t 0.002] [optim_t 1.417] [lr 0.005000] +2024-04-08 05:18:23,305 - Train: 16.50% [671700/4070000] [165.0/1000.0] [batch_t 1.430 (1.432)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-08 05:20:46,398 - Train: 16.51% [671800/4070000] [165.1/1000.0] [batch_t 1.429 (1.431)] [data_t 0.003] [optim_t 1.426] [lr 0.005000] +2024-04-08 05:23:09,438 - Train: 16.51% [671900/4070000] [165.1/1000.0] [batch_t 1.421 (1.430)] [data_t 0.003] [optim_t 1.418] [lr 0.005000] +2024-04-08 05:25:32,454 - Train: 16.51% [672000/4070000] [165.1/1000.0] [batch_t 1.425 (1.430)] [data_t 0.002] [optim_t 1.423] [lr 0.005000] +2024-04-08 05:27:55,502 - Train: 16.51% [672100/4070000] [165.1/1000.0] [batch_t 1.445 (1.430)] [data_t 0.002] [optim_t 1.443] [lr 0.005000] +2024-04-08 05:30:18,522 - Train: 16.52% [672200/4070000] [165.2/1000.0] [batch_t 1.432 (1.430)] [data_t 0.003] [optim_t 1.429] [lr 0.005000] +2024-04-08 05:32:41,665 - Train: 16.52% [672300/4070000] [165.2/1000.0] [batch_t 1.431 (1.431)] [data_t 0.002] [optim_t 1.428] [lr 0.005000] +2024-04-08 05:35:04,860 - Train: 16.52% [672400/4070000] [165.2/1000.0] [batch_t 1.437 (1.432)] [data_t 0.003] [optim_t 1.434] [lr 0.005000] +2024-04-08 05:37:28,001 - Train: 16.52% [672500/4070000] [165.2/1000.0] [batch_t 1.442 (1.431)] [data_t 0.003] [optim_t 1.439] [lr 0.005000] +2024-04-08 05:39:51,119 - Train: 16.53% [672600/4070000] [165.3/1000.0] [batch_t 1.430 (1.431)] [data_t 0.002] [optim_t 1.427] [lr 0.005000] +2024-04-08 05:42:14,164 - Train: 16.53% [672700/4070000] [165.3/1000.0] [batch_t 1.436 (1.430)] [data_t 0.002] [optim_t 1.434] [lr 0.005000] +2024-04-08 05:44:37,257 - Train: 16.53% [672800/4070000] [165.3/1000.0] [batch_t 1.425 (1.431)] [data_t 0.002] [optim_t 1.423] [lr 0.005000] +2024-04-08 05:47:00,317 - Train: 16.53% [672900/4070000] [165.3/1000.0] [batch_t 1.440 (1.430)] [data_t 0.002] [optim_t 1.438] [lr 0.005000] +2024-04-08 05:49:23,442 - Train: 16.54% [673000/4070000] [165.4/1000.0] [batch_t 1.428 (1.431)] [data_t 0.003] [optim_t 1.426] [lr 0.005000] +2024-04-08 05:51:46,457 - Train: 16.54% [673100/4070000] [165.4/1000.0] [batch_t 1.410 (1.430)] [data_t 0.003] [optim_t 1.407] [lr 0.005000] +2024-04-08 05:54:09,575 - Train: 16.54% [673200/4070000] [165.4/1000.0] [batch_t 1.431 (1.431)] [data_t 0.002] [optim_t 1.429] [lr 0.005000] +2024-04-08 05:56:32,678 - Train: 16.54% [673300/4070000] [165.4/1000.0] [batch_t 1.420 (1.431)] [data_t 0.002] [optim_t 1.418] [lr 0.005000] +2024-04-08 05:58:55,676 - Train: 16.55% [673400/4070000] [165.5/1000.0] [batch_t 1.441 (1.430)] [data_t 0.003] [optim_t 1.438] [lr 0.005000] +2024-04-08 06:01:18,646 - Train: 16.55% [673500/4070000] [165.5/1000.0] [batch_t 1.425 (1.430)] [data_t 0.003] [optim_t 1.422] [lr 0.005000] +2024-04-08 06:03:41,720 - Train: 16.55% [673600/4070000] [165.5/1000.0] [batch_t 1.452 (1.431)] [data_t 0.002] [optim_t 1.449] [lr 0.005000] +2024-04-08 06:06:04,724 - Train: 16.55% [673700/4070000] [165.5/1000.0] [batch_t 1.417 (1.430)] [data_t 0.003] [optim_t 1.414] [lr 0.005000] +2024-04-08 06:08:27,603 - Train: 16.56% [673800/4070000] [165.6/1000.0] [batch_t 1.426 (1.429)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-08 06:10:50,623 - Train: 16.56% [673900/4070000] [165.6/1000.0] [batch_t 1.444 (1.430)] [data_t 0.002] [optim_t 1.441] [lr 0.005000] +2024-04-08 06:13:13,737 - Train: 16.56% [674000/4070000] [165.6/1000.0] [batch_t 1.440 (1.431)] [data_t 0.002] [optim_t 1.438] [lr 0.005000] +2024-04-08 06:15:36,813 - Train: 16.56% [674100/4070000] [165.6/1000.0] [batch_t 1.429 (1.431)] [data_t 0.002] [optim_t 1.427] [lr 0.005000] +2024-04-08 06:17:59,868 - Train: 16.57% [674200/4070000] [165.7/1000.0] [batch_t 1.432 (1.430)] [data_t 0.003] [optim_t 1.429] [lr 0.005000] +2024-04-08 06:20:22,901 - Train: 16.57% [674300/4070000] [165.7/1000.0] [batch_t 1.440 (1.430)] [data_t 0.003] [optim_t 1.437] [lr 0.005000] +2024-04-08 06:22:45,876 - Train: 16.57% [674400/4070000] [165.7/1000.0] [batch_t 1.431 (1.430)] [data_t 0.002] [optim_t 1.429] [lr 0.005000] +2024-04-08 06:25:08,864 - Train: 16.57% [674500/4070000] [165.7/1000.0] [batch_t 1.433 (1.430)] [data_t 0.003] [optim_t 1.431] [lr 0.005000] +2024-04-08 06:27:31,984 - Train: 16.57% [674600/4070000] [165.7/1000.0] [batch_t 1.430 (1.431)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-08 06:29:55,060 - Train: 16.58% [674700/4070000] [165.8/1000.0] [batch_t 1.411 (1.431)] [data_t 0.003] [optim_t 1.409] [lr 0.005000] +2024-04-08 06:32:18,249 - Train: 16.58% [674800/4070000] [165.8/1000.0] [batch_t 1.426 (1.432)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-08 06:34:41,196 - Train: 16.58% [674900/4070000] [165.8/1000.0] [batch_t 1.431 (1.429)] [data_t 0.002] [optim_t 1.429] [lr 0.005000] +2024-04-08 06:37:04,203 - Train: 16.58% [675000/4070000] [165.8/1000.0] [batch_t 1.438 (1.430)] [data_t 0.003] [optim_t 1.435] [lr 0.005000] +2024-04-08 06:39:27,422 - Train: 16.59% [675100/4070000] [165.9/1000.0] [batch_t 1.425 (1.432)] [data_t 0.003] [optim_t 1.422] [lr 0.005000] +2024-04-08 06:41:50,470 - Train: 16.59% [675200/4070000] [165.9/1000.0] [batch_t 1.427 (1.430)] [data_t 0.002] [optim_t 1.425] [lr 0.005000] +2024-04-08 06:44:13,535 - Train: 16.59% [675300/4070000] [165.9/1000.0] [batch_t 1.421 (1.431)] [data_t 0.003] [optim_t 1.418] [lr 0.005000] +2024-04-08 06:46:36,523 - Train: 16.59% [675400/4070000] [165.9/1000.0] [batch_t 1.430 (1.430)] [data_t 0.002] [optim_t 1.428] [lr 0.005000] +2024-04-08 06:48:59,656 - Train: 16.60% [675500/4070000] [166.0/1000.0] [batch_t 1.441 (1.431)] [data_t 0.002] [optim_t 1.439] [lr 0.005000] +2024-04-08 06:51:22,818 - Train: 16.60% [675600/4070000] [166.0/1000.0] [batch_t 1.406 (1.432)] [data_t 0.003] [optim_t 1.404] [lr 0.005000] +2024-04-08 06:51:51,423 - ==> Total time: 6 days, 5:51:22 Eta: 31 days, 8:53:30 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-08 06:53:47,635 - Train: 16.60% [675700/4070000] [166.0/1000.0] [batch_t 1.424 (1.434)] [data_t 0.003] [optim_t 1.421] [lr 0.005000] +2024-04-08 06:56:10,611 - Train: 16.60% [675800/4070000] [166.0/1000.0] [batch_t 1.431 (1.430)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-08 06:58:33,679 - Train: 16.61% [675900/4070000] [166.1/1000.0] [batch_t 1.445 (1.431)] [data_t 0.002] [optim_t 1.443] [lr 0.005000] +2024-04-08 07:00:56,749 - Train: 16.61% [676000/4070000] [166.1/1000.0] [batch_t 1.435 (1.431)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-08 07:03:19,914 - Train: 16.61% [676100/4070000] [166.1/1000.0] [batch_t 1.430 (1.432)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-08 07:05:43,067 - Train: 16.61% [676200/4070000] [166.1/1000.0] [batch_t 1.426 (1.431)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-08 07:08:06,224 - Train: 16.62% [676300/4070000] [166.2/1000.0] [batch_t 1.432 (1.431)] [data_t 0.003] [optim_t 1.429] [lr 0.005000] +2024-04-08 07:10:29,288 - Train: 16.62% [676400/4070000] [166.2/1000.0] [batch_t 1.426 (1.431)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-08 07:12:52,497 - Train: 16.62% [676500/4070000] [166.2/1000.0] [batch_t 1.430 (1.432)] [data_t 0.002] [optim_t 1.427] [lr 0.005000] +2024-04-08 07:15:15,679 - Train: 16.62% [676600/4070000] [166.2/1000.0] [batch_t 1.434 (1.432)] [data_t 0.002] [optim_t 1.432] [lr 0.005000] +2024-04-08 07:17:38,873 - Train: 16.63% [676700/4070000] [166.3/1000.0] [batch_t 1.436 (1.432)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-08 07:20:02,079 - Train: 16.63% [676800/4070000] [166.3/1000.0] [batch_t 1.441 (1.432)] [data_t 0.003] [optim_t 1.438] [lr 0.005000] +2024-04-08 07:22:25,071 - Train: 16.63% [676900/4070000] [166.3/1000.0] [batch_t 1.435 (1.430)] [data_t 0.002] [optim_t 1.432] [lr 0.005000] +2024-04-08 07:24:48,161 - Train: 16.63% [677000/4070000] [166.3/1000.0] [batch_t 1.433 (1.431)] [data_t 0.002] [optim_t 1.431] [lr 0.005000] +2024-04-08 07:27:11,267 - Train: 16.64% [677100/4070000] [166.4/1000.0] [batch_t 1.439 (1.431)] [data_t 0.003] [optim_t 1.437] [lr 0.005000] +2024-04-08 07:29:34,361 - Train: 16.64% [677200/4070000] [166.4/1000.0] [batch_t 1.441 (1.431)] [data_t 0.002] [optim_t 1.439] [lr 0.005000] +2024-04-08 07:31:57,645 - Train: 16.64% [677300/4070000] [166.4/1000.0] [batch_t 1.440 (1.433)] [data_t 0.002] [optim_t 1.438] [lr 0.005000] +2024-04-08 07:34:20,819 - Train: 16.64% [677400/4070000] [166.4/1000.0] [batch_t 1.426 (1.432)] [data_t 0.003] [optim_t 1.424] [lr 0.005000] +2024-04-08 07:36:43,900 - Train: 16.65% [677500/4070000] [166.5/1000.0] [batch_t 1.428 (1.431)] [data_t 0.003] [optim_t 1.425] [lr 0.005000] +2024-04-08 07:39:07,000 - Train: 16.65% [677600/4070000] [166.5/1000.0] [batch_t 1.439 (1.431)] [data_t 0.002] [optim_t 1.437] [lr 0.005000] +2024-04-08 07:41:30,056 - Train: 16.65% [677700/4070000] [166.5/1000.0] [batch_t 1.430 (1.430)] [data_t 0.002] [optim_t 1.427] [lr 0.005000] +2024-04-08 07:43:53,042 - Train: 16.65% [677800/4070000] [166.5/1000.0] [batch_t 1.430 (1.430)] [data_t 0.002] [optim_t 1.428] [lr 0.005000] +2024-04-08 07:46:15,996 - Train: 16.66% [677900/4070000] [166.6/1000.0] [batch_t 1.422 (1.429)] [data_t 0.003] [optim_t 1.419] [lr 0.005000] +2024-04-08 07:48:39,086 - Train: 16.66% [678000/4070000] [166.6/1000.0] [batch_t 1.417 (1.431)] [data_t 0.002] [optim_t 1.415] [lr 0.005000] +2024-04-08 07:51:02,225 - Train: 16.66% [678100/4070000] [166.6/1000.0] [batch_t 1.451 (1.431)] [data_t 0.002] [optim_t 1.449] [lr 0.005000] +2024-04-08 07:53:25,111 - Train: 16.66% [678200/4070000] [166.6/1000.0] [batch_t 1.430 (1.429)] [data_t 0.002] [optim_t 1.427] [lr 0.005000] +2024-04-08 07:55:48,238 - Train: 16.67% [678300/4070000] [166.7/1000.0] [batch_t 1.413 (1.431)] [data_t 0.002] [optim_t 1.410] [lr 0.005000] +2024-04-08 07:58:11,312 - Train: 16.67% [678400/4070000] [166.7/1000.0] [batch_t 1.434 (1.431)] [data_t 0.002] [optim_t 1.431] [lr 0.005000] +2024-04-08 08:00:34,514 - Train: 16.67% [678500/4070000] [166.7/1000.0] [batch_t 1.446 (1.432)] [data_t 0.003] [optim_t 1.443] [lr 0.005000] +2024-04-08 08:02:57,420 - Train: 16.67% [678600/4070000] [166.7/1000.0] [batch_t 1.429 (1.429)] [data_t 0.003] [optim_t 1.426] [lr 0.005000] +2024-04-08 08:05:20,351 - Train: 16.68% [678700/4070000] [166.8/1000.0] [batch_t 1.431 (1.429)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-08 08:07:43,344 - Train: 16.68% [678800/4070000] [166.8/1000.0] [batch_t 1.407 (1.430)] [data_t 0.003] [optim_t 1.404] [lr 0.005000] +2024-04-08 08:10:06,416 - Train: 16.68% [678900/4070000] [166.8/1000.0] [batch_t 1.431 (1.431)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-08 08:12:29,419 - Train: 16.68% [679000/4070000] [166.8/1000.0] [batch_t 1.431 (1.430)] [data_t 0.002] [optim_t 1.429] [lr 0.005000] +2024-04-08 08:14:52,661 - Train: 16.69% [679100/4070000] [166.9/1000.0] [batch_t 1.444 (1.432)] [data_t 0.003] [optim_t 1.441] [lr 0.005000] +2024-04-08 08:17:15,728 - Train: 16.69% [679200/4070000] [166.9/1000.0] [batch_t 1.424 (1.431)] [data_t 0.003] [optim_t 1.421] [lr 0.005000] +2024-04-08 08:19:38,792 - Train: 16.69% [679300/4070000] [166.9/1000.0] [batch_t 1.415 (1.431)] [data_t 0.003] [optim_t 1.412] [lr 0.005000] +2024-04-08 08:22:01,816 - Train: 16.69% [679400/4070000] [166.9/1000.0] [batch_t 1.408 (1.430)] [data_t 0.003] [optim_t 1.405] [lr 0.005000] +2024-04-08 08:24:25,017 - Train: 16.70% [679500/4070000] [167.0/1000.0] [batch_t 1.434 (1.432)] [data_t 0.003] [optim_t 1.431] [lr 0.005000] +2024-04-08 08:26:48,052 - Train: 16.70% [679600/4070000] [167.0/1000.0] [batch_t 1.426 (1.430)] [data_t 0.002] [optim_t 1.424] [lr 0.005000] +2024-04-08 08:28:56,842 - ==> Total time: 6 days, 7:28:27 Eta: 31 days, 11:33:27 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-08 08:29:12,785 - Train: 16.70% [679700/4070000] [167.0/1000.0] [batch_t 1.435 (1.446)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-08 08:31:35,849 - Train: 16.70% [679800/4070000] [167.0/1000.0] [batch_t 1.428 (1.431)] [data_t 0.002] [optim_t 1.426] [lr 0.005000] +2024-04-08 08:33:58,952 - Train: 16.71% [679900/4070000] [167.1/1000.0] [batch_t 1.429 (1.431)] [data_t 0.002] [optim_t 1.427] [lr 0.005000] +2024-04-08 08:36:22,034 - Train: 16.71% [680000/4070000] [167.1/1000.0] [batch_t 1.426 (1.431)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-08 08:38:45,115 - Train: 16.71% [680100/4070000] [167.1/1000.0] [batch_t 1.438 (1.431)] [data_t 0.002] [optim_t 1.436] [lr 0.005000] +2024-04-08 08:41:08,189 - Train: 16.71% [680200/4070000] [167.1/1000.0] [batch_t 1.439 (1.431)] [data_t 0.002] [optim_t 1.437] [lr 0.005000] +2024-04-08 08:43:31,157 - Train: 16.71% [680300/4070000] [167.1/1000.0] [batch_t 1.427 (1.430)] [data_t 0.002] [optim_t 1.425] [lr 0.005000] +2024-04-08 08:45:54,085 - Train: 16.72% [680400/4070000] [167.2/1000.0] [batch_t 1.435 (1.429)] [data_t 0.002] [optim_t 1.432] [lr 0.005000] +2024-04-08 08:48:17,138 - Train: 16.72% [680500/4070000] [167.2/1000.0] [batch_t 1.442 (1.430)] [data_t 0.002] [optim_t 1.439] [lr 0.005000] +2024-04-08 08:50:40,368 - Train: 16.72% [680600/4070000] [167.2/1000.0] [batch_t 1.441 (1.432)] [data_t 0.002] [optim_t 1.439] [lr 0.005000] +2024-04-08 08:53:03,445 - Train: 16.72% [680700/4070000] [167.2/1000.0] [batch_t 1.412 (1.431)] [data_t 0.003] [optim_t 1.409] [lr 0.005000] +2024-04-08 08:55:26,664 - Train: 16.73% [680800/4070000] [167.3/1000.0] [batch_t 1.453 (1.432)] [data_t 0.002] [optim_t 1.451] [lr 0.005000] +2024-04-08 08:57:49,682 - Train: 16.73% [680900/4070000] [167.3/1000.0] [batch_t 1.431 (1.430)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-08 09:00:12,715 - Train: 16.73% [681000/4070000] [167.3/1000.0] [batch_t 1.426 (1.430)] [data_t 0.003] [optim_t 1.424] [lr 0.005000] +2024-04-08 09:02:35,829 - Train: 16.73% [681100/4070000] [167.3/1000.0] [batch_t 1.433 (1.431)] [data_t 0.002] [optim_t 1.431] [lr 0.005000] +2024-04-08 09:04:58,905 - Train: 16.74% [681200/4070000] [167.4/1000.0] [batch_t 1.429 (1.431)] [data_t 0.002] [optim_t 1.426] [lr 0.005000] +2024-04-08 09:07:21,894 - Train: 16.74% [681300/4070000] [167.4/1000.0] [batch_t 1.423 (1.430)] [data_t 0.003] [optim_t 1.420] [lr 0.005000] +2024-04-08 09:09:44,929 - Train: 16.74% [681400/4070000] [167.4/1000.0] [batch_t 1.435 (1.430)] [data_t 0.002] [optim_t 1.433] [lr 0.005000] +2024-04-08 09:12:08,045 - Train: 16.74% [681500/4070000] [167.4/1000.0] [batch_t 1.436 (1.431)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-08 09:14:31,037 - Train: 16.75% [681600/4070000] [167.5/1000.0] [batch_t 1.431 (1.430)] [data_t 0.002] [optim_t 1.429] [lr 0.005000] +2024-04-08 09:16:54,075 - Train: 16.75% [681700/4070000] [167.5/1000.0] [batch_t 1.416 (1.430)] [data_t 0.003] [optim_t 1.414] [lr 0.005000] +2024-04-08 09:19:17,274 - Train: 16.75% [681800/4070000] [167.5/1000.0] [batch_t 1.451 (1.432)] [data_t 0.002] [optim_t 1.449] [lr 0.005000] +2024-04-08 09:21:40,310 - Train: 16.75% [681900/4070000] [167.5/1000.0] [batch_t 1.429 (1.430)] [data_t 0.003] [optim_t 1.426] [lr 0.005000] +2024-04-08 09:24:03,566 - Train: 16.76% [682000/4070000] [167.6/1000.0] [batch_t 1.438 (1.432)] [data_t 0.002] [optim_t 1.436] [lr 0.005000] +2024-04-08 09:26:26,679 - Train: 16.76% [682100/4070000] [167.6/1000.0] [batch_t 1.420 (1.431)] [data_t 0.003] [optim_t 1.417] [lr 0.005000] +2024-04-08 09:28:49,738 - Train: 16.76% [682200/4070000] [167.6/1000.0] [batch_t 1.431 (1.430)] [data_t 0.002] [optim_t 1.429] [lr 0.005000] +2024-04-08 09:31:12,753 - Train: 16.76% [682300/4070000] [167.6/1000.0] [batch_t 1.425 (1.430)] [data_t 0.003] [optim_t 1.422] [lr 0.005000] +2024-04-08 09:33:35,926 - Train: 16.77% [682400/4070000] [167.7/1000.0] [batch_t 1.435 (1.432)] [data_t 0.003] [optim_t 1.432] [lr 0.005000] +2024-04-08 09:35:58,881 - Train: 16.77% [682500/4070000] [167.7/1000.0] [batch_t 1.425 (1.429)] [data_t 0.003] [optim_t 1.422] [lr 0.005000] +2024-04-08 09:38:22,020 - Train: 16.77% [682600/4070000] [167.7/1000.0] [batch_t 1.443 (1.431)] [data_t 0.003] [optim_t 1.440] [lr 0.005000] +2024-04-08 09:40:45,089 - Train: 16.77% [682700/4070000] [167.7/1000.0] [batch_t 1.421 (1.431)] [data_t 0.003] [optim_t 1.418] [lr 0.005000] +2024-04-08 09:43:08,038 - Train: 16.78% [682800/4070000] [167.8/1000.0] [batch_t 1.426 (1.429)] [data_t 0.002] [optim_t 1.423] [lr 0.005000] +2024-04-08 09:45:31,073 - Train: 16.78% [682900/4070000] [167.8/1000.0] [batch_t 1.398 (1.430)] [data_t 0.003] [optim_t 1.395] [lr 0.005000] +2024-04-08 09:47:54,069 - Train: 16.78% [683000/4070000] [167.8/1000.0] [batch_t 1.435 (1.430)] [data_t 0.003] [optim_t 1.432] [lr 0.005000] +2024-04-08 09:50:17,258 - Train: 16.78% [683100/4070000] [167.8/1000.0] [batch_t 1.440 (1.432)] [data_t 0.003] [optim_t 1.437] [lr 0.005000] +2024-04-08 09:52:40,183 - Train: 16.79% [683200/4070000] [167.9/1000.0] [batch_t 1.423 (1.429)] [data_t 0.002] [optim_t 1.421] [lr 0.005000] +2024-04-08 09:55:03,323 - Train: 16.79% [683300/4070000] [167.9/1000.0] [batch_t 1.441 (1.431)] [data_t 0.002] [optim_t 1.438] [lr 0.005000] +2024-04-08 09:57:26,224 - Train: 16.79% [683400/4070000] [167.9/1000.0] [batch_t 1.434 (1.429)] [data_t 0.003] [optim_t 1.432] [lr 0.005000] +2024-04-08 09:59:49,336 - Train: 16.79% [683500/4070000] [167.9/1000.0] [batch_t 1.430 (1.431)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-08 10:02:12,434 - Train: 16.80% [683600/4070000] [168.0/1000.0] [batch_t 1.434 (1.431)] [data_t 0.003] [optim_t 1.431] [lr 0.005000] +2024-04-08 10:04:35,534 - Train: 16.80% [683700/4070000] [168.0/1000.0] [batch_t 1.439 (1.431)] [data_t 0.002] [optim_t 1.436] [lr 0.005000] +2024-04-08 10:06:01,318 - ==> Total time: 6 days, 9:05:32 Eta: 31 days, 14:10:16 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-08 10:07:02,350 - Train: 16.80% [683800/4070000] [168.0/1000.0] [batch_t 1.437 (1.443)] [data_t 0.003] [optim_t 1.434] [lr 0.005000] +2024-04-08 10:09:25,391 - Train: 16.80% [683900/4070000] [168.0/1000.0] [batch_t 1.424 (1.430)] [data_t 0.002] [optim_t 1.422] [lr 0.005000] +2024-04-08 10:11:48,450 - Train: 16.81% [684000/4070000] [168.1/1000.0] [batch_t 1.428 (1.430)] [data_t 0.002] [optim_t 1.426] [lr 0.005000] +2024-04-08 10:14:11,380 - Train: 16.81% [684100/4070000] [168.1/1000.0] [batch_t 1.429 (1.429)] [data_t 0.002] [optim_t 1.427] [lr 0.005000] +2024-04-08 10:16:34,386 - Train: 16.81% [684200/4070000] [168.1/1000.0] [batch_t 1.420 (1.430)] [data_t 0.003] [optim_t 1.418] [lr 0.005000] +2024-04-08 10:18:57,322 - Train: 16.81% [684300/4070000] [168.1/1000.0] [batch_t 1.440 (1.429)] [data_t 0.003] [optim_t 1.438] [lr 0.005000] +2024-04-08 10:21:20,450 - Train: 16.82% [684400/4070000] [168.2/1000.0] [batch_t 1.438 (1.431)] [data_t 0.003] [optim_t 1.435] [lr 0.005000] +2024-04-08 10:23:43,528 - Train: 16.82% [684500/4070000] [168.2/1000.0] [batch_t 1.430 (1.431)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-08 10:26:06,627 - Train: 16.82% [684600/4070000] [168.2/1000.0] [batch_t 1.426 (1.431)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-08 10:28:29,539 - Train: 16.82% [684700/4070000] [168.2/1000.0] [batch_t 1.435 (1.429)] [data_t 0.002] [optim_t 1.433] [lr 0.005000] +2024-04-08 10:30:52,512 - Train: 16.83% [684800/4070000] [168.3/1000.0] [batch_t 1.416 (1.430)] [data_t 0.003] [optim_t 1.413] [lr 0.005000] +2024-04-08 10:33:15,548 - Train: 16.83% [684900/4070000] [168.3/1000.0] [batch_t 1.430 (1.430)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-08 10:35:38,678 - Train: 16.83% [685000/4070000] [168.3/1000.0] [batch_t 1.431 (1.431)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-08 10:38:01,688 - Train: 16.83% [685100/4070000] [168.3/1000.0] [batch_t 1.439 (1.430)] [data_t 0.003] [optim_t 1.436] [lr 0.005000] +2024-04-08 10:40:24,702 - Train: 16.84% [685200/4070000] [168.4/1000.0] [batch_t 1.400 (1.430)] [data_t 0.003] [optim_t 1.397] [lr 0.005000] +2024-04-08 10:42:47,829 - Train: 16.84% [685300/4070000] [168.4/1000.0] [batch_t 1.421 (1.431)] [data_t 0.003] [optim_t 1.418] [lr 0.005000] +2024-04-08 10:45:10,928 - Train: 16.84% [685400/4070000] [168.4/1000.0] [batch_t 1.445 (1.431)] [data_t 0.003] [optim_t 1.443] [lr 0.005000] +2024-04-08 10:47:33,994 - Train: 16.84% [685500/4070000] [168.4/1000.0] [batch_t 1.420 (1.431)] [data_t 0.002] [optim_t 1.417] [lr 0.005000] +2024-04-08 10:49:57,111 - Train: 16.85% [685600/4070000] [168.5/1000.0] [batch_t 1.438 (1.431)] [data_t 0.003] [optim_t 1.436] [lr 0.005000] +2024-04-08 10:52:20,213 - Train: 16.85% [685700/4070000] [168.5/1000.0] [batch_t 1.417 (1.431)] [data_t 0.003] [optim_t 1.414] [lr 0.005000] +2024-04-08 10:54:43,423 - Train: 16.85% [685800/4070000] [168.5/1000.0] [batch_t 1.430 (1.432)] [data_t 0.002] [optim_t 1.428] [lr 0.005000] +2024-04-08 10:57:06,442 - Train: 16.85% [685900/4070000] [168.5/1000.0] [batch_t 1.430 (1.430)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-08 10:59:29,601 - Train: 16.86% [686000/4070000] [168.6/1000.0] [batch_t 1.432 (1.431)] [data_t 0.003] [optim_t 1.429] [lr 0.005000] +2024-04-08 11:01:52,635 - Train: 16.86% [686100/4070000] [168.6/1000.0] [batch_t 1.429 (1.430)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-08 11:04:15,745 - Train: 16.86% [686200/4070000] [168.6/1000.0] [batch_t 1.402 (1.431)] [data_t 0.003] [optim_t 1.399] [lr 0.005000] +2024-04-08 11:06:38,898 - Train: 16.86% [686300/4070000] [168.6/1000.0] [batch_t 1.453 (1.431)] [data_t 0.002] [optim_t 1.451] [lr 0.005000] +2024-04-08 11:09:02,108 - Train: 16.86% [686400/4070000] [168.6/1000.0] [batch_t 1.439 (1.432)] [data_t 0.003] [optim_t 1.436] [lr 0.005000] +2024-04-08 11:11:25,077 - Train: 16.87% [686500/4070000] [168.7/1000.0] [batch_t 1.430 (1.430)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-08 11:13:48,240 - Train: 16.87% [686600/4070000] [168.7/1000.0] [batch_t 1.432 (1.432)] [data_t 0.002] [optim_t 1.429] [lr 0.005000] +2024-04-08 11:16:11,277 - Train: 16.87% [686700/4070000] [168.7/1000.0] [batch_t 1.441 (1.430)] [data_t 0.003] [optim_t 1.438] [lr 0.005000] +2024-04-08 11:18:34,395 - Train: 16.87% [686800/4070000] [168.7/1000.0] [batch_t 1.445 (1.431)] [data_t 0.002] [optim_t 1.443] [lr 0.005000] +2024-04-08 11:20:57,582 - Train: 16.88% [686900/4070000] [168.8/1000.0] [batch_t 1.419 (1.432)] [data_t 0.003] [optim_t 1.417] [lr 0.005000] +2024-04-08 11:23:20,635 - Train: 16.88% [687000/4070000] [168.8/1000.0] [batch_t 1.432 (1.430)] [data_t 0.003] [optim_t 1.429] [lr 0.005000] +2024-04-08 11:25:43,724 - Train: 16.88% [687100/4070000] [168.8/1000.0] [batch_t 1.420 (1.431)] [data_t 0.003] [optim_t 1.417] [lr 0.005000] +2024-04-08 11:28:06,782 - Train: 16.88% [687200/4070000] [168.8/1000.0] [batch_t 1.442 (1.430)] [data_t 0.003] [optim_t 1.439] [lr 0.005000] +2024-04-08 11:30:29,863 - Train: 16.89% [687300/4070000] [168.9/1000.0] [batch_t 1.440 (1.431)] [data_t 0.003] [optim_t 1.437] [lr 0.005000] +2024-04-08 11:32:52,731 - Train: 16.89% [687400/4070000] [168.9/1000.0] [batch_t 1.429 (1.429)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-08 11:35:15,826 - Train: 16.89% [687500/4070000] [168.9/1000.0] [batch_t 1.414 (1.431)] [data_t 0.003] [optim_t 1.412] [lr 0.005000] +2024-04-08 11:37:38,841 - Train: 16.89% [687600/4070000] [168.9/1000.0] [batch_t 1.437 (1.430)] [data_t 0.003] [optim_t 1.435] [lr 0.005000] +2024-04-08 11:40:01,974 - Train: 16.90% [687700/4070000] [169.0/1000.0] [batch_t 1.438 (1.431)] [data_t 0.002] [optim_t 1.436] [lr 0.005000] +2024-04-08 11:42:24,877 - Train: 16.90% [687800/4070000] [169.0/1000.0] [batch_t 1.432 (1.429)] [data_t 0.002] [optim_t 1.430] [lr 0.005000] +2024-04-08 11:43:07,831 - ==> Total time: 6 days, 10:42:38 Eta: 31 days, 16:44:14 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-08 11:44:49,816 - Train: 16.90% [687900/4070000] [169.0/1000.0] [batch_t 1.438 (1.437)] [data_t 0.002] [optim_t 1.436] [lr 0.005000] +2024-04-08 11:47:12,857 - Train: 16.90% [688000/4070000] [169.0/1000.0] [batch_t 1.400 (1.430)] [data_t 0.002] [optim_t 1.397] [lr 0.005000] +2024-04-08 11:49:35,812 - Train: 16.91% [688100/4070000] [169.1/1000.0] [batch_t 1.432 (1.429)] [data_t 0.002] [optim_t 1.430] [lr 0.005000] +2024-04-08 11:51:58,905 - Train: 16.91% [688200/4070000] [169.1/1000.0] [batch_t 1.443 (1.431)] [data_t 0.002] [optim_t 1.441] [lr 0.005000] +2024-04-08 11:54:21,892 - Train: 16.91% [688300/4070000] [169.1/1000.0] [batch_t 1.427 (1.430)] [data_t 0.002] [optim_t 1.425] [lr 0.005000] +2024-04-08 11:56:44,982 - Train: 16.91% [688400/4070000] [169.1/1000.0] [batch_t 1.430 (1.431)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-08 11:59:07,937 - Train: 16.92% [688500/4070000] [169.2/1000.0] [batch_t 1.397 (1.429)] [data_t 0.002] [optim_t 1.395] [lr 0.005000] +2024-04-08 12:01:31,006 - Train: 16.92% [688600/4070000] [169.2/1000.0] [batch_t 1.444 (1.431)] [data_t 0.003] [optim_t 1.441] [lr 0.005000] +2024-04-08 12:03:54,046 - Train: 16.92% [688700/4070000] [169.2/1000.0] [batch_t 1.436 (1.430)] [data_t 0.002] [optim_t 1.433] [lr 0.005000] +2024-04-08 12:06:17,105 - Train: 16.92% [688800/4070000] [169.2/1000.0] [batch_t 1.421 (1.431)] [data_t 0.002] [optim_t 1.419] [lr 0.005000] +2024-04-08 12:08:40,005 - Train: 16.93% [688900/4070000] [169.3/1000.0] [batch_t 1.432 (1.429)] [data_t 0.002] [optim_t 1.429] [lr 0.005000] +2024-04-08 12:11:03,096 - Train: 16.93% [689000/4070000] [169.3/1000.0] [batch_t 1.446 (1.431)] [data_t 0.002] [optim_t 1.444] [lr 0.005000] +2024-04-08 12:13:26,101 - Train: 16.93% [689100/4070000] [169.3/1000.0] [batch_t 1.430 (1.430)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-08 12:15:49,108 - Train: 16.93% [689200/4070000] [169.3/1000.0] [batch_t 1.430 (1.430)] [data_t 0.002] [optim_t 1.428] [lr 0.005000] +2024-04-08 12:18:12,206 - Train: 16.94% [689300/4070000] [169.4/1000.0] [batch_t 1.430 (1.431)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-08 12:20:35,461 - Train: 16.94% [689400/4070000] [169.4/1000.0] [batch_t 1.436 (1.432)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-08 12:22:58,663 - Train: 16.94% [689500/4070000] [169.4/1000.0] [batch_t 1.397 (1.432)] [data_t 0.002] [optim_t 1.395] [lr 0.005000] +2024-04-08 12:25:21,713 - Train: 16.94% [689600/4070000] [169.4/1000.0] [batch_t 1.424 (1.430)] [data_t 0.003] [optim_t 1.421] [lr 0.005000] +2024-04-08 12:27:44,666 - Train: 16.95% [689700/4070000] [169.5/1000.0] [batch_t 1.432 (1.429)] [data_t 0.003] [optim_t 1.429] [lr 0.005000] +2024-04-08 12:30:07,702 - Train: 16.95% [689800/4070000] [169.5/1000.0] [batch_t 1.429 (1.430)] [data_t 0.002] [optim_t 1.427] [lr 0.005000] +2024-04-08 12:32:30,781 - Train: 16.95% [689900/4070000] [169.5/1000.0] [batch_t 1.433 (1.431)] [data_t 0.003] [optim_t 1.431] [lr 0.005000] +2024-04-08 12:34:53,775 - Train: 16.95% [690000/4070000] [169.5/1000.0] [batch_t 1.436 (1.430)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-08 12:37:16,783 - Train: 16.96% [690100/4070000] [169.6/1000.0] [batch_t 1.426 (1.430)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-08 12:39:39,795 - Train: 16.96% [690200/4070000] [169.6/1000.0] [batch_t 1.441 (1.430)] [data_t 0.002] [optim_t 1.439] [lr 0.005000] +2024-04-08 12:42:02,944 - Train: 16.96% [690300/4070000] [169.6/1000.0] [batch_t 1.434 (1.431)] [data_t 0.002] [optim_t 1.432] [lr 0.005000] +2024-04-08 12:44:25,893 - Train: 16.96% [690400/4070000] [169.6/1000.0] [batch_t 1.434 (1.429)] [data_t 0.002] [optim_t 1.432] [lr 0.005000] +2024-04-08 12:46:48,962 - Train: 16.97% [690500/4070000] [169.7/1000.0] [batch_t 1.426 (1.431)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-08 12:49:12,075 - Train: 16.97% [690600/4070000] [169.7/1000.0] [batch_t 1.439 (1.431)] [data_t 0.003] [optim_t 1.436] [lr 0.005000] +2024-04-08 12:51:35,135 - Train: 16.97% [690700/4070000] [169.7/1000.0] [batch_t 1.433 (1.431)] [data_t 0.003] [optim_t 1.430] [lr 0.005000] +2024-04-08 12:53:58,175 - Train: 16.97% [690800/4070000] [169.7/1000.0] [batch_t 1.439 (1.430)] [data_t 0.002] [optim_t 1.436] [lr 0.005000] +2024-04-08 12:56:21,305 - Train: 16.98% [690900/4070000] [169.8/1000.0] [batch_t 1.430 (1.431)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-08 12:58:44,288 - Train: 16.98% [691000/4070000] [169.8/1000.0] [batch_t 1.431 (1.430)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-08 13:01:07,421 - Train: 16.98% [691100/4070000] [169.8/1000.0] [batch_t 1.435 (1.431)] [data_t 0.003] [optim_t 1.432] [lr 0.005000] +2024-04-08 13:03:30,577 - Train: 16.98% [691200/4070000] [169.8/1000.0] [batch_t 1.436 (1.431)] [data_t 0.002] [optim_t 1.434] [lr 0.005000] +2024-04-08 13:05:53,547 - Train: 16.99% [691300/4070000] [169.9/1000.0] [batch_t 1.416 (1.430)] [data_t 0.003] [optim_t 1.413] [lr 0.005000] +2024-04-08 13:08:16,666 - Train: 16.99% [691400/4070000] [169.9/1000.0] [batch_t 1.420 (1.431)] [data_t 0.003] [optim_t 1.418] [lr 0.005000] +2024-04-08 13:10:39,743 - Train: 16.99% [691500/4070000] [169.9/1000.0] [batch_t 1.442 (1.431)] [data_t 0.003] [optim_t 1.439] [lr 0.005000] +2024-04-08 13:13:02,949 - Train: 16.99% [691600/4070000] [169.9/1000.0] [batch_t 1.425 (1.432)] [data_t 0.003] [optim_t 1.422] [lr 0.005000] +2024-04-08 13:15:25,977 - Train: 17.00% [691700/4070000] [170.0/1000.0] [batch_t 1.431 (1.430)] [data_t 0.003] [optim_t 1.429] [lr 0.005000] +2024-04-08 13:17:49,051 - Train: 17.00% [691800/4070000] [170.0/1000.0] [batch_t 1.439 (1.431)] [data_t 0.002] [optim_t 1.437] [lr 0.005000] +2024-04-08 13:20:12,013 - Train: 17.00% [691900/4070000] [170.0/1000.0] [batch_t 1.424 (1.430)] [data_t 0.002] [optim_t 1.422] [lr 0.005000] +2024-04-08 13:20:12,017 - ==> Total time: 6 days, 12:19:42 Eta: 31 days, 19:15:04 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-08 13:22:36,850 - Train: 17.00% [692000/4070000] [170.0/1000.0] [batch_t 1.421 (1.433)] [data_t 0.002] [optim_t 1.419] [lr 0.005000] +2024-04-08 13:24:59,973 - Train: 17.00% [692100/4070000] [170.0/1000.0] [batch_t 1.439 (1.431)] [data_t 0.003] [optim_t 1.437] [lr 0.005000] +2024-04-08 13:27:23,084 - Train: 17.01% [692200/4070000] [170.1/1000.0] [batch_t 1.440 (1.431)] [data_t 0.002] [optim_t 1.438] [lr 0.005000] +2024-04-08 13:29:46,104 - Train: 17.01% [692300/4070000] [170.1/1000.0] [batch_t 1.424 (1.430)] [data_t 0.002] [optim_t 1.422] [lr 0.005000] +2024-04-08 13:32:09,175 - Train: 17.01% [692400/4070000] [170.1/1000.0] [batch_t 1.420 (1.431)] [data_t 0.003] [optim_t 1.417] [lr 0.005000] +2024-04-08 13:34:32,253 - Train: 17.01% [692500/4070000] [170.1/1000.0] [batch_t 1.430 (1.431)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-08 13:36:55,216 - Train: 17.02% [692600/4070000] [170.2/1000.0] [batch_t 1.440 (1.430)] [data_t 0.003] [optim_t 1.437] [lr 0.005000] +2024-04-08 13:39:18,313 - Train: 17.02% [692700/4070000] [170.2/1000.0] [batch_t 1.428 (1.431)] [data_t 0.003] [optim_t 1.425] [lr 0.005000] +2024-04-08 13:41:41,336 - Train: 17.02% [692800/4070000] [170.2/1000.0] [batch_t 1.441 (1.430)] [data_t 0.003] [optim_t 1.438] [lr 0.005000] +2024-04-08 13:44:04,260 - Train: 17.02% [692900/4070000] [170.2/1000.0] [batch_t 1.427 (1.429)] [data_t 0.002] [optim_t 1.424] [lr 0.005000] +2024-04-08 13:46:27,276 - Train: 17.03% [693000/4070000] [170.3/1000.0] [batch_t 1.426 (1.430)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-08 13:48:50,239 - Train: 17.03% [693100/4070000] [170.3/1000.0] [batch_t 1.445 (1.430)] [data_t 0.002] [optim_t 1.443] [lr 0.005000] +2024-04-08 13:51:13,347 - Train: 17.03% [693200/4070000] [170.3/1000.0] [batch_t 1.431 (1.431)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-08 13:53:36,267 - Train: 17.03% [693300/4070000] [170.3/1000.0] [batch_t 1.426 (1.429)] [data_t 0.002] [optim_t 1.424] [lr 0.005000] +2024-04-08 13:55:59,406 - Train: 17.04% [693400/4070000] [170.4/1000.0] [batch_t 1.423 (1.431)] [data_t 0.002] [optim_t 1.421] [lr 0.005000] +2024-04-08 13:58:22,578 - Train: 17.04% [693500/4070000] [170.4/1000.0] [batch_t 1.426 (1.432)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-08 14:00:45,668 - Train: 17.04% [693600/4070000] [170.4/1000.0] [batch_t 1.431 (1.431)] [data_t 0.002] [optim_t 1.429] [lr 0.005000] +2024-04-08 14:03:08,682 - Train: 17.04% [693700/4070000] [170.4/1000.0] [batch_t 1.444 (1.430)] [data_t 0.003] [optim_t 1.442] [lr 0.005000] +2024-04-08 14:05:31,777 - Train: 17.05% [693800/4070000] [170.5/1000.0] [batch_t 1.426 (1.431)] [data_t 0.002] [optim_t 1.424] [lr 0.005000] +2024-04-08 14:07:54,836 - Train: 17.05% [693900/4070000] [170.5/1000.0] [batch_t 1.429 (1.430)] [data_t 0.002] [optim_t 1.427] [lr 0.005000] +2024-04-08 14:10:17,818 - Train: 17.05% [694000/4070000] [170.5/1000.0] [batch_t 1.439 (1.430)] [data_t 0.003] [optim_t 1.437] [lr 0.005000] +2024-04-08 14:12:40,923 - Train: 17.05% [694100/4070000] [170.5/1000.0] [batch_t 1.448 (1.431)] [data_t 0.002] [optim_t 1.446] [lr 0.005000] +2024-04-08 14:15:03,953 - Train: 17.06% [694200/4070000] [170.6/1000.0] [batch_t 1.429 (1.430)] [data_t 0.002] [optim_t 1.427] [lr 0.005000] +2024-04-08 14:17:26,989 - Train: 17.06% [694300/4070000] [170.6/1000.0] [batch_t 1.425 (1.430)] [data_t 0.003] [optim_t 1.422] [lr 0.005000] +2024-04-08 14:19:50,173 - Train: 17.06% [694400/4070000] [170.6/1000.0] [batch_t 1.425 (1.432)] [data_t 0.003] [optim_t 1.422] [lr 0.005000] +2024-04-08 14:22:13,234 - Train: 17.06% [694500/4070000] [170.6/1000.0] [batch_t 1.440 (1.431)] [data_t 0.002] [optim_t 1.438] [lr 0.005000] +2024-04-08 14:24:36,512 - Train: 17.07% [694600/4070000] [170.7/1000.0] [batch_t 1.440 (1.433)] [data_t 0.002] [optim_t 1.438] [lr 0.005000] +2024-04-08 14:26:59,645 - Train: 17.07% [694700/4070000] [170.7/1000.0] [batch_t 1.429 (1.431)] [data_t 0.003] [optim_t 1.426] [lr 0.005000] +2024-04-08 14:29:22,827 - Train: 17.07% [694800/4070000] [170.7/1000.0] [batch_t 1.438 (1.432)] [data_t 0.003] [optim_t 1.436] [lr 0.005000] +2024-04-08 14:31:46,019 - Train: 17.07% [694900/4070000] [170.7/1000.0] [batch_t 1.426 (1.432)] [data_t 0.002] [optim_t 1.423] [lr 0.005000] +2024-04-08 14:34:08,882 - Train: 17.08% [695000/4070000] [170.8/1000.0] [batch_t 1.416 (1.429)] [data_t 0.003] [optim_t 1.414] [lr 0.005000] +2024-04-08 14:36:32,041 - Train: 17.08% [695100/4070000] [170.8/1000.0] [batch_t 1.436 (1.431)] [data_t 0.003] [optim_t 1.434] [lr 0.005000] +2024-04-08 14:38:55,327 - Train: 17.08% [695200/4070000] [170.8/1000.0] [batch_t 1.444 (1.433)] [data_t 0.002] [optim_t 1.442] [lr 0.005000] +2024-04-08 14:41:18,453 - Train: 17.08% [695300/4070000] [170.8/1000.0] [batch_t 1.429 (1.431)] [data_t 0.002] [optim_t 1.427] [lr 0.005000] +2024-04-08 14:43:41,460 - Train: 17.09% [695400/4070000] [170.9/1000.0] [batch_t 1.445 (1.430)] [data_t 0.002] [optim_t 1.443] [lr 0.005000] +2024-04-08 14:46:04,488 - Train: 17.09% [695500/4070000] [170.9/1000.0] [batch_t 1.416 (1.430)] [data_t 0.003] [optim_t 1.413] [lr 0.005000] +2024-04-08 14:48:27,563 - Train: 17.09% [695600/4070000] [170.9/1000.0] [batch_t 1.440 (1.431)] [data_t 0.002] [optim_t 1.438] [lr 0.005000] +2024-04-08 14:50:50,694 - Train: 17.09% [695700/4070000] [170.9/1000.0] [batch_t 1.435 (1.431)] [data_t 0.002] [optim_t 1.433] [lr 0.005000] +2024-04-08 14:53:13,836 - Train: 17.10% [695800/4070000] [171.0/1000.0] [batch_t 1.410 (1.431)] [data_t 0.002] [optim_t 1.407] [lr 0.005000] +2024-04-08 14:55:36,821 - Train: 17.10% [695900/4070000] [171.0/1000.0] [batch_t 1.430 (1.430)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-08 14:57:16,964 - ==> Total time: 6 days, 13:56:47 Eta: 31 days, 21:43:04 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-08 14:58:01,629 - Train: 17.10% [696000/4070000] [171.0/1000.0] [batch_t 1.440 (1.441)] [data_t 0.003] [optim_t 1.437] [lr 0.005000] +2024-04-08 15:00:24,692 - Train: 17.10% [696100/4070000] [171.0/1000.0] [batch_t 1.431 (1.431)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-08 15:02:47,779 - Train: 17.11% [696200/4070000] [171.1/1000.0] [batch_t 1.432 (1.431)] [data_t 0.003] [optim_t 1.429] [lr 0.005000] +2024-04-08 15:05:10,955 - Train: 17.11% [696300/4070000] [171.1/1000.0] [batch_t 1.424 (1.432)] [data_t 0.003] [optim_t 1.421] [lr 0.005000] +2024-04-08 15:07:34,140 - Train: 17.11% [696400/4070000] [171.1/1000.0] [batch_t 1.477 (1.432)] [data_t 0.003] [optim_t 1.474] [lr 0.005000] +2024-04-08 15:09:57,318 - Train: 17.11% [696500/4070000] [171.1/1000.0] [batch_t 1.433 (1.432)] [data_t 0.003] [optim_t 1.430] [lr 0.005000] +2024-04-08 15:12:20,604 - Train: 17.12% [696600/4070000] [171.2/1000.0] [batch_t 1.428 (1.433)] [data_t 0.002] [optim_t 1.426] [lr 0.005000] +2024-04-08 15:14:43,807 - Train: 17.12% [696700/4070000] [171.2/1000.0] [batch_t 1.426 (1.432)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-08 15:17:06,888 - Train: 17.12% [696800/4070000] [171.2/1000.0] [batch_t 1.432 (1.431)] [data_t 0.003] [optim_t 1.430] [lr 0.005000] +2024-04-08 15:19:29,892 - Train: 17.12% [696900/4070000] [171.2/1000.0] [batch_t 1.389 (1.430)] [data_t 0.002] [optim_t 1.386] [lr 0.005000] +2024-04-08 15:21:53,078 - Train: 17.13% [697000/4070000] [171.3/1000.0] [batch_t 1.409 (1.432)] [data_t 0.003] [optim_t 1.406] [lr 0.005000] +2024-04-08 15:24:16,260 - Train: 17.13% [697100/4070000] [171.3/1000.0] [batch_t 1.425 (1.432)] [data_t 0.003] [optim_t 1.422] [lr 0.005000] +2024-04-08 15:26:39,365 - Train: 17.13% [697200/4070000] [171.3/1000.0] [batch_t 1.440 (1.431)] [data_t 0.002] [optim_t 1.438] [lr 0.005000] +2024-04-08 15:29:12,695 - Train: 17.13% [697300/4070000] [171.3/1000.0] [batch_t 1.440 (1.533)] [data_t 0.002] [optim_t 1.437] [lr 0.005000] +2024-04-08 15:32:16,973 - Train: 17.14% [697400/4070000] [171.4/1000.0] [batch_t 1.445 (1.843)] [data_t 0.002] [optim_t 1.443] [lr 0.005000] +2024-04-08 15:35:11,403 - Train: 17.14% [697500/4070000] [171.4/1000.0] [batch_t 1.427 (1.744)] [data_t 0.003] [optim_t 1.424] [lr 0.005000] +2024-04-08 15:38:59,256 - Train: 17.14% [697600/4070000] [171.4/1000.0] [batch_t 3.897 (2.278)] [data_t 2.465] [optim_t 1.432] [lr 0.005000] +2024-04-08 15:41:25,504 - Train: 17.14% [697700/4070000] [171.4/1000.0] [batch_t 1.439 (1.462)] [data_t 0.002] [optim_t 1.437] [lr 0.005000] +2024-04-08 15:43:54,561 - Train: 17.14% [697800/4070000] [171.4/1000.0] [batch_t 1.504 (1.490)] [data_t 0.055] [optim_t 1.449] [lr 0.005000] +2024-04-08 15:46:18,389 - Train: 17.15% [697900/4070000] [171.5/1000.0] [batch_t 1.434 (1.438)] [data_t 0.002] [optim_t 1.432] [lr 0.005000] +2024-04-08 15:48:41,618 - Train: 17.15% [698000/4070000] [171.5/1000.0] [batch_t 1.417 (1.432)] [data_t 0.002] [optim_t 1.414] [lr 0.005000] +2024-04-08 15:51:04,804 - Train: 17.15% [698100/4070000] [171.5/1000.0] [batch_t 1.431 (1.432)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-08 15:53:27,826 - Train: 17.15% [698200/4070000] [171.5/1000.0] [batch_t 1.436 (1.430)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-08 15:55:50,939 - Train: 17.16% [698300/4070000] [171.6/1000.0] [batch_t 1.431 (1.431)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-08 15:58:14,139 - Train: 17.16% [698400/4070000] [171.6/1000.0] [batch_t 1.412 (1.432)] [data_t 0.003] [optim_t 1.410] [lr 0.005000] +2024-04-08 16:00:37,280 - Train: 17.16% [698500/4070000] [171.6/1000.0] [batch_t 1.426 (1.431)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-08 16:03:00,602 - Train: 17.16% [698600/4070000] [171.6/1000.0] [batch_t 1.419 (1.433)] [data_t 0.002] [optim_t 1.416] [lr 0.005000] +2024-04-08 16:05:23,707 - Train: 17.17% [698700/4070000] [171.7/1000.0] [batch_t 1.437 (1.431)] [data_t 0.003] [optim_t 1.434] [lr 0.005000] +2024-04-08 16:07:46,813 - Train: 17.17% [698800/4070000] [171.7/1000.0] [batch_t 1.428 (1.431)] [data_t 0.003] [optim_t 1.425] [lr 0.005000] +2024-04-08 16:10:09,948 - Train: 17.17% [698900/4070000] [171.7/1000.0] [batch_t 1.400 (1.431)] [data_t 0.003] [optim_t 1.398] [lr 0.005000] +2024-04-08 16:12:33,159 - Train: 17.17% [699000/4070000] [171.7/1000.0] [batch_t 1.432 (1.432)] [data_t 0.003] [optim_t 1.429] [lr 0.005000] +2024-04-08 16:14:56,362 - Train: 17.18% [699100/4070000] [171.8/1000.0] [batch_t 1.429 (1.432)] [data_t 0.003] [optim_t 1.426] [lr 0.005000] +2024-04-08 16:17:19,535 - Train: 17.18% [699200/4070000] [171.8/1000.0] [batch_t 1.434 (1.432)] [data_t 0.002] [optim_t 1.432] [lr 0.005000] +2024-04-08 16:19:42,583 - Train: 17.18% [699300/4070000] [171.8/1000.0] [batch_t 1.430 (1.430)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-08 16:22:05,703 - Train: 17.18% [699400/4070000] [171.8/1000.0] [batch_t 1.424 (1.431)] [data_t 0.002] [optim_t 1.421] [lr 0.005000] +2024-04-08 16:24:28,933 - Train: 17.19% [699500/4070000] [171.9/1000.0] [batch_t 1.420 (1.432)] [data_t 0.002] [optim_t 1.418] [lr 0.005000] +2024-04-08 16:26:52,019 - Train: 17.19% [699600/4070000] [171.9/1000.0] [batch_t 1.432 (1.431)] [data_t 0.003] [optim_t 1.429] [lr 0.005000] +2024-04-08 16:29:15,058 - Train: 17.19% [699700/4070000] [171.9/1000.0] [batch_t 1.436 (1.430)] [data_t 0.002] [optim_t 1.433] [lr 0.005000] +2024-04-08 16:31:38,200 - Train: 17.19% [699800/4070000] [171.9/1000.0] [batch_t 1.432 (1.431)] [data_t 0.002] [optim_t 1.429] [lr 0.005000] +2024-04-08 16:34:01,321 - Train: 17.20% [699900/4070000] [172.0/1000.0] [batch_t 1.432 (1.431)] [data_t 0.002] [optim_t 1.430] [lr 0.005000] +2024-04-08 16:36:24,515 - Train: 17.20% [700000/4070000] [172.0/1000.0] [batch_t 1.441 (1.432)] [data_t 0.002] [optim_t 1.439] [lr 0.005000] +2024-04-08 16:37:21,751 - ==> Total time: 6 days, 15:36:52 Eta: 32 days, 0:22:38 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-08 16:38:50,355 - Train: 17.20% [700100/4070000] [172.0/1000.0] [batch_t 1.435 (1.446)] [data_t 0.003] [optim_t 1.432] [lr 0.005000] +2024-04-08 16:41:13,440 - Train: 17.20% [700200/4070000] [172.0/1000.0] [batch_t 1.431 (1.431)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-08 16:43:36,646 - Train: 17.21% [700300/4070000] [172.1/1000.0] [batch_t 1.430 (1.432)] [data_t 0.002] [optim_t 1.427] [lr 0.005000] +2024-04-08 16:45:59,758 - Train: 17.21% [700400/4070000] [172.1/1000.0] [batch_t 1.424 (1.431)] [data_t 0.002] [optim_t 1.422] [lr 0.005000] +2024-04-08 16:48:22,942 - Train: 17.21% [700500/4070000] [172.1/1000.0] [batch_t 1.436 (1.432)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-08 16:50:46,034 - Train: 17.21% [700600/4070000] [172.1/1000.0] [batch_t 1.430 (1.431)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-08 16:53:09,170 - Train: 17.22% [700700/4070000] [172.2/1000.0] [batch_t 1.423 (1.431)] [data_t 0.003] [optim_t 1.420] [lr 0.005000] +2024-04-08 16:55:32,396 - Train: 17.22% [700800/4070000] [172.2/1000.0] [batch_t 1.422 (1.432)] [data_t 0.002] [optim_t 1.420] [lr 0.005000] +2024-04-08 16:57:55,479 - Train: 17.22% [700900/4070000] [172.2/1000.0] [batch_t 1.427 (1.431)] [data_t 0.003] [optim_t 1.424] [lr 0.005000] +2024-04-08 17:00:18,723 - Train: 17.22% [701000/4070000] [172.2/1000.0] [batch_t 1.436 (1.432)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-08 17:02:41,994 - Train: 17.23% [701100/4070000] [172.3/1000.0] [batch_t 1.424 (1.433)] [data_t 0.003] [optim_t 1.421] [lr 0.005000] +2024-04-08 17:05:05,180 - Train: 17.23% [701200/4070000] [172.3/1000.0] [batch_t 1.435 (1.432)] [data_t 0.003] [optim_t 1.432] [lr 0.005000] +2024-04-08 17:07:28,411 - Train: 17.23% [701300/4070000] [172.3/1000.0] [batch_t 1.426 (1.432)] [data_t 0.003] [optim_t 1.424] [lr 0.005000] +2024-04-08 17:09:51,515 - Train: 17.23% [701400/4070000] [172.3/1000.0] [batch_t 1.443 (1.431)] [data_t 0.002] [optim_t 1.441] [lr 0.005000] +2024-04-08 17:12:14,757 - Train: 17.24% [701500/4070000] [172.4/1000.0] [batch_t 1.431 (1.432)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-08 17:14:37,908 - Train: 17.24% [701600/4070000] [172.4/1000.0] [batch_t 1.437 (1.431)] [data_t 0.003] [optim_t 1.434] [lr 0.005000] +2024-04-08 17:17:01,027 - Train: 17.24% [701700/4070000] [172.4/1000.0] [batch_t 1.436 (1.431)] [data_t 0.002] [optim_t 1.433] [lr 0.005000] +2024-04-08 17:19:24,172 - Train: 17.24% [701800/4070000] [172.4/1000.0] [batch_t 1.435 (1.431)] [data_t 0.003] [optim_t 1.432] [lr 0.005000] +2024-04-08 17:21:47,375 - Train: 17.25% [701900/4070000] [172.5/1000.0] [batch_t 1.437 (1.432)] [data_t 0.003] [optim_t 1.434] [lr 0.005000] +2024-04-08 17:24:10,515 - Train: 17.25% [702000/4070000] [172.5/1000.0] [batch_t 1.427 (1.431)] [data_t 0.003] [optim_t 1.424] [lr 0.005000] +2024-04-08 17:26:33,753 - Train: 17.25% [702100/4070000] [172.5/1000.0] [batch_t 1.430 (1.432)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-08 17:28:57,010 - Train: 17.25% [702200/4070000] [172.5/1000.0] [batch_t 1.436 (1.432)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-08 17:31:20,177 - Train: 17.26% [702300/4070000] [172.6/1000.0] [batch_t 1.417 (1.432)] [data_t 0.002] [optim_t 1.414] [lr 0.005000] +2024-04-08 17:33:43,331 - Train: 17.26% [702400/4070000] [172.6/1000.0] [batch_t 1.431 (1.431)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-08 17:36:06,462 - Train: 17.26% [702500/4070000] [172.6/1000.0] [batch_t 1.432 (1.431)] [data_t 0.003] [optim_t 1.430] [lr 0.005000] +2024-04-08 17:38:29,738 - Train: 17.26% [702600/4070000] [172.6/1000.0] [batch_t 1.435 (1.433)] [data_t 0.003] [optim_t 1.432] [lr 0.005000] +2024-04-08 17:40:52,922 - Train: 17.27% [702700/4070000] [172.7/1000.0] [batch_t 1.451 (1.432)] [data_t 0.003] [optim_t 1.448] [lr 0.005000] +2024-04-08 17:43:19,140 - Train: 17.27% [702800/4070000] [172.7/1000.0] [batch_t 1.432 (1.462)] [data_t 0.002] [optim_t 1.429] [lr 0.005000] +2024-04-08 17:45:42,285 - Train: 17.27% [702900/4070000] [172.7/1000.0] [batch_t 1.425 (1.431)] [data_t 0.003] [optim_t 1.422] [lr 0.005000] +2024-04-08 17:48:08,397 - Train: 17.27% [703000/4070000] [172.7/1000.0] [batch_t 1.425 (1.461)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-08 17:50:31,550 - Train: 17.28% [703100/4070000] [172.8/1000.0] [batch_t 1.436 (1.431)] [data_t 0.002] [optim_t 1.433] [lr 0.005000] +2024-04-08 17:52:54,542 - Train: 17.28% [703200/4070000] [172.8/1000.0] [batch_t 1.430 (1.430)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-08 17:55:19,101 - Train: 17.28% [703300/4070000] [172.8/1000.0] [batch_t 1.431 (1.445)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-08 17:57:42,305 - Train: 17.28% [703400/4070000] [172.8/1000.0] [batch_t 1.424 (1.432)] [data_t 0.003] [optim_t 1.422] [lr 0.005000] +2024-04-08 18:00:05,995 - Train: 17.29% [703500/4070000] [172.9/1000.0] [batch_t 1.426 (1.437)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-08 18:02:29,132 - Train: 17.29% [703600/4070000] [172.9/1000.0] [batch_t 1.442 (1.431)] [data_t 0.003] [optim_t 1.439] [lr 0.005000] +2024-04-08 18:04:53,555 - Train: 17.29% [703700/4070000] [172.9/1000.0] [batch_t 1.430 (1.444)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-08 18:07:17,038 - Train: 17.29% [703800/4070000] [172.9/1000.0] [batch_t 1.436 (1.435)] [data_t 0.003] [optim_t 1.434] [lr 0.005000] +2024-04-08 18:09:40,325 - Train: 17.29% [703900/4070000] [172.9/1000.0] [batch_t 1.441 (1.433)] [data_t 0.002] [optim_t 1.438] [lr 0.005000] +2024-04-08 18:12:03,496 - Train: 17.30% [704000/4070000] [173.0/1000.0] [batch_t 1.442 (1.432)] [data_t 0.003] [optim_t 1.440] [lr 0.005000] +2024-04-08 18:14:26,755 - Train: 17.30% [704100/4070000] [173.0/1000.0] [batch_t 1.439 (1.432)] [data_t 0.003] [optim_t 1.436] [lr 0.005000] +2024-04-08 18:14:41,041 - ==> Total time: 6 days, 17:14:11 Eta: 32 days, 2:46:01 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-08 18:16:51,956 - Train: 17.30% [704200/4070000] [173.0/1000.0] [batch_t 1.441 (1.432)] [data_t 0.002] [optim_t 1.439] [lr 0.005000] +2024-04-08 18:19:15,189 - Train: 17.30% [704300/4070000] [173.0/1000.0] [batch_t 1.421 (1.432)] [data_t 0.002] [optim_t 1.418] [lr 0.005000] +2024-04-08 18:21:38,448 - Train: 17.31% [704400/4070000] [173.1/1000.0] [batch_t 1.432 (1.432)] [data_t 0.003] [optim_t 1.429] [lr 0.005000] +2024-04-08 18:24:01,683 - Train: 17.31% [704500/4070000] [173.1/1000.0] [batch_t 1.444 (1.432)] [data_t 0.003] [optim_t 1.441] [lr 0.005000] +2024-04-08 18:26:24,752 - Train: 17.31% [704600/4070000] [173.1/1000.0] [batch_t 1.430 (1.431)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-08 18:28:48,109 - Train: 17.31% [704700/4070000] [173.1/1000.0] [batch_t 1.440 (1.433)] [data_t 0.003] [optim_t 1.437] [lr 0.005000] +2024-04-08 18:31:11,268 - Train: 17.32% [704800/4070000] [173.2/1000.0] [batch_t 1.415 (1.431)] [data_t 0.003] [optim_t 1.412] [lr 0.005000] +2024-04-08 18:33:34,405 - Train: 17.32% [704900/4070000] [173.2/1000.0] [batch_t 1.430 (1.431)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-08 18:35:57,625 - Train: 17.32% [705000/4070000] [173.2/1000.0] [batch_t 1.418 (1.432)] [data_t 0.002] [optim_t 1.416] [lr 0.005000] +2024-04-08 18:38:20,953 - Train: 17.32% [705100/4070000] [173.2/1000.0] [batch_t 1.430 (1.433)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-08 18:40:44,166 - Train: 17.33% [705200/4070000] [173.3/1000.0] [batch_t 1.435 (1.432)] [data_t 0.003] [optim_t 1.432] [lr 0.005000] +2024-04-08 18:43:07,427 - Train: 17.33% [705300/4070000] [173.3/1000.0] [batch_t 1.414 (1.433)] [data_t 0.002] [optim_t 1.411] [lr 0.005000] +2024-04-08 18:45:30,515 - Train: 17.33% [705400/4070000] [173.3/1000.0] [batch_t 1.423 (1.431)] [data_t 0.003] [optim_t 1.420] [lr 0.005000] +2024-04-08 18:47:53,703 - Train: 17.33% [705500/4070000] [173.3/1000.0] [batch_t 1.431 (1.432)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-08 18:50:17,003 - Train: 17.34% [705600/4070000] [173.4/1000.0] [batch_t 1.440 (1.433)] [data_t 0.002] [optim_t 1.438] [lr 0.005000] +2024-04-08 18:52:40,228 - Train: 17.34% [705700/4070000] [173.4/1000.0] [batch_t 1.430 (1.432)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-08 18:55:03,300 - Train: 17.34% [705800/4070000] [173.4/1000.0] [batch_t 1.426 (1.431)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-08 18:57:26,562 - Train: 17.34% [705900/4070000] [173.4/1000.0] [batch_t 1.441 (1.433)] [data_t 0.002] [optim_t 1.438] [lr 0.005000] +2024-04-08 18:59:49,802 - Train: 17.35% [706000/4070000] [173.5/1000.0] [batch_t 1.431 (1.432)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-08 19:02:12,950 - Train: 17.35% [706100/4070000] [173.5/1000.0] [batch_t 1.436 (1.431)] [data_t 0.003] [optim_t 1.432] [lr 0.005000] +2024-04-08 19:04:36,115 - Train: 17.35% [706200/4070000] [173.5/1000.0] [batch_t 1.429 (1.432)] [data_t 0.003] [optim_t 1.426] [lr 0.005000] +2024-04-08 19:06:59,339 - Train: 17.35% [706300/4070000] [173.5/1000.0] [batch_t 1.420 (1.432)] [data_t 0.003] [optim_t 1.417] [lr 0.005000] +2024-04-08 19:09:22,548 - Train: 17.36% [706400/4070000] [173.6/1000.0] [batch_t 1.441 (1.432)] [data_t 0.002] [optim_t 1.439] [lr 0.005000] +2024-04-08 19:11:45,745 - Train: 17.36% [706500/4070000] [173.6/1000.0] [batch_t 1.428 (1.432)] [data_t 0.004] [optim_t 1.424] [lr 0.005000] +2024-04-08 19:14:08,848 - Train: 17.36% [706600/4070000] [173.6/1000.0] [batch_t 1.427 (1.431)] [data_t 0.002] [optim_t 1.424] [lr 0.005000] +2024-04-08 19:16:32,060 - Train: 17.36% [706700/4070000] [173.6/1000.0] [batch_t 1.441 (1.432)] [data_t 0.003] [optim_t 1.438] [lr 0.005000] +2024-04-08 19:18:55,232 - Train: 17.37% [706800/4070000] [173.7/1000.0] [batch_t 1.433 (1.432)] [data_t 0.003] [optim_t 1.430] [lr 0.005000] +2024-04-08 19:21:18,290 - Train: 17.37% [706900/4070000] [173.7/1000.0] [batch_t 1.440 (1.430)] [data_t 0.003] [optim_t 1.437] [lr 0.005000] +2024-04-08 19:23:41,380 - Train: 17.37% [707000/4070000] [173.7/1000.0] [batch_t 1.440 (1.431)] [data_t 0.003] [optim_t 1.437] [lr 0.005000] +2024-04-08 19:26:04,632 - Train: 17.37% [707100/4070000] [173.7/1000.0] [batch_t 1.435 (1.432)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-08 19:28:27,792 - Train: 17.38% [707200/4070000] [173.8/1000.0] [batch_t 1.431 (1.432)] [data_t 0.002] [optim_t 1.428] [lr 0.005000] +2024-04-08 19:30:51,233 - Train: 17.38% [707300/4070000] [173.8/1000.0] [batch_t 1.437 (1.434)] [data_t 0.002] [optim_t 1.434] [lr 0.005000] +2024-04-08 19:33:14,378 - Train: 17.38% [707400/4070000] [173.8/1000.0] [batch_t 1.434 (1.431)] [data_t 0.003] [optim_t 1.431] [lr 0.005000] +2024-04-08 19:35:37,493 - Train: 17.38% [707500/4070000] [173.8/1000.0] [batch_t 1.414 (1.431)] [data_t 0.002] [optim_t 1.411] [lr 0.005000] +2024-04-08 19:38:00,689 - Train: 17.39% [707600/4070000] [173.9/1000.0] [batch_t 1.424 (1.432)] [data_t 0.002] [optim_t 1.422] [lr 0.005000] +2024-04-08 19:40:23,994 - Train: 17.39% [707700/4070000] [173.9/1000.0] [batch_t 1.447 (1.433)] [data_t 0.002] [optim_t 1.445] [lr 0.005000] +2024-04-08 19:42:47,254 - Train: 17.39% [707800/4070000] [173.9/1000.0] [batch_t 1.436 (1.432)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-08 19:45:10,336 - Train: 17.39% [707900/4070000] [173.9/1000.0] [batch_t 1.417 (1.431)] [data_t 0.003] [optim_t 1.414] [lr 0.005000] +2024-04-08 19:47:33,480 - Train: 17.40% [708000/4070000] [174.0/1000.0] [batch_t 1.435 (1.431)] [data_t 0.002] [optim_t 1.433] [lr 0.005000] +2024-04-08 19:49:56,514 - Train: 17.40% [708100/4070000] [174.0/1000.0] [batch_t 1.436 (1.430)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-08 19:51:50,871 - ==> Total time: 6 days, 18:51:21 Eta: 32 days, 5:05:53 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-08 19:52:22,033 - Train: 17.40% [708200/4070000] [174.0/1000.0] [batch_t 1.446 (1.442)] [data_t 0.003] [optim_t 1.443] [lr 0.005000] +2024-04-08 19:54:45,245 - Train: 17.40% [708300/4070000] [174.0/1000.0] [batch_t 1.434 (1.432)] [data_t 0.003] [optim_t 1.432] [lr 0.005000] +2024-04-08 19:57:08,431 - Train: 17.41% [708400/4070000] [174.1/1000.0] [batch_t 1.432 (1.432)] [data_t 0.003] [optim_t 1.429] [lr 0.005000] +2024-04-08 19:59:31,710 - Train: 17.41% [708500/4070000] [174.1/1000.0] [batch_t 1.438 (1.433)] [data_t 0.003] [optim_t 1.435] [lr 0.005000] +2024-04-08 20:01:54,954 - Train: 17.41% [708600/4070000] [174.1/1000.0] [batch_t 1.416 (1.432)] [data_t 0.002] [optim_t 1.413] [lr 0.005000] +2024-04-08 20:04:18,181 - Train: 17.41% [708700/4070000] [174.1/1000.0] [batch_t 1.431 (1.432)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-08 20:06:41,467 - Train: 17.42% [708800/4070000] [174.2/1000.0] [batch_t 1.441 (1.433)] [data_t 0.002] [optim_t 1.438] [lr 0.005000] +2024-04-08 20:09:04,684 - Train: 17.42% [708900/4070000] [174.2/1000.0] [batch_t 1.436 (1.432)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-08 20:11:27,838 - Train: 17.42% [709000/4070000] [174.2/1000.0] [batch_t 1.428 (1.431)] [data_t 0.002] [optim_t 1.426] [lr 0.005000] +2024-04-08 20:13:51,091 - Train: 17.42% [709100/4070000] [174.2/1000.0] [batch_t 1.429 (1.432)] [data_t 0.003] [optim_t 1.426] [lr 0.005000] +2024-04-08 20:16:14,183 - Train: 17.43% [709200/4070000] [174.3/1000.0] [batch_t 1.436 (1.431)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-08 20:18:37,409 - Train: 17.43% [709300/4070000] [174.3/1000.0] [batch_t 1.433 (1.432)] [data_t 0.002] [optim_t 1.431] [lr 0.005000] +2024-04-08 20:21:00,514 - Train: 17.43% [709400/4070000] [174.3/1000.0] [batch_t 1.434 (1.431)] [data_t 0.002] [optim_t 1.431] [lr 0.005000] +2024-04-08 20:23:23,662 - Train: 17.43% [709500/4070000] [174.3/1000.0] [batch_t 1.435 (1.431)] [data_t 0.003] [optim_t 1.432] [lr 0.005000] +2024-04-08 20:25:46,807 - Train: 17.43% [709600/4070000] [174.3/1000.0] [batch_t 1.414 (1.431)] [data_t 0.003] [optim_t 1.412] [lr 0.005000] +2024-04-08 20:28:09,976 - Train: 17.44% [709700/4070000] [174.4/1000.0] [batch_t 1.444 (1.432)] [data_t 0.003] [optim_t 1.442] [lr 0.005000] +2024-04-08 20:30:32,876 - Train: 17.44% [709800/4070000] [174.4/1000.0] [batch_t 1.451 (1.429)] [data_t 0.004] [optim_t 1.448] [lr 0.005000] +2024-04-08 20:32:56,153 - Train: 17.44% [709900/4070000] [174.4/1000.0] [batch_t 1.444 (1.433)] [data_t 0.003] [optim_t 1.442] [lr 0.005000] +2024-04-08 20:35:19,308 - Train: 17.44% [710000/4070000] [174.4/1000.0] [batch_t 1.432 (1.431)] [data_t 0.003] [optim_t 1.430] [lr 0.005000] +2024-04-08 20:37:42,466 - Train: 17.45% [710100/4070000] [174.5/1000.0] [batch_t 1.435 (1.431)] [data_t 0.003] [optim_t 1.432] [lr 0.005000] +2024-04-08 20:40:05,468 - Train: 17.45% [710200/4070000] [174.5/1000.0] [batch_t 1.426 (1.430)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-08 20:42:28,691 - Train: 17.45% [710300/4070000] [174.5/1000.0] [batch_t 1.424 (1.432)] [data_t 0.003] [optim_t 1.421] [lr 0.005000] +2024-04-08 20:44:51,869 - Train: 17.45% [710400/4070000] [174.5/1000.0] [batch_t 1.436 (1.432)] [data_t 0.002] [optim_t 1.434] [lr 0.005000] +2024-04-08 20:47:14,915 - Train: 17.46% [710500/4070000] [174.6/1000.0] [batch_t 1.430 (1.430)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-08 20:49:38,181 - Train: 17.46% [710600/4070000] [174.6/1000.0] [batch_t 1.414 (1.433)] [data_t 0.003] [optim_t 1.411] [lr 0.005000] +2024-04-08 20:52:01,238 - Train: 17.46% [710700/4070000] [174.6/1000.0] [batch_t 1.429 (1.430)] [data_t 0.003] [optim_t 1.426] [lr 0.005000] +2024-04-08 20:54:24,517 - Train: 17.46% [710800/4070000] [174.6/1000.0] [batch_t 1.438 (1.433)] [data_t 0.002] [optim_t 1.435] [lr 0.005000] +2024-04-08 20:56:47,558 - Train: 17.47% [710900/4070000] [174.7/1000.0] [batch_t 1.420 (1.430)] [data_t 0.003] [optim_t 1.418] [lr 0.005000] +2024-04-08 20:59:10,956 - Train: 17.47% [711000/4070000] [174.7/1000.0] [batch_t 1.441 (1.434)] [data_t 0.003] [optim_t 1.439] [lr 0.005000] +2024-04-08 21:01:34,155 - Train: 17.47% [711100/4070000] [174.7/1000.0] [batch_t 1.445 (1.432)] [data_t 0.003] [optim_t 1.442] [lr 0.005000] +2024-04-08 21:03:57,373 - Train: 17.47% [711200/4070000] [174.7/1000.0] [batch_t 1.449 (1.432)] [data_t 0.002] [optim_t 1.446] [lr 0.005000] +2024-04-08 21:06:20,525 - Train: 17.48% [711300/4070000] [174.8/1000.0] [batch_t 1.422 (1.431)] [data_t 0.003] [optim_t 1.420] [lr 0.005000] +2024-04-08 21:08:43,564 - Train: 17.48% [711400/4070000] [174.8/1000.0] [batch_t 1.432 (1.430)] [data_t 0.002] [optim_t 1.430] [lr 0.005000] +2024-04-08 21:11:06,707 - Train: 17.48% [711500/4070000] [174.8/1000.0] [batch_t 1.436 (1.431)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-08 21:13:29,841 - Train: 17.48% [711600/4070000] [174.8/1000.0] [batch_t 1.425 (1.431)] [data_t 0.002] [optim_t 1.423] [lr 0.005000] +2024-04-08 21:15:53,049 - Train: 17.49% [711700/4070000] [174.9/1000.0] [batch_t 1.418 (1.432)] [data_t 0.003] [optim_t 1.416] [lr 0.005000] +2024-04-08 21:18:16,233 - Train: 17.49% [711800/4070000] [174.9/1000.0] [batch_t 1.428 (1.432)] [data_t 0.003] [optim_t 1.425] [lr 0.005000] +2024-04-08 21:20:39,433 - Train: 17.49% [711900/4070000] [174.9/1000.0] [batch_t 1.439 (1.432)] [data_t 0.003] [optim_t 1.436] [lr 0.005000] +2024-04-08 21:23:02,567 - Train: 17.49% [712000/4070000] [174.9/1000.0] [batch_t 1.431 (1.431)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-08 21:25:25,592 - Train: 17.50% [712100/4070000] [175.0/1000.0] [batch_t 1.428 (1.430)] [data_t 0.003] [optim_t 1.425] [lr 0.005000] +2024-04-08 21:27:48,648 - Train: 17.50% [712200/4070000] [175.0/1000.0] [batch_t 1.431 (1.430)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-08 21:29:00,222 - ==> Total time: 6 days, 20:28:31 Eta: 32 days, 7:23:00 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-08 21:30:13,600 - Train: 17.50% [712300/4070000] [175.0/1000.0] [batch_t 1.506 (1.440)] [data_t 0.003] [optim_t 1.503] [lr 0.005000] +2024-04-08 21:32:36,856 - Train: 17.50% [712400/4070000] [175.0/1000.0] [batch_t 1.432 (1.432)] [data_t 0.002] [optim_t 1.430] [lr 0.005000] +2024-04-08 21:34:59,993 - Train: 17.51% [712500/4070000] [175.1/1000.0] [batch_t 1.441 (1.431)] [data_t 0.002] [optim_t 1.439] [lr 0.005000] +2024-04-08 21:37:23,194 - Train: 17.51% [712600/4070000] [175.1/1000.0] [batch_t 1.439 (1.432)] [data_t 0.003] [optim_t 1.436] [lr 0.005000] +2024-04-08 21:39:46,263 - Train: 17.51% [712700/4070000] [175.1/1000.0] [batch_t 1.409 (1.431)] [data_t 0.002] [optim_t 1.407] [lr 0.005000] +2024-04-08 21:42:09,321 - Train: 17.51% [712800/4070000] [175.1/1000.0] [batch_t 1.426 (1.430)] [data_t 0.002] [optim_t 1.423] [lr 0.005000] +2024-04-08 21:44:32,449 - Train: 17.52% [712900/4070000] [175.2/1000.0] [batch_t 1.440 (1.431)] [data_t 0.003] [optim_t 1.437] [lr 0.005000] +2024-04-08 21:46:55,668 - Train: 17.52% [713000/4070000] [175.2/1000.0] [batch_t 1.421 (1.432)] [data_t 0.003] [optim_t 1.418] [lr 0.005000] +2024-04-08 21:49:18,851 - Train: 17.52% [713100/4070000] [175.2/1000.0] [batch_t 1.436 (1.432)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-08 21:51:41,899 - Train: 17.52% [713200/4070000] [175.2/1000.0] [batch_t 1.429 (1.430)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-08 21:54:04,982 - Train: 17.53% [713300/4070000] [175.3/1000.0] [batch_t 1.432 (1.431)] [data_t 0.003] [optim_t 1.429] [lr 0.005000] +2024-04-08 21:56:28,142 - Train: 17.53% [713400/4070000] [175.3/1000.0] [batch_t 1.407 (1.432)] [data_t 0.002] [optim_t 1.405] [lr 0.005000] +2024-04-08 21:58:51,473 - Train: 17.53% [713500/4070000] [175.3/1000.0] [batch_t 1.441 (1.433)] [data_t 0.002] [optim_t 1.439] [lr 0.005000] +2024-04-08 22:01:14,948 - Train: 17.53% [713600/4070000] [175.3/1000.0] [batch_t 1.444 (1.435)] [data_t 0.002] [optim_t 1.442] [lr 0.005000] +2024-04-08 22:03:38,214 - Train: 17.54% [713700/4070000] [175.4/1000.0] [batch_t 1.434 (1.433)] [data_t 0.003] [optim_t 1.431] [lr 0.005000] +2024-04-08 22:06:01,390 - Train: 17.54% [713800/4070000] [175.4/1000.0] [batch_t 1.445 (1.432)] [data_t 0.002] [optim_t 1.443] [lr 0.005000] +2024-04-08 22:08:24,517 - Train: 17.54% [713900/4070000] [175.4/1000.0] [batch_t 1.383 (1.431)] [data_t 0.003] [optim_t 1.381] [lr 0.005000] +2024-04-08 22:10:47,781 - Train: 17.54% [714000/4070000] [175.4/1000.0] [batch_t 1.434 (1.433)] [data_t 0.002] [optim_t 1.432] [lr 0.005000] +2024-04-08 22:13:11,097 - Train: 17.55% [714100/4070000] [175.5/1000.0] [batch_t 1.426 (1.433)] [data_t 0.003] [optim_t 1.424] [lr 0.005000] +2024-04-08 22:15:34,319 - Train: 17.55% [714200/4070000] [175.5/1000.0] [batch_t 1.436 (1.432)] [data_t 0.002] [optim_t 1.434] [lr 0.005000] +2024-04-08 22:17:57,527 - Train: 17.55% [714300/4070000] [175.5/1000.0] [batch_t 1.436 (1.432)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-08 22:20:20,657 - Train: 17.55% [714400/4070000] [175.5/1000.0] [batch_t 1.420 (1.431)] [data_t 0.003] [optim_t 1.418] [lr 0.005000] +2024-04-08 22:22:43,759 - Train: 17.56% [714500/4070000] [175.6/1000.0] [batch_t 1.430 (1.431)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-08 22:25:06,827 - Train: 17.56% [714600/4070000] [175.6/1000.0] [batch_t 1.446 (1.431)] [data_t 0.002] [optim_t 1.443] [lr 0.005000] +2024-04-08 22:27:29,731 - Train: 17.56% [714700/4070000] [175.6/1000.0] [batch_t 1.439 (1.429)] [data_t 0.003] [optim_t 1.437] [lr 0.005000] +2024-04-08 22:29:52,866 - Train: 17.56% [714800/4070000] [175.6/1000.0] [batch_t 1.433 (1.431)] [data_t 0.002] [optim_t 1.431] [lr 0.005000] +2024-04-08 22:32:16,160 - Train: 17.57% [714900/4070000] [175.7/1000.0] [batch_t 1.434 (1.433)] [data_t 0.002] [optim_t 1.432] [lr 0.005000] +2024-04-08 22:34:39,179 - Train: 17.57% [715000/4070000] [175.7/1000.0] [batch_t 1.423 (1.430)] [data_t 0.003] [optim_t 1.420] [lr 0.005000] +2024-04-08 22:37:02,214 - Train: 17.57% [715100/4070000] [175.7/1000.0] [batch_t 1.431 (1.430)] [data_t 0.002] [optim_t 1.429] [lr 0.005000] +2024-04-08 22:39:25,357 - Train: 17.57% [715200/4070000] [175.7/1000.0] [batch_t 1.437 (1.431)] [data_t 0.002] [optim_t 1.435] [lr 0.005000] +2024-04-08 22:41:48,500 - Train: 17.57% [715300/4070000] [175.7/1000.0] [batch_t 1.430 (1.431)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-08 22:44:11,486 - Train: 17.58% [715400/4070000] [175.8/1000.0] [batch_t 1.446 (1.430)] [data_t 0.002] [optim_t 1.444] [lr 0.005000] +2024-04-08 22:46:34,446 - Train: 17.58% [715500/4070000] [175.8/1000.0] [batch_t 1.420 (1.429)] [data_t 0.003] [optim_t 1.417] [lr 0.005000] +2024-04-08 22:48:57,257 - Train: 17.58% [715600/4070000] [175.8/1000.0] [batch_t 1.444 (1.428)] [data_t 0.003] [optim_t 1.441] [lr 0.005000] +2024-04-08 22:51:20,104 - Train: 17.58% [715700/4070000] [175.8/1000.0] [batch_t 1.436 (1.428)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-08 22:53:43,026 - Train: 17.59% [715800/4070000] [175.9/1000.0] [batch_t 1.430 (1.429)] [data_t 0.002] [optim_t 1.428] [lr 0.005000] +2024-04-08 22:56:05,889 - Train: 17.59% [715900/4070000] [175.9/1000.0] [batch_t 1.411 (1.429)] [data_t 0.003] [optim_t 1.409] [lr 0.005000] +2024-04-08 22:58:28,993 - Train: 17.59% [716000/4070000] [175.9/1000.0] [batch_t 1.435 (1.430)] [data_t 0.002] [optim_t 1.432] [lr 0.005000] +2024-04-08 23:00:51,907 - Train: 17.59% [716100/4070000] [175.9/1000.0] [batch_t 1.435 (1.429)] [data_t 0.002] [optim_t 1.432] [lr 0.005000] +2024-04-08 23:03:14,763 - Train: 17.60% [716200/4070000] [176.0/1000.0] [batch_t 1.429 (1.428)] [data_t 0.002] [optim_t 1.427] [lr 0.005000] +2024-04-08 23:05:37,610 - Train: 17.60% [716300/4070000] [176.0/1000.0] [batch_t 1.434 (1.428)] [data_t 0.002] [optim_t 1.432] [lr 0.005000] +2024-04-08 23:06:06,283 - ==> Total time: 6 days, 22:05:37 Eta: 32 days, 9:37:12 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-08 23:08:02,291 - Train: 17.60% [716400/4070000] [176.0/1000.0] [batch_t 1.425 (1.431)] [data_t 0.003] [optim_t 1.422] [lr 0.005000] +2024-04-08 23:10:25,035 - Train: 17.60% [716500/4070000] [176.0/1000.0] [batch_t 1.432 (1.427)] [data_t 0.002] [optim_t 1.430] [lr 0.005000] +2024-04-08 23:12:47,744 - Train: 17.61% [716600/4070000] [176.1/1000.0] [batch_t 1.425 (1.427)] [data_t 0.003] [optim_t 1.422] [lr 0.005000] +2024-04-08 23:15:10,479 - Train: 17.61% [716700/4070000] [176.1/1000.0] [batch_t 1.420 (1.427)] [data_t 0.002] [optim_t 1.418] [lr 0.005000] +2024-04-08 23:17:33,418 - Train: 17.61% [716800/4070000] [176.1/1000.0] [batch_t 1.439 (1.429)] [data_t 0.003] [optim_t 1.436] [lr 0.005000] +2024-04-08 23:19:56,489 - Train: 17.61% [716900/4070000] [176.1/1000.0] [batch_t 1.426 (1.431)] [data_t 0.002] [optim_t 1.424] [lr 0.005000] +2024-04-08 23:22:19,514 - Train: 17.62% [717000/4070000] [176.2/1000.0] [batch_t 1.425 (1.430)] [data_t 0.002] [optim_t 1.423] [lr 0.005000] +2024-04-08 23:24:42,537 - Train: 17.62% [717100/4070000] [176.2/1000.0] [batch_t 1.426 (1.430)] [data_t 0.002] [optim_t 1.424] [lr 0.005000] +2024-04-08 23:27:05,506 - Train: 17.62% [717200/4070000] [176.2/1000.0] [batch_t 1.438 (1.430)] [data_t 0.003] [optim_t 1.435] [lr 0.005000] +2024-04-08 23:29:28,326 - Train: 17.62% [717300/4070000] [176.2/1000.0] [batch_t 1.428 (1.428)] [data_t 0.002] [optim_t 1.426] [lr 0.005000] +2024-04-08 23:31:51,160 - Train: 17.63% [717400/4070000] [176.3/1000.0] [batch_t 1.424 (1.428)] [data_t 0.003] [optim_t 1.422] [lr 0.005000] +2024-04-08 23:34:14,076 - Train: 17.63% [717500/4070000] [176.3/1000.0] [batch_t 1.431 (1.429)] [data_t 0.003] [optim_t 1.429] [lr 0.005000] +2024-04-08 23:36:36,898 - Train: 17.63% [717600/4070000] [176.3/1000.0] [batch_t 1.429 (1.428)] [data_t 0.003] [optim_t 1.426] [lr 0.005000] +2024-04-08 23:38:59,820 - Train: 17.63% [717700/4070000] [176.3/1000.0] [batch_t 1.425 (1.429)] [data_t 0.002] [optim_t 1.423] [lr 0.005000] +2024-04-08 23:41:22,725 - Train: 17.64% [717800/4070000] [176.4/1000.0] [batch_t 1.434 (1.429)] [data_t 0.002] [optim_t 1.432] [lr 0.005000] +2024-04-08 23:43:45,581 - Train: 17.64% [717900/4070000] [176.4/1000.0] [batch_t 1.435 (1.428)] [data_t 0.003] [optim_t 1.432] [lr 0.005000] +2024-04-08 23:46:08,385 - Train: 17.64% [718000/4070000] [176.4/1000.0] [batch_t 1.429 (1.428)] [data_t 0.003] [optim_t 1.426] [lr 0.005000] +2024-04-08 23:48:31,272 - Train: 17.64% [718100/4070000] [176.4/1000.0] [batch_t 1.440 (1.429)] [data_t 0.003] [optim_t 1.437] [lr 0.005000] +2024-04-08 23:50:54,063 - Train: 17.65% [718200/4070000] [176.5/1000.0] [batch_t 1.432 (1.428)] [data_t 0.002] [optim_t 1.430] [lr 0.005000] +2024-04-08 23:53:17,035 - Train: 17.65% [718300/4070000] [176.5/1000.0] [batch_t 1.431 (1.430)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-08 23:55:40,133 - Train: 17.65% [718400/4070000] [176.5/1000.0] [batch_t 1.445 (1.431)] [data_t 0.002] [optim_t 1.443] [lr 0.005000] +2024-04-08 23:58:03,171 - Train: 17.65% [718500/4070000] [176.5/1000.0] [batch_t 1.433 (1.430)] [data_t 0.003] [optim_t 1.430] [lr 0.005000] +2024-04-09 00:00:26,155 - Train: 17.66% [718600/4070000] [176.6/1000.0] [batch_t 1.425 (1.430)] [data_t 0.002] [optim_t 1.422] [lr 0.005000] +2024-04-09 00:02:49,073 - Train: 17.66% [718700/4070000] [176.6/1000.0] [batch_t 1.440 (1.429)] [data_t 0.002] [optim_t 1.438] [lr 0.005000] +2024-04-09 00:05:11,806 - Train: 17.66% [718800/4070000] [176.6/1000.0] [batch_t 1.432 (1.427)] [data_t 0.003] [optim_t 1.430] [lr 0.005000] +2024-04-09 00:07:34,552 - Train: 17.66% [718900/4070000] [176.6/1000.0] [batch_t 1.438 (1.427)] [data_t 0.003] [optim_t 1.435] [lr 0.005000] +2024-04-09 00:09:57,495 - Train: 17.67% [719000/4070000] [176.7/1000.0] [batch_t 1.429 (1.429)] [data_t 0.002] [optim_t 1.426] [lr 0.005000] +2024-04-09 00:12:20,323 - Train: 17.67% [719100/4070000] [176.7/1000.0] [batch_t 1.422 (1.428)] [data_t 0.003] [optim_t 1.419] [lr 0.005000] +2024-04-09 00:14:43,366 - Train: 17.67% [719200/4070000] [176.7/1000.0] [batch_t 1.434 (1.430)] [data_t 0.003] [optim_t 1.431] [lr 0.005000] +2024-04-09 00:17:06,383 - Train: 17.67% [719300/4070000] [176.7/1000.0] [batch_t 1.433 (1.430)] [data_t 0.002] [optim_t 1.431] [lr 0.005000] +2024-04-09 00:19:29,300 - Train: 17.68% [719400/4070000] [176.8/1000.0] [batch_t 1.428 (1.429)] [data_t 0.002] [optim_t 1.425] [lr 0.005000] +2024-04-09 00:21:52,191 - Train: 17.68% [719500/4070000] [176.8/1000.0] [batch_t 1.431 (1.429)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-09 00:24:15,114 - Train: 17.68% [719600/4070000] [176.8/1000.0] [batch_t 1.418 (1.429)] [data_t 0.003] [optim_t 1.415] [lr 0.005000] +2024-04-09 00:26:37,950 - Train: 17.68% [719700/4070000] [176.8/1000.0] [batch_t 1.427 (1.428)] [data_t 0.003] [optim_t 1.425] [lr 0.005000] +2024-04-09 00:29:00,729 - Train: 17.69% [719800/4070000] [176.9/1000.0] [batch_t 1.416 (1.428)] [data_t 0.003] [optim_t 1.413] [lr 0.005000] +2024-04-09 00:31:23,637 - Train: 17.69% [719900/4070000] [176.9/1000.0] [batch_t 1.435 (1.429)] [data_t 0.003] [optim_t 1.432] [lr 0.005000] +2024-04-09 00:33:46,534 - Train: 17.69% [720000/4070000] [176.9/1000.0] [batch_t 1.426 (1.429)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-09 00:36:09,495 - Train: 17.69% [720100/4070000] [176.9/1000.0] [batch_t 1.441 (1.429)] [data_t 0.002] [optim_t 1.439] [lr 0.005000] +2024-04-09 00:38:32,270 - Train: 17.70% [720200/4070000] [177.0/1000.0] [batch_t 1.429 (1.428)] [data_t 0.002] [optim_t 1.427] [lr 0.005000] +2024-04-09 00:40:55,141 - Train: 17.70% [720300/4070000] [177.0/1000.0] [batch_t 1.428 (1.429)] [data_t 0.003] [optim_t 1.425] [lr 0.005000] +2024-04-09 00:43:03,790 - ==> Total time: 6 days, 23:42:34 Eta: 32 days, 11:48:08 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-09 00:43:20,254 - Train: 17.70% [720400/4070000] [177.0/1000.0] [batch_t 1.420 (1.444)] [data_t 0.002] [optim_t 1.418] [lr 0.005000] +2024-04-09 00:45:42,971 - Train: 17.70% [720500/4070000] [177.0/1000.0] [batch_t 1.439 (1.427)] [data_t 0.003] [optim_t 1.436] [lr 0.005000] +2024-04-09 00:48:05,903 - Train: 17.71% [720600/4070000] [177.1/1000.0] [batch_t 1.433 (1.429)] [data_t 0.002] [optim_t 1.430] [lr 0.005000] +2024-04-09 00:50:28,823 - Train: 17.71% [720700/4070000] [177.1/1000.0] [batch_t 1.435 (1.429)] [data_t 0.002] [optim_t 1.433] [lr 0.005000] +2024-04-09 00:52:51,682 - Train: 17.71% [720800/4070000] [177.1/1000.0] [batch_t 1.425 (1.428)] [data_t 0.003] [optim_t 1.422] [lr 0.005000] +2024-04-09 00:55:14,690 - Train: 17.71% [720900/4070000] [177.1/1000.0] [batch_t 1.425 (1.430)] [data_t 0.003] [optim_t 1.422] [lr 0.005000] +2024-04-09 00:57:37,617 - Train: 17.71% [721000/4070000] [177.1/1000.0] [batch_t 1.443 (1.429)] [data_t 0.002] [optim_t 1.441] [lr 0.005000] +2024-04-09 01:00:00,584 - Train: 17.72% [721100/4070000] [177.2/1000.0] [batch_t 1.429 (1.430)] [data_t 0.002] [optim_t 1.427] [lr 0.005000] +2024-04-09 01:02:23,476 - Train: 17.72% [721200/4070000] [177.2/1000.0] [batch_t 1.418 (1.429)] [data_t 0.002] [optim_t 1.416] [lr 0.005000] +2024-04-09 01:04:46,321 - Train: 17.72% [721300/4070000] [177.2/1000.0] [batch_t 1.433 (1.428)] [data_t 0.003] [optim_t 1.430] [lr 0.005000] +2024-04-09 01:07:09,252 - Train: 17.72% [721400/4070000] [177.2/1000.0] [batch_t 1.435 (1.429)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-09 01:09:32,072 - Train: 17.73% [721500/4070000] [177.3/1000.0] [batch_t 1.442 (1.428)] [data_t 0.002] [optim_t 1.440] [lr 0.005000] +2024-04-09 01:11:55,030 - Train: 17.73% [721600/4070000] [177.3/1000.0] [batch_t 1.427 (1.429)] [data_t 0.002] [optim_t 1.425] [lr 0.005000] +2024-04-09 01:14:17,820 - Train: 17.73% [721700/4070000] [177.3/1000.0] [batch_t 1.430 (1.428)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-09 01:16:40,613 - Train: 17.73% [721800/4070000] [177.3/1000.0] [batch_t 1.429 (1.428)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-09 01:19:03,483 - Train: 17.74% [721900/4070000] [177.4/1000.0] [batch_t 1.425 (1.429)] [data_t 0.003] [optim_t 1.422] [lr 0.005000] +2024-04-09 01:21:26,456 - Train: 17.74% [722000/4070000] [177.4/1000.0] [batch_t 1.431 (1.430)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-09 01:23:49,347 - Train: 17.74% [722100/4070000] [177.4/1000.0] [batch_t 1.430 (1.429)] [data_t 0.002] [optim_t 1.428] [lr 0.005000] +2024-04-09 01:26:12,295 - Train: 17.74% [722200/4070000] [177.4/1000.0] [batch_t 1.422 (1.429)] [data_t 0.002] [optim_t 1.419] [lr 0.005000] +2024-04-09 01:28:35,001 - Train: 17.75% [722300/4070000] [177.5/1000.0] [batch_t 1.398 (1.427)] [data_t 0.003] [optim_t 1.395] [lr 0.005000] +2024-04-09 01:30:58,017 - Train: 17.75% [722400/4070000] [177.5/1000.0] [batch_t 1.425 (1.430)] [data_t 0.003] [optim_t 1.422] [lr 0.005000] +2024-04-09 01:33:20,792 - Train: 17.75% [722500/4070000] [177.5/1000.0] [batch_t 1.420 (1.428)] [data_t 0.003] [optim_t 1.416] [lr 0.005000] +2024-04-09 01:35:43,560 - Train: 17.75% [722600/4070000] [177.5/1000.0] [batch_t 1.428 (1.428)] [data_t 0.002] [optim_t 1.425] [lr 0.005000] +2024-04-09 01:38:06,404 - Train: 17.76% [722700/4070000] [177.6/1000.0] [batch_t 1.421 (1.428)] [data_t 0.003] [optim_t 1.418] [lr 0.005000] +2024-04-09 01:40:29,400 - Train: 17.76% [722800/4070000] [177.6/1000.0] [batch_t 1.435 (1.430)] [data_t 0.002] [optim_t 1.433] [lr 0.005000] +2024-04-09 01:42:52,239 - Train: 17.76% [722900/4070000] [177.6/1000.0] [batch_t 1.433 (1.428)] [data_t 0.002] [optim_t 1.431] [lr 0.005000] +2024-04-09 01:45:14,995 - Train: 17.76% [723000/4070000] [177.6/1000.0] [batch_t 1.415 (1.427)] [data_t 0.003] [optim_t 1.412] [lr 0.005000] +2024-04-09 01:47:37,924 - Train: 17.77% [723100/4070000] [177.7/1000.0] [batch_t 1.430 (1.429)] [data_t 0.002] [optim_t 1.427] [lr 0.005000] +2024-04-09 01:50:00,905 - Train: 17.77% [723200/4070000] [177.7/1000.0] [batch_t 1.426 (1.430)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-09 01:52:23,877 - Train: 17.77% [723300/4070000] [177.7/1000.0] [batch_t 1.433 (1.430)] [data_t 0.003] [optim_t 1.430] [lr 0.005000] +2024-04-09 01:54:46,661 - Train: 17.77% [723400/4070000] [177.7/1000.0] [batch_t 1.436 (1.428)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-09 01:57:09,489 - Train: 17.78% [723500/4070000] [177.8/1000.0] [batch_t 1.416 (1.428)] [data_t 0.003] [optim_t 1.413] [lr 0.005000] +2024-04-09 01:59:32,485 - Train: 17.78% [723600/4070000] [177.8/1000.0] [batch_t 1.441 (1.430)] [data_t 0.002] [optim_t 1.439] [lr 0.005000] +2024-04-09 02:01:55,584 - Train: 17.78% [723700/4070000] [177.8/1000.0] [batch_t 1.433 (1.431)] [data_t 0.003] [optim_t 1.430] [lr 0.005000] +2024-04-09 02:04:18,478 - Train: 17.78% [723800/4070000] [177.8/1000.0] [batch_t 1.435 (1.429)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-09 02:06:41,464 - Train: 17.79% [723900/4070000] [177.9/1000.0] [batch_t 1.431 (1.430)] [data_t 0.002] [optim_t 1.429] [lr 0.005000] +2024-04-09 02:09:04,368 - Train: 17.79% [724000/4070000] [177.9/1000.0] [batch_t 1.435 (1.429)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-09 02:11:27,291 - Train: 17.79% [724100/4070000] [177.9/1000.0] [batch_t 1.421 (1.429)] [data_t 0.002] [optim_t 1.419] [lr 0.005000] +2024-04-09 02:13:50,249 - Train: 17.79% [724200/4070000] [177.9/1000.0] [batch_t 1.453 (1.429)] [data_t 0.003] [optim_t 1.450] [lr 0.005000] +2024-04-09 02:16:13,090 - Train: 17.80% [724300/4070000] [178.0/1000.0] [batch_t 1.435 (1.428)] [data_t 0.003] [optim_t 1.432] [lr 0.005000] +2024-04-09 02:18:36,027 - Train: 17.80% [724400/4070000] [178.0/1000.0] [batch_t 1.429 (1.429)] [data_t 0.003] [optim_t 1.426] [lr 0.005000] +2024-04-09 02:20:01,787 - ==> Total time: 7 days, 1:19:32 Eta: 32 days, 13:56:32 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-09 02:21:02,341 - Train: 17.80% [724500/4070000] [178.0/1000.0] [batch_t 1.425 (1.434)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-09 02:23:25,301 - Train: 17.80% [724600/4070000] [178.0/1000.0] [batch_t 1.448 (1.430)] [data_t 0.002] [optim_t 1.446] [lr 0.005000] +2024-04-09 02:25:48,168 - Train: 17.81% [724700/4070000] [178.1/1000.0] [batch_t 1.434 (1.429)] [data_t 0.003] [optim_t 1.431] [lr 0.005000] +2024-04-09 02:28:11,097 - Train: 17.81% [724800/4070000] [178.1/1000.0] [batch_t 1.429 (1.429)] [data_t 0.003] [optim_t 1.426] [lr 0.005000] +2024-04-09 02:30:34,077 - Train: 17.81% [724900/4070000] [178.1/1000.0] [batch_t 1.404 (1.430)] [data_t 0.002] [optim_t 1.402] [lr 0.005000] +2024-04-09 02:32:56,946 - Train: 17.81% [725000/4070000] [178.1/1000.0] [batch_t 1.427 (1.429)] [data_t 0.003] [optim_t 1.424] [lr 0.005000] +2024-04-09 02:35:19,871 - Train: 17.82% [725100/4070000] [178.2/1000.0] [batch_t 1.439 (1.429)] [data_t 0.002] [optim_t 1.436] [lr 0.005000] +2024-04-09 02:37:42,668 - Train: 17.82% [725200/4070000] [178.2/1000.0] [batch_t 1.444 (1.428)] [data_t 0.002] [optim_t 1.442] [lr 0.005000] +2024-04-09 02:40:05,521 - Train: 17.82% [725300/4070000] [178.2/1000.0] [batch_t 1.429 (1.428)] [data_t 0.003] [optim_t 1.426] [lr 0.005000] +2024-04-09 02:42:28,430 - Train: 17.82% [725400/4070000] [178.2/1000.0] [batch_t 1.432 (1.429)] [data_t 0.002] [optim_t 1.429] [lr 0.005000] +2024-04-09 02:44:51,305 - Train: 17.83% [725500/4070000] [178.3/1000.0] [batch_t 1.435 (1.429)] [data_t 0.002] [optim_t 1.433] [lr 0.005000] +2024-04-09 02:47:14,299 - Train: 17.83% [725600/4070000] [178.3/1000.0] [batch_t 1.438 (1.430)] [data_t 0.002] [optim_t 1.435] [lr 0.005000] +2024-04-09 02:49:37,262 - Train: 17.83% [725700/4070000] [178.3/1000.0] [batch_t 1.439 (1.430)] [data_t 0.002] [optim_t 1.437] [lr 0.005000] +2024-04-09 02:52:00,198 - Train: 17.83% [725800/4070000] [178.3/1000.0] [batch_t 1.425 (1.429)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-09 02:54:23,054 - Train: 17.84% [725900/4070000] [178.4/1000.0] [batch_t 1.429 (1.428)] [data_t 0.002] [optim_t 1.426] [lr 0.005000] +2024-04-09 02:56:45,874 - Train: 17.84% [726000/4070000] [178.4/1000.0] [batch_t 1.434 (1.428)] [data_t 0.002] [optim_t 1.432] [lr 0.005000] +2024-04-09 02:59:08,745 - Train: 17.84% [726100/4070000] [178.4/1000.0] [batch_t 1.444 (1.429)] [data_t 0.002] [optim_t 1.441] [lr 0.005000] +2024-04-09 03:01:31,547 - Train: 17.84% [726200/4070000] [178.4/1000.0] [batch_t 1.420 (1.428)] [data_t 0.002] [optim_t 1.418] [lr 0.005000] +2024-04-09 03:03:54,384 - Train: 17.85% [726300/4070000] [178.5/1000.0] [batch_t 1.424 (1.428)] [data_t 0.002] [optim_t 1.422] [lr 0.005000] +2024-04-09 03:06:17,251 - Train: 17.85% [726400/4070000] [178.5/1000.0] [batch_t 1.431 (1.429)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-09 03:08:40,236 - Train: 17.85% [726500/4070000] [178.5/1000.0] [batch_t 1.425 (1.430)] [data_t 0.002] [optim_t 1.423] [lr 0.005000] +2024-04-09 03:11:03,056 - Train: 17.85% [726600/4070000] [178.5/1000.0] [batch_t 1.435 (1.428)] [data_t 0.003] [optim_t 1.432] [lr 0.005000] +2024-04-09 03:13:26,030 - Train: 17.86% [726700/4070000] [178.6/1000.0] [batch_t 1.419 (1.430)] [data_t 0.003] [optim_t 1.416] [lr 0.005000] +2024-04-09 03:15:49,010 - Train: 17.86% [726800/4070000] [178.6/1000.0] [batch_t 1.431 (1.430)] [data_t 0.002] [optim_t 1.429] [lr 0.005000] +2024-04-09 03:18:11,891 - Train: 17.86% [726900/4070000] [178.6/1000.0] [batch_t 1.425 (1.429)] [data_t 0.002] [optim_t 1.423] [lr 0.005000] +2024-04-09 03:20:34,626 - Train: 17.86% [727000/4070000] [178.6/1000.0] [batch_t 1.437 (1.427)] [data_t 0.003] [optim_t 1.434] [lr 0.005000] +2024-04-09 03:22:57,515 - Train: 17.86% [727100/4070000] [178.6/1000.0] [batch_t 1.435 (1.429)] [data_t 0.002] [optim_t 1.432] [lr 0.005000] +2024-04-09 03:25:20,437 - Train: 17.87% [727200/4070000] [178.7/1000.0] [batch_t 1.430 (1.429)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-09 03:27:43,627 - Train: 17.87% [727300/4070000] [178.7/1000.0] [batch_t 1.435 (1.432)] [data_t 0.002] [optim_t 1.432] [lr 0.005000] +2024-04-09 03:30:06,612 - Train: 17.87% [727400/4070000] [178.7/1000.0] [batch_t 1.433 (1.430)] [data_t 0.002] [optim_t 1.431] [lr 0.005000] +2024-04-09 03:32:29,799 - Train: 17.87% [727500/4070000] [178.7/1000.0] [batch_t 1.436 (1.432)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-09 03:34:52,925 - Train: 17.88% [727600/4070000] [178.8/1000.0] [batch_t 1.430 (1.431)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-09 03:37:15,826 - Train: 17.88% [727700/4070000] [178.8/1000.0] [batch_t 1.401 (1.429)] [data_t 0.002] [optim_t 1.399] [lr 0.005000] +2024-04-09 03:39:38,767 - Train: 17.88% [727800/4070000] [178.8/1000.0] [batch_t 1.435 (1.429)] [data_t 0.002] [optim_t 1.433] [lr 0.005000] +2024-04-09 03:42:01,852 - Train: 17.88% [727900/4070000] [178.8/1000.0] [batch_t 1.441 (1.431)] [data_t 0.003] [optim_t 1.438] [lr 0.005000] +2024-04-09 03:44:24,958 - Train: 17.89% [728000/4070000] [178.9/1000.0] [batch_t 1.426 (1.431)] [data_t 0.003] [optim_t 1.424] [lr 0.005000] +2024-04-09 03:46:48,001 - Train: 17.89% [728100/4070000] [178.9/1000.0] [batch_t 1.434 (1.430)] [data_t 0.002] [optim_t 1.431] [lr 0.005000] +2024-04-09 03:49:11,041 - Train: 17.89% [728200/4070000] [178.9/1000.0] [batch_t 1.420 (1.430)] [data_t 0.003] [optim_t 1.417] [lr 0.005000] +2024-04-09 03:51:34,045 - Train: 17.89% [728300/4070000] [178.9/1000.0] [batch_t 1.428 (1.430)] [data_t 0.003] [optim_t 1.425] [lr 0.005000] +2024-04-09 03:53:56,953 - Train: 17.90% [728400/4070000] [179.0/1000.0] [batch_t 1.431 (1.429)] [data_t 0.002] [optim_t 1.429] [lr 0.005000] +2024-04-09 03:56:20,004 - Train: 17.90% [728500/4070000] [179.0/1000.0] [batch_t 1.438 (1.430)] [data_t 0.002] [optim_t 1.435] [lr 0.005000] +2024-04-09 03:57:02,951 - ==> Total time: 7 days, 2:56:33 Eta: 32 days, 16:02:40 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-09 03:58:46,941 - Train: 17.90% [728600/4070000] [179.0/1000.0] [batch_t 1.428 (1.441)] [data_t 0.003] [optim_t 1.426] [lr 0.005000] +2024-04-09 04:01:09,918 - Train: 17.90% [728700/4070000] [179.0/1000.0] [batch_t 1.422 (1.430)] [data_t 0.002] [optim_t 1.419] [lr 0.005000] +2024-04-09 04:03:32,933 - Train: 17.91% [728800/4070000] [179.1/1000.0] [batch_t 1.425 (1.430)] [data_t 0.003] [optim_t 1.422] [lr 0.005000] +2024-04-09 04:05:55,935 - Train: 17.91% [728900/4070000] [179.1/1000.0] [batch_t 1.424 (1.430)] [data_t 0.002] [optim_t 1.421] [lr 0.005000] +2024-04-09 04:08:19,059 - Train: 17.91% [729000/4070000] [179.1/1000.0] [batch_t 1.447 (1.431)] [data_t 0.003] [optim_t 1.445] [lr 0.005000] +2024-04-09 04:10:42,129 - Train: 17.91% [729100/4070000] [179.1/1000.0] [batch_t 1.440 (1.431)] [data_t 0.003] [optim_t 1.437] [lr 0.005000] +2024-04-09 04:13:05,331 - Train: 17.92% [729200/4070000] [179.2/1000.0] [batch_t 1.434 (1.432)] [data_t 0.002] [optim_t 1.432] [lr 0.005000] +2024-04-09 04:15:28,525 - Train: 17.92% [729300/4070000] [179.2/1000.0] [batch_t 1.430 (1.432)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-09 04:17:51,684 - Train: 17.92% [729400/4070000] [179.2/1000.0] [batch_t 1.428 (1.432)] [data_t 0.003] [optim_t 1.425] [lr 0.005000] +2024-04-09 04:20:14,661 - Train: 17.92% [729500/4070000] [179.2/1000.0] [batch_t 1.434 (1.430)] [data_t 0.002] [optim_t 1.431] [lr 0.005000] +2024-04-09 04:22:37,843 - Train: 17.93% [729600/4070000] [179.3/1000.0] [batch_t 1.427 (1.432)] [data_t 0.003] [optim_t 1.424] [lr 0.005000] +2024-04-09 04:25:00,930 - Train: 17.93% [729700/4070000] [179.3/1000.0] [batch_t 1.441 (1.431)] [data_t 0.002] [optim_t 1.439] [lr 0.005000] +2024-04-09 04:27:23,853 - Train: 17.93% [729800/4070000] [179.3/1000.0] [batch_t 1.428 (1.429)] [data_t 0.002] [optim_t 1.426] [lr 0.005000] +2024-04-09 04:29:46,837 - Train: 17.93% [729900/4070000] [179.3/1000.0] [batch_t 1.447 (1.430)] [data_t 0.002] [optim_t 1.444] [lr 0.005000] +2024-04-09 04:32:09,848 - Train: 17.94% [730000/4070000] [179.4/1000.0] [batch_t 1.420 (1.430)] [data_t 0.002] [optim_t 1.418] [lr 0.005000] +2024-04-09 04:34:33,046 - Train: 17.94% [730100/4070000] [179.4/1000.0] [batch_t 1.431 (1.432)] [data_t 0.002] [optim_t 1.429] [lr 0.005000] +2024-04-09 04:36:56,151 - Train: 17.94% [730200/4070000] [179.4/1000.0] [batch_t 1.435 (1.431)] [data_t 0.002] [optim_t 1.433] [lr 0.005000] +2024-04-09 04:39:19,221 - Train: 17.94% [730300/4070000] [179.4/1000.0] [batch_t 1.436 (1.431)] [data_t 0.002] [optim_t 1.434] [lr 0.005000] +2024-04-09 04:41:42,325 - Train: 17.95% [730400/4070000] [179.5/1000.0] [batch_t 1.429 (1.431)] [data_t 0.003] [optim_t 1.426] [lr 0.005000] +2024-04-09 04:44:05,472 - Train: 17.95% [730500/4070000] [179.5/1000.0] [batch_t 1.435 (1.431)] [data_t 0.002] [optim_t 1.432] [lr 0.005000] +2024-04-09 04:46:28,411 - Train: 17.95% [730600/4070000] [179.5/1000.0] [batch_t 1.437 (1.429)] [data_t 0.003] [optim_t 1.434] [lr 0.005000] +2024-04-09 04:48:51,614 - Train: 17.95% [730700/4070000] [179.5/1000.0] [batch_t 1.412 (1.432)] [data_t 0.003] [optim_t 1.409] [lr 0.005000] +2024-04-09 04:51:14,765 - Train: 17.96% [730800/4070000] [179.6/1000.0] [batch_t 1.443 (1.431)] [data_t 0.002] [optim_t 1.441] [lr 0.005000] +2024-04-09 04:53:37,792 - Train: 17.96% [730900/4070000] [179.6/1000.0] [batch_t 1.431 (1.430)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-09 04:56:00,845 - Train: 17.96% [731000/4070000] [179.6/1000.0] [batch_t 1.426 (1.430)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-09 04:58:23,817 - Train: 17.96% [731100/4070000] [179.6/1000.0] [batch_t 1.439 (1.430)] [data_t 0.003] [optim_t 1.437] [lr 0.005000] +2024-04-09 05:00:46,844 - Train: 17.97% [731200/4070000] [179.7/1000.0] [batch_t 1.437 (1.430)] [data_t 0.002] [optim_t 1.434] [lr 0.005000] +2024-04-09 05:03:10,067 - Train: 17.97% [731300/4070000] [179.7/1000.0] [batch_t 1.434 (1.432)] [data_t 0.003] [optim_t 1.431] [lr 0.005000] +2024-04-09 05:05:32,958 - Train: 17.97% [731400/4070000] [179.7/1000.0] [batch_t 1.410 (1.429)] [data_t 0.003] [optim_t 1.407] [lr 0.005000] +2024-04-09 05:07:56,050 - Train: 17.97% [731500/4070000] [179.7/1000.0] [batch_t 1.426 (1.431)] [data_t 0.002] [optim_t 1.423] [lr 0.005000] +2024-04-09 05:10:18,979 - Train: 17.98% [731600/4070000] [179.8/1000.0] [batch_t 1.429 (1.429)] [data_t 0.003] [optim_t 1.426] [lr 0.005000] +2024-04-09 05:12:42,091 - Train: 17.98% [731700/4070000] [179.8/1000.0] [batch_t 1.429 (1.431)] [data_t 0.003] [optim_t 1.426] [lr 0.005000] +2024-04-09 05:15:05,102 - Train: 17.98% [731800/4070000] [179.8/1000.0] [batch_t 1.416 (1.430)] [data_t 0.003] [optim_t 1.413] [lr 0.005000] +2024-04-09 05:17:28,248 - Train: 17.98% [731900/4070000] [179.8/1000.0] [batch_t 1.451 (1.431)] [data_t 0.002] [optim_t 1.449] [lr 0.005000] +2024-04-09 05:19:51,255 - Train: 17.99% [732000/4070000] [179.9/1000.0] [batch_t 1.433 (1.430)] [data_t 0.003] [optim_t 1.430] [lr 0.005000] +2024-04-09 05:22:14,128 - Train: 17.99% [732100/4070000] [179.9/1000.0] [batch_t 1.430 (1.429)] [data_t 0.002] [optim_t 1.428] [lr 0.005000] +2024-04-09 05:24:37,104 - Train: 17.99% [732200/4070000] [179.9/1000.0] [batch_t 1.435 (1.430)] [data_t 0.003] [optim_t 1.432] [lr 0.005000] +2024-04-09 05:27:00,145 - Train: 17.99% [732300/4070000] [179.9/1000.0] [batch_t 1.434 (1.430)] [data_t 0.002] [optim_t 1.431] [lr 0.005000] +2024-04-09 05:29:23,117 - Train: 18.00% [732400/4070000] [180.0/1000.0] [batch_t 1.430 (1.430)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-09 05:31:46,123 - Train: 18.00% [732500/4070000] [180.0/1000.0] [batch_t 1.421 (1.430)] [data_t 0.002] [optim_t 1.419] [lr 0.005000] +2024-04-09 05:34:08,986 - Train: 18.00% [732600/4070000] [180.0/1000.0] [batch_t 1.425 (1.429)] [data_t 0.002] [optim_t 1.423] [lr 0.005000] +2024-04-09 05:34:08,989 - ==> Total time: 7 days, 4:33:39 Eta: 32 days, 18:06:41 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-09 05:36:33,726 - Train: 18.00% [732700/4070000] [180.0/1000.0] [batch_t 1.442 (1.434)] [data_t 0.002] [optim_t 1.440] [lr 0.005000] +2024-04-09 05:38:56,905 - Train: 18.00% [732800/4070000] [180.0/1000.0] [batch_t 1.436 (1.432)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-09 05:41:20,104 - Train: 18.01% [732900/4070000] [180.1/1000.0] [batch_t 1.426 (1.432)] [data_t 0.002] [optim_t 1.424] [lr 0.005000] +2024-04-09 05:43:43,187 - Train: 18.01% [733000/4070000] [180.1/1000.0] [batch_t 1.435 (1.431)] [data_t 0.003] [optim_t 1.432] [lr 0.005000] +2024-04-09 05:46:06,206 - Train: 18.01% [733100/4070000] [180.1/1000.0] [batch_t 1.434 (1.430)] [data_t 0.002] [optim_t 1.432] [lr 0.005000] +2024-04-09 05:48:29,312 - Train: 18.01% [733200/4070000] [180.1/1000.0] [batch_t 1.414 (1.431)] [data_t 0.003] [optim_t 1.411] [lr 0.005000] +2024-04-09 05:50:52,324 - Train: 18.02% [733300/4070000] [180.2/1000.0] [batch_t 1.435 (1.430)] [data_t 0.002] [optim_t 1.433] [lr 0.005000] +2024-04-09 05:53:15,388 - Train: 18.02% [733400/4070000] [180.2/1000.0] [batch_t 1.441 (1.431)] [data_t 0.002] [optim_t 1.439] [lr 0.005000] +2024-04-09 05:55:38,441 - Train: 18.02% [733500/4070000] [180.2/1000.0] [batch_t 1.426 (1.430)] [data_t 0.003] [optim_t 1.424] [lr 0.005000] +2024-04-09 05:58:01,593 - Train: 18.02% [733600/4070000] [180.2/1000.0] [batch_t 1.424 (1.431)] [data_t 0.002] [optim_t 1.422] [lr 0.005000] +2024-04-09 06:00:24,638 - Train: 18.03% [733700/4070000] [180.3/1000.0] [batch_t 1.423 (1.430)] [data_t 0.002] [optim_t 1.421] [lr 0.005000] +2024-04-09 06:02:47,560 - Train: 18.03% [733800/4070000] [180.3/1000.0] [batch_t 1.428 (1.429)] [data_t 0.002] [optim_t 1.426] [lr 0.005000] +2024-04-09 06:05:10,717 - Train: 18.03% [733900/4070000] [180.3/1000.0] [batch_t 1.440 (1.431)] [data_t 0.002] [optim_t 1.438] [lr 0.005000] +2024-04-09 06:07:33,903 - Train: 18.03% [734000/4070000] [180.3/1000.0] [batch_t 1.430 (1.432)] [data_t 0.002] [optim_t 1.427] [lr 0.005000] +2024-04-09 06:09:56,831 - Train: 18.04% [734100/4070000] [180.4/1000.0] [batch_t 1.429 (1.429)] [data_t 0.002] [optim_t 1.426] [lr 0.005000] +2024-04-09 06:12:19,840 - Train: 18.04% [734200/4070000] [180.4/1000.0] [batch_t 1.434 (1.430)] [data_t 0.003] [optim_t 1.431] [lr 0.005000] +2024-04-09 06:14:42,855 - Train: 18.04% [734300/4070000] [180.4/1000.0] [batch_t 1.441 (1.430)] [data_t 0.003] [optim_t 1.438] [lr 0.005000] +2024-04-09 06:17:05,877 - Train: 18.04% [734400/4070000] [180.4/1000.0] [batch_t 1.434 (1.430)] [data_t 0.002] [optim_t 1.432] [lr 0.005000] +2024-04-09 06:19:28,872 - Train: 18.05% [734500/4070000] [180.5/1000.0] [batch_t 1.426 (1.430)] [data_t 0.002] [optim_t 1.424] [lr 0.005000] +2024-04-09 06:21:51,886 - Train: 18.05% [734600/4070000] [180.5/1000.0] [batch_t 1.430 (1.430)] [data_t 0.002] [optim_t 1.427] [lr 0.005000] +2024-04-09 06:24:15,009 - Train: 18.05% [734700/4070000] [180.5/1000.0] [batch_t 1.428 (1.431)] [data_t 0.003] [optim_t 1.425] [lr 0.005000] +2024-04-09 06:26:37,996 - Train: 18.05% [734800/4070000] [180.5/1000.0] [batch_t 1.438 (1.430)] [data_t 0.003] [optim_t 1.435] [lr 0.005000] +2024-04-09 06:29:01,019 - Train: 18.06% [734900/4070000] [180.6/1000.0] [batch_t 1.415 (1.430)] [data_t 0.002] [optim_t 1.413] [lr 0.005000] +2024-04-09 06:31:23,907 - Train: 18.06% [735000/4070000] [180.6/1000.0] [batch_t 1.438 (1.429)] [data_t 0.004] [optim_t 1.434] [lr 0.005000] +2024-04-09 06:33:47,124 - Train: 18.06% [735100/4070000] [180.6/1000.0] [batch_t 1.425 (1.432)] [data_t 0.002] [optim_t 1.423] [lr 0.005000] +2024-04-09 06:36:10,034 - Train: 18.06% [735200/4070000] [180.6/1000.0] [batch_t 1.432 (1.429)] [data_t 0.002] [optim_t 1.430] [lr 0.005000] +2024-04-09 06:38:33,025 - Train: 18.07% [735300/4070000] [180.7/1000.0] [batch_t 1.435 (1.430)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-09 06:40:56,303 - Train: 18.07% [735400/4070000] [180.7/1000.0] [batch_t 1.445 (1.433)] [data_t 0.003] [optim_t 1.442] [lr 0.005000] +2024-04-09 06:43:19,110 - Train: 18.07% [735500/4070000] [180.7/1000.0] [batch_t 1.440 (1.428)] [data_t 0.003] [optim_t 1.438] [lr 0.005000] +2024-04-09 06:45:42,080 - Train: 18.07% [735600/4070000] [180.7/1000.0] [batch_t 1.427 (1.430)] [data_t 0.002] [optim_t 1.424] [lr 0.005000] +2024-04-09 06:48:04,976 - Train: 18.08% [735700/4070000] [180.8/1000.0] [batch_t 1.435 (1.429)] [data_t 0.003] [optim_t 1.432] [lr 0.005000] +2024-04-09 06:50:27,993 - Train: 18.08% [735800/4070000] [180.8/1000.0] [batch_t 1.436 (1.430)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-09 06:52:51,128 - Train: 18.08% [735900/4070000] [180.8/1000.0] [batch_t 1.425 (1.431)] [data_t 0.003] [optim_t 1.422] [lr 0.005000] +2024-04-09 06:55:14,221 - Train: 18.08% [736000/4070000] [180.8/1000.0] [batch_t 1.423 (1.431)] [data_t 0.002] [optim_t 1.420] [lr 0.005000] +2024-04-09 06:57:37,114 - Train: 18.09% [736100/4070000] [180.9/1000.0] [batch_t 1.427 (1.429)] [data_t 0.002] [optim_t 1.425] [lr 0.005000] +2024-04-09 07:00:00,146 - Train: 18.09% [736200/4070000] [180.9/1000.0] [batch_t 1.439 (1.430)] [data_t 0.002] [optim_t 1.437] [lr 0.005000] +2024-04-09 07:02:23,269 - Train: 18.09% [736300/4070000] [180.9/1000.0] [batch_t 1.431 (1.431)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-09 07:04:46,352 - Train: 18.09% [736400/4070000] [180.9/1000.0] [batch_t 1.422 (1.431)] [data_t 0.002] [optim_t 1.419] [lr 0.005000] +2024-04-09 07:07:09,423 - Train: 18.10% [736500/4070000] [181.0/1000.0] [batch_t 1.425 (1.431)] [data_t 0.003] [optim_t 1.422] [lr 0.005000] +2024-04-09 07:09:32,513 - Train: 18.10% [736600/4070000] [181.0/1000.0] [batch_t 1.445 (1.431)] [data_t 0.002] [optim_t 1.443] [lr 0.005000] +2024-04-09 07:11:12,642 - ==> Total time: 7 days, 6:10:43 Eta: 32 days, 20:08:05 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-09 07:11:57,171 - Train: 18.10% [736700/4070000] [181.0/1000.0] [batch_t 1.433 (1.437)] [data_t 0.003] [optim_t 1.430] [lr 0.005000] +2024-04-09 07:14:20,350 - Train: 18.10% [736800/4070000] [181.0/1000.0] [batch_t 1.432 (1.432)] [data_t 0.002] [optim_t 1.430] [lr 0.005000] +2024-04-09 07:16:43,428 - Train: 18.11% [736900/4070000] [181.1/1000.0] [batch_t 1.438 (1.431)] [data_t 0.002] [optim_t 1.436] [lr 0.005000] +2024-04-09 07:19:06,492 - Train: 18.11% [737000/4070000] [181.1/1000.0] [batch_t 1.425 (1.431)] [data_t 0.003] [optim_t 1.422] [lr 0.005000] +2024-04-09 07:21:29,567 - Train: 18.11% [737100/4070000] [181.1/1000.0] [batch_t 1.401 (1.431)] [data_t 0.003] [optim_t 1.398] [lr 0.005000] +2024-04-09 07:23:52,487 - Train: 18.11% [737200/4070000] [181.1/1000.0] [batch_t 1.439 (1.429)] [data_t 0.002] [optim_t 1.436] [lr 0.005000] +2024-04-09 07:26:15,505 - Train: 18.12% [737300/4070000] [181.2/1000.0] [batch_t 1.447 (1.430)] [data_t 0.002] [optim_t 1.444] [lr 0.005000] +2024-04-09 07:28:38,557 - Train: 18.12% [737400/4070000] [181.2/1000.0] [batch_t 1.441 (1.430)] [data_t 0.002] [optim_t 1.438] [lr 0.005000] +2024-04-09 07:31:01,657 - Train: 18.12% [737500/4070000] [181.2/1000.0] [batch_t 1.431 (1.431)] [data_t 0.002] [optim_t 1.428] [lr 0.005000] +2024-04-09 07:33:24,653 - Train: 18.12% [737600/4070000] [181.2/1000.0] [batch_t 1.433 (1.430)] [data_t 0.003] [optim_t 1.431] [lr 0.005000] +2024-04-09 07:35:47,721 - Train: 18.13% [737700/4070000] [181.3/1000.0] [batch_t 1.427 (1.431)] [data_t 0.003] [optim_t 1.424] [lr 0.005000] +2024-04-09 07:38:10,750 - Train: 18.13% [737800/4070000] [181.3/1000.0] [batch_t 1.429 (1.430)] [data_t 0.003] [optim_t 1.426] [lr 0.005000] +2024-04-09 07:40:33,911 - Train: 18.13% [737900/4070000] [181.3/1000.0] [batch_t 1.434 (1.432)] [data_t 0.002] [optim_t 1.432] [lr 0.005000] +2024-04-09 07:42:56,971 - Train: 18.13% [738000/4070000] [181.3/1000.0] [batch_t 1.430 (1.431)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-09 07:45:20,070 - Train: 18.14% [738100/4070000] [181.4/1000.0] [batch_t 1.429 (1.431)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-09 07:47:43,309 - Train: 18.14% [738200/4070000] [181.4/1000.0] [batch_t 1.419 (1.432)] [data_t 0.003] [optim_t 1.417] [lr 0.005000] +2024-04-09 07:50:06,318 - Train: 18.14% [738300/4070000] [181.4/1000.0] [batch_t 1.433 (1.430)] [data_t 0.003] [optim_t 1.430] [lr 0.005000] +2024-04-09 07:52:29,413 - Train: 18.14% [738400/4070000] [181.4/1000.0] [batch_t 1.441 (1.431)] [data_t 0.003] [optim_t 1.438] [lr 0.005000] +2024-04-09 07:54:52,308 - Train: 18.14% [738500/4070000] [181.4/1000.0] [batch_t 1.426 (1.429)] [data_t 0.003] [optim_t 1.424] [lr 0.005000] +2024-04-09 07:57:15,263 - Train: 18.15% [738600/4070000] [181.5/1000.0] [batch_t 1.418 (1.429)] [data_t 0.002] [optim_t 1.416] [lr 0.005000] +2024-04-09 07:59:38,384 - Train: 18.15% [738700/4070000] [181.5/1000.0] [batch_t 1.437 (1.431)] [data_t 0.003] [optim_t 1.434] [lr 0.005000] +2024-04-09 08:02:01,358 - Train: 18.15% [738800/4070000] [181.5/1000.0] [batch_t 1.440 (1.430)] [data_t 0.002] [optim_t 1.438] [lr 0.005000] +2024-04-09 08:04:24,306 - Train: 18.15% [738900/4070000] [181.5/1000.0] [batch_t 1.431 (1.429)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-09 08:06:47,353 - Train: 18.16% [739000/4070000] [181.6/1000.0] [batch_t 1.439 (1.430)] [data_t 0.002] [optim_t 1.437] [lr 0.005000] +2024-04-09 08:09:10,552 - Train: 18.16% [739100/4070000] [181.6/1000.0] [batch_t 1.445 (1.432)] [data_t 0.002] [optim_t 1.443] [lr 0.005000] +2024-04-09 08:11:33,661 - Train: 18.16% [739200/4070000] [181.6/1000.0] [batch_t 1.433 (1.431)] [data_t 0.003] [optim_t 1.431] [lr 0.005000] +2024-04-09 08:13:56,617 - Train: 18.16% [739300/4070000] [181.6/1000.0] [batch_t 1.445 (1.429)] [data_t 0.003] [optim_t 1.442] [lr 0.005000] +2024-04-09 08:15:17,621 - Train: 18.17% [739400/4070000] [181.7/1000.0] [batch_t 0.605 (0.810)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 08:16:18,059 - Train: 18.17% [739500/4070000] [181.7/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 08:17:18,525 - Train: 18.17% [739600/4070000] [181.7/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 08:18:19,086 - Train: 18.17% [739700/4070000] [181.7/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 08:19:19,554 - Train: 18.18% [739800/4070000] [181.8/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 08:20:20,000 - Train: 18.18% [739900/4070000] [181.8/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 08:21:20,443 - Train: 18.18% [740000/4070000] [181.8/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 08:22:20,904 - Train: 18.18% [740100/4070000] [181.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 08:23:21,334 - Train: 18.19% [740200/4070000] [181.9/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 08:24:21,796 - Train: 18.19% [740300/4070000] [181.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 08:25:22,228 - Train: 18.19% [740400/4070000] [181.9/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 08:26:22,740 - Train: 18.19% [740500/4070000] [181.9/1000.0] [batch_t 0.600 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-09 08:27:23,199 - Train: 18.20% [740600/4070000] [182.0/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 08:28:23,671 - Train: 18.20% [740700/4070000] [182.0/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 08:28:47,862 - ==> Total time: 7 days, 7:28:18 Eta: 32 days, 20:39:33 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-09 08:29:25,827 - Train: 18.20% [740800/4070000] [182.0/1000.0] [batch_t 0.605 (0.609)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 08:30:26,267 - Train: 18.20% [740900/4070000] [182.0/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 08:31:26,853 - Train: 18.21% [741000/4070000] [182.1/1000.0] [batch_t 0.602 (0.606)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-09 08:32:27,279 - Train: 18.21% [741100/4070000] [182.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 08:33:27,700 - Train: 18.21% [741200/4070000] [182.1/1000.0] [batch_t 0.597 (0.604)] [data_t 0.002] [optim_t 0.595] [lr 0.005000] +2024-04-09 08:34:28,201 - Train: 18.21% [741300/4070000] [182.1/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 08:35:28,730 - Train: 18.22% [741400/4070000] [182.2/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 08:36:29,282 - Train: 18.22% [741500/4070000] [182.2/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-09 08:37:29,843 - Train: 18.22% [741600/4070000] [182.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 08:38:30,465 - Train: 18.22% [741700/4070000] [182.2/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 08:39:30,910 - Train: 18.23% [741800/4070000] [182.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 08:40:31,392 - Train: 18.23% [741900/4070000] [182.3/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 08:41:31,886 - Train: 18.23% [742000/4070000] [182.3/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-09 08:42:32,419 - Train: 18.23% [742100/4070000] [182.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 08:43:32,854 - Train: 18.24% [742200/4070000] [182.4/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 08:44:33,423 - Train: 18.24% [742300/4070000] [182.4/1000.0] [batch_t 0.602 (0.606)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-09 08:45:33,900 - Train: 18.24% [742400/4070000] [182.4/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 08:46:34,362 - Train: 18.24% [742500/4070000] [182.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 08:47:34,813 - Train: 18.25% [742600/4070000] [182.5/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 08:48:35,273 - Train: 18.25% [742700/4070000] [182.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 08:49:35,742 - Train: 18.25% [742800/4070000] [182.5/1000.0] [batch_t 0.600 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-09 08:50:36,198 - Train: 18.25% [742900/4070000] [182.5/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 08:51:36,701 - Train: 18.26% [743000/4070000] [182.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 08:52:37,130 - Train: 18.26% [743100/4070000] [182.6/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 08:53:37,603 - Train: 18.26% [743200/4070000] [182.6/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 08:54:38,082 - Train: 18.26% [743300/4070000] [182.6/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 08:55:38,544 - Train: 18.27% [743400/4070000] [182.7/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-09 08:56:39,117 - Train: 18.27% [743500/4070000] [182.7/1000.0] [batch_t 0.613 (0.606)] [data_t 0.002] [optim_t 0.611] [lr 0.005000] +2024-04-09 08:57:39,589 - Train: 18.27% [743600/4070000] [182.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 08:58:40,104 - Train: 18.27% [743700/4070000] [182.7/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 08:59:40,566 - Train: 18.28% [743800/4070000] [182.8/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 09:00:41,041 - Train: 18.28% [743900/4070000] [182.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 09:01:41,497 - Train: 18.28% [744000/4070000] [182.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 09:02:41,953 - Train: 18.28% [744100/4070000] [182.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 09:03:42,406 - Train: 18.29% [744200/4070000] [182.9/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 09:04:42,923 - Train: 18.29% [744300/4070000] [182.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 09:05:43,398 - Train: 18.29% [744400/4070000] [182.9/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 09:06:43,879 - Train: 18.29% [744500/4070000] [182.9/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-09 09:07:44,311 - Train: 18.29% [744600/4070000] [182.9/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 09:08:44,821 - Train: 18.30% [744700/4070000] [183.0/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-09 09:09:45,387 - Train: 18.30% [744800/4070000] [183.0/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 09:09:51,435 - ==> Total time: 7 days, 8:09:22 Eta: 32 days, 18:26:45 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-09 09:10:47,375 - Train: 18.30% [744900/4070000] [183.0/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 09:11:47,851 - Train: 18.30% [745000/4070000] [183.0/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-09 09:12:48,338 - Train: 18.31% [745100/4070000] [183.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 09:13:48,815 - Train: 18.31% [745200/4070000] [183.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 09:14:49,389 - Train: 18.31% [745300/4070000] [183.1/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 09:15:49,837 - Train: 18.31% [745400/4070000] [183.1/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 09:16:50,331 - Train: 18.32% [745500/4070000] [183.2/1000.0] [batch_t 0.600 (0.605)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-09 09:17:50,775 - Train: 18.32% [745600/4070000] [183.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 09:18:51,218 - Train: 18.32% [745700/4070000] [183.2/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 09:19:51,646 - Train: 18.32% [745800/4070000] [183.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 09:20:52,097 - Train: 18.33% [745900/4070000] [183.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 09:21:52,559 - Train: 18.33% [746000/4070000] [183.3/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-09 09:22:53,152 - Train: 18.33% [746100/4070000] [183.3/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 09:23:53,618 - Train: 18.33% [746200/4070000] [183.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 09:24:54,083 - Train: 18.34% [746300/4070000] [183.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 09:25:54,530 - Train: 18.34% [746400/4070000] [183.4/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 09:26:54,974 - Train: 18.34% [746500/4070000] [183.4/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 09:27:55,575 - Train: 18.34% [746600/4070000] [183.4/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 09:28:56,082 - Train: 18.35% [746700/4070000] [183.5/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 09:29:56,559 - Train: 18.35% [746800/4070000] [183.5/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 09:30:57,055 - Train: 18.35% [746900/4070000] [183.5/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 09:31:57,488 - Train: 18.35% [747000/4070000] [183.5/1000.0] [batch_t 0.600 (0.604)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-09 09:32:57,951 - Train: 18.36% [747100/4070000] [183.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 09:33:58,408 - Train: 18.36% [747200/4070000] [183.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 09:34:58,890 - Train: 18.36% [747300/4070000] [183.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 09:35:59,405 - Train: 18.36% [747400/4070000] [183.6/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 09:36:59,878 - Train: 18.37% [747500/4070000] [183.7/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 09:38:00,311 - Train: 18.37% [747600/4070000] [183.7/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 09:39:00,771 - Train: 18.37% [747700/4070000] [183.7/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 09:40:01,228 - Train: 18.37% [747800/4070000] [183.7/1000.0] [batch_t 0.601 (0.604)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-09 09:41:01,780 - Train: 18.38% [747900/4070000] [183.8/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-09 09:42:02,227 - Train: 18.38% [748000/4070000] [183.8/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 09:43:02,673 - Train: 18.38% [748100/4070000] [183.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 09:44:03,128 - Train: 18.38% [748200/4070000] [183.8/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 09:45:03,600 - Train: 18.39% [748300/4070000] [183.9/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-09 09:46:04,040 - Train: 18.39% [748400/4070000] [183.9/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 09:47:04,492 - Train: 18.39% [748500/4070000] [183.9/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 09:48:04,938 - Train: 18.39% [748600/4070000] [183.9/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 09:49:05,474 - Train: 18.40% [748700/4070000] [184.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 09:50:05,922 - Train: 18.40% [748800/4070000] [184.0/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-09 09:50:54,282 - ==> Total time: 7 days, 8:50:25 Eta: 32 days, 16:14:54 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-09 09:51:07,813 - Train: 18.40% [748900/4070000] [184.0/1000.0] [batch_t 0.607 (0.613)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 09:52:08,224 - Train: 18.40% [749000/4070000] [184.0/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 09:53:08,679 - Train: 18.41% [749100/4070000] [184.1/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 09:54:09,337 - Train: 18.41% [749200/4070000] [184.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 09:55:09,826 - Train: 18.41% [749300/4070000] [184.1/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-09 09:56:10,293 - Train: 18.41% [749400/4070000] [184.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 09:57:10,790 - Train: 18.42% [749500/4070000] [184.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 09:58:11,234 - Train: 18.42% [749600/4070000] [184.2/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 09:59:11,674 - Train: 18.42% [749700/4070000] [184.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 10:00:12,123 - Train: 18.42% [749800/4070000] [184.2/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-09 10:01:12,768 - Train: 18.43% [749900/4070000] [184.3/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 10:02:13,225 - Train: 18.43% [750000/4070000] [184.3/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 10:03:13,693 - Train: 18.43% [750100/4070000] [184.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 10:04:14,166 - Train: 18.43% [750200/4070000] [184.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 10:05:14,619 - Train: 18.43% [750300/4070000] [184.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 10:06:15,213 - Train: 18.44% [750400/4070000] [184.4/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 10:07:15,649 - Train: 18.44% [750500/4070000] [184.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 10:08:16,111 - Train: 18.44% [750600/4070000] [184.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 10:09:16,544 - Train: 18.44% [750700/4070000] [184.4/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 10:10:16,979 - Train: 18.45% [750800/4070000] [184.5/1000.0] [batch_t 0.600 (0.604)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-09 10:11:17,433 - Train: 18.45% [750900/4070000] [184.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 10:12:17,835 - Train: 18.45% [751000/4070000] [184.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 10:13:18,251 - Train: 18.45% [751100/4070000] [184.5/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-09 10:14:18,888 - Train: 18.46% [751200/4070000] [184.6/1000.0] [batch_t 0.601 (0.606)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-09 10:15:19,302 - Train: 18.46% [751300/4070000] [184.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 10:16:19,771 - Train: 18.46% [751400/4070000] [184.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 10:17:20,331 - Train: 18.46% [751500/4070000] [184.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 10:18:20,807 - Train: 18.47% [751600/4070000] [184.7/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-09 10:19:21,454 - Train: 18.47% [751700/4070000] [184.7/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 10:20:21,887 - Train: 18.47% [751800/4070000] [184.7/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 10:21:22,343 - Train: 18.47% [751900/4070000] [184.7/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 10:22:22,825 - Train: 18.48% [752000/4070000] [184.8/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 10:23:23,283 - Train: 18.48% [752100/4070000] [184.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 10:24:23,777 - Train: 18.48% [752200/4070000] [184.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 10:25:24,211 - Train: 18.48% [752300/4070000] [184.8/1000.0] [batch_t 0.600 (0.604)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-09 10:26:24,857 - Train: 18.49% [752400/4070000] [184.9/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 10:27:25,328 - Train: 18.49% [752500/4070000] [184.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 10:28:25,785 - Train: 18.49% [752600/4070000] [184.9/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-09 10:29:26,241 - Train: 18.49% [752700/4070000] [184.9/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 10:30:26,687 - Train: 18.50% [752800/4070000] [185.0/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 10:31:27,169 - Train: 18.50% [752900/4070000] [185.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 10:31:57,383 - ==> Total time: 7 days, 9:31:28 Eta: 32 days, 14:04:02 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-09 10:32:29,907 - Train: 18.50% [753000/4070000] [185.0/1000.0] [batch_t 0.605 (0.620)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 10:33:30,543 - Train: 18.50% [753100/4070000] [185.0/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 10:34:30,982 - Train: 18.51% [753200/4070000] [185.1/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 10:35:31,500 - Train: 18.51% [753300/4070000] [185.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 10:36:31,975 - Train: 18.51% [753400/4070000] [185.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 10:37:32,438 - Train: 18.51% [753500/4070000] [185.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 10:38:32,896 - Train: 18.52% [753600/4070000] [185.2/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-09 10:39:33,480 - Train: 18.52% [753700/4070000] [185.2/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 10:40:34,001 - Train: 18.52% [753800/4070000] [185.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 10:41:34,522 - Train: 18.52% [753900/4070000] [185.2/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 10:42:34,967 - Train: 18.53% [754000/4070000] [185.3/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 10:43:35,492 - Train: 18.53% [754100/4070000] [185.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 10:44:36,022 - Train: 18.53% [754200/4070000] [185.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 10:45:36,501 - Train: 18.53% [754300/4070000] [185.3/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 10:46:37,045 - Train: 18.54% [754400/4070000] [185.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 10:47:37,540 - Train: 18.54% [754500/4070000] [185.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 10:48:38,001 - Train: 18.54% [754600/4070000] [185.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 10:49:38,509 - Train: 18.54% [754700/4070000] [185.4/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-09 10:50:38,965 - Train: 18.55% [754800/4070000] [185.5/1000.0] [batch_t 0.608 (0.604)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-09 10:51:39,424 - Train: 18.55% [754900/4070000] [185.5/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 10:52:39,950 - Train: 18.55% [755000/4070000] [185.5/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 10:53:40,395 - Train: 18.55% [755100/4070000] [185.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 10:54:40,926 - Train: 18.56% [755200/4070000] [185.6/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 10:55:41,352 - Train: 18.56% [755300/4070000] [185.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 10:56:41,833 - Train: 18.56% [755400/4070000] [185.6/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-09 10:57:42,306 - Train: 18.56% [755500/4070000] [185.6/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 10:58:42,861 - Train: 18.57% [755600/4070000] [185.7/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 10:59:43,328 - Train: 18.57% [755700/4070000] [185.7/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 11:00:43,824 - Train: 18.57% [755800/4070000] [185.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 11:01:44,303 - Train: 18.57% [755900/4070000] [185.7/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 11:02:44,781 - Train: 18.57% [756000/4070000] [185.7/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-09 11:03:45,291 - Train: 18.58% [756100/4070000] [185.8/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 11:04:45,832 - Train: 18.58% [756200/4070000] [185.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 11:05:46,390 - Train: 18.58% [756300/4070000] [185.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 11:06:46,826 - Train: 18.58% [756400/4070000] [185.8/1000.0] [batch_t 0.601 (0.604)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-09 11:07:47,307 - Train: 18.59% [756500/4070000] [185.9/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 11:08:47,777 - Train: 18.59% [756600/4070000] [185.9/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 11:09:48,273 - Train: 18.59% [756700/4070000] [185.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 11:10:48,725 - Train: 18.59% [756800/4070000] [185.9/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-09 11:11:49,367 - Train: 18.60% [756900/4070000] [186.0/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 11:12:49,836 - Train: 18.60% [757000/4070000] [186.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 11:13:01,931 - ==> Total time: 7 days, 10:12:32 Eta: 32 days, 11:54:15 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-09 11:13:54,216 - Train: 18.60% [757100/4070000] [186.0/1000.0] [batch_t 0.603 (0.611)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 11:14:54,704 - Train: 18.60% [757200/4070000] [186.0/1000.0] [batch_t 0.597 (0.605)] [data_t 0.002] [optim_t 0.595] [lr 0.005000] +2024-04-09 11:15:55,145 - Train: 18.61% [757300/4070000] [186.1/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 11:16:55,592 - Train: 18.61% [757400/4070000] [186.1/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 11:17:56,038 - Train: 18.61% [757500/4070000] [186.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 11:18:56,634 - Train: 18.61% [757600/4070000] [186.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 11:19:57,122 - Train: 18.62% [757700/4070000] [186.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 11:20:57,586 - Train: 18.62% [757800/4070000] [186.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 11:21:58,057 - Train: 18.62% [757900/4070000] [186.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 11:22:58,514 - Train: 18.62% [758000/4070000] [186.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 11:23:59,063 - Train: 18.63% [758100/4070000] [186.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 11:24:59,519 - Train: 18.63% [758200/4070000] [186.3/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 11:25:59,970 - Train: 18.63% [758300/4070000] [186.3/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 11:27:00,421 - Train: 18.63% [758400/4070000] [186.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 11:28:00,838 - Train: 18.64% [758500/4070000] [186.4/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 11:29:01,284 - Train: 18.64% [758600/4070000] [186.4/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 11:30:01,833 - Train: 18.64% [758700/4070000] [186.4/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 11:31:02,283 - Train: 18.64% [758800/4070000] [186.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 11:32:02,770 - Train: 18.65% [758900/4070000] [186.5/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-09 11:33:03,212 - Train: 18.65% [759000/4070000] [186.5/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 11:34:03,689 - Train: 18.65% [759100/4070000] [186.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 11:35:04,186 - Train: 18.65% [759200/4070000] [186.5/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-09 11:36:04,634 - Train: 18.66% [759300/4070000] [186.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 11:37:05,167 - Train: 18.66% [759400/4070000] [186.6/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-09 11:38:05,625 - Train: 18.66% [759500/4070000] [186.6/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 11:39:06,100 - Train: 18.66% [759600/4070000] [186.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 11:40:06,549 - Train: 18.67% [759700/4070000] [186.7/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 11:41:07,025 - Train: 18.67% [759800/4070000] [186.7/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 11:42:07,481 - Train: 18.67% [759900/4070000] [186.7/1000.0] [batch_t 0.597 (0.604)] [data_t 0.002] [optim_t 0.595] [lr 0.005000] +2024-04-09 11:43:08,103 - Train: 18.67% [760000/4070000] [186.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 11:44:08,543 - Train: 18.68% [760100/4070000] [186.8/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 11:45:08,994 - Train: 18.68% [760200/4070000] [186.8/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 11:46:13,480 - Train: 18.68% [760300/4070000] [186.8/1000.0] [batch_t 0.605 (0.645)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 11:47:14,852 - Train: 18.68% [760400/4070000] [186.8/1000.0] [batch_t 0.606 (0.614)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 11:48:19,144 - Train: 18.69% [760500/4070000] [186.9/1000.0] [batch_t 0.600 (0.643)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-09 11:49:19,618 - Train: 18.69% [760600/4070000] [186.9/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 11:50:20,491 - Train: 18.69% [760700/4070000] [186.9/1000.0] [batch_t 0.605 (0.609)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 11:51:25,092 - Train: 18.69% [760800/4070000] [186.9/1000.0] [batch_t 0.604 (0.646)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 11:52:25,823 - Train: 18.70% [760900/4070000] [187.0/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 11:53:27,822 - Train: 18.70% [761000/4070000] [187.0/1000.0] [batch_t 1.240 (0.620)] [data_t 0.638] [optim_t 0.602] [lr 0.005000] +2024-04-09 11:54:26,715 - ==> Total time: 7 days, 10:53:57 Eta: 32 days, 9:46:53 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-09 11:54:35,196 - Train: 18.70% [761100/4070000] [187.0/1000.0] [batch_t 0.596 (0.675)] [data_t 0.002] [optim_t 0.594] [lr 0.005000] +2024-04-09 11:55:42,589 - Train: 18.70% [761200/4070000] [187.0/1000.0] [batch_t 0.605 (0.674)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 11:56:43,031 - Train: 18.71% [761300/4070000] [187.1/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 11:57:49,326 - Train: 18.71% [761400/4070000] [187.1/1000.0] [batch_t 0.601 (0.663)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-09 11:58:49,772 - Train: 18.71% [761500/4070000] [187.1/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 11:59:50,420 - Train: 18.71% [761600/4070000] [187.1/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 12:00:56,552 - Train: 18.71% [761700/4070000] [187.1/1000.0] [batch_t 0.604 (0.661)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 12:01:56,992 - Train: 18.72% [761800/4070000] [187.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 12:02:57,428 - Train: 18.72% [761900/4070000] [187.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 12:03:58,032 - Train: 18.72% [762000/4070000] [187.2/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 12:04:58,504 - Train: 18.72% [762100/4070000] [187.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 12:05:58,970 - Train: 18.73% [762200/4070000] [187.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 12:06:59,468 - Train: 18.73% [762300/4070000] [187.3/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 12:07:59,975 - Train: 18.73% [762400/4070000] [187.3/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-09 12:09:00,576 - Train: 18.73% [762500/4070000] [187.3/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 12:10:01,054 - Train: 18.74% [762600/4070000] [187.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 12:11:01,600 - Train: 18.74% [762700/4070000] [187.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 12:12:02,113 - Train: 18.74% [762800/4070000] [187.4/1000.0] [batch_t 0.599 (0.605)] [data_t 0.002] [optim_t 0.597] [lr 0.005000] +2024-04-09 12:13:02,677 - Train: 18.74% [762900/4070000] [187.4/1000.0] [batch_t 0.602 (0.606)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-09 12:14:03,167 - Train: 18.75% [763000/4070000] [187.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 12:15:03,654 - Train: 18.75% [763100/4070000] [187.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 12:16:04,234 - Train: 18.75% [763200/4070000] [187.5/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 12:17:04,787 - Train: 18.75% [763300/4070000] [187.5/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-09 12:18:05,292 - Train: 18.76% [763400/4070000] [187.6/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 12:19:05,851 - Train: 18.76% [763500/4070000] [187.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 12:20:06,377 - Train: 18.76% [763600/4070000] [187.6/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 12:21:06,912 - Train: 18.76% [763700/4070000] [187.6/1000.0] [batch_t 0.610 (0.605)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-09 12:22:07,542 - Train: 18.77% [763800/4070000] [187.7/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 12:23:08,081 - Train: 18.77% [763900/4070000] [187.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 12:24:08,550 - Train: 18.77% [764000/4070000] [187.7/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 12:25:09,021 - Train: 18.77% [764100/4070000] [187.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 12:26:09,519 - Train: 18.78% [764200/4070000] [187.8/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 12:27:09,996 - Train: 18.78% [764300/4070000] [187.8/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 12:28:10,484 - Train: 18.78% [764400/4070000] [187.8/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 12:29:11,021 - Train: 18.78% [764500/4070000] [187.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 12:30:11,504 - Train: 18.79% [764600/4070000] [187.9/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-09 12:31:11,986 - Train: 18.79% [764700/4070000] [187.9/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 12:32:12,474 - Train: 18.79% [764800/4070000] [187.9/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-09 12:33:12,973 - Train: 18.79% [764900/4070000] [187.9/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 12:34:13,496 - Train: 18.80% [765000/4070000] [188.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 12:35:14,032 - Train: 18.80% [765100/4070000] [188.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 12:35:50,414 - ==> Total time: 7 days, 11:35:21 Eta: 32 days, 7:40:21 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-09 12:36:16,304 - Train: 18.80% [765200/4070000] [188.0/1000.0] [batch_t 0.603 (0.608)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 12:37:16,753 - Train: 18.80% [765300/4070000] [188.0/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 12:38:17,203 - Train: 18.81% [765400/4070000] [188.1/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 12:39:17,670 - Train: 18.81% [765500/4070000] [188.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 12:40:18,159 - Train: 18.81% [765600/4070000] [188.1/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-09 12:41:18,599 - Train: 18.81% [765700/4070000] [188.1/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 12:42:19,030 - Train: 18.82% [765800/4070000] [188.2/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-09 12:43:19,619 - Train: 18.82% [765900/4070000] [188.2/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 12:44:20,122 - Train: 18.82% [766000/4070000] [188.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 12:45:20,579 - Train: 18.82% [766100/4070000] [188.2/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 12:46:21,033 - Train: 18.83% [766200/4070000] [188.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 12:47:21,468 - Train: 18.83% [766300/4070000] [188.3/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 12:48:22,001 - Train: 18.83% [766400/4070000] [188.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 12:49:22,447 - Train: 18.83% [766500/4070000] [188.3/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 12:50:22,916 - Train: 18.84% [766600/4070000] [188.4/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 12:51:23,417 - Train: 18.84% [766700/4070000] [188.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 12:52:23,856 - Train: 18.84% [766800/4070000] [188.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 12:53:24,333 - Train: 18.84% [766900/4070000] [188.4/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 12:54:24,795 - Train: 18.85% [767000/4070000] [188.5/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-09 12:55:25,258 - Train: 18.85% [767100/4070000] [188.5/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 12:56:25,912 - Train: 18.85% [767200/4070000] [188.5/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 12:57:26,387 - Train: 18.85% [767300/4070000] [188.5/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 12:58:26,837 - Train: 18.86% [767400/4070000] [188.6/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 12:59:27,259 - Train: 18.86% [767500/4070000] [188.6/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 13:00:27,701 - Train: 18.86% [767600/4070000] [188.6/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 13:01:28,285 - Train: 18.86% [767700/4070000] [188.6/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 13:02:28,755 - Train: 18.86% [767800/4070000] [188.6/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 13:03:29,229 - Train: 18.87% [767900/4070000] [188.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 13:04:29,675 - Train: 18.87% [768000/4070000] [188.7/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 13:05:30,114 - Train: 18.87% [768100/4070000] [188.7/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 13:06:30,576 - Train: 18.87% [768200/4070000] [188.7/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 13:07:31,069 - Train: 18.88% [768300/4070000] [188.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 13:08:31,553 - Train: 18.88% [768400/4070000] [188.8/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 13:09:32,173 - Train: 18.88% [768500/4070000] [188.8/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 13:10:32,693 - Train: 18.88% [768600/4070000] [188.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 13:11:33,168 - Train: 18.89% [768700/4070000] [188.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 13:12:33,712 - Train: 18.89% [768800/4070000] [188.9/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-09 13:13:34,171 - Train: 18.89% [768900/4070000] [188.9/1000.0] [batch_t 0.599 (0.604)] [data_t 0.002] [optim_t 0.597] [lr 0.005000] +2024-04-09 13:14:34,825 - Train: 18.89% [769000/4070000] [188.9/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 13:15:35,298 - Train: 18.90% [769100/4070000] [189.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 13:16:35,831 - Train: 18.90% [769200/4070000] [189.0/1000.0] [batch_t 0.596 (0.605)] [data_t 0.002] [optim_t 0.594] [lr 0.005000] +2024-04-09 13:16:54,005 - ==> Total time: 7 days, 12:16:24 Eta: 32 days, 5:33:17 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-09 13:17:37,943 - Train: 18.90% [769300/4070000] [189.0/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 13:18:38,423 - Train: 18.90% [769400/4070000] [189.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 13:19:38,869 - Train: 18.91% [769500/4070000] [189.1/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 13:20:39,512 - Train: 18.91% [769600/4070000] [189.1/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 13:21:39,965 - Train: 18.91% [769700/4070000] [189.1/1000.0] [batch_t 0.609 (0.604)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-09 13:22:40,433 - Train: 18.91% [769800/4070000] [189.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 13:23:40,936 - Train: 18.92% [769900/4070000] [189.2/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-09 13:24:41,399 - Train: 18.92% [770000/4070000] [189.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 13:25:41,873 - Train: 18.92% [770100/4070000] [189.2/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-09 13:26:42,455 - Train: 18.92% [770200/4070000] [189.2/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 13:27:42,902 - Train: 18.93% [770300/4070000] [189.3/1000.0] [batch_t 0.608 (0.604)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-09 13:28:43,372 - Train: 18.93% [770400/4070000] [189.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 13:29:43,823 - Train: 18.93% [770500/4070000] [189.3/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 13:30:44,302 - Train: 18.93% [770600/4070000] [189.3/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-09 13:31:44,749 - Train: 18.94% [770700/4070000] [189.4/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 13:32:45,229 - Train: 18.94% [770800/4070000] [189.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 13:33:45,672 - Train: 18.94% [770900/4070000] [189.4/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-09 13:34:46,241 - Train: 18.94% [771000/4070000] [189.4/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 13:35:46,666 - Train: 18.95% [771100/4070000] [189.5/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-09 13:36:47,124 - Train: 18.95% [771200/4070000] [189.5/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 13:37:47,599 - Train: 18.95% [771300/4070000] [189.5/1000.0] [batch_t 0.599 (0.605)] [data_t 0.002] [optim_t 0.597] [lr 0.005000] +2024-04-09 13:38:48,095 - Train: 18.95% [771400/4070000] [189.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 13:39:48,750 - Train: 18.96% [771500/4070000] [189.6/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 13:40:49,208 - Train: 18.96% [771600/4070000] [189.6/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-09 13:41:49,684 - Train: 18.96% [771700/4070000] [189.6/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 13:42:50,176 - Train: 18.96% [771800/4070000] [189.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 13:43:50,637 - Train: 18.97% [771900/4070000] [189.7/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 13:44:51,108 - Train: 18.97% [772000/4070000] [189.7/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 13:45:51,606 - Train: 18.97% [772100/4070000] [189.7/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 13:46:52,107 - Train: 18.97% [772200/4070000] [189.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 13:47:52,761 - Train: 18.98% [772300/4070000] [189.8/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 13:48:53,230 - Train: 18.98% [772400/4070000] [189.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 13:49:53,705 - Train: 18.98% [772500/4070000] [189.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 13:50:54,140 - Train: 18.98% [772600/4070000] [189.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 13:51:54,604 - Train: 18.99% [772700/4070000] [189.9/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-09 13:52:55,215 - Train: 18.99% [772800/4070000] [189.9/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 13:53:55,673 - Train: 18.99% [772900/4070000] [189.9/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 13:54:56,134 - Train: 18.99% [773000/4070000] [189.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 13:55:56,559 - Train: 19.00% [773100/4070000] [190.0/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 13:56:57,020 - Train: 19.00% [773200/4070000] [190.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 13:57:57,463 - Train: 19.00% [773300/4070000] [190.0/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 13:57:57,466 - ==> Total time: 7 days, 12:57:28 Eta: 32 days, 3:27:07 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-09 13:58:59,749 - Train: 19.00% [773400/4070000] [190.0/1000.0] [batch_t 0.604 (0.608)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 14:00:00,339 - Train: 19.00% [773500/4070000] [190.0/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 14:01:00,849 - Train: 19.01% [773600/4070000] [190.1/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-09 14:02:01,308 - Train: 19.01% [773700/4070000] [190.1/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 14:03:01,742 - Train: 19.01% [773800/4070000] [190.1/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 14:04:02,199 - Train: 19.01% [773900/4070000] [190.1/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 14:05:02,640 - Train: 19.02% [774000/4070000] [190.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 14:06:03,102 - Train: 19.02% [774100/4070000] [190.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 14:07:03,584 - Train: 19.02% [774200/4070000] [190.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 14:08:04,035 - Train: 19.02% [774300/4070000] [190.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 14:09:04,501 - Train: 19.03% [774400/4070000] [190.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 14:10:04,970 - Train: 19.03% [774500/4070000] [190.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 14:11:05,470 - Train: 19.03% [774600/4070000] [190.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 14:12:05,939 - Train: 19.03% [774700/4070000] [190.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 14:13:06,506 - Train: 19.04% [774800/4070000] [190.4/1000.0] [batch_t 0.602 (0.606)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-09 14:14:06,957 - Train: 19.04% [774900/4070000] [190.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 14:15:07,414 - Train: 19.04% [775000/4070000] [190.4/1000.0] [batch_t 0.608 (0.604)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-09 14:16:07,905 - Train: 19.04% [775100/4070000] [190.4/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-09 14:17:08,343 - Train: 19.05% [775200/4070000] [190.5/1000.0] [batch_t 0.601 (0.604)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-09 14:18:08,831 - Train: 19.05% [775300/4070000] [190.5/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 14:19:09,312 - Train: 19.05% [775400/4070000] [190.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 14:20:09,891 - Train: 19.05% [775500/4070000] [190.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 14:21:10,401 - Train: 19.06% [775600/4070000] [190.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 14:22:10,881 - Train: 19.06% [775700/4070000] [190.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 14:23:11,361 - Train: 19.06% [775800/4070000] [190.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 14:24:11,790 - Train: 19.06% [775900/4070000] [190.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 14:25:12,375 - Train: 19.07% [776000/4070000] [190.7/1000.0] [batch_t 0.609 (0.606)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-09 14:26:12,857 - Train: 19.07% [776100/4070000] [190.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 14:27:13,314 - Train: 19.07% [776200/4070000] [190.7/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 14:28:13,812 - Train: 19.07% [776300/4070000] [190.7/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 14:29:14,262 - Train: 19.08% [776400/4070000] [190.8/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 14:30:14,775 - Train: 19.08% [776500/4070000] [190.8/1000.0] [batch_t 0.600 (0.605)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-09 14:31:15,239 - Train: 19.08% [776600/4070000] [190.8/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 14:32:15,716 - Train: 19.08% [776700/4070000] [190.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 14:33:16,328 - Train: 19.09% [776800/4070000] [190.9/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 14:34:16,801 - Train: 19.09% [776900/4070000] [190.9/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 14:35:17,288 - Train: 19.09% [777000/4070000] [190.9/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-09 14:36:17,733 - Train: 19.09% [777100/4070000] [190.9/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 14:37:18,192 - Train: 19.10% [777200/4070000] [191.0/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 14:38:18,790 - Train: 19.10% [777300/4070000] [191.0/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-09 14:39:01,098 - ==> Total time: 7 days, 13:38:31 Eta: 32 days, 1:21:50 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-09 14:39:22,308 - Train: 19.10% [777400/4070000] [191.0/1000.0] [batch_t 0.601 (0.611)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-09 14:40:22,760 - Train: 19.10% [777500/4070000] [191.0/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 14:41:23,221 - Train: 19.11% [777600/4070000] [191.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 14:42:23,689 - Train: 19.11% [777700/4070000] [191.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 14:43:24,106 - Train: 19.11% [777800/4070000] [191.1/1000.0] [batch_t 0.609 (0.604)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-09 14:44:24,735 - Train: 19.11% [777900/4070000] [191.1/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 14:45:25,171 - Train: 19.12% [778000/4070000] [191.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 14:46:25,649 - Train: 19.12% [778100/4070000] [191.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 14:47:26,117 - Train: 19.12% [778200/4070000] [191.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 14:48:26,591 - Train: 19.12% [778300/4070000] [191.2/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 14:49:27,083 - Train: 19.13% [778400/4070000] [191.3/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 14:50:27,511 - Train: 19.13% [778500/4070000] [191.3/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-09 14:51:28,067 - Train: 19.13% [778600/4070000] [191.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 14:52:28,494 - Train: 19.13% [778700/4070000] [191.3/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 14:53:28,942 - Train: 19.14% [778800/4070000] [191.4/1000.0] [batch_t 0.598 (0.604)] [data_t 0.002] [optim_t 0.596] [lr 0.005000] +2024-04-09 14:54:29,407 - Train: 19.14% [778900/4070000] [191.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 14:55:29,888 - Train: 19.14% [779000/4070000] [191.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 14:56:30,466 - Train: 19.14% [779100/4070000] [191.4/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 14:57:30,908 - Train: 19.14% [779200/4070000] [191.4/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 14:58:31,378 - Train: 19.15% [779300/4070000] [191.5/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-09 14:59:31,834 - Train: 19.15% [779400/4070000] [191.5/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 15:00:32,294 - Train: 19.15% [779500/4070000] [191.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 15:01:32,749 - Train: 19.15% [779600/4070000] [191.5/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 15:02:33,200 - Train: 19.16% [779700/4070000] [191.6/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 15:03:33,672 - Train: 19.16% [779800/4070000] [191.6/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 15:04:34,193 - Train: 19.16% [779900/4070000] [191.6/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 15:05:34,706 - Train: 19.16% [780000/4070000] [191.6/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 15:06:35,232 - Train: 19.17% [780100/4070000] [191.7/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 15:07:35,766 - Train: 19.17% [780200/4070000] [191.7/1000.0] [batch_t 0.610 (0.605)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-09 15:08:36,327 - Train: 19.17% [780300/4070000] [191.7/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 15:09:37,015 - Train: 19.17% [780400/4070000] [191.7/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 15:10:37,483 - Train: 19.18% [780500/4070000] [191.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 15:11:37,987 - Train: 19.18% [780600/4070000] [191.8/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-09 15:12:38,484 - Train: 19.18% [780700/4070000] [191.8/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 15:13:38,981 - Train: 19.18% [780800/4070000] [191.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 15:14:39,508 - Train: 19.19% [780900/4070000] [191.9/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 15:15:40,002 - Train: 19.19% [781000/4070000] [191.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 15:16:40,477 - Train: 19.19% [781100/4070000] [191.9/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 15:17:41,168 - Train: 19.19% [781200/4070000] [191.9/1000.0] [batch_t 0.603 (0.607)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 15:18:41,680 - Train: 19.20% [781300/4070000] [192.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 15:19:42,236 - Train: 19.20% [781400/4070000] [192.0/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-09 15:20:06,451 - ==> Total time: 7 days, 14:19:37 Eta: 31 days, 23:17:34 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-09 15:20:44,500 - Train: 19.20% [781500/4070000] [192.0/1000.0] [batch_t 0.608 (0.607)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-09 15:21:45,015 - Train: 19.20% [781600/4070000] [192.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 15:22:45,616 - Train: 19.21% [781700/4070000] [192.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 15:23:46,093 - Train: 19.21% [781800/4070000] [192.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 15:24:46,588 - Train: 19.21% [781900/4070000] [192.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 15:25:47,032 - Train: 19.21% [782000/4070000] [192.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 15:26:47,481 - Train: 19.22% [782100/4070000] [192.2/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 15:27:47,940 - Train: 19.22% [782200/4070000] [192.2/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 15:28:49,620 - Train: 19.22% [782300/4070000] [192.2/1000.0] [batch_t 0.607 (0.617)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 15:29:50,990 - Train: 19.22% [782400/4070000] [192.2/1000.0] [batch_t 0.605 (0.614)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 15:30:51,459 - Train: 19.23% [782500/4070000] [192.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 15:31:52,765 - Train: 19.23% [782600/4070000] [192.3/1000.0] [batch_t 0.601 (0.613)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-09 15:32:53,266 - Train: 19.23% [782700/4070000] [192.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 15:33:53,749 - Train: 19.23% [782800/4070000] [192.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 15:34:54,260 - Train: 19.24% [782900/4070000] [192.4/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 15:35:54,811 - Train: 19.24% [783000/4070000] [192.4/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-09 15:36:55,972 - Train: 19.24% [783100/4070000] [192.4/1000.0] [batch_t 0.605 (0.612)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 15:38:09,406 - Train: 19.24% [783200/4070000] [192.4/1000.0] [batch_t 0.604 (0.734)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 15:39:09,856 - Train: 19.25% [783300/4070000] [192.5/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 15:40:10,345 - Train: 19.25% [783400/4070000] [192.5/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-09 15:41:10,829 - Train: 19.25% [783500/4070000] [192.5/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 15:42:11,314 - Train: 19.25% [783600/4070000] [192.5/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-09 15:43:11,858 - Train: 19.26% [783700/4070000] [192.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 15:44:12,304 - Train: 19.26% [783800/4070000] [192.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 15:45:12,804 - Train: 19.26% [783900/4070000] [192.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 15:46:13,318 - Train: 19.26% [784000/4070000] [192.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 15:47:13,825 - Train: 19.27% [784100/4070000] [192.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 15:48:14,395 - Train: 19.27% [784200/4070000] [192.7/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 15:49:14,847 - Train: 19.27% [784300/4070000] [192.7/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 15:50:15,302 - Train: 19.27% [784400/4070000] [192.7/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 15:51:15,780 - Train: 19.28% [784500/4070000] [192.8/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-09 15:52:16,261 - Train: 19.28% [784600/4070000] [192.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 15:53:16,757 - Train: 19.28% [784700/4070000] [192.8/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 15:54:17,235 - Train: 19.28% [784800/4070000] [192.8/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-09 15:55:17,790 - Train: 19.29% [784900/4070000] [192.9/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 15:56:18,217 - Train: 19.29% [785000/4070000] [192.9/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 15:57:18,668 - Train: 19.29% [785100/4070000] [192.9/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-09 15:58:19,086 - Train: 19.29% [785200/4070000] [192.9/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 15:59:19,610 - Train: 19.29% [785300/4070000] [192.9/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-09 16:00:20,160 - Train: 19.30% [785400/4070000] [193.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 16:01:20,846 - Train: 19.30% [785500/4070000] [193.0/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 16:01:26,900 - ==> Total time: 7 days, 15:00:57 Eta: 31 days, 21:15:13 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-09 16:02:23,429 - Train: 19.30% [785600/4070000] [193.0/1000.0] [batch_t 0.608 (0.611)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-09 16:03:23,902 - Train: 19.30% [785700/4070000] [193.0/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 16:04:24,357 - Train: 19.31% [785800/4070000] [193.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 16:05:24,846 - Train: 19.31% [785900/4070000] [193.1/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-09 16:06:25,368 - Train: 19.31% [786000/4070000] [193.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 16:07:32,360 - Train: 19.31% [786100/4070000] [193.1/1000.0] [batch_t 0.606 (0.670)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 16:08:32,962 - Train: 19.32% [786200/4070000] [193.2/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 16:09:33,474 - Train: 19.32% [786300/4070000] [193.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 16:10:34,001 - Train: 19.32% [786400/4070000] [193.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 16:11:34,484 - Train: 19.32% [786500/4070000] [193.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 16:12:34,987 - Train: 19.33% [786600/4070000] [193.3/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-09 16:13:35,582 - Train: 19.33% [786700/4070000] [193.3/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 16:14:36,063 - Train: 19.33% [786800/4070000] [193.3/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 16:15:36,595 - Train: 19.33% [786900/4070000] [193.3/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 16:16:37,090 - Train: 19.34% [787000/4070000] [193.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 16:17:37,575 - Train: 19.34% [787100/4070000] [193.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 16:18:38,054 - Train: 19.34% [787200/4070000] [193.4/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 16:19:38,499 - Train: 19.34% [787300/4070000] [193.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 16:20:39,004 - Train: 19.35% [787400/4070000] [193.5/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-09 16:21:39,707 - Train: 19.35% [787500/4070000] [193.5/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 16:22:40,185 - Train: 19.35% [787600/4070000] [193.5/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 16:23:40,668 - Train: 19.35% [787700/4070000] [193.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 16:24:41,174 - Train: 19.36% [787800/4070000] [193.6/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 16:25:41,640 - Train: 19.36% [787900/4070000] [193.6/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 16:26:42,222 - Train: 19.36% [788000/4070000] [193.6/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 16:27:42,657 - Train: 19.36% [788100/4070000] [193.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 16:28:43,193 - Train: 19.37% [788200/4070000] [193.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 16:29:43,649 - Train: 19.37% [788300/4070000] [193.7/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 16:30:44,109 - Train: 19.37% [788400/4070000] [193.7/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 16:31:44,538 - Train: 19.37% [788500/4070000] [193.7/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 16:32:44,971 - Train: 19.38% [788600/4070000] [193.8/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 16:33:50,699 - Train: 19.38% [788700/4070000] [193.8/1000.0] [batch_t 1.227 (0.657)] [data_t 0.624] [optim_t 0.603] [lr 0.005000] +2024-04-09 16:36:05,106 - Train: 19.38% [788800/4070000] [193.8/1000.0] [batch_t 7.846 (1.344)] [data_t 7.242] [optim_t 0.604] [lr 0.005000] +2024-04-09 16:39:05,148 - Train: 19.38% [788900/4070000] [193.8/1000.0] [batch_t 2.575 (1.800)] [data_t 1.970] [optim_t 0.604] [lr 0.005000] +2024-04-09 16:40:12,835 - Train: 19.39% [789000/4070000] [193.9/1000.0] [batch_t 0.602 (0.677)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-09 16:41:13,720 - Train: 19.39% [789100/4070000] [193.9/1000.0] [batch_t 0.610 (0.609)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-09 16:42:14,679 - Train: 19.39% [789200/4070000] [193.9/1000.0] [batch_t 0.607 (0.609)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 16:43:15,640 - Train: 19.39% [789300/4070000] [193.9/1000.0] [batch_t 0.609 (0.610)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-09 16:44:16,146 - Train: 19.40% [789400/4070000] [194.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 16:45:16,966 - Train: 19.40% [789500/4070000] [194.0/1000.0] [batch_t 0.605 (0.608)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 16:46:05,338 - ==> Total time: 7 days, 15:45:36 Eta: 31 days, 19:27:24 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-09 16:46:22,667 - Train: 19.40% [789600/4070000] [194.0/1000.0] [batch_t 0.605 (0.722)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 16:47:32,802 - Train: 19.40% [789700/4070000] [194.0/1000.0] [batch_t 0.604 (0.701)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 16:48:33,288 - Train: 19.41% [789800/4070000] [194.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 16:49:33,740 - Train: 19.41% [789900/4070000] [194.1/1000.0] [batch_t 0.600 (0.604)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-09 16:50:34,380 - Train: 19.41% [790000/4070000] [194.1/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 16:51:34,864 - Train: 19.41% [790100/4070000] [194.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 16:52:35,303 - Train: 19.42% [790200/4070000] [194.2/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 16:53:35,744 - Train: 19.42% [790300/4070000] [194.2/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-09 16:54:36,259 - Train: 19.42% [790400/4070000] [194.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.003] [optim_t 0.603] [lr 0.005000] +2024-04-09 16:55:36,879 - Train: 19.42% [790500/4070000] [194.2/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 16:56:37,447 - Train: 19.43% [790600/4070000] [194.3/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 16:57:37,932 - Train: 19.43% [790700/4070000] [194.3/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-09 16:58:38,396 - Train: 19.43% [790800/4070000] [194.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 16:59:38,838 - Train: 19.43% [790900/4070000] [194.3/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 17:00:41,172 - Train: 19.43% [791000/4070000] [194.3/1000.0] [batch_t 0.602 (0.623)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-09 17:01:41,659 - Train: 19.44% [791100/4070000] [194.4/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 17:02:42,159 - Train: 19.44% [791200/4070000] [194.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 17:03:42,856 - Train: 19.44% [791300/4070000] [194.4/1000.0] [batch_t 0.600 (0.607)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-09 17:04:43,327 - Train: 19.44% [791400/4070000] [194.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 17:05:43,765 - Train: 19.45% [791500/4070000] [194.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 17:06:44,299 - Train: 19.45% [791600/4070000] [194.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 17:07:44,830 - Train: 19.45% [791700/4070000] [194.5/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 17:08:45,405 - Train: 19.45% [791800/4070000] [194.5/1000.0] [batch_t 0.602 (0.606)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-09 17:09:45,901 - Train: 19.46% [791900/4070000] [194.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 17:10:53,902 - Train: 19.46% [792000/4070000] [194.6/1000.0] [batch_t 0.607 (0.680)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 17:11:54,376 - Train: 19.46% [792100/4070000] [194.6/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 17:13:05,559 - Train: 19.46% [792200/4070000] [194.6/1000.0] [batch_t 0.606 (0.712)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 17:14:06,050 - Train: 19.47% [792300/4070000] [194.7/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 17:15:09,835 - Train: 19.47% [792400/4070000] [194.7/1000.0] [batch_t 0.604 (0.638)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 17:16:10,319 - Train: 19.47% [792500/4070000] [194.7/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 17:17:10,895 - Train: 19.47% [792600/4070000] [194.7/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-09 17:18:11,454 - Train: 19.48% [792700/4070000] [194.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 17:19:12,096 - Train: 19.48% [792800/4070000] [194.8/1000.0] [batch_t 0.609 (0.606)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-09 17:20:12,557 - Train: 19.48% [792900/4070000] [194.8/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 17:21:16,704 - Train: 19.48% [793000/4070000] [194.8/1000.0] [batch_t 0.607 (0.641)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 17:22:37,018 - Train: 19.49% [793100/4070000] [194.9/1000.0] [batch_t 0.606 (0.803)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 17:23:37,526 - Train: 19.49% [793200/4070000] [194.9/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 17:25:03,619 - Train: 19.49% [793300/4070000] [194.9/1000.0] [batch_t 0.604 (0.861)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 17:26:22,141 - Train: 19.49% [793400/4070000] [194.9/1000.0] [batch_t 0.605 (0.785)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 17:27:36,359 - Train: 19.50% [793500/4070000] [195.0/1000.0] [batch_t 0.604 (0.742)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 17:28:39,641 - Train: 19.50% [793600/4070000] [195.0/1000.0] [batch_t 0.605 (0.633)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 17:29:09,881 - ==> Total time: 7 days, 16:28:40 Eta: 31 days, 17:33:46 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-09 17:29:46,409 - Train: 19.50% [793700/4070000] [195.0/1000.0] [batch_t 0.604 (0.677)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 17:30:48,139 - Train: 19.50% [793800/4070000] [195.0/1000.0] [batch_t 0.603 (0.617)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 17:31:48,582 - Train: 19.51% [793900/4070000] [195.1/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 17:32:49,014 - Train: 19.51% [794000/4070000] [195.1/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 17:33:49,487 - Train: 19.51% [794100/4070000] [195.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 17:34:59,743 - Train: 19.51% [794200/4070000] [195.1/1000.0] [batch_t 0.605 (0.702)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 17:36:00,390 - Train: 19.52% [794300/4070000] [195.2/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 17:37:00,898 - Train: 19.52% [794400/4070000] [195.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 17:38:01,419 - Train: 19.52% [794500/4070000] [195.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 17:39:01,863 - Train: 19.52% [794600/4070000] [195.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 17:40:02,353 - Train: 19.53% [794700/4070000] [195.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 17:41:02,877 - Train: 19.53% [794800/4070000] [195.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 17:42:09,194 - Train: 19.53% [794900/4070000] [195.3/1000.0] [batch_t 0.605 (0.663)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 17:43:10,364 - Train: 19.53% [795000/4070000] [195.3/1000.0] [batch_t 0.605 (0.612)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 17:44:16,430 - Train: 19.54% [795100/4070000] [195.4/1000.0] [batch_t 0.605 (0.661)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 17:45:18,025 - Train: 19.54% [795200/4070000] [195.4/1000.0] [batch_t 0.603 (0.616)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 17:46:18,867 - Train: 19.54% [795300/4070000] [195.4/1000.0] [batch_t 0.603 (0.608)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 17:47:22,973 - Train: 19.54% [795400/4070000] [195.4/1000.0] [batch_t 0.607 (0.641)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 17:48:27,719 - Train: 19.55% [795500/4070000] [195.5/1000.0] [batch_t 0.605 (0.647)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 17:49:28,352 - Train: 19.55% [795600/4070000] [195.5/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 17:50:28,821 - Train: 19.55% [795700/4070000] [195.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 17:51:29,966 - Train: 19.55% [795800/4070000] [195.5/1000.0] [batch_t 0.604 (0.611)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 17:52:30,413 - Train: 19.56% [795900/4070000] [195.6/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 17:53:30,904 - Train: 19.56% [796000/4070000] [195.6/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 17:54:42,657 - Train: 19.56% [796100/4070000] [195.6/1000.0] [batch_t 0.602 (0.717)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-09 17:55:43,146 - Train: 19.56% [796200/4070000] [195.6/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 17:56:43,789 - Train: 19.57% [796300/4070000] [195.7/1000.0] [batch_t 0.602 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 17:57:44,288 - Train: 19.57% [796400/4070000] [195.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 17:58:44,844 - Train: 19.57% [796500/4070000] [195.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 17:59:45,330 - Train: 19.57% [796600/4070000] [195.7/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-09 18:00:45,855 - Train: 19.57% [796700/4070000] [195.7/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 18:01:46,624 - Train: 19.58% [796800/4070000] [195.8/1000.0] [batch_t 0.604 (0.608)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 18:02:47,079 - Train: 19.58% [796900/4070000] [195.8/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 18:03:47,627 - Train: 19.58% [797000/4070000] [195.8/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 18:04:48,110 - Train: 19.58% [797100/4070000] [195.8/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 18:05:48,620 - Train: 19.59% [797200/4070000] [195.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 18:06:49,105 - Train: 19.59% [797300/4070000] [195.9/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 18:07:49,552 - Train: 19.59% [797400/4070000] [195.9/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 18:08:50,225 - Train: 19.59% [797500/4070000] [195.9/1000.0] [batch_t 0.602 (0.607)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-09 18:09:50,744 - Train: 19.60% [797600/4070000] [196.0/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 18:10:51,252 - Train: 19.60% [797700/4070000] [196.0/1000.0] [batch_t 0.610 (0.605)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-09 18:11:03,361 - ==> Total time: 7 days, 17:10:34 Eta: 31 days, 15:36:00 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-09 18:11:55,648 - Train: 19.60% [797800/4070000] [196.0/1000.0] [batch_t 0.605 (0.621)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 18:12:56,146 - Train: 19.60% [797900/4070000] [196.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 18:13:56,635 - Train: 19.61% [798000/4070000] [196.1/1000.0] [batch_t 0.614 (0.605)] [data_t 0.002] [optim_t 0.612] [lr 0.005000] +2024-04-09 18:14:57,161 - Train: 19.61% [798100/4070000] [196.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 18:15:57,789 - Train: 19.61% [798200/4070000] [196.1/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 18:16:58,253 - Train: 19.61% [798300/4070000] [196.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 18:17:58,729 - Train: 19.62% [798400/4070000] [196.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 18:18:59,154 - Train: 19.62% [798500/4070000] [196.2/1000.0] [batch_t 0.601 (0.604)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-09 18:19:59,628 - Train: 19.62% [798600/4070000] [196.2/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-09 18:21:00,089 - Train: 19.62% [798700/4070000] [196.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 18:22:00,653 - Train: 19.63% [798800/4070000] [196.3/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 18:23:01,151 - Train: 19.63% [798900/4070000] [196.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 18:24:01,562 - Train: 19.63% [799000/4070000] [196.3/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 18:25:02,009 - Train: 19.63% [799100/4070000] [196.3/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 18:26:02,469 - Train: 19.64% [799200/4070000] [196.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 18:27:02,947 - Train: 19.64% [799300/4070000] [196.4/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-09 18:28:03,426 - Train: 19.64% [799400/4070000] [196.4/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-09 18:29:03,938 - Train: 19.64% [799500/4070000] [196.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 18:30:04,388 - Train: 19.65% [799600/4070000] [196.5/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 18:31:04,841 - Train: 19.65% [799700/4070000] [196.5/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 18:32:05,368 - Train: 19.65% [799800/4070000] [196.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 18:33:05,831 - Train: 19.65% [799900/4070000] [196.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 18:34:06,407 - Train: 19.66% [800000/4070000] [196.6/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 18:35:06,844 - Train: 19.66% [800100/4070000] [196.6/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 18:36:07,299 - Train: 19.66% [800200/4070000] [196.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 18:37:07,755 - Train: 19.66% [800300/4070000] [196.6/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 18:38:08,212 - Train: 19.67% [800400/4070000] [196.7/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 18:39:08,679 - Train: 19.67% [800500/4070000] [196.7/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 18:40:09,246 - Train: 19.67% [800600/4070000] [196.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 18:41:09,737 - Train: 19.67% [800700/4070000] [196.7/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 18:42:10,180 - Train: 19.68% [800800/4070000] [196.8/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 18:43:10,646 - Train: 19.68% [800900/4070000] [196.8/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 18:44:11,104 - Train: 19.68% [801000/4070000] [196.8/1000.0] [batch_t 0.608 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 18:45:11,598 - Train: 19.68% [801100/4070000] [196.8/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 18:46:12,061 - Train: 19.69% [801200/4070000] [196.9/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 18:47:12,678 - Train: 19.69% [801300/4070000] [196.9/1000.0] [batch_t 0.602 (0.606)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-09 18:48:13,235 - Train: 19.69% [801400/4070000] [196.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 18:49:13,746 - Train: 19.69% [801500/4070000] [196.9/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 18:50:14,211 - Train: 19.70% [801600/4070000] [197.0/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 18:51:14,674 - Train: 19.70% [801700/4070000] [197.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 18:52:09,145 - ==> Total time: 7 days, 17:51:40 Eta: 31 days, 13:35:46 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-09 18:52:16,888 - Train: 19.70% [801800/4070000] [197.0/1000.0] [batch_t 0.608 (0.624)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-09 18:53:17,551 - Train: 19.70% [801900/4070000] [197.0/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 18:54:18,023 - Train: 19.71% [802000/4070000] [197.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 18:55:18,468 - Train: 19.71% [802100/4070000] [197.1/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 18:56:18,951 - Train: 19.71% [802200/4070000] [197.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 18:57:19,456 - Train: 19.71% [802300/4070000] [197.1/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 18:58:19,910 - Train: 19.71% [802400/4070000] [197.1/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 18:59:20,418 - Train: 19.72% [802500/4070000] [197.2/1000.0] [batch_t 0.610 (0.605)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-09 19:00:20,985 - Train: 19.72% [802600/4070000] [197.2/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 19:01:21,451 - Train: 19.72% [802700/4070000] [197.2/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 19:02:21,938 - Train: 19.72% [802800/4070000] [197.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 19:03:22,483 - Train: 19.73% [802900/4070000] [197.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 19:04:22,982 - Train: 19.73% [803000/4070000] [197.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 19:05:23,490 - Train: 19.73% [803100/4070000] [197.3/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 19:06:24,100 - Train: 19.73% [803200/4070000] [197.3/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 19:07:33,831 - Train: 19.74% [803300/4070000] [197.4/1000.0] [batch_t 0.606 (0.697)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 19:08:34,270 - Train: 19.74% [803400/4070000] [197.4/1000.0] [batch_t 0.601 (0.604)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-09 19:09:34,738 - Train: 19.74% [803500/4070000] [197.4/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 19:10:35,252 - Train: 19.74% [803600/4070000] [197.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 19:11:35,762 - Train: 19.75% [803700/4070000] [197.5/1000.0] [batch_t 0.610 (0.605)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-09 19:12:36,279 - Train: 19.75% [803800/4070000] [197.5/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 19:13:36,829 - Train: 19.75% [803900/4070000] [197.5/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 19:14:37,308 - Train: 19.75% [804000/4070000] [197.5/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 19:15:37,818 - Train: 19.76% [804100/4070000] [197.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 19:16:38,371 - Train: 19.76% [804200/4070000] [197.6/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 19:17:38,880 - Train: 19.76% [804300/4070000] [197.6/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-09 19:18:39,352 - Train: 19.76% [804400/4070000] [197.6/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 19:19:39,980 - Train: 19.77% [804500/4070000] [197.7/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-09 19:20:40,464 - Train: 19.77% [804600/4070000] [197.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 19:21:40,954 - Train: 19.77% [804700/4070000] [197.7/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 19:22:41,433 - Train: 19.77% [804800/4070000] [197.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 19:23:41,931 - Train: 19.78% [804900/4070000] [197.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 19:24:42,515 - Train: 19.78% [805000/4070000] [197.8/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-09 19:25:42,993 - Train: 19.78% [805100/4070000] [197.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 19:26:43,466 - Train: 19.78% [805200/4070000] [197.8/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-09 19:27:51,693 - Train: 19.79% [805300/4070000] [197.9/1000.0] [batch_t 0.603 (0.682)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 19:28:52,152 - Train: 19.79% [805400/4070000] [197.9/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 19:29:52,590 - Train: 19.79% [805500/4070000] [197.9/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 19:30:53,021 - Train: 19.79% [805600/4070000] [197.9/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 19:31:53,460 - Train: 19.80% [805700/4070000] [198.0/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 19:32:53,976 - Train: 19.80% [805800/4070000] [198.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 19:33:30,253 - ==> Total time: 7 days, 18:33:01 Eta: 31 days, 11:37:22 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-09 19:33:56,157 - Train: 19.80% [805900/4070000] [198.0/1000.0] [batch_t 0.605 (0.609)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 19:34:56,703 - Train: 19.80% [806000/4070000] [198.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 19:35:57,150 - Train: 19.81% [806100/4070000] [198.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 19:36:57,622 - Train: 19.81% [806200/4070000] [198.1/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-09 19:37:58,211 - Train: 19.81% [806300/4070000] [198.1/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 19:38:58,699 - Train: 19.81% [806400/4070000] [198.1/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 19:39:59,201 - Train: 19.82% [806500/4070000] [198.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 19:40:59,656 - Train: 19.82% [806600/4070000] [198.2/1000.0] [batch_t 0.608 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 19:42:00,099 - Train: 19.82% [806700/4070000] [198.2/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 19:43:00,660 - Train: 19.82% [806800/4070000] [198.2/1000.0] [batch_t 0.602 (0.606)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-09 19:44:01,122 - Train: 19.83% [806900/4070000] [198.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 19:45:01,744 - Train: 19.83% [807000/4070000] [198.3/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 19:46:02,244 - Train: 19.83% [807100/4070000] [198.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 19:47:02,731 - Train: 19.83% [807200/4070000] [198.3/1000.0] [batch_t 0.596 (0.605)] [data_t 0.002] [optim_t 0.594] [lr 0.005000] +2024-04-09 19:48:03,225 - Train: 19.84% [807300/4070000] [198.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-09 19:49:03,736 - Train: 19.84% [807400/4070000] [198.4/1000.0] [batch_t 0.607 (0.605)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-09 19:50:04,477 - Train: 19.84% [807500/4070000] [198.4/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 19:51:04,985 - Train: 19.84% [807600/4070000] [198.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 19:52:05,452 - Train: 19.85% [807700/4070000] [198.5/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-09 19:53:12,629 - Train: 19.85% [807800/4070000] [198.5/1000.0] [batch_t 0.606 (0.672)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 19:54:13,100 - Train: 19.85% [807900/4070000] [198.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 19:55:13,637 - Train: 19.85% [808000/4070000] [198.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 19:56:14,162 - Train: 19.86% [808100/4070000] [198.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 19:57:14,884 - Train: 19.86% [808200/4070000] [198.6/1000.0] [batch_t 0.611 (0.607)] [data_t 0.002] [optim_t 0.609] [lr 0.005000] +2024-04-09 19:58:15,383 - Train: 19.86% [808300/4070000] [198.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 19:59:15,864 - Train: 19.86% [808400/4070000] [198.6/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 20:00:16,319 - Train: 19.86% [808500/4070000] [198.6/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 20:01:16,837 - Train: 19.87% [808600/4070000] [198.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 20:02:17,321 - Train: 19.87% [808700/4070000] [198.7/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 20:03:17,823 - Train: 19.87% [808800/4070000] [198.7/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-09 20:04:18,359 - Train: 19.87% [808900/4070000] [198.7/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 20:05:18,996 - Train: 19.88% [809000/4070000] [198.8/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 20:06:19,451 - Train: 19.88% [809100/4070000] [198.8/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-09 20:07:19,977 - Train: 19.88% [809200/4070000] [198.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 20:08:20,520 - Train: 19.88% [809300/4070000] [198.8/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 20:09:21,031 - Train: 19.89% [809400/4070000] [198.9/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-09 20:10:21,782 - Train: 19.89% [809500/4070000] [198.9/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 20:11:22,257 - Train: 19.89% [809600/4070000] [198.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 20:12:22,701 - Train: 19.89% [809700/4070000] [198.9/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 20:13:23,171 - Train: 19.90% [809800/4070000] [199.0/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 20:14:23,672 - Train: 19.90% [809900/4070000] [199.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 20:14:41,847 - ==> Total time: 7 days, 19:14:12 Eta: 31 days, 9:39:06 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-09 20:15:25,762 - Train: 19.90% [810000/4070000] [199.0/1000.0] [batch_t 0.605 (0.608)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 20:16:26,241 - Train: 19.90% [810100/4070000] [199.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 20:17:26,862 - Train: 19.91% [810200/4070000] [199.1/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 20:18:27,344 - Train: 19.91% [810300/4070000] [199.1/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-09 20:19:27,847 - Train: 19.91% [810400/4070000] [199.1/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 20:20:28,445 - Train: 19.91% [810500/4070000] [199.1/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 20:21:28,943 - Train: 19.92% [810600/4070000] [199.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 20:22:29,697 - Train: 19.92% [810700/4070000] [199.2/1000.0] [batch_t 0.603 (0.607)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 20:23:30,201 - Train: 19.92% [810800/4070000] [199.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 20:24:30,841 - Train: 19.92% [810900/4070000] [199.2/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 20:25:31,392 - Train: 19.93% [811000/4070000] [199.3/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 20:26:31,948 - Train: 19.93% [811100/4070000] [199.3/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 20:27:32,533 - Train: 19.93% [811200/4070000] [199.3/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 20:28:33,107 - Train: 19.93% [811300/4070000] [199.3/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 20:29:38,669 - Train: 19.94% [811400/4070000] [199.4/1000.0] [batch_t 0.610 (0.656)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-09 20:31:29,522 - Train: 19.94% [811500/4070000] [199.4/1000.0] [batch_t 0.602 (1.108)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-09 20:32:33,810 - Train: 19.94% [811600/4070000] [199.4/1000.0] [batch_t 0.611 (0.643)] [data_t 0.002] [optim_t 0.609] [lr 0.005000] +2024-04-09 20:33:49,690 - Train: 19.94% [811700/4070000] [199.4/1000.0] [batch_t 0.606 (0.759)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 20:34:50,207 - Train: 19.95% [811800/4070000] [199.5/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 20:35:53,281 - Train: 19.95% [811900/4070000] [199.5/1000.0] [batch_t 0.603 (0.631)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 20:36:56,349 - Train: 19.95% [812000/4070000] [199.5/1000.0] [batch_t 0.605 (0.631)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 20:37:56,961 - Train: 19.95% [812100/4070000] [199.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 20:39:00,131 - Train: 19.96% [812200/4070000] [199.6/1000.0] [batch_t 0.604 (0.632)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 20:40:00,723 - Train: 19.96% [812300/4070000] [199.6/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 20:41:01,302 - Train: 19.96% [812400/4070000] [199.6/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 20:42:01,805 - Train: 19.96% [812500/4070000] [199.6/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 20:43:02,284 - Train: 19.97% [812600/4070000] [199.7/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 20:44:02,924 - Train: 19.97% [812700/4070000] [199.7/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 20:45:03,480 - Train: 19.97% [812800/4070000] [199.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 20:46:03,993 - Train: 19.97% [812900/4070000] [199.7/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-09 20:47:04,506 - Train: 19.98% [813000/4070000] [199.8/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 20:48:04,978 - Train: 19.98% [813100/4070000] [199.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 20:49:05,556 - Train: 19.98% [813200/4070000] [199.8/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 20:50:05,980 - Train: 19.98% [813300/4070000] [199.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 20:51:06,504 - Train: 19.99% [813400/4070000] [199.9/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 20:52:06,931 - Train: 19.99% [813500/4070000] [199.9/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 20:53:07,412 - Train: 19.99% [813600/4070000] [199.9/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-09 20:54:07,927 - Train: 19.99% [813700/4070000] [199.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 20:55:08,428 - Train: 20.00% [813800/4070000] [200.0/1000.0] [batch_t 0.610 (0.605)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-09 20:56:08,906 - Train: 20.00% [813900/4070000] [200.0/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 20:57:09,483 - Train: 20.00% [814000/4070000] [200.0/1000.0] [batch_t 0.602 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-09 20:57:22,230 - Test: 16.13% [50/310] [batch_t 0.231 (0.248)] +2024-04-09 20:57:33,180 - Test: 32.26% [100/310] [batch_t 0.231 (0.233)] +2024-04-09 20:57:44,441 - Test: 48.39% [150/310] [batch_t 0.229 (0.231)] +2024-04-09 20:57:55,597 - Test: 64.52% [200/310] [batch_t 0.233 (0.229)] +2024-04-09 20:58:07,745 - Test: 80.65% [250/310] [batch_t 0.233 (0.232)] +2024-04-09 20:58:18,869 - Test: 96.77% [300/310] [batch_t 0.240 (0.230)] +2024-04-09 20:58:21,934 - Test: 100.00% [310/310] [batch_t 1.031 (0.233)] +2024-04-09 21:28:30,887 - ==> Metric Time for coco : 0.006 (mAUROC_sp_max) 0.002 (mAP_sp_max) 0.001 (mF1_max_sp_max) 473.234 (mAUROC_px) 308.212 (mAP_px) 35.167 (mF1_max_px) 903.497 (mAUPRO_px) 12.312 (mF1_px_0.2_0.8_0.1) 12.673 (mAcc_px_0.2_0.8_0.1) 13.200 (mIoU_px_0.2_0.8_0.1) 37.535 (mIoU_max_px) +2024-04-09 21:28:31,531 - +| Name | mAUROC_sp_max | mAUROC_sp_max (Max) | mAP_sp_max | mAP_sp_max (Max) | mF1_max_sp_max | mF1_max_sp_max (Max) | mAUROC_px | mAUROC_px (Max) | mAP_px | mAP_px (Max) | mF1_max_px | mF1_max_px (Max) | mAUPRO_px | mAUPRO_px (Max) | mF1_px_0.2_0.8_0.1 | mF1_px_0.2_0.8_0.1 (Max) | mAcc_px_0.2_0.8_0.1 | mAcc_px_0.2_0.8_0.1 (Max) | mIoU_px_0.2_0.8_0.1 | mIoU_px_0.2_0.8_0.1 (Max) | mIoU_max_px | mIoU_max_px (Max) | +|:------:|:---------------:|:---------------------:|:------------:|:------------------:|:----------------:|:----------------------:|:-----------:|:------------------:|:--------:|:-----------------:|:------------:|:------------------:|:-----------:|:------------------:|:--------------------:|:--------------------------:|:---------------------:|:---------------------------:|:---------------------:|:---------------------------:|:-------------:|:-------------------:| +| coco | 55.293 | 55.293 (200 epoch) | 48.661 | 48.661 (200 epoch) | 60.970 | 60.970 (200 epoch) | 71.125 | 71.125 (200 epoch) | 6.541 | 6.541 (200 epoch) | 11.384 | 11.384 (200 epoch) | 36.589 | 36.589 (200 epoch) | 6.504 | 6.504 (200 epoch) | 43.125 | 43.125 (200 epoch) | 3.400 | 3.400 (200 epoch) | 6.036 | 6.036 (200 epoch) | +2024-04-09 21:28:32,005 - ==> Total time: 7 days, 20:28:02 Eta: 31 days, 9:52:11 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-09 21:29:37,414 - Train: 20.00% [814100/4070000] [200.0/1000.0] [batch_t 0.610 (0.627)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-09 21:30:37,998 - Train: 20.00% [814200/4070000] [200.0/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 21:31:38,572 - Train: 20.01% [814300/4070000] [200.1/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 21:32:39,217 - Train: 20.01% [814400/4070000] [200.1/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 21:33:39,991 - Train: 20.01% [814500/4070000] [200.1/1000.0] [batch_t 0.608 (0.608)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-09 21:34:40,673 - Train: 20.01% [814600/4070000] [200.1/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 21:35:41,313 - Train: 20.02% [814700/4070000] [200.2/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 21:36:41,907 - Train: 20.02% [814800/4070000] [200.2/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 21:37:42,481 - Train: 20.02% [814900/4070000] [200.2/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 21:38:43,039 - Train: 20.02% [815000/4070000] [200.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 21:39:43,597 - Train: 20.03% [815100/4070000] [200.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 21:40:44,248 - Train: 20.03% [815200/4070000] [200.3/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 21:41:44,801 - Train: 20.03% [815300/4070000] [200.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 21:42:45,356 - Train: 20.03% [815400/4070000] [200.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 21:43:45,913 - Train: 20.04% [815500/4070000] [200.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 21:44:46,474 - Train: 20.04% [815600/4070000] [200.4/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 21:45:47,052 - Train: 20.04% [815700/4070000] [200.4/1000.0] [batch_t 0.609 (0.606)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-09 21:46:47,738 - Train: 20.04% [815800/4070000] [200.4/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 21:47:48,324 - Train: 20.05% [815900/4070000] [200.5/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 21:48:48,891 - Train: 20.05% [816000/4070000] [200.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 21:49:49,449 - Train: 20.05% [816100/4070000] [200.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 21:50:50,032 - Train: 20.05% [816200/4070000] [200.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 21:51:50,745 - Train: 20.06% [816300/4070000] [200.6/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 21:52:51,309 - Train: 20.06% [816400/4070000] [200.6/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 21:53:51,906 - Train: 20.06% [816500/4070000] [200.6/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 21:54:52,501 - Train: 20.06% [816600/4070000] [200.6/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 21:55:55,681 - Train: 20.07% [816700/4070000] [200.7/1000.0] [batch_t 0.605 (0.632)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 21:56:56,282 - Train: 20.07% [816800/4070000] [200.7/1000.0] [batch_t 0.609 (0.606)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-09 21:57:56,936 - Train: 20.07% [816900/4070000] [200.7/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 21:58:57,526 - Train: 20.07% [817000/4070000] [200.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 21:59:58,173 - Train: 20.08% [817100/4070000] [200.8/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 22:00:58,750 - Train: 20.08% [817200/4070000] [200.8/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 22:01:59,340 - Train: 20.08% [817300/4070000] [200.8/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 22:02:59,945 - Train: 20.08% [817400/4070000] [200.8/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 22:04:00,542 - Train: 20.09% [817500/4070000] [200.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 22:05:01,248 - Train: 20.09% [817600/4070000] [200.9/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 22:06:01,843 - Train: 20.09% [817700/4070000] [200.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 22:07:02,440 - Train: 20.09% [817800/4070000] [200.9/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 22:08:03,003 - Train: 20.10% [817900/4070000] [201.0/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 22:09:03,580 - Train: 20.10% [818000/4070000] [201.0/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 22:09:46,019 - ==> Total time: 7 days, 21:09:16 Eta: 31 days, 7:54:48 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-09 22:10:06,478 - Train: 20.10% [818100/4070000] [201.0/1000.0] [batch_t 0.607 (0.609)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 22:11:07,074 - Train: 20.10% [818200/4070000] [201.0/1000.0] [batch_t 0.610 (0.606)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-09 22:12:07,892 - Train: 20.11% [818300/4070000] [201.1/1000.0] [batch_t 0.606 (0.608)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 22:13:08,480 - Train: 20.11% [818400/4070000] [201.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 22:14:09,081 - Train: 20.11% [818500/4070000] [201.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 22:15:09,686 - Train: 20.11% [818600/4070000] [201.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 22:16:10,291 - Train: 20.12% [818700/4070000] [201.2/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 22:17:10,893 - Train: 20.12% [818800/4070000] [201.2/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 22:18:11,581 - Train: 20.12% [818900/4070000] [201.2/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 22:19:12,164 - Train: 20.12% [819000/4070000] [201.2/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 22:20:12,783 - Train: 20.13% [819100/4070000] [201.3/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 22:21:13,398 - Train: 20.13% [819200/4070000] [201.3/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 22:22:13,998 - Train: 20.13% [819300/4070000] [201.3/1000.0] [batch_t 0.610 (0.606)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-09 22:23:14,554 - Train: 20.13% [819400/4070000] [201.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 22:24:15,123 - Train: 20.14% [819500/4070000] [201.4/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 22:25:15,762 - Train: 20.14% [819600/4070000] [201.4/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 22:26:16,361 - Train: 20.14% [819700/4070000] [201.4/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-09 22:27:16,935 - Train: 20.14% [819800/4070000] [201.4/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 22:28:17,561 - Train: 20.14% [819900/4070000] [201.4/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 22:29:18,109 - Train: 20.15% [820000/4070000] [201.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 22:30:18,836 - Train: 20.15% [820100/4070000] [201.5/1000.0] [batch_t 0.608 (0.607)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-09 22:31:19,440 - Train: 20.15% [820200/4070000] [201.5/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 22:32:20,006 - Train: 20.15% [820300/4070000] [201.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 22:33:20,674 - Train: 20.16% [820400/4070000] [201.6/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 22:34:21,245 - Train: 20.16% [820500/4070000] [201.6/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 22:35:21,829 - Train: 20.16% [820600/4070000] [201.6/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 22:36:22,418 - Train: 20.16% [820700/4070000] [201.6/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 22:37:23,292 - Train: 20.17% [820800/4070000] [201.7/1000.0] [batch_t 0.609 (0.609)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-09 22:38:23,969 - Train: 20.17% [820900/4070000] [201.7/1000.0] [batch_t 0.610 (0.607)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-09 22:39:24,621 - Train: 20.17% [821000/4070000] [201.7/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 22:40:25,201 - Train: 20.17% [821100/4070000] [201.7/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 22:41:25,789 - Train: 20.18% [821200/4070000] [201.8/1000.0] [batch_t 0.611 (0.606)] [data_t 0.002] [optim_t 0.609] [lr 0.005000] +2024-04-09 22:42:26,366 - Train: 20.18% [821300/4070000] [201.8/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 22:43:26,946 - Train: 20.18% [821400/4070000] [201.8/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 22:44:27,551 - Train: 20.18% [821500/4070000] [201.8/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 22:45:28,331 - Train: 20.19% [821600/4070000] [201.9/1000.0] [batch_t 0.607 (0.608)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 22:46:28,986 - Train: 20.19% [821700/4070000] [201.9/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 22:47:29,607 - Train: 20.19% [821800/4070000] [201.9/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 22:48:30,195 - Train: 20.19% [821900/4070000] [201.9/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 22:49:30,773 - Train: 20.20% [822000/4070000] [202.0/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 22:50:31,468 - Train: 20.20% [822100/4070000] [202.0/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 22:50:55,712 - ==> Total time: 7 days, 21:50:26 Eta: 31 days, 5:57:53 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-09 22:51:34,416 - Train: 20.20% [822200/4070000] [202.0/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 22:52:34,990 - Train: 20.20% [822300/4070000] [202.0/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 22:53:35,582 - Train: 20.21% [822400/4070000] [202.1/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 22:54:36,121 - Train: 20.21% [822500/4070000] [202.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 22:55:36,656 - Train: 20.21% [822600/4070000] [202.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 22:56:37,205 - Train: 20.21% [822700/4070000] [202.1/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 22:57:37,936 - Train: 20.22% [822800/4070000] [202.2/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 22:58:38,499 - Train: 20.22% [822900/4070000] [202.2/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 22:59:39,032 - Train: 20.22% [823000/4070000] [202.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 23:00:39,594 - Train: 20.22% [823100/4070000] [202.2/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 23:01:40,155 - Train: 20.23% [823200/4070000] [202.3/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 23:02:40,710 - Train: 20.23% [823300/4070000] [202.3/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 23:03:41,291 - Train: 20.23% [823400/4070000] [202.3/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 23:04:41,944 - Train: 20.23% [823500/4070000] [202.3/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 23:05:42,497 - Train: 20.24% [823600/4070000] [202.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 23:06:43,028 - Train: 20.24% [823700/4070000] [202.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 23:07:43,582 - Train: 20.24% [823800/4070000] [202.4/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-09 23:08:44,106 - Train: 20.24% [823900/4070000] [202.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 23:09:44,611 - Train: 20.25% [824000/4070000] [202.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 23:10:45,239 - Train: 20.25% [824100/4070000] [202.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 23:11:45,783 - Train: 20.25% [824200/4070000] [202.5/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-09 23:12:46,378 - Train: 20.25% [824300/4070000] [202.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 23:13:46,923 - Train: 20.26% [824400/4070000] [202.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 23:14:47,473 - Train: 20.26% [824500/4070000] [202.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 23:15:48,014 - Train: 20.26% [824600/4070000] [202.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 23:16:48,575 - Train: 20.26% [824700/4070000] [202.6/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 23:17:49,283 - Train: 20.27% [824800/4070000] [202.7/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 23:18:49,891 - Train: 20.27% [824900/4070000] [202.7/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 23:19:50,503 - Train: 20.27% [825000/4070000] [202.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 23:20:51,128 - Train: 20.27% [825100/4070000] [202.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 23:21:51,693 - Train: 20.28% [825200/4070000] [202.8/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 23:22:52,311 - Train: 20.28% [825300/4070000] [202.8/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-09 23:23:53,146 - Train: 20.28% [825400/4070000] [202.8/1000.0] [batch_t 0.609 (0.608)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-09 23:24:53,827 - Train: 20.28% [825500/4070000] [202.8/1000.0] [batch_t 0.609 (0.607)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-09 23:25:54,419 - Train: 20.29% [825600/4070000] [202.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 23:26:55,004 - Train: 20.29% [825700/4070000] [202.9/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 23:27:55,635 - Train: 20.29% [825800/4070000] [202.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 23:28:56,221 - Train: 20.29% [825900/4070000] [202.9/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 23:29:56,988 - Train: 20.29% [826000/4070000] [202.9/1000.0] [batch_t 0.605 (0.608)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 23:30:57,578 - Train: 20.30% [826100/4070000] [203.0/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 23:31:58,132 - Train: 20.30% [826200/4070000] [203.0/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 23:32:04,194 - ==> Total time: 7 days, 22:31:35 Eta: 31 days, 4:01:38 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-09 23:33:01,270 - Train: 20.30% [826300/4070000] [203.0/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 23:34:01,821 - Train: 20.30% [826400/4070000] [203.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 23:35:02,377 - Train: 20.31% [826500/4070000] [203.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 23:36:02,942 - Train: 20.31% [826600/4070000] [203.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 23:37:03,685 - Train: 20.31% [826700/4070000] [203.1/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 23:38:04,266 - Train: 20.31% [826800/4070000] [203.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 23:39:04,855 - Train: 20.32% [826900/4070000] [203.2/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 23:40:05,398 - Train: 20.32% [827000/4070000] [203.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 23:41:05,962 - Train: 20.32% [827100/4070000] [203.2/1000.0] [batch_t 0.609 (0.606)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-09 23:42:06,607 - Train: 20.32% [827200/4070000] [203.2/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 23:43:07,207 - Train: 20.33% [827300/4070000] [203.3/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 23:44:07,790 - Train: 20.33% [827400/4070000] [203.3/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-09 23:45:08,399 - Train: 20.33% [827500/4070000] [203.3/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 23:46:08,995 - Train: 20.33% [827600/4070000] [203.3/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 23:47:09,632 - Train: 20.34% [827700/4070000] [203.4/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 23:48:10,177 - Train: 20.34% [827800/4070000] [203.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 23:49:10,739 - Train: 20.34% [827900/4070000] [203.4/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 23:50:11,397 - Train: 20.34% [828000/4070000] [203.4/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 23:51:12,016 - Train: 20.35% [828100/4070000] [203.5/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 23:52:12,561 - Train: 20.35% [828200/4070000] [203.5/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 23:53:13,146 - Train: 20.35% [828300/4070000] [203.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-09 23:54:13,720 - Train: 20.35% [828400/4070000] [203.5/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 23:55:14,356 - Train: 20.36% [828500/4070000] [203.6/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-09 23:56:14,939 - Train: 20.36% [828600/4070000] [203.6/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-09 23:57:15,477 - Train: 20.36% [828700/4070000] [203.6/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-09 23:58:16,008 - Train: 20.36% [828800/4070000] [203.6/1000.0] [batch_t 0.610 (0.605)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-09 23:59:16,578 - Train: 20.37% [828900/4070000] [203.7/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 00:00:17,168 - Train: 20.37% [829000/4070000] [203.7/1000.0] [batch_t 0.609 (0.606)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-10 00:01:17,720 - Train: 20.37% [829100/4070000] [203.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 00:02:18,304 - Train: 20.37% [829200/4070000] [203.7/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 00:03:19,042 - Train: 20.38% [829300/4070000] [203.8/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 00:04:19,565 - Train: 20.38% [829400/4070000] [203.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 00:05:20,160 - Train: 20.38% [829500/4070000] [203.8/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 00:06:20,860 - Train: 20.38% [829600/4070000] [203.8/1000.0] [batch_t 0.603 (0.607)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 00:07:21,478 - Train: 20.39% [829700/4070000] [203.9/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 00:08:22,138 - Train: 20.39% [829800/4070000] [203.9/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-10 00:09:22,724 - Train: 20.39% [829900/4070000] [203.9/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 00:10:23,303 - Train: 20.39% [830000/4070000] [203.9/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-10 00:11:23,891 - Train: 20.40% [830100/4070000] [204.0/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 00:12:24,451 - Train: 20.40% [830200/4070000] [204.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 00:13:12,912 - ==> Total time: 7 days, 23:12:43 Eta: 31 days, 2:06:08 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-10 00:13:27,250 - Train: 20.40% [830300/4070000] [204.0/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 00:14:27,870 - Train: 20.40% [830400/4070000] [204.0/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 00:15:28,569 - Train: 20.41% [830500/4070000] [204.1/1000.0] [batch_t 0.608 (0.607)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-10 00:16:29,214 - Train: 20.41% [830600/4070000] [204.1/1000.0] [batch_t 0.609 (0.606)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-10 00:17:29,789 - Train: 20.41% [830700/4070000] [204.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 00:18:30,343 - Train: 20.41% [830800/4070000] [204.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 00:19:30,969 - Train: 20.42% [830900/4070000] [204.2/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 00:20:31,603 - Train: 20.42% [831000/4070000] [204.2/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 00:21:32,126 - Train: 20.42% [831100/4070000] [204.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 00:22:32,715 - Train: 20.42% [831200/4070000] [204.2/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 00:23:33,379 - Train: 20.43% [831300/4070000] [204.3/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 00:24:33,957 - Train: 20.43% [831400/4070000] [204.3/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 00:25:34,527 - Train: 20.43% [831500/4070000] [204.3/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 00:26:35,074 - Train: 20.43% [831600/4070000] [204.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 00:27:35,620 - Train: 20.43% [831700/4070000] [204.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 00:28:36,299 - Train: 20.44% [831800/4070000] [204.4/1000.0] [batch_t 0.608 (0.607)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-10 00:29:36,835 - Train: 20.44% [831900/4070000] [204.4/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 00:30:37,379 - Train: 20.44% [832000/4070000] [204.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.003] [optim_t 0.602] [lr 0.005000] +2024-04-10 00:31:37,942 - Train: 20.44% [832100/4070000] [204.4/1000.0] [batch_t 0.610 (0.606)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-10 00:32:38,527 - Train: 20.45% [832200/4070000] [204.5/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 00:33:39,257 - Train: 20.45% [832300/4070000] [204.5/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 00:34:39,835 - Train: 20.45% [832400/4070000] [204.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 00:35:40,420 - Train: 20.45% [832500/4070000] [204.5/1000.0] [batch_t 0.612 (0.606)] [data_t 0.002] [optim_t 0.610] [lr 0.005000] +2024-04-10 00:36:41,012 - Train: 20.46% [832600/4070000] [204.6/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 00:37:41,601 - Train: 20.46% [832700/4070000] [204.6/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 00:38:42,146 - Train: 20.46% [832800/4070000] [204.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 00:39:42,696 - Train: 20.46% [832900/4070000] [204.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 00:40:43,295 - Train: 20.47% [833000/4070000] [204.7/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 00:41:43,954 - Train: 20.47% [833100/4070000] [204.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 00:42:44,591 - Train: 20.47% [833200/4070000] [204.7/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 00:43:45,104 - Train: 20.47% [833300/4070000] [204.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 00:44:45,652 - Train: 20.48% [833400/4070000] [204.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 00:45:46,210 - Train: 20.48% [833500/4070000] [204.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 00:46:46,878 - Train: 20.48% [833600/4070000] [204.8/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 00:47:47,474 - Train: 20.48% [833700/4070000] [204.8/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 00:48:48,032 - Train: 20.49% [833800/4070000] [204.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 00:49:48,571 - Train: 20.49% [833900/4070000] [204.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 00:50:49,102 - Train: 20.49% [834000/4070000] [204.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 00:51:49,675 - Train: 20.49% [834100/4070000] [204.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 00:52:50,223 - Train: 20.50% [834200/4070000] [205.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 00:53:50,857 - Train: 20.50% [834300/4070000] [205.0/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 00:54:21,258 - ==> Total time: 7 days, 23:53:52 Eta: 31 days, 0:11:20 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-10 00:54:53,998 - Train: 20.50% [834400/4070000] [205.0/1000.0] [batch_t 0.605 (0.608)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 00:55:54,576 - Train: 20.50% [834500/4070000] [205.0/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 00:56:55,155 - Train: 20.51% [834600/4070000] [205.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 00:57:55,729 - Train: 20.51% [834700/4070000] [205.1/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 00:58:56,295 - Train: 20.51% [834800/4070000] [205.1/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 00:59:56,874 - Train: 20.51% [834900/4070000] [205.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 01:00:57,441 - Train: 20.52% [835000/4070000] [205.2/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 01:01:58,182 - Train: 20.52% [835100/4070000] [205.2/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 01:02:58,797 - Train: 20.52% [835200/4070000] [205.2/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 01:03:59,395 - Train: 20.52% [835300/4070000] [205.2/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-10 01:05:00,075 - Train: 20.53% [835400/4070000] [205.3/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 01:06:00,680 - Train: 20.53% [835500/4070000] [205.3/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 01:07:01,465 - Train: 20.53% [835600/4070000] [205.3/1000.0] [batch_t 0.608 (0.608)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-10 01:08:02,029 - Train: 20.53% [835700/4070000] [205.3/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 01:09:02,668 - Train: 20.54% [835800/4070000] [205.4/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 01:10:03,220 - Train: 20.54% [835900/4070000] [205.4/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 01:11:03,842 - Train: 20.54% [836000/4070000] [205.4/1000.0] [batch_t 0.611 (0.606)] [data_t 0.002] [optim_t 0.609] [lr 0.005000] +2024-04-10 01:12:04,463 - Train: 20.54% [836100/4070000] [205.4/1000.0] [batch_t 0.609 (0.606)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-10 01:13:05,095 - Train: 20.55% [836200/4070000] [205.5/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-10 01:14:05,856 - Train: 20.55% [836300/4070000] [205.5/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 01:15:06,453 - Train: 20.55% [836400/4070000] [205.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 01:16:07,007 - Train: 20.55% [836500/4070000] [205.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 01:17:07,621 - Train: 20.56% [836600/4070000] [205.6/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 01:18:08,333 - Train: 20.56% [836700/4070000] [205.6/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 01:19:08,917 - Train: 20.56% [836800/4070000] [205.6/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 01:20:09,658 - Train: 20.56% [836900/4070000] [205.6/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 01:21:10,232 - Train: 20.57% [837000/4070000] [205.7/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 01:22:10,806 - Train: 20.57% [837100/4070000] [205.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 01:23:11,418 - Train: 20.57% [837200/4070000] [205.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 01:24:12,017 - Train: 20.57% [837300/4070000] [205.7/1000.0] [batch_t 0.609 (0.606)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-10 01:25:12,631 - Train: 20.57% [837400/4070000] [205.7/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 01:26:13,285 - Train: 20.58% [837500/4070000] [205.8/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 01:27:14,050 - Train: 20.58% [837600/4070000] [205.8/1000.0] [batch_t 0.606 (0.608)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 01:28:14,679 - Train: 20.58% [837700/4070000] [205.8/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 01:29:15,228 - Train: 20.58% [837800/4070000] [205.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 01:30:15,798 - Train: 20.59% [837900/4070000] [205.9/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-10 01:31:16,383 - Train: 20.59% [838000/4070000] [205.9/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-10 01:32:16,969 - Train: 20.59% [838100/4070000] [205.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 01:33:17,743 - Train: 20.59% [838200/4070000] [205.9/1000.0] [batch_t 0.605 (0.608)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 01:34:18,343 - Train: 20.60% [838300/4070000] [206.0/1000.0] [batch_t 0.609 (0.606)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-10 01:35:18,919 - Train: 20.60% [838400/4070000] [206.0/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 01:35:31,036 - ==> Total time: 8 days, 0:35:01 Eta: 30 days, 22:17:21 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-10 01:36:21,849 - Train: 20.60% [838500/4070000] [206.0/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 01:37:22,421 - Train: 20.60% [838600/4070000] [206.0/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 01:38:23,122 - Train: 20.61% [838700/4070000] [206.1/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 01:39:23,690 - Train: 20.61% [838800/4070000] [206.1/1000.0] [batch_t 0.610 (0.606)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-10 01:40:24,293 - Train: 20.61% [838900/4070000] [206.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 01:41:24,939 - Train: 20.61% [839000/4070000] [206.1/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-10 01:42:25,569 - Train: 20.62% [839100/4070000] [206.2/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 01:43:26,230 - Train: 20.62% [839200/4070000] [206.2/1000.0] [batch_t 0.610 (0.607)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-10 01:44:26,844 - Train: 20.62% [839300/4070000] [206.2/1000.0] [batch_t 0.608 (0.606)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-10 01:45:27,458 - Train: 20.62% [839400/4070000] [206.2/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 01:46:28,239 - Train: 20.63% [839500/4070000] [206.3/1000.0] [batch_t 0.606 (0.608)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 01:47:28,857 - Train: 20.63% [839600/4070000] [206.3/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 01:48:29,476 - Train: 20.63% [839700/4070000] [206.3/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 01:49:30,115 - Train: 20.63% [839800/4070000] [206.3/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 01:50:30,680 - Train: 20.64% [839900/4070000] [206.4/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 01:51:31,270 - Train: 20.64% [840000/4070000] [206.4/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 01:52:31,824 - Train: 20.64% [840100/4070000] [206.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 01:53:32,569 - Train: 20.64% [840200/4070000] [206.4/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 01:54:33,175 - Train: 20.65% [840300/4070000] [206.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 01:55:33,814 - Train: 20.65% [840400/4070000] [206.5/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-10 01:56:34,474 - Train: 20.65% [840500/4070000] [206.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 01:57:35,114 - Train: 20.65% [840600/4070000] [206.5/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 01:58:35,780 - Train: 20.66% [840700/4070000] [206.6/1000.0] [batch_t 0.610 (0.607)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-10 01:59:36,541 - Train: 20.66% [840800/4070000] [206.6/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 02:00:37,198 - Train: 20.66% [840900/4070000] [206.6/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 02:01:37,758 - Train: 20.66% [841000/4070000] [206.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 02:02:38,336 - Train: 20.67% [841100/4070000] [206.7/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 02:03:38,940 - Train: 20.67% [841200/4070000] [206.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 02:04:39,719 - Train: 20.67% [841300/4070000] [206.7/1000.0] [batch_t 0.606 (0.608)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 02:05:40,323 - Train: 20.67% [841400/4070000] [206.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 02:06:40,934 - Train: 20.68% [841500/4070000] [206.8/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 02:07:41,514 - Train: 20.68% [841600/4070000] [206.8/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 02:08:42,237 - Train: 20.68% [841700/4070000] [206.8/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 02:09:42,827 - Train: 20.68% [841800/4070000] [206.8/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 02:10:43,377 - Train: 20.69% [841900/4070000] [206.9/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 02:11:43,928 - Train: 20.69% [842000/4070000] [206.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 02:12:44,565 - Train: 20.69% [842100/4070000] [206.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 02:13:45,080 - Train: 20.69% [842200/4070000] [206.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 02:14:45,604 - Train: 20.70% [842300/4070000] [207.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 02:15:46,135 - Train: 20.70% [842400/4070000] [207.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 02:16:40,620 - ==> Total time: 8 days, 1:16:11 Eta: 30 days, 20:24:03 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-10 02:16:48,805 - Train: 20.70% [842500/4070000] [207.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 02:17:49,485 - Train: 20.70% [842600/4070000] [207.0/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 02:18:50,012 - Train: 20.71% [842700/4070000] [207.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 02:19:50,541 - Train: 20.71% [842800/4070000] [207.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 02:20:51,073 - Train: 20.71% [842900/4070000] [207.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 02:21:51,613 - Train: 20.71% [843000/4070000] [207.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 02:22:52,269 - Train: 20.71% [843100/4070000] [207.1/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 02:23:52,804 - Train: 20.72% [843200/4070000] [207.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 02:24:53,365 - Train: 20.72% [843300/4070000] [207.2/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 02:25:53,895 - Train: 20.72% [843400/4070000] [207.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 02:26:54,422 - Train: 20.72% [843500/4070000] [207.2/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 02:27:54,974 - Train: 20.73% [843600/4070000] [207.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 02:28:55,548 - Train: 20.73% [843700/4070000] [207.3/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 02:29:56,182 - Train: 20.73% [843800/4070000] [207.3/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 02:30:56,753 - Train: 20.73% [843900/4070000] [207.3/1000.0] [batch_t 0.603 (0.606)] [data_t 0.003] [optim_t 0.600] [lr 0.005000] +2024-04-10 02:31:57,318 - Train: 20.74% [844000/4070000] [207.4/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 02:32:57,906 - Train: 20.74% [844100/4070000] [207.4/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 02:33:58,465 - Train: 20.74% [844200/4070000] [207.4/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 02:34:59,031 - Train: 20.74% [844300/4070000] [207.4/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 02:35:59,576 - Train: 20.75% [844400/4070000] [207.5/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 02:37:00,241 - Train: 20.75% [844500/4070000] [207.5/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 02:38:00,811 - Train: 20.75% [844600/4070000] [207.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 02:39:01,349 - Train: 20.75% [844700/4070000] [207.5/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 02:40:01,893 - Train: 20.76% [844800/4070000] [207.6/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-10 02:41:02,500 - Train: 20.76% [844900/4070000] [207.6/1000.0] [batch_t 0.611 (0.606)] [data_t 0.002] [optim_t 0.609] [lr 0.005000] +2024-04-10 02:42:03,136 - Train: 20.76% [845000/4070000] [207.6/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 02:43:03,703 - Train: 20.76% [845100/4070000] [207.6/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 02:44:04,270 - Train: 20.77% [845200/4070000] [207.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 02:45:04,839 - Train: 20.77% [845300/4070000] [207.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 02:46:05,416 - Train: 20.77% [845400/4070000] [207.7/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 02:47:05,966 - Train: 20.77% [845500/4070000] [207.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 02:48:06,688 - Train: 20.78% [845600/4070000] [207.8/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 02:49:07,243 - Train: 20.78% [845700/4070000] [207.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 02:50:07,821 - Train: 20.78% [845800/4070000] [207.8/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 02:51:08,406 - Train: 20.78% [845900/4070000] [207.8/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 02:52:08,951 - Train: 20.79% [846000/4070000] [207.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 02:53:09,504 - Train: 20.79% [846100/4070000] [207.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 02:54:10,083 - Train: 20.79% [846200/4070000] [207.9/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 02:55:10,674 - Train: 20.79% [846300/4070000] [207.9/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 02:56:11,354 - Train: 20.80% [846400/4070000] [208.0/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 02:57:11,921 - Train: 20.80% [846500/4070000] [208.0/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 02:57:48,281 - ==> Total time: 8 days, 1:57:19 Eta: 30 days, 18:31:19 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-10 02:58:14,738 - Train: 20.80% [846600/4070000] [208.0/1000.0] [batch_t 0.607 (0.608)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 02:59:15,312 - Train: 20.80% [846700/4070000] [208.0/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 03:00:15,895 - Train: 20.81% [846800/4070000] [208.1/1000.0] [batch_t 0.611 (0.606)] [data_t 0.002] [optim_t 0.609] [lr 0.005000] +2024-04-10 03:01:16,504 - Train: 20.81% [846900/4070000] [208.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 03:02:17,302 - Train: 20.81% [847000/4070000] [208.1/1000.0] [batch_t 0.604 (0.608)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 03:03:17,906 - Train: 20.81% [847100/4070000] [208.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 03:04:18,475 - Train: 20.82% [847200/4070000] [208.2/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 03:05:19,065 - Train: 20.82% [847300/4070000] [208.2/1000.0] [batch_t 0.610 (0.606)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-10 03:06:19,653 - Train: 20.82% [847400/4070000] [208.2/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 03:07:20,334 - Train: 20.82% [847500/4070000] [208.2/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 03:08:20,906 - Train: 20.83% [847600/4070000] [208.3/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 03:09:21,470 - Train: 20.83% [847700/4070000] [208.3/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 03:10:22,023 - Train: 20.83% [847800/4070000] [208.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 03:11:22,651 - Train: 20.83% [847900/4070000] [208.3/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 03:12:23,244 - Train: 20.84% [848000/4070000] [208.4/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 03:13:23,825 - Train: 20.84% [848100/4070000] [208.4/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 03:14:24,550 - Train: 20.84% [848200/4070000] [208.4/1000.0] [batch_t 0.720 (0.607)] [data_t 0.002] [optim_t 0.718] [lr 0.005000] +2024-04-10 03:15:25,118 - Train: 20.84% [848300/4070000] [208.4/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 03:16:25,698 - Train: 20.85% [848400/4070000] [208.5/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 03:17:26,248 - Train: 20.85% [848500/4070000] [208.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 03:18:26,844 - Train: 20.85% [848600/4070000] [208.5/1000.0] [batch_t 0.611 (0.606)] [data_t 0.002] [optim_t 0.609] [lr 0.005000] +2024-04-10 03:19:27,436 - Train: 20.85% [848700/4070000] [208.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 03:20:28,042 - Train: 20.86% [848800/4070000] [208.6/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 03:21:28,631 - Train: 20.86% [848900/4070000] [208.6/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 03:22:29,366 - Train: 20.86% [849000/4070000] [208.6/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 03:23:29,932 - Train: 20.86% [849100/4070000] [208.6/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 03:24:30,522 - Train: 20.86% [849200/4070000] [208.6/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 03:25:31,077 - Train: 20.87% [849300/4070000] [208.7/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 03:26:31,620 - Train: 20.87% [849400/4070000] [208.7/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 03:27:32,336 - Train: 20.87% [849500/4070000] [208.7/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 03:28:32,998 - Train: 20.87% [849600/4070000] [208.7/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 03:29:33,564 - Train: 20.88% [849700/4070000] [208.8/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 03:30:34,163 - Train: 20.88% [849800/4070000] [208.8/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 03:31:34,727 - Train: 20.88% [849900/4070000] [208.8/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 03:32:35,337 - Train: 20.88% [850000/4070000] [208.8/1000.0] [batch_t 0.611 (0.606)] [data_t 0.002] [optim_t 0.609] [lr 0.005000] +2024-04-10 03:33:35,945 - Train: 20.89% [850100/4070000] [208.9/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 03:34:36,543 - Train: 20.89% [850200/4070000] [208.9/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 03:35:37,214 - Train: 20.89% [850300/4070000] [208.9/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 03:36:37,804 - Train: 20.89% [850400/4070000] [208.9/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 03:37:38,374 - Train: 20.90% [850500/4070000] [209.0/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 03:38:38,997 - Train: 20.90% [850600/4070000] [209.0/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 03:38:57,176 - ==> Total time: 8 days, 2:38:28 Eta: 30 days, 16:39:21 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-10 03:39:41,823 - Train: 20.90% [850700/4070000] [209.0/1000.0] [batch_t 0.610 (0.606)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-10 03:40:42,544 - Train: 20.90% [850800/4070000] [209.0/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 03:41:43,095 - Train: 20.91% [850900/4070000] [209.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 03:42:43,652 - Train: 20.91% [851000/4070000] [209.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 03:43:44,232 - Train: 20.91% [851100/4070000] [209.1/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 03:44:44,798 - Train: 20.91% [851200/4070000] [209.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 03:45:45,375 - Train: 20.92% [851300/4070000] [209.2/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 03:46:45,929 - Train: 20.92% [851400/4070000] [209.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 03:47:46,574 - Train: 20.92% [851500/4070000] [209.2/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 03:48:47,167 - Train: 20.92% [851600/4070000] [209.2/1000.0] [batch_t 0.609 (0.606)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-10 03:49:47,734 - Train: 20.93% [851700/4070000] [209.3/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 03:50:48,297 - Train: 20.93% [851800/4070000] [209.3/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 03:51:48,832 - Train: 20.93% [851900/4070000] [209.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 03:52:49,344 - Train: 20.93% [852000/4070000] [209.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 03:53:49,998 - Train: 20.94% [852100/4070000] [209.4/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 03:54:50,557 - Train: 20.94% [852200/4070000] [209.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 03:55:51,109 - Train: 20.94% [852300/4070000] [209.4/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 03:56:51,673 - Train: 20.94% [852400/4070000] [209.4/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 03:57:52,223 - Train: 20.95% [852500/4070000] [209.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 03:58:52,793 - Train: 20.95% [852600/4070000] [209.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 03:59:53,326 - Train: 20.95% [852700/4070000] [209.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 04:00:53,985 - Train: 20.95% [852800/4070000] [209.5/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 04:01:54,540 - Train: 20.96% [852900/4070000] [209.6/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-10 04:02:55,072 - Train: 20.96% [853000/4070000] [209.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 04:03:55,718 - Train: 20.96% [853100/4070000] [209.6/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 04:04:56,291 - Train: 20.96% [853200/4070000] [209.6/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 04:05:56,932 - Train: 20.97% [853300/4070000] [209.7/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 04:06:57,704 - Train: 20.97% [853400/4070000] [209.7/1000.0] [batch_t 0.604 (0.608)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 04:07:58,339 - Train: 20.97% [853500/4070000] [209.7/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 04:08:58,931 - Train: 20.97% [853600/4070000] [209.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 04:09:59,612 - Train: 20.98% [853700/4070000] [209.8/1000.0] [batch_t 0.610 (0.607)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-10 04:11:00,237 - Train: 20.98% [853800/4070000] [209.8/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 04:12:00,833 - Train: 20.98% [853900/4070000] [209.8/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 04:13:01,410 - Train: 20.98% [854000/4070000] [209.8/1000.0] [batch_t 0.610 (0.606)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-10 04:14:02,168 - Train: 20.99% [854100/4070000] [209.9/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 04:15:02,799 - Train: 20.99% [854200/4070000] [209.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 04:16:03,426 - Train: 20.99% [854300/4070000] [209.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 04:17:04,084 - Train: 20.99% [854400/4070000] [209.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 04:18:04,800 - Train: 21.00% [854500/4070000] [210.0/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 04:19:05,386 - Train: 21.00% [854600/4070000] [210.0/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-10 04:20:06,133 - Train: 21.00% [854700/4070000] [210.0/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 04:20:06,137 - ==> Total time: 8 days, 3:19:36 Eta: 30 days, 14:48:04 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-10 04:21:09,993 - Train: 21.00% [854800/4070000] [210.0/1000.0] [batch_t 0.606 (0.609)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 04:22:10,558 - Train: 21.00% [854900/4070000] [210.0/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 04:23:11,150 - Train: 21.01% [855000/4070000] [210.1/1000.0] [batch_t 0.607 (0.606)] [data_t 0.003] [optim_t 0.605] [lr 0.005000] +2024-04-10 04:24:11,749 - Train: 21.01% [855100/4070000] [210.1/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 04:25:12,395 - Train: 21.01% [855200/4070000] [210.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 04:26:12,996 - Train: 21.01% [855300/4070000] [210.1/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 04:27:13,814 - Train: 21.02% [855400/4070000] [210.2/1000.0] [batch_t 0.606 (0.608)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 04:28:14,421 - Train: 21.02% [855500/4070000] [210.2/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 04:29:14,985 - Train: 21.02% [855600/4070000] [210.2/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 04:30:15,606 - Train: 21.02% [855700/4070000] [210.2/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 04:31:16,210 - Train: 21.03% [855800/4070000] [210.3/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 04:32:16,926 - Train: 21.03% [855900/4070000] [210.3/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 04:33:17,485 - Train: 21.03% [856000/4070000] [210.3/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-10 04:34:18,022 - Train: 21.03% [856100/4070000] [210.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 04:35:18,598 - Train: 21.04% [856200/4070000] [210.4/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 04:36:19,199 - Train: 21.04% [856300/4070000] [210.4/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 04:37:19,777 - Train: 21.04% [856400/4070000] [210.4/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-10 04:38:20,356 - Train: 21.04% [856500/4070000] [210.4/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 04:39:21,065 - Train: 21.05% [856600/4070000] [210.5/1000.0] [batch_t 0.605 (0.607)] [data_t 0.003] [optim_t 0.602] [lr 0.005000] +2024-04-10 04:40:21,633 - Train: 21.05% [856700/4070000] [210.5/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 04:41:22,238 - Train: 21.05% [856800/4070000] [210.5/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-10 04:42:22,787 - Train: 21.05% [856900/4070000] [210.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 04:43:23,372 - Train: 21.06% [857000/4070000] [210.6/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 04:44:23,941 - Train: 21.06% [857100/4070000] [210.6/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 04:45:24,613 - Train: 21.06% [857200/4070000] [210.6/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 04:46:25,191 - Train: 21.06% [857300/4070000] [210.6/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 04:47:25,734 - Train: 21.07% [857400/4070000] [210.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 04:48:26,269 - Train: 21.07% [857500/4070000] [210.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 04:49:26,838 - Train: 21.07% [857600/4070000] [210.7/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-10 04:50:27,369 - Train: 21.07% [857700/4070000] [210.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 04:51:27,901 - Train: 21.08% [857800/4070000] [210.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 04:52:28,572 - Train: 21.08% [857900/4070000] [210.8/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 04:53:29,145 - Train: 21.08% [858000/4070000] [210.8/1000.0] [batch_t 0.609 (0.606)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-10 04:54:29,714 - Train: 21.08% [858100/4070000] [210.8/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-10 04:55:30,264 - Train: 21.09% [858200/4070000] [210.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 04:56:30,848 - Train: 21.09% [858300/4070000] [210.9/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 04:57:31,404 - Train: 21.09% [858400/4070000] [210.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 04:58:32,144 - Train: 21.09% [858500/4070000] [210.9/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 04:59:32,701 - Train: 21.10% [858600/4070000] [211.0/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-10 05:00:33,247 - Train: 21.10% [858700/4070000] [211.0/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 05:01:15,646 - ==> Total time: 8 days, 4:00:46 Eta: 30 days, 12:57:29 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-10 05:01:35,809 - Train: 21.10% [858800/4070000] [211.0/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 05:02:36,354 - Train: 21.10% [858900/4070000] [211.0/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 05:03:36,909 - Train: 21.11% [859000/4070000] [211.1/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-10 05:04:37,599 - Train: 21.11% [859100/4070000] [211.1/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 05:05:38,169 - Train: 21.11% [859200/4070000] [211.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 05:06:38,746 - Train: 21.11% [859300/4070000] [211.1/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 05:07:39,328 - Train: 21.12% [859400/4070000] [211.2/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 05:08:39,964 - Train: 21.12% [859500/4070000] [211.2/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 05:09:40,535 - Train: 21.12% [859600/4070000] [211.2/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 05:10:41,224 - Train: 21.12% [859700/4070000] [211.2/1000.0] [batch_t 0.610 (0.607)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-10 05:11:41,822 - Train: 21.13% [859800/4070000] [211.3/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 05:12:42,438 - Train: 21.13% [859900/4070000] [211.3/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 05:13:42,999 - Train: 21.13% [860000/4070000] [211.3/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 05:14:43,555 - Train: 21.13% [860100/4070000] [211.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 05:15:44,110 - Train: 21.14% [860200/4070000] [211.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 05:16:44,707 - Train: 21.14% [860300/4070000] [211.4/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 05:17:45,357 - Train: 21.14% [860400/4070000] [211.4/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 05:18:45,994 - Train: 21.14% [860500/4070000] [211.4/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 05:19:46,595 - Train: 21.14% [860600/4070000] [211.4/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 05:20:47,209 - Train: 21.15% [860700/4070000] [211.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 05:21:47,871 - Train: 21.15% [860800/4070000] [211.5/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 05:22:48,464 - Train: 21.15% [860900/4070000] [211.5/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 05:23:49,147 - Train: 21.15% [861000/4070000] [211.5/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 05:24:49,707 - Train: 21.16% [861100/4070000] [211.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 05:25:50,273 - Train: 21.16% [861200/4070000] [211.6/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-10 05:26:50,868 - Train: 21.16% [861300/4070000] [211.6/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 05:27:51,463 - Train: 21.16% [861400/4070000] [211.6/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 05:28:52,030 - Train: 21.17% [861500/4070000] [211.7/1000.0] [batch_t 0.610 (0.606)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-10 05:29:52,594 - Train: 21.17% [861600/4070000] [211.7/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 05:30:53,294 - Train: 21.17% [861700/4070000] [211.7/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 05:31:53,869 - Train: 21.17% [861800/4070000] [211.7/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 05:32:54,520 - Train: 21.18% [861900/4070000] [211.8/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 05:33:55,256 - Train: 21.18% [862000/4070000] [211.8/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 05:34:55,846 - Train: 21.18% [862100/4070000] [211.8/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 05:35:56,415 - Train: 21.18% [862200/4070000] [211.8/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 05:36:57,091 - Train: 21.19% [862300/4070000] [211.9/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 05:37:57,637 - Train: 21.19% [862400/4070000] [211.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 05:38:58,203 - Train: 21.19% [862500/4070000] [211.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 05:39:58,790 - Train: 21.19% [862600/4070000] [211.9/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 05:40:59,408 - Train: 21.20% [862700/4070000] [212.0/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 05:42:00,055 - Train: 21.20% [862800/4070000] [212.0/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 05:42:24,294 - ==> Total time: 8 days, 4:41:55 Eta: 30 days, 11:07:30 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-10 05:43:03,143 - Train: 21.20% [862900/4070000] [212.0/1000.0] [batch_t 0.605 (0.610)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 05:44:03,856 - Train: 21.20% [863000/4070000] [212.0/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 05:45:04,411 - Train: 21.21% [863100/4070000] [212.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 05:46:04,993 - Train: 21.21% [863200/4070000] [212.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 05:47:05,556 - Train: 21.21% [863300/4070000] [212.1/1000.0] [batch_t 0.609 (0.606)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-10 05:48:06,131 - Train: 21.21% [863400/4070000] [212.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 05:49:06,767 - Train: 21.22% [863500/4070000] [212.2/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 05:50:07,416 - Train: 21.22% [863600/4070000] [212.2/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-10 05:51:07,962 - Train: 21.22% [863700/4070000] [212.2/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-10 05:52:08,479 - Train: 21.22% [863800/4070000] [212.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 05:53:09,022 - Train: 21.23% [863900/4070000] [212.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 05:54:09,592 - Train: 21.23% [864000/4070000] [212.3/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 05:55:10,160 - Train: 21.23% [864100/4070000] [212.3/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 05:56:10,756 - Train: 21.23% [864200/4070000] [212.3/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 05:57:11,402 - Train: 21.24% [864300/4070000] [212.4/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 05:58:11,972 - Train: 21.24% [864400/4070000] [212.4/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 05:59:12,596 - Train: 21.24% [864500/4070000] [212.4/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 06:00:13,154 - Train: 21.24% [864600/4070000] [212.4/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 06:01:13,729 - Train: 21.25% [864700/4070000] [212.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 06:02:14,412 - Train: 21.25% [864800/4070000] [212.5/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 06:03:14,961 - Train: 21.25% [864900/4070000] [212.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 06:04:15,517 - Train: 21.25% [865000/4070000] [212.5/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-10 06:05:16,112 - Train: 21.26% [865100/4070000] [212.6/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 06:06:16,698 - Train: 21.26% [865200/4070000] [212.6/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 06:07:17,229 - Train: 21.26% [865300/4070000] [212.6/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 06:08:17,787 - Train: 21.26% [865400/4070000] [212.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 06:09:18,482 - Train: 21.27% [865500/4070000] [212.7/1000.0] [batch_t 0.687 (0.607)] [data_t 0.002] [optim_t 0.685] [lr 0.005000] +2024-04-10 06:10:19,045 - Train: 21.27% [865600/4070000] [212.7/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 06:11:19,633 - Train: 21.27% [865700/4070000] [212.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 06:12:20,178 - Train: 21.27% [865800/4070000] [212.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 06:13:20,733 - Train: 21.28% [865900/4070000] [212.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 06:14:21,285 - Train: 21.28% [866000/4070000] [212.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 06:15:21,939 - Train: 21.28% [866100/4070000] [212.8/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 06:16:22,537 - Train: 21.28% [866200/4070000] [212.8/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 06:17:23,114 - Train: 21.29% [866300/4070000] [212.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 06:18:23,680 - Train: 21.29% [866400/4070000] [212.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 06:19:24,244 - Train: 21.29% [866500/4070000] [212.9/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 06:20:24,825 - Train: 21.29% [866600/4070000] [212.9/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 06:21:25,563 - Train: 21.29% [866700/4070000] [212.9/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 06:22:26,099 - Train: 21.30% [866800/4070000] [213.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 06:23:26,653 - Train: 21.30% [866900/4070000] [213.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 06:23:32,713 - ==> Total time: 8 days, 5:23:03 Eta: 30 days, 9:18:09 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-10 06:24:29,510 - Train: 21.30% [867000/4070000] [213.0/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 06:25:30,042 - Train: 21.30% [867100/4070000] [213.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 06:26:30,606 - Train: 21.31% [867200/4070000] [213.1/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 06:27:31,185 - Train: 21.31% [867300/4070000] [213.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 06:28:31,885 - Train: 21.31% [867400/4070000] [213.1/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 06:29:32,430 - Train: 21.31% [867500/4070000] [213.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 06:30:33,033 - Train: 21.32% [867600/4070000] [213.2/1000.0] [batch_t 0.613 (0.606)] [data_t 0.002] [optim_t 0.611] [lr 0.005000] +2024-04-10 06:31:33,621 - Train: 21.32% [867700/4070000] [213.2/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 06:32:34,223 - Train: 21.32% [867800/4070000] [213.2/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 06:33:35,044 - Train: 21.32% [867900/4070000] [213.2/1000.0] [batch_t 0.604 (0.608)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 06:34:35,641 - Train: 21.33% [868000/4070000] [213.3/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 06:35:36,266 - Train: 21.33% [868100/4070000] [213.3/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 06:36:36,864 - Train: 21.33% [868200/4070000] [213.3/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 06:37:37,410 - Train: 21.33% [868300/4070000] [213.3/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-10 06:38:38,043 - Train: 21.34% [868400/4070000] [213.4/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 06:39:38,673 - Train: 21.34% [868500/4070000] [213.4/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 06:40:39,275 - Train: 21.34% [868600/4070000] [213.4/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 06:41:40,027 - Train: 21.34% [868700/4070000] [213.4/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 06:42:40,592 - Train: 21.35% [868800/4070000] [213.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 06:43:41,143 - Train: 21.35% [868900/4070000] [213.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 06:44:41,708 - Train: 21.35% [869000/4070000] [213.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 06:45:42,255 - Train: 21.35% [869100/4070000] [213.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 06:46:42,919 - Train: 21.36% [869200/4070000] [213.6/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 06:47:43,461 - Train: 21.36% [869300/4070000] [213.6/1000.0] [batch_t 0.607 (0.605)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-10 06:48:44,019 - Train: 21.36% [869400/4070000] [213.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 06:49:44,577 - Train: 21.36% [869500/4070000] [213.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 06:50:45,128 - Train: 21.37% [869600/4070000] [213.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 06:51:45,677 - Train: 21.37% [869700/4070000] [213.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 06:52:46,261 - Train: 21.37% [869800/4070000] [213.7/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 06:53:46,829 - Train: 21.37% [869900/4070000] [213.7/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 06:54:47,552 - Train: 21.38% [870000/4070000] [213.8/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 06:55:48,096 - Train: 21.38% [870100/4070000] [213.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 06:56:48,665 - Train: 21.38% [870200/4070000] [213.8/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 06:57:49,222 - Train: 21.38% [870300/4070000] [213.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 06:58:49,787 - Train: 21.39% [870400/4070000] [213.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 06:59:50,452 - Train: 21.39% [870500/4070000] [213.9/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 07:00:51,050 - Train: 21.39% [870600/4070000] [213.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 07:01:51,598 - Train: 21.39% [870700/4070000] [213.9/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 07:02:52,144 - Train: 21.40% [870800/4070000] [214.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 07:03:52,702 - Train: 21.40% [870900/4070000] [214.0/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 07:04:41,160 - ==> Total time: 8 days, 6:04:12 Eta: 30 days, 7:29:26 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-10 07:04:55,280 - Train: 21.40% [871000/4070000] [214.0/1000.0] [batch_t 0.603 (0.609)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 07:05:55,956 - Train: 21.40% [871100/4070000] [214.0/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 07:06:56,493 - Train: 21.41% [871200/4070000] [214.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 07:07:57,041 - Train: 21.41% [871300/4070000] [214.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 07:08:57,591 - Train: 21.41% [871400/4070000] [214.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 07:09:58,150 - Train: 21.41% [871500/4070000] [214.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 07:10:58,694 - Train: 21.42% [871600/4070000] [214.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 07:11:59,259 - Train: 21.42% [871700/4070000] [214.2/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 07:12:59,819 - Train: 21.42% [871800/4070000] [214.2/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 07:14:00,452 - Train: 21.42% [871900/4070000] [214.2/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 07:15:01,001 - Train: 21.43% [872000/4070000] [214.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 07:16:01,532 - Train: 21.43% [872100/4070000] [214.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 07:17:02,084 - Train: 21.43% [872200/4070000] [214.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 07:18:02,633 - Train: 21.43% [872300/4070000] [214.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 07:19:03,299 - Train: 21.43% [872400/4070000] [214.3/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 07:20:03,887 - Train: 21.44% [872500/4070000] [214.4/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 07:21:04,435 - Train: 21.44% [872600/4070000] [214.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 07:22:05,042 - Train: 21.44% [872700/4070000] [214.4/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 07:23:05,616 - Train: 21.44% [872800/4070000] [214.4/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 07:24:06,186 - Train: 21.45% [872900/4070000] [214.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 07:25:06,740 - Train: 21.45% [873000/4070000] [214.5/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 07:26:07,279 - Train: 21.45% [873100/4070000] [214.5/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 07:27:07,973 - Train: 21.45% [873200/4070000] [214.5/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 07:28:08,563 - Train: 21.46% [873300/4070000] [214.6/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 07:29:09,127 - Train: 21.46% [873400/4070000] [214.6/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 07:30:09,690 - Train: 21.46% [873500/4070000] [214.6/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 07:31:10,236 - Train: 21.46% [873600/4070000] [214.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 07:32:10,884 - Train: 21.47% [873700/4070000] [214.7/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 07:33:11,420 - Train: 21.47% [873800/4070000] [214.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 07:34:11,992 - Train: 21.47% [873900/4070000] [214.7/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 07:35:12,536 - Train: 21.47% [874000/4070000] [214.7/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 07:36:13,103 - Train: 21.48% [874100/4070000] [214.8/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 07:37:13,665 - Train: 21.48% [874200/4070000] [214.8/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 07:38:14,230 - Train: 21.48% [874300/4070000] [214.8/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 07:39:14,810 - Train: 21.48% [874400/4070000] [214.8/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 07:40:15,463 - Train: 21.49% [874500/4070000] [214.9/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 07:41:15,998 - Train: 21.49% [874600/4070000] [214.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 07:42:16,555 - Train: 21.49% [874700/4070000] [214.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 07:43:17,108 - Train: 21.49% [874800/4070000] [214.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 07:44:17,654 - Train: 21.50% [874900/4070000] [215.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 07:45:18,326 - Train: 21.50% [875000/4070000] [215.0/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 07:45:48,607 - ==> Total time: 8 days, 6:45:19 Eta: 30 days, 5:41:18 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-10 07:46:21,156 - Train: 21.50% [875100/4070000] [215.0/1000.0] [batch_t 0.606 (0.608)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 07:47:21,693 - Train: 21.50% [875200/4070000] [215.0/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-10 07:48:22,244 - Train: 21.51% [875300/4070000] [215.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 07:49:22,801 - Train: 21.51% [875400/4070000] [215.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 07:50:23,488 - Train: 21.51% [875500/4070000] [215.1/1000.0] [batch_t 0.603 (0.607)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 07:51:24,026 - Train: 21.51% [875600/4070000] [215.1/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 07:52:24,576 - Train: 21.52% [875700/4070000] [215.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 07:53:25,139 - Train: 21.52% [875800/4070000] [215.2/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 07:54:25,681 - Train: 21.52% [875900/4070000] [215.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 07:55:26,228 - Train: 21.52% [876000/4070000] [215.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 07:56:26,765 - Train: 21.53% [876100/4070000] [215.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 07:57:27,282 - Train: 21.53% [876200/4070000] [215.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 07:58:27,892 - Train: 21.53% [876300/4070000] [215.3/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 07:59:28,451 - Train: 21.53% [876400/4070000] [215.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 08:00:29,024 - Train: 21.54% [876500/4070000] [215.4/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 08:01:29,569 - Train: 21.54% [876600/4070000] [215.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 08:02:30,137 - Train: 21.54% [876700/4070000] [215.4/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 08:03:30,804 - Train: 21.54% [876800/4070000] [215.4/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 08:04:31,372 - Train: 21.55% [876900/4070000] [215.5/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-10 08:05:31,961 - Train: 21.55% [877000/4070000] [215.5/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 08:06:32,528 - Train: 21.55% [877100/4070000] [215.5/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 08:07:33,093 - Train: 21.55% [877200/4070000] [215.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 08:08:33,660 - Train: 21.56% [877300/4070000] [215.6/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 08:09:34,212 - Train: 21.56% [877400/4070000] [215.6/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 08:10:34,798 - Train: 21.56% [877500/4070000] [215.6/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 08:11:35,447 - Train: 21.56% [877600/4070000] [215.6/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 08:12:36,015 - Train: 21.57% [877700/4070000] [215.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 08:13:36,580 - Train: 21.57% [877800/4070000] [215.7/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 08:14:37,140 - Train: 21.57% [877900/4070000] [215.7/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 08:15:37,701 - Train: 21.57% [878000/4070000] [215.7/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 08:16:38,334 - Train: 21.57% [878100/4070000] [215.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 08:17:38,917 - Train: 21.58% [878200/4070000] [215.8/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 08:18:39,494 - Train: 21.58% [878300/4070000] [215.8/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 08:19:40,031 - Train: 21.58% [878400/4070000] [215.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 08:20:40,591 - Train: 21.58% [878500/4070000] [215.8/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 08:21:41,165 - Train: 21.59% [878600/4070000] [215.9/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 08:22:41,724 - Train: 21.59% [878700/4070000] [215.9/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 08:23:42,243 - Train: 21.59% [878800/4070000] [215.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 08:24:42,989 - Train: 21.59% [878900/4070000] [215.9/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 08:25:43,570 - Train: 21.60% [879000/4070000] [216.0/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 08:26:44,175 - Train: 21.60% [879100/4070000] [216.0/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 08:26:56,302 - ==> Total time: 8 days, 7:26:27 Eta: 30 days, 3:53:47 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-10 08:27:47,411 - Train: 21.60% [879200/4070000] [216.0/1000.0] [batch_t 0.605 (0.610)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 08:28:47,978 - Train: 21.60% [879300/4070000] [216.0/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 08:29:48,664 - Train: 21.61% [879400/4070000] [216.1/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 08:30:49,226 - Train: 21.61% [879500/4070000] [216.1/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 08:31:49,789 - Train: 21.61% [879600/4070000] [216.1/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 08:32:50,378 - Train: 21.61% [879700/4070000] [216.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 08:33:50,985 - Train: 21.62% [879800/4070000] [216.2/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 08:34:51,566 - Train: 21.62% [879900/4070000] [216.2/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 08:35:52,225 - Train: 21.62% [880000/4070000] [216.2/1000.0] [batch_t 0.616 (0.606)] [data_t 0.002] [optim_t 0.614] [lr 0.005000] +2024-04-10 08:36:52,831 - Train: 21.62% [880100/4070000] [216.2/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 08:37:53,407 - Train: 21.63% [880200/4070000] [216.3/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 08:38:53,946 - Train: 21.63% [880300/4070000] [216.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 08:39:54,501 - Train: 21.63% [880400/4070000] [216.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 08:40:55,054 - Train: 21.63% [880500/4070000] [216.3/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 08:41:55,653 - Train: 21.64% [880600/4070000] [216.4/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 08:42:56,347 - Train: 21.64% [880700/4070000] [216.4/1000.0] [batch_t 0.603 (0.607)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 08:43:56,907 - Train: 21.64% [880800/4070000] [216.4/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 08:44:57,495 - Train: 21.64% [880900/4070000] [216.4/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 08:45:58,116 - Train: 21.65% [881000/4070000] [216.5/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 08:46:58,690 - Train: 21.65% [881100/4070000] [216.5/1000.0] [batch_t 0.609 (0.606)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-10 08:47:59,266 - Train: 21.65% [881200/4070000] [216.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 08:49:00,034 - Train: 21.65% [881300/4070000] [216.5/1000.0] [batch_t 0.606 (0.608)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 08:50:00,589 - Train: 21.66% [881400/4070000] [216.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 08:51:01,145 - Train: 21.66% [881500/4070000] [216.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 08:52:01,681 - Train: 21.66% [881600/4070000] [216.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 08:53:02,263 - Train: 21.66% [881700/4070000] [216.6/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 08:54:02,938 - Train: 21.67% [881800/4070000] [216.7/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 08:55:03,516 - Train: 21.67% [881900/4070000] [216.7/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 08:56:04,274 - Train: 21.67% [882000/4070000] [216.7/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 08:57:04,826 - Train: 21.67% [882100/4070000] [216.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 08:58:05,387 - Train: 21.68% [882200/4070000] [216.8/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 08:59:05,964 - Train: 21.68% [882300/4070000] [216.8/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 09:00:06,524 - Train: 21.68% [882400/4070000] [216.8/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 09:01:07,326 - Train: 21.68% [882500/4070000] [216.8/1000.0] [batch_t 0.604 (0.608)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 09:02:07,883 - Train: 21.69% [882600/4070000] [216.9/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 09:03:08,451 - Train: 21.69% [882700/4070000] [216.9/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 09:04:09,093 - Train: 21.69% [882800/4070000] [216.9/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 09:05:09,678 - Train: 21.69% [882900/4070000] [216.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 09:06:10,271 - Train: 21.70% [883000/4070000] [217.0/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 09:07:10,821 - Train: 21.70% [883100/4070000] [217.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 09:08:05,335 - ==> Total time: 8 days, 8:07:36 Eta: 30 days, 2:06:58 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-10 09:08:13,560 - Train: 21.70% [883200/4070000] [217.0/1000.0] [batch_t 0.604 (0.608)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 09:09:14,103 - Train: 21.70% [883300/4070000] [217.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 09:10:14,688 - Train: 21.71% [883400/4070000] [217.1/1000.0] [batch_t 0.609 (0.606)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-10 09:11:15,237 - Train: 21.71% [883500/4070000] [217.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 09:12:15,826 - Train: 21.71% [883600/4070000] [217.1/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 09:13:16,378 - Train: 21.71% [883700/4070000] [217.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 09:14:16,919 - Train: 21.71% [883800/4070000] [217.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 09:15:17,475 - Train: 21.72% [883900/4070000] [217.2/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 09:16:18,189 - Train: 21.72% [884000/4070000] [217.2/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 09:17:18,738 - Train: 21.72% [884100/4070000] [217.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 09:18:19,321 - Train: 21.72% [884200/4070000] [217.2/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 09:19:19,885 - Train: 21.73% [884300/4070000] [217.3/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 09:20:20,435 - Train: 21.73% [884400/4070000] [217.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 09:21:21,228 - Train: 21.73% [884500/4070000] [217.3/1000.0] [batch_t 0.605 (0.608)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 09:22:21,835 - Train: 21.73% [884600/4070000] [217.3/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 09:23:22,432 - Train: 21.74% [884700/4070000] [217.4/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 09:24:23,054 - Train: 21.74% [884800/4070000] [217.4/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 09:25:23,653 - Train: 21.74% [884900/4070000] [217.4/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 09:26:24,241 - Train: 21.74% [885000/4070000] [217.4/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 09:27:24,853 - Train: 21.75% [885100/4070000] [217.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 09:28:25,428 - Train: 21.75% [885200/4070000] [217.5/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 09:29:26,115 - Train: 21.75% [885300/4070000] [217.5/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 09:30:26,689 - Train: 21.75% [885400/4070000] [217.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 09:31:27,295 - Train: 21.76% [885500/4070000] [217.6/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 09:32:27,844 - Train: 21.76% [885600/4070000] [217.6/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-10 09:33:28,462 - Train: 21.76% [885700/4070000] [217.6/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 09:34:29,197 - Train: 21.76% [885800/4070000] [217.6/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 09:35:29,726 - Train: 21.77% [885900/4070000] [217.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 09:36:30,349 - Train: 21.77% [886000/4070000] [217.7/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-10 09:37:30,918 - Train: 21.77% [886100/4070000] [217.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 09:38:31,475 - Train: 21.77% [886200/4070000] [217.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 09:39:32,082 - Train: 21.78% [886300/4070000] [217.8/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 09:40:32,665 - Train: 21.78% [886400/4070000] [217.8/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 09:41:33,278 - Train: 21.78% [886500/4070000] [217.8/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 09:42:34,000 - Train: 21.78% [886600/4070000] [217.8/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 09:43:34,563 - Train: 21.79% [886700/4070000] [217.9/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 09:44:35,140 - Train: 21.79% [886800/4070000] [217.9/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-10 09:45:35,738 - Train: 21.79% [886900/4070000] [217.9/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 09:46:36,351 - Train: 21.79% [887000/4070000] [217.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 09:47:37,127 - Train: 21.80% [887100/4070000] [218.0/1000.0] [batch_t 0.607 (0.608)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 09:48:37,753 - Train: 21.80% [887200/4070000] [218.0/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 09:49:14,088 - ==> Total time: 8 days, 8:48:44 Eta: 30 days, 0:20:44 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-10 09:49:41,123 - Train: 21.80% [887300/4070000] [218.0/1000.0] [batch_t 0.606 (0.616)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 09:50:41,694 - Train: 21.80% [887400/4070000] [218.0/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 09:51:42,288 - Train: 21.81% [887500/4070000] [218.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 09:52:42,846 - Train: 21.81% [887600/4070000] [218.1/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 09:53:43,561 - Train: 21.81% [887700/4070000] [218.1/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 09:54:44,138 - Train: 21.81% [887800/4070000] [218.1/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 09:55:44,745 - Train: 21.82% [887900/4070000] [218.2/1000.0] [batch_t 0.610 (0.606)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-10 09:56:45,324 - Train: 21.82% [888000/4070000] [218.2/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 09:57:45,901 - Train: 21.82% [888100/4070000] [218.2/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 09:58:46,563 - Train: 21.82% [888200/4070000] [218.2/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 09:59:47,154 - Train: 21.83% [888300/4070000] [218.3/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 10:00:47,830 - Train: 21.83% [888400/4070000] [218.3/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 10:01:48,446 - Train: 21.83% [888500/4070000] [218.3/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 10:02:49,011 - Train: 21.83% [888600/4070000] [218.3/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 10:03:49,569 - Train: 21.84% [888700/4070000] [218.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 10:04:50,128 - Train: 21.84% [888800/4070000] [218.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 10:05:50,683 - Train: 21.84% [888900/4070000] [218.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 10:06:51,381 - Train: 21.84% [889000/4070000] [218.4/1000.0] [batch_t 0.609 (0.607)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-10 10:07:51,956 - Train: 21.85% [889100/4070000] [218.5/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 10:08:52,555 - Train: 21.85% [889200/4070000] [218.5/1000.0] [batch_t 0.609 (0.606)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-10 10:09:53,165 - Train: 21.85% [889300/4070000] [218.5/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 10:10:53,743 - Train: 21.85% [889400/4070000] [218.5/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 10:11:54,315 - Train: 21.86% [889500/4070000] [218.6/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 10:12:54,891 - Train: 21.86% [889600/4070000] [218.6/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 10:13:55,500 - Train: 21.86% [889700/4070000] [218.6/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 10:14:56,077 - Train: 21.86% [889800/4070000] [218.6/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 10:15:56,666 - Train: 21.86% [889900/4070000] [218.6/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 10:17:11,333 - Train: 21.87% [890000/4070000] [218.7/1000.0] [batch_t 0.606 (0.747)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 10:18:21,824 - Train: 21.87% [890100/4070000] [218.7/1000.0] [batch_t 0.604 (0.705)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 10:19:32,890 - Train: 21.87% [890200/4070000] [218.7/1000.0] [batch_t 0.604 (0.711)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 10:21:35,492 - Train: 21.87% [890300/4070000] [218.7/1000.0] [batch_t 0.605 (1.226)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 10:22:36,065 - Train: 21.88% [890400/4070000] [218.8/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 10:23:36,714 - Train: 21.88% [890500/4070000] [218.8/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 10:24:37,250 - Train: 21.88% [890600/4070000] [218.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 10:25:37,788 - Train: 21.88% [890700/4070000] [218.8/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 10:26:38,334 - Train: 21.89% [890800/4070000] [218.9/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 10:27:38,920 - Train: 21.89% [890900/4070000] [218.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 10:28:39,581 - Train: 21.89% [891000/4070000] [218.9/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 10:29:40,192 - Train: 21.89% [891100/4070000] [218.9/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 10:30:40,731 - Train: 21.90% [891200/4070000] [219.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 10:31:41,279 - Train: 21.90% [891300/4070000] [219.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 10:31:59,476 - ==> Total time: 8 days, 9:31:30 Eta: 29 days, 22:40:50 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-10 10:32:44,010 - Train: 21.90% [891400/4070000] [219.0/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 10:33:44,717 - Train: 21.90% [891500/4070000] [219.0/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 10:34:45,293 - Train: 21.91% [891600/4070000] [219.1/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 10:35:45,877 - Train: 21.91% [891700/4070000] [219.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 10:36:46,445 - Train: 21.91% [891800/4070000] [219.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 10:37:46,966 - Train: 21.91% [891900/4070000] [219.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 10:38:47,534 - Train: 21.92% [892000/4070000] [219.2/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 10:39:48,212 - Train: 21.92% [892100/4070000] [219.2/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 10:40:48,765 - Train: 21.92% [892200/4070000] [219.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 10:41:49,325 - Train: 21.92% [892300/4070000] [219.2/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 10:42:49,869 - Train: 21.93% [892400/4070000] [219.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 10:43:50,427 - Train: 21.93% [892500/4070000] [219.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 10:44:51,041 - Train: 21.93% [892600/4070000] [219.3/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 10:45:51,637 - Train: 21.93% [892700/4070000] [219.3/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 10:46:52,325 - Train: 21.94% [892800/4070000] [219.4/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 10:47:52,897 - Train: 21.94% [892900/4070000] [219.4/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 10:48:53,477 - Train: 21.94% [893000/4070000] [219.4/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 10:49:56,421 - Train: 21.94% [893100/4070000] [219.4/1000.0] [batch_t 0.605 (0.629)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 10:50:58,495 - Train: 21.95% [893200/4070000] [219.5/1000.0] [batch_t 0.606 (0.621)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 10:51:59,064 - Train: 21.95% [893300/4070000] [219.5/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 10:52:59,792 - Train: 21.95% [893400/4070000] [219.5/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 10:54:00,357 - Train: 21.95% [893500/4070000] [219.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 10:55:00,952 - Train: 21.96% [893600/4070000] [219.6/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 10:56:01,515 - Train: 21.96% [893700/4070000] [219.6/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 10:57:02,097 - Train: 21.96% [893800/4070000] [219.6/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 10:58:02,662 - Train: 21.96% [893900/4070000] [219.6/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 10:59:03,222 - Train: 21.97% [894000/4070000] [219.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 11:00:03,976 - Train: 21.97% [894100/4070000] [219.7/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 11:01:04,597 - Train: 21.97% [894200/4070000] [219.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 11:02:05,212 - Train: 21.97% [894300/4070000] [219.7/1000.0] [batch_t 0.610 (0.606)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-10 11:03:05,784 - Train: 21.98% [894400/4070000] [219.8/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 11:04:06,342 - Train: 21.98% [894500/4070000] [219.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 11:05:06,905 - Train: 21.98% [894600/4070000] [219.8/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 11:06:07,626 - Train: 21.98% [894700/4070000] [219.8/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 11:07:08,222 - Train: 21.99% [894800/4070000] [219.9/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-10 11:08:08,779 - Train: 21.99% [894900/4070000] [219.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 11:09:09,357 - Train: 21.99% [895000/4070000] [219.9/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 11:10:09,906 - Train: 21.99% [895100/4070000] [219.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 11:11:10,473 - Train: 22.00% [895200/4070000] [220.0/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 11:12:11,015 - Train: 22.00% [895300/4070000] [220.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 11:13:11,735 - Train: 22.00% [895400/4070000] [220.0/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 11:13:11,739 - ==> Total time: 8 days, 10:12:42 Eta: 29 days, 20:55:58 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-10 11:14:14,404 - Train: 22.00% [895500/4070000] [220.0/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 11:15:14,945 - Train: 22.00% [895600/4070000] [220.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 11:16:16,524 - Train: 22.01% [895700/4070000] [220.1/1000.0] [batch_t 0.606 (0.616)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 11:17:17,079 - Train: 22.01% [895800/4070000] [220.1/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 11:18:17,774 - Train: 22.01% [895900/4070000] [220.1/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 11:19:18,365 - Train: 22.01% [896000/4070000] [220.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 11:20:18,975 - Train: 22.02% [896100/4070000] [220.2/1000.0] [batch_t 0.609 (0.606)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-10 11:21:19,562 - Train: 22.02% [896200/4070000] [220.2/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-10 11:22:20,116 - Train: 22.02% [896300/4070000] [220.2/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 11:23:20,659 - Train: 22.02% [896400/4070000] [220.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 11:24:21,322 - Train: 22.03% [896500/4070000] [220.3/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 11:25:21,879 - Train: 22.03% [896600/4070000] [220.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 11:26:22,421 - Train: 22.03% [896700/4070000] [220.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 11:27:22,946 - Train: 22.03% [896800/4070000] [220.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 11:28:23,521 - Train: 22.04% [896900/4070000] [220.4/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 11:29:24,096 - Train: 22.04% [897000/4070000] [220.4/1000.0] [batch_t 0.602 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 11:30:24,767 - Train: 22.04% [897100/4070000] [220.4/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 11:31:25,321 - Train: 22.04% [897200/4070000] [220.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 11:32:25,854 - Train: 22.05% [897300/4070000] [220.5/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-10 11:33:26,423 - Train: 22.05% [897400/4070000] [220.5/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 11:34:26,977 - Train: 22.05% [897500/4070000] [220.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 11:35:27,552 - Train: 22.05% [897600/4070000] [220.5/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-10 11:36:28,135 - Train: 22.06% [897700/4070000] [220.6/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 11:37:28,910 - Train: 22.06% [897800/4070000] [220.6/1000.0] [batch_t 0.605 (0.608)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 11:38:29,475 - Train: 22.06% [897900/4070000] [220.6/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 11:39:30,086 - Train: 22.06% [898000/4070000] [220.6/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 11:40:30,651 - Train: 22.07% [898100/4070000] [220.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 11:41:31,289 - Train: 22.07% [898200/4070000] [220.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 11:42:32,288 - Train: 22.07% [898300/4070000] [220.7/1000.0] [batch_t 0.605 (0.610)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 11:43:33,020 - Train: 22.07% [898400/4070000] [220.7/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 11:44:33,654 - Train: 22.08% [898500/4070000] [220.8/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-10 11:45:34,224 - Train: 22.08% [898600/4070000] [220.8/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 11:46:34,782 - Train: 22.08% [898700/4070000] [220.8/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 11:47:35,339 - Train: 22.08% [898800/4070000] [220.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 11:48:35,936 - Train: 22.09% [898900/4070000] [220.9/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 11:49:39,107 - Train: 22.09% [899000/4070000] [220.9/1000.0] [batch_t 0.603 (0.632)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 11:50:43,463 - Train: 22.09% [899100/4070000] [220.9/1000.0] [batch_t 0.605 (0.643)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 11:51:44,092 - Train: 22.09% [899200/4070000] [220.9/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 11:52:45,197 - Train: 22.10% [899300/4070000] [221.0/1000.0] [batch_t 0.605 (0.611)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 11:53:45,809 - Train: 22.10% [899400/4070000] [221.0/1000.0] [batch_t 0.610 (0.606)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-10 11:54:28,211 - ==> Total time: 8 days, 10:53:59 Eta: 29 days, 19:11:55 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-10 11:54:48,745 - Train: 22.10% [899500/4070000] [221.0/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 11:55:50,304 - Train: 22.10% [899600/4070000] [221.0/1000.0] [batch_t 0.606 (0.615)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 11:56:51,707 - Train: 22.11% [899700/4070000] [221.1/1000.0] [batch_t 0.605 (0.614)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 11:57:52,267 - Train: 22.11% [899800/4070000] [221.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 11:58:52,841 - Train: 22.11% [899900/4070000] [221.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 11:59:53,390 - Train: 22.11% [900000/4070000] [221.1/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 12:00:53,978 - Train: 22.12% [900100/4070000] [221.2/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 12:01:54,540 - Train: 22.12% [900200/4070000] [221.2/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 12:02:55,220 - Train: 22.12% [900300/4070000] [221.2/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 12:03:55,796 - Train: 22.12% [900400/4070000] [221.2/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 12:04:56,355 - Train: 22.13% [900500/4070000] [221.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 12:05:58,190 - Train: 22.13% [900600/4070000] [221.3/1000.0] [batch_t 0.611 (0.618)] [data_t 0.002] [optim_t 0.609] [lr 0.005000] +2024-04-10 12:06:59,035 - Train: 22.13% [900700/4070000] [221.3/1000.0] [batch_t 0.606 (0.608)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 12:07:59,697 - Train: 22.13% [900800/4070000] [221.3/1000.0] [batch_t 0.603 (0.607)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 12:09:00,309 - Train: 22.14% [900900/4070000] [221.4/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 12:10:00,874 - Train: 22.14% [901000/4070000] [221.4/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 12:11:01,446 - Train: 22.14% [901100/4070000] [221.4/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 12:12:02,031 - Train: 22.14% [901200/4070000] [221.4/1000.0] [batch_t 0.607 (0.606)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-10 12:13:02,597 - Train: 22.14% [901300/4070000] [221.4/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 12:14:03,305 - Train: 22.15% [901400/4070000] [221.5/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 12:15:03,895 - Train: 22.15% [901500/4070000] [221.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 12:16:04,521 - Train: 22.15% [901600/4070000] [221.5/1000.0] [batch_t 0.610 (0.606)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-10 12:17:05,131 - Train: 22.15% [901700/4070000] [221.5/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 12:18:05,683 - Train: 22.16% [901800/4070000] [221.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 12:19:06,295 - Train: 22.16% [901900/4070000] [221.6/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 12:20:06,873 - Train: 22.16% [902000/4070000] [221.6/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 12:21:07,428 - Train: 22.16% [902100/4070000] [221.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 12:22:08,032 - Train: 22.17% [902200/4070000] [221.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 12:23:08,567 - Train: 22.17% [902300/4070000] [221.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 12:24:09,124 - Train: 22.17% [902400/4070000] [221.7/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 12:25:09,690 - Train: 22.17% [902500/4070000] [221.7/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 12:26:10,239 - Train: 22.18% [902600/4070000] [221.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 12:27:10,907 - Train: 22.18% [902700/4070000] [221.8/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 12:28:11,462 - Train: 22.18% [902800/4070000] [221.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 12:29:12,052 - Train: 22.18% [902900/4070000] [221.8/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 12:30:12,617 - Train: 22.19% [903000/4070000] [221.9/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-10 12:31:13,177 - Train: 22.19% [903100/4070000] [221.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 12:32:13,775 - Train: 22.19% [903200/4070000] [221.9/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 12:33:14,339 - Train: 22.19% [903300/4070000] [221.9/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 12:34:14,903 - Train: 22.20% [903400/4070000] [222.0/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 12:35:15,603 - Train: 22.20% [903500/4070000] [222.0/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 12:35:39,835 - ==> Total time: 8 days, 11:35:10 Eta: 29 days, 17:28:08 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-10 12:36:18,216 - Train: 22.20% [903600/4070000] [222.0/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 12:37:18,781 - Train: 22.20% [903700/4070000] [222.0/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 12:38:19,348 - Train: 22.21% [903800/4070000] [222.1/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 12:39:19,926 - Train: 22.21% [903900/4070000] [222.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 12:40:20,656 - Train: 22.21% [904000/4070000] [222.1/1000.0] [batch_t 0.603 (0.607)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 12:41:21,219 - Train: 22.21% [904100/4070000] [222.1/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 12:42:21,818 - Train: 22.22% [904200/4070000] [222.2/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 12:43:22,371 - Train: 22.22% [904300/4070000] [222.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 12:44:22,930 - Train: 22.22% [904400/4070000] [222.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 12:45:23,470 - Train: 22.22% [904500/4070000] [222.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 12:46:24,046 - Train: 22.23% [904600/4070000] [222.3/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 12:47:24,734 - Train: 22.23% [904700/4070000] [222.3/1000.0] [batch_t 0.608 (0.607)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-10 12:48:25,358 - Train: 22.23% [904800/4070000] [222.3/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 12:49:25,919 - Train: 22.23% [904900/4070000] [222.3/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 12:50:26,447 - Train: 22.24% [905000/4070000] [222.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 12:51:27,063 - Train: 22.24% [905100/4070000] [222.4/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 12:52:27,749 - Train: 22.24% [905200/4070000] [222.4/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 12:53:28,349 - Train: 22.24% [905300/4070000] [222.4/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-10 12:54:28,926 - Train: 22.25% [905400/4070000] [222.5/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 12:55:29,541 - Train: 22.25% [905500/4070000] [222.5/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 12:56:30,086 - Train: 22.25% [905600/4070000] [222.5/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 12:57:30,640 - Train: 22.25% [905700/4070000] [222.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 12:58:31,224 - Train: 22.26% [905800/4070000] [222.6/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 12:59:31,831 - Train: 22.26% [905900/4070000] [222.6/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 13:00:32,513 - Train: 22.26% [906000/4070000] [222.6/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 13:01:33,103 - Train: 22.26% [906100/4070000] [222.6/1000.0] [batch_t 0.609 (0.606)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-10 13:02:33,653 - Train: 22.27% [906200/4070000] [222.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 13:03:34,231 - Train: 22.27% [906300/4070000] [222.7/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 13:04:34,807 - Train: 22.27% [906400/4070000] [222.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 13:05:35,465 - Train: 22.27% [906500/4070000] [222.7/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 13:06:36,053 - Train: 22.28% [906600/4070000] [222.8/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 13:07:36,584 - Train: 22.28% [906700/4070000] [222.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 13:08:37,149 - Train: 22.28% [906800/4070000] [222.8/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 13:09:37,705 - Train: 22.28% [906900/4070000] [222.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 13:10:38,255 - Train: 22.29% [907000/4070000] [222.9/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 13:11:38,812 - Train: 22.29% [907100/4070000] [222.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 13:12:39,413 - Train: 22.29% [907200/4070000] [222.9/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 13:13:40,081 - Train: 22.29% [907300/4070000] [222.9/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 13:14:40,714 - Train: 22.29% [907400/4070000] [222.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 13:15:41,269 - Train: 22.30% [907500/4070000] [223.0/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 13:16:41,815 - Train: 22.30% [907600/4070000] [223.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 13:16:47,883 - ==> Total time: 8 days, 12:16:18 Eta: 29 days, 15:44:43 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-10 13:17:44,759 - Train: 22.30% [907700/4070000] [223.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 13:18:45,462 - Train: 22.30% [907800/4070000] [223.0/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 13:19:46,017 - Train: 22.31% [907900/4070000] [223.1/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 13:20:46,562 - Train: 22.31% [908000/4070000] [223.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 13:21:47,148 - Train: 22.31% [908100/4070000] [223.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 13:22:47,728 - Train: 22.31% [908200/4070000] [223.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 13:23:48,328 - Train: 22.32% [908300/4070000] [223.2/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 13:24:48,921 - Train: 22.32% [908400/4070000] [223.2/1000.0] [batch_t 0.604 (0.606)] [data_t 0.003] [optim_t 0.601] [lr 0.005000] +2024-04-10 13:25:49,623 - Train: 22.32% [908500/4070000] [223.2/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 13:26:50,257 - Train: 22.32% [908600/4070000] [223.2/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 13:27:50,842 - Train: 22.33% [908700/4070000] [223.3/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 13:28:51,403 - Train: 22.33% [908800/4070000] [223.3/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 13:29:51,969 - Train: 22.33% [908900/4070000] [223.3/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 13:30:52,570 - Train: 22.33% [909000/4070000] [223.3/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 13:31:53,347 - Train: 22.34% [909100/4070000] [223.4/1000.0] [batch_t 0.605 (0.608)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 13:32:53,930 - Train: 22.34% [909200/4070000] [223.4/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 13:33:54,509 - Train: 22.34% [909300/4070000] [223.4/1000.0] [batch_t 0.614 (0.606)] [data_t 0.002] [optim_t 0.612] [lr 0.005000] +2024-04-10 13:34:55,139 - Train: 22.34% [909400/4070000] [223.4/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 13:35:55,738 - Train: 22.35% [909500/4070000] [223.5/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 13:36:56,290 - Train: 22.35% [909600/4070000] [223.5/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 13:37:56,856 - Train: 22.35% [909700/4070000] [223.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 13:38:57,566 - Train: 22.35% [909800/4070000] [223.5/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 13:39:58,124 - Train: 22.36% [909900/4070000] [223.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 13:40:58,679 - Train: 22.36% [910000/4070000] [223.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 13:41:59,292 - Train: 22.36% [910100/4070000] [223.6/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 13:42:59,870 - Train: 22.36% [910200/4070000] [223.6/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 13:44:00,458 - Train: 22.37% [910300/4070000] [223.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 13:45:01,234 - Train: 22.37% [910400/4070000] [223.7/1000.0] [batch_t 0.605 (0.608)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 13:46:01,799 - Train: 22.37% [910500/4070000] [223.7/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 13:47:02,383 - Train: 22.37% [910600/4070000] [223.7/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 13:48:02,968 - Train: 22.38% [910700/4070000] [223.8/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 13:49:03,541 - Train: 22.38% [910800/4070000] [223.8/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 13:50:04,080 - Train: 22.38% [910900/4070000] [223.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 13:51:04,634 - Train: 22.38% [911000/4070000] [223.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 13:52:05,288 - Train: 22.39% [911100/4070000] [223.9/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 13:53:05,863 - Train: 22.39% [911200/4070000] [223.9/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 13:54:06,495 - Train: 22.39% [911300/4070000] [223.9/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 13:55:07,048 - Train: 22.39% [911400/4070000] [223.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 13:56:07,623 - Train: 22.40% [911500/4070000] [224.0/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 13:57:08,200 - Train: 22.40% [911600/4070000] [224.0/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 13:57:56,834 - ==> Total time: 8 days, 12:57:27 Eta: 29 days, 14:01:55 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-10 13:58:11,122 - Train: 22.40% [911700/4070000] [224.0/1000.0] [batch_t 0.604 (0.609)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 13:59:11,683 - Train: 22.40% [911800/4070000] [224.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 14:00:12,282 - Train: 22.41% [911900/4070000] [224.1/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 14:01:12,847 - Train: 22.41% [912000/4070000] [224.1/1000.0] [batch_t 0.609 (0.606)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-10 14:02:13,400 - Train: 22.41% [912100/4070000] [224.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 14:03:13,943 - Train: 22.41% [912200/4070000] [224.1/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 14:04:14,640 - Train: 22.42% [912300/4070000] [224.2/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 14:05:15,211 - Train: 22.42% [912400/4070000] [224.2/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 14:06:15,779 - Train: 22.42% [912500/4070000] [224.2/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 14:07:16,345 - Train: 22.42% [912600/4070000] [224.2/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 14:08:16,967 - Train: 22.43% [912700/4070000] [224.3/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 14:09:17,554 - Train: 22.43% [912800/4070000] [224.3/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 14:10:18,292 - Train: 22.43% [912900/4070000] [224.3/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 14:11:18,853 - Train: 22.43% [913000/4070000] [224.3/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 14:12:19,401 - Train: 22.43% [913100/4070000] [224.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 14:13:19,984 - Train: 22.44% [913200/4070000] [224.4/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 14:14:20,579 - Train: 22.44% [913300/4070000] [224.4/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-10 14:15:21,154 - Train: 22.44% [913400/4070000] [224.4/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 14:16:21,743 - Train: 22.44% [913500/4070000] [224.4/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 14:17:22,460 - Train: 22.45% [913600/4070000] [224.5/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 14:18:23,032 - Train: 22.45% [913700/4070000] [224.5/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 14:19:23,584 - Train: 22.45% [913800/4070000] [224.5/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 14:20:24,148 - Train: 22.45% [913900/4070000] [224.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 14:21:24,694 - Train: 22.46% [914000/4070000] [224.6/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 14:22:25,249 - Train: 22.46% [914100/4070000] [224.6/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 14:23:25,935 - Train: 22.46% [914200/4070000] [224.6/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 14:24:26,482 - Train: 22.46% [914300/4070000] [224.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 14:25:27,061 - Train: 22.47% [914400/4070000] [224.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 14:26:27,611 - Train: 22.47% [914500/4070000] [224.7/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 14:27:28,201 - Train: 22.47% [914600/4070000] [224.7/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 14:28:28,872 - Train: 22.47% [914700/4070000] [224.7/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 14:29:29,428 - Train: 22.48% [914800/4070000] [224.8/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 14:30:29,986 - Train: 22.48% [914900/4070000] [224.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 14:31:30,562 - Train: 22.48% [915000/4070000] [224.8/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 14:32:31,136 - Train: 22.48% [915100/4070000] [224.8/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 14:33:31,697 - Train: 22.49% [915200/4070000] [224.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 14:34:32,242 - Train: 22.49% [915300/4070000] [224.9/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 14:35:32,793 - Train: 22.49% [915400/4070000] [224.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 14:36:33,397 - Train: 22.49% [915500/4070000] [224.9/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 14:37:33,933 - Train: 22.50% [915600/4070000] [225.0/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 14:38:34,467 - Train: 22.50% [915700/4070000] [225.0/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 14:39:04,737 - ==> Total time: 8 days, 13:38:35 Eta: 29 days, 12:19:35 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-10 14:39:38,372 - Train: 22.50% [915800/4070000] [225.0/1000.0] [batch_t 0.606 (0.608)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 14:40:38,935 - Train: 22.50% [915900/4070000] [225.0/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 14:41:39,612 - Train: 22.51% [916000/4070000] [225.1/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 14:42:40,164 - Train: 22.51% [916100/4070000] [225.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 14:43:40,706 - Train: 22.51% [916200/4070000] [225.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 14:44:41,281 - Train: 22.51% [916300/4070000] [225.1/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-10 14:45:41,817 - Train: 22.52% [916400/4070000] [225.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 14:46:42,374 - Train: 22.52% [916500/4070000] [225.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 14:47:42,915 - Train: 22.52% [916600/4070000] [225.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 14:48:43,461 - Train: 22.52% [916700/4070000] [225.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 14:49:44,087 - Train: 22.53% [916800/4070000] [225.3/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 14:50:44,643 - Train: 22.53% [916900/4070000] [225.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 14:51:45,172 - Train: 22.53% [917000/4070000] [225.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 14:52:45,697 - Train: 22.53% [917100/4070000] [225.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 14:53:46,284 - Train: 22.54% [917200/4070000] [225.4/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 14:54:46,930 - Train: 22.54% [917300/4070000] [225.4/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 14:55:47,478 - Train: 22.54% [917400/4070000] [225.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 14:56:48,019 - Train: 22.54% [917500/4070000] [225.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 14:57:48,577 - Train: 22.55% [917600/4070000] [225.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 14:58:49,132 - Train: 22.55% [917700/4070000] [225.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 14:59:49,661 - Train: 22.55% [917800/4070000] [225.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 15:00:50,245 - Train: 22.55% [917900/4070000] [225.5/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 15:01:50,813 - Train: 22.56% [918000/4070000] [225.6/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-10 15:02:51,490 - Train: 22.56% [918100/4070000] [225.6/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 15:03:52,078 - Train: 22.56% [918200/4070000] [225.6/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 15:04:52,648 - Train: 22.56% [918300/4070000] [225.6/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 15:05:53,194 - Train: 22.57% [918400/4070000] [225.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 15:06:53,744 - Train: 22.57% [918500/4070000] [225.7/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 15:07:54,444 - Train: 22.57% [918600/4070000] [225.7/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 15:08:54,991 - Train: 22.57% [918700/4070000] [225.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 15:09:55,531 - Train: 22.57% [918800/4070000] [225.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 15:10:56,098 - Train: 22.58% [918900/4070000] [225.8/1000.0] [batch_t 0.609 (0.606)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-10 15:11:56,655 - Train: 22.58% [919000/4070000] [225.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 15:12:57,230 - Train: 22.58% [919100/4070000] [225.8/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 15:13:57,769 - Train: 22.58% [919200/4070000] [225.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 15:14:58,411 - Train: 22.59% [919300/4070000] [225.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 15:15:58,980 - Train: 22.59% [919400/4070000] [225.9/1000.0] [batch_t 0.609 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-10 15:16:59,553 - Train: 22.59% [919500/4070000] [225.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 15:18:00,150 - Train: 22.59% [919600/4070000] [225.9/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 15:19:00,755 - Train: 22.60% [919700/4070000] [226.0/1000.0] [batch_t 0.609 (0.606)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-10 15:20:01,310 - Train: 22.60% [919800/4070000] [226.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 15:20:13,427 - ==> Total time: 8 days, 14:19:44 Eta: 29 days, 10:37:51 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-10 15:21:04,209 - Train: 22.60% [919900/4070000] [226.0/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 15:22:04,779 - Train: 22.60% [920000/4070000] [226.0/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 15:23:05,350 - Train: 22.61% [920100/4070000] [226.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 15:24:05,894 - Train: 22.61% [920200/4070000] [226.1/1000.0] [batch_t 0.610 (0.605)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-10 15:25:06,432 - Train: 22.61% [920300/4070000] [226.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 15:26:07,178 - Train: 22.61% [920400/4070000] [226.1/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 15:27:07,771 - Train: 22.62% [920500/4070000] [226.2/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 15:28:08,393 - Train: 22.62% [920600/4070000] [226.2/1000.0] [batch_t 0.614 (0.606)] [data_t 0.002] [optim_t 0.612] [lr 0.005000] +2024-04-10 15:29:08,912 - Train: 22.62% [920700/4070000] [226.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 15:30:09,477 - Train: 22.62% [920800/4070000] [226.2/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 15:31:15,288 - Train: 22.63% [920900/4070000] [226.3/1000.0] [batch_t 0.606 (0.658)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 15:32:15,871 - Train: 22.63% [921000/4070000] [226.3/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 15:33:16,556 - Train: 22.63% [921100/4070000] [226.3/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 15:34:17,203 - Train: 22.63% [921200/4070000] [226.3/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 15:35:17,759 - Train: 22.64% [921300/4070000] [226.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 15:36:18,391 - Train: 22.64% [921400/4070000] [226.4/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 15:37:18,960 - Train: 22.64% [921500/4070000] [226.4/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 15:38:19,568 - Train: 22.64% [921600/4070000] [226.4/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 15:39:20,181 - Train: 22.65% [921700/4070000] [226.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 15:40:20,791 - Train: 22.65% [921800/4070000] [226.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 15:41:21,376 - Train: 22.65% [921900/4070000] [226.5/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 15:42:21,968 - Train: 22.65% [922000/4070000] [226.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 15:43:22,541 - Train: 22.66% [922100/4070000] [226.6/1000.0] [batch_t 0.610 (0.606)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-10 15:44:23,177 - Train: 22.66% [922200/4070000] [226.6/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 15:45:23,748 - Train: 22.66% [922300/4070000] [226.6/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 15:46:24,460 - Train: 22.66% [922400/4070000] [226.6/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 15:47:25,052 - Train: 22.67% [922500/4070000] [226.7/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 15:48:25,612 - Train: 22.67% [922600/4070000] [226.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 15:49:26,187 - Train: 22.67% [922700/4070000] [226.7/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 15:50:26,781 - Train: 22.67% [922800/4070000] [226.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 15:51:27,327 - Train: 22.68% [922900/4070000] [226.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 15:52:28,042 - Train: 22.68% [923000/4070000] [226.8/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 15:53:28,590 - Train: 22.68% [923100/4070000] [226.8/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 15:54:29,439 - Train: 22.68% [923200/4070000] [226.8/1000.0] [batch_t 0.605 (0.608)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 15:55:29,969 - Train: 22.69% [923300/4070000] [226.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 15:56:30,509 - Train: 22.69% [923400/4070000] [226.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 15:57:31,048 - Train: 22.69% [923500/4070000] [226.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 15:58:31,755 - Train: 22.69% [923600/4070000] [226.9/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 15:59:32,302 - Train: 22.70% [923700/4070000] [227.0/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 16:00:32,857 - Train: 22.70% [923800/4070000] [227.0/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 16:01:27,359 - ==> Total time: 8 days, 15:00:58 Eta: 29 days, 8:56:57 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-10 16:01:35,840 - Train: 22.70% [923900/4070000] [227.0/1000.0] [batch_t 0.605 (0.638)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 16:02:36,375 - Train: 22.70% [924000/4070000] [227.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 16:03:36,932 - Train: 22.71% [924100/4070000] [227.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 16:04:45,073 - Train: 22.71% [924200/4070000] [227.1/1000.0] [batch_t 0.603 (0.681)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 16:05:53,381 - Train: 22.71% [924300/4070000] [227.1/1000.0] [batch_t 3.905 (0.683)] [data_t 3.300] [optim_t 0.605] [lr 0.005000] +2024-04-10 16:07:10,070 - Train: 22.71% [924400/4070000] [227.1/1000.0] [batch_t 2.499 (0.767)] [data_t 1.892] [optim_t 0.607] [lr 0.005000] +2024-04-10 16:08:51,605 - Train: 22.71% [924500/4070000] [227.1/1000.0] [batch_t 0.605 (1.015)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 16:11:45,594 - Train: 22.72% [924600/4070000] [227.2/1000.0] [batch_t 0.606 (1.740)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 16:12:51,306 - Train: 22.72% [924700/4070000] [227.2/1000.0] [batch_t 0.605 (0.657)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 16:15:37,650 - Train: 22.72% [924800/4070000] [227.2/1000.0] [batch_t 0.607 (1.663)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 16:16:44,110 - Train: 22.72% [924900/4070000] [227.2/1000.0] [batch_t 3.039 (0.664)] [data_t 2.433] [optim_t 0.606] [lr 0.005000] +2024-04-10 16:18:14,208 - Train: 22.73% [925000/4070000] [227.3/1000.0] [batch_t 0.605 (0.901)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 16:19:21,908 - Train: 22.73% [925100/4070000] [227.3/1000.0] [batch_t 0.605 (0.677)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 16:20:25,137 - Train: 22.73% [925200/4070000] [227.3/1000.0] [batch_t 0.606 (0.632)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 16:21:27,471 - Train: 22.73% [925300/4070000] [227.3/1000.0] [batch_t 0.605 (0.623)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 16:22:53,313 - Train: 22.74% [925400/4070000] [227.4/1000.0] [batch_t 0.604 (0.858)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 16:23:54,156 - Train: 22.74% [925500/4070000] [227.4/1000.0] [batch_t 0.606 (0.608)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 16:26:19,464 - Train: 22.74% [925600/4070000] [227.4/1000.0] [batch_t 0.637 (1.453)] [data_t 0.033] [optim_t 0.604] [lr 0.005000] +2024-04-10 16:28:43,223 - Train: 22.74% [925700/4070000] [227.4/1000.0] [batch_t 0.605 (1.437)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 16:29:43,812 - Train: 22.75% [925800/4070000] [227.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 16:30:44,789 - Train: 22.75% [925900/4070000] [227.5/1000.0] [batch_t 0.606 (0.610)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 16:31:46,418 - Train: 22.75% [926000/4070000] [227.5/1000.0] [batch_t 0.607 (0.616)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 16:32:49,141 - Train: 22.75% [926100/4070000] [227.5/1000.0] [batch_t 0.606 (0.627)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 16:33:49,702 - Train: 22.76% [926200/4070000] [227.6/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 16:34:50,877 - Train: 22.76% [926300/4070000] [227.6/1000.0] [batch_t 0.605 (0.612)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 16:35:51,729 - Train: 22.76% [926400/4070000] [227.6/1000.0] [batch_t 0.606 (0.608)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 16:36:53,382 - Train: 22.76% [926500/4070000] [227.6/1000.0] [batch_t 0.606 (0.616)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 16:37:54,769 - Train: 22.77% [926600/4070000] [227.7/1000.0] [batch_t 0.605 (0.614)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 16:38:55,787 - Train: 22.77% [926700/4070000] [227.7/1000.0] [batch_t 0.606 (0.610)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 16:39:56,542 - Train: 22.77% [926800/4070000] [227.7/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 16:40:57,778 - Train: 22.77% [926900/4070000] [227.7/1000.0] [batch_t 0.606 (0.612)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 16:41:58,908 - Train: 22.78% [927000/4070000] [227.8/1000.0] [batch_t 0.606 (0.611)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 16:43:00,837 - Train: 22.78% [927100/4070000] [227.8/1000.0] [batch_t 0.605 (0.619)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 16:44:01,613 - Train: 22.78% [927200/4070000] [227.8/1000.0] [batch_t 0.605 (0.608)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 16:45:02,218 - Train: 22.78% [927300/4070000] [227.8/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 16:46:03,077 - Train: 22.79% [927400/4070000] [227.9/1000.0] [batch_t 0.605 (0.608)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 16:47:03,694 - Train: 22.79% [927500/4070000] [227.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 16:48:04,248 - Train: 22.79% [927600/4070000] [227.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 16:49:04,770 - Train: 22.79% [927700/4070000] [227.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 16:50:05,300 - Train: 22.80% [927800/4070000] [228.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 16:51:05,821 - Train: 22.80% [927900/4070000] [228.0/1000.0] [batch_t 0.600 (0.605)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-10 16:51:42,258 - ==> Total time: 8 days, 15:51:13 Eta: 29 days, 7:47:06 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-10 16:52:08,476 - Train: 22.80% [928000/4070000] [228.0/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 16:53:09,007 - Train: 22.80% [928100/4070000] [228.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 16:54:09,584 - Train: 22.81% [928200/4070000] [228.1/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 16:55:10,133 - Train: 22.81% [928300/4070000] [228.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 16:56:10,708 - Train: 22.81% [928400/4070000] [228.1/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 16:57:11,294 - Train: 22.81% [928500/4070000] [228.1/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 16:58:11,957 - Train: 22.82% [928600/4070000] [228.2/1000.0] [batch_t 0.610 (0.607)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-10 16:59:12,523 - Train: 22.82% [928700/4070000] [228.2/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 17:00:13,073 - Train: 22.82% [928800/4070000] [228.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 17:01:13,625 - Train: 22.82% [928900/4070000] [228.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 17:02:14,178 - Train: 22.83% [929000/4070000] [228.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 17:03:14,792 - Train: 22.83% [929100/4070000] [228.3/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 17:04:15,332 - Train: 22.83% [929200/4070000] [228.3/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 17:05:15,886 - Train: 22.83% [929300/4070000] [228.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 17:06:16,418 - Train: 22.84% [929400/4070000] [228.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 17:07:16,977 - Train: 22.84% [929500/4070000] [228.4/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 17:08:17,544 - Train: 22.84% [929600/4070000] [228.4/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 17:09:18,072 - Train: 22.84% [929700/4070000] [228.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 17:10:18,651 - Train: 22.85% [929800/4070000] [228.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 17:11:19,261 - Train: 22.85% [929900/4070000] [228.5/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 17:12:19,811 - Train: 22.85% [930000/4070000] [228.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 17:13:20,422 - Train: 22.85% [930100/4070000] [228.5/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 17:14:21,009 - Train: 22.86% [930200/4070000] [228.6/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 17:15:21,588 - Train: 22.86% [930300/4070000] [228.6/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 17:16:22,236 - Train: 22.86% [930400/4070000] [228.6/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 17:17:22,804 - Train: 22.86% [930500/4070000] [228.6/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 17:18:23,350 - Train: 22.86% [930600/4070000] [228.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 17:19:23,914 - Train: 22.87% [930700/4070000] [228.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 17:20:24,474 - Train: 22.87% [930800/4070000] [228.7/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 17:21:25,201 - Train: 22.87% [930900/4070000] [228.7/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 17:22:33,914 - Train: 22.87% [931000/4070000] [228.7/1000.0] [batch_t 0.606 (0.687)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 17:23:34,561 - Train: 22.88% [931100/4070000] [228.8/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 17:24:35,110 - Train: 22.88% [931200/4070000] [228.8/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 17:25:35,665 - Train: 22.88% [931300/4070000] [228.8/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 17:26:36,230 - Train: 22.88% [931400/4070000] [228.8/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 17:27:38,830 - Train: 22.89% [931500/4070000] [228.9/1000.0] [batch_t 0.607 (0.626)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 17:28:39,366 - Train: 22.89% [931600/4070000] [228.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 17:29:39,899 - Train: 22.89% [931700/4070000] [228.9/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-10 17:30:40,448 - Train: 22.89% [931800/4070000] [228.9/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 17:31:41,084 - Train: 22.90% [931900/4070000] [229.0/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 17:32:41,681 - Train: 22.90% [932000/4070000] [229.0/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 17:32:59,846 - ==> Total time: 8 days, 16:32:30 Eta: 29 days, 6:07:16 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-10 17:33:48,497 - Train: 22.90% [932100/4070000] [229.0/1000.0] [batch_t 0.606 (0.663)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 17:34:49,121 - Train: 22.90% [932200/4070000] [229.0/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 17:35:49,683 - Train: 22.91% [932300/4070000] [229.1/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 17:36:50,270 - Train: 22.91% [932400/4070000] [229.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 17:37:50,936 - Train: 22.91% [932500/4070000] [229.1/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 17:38:51,477 - Train: 22.91% [932600/4070000] [229.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 17:39:52,040 - Train: 22.92% [932700/4070000] [229.2/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 17:40:52,603 - Train: 22.92% [932800/4070000] [229.2/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 17:41:53,132 - Train: 22.92% [932900/4070000] [229.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 17:42:53,797 - Train: 22.92% [933000/4070000] [229.2/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 17:43:54,376 - Train: 22.93% [933100/4070000] [229.3/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 17:44:54,919 - Train: 22.93% [933200/4070000] [229.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 17:45:55,485 - Train: 22.93% [933300/4070000] [229.3/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 17:46:56,069 - Train: 22.93% [933400/4070000] [229.3/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 17:47:56,648 - Train: 22.94% [933500/4070000] [229.4/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 17:48:57,202 - Train: 22.94% [933600/4070000] [229.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 17:49:57,884 - Train: 22.94% [933700/4070000] [229.4/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 17:50:58,481 - Train: 22.94% [933800/4070000] [229.4/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 17:51:59,021 - Train: 22.95% [933900/4070000] [229.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 17:53:00,051 - Train: 22.95% [934000/4070000] [229.5/1000.0] [batch_t 0.607 (0.610)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 17:54:00,638 - Train: 22.95% [934100/4070000] [229.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 17:55:01,882 - Train: 22.95% [934200/4070000] [229.5/1000.0] [batch_t 0.606 (0.612)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 17:56:02,402 - Train: 22.96% [934300/4070000] [229.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 17:57:03,060 - Train: 22.96% [934400/4070000] [229.6/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 17:58:03,616 - Train: 22.96% [934500/4070000] [229.6/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 17:59:04,148 - Train: 22.96% [934600/4070000] [229.6/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 18:00:04,699 - Train: 22.97% [934700/4070000] [229.7/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-10 18:01:05,296 - Train: 22.97% [934800/4070000] [229.7/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-10 18:02:05,831 - Train: 22.97% [934900/4070000] [229.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 18:03:06,492 - Train: 22.97% [935000/4070000] [229.7/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 18:04:07,045 - Train: 22.98% [935100/4070000] [229.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 18:05:07,636 - Train: 22.98% [935200/4070000] [229.8/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 18:06:08,758 - Train: 22.98% [935300/4070000] [229.8/1000.0] [batch_t 0.603 (0.611)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 18:07:09,315 - Train: 22.98% [935400/4070000] [229.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 18:08:09,965 - Train: 22.99% [935500/4070000] [229.9/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 18:09:10,535 - Train: 22.99% [935600/4070000] [229.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 18:10:11,082 - Train: 22.99% [935700/4070000] [229.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 18:11:11,620 - Train: 22.99% [935800/4070000] [229.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 18:12:12,168 - Train: 23.00% [935900/4070000] [230.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 18:13:12,709 - Train: 23.00% [936000/4070000] [230.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 18:14:13,330 - Train: 23.00% [936100/4070000] [230.0/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 18:14:13,334 - ==> Total time: 8 days, 17:13:44 Eta: 29 days, 4:27:43 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-10 18:15:16,395 - Train: 23.00% [936200/4070000] [230.0/1000.0] [batch_t 0.607 (0.610)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 18:16:16,923 - Train: 23.00% [936300/4070000] [230.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 18:17:17,450 - Train: 23.01% [936400/4070000] [230.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 18:18:18,011 - Train: 23.01% [936500/4070000] [230.1/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 18:19:18,532 - Train: 23.01% [936600/4070000] [230.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 18:20:19,182 - Train: 23.01% [936700/4070000] [230.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 18:21:19,744 - Train: 23.02% [936800/4070000] [230.2/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 18:22:20,358 - Train: 23.02% [936900/4070000] [230.2/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 18:23:20,885 - Train: 23.02% [937000/4070000] [230.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 18:24:21,442 - Train: 23.02% [937100/4070000] [230.2/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 18:25:21,999 - Train: 23.03% [937200/4070000] [230.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 18:26:22,534 - Train: 23.03% [937300/4070000] [230.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 18:27:23,067 - Train: 23.03% [937400/4070000] [230.3/1000.0] [batch_t 0.610 (0.605)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-10 18:28:23,714 - Train: 23.03% [937500/4070000] [230.3/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 18:29:24,267 - Train: 23.04% [937600/4070000] [230.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 18:30:24,793 - Train: 23.04% [937700/4070000] [230.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 18:31:25,326 - Train: 23.04% [937800/4070000] [230.4/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 18:32:25,885 - Train: 23.04% [937900/4070000] [230.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 18:33:26,530 - Train: 23.05% [938000/4070000] [230.5/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 18:34:27,098 - Train: 23.05% [938100/4070000] [230.5/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 18:35:27,621 - Train: 23.05% [938200/4070000] [230.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 18:36:28,149 - Train: 23.05% [938300/4070000] [230.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 18:37:28,728 - Train: 23.06% [938400/4070000] [230.6/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 18:38:29,282 - Train: 23.06% [938500/4070000] [230.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 18:39:29,815 - Train: 23.06% [938600/4070000] [230.6/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 18:40:30,333 - Train: 23.06% [938700/4070000] [230.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 18:41:30,952 - Train: 23.07% [938800/4070000] [230.7/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 18:42:31,490 - Train: 23.07% [938900/4070000] [230.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 18:43:32,030 - Train: 23.07% [939000/4070000] [230.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 18:44:32,568 - Train: 23.07% [939100/4070000] [230.7/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 18:45:33,125 - Train: 23.08% [939200/4070000] [230.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 18:46:33,884 - Train: 23.08% [939300/4070000] [230.8/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 18:47:34,444 - Train: 23.08% [939400/4070000] [230.8/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 18:48:35,049 - Train: 23.08% [939500/4070000] [230.8/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-10 18:49:35,617 - Train: 23.09% [939600/4070000] [230.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 18:50:36,268 - Train: 23.09% [939700/4070000] [230.9/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 18:51:36,869 - Train: 23.09% [939800/4070000] [230.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 18:52:37,430 - Train: 23.09% [939900/4070000] [230.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 18:53:38,029 - Train: 23.10% [940000/4070000] [231.0/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 18:54:38,761 - Train: 23.10% [940100/4070000] [231.0/1000.0] [batch_t 0.608 (0.607)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-10 18:55:21,146 - ==> Total time: 8 days, 17:54:52 Eta: 29 days, 2:48:21 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-10 18:55:41,325 - Train: 23.10% [940200/4070000] [231.0/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 18:56:41,840 - Train: 23.10% [940300/4070000] [231.0/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 18:57:42,388 - Train: 23.11% [940400/4070000] [231.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 18:58:42,943 - Train: 23.11% [940500/4070000] [231.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 18:59:43,480 - Train: 23.11% [940600/4070000] [231.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 19:00:44,157 - Train: 23.11% [940700/4070000] [231.1/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 19:01:44,744 - Train: 23.12% [940800/4070000] [231.2/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-10 19:02:45,313 - Train: 23.12% [940900/4070000] [231.2/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 19:03:45,863 - Train: 23.12% [941000/4070000] [231.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 19:04:46,537 - Train: 23.12% [941100/4070000] [231.2/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 19:05:47,135 - Train: 23.13% [941200/4070000] [231.3/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 19:06:47,888 - Train: 23.13% [941300/4070000] [231.3/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 19:07:48,478 - Train: 23.13% [941400/4070000] [231.3/1000.0] [batch_t 0.610 (0.606)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-10 19:08:49,053 - Train: 23.13% [941500/4070000] [231.3/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 19:09:49,614 - Train: 23.14% [941600/4070000] [231.4/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 19:10:50,173 - Train: 23.14% [941700/4070000] [231.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 19:11:50,804 - Train: 23.14% [941800/4070000] [231.4/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 19:12:51,417 - Train: 23.14% [941900/4070000] [231.4/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 19:13:52,147 - Train: 23.14% [942000/4070000] [231.4/1000.0] [batch_t 0.608 (0.607)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-10 19:14:52,739 - Train: 23.15% [942100/4070000] [231.5/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 19:15:53,320 - Train: 23.15% [942200/4070000] [231.5/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 19:16:53,918 - Train: 23.15% [942300/4070000] [231.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 19:17:54,527 - Train: 23.15% [942400/4070000] [231.5/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 19:18:55,143 - Train: 23.16% [942500/4070000] [231.6/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 19:19:56,020 - Train: 23.16% [942600/4070000] [231.6/1000.0] [batch_t 0.606 (0.609)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 19:20:56,601 - Train: 23.16% [942700/4070000] [231.6/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 19:21:57,175 - Train: 23.16% [942800/4070000] [231.6/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 19:22:57,766 - Train: 23.17% [942900/4070000] [231.7/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 19:23:58,365 - Train: 23.17% [943000/4070000] [231.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 19:24:58,933 - Train: 23.17% [943100/4070000] [231.7/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 19:25:59,514 - Train: 23.17% [943200/4070000] [231.7/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 19:27:00,265 - Train: 23.18% [943300/4070000] [231.8/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 19:28:00,910 - Train: 23.18% [943400/4070000] [231.8/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-10 19:29:01,512 - Train: 23.18% [943500/4070000] [231.8/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 19:30:02,149 - Train: 23.18% [943600/4070000] [231.8/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 19:31:02,783 - Train: 23.19% [943700/4070000] [231.9/1000.0] [batch_t 0.610 (0.606)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-10 19:32:03,361 - Train: 23.19% [943800/4070000] [231.9/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 19:33:03,941 - Train: 23.19% [943900/4070000] [231.9/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 19:34:04,542 - Train: 23.19% [944000/4070000] [231.9/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 19:35:05,298 - Train: 23.20% [944100/4070000] [232.0/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 19:36:05,857 - Train: 23.20% [944200/4070000] [232.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 19:36:30,076 - ==> Total time: 8 days, 18:36:00 Eta: 29 days, 1:09:34 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-10 19:37:08,640 - Train: 23.20% [944300/4070000] [232.0/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 19:38:09,208 - Train: 23.20% [944400/4070000] [232.0/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 19:39:09,801 - Train: 23.21% [944500/4070000] [232.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 19:40:10,406 - Train: 23.21% [944600/4070000] [232.1/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 19:41:11,097 - Train: 23.21% [944700/4070000] [232.1/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 19:42:11,659 - Train: 23.21% [944800/4070000] [232.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 19:43:12,203 - Train: 23.22% [944900/4070000] [232.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 19:44:12,963 - Train: 23.22% [945000/4070000] [232.2/1000.0] [batch_t 0.605 (0.608)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 19:45:13,542 - Train: 23.22% [945100/4070000] [232.2/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 19:46:14,077 - Train: 23.22% [945200/4070000] [232.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 19:47:14,807 - Train: 23.23% [945300/4070000] [232.3/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 19:48:15,378 - Train: 23.23% [945400/4070000] [232.3/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 19:49:15,945 - Train: 23.23% [945500/4070000] [232.3/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 19:50:16,731 - Train: 23.23% [945600/4070000] [232.3/1000.0] [batch_t 0.606 (0.608)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 19:51:17,277 - Train: 23.24% [945700/4070000] [232.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 19:52:17,789 - Train: 23.24% [945800/4070000] [232.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 19:53:18,332 - Train: 23.24% [945900/4070000] [232.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 19:54:19,024 - Train: 23.24% [946000/4070000] [232.4/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 19:55:19,593 - Train: 23.25% [946100/4070000] [232.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 19:56:20,164 - Train: 23.25% [946200/4070000] [232.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 19:57:20,720 - Train: 23.25% [946300/4070000] [232.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 19:58:21,297 - Train: 23.25% [946400/4070000] [232.5/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 19:59:22,019 - Train: 23.26% [946500/4070000] [232.6/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 20:00:22,590 - Train: 23.26% [946600/4070000] [232.6/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 20:01:23,153 - Train: 23.26% [946700/4070000] [232.6/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 20:02:23,674 - Train: 23.26% [946800/4070000] [232.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 20:03:24,261 - Train: 23.27% [946900/4070000] [232.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 20:04:24,838 - Train: 23.27% [947000/4070000] [232.7/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 20:05:25,375 - Train: 23.27% [947100/4070000] [232.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 20:06:25,917 - Train: 23.27% [947200/4070000] [232.7/1000.0] [batch_t 0.612 (0.605)] [data_t 0.002] [optim_t 0.610] [lr 0.005000] +2024-04-10 20:07:26,640 - Train: 23.28% [947300/4070000] [232.8/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 20:08:27,235 - Train: 23.28% [947400/4070000] [232.8/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 20:09:27,813 - Train: 23.28% [947500/4070000] [232.8/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 20:10:28,445 - Train: 23.28% [947600/4070000] [232.8/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 20:11:29,012 - Train: 23.29% [947700/4070000] [232.9/1000.0] [batch_t 0.610 (0.606)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-10 20:12:29,733 - Train: 23.29% [947800/4070000] [232.9/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 20:13:30,283 - Train: 23.29% [947900/4070000] [232.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 20:14:30,844 - Train: 23.29% [948000/4070000] [232.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 20:15:31,386 - Train: 23.29% [948100/4070000] [232.9/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 20:16:31,946 - Train: 23.30% [948200/4070000] [233.0/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 20:17:32,490 - Train: 23.30% [948300/4070000] [233.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 20:17:38,549 - ==> Total time: 8 days, 19:17:09 Eta: 28 days, 23:31:14 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-10 20:18:35,293 - Train: 23.30% [948400/4070000] [233.0/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 20:19:35,978 - Train: 23.30% [948500/4070000] [233.0/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 20:20:36,541 - Train: 23.31% [948600/4070000] [233.1/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 20:21:37,076 - Train: 23.31% [948700/4070000] [233.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 20:22:37,622 - Train: 23.31% [948800/4070000] [233.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 20:23:38,149 - Train: 23.31% [948900/4070000] [233.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 20:24:38,697 - Train: 23.32% [949000/4070000] [233.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 20:25:39,352 - Train: 23.32% [949100/4070000] [233.2/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-10 20:26:39,896 - Train: 23.32% [949200/4070000] [233.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 20:27:40,463 - Train: 23.32% [949300/4070000] [233.2/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 20:28:41,012 - Train: 23.33% [949400/4070000] [233.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 20:29:41,907 - Train: 23.33% [949500/4070000] [233.3/1000.0] [batch_t 0.607 (0.609)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 20:30:42,481 - Train: 23.33% [949600/4070000] [233.3/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 20:31:43,009 - Train: 23.33% [949700/4070000] [233.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 20:32:43,622 - Train: 23.34% [949800/4070000] [233.4/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 20:33:44,191 - Train: 23.34% [949900/4070000] [233.4/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-10 20:34:44,771 - Train: 23.34% [950000/4070000] [233.4/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-10 20:35:45,393 - Train: 23.34% [950100/4070000] [233.4/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 20:36:45,955 - Train: 23.35% [950200/4070000] [233.5/1000.0] [batch_t 0.611 (0.606)] [data_t 0.002] [optim_t 0.609] [lr 0.005000] +2024-04-10 20:37:46,707 - Train: 23.35% [950300/4070000] [233.5/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 20:38:47,803 - Train: 23.35% [950400/4070000] [233.5/1000.0] [batch_t 0.606 (0.611)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 20:39:48,347 - Train: 23.35% [950500/4070000] [233.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 20:40:48,901 - Train: 23.36% [950600/4070000] [233.6/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 20:41:49,492 - Train: 23.36% [950700/4070000] [233.6/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 20:42:50,065 - Train: 23.36% [950800/4070000] [233.6/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 20:43:50,616 - Train: 23.36% [950900/4070000] [233.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 20:44:51,160 - Train: 23.37% [951000/4070000] [233.7/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 20:45:51,813 - Train: 23.37% [951100/4070000] [233.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 20:46:52,401 - Train: 23.37% [951200/4070000] [233.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 20:47:52,967 - Train: 23.37% [951300/4070000] [233.7/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 20:48:53,534 - Train: 23.38% [951400/4070000] [233.8/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 20:49:54,130 - Train: 23.38% [951500/4070000] [233.8/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 20:50:54,661 - Train: 23.38% [951600/4070000] [233.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 20:51:55,298 - Train: 23.38% [951700/4070000] [233.8/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 20:52:56,701 - Train: 23.39% [951800/4070000] [233.9/1000.0] [batch_t 0.605 (0.614)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 20:53:57,237 - Train: 23.39% [951900/4070000] [233.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 20:54:57,777 - Train: 23.39% [952000/4070000] [233.9/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 20:55:58,304 - Train: 23.39% [952100/4070000] [233.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 20:56:59,003 - Train: 23.40% [952200/4070000] [234.0/1000.0] [batch_t 0.609 (0.607)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-10 20:57:59,551 - Train: 23.40% [952300/4070000] [234.0/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 20:58:47,977 - ==> Total time: 8 days, 19:58:18 Eta: 28 days, 21:53:27 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-10 20:59:03,411 - Train: 23.40% [952400/4070000] [234.0/1000.0] [batch_t 0.606 (0.617)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 21:00:04,042 - Train: 23.40% [952500/4070000] [234.0/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 21:01:04,640 - Train: 23.41% [952600/4070000] [234.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 21:02:05,191 - Train: 23.41% [952700/4070000] [234.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 21:03:05,887 - Train: 23.41% [952800/4070000] [234.1/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 21:04:06,441 - Train: 23.41% [952900/4070000] [234.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 21:05:06,996 - Train: 23.42% [953000/4070000] [234.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 21:06:07,588 - Train: 23.42% [953100/4070000] [234.2/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 21:07:08,150 - Train: 23.42% [953200/4070000] [234.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 21:08:08,728 - Train: 23.42% [953300/4070000] [234.2/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 21:09:09,268 - Train: 23.43% [953400/4070000] [234.3/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 21:10:09,818 - Train: 23.43% [953500/4070000] [234.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 21:11:10,477 - Train: 23.43% [953600/4070000] [234.3/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 21:12:11,060 - Train: 23.43% [953700/4070000] [234.3/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 21:13:11,655 - Train: 23.43% [953800/4070000] [234.3/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 21:14:12,231 - Train: 23.44% [953900/4070000] [234.4/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 21:15:12,828 - Train: 23.44% [954000/4070000] [234.4/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 21:16:13,400 - Train: 23.44% [954100/4070000] [234.4/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 21:17:14,068 - Train: 23.44% [954200/4070000] [234.4/1000.0] [batch_t 0.603 (0.607)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 21:18:14,708 - Train: 23.45% [954300/4070000] [234.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 21:19:15,282 - Train: 23.45% [954400/4070000] [234.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 21:20:15,970 - Train: 23.45% [954500/4070000] [234.5/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 21:21:16,567 - Train: 23.45% [954600/4070000] [234.5/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 21:22:17,219 - Train: 23.46% [954700/4070000] [234.6/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 21:23:17,814 - Train: 23.46% [954800/4070000] [234.6/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 21:24:18,399 - Train: 23.46% [954900/4070000] [234.6/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 21:25:18,976 - Train: 23.46% [955000/4070000] [234.6/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 21:26:19,540 - Train: 23.47% [955100/4070000] [234.7/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 21:27:20,081 - Train: 23.47% [955200/4070000] [234.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 21:28:20,630 - Train: 23.47% [955300/4070000] [234.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 21:29:21,171 - Train: 23.47% [955400/4070000] [234.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 21:30:21,903 - Train: 23.48% [955500/4070000] [234.8/1000.0] [batch_t 0.612 (0.607)] [data_t 0.003] [optim_t 0.609] [lr 0.005000] +2024-04-10 21:31:22,452 - Train: 23.48% [955600/4070000] [234.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 21:32:23,075 - Train: 23.48% [955700/4070000] [234.8/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 21:33:23,615 - Train: 23.48% [955800/4070000] [234.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 21:34:24,182 - Train: 23.49% [955900/4070000] [234.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 21:35:24,924 - Train: 23.49% [956000/4070000] [234.9/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 21:36:25,471 - Train: 23.49% [956100/4070000] [234.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 21:37:26,850 - Train: 23.49% [956200/4070000] [234.9/1000.0] [batch_t 0.606 (0.614)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 21:38:27,420 - Train: 23.50% [956300/4070000] [235.0/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 21:39:27,976 - Train: 23.50% [956400/4070000] [235.0/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 21:39:58,263 - ==> Total time: 8 days, 20:39:29 Eta: 28 days, 20:16:11 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-10 21:40:31,525 - Train: 23.50% [956500/4070000] [235.0/1000.0] [batch_t 0.604 (0.610)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 21:41:32,246 - Train: 23.50% [956600/4070000] [235.0/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 21:42:32,842 - Train: 23.51% [956700/4070000] [235.1/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 21:43:33,393 - Train: 23.51% [956800/4070000] [235.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 21:44:33,946 - Train: 23.51% [956900/4070000] [235.1/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 21:45:34,496 - Train: 23.51% [957000/4070000] [235.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 21:46:35,038 - Train: 23.52% [957100/4070000] [235.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 21:47:35,600 - Train: 23.52% [957200/4070000] [235.2/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 21:48:36,265 - Train: 23.52% [957300/4070000] [235.2/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 21:49:36,817 - Train: 23.52% [957400/4070000] [235.2/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 21:50:37,364 - Train: 23.53% [957500/4070000] [235.3/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 21:51:37,903 - Train: 23.53% [957600/4070000] [235.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 21:52:38,461 - Train: 23.53% [957700/4070000] [235.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 21:53:39,015 - Train: 23.53% [957800/4070000] [235.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 21:54:39,570 - Train: 23.54% [957900/4070000] [235.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 21:55:40,139 - Train: 23.54% [958000/4070000] [235.4/1000.0] [batch_t 0.609 (0.606)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-10 21:56:40,763 - Train: 23.54% [958100/4070000] [235.4/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-10 21:57:41,357 - Train: 23.54% [958200/4070000] [235.4/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 21:58:41,945 - Train: 23.55% [958300/4070000] [235.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 21:59:42,533 - Train: 23.55% [958400/4070000] [235.5/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 22:00:43,151 - Train: 23.55% [958500/4070000] [235.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 22:01:43,872 - Train: 23.55% [958600/4070000] [235.5/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 22:02:44,454 - Train: 23.56% [958700/4070000] [235.6/1000.0] [batch_t 0.610 (0.606)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-10 22:03:45,066 - Train: 23.56% [958800/4070000] [235.6/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 22:04:45,648 - Train: 23.56% [958900/4070000] [235.6/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 22:05:46,259 - Train: 23.56% [959000/4070000] [235.6/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 22:06:46,827 - Train: 23.57% [959100/4070000] [235.7/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-10 22:07:47,474 - Train: 23.57% [959200/4070000] [235.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 22:08:48,206 - Train: 23.57% [959300/4070000] [235.7/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 22:09:48,856 - Train: 23.57% [959400/4070000] [235.7/1000.0] [batch_t 0.609 (0.606)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-10 22:10:49,503 - Train: 23.57% [959500/4070000] [235.7/1000.0] [batch_t 0.602 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 22:11:50,048 - Train: 23.58% [959600/4070000] [235.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 22:12:50,671 - Train: 23.58% [959700/4070000] [235.8/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 22:13:51,295 - Train: 23.58% [959800/4070000] [235.8/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 22:14:51,869 - Train: 23.58% [959900/4070000] [235.8/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 22:15:52,619 - Train: 23.59% [960000/4070000] [235.9/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 22:16:53,202 - Train: 23.59% [960100/4070000] [235.9/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 22:17:53,801 - Train: 23.59% [960200/4070000] [235.9/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 22:18:54,395 - Train: 23.59% [960300/4070000] [235.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 22:19:54,992 - Train: 23.60% [960400/4070000] [236.0/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 22:20:55,606 - Train: 23.60% [960500/4070000] [236.0/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 22:21:07,763 - ==> Total time: 8 days, 21:20:38 Eta: 28 days, 18:39:22 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-10 22:21:58,548 - Train: 23.60% [960600/4070000] [236.0/1000.0] [batch_t 0.606 (0.608)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 22:22:59,323 - Train: 23.60% [960700/4070000] [236.0/1000.0] [batch_t 0.605 (0.608)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 22:23:59,867 - Train: 23.61% [960800/4070000] [236.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 22:25:00,459 - Train: 23.61% [960900/4070000] [236.1/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-10 22:26:01,030 - Train: 23.61% [961000/4070000] [236.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 22:27:01,646 - Train: 23.61% [961100/4070000] [236.1/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 22:28:02,215 - Train: 23.62% [961200/4070000] [236.2/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 22:29:02,779 - Train: 23.62% [961300/4070000] [236.2/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 22:30:03,527 - Train: 23.62% [961400/4070000] [236.2/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 22:31:04,065 - Train: 23.62% [961500/4070000] [236.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 22:32:04,634 - Train: 23.63% [961600/4070000] [236.3/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 22:33:05,161 - Train: 23.63% [961700/4070000] [236.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 22:34:05,742 - Train: 23.63% [961800/4070000] [236.3/1000.0] [batch_t 0.609 (0.606)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-10 22:35:06,294 - Train: 23.63% [961900/4070000] [236.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 22:36:06,862 - Train: 23.64% [962000/4070000] [236.4/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 22:37:07,414 - Train: 23.64% [962100/4070000] [236.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 22:38:08,101 - Train: 23.64% [962200/4070000] [236.4/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 22:39:08,633 - Train: 23.64% [962300/4070000] [236.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 22:40:09,175 - Train: 23.65% [962400/4070000] [236.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 22:41:09,822 - Train: 23.65% [962500/4070000] [236.5/1000.0] [batch_t 0.601 (0.606)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-10 22:42:10,354 - Train: 23.65% [962600/4070000] [236.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 22:43:10,903 - Train: 23.65% [962700/4070000] [236.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 22:44:11,457 - Train: 23.66% [962800/4070000] [236.6/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-10 22:45:12,158 - Train: 23.66% [962900/4070000] [236.6/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 22:46:12,768 - Train: 23.66% [963000/4070000] [236.6/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 22:47:13,325 - Train: 23.66% [963100/4070000] [236.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 22:48:13,962 - Train: 23.67% [963200/4070000] [236.7/1000.0] [batch_t 0.609 (0.606)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-10 22:49:14,606 - Train: 23.67% [963300/4070000] [236.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 22:50:15,297 - Train: 23.67% [963400/4070000] [236.7/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 22:51:15,846 - Train: 23.67% [963500/4070000] [236.7/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 22:52:16,396 - Train: 23.68% [963600/4070000] [236.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 22:53:16,984 - Train: 23.68% [963700/4070000] [236.8/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 22:54:17,592 - Train: 23.68% [963800/4070000] [236.8/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 22:55:18,205 - Train: 23.68% [963900/4070000] [236.8/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 22:56:18,740 - Train: 23.69% [964000/4070000] [236.9/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 22:57:19,309 - Train: 23.69% [964100/4070000] [236.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 22:58:19,964 - Train: 23.69% [964200/4070000] [236.9/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 22:59:20,570 - Train: 23.69% [964300/4070000] [236.9/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 23:00:21,291 - Train: 23.70% [964400/4070000] [237.0/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 23:01:21,916 - Train: 23.70% [964500/4070000] [237.0/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 23:02:16,418 - ==> Total time: 8 days, 22:01:47 Eta: 28 days, 17:02:58 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-10 23:02:24,670 - Train: 23.70% [964600/4070000] [237.0/1000.0] [batch_t 0.605 (0.609)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 23:03:25,361 - Train: 23.70% [964700/4070000] [237.0/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 23:04:25,911 - Train: 23.71% [964800/4070000] [237.1/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 23:05:26,480 - Train: 23.71% [964900/4070000] [237.1/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 23:06:27,052 - Train: 23.71% [965000/4070000] [237.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 23:07:27,625 - Train: 23.71% [965100/4070000] [237.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 23:08:28,311 - Train: 23.71% [965200/4070000] [237.1/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 23:09:28,861 - Train: 23.72% [965300/4070000] [237.2/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 23:10:29,424 - Train: 23.72% [965400/4070000] [237.2/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 23:11:29,985 - Train: 23.72% [965500/4070000] [237.2/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 23:12:30,511 - Train: 23.72% [965600/4070000] [237.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 23:13:31,057 - Train: 23.73% [965700/4070000] [237.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 23:14:31,635 - Train: 23.73% [965800/4070000] [237.3/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 23:15:32,260 - Train: 23.73% [965900/4070000] [237.3/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 23:16:32,836 - Train: 23.73% [966000/4070000] [237.3/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 23:17:33,426 - Train: 23.74% [966100/4070000] [237.4/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 23:18:34,036 - Train: 23.74% [966200/4070000] [237.4/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 23:19:34,650 - Train: 23.74% [966300/4070000] [237.4/1000.0] [batch_t 0.611 (0.606)] [data_t 0.002] [optim_t 0.609] [lr 0.005000] +2024-04-10 23:20:35,249 - Train: 23.74% [966400/4070000] [237.4/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 23:21:35,853 - Train: 23.75% [966500/4070000] [237.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 23:22:36,414 - Train: 23.75% [966600/4070000] [237.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 23:23:37,100 - Train: 23.75% [966700/4070000] [237.5/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 23:24:37,673 - Train: 23.75% [966800/4070000] [237.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 23:25:38,271 - Train: 23.76% [966900/4070000] [237.6/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 23:26:38,821 - Train: 23.76% [967000/4070000] [237.6/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 23:27:39,428 - Train: 23.76% [967100/4070000] [237.6/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 23:28:40,096 - Train: 23.76% [967200/4070000] [237.6/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 23:29:40,686 - Train: 23.77% [967300/4070000] [237.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 23:30:41,247 - Train: 23.77% [967400/4070000] [237.7/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 23:31:41,811 - Train: 23.77% [967500/4070000] [237.7/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-10 23:32:42,431 - Train: 23.77% [967600/4070000] [237.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 23:33:42,989 - Train: 23.78% [967700/4070000] [237.8/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 23:34:43,603 - Train: 23.78% [967800/4070000] [237.8/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 23:35:44,202 - Train: 23.78% [967900/4070000] [237.8/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 23:36:44,923 - Train: 23.78% [968000/4070000] [237.8/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 23:37:45,474 - Train: 23.79% [968100/4070000] [237.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 23:38:46,062 - Train: 23.79% [968200/4070000] [237.9/1000.0] [batch_t 0.602 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-10 23:39:46,644 - Train: 23.79% [968300/4070000] [237.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 23:40:47,252 - Train: 23.79% [968400/4070000] [237.9/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 23:41:47,916 - Train: 23.80% [968500/4070000] [238.0/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 23:42:48,478 - Train: 23.80% [968600/4070000] [238.0/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 23:43:24,905 - ==> Total time: 8 days, 22:42:55 Eta: 28 days, 15:27:01 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-10 23:43:51,500 - Train: 23.80% [968700/4070000] [238.0/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 23:44:52,101 - Train: 23.80% [968800/4070000] [238.0/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 23:45:52,665 - Train: 23.81% [968900/4070000] [238.1/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 23:46:53,228 - Train: 23.81% [969000/4070000] [238.1/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-10 23:47:53,759 - Train: 23.81% [969100/4070000] [238.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 23:48:54,314 - Train: 23.81% [969200/4070000] [238.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 23:49:54,879 - Train: 23.82% [969300/4070000] [238.2/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 23:50:55,549 - Train: 23.82% [969400/4070000] [238.2/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-10 23:51:56,135 - Train: 23.82% [969500/4070000] [238.2/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 23:52:56,672 - Train: 23.82% [969600/4070000] [238.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 23:53:57,248 - Train: 23.83% [969700/4070000] [238.3/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 23:54:57,833 - Train: 23.83% [969800/4070000] [238.3/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-10 23:55:58,495 - Train: 23.83% [969900/4070000] [238.3/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 23:56:59,057 - Train: 23.83% [970000/4070000] [238.3/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-10 23:57:59,627 - Train: 23.84% [970100/4070000] [238.4/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-10 23:59:00,207 - Train: 23.84% [970200/4070000] [238.4/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-11 00:00:00,756 - Train: 23.84% [970300/4070000] [238.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-11 00:01:01,369 - Train: 23.84% [970400/4070000] [238.4/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-11 00:02:01,987 - Train: 23.85% [970500/4070000] [238.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-11 00:03:02,589 - Train: 23.85% [970600/4070000] [238.5/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-11 00:04:03,247 - Train: 23.85% [970700/4070000] [238.5/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-11 00:05:03,826 - Train: 23.85% [970800/4070000] [238.5/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-11 00:06:04,400 - Train: 23.86% [970900/4070000] [238.6/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-11 00:07:05,015 - Train: 23.86% [971000/4070000] [238.6/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-11 00:08:05,566 - Train: 23.86% [971100/4070000] [238.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-11 00:09:06,241 - Train: 23.86% [971200/4070000] [238.6/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-11 00:10:06,860 - Train: 23.86% [971300/4070000] [238.6/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-11 00:11:07,445 - Train: 23.87% [971400/4070000] [238.7/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-11 00:12:08,064 - Train: 23.87% [971500/4070000] [238.7/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-11 00:13:08,621 - Train: 23.87% [971600/4070000] [238.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-11 00:14:09,217 - Train: 23.87% [971700/4070000] [238.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-11 00:15:09,837 - Train: 23.88% [971800/4070000] [238.8/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-11 00:16:10,429 - Train: 23.88% [971900/4070000] [238.8/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-11 00:17:11,091 - Train: 23.88% [972000/4070000] [238.8/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-11 00:18:11,679 - Train: 23.88% [972100/4070000] [238.8/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-11 00:19:12,300 - Train: 23.89% [972200/4070000] [238.9/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-11 00:20:12,877 - Train: 23.89% [972300/4070000] [238.9/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-11 00:21:13,435 - Train: 23.89% [972400/4070000] [238.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-11 00:22:13,985 - Train: 23.89% [972500/4070000] [238.9/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-11 00:23:14,570 - Train: 23.90% [972600/4070000] [239.0/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-11 00:24:15,307 - Train: 23.90% [972700/4070000] [239.0/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-11 00:24:33,501 - ==> Total time: 8 days, 23:24:04 Eta: 28 days, 13:51:32 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco1_nhcs57c1_20240402-010029' +2024-04-11 00:25:18,002 - Train: 23.90% [972800/4070000] [239.0/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-11 00:26:18,586 - Train: 23.90% [972900/4070000] [239.0/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-11 00:28:26,406 - Train: 23.91% [973000/4070000] [239.1/1000.0] [batch_t 1.444 (1.278)] [data_t 0.004] [optim_t 1.440] [lr 0.005000]