Saving weights and logs of step 500

events.out.tfevents.1625915366.t1v-n-112df4a9-w-0.54625.3.v2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4dd43ada80ff05b64bc410a794e4619c8614294acada493832cd76e0997eff1f
+size 9069

flax_model.msgpack

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:76d55c57630984430801c3b6d5cc75703bd2a61ef6bb8f9d20f7c6150f2dfbd3
+oid sha256:0f6552dbdda3b1eba8c96bfd83fb06cb63770ea852b6779eefb7346da560de2c
 size 498796983

mr_clean_text.py

@@ -0,0 +1,39 @@
+import sys
+import string
+import regex as re
+# import re
+
+from datasets import load_dataset
+
+split_type = sys.argv[1]
+
+def clean_marathi_text(text):
+    clean_text = "".join([tok.group() for tok in re.finditer(r'[\u0900-\u097F\s]', text)])
+    clean_text = re.sub(r"\s+", " ", clean_text)
+    return clean_text
+
+def clean_marathi_text2(text):
+    pattern1 = r'\P{Devanagari}+'
+    pattern2 = r'[\p{Devanagari}\s\.\!]+'
+    punct_rm = re.compile("[" + re.escape(string.punctuation) + "]")
+    # cleaned = re.sub(pattern1, "", text)
+
+    cleaned = "".join([tok.group() for tok in re.finditer(pattern2, text)])
+    cleaned = re.sub(r"[ ]+", " ", cleaned)
+    clean_puct_rm = re.sub(punct_rm, "", cleaned)
+    return cleaned, clean_puct_rm.strip()
+
+
+mr_mc4 = load_dataset("mc4", "mr", split=split_type)
+
+print(mr_mc4)
+
+count = 0
+with open(f"mr_{split_type}_clean2.txt", 'w', encoding="utf-8") as f:
+    for ind, each in enumerate(mr_mc4):
+        clean_text, only_punct = clean_marathi_text2(each["text"])
+        if only_punct:
+            count += 1
+            f.write(clean_text.strip())
+            f.write("\n\n")
+print(f"{split_type} clean docs count {count} out of {len(mr_mc4)}")

run.sh

@@ -4,8 +4,8 @@ python run_mlm_flax.py \
     --model_type="roberta" \
     --config_name="./" \
     --tokenizer_name="./" \
-    --dataset_name="oscar" \
-    --dataset_config_name="unshuffled_deduplicated_als" \
+    --train_file="/home/nipunsadvilkar/mr_data/mr_train_df.csv" \
+    --validation_file="/home/nipunsadvilkar/mr_data/mr_validation_df.csv"\
     --max_seq_length="128" \
     --per_device_train_batch_size="4" \
     --per_device_eval_batch_size="4" \
@@ -17,4 +17,4 @@ python run_mlm_flax.py \
     --run_name hf-flax-robert-base-mr \
     --push_to_hub_model_id="flax-community/roberta-base-mr" \
     --push_to_hub_token=$HUB_TOKEN \
-    --push_to_hub 2>&1 | tee run.log
+    --push_to_hub 2>&1 | tee run.log

run_mlm_flax.py

@@ -457,12 +457,6 @@ if __name__ == "__main__":
     has_tensorboard = is_tensorboard_available()
     if has_tensorboard and jax.process_index() == 0:
         try:
-            import wandb
-            wandb.init(
-            entity='nipunsadvilkar', 
-            project='hf-flax-robert-base-mr',
-            sync_tensorboard=True
-            )
             from flax.metrics.tensorboard import SummaryWriter
 
             summary_writer = SummaryWriter(log_dir=Path(training_args.output_dir))

test_mr_tokenizer.py

@@ -5,4 +5,4 @@ tokenizer = Tokenizer.from_file("tokenizer.json")
 text = "गेल्या अनेक दिवसांपासून केंद्र सरकरा विरुद्ध ट्विटर इंडिया यांच्यामध्ये सरकारच्या नव्या मार्गदर्शक सूचना किंवा नियमावलीवरून वाद सुरू आहे."
 
 output = tokenizer.encode(text)
-print(tokenizer.decode(output.ids))
+print(tokenizer.decode(output.ids))

tokens.py

@@ -2,12 +2,14 @@
 from datasets import load_dataset
 from tokenizers import ByteLevelBPETokenizer
 # Load dataset
-dataset = load_dataset("oscar", "unshuffled_deduplicated_als", split="train")
+dataset = load_dataset('csv', data_files={'train': ["/home/nipunsadvilkar/mr_data/mr_train_df.csv"], "validation": ["/home/nipunsadvilkar/mr_data/mr_validation_df.csv"]}, split="train")
 # Instantiate tokenizer
 tokenizer = ByteLevelBPETokenizer()
-def batch_iterator(batch_size=100_000):
+
+def batch_iterator(batch_size=1000):
     for i in range(0, len(dataset), batch_size):
         yield dataset["text"][i: i + batch_size]
+
 # Customized training
 tokenizer.train_from_iterator(batch_iterator(), vocab_size=50265, min_frequency=2, special_tokens=[
     "<s>",

wandb/run-20210709_132941-xrzp5klf/files/config.yaml

@@ -13,6 +13,9 @@ _wandb:
       1:
       - 3
       - 11
+      2:
+      - 3
+      - 11
       4: 3.8.10
       5: 0.10.33
       6: 4.9.0.dev0

wandb/run-20210709_132941-xrzp5klf/files/output.log

@@ -32,3 +32,7 @@ Step... (500 | Loss: 8.205772399902344, Acc: 0.0773010179400444):  88%|███
 Training...:   3%|███▍                                                                                                                        | 4/142 [00:00<00:03, 35.60it/s]
 Evaluating ...:   0%|                                                                                                                                  | 0/10 [00:00<?, ?it/s]
 git-lfs/2.9.2 (GitHub; linux amd64; go 1.13.5)████████████████████████████████████████████████████████████████████████▌                        | 8/10 [00:00<00:00, 76.30it/s]
+[13:31:54] - DEBUG - huggingface_hub.repository - [Repository] is a valid git repo
+[13:32:05] - INFO - huggingface_hub.repository - Uploading LFS objects: 100% (2/2), 499 MB | 0 B/s, done.
+
+Step... (1000 | Loss: 7.925442218780518, Acc: 0.09575153142213821): 100%|███████████████████████████████████████████████████████████████████████| 8/8 [02:21<00:00, 17.68s/it]

wandb/run-20210709_132941-xrzp5klf/files/wandb-summary.json

@@ -1 +1 @@
-{"global_step": 1000, "_timestamp": 1625837513.455174, "train_time": 4.0346198081970215, "train_learning_rate": 0.0003000000142492354, "_step": 156, "train_loss": 7.2498674392700195, "eval_accuracy": 0.0773010179400444, "eval_loss": 8.205772399902344}
+{"global_step": 1000, "_timestamp": 1625837513.610818, "train_time": 4.0346198081970215, "train_learning_rate": 0.0003000000142492354, "_step": 157, "train_loss": 7.807257652282715, "eval_accuracy": 0.09575153142213821, "eval_loss": 7.925442218780518}

wandb/run-20210709_132941-xrzp5klf/logs/debug-internal.log

@@ -562,3 +562,113 @@
 2021-07-09 13:31:56,216 INFO    Thread-8  :41762 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_132941-xrzp5klf/files/output.log
 2021-07-09 13:31:58,588 DEBUG   HandlerThread:41762 [handler.py:handle_request():124] handle_request: stop_status
 2021-07-09 13:31:58,589 DEBUG   SenderThread:41762 [sender.py:send_request():193] send_request: stop_status
+2021-07-09 13:32:06,221 INFO    Thread-8  :41762 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_132941-xrzp5klf/files/output.log
+2021-07-09 13:32:08,222 INFO    Thread-8  :41762 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_132941-xrzp5klf/files/output.log
+2021-07-09 13:32:10,223 INFO    Thread-8  :41762 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_132941-xrzp5klf/files/output.log
+2021-07-09 13:32:11,569 DEBUG   SenderThread:41762 [sender.py:send():179] send: stats
+2021-07-09 13:32:11,972 DEBUG   SenderThread:41762 [sender.py:send():179] send: telemetry
+2021-07-09 13:32:11,972 DEBUG   HandlerThread:41762 [handler.py:handle_request():124] handle_request: poll_exit
+2021-07-09 13:32:11,972 DEBUG   SenderThread:41762 [sender.py:send():179] send: exit
+2021-07-09 13:32:11,973 INFO    SenderThread:41762 [sender.py:send_exit():287] handling exit code: 0
+2021-07-09 13:32:11,973 INFO    SenderThread:41762 [sender.py:send_exit():295] send defer
+2021-07-09 13:32:11,973 DEBUG   SenderThread:41762 [sender.py:send_request():193] send_request: poll_exit
+2021-07-09 13:32:11,974 DEBUG   HandlerThread:41762 [handler.py:handle_request():124] handle_request: defer
+2021-07-09 13:32:11,974 INFO    HandlerThread:41762 [handler.py:handle_request_defer():141] handle defer: 0
+2021-07-09 13:32:11,974 DEBUG   SenderThread:41762 [sender.py:send_request():193] send_request: defer
+2021-07-09 13:32:11,974 INFO    SenderThread:41762 [sender.py:send_request_defer():304] handle sender defer: 0
+2021-07-09 13:32:11,974 INFO    SenderThread:41762 [sender.py:transition_state():308] send defer: 1
+2021-07-09 13:32:11,974 DEBUG   HandlerThread:41762 [handler.py:handle_request():124] handle_request: defer
+2021-07-09 13:32:11,974 INFO    HandlerThread:41762 [handler.py:handle_request_defer():141] handle defer: 1
+2021-07-09 13:32:12,071 DEBUG   SenderThread:41762 [sender.py:send_request():193] send_request: defer
+2021-07-09 13:32:12,071 INFO    SenderThread:41762 [sender.py:send_request_defer():304] handle sender defer: 1
+2021-07-09 13:32:12,071 INFO    SenderThread:41762 [sender.py:transition_state():308] send defer: 2
+2021-07-09 13:32:12,071 DEBUG   SenderThread:41762 [sender.py:send():179] send: stats
+2021-07-09 13:32:12,072 DEBUG   HandlerThread:41762 [handler.py:handle_request():124] handle_request: defer
+2021-07-09 13:32:12,072 INFO    HandlerThread:41762 [handler.py:handle_request_defer():141] handle defer: 2
+2021-07-09 13:32:12,224 INFO    Thread-8  :41762 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_132941-xrzp5klf/files/output.log
+2021-07-09 13:32:13,225 INFO    Thread-8  :41762 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_132941-xrzp5klf/files/config.yaml
+2021-07-09 13:32:18,572 DEBUG   HandlerThread:41762 [handler.py:handle_request():124] handle_request: poll_exit
+2021-07-09 13:32:18,572 DEBUG   HandlerThread:41762 [handler.py:handle_request():124] handle_request: poll_exit
+2021-07-09 13:32:18,573 DEBUG   SenderThread:41762 [sender.py:send_request():193] send_request: defer
+2021-07-09 13:32:18,573 INFO    SenderThread:41762 [sender.py:send_request_defer():304] handle sender defer: 2
+2021-07-09 13:32:18,573 INFO    SenderThread:41762 [sender.py:transition_state():308] send defer: 3
+2021-07-09 13:32:18,573 DEBUG   SenderThread:41762 [sender.py:send_request():193] send_request: poll_exit
+2021-07-09 13:32:18,573 DEBUG   SenderThread:41762 [sender.py:send_request():193] send_request: poll_exit
+2021-07-09 13:32:18,574 DEBUG   SenderThread:41762 [sender.py:send():179] send: history
+2021-07-09 13:32:18,574 DEBUG   SenderThread:41762 [sender.py:send():179] send: summary
+2021-07-09 13:32:18,576 DEBUG   HandlerThread:41762 [handler.py:handle_request():124] handle_request: defer
+2021-07-09 13:32:18,576 INFO    HandlerThread:41762 [handler.py:handle_request_defer():141] handle defer: 3
+2021-07-09 13:32:18,577 INFO    SenderThread:41762 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-09 13:32:18,577 DEBUG   SenderThread:41762 [sender.py:send():179] send: summary
+2021-07-09 13:32:18,578 INFO    SenderThread:41762 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-09 13:32:18,578 DEBUG   SenderThread:41762 [sender.py:send_request():193] send_request: defer
+2021-07-09 13:32:18,578 INFO    SenderThread:41762 [sender.py:send_request_defer():304] handle sender defer: 3
+2021-07-09 13:32:18,578 INFO    SenderThread:41762 [sender.py:transition_state():308] send defer: 4
+2021-07-09 13:32:18,578 DEBUG   HandlerThread:41762 [handler.py:handle_request():124] handle_request: defer
+2021-07-09 13:32:18,578 INFO    HandlerThread:41762 [handler.py:handle_request_defer():141] handle defer: 4
+2021-07-09 13:32:18,579 DEBUG   SenderThread:41762 [sender.py:send_request():193] send_request: defer
+2021-07-09 13:32:18,579 INFO    SenderThread:41762 [sender.py:send_request_defer():304] handle sender defer: 4
+2021-07-09 13:32:18,579 INFO    SenderThread:41762 [sender.py:transition_state():308] send defer: 5
+2021-07-09 13:32:18,579 DEBUG   HandlerThread:41762 [handler.py:handle_request():124] handle_request: defer
+2021-07-09 13:32:18,579 INFO    HandlerThread:41762 [handler.py:handle_request_defer():141] handle defer: 5
+2021-07-09 13:32:18,579 DEBUG   SenderThread:41762 [sender.py:send_request():193] send_request: defer
+2021-07-09 13:32:18,579 INFO    SenderThread:41762 [sender.py:send_request_defer():304] handle sender defer: 5
+2021-07-09 13:32:18,579 INFO    SenderThread:41762 [dir_watcher.py:finish():282] shutting down directory watcher
+2021-07-09 13:32:18,678 DEBUG   HandlerThread:41762 [handler.py:handle_request():124] handle_request: poll_exit
+2021-07-09 13:32:19,227 INFO    SenderThread:41762 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_132941-xrzp5klf/files/wandb-summary.json
+2021-07-09 13:32:19,227 INFO    SenderThread:41762 [dir_watcher.py:finish():312] scan: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_132941-xrzp5klf/files
+2021-07-09 13:32:19,227 INFO    SenderThread:41762 [dir_watcher.py:finish():318] scan save: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_132941-xrzp5klf/files/requirements.txt requirements.txt
+2021-07-09 13:32:19,228 INFO    SenderThread:41762 [dir_watcher.py:finish():318] scan save: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_132941-xrzp5klf/files/events.out.tfevents.1625837383.t1v-n-112df4a9-w-0.40707.3.v2 events.out.tfevents.1625837383.t1v-n-112df4a9-w-0.40707.3.v2
+2021-07-09 13:32:19,228 INFO    SenderThread:41762 [dir_watcher.py:finish():318] scan save: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_132941-xrzp5klf/files/output.log output.log
+2021-07-09 13:32:19,228 INFO    SenderThread:41762 [dir_watcher.py:finish():318] scan save: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_132941-xrzp5klf/files/wandb-metadata.json wandb-metadata.json
+2021-07-09 13:32:19,228 INFO    SenderThread:41762 [dir_watcher.py:finish():318] scan save: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_132941-xrzp5klf/files/config.yaml config.yaml
+2021-07-09 13:32:19,229 INFO    SenderThread:41762 [dir_watcher.py:finish():318] scan save: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_132941-xrzp5klf/files/wandb-summary.json wandb-summary.json
+2021-07-09 13:32:19,229 INFO    SenderThread:41762 [sender.py:transition_state():308] send defer: 6
+2021-07-09 13:32:19,229 DEBUG   SenderThread:41762 [sender.py:send_request():193] send_request: poll_exit
+2021-07-09 13:32:19,233 DEBUG   HandlerThread:41762 [handler.py:handle_request():124] handle_request: defer
+2021-07-09 13:32:19,233 INFO    HandlerThread:41762 [handler.py:handle_request_defer():141] handle defer: 6
+2021-07-09 13:32:19,233 DEBUG   SenderThread:41762 [sender.py:send_request():193] send_request: defer
+2021-07-09 13:32:19,233 INFO    SenderThread:41762 [sender.py:send_request_defer():304] handle sender defer: 6
+2021-07-09 13:32:19,233 INFO    SenderThread:41762 [file_pusher.py:finish():177] shutting down file pusher
+2021-07-09 13:32:19,331 DEBUG   HandlerThread:41762 [handler.py:handle_request():124] handle_request: poll_exit
+2021-07-09 13:32:19,331 DEBUG   SenderThread:41762 [sender.py:send_request():193] send_request: poll_exit
+2021-07-09 13:32:19,433 DEBUG   HandlerThread:41762 [handler.py:handle_request():124] handle_request: poll_exit
+2021-07-09 13:32:19,433 DEBUG   SenderThread:41762 [sender.py:send_request():193] send_request: poll_exit
+2021-07-09 13:32:19,444 INFO    Thread-19 :41762 [upload_job.py:push():137] Uploaded file /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_132941-xrzp5klf/files/config.yaml
+2021-07-09 13:32:19,448 INFO    Thread-18 :41762 [upload_job.py:push():137] Uploaded file /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_132941-xrzp5klf/files/output.log
+2021-07-09 13:32:19,449 INFO    Thread-16 :41762 [upload_job.py:push():137] Uploaded file /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_132941-xrzp5klf/files/requirements.txt
+2021-07-09 13:32:19,460 INFO    Thread-17 :41762 [upload_job.py:push():137] Uploaded file /tmp/tmp2vrml8zwwandb/1110a0ll-events.out.tfevents.1625837383.t1v-n-112df4a9-w-0.40707.3.v2
+2021-07-09 13:32:19,462 INFO    Thread-20 :41762 [upload_job.py:push():137] Uploaded file /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_132941-xrzp5klf/files/wandb-summary.json
+2021-07-09 13:32:19,535 DEBUG   HandlerThread:41762 [handler.py:handle_request():124] handle_request: poll_exit
+2021-07-09 13:32:19,535 DEBUG   SenderThread:41762 [sender.py:send_request():193] send_request: poll_exit
+2021-07-09 13:32:19,636 DEBUG   HandlerThread:41762 [handler.py:handle_request():124] handle_request: poll_exit
+2021-07-09 13:32:19,637 DEBUG   SenderThread:41762 [sender.py:send_request():193] send_request: poll_exit
+2021-07-09 13:32:19,663 INFO    Thread-7  :41762 [sender.py:transition_state():308] send defer: 7
+2021-07-09 13:32:19,663 DEBUG   HandlerThread:41762 [handler.py:handle_request():124] handle_request: defer
+2021-07-09 13:32:19,663 INFO    HandlerThread:41762 [handler.py:handle_request_defer():141] handle defer: 7
+2021-07-09 13:32:19,664 DEBUG   SenderThread:41762 [sender.py:send_request():193] send_request: defer
+2021-07-09 13:32:19,664 INFO    SenderThread:41762 [sender.py:send_request_defer():304] handle sender defer: 7
+2021-07-09 13:32:19,711 INFO    SenderThread:41762 [sender.py:transition_state():308] send defer: 8
+2021-07-09 13:32:19,712 DEBUG   HandlerThread:41762 [handler.py:handle_request():124] handle_request: defer
+2021-07-09 13:32:19,712 INFO    HandlerThread:41762 [handler.py:handle_request_defer():141] handle defer: 8
+2021-07-09 13:32:19,712 DEBUG   SenderThread:41762 [sender.py:send_request():193] send_request: defer
+2021-07-09 13:32:19,712 INFO    SenderThread:41762 [sender.py:send_request_defer():304] handle sender defer: 8
+2021-07-09 13:32:19,712 INFO    SenderThread:41762 [sender.py:transition_state():308] send defer: 9
+2021-07-09 13:32:19,713 DEBUG   HandlerThread:41762 [handler.py:handle_request():124] handle_request: defer
+2021-07-09 13:32:19,713 INFO    HandlerThread:41762 [handler.py:handle_request_defer():141] handle defer: 9
+2021-07-09 13:32:19,713 DEBUG   SenderThread:41762 [sender.py:send():179] send: final
+2021-07-09 13:32:19,713 DEBUG   SenderThread:41762 [sender.py:send():179] send: footer
+2021-07-09 13:32:19,713 DEBUG   SenderThread:41762 [sender.py:send_request():193] send_request: defer
+2021-07-09 13:32:19,713 INFO    SenderThread:41762 [sender.py:send_request_defer():304] handle sender defer: 9
+2021-07-09 13:32:19,738 DEBUG   HandlerThread:41762 [handler.py:handle_request():124] handle_request: poll_exit
+2021-07-09 13:32:19,739 DEBUG   SenderThread:41762 [sender.py:send_request():193] send_request: poll_exit
+2021-07-09 13:32:19,739 INFO    SenderThread:41762 [file_pusher.py:join():182] waiting for file pusher
+2021-07-09 13:32:19,740 DEBUG   HandlerThread:41762 [handler.py:handle_request():124] handle_request: get_summary
+2021-07-09 13:32:19,741 DEBUG   HandlerThread:41762 [handler.py:handle_request():124] handle_request: sampled_history
+2021-07-09 13:32:19,743 DEBUG   HandlerThread:41762 [handler.py:handle_request():124] handle_request: shutdown
+2021-07-09 13:32:19,743 INFO    HandlerThread:41762 [handler.py:finish():638] shutting down handler
+2021-07-09 13:32:20,713 INFO    WriterThread:41762 [datastore.py:close():288] close: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_132941-xrzp5klf/run-xrzp5klf.wandb
+2021-07-09 13:32:20,739 INFO    SenderThread:41762 [sender.py:finish():945] shutting down sender
+2021-07-09 13:32:20,740 INFO    SenderThread:41762 [file_pusher.py:finish():177] shutting down file pusher
+2021-07-09 13:32:20,740 INFO    SenderThread:41762 [file_pusher.py:join():182] waiting for file pusher
+2021-07-09 13:32:20,742 INFO    MainThread:41762 [internal.py:handle_exit():78] Internal process exited

wandb/run-20210709_132941-xrzp5klf/logs/debug.log

@@ -21,3 +21,84 @@ config: {}
 2021-07-09 13:29:43,218 INFO    MainThread:40707 [wandb_run.py:_redirect():1558] Redirects installed.
 2021-07-09 13:29:43,218 INFO    MainThread:40707 [wandb_init.py:init():554] run started, returning control to user process
 2021-07-09 13:29:43,276 INFO    MainThread:40707 [wandb_run.py:_tensorboard_callback():943] tensorboard callback: ., None
+2021-07-09 13:32:09,325 INFO    MainThread:40707 [wandb_run.py:_atexit_cleanup():1593] got exitcode: 0
+2021-07-09 13:32:09,325 INFO    MainThread:40707 [wandb_run.py:_restore():1565] restore
+2021-07-09 13:32:11,974 INFO    MainThread:40707 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts {
+  wandb_count: 1
+  other_count: 1
+}
+pusher_stats {
+  uploaded_bytes: 12680
+  total_bytes: 12680
+}
+
+2021-07-09 13:32:17,075 INFO    MainThread:40707 [wandb_run.py:_wait_for_finish():1715] got exit ret: None
+2021-07-09 13:32:18,577 INFO    MainThread:40707 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts {
+  wandb_count: 1
+  other_count: 1
+}
+pusher_stats {
+  uploaded_bytes: 12680
+  total_bytes: 12680
+}
+
+2021-07-09 13:32:19,230 INFO    MainThread:40707 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts {
+  wandb_count: 2
+  other_count: 1
+}
+pusher_stats {
+  uploaded_bytes: 12680
+  total_bytes: 14613
+}
+
+2021-07-09 13:32:19,332 INFO    MainThread:40707 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts {
+  wandb_count: 5
+  other_count: 1
+}
+pusher_stats {
+  uploaded_bytes: 21518
+  total_bytes: 21518
+}
+
+2021-07-09 13:32:19,434 INFO    MainThread:40707 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts {
+  wandb_count: 5
+  other_count: 1
+}
+pusher_stats {
+  uploaded_bytes: 21518
+  total_bytes: 21518
+}
+
+2021-07-09 13:32:19,536 INFO    MainThread:40707 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts {
+  wandb_count: 5
+  other_count: 1
+}
+pusher_stats {
+  uploaded_bytes: 21518
+  total_bytes: 21518
+}
+
+2021-07-09 13:32:19,637 INFO    MainThread:40707 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts {
+  wandb_count: 5
+  other_count: 1
+}
+pusher_stats {
+  uploaded_bytes: 21518
+  total_bytes: 21518
+}
+
+2021-07-09 13:32:19,740 INFO    MainThread:40707 [wandb_run.py:_wait_for_finish():1715] got exit ret: done: true
+exit_result {
+}
+file_counts {
+  wandb_count: 5
+  other_count: 1
+}
+pusher_stats {
+  uploaded_bytes: 21518
+  total_bytes: 21518
+}
+
+2021-07-09 13:32:20,981 INFO    MainThread:40707 [wandb_run.py:_show_summary():1870] rendering summary
+2021-07-09 13:32:20,982 INFO    MainThread:40707 [wandb_run.py:_show_history():1908] rendering history
+2021-07-09 13:32:20,982 INFO    MainThread:40707 [wandb_run.py:_show_files():1937] logging synced files