Update config.json

config.json

@@ -1,58 +1,133 @@
 {
+  "accumulate_gradients": 4,
+  "ae_steps": [],
+  "amp": 2,
   "architectures": [
     "XLMWithLMHeadModel"
   ],
+  "asm": false,
+  "attention_dropout": 0.1,
+  "batch_size": 16,
+  "beam_size": 1,
+  "bos_index": 0,
+  "bos_token_id": 0,
+  "bptt": 256,
+  "bt_src_langs": [],
+  "bt_steps": [],
+  "causal": false,
+  "clip_grad_norm": 1.0,
+  "clm_steps": [],
+  "command": "python /private/home/aconneau/workdir/xlm_17_100_big.3/2019_08_10_19_23_42/train.py --n_heads 16 --bt_steps '' --max_vocab 200000 --word_mask_keep_rand '0.8,0.1,0.1' --use_lang_emb false --data_path '/private/home/aconneau/projects/XLM/data/wiki/17/175k' --save_periodic 0 --max_len 200 --bptt 256 --ae_steps '' --fp16 true --share_inout_emb true --sinusoidal_embeddings false --word_shuffle 0 --tokens_per_batch '-1' --accumulate_gradients 4 --validation_metrics '_valid_en_mlm_ppl,_valid_mlm_ppl,_valid_zh_mlm_ppl' --attention_dropout '0.1' --split_data true --max_epoch 100000 --stopping_criterion '_valid_zh_mlm_ppl,25' --dump_path '/checkpoint/aconneau/dumped' --epoch_size 200000 --word_blank 0 --gelu_activation true --n_layers 16 --optimizer 'adam_inverse_sqrt,lr=0.00005,warmup_updates=30000,beta1=0.9,beta2=0.999,weight_decay=0.01,eps=0.000001' --mlm_steps 'en,fr,es,de,it,pt,nl,sv,pl,ru,ar,tr,zh,ja,ko,hi,vi' --eval_bleu false --dropout '0.1' --mt_steps '' --batch_size 16 --word_dropout 0 --reload_model '/checkpoint/aconneau/dumped/xlm_17_100_240_big_model_upper.2/14884510/best-valid_zh_mlm_ppl.pth' --min_count 0 --amp 2 --group_by_size true --asm false --sample_alpha '0.5' --word_pred '0.15' --clip_grad_norm 1 --emb_dim 1280 --encoder_only true --lgs 'en-fr-es-de-it-pt-nl-sv-pl-ru-ar-tr-zh-ja-ko-hi-vi' --clm_steps '' --exp_name 'xlm_17_100_big.3' --lg_sampling_factor '0.7' --eval_only false --exp_id 16656237 --master_port 14148 --exp_id \"16656237\"",
+  "context_size": 0,
+  "data_path": "/private/home/aconneau/projects/XLM/data/wiki/17/175k",
+  "debug": false,
+  "debug_slurm": false,
+  "debug_train": false,
+  "dropout": 0.1,
   "dump_path": "/checkpoint/aconneau/dumped/xlm_17_100_big.3/16656237",
-  "exp_name": "xlm_17_100_big.3",
-  "save_periodic": 0,
+  "emb_dim": 1280,
+  "embed_init_std": 0.02209708691207961,
+  "encoder_only": true,
+  "end_n_top": 5,
+  "eos_index": 1,
+  "epoch_size": 200000,
+  "eval_bleu": false,
+  "eval_only": false,
   "exp_id": "16656237",
+  "exp_name": "xlm_17_100_big.3",
   "fp16": true,
-  "amp": 2,
-  "encoder_only": true,
-  "emb_dim": 1280,
-  "n_layers": 16,
-  "n_heads": 16,
-  "dropout": 0.1,
-  "attention_dropout": 0.1,
   "gelu_activation": true,
-  "share_inout_emb": true,
-  "sinusoidal_embeddings": false,
-  "use_lang_emb": false,
-  "use_memory": false,
-  "asm": false,
-  "context_size": 0,
-  "word_pred": 0.15,
-  "sample_alpha": 0.5,
-  "word_mask_keep_rand": "0.8,0.1,0.1",
-  "word_shuffle": 0.0,
-  "word_dropout": 0.0,
-  "word_blank": 0.0,
-  "data_path": "/private/home/aconneau/projects/XLM/data/wiki/17/175k",
-  "lgs": "en-fr-es-de-it-pt-nl-sv-pl-ru-ar-tr-zh-ja-ko-hi-vi",
-  "max_vocab": 200000,
-  "min_count": 0,
-  "lg_sampling_factor": 0.7,
-  "bptt": 256,
-  "max_len": 200,
+  "global_rank": 0,
   "group_by_size": true,
-  "batch_size": 16,
-  "max_batch_size": 0,
-  "tokens_per_batch": -1,
-  "split_data": true,
-  "optimizer": "adam_inverse_sqrt,lr=0.00005,warmup_updates=30000,beta1=0.9,beta2=0.999,weight_decay=0.01,eps=0.000001",
-  "clip_grad_norm": 1.0,
-  "epoch_size": 200000,
-  "max_epoch": 100000,
-  "stopping_criterion": "_valid_zh_mlm_ppl,25",
-  "validation_metrics": "_valid_en_mlm_ppl,_valid_mlm_ppl,_valid_zh_mlm_ppl",
-  "accumulate_gradients": 4,
-  "lambda_mlm": 1.0,
-  "lambda_clm": 1.0,
-  "lambda_pc": 1.0,
+  "hyp_path": "/checkpoint/aconneau/dumped/xlm_17_100_big.3/16656237/hypotheses",
+  "id2lang": {
+    "0": "ar",
+    "1": "de",
+    "10": "pl",
+    "11": "pt",
+    "12": "ru",
+    "13": "sv",
+    "14": "tr",
+    "15": "vi",
+    "16": "zh",
+    "2": "en",
+    "3": "es",
+    "4": "fr",
+    "5": "hi",
+    "6": "it",
+    "7": "ja",
+    "8": "ko",
+    "9": "nl"
+  },
+  "init_std": 0.02,
+  "is_encoder": true,
+  "is_master": true,
+  "is_slurm_job": true,
   "lambda_ae": 1.0,
-  "lambda_mt": 1.0,
+  "lambda_ae_config": null,
   "lambda_bt": 1.0,
-  "clm_steps": [],
+  "lambda_bt_config": null,
+  "lambda_clm": 1.0,
+  "lambda_clm_config": null,
+  "lambda_mlm": 1.0,
+  "lambda_mlm_config": null,
+  "lambda_mt": 1.0,
+  "lambda_mt_config": null,
+  "lambda_pc": 1.0,
+  "lambda_pc_config": null,
+  "lang2id": {
+    "ar": 0,
+    "de": 1,
+    "en": 2,
+    "es": 3,
+    "fr": 4,
+    "hi": 5,
+    "it": 6,
+    "ja": 7,
+    "ko": 8,
+    "nl": 9,
+    "pl": 10,
+    "pt": 11,
+    "ru": 12,
+    "sv": 13,
+    "tr": 14,
+    "vi": 15,
+    "zh": 16
+  },
+  "lang_id": 0,
+  "langs": [
+    "en",
+    "fr",
+    "es",
+    "de",
+    "it",
+    "pt",
+    "nl",
+    "sv",
+    "pl",
+    "ru",
+    "ar",
+    "tr",
+    "zh",
+    "ja",
+    "ko",
+    "hi",
+    "vi"
+  ],
+  "layer_norm_eps": 1e-12,
+  "lg_sampling_factor": 0.7,
+  "lgs": "en-fr-es-de-it-pt-nl-sv-pl-ru-ar-tr-zh-ja-ko-hi-vi",
+  "local_rank": 0,
+  "mask_index": 5,
+  "mask_token_id": 0,
+  "master_addr": "learnfair1605",
+  "master_port": 14148,
+  "max_batch_size": 0,
+  "max_epoch": 100000,
+  "max_len": 200,
+  "max_position_embeddings": 512,
+  "max_vocab": 200000,
+  "min_count": 0,
   "mlm_steps": [
     [
       "en",
@@ -123,198 +198,94 @@
       null
     ]
   ],
-  "mt_steps": [],
-  "ae_steps": [],
-  "bt_steps": [],
-  "pc_steps": [],
-  "reload_emb": "",
-  "reload_model": "/checkpoint/aconneau/dumped/xlm_17_100_240_big_model_upper.2/14884510/best-valid_zh_mlm_ppl.pth",
-  "reload_checkpoint": "",
-  "beam_size": 1,
-  "length_penalty": 1,
-  "early_stopping": false,
-  "eval_bleu": false,
-  "eval_only": false,
-  "debug_train": false,
-  "debug_slurm": false,
-  "debug": false,
-  "local_rank": 0,
-  "master_port": 14148,
-  "langs": [
-    "en",
-    "fr",
-    "es",
-    "de",
-    "it",
-    "pt",
-    "nl",
-    "sv",
-    "pl",
-    "ru",
-    "ar",
-    "tr",
-    "zh",
-    "ja",
-    "ko",
-    "hi",
-    "vi"
-  ],
-  "id2lang": {
-    "0": "ar",
-    "1": "de",
-    "2": "en",
-    "3": "es",
-    "4": "fr",
-    "5": "hi",
-    "6": "it",
-    "7": "ja",
-    "8": "ko",
-    "9": "nl",
-    "10": "pl",
-    "11": "pt",
-    "12": "ru",
-    "13": "sv",
-    "14": "tr",
-    "15": "vi",
-    "16": "zh"
-  },
-  "lang2id": {
-    "ar": 0,
-    "de": 1,
-    "en": 2,
-    "es": 3,
-    "fr": 4,
-    "hi": 5,
-    "it": 6,
-    "ja": 7,
-    "ko": 8,
-    "nl": 9,
-    "pl": 10,
-    "pt": 11,
-    "ru": 12,
-    "sv": 13,
-    "tr": 14,
-    "vi": 15,
-    "zh": 16
-  },
-  "n_langs": 17,
-  "bt_src_langs": [],
+  "model_type": "xlm",
   "mono_dataset": {
+    "ar": {
+      "test": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/test.ar.pth",
+      "train": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/train.ar.pth",
+      "valid": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/valid.ar.pth"
+    },
+    "de": {
+      "test": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/test.de.pth",
+      "train": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/train.de.pth",
+      "valid": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/valid.de.pth"
+    },
     "en": {
+      "test": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/test.en.pth",
       "train": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/train.en.pth",
-      "valid": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/valid.en.pth",
-      "test": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/test.en.pth"
-    },
-    "fr": {
-      "train": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/train.fr.pth",
-      "valid": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/valid.fr.pth",
-      "test": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/test.fr.pth"
+      "valid": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/valid.en.pth"
     },
     "es": {
+      "test": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/test.es.pth",
       "train": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/train.es.pth",
-      "valid": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/valid.es.pth",
-      "test": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/test.es.pth"
+      "valid": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/valid.es.pth"
     },
-    "de": {
-      "train": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/train.de.pth",
-      "valid": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/valid.de.pth",
-      "test": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/test.de.pth"
+    "fr": {
+      "test": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/test.fr.pth",
+      "train": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/train.fr.pth",
+      "valid": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/valid.fr.pth"
+    },
+    "hi": {
+      "test": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/test.hi.pth",
+      "train": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/train.hi.pth",
+      "valid": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/valid.hi.pth"
     },
     "it": {
+      "test": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/test.it.pth",
       "train": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/train.it.pth",
-      "valid": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/valid.it.pth",
-      "test": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/test.it.pth"
+      "valid": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/valid.it.pth"
     },
-    "pt": {
-      "train": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/train.pt.pth",
-      "valid": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/valid.pt.pth",
-      "test": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/test.pt.pth"
+    "ja": {
+      "test": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/test.ja.pth",
+      "train": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/train.ja.pth",
+      "valid": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/valid.ja.pth"
+    },
+    "ko": {
+      "test": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/test.ko.pth",
+      "train": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/train.ko.pth",
+      "valid": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/valid.ko.pth"
     },
     "nl": {
+      "test": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/test.nl.pth",
       "train": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/train.nl.pth",
-      "valid": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/valid.nl.pth",
-      "test": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/test.nl.pth"
-    },
-    "sv": {
-      "train": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/train.sv.pth",
-      "valid": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/valid.sv.pth",
-      "test": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/test.sv.pth"
+      "valid": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/valid.nl.pth"
     },
     "pl": {
+      "test": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/test.pl.pth",
       "train": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/train.pl.pth",
-      "valid": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/valid.pl.pth",
-      "test": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/test.pl.pth"
+      "valid": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/valid.pl.pth"
+    },
+    "pt": {
+      "test": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/test.pt.pth",
+      "train": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/train.pt.pth",
+      "valid": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/valid.pt.pth"
     },
     "ru": {
+      "test": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/test.ru.pth",
       "train": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/train.ru.pth",
-      "valid": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/valid.ru.pth",
-      "test": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/test.ru.pth"
+      "valid": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/valid.ru.pth"
     },
-    "ar": {
-      "train": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/train.ar.pth",
-      "valid": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/valid.ar.pth",
-      "test": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/test.ar.pth"
+    "sv": {
+      "test": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/test.sv.pth",
+      "train": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/train.sv.pth",
+      "valid": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/valid.sv.pth"
     },
     "tr": {
+      "test": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/test.tr.pth",
       "train": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/train.tr.pth",
-      "valid": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/valid.tr.pth",
-      "test": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/test.tr.pth"
-    },
-    "zh": {
-      "train": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/train.zh.pth",
-      "valid": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/valid.zh.pth",
-      "test": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/test.zh.pth"
-    },
-    "ja": {
-      "train": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/train.ja.pth",
-      "valid": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/valid.ja.pth",
-      "test": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/test.ja.pth"
-    },
-    "ko": {
-      "train": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/train.ko.pth",
-      "valid": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/valid.ko.pth",
-      "test": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/test.ko.pth"
-    },
-    "hi": {
-      "train": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/train.hi.pth",
-      "valid": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/valid.hi.pth",
-      "test": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/test.hi.pth"
+      "valid": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/valid.tr.pth"
     },
     "vi": {
+      "test": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/test.vi.pth",
       "train": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/train.vi.pth",
-      "valid": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/valid.vi.pth",
-      "test": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/test.vi.pth"
+      "valid": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/valid.vi.pth"
+    },
+    "zh": {
+      "test": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/test.zh.pth",
+      "train": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/train.zh.pth",
+      "valid": "/private/home/aconneau/projects/XLM/data/wiki/17/175k/valid.zh.pth"
     }
   },
-  "para_dataset": {},
-  "word_mask": 0.8,
-  "word_keep": 0.1,
-  "word_rand": 0.1,
-  "is_slurm_job": true,
-  "n_nodes": 4,
-  "node_id": 0,
-  "global_rank": 0,
-  "world_size": 32,
-  "n_gpu_per_node": 8,
-  "master_addr": "learnfair1605",
-  "is_master": true,
-  "multi_node": true,
-  "multi_gpu": true,
-  "command": "python /private/home/aconneau/workdir/xlm_17_100_big.3/2019_08_10_19_23_42/train.py --n_heads 16 --bt_steps '' --max_vocab 200000 --word_mask_keep_rand '0.8,0.1,0.1' --use_lang_emb false --data_path '/private/home/aconneau/projects/XLM/data/wiki/17/175k' --save_periodic 0 --max_len 200 --bptt 256 --ae_steps '' --fp16 true --share_inout_emb true --sinusoidal_embeddings false --word_shuffle 0 --tokens_per_batch '-1' --accumulate_gradients 4 --validation_metrics '_valid_en_mlm_ppl,_valid_mlm_ppl,_valid_zh_mlm_ppl' --attention_dropout '0.1' --split_data true --max_epoch 100000 --stopping_criterion '_valid_zh_mlm_ppl,25' --dump_path '/checkpoint/aconneau/dumped' --epoch_size 200000 --word_blank 0 --gelu_activation true --n_layers 16 --optimizer 'adam_inverse_sqrt,lr=0.00005,warmup_updates=30000,beta1=0.9,beta2=0.999,weight_decay=0.01,eps=0.000001' --mlm_steps 'en,fr,es,de,it,pt,nl,sv,pl,ru,ar,tr,zh,ja,ko,hi,vi' --eval_bleu false --dropout '0.1' --mt_steps '' --batch_size 16 --word_dropout 0 --reload_model '/checkpoint/aconneau/dumped/xlm_17_100_240_big_model_upper.2/14884510/best-valid_zh_mlm_ppl.pth' --min_count 0 --amp 2 --group_by_size true --asm false --sample_alpha '0.5' --word_pred '0.15' --clip_grad_norm 1 --emb_dim 1280 --encoder_only true --lgs 'en-fr-es-de-it-pt-nl-sv-pl-ru-ar-tr-zh-ja-ko-hi-vi' --clm_steps '' --exp_name 'xlm_17_100_big.3' --lg_sampling_factor '0.7' --eval_only false --exp_id 16656237 --master_port 14148 --exp_id \"16656237\"",
-  "n_words": 200000,
-  "bos_index": 0,
-  "eos_index": 1,
-  "pad_index": 2,
-  "unk_index": 3,
-  "mask_index": 5,
-  "lambda_clm_config": null,
-  "lambda_mlm_config": null,
-  "lambda_pc_config": null,
-  "lambda_ae_config": null,
-  "lambda_mt_config": null,
-  "lambda_bt_config": null,
-  "hyp_path": "/checkpoint/aconneau/dumped/xlm_17_100_big.3/16656237/hypotheses",
-  "ref_paths": {},
   "mono_list": [
     "en",
     "fr",
@@ -334,5 +305,50 @@
     "hi",
     "vi"
   ],
-  "para_list": []
+  "mt_steps": [],
+  "multi_gpu": true,
+  "multi_node": true,
+  "n_gpu_per_node": 8,
+  "n_heads": 16,
+  "n_langs": 17,
+  "n_layers": 16,
+  "n_nodes": 4,
+  "node_id": 0,
+  "optimizer": "adam_inverse_sqrt,lr=0.00005,warmup_updates=30000,beta1=0.9,beta2=0.999,weight_decay=0.01,eps=0.000001",
+  "pad_index": 2,
+  "pad_token_id": 2,
+  "para_dataset": {},
+  "para_list": [],
+  "pc_steps": [],
+  "ref_paths": {},
+  "reload_checkpoint": "",
+  "reload_emb": "",
+  "reload_model": "/checkpoint/aconneau/dumped/xlm_17_100_240_big_model_upper.2/14884510/best-valid_zh_mlm_ppl.pth",
+  "sample_alpha": 0.5,
+  "save_periodic": 0,
+  "share_inout_emb": true,
+  "sinusoidal_embeddings": false,
+  "split_data": true,
+  "start_n_top": 5,
+  "stopping_criterion": "_valid_zh_mlm_ppl,25",
+  "summary_activation": null,
+  "summary_first_dropout": 0.1,
+  "summary_proj_to_labels": true,
+  "summary_type": "first",
+  "summary_use_proj": true,
+  "tokens_per_batch": -1,
+  "unk_index": 3,
+  "use_lang_emb": false,
+  "use_memory": false,
+  "validation_metrics": "_valid_en_mlm_ppl,_valid_mlm_ppl,_valid_zh_mlm_ppl",
+  "vocab_size": 200000,
+  "word_blank": 0.0,
+  "word_dropout": 0.0,
+  "word_keep": 0.1,
+  "word_mask": 0.8,
+  "word_mask_keep_rand": "0.8,0.1,0.1",
+  "word_pred": 0.15,
+  "word_rand": 0.1,
+  "word_shuffle": 0.0,
+  "world_size": 32
 }