Update BertForJointParsing.py
Browse files- BertForJointParsing.py +1 -0
BertForJointParsing.py
CHANGED
|
@@ -314,6 +314,7 @@ def lex_parse_logits(input_ids: List[List[int]], sentences: List[str], tokenizer
|
|
| 314 |
|
| 315 |
special_toks = tokenizer.all_special_tokens
|
| 316 |
special_toks.remove(tokenizer.unk_token)
|
|
|
|
| 317 |
for batch_idx in range(len(sentences)):
|
| 318 |
intermediate_ret = []
|
| 319 |
tokens = tokenizer.convert_ids_to_tokens(input_ids[batch_idx])
|
|
|
|
| 314 |
|
| 315 |
special_toks = tokenizer.all_special_tokens
|
| 316 |
special_toks.remove(tokenizer.unk_token)
|
| 317 |
+
special_toks.remove(tokenizer.mask_token)
|
| 318 |
for batch_idx in range(len(sentences)):
|
| 319 |
intermediate_ret = []
|
| 320 |
tokens = tokenizer.convert_ids_to_tokens(input_ids[batch_idx])
|