|
|
@@ -973,8 +973,6 @@ class XverseModel(Model):
|
|
|
if max_vocab_index >= vocab_size:
|
|
|
raise ValueError("Vocabulary size exceeds expected maximum size.")
|
|
|
|
|
|
-
|
|
|
-
|
|
|
reverse_vocab: dict[int, str] = {id_: encoded_tok for encoded_tok, id_ in tokenizer.vocab.items()}
|
|
|
added_vocab = tokenizer.get_added_vocab()
|
|
|
|