|
|
@@ -266,11 +266,10 @@ class Model:
|
|
|
toktypes.append(gguf.TokenType.USER_DEFINED)
|
|
|
elif reverse_vocab[i] in added_vocab:
|
|
|
tokens.append(reverse_vocab[i])
|
|
|
- if hasattr(tokenizer, "added_tokens_decoder"):
|
|
|
- if tokenizer.added_tokens_decoder[i].special:
|
|
|
- toktypes.append(gguf.TokenType.CONTROL)
|
|
|
- else:
|
|
|
- toktypes.append(gguf.TokenType.USER_DEFINED)
|
|
|
+ if tokenizer.added_tokens_decoder[i].special:
|
|
|
+ toktypes.append(gguf.TokenType.CONTROL)
|
|
|
+ else:
|
|
|
+ toktypes.append(gguf.TokenType.USER_DEFINED)
|
|
|
else:
|
|
|
tokens.append(reverse_vocab[i])
|
|
|
toktypes.append(gguf.TokenType.NORMAL)
|