|
|
@@ -3078,7 +3078,7 @@ int32_t llama_vocab::tokenize(
|
|
|
LLAMA_LOG_ERROR("%s: tokenization result size %zu exceeds int32_t limit\n", __func__, res.size());
|
|
|
return std::numeric_limits<int32_t>::min();
|
|
|
}
|
|
|
-
|
|
|
+
|
|
|
if (n_tokens_max < (int) res.size()) {
|
|
|
// LLAMA_LOG_ERROR("%s: too many tokens\n", __func__);
|
|
|
return -((int) res.size());
|