|
@@ -221,7 +221,7 @@ struct llm_tokenizer_spm_session {
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
// seed the work queue with all possible 2-character tokens.
|
|
// seed the work queue with all possible 2-character tokens.
|
|
|
- for (size_t i = 1; i < symbols.size(); ++i) {
|
|
|
|
|
|
|
+ for (int i = 1; i < (int) symbols.size(); ++i) {
|
|
|
try_add_bigram(i - 1, i);
|
|
try_add_bigram(i - 1, i);
|
|
|
}
|
|
}
|
|
|
|
|
|
|
@@ -563,7 +563,7 @@ struct llm_tokenizer_bpe_session {
|
|
|
index++;
|
|
index++;
|
|
|
symbols.emplace_back(sym);
|
|
symbols.emplace_back(sym);
|
|
|
}
|
|
}
|
|
|
- for (size_t i = 1; i < symbols.size(); ++i) {
|
|
|
|
|
|
|
+ for (int i = 1; i < (int) symbols.size(); ++i) {
|
|
|
add_new_bigram(i - 1, i);
|
|
add_new_bigram(i - 1, i);
|
|
|
}
|
|
}
|
|
|
|
|
|