Browse Source

server: fix crash with model not having BOS/EOS (#18321)

Xuan-Son Nguyen 1 month ago
parent
commit
849d021104
1 changed files with 8 additions and 2 deletions
  1. 8 2
      tools/server/server-context.cpp

+ 8 - 2
tools/server/server-context.cpp

@@ -2784,6 +2784,12 @@ server_response_reader server_context::get_response_reader() {
 
 
 server_context_meta server_context::get_meta() const {
 server_context_meta server_context::get_meta() const {
     auto tool_use_src = common_chat_templates_source(impl->chat_templates.get(), "tool_use");
     auto tool_use_src = common_chat_templates_source(impl->chat_templates.get(), "tool_use");
+
+    auto bos_id = llama_vocab_bos(impl->vocab);
+    auto eos_id = llama_vocab_eos(impl->vocab);
+    auto bos_token_str = bos_id != LLAMA_TOKEN_NULL ? common_token_to_piece(impl->ctx, bos_id, true) : "";
+    auto eos_token_str = eos_id != LLAMA_TOKEN_NULL ? common_token_to_piece(impl->ctx, eos_id, true) : "";
+
     return server_context_meta {
     return server_context_meta {
         /* build_info             */ build_info,
         /* build_info             */ build_info,
         /* model_name             */ impl->model_name,
         /* model_name             */ impl->model_name,
@@ -2798,8 +2804,8 @@ server_context_meta server_context::get_meta() const {
         /* chat_template          */ common_chat_templates_source(impl->chat_templates.get()),
         /* chat_template          */ common_chat_templates_source(impl->chat_templates.get()),
         /* chat_template_tool_use */ tool_use_src ? tool_use_src : "",
         /* chat_template_tool_use */ tool_use_src ? tool_use_src : "",
 
 
-        /* bos_token_str          */ common_token_to_piece(impl->ctx, llama_vocab_bos(impl->vocab), true),
-        /* eos_token_str          */ common_token_to_piece(impl->ctx, llama_vocab_eos(impl->vocab), true),
+        /* bos_token_str          */ bos_token_str,
+        /* eos_token_str          */ eos_token_str,
         /* fim_pre_token          */ llama_vocab_fim_pre(impl->vocab),
         /* fim_pre_token          */ llama_vocab_fim_pre(impl->vocab),
         /* fim_sub_token          */ llama_vocab_fim_suf(impl->vocab),
         /* fim_sub_token          */ llama_vocab_fim_suf(impl->vocab),
         /* fim_mid_token          */ llama_vocab_fim_mid(impl->vocab),
         /* fim_mid_token          */ llama_vocab_fim_mid(impl->vocab),