Просмотр исходного кода

server : send token probs for "stream == false" (#4714)

Georgi Gerganov 2 лет назад
Родитель
Сommit
012cf349ae
1 измененных файлов с 2 добавлено и 2 удалено
  1. 2 2
      examples/server/server.cpp

+ 2 - 2
examples/server/server.cpp

@@ -1265,7 +1265,7 @@ struct llama_server_context
         {
         {
             std::vector<completion_token_output> probs_output = {};
             std::vector<completion_token_output> probs_output = {};
             const std::vector<llama_token> to_send_toks = llama_tokenize(ctx, tkn.text_to_send, false);
             const std::vector<llama_token> to_send_toks = llama_tokenize(ctx, tkn.text_to_send, false);
-            size_t probs_pos = std::min(slot.sent_token_probs_index, slot.generated_token_probs.size());
+            size_t probs_pos      = std::min(slot.sent_token_probs_index,                       slot.generated_token_probs.size());
             size_t probs_stop_pos = std::min(slot.sent_token_probs_index + to_send_toks.size(), slot.generated_token_probs.size());
             size_t probs_stop_pos = std::min(slot.sent_token_probs_index + to_send_toks.size(), slot.generated_token_probs.size());
             if (probs_pos < probs_stop_pos)
             if (probs_pos < probs_stop_pos)
             {
             {
@@ -1325,7 +1325,7 @@ struct llama_server_context
             {
             {
                 probs = std::vector<completion_token_output>(
                 probs = std::vector<completion_token_output>(
                                     slot.generated_token_probs.begin(),
                                     slot.generated_token_probs.begin(),
-                                    slot.generated_token_probs.begin() + slot.sent_token_probs_index);
+                                    slot.generated_token_probs.end());
             }
             }
             res.result_json["completion_probabilities"] = probs_vector_to_json(ctx, probs);
             res.result_json["completion_probabilities"] = probs_vector_to_json(ctx, probs);
         }
         }