Browse Source

server : do context shift only while generating (#17000)

Georgi Gerganov 2 months ago
parent
commit
66d8eccd42
1 changed files with 1 additions and 1 deletions
  1. 1 1
      tools/server/server.cpp

+ 1 - 1
tools/server/server.cpp

@@ -3587,7 +3587,7 @@ struct server_context {
         // apply context-shift if needed
         // TODO: simplify and improve
         for (server_slot & slot : slots) {
-            if (slot.is_processing() && slot.prompt.n_tokens() + 1 >= slot.n_ctx) {
+            if (slot.state == SLOT_STATE_GENERATING && slot.prompt.n_tokens() + 1 >= slot.n_ctx) {
                 if (!params_base.ctx_shift) {
                     // this check is redundant (for good)
                     // we should never get here, because generation should already stopped in process_token()