|
|
@@ -202,6 +202,7 @@ enum llm_arch {
|
|
|
LLM_ARCH_COMMAND_R,
|
|
|
LLM_ARCH_DBRX,
|
|
|
LLM_ARCH_OLMO,
|
|
|
+ LLM_ARCH_OLMOE,
|
|
|
LLM_ARCH_OPENELM,
|
|
|
LLM_ARCH_ARCTIC,
|
|
|
LLM_ARCH_DEEPSEEK2,
|
|
|
@@ -251,6 +252,7 @@ static const std::map<llm_arch, const char *> LLM_ARCH_NAMES = {
|
|
|
{ LLM_ARCH_COMMAND_R, "command-r" },
|
|
|
{ LLM_ARCH_DBRX, "dbrx" },
|
|
|
{ LLM_ARCH_OLMO, "olmo" },
|
|
|
+ { LLM_ARCH_OLMOE, "olmoe" },
|
|
|
{ LLM_ARCH_OPENELM, "openelm" },
|
|
|
{ LLM_ARCH_ARCTIC, "arctic" },
|
|
|
{ LLM_ARCH_DEEPSEEK2, "deepseek2" },
|
|
|
@@ -1193,6 +1195,26 @@ static const std::map<llm_arch, std::map<llm_tensor, std::string>> LLM_TENSOR_NA
|
|
|
{ LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },
|
|
|
},
|
|
|
},
|
|
|
+ {
|
|
|
+ LLM_ARCH_OLMOE,
|
|
|
+ {
|
|
|
+ { LLM_TENSOR_TOKEN_EMBD, "token_embd" },
|
|
|
+ { LLM_TENSOR_OUTPUT_NORM, "output_norm" },
|
|
|
+ { LLM_TENSOR_OUTPUT, "output" },
|
|
|
+ { LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" },
|
|
|
+ { LLM_TENSOR_ATTN_Q, "blk.%d.attn_q" },
|
|
|
+ { LLM_TENSOR_ATTN_K, "blk.%d.attn_k" },
|
|
|
+ { LLM_TENSOR_ATTN_V, "blk.%d.attn_v" },
|
|
|
+ { LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" },
|
|
|
+ { LLM_TENSOR_ATTN_Q_NORM, "blk.%d.attn_q_norm" },
|
|
|
+ { LLM_TENSOR_ATTN_K_NORM, "blk.%d.attn_k_norm" },
|
|
|
+ { LLM_TENSOR_FFN_NORM, "blk.%d.ffn_norm" },
|
|
|
+ { LLM_TENSOR_FFN_GATE_INP, "blk.%d.ffn_gate_inp" },
|
|
|
+ { LLM_TENSOR_FFN_GATE_EXPS, "blk.%d.ffn_gate_exps" },
|
|
|
+ { LLM_TENSOR_FFN_DOWN_EXPS, "blk.%d.ffn_down_exps" },
|
|
|
+ { LLM_TENSOR_FFN_UP_EXPS, "blk.%d.ffn_up_exps" },
|
|
|
+ },
|
|
|
+ },
|
|
|
{
|
|
|
LLM_ARCH_OPENELM,
|
|
|
{
|
|
|
@@ -2277,6 +2299,7 @@ enum e_model {
|
|
|
MODEL_MEDIUM,
|
|
|
MODEL_LARGE,
|
|
|
MODEL_XL,
|
|
|
+ MODEL_A1_7B,
|
|
|
MODEL_A2_7B,
|
|
|
MODEL_8x7B,
|
|
|
MODEL_8x22B,
|
|
|
@@ -5241,6 +5264,7 @@ static const char * llama_model_type_name(e_model type) {
|
|
|
case MODEL_MEDIUM: return "0.4B";
|
|
|
case MODEL_LARGE: return "0.8B";
|
|
|
case MODEL_XL: return "1.5B";
|
|
|
+ case MODEL_A1_7B: return "A1.7B";
|
|
|
case MODEL_A2_7B: return "A2.7B";
|
|
|
case MODEL_8x7B: return "8x7B";
|
|
|
case MODEL_8x22B: return "8x22B";
|
|
|
@@ -5791,6 +5815,14 @@ static void llm_load_hparams(
|
|
|
default: model.type = e_model::MODEL_UNKNOWN;
|
|
|
}
|
|
|
} break;
|
|
|
+ case LLM_ARCH_OLMOE:
|
|
|
+ {
|
|
|
+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
|
|
|
+ switch (hparams.n_layer) {
|
|
|
+ case 16: model.type = e_model::MODEL_A1_7B; break;
|
|
|
+ default: model.type = e_model::MODEL_UNKNOWN;
|
|
|
+ }
|
|
|
+ } break;
|
|
|
case LLM_ARCH_OPENELM:
|
|
|
{
|
|
|
ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
|
|
|
@@ -8018,6 +8050,44 @@ static bool llm_load_tensors(
|
|
|
layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff});
|
|
|
}
|
|
|
} break;
|
|
|
+ case LLM_ARCH_OLMOE:
|
|
|
+ {
|
|
|
+ model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab});
|
|
|
+
|
|
|
+ // output
|
|
|
+ {
|
|
|
+ model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd});
|
|
|
+ model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab});
|
|
|
+ }
|
|
|
+
|
|
|
+ for (int i = 0; i < n_layer; ++i) {
|
|
|
+ ggml_context * ctx_layer = ctx_for_layer(i);
|
|
|
+ ggml_context * ctx_split = ctx_for_layer_split(i);
|
|
|
+
|
|
|
+ auto & layer = model.layers[i];
|
|
|
+
|
|
|
+ layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd});
|
|
|
+
|
|
|
+ layer.wq = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd});
|
|
|
+ layer.wk = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_gqa});
|
|
|
+ layer.wv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa});
|
|
|
+ layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd});
|
|
|
+ layer.attn_q_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_Q_NORM, "weight", i), {n_embd});
|
|
|
+ layer.attn_k_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_K_NORM, "weight", i), {n_embd});
|
|
|
+
|
|
|
+ layer.ffn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd});
|
|
|
+
|
|
|
+ layer.ffn_gate_inp = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert});
|
|
|
+
|
|
|
+ GGML_ASSERT(n_expert > 0);
|
|
|
+ GGML_ASSERT(n_expert_used > 0);
|
|
|
+
|
|
|
+ // MoE branch
|
|
|
+ layer.ffn_gate_exps = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), {n_embd, n_ff, n_expert});
|
|
|
+ layer.ffn_down_exps = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff, n_embd, n_expert});
|
|
|
+ layer.ffn_up_exps = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), {n_embd, n_ff, n_expert});
|
|
|
+ }
|
|
|
+ } break;
|
|
|
case LLM_ARCH_OPENELM:
|
|
|
{
|
|
|
model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab});
|
|
|
@@ -13832,6 +13902,134 @@ struct llm_build_context {
|
|
|
return gf;
|
|
|
}
|
|
|
|
|
|
+ // based on the build_qwen2moe() function, changes:
|
|
|
+ // * removed shared experts
|
|
|
+ // * removed bias
|
|
|
+ // * added q, k norm
|
|
|
+ struct ggml_cgraph * build_olmoe() {
|
|
|
+ struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, llama_model_max_nodes(model), false);
|
|
|
+
|
|
|
+ // mutable variable, needed during the last layer of the computation to skip unused tokens
|
|
|
+ int32_t n_tokens = this->n_tokens;
|
|
|
+
|
|
|
+ const int64_t n_embd_head = hparams.n_embd_head_v;
|
|
|
+ GGML_ASSERT(n_embd_head == hparams.n_embd_head_k);
|
|
|
+ GGML_ASSERT(n_embd_head == hparams.n_rot);
|
|
|
+
|
|
|
+ struct ggml_tensor * cur;
|
|
|
+ struct ggml_tensor * inpL;
|
|
|
+
|
|
|
+ inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb);
|
|
|
+
|
|
|
+ // inp_pos - contains the positions
|
|
|
+ struct ggml_tensor * inp_pos = build_inp_pos();
|
|
|
+
|
|
|
+ // KQ_mask (mask for 1 head, it will be broadcasted to all heads)
|
|
|
+ struct ggml_tensor * KQ_mask = build_inp_KQ_mask();
|
|
|
+
|
|
|
+ for (int il = 0; il < n_layer; ++il) {
|
|
|
+ struct ggml_tensor * inpSA = inpL;
|
|
|
+
|
|
|
+ // norm
|
|
|
+ cur = llm_build_norm(ctx0, inpL, hparams,
|
|
|
+ model.layers[il].attn_norm, NULL,
|
|
|
+ LLM_NORM_RMS, cb, il);
|
|
|
+ cb(cur, "attn_norm", il);
|
|
|
+
|
|
|
+ // self_attention
|
|
|
+ {
|
|
|
+ // compute Q and K and RoPE them
|
|
|
+ struct ggml_tensor * Qcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wq, cur);
|
|
|
+ cb(Qcur, "Qcur", il);
|
|
|
+
|
|
|
+ struct ggml_tensor * Kcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wk, cur);
|
|
|
+ cb(Kcur, "Kcur", il);
|
|
|
+
|
|
|
+ struct ggml_tensor * Vcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wv, cur);
|
|
|
+ cb(Vcur, "Vcur", il);
|
|
|
+
|
|
|
+ Qcur = llm_build_norm(ctx0, Qcur, hparams, model.layers[il].attn_q_norm, NULL,
|
|
|
+ LLM_NORM_RMS, cb, il);
|
|
|
+ cb(Qcur, "Qcur_normed", il);
|
|
|
+
|
|
|
+ Kcur = llm_build_norm(ctx0, Kcur, hparams, model.layers[il].attn_k_norm, NULL,
|
|
|
+ LLM_NORM_RMS, cb, il);
|
|
|
+ cb(Kcur, "Kcur_normed", il);
|
|
|
+
|
|
|
+ Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
|
|
|
+ Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
|
|
|
+
|
|
|
+ Qcur = ggml_rope_ext(
|
|
|
+ ctx0, Qcur, inp_pos, nullptr,
|
|
|
+ n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
|
|
|
+ ext_factor, attn_factor, beta_fast, beta_slow
|
|
|
+ );
|
|
|
+ cb(Qcur, "Qcur_rope", il);
|
|
|
+
|
|
|
+ Kcur = ggml_rope_ext(
|
|
|
+ ctx0, Kcur, inp_pos, nullptr,
|
|
|
+ n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
|
|
|
+ ext_factor, attn_factor, beta_fast, beta_slow
|
|
|
+ );
|
|
|
+ cb(Kcur, "Kcur_rope", il);
|
|
|
+
|
|
|
+ cur = llm_build_kv(ctx0, lctx, kv_self, gf,
|
|
|
+ model.layers[il].wo, NULL,
|
|
|
+ Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);
|
|
|
+ }
|
|
|
+
|
|
|
+ if (il == n_layer - 1) {
|
|
|
+ // skip computing output for unused tokens
|
|
|
+ struct ggml_tensor * inp_out_ids = build_inp_out_ids();
|
|
|
+ n_tokens = n_outputs;
|
|
|
+ cur = ggml_get_rows(ctx0, cur, inp_out_ids);
|
|
|
+ inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);
|
|
|
+ }
|
|
|
+
|
|
|
+ struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);
|
|
|
+ cb(ffn_inp, "ffn_inp", il);
|
|
|
+
|
|
|
+ // MoE branch
|
|
|
+ cur = llm_build_norm(ctx0, ffn_inp, hparams,
|
|
|
+ model.layers[il].ffn_norm, NULL,
|
|
|
+ LLM_NORM_RMS, cb, il);
|
|
|
+ cb(cur, "ffn_norm", il);
|
|
|
+
|
|
|
+ cur = llm_build_moe_ffn(ctx0, lctx, cur,
|
|
|
+ model.layers[il].ffn_gate_inp,
|
|
|
+ model.layers[il].ffn_up_exps,
|
|
|
+ model.layers[il].ffn_gate_exps,
|
|
|
+ model.layers[il].ffn_down_exps,
|
|
|
+ n_expert, n_expert_used,
|
|
|
+ LLM_FFN_SILU, false,
|
|
|
+ false, 0.0,
|
|
|
+ cb, il);
|
|
|
+ cb(cur, "ffn_moe_out", il);
|
|
|
+
|
|
|
+ cur = ggml_add(ctx0, cur, ffn_inp);
|
|
|
+ cur = lctx.cvec.apply_to(ctx0, cur, il);
|
|
|
+ cb(cur, "l_out", il);
|
|
|
+
|
|
|
+ // input for next layer
|
|
|
+ inpL = cur;
|
|
|
+ }
|
|
|
+
|
|
|
+ cur = inpL;
|
|
|
+
|
|
|
+ cur = llm_build_norm(ctx0, cur, hparams,
|
|
|
+ model.output_norm, NULL,
|
|
|
+ LLM_NORM_RMS, cb, -1);
|
|
|
+ cb(cur, "result_norm", -1);
|
|
|
+
|
|
|
+ // lm_head
|
|
|
+ cur = llm_build_lora_mm(lctx, ctx0, model.output, cur);
|
|
|
+ cb(cur, "result_output", -1);
|
|
|
+
|
|
|
+ ggml_build_forward_expand(gf, cur);
|
|
|
+
|
|
|
+ return gf;
|
|
|
+ }
|
|
|
+
|
|
|
struct ggml_cgraph * build_openelm() {
|
|
|
struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, llama_model_max_nodes(model), false);
|
|
|
|
|
|
@@ -15712,6 +15910,10 @@ static struct ggml_cgraph * llama_build_graph(
|
|
|
{
|
|
|
result = llm.build_olmo();
|
|
|
} break;
|
|
|
+ case LLM_ARCH_OLMOE:
|
|
|
+ {
|
|
|
+ result = llm.build_olmoe();
|
|
|
+ } break;
|
|
|
case LLM_ARCH_OPENELM:
|
|
|
{
|
|
|
result = llm.build_openelm();
|
|
|
@@ -18896,6 +19098,7 @@ enum llama_rope_type llama_rope_type(const struct llama_model * model) {
|
|
|
case LLM_ARCH_QWEN:
|
|
|
case LLM_ARCH_QWEN2:
|
|
|
case LLM_ARCH_QWEN2MOE:
|
|
|
+ case LLM_ARCH_OLMOE:
|
|
|
case LLM_ARCH_PHI2:
|
|
|
case LLM_ARCH_PHI3:
|
|
|
case LLM_ARCH_GEMMA:
|
Shane A