Skip to content

fixes #7999 (adds control vectors to all build_XXX() functions in llama.cpp [needs testing] #8060

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 8 commits into from
Jun 25, 2024
Merged

fixes #7999 (adds control vectors to all build_XXX() functions in llama.cpp [needs testing] #8060

Changes from 7 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
8e09261
fixes #7999
jukofyork Jun 22, 2024
af2e033
Fixes qwen2 too
jukofyork Jun 22, 2024
2c42c51
Merge branch 'ggerganov:master' into jukofyork-command_r-control-vect…
jukofyork Jun 22, 2024
f57ebf2
Fixed all models' control vectors
jukofyork Jun 22, 2024
aac1e66
Merge branch 'ggerganov:master' into jukofyork-command_r-control-vect…
jukofyork Jun 22, 2024
f393795
Removed double calls to `cb(cur, "l_out", il)`
jukofyork Jun 22, 2024
ed90e43
Merge branch 'ggerganov:master' into jukofyork-command_r-control-vect…
jukofyork Jun 25, 2024
a5a5319
Moved control vector logic to llama_control_vector:apply_to()
jukofyork Jun 25, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
144 changes: 116 additions & 28 deletions llama.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7901,8 +7901,7 @@ struct llm_build_context {
cur = ggml_add(ctx0, cur, ffn_inp);
cb(cur, "ffn_out", il);

ggml_tensor * layer_dir = lctx.cvec.tensor_for(il);
if (layer_dir != nullptr) {
if (ggml_tensor * layer_dir = lctx.cvec.tensor_for(il)) {
cur = ggml_add(ctx0, cur, layer_dir);
}
cb(cur, "l_out", il);
Expand Down Expand Up @@ -8019,6 +8018,9 @@ struct llm_build_context {
}

cur = ggml_add(ctx0, cur, ffn_inp);
if (ggml_tensor * layer_dir = lctx.cvec.tensor_for(il)) {
cur = ggml_add(ctx0, cur, layer_dir);
}
cb(cur, "l_out", il);

// input for next layer
Expand Down Expand Up @@ -8123,6 +8125,9 @@ struct llm_build_context {
}

cur = ggml_add(ctx0, cur, ffn_inp);
if (ggml_tensor * layer_dir = lctx.cvec.tensor_for(il)) {
cur = ggml_add(ctx0, cur, layer_dir);
}
cb(cur, "l_out", il);

// input for next layer
Expand Down Expand Up @@ -8238,9 +8243,10 @@ struct llm_build_context {
}

cur = ggml_add(ctx0, cur, ffn_inp);
cb(cur, "l_out", il);

cur = ggml_add(ctx0, cur, inpL);
if (ggml_tensor * layer_dir = lctx.cvec.tensor_for(il)) {
cur = ggml_add(ctx0, cur, layer_dir);
}
cb(cur, "l_out", il);

// input for next layer
Expand Down Expand Up @@ -8392,8 +8398,7 @@ struct llm_build_context {
cur = ggml_add(ctx0, cur, ffn_inp);
cb(cur, "ffn_out", il);

ggml_tensor * layer_dir = lctx.cvec.tensor_for(il);
if (layer_dir != nullptr) {
if (ggml_tensor * layer_dir = lctx.cvec.tensor_for(il)) {
cur = ggml_add(ctx0, cur, layer_dir);
}
cb(cur, "l_out", il);
Expand Down Expand Up @@ -8526,8 +8531,7 @@ struct llm_build_context {
cur = ggml_add(ctx0, cur, ffn_inp);
cb(cur, "ffn_out", il);

ggml_tensor * layer_dir = lctx.cvec.tensor_for(il);
if (layer_dir != nullptr) {
if (ggml_tensor * layer_dir = lctx.cvec.tensor_for(il)) {
cur = ggml_add(ctx0, cur, layer_dir);
}
cb(cur, "l_out", il);
Expand Down Expand Up @@ -8635,8 +8639,14 @@ struct llm_build_context {
cb(cur, "ffn_out", il);
}

inpL = ggml_add(ctx0, cur, ffn_inp);
cb(inpL, "l_out", il);
cur = ggml_add(ctx0, cur, ffn_inp);
if (ggml_tensor * layer_dir = lctx.cvec.tensor_for(il)) {
cur = ggml_add(ctx0, cur, layer_dir);
}
cb(cur, "l_out", il);

// input for next layer
inpL = cur;
}

cur = llm_build_norm(ctx0, inpL, hparams,
Expand Down Expand Up @@ -8724,6 +8734,9 @@ struct llm_build_context {
}

cur = ggml_add(ctx0, cur, ffn_inp);
if (ggml_tensor * layer_dir = lctx.cvec.tensor_for(il)) {
cur = ggml_add(ctx0, cur, layer_dir);
}
cb(cur, "l_out", il);

// input for next layer
Expand Down Expand Up @@ -9019,8 +9032,14 @@ struct llm_build_context {
cb(cur, "ffn_out", il);
}

inpL = ggml_add(ctx0, cur, ffn_inp);
cb(inpL, "l_out", il);
cur = ggml_add(ctx0, cur, ffn_inp);
if (ggml_tensor * layer_dir = lctx.cvec.tensor_for(il)) {
cur = ggml_add(ctx0, cur, layer_dir);
}
cb(cur, "l_out", il);

// input for next layer
inpL = cur;
}

cur = llm_build_norm(ctx0, inpL, hparams,
Expand Down Expand Up @@ -9154,6 +9173,9 @@ struct llm_build_context {
}

cur = ggml_add(ctx0, cur, ffn_inp);
if (ggml_tensor * layer_dir = lctx.cvec.tensor_for(il)) {
cur = ggml_add(ctx0, cur, layer_dir);
}
cb(cur, "l_out", il);

// input for next layer
Expand Down Expand Up @@ -9302,6 +9324,9 @@ struct llm_build_context {
}

cur = ggml_add(ctx0, cur, ffn_inp);
if (ggml_tensor * layer_dir = lctx.cvec.tensor_for(il)) {
cur = ggml_add(ctx0, cur, layer_dir);
}
cb(cur, "l_out", il);

// input for next layer
Expand Down Expand Up @@ -9414,6 +9439,9 @@ struct llm_build_context {
}

cur = ggml_add(ctx0, cur, ffn_inp);
if (ggml_tensor * layer_dir = lctx.cvec.tensor_for(il)) {
cur = ggml_add(ctx0, cur, layer_dir);
}
cb(cur, "l_out", il);

// input for next layer
Expand Down Expand Up @@ -9525,6 +9553,9 @@ struct llm_build_context {
cb(cur, "ffn_out", il);

cur = ggml_add(ctx0, cur, ffn_inp);
if (ggml_tensor * layer_dir = lctx.cvec.tensor_for(il)) {
cur = ggml_add(ctx0, cur, layer_dir);
}
cb(cur, "l_out", il);

// input for next layer
Expand Down Expand Up @@ -9670,6 +9701,9 @@ struct llm_build_context {
}

cur = ggml_add(ctx0, cur, ffn_inp);
if (ggml_tensor * layer_dir = lctx.cvec.tensor_for(il)) {
cur = ggml_add(ctx0, cur, layer_dir);
}
cb(cur, "l_out", il);

// input for next layer
Expand Down Expand Up @@ -9790,11 +9824,13 @@ struct llm_build_context {
}

cur = ggml_add(ctx0, cur, ffn_output);
cb(cur, "l_out", il);

cur = ggml_add(ctx0, cur, inpL);
if (ggml_tensor * layer_dir = lctx.cvec.tensor_for(il)) {
cur = ggml_add(ctx0, cur, layer_dir);
}
cb(cur, "l_out", il);

// input for next layer
inpL = cur;
}

Expand Down Expand Up @@ -9926,8 +9962,12 @@ struct llm_build_context {
}

cur = ggml_add(ctx0, residual, cur);
if (ggml_tensor * layer_dir = lctx.cvec.tensor_for(il)) {
cur = ggml_add(ctx0, cur, layer_dir);
}
cb(cur, "l_out", il);

// input for next layer
inpL = cur;
}

Expand Down Expand Up @@ -10026,9 +10066,10 @@ struct llm_build_context {
}

cur = ggml_add(ctx0, cur, sa_out);
cb(cur, "l_out", il);

cur = ggml_add(ctx0, cur, inpL);
if (ggml_tensor * layer_dir = lctx.cvec.tensor_for(il)) {
cur = ggml_add(ctx0, cur, layer_dir);
}
cb(cur, "l_out", il);

// input for next layer
Expand Down Expand Up @@ -10134,8 +10175,14 @@ struct llm_build_context {
cb(cur, "ffn_out", il);
}

inpL = ggml_add(ctx0, cur, ffn_inp);
cb(inpL, "l_out", il);
cur = ggml_add(ctx0, cur, ffn_inp);
if (ggml_tensor * layer_dir = lctx.cvec.tensor_for(il)) {
cur = ggml_add(ctx0, cur, layer_dir);
}
cb(cur, "l_out", il);

// input for next layer
inpL = cur;
}

cur = llm_build_norm(ctx0, inpL, hparams,
Expand Down Expand Up @@ -10241,8 +10288,14 @@ struct llm_build_context {
cb(cur, "ffn_out", il);
}

inpL = ggml_add(ctx0, cur, ffn_inp);
cb(inpL, "l_out", il);
cur = ggml_add(ctx0, cur, ffn_inp);
if (ggml_tensor * layer_dir = lctx.cvec.tensor_for(il)) {
cur = ggml_add(ctx0, cur, layer_dir);
}
cb(cur, "l_out", il);

// input for next layer
inpL = cur;
}

cur = llm_build_norm(ctx0, inpL, hparams,
Expand Down Expand Up @@ -10354,6 +10407,9 @@ struct llm_build_context {
cb(cur, "ffn_out", il);

cur = ggml_add(ctx0, cur, ffn_inp);
if (ggml_tensor * layer_dir = lctx.cvec.tensor_for(il)) {
cur = ggml_add(ctx0, cur, layer_dir);
}
cb(cur, "l_out", il);

// input for next layer
Expand Down Expand Up @@ -10471,6 +10527,9 @@ struct llm_build_context {
cb(cur, "ffn_out", il);

cur = ggml_add(ctx0, cur, ffn_inp);
if (ggml_tensor * layer_dir = lctx.cvec.tensor_for(il)) {
cur = ggml_add(ctx0, cur, layer_dir);
}
cb(cur, "l_out", il);

// input for next layer
Expand Down Expand Up @@ -10612,6 +10671,9 @@ struct llm_build_context {
cb(cur, "hidden_scaled_ffn", -1);

cur = ggml_add(ctx0, cur, ffn_inp);
if (ggml_tensor * layer_dir = lctx.cvec.tensor_for(il)) {
cur = ggml_add(ctx0, cur, layer_dir);
}
cb(cur, "l_out", il);

// input for next layer
Expand Down Expand Up @@ -10724,6 +10786,9 @@ struct llm_build_context {
}

cur = ggml_add(ctx0, cur, sa_out);
if (ggml_tensor * layer_dir = lctx.cvec.tensor_for(il)) {
cur = ggml_add(ctx0, cur, layer_dir);
}
cb(cur, "l_out", il);

// input for next layer
Expand Down Expand Up @@ -10840,7 +10905,11 @@ struct llm_build_context {
NULL,
LLM_FFN_GELU, LLM_FFN_SEQ, cb, il);
cb(cur, "ffn_out", il);

cur = ggml_add(ctx0, cur, ffn_inp);
if (ggml_tensor * layer_dir = lctx.cvec.tensor_for(il)) {
cur = ggml_add(ctx0, cur, layer_dir);
}
cb(cur, "l_out", il);

// input for next layer
Expand Down Expand Up @@ -10989,6 +11058,9 @@ struct llm_build_context {

// residual
cur = ggml_add(ctx0, cur, inpL);
if (ggml_tensor * layer_dir = lctx.cvec.tensor_for(il)) {
cur = ggml_add(ctx0, cur, layer_dir);
}
cb(cur, "l_out", il);

// input for next layer
Expand Down Expand Up @@ -11130,6 +11202,9 @@ struct llm_build_context {
// add together residual + FFN + self-attention
cur = ggml_add(ctx0, cur, inpL);
cur = ggml_add(ctx0, cur, attn_out);
if (ggml_tensor * layer_dir = lctx.cvec.tensor_for(il)) {
cur = ggml_add(ctx0, cur, layer_dir);
}
cb(cur, "l_out", il);

// input for next layer
Expand Down Expand Up @@ -11265,8 +11340,7 @@ struct llm_build_context {
cur = ggml_add(ctx0, cur, ffn_inp);
cb(cur, "ffn_out", il);

ggml_tensor * layer_dir = lctx.cvec.tensor_for(il);
if (layer_dir != nullptr) {
if (ggml_tensor * layer_dir = lctx.cvec.tensor_for(il)) {
cur = ggml_add(ctx0, cur, layer_dir);
}
cb(cur, "l_out", il);
Expand Down Expand Up @@ -11382,8 +11456,14 @@ struct llm_build_context {
cur = ggml_add(ctx0, cur, inpL);
cb(cur, "ffn_out", il);

inpL = ggml_add(ctx0, cur, attn_out);
cb(inpL, "l_out", il);
cur = ggml_add(ctx0, cur, attn_out);
if (ggml_tensor * layer_dir = lctx.cvec.tensor_for(il)) {
cur = ggml_add(ctx0, cur, layer_dir);
}
cb(cur, "l_out", il);

// input for next layer
inpL = cur;
} else {
// attention and ffn are computed sequentially
// x = x + attn(ln1(x))
Expand All @@ -11406,8 +11486,14 @@ struct llm_build_context {
LLM_FFN_GELU, LLM_FFN_SEQ, cb, il);
cb(cur, "ffn_out", il);

inpL = ggml_add(ctx0, cur, ffn_inp);
cb(inpL, "l_out", il);
cur = ggml_add(ctx0, cur, ffn_inp);
if (ggml_tensor * layer_dir = lctx.cvec.tensor_for(il)) {
cur = ggml_add(ctx0, cur, layer_dir);
}
cb(cur, "l_out", il);

// input for next layer
inpL = cur;
}
}

Expand Down Expand Up @@ -11534,8 +11620,7 @@ struct llm_build_context {
cur = ggml_add(ctx0, cur, ffn_out);
cb(cur, "ffn_out", il);

ggml_tensor * layer_dir = lctx.cvec.tensor_for(il);
if (layer_dir != nullptr) {
if (ggml_tensor * layer_dir = lctx.cvec.tensor_for(il)) {
cur = ggml_add(ctx0, cur, layer_dir);
}
cb(cur, "l_out", il);
Expand Down Expand Up @@ -11770,6 +11855,9 @@ struct llm_build_context {
}

cur = ggml_add(ctx0, cur, ffn_inp);
if (ggml_tensor * layer_dir = lctx.cvec.tensor_for(il)) {
cur = ggml_add(ctx0, cur, layer_dir);
}
cb(cur, "l_out", il);

// input for next layer
Expand Down
Loading