Skip to content

Commit a03ce38

Browse files
authored
finetune : fix #3404 (#3437)
the shapes for init model of gqa models was wrong
1 parent a847676 commit a03ce38

File tree

1 file changed

+2
-2
lines changed

1 file changed

+2
-2
lines changed

examples/finetune/finetune.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -332,8 +332,8 @@ static void init_model(struct llama_model * input, struct my_llama_model * model
332332

333333
assert_shape_1d(layer.attention_norm, hparams.n_embd);
334334
assert_shape_2d(layer.wq, hparams.n_embd, hparams.n_embd);
335-
assert_shape_2d(layer.wk, hparams.n_embd, hparams.n_embd);
336-
assert_shape_2d(layer.wv, hparams.n_embd, hparams.n_embd);
335+
assert_shape_2d(layer.wk, hparams.n_embd, hparams.n_embd_gqa());
336+
assert_shape_2d(layer.wv, hparams.n_embd, hparams.n_embd_gqa());
337337
assert_shape_2d(layer.wo, hparams.n_embd, hparams.n_embd);
338338
assert_shape_1d(layer.ffn_norm, hparams.n_embd);
339339
assert_shape_2d(layer.w1, hparams.n_embd, hparams.n_ff);

0 commit comments

Comments
 (0)