fix bug

Author: lugimzzz

llm/run_finetune.py

@@ -162,6 +162,13 @@ def main():
         qlora_weight_blocksize=model_args.qlora_weight_blocksize,
         qlora_weight_double_quant=model_args.qlora_weight_double_quant,
         qlora_weight_double_quant_block_size=model_args.qlora_weight_double_quant_block_size,
+        apply_hadamard=model_args.apply_hadamard,
+        hadamard_is_block=model_args.hadamard_is_block,
+        hadamard_block_size=model_args.hadamard_block_size,
+        quant_input_grad=model_args.quant_input_grad,
+        apply_online_actscale_step=model_args.apply_online_actscale_step,
+        scale_epsilon=model_args.scale_epsilon,
+        moving_rate=model_args.moving_rate,
     )
 
     model_config = AutoConfig.from_pretrained(
@@ -291,7 +298,7 @@ def neft_post_hook(module, input, output):
         logging.info("Using ReFT with layers: ", reft_layers)
     # init chat_template for tokenizer
     init_chat_template(tokenizer, model_args.model_name_or_path, data_args.chat_template)
-
+    tokenizer.chat_template = None
     # if using chat_template, data_args.eval_with_do_generation must be false
     if tokenizer.chat_template is not None:
         data_args.eval_with_do_generation = False

ops/src/paddlenlp_kernel/triton/optimizer/adamw_triton.py

@@ -12,9 +12,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import paddle
 import triton
 import triton.language as tl
 
+DTYPE_MAPPING = {
+    paddle.bfloat16: tl.bfloat16,
+    paddle.float32: tl.float32,
+    paddle.float16: tl.float16,
+}
+
 
 @triton.jit
 def adamw_kernel(
@@ -30,10 +37,11 @@ def adamw_kernel(
     beta1_pow_ptr,
     beta2_pow_ptr,
     master_weight_ptr,
-    dtype,
     N,
-    BLOCK_SIZE,
     skip_update_param,
+    param_dtype: tl.constexpr,
+    moment_dtype: tl.constexpr,
+    BLOCK_SIZE: tl.constexpr,
 ):
     pid = tl.program_id(0)
     offsets = pid * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
@@ -45,11 +53,8 @@ def adamw_kernel(
         param = tl.load(param_ptr + offsets, mask=mask).to(tl.float32)
     grad = tl.load(grad_ptr + offsets, mask=mask).to(tl.float32)
 
-    moment1 = tl.load(moment1_ptr + offsets, mask=mask)
-    moment2 = tl.load(moment2_ptr + offsets, mask=mask)
-    moment_dtype = moment1.dtype
-    moment1 = moment1.to(tl.float32)
-    moment2 = moment2.to(tl.float32)
+    moment1 = tl.load(moment1_ptr + offsets, mask=mask).to(tl.float32)
+    moment2 = tl.load(moment2_ptr + offsets, mask=mask).to(tl.float32)
     lr = tl.load(lr_ptr)
     beta1_pow = tl.load(beta1_pow_ptr)
     beta2_pow = tl.load(beta2_pow_ptr)
@@ -62,13 +67,6 @@ def adamw_kernel(
     moment2 = beta2 * moment2 + (1.0 - beta2) * grad * grad
     denom = tl.sqrt(moment2) / tl.sqrt(1.0 - beta2_pow) + epsilon
     param += (moment1 / denom) * (-lr / (1 - beta1_pow))
-    if dtype == 0:
-        param_dtype = tl.float16
-    elif dtype == 1:
-        param_dtype = tl.bfloat16
-    else:
-        param_dtype = tl.float32
-
     # Update param
     if master_weight_ptr is not None:
         tl.store(master_weight_ptr + offsets, param, mask=mask)
@@ -110,12 +108,6 @@ def adamw_triton(
     N = param.numel().item()
     BLOCK_SIZE = 512
     grid = lambda meta: (triton.cdiv(N, BLOCK_SIZE),)
-    if str(param.dtype) == "paddle.float16":
-        dtype = 0
-    elif str(param.dtype) == "paddle.bfloat16":
-        dtype = 1
-    else:
-        dtype = 2
     adamw_kernel[grid](
         param,
         grad,
@@ -129,9 +121,10 @@ def adamw_triton(
         beta1_pow,
         beta2_pow,
         master_weight,
-        dtype,
         N,
-        BLOCK_SIZE,
         skip_update_param,
+        DTYPE_MAPPING[param.dtype],
+        DTYPE_MAPPING[moment1.dtype],
+        BLOCK_SIZE,
     )
     beta1_pow[:], beta2_pow[:] = beta1 * beta1_pow[:], beta2 * beta2_pow[:]

paddlenlp/quantization/hadamard_utils.py

@@ -14,6 +14,8 @@
 
 import paddle
 
+from paddlenlp.utils import infohub
+
 
 def matmul_hadU(X):
 
@@ -74,3 +76,28 @@ def hadamard_matmul(input, side, hadamard_maxtrix, block_size):
     output = output.reshape(origin_shape)
 
     return output
+
+
+def apply_hadamard_matmul(x, side, quantization_config=None, dequant=False):
+    if getattr(infohub, "hadamard") is None:
+        setattr(infohub, "hadamard", {})
+    if side == "left":
+        x_shape = x.shape[0]
+    else:
+        x_shape = x.shape[-1]
+    if x_shape in infohub.hadamard:
+        hadamard_maxtrix, block_size = infohub.hadamard[x_shape]
+    else:
+        hadamard_matrix, block_size = random_hadamard_matrix(x_shape, x.dtype, quantization_config)
+        infohub.hadamard[x_shape] = (hadamard_matrix, block_size)
+    if block_size > 1:
+        target_x = hadamard_matmul(x, side, hadamard_maxtrix, block_size)
+    else:
+        if dequant:
+            hadamard_matrix = hadamard_matrix.T
+        if side == "right":
+            target_x = x @ hadamard_matrix
+        else:
+            target_x = hadamard_matrix.T @ x
+
+    return target_x, block_size