Add qwen baichuan ci (#7324)

* add qwen alibi-baichuan ci

* attention mask fix

* add tiktoken requirement

---------

Co-authored-by: 骑马小猫 <[email protected]>

Author: wtmlon

paddlenlp/transformers/qwen/modeling.py

@@ -841,7 +841,13 @@ def update_model_kwargs_for_generation(outputs, model_kwargs, is_encoder_decoder
 
         # update attention_mask
         if not is_encoder_decoder and "attention_mask" in model_kwargs:
-            model_kwargs["attention_mask"] = None
+            attention_mask = model_kwargs["attention_mask"]
+            if attention_mask is not None and len(attention_mask.shape) == 2:
+                model_kwargs["attention_mask"] = paddle.concat(
+                    [attention_mask, paddle.ones([attention_mask.shape[0], 1], dtype=attention_mask.dtype)], axis=-1
+                )
+            else:
+                model_kwargs["attention_mask"] = None
 
         return model_kwargs
 

scripts/regression/requirements_ci.txt

@@ -41,4 +41,4 @@ sacremoses
 soundfile
 librosa
 gradio
-tiktoken
+tiktoken

tests/fixtures/llm/finetune.yaml

@@ -34,6 +34,10 @@ finetune:
       model_name_or_path: __internal_testing__/tiny-random-chatglm2
     bloom:
       model_name_or_path: __internal_testing__/tiny-fused-bloom
+    qwen:
+      model_name_or_path: __internal_testing__/tiny-fused-qwen
+    baichuan:
+      model_name_or_path: __internal_testing__/tiny-fused-baichuan
 
 inference-predict:
   default:

tests/fixtures/llm/lora.yaml

@@ -36,6 +36,10 @@ lora:
       model_name_or_path: __internal_testing__/tiny-random-chatglm2
     bloom:
       model_name_or_path: __internal_testing__/tiny-fused-bloom
+    qwen:
+      model_name_or_path: __internal_testing__/tiny-fused-qwen
+    baichuan:
+      model_name_or_path: __internal_testing__/tiny-fused-baichuan
 
 inference-predict:
   default:

tests/fixtures/llm/prefix_tuning.yaml

@@ -35,6 +35,10 @@ prefix_tuning:
       model_name_or_path: __internal_testing__/tiny-random-chatglm2
     bloom:
       model_name_or_path: __internal_testing__/tiny-fused-bloom
+    qwen:
+      model_name_or_path: __internal_testing__/tiny-fused-qwen
+    baichuan:
+      model_name_or_path: __internal_testing__/tiny-fused-baichuan
 
 inference-predict:
   default:

tests/llm/test_finetune.py

@@ -25,7 +25,7 @@
 
 @parameterized_class(
     ["model_dir"],
-    [["llama"], ["chatglm"], ["bloom"], ["chatglm2"]],
+    [["llama"], ["chatglm"], ["bloom"], ["chatglm2"], ["qwen"], ["baichuan"]],
 )
 class FinetuneTest(LLMTest, unittest.TestCase):
     config_path: str = "./tests/fixtures/llm/finetune.yaml"
@@ -50,7 +50,7 @@ def test_finetune(self):
 
             main()
 
-        if self.model_dir != "opt" and self.model_dir != "chatglm2":
+        if self.model_dir not in ["opt", "chatglm2", "qwen", "baichuan"]:
             self.run_predictor({"inference_model": True})
 
         self.run_predictor({"inference_model": False})

tests/llm/test_lora.py

@@ -32,6 +32,8 @@
         ["chatglm"],
         ["chatglm2"],
         ["bloom"],
+        ["qwen"],
+        ["baichuan"],
     ],
 )
 class LoraTest(LLMTest, unittest.TestCase):
@@ -73,7 +75,7 @@ def test_lora(self):
 
             merge()
 
-        if self.model_dir not in ["chatglm2"]:
+        if self.model_dir not in ["chatglm2", "qwen", "baichuan"]:
             self.run_predictor({"inference_model": True})
 
         self.run_predictor({"inference_model": False})

tests/llm/test_prefix_tuning.py

@@ -31,6 +31,8 @@
         ["bloom"],
         ["chatglm"],
         ["chatglm2"],
+        ["qwen"],
+        ["baichuan"],
     ],
 )
 class PrefixTuningTest(LLMTest, unittest.TestCase):
@@ -58,7 +60,7 @@ def test_prefix_tuning(self):
 
             main()
 
-        if self.model_dir not in ["chatglm2"]:
+        if self.model_dir not in ["chatglm2", "qwen", "baichuan"]:
             self.run_predictor(
                 {
                     "inference_model": True,