基于开发者空间Notebook进行LoRA微调Whisper_base实现语音识别

pip install kagglehub 
pip install datasets transformers librosa 
pip install peft torch 
pip install ffmpeg-python

• 1.
• 2.
• 3.
• 4.

import kagglehub

# Download latest version
path = kagglehub.dataset_download("tenffe/common-voice-zh-cn")

print("Path to dataset files:", path)

• 1.
• 2.
• 3.
• 4.
• 5.
• 6.

cd data
python data.py

• 1.
• 2.

mv /home/service/.cache/kagglehub/datasets/tenffe/common-voice-zh-cn/versions/1 /opt/huawei/edu-apaas/src/init/data

• 1.

import kagglehub

# Download latest version
path = kagglehub.dataset_download("shawnchile/voice-female-yuri")

print("Path to dataset files:", path)

• 1.
• 2.
• 3.
• 4.
• 5.
• 6.

python data_ce.py

• 1.

mv /home/service/.cache/kagglehub/datasets/shawnchile/voice-female-yuri/versions/1 /opt/huawei/edu-apaas/src/init/data/wavs

• 1.

wget -P /opt/huawei/edu-apaas/src/init/pretrained_model wget https://dtse-mirrors.obs.cn-north-4.myhuaweicloud.com/case/0002/whisper_base.tar.gz

• 1.

tar -zxvf whisper_base.tar.gz

• 1.

import os
from datasets import load_from_disk, Audio
from transformers import WhisperFeatureExtractor, WhisperTokenizer, WhisperProcessor

# 配置参数
model_name_or_path = "/opt/huawei/edu-apaas/src/init/pretrained_model/whisper_base"
language = "chinese"
language_abbr = "zh"
task = "transcribe"
dataset_name = "/opt/huawei/edu-apaas/src/init/data/1/common_voice_zh_CN"
prepared_dataset_path = "/opt/huawei/edu-apaas/src/init/pretrained_model/prepared_common_voice"
# 加载数据集
common_voice = load_from_disk(dataset_name)

# 选择前1000个样本
common_voice['train'] = common_voice['train'].select(range(100))
if 'test' in common_voice:
    common_voice['test'] = common_voice['test'].select(range(100))
if 'validation' in common_voice:
    common_voice['validation'] = common_voice['validation'].select(range(100))

# 移除不需要的列
common_voice = common_voice.remove_columns(
    ["accent", "age", "client_id", "down_votes", "gender", "locale", "path", "segment", "up_votes"]
)

# 将采样率改为16kHz
common_voice = common_voice.cast_column("audio", Audio(sampling_rate=16000))

# 预处理音频数据和对应的文本
def prepare_dataset(batch):
    audio = batch["audio"]
    batch["input_features"] = feature_extractor(audio["array"], sampling_rate=audio["sampling_rate"]).input_features[0]
    batch["labels"] = tokenizer(batch["sentence"]).input_ids
    return batch

# 加载特征提取器和分词器
feature_extractor = WhisperFeatureExtractor.from_pretrained(model_name_or_path)
tokenizer = WhisperTokenizer.from_pretrained(model_name_or_path, language=language, task=task)
processor = WhisperProcessor.from_pretrained(model_name_or_path, language=language, task=task)

# 预处理数据集
common_voice = common_voice.map(prepare_dataset, remove_columns=common_voice.column_names["train"], num_proc=1)

# 保存预处理后的数据集
common_voice.save_to_disk(prepared_dataset_path)

• 1.
• 2.
• 3.
• 4.
• 5.
• 6.
• 7.
• 8.
• 9.
• 10.
• 11.
• 12.
• 13.
• 14.
• 15.
• 16.
• 17.
• 18.
• 19.
• 20.
• 21.
• 22.
• 23.
• 24.
• 25.
• 26.
• 27.
• 28.
• 29.
• 30.
• 31.
• 32.
• 33.
• 34.
• 35.
• 36.
• 37.
• 38.
• 39.
• 40.
• 41.
• 42.
• 43.
• 44.
• 45.
• 46.

pip install --upgrade soundfile

• 1.

cd whisper_api/
python preprocess_data.py

• 1.
• 2.

pip install torch_npu -f https://ascend-repository.obs.cn-east-2.myhuaweicloud.com/ascend-toolkit/23.0.RC3/whl/npu-cpu/

• 1.

pip install --upgrade transformers

• 1.

from transformers import WhisperForConditionalGeneration, WhisperProcessor, WhisperTokenizer, AutoConfig
from peft import LoraConfig, get_peft_model
import os
import torch

# 禁用 safetensors
os.environ["TRANSFORMERS_SAFE_TENSORS"] = "false"

model_name_or_path = "/opt/huawei/edu-apaas/src/init/pretrained_model/whisper_base" # 加载模型配置
config = AutoConfig.from_pretrained(model_name_or_path)

# 手动加载模型文件
state_dict = torch.load(f"{model_name_or_path}/pytorch_model.bin", weights_only=True)

# 初始化模型
model = WhisperForConditionalGeneration(config)

# 尝试加载 state_dict，允许忽略缺失的键
model.load_state_dict(state_dict, strict=False)

# 打印模型信息
print(model)

# 定义 LoRA 配置
config = LoraConfig(r=32, lora_alpha=64, target_modules=["q_proj", "v_proj"], lora_dropout=0.05)

# 获取 LoRA 模型
model = get_peft_model(model, config)

# 打印可训练参数
model.print_trainable_parameters()

• 1.
• 2.
• 3.
• 4.
• 5.
• 6.
• 7.
• 8.
• 9.
• 10.
• 11.
• 12.
• 13.
• 14.
• 15.
• 16.
• 17.
• 18.
• 19.
• 20.
• 21.
• 22.
• 23.
• 24.
• 25.
• 26.
• 27.
• 28.
• 29.
• 30.
• 31.

python load_model.py

• 1.

pip install transformers==4.38.0 peft==0.10.0 accelerate==0.26.0 torch_npu==2.1.0

• 1.

from transformers import Seq2SeqTrainingArguments, Seq2SeqTrainer
from dataclasses import dataclass
from typing import Any, Dict, List, Union
from transformers import WhisperFeatureExtractor, WhisperTokenizer, WhisperProcessor
from datasets import load_from_disk  # 导入 load_from_disk 函数
import torch
from peft import LoraConfig, get_peft_model, PeftModel

@dataclass
class DataCollatorSpeechSeq2SeqWithPadding:
    def __init__(self, processor: Any):
        self.processor = processor

    def __call__(self, features: List[Dict[str, Union[List[int], torch.Tensor]]]) -> Dict[str, torch.Tensor]:
        input_features = [{"input_features": feature["input_features"]} for feature in features]
        batch = self.processor.feature_extractor.pad(input_features, return_tensors="pt")

        label_features = [{"input_ids": feature["labels"]} for feature in features]
        labels_batch = self.processor.tokenizer.pad(label_features, return_tensors="pt")

        labels = labels_batch["input_ids"].masked_fill(labels_batch.attention_mask.ne(1), -100)

        if (labels[:, 0] == self.processor.tokenizer.bos_token_id).all().cpu().item():
            labels = labels[:, 1:]

        batch["labels"] = labels

        return batch

# 配置参数
model_name_or_path = "/opt/huawei/edu-apaas/src/init/pretrained_model/whisper_base"
language = "chinese"
task = "transcribe"
prepared_dataset_path = "/opt/huawei/edu-apaas/src/init/pretrained_model/prepared_common_voice"   # 预处理后的数据集路径

# 加载特征提取器和分词器
feature_extractor = WhisperFeatureExtractor.from_pretrained(model_name_or_path)
tokenizer = WhisperTokenizer.from_pretrained(model_name_or_path, language=language, task=task)
processor = WhisperProcessor.from_pretrained(model_name_or_path, language=language, task=task)

# 加载预处理后的数据集
common_voice = load_from_disk(prepared_dataset_path)

# 定义 LoRA 配置
lora_config = LoraConfig(
    r=8,  # LoRA attention dimension
    lora_alpha=32,  # Alpha scaling
    target_modules=["q_proj", "v_proj"],  # Target modules to apply LoRA
    lora_dropout=0.1,  # Dropout probability for LoRA layers
)

# 加载模型
from transformers import WhisperForConditionalGeneration
model = WhisperForConditionalGeneration.from_pretrained(model_name_or_path)

# 将 LoRA 配置应用到模型
model = get_peft_model(model, lora_config)

# 设置 Trainer 参数
training_args = Seq2SeqTrainingArguments(
    output_dir="your-name/int8-whisper-base-v2-asr",
    per_device_train_batch_size=2,
    gradient_accumulation_steps=1,
    learning_rate=2e-3,
    warmup_steps=50,
    num_train_epochs=3,
    evaluation_strategy="epoch",  # 需要 transformers ≥ 4.0.0
    fp16=True,
    per_device_eval_batch_size=4,
    generation_max_length=128,
    logging_steps=25,
    remove_unused_columns=False,
    label_names=["labels"],
    no_cuda=False,  # 允许使用 NPU/GPU
)
# 创建数据收集器
data_collator = DataCollatorSpeechSeq2SeqWithPadding(processor=processor)

# 创建 Trainer
trainer = Seq2SeqTrainer(
    args=training_args,
    model=model,
    train_dataset=common_voice["train"].select(range(100)),  # 仅使用前100个样本进行训练
    eval_dataset=common_voice["test"].select(range(100)),  # 仅使用前100个样本进行验证
    data_collator=data_collator,
    tokenizer=processor.feature_extractor,
)

# 开始训练
model.config.use_cache = False
trainer.train()

# 保存 LoRA 模型
model.save_pretrained("/opt/huawei/edu-apaas/src/init/pretrained_model/checkpoint_1000_samples")

• 1.
• 2.
• 3.
• 4.
• 5.
• 6.
• 7.
• 8.
• 9.
• 10.
• 11.
• 12.
• 13.
• 14.
• 15.
• 16.
• 17.
• 18.
• 19.
• 20.
• 21.
• 22.
• 23.
• 24.
• 25.
• 26.
• 27.
• 28.
• 29.
• 30.
• 31.
• 32.
• 33.
• 34.
• 35.
• 36.
• 37.
• 38.
• 39.
• 40.
• 41.
• 42.
• 43.
• 44.
• 45.
• 46.
• 47.
• 48.
• 49.
• 50.
• 51.
• 52.
• 53.
• 54.
• 55.
• 56.
• 57.
• 58.
• 59.
• 60.
• 61.
• 62.
• 63.
• 64.
• 65.
• 66.
• 67.
• 68.
• 69.
• 70.
• 71.
• 72.
• 73.
• 74.
• 75.
• 76.
• 77.
• 78.
• 79.
• 80.
• 81.
• 82.
• 83.
• 84.
• 85.
• 86.
• 87.
• 88.
• 89.
• 90.
• 91.
• 92.
• 93.
• 94.

python train_model.py

• 1.

from transformers import WhisperForConditionalGeneration, WhisperTokenizer, WhisperProcessor
from peft import PeftModel, PeftConfig
from transformers import AutomaticSpeechRecognitionPipeline
import torch
import os
import json

# 配置参数
peft_model_id = "/opt/huawei/edu-apaas/src/init/pretrained_model/checkpoint_1000_samples"
language = "chinese"
task = "transcribe"
output_directory = "/opt/huawei/edu-apaas/src/init/output"  # 输出文件夹路径

# 确保输出目录存在
os.makedirs(output_directory, exist_ok=True)

# 加载 LoRA 模型配置
peft_config = PeftConfig.from_pretrained(peft_model_id)

# 加载基础模型
model = WhisperForConditionalGeneration.from_pretrained(
    peft_config.base_model_name_or_path
)

# 基于基础模型和 LoRA 模型路径构建 LoRA 模型
model = PeftModel.from_pretrained(model, peft_model_id)

# 将 PeftModel 转换为普通的 WhisperForConditionalGeneration 模型
model = model.merge_and_unload()

# 加载分词器(处理文本)和处理器(封装特征抽取器和分词器)、以及特征抽取器(处理音频)
tokenizer = WhisperTokenizer.from_pretrained(peft_config.base_model_name_or_path, language=language, task=task)
processor = WhisperProcessor.from_pretrained(peft_config.base_model_name_or_path, language=language, task=task)
feature_extractor = processor.feature_extractor
forced_decoder_ids = processor.get_decoder_prompt_ids(language=language, task=task)

# 构建一个 pipeline 进行测试
pipeline = AutomaticSpeechRecognitionPipeline(model=model, tokenizer=tokenizer, feature_extractor=feature_extractor)

# 测试音频文件路径
audio_directory = "/opt/huawei/edu-apaas/src/init/data/wavs/1/wavs"

# 获取目录下的所有 .wav 文件
audio_files = [os.path.join(audio_directory, f) for f in os.listdir(audio_directory) if f.endswith('.wav')]

# 只处理前100个文件
audio_files = audio_files[:100]

# 存储结果的列表
results = []

# 遍历所有音频文件并进行语音识别
for audio_file in audio_files:
    with torch.no_grad():
        text = pipeline(audio_file, generate_kwargs={"forced_decoder_ids": forced_decoder_ids}, max_new_tokens=255)["text"]
        results.append({"file": audio_file, "text": text})
        print(f"Transcribed text for {audio_file}: {text}")

# 将结果保存到 JSON 文件中
output_file = os.path.join(output_directory, "transcription_results.json")
with open(output_file, "w", encoding="utf-8") as f:
    json.dump(results, f, ensure_ascii=False, indent=4)

print(f"Results saved to {output_file}")

• 1.
• 2.
• 3.
• 4.
• 5.
• 6.
• 7.
• 8.
• 9.
• 10.
• 11.
• 12.
• 13.
• 14.
• 15.
• 16.
• 17.
• 18.
• 19.
• 20.
• 21.
• 22.
• 23.
• 24.
• 25.
• 26.
• 27.
• 28.
• 29.
• 30.
• 31.
• 32.
• 33.
• 34.
• 35.
• 36.
• 37.
• 38.
• 39.
• 40.
• 41.
• 42.
• 43.
• 44.
• 45.
• 46.
• 47.
• 48.
• 49.
• 50.
• 51.
• 52.
• 53.
• 54.
• 55.
• 56.
• 57.
• 58.
• 59.
• 60.
• 61.
• 62.
• 63.
• 64.

python save_and_infer.py

• 1.