迁移学习与FastText精简版

import fasttext

# 1- 基础训练 + 预测 + 测试
model = fasttext.train_supervised(input="data/cooking_train.txt")
print(model.predict("Which baking dish is best to bake a banana bread ?"))
print(model.test(path="data/cooking_valid.txt"))  # (样本数, 精确率, 召回率)

# 2- 综合调参训练
model = fasttext.train_supervised(
    input="data/cooking.pre.train",
    epoch=20, lr=1, wordNgrams=2, loss="hs"
)

# 3- 自动超参调优
model = fasttext.train_supervised(
    input="data/cooking.pre.train",
    autotuneValidationFile="data/cooking.pre.valid",
    autotuneDuration=60 * 2
)

# 4- 多标签分类（ova）
model = fasttext.train_supervised(
    input="data/cooking.pre.train",
    epoch=20, lr=0.1, wordNgrams=2, loss="ova"
)
# k：最多展示目标值数量，-1 表示全部；threshold：概率阈值
print(model.predict("Which baking dish is best to bake a banana bread ?", k=3, threshold=0.5))

# 5- 保存与加载
model.save_model("model/cooking_model.pkl")
model2 = fasttext.load_model("model/cooking_model.pkl")

from transformers import AutoTokenizer, AutoModelForSequenceClassification

model_path = r"D:\PretrainedModel\chinese_sentiment"
tokenizer = AutoTokenizer.from_pretrained(model_path)
model = AutoModelForSequenceClassification.from_pretrained(model_path)

data_tensor = tokenizer.encode(
    text="我爱黑马",
    return_tensors="pt",
    padding="max_length",
    truncation=True,
    max_length=10
)

model.eval()
result = model(data_tensor)
print(result[0].argmax(dim=-1))  # 预测分类 ID

from transformers import AutoTokenizer, AutoModel

model_path = r"D:\soft\PretrainedModel\bert-base-chinese"
tokenizer = AutoTokenizer.from_pretrained(model_path)
model = AutoModel.from_pretrained(model_path)

data_tensor = tokenizer.encode_plus(
    text=['你是谁', '人生该如何起头'],
    return_tensors="pt",
    padding="max_length",
    truncation=True,
    max_length=30
)

model.eval()
result = model(**data_tensor)  # 字典解包
print("最后一个隐藏状态信息", result.last_hidden_state.shape)
print("池化层信息", result.pooler_output.shape)

import torch
import torch.nn as nn
from datasets import load_dataset
from torch.utils.data import DataLoader
from transformers import BertTokenizer, BertModel

device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

model_path = r"D:\PretrainedModel\bert-base-chinese"
# 分词器不涉及张量计算，不需要发送到 GPU
bert_tokenizer = BertTokenizer.from_pretrained(model_path)
bert_model = BertModel.from_pretrained(model_path).to(device)

def collate_fn(data):
    sents = [item["text"] for item in data]
    labels = [item["label"] for item in data]

    data_tensor = bert_tokenizer.batch_encode_plus(
        sents,
        return_tensors="pt",
        padding=True,
        truncation=True,
        max_length=300
    )
    labels = torch.LongTensor(labels)
    return data_tensor["input_ids"], data_tensor["token_type_ids"], data_tensor["attention_mask"], labels

def get_dataloader(file_path):
    dataset = load_dataset(path="data", data_files=file_path, split="train")
    # batch_size 只能为 1 时，说明样本长度不一致
    return DataLoader(dataset, batch_size=8, shuffle=True, drop_last=True, collate_fn=collate_fn)

class AiModel(nn.Module):
    def __init__(self):
        super().__init__()
        self.linear = nn.Linear(in_features=768, out_features=2)

    def forward(self, input_ids, token_type_ids, attention_mask):
        with torch.no_grad():
            bert_output = bert_model(
                input_ids=input_ids,
                token_type_ids=token_type_ids,
                attention_mask=attention_mask
            )
        # 取 [CLS] 的隐藏状态作为句子表示
        return self.linear(bert_output.last_hidden_state[:, 0])

from tqdm import tqdm

def train():
    dataloader = get_dataloader("train.csv")

    # 禁用 BERT 梯度
    for param in bert_model.parameters():
        param.requires_grad_(False)

    model = AiModel().to(device=device)
    optimizer = torch.optim.AdamW(params=model.parameters(), lr=1e-3)
    loss = nn.CrossEntropyLoss()

    model.train()
    for epoch in range(1):
        for i, (input_ids, token_type_ids, attention_mask, labels) in enumerate(tqdm(dataloader), start=1):
            input_ids = input_ids.to(device)
            token_type_ids = token_type_ids.to(device)
            attention_mask = attention_mask.to(device)
            labels = labels.to(device)

            pred_result = model(input_ids, token_type_ids, attention_mask)
            loss_value = loss(pred_result, labels)

            optimizer.zero_grad()
            loss_value.sum().backward()
            optimizer.step()  # 只更新自定义线性层

            if i % 20 == 0:
                pred_index = torch.argmax(pred_result, dim=-1)
                acc = (pred_index == labels).sum().item() / len(labels)
                print(f"第{epoch+1}轮次，批次{i}，准确率{round(acc, 4)}")

    torch.save(model.state_dict(), "model/bert.pkl")

def predict():
    dataloader = get_dataloader("test.csv")

    model = AiModel().to(device)
    model.load_state_dict(torch.load("model/bert.pkl"))

    correct_count = 0
    total_sample_count = 0

    model.eval()
    with torch.no_grad():
        for i, (input_ids, token_type_ids, attention_mask, labels) in enumerate(tqdm(dataloader), start=1):
            input_ids = input_ids.to(device)
            token_type_ids = token_type_ids.to(device)
            attention_mask = attention_mask.to(device)
            labels = labels.to(device)

            pred_result = model(input_ids, token_type_ids, attention_mask)
            pred_index = torch.argmax(pred_result, dim=-1)
            correct_count += (pred_index == labels).sum().item()
            total_sample_count += len(labels)

            if i % 20 == 0:
                acc = correct_count / total_sample_count
                print(f"已预测批次{i}，累计平均准确率{round(acc, 4)}")