import os
import pandas as pd
import torch
from torch import nn, optim
from torch.utils.data import DataLoader, Dataset
from transformers import AutoTokenizer, AutoModel, get_linear_schedule_with_warmup
from sklearn.metrics import accuracy_score, f1_score, classification_report
from sklearn.model_selection import StratifiedKFold, train_test_split
import random
import numpy as np
import statistics
#Mudanças principais:
#1400
#Modelo Bertimbau Large: Alterado o model_name para 'neuralmind/bert-large-portuguese-cased'.
#LR= 3e-5.
#Descongelamento das camadas: Parametrizamos o número de camadas finais do BERT a descongelar, via unfreeze_layers. Por exemplo, se definirmos unfreeze_layers=8, descongelamos as últimas 8 camadas.
#Outros otimizadores e LR Schedulers: Mantemos o AdamW como otimizador principal, mas agora adicionamos um scheduler (get_linear_schedule_with_warmup do transformers) para ajustar a taxa de aprendizado durante o treino. Caso queira testar outro otimizador, basta substituir a linha do optimizador. Também deixamos comentado outro exemplo (SGD) para referência.
#Para testar diferentes taxas de aprendizado, basta alterar learning_rate no código.
#Para testar diferentes números de camadas a descongelar, altere unfreeze_layers.
#4
#processo de treinamento e avaliação várias vezes (uma para cada fold).
#diminuindo épocas ou early stopping, se necessário.
#O early stopping agora é feito com base no conjunto de validação interno a cada fold.
#Esse processo é mais demorado, pois treinaremos o modelo K vezes.
#Ajuste parâmetros (como número de épocas, taxa de aprendizado, etc.) conforme necessário.
# Semente para reprodutibilidade
seed = 42
random.seed(seed)
np.random.seed(seed)
torch.manual_seed(seed)
if torch.cuda.is_available():
torch.cuda.manual_seed_all(seed)
# Configurações gerais
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
print(f'Usando dispositivo: {device}')
model_name = 'neuralmind/bert-large-portuguese-cased'
learning_rate = 3e-5
unfreeze_layers = 4
nclasses = 2
nepochs = 5
batch_size = 8
batch_status = 32
early_stop = 2
max_length = 360
write_path = 'model_cv'
if not os.path.exists(write_path):
os.makedirs(write_path)
Usando dispositivo: cuda
# Carregar os dados
data = pd.read_csv("DATAFRAME1400.csv")
# Dataset Customizado
class CustomDataset(Dataset):
def __init__(self, data, tokenizer, max_length):
self.data = data.reset_index(drop=True)
self.tokenizer = tokenizer
self.max_length = max_length
def __len__(self):
return len(self.data)
def __getitem__(self, idx):
text = self.data.iloc[idx]['text']
label = self.data.iloc[idx]['contra']
inputs = self.tokenizer(text, return_tensors='pt',
padding='max_length', truncation=True,
max_length=self.max_length)
return {key: val.squeeze(0) for key, val in inputs.items()}, torch.tensor(label)
# Modelo
class CustomBERTModel(nn.Module):
def __init__(self, model_name, nclasses, unfreeze_layers):
super(CustomBERTModel, self).__init__()
self.bert = AutoModel.from_pretrained(model_name)
self.dropout = nn.Dropout(0.3)
self.classifier = nn.Linear(self.bert.config.hidden_size, nclasses)
# Congelar tudo inicialmente
for param in self.bert.parameters():
param.requires_grad = False
# Descongelar as últimas unfreeze_layers camadas
if unfreeze_layers > 0:
for param in self.bert.encoder.layer[-unfreeze_layers:].parameters():
param.requires_grad = True
def forward(self, input_ids, attention_mask, token_type_ids=None):
outputs = self.bert(input_ids=input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids)
pooled_output = outputs.pooler_output
dropped_out = self.dropout(pooled_output)
logits = self.classifier(dropped_out)
return logits
def evaluate(model, dataloader):
model.eval()
y_real, y_pred = [], []
with torch.no_grad():
for inputs, labels in dataloader:
inputs = {key: val.to(device) for key, val in inputs.items()}
labels = labels.to(device)
logits = model(**inputs)
pred_labels = torch.argmax(logits, 1)
y_real.extend(labels.cpu().tolist())
y_pred.extend(pred_labels.cpu().tolist())
f1 = f1_score(y_real, y_pred, average='weighted')
acc = accuracy_score(y_real, y_pred)
return f1, acc, (y_real, y_pred)
# Cross-validation
k = 5
skf = StratifiedKFold(n_splits=k, shuffle=True, random_state=seed)
X = data.index.values
y = data['contra'].values
f1_scores = []
acc_scores = []
tokenizer = AutoTokenizer.from_pretrained(model_name, do_lower_case=False)
fold_num = 1
for train_val_idx, test_idx in skf.split(X, y):
print(f"\n=== Fold {fold_num}/{k} ===")
# Separamos test fold
test_data = data.iloc[test_idx]
# A partir do train_val_idx, dividimos em train e val
train_val_data = data.iloc[train_val_idx]
train_data, val_data = train_test_split(train_val_data,
test_size=0.1,
random_state=seed,
stratify=train_val_data['contra'])
# Criar datasets e dataloaders
train_dataset = CustomDataset(train_data, tokenizer, max_length)
val_dataset = CustomDataset(val_data, tokenizer, max_length)
test_dataset = CustomDataset(test_data, tokenizer, max_length)
traindata = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)
valdata = DataLoader(val_dataset, batch_size=batch_size, shuffle=False)
testdata = DataLoader(test_dataset, batch_size=batch_size, shuffle=False)
model = CustomBERTModel(model_name, nclasses, unfreeze_layers).to(device)
optimizer = optim.AdamW(filter(lambda p: p.requires_grad, model.parameters()), lr=learning_rate)
loss_fn = nn.CrossEntropyLoss()
total_steps = len(traindata) * nepochs
scheduler = get_linear_schedule_with_warmup(optimizer,
num_warmup_steps=int(0.1 * total_steps),
num_training_steps=total_steps)
max_f1, repeat = 0, 0
best_model_path = os.path.join(write_path, f'best_model_fold{fold_num}.pth')
for epoch in range(nepochs):
model.train()
losses = []
for batch_idx, (inputs, labels) in enumerate(traindata):
inputs = {key: val.to(device) for key, val in inputs.items()}
labels = labels.to(device)
logits = model(**inputs)
loss = loss_fn(logits, labels)
losses.append(float(loss))
# Backprop
loss.backward()
optimizer.step()
scheduler.step()
optimizer.zero_grad()
if (batch_idx + 1) % batch_status == 0:
print(f'Epoch: {epoch} [{batch_idx + 1}/{len(traindata)}]\tLoss: {loss:.6f}')
f1_val, acc_val, _ = evaluate(model, valdata)
print(f'Epoch {epoch} - Val F1: {f1_val:.4f}, Val Accuracy: {acc_val:.4f}')
if f1_val > max_f1:
torch.save(model.state_dict(), best_model_path)
max_f1 = f1_val
repeat = 0
print('Novo melhor modelo salvo.')
else:
repeat += 1
if repeat == early_stop:
print('Early stopping atingido.')
break
# Avaliar no teste
state_dict = torch.load(best_model_path, weights_only=True)
model.load_state_dict(state_dict)
f1_test, acc_test, (y_real, y_pred) = evaluate(model, testdata)
print("Desempenho no conjunto de teste desta dobra:")
print(classification_report(y_real, y_pred, target_names=['0', '1']))
print(f"F1 (teste): {f1_test:.4f}, Accuracy (teste): {acc_test:.4f}")
f1_scores.append(f1_test)
acc_scores.append(acc_test)
fold_num += 1
# Resultados médios da validação cruzada
print("\n=== Resultados Médios da Validação Cruzada ===")
print(f"F1 médio: {statistics.mean(f1_scores):.4f} (+/- {statistics.pstdev(f1_scores):.4f})")
print(f"Acurácia média: {statistics.mean(acc_scores):.4f} (+/- {statistics.pstdev(acc_scores):.4f})")
=== Fold 1/5 ===
Epoch: 0 [32/135] Loss: 0.943423
Epoch: 0 [64/135] Loss: 0.659408
Epoch: 0 [96/135] Loss: 0.393854
Epoch: 0 [128/135] Loss: 0.469751
Epoch 0 - Val F1: 0.8000, Val Accuracy: 0.8000
Novo melhor modelo salvo.
Epoch: 1 [32/135] Loss: 0.371063
Epoch: 1 [64/135] Loss: 0.466771
Epoch: 1 [96/135] Loss: 0.271778
Epoch: 1 [128/135] Loss: 0.209255
Epoch 1 - Val F1: 0.8300, Val Accuracy: 0.8333
Novo melhor modelo salvo.
Epoch: 2 [32/135] Loss: 0.078391
Epoch: 2 [64/135] Loss: 0.239022
Epoch: 2 [96/135] Loss: 0.283219
Epoch: 2 [128/135] Loss: 0.061773
Epoch 2 - Val F1: 0.8483, Val Accuracy: 0.8500
Novo melhor modelo salvo.
Epoch: 3 [32/135] Loss: 0.081003
Epoch: 3 [64/135] Loss: 0.289715
Epoch: 3 [96/135] Loss: 0.047254
Epoch: 3 [128/135] Loss: 0.025544
Epoch 3 - Val F1: 0.8418, Val Accuracy: 0.8417
Epoch: 4 [32/135] Loss: 0.096558
Epoch: 4 [64/135] Loss: 0.023162
Epoch: 4 [96/135] Loss: 0.028982
Epoch: 4 [128/135] Loss: 0.015242
Epoch 4 - Val F1: 0.8663, Val Accuracy: 0.8667
Novo melhor modelo salvo.
Desempenho no conjunto de teste desta dobra:
precision recall f1-score support
0 0.90 0.93 0.91 157
1 0.92 0.88 0.90 143
accuracy 0.91 300
macro avg 0.91 0.91 0.91 300
weighted avg 0.91 0.91 0.91 300
F1 (teste): 0.9065, Accuracy (teste): 0.9067
=== Fold 2/5 ===
Epoch: 0 [32/135] Loss: 0.750314
Epoch: 0 [64/135] Loss: 0.607893
Epoch: 0 [96/135] Loss: 0.476860
Epoch: 0 [128/135] Loss: 0.436566
Epoch 0 - Val F1: 0.8496, Val Accuracy: 0.8500
Novo melhor modelo salvo.
Epoch: 1 [32/135] Loss: 0.266623
Epoch: 1 [64/135] Loss: 0.167054
Epoch: 1 [96/135] Loss: 0.574422
Epoch: 1 [128/135] Loss: 0.136387
Epoch 1 - Val F1: 0.8834, Val Accuracy: 0.8833
Novo melhor modelo salvo.
Epoch: 2 [32/135] Loss: 0.054383
Epoch: 2 [64/135] Loss: 0.190985
Epoch: 2 [96/135] Loss: 0.210877
Epoch: 2 [128/135] Loss: 0.279167
Epoch 2 - Val F1: 0.8833, Val Accuracy: 0.8833
Epoch: 3 [32/135] Loss: 0.319686
Epoch: 3 [64/135] Loss: 0.331613
Epoch: 3 [96/135] Loss: 0.031250
Epoch: 3 [128/135] Loss: 0.026889
Epoch 3 - Val F1: 0.8917, Val Accuracy: 0.8917
Novo melhor modelo salvo.
Epoch: 4 [32/135] Loss: 0.051201
Epoch: 4 [64/135] Loss: 0.115116
Epoch: 4 [96/135] Loss: 0.014849
Epoch: 4 [128/135] Loss: 0.025389
Epoch 4 - Val F1: 0.8830, Val Accuracy: 0.8833
Desempenho no conjunto de teste desta dobra:
precision recall f1-score support
0 0.90 0.88 0.89 156
1 0.87 0.90 0.88 143
accuracy 0.89 299
macro avg 0.89 0.89 0.89 299
weighted avg 0.89 0.89 0.89 299
F1 (teste): 0.8863, Accuracy (teste): 0.8863
=== Fold 3/5 ===
Epoch: 0 [32/135] Loss: 0.830914
Epoch: 0 [64/135] Loss: 0.683001
Epoch: 0 [96/135] Loss: 0.577303
Epoch: 0 [128/135] Loss: 0.676515
Epoch 0 - Val F1: 0.8501, Val Accuracy: 0.8500
Novo melhor modelo salvo.
Epoch: 1 [32/135] Loss: 0.210992
Epoch: 1 [64/135] Loss: 0.217102
Epoch: 1 [96/135] Loss: 0.643290
Epoch: 1 [128/135] Loss: 0.511327
Epoch 1 - Val F1: 0.8748, Val Accuracy: 0.8750
Novo melhor modelo salvo.
Epoch: 2 [32/135] Loss: 0.392438
Epoch: 2 [64/135] Loss: 0.138488
Epoch: 2 [96/135] Loss: 0.379189
Epoch: 2 [128/135] Loss: 0.057960
Epoch 2 - Val F1: 0.9250, Val Accuracy: 0.9250
Novo melhor modelo salvo.
Epoch: 3 [32/135] Loss: 0.094107
Epoch: 3 [64/135] Loss: 0.095579
Epoch: 3 [96/135] Loss: 0.070371
Epoch: 3 [128/135] Loss: 0.038925
Epoch 3 - Val F1: 0.9250, Val Accuracy: 0.9250
Epoch: 4 [32/135] Loss: 0.054453
Epoch: 4 [64/135] Loss: 0.027761
Epoch: 4 [96/135] Loss: 0.014768
Epoch: 4 [128/135] Loss: 0.069529
Epoch 4 - Val F1: 0.9333, Val Accuracy: 0.9333
Novo melhor modelo salvo.
Desempenho no conjunto de teste desta dobra:
precision recall f1-score support
0 0.91 0.86 0.89 157
1 0.85 0.91 0.88 142
accuracy 0.88 299
macro avg 0.88 0.88 0.88 299
weighted avg 0.88 0.88 0.88 299
F1 (teste): 0.8830, Accuracy (teste): 0.8829
=== Fold 4/5 ===
Epoch: 0 [32/135] Loss: 0.743773
Epoch: 0 [64/135] Loss: 0.690046
Epoch: 0 [96/135] Loss: 0.623647
Epoch: 0 [128/135] Loss: 0.497060
Epoch 0 - Val F1: 0.8583, Val Accuracy: 0.8583
Novo melhor modelo salvo.
Epoch: 1 [32/135] Loss: 0.449734
Epoch: 1 [64/135] Loss: 0.170243
Epoch: 1 [96/135] Loss: 0.075471
Epoch: 1 [128/135] Loss: 0.299446
Epoch 1 - Val F1: 0.8416, Val Accuracy: 0.8417
Epoch: 2 [32/135] Loss: 0.488727
Epoch: 2 [64/135] Loss: 0.095850
Epoch: 2 [96/135] Loss: 0.157705
Epoch: 2 [128/135] Loss: 0.141444
Epoch 2 - Val F1: 0.8416, Val Accuracy: 0.8417
Early stopping atingido.
Desempenho no conjunto de teste desta dobra:
precision recall f1-score support
0 0.81 0.91 0.86 157
1 0.89 0.77 0.82 142
accuracy 0.84 299
macro avg 0.85 0.84 0.84 299
weighted avg 0.85 0.84 0.84 299
F1 (teste): 0.8417, Accuracy (teste): 0.8428
=== Fold 5/5 ===
Epoch: 0 [32/135] Loss: 0.598468
Epoch: 0 [64/135] Loss: 0.593075
Epoch: 0 [96/135] Loss: 0.494841
Epoch: 0 [128/135] Loss: 0.455896
Epoch 0 - Val F1: 0.8500, Val Accuracy: 0.8500
Novo melhor modelo salvo.
Epoch: 1 [32/135] Loss: 0.359274
Epoch: 1 [64/135] Loss: 0.157859
Epoch: 1 [96/135] Loss: 0.085530
Epoch: 1 [128/135] Loss: 0.029607
Epoch 1 - Val F1: 0.8748, Val Accuracy: 0.8750
Novo melhor modelo salvo.
Epoch: 2 [32/135] Loss: 0.119052
Epoch: 2 [64/135] Loss: 0.519205
Epoch: 2 [96/135] Loss: 0.403141
Epoch: 2 [128/135] Loss: 0.155865
Epoch 2 - Val F1: 0.9000, Val Accuracy: 0.9000
Novo melhor modelo salvo.
Epoch: 3 [32/135] Loss: 0.095529
Epoch: 3 [64/135] Loss: 0.024375
Epoch: 3 [96/135] Loss: 0.047616
Epoch: 3 [128/135] Loss: 0.028527
Epoch 3 - Val F1: 0.9250, Val Accuracy: 0.9250
Novo melhor modelo salvo.
Epoch: 4 [32/135] Loss: 0.019617
Epoch: 4 [64/135] Loss: 0.032093
Epoch: 4 [96/135] Loss: 0.128916
Epoch: 4 [128/135] Loss: 0.042773
Epoch 4 - Val F1: 0.9084, Val Accuracy: 0.9083
Desempenho no conjunto de teste desta dobra:
precision recall f1-score support
0 0.88 0.87 0.88 157
1 0.86 0.87 0.86 142
accuracy 0.87 299
macro avg 0.87 0.87 0.87 299
weighted avg 0.87 0.87 0.87 299
F1 (teste): 0.8696, Accuracy (teste): 0.8696
=== Resultados Médios da Validação Cruzada ===
F1 médio: 0.8774 (+/- 0.0214)
Acurácia média: 0.8777 (+/- 0.0211)