Zeker, hier is een gedetailleerde handleiding voor mixed precision training in PyTorch voor uw ResNet50-model: ## Basisimplementatie met Automatic Mixed Precision (AMP) ```python import torch import torch.nn as nn import torch.optim as optim from torch.cuda import amp from torch.utils.data import DataLoader import torchvision.models as models import torchvision.transforms as transforms from torchvision.datasets import ImageFolder # Configuratie device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') batch_size = 128 # Hoger door geheugenbesparing # Data augmentatie en transformaties train_transform = transforms.Compose([ transforms.RandomResizedCrop(224), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) ]) # Data loading train_dataset = ImageFolder('pad/naar/train_data', transform=train_transform) train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=4, pin_memory=True) # Model initialisatie model = models.resnet50(pretrained=True) num_ftrs = model.fc.in_features model.fc = nn.Linear(num_ftrs, 10) # 10 klassen model = model.to(device) # Optimizer en loss optimizer = optim.Adam(model.parameters(), lr=0.001) criterion = nn.CrossEntropyLoss() # GradScaler voor mixed precision scaler = amp.GradScaler() # Training loop met AMP def train_epoch(model, train_loader, optimizer, criterion, scaler): model.train() running_loss = 0.0 correct = 0 total = 0 for inputs, labels in train_loader: inputs, labels = inputs.to(device), labels.to(device) optimizer.zero_grad() # Mixed precision forward pass with amp.autocast(): outputs = model(inputs) loss = criterion(outputs, labels) # Backward pass met scaling scaler.scale(loss).backward() # Gradient clipping voor stabiliteit scaler.unscale_(optimizer) torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0) # Optimizer step met scaling scaler.step(optimizer) scaler.update() # Statistics running_loss += loss.item() _, predicted = outputs.max(1) total += labels.size(0) correct += predicted.eq(labels).sum().item() return running_loss / len(train_loader), 100. * correct / total # Training loop num_epochs = 50 for epoch in range(num_epochs): train_loss, train_acc = train_epoch(model, train_loader, optimizer, criterion, scaler) print(f'Epoch {epoch+1}/{num_epochs}, Loss: {train_loss:.4f}, Acc: {train_acc:.2f}%') ``` ## Geavanceerde Best Practices ### 1. Gradient Scaling en Clipping ```python # Optimalisatie van gradient scaling scaler = amp.GradScaler(init_scale=2.**16, growth_interval=2000) # Aanpasbare gradient clipping def adaptive_gradient_clipping(parameters, scaler): # Schaal gradients terug naar FP32 voor clipping scaler.unscale_(optimizer) # Dynamische clipping threshold total_norm = torch.nn.utils.clip_grad_norm_( parameters, max_norm=1.0, norm_type=2.0 ) # Pas scaling factor aan based op gradient norm if total_norm > 1.0: scaler.update(2.0) # Verhoog scaling factor ``` ### 2. Numerieke Stabiliteit ```python # Loss scaling monitoring class StableTraining: def __init__(self, scaler): self.scaler = scaler self.nan_count = 0 self.max_nan = 5 def check_stability(self, loss): if torch.isnan(loss) or torch.isinf(loss): self.nan_count += 1 if self.nan_count > self.max_nan: # Reset training state self.scaler.update(2.0) self.nan_count = 0 return False return True # Gebruik in training stability_monitor = StableTraining(scaler) with amp.autocast(): outputs = model(inputs) loss = criterion(outputs, labels) if not stability_monitor.check_stability(loss): continue # Sla batch over ``` ### 3. Geheugenoptimalisatie ```python # Memory-efficient configuratie torch.backends.cudnn.benchmark = True torch.backends.cudnn.deterministic = False # Gradient checkpointing voor extra geheugenbesparing from torch.utils.checkpoint import checkpoint class MemoryEfficientResNet(nn.Module): def forward(self, x): # Gebruik checkpointing voor resource-intensive layers x = checkpoint(self.layer1, x) x = checkpoint(self.layer2, x) x = checkpoint(self.layer3, x) x = checkpoint(self.layer4, x) return x ``` ### 4. Geavanceerde Training Loop ```python def advanced_train_epoch(model, loader, optimizer, criterion, scaler): model.train() for batch_idx, (inputs, labels) in enumerate(loader): inputs, labels = inputs.to(device), labels.to(device) # Forward pass met mixed precision with amp.autocast(): outputs = model(inputs) loss = criterion(outputs, labels) # Backward pass scaler.scale(loss).backward() # Gradient accumulation if (batch_idx + 1) % 2 == 0: # Accumuleer 2 batches # Gradient clipping scaler.unscale_(optimizer) torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0) # Optimizer step scaler.step(optimizer) scaler.update() optimizer.zero_grad() ``` ## Monitoring en Debugging ```python # Monitoring tools def monitor_memory_usage(): print(f"VRAM gebruikt: {torch.cuda.memory_allocated()/1024**3:.2f} GB") print(f"VRAM gereserveerd: {torch.cuda.memory_reserved()/1024**3:.2f} GB") # NaN detection def check_nan_parameters(model): for name, param in model.named_parameters(): if torch.isnan(param).any(): print(f"NaN gevonden in: {name}") return True return False ``` ## Aanbevolen Instellingen voor RTX 3090 ```python # Optimale instellingen voor 24GB VRAM config = { 'batch_size': 128, # Start met 128, verhoog indien mogelijk 'mixed_precision': True, 'gradient_accumulation': 2, # Voor effectieve batch size van 256 'learning_rate': 0.001, 'weight_decay': 1e-4, 'gradient_clip': 1.0 } ``` ## Belangrijke Tips: 1. **Begin met kleine batch size** en verhoog geleidelijk 2. **Monitor GPU-geheugen** tijdens training 3. **Gebruik gradient clipping** voor stabiliteit 4. **Test zonder AMP eerst** voor baseline performance 5. **Controleer op NaN values** regelmatig 6. **Pas learning rate aan** indien nodig voor mixed precision Deze implementatie zou uw trainingssnelheid met 1.5-3x moeten verbeteren en het geheugengebruik met ongeveer 50% verminderen op uw RTX 3090.