Zeker, hier is een complete handleiding voor het ontwerpen, implementeren en integreren van uw aangepaste activatielaag in PyTorch. ### Aangepaste ActivatieLaag: ReLU + Sigmoid met Leerbare Parameters ```python import torch import torch.nn as nn import torch.nn.functional as F class CustomActivation(nn.Module): """ Aangepaste activatiefunctie die ReLU en Sigmoid combineert met leerbare parameters. Formule: alpha * relu(x) + beta * sigmoid(x) + gamma """ def __init__(self, alpha_init=1.0, beta_init=1.0, gamma_init=0.0): super(CustomActivation, self).__init__() # Leerbare parameters initialiseren self.alpha = nn.Parameter(torch.tensor(alpha_init, dtype=torch.float32)) self.beta = nn.Parameter(torch.tensor(beta_init, dtype=torch.float32)) self.gamma = nn.Parameter(torch.tensor(gamma_init, dtype=torch.float32)) def forward(self, x): # Combineer ReLU en Sigmoid met leerbare gewichten relu_component = F.relu(x) sigmoid_component = torch.sigmoid(x) return self.alpha * relu_component + self.beta * sigmoid_component + self.gamma def extra_repr(self): # Toon parameterwaarden voor debugging return f'alpha={self.alpha.item():.3f}, beta={self.beta.item():.3f}, gamma={self.gamma.item():.3f}' # Alternatieve versie met meerdere kanalen (indien nodig) class ChannelWiseCustomActivation(nn.Module): """ Versie met verschillende parameters per kanaal voor complexere modellen """ def __init__(self, num_channels, alpha_init=1.0, beta_init=1.0, gamma_init=0.0): super(ChannelWiseCustomActivation, self).__init__() self.alpha = nn.Parameter(torch.full((num_channels,), alpha_init, dtype=torch.float32)) self.beta = nn.Parameter(torch.full((num_channels,), beta_init, dtype=torch.float32)) self.gamma = nn.Parameter(torch.full((num_channels,), gamma_init, dtype=torch.float32)) def forward(self, x): # Zorg dat de parameters de juiste vorm hebben voor broadcasting alpha = self.alpha.view(1, -1, 1, 1) if x.dim() == 4 else self.alpha.view(1, -1) beta = self.beta.view(1, -1, 1, 1) if x.dim() == 4 else self.beta.view(1, -1) gamma = self.gamma.view(1, -1, 1, 1) if x.dim() == 4 else self.gamma.view(1, -1) relu_component = F.relu(x) sigmoid_component = torch.sigmoid(x) return alpha * relu_component + beta * sigmoid_component + gamma ``` ### Integratie in Bestaande Modelarchitectuur ```python class YourExistingModel(nn.Module): def __init__(self, input_size, hidden_size, num_classes): super(YourExistingModel, self).__init__() # Bestaande lagen self.fc1 = nn.Linear(input_size, hidden_size) self.fc2 = nn.Linear(hidden_size, hidden_size) self.fc3 = nn.Linear(hidden_size, num_classes) # Voeg aangepaste activatie toe self.custom_activation = CustomActivation(alpha_init=0.8, beta_init=0.5, gamma_init=0.1) # Of voor CNN-architecturen: # self.custom_activation = ChannelWiseCustomActivation(num_channels=64) def forward(self, x): x = self.fc1(x) x = self.custom_activation(x) # Vervang standaard activatie x = self.fc2(x) x = self.custom_activation(x) # Hergebruik dezelfde instantie x = self.fc3(x) return x # Gebruik in CNN class SimpleCNN(nn.Module): def __init__(self): super(SimpleCNN, self).__init__() self.conv1 = nn.Conv2d(3, 64, kernel_size=3, padding=1) self.custom_act = ChannelWiseCustomActivation(num_channels=64) self.pool = nn.MaxPool2d(2, 2) self.fc = nn.Linear(64 * 16 * 16, 10) def forward(self, x): x = self.pool(self.custom_act(self.conv1(x))) x = x.view(x.size(0), -1) x = self.fc(x) return x ``` ### Debugging en Test Tips ```python # 1. Test de activatiefunctie geïsoleerd def test_custom_activation(): activation = CustomActivation() test_input = torch.tensor([-2.0, -1.0, 0.0, 1.0, 2.0]) output = activation(test_input) print("Input:", test_input) print("Output:", output) print("Parameters:", dict(activation.named_parameters())) # 2. Gradient checking def gradient_check(): model = CustomActivation() x = torch.tensor([1.0], requires_grad=True) y = model(x) y.backward() print("Gradients:", x.grad) # 3. Integratie test def integration_test(): model = YourExistingModel(input_size=784, hidden_size=128, num_classes=10) test_input = torch.randn(32, 784) # Batch size 32 output = model(test_input) print("Output shape:", output.shape) print("Model parameters werken correct") # 4. Training test met dummy data def training_test(): model = YourExistingModel(784, 128, 10) optimizer = torch.optim.Adam(model.parameters(), lr=0.001) criterion = nn.CrossEntropyLoss() # Dummy training loop for epoch in range(3): data = torch.randn(64, 784) target = torch.randint(0, 10, (64,)) optimizer.zero_grad() output = model(data) loss = criterion(output, target) loss.backward() optimizer.step() print(f"Epoch {epoch}, Loss: {loss.item():.4f}") print("Activation params:", {name: param.item() for name, param in model.custom_activation.named_parameters()}) # Uitvoeren van tests if __name__ == "__main__": print("=== Geïsoleerde test ===") test_custom_activation() print("\n=== Gradient check ===") gradient_check() print("\n=== Integratie test ===") integration_test() print("\n=== Training test ===") training_test() ``` ### Belangrijke Tips voor Implementatie: 1. **Parameter Initialisatie**: Begin met redelijke initiële waarden (bijv. alpha=1.0, beta=0.5) 2. **Gradient Flow**: Controleer of gradients correct doorstromen met `torch.autograd.gradcheck` 3. **Device Compatibility**: Zorg dat parameters automatisch op de juiste device komen: ```python device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') model = YourExistingModel(...).to(device) ``` 4. **Serialisatie**: Aangepaste modules werken prima met `torch.save()` en `torch.load()` 5. **Monitoring**: Gebruik TensorBoard of print statements om parameterwaarden tijdens training te volgen ### Gebruik in Practice: ```python # In uw hoofd training script model = YourExistingModel(input_size, hidden_size, num_classes) optimizer = torch.optim.Adam(model.parameters(), lr=0.001) # Tijdens training kunt u parameters monitoren: for epoch in range(num_epochs): # ... training code ... if epoch % 10 == 0: print(f"Activation parameters: {model.custom_activation.alpha.item():.3f}, " f"{model.custom_activation.beta.item():.3f}, " f"{model.custom_activation.gamma.item():.3f}") ``` Deze implementatie behoudt volledige differentieerbaarheid en integreert naadloos met bestaande PyTorch functionaliteiten zoals automatische differentiatie en model serialisatie.