Lädt...


🔧 Batch, Mini-Batch & Stochastic Gradient Descent examples with `DataLoader()` in PyTorch


Nachrichtenbereich: 🔧 Programmierung
🔗 Quelle: dev.to

Buy Me a Coffee

*Memos:

  • My post explains Batch, Mini-Batch and Stochastic Gradient Descent.
  • My post explains Deep Learning Workflow in PyTorch.

This is Batch Gradient Descent(BGD), Mini-Batch Gradient Descent(MBGD) or Stochastic Gradient Descent(SGD) with DataLoader() and non-shuffled or shuffled dataset as shown below:

*Memos:

import torch
from torch import nn
from torch import optim
from torch.utils.data import DataLoader
import matplotlib.pyplot as plt

# Set device
device = "cuda" if torch.cuda.is_available() else "cpu"

""" Prepare dataset """
weight = 0.8
bias = 0.5

X = torch.tensor([[0.00], [0.02], [0.04], [0.06], [0.08], # Size(50, 1)
                  [0.10], [0.12], [0.14], [0.16], [0.18],
                  [0.20], [0.22], [0.24], [0.26], [0.28],
                  [0.30], [0.32], [0.34], [0.36], [0.38],
                  [0.40], [0.42], [0.44], [0.46], [0.48],
                  [0.50], [0.52], [0.54], [0.56], [0.58],
                  [0.60], [0.62], [0.64], [0.66], [0.68],
                  [0.70], [0.72], [0.74], [0.76], [0.78],
                  [0.80], [0.82], [0.84], [0.86], [0.88],
                  [0.90], [0.92], [0.94], [0.96], [0.98]])
torch.manual_seed(42)

# ↓ ↓ ↓ ↓ ↓ ↓ ↓ ↓ Uncomment it to shuffle dataset ↓ ↓ ↓ ↓ ↓ ↓ ↓ ↓
# X = list(DataLoader(dataset=X, batch_size=50, shuffle=True))[0] 

Y = weight * X + bias

l = int(0.8 * len(X))

X_train, Y_train, X_test, Y_test = X[:l], Y[:l], X[l:], Y[l:]
""" Prepare dataset """

""" Prepare model, loss function and optimizer """
class MyModel(nn.Module):
    def __init__(self):
        super().__init__()
        self.linear_layer = nn.Linear(in_features=1, out_features=1)

    def forward(self, x):
        return self.linear_layer(x)

torch.manual_seed(42)

my_model = MyModel().to(device)

loss_fn = nn.L1Loss()

optimizer = optim.SGD(params=my_model.parameters(), lr=0.01)
""" Prepare model, loss function and optimizer """

""" Train and test model """
epochs = 100

epoch_count = []
loss_values = []
test_loss_values = []

for epoch in range(epochs):
    X_train_dl = DataLoader(dataset=X_train, batch_size=40)   # BGD
    Y_train_dl = DataLoader(dataset=Y_train, batch_size=40)   # BGD
    # X_train_dl = DataLoader(dataset=X_train, batch_size=10) # MBGD
    # Y_train_dl = DataLoader(dataset=Y_train, batch_size=10) # MBGD
    # X_train_dl = DataLoader(dataset=X_train, batch_size=1)  # SGD
    # Y_train_dl = DataLoader(dataset=Y_train, batch_size=1)  # SGD
    X_Y_train_dl = zip(X_train_dl, Y_train_dl)
    for X_train_batch, Y_train_batch in X_Y_train_dl:

        """ Train """
        my_model.train()

        # 1. Calculate predictions
        Y_pred = my_model(X_train_batch)

        # 2. Calculate loss
        loss = loss_fn(Y_pred, Y_train_batch)

        # 3. Zero out gradients
        optimizer.zero_grad()

        # 4. Backpropagation
        loss.backward()

        # 5. Update parameters
        optimizer.step()
        """ Train """        

        """ Test """
        my_model.eval()

        with torch.inference_mode():
            Y_test_pred = my_model(x=X_test)
            test_loss = loss_fn(Y_test_pred, Y_test)
        if epoch % 10 == 0:
            epoch_count.append(epoch)
            loss_values.append(loss)
            test_loss_values.append(test_loss)
            # print(f"Epoch: {epoch} | Loss: {loss} | Test loss: {test_loss}")
        """ Test """

""" Train and test model """

""" Visualize train and test data and predictions"""
with torch.inference_mode():
    y_pred = my_model(x=X_test)

def plot_predictions(X_train, Y_train, X_test, Y_test, predictions=None):
    plt.figure(figsize=[6, 4])
    plt.scatter(X_train, Y_train, c='g', s=5, label='Train data(Green)')
    plt.scatter(X_test, Y_test, c='b', s=15, label='Test data(Blue)')
    if predictions is not None:
        plt.scatter(X_test, predictions, c='r', s=15, label='Predictions(Red)')
    plt.title(label="Train and test data and predictions", fontsize=14)
    plt.legend(fontsize=14)

plot_predictions(X_train=X_train.cpu(),
                 Y_train=Y_train.cpu(),
                 X_test=X_test.cpu(),
                 Y_test=Y_test.cpu(),
                 predictions=y_pred.cpu())
""" Visualize train and test data, predictions"""

""" Visualize train and test loss """
def plot_loss_curves(epoch_count, loss_values, test_loss_values):
    plt.figure(figsize=[6, 4])
    plt.plot(epoch_count, loss_values, label="Train loss")
    plt.plot(epoch_count, test_loss_values, label="Test loss")
    plt.title(label="Train and test loss curves", fontsize=14)
    plt.ylabel(ylabel="Loss", fontsize=14)
    plt.xlabel(xlabel="Epochs", fontsize=14)
    plt.legend(fontsize=14)

plot_loss_curves(epoch_count=epoch_count,
                 loss_values=torch.tensor(loss_values).cpu(),
                 test_loss_values=torch.tensor(test_loss_values).cpu())
""" Visualize train and test loss """

BGD with DataLoader() and non-shuffled dataset:

Image description

Image description

BGD with DataLoader() and shuffled dataset:

Image description

Image description

MBGD with DataLoader() and non-shuffled dataset:

Image description

Image description

MBGD with DataLoader() and shuffled dataset:

Image description

Image description

SGD with DataLoader() and non-shuffled dataset:

Image description

Image description

SGD with DataLoader() and shuffled dataset:

Image description

Image description

...

📰 Stochastic Gradient Descent: Math and Python Code


📈 56.31 Punkte
🔧 AI Nachrichten

🕵️ QGit 1.5.6 2pre1 dataloader.cpp dataloader::dostart memory corruption


📈 53.04 Punkte
🕵️ Sicherheitslücken

📰 Implementing Gradient Descent in PyTorch


📈 46.43 Punkte
🔧 AI Nachrichten

🔧 DataLoader in PyTorch


📈 39 Punkte
🔧 Programmierung

🎥 Machine Learning Crash Course: Gradient Descent


📈 33.94 Punkte
🎥 Videos

🔧 Gradient Descent: Optimizer Behind Machine Learning


📈 33.94 Punkte
🔧 Programmierung

📰 Visualizing Gradient Descent Parameters in Torch


📈 33.94 Punkte
🔧 AI Nachrichten

📰 Courage to Learn ML: A Detailed Exploration of Gradient Descent and Popular Optimizers


📈 33.94 Punkte
🔧 AI Nachrichten

📰 Newton’s Laws of Motion: The Original Gradient Descent


📈 33.94 Punkte
🔧 AI Nachrichten

📰 DL Notes: Advanced Gradient Descent


📈 33.94 Punkte
🔧 AI Nachrichten

📰 Creating a Gradient Descent Animation in Python


📈 33.94 Punkte
🔧 AI Nachrichten

📰 DL Notes: Gradient Descent


📈 33.94 Punkte
🔧 AI Nachrichten

📰 Gradient Descent Algorithm 101


📈 33.94 Punkte
🔧 AI Nachrichten

📰 Train ImageNet without Hyperparameters with Automatic Gradient Descent


📈 33.94 Punkte
🔧 AI Nachrichten

📰 Gradient Descent: Optimisation and Initialisation Explained


📈 33.94 Punkte
🔧 AI Nachrichten

📰 A Visual Approach to Gradient Descent and other Optimization Algorithms


📈 33.94 Punkte
🔧 AI Nachrichten

🔧 How Gradient Descent Powers Machine Learning Models


📈 33.94 Punkte
🔧 Programmierung

📰 Camera Radial Distortion Compensation with Gradient Descent


📈 33.94 Punkte
🔧 AI Nachrichten

🔧 State-Space Models Adapt by Gradient Descent, Learning in Context


📈 33.94 Punkte
🔧 Programmierung

🔧 Today's Trending Projects: Optimizing Gradient Descent for Global Optimization and More


📈 33.94 Punkte
🔧 Programmierung

🔧 Vanishing & Exploding Gradient Problem & Dying ReLU Problem in PyTorch


📈 32.69 Punkte
🔧 Programmierung

📰 Gradient AI Introduces Llama-3 8B Gradient Instruct 1048k: Setting New Standards in Long-Context AI


📈 32.38 Punkte
🔧 AI Nachrichten

🔧 Customize dataset so that Dataloader can apply...


📈 26.52 Punkte
🔧 Programmierung

🔧 Implementing DataLoader and Understanding Its Advantages Over Lookup


📈 26.52 Punkte
🔧 Programmierung

🔧 Implementing DataLoader and Understanding Its Advantages Over Lookup


📈 26.52 Punkte
🔧 Programmierung

📰 ML Engineering 101: A Thorough Explanation of The Error “DataLoader worker (pid(s) xxx) exited…


📈 26.52 Punkte
🔧 AI Nachrichten

🕵️ CVE-2018-11797 | Oracle Retail Xstore Point of Service 17 Dataloader input validation


📈 26.52 Punkte
🕵️ Sicherheitslücken

📰 Gradient Boosting Regressor, Explained: A Visual Guide with Code Examples


📈 26.1 Punkte
🔧 AI Nachrichten

📰 Pytorch: Microsoft startet neuen Azure-Service Pytorch Enterprise


📈 24.97 Punkte
📰 IT Nachrichten

📰 PyTorch Lightning 1.0: PyTorch, nur schneller und flexibler


📈 24.97 Punkte
📰 IT Nachrichten

🔧 PyTorch on Azure: Full support for PyTorch 1.2


📈 24.97 Punkte
🔧 Programmierung

🔧 Learn You a PyTorch! (aka Introduction Into PyTorch)


📈 24.97 Punkte
🔧 Programmierung

matomo