Select Device

1
2
3
4
5
6
def get_device():
    ''' Get device (if GPU is available, use GPU) '''
    return 'cuda' if torch.cuda.is_available() else 'cpu'

#To get the device:
device = get_device()  # get the current available device ('cpu' or 'cuda')
  • Default: tensors & modules will be computed with CPU
  • To check if computer has a NVIDIA CPU: torch.cuda.is_available()

Use Device

You can force a compution in specific device.

1
2
3
4
5
6
7
8
#Hardcode to use CPU
x = x.to('cpu')

#Hardcode to use GPU
x = x.to('cuda')

#Let system decide, using above function
x = x.to(device)

Load Data

PyTorch has two primitives to work with data: torch.utils.data.DataLoader and torch.utils.data.Dataset. Dataset stores the samples and their corresponding labels, and DataLoader wraps an iterable around the Dataset.

Dataset and DataLoader

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
from torch.utils.data import Dataset, DataLoader

class MyDataset(Dataset): # A subclass of Dataset
  def __init__(self, file, ...):
    # Read data & preprocess
    self.data = '...'
    
  def __getitem__(self, index): #Used by DataLoader
    # Returns one sample at a time
    return self.data[index]
  
  def __len__(self):
    # Return the size of the dataset
		return len(self.data)

To use the class

1
2
dataset = MyDataset(file)
dataloader = DataLoader(dataset, batch_size, shuffle=True)

DataLoader(dataset, batch_size, shuffle=True

  • shuffle=False when we use it for Testing or Vaildation
  • shuffle=True when we use it for Training
    • Only shuffle the training data

Example: Preparing Train and Valid and Test dataloader

1
2
3
4
5
6
7
8
9
10
11
BATCH_SIZE = 64

# Before feeding into Dataset class, split the data by yourself
train_dataset = MyDataset(train_x, train_y)
val_dataset = MyDataset(val_x, val_y)
test_dataset = MyDataset(test_x, val_y)

# Feed to DataLoader class (only shuffle the training data)
tr_set = DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True) 
dv_set = DataLoader(val_dataset, batch_size=BATCH_SIZE, shuffle=False)
tt_set = DataLoader(test_dataset, batch_size=BATCH_SIZE, shuffle=False)

Concat Data

1
from torch.utils.data import ConcatDataset, DataLoader, Subset

Image Augmentation (For Image only)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
import torchvision.transforms as transforms

# It is important to do data augmentation in training.
# However, not every augmentation is useful.
# Please think about what kind of augmentation is helpful for food recognition.
train_tfm = transforms.Compose([
    # Resize the image into a fixed shape (height = width = 128)
    transforms.Resize((128, 128)),
    # You may add some transforms here.
    # ToTensor() should be the last one of the transforms.
    transforms.ToTensor(),
])

# We don't need augmentations in testing and validation.
# All we need here is to resize the PIL image and transform it into Tensor.
test_tfm = transforms.Compose([
    transforms.Resize((128, 128)),
    transforms.ToTensor(),
])



# Use it in datasetfolder:
from torchvision.datasets import DatasetFolder
train_set = DatasetFolder(path, loader=lambda x: Image.open(x), extensions="jpg", transform=train_tfm)

# Construct data loader:
train_loader = DataLoader(train_set, batch_size=batch_size, shuffle=True, num_workers=8, pin_memory=True)

Model Creation

1
import torch.nn as nn

PyTorch offers domain-specific libraries such as TorchText, TorchVision, and TorchAudio, all of which include datasets.

  • TorchVision - for Computer Vision
  • TorchText - for Natural Langauge Processing
  • TorchAudio - for Speech/Audio processing

Model Define

Common used items:

Layers

  • Fully-connected Layer - nn.Linear(in_features, out_features)

Activations

  • Sigmoid - nn.Sigmoid()
  • ReLU - nn.ReLU()
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
# PyTorch - Model Creation
import torch.nn as nn

class MyModel(nn.Module): #A subclass of Module
		def __init__(self):
      	super(MyModel, self).__init__() # Initialize model
        
        # Define Layers (Sequential, layer by layer)
        self.net = nn.Sequential(
          	nn.Linear(10, 32),
          	nn.Sigmoid(),
          	nn.Linear(32, 1)
        )
        
    def forward(self, x): #To compute output
      	return self.net(x)

Which is same as:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
# PyTorch - Model Creation
import torch.nn as nn

class MyModel(nn.Module): #A subclass of Module
		def __init__(self):
      	super(MyModel, self).__init__() # Initialize model
        
        # Define Layers
        self.layer1 = nn.Linear(10, 32)
        self.act_fn = nn.Sigmoid()
        self.out = nn.Linear(32, 1)
        
    def forward(self, x): #To compute output
      	x = self.layer1(x)
        x = self.act_fn(x)
        x = self.out(x)
      	return x

Example - Regression Model

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
import torch
import torch.nn as nn

class Classifier(nn.Module):
    def __init__(self):
        super(Classifier, self).__init__()
        self.layer1 = nn.Linear(429, 1024)
        self.layer2 = nn.Linear(1024, 512)
        self.layer3 = nn.Linear(512, 128)
        self.out = nn.Linear(128, 39) 

        self.act_fn = nn.Sigmoid()

    def forward(self, x):
        x = self.layer1(x)
        x = self.act_fn(x)

        x = self.layer2(x)
        x = self.act_fn(x)

        x = self.layer3(x)
        x = self.act_fn(x)

        x = self.out(x)
        
        return x

Training

1
import torch.nn as nn

Loss Functions

  • Mean Squared Error (for linear regression) - nn.MSELoss()
  • Cross Entropy (for classification) - nn.CrossEntropyLoss()
    • Softmax is included

Optimizers

  • Adam - torch.optim.Adam(model.parameters(), lr=learning_rate)
  • SGD - torch.optim.SGD(model.parameters(), lr=learning_rate)
1
2
3
4
5
6
7
8
dataset = MyDataset(file) # read data via MyDataset
tr_set = DataLoader(dataset, 16, shuffle=True) # put dataset into Dataloader
#dv_set for validation set, tt_set for test set
...

model = MyModel().to(device) # contruct model and move to device (cpu/cuda)
criterion = nn.MSELoss() #set loss function
optimizer = torch.optim.SGD(model.parameters(), 0.1) #set optimizer
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
for epoch in range(n_epochs): # iterate n_epochs
  
  	# Training
  	model.train() # Set model to train mode
    for x, y in tr_set: # iterate through the DataLoader
      	optimizer.zero_grad() # set gradient to 0
        x, y = x.to(device), y.to(device) # move data to device (cpu/cuda)
        pred = model(x) # forward pass (compute output)
        loss = criterion(pred, y) # compute loss
        loss.backward() # compute gradient (backpropagation)
        optimizer.step() # update model with optimizer
    
    # Validation
    model.eval()
    total_loss = 0
    for x, y in dv_set:
      	x, y = x.to(device), y.to(device) # move data to device (cpu/cuda)
        with torch.no_grad(): # disable gradient calculation
          	pred = model(x) # forward pass (compute output)
            loss = criterion(pred, y) # compute loss
        total_loss = loss.cpu().item() * len(x) # accumulate loss
        avg_loss = total_loss / len(dv_set.dataset) compute average loss

Inference

1
2
3
4
5
6
7
model.eval() # set the model to evaluation mode
preds = []
for x in tt_set:
    x = x.to(device) # move data to device (cpu/cuda)
    with torch.no_grad(): # disable gradient calculation
        pred = model(x) # forward pass (compute output)
        preds.append(pred.cpu()) # collect prediction

Misc

Delete Variables to save memory usage

Example - Delete Variables and collect garbage

1
2
3
4
import gc

del train, train_label, train_x, train_y, val_x, val_y
gc.collect()

Cuda Out of Memory

1
CUDA out of memory. Tried to allocate ...
  • The batch size of data is too large to fit in the GPU. Reduce the batch size.

Saving / Loading Model

Save

1
torch.save(model.state_dict(), path)

Load

1
2
ckpt = torch.load(path)
model.load_state_dic(cpkt)

Reference