如何使用 torch vision 在 Google Colab 上加载 CelebA 数据集，而不 运行 内存不足？
How do I load the CelebA dataset on Google Colab, using torch vision, without running out of memory?
我正在学习 DCGAN 上的教程。每当我尝试加载 CelebA 数据集时，torchvision 都会用完我所有 运行-time 的内存 (12GB) 并且 运行time 崩溃。我正在寻找如何加载和应用转换到数据集的方法，而不会占用我 运行-time 的资源。
复制
这是导致问题的代码部分。
# Root directory for the dataset
data_root = 'data/celeba'
# Spatial size of training images, images are resized to this size.
image_size = 64
celeba_data = datasets.CelebA(data_root,
download=True,
transform=transforms.Compose([
transforms.Resize(image_size),
transforms.CenterCrop(image_size),
transforms.ToTensor(),
transforms.Normalize(mean=[0.5, 0.5, 0.5],
std=[0.5, 0.5, 0.5])
]))
可以找到完整的笔记本here
环境
PyTorch 版本：1.7.1+cu101
是否为调试版本：False
用于构建 PyTorch 的 CUDA：10.1
用于构建 PyTorch 的 ROCM：N/A
OS：Ubuntu 18.04.5 LTS (x86_64)
GCC 版本: (Ubuntu 7.5.0-3ubuntu1~18.04) 7.5.0
Clang 版本：6.0.0-1ubuntu2 (tags/RELEASE_600/final)
CMake版本：3.12.0版
Python版本：3.6（64位运行时间）
CUDA 是否可用：是
CUDA 运行时间版本：10.1.243
GPU型号及配置：GPU 0：Tesla T4
Nvidia驱动版本：418.67
cuDNN 版本：/usr/lib/x86_64-linux-gnu/libcudnn.so.7.6.5
HIP 运行时间版本：N/A
MIOpen 运行时间版本：N/A
相关库的版本：
- [pip3] numpy==1.19.4
- [pip3]火炬==1.7.1+cu101
- [pip3] torchaudio==0.7.2
- pip3] torchsummary==1.5.1
- [pip3] torchtext==0.3.1
- [pip3] torchvision==0.8.2+cu101
- [conda] 无法收集
其他上下文
我尝试过的一些事情是：
- 在不同的行上下载和加载数据集。例如：
# Download the dataset only
datasets.CelebA(data_root, download=True)
# Load the dataset here
celeba_data = datasets.CelebA(data_root, download=False, transforms=...)
- 使用
ImageFolder 数据集 class 而不是 CelebA class。例如：
# Download the dataset only
datasets.CelebA(data_root, download=True)
# Load the dataset using the ImageFolder class
celeba_data = datasets.ImageFolder(data_root, transforms=...)
在这两种情况下，内存问题仍然存在。
尝试以下操作：
from torchvision.datasets import ImageFolder
from torch.utils.data import DataLoader
from torchvision import transforms
# Root directory for the dataset
data_root = 'data/celeba'
# Spatial size of training images, images are resized to this size.
image_size = 64
# batch size
batch_size = 10
transform=transforms.Compose([
transforms.Resize(image_size),
transforms.CenterCrop(image_size),
transforms.ToTensor(),
transforms.Normalize(mean=[0.5, 0.5, 0.5],
std=[0.5, 0.5, 0.5])
dataset = ImageFolder(data_root, transform)
data_loader = DataLoader(dataset=dataset, batch_size=batch_size, shuffle=True, num_workers=8, drop_last=True)
Dataloader class 的更多详细信息可以查找 here。
以上回答，礼貌 this kaggle notebook.
我没能找到内存问题的解决方案。但是，我想出了一个解决方法，即自定义数据集。这是我的实现：
import os
import zipfile
import gdown
import torch
from natsort import natsorted
from PIL import Image
from torch.utils.data import Dataset
from torchvision import transforms
## Setup
# Number of gpus available
ngpu = 1
device = torch.device('cuda:0' if (
torch.cuda.is_available() and ngpu > 0) else 'cpu')
## Fetch data from Google Drive
# Root directory for the dataset
data_root = 'data/celeba'
# Path to folder with the dataset
dataset_folder = f'{data_root}/img_align_celeba'
# URL for the CelebA dataset
url = 'https://drive.google.com/uc?id=1cNIac61PSA_LqDFYFUeyaQYekYPc75NH'
# Path to download the dataset to
download_path = f'{data_root}/img_align_celeba.zip'
# Create required directories
if not os.path.exists(data_root):
os.makedirs(data_root)
os.makedirs(dataset_folder)
# Download the dataset from google drive
gdown.download(url, download_path, quiet=False)
# Unzip the downloaded file
with zipfile.ZipFile(download_path, 'r') as ziphandler:
ziphandler.extractall(dataset_folder)
## Create a custom Dataset class
class CelebADataset(Dataset):
def __init__(self, root_dir, transform=None):
"""
Args:
root_dir (string): Directory with all the images
transform (callable, optional): transform to be applied to each image sample
"""
# Read names of images in the root directory
image_names = os.listdir(root_dir)
self.root_dir = root_dir
self.transform = transform
self.image_names = natsorted(image_names)
def __len__(self):
return len(self.image_names)
def __getitem__(self, idx):
# Get the path to the image
img_path = os.path.join(self.root_dir, self.image_names[idx])
# Load image and convert it to RGB
img = Image.open(img_path).convert('RGB')
# Apply transformations to the image
if self.transform:
img = self.transform(img)
return img
## Load the dataset
# Path to directory with all the images
img_folder = f'{dataset_folder}/img_align_celeba'
# Spatial size of training images, images are resized to this size.
image_size = 64
# Transformations to be applied to each individual image sample
transform=transforms.Compose([
transforms.Resize(image_size),
transforms.CenterCrop(image_size),
transforms.ToTensor(),
transforms.Normalize(mean=[0.5, 0.5, 0.5],
std=[0.5, 0.5, 0.5])
])
# Load the dataset from file and apply transformations
celeba_dataset = CelebADataset(img_folder, transform)
## Create a dataloader
# Batch size during training
batch_size = 128
# Number of workers for the dataloader
num_workers = 0 if device.type == 'cuda' else 2
# Whether to put fetched data tensors to pinned memory
pin_memory = True if device.type == 'cuda' else False
celeba_dataloader = torch.utils.data.DataLoader(celeba_dataset,
batch_size=batch_size,
num_workers=num_workers,
pin_memory=pin_memory,
shuffle=True)
这个实现是内存高效的，适用于我的用例，即使在训练期间使用的内存平均约为 (4GB)。但是，我希望进一步了解可能导致内存问题的原因。
我正在学习 DCGAN 上的教程。每当我尝试加载 CelebA 数据集时，torchvision 都会用完我所有 运行-time 的内存 (12GB) 并且 运行time 崩溃。我正在寻找如何加载和应用转换到数据集的方法，而不会占用我 运行-time 的资源。
复制
这是导致问题的代码部分。
# Root directory for the dataset
data_root = 'data/celeba'
# Spatial size of training images, images are resized to this size.
image_size = 64
celeba_data = datasets.CelebA(data_root,
download=True,
transform=transforms.Compose([
transforms.Resize(image_size),
transforms.CenterCrop(image_size),
transforms.ToTensor(),
transforms.Normalize(mean=[0.5, 0.5, 0.5],
std=[0.5, 0.5, 0.5])
]))
可以找到完整的笔记本here
环境
PyTorch 版本：1.7.1+cu101
是否为调试版本：False
用于构建 PyTorch 的 CUDA：10.1
用于构建 PyTorch 的 ROCM：N/A
OS：Ubuntu 18.04.5 LTS (x86_64)
GCC 版本: (Ubuntu 7.5.0-3ubuntu1~18.04) 7.5.0
Clang 版本：6.0.0-1ubuntu2 (tags/RELEASE_600/final)
CMake版本：3.12.0版
Python版本：3.6（64位运行时间）
CUDA 是否可用：是
CUDA 运行时间版本：10.1.243
GPU型号及配置：GPU 0：Tesla T4
Nvidia驱动版本：418.67
cuDNN 版本：/usr/lib/x86_64-linux-gnu/libcudnn.so.7.6.5
HIP 运行时间版本：N/A
MIOpen 运行时间版本：N/A
相关库的版本：
- [pip3] numpy==1.19.4
- [pip3]火炬==1.7.1+cu101
- [pip3] torchaudio==0.7.2
- pip3] torchsummary==1.5.1
- [pip3] torchtext==0.3.1
- [pip3] torchvision==0.8.2+cu101
- [conda] 无法收集
其他上下文
我尝试过的一些事情是：
- 在不同的行上下载和加载数据集。例如：
# Download the dataset only
datasets.CelebA(data_root, download=True)
# Load the dataset here
celeba_data = datasets.CelebA(data_root, download=False, transforms=...)
- 使用
ImageFolder数据集 class 而不是CelebAclass。例如：
# Download the dataset only
datasets.CelebA(data_root, download=True)
# Load the dataset using the ImageFolder class
celeba_data = datasets.ImageFolder(data_root, transforms=...)
在这两种情况下，内存问题仍然存在。
尝试以下操作：
from torchvision.datasets import ImageFolder
from torch.utils.data import DataLoader
from torchvision import transforms
# Root directory for the dataset
data_root = 'data/celeba'
# Spatial size of training images, images are resized to this size.
image_size = 64
# batch size
batch_size = 10
transform=transforms.Compose([
transforms.Resize(image_size),
transforms.CenterCrop(image_size),
transforms.ToTensor(),
transforms.Normalize(mean=[0.5, 0.5, 0.5],
std=[0.5, 0.5, 0.5])
dataset = ImageFolder(data_root, transform)
data_loader = DataLoader(dataset=dataset, batch_size=batch_size, shuffle=True, num_workers=8, drop_last=True)
Dataloader class 的更多详细信息可以查找 here。
以上回答，礼貌 this kaggle notebook.
我没能找到内存问题的解决方案。但是，我想出了一个解决方法，即自定义数据集。这是我的实现：
import os
import zipfile
import gdown
import torch
from natsort import natsorted
from PIL import Image
from torch.utils.data import Dataset
from torchvision import transforms
## Setup
# Number of gpus available
ngpu = 1
device = torch.device('cuda:0' if (
torch.cuda.is_available() and ngpu > 0) else 'cpu')
## Fetch data from Google Drive
# Root directory for the dataset
data_root = 'data/celeba'
# Path to folder with the dataset
dataset_folder = f'{data_root}/img_align_celeba'
# URL for the CelebA dataset
url = 'https://drive.google.com/uc?id=1cNIac61PSA_LqDFYFUeyaQYekYPc75NH'
# Path to download the dataset to
download_path = f'{data_root}/img_align_celeba.zip'
# Create required directories
if not os.path.exists(data_root):
os.makedirs(data_root)
os.makedirs(dataset_folder)
# Download the dataset from google drive
gdown.download(url, download_path, quiet=False)
# Unzip the downloaded file
with zipfile.ZipFile(download_path, 'r') as ziphandler:
ziphandler.extractall(dataset_folder)
## Create a custom Dataset class
class CelebADataset(Dataset):
def __init__(self, root_dir, transform=None):
"""
Args:
root_dir (string): Directory with all the images
transform (callable, optional): transform to be applied to each image sample
"""
# Read names of images in the root directory
image_names = os.listdir(root_dir)
self.root_dir = root_dir
self.transform = transform
self.image_names = natsorted(image_names)
def __len__(self):
return len(self.image_names)
def __getitem__(self, idx):
# Get the path to the image
img_path = os.path.join(self.root_dir, self.image_names[idx])
# Load image and convert it to RGB
img = Image.open(img_path).convert('RGB')
# Apply transformations to the image
if self.transform:
img = self.transform(img)
return img
## Load the dataset
# Path to directory with all the images
img_folder = f'{dataset_folder}/img_align_celeba'
# Spatial size of training images, images are resized to this size.
image_size = 64
# Transformations to be applied to each individual image sample
transform=transforms.Compose([
transforms.Resize(image_size),
transforms.CenterCrop(image_size),
transforms.ToTensor(),
transforms.Normalize(mean=[0.5, 0.5, 0.5],
std=[0.5, 0.5, 0.5])
])
# Load the dataset from file and apply transformations
celeba_dataset = CelebADataset(img_folder, transform)
## Create a dataloader
# Batch size during training
batch_size = 128
# Number of workers for the dataloader
num_workers = 0 if device.type == 'cuda' else 2
# Whether to put fetched data tensors to pinned memory
pin_memory = True if device.type == 'cuda' else False
celeba_dataloader = torch.utils.data.DataLoader(celeba_dataset,
batch_size=batch_size,
num_workers=num_workers,
pin_memory=pin_memory,
shuffle=True)
这个实现是内存高效的，适用于我的用例，即使在训练期间使用的内存平均约为 (4GB)。但是，我希望进一步了解可能导致内存问题的原因。