A tutorial on writing custom Datasets + Samplers and using transforms

[soumith]

takeaway from thread: https://discuss.pytorch.org/t/feedback-on-pytorch-for-kaggle-competitions/2252/2

[mratsim]

My code from https://www.kaggle.com/mratsim/planet-understanding-the-amazon-from-space/starting-kit-for-pytorch-deep-learning

class KaggleAmazonDataset(Dataset):
    """Dataset wrapping images and target labels for Kaggle - Planet Amazon from Space competition.

    Arguments:
        A CSV file path
        Path to image folder
        Extension of images
        PIL transforms
    """

    def __init__(self, csv_path, img_path, img_ext, transform=None):
    
        tmp_df = pd.read_csv(csv_path)
        assert tmp_df['image_name'].apply(lambda x: os.path.isfile(img_path + x + img_ext)).all(), \
"Some images referenced in the CSV file were not found"
        
        self.mlb = MultiLabelBinarizer()
        self.img_path = img_path
        self.img_ext = img_ext
        self.transform = transform

        self.X_train = tmp_df['image_name']
        self.y_train = self.mlb.fit_transform(tmp_df['tags'].str.split()).astype(np.float32)

    def __getitem__(self, index):
        img = Image.open(self.img_path + self.X_train[index] + self.img_ext)
        img = img.convert('RGB')
        if self.transform is not None:
            img = self.transform(img)
        
        label = torch.from_numpy(self.y_train[index])
        return img, label

    def __len__(self):
        return len(self.X_train.index)

Loading the data set

transformations = transforms.Compose([transforms.Scale(32),transforms.ToTensor()])

dset = KaggleAmazonDataset(train_data,img_path,img_ext,transformations)
train_loader = DataLoader(dset,
                          batch_size=256,
                          shuffle=True,
                          num_workers=4
                         # pin_memory=True # CUDA only
                         )

[chsasank]

I can write a dataloader tutorial, but I don't know which dataset to use. I'm not sure if we can use a kaggle dataset. We need a fairly complex dataset which is kinda open, so that we can put a subset of it available for download on our servers. Any ideas?

…

On Thu 27 Apr, 2017, 06:41 Mamy Ratsimbazafy, ***@***.***> wrote: My code from https://www.kaggle.com/mratsim/planet-understanding-the-amazon-from-space/starting-kit-for-pytorch-deep-learning class KaggleAmazonDataset(Dataset): """Dataset wrapping images and target labels for Kaggle - Planet Amazon from Space competition. Arguments: A CSV file path Path to image folder Extension of images PIL transforms """ def __init__(self, csv_path, img_path, img_ext, transform=None): tmp_df = pd.read_csv(csv_path) assert tmp_df['image_name'].apply(lambda x: os.path.isfile(img_path + x + img_ext)).all(), \"Some images referenced in the CSV file were not found" self.mlb = MultiLabelBinarizer() self.img_path = img_path self.img_ext = img_ext self.transform = transform self.X_train = tmp_df['image_name'] self.y_train = self.mlb.fit_transform(tmp_df['tags'].str.split()).astype(np.float32) def __getitem__(self, index): img = Image.open(self.img_path + self.X_train[index] + self.img_ext) img = img.convert('RGB') if self.transform is not None: img = self.transform(img) label = torch.from_numpy(self.y_train[index]) return img, label def __len__(self): return len(self.X_train.index) Loading the data set transformations = transforms.Compose([transforms.Scale(32),transforms.ToTensor()]) dset = KaggleAmazonDataset(train_data,img_path,img_ext,transformations) train_loader = DataLoader(dset, batch_size=256, shuffle=True, num_workers=4 # pin_memory=True # CUDA only ) — You are receiving this because you are subscribed to this thread. Reply to this email directly, view it on GitHub <#78 (comment)>, or mute the thread <https://github.com/notifications/unsubscribe-auth/AI3_Ezf8uRMVGfaacsGygsd6-MBz2dt2ks5rz-tEgaJpZM4NJlGq> .

[hengck23]

i think a tutorial on simple data is good enough. For example, I try to custom the cifar10 data loader:

#  what is pinned memory
#       https://devblogs.nvidia.com/parallelforall/how-optimize-data-transfers-cuda-fortran/
#


# pytorch custom data
#       http://stackoverflow.com/questions/43441673/trying-to-load-a-custom-dataset-in-pytorch
#       http://forums.fast.ai/t/how-do-you-use-custom-dataset-with-pytorch/2040
#       https://discuss.pytorch.org/t/questions-about-imagefolder/774/6


## https://discuss.pytorch.org/t/feedback-on-pytorch-for-kaggle-competitions/2252/3
## https://discuss.pytorch.org/t/questions-about-imagefolder/774
## https://github.com/pytorch/tutorials/issues/78
## https://www.kaggle.com/mratsim/planet-understanding-the-amazon-from-space/starting-kit-for-pytorch-deep-learning
## http://pytorch.org/docs/_modules/torch/utils/data/dataset.html
## https://www.kaggle.com/mratsim/planet-understanding-the-amazon-from-space/starting-kit-for-pytorch-deep-learning/notebook
## https://devhub.io/repos/pytorch-vision
## https://github.com/ClementPinard/FlowNetPytorch/blob/master/balancedsampler.py


#my libs
from net.common import *

#pytorch
import torch
import torchvision.transforms as transforms

from torch.utils.data.dataset import Dataset
from torch.utils.data import DataLoader
from torch.utils.data.sampler import Sampler



## helper functions ----------------------------------------------
def im_show(name, image, resize=1):
    H,W = image.shape[0:2]
    cv2.namedWindow(name, cv2.WINDOW_NORMAL)
    cv2.imshow(name, image.astype(np.uint8))
    cv2.resizeWindow(name, round(resize*W), round(resize*H))


def tensor_to_img(img, mean=0, std=1):
    img = np.transpose(img.numpy(), (1, 2, 0))
    img = (img*std+ mean)*255
    img = img.astype(np.uint8)
    #img = cv2.cvtColor(img , cv2.COLOR_BGR2RGB)
    return img


## Augmentation code here -----------------------------------

# geometric transform
# illumination transform
# etc

def dummy_transform1(image):
    print ('\t\tdummy_transform1')
    return image
def dummy_transform2(image):
    print ('\t\tdummy_transform2')
    return image


## Sampler code here -----------------------------------

# see trorch/utils/data/sampler.py
class DummySampler(Sampler):
    def __init__(self, data):
        self.num_samples = len(data)

    def __iter__(self):
        print ('\tcalling Sampler:__iter__')
        return iter(range(self.num_samples))

    def __len__(self):
        print ('\tcalling Sampler:__len__')
        return self.num_samples




## custom data loader -----------------------------------
# see torchvision/dataset/cifar.py
class CifarDataset(Dataset):
    DATA_DIR = '/root/share/project/pytorch/data/cifar10/rgb'

    def __init__(self, split, transform=None, data_dir=None):
        data_dir = self.DATA_DIR if data_dir is None else data_dir

        list = data_dir +'/split/'+ split
        with open(list) as f:
            names = f.readlines()
        names = [x.strip() for x in names]

        #dummy image loader
        N = len(names)
        images = np.zeros((N,32,32,3),dtype=np.uint8)
        labels = []  #np.zeros((N),dtype=np.int32)

        for n in range(N):
            images[n] = cv2.imread(data_dir + '/image/' + names[n],1)
            s = names[n].split('/')
            labels += [int(s[-2])]


        self.transform=transform
        self.images=images  #torch.from_numpy(images)
        self.labels=labels  #torch.from_numpy(labels)
        self.classes=('plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck')



    def __getitem__(self, index):
        print ('\tcalling Dataset:__getitem__ @ idx=%d'%index)
        img   = self.images[index]
        label = self.labels[index]

        #img = PIL.Image.fromarray(img)
        if self.transform is not None:
            img = self.transform(img)

        return img, label


    def __len__(self):
        print ('\tcalling Dataset:__len__')
        return len(self.images)







# main #################################################################
if __name__ == '__main__':
    print( '%s: calling main function ... ' % os.path.basename(__file__))


    transform = transforms.Compose([
                    # transforms.ToTensor(): Converts a PIL.Image or numpy.ndarray (H x W x C) in the range [0, 255] to a torch.FloatTensor of shape (C x H x W) in the range [0.0, 1.0].
                    transforms.ToTensor(),
                    transforms.Lambda(lambda x: dummy_transform1(x)),
                    transforms.Lambda(lambda x: dummy_transform2(x)),
                ])

    dataset = CifarDataset('train10', transform=transform)
    sampler = DummySampler(dataset)  #DummySampler(dataset)  #None #
    loader  = DataLoader(dataset, batch_size=4, sampler=sampler, shuffle=False, num_workers=2)  ##shuffle=True #False
                         # pin_memory=True
                         # drop_last=False


    classes = dataset.classes
    for epoch in range(10):
        print('epoch=%d -------------------------'%(epoch))
        for i, data in enumerate(loader, 0):
            print('i=%d: '%(i))

            # get the inputs
            images, labels = data
            num = len(images)
            for n in range(num):
                image=images[n]
                label=labels[n]
                print('\t\tlabel=%d : %s'%(label,classes[label]))
                #print('')

                im_show('image', tensor_to_img(image), resize=6 )
                cv2.waitKey(1)
    dd=0

[zachmaurer]

I just wanted to express my support for a tutorial on these topics using a more complex dataset than CIFAR10.

For me, the confusion is less about the difference between the Dataset and DataLoader, but more on how to sample efficiently (from a memory and throughput standpoint) from datasets that do not all fit in memory (and perhaps have other conditions like multiple labels or data augmentation)

@mratsim 's example was very helpful. @soumith thank you for opening this as an issue.

[harveyslash]

is this going to be multi threaded ? or will the author of the class have to handle that by himself/herself ?

[chsasank]

@harveyslash A tutorial on data loading is waiting to be merged at #96. Please go through it to make things clear.

[brando90]

where is the link to the custom data set tutorial?

[miguelvr]

@brando90 I think it is this one:
https://pytorch.org/tutorials/beginner/data_loading_tutorial.html

[lcyuanjiang]

i think a tutorial on simple data is good enough. For example, I try to custom the cifar10 data loader:

#  what is pinned memory
#       https://devblogs.nvidia.com/parallelforall/how-optimize-data-transfers-cuda-fortran/
#


# pytorch custom data
#       http://stackoverflow.com/questions/43441673/trying-to-load-a-custom-dataset-in-pytorch
#       http://forums.fast.ai/t/how-do-you-use-custom-dataset-with-pytorch/2040
#       https://discuss.pytorch.org/t/questions-about-imagefolder/774/6


## https://discuss.pytorch.org/t/feedback-on-pytorch-for-kaggle-competitions/2252/3
## https://discuss.pytorch.org/t/questions-about-imagefolder/774
## https://github.com/pytorch/tutorials/issues/78
## https://www.kaggle.com/mratsim/planet-understanding-the-amazon-from-space/starting-kit-for-pytorch-deep-learning
## http://pytorch.org/docs/_modules/torch/utils/data/dataset.html
## https://www.kaggle.com/mratsim/planet-understanding-the-amazon-from-space/starting-kit-for-pytorch-deep-learning/notebook
## https://devhub.io/repos/pytorch-vision
## https://github.com/ClementPinard/FlowNetPytorch/blob/master/balancedsampler.py


#my libs
from net.common import *

#pytorch
import torch
import torchvision.transforms as transforms

from torch.utils.data.dataset import Dataset
from torch.utils.data import DataLoader
from torch.utils.data.sampler import Sampler



## helper functions ----------------------------------------------
def im_show(name, image, resize=1):
    H,W = image.shape[0:2]
    cv2.namedWindow(name, cv2.WINDOW_NORMAL)
    cv2.imshow(name, image.astype(np.uint8))
    cv2.resizeWindow(name, round(resize*W), round(resize*H))


def tensor_to_img(img, mean=0, std=1):
    img = np.transpose(img.numpy(), (1, 2, 0))
    img = (img*std+ mean)*255
    img = img.astype(np.uint8)
    #img = cv2.cvtColor(img , cv2.COLOR_BGR2RGB)
    return img


## Augmentation code here -----------------------------------

# geometric transform
# illumination transform
# etc

def dummy_transform1(image):
    print ('\t\tdummy_transform1')
    return image
def dummy_transform2(image):
    print ('\t\tdummy_transform2')
    return image


## Sampler code here -----------------------------------

# see trorch/utils/data/sampler.py
class DummySampler(Sampler):
    def __init__(self, data):
        self.num_samples = len(data)

    def __iter__(self):
        print ('\tcalling Sampler:__iter__')
        return iter(range(self.num_samples))

    def __len__(self):
        print ('\tcalling Sampler:__len__')
        return self.num_samples




## custom data loader -----------------------------------
# see torchvision/dataset/cifar.py
class CifarDataset(Dataset):
    DATA_DIR = '/root/share/project/pytorch/data/cifar10/rgb'

    def __init__(self, split, transform=None, data_dir=None):
        data_dir = self.DATA_DIR if data_dir is None else data_dir

        list = data_dir +'/split/'+ split
        with open(list) as f:
            names = f.readlines()
        names = [x.strip() for x in names]

        #dummy image loader
        N = len(names)
        images = np.zeros((N,32,32,3),dtype=np.uint8)
        labels = []  #np.zeros((N),dtype=np.int32)

        for n in range(N):
            images[n] = cv2.imread(data_dir + '/image/' + names[n],1)
            s = names[n].split('/')
            labels += [int(s[-2])]


        self.transform=transform
        self.images=images  #torch.from_numpy(images)
        self.labels=labels  #torch.from_numpy(labels)
        self.classes=('plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck')



    def __getitem__(self, index):
        print ('\tcalling Dataset:__getitem__ @ idx=%d'%index)
        img   = self.images[index]
        label = self.labels[index]

        #img = PIL.Image.fromarray(img)
        if self.transform is not None:
            img = self.transform(img)

        return img, label


    def __len__(self):
        print ('\tcalling Dataset:__len__')
        return len(self.images)







# main #################################################################
if __name__ == '__main__':
    print( '%s: calling main function ... ' % os.path.basename(__file__))


    transform = transforms.Compose([
                    # transforms.ToTensor(): Converts a PIL.Image or numpy.ndarray (H x W x C) in the range [0, 255] to a torch.FloatTensor of shape (C x H x W) in the range [0.0, 1.0].
                    transforms.ToTensor(),
                    transforms.Lambda(lambda x: dummy_transform1(x)),
                    transforms.Lambda(lambda x: dummy_transform2(x)),
                ])

    dataset = CifarDataset('train10', transform=transform)
    sampler = DummySampler(dataset)  #DummySampler(dataset)  #None #
    loader  = DataLoader(dataset, batch_size=4, sampler=sampler, shuffle=False, num_workers=2)  ##shuffle=True #False
                         # pin_memory=True
                         # drop_last=False


    classes = dataset.classes
    for epoch in range(10):
        print('epoch=%d -------------------------'%(epoch))
        for i, data in enumerate(loader, 0):
            print('i=%d: '%(i))

            # get the inputs
            images, labels = data
            num = len(images)
            for n in range(num):
                image=images[n]
                label=labels[n]
                print('\t\tlabel=%d : %s'%(label,classes[label]))
                #print('')

                im_show('image', tensor_to_img(image), resize=6 )
                cv2.waitKey(1)
    dd=0

when the memory can not handle the big data ,how to implement with pytorch dataloader? i see not any iterator in your code