1. 数据集下载
https://www.kaggle.com/c/dog-breed-identification
2. 目录结构
3. config.py
FILE_PATH = './dog-breed-identification' TEST_PATH = './dog-breed-identification/test' TRAIN_PATH = './dog-breed-identification/train' CSV_PATH = './dog-breed-identification/labels.csv'4. dataloader.py
import pandas as pd import dog_loader.config as conf from pandas import Series, DataFrame import numpy as np import os import PIL from torch.utils.data import Dataset, DataLoader from torchvision import transforms, utils # 一. 查看表格信息 df = pd.read_csv(conf.CSV_PATH) print(df.info()) print(df.head()) print('*' * 60) # 二.预处理 # 1)得到一个长 list1 : 里面是每张图片的路径 # 2)另外一个长list2: 里面是每张图片对应的标签（整数），顺序要和list1对应。 # 3)把这两个list切分出来一部分作为验证集 breed = df['breed'] print(type(breed)) breed_np = breed.values print(type(breed_np)) print(breed_np.shape) # 看一下一共多少不同种类 #120 breed_set = set(breed_np) print('breed_set_size', len(breed_set)) # 构建一个编号与名称对应的字典，以后输出的数字要变成名字的时候用： breed_120_list = list(breed_set) dic = {} for i in range(120): dic[breed_120_list[i]] = i print('*' * 60) # 处理id那一列，分割成两段： file = df["id"].values print('file.shape', file.shape) file = [i + ".jpg" for i in file] file = [os.path.join(conf.TRAIN_PATH, i) for i in file] file_train = file[:8000] file_val = file[8000:] # print(file_train) np.save("file_train.npy", file_train) np.save("file_val.npy", file_val) # 处理breed那一列，分成两段： breed = df["breed"].values print('breed.shape', breed.shape) number = [] for i in range(10222): number.append(dic[breed[i]]) number = np.array(number) number_train = number[:8000] number_val = number[8000:] np.save("number_train.npy", number_train) np.save("number_val.npy", number_val) normalize = transforms.Normalize( mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225] ) preprocess = transforms.Compose([ # transforms.Scale(256), # transforms.CenterCrop(224), transforms.ToTensor(), normalize ]) def default_loader(path): img_pil = PIL.Image.open(path) img_pil = img_pil.resize((224, 224)) img_tensor = preprocess(img_pil) return img_tensor # 当然出来的时候已经全都变成了tensor class trainset(Dataset): def __init__(self, loader=default_loader): # 定义好 image 的路径 self.images = file_train self.target = number_train self.loader = loader def __getitem__(self, index): fn = self.images[index] img = self.loader(fn) target = self.target[index] return img, target def __len__(self): return len(self.images) # 自定义Dataset只需要最下面一个class,继承自Dataset类。有三个私有函数 # def __init__(self, loader=default_loader): # 这个里面一般要初始化一个loader(代码见上面),一个images_path的列表，一个target的列表 # def __getitem__(self, index)： # 这里吗就是在给你一个index的时候，你返回一个图片的tensor和target的tensor,使用了loader方法，经过 归一化，剪裁，类型转化，从图像变成tensor # def __len__(self): # return 你所有数据的个数5. dataset.py, 测试dataloader:
import dog_loader.dataloader as dat from torch.utils.data import Dataset, DataLoader train_data = dat.trainset() trainloader = DataLoader(train_data, batch_size=128, shuffle=True) print('type(trainloader)', type(trainloader)) # 使用enumerate读取 # for index, data in enumerate(trainloader): # image, label = data # print(image.size(), label.size()) # print('image_tensor', image[0][0][0][:5]) # print('label', label[:5]) for index, (image, label) in enumerate(trainloader): print(image.size(), label.size()) # print('image_tensor', image[0][0][0][:5]) # print('label', label[:5]) print('-'*25, index, '-'*25)6. 参考：
https://zhuanlan.zhihu.com/p/35698470
