kaggle Dogs_and_Cats实践记录

• 数据下载
• transfer learning
• • train-set, dev-set准备
  • fine-tuning resnet18
  • 预测并将结果存至submission.csv
• 代码下载

数据下载

kaggle Dogs_and_Cats数据集下载，可以在配置好kaggle 好之后，从相应的Competition:Dogs_and_Cats 中下载数据。

原数据包含了train.zip & test1.zip & sampleSubmission.csv

解压后，train.zip包含25000张图片，12500为Dogs, 12500为Cats, test1.zip解压之后为待预测的数据,12500张。

transfer learning

总体思路： Dogs, Cats与ImageNet中的数据有重合，因此，采用 transfer learning的方式来做。

train-set, dev-set准备

首先，训练需使用train.zip中的数据，为更快地操作，结合torchvision ImageFolder(folder.py)类，将原数据分别放进dogs , cats目录，并且使用下述脚本，将train分成train-set, dev-set.

import os
import sys
import shutil
import glob
import os.path as osp

dev_num = 2500

def split_data(root, target_dir):
    img_files = glob.glob(root+"/*.jpg")
    dev_files = img_files[-dev_num:]
    print('dev_files:', dev_files)

    for dev_file in dev_files:
        shutil.move(dev_file, osp.join(target_dir, osp.basename(dev_file)))
        


if __name__ == "__main__":
    root = sys.argv[1]
    target_dir = sys.argv[2]
    split_data(root, target_dir)
           

fine-tuning resnet18

构建模型，因要分类的数据不复杂，现采用resnet18作为基础网格，并且借用已有的预训练权重，只对最后一层fc层进行fine-tune.代码如下：

import torch
import torchvision

import torch.optim as optim
from torch.utils.data import *
from torchvision import models, transforms, datasets
import torch.nn as nn
import matplotlib.pyplot as plt

from torch.utils.tensorboard import SummaryWriter


writer = SummaryWriter(log_dir='runs')


def imshow(inp, title=None):
    """ Imshow for Tensor"""
    print(inp.size())
    inp = inp.numpy().transpose((1,2,0))
    mean = np.array([0.485, 0.456, 0.406])
    std = np.array([0.229, 0.224, 0.225])

    inp = inp*std+mean
    inp = np.clip(inp, 0, 1)
    plt.imshow(inp)
    if title is not None:
        plt.title(title)
    plt.show()

def test_show_data():
    # Test Showing, Get a batch of training data
    inputs, classes = next(iter(train_dataloader))
    # Make a grid from batch
    out = torchvision.utils.make_grid(inputs)
    imshow(out, title=[class_names[x] for x in classes])

data_transforms = {
    'train' : transforms.Compose([
        transforms.RandomResizedCrop(224),
        transforms.RandomHorizontalFlip(),
        transforms.ToTensor(),
        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
    ]),
    'val' : transforms.Compose([
        transforms.Resize(256),
        transforms.CenterCrop(224),
        transforms.ToTensor(),
        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
    ])
}

train_dataset = datasets.ImageFolder(root="/home/jingwenlai/data/kaggle/dogs_and_cats/train", transform=data_transforms['train'])
train_dataloader = DataLoader(train_dataset, batch_size=64, shuffle=True, num_workers=4)
val_dataset = datasets.ImageFolder(root="/home/jingwenlai/data/kaggle/dogs_and_cats/dev", transform=data_transforms['val'])
val_dataloader = DataLoader(val_dataset, batch_size=32, shuffle=False, num_workers=4)

class_names = ['cat', 'dog']
device = torch.device('cuda:0' if torch.cuda.is_available() else "cpu")

# Now, because the val set have problem, only use trainset
def train_model(model, criterion, optimizer, lr_scheduler, num_epoches):
    best_model = None
    best_acc = 0.0

    for epoch in range(0, num_epoches):
        print("---------Training {}/{}---------".format(epoch, num_epoches))
        model.train()
        running_loss = 0.0
        running_corrects = 0

        #Iterate over data
        for inputs, labels in train_dataloader:
            inputs = inputs.to(device)
            labels = labels.to(device)

            # Zero the parameters gradients
            optimizer.zero_grad()

            #forward 
            outputs = model(inputs)            
            _, preds = torch.max(outputs, 1)
            
            loss = criterion(outputs, labels)

            #backward + optimize only in training phase
            loss.backward()
            optimizer.step()

            # statistics
            running_loss += loss.item() * inputs.size(0)
            running_corrects += torch.sum(preds==labels.data)

        lr_scheduler.step()

        epoch_train_loss = running_loss/len(train_dataset)
        epoch_train_acc = running_corrects.double()/len(train_dataset)
        print('epoch train loss:{:.4f}, epoch train acc: {:.4f}'.format(epoch_train_loss, epoch_train_acc))
        #writer.add_scalar('epoch_train_loss',epoch_train_loss, epoch)
        #writer.add_scalar('epoch_train_acc', epoch_train_acc, epoch)

        model.eval()
        val_loss = 0.0
        val_corrects = 0

        with torch.no_grad():
            for inputs, labels in val_dataloader:
                inputs = inputs.to(device)
                labels = labels.to(device)

                outputs = model(inputs)
                _, preds = torch.max(outputs, 1)

                loss = criterion(outputs, labels)

                val_loss += loss.item() * inputs.size(0)
                val_corrects += torch.sum(preds == labels.data)

        epoch_val_loss = val_loss/len(val_dataset)
        epoch_val_acc = val_corrects.double()/len(val_dataset)
        print('epoch val loss: {:.4f}, epoch val acc: {:.4f}'.format(epoch_val_loss, epoch_val_acc))
        #writer.add_scalar('epoch_val_loss', epoch_val_loss, epoch)
        #writer.add_scalar('epoch_val_acc', epoch_val_acc, epoch)

        writer.add_scalars('loss', {'train_loss': epoch_train_loss, 'val_loss': epoch_val_loss}, epoch)
        writer.add_scalars('acc', {'train_acc': epoch_train_acc, 'val_acc': epoch_val_acc}, epoch)

        if epoch_val_acc > best_acc:
            best_acc = epoch_val_acc
            ckpt_state = {
                'epoch': epoch,
                'best_acc' : best_acc,
                'model_state': model.state_dict(),
                'optimizer_state' : optimizer.state_dict(),
                'loss': loss
            }
            torch.save(ckpt_state, 'ckpt_{}_model.pt'.format(epoch));
            best_model = model.state_dict()
        
    return best_model

model = models.resnet18(pretrained=True)
for param in model.parameters(): # set all pretrained weights are kept & not update that gradients
    param.requires_grad = False

numfts = model.fc.in_features
model.fc = nn.Linear(numfts, 2)
model = model.to(device)
print("modified model:", model)

criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.SGD(model.fc.parameters(), lr=0.001, momentum = 0.9)
lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=5, gamma=0.1)

best_model = train_model(model, criterion, optimizer, lr_scheduler, num_epoches=25)

torch.save(best_model, 'best_model.pt')
           

训练出来的结果，达到97%~98%的准确率

---------Training 0/25---------
epoch train loss:0.3792, epoch train acc: 0.8478
epoch val loss: 0.1733, epoch val acc: 0.9582
---------Training 1/25---------
epoch train loss:0.2315, epoch train acc: 0.9216
epoch val loss: 0.1223, epoch val acc: 0.9684
---------Training 2/25---------
epoch train loss:0.2033, epoch train acc: 0.9257
epoch val loss: 0.1008, epoch val acc: 0.9732
---------Training 3/25---------
epoch train loss:0.1848, epoch train acc: 0.9318
epoch val loss: 0.0871, epoch val acc: 0.9764
---------Training 4/25---------
epoch train loss:0.1739, epoch train acc: 0.9332
epoch val loss: 0.0825, epoch val acc: 0.9764
---------Training 5/25---------
epoch train loss:0.1670, epoch train acc: 0.9355
epoch val loss: 0.0759, epoch val acc: 0.9774
---------Training 6/25---------
epoch train loss:0.1631, epoch train acc: 0.9352
epoch val loss: 0.0712, epoch val acc: 0.9786
---------Training 7/25---------
epoch train loss:0.1623, epoch train acc: 0.9341
epoch val loss: 0.0731, epoch val acc: 0.9778
---------Training 8/25---------
epoch train loss:0.1615, epoch train acc: 0.9353
epoch val loss: 0.0718, epoch val acc: 0.9782
---------Training 9/25---------
epoch train loss:0.1600, epoch train acc: 0.9360
epoch val loss: 0.0721, epoch val acc: 0.9770
---------Training 10/25---------
epoch train loss:0.1590, epoch train acc: 0.9356
epoch val loss: 0.0710, epoch val acc: 0.9776
---------Training 11/25---------
epoch train loss:0.1624, epoch train acc: 0.9354
epoch val loss: 0.0714, epoch val acc: 0.9776
---------Training 12/25---------
epoch train loss:0.1611, epoch train acc: 0.9355
epoch val loss: 0.0698, epoch val acc: 0.9786
---------Training 13/25---------
epoch train loss:0.1566, epoch train acc: 0.9371
epoch val loss: 0.0708, epoch val acc: 0.9782
---------Training 14/25---------
epoch train loss:0.1547, epoch train acc: 0.9397
epoch val loss: 0.0704, epoch val acc: 0.9780
---------Training 15/25---------
epoch train loss:0.1590, epoch train acc: 0.9371
epoch val loss: 0.0707, epoch val acc: 0.9780
---------Training 16/25---------
epoch train loss:0.1612, epoch train acc: 0.9358
epoch val loss: 0.0707, epoch val acc: 0.9770
---------Training 17/25---------
epoch train loss:0.1611, epoch train acc: 0.9358
epoch val loss: 0.0707, epoch val acc: 0.9784
---------Training 18/25---------
epoch train loss:0.1596, epoch train acc: 0.9361
epoch val loss: 0.0698, epoch val acc: 0.9780
---------Training 19/25---------
epoch train loss:0.1590, epoch train acc: 0.9354
epoch val loss: 0.0701, epoch val acc: 0.9780
---------Training 20/25---------
epoch train loss:0.1599, epoch train acc: 0.9347
epoch val loss: 0.0701, epoch val acc: 0.9774
---------Training 21/25---------
epoch train loss:0.1588, epoch train acc: 0.9381
epoch val loss: 0.0695, epoch val acc: 0.9776
---------Training 22/25---------
epoch train loss:0.1567, epoch train acc: 0.9385
epoch val loss: 0.0691, epoch val acc: 0.9784
---------Training 23/25---------
epoch train loss:0.1591, epoch train acc: 0.9358
epoch val loss: 0.0695, epoch val acc: 0.9790
---------Training 24/25---------
epoch train loss:0.1618, epoch train acc: 0.9345
epoch val loss: 0.0745, epoch val acc: 0.9760
           

预测并将结果存至submission.csv

使用predict_and_save.py，对test1.zip解压出的图片进行预测，留意到这里用了list.sort(key=…)这个接口，对文件进行排序。默认情况下glob.glob(…)接口获取的文件列表不是数字升序的。

import torch
import torchvision

import torch.optim as optim
from torch.utils.data import *
from torchvision import models, transforms, datasets
import torch.nn as nn
import matplotlib.pyplot as plt
import glob
import os.path as osp

eval_transform = transforms.Compose([
        transforms.Resize(256),
        transforms.CenterCrop(224),
        transforms.ToTensor(),
        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
    ])

device = torch.device('cuda:0' if torch.cuda.is_available() else "cpu")
model = models.resnet18(pretrained=True)
for param in model.parameters(): # set all pretrained weights are kept & not update that gradients
    param.requires_grad = False

numfts = model.fc.in_features
model.fc = nn.Linear(numfts, 2)
model = model.to(device)
model.load_state_dict(torch.load('best_model.pt'))

from PIL import Image

csv_filename = "submission.csv"
csv_file = open(csv_filename, 'w')
csv_file.write("id,label"+"\n")

root="/home/jingwenlai/data/kaggle/dogs_and_cats/test1"
img_files = glob.glob(osp.join(root,"*.jpg"))
img_files.sort(key=lambda x: int(osp.basename(x)[:-4]))
    
model.eval()
with torch.no_grad():
    for img_file in img_files:
        basename = osp.basename(img_file)
        print('predicting...', basename)
        file_id = basename.split('.')[0]
        with open(img_file, 'rb') as f:
            img = Image.open(f)
            img = img.convert('RGB')
            img_tensor = eval_transform(img)
            img_tensor = img_tensor.unsqueeze(0).to(device)            

            outputs = model(img_tensor)
            _, preds = torch.max(outputs, 1)

            predict_result = preds.cpu().item()
            csv_file.write(str(file_id)+","+str(predict_result)+"\n")

csv_file.close()
           

代码下载

https://github.com/dreamway/Dogs_vs_Cats