title: KeyError when trying to modify pytorch-example

body:
I'm trying to modify this pytorch-example (https://github.com/pytorch/examples/blob/master/mnist/main.py) to work with my own dataset.

I tried to feed my data into a dataloader. I encapsulated the data in two different ways: one time as an extension of torch.utils.data.Dataset and one time as torch.utils.data.TensorDataset. Unfortunately I always get the same error that I don't understand:

Traceback (most recent call last):
  File "main.py", line 142, in <module>
    train(epoch)
  File "main.py", line 112, in train
    output = model(data)
  File "/usr/local/lib/python2.7/dist-packages/torch/nn/modules/module.py", line 210, in __call__
    result = self.forward(*input, **kwargs)
  File "main.py", line 90, in forward
    x = F.relu(F.max_pool2d(self.conv1(x), 2))
  File "/usr/local/lib/python2.7/dist-packages/torch/nn/modules/module.py", line 210, in __call__
    result = self.forward(*input, **kwargs)
  File "/usr/local/lib/python2.7/dist-packages/torch/nn/modules/conv.py", line 235, in forward
    self.padding, self.dilation, self.groups)
  File "/usr/local/lib/python2.7/dist-packages/torch/nn/functional.py", line 54, in conv2d
    return f(input, weight, bias) if bias is not None else f(input, weight)
  File "/usr/local/lib/python2.7/dist-packages/torch/nn/_functions/conv.py", line 33, in forward
    output = self._update_output(input, weight, bias)
  File "/usr/local/lib/python2.7/dist-packages/torch/nn/_functions/conv.py", line 88, in _update_output
    return self._thnn('update_output', input, weight, bias)
  File "/usr/local/lib/python2.7/dist-packages/torch/nn/_functions/conv.py", line 147, in _thnn
    return impl[fn_name](self, self._bufs[0], input, weight, *args)
  File "/usr/local/lib/python2.7/dist-packages/torch/nn/_functions/conv.py", line 213, in call_update_output
    backend = type2backend[type(input)]
  File "/usr/local/lib/python2.7/dist-packages/torch/_thnn/__init__.py", line 13, in __getitem__
    return self.backends[name].load()
KeyError: <class 'torch.cuda.ByteTensor'>

Here's my main.py, which is basically this example: https://github.com/pytorch/examples/blob/master/mnist/main.py

from __future__ import print_function
import argparse
import os
import glob
import numpy
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
import torch.utils.data as data_utils
from PIL import Image
from torchvision import datasets, transforms
from torch.autograd import Variable
from InputData import InputData

# Training settings
parser = argparse.ArgumentParser(description='PyTorch MNIST Example')
parser.add_argument('--batch-size', type=int, default=64, metavar='N',
                    help='input batch size for training (default: 64)')
parser.add_argument('--test-batch-size', type=int, default=10, metavar='N',
                    help='input batch size for testing (default: 1000)')
parser.add_argument('--epochs', type=int, default=1, metavar='N',
                    help='number of epochs to train (default: 10)')
parser.add_argument('--lr', type=float, default=0.01, metavar='LR',
                    help='learning rate (default: 0.01)')
parser.add_argument('--momentum', type=float, default=0.5, metavar='M',
                    help='SGD momentum (default: 0.5)')
parser.add_argument('--no-cuda', action='store_true', default=False,
                    help='disables CUDA training')
parser.add_argument('--seed', type=int, default=1, metavar='S',
                    help='random seed (default: 1)')
parser.add_argument('--log-interval', type=int, default=10, metavar='N',
                    help='how many batches to wait before logging training status')
args = parser.parse_args()
args.cuda = not args.no_cuda and torch.cuda.is_available()

torch.manual_seed(args.seed)
if args.cuda:
    torch.cuda.manual_seed(args.seed)

kwargs = {'num_workers': 1, 'pin_memory': True} if args.cuda else {}

# Original DataLoader - WORKS:
'''
train_loader = torch.utils.data.DataLoader(
    datasets.MNIST('../data', train=True, download=True,
                   transform=transforms.Compose([
                       transforms.ToTensor(),
                       transforms.Normalize((0.1307,), (0.3081,))
                   ])),
    batch_size=args.batch_size, shuffle=True, **kwargs)
test_loader = torch.utils.data.DataLoader(
    datasets.MNIST('../data', train=False, transform=transforms.Compose([
                       transforms.ToTensor(),
                       transforms.Normalize((0.1307,), (0.3081,))
                   ])),
    batch_size=args.batch_size, shuffle=True, **kwargs)
'''

# DataLoader as extension of data.Dataset:

train_loader = torch.utils.data.DataLoader(InputData('~/bakk-arbeit/data', train=True),
                                           batch_size=args.batch_size, shuffle=True, **kwargs)
test_loader = torch.utils.data.DataLoader(InputData('~/bakk-arbeit/data', train=False),
                                          batch_size=args.batch_size, shuffle=True, **kwargs)

# DataLoader as data.TensorDataset:
'''
data_folder = os.path.expanduser('~/bakk-arbeit/data')
InputData = InputData()
train = data_utils.TensorDataset(InputData.read_image_files(os.path.join(data_folder, 'training')),InputData.read_label_files(os.path.join(data_folder, 'training')))
test = data_utils.TensorDataset(InputData.read_image_files(os.path.join(data_folder, 'test')),InputData.read_label_files(os.path.join(data_folder, 'test')))
train_loader = data_utils.DataLoader(train, batch_size=args.batch_size, shuffle=True, **kwargs)
test_loader = data_utils.DataLoader(test, batch_size=args.batch_size, shuffle=True, **kwargs)
'''

class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        self.conv1 = nn.Conv2d(1, 10, kernel_size=5) # change to 3 input channels for InputData!
        self.conv2 = nn.Conv2d(10, 20, kernel_size=5)
        self.conv2_drop = nn.Dropout2d()
        self.fc1 = nn.Linear(320, 50) # change 320 to 500 for InputData to match 32x32
        self.fc2 = nn.Linear(50, 10)

    def forward(self, x):
        x = F.relu(F.max_pool2d(self.conv1(x), 2))
        x = F.relu(F.max_pool2d(self.conv2_drop(self.conv2(x)), 2))
        x = x.view(-1, 320) # # change 320 to 500 for InputData to match 32x32
        x = F.relu(self.fc1(x))
        x = F.dropout(x, training=self.training)
        x = self.fc2(x)
        return F.log_softmax(x)

model = Net()
if args.cuda:
    model.cuda()

optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum)

def train(epoch):
    model.train()
    for batch_idx, (data, target) in enumerate(train_loader):
        # data = data.numpy()
        if args.cuda:
            data, target = data.cuda(), target.cuda()
        data, target = Variable(data), Variable(target)
        optimizer.zero_grad()
        output = model(data)
        loss = F.nll_loss(output, target)
        loss.backward()
        optimizer.step()
        if batch_idx % args.log_interval == 0:
            print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(
                epoch, batch_idx * len(data), len(train_loader.dataset),
                100. * batch_idx / len(train_loader), loss.data[0]))

def test():
    model.eval()
    test_loss = 0
    correct = 0
    for data, target in test_loader:
        if args.cuda:
            data, target = data.cuda(), target.cuda()
        data, target = Variable(data, volatile=True), Variable(target)
        output = model(data)
        test_loss += F.nll_loss(output, target, size_average=False).data[0] # sum up batch loss
        pred = output.data.max(1)[1] # get the index of the max log-probability
        correct += pred.eq(target.data.view_as(pred)).cpu().sum()

    test_loss /= len(test_loader.dataset)
    print('\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n'.format(
        test_loss, correct, len(test_loader.dataset),
        100. * correct / len(test_loader.dataset)))

for epoch in range(1, args.epochs + 1):
    train(epoch)
    test()

...and here's my InputData.py that extends data.Dataset:

import torch
import numpy
import torch.utils.data as data
import glob
import os
from PIL import Image

class InputData(data.Dataset):
    train_folder = 'training'
    test_folder = 'test'

    def __init__(self, root='', train=True):
        self.root = os.path.expanduser(root)
        self.train = train  # training set or test set
        if root:
            if self.train:
                self.training_labels = self.read_label_files(os.path.join(self.root, self.train_folder))
                #with open(os.path.join(self.root, 'training_labels.pt'), 'wb') as f:
                    # torch.save(self.read_label_files(os.path.join(self.root, self.train_folder)), f)
                # with open(os.path.join(self.root, 'training_images.pt'), 'wb') as f:
                    #torch.save(self.read_image_files(os.path.join(self.root, self.train_folder)), f)
                self.training_images = self.read_image_files(os.path.join(self.root, self.train_folder))
            else:
                self.test_images = self.read_image_files(os.path.join(self.root, self.test_folder))
                self.test_labels = self.read_label_files(os.path.join(self.root, self.test_folder))
        print('initialized')

    def read_image_files(self, path):
        print('reading image files...')
        image_list = []
        # ten = torch.ByteTensor(3,32,32)
        for filename in glob.glob(path + '/*.png'):
            im = Image.open(filename)
            data = numpy.asarray(im)
            data = numpy.swapaxes(data,0,2)
            image_list.append(data)
        image_list = numpy.asarray(image_list)
        t = torch.from_numpy(image_list)
        # ten = torch.stack([ten, t])
        print('done!')
        return t

    def read_label_files(self, path):
        print('reading labels...')
        labels = []
        for filename in glob.glob(path + '/*.png'):
            base = os.path.basename(filename)
            im_class = int(base[:1])
            labels.append(im_class)
        print('done!')
        return torch.LongTensor(labels)

    def __getitem__(self, index):
        """
        Args:
            index (int): Index

        Returns:
            tuple: (image, target) where target is index of the target class.
        """
        if self.train:
            img, target = self.training_images[Index], self.training_labels[Index]
        else:
            img, target = self.test_images[Index], self.test_labels[Index]

        # img = Image.fromarray(img.numpy(), mode='RGB')
        # -> won't work for me??? returns TypeError: batch must contain tensors, numbers, or lists; found <class 'PIL.Image.Image'>

        return img, target

    def __len__(self):
        if self.train:
            return len(self.training_images)
        else:
            return len(self.test_images)

What am I doing wrong?


title: about torch.nn.CrossEntropyLoss parameter shape

body:
i'm learning pytorch, and taking the anpr project,which is based tensorflow
(https://github.com/matthewearl/deep-anpr,
http://matthewearl.github.io/2016/05/06/cnn-anpr/)
as a exercise, transplant it to pytorch platform.

there is a problem,i'm using nn.CrossEntropyLoss() as loss function:

criterion=nn.CrossEntropyLoss()

the output.data of model is:

  - 1.00000e-02 *
   - 2.5552 2.7582 2.5368 ... 5.6184 1.2288 -0.0076
   - 0.7033 1.3167 -1.0966 ... 4.7249 1.3217 1.8367
   - 0.7592 1.4777 1.8095 ... 0.8733 1.2417 1.1521
   - 0.1040 -0.7054 -3.4862 ... 4.7703 2.9595 1.4263
   - [torch.FloatTensor of size 4x253]

and targets.data is:

 - 1 0 0 ... 0 0 0
 - 1 0 0 ... 0 0 0
 - 1 0 0 ... 0 0 0
 - 1 0 0 ... 0 0 0
 - [torch.DoubleTensor of size 4x253]

when i call:

loss=criterion(output,targets)

error occured,information is:

TypeError: FloatClassNLLCriterion_updateOutput received an invalid combination of arguments - got (int, torch.FloatTensor, **torch.DoubleTensor**, torch.FloatTensor, bool, NoneType, torch.FloatTensor), but expected (int state, torch.FloatTensor input, **torch.LongTensor** target, torch.FloatTensor output, bool sizeAverage, [torch.FloatTensor weights or None], torch.FloatTensor total_weight)

'expected torch.LongTensor'......'got torch.DoubleTensor',but if i convert the targets into LongTensor:
torch.LongTensor(numpy.array(targets.data.numpy(),numpy.long))

call loss=criterion(output,targets), the error is:

RuntimeError: multi-target not supported at /data/users/soumith/miniconda2/conda-bld/pytorch-0.1.10_1488752595704/work/torch/lib/THNN/generic/ClassNLLCriterion.c:20

my last exercise is mnist, a example from pytorch,i made a bit modification,batch_size is 4,the loss function:

loss = F.nll_loss(outputs, labels)

outputs.data:

 - -2.3220 -2.1229 -2.3395 -2.3391 -2.5270 -2.3269 -2.1055 -2.2321 -2.4943 -2.2996

   -2.3653 -2.2034 -2.4437 -2.2708 -2.5114 -2.3286 -2.1921 -2.1771 -2.3343 -2.2533

   -2.2809 -2.2119 -2.3872 -2.2190 -2.4610 -2.2946 -2.2053 -2.3192 -2.3674 -2.3100

   -2.3715 -2.1455 -2.4199 -2.4177 -2.4565 -2.2812 -2.2467 -2.1144 -2.3321 -2.3009

   [torch.FloatTensor of size 4x10]

labels.data:

 - 8
 - 6
 - 0
 - 1
 - [torch.LongTensor of size 4]

the labels, for a input image,must be a single element, in upper example, there is 253 numbers, and in 'mnist',there is only one number, the shape of outputs is difference from labels.

i review the tensorflow manual, tf.nn.softmax_cross_entropy_with_logits,
'Logits and labels must have the sameshape [batch_size, num_classes] and the same dtype (either float32 or float64).'

does pytorch support the same function in tensorflow?

many thks


title: Conv1D with kernel_size=1 vs Linear layer

body:
I'm working on very sparse vectors as input. I started working with simple Linear (dense/fully connected layers) and my network yielded pretty good results (let's take accuracy as my metric here, 95.8%).
I later tried to use a Conv1d with a kernel_size=1 and a MaxPool1d, and this network works slightly better (96.4% accuracy).
Question: How are these two implementation different ? Shouldn't a Conv1d with a unit kernel_size do the same as a Linear layer?
I've tried multiple runs, the CNN always yields slightly better results.


title: RuntimeError: cuDNN error: CUDNN_STATUS_NOT_INITIALIZED using pytorch

body:
I am trying to run a simple pytorch sample code. It's works fine using CPU. But when using GPU, i get this error message:
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/usr/local/lib/python3.6/dist-packages/torch/nn/modules/module.py", line 889, in _call_impl
    result = self.forward(*input, **kwargs)
  File "/usr/local/lib/python3.6/dist-packages/torch/nn/modules/conv.py", line 263, in forward
    return self._conv_forward(input, self.weight, self.bias)
  File "/usr/local/lib/python3.6/dist-packages/torch/nn/modules/conv.py", line 260, in _conv_forward
    self.padding, self.dilation, self.groups)
RuntimeError: cuDNN error: CUDNN_STATUS_NOT_INITIALIZED

The code i am trying to run is the following:
import torch
from torch import nn
m = nn.Conv1d(16, 33, 3, stride=2)
m=m.to('cuda')
input = torch.randn(20, 16, 50)
input=input.to('cuda')
output = m(input)

I am running this code in a NVIDIA docker with CUDA version 10.2 and my GPU is a RTX 2070


title: Runtime crashes on Google Colab

body:
Why does the runtime keep crashing on Google Colab.

I have a simple MLP code that runs on my machine. I tried running the same code on Colab but it crashes immediately after loading the data files.

The data files are around 3GB total. The CPU and the GPU memory for the Colab virtual machine are easily above that.

Then why does my program crash before it can even start training.

My Code:

def load_raw(name):
  return (np.load(name + '.npy', encoding='bytes'), np.load(name + '_labels.npy', encoding='bytes'))

class WSJ():
 def __init__(self):
    self.dev_set = None
    self.train_set = None
    self.test_set = None

@property
def dev(self):
    if self.dev_set is None:
        self.dev_set = load_raw('dev')
    return self.dev_set

@property

def train(self):
    if self.train_set is None:
        self.train_set = load_raw('train')
    return self.train_set

@property
def test(self):
    if self.test_set is None:
        self.test_set = (np.load('test.npy', encoding='bytes'), None)
    return self.test_set

def preprocess_data(self, trainX, trainY, k):
     # some form of preprocessing that pads and flattens the data into the format required

    return trainX_padded, trainY, y_to_x_map

def main():

 global index
 padding = 3
 epochs = 1
 batch_size = 512
 lr = 0.1
 momentum = 0.9

 input_dim = 40 * ((2*padding) + 1)
 output_dim = 138

 neural_net = MLP(input_dim, output_dim)
 !free -g

 print("Starting...")
 loader = WSJ()
 trainX, trainY = loader.train
 print("Training Data obtained...")
 !free -g
 trainX, trainY, y_to_x_map = loader.preprocess_data(trainX, trainY, k = padding)
 print("Training Data preprocessed...")

 !free -g

 devX, devY = loader.dev
 devX, devY, y_to_x_map_dev = loader.preprocess_data(devX, devY, k = padding)
 print("Development data preprocessed...")

 !free -g

 print("Scaling...")
 input_scaler = preprocessing.StandardScaler().fit(trainX)

 !free -g

 trainX = input_scaler.transform(trainX)
 devX = input_scaler.transform(devX)

It crashes immediately after after printing Scaling...


title: Where is the len function used in PyTorch Dataset?

body:
I am looking to use the code from here .
However, I am looking at box 5, where there is the following function;

def __len__(self):
    # Default epoch size is 10 000 samples
    return 10000

I do not see  anywhere throughout this script where this function is being used.
Clarification on this would be appreciated.

Also, I want to determine the number of image patches used for training  this convolutional neural network. Is this len function linked to the number of patches?


title: How can I download and skip VGG weights that have no counterpart with my CNN in Keras?

body:
I would like to follow the Convolutional Neural Net (CNN) approach here. However, this code in github uses Pytorch, whereas I am using Keras.

I want to reproduce boxes 6,7 and 8 where pre-trained weights from VGG-16 on ImageNet is downloaded and is used to make the CNN converge faster.

In particular, there is a portion (box 8) where weights are downloaded and skipped from VGG-16 that have no counterpart in SegNet (the CNN model). In my work, I am using a CNN model called U-Net instead of Segnet. The U-Net Keras code that I am using can be found here.

I am new to Keras and would appreciate any insight in Keras code on how I can go about downloading and skipping the VGG weights that have no counterpart with my U-Netmodel.


title: Python matplotlib, invalid shape for image data

body:
Currently I have this code to show three images:

imshow(image1, title='1')
imshow(image2, title='2')
imshow(image3, title='3')

And it works fine. But I am trying to put them all three in a row instead of column.

Here is the code I have tried:

f = plt.figure()
f.add_subplot(1,3,1)
plt.imshow(image1)
f.add_subplot(1,3,2)
plt.imshow(image2)
f.add_subplot(1,3,3)
plt.imshow(image3)

It throws

  TypeError: can't convert CUDA tensor to numpy. Use Tensor.cpu() to copy the tensor to host memory first.

If I do

f = plt.figure()
f.add_subplot(1,3,1)
plt.imshow(image1.cpu())
f.add_subplot(1,3,2)
plt.imshow(image2.cpu())
f.add_subplot(1,3,3)
plt.imshow(image3.cpu())

It throws

  TypeError: Invalid shape (1, 3, 128, 128) for image data

How should I fix this or is there an easier way to implement it?


title: Convert a python list of python lists to pytorch tensor

body:
What is the conventional way to convert python list of lists to PyTorch tensors?

a = [0,0]
b = [1,1]
c = [2]
c = [a, b, c]

I want c to be converted to a flattened Torch tensor as below:

torch([ 0, 0, 1, 1, 2])


title: Pytorch GPU memory allocation

body:
I have self-modified car recognition model, taken from https://github.com/Helias/Car-Model-Recognition. I use Cuda and Pytorch:1.4.0. But even though I tried to use the answer on my previous question, it still doesn't want to work with BATCH_SIZE>=7.The same thing happens:

CUDA out of memory. Tried to allocate 20.00 MiB (GPU 0; 4.00 GiB total capacity; 2.74 GiB already allocated; 7.80 MiB free; 2.96 GiB reserved in total by PyTorch)

If I increase my BATCH_SIZE,pytorch gives me more, but not enough:
BATCH_SIZE=256

CUDA out of memory. Tried to allocate 196.00 MiB (GPU 0; 4.00 GiB total capacity; 2.85 GiB already allocated; 93.80 MiB free; 2.87 GiB reserved in total by PyTorch)

BATCH_SIZE=512

CUDA out of memory. Tried to allocate 1.53 GiB (GPU 0; 4.00 GiB total capacity; 2.04 GiB already allocated; 927.80 MiB free; 2.06 GiB reserved in total by PyTorch)

My code is the following:
main.py

from dataset import torch, os, LocalDataset, transforms, np, get_class, num_classes, preprocessing, Image, m, s, dataset_main
from config import *

from torch import nn
from torch.optim import SGD
from torch.optim.lr_scheduler import StepLR
from torch.autograd import Variable
from torch.utils.data import DataLoader
from torchvision.models import resnet

from sklearn.metrics import accuracy_score
from sklearn.metrics import confusion_matrix
from sklearn.metrics import f1_score

from matplotlib import pyplot as plt
from numpy import unravel_index
import gc
import argparse
import cv2 as cv2
cuda_available = torch.cuda.is_available()
torch.cuda.set_device(0)
def train_model(model_name, model, train_loader, test_loader, lr=LEARNING_RATE, epochs=EPOCHS, momentum=MOMENTUM, weight_decay=0,):

    if not os.path.exists(RESULTS_PATH + "/" + model_name):
        os.makedirs(RESULTS_PATH + "/" + model_name)
    criterion = nn.CrossEntropyLoss()
    optimizer = SGD(model.parameters(), lr, momentum=momentum, weight_decay=weight_decay)
    loaders = {'train':train_loader, 'test':test_loader}
    losses = {'train':[], 'test':[]}
    accuracies = {'train':[], 'test':[]}

    #testing variables
    y_testing = []
    preds = []

    if USE_CUDA and cuda_available:
        model=model.cuda()
    for e in range(epochs):
        for mode in ['train', 'test']:
            if mode=='train':
                model.train()
            else:
                model.eval()

            epoch_loss = 0
            epoch_acc = 0
            samples = 0

            try:
                for i, batch in enumerate(loaders[mode]):
                    # convert tensor to variable
                    x=Variable(batch['image'], requires_grad=(mode=='train'))
                    y=Variable(batch['label'])

                    if USE_CUDA and cuda_available:
                        x = x.cuda()
                        y = y.cuda()

                    output = model(x)
                    l = criterion(output, y) # loss

                    if mode=='train':
                        l.backward()
                        optimizer.step()
                        optimizer.zero_grad()
                    else:
                        y_testing.extend(y.data.tolist())
                        preds.extend(output.max(1)[1].tolist())

                    if USE_CUDA and cuda_available:
                        acc = accuracy_score(y.data.cuda().cpu().numpy(), output.max(1)[1].cuda().cpu().numpy())
                    else:
                        acc = accuracy_score(y.data, output.max(1)[1])

                    epoch_loss += l.data.item()*x.shape[0] # l.data[0]
                    epoch_acc += acc*x.shape[0]
                    samples += x.shape[0]

                    print ("\r[%s] Epoch %d/%d. Iteration %d/%d. Loss: %0.2f. Accuracy: %0.2f Learning Rate: %s" % \
                        (mode, e+1, epochs, i, len(loaders[mode]), epoch_loss/samples, epoch_acc/samples, 'test'))

                    if DEBUG and i == 2:
                        break
            except Exception as err:
                print ("\n\n######### ERROR #######")
                print (str(err))
                print ("\n\n######### batch #######")
                print (batch['img_name'])
                print ("\n\n")

            epoch_loss /= samples
            epoch_acc /= samples

            losses[mode].append(epoch_loss)
            accuracies[mode].append(epoch_acc)

            print ("\r[%s] Epoch %d/%d. Iteration %d/%d. Loss: %0.2f. Accuracy: %0.2f Learning Rate: %s" % \
                  (mode, e+1, epochs, i, len(loaders[mode]), epoch_loss, epoch_acc, 'test'))

    torch.save(model.state_dict(), str(RESULTS_PATH) + "/" + str(model_name) + "/" + str(model_name) + ".pt")
    return model, (losses, accuracies), y_testing, preds

def test_model(model_name, model, test_loader):
    model.load_state_dict(torch.load(str(RESULTS_PATH) + "/" + str(model_name) + "/" + str(model_name) + ".pt"))

    if USE_CUDA and cuda_available:
        model = model.cuda()

    model.eval()

    preds = []
    gts = []

    #debug
    i = 0

    for batch in test_loader:
        x = Variable(batch['image'])

        if USE_CUDA and cuda_available:
            x = x.cuda()
            pred = model(x).data.cuda().cpu().numpy().copy()
        else:
            pred = model(x).data.numpy().copy()

        gt = batch['label'].numpy().copy()
        preds.append(pred)
        gts.append(gt)

        # debug
        if DEBUG:
            if i == 2:
                break
            else:
                i+=1

    return np.concatenate(preds), np.concatenate(gts)

def write_stats(model_name, y, predictions, gts, predictions2):
    if not os.path.exists(RESULTS_PATH + "/" + model_name):
        os.makedirs(RESULTS_PATH + "/" + model_name)

    acc = accuracy_score(gts, predictions2.argmax(1))
    cm = confusion_matrix(y, predictions)

    if DEBUG:
      score = "00 F1_SCORE 00"
    else:
      score = f1_score(y, predictions, average=None)

    file = open(str(RESULTS_PATH) + "/" + str(model_name) + "/" + str(model_name) + "_stats.txt", "w+")
    file.write ("Accuracy: " + str(acc) + "\n\n")
    file.write("Confusion Matrix: \n" + str(cm) + "\n\n")
    file.write("F1 Score: \n" + str(score))
    file.close()

def plot_logs_classification(model_name, logs):
    if not os.path.exists(RESULTS_PATH + "/" + model_name):
        os.makedirs(RESULTS_PATH + "/" + model_name)

    training_losses, training_accuracies, test_losses, test_accuracies = \
        logs[0]['train'], logs[1]['train'], logs[0]['test'], logs[1]['test']

    plt.figure(figsize=(18,6))
    plt.subplot(121)
    plt.plot(training_losses)
    plt.plot(test_losses)
    plt.legend(['Training Loss','Test Losses'])
    plt.grid()
    plt.subplot(122)
    plt.plot(training_accuracies)
    plt.plot(test_accuracies)
    plt.legend(['Training Accuracy','Test Accuracy'])
    plt.grid()
    #plt.show()
    plt.savefig(str(RESULTS_PATH) + "/" + str(model_name) + "/" + str(model_name) + "_graph.png")

def train_model_iter(model_name, model, args, training_set_loader, validation_set_loader, weight_decay=0):

    if args.train:
        model, loss_acc, y_testing, preds = train_model(model_name=model_name, model=model, train_loader=training_set_loader, test_loader=validation_set_loader, weight_decay=weight_decay)

        preds_test, gts = test_model(model_name, model=model, test_loader=validation_set_loader)

        write_stats(model_name, y_testing, preds, gts, preds_test)
        plot_logs_classification(model_name, loss_acc)

    gc.collect()
def main():
    torch.cuda.empty_cache()
    print('main')
    print(m, s)
    parser = argparse.ArgumentParser(description='car model recognition')

    parser.add_argument("-i", "--input",      action="store",       dest="inp",   help="Take a sample image and classify it", type=str)
    parser.add_argument("-t", "--train",      action="store_true",                help="Run the training of the model")
    parser.add_argument("-p", "--preprocess", action="store_true",                help="Update the train and test csv files with the new images in dataset, used this if you added new images in dataset")

    args = parser.parse_args()

    if args.preprocess:
        print ("Preprocessing..")
        preprocessing()
        print ("Preprocessing finished!")
    print(cuda_available)
    # directory results
    if not os.path.exists(RESULTS_PATH):
        os.makedirs(RESULTS_PATH)

    # Load dataset
    mean=m
    std_dev=s

    transform = transforms.Compose([transforms.Resize((224,224)),
                                        transforms.ToTensor(),
                                        transforms.Normalize(mean, std_dev)])

    training_set = LocalDataset(IMAGES_PATH, TRAINING_PATH, transform=transform)
    validation_set = LocalDataset(IMAGES_PATH, VALIDATION_PATH, transform=transform)

    training_set_loader = DataLoader(dataset=training_set, batch_size=BATCH_SIZE, num_workers=8, shuffle=True)
    validation_set_loader = DataLoader(dataset=validation_set, batch_size=BATCH_SIZE, num_workers=8, shuffle=False)
    classes = {"num_classes": len(num_classes)}
    torch.cuda.empty_cache()
    resnet152_model = resnet.resnet152(pretrained=False, **classes)
    train_model_iter("resnet152", resnet152_model, args, training_set_loader, validation_set_loader)

    model_name="resnet152"
    model=resnet152_model

    if args.inp:
        print ("input: ", args.inp)

        image_path = args.inp
        im = Image.open(image_path).convert("RGB")
        im = transform(im)

        print (str(RESULTS_PATH) + "/" + str(model_name) + "/" + str(model_name) + ".pt")

        model.load_state_dict(torch.load(str(RESULTS_PATH) + "/" + str(model_name) + "/" + str(model_name) + ".pt"))

        if USE_CUDA and cuda_available:
            model = model.cuda()
        model.eval()

        x = Variable(im.unsqueeze(0))

        if USE_CUDA and cuda_available:
            x = x.cuda()
            pred = model(x).data.cuda().cpu().numpy().copy()
        else:
            pred = model(x).data.numpy().copy()

        print (pred)

        idx_max_pred = np.argmax(pred)
        idx_classes = idx_max_pred % classes["num_classes"]
        print(get_class(idx_classes))
if __name__ == '__main__':
    print('main1')
    result = dataset_main()
    m = result[0]
    s = result[1]
    num_classes = result[2]
    torch.cuda.empty_cache()
    main()

dataset.py

import numpy as np
import torch
import os
import cv2 as cv2

from torch.utils.data.dataset import Dataset
from torch.utils.data import DataLoader
from torchvision import transforms

from PIL import Image
Image.LOAD_TRUNCATED_IMAGES = True
from os import path
from glob import glob
import random

from config import *

num_classes = { 'ford_explorer': 0, 'nissan_altima': 1, 'volkswagen_jetta': 2 }
# read mean and dev. standard pre-computed
m = 0
s = 0
def dataset_main():
    # dirs = glob(IMAGES_PATH + "/*/")
    # i = 0
    # for d in dirs:
    #     d = d.replace(IMAGES_PATH, "")
    #     d = d.replace("/", "")
    #     if " " in d:
    #         d = d.replace(" ", "_")
    #     num_classes[d] = i
    #     i+=1

    print ("Classes: ")
    print (num_classes)
    print ("")

    if os.path.isfile('./mean_devstd.txt'):
        m_s = open("mean_devstd.txt", "r").read()
        if "," in m_s:
            m_s = m_s.replace("\n", "")
            m_s = m_s.replace("tensor", "")
            m_s = m_s.replace("(", "")
            m_s = m_s.replace(")", "")
            m_s = m_s.split(",")
            m = torch.Tensor( [float(m_s[0]), float(m_s[1]), float(m_s[2])] )
            s = torch.Tensor( [float(m_s[3]), float(m_s[4]), float(m_s[5])] )
        return m, s, num_classes
def get_class(idx):
    #print (num_classes)
    for key in num_classes:
        if idx == num_classes[key]:
            return key

def preprocessing():
    train_csv = ""
    test_csv  = ""
    train_csv_supp = []
    test_csv_supp = []
    class_files_training = []
    class_files_testing  = []

    for key in num_classes:
        if " " in key:
            os.rename(IMAGES_PATH+"/"+key, IMAGES_PATH+"/"+key.replace(" ", "_"))
            key = key.replace(" ", "_")

        class_files = glob(IMAGES_PATH+"/"+str(key)+"/*")
        class_files = [w.replace(IMAGES_PATH+"/"+str(key)+"/", "") for w in class_files]
        class_files.sort()

        class_files_training = class_files[: int(len(class_files)*.66)] # get 66% class images fo training
        class_files_testing = class_files[int(len(class_files)*.66)+1 :] # get 33% class images fo testing

        for f in class_files_training:
            if "," in f or "#" in f or " " in f:
                tmp_f = f.replace(",", "")
                tmp_f = tmp_f.replace("#", "")
                tmp_f = tmp_f.replace(" ", "_")
                os.rename(IMAGES_PATH+"/"+key+"/"+f, IMAGES_PATH+"/"+key+"/"+tmp_f)
                f = tmp_f
            train_csv_supp.append(f + ","+str(key))

        for f in class_files_testing:
            if "," in f or "#" in f or " " in f:
                tmp_f = f.replace(",", "")
                tmp_f = tmp_f.replace("#", "")
                tmp_f = tmp_f.replace(" ", "_")
                os.rename(IMAGES_PATH+"/"+key+"/"+f, IMAGES_PATH+"/"+key+"/"+tmp_f)
                f = tmp_f
            test_csv_supp.append(f + ","+str(key))

    random.shuffle(train_csv_supp)
    random.shuffle(test_csv_supp)

    for t in train_csv_supp:
        train_csv += t + "\n"

    for t in test_csv_supp:
        test_csv += t + "\n"

    train_csv_file = open("train_file.csv", "w+")
    train_csv_file.write(train_csv)
    train_csv_file.close()

    test_csv_file = open("test_file.csv", "w+")
    test_csv_file.write(test_csv)
    test_csv_file.close()

    # Algorithms to calculate mean and standard_deviation
    print("Loading dataset...")
    dataset = LocalDataset(IMAGES_PATH, TRAINING_PATH, transform=transforms.ToTensor())
    print("Calculating mean & dev std...")

    m = torch.zeros(3) # Mean
    s = torch.zeros(3) # Standard Deviation
    for sample in dataset:
        m += sample['image'].sum(1).sum(1)
        s += ((sample['image']-m.view(3,1,1))**2).sum(1).sum(1)
    m /= len(dataset)*256*144
    s = torch.sqrt(s/(len(dataset)*256*144))

    print("Calculated mean and standard deviation!")
    str_m = str(m[0])+","+str(m[1])+","+str(m[2])
    str_s = str(s[0])+","+str(s[1])+","+str(s[2])
    file = open("mean_devstd.txt", "w+")
    file.write(str(str_m)+","+str(str_s))
    file.close()
#preprocessing()

class LocalDataset(Dataset):

    def __init__(self, base_path, txt_list, transform=None):
        self.base_path=base_path
        self.images = np.loadtxt(txt_list,delimiter=',',dtype='str') # use np.genfrom() instead of np.loadtxt() to skip errors

        self.transform = transform

    def __getitem__(self, index):
        f,c = self.images[index]

        image_path = path.join(self.base_path + "/" + str(c), f)
        # image_path = path.join(str(c), f)
        im = cv2.cvtColor(cv2.imread(image_path), cv2.COLOR_BGR2RGB)

        if self.transform is not None:
            im = self.transform(Image.fromarray(im))
        label = num_classes[c]

        return { 'image' : im, 'label':label, 'img_name': f }

    def __len__(self):
        return len(self.images)

I can't understand, where my gpu is fluded, because everything seems to work fine.Is it just a problem in my video card? I have GTX 1050 TI and it should work well


title: Accuracy score in pyTorch LSTM

body:
I have been running this LSTM tutorial on the wikigold.conll NER data set

training_data contains a list of tuples of sequences and tags, for example:

training_data = [
    ("They also have a song called \" wake up \"".split(), ["O", "O", "O", "O", "O", "O", "I-MISC", "I-MISC", "I-MISC", "I-MISC"]),
    ("Major General John C. Scheidt Jr.".split(), ["O", "O", "I-PER", "I-PER", "I-PER"])
]

And I wrote down this function

def predict(indices):
    """Gets a list of indices of training_data, and returns a list of predicted lists of tags"""
    for index in indicies:
        inputs = prepare_sequence(training_data[index][0], word_to_ix)
        tag_scores = model(inputs)
        values, target = torch.max(tag_scores, 1)
        yield target

This way I can get the predicted labels for specific indices in the training data.

However, how do I evaluate the accuracy score across all training data.

Accuracy being, the amount of words correctly classified across all sentences divided by the word count.

This is what I came up with, which is extremely slow and ugly:

y_pred = list(predict([s for s, t in training_data]))
y_true = [t for s, t in training_data]
c=0
s=0
for i in range(len(training_data)):
    n = len(y_true[i])
    #super ugly and ineffiicient
    s+=(sum(sum(list(y_true[i].view(-1, n) == y_pred[i].view(-1, n).data))))
    c+=n

print ('Training accuracy:{a}'.format(a=float(s)/c))

How can this be done efficiently in pytorch ?

P.S:
I've been trying to use sklearn's accuracy_score unsuccessfully


title: I have a GPU and CUDA installed in Windows 10 but Pytorch's torch.cuda.is_avaiable() returns false; how can I correct this?

body:
I have PyTorch installed on a Windows 10 machine with a Nvidia GTX 1050 GPU. I have installed the CUDA Toolkit and tested it using Nvidia instructions and that has gone smoothly, including execution of the suggested tests.

However, torch.cuda.is_available() returns False. How can I fix this?


title: Why does dim=1 return row indices in torch.argmax?

body:
I am working on argmax function of PyTorch which is defined as:

torch.argmax(input, dim=None, keepdim=False)

Consider an example

a = torch.randn(4, 4)
print(a)
print(torch.argmax(a, dim=1))

Here when I use dim=1 instead of searching column vectors, the function searches for row vectors as shown below.

print(a) :
tensor([[-1.7739,  0.8073,  0.0472, -0.4084],
        [ 0.6378,  0.6575, -1.2970, -0.0625],
        [ 1.7970, -1.3463,  0.9011, -0.8704],
        [ 1.5639,  0.7123,  0.0385,  1.8410]])

print(torch.argmax(a, dim=1))
tensor([1, 1, 0, 3])

As far as my assumption goes dim = 0 represents rows and dim =1 represent columns.


title: In-place operations with PyTorch

body:
I was wondering how to deal with in-place operations in PyTorch. As I remember using in-place operation with autograd has always been problematic.

And actually I’m surprised that this code below works, even though I haven’t tested it I believe this code would have raised an error in version 0.3.1.

Basically I want do is set a certain position of a tensor vector to a certain value in a like:

my_tensor[i] = 42

Working example code:

# test parameter a
a = torch.rand((2), requires_grad=True)
print('a ', a)
b = torch.rand(2)

# calculation
c = a + b

# performing in-place operation
c[0] = 0
print('c ', c)
s = torch.sum(c)
print('s ', s)

# calling backward()
s.backward()

# optimizer step
optim = torch.optim.Adam(params=[a], lr=0.5)
optim.step()

# changed parameter a
print('changed a', a)

Output:

a  tensor([0.2441, 0.2589], requires_grad=True)
c  tensor([0.0000, 1.1511], grad_fn=<CopySlices>)
s  tensor(1.1511, grad_fn=<SumBackward0>)
changed a tensor([ 0.2441, -0.2411], requires_grad=True)

So obviously in version 0.4.1. this works just fine without warnings or errors.

Referring to this article in the documentation: autograd-mechanics

  Supporting in-place operations in autograd is a hard matter, and we
  discourage their use in most cases. Autograd’s aggressive buffer
  freeing and reuse makes it very efficient and there are very few
  occasions when in-place operations actually lower memory usage by any
  significant amount. Unless you’re operating under heavy memory
  pressure, you might never need to use them.

But even though it works, the use of in-place operations is discouraged in most cases.

So my questions are:

How much does the usage of in-place operations affect performance?
How do I get around using in-place operations in such cases where I want to set one element of a tensor to a certain value?

Thanks in advance!


title: Is One-Hot Encoding required for using PyTorch's Cross Entropy Loss Function?

body: For example, if I want to solve the MNIST classification problem, we have 10 output classes. With PyTorch, I would like to use the torch.nn.CrossEntropyLoss function. Do I have to format the targets so that they are one-hot encoded or can I simply use their class labels that come with the dataset?


title: How to convert some tensorflow code into pytorch version

body:
I recently started using pytorch. I've been using tensorflow framework before. I have a piece of code that I implemented with tensorflow, which I now want to convert to the pytorch version.

I'm new to pytorch and I'm not familiar with its functions and the transformation process is not smooth, so I'd like to consult.

Here's the code I want to convert：

def kl_loss_compute(logits1, logits2):
    """ KL loss
    """
    pred1 = tf.nn.softmax(logits1)
    pred2 = tf.nn.softmax(logits2)
    loss = tf.reduce_mean(tf.reduce_sum(pred2 * tf.log(1e-8 + pred2 / (pred1 + 1e-8)), 1))

    return loss

python: 3.6, ubuntu: 16.04

logits1 and logits2 are FC layer's outputs. Their shape is [batch, n]


title: In Pytorch F.nll_loss() Expected object of type torch.LongTensor but found type torch.FloatTensor for argument #2 'target'

body:
Why does this error occur.

I am trying to write a custom loss function, that finally has a negative log likelihood.

As per my understanding the NLL is calculated between two probability values?

>>> loss = F.nll_loss(sigm, trg_, ignore_index=250, weight=None, size_average=True)
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/home//lib/python3.5/site-packages/torch/nn/functional.py", line 1332, in nll_loss
    return torch._C._nn.nll_loss(input, target, weight, size_average, ignore_index, reduce)
RuntimeError: Expected object of type torch.LongTensor but found type torch.FloatTensor for argument #2 'target'

The inputs here being the following:

>>> sigm.size()
torch.Size([151414, 80])
>>> sigm
tensor([[ 0.3283,  0.6472,  0.8278,  ...,  0.6756,  0.2168,  0.5659],
        [ 0.6603,  0.5957,  0.8375,  ...,  0.2274,  0.4523,  0.4665],
        [ 0.5262,  0.4223,  0.5009,  ...,  0.5734,  0.3151,  0.2076],
        ...,
        [ 0.4083,  0.2479,  0.5996,  ...,  0.8355,  0.6681,  0.7900],
        [ 0.6373,  0.3771,  0.6568,  ...,  0.4356,  0.8143,  0.4704],
        [ 0.5888,  0.4365,  0.8587,  ...,  0.2233,  0.8264,  0.5411]])

And my target tensor is:

>>> trg_.size()
torch.Size([151414])
>>> trg_
tensor([-7.4693e-01,  3.5152e+00,  2.9679e-02,  ...,  1.6316e-01,
         3.6594e+00,  1.3366e-01])

If I convert this to long I loose all data:

>>> sigm.long()
tensor([[ 0,  0,  0,  ...,  0,  0,  0],
        [ 0,  0,  0,  ...,  0,  0,  0],
        [ 0,  0,  0,  ...,  0,  0,  0],
        ...,
        [ 0,  0,  0,  ...,  0,  0,  0],
        [ 0,  0,  0,  ...,  0,  0,  0],
        [ 0,  0,  0,  ...,  0,  0,  0]])
>>> trg_.long()
tensor([ 0,  3,  0,  ...,  0,  3,  0])

If I convert the raw values of target tensor to sigmoid too:

>>> F.sigmoid(trg_)
tensor([ 0.3215,  0.9711,  0.5074,  ...,  0.5407,  0.9749,  0.5334])
>>> loss = F.nll_loss(sigm, F.sigmoid(trg_), ignore_index=250, weight=None, size_average=True)
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/home/lib/python3.5/site-packages/torch/nn/functional.py", line 1332, in nll_loss
    return torch._C._nn.nll_loss(input, target, weight, size_average, ignore_index, reduce)
RuntimeError: Expected object of type torch.LongTensor but found type torch.FloatTensor for argument #2 'target'

This does calculate the loss happily, but it is again just make belief as I have lost data in the long conversion:

>>> loss = F.nll_loss(sigm, F.sigmoid(trg_).long(), ignore_index=250, weight=None, size_average=True)
>>> loss
tensor(-0.5010)

>>> F.sigmoid(trg_).long()
tensor([ 0,  0,  0,  ...,  0,  0,  0])


title: Apply a PyTorch CrossEntropy method for multiclass segmentation

body:
I am trying to implement a simple example of how to apply cross-entropy to what is supposed to be the output of my semantic segmentation CNN.

Using the pytorch format I would have something like this:

out = np.array([[
    [
        [1.,1, 1],
        [0, 0, 0],
        [0, 0, 0],
        [0, 0, 0]
    ],
    [
        [0, 0, 0],
        [1, 1, 1],
        [0, 0.,0],
        [0, 0, 0]
    ],
    [
        [0, 0, 0],
        [0, 0, 0],
        [1, 1, 1],
        [0, 0, 0]
    ],
    [
        [0, 0, 0],
        [0, 0, 0],
        [0, 0, 0],
        [1, 1, 1]
    ]
]])

out = torch.tensor(out)

So, my output here has dimensions (1, 4, 4, 3), being 1 element batch, 4 channels representing the 4 possible classes, and 4 by 3 data in each, storing the probability of that cell being from it's class.

Now my target is like this:

target=[
        [0, 0, 0],
        [1, 1, 1],
        [2, 2, 2],
        [3, 3, 3]
    ]

Please notice how in the 'out' tensor each row has a 1.0 probability of being from that class resulting in a perfect match with the target.

For example, the third channel (channel 2) has its whole 3rd row (row 2) with 1.0 probabilities of being from that channel, and zero's in any other place; so it matches the 2's on the target in third row as well.

With this example I expect a minimal loss value between the two tensors.

My question are:

What's the best way to use a cross-entropy loss method in PyTorch in order to reflect that this case has no difference between the target and its prediction?
What loss value should I expect from this?

This is what I got so far:

import torch
from torch.nn import CrossEntropyLoss
import numpy as np

out = torch.Tensor(np.array([[
    [
        [1.,1, 1],
        [0, 0, 0],
        [0, 0, 0],
        [0, 0, 0]
    ],
    [
        [0, 0, 0],
        [1, 1, 1],
        [0, 0.,0],
        [0, 0, 0]
    ],
    [
        [0, 0, 0],
        [0, 0, 0],
        [1, 1, 1],
        [0, 0, 0]
    ],
    [
        [0, 0, 0],
        [0, 0, 0],
        [0, 0, 0],
        [1, 1, 1]
    ]
]]))

target = torch.Tensor([[
    [0, 0, 0],
    [1, 1, 1],
    [2, 2, 2],
    [3, 3, 3]
]]).type('torch.LongTensor')

criterion = CrossEntropyLoss()
print(criterion(out, target))

And outputs: tensor(0.7437)

Shouldn't I expect a value closer to cero?

Thank you in advance


title: Why can't I append a PyTorch tensor with torch.cat?

body:
I have:

import torch

input_sliced = torch.rand(180, 161)
output_sliced = torch.rand(180,)

batched_inputs = torch.Tensor()
batched_outputs = torch.Tensor()

print('input_sliced.size', input_sliced.size())
print('output_sliced.size', output_sliced.size())

batched_inputs = torch.cat((batched_inputs, input_sliced))
batched_outputs = torch.cat((batched_outputs, output_sliced))

print('batched_inputs.size', batched_inputs.size())
print('batched_outputs.size', batched_outputs.size())

This outputs:

input_sliced.size torch.Size([180, 161])
output_sliced.size torch.Size([180])

batched_inputs.size torch.Size([180, 161])
batched_outputs.size torch.Size([180])

I need the batched ones to append, but the torch.cat isn't working. What am I doing wrong?


title: How to load and use a pretained PyTorch InceptionV3 model to classify an image

body:
I have the same problem as How can I load and use a PyTorch (.pth.tar) model which does not have an accepted answer or one I can figure out how to follow the advice given.

I'm new to PyTorch. I am trying to load the pretrained PyTorch model referenced here: https://github.com/macaodha/inat_comp_2018

I'm pretty sure I am missing some glue.

# load the model
import torch
model=torch.load("iNat_2018_InceptionV3.pth.tar",map_location='cpu')

# try to get it to classify an image
imsize = 256
loader = transforms.Compose([transforms.Scale(imsize), transforms.ToTensor()])

def image_loader(image_name):
    """load image, returns cuda tensor"""
    image = Image.open(image_name)
    image = loader(image).float()
    image = Variable(image, requires_grad=True)
    image = image.unsqueeze(0)
    return image.cpu()  #assumes that you're using CPU

image = image_loader("test-image.jpg")

Produces the error:

   in ()
  ----> 1 model.predict(image)

  AttributeError: 'dict' object has no attribute 'predict


title: what is the fastest way of loading images?

body:
I have about 200,000 high resolution images, and loading such high quality images every time is time consuming.
Preloading all images might occupy too much memory.
How about saving each images into .npz file format and loading .npz instead of .jpg?  Would it be boosting speed?


title: Dataframe as datasource in torchtext

body:
I have a dataframe, which has two columns (review and sentiment). I am using pytorch and torchtext library for preprocessing data.
Is it possible to use dataframe as source to read data from, in torchtext?
I am looking for something similar to, but not

data.TabularDataset.splits(path='./data')

I have performed some operation (clean, change to required format) on data and final data is in a dataframe.

If not torchtext, what other package would you suggest that would help in preprocessing text data present in a datarame. I could not find anything online. Any help would be great.


title: Only getting 0 second video from First Order Motion Model

body:
I'm pretty new to this community and I can't figure out how to get the First Order Motion Model algorithm working. The only output I have had so far is just a video with a single frame. Here is the link to the website:
https://github.com/AliaksandrSiarohin/first-order-model

I'm attempting to use the VoxCeleb dataset pre-trained checkpoint, provided by github page above.

I'm running a windows laptop machine with an Intel i5-6200U and integrated graphics.

This is what I have entered into the command prompt:

python run.py --config config/vox-256.yaml --mode animate --checkpoint checkpoint/vox-cpk.pth.tar

This is the result:

C:\Users\caleb\Desktop\Python\Deepfake_files\first-order-model-master>python run.py --config config/vox-256.yaml --checkpoint checkpoint/vox-cpk.pth.tar --mode animate
Use random train-test split.
Animate...
0it [00:00, ?it/s]C:\Users\caleb\AppData\Local\Programs\Python\Python38\lib\site-packages\torch\nn\functional.py:2705: UserWarning: Default grid_sample and affine_grid behavior has changed to align_corners=False since 1.3.0. Please specify align_corners=True if the old behavior is desired. See the documentation of grid_sample for details.
  warnings.warn("Default grid_sample and affine_grid behavior has changed "
C:\Users\caleb\AppData\Local\Programs\Python\Python38\lib\site-packages\torch\nn\functional.py:2503: UserWarning: Default upsampling behavior when mode=bilinear is changed to align_corners=False since 0.4.0. Please specify align_corners=True if the old behavior is desired. See the documentation of nn.Upsample for details.
  warnings.warn("Default upsampling behavior when mode={} is changed "
C:\Users\caleb\AppData\Local\Programs\Python\Python38\lib\site-packages\torch\nn\functional.py:1351: UserWarning: nn.functional.sigmoid is deprecated. Use torch.sigmoid instead.
  warnings.warn("nn.functional.sigmoid is deprecated. Use torch.sigmoid instead.")
1it [00:09,  9.26s/it]

The algorithm then outputs a video with only a single frame.

The input mp4 video is about 1 min long and both the driving video and source image are sized to 256x256

If anyone has any experience with this algorithm, please resond!


title: RuntimeError: Expected object of scalar type Long but got scalar type Float for argument #2 'target'

body:
I'm running into an issue while calculating the loss for my Neural Net. I'm not sure why the program expects a long object because all my Tensors are in float form. I looked at threads with similar errors and the solution was to cast Tensors as floats instead of longs, but that wouldn't work in my case because all my data is already in float form when passed to the network.
Here's my code:
# Dataloader
from torch.utils.data import Dataset, DataLoader

class LoadInfo(Dataset):
    def __init__(self, prediction, indicator):
        self.prediction = prediction
        self.indicator = indicator
    def __len__(self):
        return len(self.prediction)
    def __getitem__(self, idx):
        data = torch.tensor(self.indicator.iloc[idx, :],dtype=torch.float)
        data = torch.unsqueeze(data, 0)
        label = torch.tensor(self.prediction.iloc[idx, :],dtype=torch.float)
        sample = {'data': data, 'label': label}
        return sample

# Trainloader
test_train = LoadInfo(train_label, train_indicators)
trainloader = DataLoader(test_train, batch_size=64,shuffle=True, num_workers=1,pin_memory=True)

# The Network
class NetDense2(nn.Module):

    def __init__(self):
        super(NetDense2, self).__init__()
        self.rnn1 = nn.RNN(11, 100, 3)
        self.rnn2 = nn.RNN(100, 500, 3)
        self.fc1 = nn.Linear(500, 100)
        self.fc2 = nn.Linear(100, 20)
        self.fc3 = nn.Linear(20, 3)

    def forward(self, x):
        x1, h1 = self.rnn1(x)
        x2, h2 = self.rnn2(x1)
        x = F.relu(self.fc1(x2))
        x = F.relu(self.fc2(x))
        x = self.fc3(x)
        return x

# Allocate / Transfer to GPU
dense2 = NetDense2()
dense2.cuda()

# Optimizer
import torch.optim as optim
criterion = nn.CrossEntropyLoss()                                 # specify the loss function
optimizer = optim.SGD(dense2.parameters(), lr=0.001, momentum=0.9,weight_decay=0.001)

# Training
dense2.train()
loss_memory = []
for epoch in range(50):  # loop over the dataset multiple times
    running_loss = 0.0
    for i, samp in enumerate(trainloader):
        # get the inputs
        ins = samp['data']
        targets = samp['label']
        tmp = []
        tmp = torch.squeeze(targets.float())
        ins, targets = ins.cuda(),  tmp.cuda()
        # zero the parameter gradients
        optimizer.zero_grad()
        # forward + backward + optimize
        outputs = dense2(ins)
        loss = criterion(outputs, targets)     # The loss
        loss.backward()
        optimizer.step()
        # keep track of loss
        running_loss += loss.data.item()

I get the error from above in the line " loss = criterion(outputs, targets) "


title: Using WeightedRandomSampler in PyTorch

body:
I need to implement a multi-label image classification model in PyTorch. However my data is not balanced, so I used the WeightedRandomSampler in PyTorch to create a custom dataloader. But when I iterate through the custom dataloader, I get the error : IndexError: list index out of range

Implemented the following code using this link :https://discuss.pytorch.org/t/balanced-sampling-between-classes-with-torchvision-dataloader/2703/3?u=surajsubramanian

def make_weights_for_balanced_classes(images, nclasses):
    count = [0] * nclasses
    for item in images:
        count[item[1]] += 1
    weight_per_class = [0.] * nclasses
    N = float(sum(count))
    for i in range(nclasses):
        weight_per_class[i] = N/float(count[i])
    weight = [0] * len(images)
    for idx, val in enumerate(images):
        weight[idx] = weight_per_class[val[1]]
    return weight

weights = make_weights_for_balanced_classes(train_dataset.imgs, len(full_dataset.classes))
weights = torch.DoubleTensor(weights)
sampler = WeightedRandomSampler(weights, len(weights))

train_loader = DataLoader(train_dataset, batch_size=4,sampler = sampler, pin_memory=True)

Based on the answer in https://stackoverflow.com/a/60813495/10077354, the following is my updated code. But then too when I create a dataloader :loader = DataLoader(full_dataset, batch_size=4, sampler=sampler), len(loader) returns 1.

class_counts = [1691, 743, 2278, 1271]
num_samples = np.sum(class_counts)
labels = [tag for _,tag in full_dataset.imgs]

class_weights = [num_samples/class_counts[i] for i in range(len(class_counts)]
weights = [class_weights[labels[i]] for i in range(num_samples)]
sampler = WeightedRandomSampler(torch.DoubleTensor(weights), num_samples)

Thanks a lot in advance !

I included an utility function based on the accepted answer below :

def sampler_(dataset):
    dataset_counts = imageCount(dataset)
    num_samples = sum(dataset_counts)
    labels = [tag for _,tag in dataset]

    class_weights = [num_samples/dataset_counts[i] for i in range(n_classes)]
    weights = [class_weights[labels[i]] for i in range(num_samples)]
    sampler = WeightedRandomSampler(torch.DoubleTensor(weights), int(num_samples))
    return sampler

The imageCount function finds number of images of each class in the dataset. Each row in the dataset contains the image and the class, so we take the second element in the tuple into consideration.

def imageCount(dataset):
    image_count = [0]*(n_classes)
    for img in dataset:
        image_count[img[1]] += 1
    return image_count


title: Pytorch: [TypeError: __init__() takes 1 positional argument but 2 were given]

body:
I searched StackOverflow and visited other websites for help, but I can´t find a solution to my problem.
I will leave the whole code to make it understandable for you. It´s about 110 lines, written with PyTorch.

Each time, I will compile and calculate a prediction, this error-code will show up:

Traceback (most recent call last):
File "/Users/MacBookPro/Dropbox/01 GST h_da Privat/BA/06_KNN/PyTorchV1/BesucherV5.py", line 108, in <module>
result = Network(test_exp).data[0][0].item()
TypeError: __init__() takes 1 positional argument but 2 were given

I know, other users had this too, but none of their solutions helped me out. I guess the mistake is either in my class "Network" or in variable "result".
I hope that someone of you had this problem and know how to fix it or can help me in a different way.

Short information about the Dataset:

My Dataset has 10 columns and gets splitted into two sets. X and Y. X has 9 columns, Y just one. These are then used to train the network.

Thank you in advance!

Kind regards
Christian Richter

My Code:

import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.autograd import Variable
import torch.optim as optim
import pandas as pd

### Dataset ###

dataset = pd.read_csv('./data/train_data_csv.csv')

x_temp = dataset.iloc[:, :-1].values

print(x_temp)
print()
print(x_temp.size)
print()

y_temp = dataset.iloc[:, 9:].values

print(y_temp)
print()
print(y_temp.size)
print()

x_train_tensor = torch.FloatTensor(x_temp)
y_train_tensor = torch.FloatTensor(y_temp)

### Network Architecture ###

class Network(nn.Module):
    def __init__(self):
        super(Network, self).__init__()
        self.linear1 = nn.Linear(9, 9)  #10 Input-Neurons, 10 Output-Neurons, Linearer Layer
        self.linear2 = nn.Linear(9, 1)

    def forward(self, x):
        pax_predict = F.relu(self.linear1(x))
        pax_predict = self.linear2(x)
        return pax_predict

    def num_flat_features(self, pax_predict):
        size = pax_predict.size()[1:]
        num = 1
        for i in size:
            num *= i
        return num

network = Network()
print(network)

criterion = nn.MSELoss()
target = Variable(y_train_tensor)
optimizer = torch.optim.SGD(network.parameters(), lr=0.0001)

### Training

for epoch in range(200):
    input = Variable(x_train_tensor)
    y_pred = network(input)

    loss = criterion(y_pred, target)

    optimizer.zero_grad()

    loss.backward()
    optimizer.step()

test_exp = torch.Tensor([[40116]])

result = Network(test_exp).data[0][0].item()

print('Result is: ', result)


title: calculate perplexity in pytorch

body:
I've just trained an LSTM language model using pytorch. The main body of the class is this:

class LM(nn.Module):
    def __init__(self, n_vocab,
                       seq_size,
                       embedding_size,
                       lstm_size,
                       pretrained_embed):

        super(LM, self).__init__()
        self.seq_size = seq_size
        self.lstm_size = lstm_size
        self.embedding = nn.Embedding.from_pretrained(pretrained_embed, freeze = True)
        self.lstm = nn.LSTM(embedding_size,
                            lstm_size,
                            batch_first=True)
        self.fc = nn.Linear(lstm_size, n_vocab)

    def forward(self, x, prev_state):
        embed = self.embedding(x)
        output, state = self.lstm(embed, prev_state)
        logits = self.fc(output)

        return logits, state

Now I want to write a function which calculates how good a sentence is, based on the trained language model (some score like perplexity, etc.).

I'm a bit confused and I don't know how should I calculate this. A similar sample would be of greate use.


title: Using nn.Linear() and nn.BatchNorm1d() together

body:
I don't understand how BatchNorm1d works when the data is 3D, (batch size, H, W).

Example

Input size: (2,50,70)
Layer: nn.Linear(70,20)
Output size: (2,50,20)

If I then include a batch normalisation layer it requires num_features=50:

BN    : nn.BatchNorm1d(50)

and I don't understand why it isn't 20:

BN    : nn.BatchNorm1d(20)

Example 1)

class Net(nn.Module):
    def __init__(self):
        super(Net,self).__init__()
        self.bn11 = nn.BatchNorm1d(50)
        self.fc11 = nn.Linear(70,20)

    def forward(self, inputs):
        out = self.fc11(inputs)
        out = torch.relu(self.bn11(out))
        return out

model = Net()
inputs = torch.Tensor(2,50,70)
outputs = model(inputs)

Example 2)

class Net(nn.Module):
    def __init__(self):
        super(Net,self).__init__()
        self.bn11 = nn.BatchNorm1d(20)
        self.fc11 = nn.Linear(70,20)

    def forward(self, inputs):
        out = self.fc11(inputs)
        out = torch.relu(self.bn11(out))
        return out

model = Net()
inputs = torch.Tensor(2,50,70)
outputs = model(inputs)

Example 1 works.
Example 2 throws the error:

RuntimeError: running_mean should contain 50 elements not 20

2D example:

Input size: (2,70)
Layer: nn.Linear(70,20)
BN: nn.BatchNorm1d(20)

I thought the 20 in the BN layer was due to there being 20 nodes output by the linear layer and each one requires a running means/std for the incoming values.

Why in the 3D case, if the linear layer has 20 output nodes, the BN layer doesn't have 20 features?


title: expected Double tensor (got Float tensor) in Pytorch

body:
I want to create nn.Module in Pytorch. I used following code for text related problem (In fact I use Glove 300d pre-trained embedding and weighted average of words in a sentence to do classification).

class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        self.conv1 = nn.Conv1d(300, 128, kernel_size=5)
        self.conv2 = nn.Conv1d(128, 64, kernel_size=2)
        self.conv2_drop = nn.Dropout()
        self.fc1 = nn.Linear(64, 20)
        self.fc2 = nn.Linear(20, 2)

    def forward(self, x):
        x = F.relu(F.avg_pool1d(self.conv1(x), 2))
        x = F.relu(F.avg_pool1d(self.conv2_drop(self.conv2(x)), 2))
        x = x.view(-1, 1)
        x = F.relu(self.fc1(x))
        x = F.dropout(x, training=self.training)
        return self.fc2(x)

But it gives me following error:

Traceback (most recent call last):
    x = F.relu(F.avg_pool1d(self.conv1(x), 2))
  File "/Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5/site-packages/torch/nn/modules/module.py", line 224, in __call__
    result = self.forward(*input, **kwargs)
  File "/Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5/site-packages/torch/nn/modules/conv.py", line 154, in forward
    self.padding, self.dilation, self.groups)
  File "/Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5/site-packages/torch/nn/functional.py", line 83, in conv1d
    return f(input, weight, bias)
RuntimeError: expected Double tensor (got Float tensor)

I fairly new to Conv1d, and most of tutorial used Conv1d for image problems. Can anybody give me some idea what's the problem?

I also added model.double() inside forward method but gives me another error:

RuntimeError: Given input size: (300 x 1 x 1). Calculated output size: (128 x 1 x -3). Output size is too small


title: tensor division in pytorch. Assertion error

body:
This is kind of a pytorch beginner question. In pytorch I'm trying to do element wise division with two tensors of size [5,5,3]. In numpy it works fine using np.divide(), but somehow I get an error here. I'm using PyTorch version 0.1.12 for Python 3.5.

c = [torch.DoubleTensor of size 5x5x3]

input_patch = [torch.FloatTensor of size 5x5x3]

input_patch is a slice of a torch.autograd Variable, and c is made by doing c = torch.from_numpy(self.patch_filt[:, :, :, 0]).float()

When doing:

torch.div(input_patch, c)

I get this error that I don't understand.

line 317, in div
assert not torch.is_tensor(other)
AssertionError

Does it mean that variable c should not be a torch_tensor? After casting c to also be a FloatTensor gives the same error still.

Thank you!


title: Can't we run an onnx model imported to pytorch?

body:
I have been trying to import a model from onnx format to work with pytorch. I am finding it difficult to get an example for the same. As most of the resources in Internet talks about exporting a pytorch model to onnx.

I found that torch.onnx() can only export the model and the import method hasn't been implemented yet. A direct installation of onnx library, helps me to do onnx.load("model_name.onnx"). How do I use this model with pytorch? I am not able to move the model to GPU with model.to(device="GPU")


title: Can nvidia-docker be run without a GPU?

body: The official PyTorch Docker image is based on nvidia/cuda, which is able to run on Docker CE, without any GPU. It can also run on nvidia-docker, I presume with CUDA support enabled. Is it possible to run nvidia-docker itself on an x86 CPU, without any GPU? Is there a way to build a single Docker image that takes advantage of CUDA support when it is available (e.g. when running inside nvidia-docker) and uses the CPU otherwise? What happens when you use torch.cuda from inside Docker CE? What exactly is the difference between Docker CE and why can't nvidia-docker be merged into Docker CE?


title: Understanding input shape to PyTorch LSTM

body:
This seems to be one of the most common questions about LSTMs in PyTorch, but I am still unable to figure out what should be the input shape to PyTorch LSTM.

Even after following several posts (1, 2, 3) and trying out the solutions, it doesn't seem to work.

Background: I have encoded text sequences (variable length) in a batch of size 12 and the sequences are padded and packed using pad_packed_sequence functionality. MAX_LEN for each sequence is 384 and each token (or word) in the sequence has a dimension of 768. Hence my batch tensor could have one of the following shapes: [12, 384, 768] or [384, 12, 768].

The batch will be my input to the PyTorch rnn module (lstm here).

According to the PyTorch documentation for LSTMs, its input dimensions are (seq_len, batch, input_size) which I understand as following.
seq_len - the number of time steps in each input stream (feature vector length).
batch - the size of each batch of input sequences.
input_size - the dimension for each input token or time step.

lstm = nn.LSTM(input_size=?, hidden_size=?, batch_first=True)

What should be the exact input_size and hidden_size values here?


title: How do you invert a tensor of boolean values in Pytorch?

body:
With NumPy, you can do it with np.invert(array), but there's no invert function in Pytorch. Let's say I have a 2D tensor of boolean values:
import torch

ts = torch.rand((10, 4)) < .5

tensor([[ True,  True, False,  True],
        [ True,  True,  True,  True],
        [ True, False,  True,  True],
        [False,  True,  True, False],
        [False,  True,  True,  True],
        [ True,  True,  True,  True],
        [ True, False,  True,  True],
        [False,  True, False,  True],
        [ True,  True, False,  True],
        [False, False,  True, False]])

How do I transform the False into True and vice versa?


title: Dimension of hidden layer LSTM Pytorch

body:
I was reading the implementation of LSTM in Pytorch.
The code goes like this:

lstm = nn.LSTM(3, 3)  # Input dim is 3, output dim is 3
inputs = [torch.randn(1, 3) for _ in range(5)]  # make a sequence of length 5

# initialize the hidden state.
hidden = (torch.randn(1, 1, 3),
          torch.randn(1, 1, 3))
for i in inputs:
    # Step through the sequence one element at a time.
    # after each step, hidden contains the hidden state.
    out, hidden = lstm(i.view(1, 1, -1), hidden)

I don't understand why the hidden state is defined by a tuple of two tensors instead of one? Since the hidden layer is simply a layer of the feed-forward neural network which is a vector.


title: Does Google-Colab continue running the script when "Runtime disconnected"?

body:
I am training a neural network for Neural Machine Traslation on Google Colaboratory. I know that the limit before disconnection is 12 hrs, but I am frequently disconnected before (4 or 6 hrs). The amount of time required for the training is more then 12 hrs, so I add some savings each 5000 epochs.

I don't understand if when I am disconnected from Runtime (GPU is used) the code is still execute by Google on the VM? I ask it because I can easily save the intermediate models on Drive, and so continue the train also if I am disconnected.

Does anyone know it?


title: How two rows can be swapped in a torch tensor?

body:
var = [[0, 1, -4, 8],
       [2, -3, 2, 1],
       [5, -8, 7, 1]]

var = torch.Tensor(var)

Here, var is a 3 x 4 (2d) tensor. How the first and second row can be swapped to get the following 2d tensor?

2, -3, 2, 1
0, 1, -4, 8
5, -8, 7, 1


title: error occurs when loading data in pytorch: 'Image' object has no attribute 'shape'

body:
I am finetuning resnet152 using the code based on ImageNet training in PyTorch, and error occurs when I load the data, and it occured only after handling several batches of images. How can I solve the problem.
Following code is  the simplified code that produces the same error :

code

# Data loading code
normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
                                 std=[0.229, 0.224, 0.225])

train_loader = torch.utils.data.DataLoader(
    datasets.ImageFolder(train_img_dir, transforms.Compose([
        transforms.RandomSizedCrop(224),
        transforms.RandomHorizontalFlip(),
        transforms.ToTensor(),
        normalize,
    ])),
    batch_size=256, shuffle=True,
    num_workers=1, pin_memory=True)
for i, (input_x, target) in enumerate(train_loader):
    if i % 10 == 0:
        print(i)
        print(input_x.shape)
        print(target.shape)

error

0
torch.Size([256, 3, 224, 224])
torch.Size([256])
10
torch.Size([256, 3, 224, 224])
torch.Size([256])
20
torch.Size([256, 3, 224, 224])
torch.Size([256])
30
torch.Size([256, 3, 224, 224])
torch.Size([256])
----------------------------------------------------------------------
AttributeError                       Traceback (most recent call last)
<ipython-input-48-792d6ca206df> in <module>()
----> 1 for i, (input_x, target) in enumerate(train_loader):
      2     if i % 10 == 0:
      3 #     sample_img = input_x[0]
      4         print(i)
      5         print(input_x.shape)

/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py in __next__(self)
    200                 self.reorder_dict[idx] = batch
    201                 continue
--> 202             return self._process_next_batch(batch)
    203
    204     next = __next__  # Python 2 compatibility

/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py in _process_next_batch(self, batch)
    220         self._put_indices()
    221         if isinstance(batch, ExceptionWrapper):
--> 222             raise batch.exc_type(batch.exc_msg)
    223         return batch
    224

AttributeError: Traceback (most recent call last):
  File "/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py", line 41, in _worker_loop
    samples = collate_fn([dataset[i] for i in batch_indices])
  File "/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py", line 41, in <listcomp>
    samples = collate_fn([dataset[i] for i in batch_indices])
  File "/usr/local/lib/python3.5/dist-packages/torchvision-0.1.9-py3.5.egg/torchvision/datasets/folder.py", line 118, in __getitem__
    img = self.transform(img)
  File "/usr/local/lib/python3.5/dist-packages/torchvision-0.1.9-py3.5.egg/torchvision/transforms.py", line 369, in __call__
    img = t(img)
  File "/usr/local/lib/python3.5/dist-packages/torchvision-0.1.9-py3.5.egg/torchvision/transforms.py", line 706, in __call__
    i, j, h, w = self.get_params(img)
  File "/usr/local/lib/python3.5/dist-packages/torchvision-0.1.9-py3.5.egg/torchvision/transforms.py", line 693, in get_params
    w = min(img.size[0], img.shape[1])
AttributeError: 'Image' object has no attribute 'shape'


title: Why do we do batch matrix-matrix product?

body:
I'm following Pytorch seq2seq tutorial and ittorch.bmm method is used like below:

attn_applied = torch.bmm(attn_weights.unsqueeze(0),
                         encoder_outputs.unsqueeze(0))

I understand why we need to multiply attention weight and encoder outputs.

What I don't quite understand is the reason why we need bmm method here.
torch.bmm document says

  Performs a batch matrix-matrix product of matrices stored in batch1 and batch2.

  batch1 and batch2 must be 3-D tensors each containing the same number of matrices.

  If batch1 is a (b×n×m) tensor, batch2 is a (b×m×p) tensor, out will be a (b×n×p) tensor.


title: How to solve the run time error "Only Tensors created explicitly by the user (graph leaves) support the deepcopy protocol at the moment"

body:
I want to use output variables of NN as an input in another function,but met with error like this 'Only Tensors created explicitly by the user (graph leaves) support the deepcopy protocol at the moment'.The out variables require gradient.

I tried by changing the output variables to numpy values, but in that case the back propagataion does not work because it see numpy values as variables which does not need gradient.

output = model(SOC[13])

# Three output values of NN
Rs=output[0]
R1=output[1]
C1=output[2]

# Using these variables in another function

num1=[Rs*R1*C1,R1+Rs]
den1=[C1*R1,1]
G = control.tf(num,den)

It should work, but it gives error.

     14             num=[Rs*R1*C1,R1+Rs]
     15             den=[C1*R1,1]
---> 16             G = control.tf(num,den)
~\Anaconda3\lib\site-packages\control\xferfcn.py in __init__(self, *args)
    106
    107         """
--> 108         args = deepcopy(args)
    109         if len(args) == 2:
    110             # The user provided a numerator and a denominator.
~\Anaconda3\lib\site-packages\torch\tensor.py in __deepcopy__(self, memo)
     16     def __deepcopy__(self, memo):
     17         if not self.is_leaf:
---> 18             raise RuntimeError("Only Tensors created explicitly by the user "
     19                                "(graph leaves) support the deepcopy protocol at the moment")
     20         if id(self) in memo:


title: How can I update the parameters of a neural network in PyTorch?

body: Let's say I wanted to multiply all parameters of a neural network in PyTorch (an instance of a class inheriting from torch.nn.Module) by 0.9. How would I do that?


I am getting the following error while trying to load a saved model.

KeyError: 'unexpected key "module.encoder.embedding.weight" in state_dict'

This is the function I am using to load a saved model.

def load_model_states(model, tag):
    """Load a previously saved model states."""
    filename = os.path.join(args.save_path, tag)
    with open(filename, 'rb') as f:
        model.load_state_dict(torch.load(f))

The model is a sequence-to-sequence network whose init function (constructor) is given below.

def __init__(self, dictionary, embedding_index, max_sent_length, args):
    """"Constructor of the class."""
    super(Sequence2Sequence, self).__init__()
    self.dictionary = dictionary
    self.embedding_index = embedding_index
    self.config = args
    self.encoder = Encoder(len(self.dictionary), self.config)
    self.decoder = AttentionDecoder(len(self.dictionary), max_sent_length, self.config)
    self.criterion = nn.NLLLoss()  # Negative log-likelihood loss

    # Initializing the weight parameters for the embedding layer in the encoder.
    self.encoder.init_embedding_weights(self.dictionary, self.embedding_index, self.config.emsize)

When I print the model (sequence-to-sequence network), I get the following.

Sequence2Sequence (
  (encoder): Encoder (
    (drop): Dropout (p = 0.25)
    (embedding): Embedding(43723, 300)
    (rnn): LSTM(300, 300, batch_first=True, dropout=0.25)
  )
  (decoder): AttentionDecoder (
    (embedding): Embedding(43723, 300)
    (attn): Linear (600 -> 12)
    (attn_combine): Linear (600 -> 300)
    (drop): Dropout (p = 0.25)
    (out): Linear (300 -> 43723)
    (rnn): LSTM(300, 300, batch_first=True, dropout=0.25)
  )
  (criterion): NLLLoss (
  )
)

So, module.encoder.embedding is an embedding layer, and module.encoder.embedding.weight represents the associated weight matrix. So, why it says- unexpected key "module.encoder.embedding.weight" in state_dict?


title: Pytorch: Intermediate testing during training

body:
How can I test my pytorch model on validation data during training?
I know that there is the function myNet.eval() which apparantly switches of any dropout layers, but is it also preventing the gradients from being accumulated?
Also how would I undo the myNet.eval() command in order to continue with the training?

If anyone has some code snippet / toy example I would be grateful!


title: multi-variable linear regression with pytorch

body:
I'm working on a linear regression problem with Pytorch.
I've had success with the single variable case, however when I perform multi-variable linear regression I get the following error. How should I perform linear regression with multiple variables?

  TypeError                                 Traceback (most recent call
  last)  in ()
        9     optimizer.zero_grad() #gradient
       10     outputs = model(inputs) #output
  ---> 11     loss = criterion(outputs,targets) #loss function
       12     loss.backward() #backward propogation
       13     optimizer.step() #1-step optimization(gradeint descent)

  /anaconda/envs/tensorflow/lib/python3.6/site-packages/torch/nn/modules/module.py
  in call(self, *input, **kwargs)
      204
      205     def call(self, *input, **kwargs):
  --> 206         result = self.forward(*input, **kwargs)
      207         for hook in self._forward_hooks.values():
      208             hook_result = hook(self, input, result)

  /anaconda/envs/tensorflow/lib/python3.6/site-packages/torch/nn/modules/loss.py
  in forward(self, input, target)
       22         _assert_no_grad(target)
       23         backend_fn = getattr(self._backend, type(self).name)
  ---> 24         return backend_fn(self.size_average)(input, target)
       25
       26

  /anaconda/envs/tensorflow/lib/python3.6/site-packages/torch/nn/_functions/thnn/auto.py
  in forward(self, input, target)
       39         output = input.new(1)
       40         getattr(self._backend, update_output.name)(self._backend.library_state, input, target,
  ---> 41                                                    output, *self.additional_args)
       42         return output
       43

  TypeError: FloatMSECriterion_updateOutput received an invalid
  combination of arguments - got (int, torch.FloatTensor,
  torch.DoubleTensor, torch.FloatTensor, bool), but expected (int state,
  torch.FloatTensor input, torch.FloatTensor target, torch.FloatTensor
  output, bool sizeAverage)

here is code

#import
import torch
import torch.nn as nn
import numpy as np
import matplotlib.pyplot as plt
from torch.autograd import Variable

#input_size = 1
input_size = 3
output_size = 1
num_epochs = 300
learning_rate = 0.002

#Data set
#x_train = np.array([[1.564],[2.11],[3.3],[5.4]], dtype=np.float32)
x_train = np.array([[73.,80.,75.],[93.,88.,93.],[89.,91.,90.],[96.,98.,100.],[73.,63.,70.]],dtype=np.float32)
#y_train = np.array([[8.0],[19.0],[25.0],[34.45]], dtype= np.float32)
y_train = np.array([[152.],[185.],[180.],[196.],[142.]])
print('x_train:\n',x_train)
print('y_train:\n',y_train)

class LinearRegression(nn.Module):
    def __init__(self,input_size,output_size):
        super(LinearRegression,self).__init__()
        self.linear = nn.Linear(input_size,output_size)

    def forward(self,x):
        out = self.linear(x) #Forward propogation
        return out

model = LinearRegression(input_size,output_size)

#Lost and Optimizer
criterion = nn.MSELoss()
optimizer = torch.optim.SGD(model.parameters(),lr=learning_rate)

#train the Model
for epoch in range(num_epochs):
    #convert numpy array to torch Variable
    inputs = Variable(torch.from_numpy(x_train)) #convert numpy array to torch tensor
    #inputs = Variable(torch.Tensor(x_train))
    targets = Variable(torch.from_numpy(y_train)) #convert numpy array to torch tensor

    #forward+ backward + optimize
    optimizer.zero_grad() #gradient
    outputs = model(inputs) #output
    loss = criterion(outputs,targets) #loss function
    loss.backward() #backward propogation
    optimizer.step() #1-step optimization(gradeint descent)

    if(epoch+1) %5 ==0:
        print('epoch [%d/%d], Loss: %.4f' % (epoch +1, num_epochs, loss.data[0]))
        predicted = model(Variable(torch.from_numpy(x_train))).data.numpy()
        plt.plot(x_train,y_train,'ro',label='Original Data')
        plt.plot(x_train,predicted,label='Fitted Line')
        plt.legend()
        plt.show()


title: Treat a tuple/list of Tensors as a single Tensor

body:
I'm using Pytorch for some robotics Reinforcement Learning tasks. I'd like to use both images and information about the state as observations for this task. The implementation I'm using does not directly support this so I'm making some amendments. Expected observations are either state, as a 1 dimensional Tensor, or images as a 3 dimensional Tensor (channels, width, height). In my task I would like the observation to be a tuple of Tensors.

In many places in my codebase, the observation is of course expected to be a single Tensor, not a tuple of Tensors. Is there an easy way to treat a tuple of Tensors as a single Tensor?

For example, I would like:

observation.to(device)

to work as normal when observation is a single Tensor, and call .to(device) on each Tensor when observation is a tuple of Tensors.

It should be simple enough to create a data type that can support this, but I'm wondering does such a data type already exist? I haven't found anything so far.


title: LSTM time sequence generation using PyTorch

body:
For several days now, I am trying to build a simple sine-wave sequence generation using LSTM, without any glimpse of success so far.

I started from the time sequence prediction example

All what I wanted to do differently is:

Use different optimizers (e.g RMSprob) than LBFGS
Try different signals (more sine-wave components)

This is the link to my code. "experiment.py" is the main file

What I do is:

I generate artificial time-series data (sine waves)
I cut those time-series data into small sequences
The input to my model is a sequence of time 0...T, and the output is a sequence of time 1...T+1

What happens is:

The training and the validation losses goes down smoothly
The test loss is very low
However, when I try to generate arbitrary-length sequences, starting from a seed (a random sequence from the test data), everything goes wrong. The output always flats out

I simply don't see what the problem is. I am playing with this for a week now, with no progress in sight.
I would be very grateful for any help.

Thank you


title: How does the apply(fn) function in pytorch work with a function without return statement as argument?

body:
I have some questions about the following code fragments:

>>> def init_weights(m):
        print(m)
        if type(m) == nn.Linear:
            m.weight.data.fill_(1.0)
            print(m.weight)

>>> net = nn.Sequential(nn.Linear(2, 2), nn.Linear(2, 2))
>>> net.apply(init_weights)

apply() is part of the pytorch.nn package. You find the code in the documentation of this package. The final questions:
1. Why does this code sample work, although there is no argument or brackets added to init_weights() when it is given to apply()?
2. Where does the function init_weights(m) gets its argument m from, when it's given as a parameter to the function apply() without brackets and an m?


I'm trying to build a neural network using the pre-trained network VGG16 on Pytorch.

I understand that I need to adjust the classifier part of the network, so I have frozen the parameters to prevent backpropagation through them.

Code:

%matplotlib inline
%config InlineBackend.figure_format = 'retina'

import matplotlib.pyplot as plt
import numpy as np
import time

import torch
from torch import nn
from torch import optim
import torch.nn.functional as F
from torch.autograd import Variable
from torchvision import datasets, transforms
import torchvision.models as models
from collections import OrderedDict

data_dir = 'flowers'
train_dir = data_dir + '/train'
valid_dir = data_dir + '/valid'
test_dir = data_dir + '/test'

train_transforms = transforms.Compose([transforms.Resize(224),
                                       transforms.RandomRotation(30),
                                       transforms.RandomResizedCrop(224),
                                       transforms.RandomHorizontalFlip(),
                                       transforms.ToTensor(),
                                       transforms.Normalize(mean=[0.485, 0.456, 0.406],
                                                            std=[0.229, 0.224, 0.225])])

validn_transforms = transforms.Compose([transforms.Resize(224),
                                        transforms.CenterCrop(224),
                                        transforms.ToTensor(),
                                        transforms.Normalize((0.485, 0.456, 0.406),
                                                            (0.229, 0.224, 0.225))])

test_transforms = transforms.Compose([ transforms.Resize(224),
                                       transforms.RandomResizedCrop(224),
                                       transforms.ToTensor(),
                                       transforms.Normalize((0.485, 0.456, 0.406),
                                                            (0.229, 0.224, 0.225))])

train_data = datasets.ImageFolder(train_dir,
                                transform=train_transforms)

validn_data = datasets.ImageFolder(valid_dir,
                                transform=validn_transforms)

test_data = datasets.ImageFolder(test_dir,
                                transform=test_transforms)

trainloader = torch.utils.data.DataLoader(train_data, batch_size=32, shuffle=True)
validnloader = torch.utils.data.DataLoader(validn_data, batch_size=32, shuffle=True)
testloader = torch.utils.data.DataLoader(test_data, batch_size=32, shuffle=True)

model = models.vgg16(pretrained=True)
model

for param in model.parameters():
    param.requires_grad = False

classifier = nn.Sequential(OrderedDict([
                          ('fc1', nn.Linear(3*224*224, 10000)),
                          ('relu', nn.ReLU()),
                          ('fc2', nn.Linear(10000, 5000)),
                          ('relu', nn.ReLU()),
                          ('fc3', nn.Linear(5000, 102)),
                          ('output', nn.LogSoftmax(dim=1))
                          ]))

model.classifier = classifier

classifier

criterion = nn.NLLLoss()
optimizer = optim.Adam(model.classifier.parameters(), lr=0.001)
model.cuda()

epochs = 1
steps = 0
training_loss = 0
print_every = 300
for e in range(epochs):
    model.train()
    for images, labels in iter(trainloader):
        steps == 1

        images.resize_(32,3*224*224)

        inputs = Variable(images.cuda())
        targets = Variable(labels.cuda())
        optimizer.zero_grad()

        output = model.forward(inputs)
        loss = criterion(output, targets)
        loss.backward()
        optimizer.step()

        training_loss += loss.data[0]

        if steps % print_every == 0:
            print("Epoch: {}/{}... ".format(e+1, epochs),
                  "Loss: {:.4f}".format(training_loss/print_every))

            running_loss = 0

  Traceback

ValueError                                Traceback (most recent call last)
<ipython-input-17-30552f4b46e8> in <module>()
     15         optimizer.zero_grad()
     16
---> 17         output = model.forward(inputs)
     18         loss = criterion(output, targets)
     19         loss.backward()

/opt/conda/lib/python3.6/site-packages/torchvision-0.2.0-py3.6.egg/torchvision/models/vgg.py in forward(self, x)
     39
     40     def forward(self, x):
---> 41         x = self.features(x)
     42         x = x.view(x.size(0), -1)
     43         x = self.classifier(x)

/opt/conda/lib/python3.6/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    323         for hook in self._forward_pre_hooks.values():
    324             hook(self, input)
--> 325         result = self.forward(*input, **kwargs)
    326         for hook in self._forward_hooks.values():
    327             hook_result = hook(self, input, result)

/opt/conda/lib/python3.6/site-packages/torch/nn/modules/container.py in forward(self, input)
     65     def forward(self, input):
     66         for module in self._modules.values():
---> 67             input = module(input)
     68         return input
     69

/opt/conda/lib/python3.6/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    323         for hook in self._forward_pre_hooks.values():
    324             hook(self, input)
--> 325         result = self.forward(*input, **kwargs)
    326         for hook in self._forward_hooks.values():
    327             hook_result = hook(self, input, result)

/opt/conda/lib/python3.6/site-packages/torch/nn/modules/conv.py in forward(self, input)
    275     def forward(self, input):
    276         return F.conv2d(input, self.weight, self.bias, self.stride,
--> 277                         self.padding, self.dilation, self.groups)
    278
    279

/opt/conda/lib/python3.6/site-packages/torch/nn/functional.py in conv2d(input, weight, bias, stride, padding, dilation, groups)
     83     """
     84     if input is not None and input.dim() != 4:
---> 85         raise ValueError("Expected 4D tensor as input, got {}D tensor instead.".format(input.dim()))
     86
     87     f = _ConvNd(_pair(stride), _pair(padding), _pair(dilation), False,

ValueError: Expected 4D tensor as input, got 2D tensor instead.

Could it be because I am using Linear operation in my layer definitions?


title: Display a tensor image in matplotlib

body:
I'm doing a project for Udacity's AI with Python nanodegree.

I'm trying to display a torch.cuda.FloatTensor that I obtained from an image file path. Below that image will be a bar chart showing the top 5 most likely flower names with their associated probabilities.

plt.figure(figsize=(3,3))
path = 'flowers/test/1/image_06743.jpg'

top5_probs, top5_class_names = predict(path, model,5)

print(top5_probs)
print(top5_class_names)

flower_np_image = process_image(Image.open(path))
flower_tensor_image = torch.from_numpy(flower_np_image).type(torch.cuda.FloatTensor)
flower_tensor_image = flower_tensor_image.unsqueeze_(0)

axs = imshow(flower_tensor_image, ax = plt)
axs.axis('off')
axs.title(top5_class_names[0])
axs.show()
fig, ax = plt.subplots()
y_pos = np.arange(len(top5_class_names))
plt.barh(y_pos, list(reversed(top5_probs)))
plt.yticks(y_pos, list(reversed(top5_class_names)))
plt.ylabel('Flower Type')
plt.xlabel('Class Probability')

The imshow function was given to me as

def imshow(image, ax=None, title=None):
    if ax is None:
        fig, ax = plt.subplots()

    # PyTorch tensors assume the color channel is the first dimension
    # but matplotlib assumes is the third dimension
    image = image.transpose((1, 2, 0))

    # Undo preprocessing
    mean = np.array([0.485, 0.456, 0.406])
    std = np.array([0.229, 0.224, 0.225])
    image = std * image + mean

    # Image needs to be clipped between 0 and 1 or it looks like noise when displayed
    image = np.clip(image, 0, 1)

    ax.imshow(image)

    return ax

But I get this output

[0.8310797810554504, 0.14590543508529663, 0.013837042264640331, 0.005048676859587431, 0.0027143193874508142]
['petunia', 'pink primrose', 'balloon flower', 'hibiscus', 'tree mallow']

---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<ipython-input-17-f54be68feb7a> in <module>()
     12 flower_tensor_image = flower_tensor_image.unsqueeze_(0)
     13
---> 14 axs = imshow(flower_tensor_image, ax = plt)
     15 axs.axis('off')
     16 axs.title(top5_class_names[0])

<ipython-input-15-9c543acc89cc> in imshow(image, ax, title)
      5     # PyTorch tensors assume the color channel is the first dimension
      6     # but matplotlib assumes is the third dimension
----> 7     image = image.transpose((1, 2, 0))
      8
      9     # Undo preprocessing

TypeError: transpose(): argument 'dim0' (position 1) must be int, not tuple

<matplotlib.figure.Figure at 0x7f5855792160>

My predict function works, but the imshow just chokes with the call to transpose. Any ideas on how to fix this? I think it vaguely has something to do with converting back to a numpy array.

The notebook that I'm working on can be found at https://github.com/BozSteinkalt/ImageClassificationProject

Thanks!


title: Understanding Bilinear Layers

body:
When having a bilinear layer in PyTorch I can't wrap my head around how the calculation is done.

Here is a small example where I tried to figure out how it works:

In:

import torch.nn as nn
B = nn.Bilinear(2, 2, 1)
print(B.weight)

Out:

Parameter containing:
tensor([[[-0.4394, -0.4920],
         [ 0.6137,  0.4174]]], requires_grad=True)

I am putting through a zero-vector and a one-vector.

In:

print(B(torch.ones(2), torch.zeros(2)))
print(B(torch.zeros(2), torch.ones(2)))

Out:

tensor([0.2175], grad_fn=<ThAddBackward>)
tensor([0.2175], grad_fn=<ThAddBackward>)

I tried adding up the weights in various ways but I'm not getting the same result.

Thanks in advance!


I was trying to Implement simple NN using the MNIST datasets and I keep getting this error
    import matplotlib.pyplot as plt

import torch
from torchvision import models
from torchvision import datasets, transforms
from torch import nn, optim
import torch.nn.functional as F
import helper

transform = transforms.ToTensor()

train_data = datasets.MNIST(root='data', train=True,
                                   download=True, transform=transform)
test_data = datasets.MNIST(root='data', train=False,
                                  download=True, transform=transform)
train_loader = torch.utils.data.DataLoader(train_data, batch_size = 20, shuffle=True)
test_loader = torch.utils.data.DataLoader(test_data, batch_size = 20, shuffle=True)
class Net(nn.Module):
    def __init__(self):
        super(Net,self).__init__()
        self.fc1 = nn.Linear(784,10)

    def forward(self,x):

        x = x.view(-1,784)
        x = F.relu(self.fc1(x))
        x = F.log_softmax(x, dim = 1)
        return x

model = Net()
criterion = nn.NLLLoss()
optimizer = optim.Adam(model.parameters(), lr = 0.003)

epochs = 20
model.train()
for epoch in range(epochs):
    train_loss = 0

    for image, lables in train_data:

        optimizer.zero_grad()

        output = model(image)
        loss = criterion(output, lables)
        loss.backwards()
        optimizer.step()
        train_loss += loss.item()*image.size(0)
    train_loss = train_loss/len(train_data.dataset)

    print('Epoch: {} \tTraining Loss: {:.6f}'.format(epoch+1, train_loss))

Here is the error
    RuntimeError: dimension specified as 0 but tensor has no dimensions


title: PyTorch Model is not training

body:
I have a problem that I can't fix already for a week. I am trying to build CIFAR-10 classifier, but my loss value after each batch is jumping randomly, and accuracy is not improving even on same batch (I can't even overfit model with one batch), so i guess the only possible reason is - weights are not updating.

My Module class

class Net(nn.Module):
def __init__(self):
    super(Net, self).__init__()
    self.conv_pool = nn.Sequential(
        nn.Conv2d(3, 64, 3, padding=1),
        nn.ReLU(),
        nn.MaxPool2d(2, 2),
        nn.Conv2d(64, 128, 3, padding=1),
        nn.ReLU(),
        nn.MaxPool2d(2, 2),
        nn.Conv2d(128, 256, 3, padding=1),
        nn.ReLU(),
        nn.MaxPool2d(2, 2),
        nn.Conv2d(256, 512, 3, padding=1),
        nn.ReLU(),
        nn.MaxPool2d(2, 2),
        nn.Conv2d(512, 512, 1),
        nn.ReLU(),
        nn.MaxPool2d(2, 2))

    self.fcnn = nn.Sequential(
        nn.Linear(512, 2048),
        nn.ReLU(),
        nn.Linear(2048, 2048),
        nn.ReLU(),
        nn.Linear(2048, 10)
    )

def forward(self, x):
    x = self.conv_pool(x)
    x = x.view(-1, 512)
    x = self.fcnn(x)
    return x

Optimizers I am using:

net = Net()
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(net.parameters(), lr=0.001, momentum=0.9)

My Train function:

def train():
for epoch in range(5):  # loop over the dataset multiple times
    for i in range(0, df_size):
        # get the data

        try:
            images, labels = loadBatch(ds, i)
        except BaseException:
            continue

        # wrap
        inputs = Variable(images)

        optimizer.zero_grad()

        outputs = net(inputs)

        loss = criterion(outputs, Variable(labels))

        loss.backward()
        optimizer.step()
        acc = test(images,labels)
        print("Loss: " + str(loss.data[0]) + " Accuracy %: " + str(acc) + " Iteration: " + str(i))

        if i % 40 == 39:
            torch.save(net.state_dict(), "model_save_cifar")

    print("Finished epoch " + str(epoch))

I am using batch_size = 20, image_size = 32 (CIFAR-10)

loadBatch functions returns tuple of LongTensor 20x3x32x32 for images and LongTensor 20x1 for labels

I would be really happy if you can help me, or suggest possible solution ( I have a guess that it's because sequential modules in NN, but parameters that I am passing to optimizer seems to be right)