title: pytorch loss value not change

body:
I wrote a module based on this article: http://www.wildml.com/2015/12/implementing-a-cnn-for-text-classification-in-tensorflow/

The idea is pass the input into multiple streams then concat together and connect to a FC layer. I divided my source code into 3 custom modules: TextClassifyCnnNet >> FlatCnnLayer >> FilterLayer

FilterLayer:

class FilterLayer(nn.Module):
    def __init__(self, filter_size, embedding_size, sequence_length, out_channels=128):
        super(FilterLayer, self).__init__()

        self.model = nn.Sequential(
            nn.Conv2d(1, out_channels, (filter_size, embedding_size)),
            nn.ReLU(inplace=True),
            nn.MaxPool2d((sequence_length - filter_size + 1, 1), stride=1)
        )

        for m in self.modules():
            if isinstance(m, nn.Conv2d):
                n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
                m.weight.data.normal_(0, math.sqrt(2. / n))

    def forward(self, x):
        return self.model(x)

FlatCnnLayer:

class FlatCnnLayer(nn.Module):
    def __init__(self, embedding_size, sequence_length, filter_sizes=[3, 4, 5], out_channels=128):
        super(FlatCnnLayer, self).__init__()

        self.filter_layers = nn.ModuleList(
            [FilterLayer(filter_size, embedding_size, sequence_length, out_channels=out_channels) for
             filter_size in filter_sizes])

    def forward(self, x):
        pools = []
        for filter_layer in self.filter_layers:
            out_filter = filter_layer(x)
            # reshape from (batch_size, out_channels, h, w) to (batch_size, h, w, out_channels)
            pools.append(out_filter.view(out_filter.size()[0], 1, 1, -1))
        x = torch.cat(pools, dim=3)

        x = x.view(x.size()[0], -1)
        x = F.dropout(x, p=dropout_prob, training=True)

        return x

TextClassifyCnnNet (main module):

class TextClassifyCnnNet(nn.Module):
    def __init__(self, embedding_size, sequence_length, num_classes, filter_sizes=[3, 4, 5], out_channels=128):
        super(TextClassifyCnnNet, self).__init__()

        self.flat_layer = FlatCnnLayer(embedding_size, sequence_length, filter_sizes=filter_sizes,
                                       out_channels=out_channels)

        self.model = nn.Sequential(
            self.flat_layer,
            nn.Linear(out_channels * len(filter_sizes), num_classes)
        )

    def forward(self, x):
        x = self.model(x)

        return x

def fit(net, data, save_path):
    if torch.cuda.is_available():
        net = net.cuda()

    for param in list(net.parameters()):
        print(type(param.data), param.size())

    optimizer = optim.Adam(net.parameters(), lr=0.01, weight_decay=0.1)

    X_train, X_test = data['X_train'], data['X_test']
    Y_train, Y_test = data['Y_train'], data['Y_test']

    X_valid, Y_valid = data['X_valid'], data['Y_valid']

    n_batch = len(X_train) // batch_size

    for epoch in range(1, n_epochs + 1):  # loop over the dataset multiple times
        net.train()
        start = 0
        end = batch_size

        for batch_idx in range(1, n_batch + 1):
            # get the inputs
            x, y = X_train[start:end], Y_train[start:end]
            start = end
            end = start + batch_size

            # zero the parameter gradients
            optimizer.zero_grad()

            # forward + backward + optimize
            predicts = _get_predict(net, x)
            loss = _get_loss(predicts, y)
            loss.backward()
            optimizer.step()

            if batch_idx % display_step == 0:
                print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(
                    epoch, batch_idx * len(x), len(X_train), 100. * batch_idx / (n_batch + 1), loss.data[0]))

        # print statistics
        if epoch % display_step == 0 or epoch == 1:
            net.eval()
            valid_predicts = _get_predict(net, X_valid)
            valid_loss = _get_loss(valid_predicts, Y_valid)
            valid_accuracy = _get_accuracy(valid_predicts, Y_valid)
            print('\r[%d] loss: %.3f - accuracy: %.2f' % (epoch, valid_loss.data[0], valid_accuracy * 100))

    print('\rFinished Training\n')

    net.eval()

    test_predicts = _get_predict(net, X_test)
    test_loss = _get_loss(test_predicts, Y_test).data[0]
    test_accuracy = _get_accuracy(test_predicts, Y_test)
    print('Test loss: %.3f - Test accuracy: %.2f' % (test_loss, test_accuracy * 100))

    torch.save(net.flat_layer.state_dict(), save_path)

def _get_accuracy(predicts, labels):
    predicts = torch.max(predicts, 1)[1].data[0]
    return np.mean(predicts == labels)

def _get_predict(net, x):
    # wrap them in Variable
    inputs = torch.from_numpy(x).float()
    # convert to cuda tensors if cuda flag is true
    if torch.cuda.is_available:
        inputs = inputs.cuda()
    inputs = Variable(inputs)
    return net(inputs)

def _get_loss(predicts, labels):
    labels = torch.from_numpy(labels).long()
    # convert to cuda tensors if cuda flag is true
    if torch.cuda.is_available:
        labels = labels.cuda()
    labels = Variable(labels)
    return F.cross_entropy(predicts, labels)

It seems that parameters 're just updated slightly each epoch, the accuracy remains for all the process. While with the same implementation and the same params in Tensorflow, it runs correctly.

I'm new to Pytorch, so maybe my instructions has something wrong, please help me to find out. Thank you!

P.s: I try to use F.nll_loss + F.log_softmax instead of F.cross_entropy. Theoretically, it should return the same, but in fact another result is printed out (but it still be a wrong loss value)


title: PyTorch Datasets: Converting entire Dataset to NumPy

body:
I'm trying to convert the Torchvision MNIST train and test datasets into NumPy arrays but can't find documentation to actually perform the conversion.

My goal would be to take an entire dataset and convert it into a single NumPy array, preferably without iterating through the entire dataset.

I've looked at How do I turn a Pytorch Dataloader into a numpy array to display image data with matplotlib? but it doesn't address my issue.

So my question is, utilizing torch.utils.data.DataLoader, how would I go about converting the datasets (train/test) into two NumPy arrays such that all of the examples are present?

Note: I've left the batch size as the default of 1 for now; I could set it to 60,000 for train and 10,000 for test, but I'd prefer to not use magic numbers of that sort.

Thank you.


title: Loss not decreasing - Pytorch

body:
I am using dice loss for my implementation of a Fully Convolutional Network(FCN) which involves hypernetworks. The model has two inputs and one output which is a binary segmentation map. The model is updating weights but loss is constant.
It is not even overfitting on only three training examples

I have used other loss functions as well like dice+binarycrossentropy loss, jacard loss and MSE loss but the loss is almost constant.
I have also tried almost every activation function like ReLU, LeakyReLU, Tanh. Moreover I have to use sigmoid at the the output because I need my outputs to be in range [0,1]
Learning rate is 0.01. Moreover, I have tried different learning rates as well like 0.0001, 0.001, 0.1. And no matter what loss the training starts at, it always comes at this value

This shows gradients for three training examples. And overall loss

tensor(0.0010, device='cuda:0')
tensor(0.1377, device='cuda:0')
tensor(0.1582, device='cuda:0')
Epoch 9, Overall loss = 0.9604763123724196, mIOU=0.019766070265581623
tensor(0.0014, device='cuda:0')
tensor(0.0898, device='cuda:0')
tensor(0.0455, device='cuda:0')
Epoch 10, Overall loss = 0.9616242945194244, mIOU=0.01919178702228237
tensor(0.0886, device='cuda:0')
tensor(0.2561, device='cuda:0')
tensor(0.0108, device='cuda:0')
Epoch 11, Overall loss = 0.960331304506822, mIOU=0.01983801422510155

I expect the loss to converge in few epochs.
What should I do?


title: How do we analyse a loss vs epochs graph?

body:
I'm training a language model and the loss vs epochs is plotted each time of training. I'm attaching two samples from it.

Obviously, the second one is showing better performance. But, from these graphs, when do we take a decision to stop training (early stopping)?

Can we understand overfitting and underfitting from these graphs or do I need to plot additional learning curves?

What are the additional inferences that can be made from these plots?


title: Channel wise CrossEntropyLoss for image segmentation in pytorch

body:
I am doing an image segmentation task. There are 7 classes in total so the final outout is a tensor like [batch, 7, height, width] which is a softmax output. Now intuitively I wanted to use CrossEntropy loss but the pytorch implementation doesn't work on channel wise one-hot encoded vector

So I was planning to make a function on my own. With a help from some stackoverflow, My code so far looks like this

from torch.autograd import Variable
import torch
import torch.nn.functional as F

def cross_entropy2d(input, target, weight=None, size_average=True):
    # input: (n, c, w, z), target: (n, w, z)
    n, c, w, z = input.size()
    # log_p: (n, c, w, z)
    log_p = F.log_softmax(input, dim=1)
    # log_p: (n*w*z, c)
    log_p = log_p.permute(0, 3, 2, 1).contiguous().view(-1, c)  # make class dimension last dimension
    log_p = log_p[
       target.view(n, w, z, 1).repeat(0, 0, 0, c) >= 0]  # this looks wrong -> Should rather be a one-hot vector
    log_p = log_p.view(-1, c)
    # target: (n*w*z,)
    mask = target >= 0
    target = target[mask]
    loss = F.nll_loss(log_p, target.view(-1), weight=weight, size_average=False)
    if size_average:
        loss /= mask.data.sum()
    return loss

images = Variable(torch.randn(5, 3, 4, 4))
labels = Variable(torch.LongTensor(5, 3, 4, 4).random_(3))
cross_entropy2d(images, labels)

I get two errors. One is mentioned on the code itself, where it expects one-hot vector. The 2nd one says the following

RuntimeError: invalid argument 2: size '[5 x 4 x 4 x 1]' is invalid for input with 3840 elements at ..\src\TH\THStorage.c:41

For example purpose I was trying to make it work on a 3 class problem. So the targets and labels are (excluding the batch parameter for simplification ! )

Target:

 Channel 1     Channel 2  Channel 3

[[0 1 1 0 ]   [0 0 0 1 ]  [1 0 0 0 ]
  [0 0 1 1 ]   [0 0 0 0 ]  [1 1 0 0 ]
  [0 0 0 1 ]   [0 0 0 0 ]  [1 1 1 0 ]
  [0 0 0 0 ]   [0 0 0 1 ]  [1 1 1 0 ]

Labels:

 Channel 1     Channel 2  Channel 3

[[0 1 1 0 ]   [0 0 0 1 ]  [1 0 0 0 ]
  [0 0 1 1 ]   [.2 0 0 0] [.8 1 0 0 ]
  [0 0 0 1 ]   [0 0 0 0 ]  [1 1 1 0 ]
  [0 0 0 0 ]   [0 0 0 1 ]  [1 1 1 0 ]

So how can I fix my code to calculate channel wise CrossEntropy loss ?


title: Error libtorch_python.so: cannot open shared object file: No such file or directory

body:
I'm trying to implement fastai pretrain language model and it requires torch to work. After run the code, I got some problem about the import torch._C

I run it on my linux, python 3.7.1, via pip: torch 1.0.1.post2, cuda V7.5.17. I'm getting this error:

Traceback (most recent call last):
  File "pretrain_lm.py", line 7, in <module>
    import fastai
  File "/home/andira/anaconda3/lib/python3.7/site-packages/fastai/__init__.py", line 1, in <module>
    from .basic_train import *
  File "/home/andira/anaconda3/lib/python3.7/site-packages/fastai/basic_train.py", line 2, in <module>
    from .torch_core import *
  File "/home/andira/anaconda3/lib/python3.7/site-packages/fastai/torch_core.py", line 2, in <module>
    from .imports.torch import *
  File "/home/andira/anaconda3/lib/python3.7/site-packages/fastai/imports/__init__.py", line 2, in <module>
    from .torch import *
  File "/home/andira/anaconda3/lib/python3.7/site-packages/fastai/imports/torch.py", line 1, in <module>
    import torch, torch.nn.functional as F
  File "/home/andira/anaconda3/lib/python3.7/site-packages/torch/__init__.py", line 84, in <module>
    from torch._C import *
ImportError: libtorch_python.so: cannot open shared object file: No such file or directory

So I tried to run this line:

from torch._C import *

and got same result

ImportError: libtorch_python.so: cannot open shared object file: No such file or directory

I checked /home/andira/anaconda3/lib/python3.7/site-packages/torch/lib and there are only libcaffe2_gpu.so and libshm.so file, and I can't find libtorch_python.so either. My question is, what is actually libtorch_python.so? I've read some of article and like most of it talked about undefined symbol, not cannot open shared object file: No such file or directory like mine. I'm new at python and torch so I really appreciate your answer.


title: Pytorch saving model UserWarning: Couldn't retrieve source code for container of type Network

body:
When saving model in Pytorch by using:

torch.save(model, 'checkpoint.pth')

I get the following warning:

  /opt/conda/lib/python3.6/site-packages/torch/serialization.py:193:
  UserWarning: Couldn't retrieve source code for container of type
  Network. It won't be checked for correctness upon loading.   "type " +
  obj.name + ". It won't be checked "

When I load it I get the following error:

state_dict = torch.load('checkpoint_state_dict.pth')
model = torch.load('checkpoint.pth')
model.load_state_dict(state_dict)

AttributeError                            Traceback (most recent call last)
<ipython-input-2-6a79854aef0f> in <module>()
      2 state_dict = torch.load('checkpoint_state_dict.pth')
      3 model = 0
----> 4 model = torch.load('checkpoint.pth')
      5 model.load_state_dict(state_dict)

/opt/conda/lib/python3.6/site-packages/torch/serialization.py in load(f, map_location, pickle_module)
    301         f = open(f, 'rb')
    302     try:
--> 303         return _load(f, map_location, pickle_module)
    304     finally:
    305         if new_fd:

/opt/conda/lib/python3.6/site-packages/torch/serialization.py in _load(f, map_location, pickle_module)
    467     unpickler = pickle_module.Unpickler(f)
    468     unpickler.persistent_load = persistent_load
--> 469     result = unpickler.load()
    470
    471     deserialized_storage_keys = pickle_module.load(f)

AttributeError: Can't get attribute 'Network' on <module '__main__'>

Why is not possible to save a model and reload it entirely?


title: How do I re-use trained fastai models?

body:
How do I load pretrained model using fastai implementation over PyTorch? Like in SkLearn I can use pickle to dump a model in file then load and use later. I've use .load() method after declaring learn instance like bellow to load previously saved weights:

arch=resnet34
data = ImageClassifierData.from_paths(PATH, tfms=tfms_from_model(arch, sz))
learn = ConvLearner.pretrained(arch, data, precompute=False)
learn.load('resnet34_test')

Then to predict the class of an image:

trn_tfms, val_tfms = tfms_from_model(arch,100)
img = open_image('circle/14.png')
im = val_tfms(img)
preds = learn.predict_array(im[None])
print(np.argmax(preds))

But It gets me the error:

ValueError: Expected more than 1 value per channel when training, got input size [1, 1024]

This code works if I use learn.fit(0.01, 3) instead of learn.load(). What I really want is to avoid the training step In my application.


title: Difference in shape of tensor torch.Size([]) and torch.Size([1]) in pytorch

body:
I am new to pytorch. While playing around with tensors I observed 2 types of tensors-

tensor(58)
tensor([57.3895])

I printed their shape and the output was respectively -

torch.Size([])
torch.Size([1])

What is the difference between the two?


title: Pytorch - TypeError: 'torch.Size' object cannot be interpreted as an integer

body:
Hi I am training a PyTorch model and occurred this error:

----> 5     for i, data in enumerate(trainloader, 0):

TypeError: 'torch.Size' object cannot be interpreted as an integer

Not sure what this error means.

You can find my code here :

model.train()
for epoch in range(10):
    running_loss = 0

    for i, data in enumerate(trainloader, 0):

        inputs, labels = data

        optimizer.zero_grad()

        outputs = model(inputs)
        loss = criterion(outputs, labels)
        loss.backward()
        optimizer.step()

        if i % 2000 == 0:
          print (loss.item())
        running_loss += loss.item()
        if i % 1000 == 0:
            print ('[%d, %5d] loss: %.3f' % (epoch, i, running_loss/ 1000))
            running_loss = 0

torch.save(model, 'FeatureNet.pkl')

Update

This is the codeblock for DataLoader. I am using a customized dataloader and datasets, which x are pictures with size (1025, 16) and y are one-hot encoded vectors for classification.

x_train.shape = (1100, 1025, 16)

y_train.shape = (1100, 10)

clean_dir = '/home/tk/Documents/clean/'
mix_dir = '/home/tk/Documents/mix/'
clean_label_dir = '/home/tk/Documents/clean_labels/'
mix_label_dir = '/home/tk/Documents/mix_labels/'

class MSourceDataSet(Dataset):

    def __init__(self, clean_dir, mix_dir, clean_label_dir, mix_label_dir):

        with open(clean_dir + 'clean0.json') as f:
            clean0 = torch.Tensor(json.load(f))

        with open(mix_dir + 'mix0.json') as f:
            mix0 = torch.Tensor(json.load(f))

        with open(clean_label_dir + 'clean_label0.json') as f:
            clean_label0 = torch.Tensor(json.load(f))

        with open(mix_label_dir + 'mix_label0.json') as f:
            mix_label0 = torch.Tensor(json.load(f))

        self.spec = torch.cat([clean0, mix0], 0)
        self.label = torch.cat([clean_label0, mix_label0], 0)

    def __len__(self):
        return self.spec.shape

    def __getitem__(self, index):

        spec = self.spec[index]
        label = self.label[index]
        return spec, label

getitem

a, b = trainset.__getitem__(1000)
print (a.shape)
print (b.shape)

a.shape = torch.Size([1025, 16]);
b.shape = torch.Size([10])

Error message

---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<ipython-input-9-3bd71e5c00e1> in <module>()
      3     running_loss = 0
      4
----> 5     for i, data in enumerate(trainloader, 0):
      6
      7         inputs, labels = data

~/anaconda3/lib/python3.7/site-packages/torch/utils/data/dataloader.py in __next__(self)
    311     def __next__(self):
    312         if self.num_workers == 0:  # same-process loading
--> 313             indices = next(self.sample_iter)  # may raise StopIteration
    314             batch = self.collate_fn([self.dataset[i] for i in indices])
    315             if self.pin_memory:

~/anaconda3/lib/python3.7/site-packages/torch/utils/data/sampler.py in __iter__(self)
    136     def __iter__(self):
    137         batch = []
--> 138         for idx in self.sampler:
    139             batch.append(idx)
    140             if len(batch) == self.batch_size:

~/anaconda3/lib/python3.7/site-packages/torch/utils/data/sampler.py in __iter__(self)
     32
     33     def __iter__(self):
---> 34         return iter(range(len(self.data_source)))
     35
     36     def __len__(self):

TypeError: 'torch.Size' object cannot be interpreted as an integer


title: how to install pytorch in python2.7?

body:
i am using python2.7 in virtual environment. i tried to install pytorch in python2.7 but i got error belows:

UnsatisfiableError: The following specifications were found
to be incompatible with the existing python installation in your environment:

  - pytorch-cpu -> python[version='3.5.*|3.6.*']
  - pytorch-cpu -> python[version='>=3.5,<3.6.0a0|>=3.6,<3.7.0a0|>=3.7,<3.8.0a0']

If python is on the left-most side of the chain, that's the version you've asked for.
When python appears to the right, that indicates that the thing on the left is somehow
not available for the python version you are constrained to.  Your current python version
is (python=2.7).  Note that conda will not change your python version to a different minor version
unless you explicitly specify that.

The following specifications were found to be incompatible with each other:

Package wheel conflicts for:
python=2.7 -> pip -> wheel
pytorch-cpu -> python[version='>=3.6,<3.7.0a0'] -> pip -> wheel
Package vc conflicts for:
python=2.7 -> sqlite[version='>=3.27.2,<4.0a0'] -> vc[version='14.*|>=14,<15.0a0|>=14.1,<15.0a0']
python=2.7 -> vc[version='9.*|>=9,<10.0a0']
pytorch-cpu -> numpy[version='>=1.11'] -> vc[version='14|14.*|>=14,<15.0a0']
pytorch-cpu -> vc[version='>=14.1,<15.0a0']
Package cffi conflicts for:
pytorch-cpu -> cffi
pytorch-cpu -> python[version='>=3.6,<3.7.0a0'] -> pip -> requests -> urllib3[version='>=1.21.1,<1.25'] -> cryptography[version='>=1.3.4'] -> cffi[version='>=1.7']
python=2.7 -> pip -> requests -> urllib3[version='>=1.21.1,<1.25'] -> cryptography[version='>=1.3.4'] -> cffi[version='>=1.7']
Package pip conflicts for:
python=2.7 -> pip
pytorch-cpu -> python[version='>=3.6,<3.7.0a0'] -> pip
Package setuptools conflicts for:
python=2.7 -> pip -> setuptools
pytorch-cpu -> python[version='>=3.6,<3.7.0a0'] -> pip -> setuptools
Package msgpack-python conflicts for:
python=2.7 -> pip -> cachecontrol -> msgpack-python
pytorch-cpu -> python[version='>=3.6,<3.7.0a0'] -> pip -> cachecontrol -> msgpack-python

i tried conda install pytorch-cpu -c pytorch and link(https://pytorch.org/get-started/locally/). but it is not worked. so what should i do for install torch in python version2.7? i want to install pytorch cpu version.

plz help:)


title: How do I add some Gaussian noise to a tensor in PyTorch?

body:
I have a tensor I created using

    temp = torch.zeros(5, 10, 20, dtype=torch.float64)
    ## some values I set in temp

Now I want to add to each temp[i,j,k] a Gaussian noise (sampled from normal distribution with mean 0 and variance 0.1). How do I do it? I would expect there is a function to noise a tensor, but couldn't find anything. I did find this:

How to add Poisson noise and Gaussian noise?

but it seems to be related to images.


Im trying to make my first CNN using pyTorch and am following online help and code already people wrote. i am trying to reproduce their results. I'm using the Kaggle Dogs Breed Dataset for this and below is the error I get. The trainloader does not return my images and labels and any attempt to get them leads in an error:

Traceback (most recent call last):
  File "E:\Program Files\JetBrains\PyCharm Community Edition 2018.2.4\helpers\pydev\pydevd.py", line 1664, in <module>
    main()
  File "E:\Program Files\JetBrains\PyCharm Community Edition 2018.2.4\helpers\pydev\pydevd.py", line 1658, in main
    globals = debugger.run(setup['file'], None, None, is_module)
  File "E:\Program Files\JetBrains\PyCharm Community Edition 2018.2.4\helpers\pydev\pydevd.py", line 1068, in run
    pydev_imports.execfile(file, globals, locals)  # execute the script
  File "E:\Program Files\JetBrains\PyCharm Community Edition 2018.2.4\helpers\pydev\_pydev_imps\_pydev_execfile.py", line 18, in execfile
    exec(compile(contents+"\n", file, 'exec'), glob, loc)
  File "C:/Users/sbzfk/PycharmProjects/my_FCN_attempt/Kaggle_Dogs_Competition.py", line 85, in <module>
    img, label = next(iter(train_loader))
  File "C:\Users\sbzfk\AppData\Local\Programs\Python\Python37\lib\site-packages\torch\utils\data\dataloader.py", line 314, in __next__
    batch = self.collate_fn([self.dataset[i] for i in indices])
  File "C:\Users\sbzfk\AppData\Local\Programs\Python\Python37\lib\site-packages\torch\utils\data\dataloader.py", line 314, in <listcomp>
    batch = self.collate_fn([self.dataset[i] for i in indices])
  File "C:/Users/sbzfk/PycharmProjects/my_FCN_attempt/Kaggle_Dogs_Competition.py", line 42, in __getitem__
    img = self.transform(img)
  File "C:\Users\sbzfk\AppData\Local\Programs\Python\Python37\lib\site-packages\torchvision\transforms.py", line 34, in __call__
    img = t(img)
  File "C:\Users\sbzfk\AppData\Local\Programs\Python\Python37\lib\site-packages\torchvision\transforms.py", line 187, in __call__
    w, h = img.size
TypeError: cannot unpack non-iterable int object

Below is my code:

class DogsDataset(Dataset):

    def __init__(self, filenames, labels, root_dir, transform=None):
        assert len(filenames) == len(labels)        # if the two are not of equal length throw an error
        self.filenames = filenames
        self.labels = labels
        self.root_dir = root_dir
        self.transform = transform

    def __len__(self):
        return len(self.filenames)

    def __getitem__(self, idx):
        this_img = join(self.root_dir, 'train', self.filenames[idx]+'.jpg')
        print(this_img)
        img = io.imread(this_img)
        label = self.labels[idx]
        print(label)

        if self.transform:
            img = self.transform(img)

        return [img, label]

batch_size = 64
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

dataset_root = expanduser(join('~', 'Documents', 'kaggle_dogs_dataset'))
# join will intelligently join directories irrespective of OS, and expanduser will
# replace with /home/ in linux or the username in Windows

csv_file = pd.read_csv(join(dataset_root, 'labels.csv'))        # csv file has two columns, id which are filenames and breed which are labels
filenames = csv_file.id.values              # convert that column to an array, id is the column name and values converty to numpy array
# le = LabelEncoder()
# labels = le.fit_transform(csv_file.breed)           # this will just encode the names between 0 to models-1 , basically changing strings to integers
labels = csv_file.breed.values

filenames_train, filenames_eval, labels_train, labels_eval = train_test_split(filenames, labels,
                                                                              test_size=0.1, stratify=labels)       # this is an import from sklearn as the name implies, it randomly splits data into train and eval, 10% of it to test and rest train

data_transform = transforms.Compose([transforms.Scale(224),
                                     transforms.CenterCrop(224),
                                     transforms.ToTensor(),
                                     transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))])

dog_train = DogsDataset(filenames_train, labels_train, dataset_root, transform=data_transform)
train_loader = DataLoader(dog_train, batch_size, shuffle=True)

dog_eval = DogsDataset(filenames_eval, labels_eval, dataset_root, transform=data_transform)
eval_loader = DataLoader(dog_eval, batch_size, shuffle=True)

def im_show(axis, inp):
    """Denormalize and show"""
    inp = inp.numpy().transpose((1, 2, 0))
    mean = np.array([0.485, 0.456, 0.406])
    std = np.array([0.229, 0.224, 0.225])
    inp = std * inp + mean
    axis.imshow(inp)

img, label = next(iter(train_loader))
print(img.size(), label.size())
fig = plt.figure(1, figsize=(16, 4))
grid = ImageGrid(fig, 111, nrows_ncols=(1, 4), axes_pad=0.05)
for i in range(img.size()[0]):
    ax = grid[i]
    im_show(ax, img[i])

Ive tried debugging it line by line and with transform=none I seem to read all the images, only with transform=data_transform I seem to get this error.


When I run a project used Pytorch I came up with this error:

Traceback (most recent call last):
File "train_drnet.py", line 10, in <module>
import utils
File "/home/muse/drnet-py/utils.py", line 18, in <module>
from data.kth import KTH
File "/home/muse/drnet-py/data/kth.py", line 7, in <module>
from torch.utils.serialization import load_lua
ModuleNotFoundError: No module named 'torch.utils.serialization'

how to solve this,please?


title: How can I fix this pytorch error on Windows? (ModuleNotFoundError: No module named 'torch')

body:
Edit: You might want to skip to the end of the question first, I've followed some advice in comments / answers and the current error is different from the original (appears to be related to numpy possibly).

This error ModuleNotFoundError: No module named 'torch' shows up in tons of threads, I've been trying solutions all day. I'll go through my troubleshooting steps one by one, using the solutions suggested in threads.

System info:
Windows 10

First thing I did was follow the instructions on Pytorch, installed Anaconda and did this using the correct settings for my machine (Note: I tried Python v3.7 before trying v3.8 in these screenshots, none of the solutions worked with that either):

As you can see, that should be good to go, according to the instructions.

So I go into the python terminal and try to import pytorch, like so:

ModuleNotFoundError: No module named 'torch' Great, so what now? Well I paste the error into Google and begin my 4 hour wild goose chase.

First result, stack overflow answer: No module named "Torch"
 Let's try the selected answer, it requires some version-related syntax so lets check my python version:

Alright so as directed by the answer:

  Try to install PyTorch using pip:

  First create a conda environment using:

conda create -n env_pytorch python=3.6

Ok:

  Activate the environment using:

source activate env_pytorch

That doesnt work, but if we activate using the instructions given by the prompt, we can do so:

  Now install PyTorch using pip:

pip install torchvision --user ( this will install both torch and torchvision)

Hmmm.. well that went up in flames, so the following...

  Now go to python shell and import using the command:

import torch
import torchvision

...doesn't do anything new, same error as before.

Well, to the next thread, on PyTorch GitHub: https://github.com/pytorch/pytorch/issues/4827

They're trying to use Jupyter, so I tried this, is was another long process like the above that went up in flames, and I really dont want to need to use Jupyter anyway, so we'll skip this one.

Another Pytorch GitHub thread: https://github.com/pytorch/pytorch/issues/12004

  @edtky Could you please give me the output of the following commands
  in CMD?

  where conda.exe
  where pip.exe
  where python.exe

Sure I'll give it a shot:

  @edtky Looks like you have two Python environments. Please try
  importing torch in Anaconda Prompt.

Oh well, I already did that. No bueno.

Another thread: https://discuss.pytorch.org/t/modulenotfounderror-no-module-named-torch/7309 suggests:

  In that case you’ve probably forgotten to activate the environment
  where pytorch is installed. It can also be the library missing in your
  PYTHONPATH variable.

Well I did activate the environment as shown above, but I dont know anything about a PYTHONPTH variable, seems like the PyTorch setup guide wouldve mentioned if I needed to manually do that, I have no clue how to do it and you aren't explaining, so lets look for other answers.

Someone made a whole article to give us this little gym of advice: https://medium.com/@valeryyakovlev/anaconda-no-module-named-torch-ead10946de66

  Another beginner error I encountered when started to use pytorch in
  anaconda environment
      import torch ModuleNotFoundError: No module named ‘torch’ the proper way to install pytorch to anaconda is following
      conda install -c pytorch pytorch It’s not enough to simply run “conda install pytorch” — the package won’t be found. So first
  activate your conda profile with “source activate {your_profile}” and
  then run the command conda install -c...

Ok thats new info, let's try that command again now that our env is activated:

Ok that's a lot of green, let's try now...

Well we can't win 'em all, so lets go onto the next thread: https://forums.fast.ai/t/modulenotfounderror-no-module-named-torch-windows-10/12438/2

  I had also faced the similar problem , I just installed torch and torchvision using pip and it worked …

Ok! Let's try:

Oh well, another solution up in flames..

  I ran into a similar issue with Windows 10. In the end I could only get torch installed with Miniconda.

Alrighty, lets try it!

Alright, cool, moment of truth:

Awesome! You just read through 25 minutes of me re-producing all my attempts to solve this problem, and it doesnt even include the hour I spend down a rabbit hole trying to use Jupyter, which failed equally as miserably. I think it's time to post the question to StackOverflow!

Edit 1:

An answer points out that one of my logs was an error python 3.8 isn't compatible with pytorch, good point I'll fix that. After unintalling 3.8 and installing 3.7:

And no luck! Remember I actually mentioned in my first paragraph that while I was trying 3.8 in these screenshots, the first time around I did all of this with 3.7

Edit 2:

I forgot to install after activating the environment in the previous edit. Once I fixed that, there's a new error:


title: Output and Broadcast shape mismatch in MNIST, torchvision

body:
I am getting following error when using MNIST dataset in Torchvision

RuntimeError: output with shape [1, 28, 28] doesn't match the broadcast shape [3, 28, 28]

Here is my code:

import torch
from torchvision import datasets, transforms

transform = transforms.Compose([transforms.ToTensor(),
                            transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),
                          ])
trainset = datasets.MNIST('~/.pytorch/MNIST_data/', download=True, train=True, transform=transform)
trainloader = torch.utils.data.DataLoader(trainset, batch_size=64, shuffle=True)
images, labels = next(iter(trainloader))


title: Comparing Conv2D with padding between Tensorflow and PyTorch

body:
I am trying to import weights saved from a Tensorflow model to PyTorch. So far the results have been very similar. I ran into a snag when the model calls for conv2d with stride=2.

To verify the mismatch, I set up a very simple comparison between TF and PyTorch. First, I compare conv2d with stride=1.

import tensorflow as tf
import numpy as np
import torch
import torch.nn.functional as F

np.random.seed(0)
sess = tf.Session()

# Create random weights and input
weights = torch.empty(3, 3, 3, 8)
torch.nn.init.constant_(weights, 5e-2)
x = np.random.randn(1, 3, 10, 10)

weights_tf = tf.convert_to_tensor(weights.numpy(), dtype=tf.float32)
# PyTorch adopts [outputC, inputC, kH, kW]
weights_torch = torch.Tensor(weights.permute((3, 2, 0, 1)))

# Tensorflow defaults to NHWC
x_tf = tf.convert_to_tensor(x.transpose((0, 2, 3, 1)), dtype=tf.float32)
x_torch = torch.Tensor(x)

# TF Conv2D
tf_conv2d = tf.nn.conv2d(x_tf,
                         weights_tf,
                         strides=[1, 1, 1, 1],
                         padding="SAME")

# PyTorch Conv2D
torch_conv2d = F.conv2d(x_torch, weights_torch, padding=1, stride=1)

sess.run(tf.global_variables_initializer())
tf_result = sess.run(tf_conv2d)

diff = np.mean(np.abs(tf_result.transpose((0, 3, 1, 2)) - torch_conv2d.detach().numpy()))
print('Mean of Abs Diff: {0}'.format(diff))

The result of this execution is:

Mean of Abs Diff: 2.0443112092038973e-08

When I change stride to 2, the results start to vary.

# TF Conv2D
tf_conv2d = tf.nn.conv2d(x_tf,
                         weights_tf,
                         strides=[1, 2, 2, 1],
                         padding="SAME")

# PyTorch Conv2D
torch_conv2d = F.conv2d(x_torch, weights_torch, padding=1, stride=2)

The result of this execution is:

Mean of Abs Diff: 0.2104552686214447

According to PyTorch documentation, conv2d uses zero-padding defined by the padding argument. Thus, zeros are added to the left, top, right, and bottom of the input in my example.

If PyTorch simply adds padding on both sides based on the input parameter, it should be easy to replicate in Tensorflow.

# Manually add padding - consistent with PyTorch
paddings = tf.constant([[0, 0], [1, 1], [1, 1], [0, 0]])
x_tf = tf.convert_to_tensor(x.transpose((0, 2, 3, 1)), dtype=tf.float32)
x_tf = tf.pad(x_tf, paddings, "CONSTANT")

# TF Conv2D
tf_conv2d = tf.nn.conv2d(x_tf,
                         weights_tf,
                         strides=[1, 2, 2, 1],
                         padding="VALID")

The result of this comparison is:

Mean of Abs Diff: 1.6035047067930464e-08

What this tells me is that if I am somehow able to replicate the default padding behavior from Tensorflow into PyTorch, then my results will be similar.

This question inspected the behavior of padding in Tensorflow. TF documentation explains how padding is added for "SAME" convolutions. I discovered these links while writing this question.

Now that I know the padding strategy of Tensorflow, I can implement it in PyTorch.


title: PyTorch [1 if x > 0.5 else 0 for x in outputs ] with tensors

body:
I have a list outputs from a sigmoid function as a tensor in PyTorch

E.g

output (type) = torch.Size([4]) tensor([0.4481, 0.4014, 0.5820, 0.2877], device='cuda:0',

As I'm doing binary classification I want to turn all values bellow 0.5 to 0 and above 0.5 to 1.

Traditionally with a NumPy array you can use list iterators:

output_prediction = [1 if x > 0.5 else 0 for x in outputs ]

This would work, however I have to later convert output_prediction back to a tensor to use

torch.sum(ouput_prediction == labels.data)

Where labels.data is a binary tensor of labels.

Is there a way to use list iterators with tensors?


title: name '_C' is not defined pytorch+jupyter notebook

body:
I have some code that uses pytorch, that runs fine from my IDE (pycharm).

For research, I tried to run it from a jupyter notebook.

The code in the notebook:

from algorithms import Argparser
from algorithms import Session
def main():
    print("main started")
    args = Argparser.parse()
    session = Session(args)
    session.run()

The package looks like:

|-algorithms
|---__init__.py
|---Argparser.py
|---Session.py
|---<many more files that are being used by Session>.py

some of those files do import torch

When running the code in the notebook, I get

  NameError                                 Traceback (most recent call
  last)  in
        1 from algorithms import Argparser
  ----> 2 from algorithms import Session
        3 def main():
        4     print("main started")
        5     args = Argparser.parse()

  D:\git\stav\stav-rl\algorithms\Session.py in
       12
       13
  ---> 14 from algorithms.Episode import Episode
       15 from algorithms.Agent import Agent
       16 import torch

  D:\git\stav\stav-rl\algorithms\Episode.py in
        1 author = 'Noam'
        2
  ----> 3 import torch
        4 import numpy as np
        5 import cv2

  c:\anaconda3\envs\threadartrl\lib\site-packages\torch__init__.py in

       84 from torch._C import *
       85
  ---> 86 all += [name for name in dir(C)
       87             if name[0] != '' and
       88             not name.endswith('Base')]

  NameError: name '_C' is not defined

The error is on from algorithms import Session-->...-->import torch

How can i get the code to run?


In the code below:

    model_ft.eval()
    test_data, test_target = image_datasets['train'][idx]
    test_data = test_data.cuda()
    #test_target = test_target.cuda()
    test_target = torch.tensor(test_target)
    test_target = test_target.cuda()
    test_data.unsqueeze_(1)
    test_target.unsqueeze_(0)
    print(test_data.shape)
    output = model_ft(test_data)

I get the following error:

Traceback (most recent call last):
  File "test_loocv.py", line 245, in <module>
    output = model_ft(test_data)
  File "/scratch/sjn-p3/anaconda/anaconda3/lib/python3.6/site-packages/torch/nn/modules/module.py", line 477, in __call__
    result = self.forward(*input, **kwargs)
  File "/scratch/sjn-p3/anaconda/anaconda3/lib/python3.6/site-packages/torchvision-0.2.1-py3.6.egg/torchvision/models/resnet.py", line 139, in forward
  File "/scratch/sjn-p3/anaconda/anaconda3/lib/python3.6/site-packages/torch/nn/modules/module.py", line 477, in __call__
    result = self.forward(*input, **kwargs)
  File "/scratch/sjn-p3/anaconda/anaconda3/lib/python3.6/site-packages/torch/nn/modules/conv.py", line 301, in forward
    self.padding, self.dilation, self.groups)
RuntimeError: Given groups=1, weight of size [64, 3, 7, 7], expected input[3, 1, 224, 224] to have 3 channels, but got 1 channels instead

Also, test_data has the shape: torch.Size([3, 1, 224, 224]).

How should I fix this?


title: How do i iterate over pytorch 2d tensors?

body:
X = np.array([
    [-2,4,-1],
    [4,1,-1],
    [1, 6, -1],
    [2, 4, -1],
    [6, 2, -1],
    ])

for epoch in range(1,epochs):
    for i, x in enumerate(X):

X = torch.tensor([
    [-2,4,-1],
    [4,1,-1],
    [1, 6, -1],
    [2, 4, -1],
    [6, 2, -1],
    ])

The looping was fine when it was an numpy array. But i want to work with pytorch tensors so what's alternative to enumerate or How can i loop through the above tensor in the 2nd line?


title: Multiply all elements of PyTorch tensor

body: I wanted to do something like this question in PyTorch i.e. multiply all elements with each other keeping a certain axis constant. Does PyTorch has any pre-defined function for this?


title: How to properly update the weights in PyTorch?

body:
I'm trying to implement the gradient descent with PyTorch according to this schema but can't figure out how to properly update the weights. It is just a toy example with 2 linear layers with 2 nodes in hidden layer and one output.
Learning rate = 0.05;
target output = 1

https://hmkcode.github.io/ai/backpropagation-step-by-step/
Forward
Backward
My code is as following:
    import torch
    import torch.nn as nn
    import torch.nn.functional as F
    import torch.optim as optim

    class MyNet(nn.Module):

    def __init__(self):
         super(MyNet, self).__init__()
         self.linear1 = nn.Linear(2, 2,  bias=None)
         self.linear1.weight = torch.nn.Parameter(torch.tensor([[0.11, 0.21], [0.12, 0.08]]))
         self.linear2 = nn.Linear(2, 1,  bias=None)
         self.linear2.weight = torch.nn.Parameter(torch.tensor([[0.14, 0.15]]))

    def forward(self, inputs):
         out = self.linear1(inputs)
         out = self.linear2(out)
         return out

    losses = []
    loss_function = nn.L1Loss()
    model = MyNet()
    optimizer = optim.SGD(model.parameters(), lr=0.05)
    input = torch.tensor([2.0,3.0])
    print('weights before backpropagation = ',   list(model.parameters()))
    for epoch in range(1):
       result = model(input )
       loss = loss_function(result , torch.tensor([1.00],dtype=torch.float))
       print('result = ', result)
       print("loss = ",   loss)
       model.zero_grad()
       loss.backward()
       print('gradients =', [x.grad.data  for x in model.parameters()] )
       optimizer.step()
       print('weights after backpropagation = ',   list(model.parameters()))

The result is following :
    weights before backpropagation =  [Parameter containing:
    tensor([[0.1100, 0.2100],
            [0.1200, 0.0800]], requires_grad=True), Parameter containing:
    tensor([[0.1400, 0.1500]], requires_grad=True)]

    result =  tensor([0.1910], grad_fn=<SqueezeBackward3>)
    loss =  tensor(0.8090, grad_fn=<L1LossBackward>)

    gradients = [tensor([[-0.2800, -0.4200], [-0.3000, -0.4500]]),
                 tensor([[-0.8500, -0.4800]])]

    weights after backpropagation =  [Parameter containing:
    tensor([[0.1240, 0.2310],
            [0.1350, 0.1025]], requires_grad=True), Parameter containing:
    tensor([[0.1825, 0.1740]], requires_grad=True)]

Forward pass values:
2x0.11 + 3*0.21=0.85 ->
2x0.12 + 3*0.08=0.48 -> 0.85x0.14 + 0.48*0.15=0.191 -> loss =0.191-1 = -0.809

Backward pass: let's calculate w5 and w6 (output node weights)
w = w - (prediction-target)x(gradient)x(output of previous node)x(learning rate)
w5= 0.14 -(0.191-1)*1*0.85*0.05= 0.14 + 0.034= 0.174
w6= 0.15 -(0.191-1)*1*0.48*0.05= 0.15 + 0.019= 0.169

In my example Torch doesn't multiply the loss by derivative so we get wrong weights after updating. For the output node we got new weights w5,w6 [0.1825, 0.1740] , when it should be [0.174, 0.169]
Moving backward to update the first weight of the output node (w5) we need to calculate: (prediction-target)x(gradient)x(output of previous node)x(learning rate)=-0.809*1*0.85*0.05=-0.034. Updated weight w5 = 0.14-(-0.034)=0.174. But instead pytorch calculated new weight = 0.1825. It forgot to multiply by (prediction-target)=-0.809. For the output node we got gradients -0.8500 and -0.4800. But we still need to multiply them by loss 0.809 and learning rate 0.05 before we can update the weights.
What is the proper way of doing this?
Should we pass 'loss' as an argument to backward() as following: loss.backward(loss) .
That seems to fix it. But I couldn't find any example on this in documentation.


title: ImportError: No module named 'torchvision.datasets.mnist'

body:
Even after installing pytorch, this error is coming for this line.

from torchvision import datasets


title: How to calculate unbalanced weights for BCEWithLogitsLoss in pytorch

body:
I am trying to solve one multilabel problem with 270 labels and i have converted target labels into one hot encoded form. I am using BCEWithLogitsLoss(). Since training data is unbalanced, I am using pos_weight argument but i am bit confused.

pos_weight (Tensor, optional) – a weight of positive examples. Must be a vector with length equal to the number of classes.

Do i need to give total count of positive values of each label as a tensor or they mean something else by weights?


title: how to load the gpu trained model into the cpu?

body:
I am using PyTorch. I am going to use the already trained model on multiple GPUs with CPU. how to do this task?

I tried on Anaconda 3 and pytorch with cpu only i dont have gpu

model = models.get_pose_net(config, is_train=False)
gpus = [int(i) for i in config.GPUS.split(',')]
model = torch.nn.DataParallel(model, device_ids=gpus).cuda()

print('Created model...')
print(model)
checkpoint = torch.load(config.MODEL.RESUME)
model.load_state_dict(checkpoint)
model.eval()
print('Loaded pretrained weights...')

the error i got is

    AssertionError                            Traceback (most recent call  last)
<ipython-input-15-bbfcd201d332> in <module>()
      2 model = models.get_pose_net(config, is_train=False)
      3 gpus = [int(i) for i in config.GPUS.split(',')]
----> 4 model = torch.nn.DataParallel(model, device_ids=gpus).cuda()
      5 print('Created model...')
      6 print(model)

C:\Users\psl\Anaconda3\lib\site-packages\torch\nn\modules\module.py in cuda(self, device)
    258             Module: self
    259         """
--> 260         return self._apply(lambda t: t.cuda(device))
    261
    262     def cpu(self):

C:\Users\psl\Anaconda3\lib\site-packages\torch\nn\modules\module.py in

_apply(self, fn)
        185     def _apply(self, fn):
        186         for module in self.children():
    --> 187             module._apply(fn)
        188
        189         for param in self._parameters.values():

C:\Users\psl\Anaconda3\lib\site-packages\torch\nn\modules\module.py in _apply(self, fn)
    185     def _apply(self, fn):
    186         for module in self.children():
--> 187             module._apply(fn)
    188
    189         for param in self._parameters.values():

C:\Users\psl\Anaconda3\lib\site-packages\torch\nn\modules\module.py in _apply(self, fn)
    191                 # Tensors stored in modules are graph leaves, and we don't
    192                 # want to create copy nodes, so we have to unpack the data.
--> 193                 param.data = fn(param.data)
    194                 if param._grad is not None:
    195                     param._grad.data = fn(param._grad.data)

C:\Users\psl\Anaconda3\lib\site-packages\torch\nn\modules\module.py in <lambda>(t)
    258             Module: self
    259         """
--> 260         return self._apply(lambda t: t.cuda(device))
    261
    262     def cpu(self):

 C:\Users\psl\Anaconda3\lib\site-packages\torch\cuda\__init__.py in _lazy_init()
    159         raise RuntimeError(
    160             "Cannot re-initialize CUDA in forked subprocess. " + msg)
--> 161     _check_driver()
    162     torch._C._cuda_init()
    163     _cudart = _load_cudart()

C:\Users\psl\Anaconda3\lib\site-packages\torch\cuda\__init__.py in _check_driver()
     80 Found no NVIDIA driver on your system. Please check that you
     81 have an NVIDIA GPU and installed a driver from
---> 82 http://www.nvidia.com/Download/index.aspx""")
     83         else:
     84             # TODO: directly link to the alternative bin that needs install

AssertionError:
Found no NVIDIA driver on your system. Please check that you
have an NVIDIA GPU and installed a driver from
http://www.nvidia.com/Download/index.aspx


title: Pytorch tensor, how to switch channel position - Runtime error

body:
I have my training dataset as below, where X_train is 3D with 3 channels

Shape of X_Train:  (708, 256, 3)
Shape of Y_Train:  (708, 4)

Then I convert them into a tensor and input into the dataloader:

X_train=torch.from_numpy(X_data)
y_train=torch.from_numpy(y_data)
training_dataset = torch.utils.data.TensorDataset(X_train, y_train)
train_loader = torch.utils.data.DataLoader(training_dataset, batch_size=50, shuffle=False)

However when training the model, I get the following error:
RuntimeError: Given groups=1, weight of size 24 3 5, expected input[708, 256, 3] to have 3 channels, but got 256 channels instead

I suppose this is due to the position of the channel? In Tensorflow, the channel position is at the end, but in PyTorch the format is "Batch Size x Channel x Height x Width"? So how do I swap the positions in the x_train tensor to match the expected format in the dataloader?

class TwoLayerNet(torch.nn.Module):
    def __init__(self):
        super(TwoLayerNet,self).__init__()
        self.conv1 = nn.Sequential(
            nn.Conv1d(3, 3*8, kernel_size=5, stride=1),
            nn.Sigmoid(),
            nn.AvgPool1d(kernel_size=2, stride=0))
        self.conv2 = nn.Sequential(
            nn.Conv1d(3*8, 12, kernel_size=5, stride=1),
            nn.Sigmoid(),
            nn.AvgPool1d(kernel_size=2, stride = 0))
        #self.drop_out = nn.Dropout()

        self.fc1 = nn.Linear(708, 732)
        self.fc2 = nn.Linear(732, 4)

    def forward(self, x):
        out = self.conv1(x)
        out = self.conv2(out)
        out = out.reshape(out.size(0), -1)
        out = self.drop_out(out)
        out = self.fc1(out)
        out = self.fc2(out)
        return out


title: Extracting the top-k value-indices from a 1-D Tensor

body:
Given a 1-D tensor in Torch (torch.Tensor), containing values which can be compared (say floating point), how can we extract the indices of the top-k values in that tensor?
Apart from the brute-force method, I am looking for some API call, that Torch/lua provides, which can perform this task efficiently.


title: Pytorch gradients exist but weights not updating

body:
So, I have a deep convolutional network with an lstm layer, and after the ltsm layer it splits off to compute two different functions (using two different linear layers) whose results are then added together to form the final network output.

When I compute the loss of the network so that I can have it compute the gradients and update the weights, I have it do a few operations and then have it compute the loss between the derived value and the calculated target value.

def update(output, target):
    # target output is calculated outside the function
    # operations on output
    loss(output, target).backward()
    self.optimizer.step()

The network has some loss (sometimes in a very small order of magnitude, but sometimes also on higher orders of magnitude), for example a few of the losses:

tensor(1.00000e-04 *
   5.7420)
tensor(2.7190)
tensor(0.9684)

It also has gradients as calculated here:

for param in self.parameters():
    print(param.grad.data.sum())

Which outputs:

tensor(1.00000e-03 *
   1.9996)
tensor(1.00000e-03 *
   2.6101)
tensor(1.00000e-02 *
   -1.3879)
tensor(1.00000e-03 *
   -4.5834)
tensor(1.00000e-02 *
   2.1762)
tensor(1.00000e-03 *
   3.6246)
tensor(1.00000e-03 *
   6.6234)
tensor(1.00000e-02 *
   2.9373)
tensor(1.00000e-02 *
   1.2680)
tensor(1.00000e-03 *
   1.8791)
tensor(1.00000e-02 *
   1.7322)
tensor(1.00000e-02 *
   1.7322)
tensor(0.)
tensor(0.)
tensor(1.00000e-03 *
   -6.7885)
tensor(1.00000e-02 *
   9.7793)

And:

tensor(2.4620)
tensor(0.9544)
tensor(-26.2465)
tensor(0.2280)
tensor(-219.2602)
tensor(-2.7870)
tensor(-50.8203)
tensor(3.2548)
tensor(19.6163)
tensor(-18.6029)
tensor(3.8564)
tensor(3.8564)
tensor(0.)
tensor(0.)
tensor(0.8040)
tensor(-0.1157)

But when I compare the weight before and after running the optimizer, I get the result that the weights are equal to each other.

Code to see if weights change:

before = list(neuralnet.parameters())
neuralnet.update()
after = list(neuralnet.parameters())
for i in range(len(before)):
    print(torch.equal(before[i].data, after[i].data))

The above returns True for each iteration.


title: How to use groups parameter in PyTorch conv2d function

body:
I am trying to compute a per-channel gradient image in PyTorch. To do this, I want to perform a standard 2D convolution with a Sobel filter on each channel of an image. I am using the torch.nn.functional.conv2d function for this

In my minimum working example code below, I get an error:

import torch
import torch.nn.functional as F

filters = torch.autograd.Variable(torch.randn(1,1,3,3))
inputs = torch.autograd.Variable(torch.randn(1,3,10,10))
out = F.conv2d(inputs, filters, padding=1)

  RuntimeError: Given groups=1, weight[1, 1, 3, 3], so expected
  input[1, 3, 10, 10]  to have 1 channels, but got 3 channels instead

This suggests that groups need to be 3. However, when I make groups=3, I get a different error:

import torch
import torch.nn.functional as F

filters = torch.autograd.Variable(torch.randn(1,1,3,3))
inputs = torch.autograd.Variable(torch.randn(1,3,10,10))
out = F.conv2d(inputs, filters, padding=1, groups=3)

  RuntimeError: invalid argument 4: out of range at
  /usr/local/src/pytorch/torch/lib/TH/generic/THTensor.c:440

When I check that code snippet in the THTensor class, it refers to a bunch of dimension checks, but I don't know where I'm going wrong.

What does this error mean? How can I perform my intended convolution with this conv2d function? I believe I am misunderstanding the groups parameter.


title: I get this error on pytorch "RuntimeError: invalid argument 2: size '[-1 x 400]"

body:
Can anyone help me how to fix this error?

  RuntimeError: invalid argument 2: size '[-1 x 400]' is invalid for input of with 1597248 elements at /Users/soumith/miniconda2/conda-bld/pytorch_1503975723910/work/torch/lib/TH/THStorage.c:37

class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        self.conv1 = nn.Conv2d(3, 6, 5)
        self.pool = nn.MaxPool2d(2, 2)
        self.conv2 = nn.Conv2d(6, 16, 5)
        self.fc1 = nn.Linear(16 * 5 * 5, 120)
        self.fc2 = nn.Linear(120, 84)
        self.fc3 = nn.Linear(84, 10)

    def forward(self, x):
        x = self.pool(F.relu(self.conv1(x)))
        x = self.pool(F.relu(self.conv2(x)))
        x = x.view(-1, 16 * 5 * 5)
        x = F.relu(self.fc1(x))
        x = F.relu(self.fc2(x))
        x = self.fc3(x)
        return x

net = Net()

import torch.optim as optim

criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(net.parameters(), lr=0.001, momentum=0.9)
for epoch in range(2):  # loop over the dataset multiple times

    running_loss = 0.0
    for i, data in enumerate(trainloader, 0):
        # get the inputs
        inputs, labels = data

        # wrap them in Variable
        inputs, labels = Variable(inputs), Variable(labels)

        # zero the parameter gradients
        optimizer.zero_grad()

        # forward + backward + optimize
        outputs = net(inputs)
        loss = criterion(outputs, labels)
        loss.backward()
        optimizer.step()

        # print statistics
        running_loss += loss.data[0]
        if i % 5 == 4:    # print every 2000 mini-batches
            print('[%d, %5d] loss: %.3f' %
                  (epoch + 1, i + 1, running_loss / 5))
            running_loss = 0.0

print('Finished Training')


title: Extract features from last hidden layer Pytorch Resnet18

body:
I am implementing an image classifier using the Oxford Pet dataset with the pre-trained Resnet18 CNN.
The dataset consists of 37 categories with ~200 images in each of them.
Rather than using the final fc layer of the CNN as output to make predictions I want to use the CNN as a feature extractor to classify the pets.
For each image i'd like to grab features from the last hidden layer (which should be before the 1000-dimensional output layer). My model is using Relu activation so I should grab the output just after the ReLU (so all values will be non-negative)
Here is code (following the transfer learning tutorial on Pytorch):
loading data
normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
                                std=[0.229, 0.224, 0.225])

image_datasets = {"train": datasets.ImageFolder('images_new/train', transforms.Compose([
        transforms.RandomResizedCrop(224),
        transforms.RandomHorizontalFlip(),
        transforms.ToTensor(),
        normalize
    ])), "test": datasets.ImageFolder('images_new/test', transforms.Compose([
        transforms.Resize(256),
        transforms.CenterCrop(224),
        transforms.ToTensor(),
        normalize
    ]))
               }

dataloaders = {x: torch.utils.data.DataLoader(image_datasets[x], batch_size=4,
                                             shuffle=True, num_workers=4, pin_memory=True)
              for x in ['train', 'test']}

dataset_sizes = {x: len(image_datasets[x]) for x in ['train', 'test']}

train_class_names = image_datasets['train'].classes

device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")

train function
def train_model(model, criterion, optimizer, scheduler, num_epochs=25):
    since = time.time()

    best_model_wts = copy.deepcopy(model.state_dict())
    best_acc = 0.0

    for epoch in range(num_epochs):
        print('Epoch {}/{}'.format(epoch, num_epochs - 1))
        print('-' * 10)

        # Each epoch has a training and validation phase
        for phase in ['train', 'test']:
            if phase == 'train':
                scheduler.step()
                model.train()  # Set model to training mode
            else:
                model.eval()   # Set model to evaluate mode

            running_loss = 0.0
            running_corrects = 0

            # Iterate over data.
            for inputs, labels in dataloaders[phase]:
                inputs = inputs.to(device)
                labels = labels.to(device)

                # zero the parameter gradients
                optimizer.zero_grad()

                # forward
                # track history if only in train
                with torch.set_grad_enabled(phase == 'train'):

                    outputs = model(inputs)
                    _, preds = torch.max(outputs, 1)

                    loss = criterion(outputs, labels)

                    # backward + optimize only if in training phase
                    if phase == 'train':
                        loss.backward()
                        optimizer.step()

                # statistics
                running_loss += loss.item() * inputs.size(0)
                running_corrects += torch.sum(preds == labels.data)

            epoch_loss = running_loss / dataset_sizes[phase]
            epoch_acc = running_corrects.double() / dataset_sizes[phase]

            print('{} Loss: {:.4f} Acc: {:.4f}'.format(
                phase, epoch_loss, epoch_acc))

            # deep copy the model
            if phase == 'test' and epoch_acc > best_acc:
                best_acc = epoch_acc
                best_model_wts = copy.deepcopy(model.state_dict())

        print()

    time_elapsed = time.time() - since
    print('Training complete in {:.0f}m {:.0f}s'.format(
        time_elapsed // 60, time_elapsed % 60))
    print('Best val Acc: {:4f}'.format(best_acc))

    # load best model weights
    model.load_state_dict(best_model_wts)
    return model

Compute SGD cross-entropy loss
model_ft = models.resnet18(pretrained=True)
num_ftrs = model_ft.fc.in_features

print("number of features: ", num_ftrs)

model_ft.fc = nn.Linear(num_ftrs, len(train_class_names))

model_ft = model_ft.to(device)
criterion = nn.CrossEntropyLoss()

# Observe that all parameters are being optimized
optimizer_ft = optim.SGD(model_ft.parameters(), lr=0.001, momentum=0.9)

# Decay LR by a factor of 0.1 every 7 epochs
exp_lr_scheduler = lr_scheduler.StepLR(optimizer_ft, step_size=7, gamma=0.1)
model_ft = train_model(model_ft, criterion, optimizer_ft, exp_lr_scheduler,
                       num_epochs=24)

Now how do I get a feature vector from the last hidden layer for each of my images? I know I have to freeze the previous layer so that gradient isn't computed on them but I'm having trouble extracting the feature vectors.
My ultimate goal is to use those feature vectors to train a linear classifier such as Ridge or something like that.
Thanks!


title: Is there an efficient way to create a random bit mask in Pytorch?

body:
I want to have a random bit mask that has some specified percent of 0s. The function I devised is:

def create_mask(shape, rate):
    """
    The idea is, you take a random permutations of numbers. You then mod then
    mod it by the [number of entries in the bitmask] / [percent of 0s you
    want]. The number of zeros will be exactly the rate of zeros need. You
    can clamp the values for a bitmask.
    """
    mask = torch.randperm(reduce(operator.mul, shape, 1)).float().cuda()
    # Mod it by the percent to get an even dist of 0s.
    mask = torch.fmod(mask, reduce(operator.mul, shape, 1) / rate)
    # Anything not zero should be put to 1
    mask = torch.clamp(mask, 0, 1)
    return mask.view(shape)

To illustrate:

>>> x = create_mask((10, 10), 10)
>>> x

    1     1     1     1     1     1     1     1     1     1
    1     1     1     1     1     1     0     1     1     1
    0     1     1     1     1     0     1     1     1     1
    0     1     1     1     1     1     1     1     1     1
    1     1     1     1     1     1     1     1     1     0
    1     1     1     1     1     1     1     1     1     1
    1     1     1     0     1     1     1     0     1     1
    0     1     1     1     1     1     1     1     1     1
    1     1     1     0     1     1     0     1     1     1
    1     1     1     1     1     1     1     1     1     1
[torch.cuda.FloatTensor of size 10x10 (GPU 0)]

The main issue I have with this method is it requires the rate to divide the shape. I want a function that accepts an arbitrary decimal and gives approximately rate percent of 0s in the bitmask. Furthermore, I am trying to find a relatively efficient way of doing so. Hence, I would rather not move a numpy array from the CPU to the GPU. Is there an effiecient way of doing so that allows for a decimal rate?


title: Calculating Euclidian Norm in Pytorch.. Trouble understanding an implementation

body:
I've seen another StackOverflow thread talking about the various implementations for calculating the Euclidian norm and I'm having trouble seeing why/how a particular implementation works.

The code is found in an implementation of the MMD metric: https://github.com/josipd/torch-two-sample/blob/master/torch_two_sample/statistics_diff.py

Here is some beginning boilerplate:

import torch
sample_1, sample_2 = torch.ones((10,2)), torch.zeros((10,2))

Then the next part is where we pick up from the code above.. I'm unsure why the samples are being concatenated together..

sample_12 = torch.cat((sample_1, sample_2), 0)
distances = pdist(sample_12, sample_12, norm=2)

and are then passed to the pdist function:

def pdist(sample_1, sample_2, norm=2, eps=1e-5):
    r"""Compute the matrix of all squared pairwise distances.
    Arguments
    ---------
    sample_1 : torch.Tensor or Variable
        The first sample, should be of shape ``(n_1, d)``.
    sample_2 : torch.Tensor or Variable
        The second sample, should be of shape ``(n_2, d)``.
    norm : float
        The l_p norm to be used.
    Returns
    -------
    torch.Tensor or Variable
        Matrix of shape (n_1, n_2). The [i, j]-th entry is equal to
        ``|| sample_1[i, :] - sample_2[j, :] ||_p``."""

here we get to the meat of the calculation

    n_1, n_2 = sample_1.size(0), sample_2.size(0)
    norm = float(norm)
    if norm == 2.:
        norms_1 = torch.sum(sample_1**2, dim=1, keepdim=True)
        norms_2 = torch.sum(sample_2**2, dim=1, keepdim=True)
        norms = (norms_1.expand(n_1, n_2) +
             norms_2.transpose(0, 1).expand(n_1, n_2))
        distances_squared = norms - 2 * sample_1.mm(sample_2.t())
        return torch.sqrt(eps + torch.abs(distances_squared))

I am at a loss for why the euclidian norm would be calculated this way. Any insight would be greatly appreciated


title: convert cv2.umat to numpy array

body:
Processed_image() function returns a cv2.Umat type value which is to be reshaped from
3 dimensions(h, ch, w) to 4 dimensions(h, ch, w, 1) so i need it to be converted
to numpy array or also if possible help me to directally rehshape cv2.umat
type variable to be directally reshaped and converted to a pytorch tensor and can be
assigned to reshaped_image_tensor.

img_w=640
img_h=640
img_ch=3
umat_img = cv2.UMat(img)
display_one(umat_img, "RESPONSE")    #function created by me to display image
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
with torch.no_grad():
    processed_img = preprocess_image(umat_img, model_image_size = (img_h, img_ch, img_w))
    #___________write YOUR CODE here________
    reshaped_images_tensor = torch.from_numpy(processed_img.reshape(img_h, img_ch, img_w, 1)).float().to(device)      #images_tensor.reshape(img_h, img_ch, img_w, 1)
    outputs = model(reshaped_images_tensor)
    _, predicted = torch.max(outputs, 1)
    c = predicted.squeeze()
    output_probability(predicted, processed_img, umat_img)
if ord('q')==cv2.waitKey(10):
    exit(0)


title: Pytorch LSTM vs LSTMCell

body:
What is the difference between LSTM and LSTMCell in Pytorch (currently version 1.1)? It seems that LSTMCell is a special case of LSTM (i.e. with only one layer, unidirectional, no dropout).

Then, what's the purpose of having both implementations? Unless I'm missing something, it's trivial to use an LSTM object as an LSTMCell (or alternatively, it's pretty easy to use multiple LSTMCells to create the LSTM object)


title: PyTorch giving cuda runtime error

body:
I have made a slight modification in my code so that it does not use DataParallel and DistributedDataParallel. The code is as follows:

import argparse
import os
import shutil
import time

import torch
import torch.nn as nn
import torch.nn.parallel
import torch.backends.cudnn as cudnn
import torch.distributed as dist
import torch.optim
import torch.utils.data
import torch.utils.data.distributed
import torchvision.transforms as transforms
import torchvision.datasets as datasets
import torchvision.models as models

model_names = sorted(name for name in models.__dict__
    if name.islower() and not name.startswith("__")
    and callable(models.__dict__[name]))

parser = argparse.ArgumentParser(description='PyTorch ImageNet Training')
parser.add_argument('data', metavar='DIR',
                    help='path to dataset')
parser.add_argument('--arch', '-a', metavar='ARCH', default='resnet18',
                    choices=model_names,
                    help='model architecture: ' +
                        ' | '.join(model_names) +
                        ' (default: resnet18)')
parser.add_argument('-j', '--workers', default=4, type=int, metavar='N',
                    help='number of data loading workers (default: 4)')
parser.add_argument('--epochs', default=90, type=int, metavar='N',
                    help='number of total epochs to run')
parser.add_argument('--start-epoch', default=0, type=int, metavar='N',
                    help='manual epoch number (useful on restarts)')
parser.add_argument('-b', '--batch-size', default=256, type=int,
                    metavar='N', help='mini-batch size (default: 256)')
parser.add_argument('--lr', '--learning-rate', default=0.1, type=float,
                    metavar='LR', help='initial learning rate')
parser.add_argument('--momentum', default=0.9, type=float, metavar='M',
                    help='momentum')
parser.add_argument('--weight-decay', '--wd', default=1e-4, type=float,
                    metavar='W', help='weight decay (default: 1e-4)')
parser.add_argument('--print-freq', '-p', default=10, type=int,
                    metavar='N', help='print frequency (default: 10)')
parser.add_argument('--resume', default='', type=str, metavar='PATH',
                    help='path to latest checkpoint (default: none)')
parser.add_argument('-e', '--evaluate', dest='evaluate', action='store_true',
                    help='evaluate model on validation set')
parser.add_argument('--pretrained', dest='pretrained', action='store_true',
                    help='use pre-trained model')
parser.add_argument('--world-size', default=1, type=int,
                    help='number of distributed processes')
parser.add_argument('--dist-url', default='tcp://224.66.41.62:23456', type=str,
                    help='url used to set up distributed training')
parser.add_argument('--dist-backend', default='gloo', type=str,
                    help='distributed backend')

best_prec1 = 0

def main():
    global args, best_prec1
    args = parser.parse_args()

    args.distributed = args.world_size > 1

    if args.distributed:
        dist.init_process_group(backend=args.dist_backend, init_method=args.dist_url,
                                world_size=args.world_size)

    # create model
    if args.pretrained:
        print("=> using pre-trained model '{}'".format(args.arch))
        model = models.__dict__[args.arch](pretrained=True)
    else:
        print("=> creating model '{}'".format(args.arch))
        model = models.__dict__[args.arch]()

    if not args.distributed:
        if args.arch.startswith('alexnet') or args.arch.startswith('vgg'):
            #model.features = torch.nn.DataParallel(model.features)
            model.cuda()
        #else:
            #model = torch.nn.DataParallel(model).cuda()
    else:
        model.cuda()
        #model = torch.nn.parallel.DistributedDataParallel(model)

    # define loss function (criterion) and optimizer
    criterion = nn.CrossEntropyLoss().cuda()

    optimizer = torch.optim.SGD(model.parameters(), args.lr,
                                momentum=args.momentum,
                                weight_decay=args.weight_decay)

    # optionally resume from a checkpoint
    if args.resume:
        if os.path.isfile(args.resume):
            print("=> loading checkpoint '{}'".format(args.resume))
            checkpoint = torch.load(args.resume)
            args.start_epoch = checkpoint['epoch']
            best_prec1 = checkpoint['best_prec1']
            model.load_state_dict(checkpoint['state_dict'])
            optimizer.load_state_dict(checkpoint['optimizer'])
            print("=> loaded checkpoint '{}' (epoch {})"
                  .format(args.resume, checkpoint['epoch']))
        else:
            print("=> no checkpoint found at '{}'".format(args.resume))

    cudnn.benchmark = True

    # Data loading code
    traindir = os.path.join(args.data, 'train')
    valdir = os.path.join(args.data, 'val')
    normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
                                     std=[0.229, 0.224, 0.225])

    train_dataset = datasets.ImageFolder(
        traindir,
        transforms.Compose([
            transforms.RandomSizedCrop(224),
            transforms.RandomHorizontalFlip(),
            transforms.ToTensor(),
            normalize,
        ]))

    if args.distributed:
        train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset)
    else:
        train_sampler = None

    train_loader = torch.utils.data.DataLoader(
        train_dataset, batch_size=args.batch_size, shuffle=(train_sampler is None),
        num_workers=args.workers, pin_memory=True, sampler=train_sampler)

    val_loader = torch.utils.data.DataLoader(
        datasets.ImageFolder(valdir, transforms.Compose([
            transforms.Scale(256),
            transforms.CenterCrop(224),
            transforms.ToTensor(),
            normalize,
        ])),
        batch_size=args.batch_size, shuffle=False,
        num_workers=args.workers, pin_memory=True)

    if args.evaluate:
        validate(val_loader, model, criterion)
        return

    for epoch in range(args.start_epoch, args.epochs):
        if args.distributed:
            train_sampler.set_epoch(epoch)
        adjust_learning_rate(optimizer, epoch)

        # train for one epoch
        train(train_loader, model, criterion, optimizer, epoch)

        # evaluate on validation set
        prec1 = validate(val_loader, model, criterion)

        # remember best prec@1 and save checkpoint
        is_best = prec1 > best_prec1
        best_prec1 = max(prec1, best_prec1)
        save_checkpoint({
            'epoch': epoch + 1,
            'arch': args.arch,
            'state_dict': model.state_dict(),
            'best_prec1': best_prec1,
            'optimizer' : optimizer.state_dict(),
        }, is_best)

def train(train_loader, model, criterion, optimizer, epoch):
    batch_time = AverageMeter()
    data_time = AverageMeter()
    losses = AverageMeter()
    top1 = AverageMeter()
    top5 = AverageMeter()

    # switch to train mode
    model.train()

    end = time.time()
    for i, (input, target) in enumerate(train_loader):
        # measure data loading time
        data_time.update(time.time() - end)

        target = target.cuda(async=True)
        input_var = torch.autograd.Variable(input)
        target_var = torch.autograd.Variable(target)

        # compute output
        output = model(input_var)
        loss = criterion(output, target_var)

        # measure accuracy and record loss
        prec1, prec5 = accuracy(output.data, target, topk=(1, 5))
        losses.update(loss.data[0], input.size(0))
        top1.update(prec1[0], input.size(0))
        top5.update(prec5[0], input.size(0))

        # compute gradient and do SGD step
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

        # measure elapsed time
        batch_time.update(time.time() - end)
        end = time.time()

        if i % args.print_freq == 0:
            print('Epoch: [{0}][{1}/{2}]\t'
                  'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
                  'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
                  'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
                  'Prec@1 {top1.val:.3f} ({top1.avg:.3f})\t'
                  'Prec@5 {top5.val:.3f} ({top5.avg:.3f})'.format(
                   epoch, i, len(train_loader), batch_time=batch_time,
                   data_time=data_time, loss=losses, top1=top1, top5=top5))

def validate(val_loader, model, criterion):
    batch_time = AverageMeter()
    losses = AverageMeter()
    top1 = AverageMeter()
    top5 = AverageMeter()

    # switch to evaluate mode
    model.eval()

    end = time.time()
    for i, (input, target) in enumerate(val_loader):
        target = target.cuda(async=True)
        input_var = torch.autograd.Variable(input, volatile=True)
        target_var = torch.autograd.Variable(target, volatile=True)

        # compute output
        output = model(input_var)
        loss = criterion(output, target_var)

        # measure accuracy and record loss
        prec1, prec5 = accuracy(output.data, target, topk=(1, 5))
        losses.update(loss.data[0], input.size(0))
        top1.update(prec1[0], input.size(0))
        top5.update(prec5[0], input.size(0))

        # measure elapsed time
        batch_time.update(time.time() - end)
        end = time.time()

        if i % args.print_freq == 0:
            print('Test: [{0}/{1}]\t'
                  'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
                  'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
                  'Prec@1 {top1.val:.3f} ({top1.avg:.3f})\t'
                  'Prec@5 {top5.val:.3f} ({top5.avg:.3f})'.format(
                   i, len(val_loader), batch_time=batch_time, loss=losses,
                   top1=top1, top5=top5))

    print(' * Prec@1 {top1.avg:.3f} Prec@5 {top5.avg:.3f}'
          .format(top1=top1, top5=top5))

    return top1.avg

def save_checkpoint(state, is_best, filename='checkpoint.pth.tar'):
    torch.save(state, filename)
    if is_best:
        shutil.copyfile(filename, 'model_best.pth.tar')

class AverageMeter(object):
    """Computes and stores the average and current value"""
    def __init__(self):
        self.reset()

    def reset(self):
        self.val = 0
        self.avg = 0
        self.sum = 0
        self.count = 0

    def update(self, val, n=1):
        self.val = val
        self.sum += val * n
        self.count += n
        self.avg = self.sum / self.count

def adjust_learning_rate(optimizer, epoch):
    """Sets the learning rate to the initial LR decayed by 10 every 30 epochs"""
    lr = args.lr * (0.1 ** (epoch // 30))
    for param_group in optimizer.param_groups:
        param_group['lr'] = lr

def accuracy(output, target, topk=(1,)):
    """Computes the precision@k for the specified values of k"""
    maxk = max(topk)
    batch_size = target.size(0)

    _, pred = output.topk(maxk, 1, True, True)
    pred = pred.t()
    correct = pred.eq(target.view(1, -1).expand_as(pred))

    res = []
    for k in topk:
        correct_k = correct[:k].view(-1).float().sum(0, keepdim=True)
        res.append(correct_k.mul_(100.0 / batch_size))
    return res

if __name__ == '__main__':
    main()

And, when I run this code on a set of images with the alexnet neuralnet architecture, it gives a weird cuda error, which is as follows:

=> creating model 'alexnet'
THCudaCheck FAIL file=/pytorch/torch/lib/THC/THCGeneral.c line=70 error=30 : unknown error
Traceback (most recent call last):
  File "imagenet2.py", line 319, in <module>
    main()
  File "imagenet2.py", line 87, in main
    model.cuda()
  File "/usr/local/lib/python2.7/dist-packages/torch/nn/modules/module.py", line 147, in cuda
    return self._apply(lambda t: t.cuda(device_id))
  File "/usr/local/lib/python2.7/dist-packages/torch/nn/modules/module.py", line 118, in _apply
    module._apply(fn)
  File "/usr/local/lib/python2.7/dist-packages/torch/nn/modules/module.py", line 118, in _apply
    module._apply(fn)
  File "/usr/local/lib/python2.7/dist-packages/torch/nn/modules/module.py", line 118, in _apply
    module._apply(fn)
  File "/usr/local/lib/python2.7/dist-packages/torch/nn/modules/module.py", line 124, in _apply
    param.data = fn(param.data)
  File "/usr/local/lib/python2.7/dist-packages/torch/nn/modules/module.py", line 147, in <lambda>
    return self._apply(lambda t: t.cuda(device_id))
  File "/usr/local/lib/python2.7/dist-packages/torch/_utils.py", line 66, in _cuda
    return new_type(self.size()).copy_(self, async)
  File "/usr/local/lib/python2.7/dist-packages/torch/cuda/__init__.py", line 266, in _lazy_new
    _lazy_init()
  File "/usr/local/lib/python2.7/dist-packages/torch/cuda/__init__.py", line 85, in _lazy_init
    torch._C._cuda_init()
RuntimeError: cuda runtime error (30) : unknown error at /pytorch/torch/lib/THC/THCGeneral.c:70

Command used for running the code: python imagenet.py --world-size 1 --arch 'alexnet' <image_folder>

Where did I go wrong?

PS: Running on an AWS g2.2xlarge Ubuntu instance.

The CUDA version is as follows:

nvcc: NVIDIA (R) Cuda compiler driver
Copyright (c) 2005-2016 NVIDIA Corporation
Built on Tue_Jan_10_13:22:03_CST_2017
Cuda compilation tools, release 8.0, V8.0.61


title: cnn IndexError: Target 2 is out of bounds

body:
I got this error after I executed my code and it seems that the below portion of the code is throwing this error. I tried different ways but nothing could solve it. The error is given by the loss function.

for i, data in enumerate(train_loader, 0):

      #  import pdb;pdb.set_trace()
        inputs, labels = data
        print(type(inputs))
        for input in inputs:
            inputs = torch.Tensor(input)
        inputs, labels= Variable(inputs), Variable(labels)
        inputs=inputs.unsqueeze(1)
        optimizer.zero_grad()
        outputs = net(inputs)
        #import pdb;pdb.set_trace()
        loss_size = loss(outputs, labels)
        loss_size.backward()
        optimizer.step()

        running_loss += loss_size.data[0]
        total_train_loss += loss_size.data[0]

        if (i + 1) % (print_every + 1) == 0:
            print("Epoch {}, {:d}% \t train_loss: {:.2f} took: {:.2f}s".format(
                    epoch+1, int(100 * (i+1) / n_batches), running_loss / print_every, time.time() - start_time))
            running_loss = 0.0
            start_time = time.time()

--------------------------------------------------------------------------- IndexError                                Traceback (most recent call
last) <ipython-input-10-7d1b8710defa> in <module>
      1 CNN = Net()
----> 2 trainNet(CNN, learning_rate=0.001)
      3 #test()

<ipython-input-7-3208c0794681> in trainNet(net, learning_rate)
     23             outputs = net(inputs)
     24             #import pdb;pdb.set_trace()
---> 25             loss_size = loss(outputs, labels)
     26             loss_size.backward()
     27             optimizer.step()

~\Documents\Anaconda3\lib\site-packages\torch\nn\modules\module.py in
__call__(self, *input, **kwargs)
    530             result = self._slow_forward(*input, **kwargs)
    531         else:
--> 532             result = self.forward(*input, **kwargs)
    533         for hook in self._forward_hooks.values():
    534             hook_result = hook(self, input, result)

~\Documents\Anaconda3\lib\site-packages\torch\nn\modules\loss.py in
forward(self, input, target)
    914     def forward(self, input, target):
    915         return F.cross_entropy(input, target, weight=self.weight,
--> 916                                ignore_index=self.ignore_index, reduction=self.reduction)
    917
    918

~\Documents\Anaconda3\lib\site-packages\torch\nn\functional.py in
cross_entropy(input, target, weight, size_average, ignore_index,
reduce, reduction)    2019     if size_average is not None or reduce
is not None:    2020         reduction =
_Reduction.legacy_get_string(size_average, reduce)
-> 2021     return nll_loss(log_softmax(input, 1), target, weight, None, ignore_index, None, reduction)    2022     2023

~\Documents\Anaconda3\lib\site-packages\torch\nn\functional.py in
nll_loss(input, target, weight, size_average, ignore_index, reduce,
reduction)    1836                          .format(input.size(0),
target.size(0)))    1837     if dim == 2:
-> 1838         ret = torch._C._nn.nll_loss(input, target, weight, _Reduction.get_enum(reduction), ignore_index)    1839     elif dim == 4:    1840         ret = torch._C._nn.nll_loss2d(input, target,
weight, _Reduction.get_enum(reduction), ignore_index)
IndexError: Target 2 is out of bounds.

  IndexError: Target 2 is out of bounds.


title: AttributeError: 'list' object has no attribute 'dim' when predicting in pytorch

body:
I'm currently loading in a model and 11 input values. Then I'm sending those 11 values into a tensor and attempting to predict outputs.
Here is my code:

# coding: utf-8

# In[5]:

import torch
import torchvision
from torchvision import transforms, datasets
import torch.nn as nn
import torch.nn.functional as F
import torch.utils.data as utils
import numpy as np

data_np = np.loadtxt('input_preds.csv', delimiter=',')

train_ds = utils.TensorDataset(torch.tensor(data_np, dtype=torch.float32).view(-1,11))

trainset = torch.utils.data.DataLoader(train_ds, batch_size=1, shuffle=True)

# setting device on GPU if available, else CPU, replace .cuda() with .to(device)
device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')

class Net(nn.Module):
    def __init__(self):
        super().__init__()
        #self.bn = nn.BatchNorm2d(11)
        self.fc1 = nn.Linear(11, 22)
        self.fc2 = nn.Linear(22, 44)
        self.fc3 = nn.Linear(44, 22)
        self.fc4 = nn.Linear(22, 11)

    def forward(self, x):
        #x = x.view(-1, 11)
        x = F.relu(self.fc1(x))
        x = F.relu(self.fc2(x))
        x = F.relu(self.fc3(x))
        x = self.fc4(x)
        #return F.log_softmax(x, dim=1)
        return x
model1 = torch.load('./1e-2')
model2 = torch.load('./1e-3')

for data in trainset:
        X = data
        X = X

        output = model1(X).to(device)
        print(output)

However, I get this error

Traceback (most recent call last):
  File "inference.py", line 53, in <module>
    output = model1(X).to(device)
  File "C:\Users\Happy\Miniconda3\envs\torch\lib\site-packages\torch\nn\modules\module.py", line 477, in __call__
    result = self.forward(*input, **kwargs)
  File "inference.py", line 40, in forward
    x = F.relu(self.fc1(x))
  File "C:\Users\Happy\Miniconda3\envs\torch\lib\site-packages\torch\nn\modules\module.py", line 477, in __call__
    result = self.forward(*input, **kwargs)
  File "C:\Users\Happy\Miniconda3\envs\torch\lib\site-packages\torch\nn\modules\linear.py", line 55, in forward
    return F.linear(input, self.weight, self.bias)
  File "C:\Users\Happy\Miniconda3\envs\torch\lib\site-packages\torch\nn\functional.py", line 1022, in linear
    if input.dim() == 2 and bias is not None:
AttributeError: 'list' object has no attribute 'dim'

I've tried to convert the batch to a numpy array but that didn't help. How do I resolve this error? Thank you for your help.


title: What's the reason of the error ValueError: Expected more than 1 value per channel?

body:
reference fast.ai

github repository of fast.ai
(as the code elevates the library which is built on top of PyTorch)

  Please scroll the discussion a bit

I am running the following code, and get an error while trying to pass the data to the predict_array function

The code is failing when i am trying to use it to predict directly on a single image but it run's perfectly when that same image is in a test folder

from fastai.conv_learner import *
from planet import f2

PATH = 'data/shopstyle/'

metrics=[f2]
f_model = resnet34

def get_data(sz):
    tfms = tfms_from_model(f_model, sz, aug_tfms=transforms_side_on, max_zoom=1.05)
    return ImageClassifierData.from_csv(PATH, 'train', label_csv, tfms=tfms, suffix='.jpg', val_idxs=val_idxs, test_name='test')

def print_list(list_or_iterator):
        return "[" + ", ".join( str(x) for x in list_or_iterator) + "]"

label_csv = f'{PATH}prod_train.csv'
n = len(list(open(label_csv)))-1
val_idxs = get_cv_idxs(n)

sz = 64
data = get_data(sz)

print("Loading model...")
learn = ConvLearner.pretrained(f_model, data, metrics=metrics)
learn.load(f'{sz}')
#learn.load("tmp")

print("Predicting...")
learn.precompute=False
trn_tfms, val_tfrms = tfms_from_model(f_model, sz)
#im = val_tfrms(open_image(f'{PATH}valid/4500132.jpg'))
im = val_tfrms(np.array(PIL.Image.open(f'{PATH}valid/4500132.jpg')))
preds = learn.predict_array(im[None])
p=list(zip(data.classes, preds))
print("predictions = " + print_list(p))

Here's the Traceback I am Getting

  Traceback (most recent call last):
  File "predict.py", line 34, in <module>
    preds = learn.predict_array(im[None])
  File "/home/ubuntu/fastai/courses/dl1/fastai/learner.py", line 266, in predict_array
    def predict_array(self, arr): return to_np(self.model(V(T(arr).cuda())))
  File "/home/ubuntu/src/anaconda3/envs/fastai/lib/python3.6/site-packages/torch/nn/modules/module.py", line 325, in __call__
    result = self.forward(*input, **kwargs)
  File "/home/ubuntu/src/anaconda3/envs/fastai/lib/python3.6/site-packages/torch/nn/modules/container.py", line 67, in forward
    input = module(input)
  File "/home/ubuntu/src/anaconda3/envs/fastai/lib/python3.6/site-packages/torch/nn/modules/module.py", line 325, in __call__
    result = self.forward(*input, **kwargs)
  File "/home/ubuntu/src/anaconda3/envs/fastai/lib/python3.6/site-packages/torch/nn/modules/batchnorm.py", line 37, in forward
    self.training, self.momentum, self.eps)
  File "/home/ubuntu/src/anaconda3/envs/fastai/lib/python3.6/site-packages/torch/nn/functional.py", line 1011, in batch_norm
    raise ValueError('Expected more than 1 value per channel when training, got input size {}'.format(size))
ValueError: Expected more than 1 value per channel when training, got input size [1, 1024]

Things I have Tried

np.expand_dims(IMG,axis=0) or image = image[..., np.newaxis]
Tried a different way of reading the image

img = cv2.imread(img_path)
img = cv2.resize(img, dsize = (200,200))
img = np.einsum('ijk->kij', img)
img = np.expand_dims(img, axis =0)
img = torch.from_numpy(img)
learn.model(Variable(img.float()).cuda())

BTW the error still remains

ValueError: Expected more than 1 value per channel when training, got input size [1, 1024]

Can't find any reference in The Google search also..


title: Pytorch speed comparison - GPU slower than CPU

body:
I was trying to find out if GPU tensor operations are actually faster than CPU ones. So, I wrote this particular code below to implement a simple 2D addition of CPU tensors and GPU cuda tensors successively to see the speed difference:

import torch
import time

###CPU
start_time = time.time()
a = torch.ones(4,4)
for _ in range(1000000):
    a += a
elapsed_time = time.time() - start_time

print('CPU time = ',elapsed_time)

###GPU
start_time = time.time()
b = torch.ones(4,4).cuda()
for _ in range(1000000):
    b += b
elapsed_time = time.time() - start_time

print('GPU time = ',elapsed_time)

To my surprise, the CPU time was 0.93 sec and the GPU time was as high as 63 seconds. Am I doing the cuda tensor operation properly or is the concept of cuda tensors works faster only in very highly complex operations, like in neural networks?

Note: My GPU is NVIDIA 940MX and torch.cuda.is_available() call returns True.


title: How to access the network weights while using PyTorch 'nn.Sequential'?

body:
I'm building a neural network and I don't know how to access the model weights for each layer.

I've tried

model.input_size.weight

Code:

input_size = 784
hidden_sizes = [128, 64]
output_size = 10

# Build a feed-forward network
model = nn.Sequential(nn.Linear(input_size, hidden_sizes[0]),
                      nn.ReLU(),
                      nn.Linear(hidden_sizes[0], hidden_sizes[1]),
                      nn.ReLU(),
                      nn.Linear(hidden_sizes[1], output_size),
                      nn.Softmax(dim=1))

I expected to get the weights but I got

  'Sequential' object has no attribute 'input_size'


title: Change Tanh activation in LSTM to ReLU

body: The default non-linear activation function in LSTM class is tanh. I wish to use ReLU for my project. Browsing through the documentation and other resources, I'm unable to find a way to do this in a simple manner. The only way I could find was to define my own custom LSTMCell, but here the author says that custom LSTMCells don't support GPU acceleration capabilities(or has that changed since the article was published?). I need to use CUDA to speed up my training. Any help would be appreciated.


title: Google Colaboratory local runtime using local GPU

body:
I'm using Colaboratory and Pytorch to run a GAN that uses an unusual dataset, which is currently stored locally on my machine. To access these files I connected to a local runtime (as per https://research.google.com/colaboratory/local-runtimes.html). However, Colaboratory now uses my own GPU when running now, which it did not do on previous runs. I know this because current runs run much much slower, as they are using my GTX 1060 6GB instead of Colab's Tesla K80.

I checked this using

torch.cuda.get_device_name(0)

Which returns "GeForce GTX 1060 6G" when I am connected locally. This is the case even with Edit -> Notebook Settings -> Hardware Accelerator -> "GPU" selected.

However, when I am not connected locally, and instead use the (default) "Connect to hosted runtime" option,

torch.cuda.get_device_name(0) does return "Tesla K80".

I've had trouble uploading my dataset to Drive, as it is a large image dataset, and would like to carry on using the local runtime.

How do I use both the local runtime and Colab's amazing Tesla K80?
Any help would be much appreciated.


title: Any pytorch tools to monitor neural network's training?

body: Are there any tools to monitor network's training in PyTorch? Like tensorboard in tensorflow.


title: Pytorch ValueError: optimizer got an empty parameter list

body:
When trying to create a neural network and optimize it using Pytorch, I am getting

  ValueError: optimizer got an empty parameter list

Here is the code.

import torch.nn as nn
import torch.nn.functional as F
from os.path import dirname
from os import getcwd
from os.path import realpath
from sys import argv

class NetActor(nn.Module):

    def __init__(self, args, state_vector_size, action_vector_size, hidden_layer_size_list):
        super(NetActor, self).__init__()
        self.args = args

        self.state_vector_size = state_vector_size
        self.action_vector_size = action_vector_size
        self.layer_sizes = hidden_layer_size_list
        self.layer_sizes.append(action_vector_size)

        self.nn_layers = []
        self._create_net()

    def _create_net(self):
        prev_layer_size = self.state_vector_size
        for next_layer_size in self.layer_sizes:
            next_layer = nn.Linear(prev_layer_size, next_layer_size)
            prev_layer_size = next_layer_size
            self.nn_layers.append(next_layer)

    def forward(self, torch_state):
        activations = torch_state
        for i,layer in enumerate(self.nn_layers):
            if i != len(self.nn_layers)-1:
                activations = F.relu(layer(activations))
            else:
                activations = layer(activations)

        probs = F.softmax(activations, dim=-1)
        return probs

and then the call

        self.actor_nn = NetActor(self.args, 4, 2, [128])
        self.actor_optimizer = optim.Adam(self.actor_nn.parameters(), lr=args.learning_rate)

gives the very informative error

  ValueError: optimizer got an empty parameter list

I find it hard to understand what exactly in the network's definition makes the network have parameters.

I am following and expanding the example I found in Pytorch's tutorial code.

I can't really tell the difference between my code and theirs that makes mine think it has no parameters to optimize.

How to make my network have parameters like the linked example?


title: Pytorch 0.4.0: There are three ways to create tensors on CUDA device. Is there some difference between them?

body:
I failed in the third way. t3 is still on CPU. No idea why.

a = np.random.randn(1, 1, 2, 3)

t1 = torch.tensor(a)
t1 = t3.to(torch.device('cuda'))

t2 = torch.tensor(a)
t2 = t2.cuda()

t3 = torch.tensor(a, device=torch.device('cuda'))


title: How to save GPU memory usage in PyTorch

body:
In PyTorch I wrote a very simple CNN discriminator and trained it. Now I need to deploy it to make predictions. But the target machine has a small GPU memory and got out of memory error. So I think that I can set requires_grad = False to prevent PyTorch from storing the gradient values. However I didn't find it making any difference.

There are about 5 millions of parameters in my model. But when predicting a single batch of input, it consumes about 1.2GB of memory. I think there should be no need for such large memory.

The question is how to save GPU memory usage when I just want to use my model to make predictions?

Here is a demo, I use discriminator.requires_grad_ to disable/enable autograd of all parameters. But it seems to be no use.

import torch
import numpy as np
import torch.nn as nn
import torch.nn.functional as functional

from pynvml.smi import nvidia_smi
nvsmi = nvidia_smi.getInstance()

def getMemoryUsage():
    usage = nvsmi.DeviceQuery("memory.used")["gpu"][0]["fb_memory_usage"]
    return "%d %s" % (usage["used"], usage["unit"])

print("Before GPU Memory: %s" % getMemoryUsage())

class Discriminator(nn.Module):
    def __init__(self):
        super().__init__()
        # trainable layers
        # input: 2x256x256
        self.conv1 = nn.Conv2d(2, 8, 5, padding=2) # 8x256x256
        self.pool1 = nn.MaxPool2d(2) # 8x128x128
        self.conv2 = nn.Conv2d(8, 32, 5, padding=2) # 32x128x128
        self.pool2 = nn.MaxPool2d(2) # 32x64x64
        self.conv3 = nn.Conv2d(32, 96, 5, padding=2) # 96x64x64
        self.pool3 = nn.MaxPool2d(4) # 96x16x16
        self.conv4 = nn.Conv2d(96, 256, 5, padding=2) # 256x16x16
        self.pool4 = nn.MaxPool2d(4) # 256x4x4
        self.num_flat_features = 4096
        self.fc1 = nn.Linear(4096, 1024)
        self.fc2 = nn.Linear(1024, 256)
        self.fc3 = nn.Linear(256, 1)
        # loss function
        self.loss = nn.MSELoss()
        # other properties
        self.requires_grad = True
    def forward(self, x):
        y = x
        y = self.conv1(y)
        y = self.pool1(y)
        y = functional.relu(y)
        y = self.conv2(y)
        y = self.pool2(y)
        y = functional.relu(y)
        y = self.conv3(y)
        y = self.pool3(y)
        y = functional.relu(y)
        y = self.conv4(y)
        y = self.pool4(y)
        y = functional.relu(y)
        y = y.view((-1,self.num_flat_features))
        y = self.fc1(y)
        y = functional.relu(y)
        y = self.fc2(y)
        y = functional.relu(y)
        y = self.fc3(y)
        y = torch.sigmoid(y)
        return y
    def predict(self, x, score_th=0.5):
        if len(x.shape) == 3:
            singlebatch = True
            x = x.view([1]+list(x.shape))
        else:
            singlebatch = False
        y = self.forward(x)
        label = (y > float(score_th))
        if singlebatch:
            y = y.view(list(y.shape)[1:])
        return label, y
    def requires_grad_(self, requires_grad=True):
        for parameter in self.parameters():
            parameter.requires_grad_(requires_grad)
        self.requires_grad = requires_grad

x = torch.cuda.FloatTensor(np.zeros([2, 256, 256]))
discriminator = Discriminator()
discriminator.to("cuda:0")

# comment/uncomment this line to make difference
discriminator.requires_grad_(False)

discriminator.predict(x)

print("Requires grad", discriminator.requires_grad)
print("After GPU Memory: %s" % getMemoryUsage())

By comment out the line discriminator.requires_grad_(False), I got output:

Before GPU Memory: 6350MiB
Requires grad True
After GPU Memory: 7547MiB

While by uncomment the line, I got:

Before GPU Memory: 6350MiB
Requires grad False
After GPU Memory: 7543MiB


title: How to get the output from a specific layer from a PyTorch model?

body: How to extract the features from a specific layer from a pre-trained PyTorch model (such as ResNet or VGG), without doing a forward pass again?