title: RuntimeError: Found dtype Double but expected Float Exception raised from compute_types at ...aten/src/ATen/native/TensorIterator.cpp:183

body:
For the following code block:
##torch.autograd.set_detect_anomaly(True)
network = Network()
network.cuda()

criterion = nn.MSELoss()
optimizer = optim.Adam(network.parameters(), lr=0.0001)

loss_min = np.inf
num_epochs = 10

start_time = time.time()
for epoch in range(1,num_epochs+1):

    loss_train = 0
    loss_valid = 0
    running_loss = 0

    network.train()
    for step in range(1,len(train_loader)+1):

        ##images, landmarks = next(iter(train_loader))
        ##print(type(images))

        batch = next(iter(train_loader))
        images, landmarks = batch['image'], batch['landmarks']

        images = images.cuda()
        landmarks = landmarks.view(landmarks.size(0),-1).cuda()

        predictions = network(images)

        # clear all the gradients before calculating them
        optimizer.zero_grad()

        # find the loss for the current step
        loss_train_step = criterion(predictions, landmarks)

        print("type(loss_train_step) is: ", type(loss_train_step))

        print("loss_train_step.dtype is: ",loss_train_step.dtype)

        ##loss_train_step = loss_train_step.to(torch.float32)

        # calculate the gradients
        loss_train_step.backward()

        # update the parameters
        optimizer.step()

        loss_train += loss_train_step.item()
        running_loss = loss_train/step

        print_overwrite(step, len(train_loader), running_loss, 'train')

    network.eval()
    with torch.no_grad():

        for step in range(1,len(valid_loader)+1):

            batch = next(iter(train_loader))
            images, landmarks = batch['image'], batch['landmarks']
            images = images.cuda()
            landmarks = landmarks.view(landmarks.size(0),-1).cuda()

            predictions = network(images)

            # find the loss for the current step
            loss_valid_step = criterion(predictions, landmarks)

            loss_valid += loss_valid_step.item()
            running_loss = loss_valid/step

            print_overwrite(step, len(valid_loader), running_loss, 'valid')

    loss_train /= len(train_loader)
    loss_valid /= len(valid_loader)

    print('\n--------------------------------------------------')
    print('Epoch: {}  Train Loss: {:.4f}  Valid Loss: {:.4f}'.format(epoch, loss_train, loss_valid))
    print('--------------------------------------------------')

    if loss_valid < loss_min:
        loss_min = loss_valid
        torch.save(network.state_dict(), '/content/face_landmarks.pth')
        print("\nMinimum Validation Loss of {:.4f} at epoch {}/{}".format(loss_min, epoch, num_epochs))
        print('Model Saved\n')

print('Training Complete')
print("Total Elapsed Time : {} s".format(time.time()-start_time))

The error is:
type(loss_train_step) is:  <class 'torch.Tensor'>
loss_train_step.dtype is:  torch.float64

---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-23-0b0a62a9f8d8> in <module>
     43
     44         # calculate the gradients
---> 45         loss_train_step.backward()
     46
     47         # update the parameters

~/anaconda3/lib/python3.7/site-packages/torch/tensor.py in backward(self, gradient, retain_graph, create_graph)
    183                 products. Defaults to ``False``.
    184         """
--> 185         torch.autograd.backward(self, gradient, retain_graph, create_graph)
    186
    187     def register_hook(self, hook):

~/anaconda3/lib/python3.7/site-packages/torch/autograd/__init__.py in backward(tensors, grad_tensors, retain_graph, create_graph, grad_variables)
    125     Variable._execution_engine.run_backward(
    126         tensors, grad_tensors, retain_graph, create_graph,
--> 127         allow_unreachable=True)  # allow_unreachable flag
    128
    129

RuntimeError: Found dtype Double but expected Float
Exception raised from compute_types at /opt/conda/conda-bld/pytorch_1595629403081/work/aten/src/ATen/native/TensorIterator.cpp:183 (most recent call first):
frame #0: c10::Error::Error(c10::SourceLocation, std::string) + 0x4d (0x7fa4ec61c77d in /home/mona/anaconda3/lib/python3.7/site-packages/torch/lib/libc10.so)
frame #1: at::TensorIterator::compute_types(at::TensorIteratorConfig const&) + 0x259 (0x7fa51f673ca9 in /home/mona/anaconda3/lib/python3.7/site-packages/torch/lib/libtorch_cpu.so)
frame #2: at::TensorIterator::build(at::TensorIteratorConfig&) + 0x6b (0x7fa51f67744b in /home/mona/anaconda3/lib/python3.7/site-packages/torch/lib/libtorch_cpu.so)
frame #3: at::TensorIterator::TensorIterator(at::TensorIteratorConfig&) + 0xdd (0x7fa51f677abd in /home/mona/anaconda3/lib/python3.7/site-packages/torch/lib/libtorch_cpu.so)
frame #4: at::native::mse_loss_backward_out(at::Tensor&, at::Tensor const&, at::Tensor const&, at::Tensor const&, long) + 0x18a (0x7fa51f4dc71a in /home/mona/anaconda3/lib/python3.7/site-packages/torch/lib/libtorch_cpu.so)
frame #5: <unknown function> + 0xd1d610 (0x7fa4ed79f610 in /home/mona/anaconda3/lib/python3.7/site-packages/torch/lib/libtorch_cuda.so)
frame #6: at::native::mse_loss_backward(at::Tensor const&, at::Tensor const&, at::Tensor const&, long) + 0x90 (0x7fa51f4d9140 in /home/mona/anaconda3/lib/python3.7/site-packages/torch/lib/libtorch_cpu.so)
frame #7: <unknown function> + 0xd1d6b0 (0x7fa4ed79f6b0 in /home/mona/anaconda3/lib/python3.7/site-packages/torch/lib/libtorch_cuda.so)
frame #8: <unknown function> + 0xd3f936 (0x7fa4ed7c1936 in /home/mona/anaconda3/lib/python3.7/site-packages/torch/lib/libtorch_cuda.so)
frame #9: at::mse_loss_backward(at::Tensor const&, at::Tensor const&, at::Tensor const&, long) + 0x119 (0x7fa51f99bda9 in /home/mona/anaconda3/lib/python3.7/site-packages/torch/lib/libtorch_cpu.so)
frame #10: <unknown function> + 0x2b5e8c9 (0x7fa5215f48c9 in /home/mona/anaconda3/lib/python3.7/site-packages/torch/lib/libtorch_cpu.so)
frame #11: <unknown function> + 0x7f60d6 (0x7fa51f28c0d6 in /home/mona/anaconda3/lib/python3.7/site-packages/torch/lib/libtorch_cpu.so)
frame #12: at::mse_loss_backward(at::Tensor const&, at::Tensor const&, at::Tensor const&, long) + 0x119 (0x7fa51f99bda9 in /home/mona/anaconda3/lib/python3.7/site-packages/torch/lib/libtorch_cpu.so)
frame #13: torch::autograd::generated::MseLossBackward::apply(std::vector<at::Tensor, std::allocator<at::Tensor> >&&) + 0x1af (0x7fa52153052f in /home/mona/anaconda3/lib/python3.7/site-packages/torch/lib/libtorch_cpu.so)
frame #14: <unknown function> + 0x30d1017 (0x7fa521b67017 in /home/mona/anaconda3/lib/python3.7/site-packages/torch/lib/libtorch_cpu.so)
frame #15: torch::autograd::Engine::evaluate_function(std::shared_ptr<torch::autograd::GraphTask>&, torch::autograd::Node*, torch::autograd::InputBuffer&, std::shared_ptr<torch::autograd::ReadyQueue> const&) + 0x1400 (0x7fa521b62860 in /home/mona/anaconda3/lib/python3.7/site-packages/torch/lib/libtorch_cpu.so)
frame #16: torch::autograd::Engine::thread_main(std::shared_ptr<torch::autograd::GraphTask> const&) + 0x451 (0x7fa521b63401 in /home/mona/anaconda3/lib/python3.7/site-packages/torch/lib/libtorch_cpu.so)
frame #17: torch::autograd::Engine::thread_init(int, std::shared_ptr<torch::autograd::ReadyQueue> const&, bool) + 0x89 (0x7fa521b5b579 in /home/mona/anaconda3/lib/python3.7/site-packages/torch/lib/libtorch_cpu.so)
frame #18: torch::autograd::python::PythonEngine::thread_init(int, std::shared_ptr<torch::autograd::ReadyQueue> const&, bool) + 0x4a (0x7fa5407f699a in /home/mona/anaconda3/lib/python3.7/site-packages/torch/lib/libtorch_python.so)
frame #19: <unknown function> + 0xc9067 (0x7fa55cb45067 in /home/mona/anaconda3/lib/python3.7/site-packages/zmq/backend/cython/../../../../.././libstdc++.so.6)
frame #20: <unknown function> + 0x9609 (0x7fa55fc5c609 in /lib/x86_64-linux-gnu/libpthread.so.0)
frame #21: clone + 0x43 (0x7fa55fb83103 in /lib/x86_64-linux-gnu/libc.so.6)

Even after I converted to float32, I get the same error:
loss_train_step = loss_train_step.to(torch.float32)
Also,
num_classes = 68 * 2 #68 coordinates X and Y flattened

class Network(nn.Module):
    def __init__(self,num_classes=136):
        super().__init__()
        self.model_name = 'resnet18'
        self.model = models.resnet18()
        self.model.fc = nn.Linear(self.model.fc.in_features, num_classes)

    def forward(self, x):
        x = x.float()
        out = self.model(x)
        return out


Has anyone been able to resolve PyTorch specific inspection issues in PyCharm? Previous posts for non-PyTorch related issues suggest upgrading PyCharm, but I'm currently at the latest version. One option is to of course disable certain inspections entirely, but I'd rather avoid that.

Example: torch.LongTensor(x) gives me "Unexpected argument...", whereas both call signatures (with and without x) are both supported.


title: Why does nn.CrossEntropyLoss throw "TypeError: iteration over a 0-d tensor" when I verify inputs to be non-0-dimensional?

body:
I am using PyTorch version 1.5.0.
When I pass an input torch tensor of size [8,21,400,400] with a target of size [8,400,400], the program raises a TypeError: iteration over a 0-d tensor. However, the dimensions of the arguments are 4 and 3 respectively.

What could be causing this error?

The traceback points to torch\tensor.py's iter function.

Traceback (most recent call last):
  File "train.py", line 108, in <module>
    loss, accuracy = lossLayer(pred2, targetBatch)
  File "C:\Users\PC\anaconda3\lib\site-packages\torch\tensor.py", line 462, in __iter__
    raise TypeError('iteration over a 0-d tensor')
TypeError: iteration over a 0-d tensor


title: transforms.Normalize() between 0 and 1 when using Lab

body:
which mean, std should I use when I want to normalize a tensor to a range of 0 to 1? But I work with images with 2 channels (a, b channel -> -128 to 127) only instead of 3 channels. Thus, the usual mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225] will not do the job.
transform = transforms.Compose([
   transforms.ToTensor(),
   transforms.Normalize(mean=[0.485, 0.456, 0.406],
                        std=[0.229, 0.224, 0.225])
])

This leads to this error message:

tensor.sub_(mean[:, None, None]).div_(std[:, None, None])
RuntimeError: The size of tensor a (2) must match the size of tensor b
(3) at non-singleton dimension 0


title: Error when running (open-mmlab) C:\mmdetection>python setup.py develop - raise RuntimeError(message)

body:
I get the following error in the last stage of the mmdetection install from
https://github.com/open-mmlab/mmdetection/blob/master/docs/install.md

when running

C:\...\mmdetection\python setup.py develop
running develop
running egg_info
writing mmdet.egg-info\PKG-INFO
writing dependency_links to mmdet.egg-info\dependency_links.txt
writing requirements to mmdet.egg-info\requires.txt
writing top-level names to mmdet.egg-info\top_level.txt
reading manifest file 'mmdet.egg-info\SOURCES.txt'
writing manifest file 'mmdet.egg-info\SOURCES.txt'
running build_ext
C:\...\torch\utils\cpp_extension.py:237: UserWarning: Error checking compiler version for cl: [WinError 2] The system cannot find the file specified

warnings.warn('Error checking compiler version for {}: {}'.format(compiler, error))

building 'mmdet.ops.utils.compiling_info' extension
Emitting ninja build file C:\...\mmdetection\build\temp.win-amd64-3.7\Release\build.ninja...
Compiling objects...
Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N)

FAILED: C:/.../mmdetection/build/temp.win-amd64-3.7/Release/mmdet/ops/utils/src/compiling_info.obj
cl /showIncludes /nologo /Ox /W3 /GL /DNDEBUG /MD /MD /wd4819 /EHsc -DWITH_CUDA -IC:\torch\include -

CreateProcess failed: The system cannot find the file specified.
ninja: build stopped: subcommand failed.

Traceback (most recent call last):
  File "C:...\torch\utils\cpp_extension.py", line 1400, in _run_ninja_build
    check=True)
  File "C:\anaconda3\envs\open-mmlab\lib\subprocess.py", line 512, in run
    output=stdout, stderr=stderr)
subprocess.CalledProcessError: Command '['ninja', '-v']' returned non-zero exit status 1.

During handling of the above exception, another exception occurred:
Traceback (most recent call last):
  File "setup.py", line 300, in <module>
    zip_safe=False)
  File "C:...\setuptools\__init__.py", line 144, in setup
    return distutils.core.setup(**attrs)
  File "C:\...\anaconda3\envs\open-mmlab\lib\distutils\core.py", line 148, in
  File "C:\...\torch\utils\cpp_extension.py", line 1140, in _write_ninja_file_and_compile_objects
    ..............
    error_prefix='Error compiling objects for extension')
  File "C:\...\torch\utils\cpp_extension.py", line 1413, in _run_ninja_build

raise RuntimeError(message)
RuntimeError: Error compiling objects for extension

I was wondering if anyone knows what could be the cause of this error message ?

I am using windows 10


title: Applying Kullback-Leibler (aka kl divergence) element-wise in Pytorch

body:
I have two tensors named x_t, x_k with follwing shapes NxHxW and KxNxHxW respectively, where K, is the number of autoencoders used to reconstruct x_t (if you have no idea what is this, assume they're K different nets aiming to predict x_t, this probably has nothing to do with the question anyways) N is batch size, H matrix height, W matrix width.

I'm trying to apply Kullback-Leibler divergence algorithm to both tensors (after broadcasting x_t as x_k along the Kth dimension) using Pytorch's nn.functional.kl_div method.

However, it does not seem to be working as I expected. I'm looking to calcualte the kl_div between each observation in x_t and x_k resulting in a tensor of size KxN (i.e., kl_div of each observation for each K autoencoder).

The actual output is a single value if I use the reduction argument, and the same tensor size (i.e., KxNxHxW) if I do not use it.

Has anyone tried something similar?

Reproducible example:

import torch
import torch.nn.functional as F
#                  K   N   H  W
x_t = torch.randn(    10, 5, 5)
x_k = torch.randn( 3, 10, 5, 5)

x_broadcasted = x_t.expand_as(x_k)

loss = F.kl_div(x_t, x_k, reduction="none") # or "batchmean", or there are many options


title: Pixel RNN Pytorch Implementation

body: I am trying to implement Pixel RNN in pytorch, but I cannot seem to find any documentation on this. The main parts of Pixel RNN are Row LSTM and BiDiagonal LSTM, so I am looking for some code of these algorithms to better understand what they are doing. Specifically, I am confused as to these algorithms calculate one row and diagonal at once, respectively. Any help would be much appreciated.


title: Image similarity using Tensorflow or PyTorch

body:
I want to compare two images for similarity. Since my purpose is to match a given image against a massive collection of images, I want to run the comparisons on GPU.

I came across tf.image.ssim and tf.image.psnr functions but I am unable to find and working examples only. The solutions in PyTorch is also appreciated. Since I don't have a good understanding of CUDA and C language, I am hesitant to try kernels in PyCuda.

Will it be helpful in terms of processing if I read the entire image collection and store as Tensorflow Records for future processing?

Any guidance or solution, greatly appreciated. Thank you.

Edit:- I am matching images of same size only. I don't want to do mere histogram match. I want to do SSIM or PSNR implementation for image similarity. So, I am assuming it would be similar in color, content etc


title: Using flatten in pytorch v1.0 Sequential module

body:
Due to my CUDA version being 8, I am using torch 1.0.0

I need to use the Flatten layer for Sequential model. Here's my code :

import torch
import torch.nn as nn
import torch.nn.functional as F
print(torch.__version__)
# 1.0.0
from collections import OrderedDict

layers = OrderedDict()
layers['conv1'] = nn.Conv2d(1, 5, 3)
layers['relu1'] = nn.ReLU()
layers['conv2'] = nn.Conv2d(5, 1, 3)
layers['relu2'] = nn.ReLU()
layers['flatten'] = nn.Flatten()
layers['linear1'] = nn.Linear(3600, 1)
model = nn.Sequential(
layers
).cuda()

It gives me the following error:

---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
<ipython-input-38-080f7c5f5037> in <module>
      6 layers['conv2'] = nn.Conv2d(5, 1, 3)
      7 layers['relu2'] = nn.ReLU()
----> 8 layers['flatten'] = nn.Flatten()
      9 layers['linear1'] = nn.Linear(3600, 1)
     10 model = nn.Sequential(

AttributeError: module 'torch.nn' has no attribute 'Flatten'

How can I flatten my conv layer output in pytorch 1.0.0?


title: How do I load the CelebA dataset on Google Colab, using torch vision, without running out of memory?

body:
I am following a tutorial on DCGAN. Whenever I try to load the CelebA dataset, torchvision uses up all my run-time's memory(12GB) and the runtime crashes. Am looking for ways on how I can load and apply transformations to the dataset without hogging my run-time's resources.
To Reproduce
Here is the part of the code that is causing issues.
# Root directory for the dataset
data_root = 'data/celeba'
# Spatial size of training images, images are resized to this size.
image_size = 64

celeba_data = datasets.CelebA(data_root,
                              download=True,
                              transform=transforms.Compose([
                                  transforms.Resize(image_size),
                                  transforms.CenterCrop(image_size),
                                  transforms.ToTensor(),
                                  transforms.Normalize(mean=[0.5, 0.5, 0.5],
                                                       std=[0.5, 0.5, 0.5])
                              ]))

The full notebook can be found here
Environment

PyTorch version: 1.7.1+cu101

Is debug build: False

CUDA used to build PyTorch: 10.1

ROCM used to build PyTorch: N/A

OS: Ubuntu 18.04.5 LTS (x86_64)

GCC version: (Ubuntu 7.5.0-3ubuntu1~18.04) 7.5.0

Clang version: 6.0.0-1ubuntu2 (tags/RELEASE_600/final)

CMake version: version 3.12.0

Python version: 3.6 (64-bit runtime)

Is CUDA available: True

CUDA runtime version: 10.1.243

GPU models and configuration: GPU 0: Tesla T4

Nvidia driver version: 418.67

cuDNN version: /usr/lib/x86_64-linux-gnu/libcudnn.so.7.6.5

HIP runtime version: N/A

MIOpen runtime version: N/A

Versions of relevant libraries:

[pip3] numpy==1.19.4
[pip3] torch==1.7.1+cu101
[pip3] torchaudio==0.7.2
pip3] torchsummary==1.5.1
[pip3] torchtext==0.3.1
[pip3] torchvision==0.8.2+cu101
[conda] Could not collect

Additional Context
Some of the things I have tried are:

Downloading and loading the dataset on seperate lines. e.g:

# Download the dataset only
datasets.CelebA(data_root, download=True)
# Load the dataset here
celeba_data = datasets.CelebA(data_root, download=False, transforms=...)

Using the ImageFolder dataset class instead of the CelebA class. e.g:

# Download the dataset only
datasets.CelebA(data_root, download=True)
# Load the dataset using the ImageFolder class
celeba_data = datasets.ImageFolder(data_root, transforms=...)

The memory problem is still persistent in either of the cases.


title: What is the meaning of the second output of Huggingface's Bert?

body:
Using the vanilla configuration of base BERT model in the huggingface implementation, I get a tuple of length 2.

import torch

import transformers
from transformers import AutoModel,AutoTokenizer

bert_name="bert-base-uncased"
tokenizer = AutoTokenizer.from_pretrained(bert_name)
BERT = AutoModel.from_pretrained(bert_name)

e=tokenizer.encode('I am hoping for the best', add_special_tokens=True)

q=BERT(torch.tensor([e]))

print (len(q)) #Output: 2

The first element is what I expect to receive - the 768 dimension embedding of each input token.

print (e) #Output : [101, 1045, 2572, 5327, 2005, 1996, 2190, 102]
print (q[0].shape) #Output : torch.Size([1, 8, 768])

But what is the second element in the tuple?

print (q[1].shape) # torch.Size([1, 768])

It has the same size as the encoding of each token.
But what is it?

Maybe a copy of the [CLS] token, a representation for the classification of the entire encoded text?

Let's check.

a= q[0][:,0,:]
b=q[1]

print (torch.eq(a,b)) #Output : Tensor([[False, False, False, .... False]])

Nope!

What about a copy the embedding of the last token (for whatever reason)?

c= q[0][:,-1,:]
b=q[1]

print (torch.eq(a,c)) #Output : Tensor([[False, False, False, .... False]])

So, also not that.

The documentation talks about how changing the config can result in more tuple elements (like hidden states), but I did not find any description of this "mysterious" tuple element outputted by the default configuration.

Any ideas as to what is it and what is its usage?


title: ValueError: too many values to unpack while using torch tensors

body:
For a project on neural networks, I am using Pytorch and am working with the EMNIST dataset.

The code that is already given loads in the dataset:

train_dataset = dsets.MNIST(root='./data',
                            train=True,
                            transform=transforms.ToTensor(),
                            download=True)

And prepares it:

train_loader = torch.utils.data.DataLoader(dataset=train_dataset,
                                           batch_size=batch_size,
                                           shuffle=True)

Then, when all the configurations of the network are defined, there is a for loop to train the model per epoch:

 for i, (images, labels) in enumerate(train_loader):

In the example code this works fine.

For my task, I am given a dataset that I load as follows:

emnist = scipy.io.loadmat("DIRECTORY/emnist-letters.mat")

data = emnist ['dataset']
X_train = data ['train'][0, 0]['images'][0, 0]
y_train = data ['train'][0, 0]['labels'][0, 0]

Then, I create the train_dataset as follows:

train_dataset = np.concatenate((X_train, y_train), axis = 1)
train_dataset = torch.from_numpy(train_dataset)

And use the same step to prepare it:

train_loader = torch.utils.data.DataLoader(dataset=train_dataset,
                                           batch_size=batch_size,
                                           shuffle=True)

However, when I try to use the same loop as before:

for i, (images, labels) in enumerate(train_loader):

I get the following error:

ValueError: too many values to unpack (expected 2)

Who knows what I can do so that I can train my dataset with this loop?


title: AttributeError on variable input of custom loss function in PyTorch

body:
I've made a custom loss function to compute cross-entropy (CE) for a multi-output multi-label problem. Within the class, I want to set the target variable I'm feeding to not require a gradient. I do this within the forward function using a pre-defined function (taken from pytorch source code) outside the class.

    def _assert_no_grad(variable):
        assert not variable.requires_grad

    def forward(self, predicted, target):
        """
        Computes cross entropy between targets and predictions.
        """
        # No gradient over target
        _assert_no_grad(target)

        # Define variables
        p = predicted.clamp(0.01, 0.99)
        t = target.float()

        #Compute cross entropy
        h1 = p.log()*t
        h2 = (1-t)*((1-p).log())
        ce = torch.add(h1, h2)
        ce_out = torch.mean(ce, 1)
        ce_out = torch.mean(ce_out, 0)

        # Save for backward step
        self.save_for_backward(ce_out)

At this point when I run the code in a batched for-loop (see below), I get the following error:

  AttributeError: 'torch.FloatTensor' object has no attribute 'requires_grad'

It seems simple enough as we should be passing a torch.autograd.Variable, however I am already doing this as can be seen in the snippet below.

for t in range(50):

    print('Epoch {}'.format(t))
    if t > 0:
        print('Loss ->', loss)

    for batch_idx, (x_batch, y_batch) in enumerate(train_loader):
        # Wrap in Variable
        x_in, target = Variable(x_batch), Variable(y_batch)

        predicted = model(x_in)

        # Compute and print loss
        loss = criterion(predicted, target)

        # Zero gradients, perform a backward pass, and update the weights.
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

To add a note, my final goal is to generate a class which behaves like BCELoss except for multiple labels and not just binary. I feel like I've already scoured the entire PyTorch docs and primarily been using this and some forum entries.
http://pytorch.org/docs/master/notes/extending.html

So


My input label size is torch.size([30, 2, 96, 96, 96])

My labels size is torch.size([30, 96, 96, 96]), I am feeding them to my loss function which goes like this :

loss = F.binary_cross_entropy(F.sigmoid(output),labels,torch.FloatTensor(CLASS_WEIGHTS).cuda())

When I run this, I however get

Value error:Target and input must have the same number of elements.target nelement(26542080) != input nelement(53084160)

I am a little confused here. I get the input value is twice the target value because it multiplies [30,96,96] by the number of classes, but I am not sure why it is so, and how to rectify it. Any suggestions will be helpful, thanks in advance.


title: pytorch cannot install using anaconda prompt

body:
C:\Users\User>pip install pytorch

When I run this command anaconda prompt display this error.

Collecting pytorch
  Downloading https://files.pythonhosted.org/packages/ee/67/f403d4ae6e9cd74b546ee88cccdb29b8415a9c1b3d80aebeb20c9ea91d96/pytorch-1.0.2.tar.gz
Building wheels for collected packages: pytorch
  Building wheel for pytorch (setup.py) ... error
  ERROR: Complete output from command 'C:\Users\User\Anaconda3\python.exe' -u -c 'import setuptools, tokenize;__file__='"'"'C:\\Users\\User\\AppData\\Local\\Temp\\pip-install-yto7o19p\\pytorch\\setup.py'"'"';f=getattr(tokenize, '"'"'open'"'"', open)(__file__);code=f.read().replace('"'"'\r\n'"'"', '"'"'\n'"'"');f.close();exec(compile(code, __file__, '"'"'exec'"'"'))' bdist_wheel -d 'C:\Users\User\AppData\Local\Temp\pip-wheel-lqbohuk2' --python-tag cp37:
  ERROR: Traceback (most recent call last):
    File "<string>", line 1, in <module>
    File "C:\Users\User\AppData\Local\Temp\pip-install-yto7o19p\pytorch\setup.py", line 15, in <module>
      raise Exception(message)
  Exception: You tried to install "pytorch". The package named for PyTorch is "torch"
  ----------------------------------------
  ERROR: Failed building wheel for pytorch
  Running setup.py clean for pytorch
Failed to build pytorch
Installing collected packages: pytorch
  Running setup.py install for pytorch ... error
    ERROR: Complete output from command 'C:\Users\User\Anaconda3\python.exe' -u -c 'import setuptools, tokenize;__file__='"'"'C:\\Users\\User\\AppData\\Local\\Temp\\pip-install-yto7o19p\\pytorch\\setup.py'"'"';f=getattr(tokenize, '"'"'open'"'"', open)(__file__);code=f.read().replace('"'"'\r\n'"'"', '"'"'\n'"'"');f.close();exec(compile(code, __file__, '"'"'exec'"'"'))' install --record 'C:\Users\User\AppData\Local\Temp\pip-record-oah_3fo2\install-record.txt' --single-version-externally-managed --compile:
    ERROR: Traceback (most recent call last):
      File "<string>", line 1, in <module>
      File "C:\Users\User\AppData\Local\Temp\pip-install-yto7o19p\pytorch\setup.py", line 11, in <module>
        raise Exception(message)
    Exception: You tried to install "pytorch". The package named for PyTorch is "torch"
    ----------------------------------------
ERROR: Command "'C:\Users\User\Anaconda3\python.exe' -u -c 'import setuptools, tokenize;__file__='"'"'C:\\Users\\User\\AppData\\Local\\Temp\\pip-install-yto7o19p\\pytorch\\setup.py'"'"';f=getattr(tokenize, '"'"'open'"'"', open)(__file__);code=f.read().replace('"'"'\r\n'"'"', '"'"'\n'"'"');f.close();exec(compile(code, __file__, '"'"'exec'"'"'))' install --record 'C:\Users\User\AppData\Local\Temp\pip-record-oah_3fo2\install-record.txt' --single-version-externally-managed --compile" failed with error code 1 in C:\Users\User\AppData\Local\Temp\pip-install-yto7o19p\pytorch\

C:\Users\User>pip install torch

When I tried to install torch it gives me this error message

Collecting torch
  Downloading https://files.pythonhosted.org/packages/f8/02/880b468bd382dc79896eaecbeb8ce95e9c4b99a24902874a2cef0b562cea/torch-0.1.2.post2.tar.gz (128kB)
     |████████████████████████████████| 133kB 544kB/s
Requirement already satisfied: pyyaml in c:\users\user\anaconda3\lib\site-packages (from torch) (5.1.1)
Building wheels for collected packages: torch
  Building wheel for torch (setup.py) ... error
  ERROR: Complete output from command 'C:\Users\User\Anaconda3\python.exe' -u -c 'import setuptools, tokenize;__file__='"'"'C:\\Users\\User\\AppData\\Local\\Temp\\pip-install-5hkp2cz9\\torch\\setup.py'"'"';f=getattr(tokenize, '"'"'open'"'"', open)(__file__);code=f.read().replace('"'"'\r\n'"'"', '"'"'\n'"'"');f.close();exec(compile(code, __file__, '"'"'exec'"'"'))' bdist_wheel -d 'C:\Users\User\AppData\Local\Temp\pip-wheel-64iazehi' --python-tag cp37:
  ERROR: running bdist_wheel
  running build
  running build_deps
  Traceback (most recent call last):
    File "<string>", line 1, in <module>
    File "C:\Users\User\AppData\Local\Temp\pip-install-5hkp2cz9\torch\setup.py", line 265, in <module>
      description="Tensors and Dynamic neural networks in Python with strong GPU acceleration",
    File "C:\Users\User\Anaconda3\lib\site-packages\setuptools\__init__.py", line 145, in setup
      return distutils.core.setup(**attrs)
    File "C:\Users\User\Anaconda3\lib\distutils\core.py", line 148, in setup
      dist.run_commands()
    File "C:\Users\User\Anaconda3\lib\distutils\dist.py", line 966, in run_commands
      self.run_command(cmd)
    File "C:\Users\User\Anaconda3\lib\distutils\dist.py", line 985, in run_command
      cmd_obj.run()
    File "C:\Users\User\Anaconda3\lib\site-packages\wheel\bdist_wheel.py", line 192, in run
      self.run_command('build')
    File "C:\Users\User\Anaconda3\lib\distutils\cmd.py", line 313, in run_command
      self.distribution.run_command(command)
    File "C:\Users\User\Anaconda3\lib\distutils\dist.py", line 985, in run_command
      cmd_obj.run()
    File "C:\Users\User\Anaconda3\lib\distutils\command\build.py", line 135, in run
      self.run_command(cmd_name)
    File "C:\Users\User\Anaconda3\lib\distutils\cmd.py", line 313, in run_command
      self.distribution.run_command(command)
    File "C:\Users\User\Anaconda3\lib\distutils\dist.py", line 985, in run_command
      cmd_obj.run()
    File "C:\Users\User\AppData\Local\Temp\pip-install-5hkp2cz9\torch\setup.py", line 51, in run
      from tools.nnwrap import generate_wrappers as generate_nn_wrappers
  ModuleNotFoundError: No module named 'tools.nnwrap'
  ----------------------------------------
  ERROR: Failed building wheel for torch
  Running setup.py clean for torch
  ERROR: Complete output from command 'C:\Users\User\Anaconda3\python.exe' -u -c 'import setuptools, tokenize;__file__='"'"'C:\\Users\\User\\AppData\\Local\\Temp\\pip-install-5hkp2cz9\\torch\\setup.py'"'"';f=getattr(tokenize, '"'"'open'"'"', open)(__file__);code=f.read().replace('"'"'\r\n'"'"', '"'"'\n'"'"');f.close();exec(compile(code, __file__, '"'"'exec'"'"'))' clean --all:
  ERROR: running clean
  error: [Errno 2] No such file or directory: '.gitignore'
  ----------------------------------------
  ERROR: Failed cleaning build dir for torch
Failed to build torch
Installing collected packages: torch
  Running setup.py install for torch ... error
    ERROR: Complete output from command 'C:\Users\User\Anaconda3\python.exe' -u -c 'import setuptools, tokenize;__file__='"'"'C:\\Users\\User\\AppData\\Local\\Temp\\pip-install-5hkp2cz9\\torch\\setup.py'"'"';f=getattr(tokenize, '"'"'open'"'"', open)(__file__);code=f.read().replace('"'"'\r\n'"'"', '"'"'\n'"'"');f.close();exec(compile(code, __file__, '"'"'exec'"'"'))' install --record 'C:\Users\User\AppData\Local\Temp\pip-record-en00_2m3\install-record.txt' --single-version-externally-managed --compile:
    ERROR: running install
    running build_deps
    Traceback (most recent call last):
      File "<string>", line 1, in <module>
      File "C:\Users\User\AppData\Local\Temp\pip-install-5hkp2cz9\torch\setup.py", line 265, in <module>
        description="Tensors and Dynamic neural networks in Python with strong GPU acceleration",
      File "C:\Users\User\Anaconda3\lib\site-packages\setuptools\__init__.py", line 145, in setup
        return distutils.core.setup(**attrs)
      File "C:\Users\User\Anaconda3\lib\distutils\core.py", line 148, in setup
        dist.run_commands()
      File "C:\Users\User\Anaconda3\lib\distutils\dist.py", line 966, in run_commands
        self.run_command(cmd)
      File "C:\Users\User\Anaconda3\lib\distutils\dist.py", line 985, in run_command
        cmd_obj.run()
      File "C:\Users\User\AppData\Local\Temp\pip-install-5hkp2cz9\torch\setup.py", line 99, in run
        self.run_command('build_deps')
      File "C:\Users\User\Anaconda3\lib\distutils\cmd.py", line 313, in run_command
        self.distribution.run_command(command)
      File "C:\Users\User\Anaconda3\lib\distutils\dist.py", line 985, in run_command
        cmd_obj.run()
      File "C:\Users\User\AppData\Local\Temp\pip-install-5hkp2cz9\torch\setup.py", line 51, in run
        from tools.nnwrap import generate_wrappers as generate_nn_wrappers
    ModuleNotFoundError: No module named 'tools.nnwrap'
    ----------------------------------------
ERROR: Command "'C:\Users\User\Anaconda3\python.exe' -u -c 'import setuptools, tokenize;__file__='"'"'C:\\Users\\User\\AppData\\Local\\Temp\\pip-install-5hkp2cz9\\torch\\setup.py'"'"';f=getattr(tokenize, '"'"'open'"'"', open)(__file__);code=f.read().replace('"'"'\r\n'"'"', '"'"'\n'"'"');f.close();exec(compile(code, __file__, '"'"'exec'"'"'))' install --record 'C:\Users\User\AppData\Local\Temp\pip-record-en00_2m3\install-record.txt' --single-version-externally-managed --compile" failed with error code 1 in C:\Users\User\AppData\Local\Temp\pip-install-5hkp2cz9\torch\

Other packsges can install by using pip install command but pytorch and torch didn't work like that


title: Assign the new value to a tensor at specific indices

body:
This is a simple example.

Assume that I have an input tensor M. Now I have a tensor of indices of M with size 2 x 3 such as [[0, 1], [2,2], [0,1]] and a new array of values which is corresponding with the index tensor is [1, 2, 3]. I want to assign these values to the input M satisfying that the value is assigned to the element of M at index [0,1] will be the min value (1 in this example).

It means M[0,1] = 1 and M[2,2] = 2.

Can I do that by using some available functions in Pytorch without a loop?


title: PyTorch doesn't import after installing Anaconda

body:
I just installed PyTorch after installing Anaconda, and when I run iPython using Anaconda Python, it won't find PyTorch.

However I can verify PyTorch is in the pkgs directory of my anaconda folder.
What's going on?


title: Custom distance loss function in Pytorch?

body:
I want to implement the following distance loss function in pytorch. I was following this https://discuss.pytorch.org/t/custom-loss-functions/29387/4 thread from the pytorch forum

np.linalg.norm(output - target)
# where output.shape = [1, 2] and target.shape = [1, 2]

So I have implemented the loss function like this

def my_loss(output, target):
    loss = torch.tensor(np.linalg.norm(output.detach().numpy() - target.detach().numpy()))
    return loss

with this loss function, calling backwards gives runtime error

RuntimeError: element 0 of tensors does not require grad and does not have a grad_fn

My entire code looks like this

model = nn.Linear(2, 2)

x = torch.randn(1, 2)
target = torch.randn(1, 2)
output = model(x)

loss = my_loss(output, target)
loss.backward()   <----- Error here

print(model.weight.grad)

PS: I am aware of the pairwise loss of pytorch but due to some limitation of it, I have to implement it myself.

Following the pytorch source code I have tried the following,

class my_function(torch.nn.Module): # forgot to define backward()
    def forward(self, output, target):

        loss = torch.tensor(np.linalg.norm(output.detach().numpy() - target.detach().numpy()))
        return loss

model = nn.Linear(2, 2)
x = torch.randn(1, 2)
target = torch.randn(1, 2)
output = model(x)

criterion = my_function()

loss = criterion(output, target)

loss.backward()
print(model.weight.grad)

And I get the Run time error

RuntimeError: element 0 of tensors does not require grad and does not have a grad_fn

How can I implement the loss function correctly?


title: LSTM layer returns nan when fed by its own output in PyTorch

body:
I’m trying to generate time-series data with an LSTM and a Mixture Density Network as described in https://arxiv.org/pdf/1308.0850.pdf
Here is a link to my implementation: https://github.com/NeoVand/MDNLSTM
The repository contains a toy dataset to train the network.
On training, the LSTM layer returns nan for its hidden state after one iteration. A similar issue is reported here.
For your convenience, here is the code:

import torch
import torch.nn as nn
import torch.optim as optim
from torch.autograd import Variable
import torch.nn.functional as F
import matplotlib.pyplot as plt
import numpy as np
import numpy.random as npr

device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
ts = torch.load('LDS_Toy_Data.pt')

def detach(states):
    return [state.detach() for state in states]

class MDNLSTM(nn.Module):
    def __init__(self, d_obs, d_lat=2, n_gaussians=2, n_layers=1):
        super(MDNLSTM, self).__init__()

        self.d_obs = d_obs
        self.d_lat = d_lat
        self.n_gaussians = n_gaussians
        self.n_layers = n_layers

        self.lstm = nn.LSTM(d_obs, d_lat, n_layers, batch_first=True)
        self.fcPi = nn.Linear(d_lat, n_gaussians*d_obs)
        self.fcMu = nn.Linear(d_lat, n_gaussians*d_obs)
        self.fcSigma = nn.Linear(d_lat, n_gaussians*d_obs)

    def get_mixture_coef(self, y):
        time_steps = y.size(1)
        pi, mu, sigma = self.fcPi(y), self.fcMu(y), self.fcSigma(y)

        pi = pi.view(-1, time_steps, self.n_gaussians, self.d_obs)
        mu = mu.view(-1, time_steps, self.n_gaussians, self.d_obs)
        sigma = sigma.view(-1, time_steps, self.n_gaussians, self.d_obs)

        pi = F.softmax(pi, 2)
        sigma = torch.exp(sigma)
        return pi, mu, sigma

    def forward(self, x, h):
        y, (h, c) = self.lstm(x, h)
        #print(h)
        pi, mu, sigma = self.get_mixture_coef(y)
        return (pi, mu, sigma), (h, c)

    def init_hidden(self, bsz):
        return (torch.zeros(self.n_layers, bsz, self.d_lat).to(device),
                torch.zeros(self.n_layers, bsz, self.d_lat).to(device))

def mdn_loss_fn(y, pi, mu, sigma):
    m = torch.distributions.Normal(loc=mu, scale=sigma)
    loss = torch.exp(m.log_prob(y))
    loss = torch.sum(loss * pi, dim=2)
    loss = -torch.log(loss)
    return loss.mean()

def criterion(y, pi, mu, sigma):
    y = y.unsqueeze(2)
    return mdn_loss_fn(y, pi, mu, sigma)

DOBS = 10
DLAT = 2
INSTS = 100
seqlen = 30
epochs = 200

mdnlstm = MDNLSTM(DOBS, DLAT).to(device)
optimizer = torch.optim.Adam(mdnlstm.parameters())

z =  torch.from_numpy(ts[:INSTS,:,:]).float().to(device)

# hiddens=[]
# Train the model
for epoch in range(epochs):
    # Set initial hidden and cell states
    hidden = mdnlstm.init_hidden(INSTS)

    for i in range(0, z.size(1) - seqlen, seqlen):
        # Get mini-batch inputs and targets
        inputs = z[:, i:i+seqlen, :]
        targets = z[:, (i+1):(i+1)+seqlen, :]

        hidden = detach(hidden)
#         hiddens.append(hidden)
        (pi, mu, sigma), hidden = mdnlstm(inputs, hidden)
        loss = criterion(targets, pi, mu, sigma)
        mdnlstm.zero_grad()
        loss.backward()
        optimizer.step()

    if epoch % 100 == 0:
        print ('Epoch [{}/{}], Loss: {:.4f}'
               .format(epoch, epochs, loss.item()))

I would appreciate any help on this.


title: pytorch - loss.backward() and optimizer.step() in eval mode with batch norm layers?

body:
I have a ResNet-8 network I am using for a project of Domain Adaptation over images, basically I have trained the network over a dataset and now I want to evaluate it over another dataset simulating a real time environment where I try to predict one image at a time, but here comes the fun part:

The way I want to do the evaluation on the target dataset is by doing, for each image, a forward pass in train mode so that the batch norm layers statistics are updated (with torch.no_grad(), since I don't want to update the network parameters then, but only "adapt" the batch norm layers), and then do another forward pass in eval mode to get the actual prediction so that the batch norm layers will use mean and variance based on the whole set of images seen so far (and not only those of that batch, a single image in this case):

optimizer.zero_grad()
model.train()
with torch.no_grad():
  output_train = model(inputs)
model.eval()
output_eval = model(inputs)
loss = criterion(output_eval, targets)

The idea is that I do domain adaptation just by updating the batch norm layers to the new target distribution.

Then after doing this let's say I get an accuracy of 60%.
Now if I add this two other lines I am able to achieve something like 80% accuracy:

loss.backward()
optimizer.step()

Therefore my question is what happens if I do backward() and step() while in eval mode? Because I know about the different behaviour of batch norm and dropout layers between train and eval mode and I know about torch.no_grad() and how gradient are calculated and then parameters updated by the optimizer, but I wasn't able to find any information about my specific problem.

I think that since the model is then set in eval mode, those two line should be useless, but something clearly happens, does this have something to do with the affine parameters of the batch norm layers?

UPDATE: Ok I misunderstood something: eval mode does not block parameters to be updated, it only changes the behaviour of some layers (batch norm and dropout) during the forward pass, am I right? Therefore with those two lines I am actually training the network, hence the better accuracy. Anyway does this change something if batch norm affine is set to true? Are those parameters considered as "normal" parameters to be updated during optimizer.step() or is it different?


title: How to get a particular layer output of a pretrained VGG16 in pytorch

body:
I am very new to pytorch and I am trying to get the output of the pretrained model VGG16 feature vector in 1*4096 format which is returned by the layers just before the final layer. I found that there are similar features available in keras. Is there any direct command in pytorch for the same?

The code I am using:

import torch
from torch import nn
from torch import optim
import torch.nn.functional as F
from torchvision import models
from torch.autograd import Variable
from PIL import Image

image1 = Image.open(r"C:\Users\user\Pictures\user.png")

model = models.vgg16(pretrained=True)

scaler = transforms.Resize((224, 224))
to_tensor = transforms.ToTensor()

img = to_tensor(scaler(image1)).unsqueeze(0)

model(img).shape
model(img)


title: How to store and load training data comprised 50 millions 25x25 numpy arrays while training a multi-class CNN model?

body:
I have an image processing problem where there are five classes, each class has approximately 10 millions examples as training data where an image is a z-scored 25x25 numpy array.

Obviously, I can’t load all the training data into memory, so I have to use fit_generator.

I also the one who generates and augments these training data matrices, but I can’t do it in real time within fit_generator because it will be too slow to train the model.

First, how to store 50 millions 25x25 .npy arrays on disk? What would be the best practice?

Second, Should I use a database to store these matrices and to query from it during training? I don’t think SQLite supports multi threads, and SQL datasets support is still experimental in tensorflow.

I would love to know if there is a neat way to store these 50 million matrices, so the retrieval during training will be optimal.

Third, what about using HDF5 format? Should I switch to pytorch instead?


title: Cannot find conflict in Conda UnsatisfiableError message

body:
Trying to install pytorch in conda in Docker, and get UnsatisfiableError. However, I cannot find any actual conflict in the error message or I may not understand it correctly. The Docker image used is nvidia/cuda:10.1-cudnn7-devel-ubuntu18.04. There will be no problem if I do not specify the pytorch version to install. But I do need this version of pytorch for some legacy code.

Here is the conda command and error message.

(Python3.6) root@0cb9aad73116:/# conda install -c pytorch pytorch=0.3.1
Collecting package metadata (current_repodata.json): done
Solving environment: failed with initial frozen solve. Retrying with flexible solve.
Solving environment: failed with repodata from current_repodata.json, will retry with next repodata source.
Collecting package metadata (repodata.json): done
Solving environment: failed with initial frozen solve. Retrying with flexible solve.
Solving environment: |
Found conflicts! Looking for incompatible packages.
This can take several minutes.  Press CTRL-C to abort.
failed

UnsatisfiableError: The following specifications were found to be incompatible with each other:

Package libgcc-ng conflicts for:
python==3.6.9 -> libgcc-ng[version='>=7.3.0']
zlib -> libgcc-ng[version='>=7.2.0|>=7.3.0']
tk -> libgcc-ng[version='>=7.2.0|>=7.3.0']
xz -> libgcc-ng[version='>=7.2.0']
pytorch=0.3.1 -> libgcc-ng[version='>=5.4.0']
libedit -> libgcc-ng[version='>=7.2.0|>=7.3.0']
libffi -> libgcc-ng[version='>=7.2.0']
ncurses -> libgcc-ng[version='>=7.2.0|>=7.3.0']
sqlite -> libgcc-ng[version='>=7.2.0|>=7.3.0']
readline -> libgcc-ng[version='>=7.2.0|>=7.3.0']
openssl -> libgcc-ng[version='>=7.2.0|>=7.3.0']
Package libstdcxx-ng conflicts for:
ncurses -> libstdcxx-ng[version='>=7.2.0|>=7.3.0']
pytorch=0.3.1 -> libstdcxx-ng[version='>=5.4.0']
python==3.6.9 -> libstdcxx-ng[version='>=7.3.0']
libffi -> libstdcxx-ng[version='>=7.2.0']
Package xz conflicts for:
python==3.6.9 -> xz[version='>=5.2.4,<6.0a0']
Package cudnn conflicts for:
pytorch=0.3.1 -> cudnn[version='7.0.*|>=7.0.5,<=8.0a0']
Package nccl conflicts for:
pytorch=0.3.1 -> nccl[version='<2']
Package setuptools conflicts for:
wheel -> setuptools
pip -> setuptools
Package zlib conflicts for:
tk -> zlib[version='>=1.2.11,<1.3.0a0']
python==3.6.9 -> zlib[version='>=1.2.11,<1.3.0a0']
Package ca-certificates conflicts for:
openssl -> ca-certificates
Package libffi conflicts for:
python==3.6.9 -> libffi[version='>=3.2.1,<4.0a0']
Package cudatoolkit conflicts for:
pytorch=0.3.1 -> cudatoolkit[version='8.*|8.0.*']
Package _libgcc_mutex conflicts for:
libgcc-ng -> _libgcc_mutex=[build=main]
Package numpy conflicts for:
pytorch=0.3.1 -> numpy[version='>=1.11|>=1.11.3,<2.0a0']
Package openssl conflicts for:
python==3.6.9 -> openssl[version='>=1.1.1c,<1.1.2a']
Package libedit conflicts for:
sqlite -> libedit[version='>=3.1.20170329,<3.2.0a0|>=3.1.20181209,<3.2.0a0']
Package ncurses conflicts for:
python==3.6.9 -> ncurses[version='>=6.1,<7.0a0']
readline -> ncurses[version='6.0.*|>=6.0,<7.0a0|>=6.1,<7.0a0']
libedit -> ncurses[version='6.0.*|>=6.1,<7.0a0']
Package cffi conflicts for:
pytorch=0.3.1 -> cffi
Package sqlite conflicts for:
python==3.6.9 -> sqlite[version='>=3.29.0,<4.0a0']
Package * conflicts for:
pytorch=0.3.1 -> *[track_features=cuda90]
Package tk conflicts for:
python==3.6.9 -> tk[version='>=8.6.8,<8.7.0a0']
Package wheel conflicts for:
pip -> wheel
Package certifi conflicts for:
setuptools -> certifi[version='>=2016.09|>=2016.9.26']
Package mkl conflicts for:
pytorch=0.3.1 -> mkl[version='>=2018|>=2018.0.2,<2019.0a0']
Package readline conflicts for:
python==3.6.9 -> readline[version='>=7.0,<8.0a0']
Package pip conflicts for:
python==3.6.9 -> pip

Packages installed

(Python3.6) root@0cb9aad73116:/# conda list
# packages in environment at /opt/conda/envs/Python3.6:
#
# Name                    Version                   Build  Channel
_libgcc_mutex             0.1                        main
ca-certificates           2019.11.27                    0
certifi                   2019.11.28               py36_0
libedit                   3.1.20181209         hc058e9b_0
libffi                    3.2.1                hd88cf55_4
libgcc-ng                 9.1.0                hdf63c60_0
libstdcxx-ng              9.1.0                hdf63c60_0
ncurses                   6.1                  he6710b0_1
openssl                   1.1.1d               h7b6447c_3
pip                       19.3.1                   py36_0
python                    3.6.9                h265db76_0
readline                  7.0                  h7b6447c_5
setuptools                42.0.2                   py36_0
sqlite                    3.30.1               h7b6447c_0
tk                        8.6.8                hbc83047_0
wheel                     0.33.6                   py36_0
xz                        5.2.4                h14c3975_4
zlib                      1.2.11               h7b6447c_3

Similar error remains if use Docker image continuumio/miniconda3:4.7.12.


title: Is there an equivalent PyTorch function for `tf.nn.space_to_depth`

body: As the title says, is there an equivalent PyTorch function for tf.nn.space_to_depth?


title: Validation loss for pytorch Faster-RCNN

body:
I’m currently doing object detection on a custom dataset using transfer learning from a pytorch pretrained Faster-RCNN model (like in torchvision tutorial).
I would like to compute validation loss dict (as in train mode)  at the end of each epoch.
I can just run model in train mode for validation like this:

model.train()
for images, targets in data_loader_val:
    images = [image.to(device) for image in images]
    targets = [{k: v.to(device) for k, v in t.items()} for t in targets]

    with torch.no_grad():
         val_loss_dict = model(images, targets)
         print(val_loss_dict)

but I don't think, that it's "correct" way to validate (cause some special layers like dropout and batch norm works different in eval/train mode). And in eval mode model returns predicted bboxes (as expected).  Can I use some build-in function for this?

Thanks.


title: Reducing batch size in pytorch

body:
I am new to programming in pytorch. I am getting this error which says cuda out of memory. So I have to reduce the batch size. Can someone tell me how to do it in python code? I also don't know my current batch size.

p.s. I am trying to run the Deep Image Prior's super-resolution. Here's the code.

The error I am getting is when running the optimization. It says

  RuntimeError: Cuda out of memory.

from __future__ import print_function
import matplotlib.pyplot as plt
%matplotlib inline

import argparse
import os
os.environ['CUDA_VISIBLE_DEVICES'] = '0'

import numpy as np
from models import *

import torch
import torch.optim
import torch.nn as nn
from torch.utils.data import Dataset, DataLoader

import warnings
warnings.filterwarnings("ignore")

from skimage.measure import compare_psnr
from models.downsampler import Downsampler

from utils.sr_utils import *

torch.backends.cudnn.enabled = True
torch.backends.cudnn.benchmark =True
dtype = torch.cuda.FloatTensor

imsize = -1
factor = 16 # 8
enforse_div32 = 'CROP' # we usually need the dimensions to be divisible by a power of two (32 in this case)
PLOT = True
path_to_image = '/home/smitha/deep-image-prior/tnew.tif'
imgs = load_LR_HR_imgs_sr(path_to_image , imsize, factor, enforse_div32)
imgs['bicubic_np'], imgs['sharp_np'], imgs['nearest_np'] = get_baselines(imgs['LR_pil'], imgs['HR_pil'])
if PLOT:
    plot_image_grid([imgs['HR_np'], imgs['bicubic_np'], imgs['sharp_np'], imgs['nearest_np']], 4,12);
    print ('PSNR bicubic: %.4f   PSNR nearest: %.4f' %  (
                                    compare_psnr(imgs['HR_np'], imgs['bicubic_np']),
                                    compare_psnr(imgs['HR_np'], imgs['nearest_np'])))
input_depth = 8
INPUT =     'noise'
pad   =     'reflection'
OPT_OVER =  'net'
KERNEL_TYPE='lanczos2'
LR = 5
tv_weight = 0.0
OPTIMIZER = 'adam'
if factor == 16:
    num_iter = 10
    reg_noise_std = 0.01
elif factor == 8:
    num_iter = 40
    reg_noise_std = 0.05
else:
    assert False, 'We did not experiment with other factors'
net_input = get_noise(input_depth, INPUT, (imgs['HR_pil'].size[1],  imgs['HR_pil'].size[0])).type(dtype).detach()
NET_TYPE = 'skip' # UNet, ResNet
net = get_net(input_depth, 'skip', pad,
          skip_n33d=128,
          skip_n33u=128,
          skip_n11=4,
          num_scales=5,
          upsample_mode='bilinear').type(dtype)
mse = torch.nn.MSELoss().type(dtype)

img_LR_var = np_to_torch(imgs['LR_np']).type(dtype)

downsampler = Downsampler(n_planes=3, factor=factor,  kernel_type=KERNEL_TYPE, phase=0.5, preserve_size=True).type(dtype)
def closure():
    global i, net_input
    if reg_noise_std > 0:
        net_input = net_input_saved + (noise.normal_() * reg_noise_std)

    out_HR = net(net_input)
    out_LR = downsampler(out_HR)

    total_loss = mse(out_LR, img_LR_var)

    if tv_weight > 0:
        total_loss += tv_weight * tv_loss(out_HR)

    total_loss.backward()

    # Log
    psnr_LR = compare_psnr(imgs['LR_np'], torch_to_np(out_LR))
    psnr_HR = compare_psnr(imgs['HR_np'], torch_to_np(out_HR))
    print ('Iteration %05d    PSNR_LR %.3f   PSNR_HR %.3f' % (i, psnr_LR, psnr_HR), '\r', end='')

    # History
    psnr_history.append([psnr_LR, psnr_HR])

    if PLOT and i % 100 == 0:
        out_HR_np = torch_to_np(out_HR)
        plot_image_grid([imgs['HR_np'], imgs['bicubic_np'], np.clip(out_HR_np, 0, 1)], factor=13, nrow=3)

    i += 1

    return total_loss

psnr_history = []
volatile=True
net_input_saved = net_input.detach().clone()
noise = net_input.clone()
i = 0
p = get_params(OPT_OVER, net, net_input)
optimize(OPTIMIZER, p, closure, LR, num_iter)
out_HR_np = np.clip(torch_to_np(net(net_input)), 0, 1)
result_deep_prior = put_in_center(out_HR_np, imgs['orig_np'].shape[1:])
plot_image_grid([imgs['HR_np'],
             imgs['bicubic_np'],
             out_HR_np], factor=4, nrow=1);


title: Pytorch : Applying lambda transformations in case of segmentation tasks in custom transformation function

body:
I am trying to perform multiclass image segmentation. There are some transformations which should be applied to both image and masks like rotations and flips etc. I followed this
post and applied it but the issue is I have certain lambda transformations also which I need to apply but I can't apply using the above method.How to apply them.
Below are the transformations I need-

data_transforms = transforms.Compose([transforms.RandomCrop((512,512)),
                             transforms.Lambda(gaussian_blur),
                             transforms.Lambda(elastic_transform),
                             transforms.RandomRotation([+90,+180]),
                             transforms.RandomRotation([+180,+270]),
                             transforms.RandomHorizontalFlip(),
                             transforms.ToTensor(),
                             transforms.Normalize(mean=train_mean, std=train_std)
                           ])

My Dataloader for identical transformations for the image and mask is below, I wasn't able to apply my lambda transformations in it, it was throwing an error
AttributeError: module 'torchvision.transforms.functional' has no attribute 'Lambda'

class DataLoaderSegmentation(data.Dataset):
def __init__(self,folder_path,transform = None):
    super(DataLoaderSegmentation, self).__init__()
    self.img_files = glob.glob(os.path.join(folder_path,'images','*.tif'))
    self.mask_files = glob.glob(os.path.join(folder_path,'mask','*.bmp'))
    # self.transform = transform

def mask_to_class(self,mask):
    target = torch.from_numpy(mask)
    h,w = target.shape[0],target.shape[1]
    masks = torch.empty(h, w, dtype=torch.long)
    colors = torch.unique(target.view(-1,target.size(2)),dim=0).numpy()
    target = target.permute(2, 0, 1).contiguous()
    mapping = {tuple(c): t for c, t in zip(colors.tolist(), range(len(colors)))}
    for k in mapping:
        idx = (target==torch.tensor(k, dtype=torch.uint8).unsqueeze(1).unsqueeze(2))
        validx = (idx.sum(0) == 3)
        masks[validx] = torch.tensor(mapping[k], dtype=torch.long)
    return masks

def transform(self,image,mask):
    i, j, h, w = transforms.RandomCrop.get_params(
    image, output_size=(512, 512))
    image = TF.crop(image, i, j, h, w)
    mask = TF.crop(mask, i, j, h, w)

    #image = TF.Lambda(gaussian_blur),

    #image = TF.Lambda(elastic_transform)
    # Random horizontal flipping
    if random.random() > 0.5:
        image = TF.hflip(image)
        mask = TF.hflip(mask)

    image = TF.rotate(image,90)
    mask = TF.rotate(mask,90)
    image = TF.rotate(image,180)
    mask = TF.rotate(mask,180)
    image = TF.rotate(image,270)
    mask = TF.rotate(mask,270)

    # Transform to tensor
    image = TF.to_tensor(image)
    return image, mask

def __getitem__(self, index):
    img_path = self.img_files[index]
    mask_path = self.mask_files[index]
    data = Image.open(img_path)
    label = Image.open(mask_path)
    data,label = self.transform(data,label)
    label = np.array(label)
    mask = self.mask_to_class(label)
    return data,mask

def __len__(self):
    return len(self.img_files)

How to do it, or is there any other way to do it?

Edit---
Following is my solution for it-

class DataLoaderSegmentation(data.Dataset):
def __init__(self,folder_path,transform = None):
    super(DataLoaderSegmentation, self).__init__()
    self.img_files = glob.glob(os.path.join(folder_path,'images','*.tif'))
    self.mask_files = glob.glob(os.path.join(folder_path,'mask','*.bmp'))
    self.transforms = transform
    #for img_path in img_files:
     #   self.mask_files.append(os.path.join(folder_path,'masks',os.path.basename(img_path))

def mask_to_class(self,mask):
    target = torch.from_numpy(mask)
    h,w = target.shape[0],target.shape[1]
    masks = torch.empty(h, w, dtype=torch.long)
    colors = torch.unique(target.view(-1,target.size(2)),dim=0).numpy()
    target = target.permute(2, 0, 1).contiguous()
    mapping = {tuple(c): t for c, t in zip(colors.tolist(), range(len(colors)))}
    for k in mapping:
        idx = (target==torch.tensor(k, dtype=torch.uint8).unsqueeze(1).unsqueeze(2))
        validx = (idx.sum(0) == 3)
        masks[validx] = torch.tensor(mapping[k], dtype=torch.long)
    return masks

def transform(self,image,mask):
    i, j, h, w = transforms.RandomCrop.get_params(
    image, output_size=(512, 512))
    image = TF.crop(image, i, j, h, w)
    mask = TF.crop(mask, i, j, h, w)

    #image = TF.Lambda(gaussian_blur),
   # mask =
    #image = TF.Lambda(elastic_transform)
    # Random horizontal flipping
    image = F.Lambda(gaussian_blur)
    if random.random() > 0.5:
        image = TF.hflip(image)
        mask = TF.hflip(mask)

    image = TF.rotate(image,90)
    mask = TF.rotate(mask,90)
    image = TF.rotate(image,180)
    mask = TF.rotate(mask,180)
    image = TF.rotate(image,270)
    mask = TF.rotate(mask,270)

    # Transform to tensor
    #image = TF.to_tensor(image)
    #mask = TF.to_tensor(mask)
    return image, mask

def __getitem__(self, index):
    img_path = self.img_files[index]
    mask_path = self.mask_files[index]
    data = Image.open(img_path)
    label = Image.open(mask_path)
   # label = np.array(label)
    data,label = self.transform(data,label)
    label = np.array(label)
    data = np.array(data)
    #label = np.transpose(label,(2,0,1))
    mask = self.mask_to_class(label)
    if transforms is not None:
        data = self.transforms(data)
    return data,mask
   # return data, torch.from_numpy(label).long()

def __len__(self):
    return len(self.img_files)

and the transformation are below-

data_transforms = transforms.Compose([
                             transforms.Lambda(gaussian_blur),
                             transforms.Lambda(elastic_transform),
                             transforms.ToTensor(),
                             transforms.Normalize(mean=train_mean, std=train_std)
                           ])


title: PyTorch: Convolving a single channel image using torch.nn.Conv2d

body:
I am trying to use a convolution layer to convolve a grayscale (single layer) image (stored as a numpy array). Here is the code:

conv1 = torch.nn.Conv2d(in_channels = 1, out_channels = 1, kernel_size = 33)
tensor1 = torch.from_numpy(img_gray)
out_2d_np = conv1(tensor1)

out_2d_np = np.asarray(out_2d_np)

I want my kernel to be 33x33 and the number of output layers should be equal to the number of input layers, which is 1 as the image's RGB channels are summed. Whenout_2d_np = conv1(tensor1) is run it yields the following runtime error:

  RuntimeError: Expected 4-dimensional input for 4-dimensional weight 1 1 33 33, but got 2-dimensional input of size [246, 248] instead

Any idea on how I can solve this? I specifically want to use the torch.nn.Conv2d() class/function.

Thanks in advance for any help!


I have tried other answers but the error is not getting removed. Difference from the other question I am getting is that the last term used in error is 'source' which I did not find in any question. If possible please also explain the term 'source' in error. And running code without CPU is working fine.

  I am using Google Colab with GPU enabled.

import torch
from torch import nn
import syft as sy

hook = sy.TorchHook(torch)

device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

model = nn.Sequential(nn.Linear(784,256),
                     nn.ReLU(),
                     nn.Linear(256,128),
                     nn.ReLU(),
                     nn.Linear(128,64),
                     nn.ReLU(),
                     nn.Linear(64,10),
                     nn.LogSoftmax(dim = 1))

model = model.to(device)

output :

---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-42-136ec343040a> in <module>()
      8                      nn.LogSoftmax(dim = 1))
      9
---> 10 model = model.to(device)

3 frames
/usr/local/lib/python3.6/dist-packages/syft/frameworks/torch/hook/hook.py in data(self, new_data)
    368
    369                 with torch.no_grad():
--> 370                     self.set_(new_data)
    371             return self
    372

RuntimeError: Expected object of backend CPU but got backend CUDA for argument #2 'source'


title: Extracting Intermediate layer outputs of a CNN in PyTorch

body:
I am using a Resnet18 model.
ResNet(
  (conv1): Conv2d(3, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)
  (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
  (relu): ReLU(inplace=True)
  (maxpool): MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1, ceil_mode=False)
  (layer1): Sequential(
    (0): BasicBlock(
      (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
      (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (relu): ReLU(inplace=True)
      (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
      (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
    )
    (1): BasicBlock(
      (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
      (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (relu): ReLU(inplace=True)
      (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
      (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
    )
  )
  (layer2): Sequential(
    (0): BasicBlock(
      (conv1): Conv2d(64, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
      (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (relu): ReLU(inplace=True)
      (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
      (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (downsample): Sequential(
        (0): Conv2d(64, 128, kernel_size=(1, 1), stride=(2, 2), bias=False)
        (1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      )
    )
    (1): BasicBlock(
      (conv1): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
      (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (relu): ReLU(inplace=True)
      (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
      (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
    )
  )
  (layer3): Sequential(
    (0): BasicBlock(
      (conv1): Conv2d(128, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (relu): ReLU(inplace=True)
      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (downsample): Sequential(
        (0): Conv2d(128, 256, kernel_size=(1, 1), stride=(2, 2), bias=False)
        (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      )
    )
    (1): BasicBlock(
      (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (relu): ReLU(inplace=True)
      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
    )
  )
  (layer4): Sequential(
    (0): BasicBlock(
      (conv1): Conv2d(256, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
      (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (relu): ReLU(inplace=True)
      (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
      (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (downsample): Sequential(
        (0): Conv2d(256, 512, kernel_size=(1, 1), stride=(2, 2), bias=False)
        (1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      )
    )
    (1): BasicBlock(
      (conv1): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
      (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (relu): ReLU(inplace=True)
      (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
      (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
    )
  )
  (avgpool): AdaptiveAvgPool2d(output_size=(1, 1))
  (fc): Linear(in_features=512, out_features=1000, bias=True)
)

I want to extract the outputs only from layer2, layer3, layer4 & I don't want the avgpool and fc outputs.
How do I achieve this ?
class BasicBlock(nn.Module):
    def __init__(self, in_channels, out_channels, stride=1, padding=1) -> None:
        super(BasicBlock, self).__init__()
        self.conv1 = nn.Conv2d(in_channels, out_channels,
                               3, stride, padding=padding, bias=False)
        self.bn1 = nn.BatchNorm2d(out_channels)
        self.relu = nn.ReLU(inplace=True)
        self.conv2 = nn.Conv2d(out_channels, out_channels,
                               3, stride=1, padding=1, bias=False)
        self.bn2 = nn.BatchNorm2d(out_channels)
        if in_channels != out_channels:
            l1 = nn.Conv2d(in_channels, out_channels,
                           kernel_size=1, stride=stride, bias=False)
            l2 = nn.BatchNorm2d(out_channels)
            self.downsample = nn.Sequential(l1, l2)
        else:
            self.downsample = None

    def forward(self, xb):
        prev = xb
        x = self.relu(self.bn1(self.conv1(xb)))
        x = self.bn2(self.conv2(x))
        if self.downsample is not None:
            prev = self.downsample(xb)
        x = x + prev
        return self.relu(x)

class CustomResnet(nn.Module):
    def __init__(self, pretrained:bool=True) -> None:
        super(CustomResnet, self).__init__()
        self.conv1 = nn.Conv2d(3, 64, kernel_size=7,stride=2, padding=3, bias=False)
        self.bn1 = nn.BatchNorm2d(64)
        self.relu = nn.ReLU(inplace=True)
        self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
        self.layer1 = nn.Sequential(BasicBlock( 64, 64, stride=1), BasicBlock(64, 64))
        self.layer2 = nn.Sequential(BasicBlock(64, 128, stride=2), BasicBlock(128, 128))
        self.layer3 = nn.Sequential(BasicBlock(128, 256, stride=2), BasicBlock(256, 256))
        self.layer4 = nn.Sequential(BasicBlock(256, 512, stride=2), BasicBlock(512, 512))

    def forward(self, xb):
        x = self.maxpool(self.relu(self.bn1(self.conv1(xb))))
        x = self.layer1(x)
        x2 = x = self.layer2(x)
        x3 = x = self.layer3(x)
        x4 = x = self.layer4(x)
        return [x2, x3, x4]

I guess one solution would be this .. But is there any other way without writing this while lot of code? Also is it possible to load in the pre-trained weights given by torchvision in the above modified ResNet model.


title: PyTorch + CUDA 7.5 error

body:
I have non-sudo access to a machine with NVIDIA GPUs and CUDA 7.5 installed. I installed PyTorch with CUDA 7.5 support, which seems to have worked:

>>> import torch
>>> torch.cuda.is_available()
True

To get some practice, I followed tutorial for machine translation using RNNs. When I set USE_CUDA = False and the CPUs are used, everything works quite alright. However, when want to utilize the GPUs with USE_CUDA = True I get the following error:

Traceback (most recent call last):
  ...
  File "seq2seq.py", line 229, in train
    encoder_output, encoder_hidden = encoder(input_variable[ei], encoder_hidden)
  File "/.../python2.7/site-packages/torch/nn/modules/module.py", line 206, in __call__
    result = self.forward(*input, **kwargs)
  File "seq2seq.py", line 144, in forward
    output, hidden = self.gru(embedded, hidden)
  File "/.../python2.7/site-packages/torch/nn/modules/module.py", line 206, in __call__
    result = self.forward(*input, **kwargs)
  File "/.../python2.7/site-packages/torch/nn/modules/rnn.py", line 91, in forward
    output, hidden = func(input, self.all_weights, hx)
  ...
  File "/.../python2.7/site-packages/torch/backends/cudnn/rnn.py", line 42, in init_rnn_descriptor
    cudnn.DropoutDescriptor(handle, dropout_p, fn.dropout_seed)
  File "/usr/lib/python2.7/ctypes/__init__.py", line 383, in __getitem__
    func = self._FuncPtr((name_or_ordinal, self))
AttributeError: python: undefined symbol: cudnnCreateDropoutDescriptor
Exception AttributeError: 'python: undefined symbol: cudnnDestroyDropoutDescriptor' in <bound method DropoutDescriptor.__del__ of <torch.backends.cudnn.DropoutDescriptor object at 0x7fe540efec10>> ignored

I've tried to use Google to search for that error but got no meaningful results. Since I'm rather a newbie with PyTorch and CUDA, I have no idea how to go on from here. The full setup is Ubuntu 14.04, Python 2.7, CUDA 7.5.


title: pytorch vgg model test on one image

body:
I've trained a vgg model, this is how I transformed the test data

test_transform_2= transforms.Compose([transforms.RandomResizedCrop(224),
                                     transforms.ToTensor()])

test_data = datasets.ImageFolder(test_dir, transform=test_transform_2)

the model's finished training now I want to test it on a single image

from scipy import misc

test_image = misc.imread('flower_data/valid/1/image_06739.jpg')
vgg16(torch.from_numpy(test_image))

Error

---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-60-b83587325fea> in <module>
----> 1 vgg16(torch.from_numpy(test_image))

c:\users\sam\mydocu~1\code\envs\data-science\lib\site-packages\torch\nn\modules\module.py in __call__(self, *input, **kwargs)
    475             result = self._slow_forward(*input, **kwargs)
    476         else:
--> 477             result = self.forward(*input, **kwargs)
    478         for hook in self._forward_hooks.values():
    479             hook_result = hook(self, input, result)

c:\users\sam\mydocu~1\code\envs\data-science\lib\site-packages\torchvision\models\vgg.py in forward(self, x)
     40
     41     def forward(self, x):
---> 42         x = self.features(x)
     43         x = x.view(x.size(0), -1)
     44         x = self.classifier(x)

c:\users\sam\mydocu~1\code\envs\data-science\lib\site-packages\torch\nn\modules\module.py in __call__(self, *input, **kwargs)
    475             result = self._slow_forward(*input, **kwargs)
    476         else:
--> 477             result = self.forward(*input, **kwargs)
    478         for hook in self._forward_hooks.values():
    479             hook_result = hook(self, input, result)

c:\users\sam\mydocu~1\code\envs\data-science\lib\site-packages\torch\nn\modules\container.py in forward(self, input)
     89     def forward(self, input):
     90         for module in self._modules.values():
---> 91             input = module(input)
     92         return input
     93

c:\users\sam\mydocu~1\code\envs\data-science\lib\site-packages\torch\nn\modules\module.py in __call__(self, *input, **kwargs)
    475             result = self._slow_forward(*input, **kwargs)
    476         else:
--> 477             result = self.forward(*input, **kwargs)
    478         for hook in self._forward_hooks.values():
    479             hook_result = hook(self, input, result)

c:\users\sam\mydocu~1\code\envs\data-science\lib\site-packages\torch\nn\modules\conv.py in forward(self, input)
    299     def forward(self, input):
    300         return F.conv2d(input, self.weight, self.bias, self.stride,
--> 301                         self.padding, self.dilation, self.groups)
    302
    303

RuntimeError: Expected 4-dimensional input for 4-dimensional weight [64, 3, 3, 3], but got input of size [628, 500, 3] instead

I can tell I need to shape the input, however I don't know how to based on the way it seems to expect the input to be inform of a batch.


title: Best practices to benchmark deep models on CPU (and not GPU) in PyTorch?

body:
I am little uncertain about how to measure execution time of deep models on CPU in PyTorch ONLY FOR INFERENCE. I list here some of them but they maybe inaccurate. Please correct them if required and mention more if required. I am running on PyTorch version 1.3.1 and Intel Xeon with 64GB RAM, 3.5GHz processor and 8 cores.

Should we use time.time()?

I know that for GPU this is a very bad idea. For GPU I do as follows

with torch.no_grad():
    wTime = 0
    start = torch.cuda.Event(enable_timing=True)
    end = torch.cuda.Event(enable_timing=True)
    out = model(input) # JUST FOR WARMUP

    start.record()
    for i in range(200):
        input = torch.rand(1,1,2048,2048).to(device)

#        beg = time.time() DO NOT USE FOR GPU

        got = net_amplifier(low,for_amplifier)

#        wTime+=time.time()-beg DO NOT USE FOR GPU
    end.record()
    torch.cuda.synchronize()

    print('execution time in MILLISECONDS: {}'.format(start.elapsed_time(end)/200))

For this code execution was done on GPU. If I have to run them on CPU what changes should be made? Will time.time() do?

Should we use volatile?

I think the use if volatile is now discouraged after v0.3. But will it still help if I use the eval mode and no_grad()

input = Variable(torch.randn(1, 3, 227, 227), volatile=True)
 model(input)

Should the page cache be cleared?

One way of doing this that I know is using sudo sh -c "/bin/echo 1 > /proc/sys/vm/drop_caches"

Should I remove nn.Sequential() and directly put in forward part

According to this link

  All the methods using copy_ take some time to execute, especially on CPU this might be slow. Also the nn.Sequential() modules are slower than just executing them on the forward pass. I think this is due to some overhead that needs to be created when executing the Sequential module.

Another thing which i do not understand on the same link is

  If you are running into performance issues with these small numbers, you might try to use torch.set_flush_denormal(True) to disable denormal floating point numbers on the CPU.

Should torch.set_num_threads(int) be used? If yes can a demo code be provided?
What does These context managers are thread local, so they won’t work if you send work to another thread using the :module:`threading` module, etc. mean as given in the documentation.

Please list any more issues for calculating execution time in CPU.
Thankyou


title: How to run pytorch with NVIDIA "cuda toolkit" version instead of the official conda "cudatoolkit" version?

body:
Some questions came up from https://superuser.com/questions/1572640/do-i-need-to-install-cuda-separately-after-installing-the-nvidia-display-driver. One of these questions:
Does conda pytorch need a different version than the official non-conda / non-pip cuda toolkit at https://developer.nvidia.com/cuda-toolkit?
In other words: Can I use the NVIDIA "cuda toolkit" for a pytorch installation?
Context:
If you go through the "command helper" at https://pytorch.org/get-started/locally/, you can choose between cuda versions 9.2, 10.1, 10.2 and None.
Taking 10.2 can result in:
conda install pytorch torchvision cudatoolkit=10.2 -c pytorch

Taking "None" builds the following command, but then you also cannot use cuda in pytorch:
conda install pytorch torchvision cpuonly -c pytorch

Could I then use NVIDIA "cuda toolkit" version 10.2 as the conda cudatoolkit in order to make this command the same as if it was executed with cudatoolkit=10.2 parameter?
The question arose since pytorch installs a different version (10.2 instead of the most recent NVIDIA 11.0), and the conda install takes additional 325 MB. If both versions were 11.0 and the installation size was smaller, you might not even notice the possible difference. But now it is clear that conda carries its own cuda version which is independent from the NVIDIA one.


title: PyTorch: training with GPU gives worse error than training the same thing with CPU

body:
I have a next step prediction model on times series which is simply a GRU with a fully-connected layer on top of it. When I train it using CPU after 50 epochs I get a loss of 0.10 but when I train it with GPU the loss is 0.15 after 50 epochs. Doing more epochs doesnt really lower the losses in either cases.

Why is performance after training on CPU better than GPU?

I have tried changing the random seeds for both data and model, and these results are independent of the random seeds.

I have:

Python 3.6.2

PyTorch 0.3.0

CUDNN_MAJOR 7

CUDNN_MINOR 0

CUDNN_PATCHLEVEL 5

Edit:

I also use PyTorch's weight normalizaton torch.nn.utils.weight_norm on the GRU and on the fully-connected layer.


title: Missing Keys in state_dict

body:
I am having problems loading my model on google colab. here is the code:

I have attached the code below

I have tried changing the name of the statedict and it does not help
basically, I am trying to save my model for later use, but, this is becoming extremely difficult since I am not being able to properly save and load it. Please help me with the problem. After the section of the code, you will also find the error that I have attached below.

here is the code

from zipfile import ZipFile
file_name = 'data.zip'
with ZipFile(file_name, 'r') as zip:
  zip.extractall()

from zipfile import ZipFile
file_name = 'results.zip'
with ZipFile(file_name, 'r') as zip:
  zip.extractall()

!pip install tensorflow-gpu

from __future__ import print_function
import torch
import torch.nn as nn
import torch.nn.parallel
import torch.optim as optim
import torch.utils.data
import torchvision.datasets as dset
import torchvision.transforms as transforms
import torchvision.utils as vutils
from torch.autograd import Variable

batchSize = 64
imageSize = 64

transform = transforms.Compose([transforms.Resize(imageSize), transforms.ToTensor(), transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),])

dataset = dset.CIFAR10(root = './data', download = True, transform = transform)
dataloader = torch.utils.data.DataLoader(dataset, batch_size = batchSize, shuffle = True, num_workers = 2)

def weights_init(m):
    classname = m.__class__.__name__
    if classname.find('Conv') != -1:
        m.weight.data.normal_(0.0, 0.02)
    elif classname.find('BatchNorm') != -1:
        m.weight.data.normal_(1.0, 0.02)
        m.bias.data.fill_(0)

class G(nn.Module):

    def __init__(self):
        super(G, self).__init__()
        self.main = nn.Sequential(
            nn.ConvTranspose2d(100, 512, 4, 1, 0, bias = False),
            nn.BatchNorm2d(512),
            nn.ReLU(True),
            nn.ConvTranspose2d(512, 256, 4, 2, 1, bias = False),
            nn.BatchNorm2d(256),
            nn.ReLU(True),
            nn.ConvTranspose2d(256, 128, 4, 2, 1, bias = False),
            nn.BatchNorm2d(128),
            nn.ReLU(True),
            nn.ConvTranspose2d(128, 64, 4, 2, 1, bias = False),
            nn.BatchNorm2d(64),
            nn.ReLU(True),
            nn.ConvTranspose2d(64, 3, 4, 2, 1, bias = False),
            nn.Tanh()
        )

    def forward(self, input):
        output = self.main(input)
        return output

netG = G()
netG.load_state_dict(torch.load('generator.pth'))
netG.eval()
#netG.apply(weights_init)

class D(nn.Module):

    def __init__(self):
        super(D, self).__init__()
        self.main = nn.Sequential(
            nn.Conv2d(3, 64, 4, 2, 1, bias = False),
            nn.LeakyReLU(0.2, inplace = True),
            nn.Conv2d(64, 128, 4, 2, 1, bias = False),
            nn.BatchNorm2d(128),
            nn.LeakyReLU(0.2, inplace = True),
            nn.Conv2d(128, 256, 4, 2, 1, bias = False),
            nn.BatchNorm2d(256),
            nn.LeakyReLU(0.2, inplace = True),
            nn.Conv2d(256, 512, 4, 2, 1, bias = False),
            nn.BatchNorm2d(512),
            nn.LeakyReLU(0.2, inplace = True),
            nn.Conv2d(512, 1, 4, 1, 0, bias = False),
            nn.Sigmoid()
        )

    def forward(self, input):
        output = self.main(input)
        return output.view(-1)

netD = D()
netD.load_state_dict(torch.load('discriminator.pth'))
netD.eval()
#netD.apply(weights_init)

criterion = nn.BCELoss()
checkpoint = torch.load('discriminator.pth')
optimizerD = optim.Adam(netD.parameters(), lr = 0.0002, betas = (0.5, 0.999))
optimizerD.load_state_dict(checkpoint['optimizer_state_dict'])
epoch = checkpoint['epoch']
errD = checkpoint['loss']
checkpoint1 = torch.load('genrator.pth')
optimizerG = optim.Adam(netG.parameters(), lr = 0.0002, betas = (0.5, 0.999))
optimizerG.load_state_dict(checkpoint1['optimizer_state_dict'])
errG = checkpoint1['loss']
k = epoch
for j in range(k, 10):

    for i, data in enumerate(dataloader, 0):

        netD.zero_grad()

        real, _ = data
        input = Variable(real)
        target = Variable(torch.ones(input.size()[0]))
        output = netD(input)
        errD_real = criterion(output, target)

        noise = Variable(torch.randn(input.size()[0], 100, 1, 1))
        fake = netG(noise)
        target = Variable(torch.zeros(input.size()[0]))
        output = netD(fake.detach())
        errD_fake = criterion(output, target)

        errD = errD_real + errD_fake
        errD.backward()
        optimizerD.step()

        netG.zero_grad()
        target = Variable(torch.ones(input.size()[0]))
        output = netD(fake)
        errG = criterion(output, target)
        errG.backward()
        optimizerG.step()

        print('[%d/%d][%d/%d] Loss_D: %.4f Loss_G: %.4f' % (epoch+1, 10, i+1, len(dataloader), errD.data, errG.data))
        if i % 100 == 0:
            vutils.save_image(real, '%s/real_samples.png' % "./results", normalize = True)
            fake = netG(noise)
            vutils.save_image(fake.data, '%s/fake_samples_epoch_%03d.png' % ("./results", epoch+1), normalize = True)

torch.save({
            'epoch': epoch,
            'model_state_dict': netD.state_dict(),
            'optimizer_state_dict': optimizerD.state_dict(),
            'loss': errD
            }, 'discriminator.pth')
torch.save({
            'epoch': epoch,
            'model_state_dict': netG.state_dict(),
            'optimizer_state_dict': optimizerG.state_dict(),
            'loss': errG
            }, 'generator.pth')

here is the error

RuntimeError                              Traceback (most recent call last)
<ipython-input-23-3e55546152c7> in <module>()
     26 # Creating the generator
     27 netG = G()
---> 28 netG.load_state_dict(torch.load('generator.pth'))
     29 netG.eval()
     30 #netG.apply(weights_init)

/usr/local/lib/python3.6/dist-packages/torch/nn/modules/module.py in load_state_dict(self, state_dict, strict)
    767         if len(error_msgs) > 0:
    768             raise RuntimeError('Error(s) in loading state_dict for {}:\n\t{}'.format(
--> 769                                self.__class__.__name__, "\n\t".join(error_msgs)))
    770
    771     def _named_members(self, get_members_fn, prefix='', recurse=True):

RuntimeError: Error(s) in loading state_dict for G:
    Missing key(s) in state_dict: "main.0.weight", "main.1.weight", "main.1.bias", "main.1.running_mean", "main.1.running_var", "main.3.weight", "main.4.weight", "main.4.bias", "main.4.running_mean", "main.4.running_var", "main.6.weight", "main.7.weight", "main.7.bias", "main.7.running_mean", "main.7.running_var", "main.9.weight", "main.10.weight", "main.10.bias", "main.10.running_mean", "main.10.running_var", "main.12.weight".
    Unexpected key(s) in state_dict: "epoch", "model_state_dict", "optimizer_state_dict", "loss".


When I run AdaIN code
def adaptive_instance_normalization(content_feat, style_mean, style_std):
    size = content_feat.size()
    content_mean, content_std = calc_mean_std(content_feat)

    normalized_feat = (content_feat - content_mean.expand(
        size)) / content_std.expand(size)
    return normalized_feat * style_std.expand(size) + style_mean.expand(size)

I got the following error
RuntimeError: The expanded size of the tensor (7) must match the existing size (128) at non-singleton dimension 3.  Target sizes: [100, 128, 7, 7].  Tensor sizes: [100, 128]


title: load pickle file obtained from GPU to CPU

body:
I meet a problem when I load a pickle file to CPU. I search it on the internet, and they say I need to add map_location parameter. However, after I add this parameter, the problem still exists.

the code is as follows:

torch.__version__
torch.load('featurs.pkl',map_location='cpu')

>>>

'1.0.1.post2'
Attempting to deserialize object on a CUDA device
but torch.cuda.is_available() is False. If you are running
on a CPU-only machine, please use torch.load with map_location='cpu'
to map your storages to the CPU.

I know it is because of the different devices, but I use the instruction in the error message, so I do not know how to solve it in the next step.

Thanks in advance!


title: using transforms.LinearTransformation to apply whitening in PyTorch

body:
I need to apply ZCA whitening in PyTorch. I think I have found a way this can be done by using transforms.LinearTransformation and I have found a test in the PyTorch repo which gives some insight into how this is done (see final code block or link below)

https://github.com/pytorch/vision/blob/master/test/test_transforms.py

I am struggling to work out how I apply something like this myself.

Currently I have transforms along the lines of:

    transform_test = transforms.Compose([
    transforms.ToTensor(),
    transforms.Normalize(np.array([125.3, 123.0, 113.9]) / 255.0,
                         np.array([63.0, 62.1, 66.7]) / 255.0),
])

The documents say they way to use LinearTransformation is as follows:

torchvision.transforms.LinearTransformation(transformation_matrix, mean_vector)

  whitening transformation: Suppose X is a column vector zero-centered
  data. Then compute the data covariance matrix [D x D] with
  torch.mm(X.t(), X), perform SVD on this matrix and pass it as
  transformation_matrix.

I can see from the tests I linked above and copied below that they are using torch.mm to calculate what they call a principal_components:

def test_linear_transformation(self):
    num_samples = 1000
    x = torch.randn(num_samples, 3, 10, 10)
    flat_x = x.view(x.size(0), x.size(1) * x.size(2) * x.size(3))
    # compute principal components
    sigma = torch.mm(flat_x.t(), flat_x) / flat_x.size(0)
    u, s, _ = np.linalg.svd(sigma.numpy())
    zca_epsilon = 1e-10  # avoid division by 0
    d = torch.Tensor(np.diag(1. / np.sqrt(s + zca_epsilon)))
    u = torch.Tensor(u)
    principal_components = torch.mm(torch.mm(u, d), u.t())
    mean_vector = (torch.sum(flat_x, dim=0) / flat_x.size(0))
    # initialize whitening matrix
    whitening = transforms.LinearTransformation(principal_components, mean_vector)
    # estimate covariance and mean using weak law of large number
    num_features = flat_x.size(1)
    cov = 0.0
    mean = 0.0
    for i in x:
        xwhite = whitening(i)
        xwhite = xwhite.view(1, -1).numpy()
        cov += np.dot(xwhite, xwhite.T) / num_features
        mean += np.sum(xwhite) / num_features
    # if rtol for std = 1e-3 then rtol for cov = 2e-3 as std**2 = cov
    assert np.allclose(cov / num_samples, np.identity(1), rtol=2e-3), "cov not close to 1"
    assert np.allclose(mean / num_samples, 0, rtol=1e-3), "mean not close to 0"

    # Checking if LinearTransformation can be printed as string
    whitening.__repr__()

How do I apply something like this? do I use it where I define my transforms or apply it in my training loop where I am iterating over my training loop?

Thanks in advance


title: Construct a rotation matrix in Pytorch

body:
I want to construct a rotation matrix, which have unknown Eular angles. I want to build some regression solution to find the value of Eular angles. My code is here.

roll = yaw = pitch = torch.randn(1,requires_grad=True)
RX = torch.tensor([
                [1, 0, 0],
                [0, cos(roll), -sin(roll)],
                [0, sin(roll), cos(roll)]
            ],requires_grad=True)
RY = torch.tensor([
                [cos(pitch), 0, sin(pitch)],
                [0, 1, 0],
                [-sin(pitch), 0, cos(pitch)]
            ],requires_grad=True)
RZ = torch.tensor([
                [cos(yaw), -sin(yaw), 0],
                [sin(yaw), cos(yaw), 0],
                [0, 0, 1]
            ],requires_grad=True)
R = torch.mm(RZ, RY).requires_grad_()
R = torch.mm(R, RX).requires_grad_()
R = R.mean().requires_grad_()
R.backward()

Matrix cannot differentiate the Euler angles.
There isn't any gradient value of matrix. Can anyone fix my problems? Thanks!
debug result


title: Can a Neural Network learn a simple interpolation?

body:
I’ve tried to train a 2 layer neural network on a simple linear interpolation for a discrete function, I’ve tried lots of different learning rates as well as different activation functions, and it seems like nothing is being learned!

I’ve literally spent the last 6 hours trying to debug the following code, but it seems like there’s no bug! What's the explanation?

    from torch.utils.data import Dataset
    import os
    import torch
    import numpy as np
    import torch.nn as nn
    import torch.optim as optim
    import random

    LOW_X=255

    MID_X=40000

    HIGH_X=200000

    LOW_Y=torch.Tensor([0,0,1])
    MID_Y=torch.Tensor([0.2,0.5,0.3])
    HIGH_Y=torch.Tensor([1,0,0])

    BATCH_SIZE=4

    def x_to_tensor(x):
        if x<=MID_X:
            return LOW_Y+(x-LOW_X)*(MID_Y-LOW_Y)/(MID_X-LOW_X)
        if x<=HIGH_X:
            return MID_Y+(x-MID_X)*(HIGH_Y-MID_Y)/(HIGH_X-MID_X)
        return HIGH_Y

    class XYDataset(Dataset):
        LENGTH=10000
        def __len__(self):
            return self.LENGTH

        def __getitem__(self, idx):
            x=random.randint(LOW_X,HIGH_X)
            y=x_to_tensor(x)
            return x,y

    class Interpolate(nn.Module):

        def __init__(self, num_outputs,hidden_size=10):
            super(Interpolate, self).__init__()
            self.hidden_size=hidden_size
            self.x_to_hidden = nn.Linear(1, hidden_size)
            self.hidden_to_out = nn.Linear(hidden_size,num_outputs)
            self.activation = nn.Tanh() #I have tried Sigmoid and Relu activations as well
            self.softmax=torch.nn.Softmax(dim=1)

        def forward(self, x):
            out = self.x_to_hidden(x)
            out = self.activation(out)
            out = self.hidden_to_out(out)
            out = self.softmax(out)
            return out

    dataset=XYDataset()

    trainloader = torch.utils.data.DataLoader(dataset, batch_size=BATCH_SIZE,
                                              shuffle=True, num_workers=4)

    criterion= nn.MSELoss()

    def train_net(net,epochs=10,lr=5.137871216190041e-05,l2_regularization=2.181622809797563e-12):
            optimizer= optim.Adam(net.parameters(),lr=lr,weight_decay=l2_regularization)
            net.train(True)
            running_loss=0.0
            for epoch in range(epochs):

                for i,data in enumerate(trainloader):
                    inputs,targets=data
                    inputs,targets=torch.FloatTensor(inputs.float()).view(-1,1),torch.FloatTensor(targets.float())
                    optimizer.zero_grad()
                    outputs=net(inputs)
                    loss=criterion(outputs,targets)
                    loss.backward()
                    optimizer.step()
                    running_loss+=loss.item()
                    if (len(trainloader)*epoch+i)%200==199:
                        running_loss=running_loss/(200*BATCH_SIZE)
                        print('[%d,%5d] loss: %.6f ' % (epoch+1,i+1,running_loss))
                        running_loss=0.0

    for i in range(-11,3):
        net=Interpolate(num_outputs=3)
        train_net(net,lr=10**i,epochs=1)
        print('for learning rate {} net output on low x is {}'.format(i,net(torch.Tensor([255]).view(-1,1))))


title: Saving/Loading models in AllenNLP package

body:
I am trying to load an AllenNLP model weights. I could not find any documentation on how to save/load a whole model, so playing with weights only.

from allennlp.nn import util
model_state = torch.load(filename_model, map_location=util.device_mapping(-1))
model.load_state_dict(model_state)

I modified my input corpus a bit and I am guessing because of this I am getting corpus-size mismatch:

RuntimeError: Error(s) in loading state_dict for BasicTextFieldEmbedder:

    size mismatch for token_embedder_tokens.weight:
    copying a param with shape torch.Size([2117, 16]) from checkpoint,
    the shape in current model is torch.Size([2129, 16]).

Seemingly there is no official way to save model with corpus vocabulary. Any hacks around it?