title: What's the workaround for "ragged/jagged tensors" in PyTorch?

body:
Tensorflow provides ragged tensors (https://www.tensorflow.org/guide/ragged_tensor). PyTorch however doesn't provide such a data structure. Is there a workaround to construct something similar in PyTorch?

import numpy as np
x = np.array([[0], [0, 1]])
print(x)  # [list([0]) list([0, 1])]

import tensorflow as tf
x = tf.ragged.constant([[0], [0, 1]])
print(x)  # <tf.RaggedTensor [[0], [0, 1]]>

import torch
# x = torch.Tensor([[0], [0, 1]])  # ValueError


title: Unable to load model from checkpoint in Pytorch-Lightning

body:
I am working with a U-Net in Pytorch Lightning. I am able to train the model successfully but after training when I try to load the model from checkpoint I get this error:
Complete Traceback:
Traceback (most recent call last):
  File "src/train.py", line 269, in <module>
    main(sys.argv[1:])
  File "src/train.py", line 263, in main
    model = Unet.load_from_checkpoint(checkpoint_callback.best_model_path)
  File "/home/africa_wikilimo/miniconda3/envs/xarray_test/lib/python3.8/site-packages/pytorch_lightning/core/saving.py", line 153, in load_from_checkpoint
    model = cls._load_model_state(checkpoint, *args, strict=strict, **kwargs)
  File "/home/africa_wikilimo/miniconda3/envs/xarray_test/lib/python3.8/site-packages/pytorch_lightning/core/saving.py", line 190, in _load_model_state
    model = cls(*cls_args, **cls_kwargs)
  File "src/train.py", line 162, in __init__
    self.inc = double_conv(self.n_channels, 64)
  File "src/train.py", line 122, in double_conv
    nn.Conv2d(in_channels, out_channels, kernel_size=3, padding=1),
  File "/home/africa_wikilimo/miniconda3/envs/xarray_test/lib/python3.8/site-packages/torch/nn/modules/conv.py", line 406, in __init__
    super(Conv2d, self).__init__(
  File "/home/africa_wikilimo/miniconda3/envs/xarray_test/lib/python3.8/site-packages/torch/nn/modules/conv.py", line 50, in __init__
    if in_channels % groups != 0:
TypeError: unsupported operand type(s) for %: 'dict' and 'int'

I tried surfing the github issues and forums, am not able to figure out what the issue is. Please help.
Here's the code of my model and the checkpoint loading step:
Model:
class Unet(pl.LightningModule):
    def __init__(self, n_channels, n_classes=5):
        super(Unet, self).__init__()
        # self.hparams = hparams

        self.n_channels = n_channels
        self.n_classes = n_classes
        self.bilinear = True
        self.logger = WandbLogger(name="Adam", project="pytorchlightning")

        def double_conv(in_channels, out_channels):
            return nn.Sequential(
                nn.Conv2d(in_channels, out_channels, kernel_size=3, padding=1),
                nn.BatchNorm2d(out_channels),
                nn.ReLU(inplace=True),
                nn.Conv2d(out_channels, out_channels, kernel_size=3, padding=1),
                nn.BatchNorm2d(out_channels),
                nn.ReLU(inplace=True),
            )

        def down(in_channels, out_channels):
            return nn.Sequential(
                nn.MaxPool2d(2), double_conv(in_channels, out_channels)
            )

        class up(nn.Module):
            def __init__(self, in_channels, out_channels, bilinear=False):
                super().__init__()

                if bilinear:
                    self.up = nn.Upsample(
                        scale_factor=2, mode="bilinear", align_corners=True
                    )
                else:
                    self.up = nn.ConvTranspose2d(
                        in_channels // 2, in_channels // 2, kernel_size=2, stride=2
                    )

                self.conv = double_conv(in_channels, out_channels)

            def forward(self, x1, x2):
                x1 = self.up(x1)
                # [?, C, H, W]
                diffY = x2.size()[2] - x1.size()[2]
                diffX = x2.size()[3] - x1.size()[3]

                x1 = F.pad(
                    x1, [diffX // 2, diffX - diffX // 2, diffY // 2, diffY - diffY // 2]
                )
                x = torch.cat([x2, x1], dim=1)
                return self.conv(x)

        self.inc = double_conv(self.n_channels, 64)
        self.down1 = down(64, 128)
        self.down2 = down(128, 256)
        self.down3 = down(256, 512)
        self.down4 = down(512, 512)
        self.up1 = up(1024, 256)
        self.up2 = up(512, 128)
        self.up3 = up(256, 64)
        self.up4 = up(128, 64)
        self.out = nn.Conv2d(64, self.n_classes, kernel_size=1)

    def forward(self, x):
        x1 = self.inc(x)

        x2 = self.down1(x1)

        x3 = self.down2(x2)

        x4 = self.down3(x3)

        x5 = self.down4(x4)

        x = self.up1(x5, x4)
        x = self.up2(x, x3)
        x = self.up3(x, x2)
        x = self.up4(x, x1)

        return self.out(x)

    def training_step(self, batch, batch_nb):
        x, y = batch

        y_hat = self.forward(x)
        loss = self.MSE(y_hat, y)

        # wandb_logger.log_metrics({"loss":loss})
        return {"loss": loss}

    def training_epoch_end(self, outputs):
        avg_train_loss = torch.stack([x["loss"] for x in outputs]).mean()
        self.logger.log_metrics({"train_loss": avg_train_loss})
        return {"average_loss": avg_train_loss}

    def test_step(self, batch, batch_nb):
        x, y = batch
        y_hat = self.forward(x)
        loss = self.MSE(y_hat, y)
        return {"test_loss": loss, "pred": y_hat}

    def test_end(self, outputs):

        avg_loss = torch.stack([x["test_loss"] for x in outputs]).mean()

        return {"avg_test_loss": avg_loss}

    def MSE(self, logits, labels):

        return torch.mean((logits - labels) ** 2)

    def configure_optimizers(self):
        return torch.optim.Adam(self.parameters(), lr=0.1, weight_decay=1e-8)

Main Function:
def main(expconfig):
    # Define checkpoint callback
    checkpoint_callback = ModelCheckpoint(
        filepath="/home/africa_wikilimo/data/model_checkpoint/",
        save_top_k=1,
        verbose=True,
        monitor="loss",
        mode="min",
        prefix="",
    )

    # Initialise datasets
    print("Initializing Climate Dataset....")
    clima_train = Clima_Dataset(expconfig[0])

    # Initialise dataloaders
    print("Initializing train_loader....")
    train_dataloader = DataLoader(clima_train, batch_size=2, num_workers=4)

    # Initialise model and trainer
    print("Initializing model...")
    model = Unet(n_channels=9, n_classes=5)
    print("Initializing Trainer....")
    if torch.cuda.is_available():

        model.cuda()

        trainer = pl.Trainer(
            max_epochs=1,
            gpus=1,
            checkpoint_callback=checkpoint_callback,
            early_stop_callback=None,
        )
    else:

        trainer = pl.Trainer(max_epochs=1, checkpoint_callback=checkpoint_callback)

    trainer.fit(model, train_dataloader=train_dataloader)
    print(checkpoint_callback.best_model_path)
    model = Unet.load_from_checkpoint(checkpoint_callback.best_model_path)


title: Loading image data from pandas to pytorch

body:
I am completely new to pytorch and have previously worked on keras and fastai.
Currently trying an image regression task and the challenge is I have to load the data from pandas dataframe.
Data frame structure:

ID   Path   Score
fig1  /folder/fig1.jpg  2
fig2  /folder/fig2.jpg  3
.....

I have previously worked on loading images to pytorch directly from folders because it was a simple classification task but kind of stuck now.

I looked into pytorch forums but didn't quiet understand how to implement.
Any help would be appreciated.


title: Pytorch gpu usage is low

body:
I am trying the example of pytorch from https://pytorch.org/tutorials/beginner/blitz/cifar10_tutorial.html.
When i run this example, the GPU usage is ~1% and finish time is 130s
While for CPU case, the CPU usage get ~90% and finish time is 79s
My CPU is Intel(R) Core(TM) i7-8700
and my GPU is NVIDIA GeForce RTX 2070.

I want to ask if it is normal that CPU is running faster than GPU ?
Because the GPU usage is so small (compare with what i see from another website)
Here is the code i run (similar from website).

import torch
import torchvision
import torchvision.transforms as transforms
import matplotlib.pyplot as plt
import numpy as np
import torch.nn as nn
import torch.nn.functional as F

def run():
    # ==================================================================================
    transform = transforms.Compose(
        [transforms.ToTensor(),
         transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])

    trainset = torchvision.datasets.CIFAR10(root='./data', train=True,
                                            download=True, transform=transform)
    trainloader = torch.utils.data.DataLoader(trainset, batch_size=4,
                                              shuffle=True, num_workers=2)

    testset = torchvision.datasets.CIFAR10(root='./data', train=False,
                                           download=True, transform=transform)
    testloader = torch.utils.data.DataLoader(testset, batch_size=4,
                                             shuffle=False, num_workers=2)

    classes = ('plane', 'car', 'bird', 'cat',
               'deer', 'dog', 'frog', 'horse', 'ship', 'truck')

    # ==================================================================================
    # # functions to show an image

    # def imshow(img):
    #     img = img / 2 + 0.5     # unnormalize
    #     npimg = img.numpy()
    #     plt.imshow(np.transpose(npimg, (1, 2, 0)))
    #     plt.show()

    # # get some random training images
    # dataiter = iter(trainloader)
    # images, labels = dataiter.next()

    # # show images
    # imshow(torchvision.utils.make_grid(images))
    # # print labels
    # print(' '.join('%5s' % classes[labels[j]] for j in range(4)))
    # ==================================================================================
    class Net(nn.Module):
        def __init__(self):
            super(Net, self).__init__()
            self.conv1 = nn.Conv2d(3, 6, 5)
            self.pool = nn.MaxPool2d(2, 2)
            self.conv2 = nn.Conv2d(6, 16, 5)
            self.fc1 = nn.Linear(16 * 5 * 5, 120)
            self.fc2 = nn.Linear(120, 84)
            self.fc3 = nn.Linear(84, 10)

        def forward(self, x):
            x = self.pool(F.relu(self.conv1(x)))
            x = self.pool(F.relu(self.conv2(x)))
            x = x.view(-1, 16 * 5 * 5)
            x = F.relu(self.fc1(x))
            x = F.relu(self.fc2(x))
            x = self.fc3(x)
            return x
    net = Net()
    device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
    print(device)
    net.to(device)
    # ==================================================================================
    import torch.optim as optim

    criterion = nn.CrossEntropyLoss().to(device)
    optimizer = optim.SGD(net.parameters(), lr=0.001, momentum=0.9)
    # ==================================================================================
    for epoch in range(2):  # loop over the dataset multiple times

        running_loss = 0.0
        for i, data in enumerate(trainloader, 0):
            # get the inputs; data is a list of [inputs, labels]
            inputs, labels = data[0].to(device), data[1].to(device)

            # zero the parameter gradients
            optimizer.zero_grad()

            # forward + backward + optimize
            outputs = net(inputs).to(device)
            loss = criterion(outputs, labels).to(device)
            loss.backward()
            optimizer.step()

            # print statistics
            running_loss += loss.item()
            if i % 2000 == 1999:    # print every 2000 mini-batches
                print('[%d, %5d] loss: %.3f' %
                      (epoch + 1, i + 1, running_loss / 2000))
                running_loss = 0.0

    print('Finished Training')
    # ==================================================================================
    PATH = './cifar_net.pth'
    torch.save(net.state_dict(), PATH)

    # ==================================================================================
    # ==================================================================================
    # ==================================================================================

if __name__ == '__main__':
    run()


title: What are saved in optimizer's state_dict? what "state","param_groups" stands for?

body:
When we use Adam optimizer, if we want to continue train a network from a pretrained model, we not only should load "model.state_dict", but also "optimizer.state_dict". And, if we modified our network's structure, we should also modify saved optimizer's state_dict to make our loading successful.

But I don't understand some params in saved "optimizer.state_dict". like optim_dict["state"] (dict_keys(['step', 'exp_avg', 'exp_avg_sq', 'max_exp_avg_sq']))and optim_dict['param_groups'][0]['params']. There are many of numbers like these:

 b['optimizer_state_dict']['state'].keys()
Out[71]: dict_keys([140623218628000, 140623218628072, 140623218628216, 140623218628360, 140623218628720, 140623218628792, 140623218628936, 140623218629080, 140623218629656, 140623218629728, 140623218629872, 140623218630016, 140623218630376, 140623218630448, 140623218716744, 140623218716816, 140623218717392, 140623218717464, 140623218717608, 140623218717752, 140623218718112, 140623218718184, 140623218718328, 140623218718472, 140623218719048, 140623218719120, 140623218719264, 140623218719408, 140623218719768, 140623218719840, 140623218719984, 140623218720128, 140623218720704, 140623209943112, 140623209943256, 140623209943400, 140623209943760, 140623209943832, 140623209943976, 140623209944120, 140623209944696, 140623209944768, 140623209944912, 140623209945056, 140623209945416, 140623209945488, 140623209945632, 140623209945776, 140623209946352, 140623209946424, 140623209946568, 140623209946712, 140623209947072, 140623210041416, 140623210041560, 140623210041704, 140623244033768, 140623244033840, 140623244033696, 140623244033912, 140623244033984, 140623244070984, 140623244071056, 140623244071128, 140623429501576, 140623244071200, 140623244071272, 140623244071344, 140623244071416, 140623244071488, 140623244071560, 140623244071632, 140623244071848, 140623244071920, 140623244072064, 140623244072208, 140623244072424, 140623244072496, 140623244072640, 140623244072784, 140623244073216, 140623244073288, 140623244073432, 140623244073576, 140623244073792, 140623244073864, 140623244074008, 140623244074152, 140623244074584, 140623244074656, 140623244074800, 140623244074944, 140623218540760, 140623218540832, 140623218540976, 140623218541120, 140623218541552, 140623218541624, 140623218541768, 140623218541912, 140623218542128, 140623218542200, 140623218542344, 140623218542488, 140623218542920, 140623218542992, 140623218543136, 140623218543280, 140623218543496, 140623218543568, 140623218543712, 140623218543856, 140623218544288, 140623218544360, 140623218544504, 140623218626632, 140623218626992, 140623218627064, 140623218627208, 140623218627352, 140623218627784, 140623218629440, 140623218717176, 140623218718832, 140623218720488, 140623209944480, 140623209946136, 140623210043000])

In [44]: b['optimizer_state_dict']['state'][140623218628072].keys()
Out[44]: dict_keys(['step', 'exp_avg', 'exp_avg_sq', 'max_exp_avg_sq'])

In [45]: b['optimizer_state_dict']['state'][140623218628072]['exp_avg'].shape
Out[45]: torch.Size([480])


title: Unable to import pytorch_lightning on google colab

body:
I have done the following:
!pip install pytorch_lightning -qqq
import pytorch_lightning

But get the following error:
ImportError                               Traceback (most recent call last)
<ipython-input-7-d883b15aac58> in <module>()
----> 1 import pytorch_lightning

----------------------------------9 frames------------------------------------------------
/usr/local/lib/python3.7/dist-packages/pytorch_lightning/utilities/apply_func.py in <module>()
     26
     27 if _TORCHTEXT_AVAILABLE:
---> 28     from torchtext.data import Batch
     29 else:
     30     Batch = type(None)

ImportError: cannot import name 'Batch' from 'torchtext.data' (/usr/local/lib/python3.7/dist-packages/torchtext/data/__init__.py)

What could the issue be?


title: How to mask a 3D tensor with 2D mask and keep the dimensions of original vector?

body:
Suppose, I have a 3D tensor A

A = torch.arange(24).view(4, 3, 2)
print(A)

and require masking it using 2D tensor

mask = torch.zeros((4, 3), dtype=torch.int64)  # or dtype=torch.ByteTensor
mask[0, 0] = 1
mask[1, 1] = 1
mask[3, 0] = 1
print('Mask: ', mask)

Using masked_select functionality from PyTorch leads to the following error.

torch.masked_select(X, (mask == 1))

---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-72-fd6809d2c4cc> in <module>
     12
     13 # Select based on new mask
---> 14 Y = torch.masked_select(X, (mask == 1))
     15 #Y = X * mask_
     16 print(Y)

RuntimeError: The size of tensor a (2) must match the size of tensor b (3) at non-singleton dimension 2

How to mask a 3D tensor with a 2D mask and keep the dimensions of the original vector? Any hints will be appreciated.


title: RuntimeError: leaf variable has been moved into the graph interior

body:
I try to use pytorch for autogradient. When I'm testing is, I met the error. My code is as below:

w11 = torch.rand((100,2), requires_grad=True)
w12 = torch.rand((100,2), requires_grad=True)
w12[:,1] = w12[:,1] + 1
w13 = torch.rand((100,2), requires_grad=True)
w13[:,1] = w13[:,1] + 2
out1=(w11-w12)**2
out2=out1.mean()
out2.backward(retain_graph=True)


title: PyTorch LSTM input dimension

body:
I'm trying train a simple 2 layer neural network with PyTorch LSTMs and I'm having trouble interpreting the PyTorch documentation. Specifically, I'm not too sure how to go about with the shape of my training data.

What I want to do is train my network on a very large dataset through mini-batches, where each batch is say, 100 elements long. Each data element will have 5 features. The documentation states that the input to the layer should be of shape (seq_len, batch_size, input_size). How should I go about shaping the input?

I've been following this post: https://discuss.pytorch.org/t/understanding-lstm-input/31110/3
and if I'm interpreting this correctly, each minibatch should be of shape (100, 100, 5). But in this case, what's the difference between seq_len and batch_size? Also, would this mean that the first layer that the input LSTM layer should have 5 units?

Thank you!


title: How to add to pytorch tensor at indices?

body:
I have to admit, I'm a bit confused by the scatter* and index* operations - I'm not sure any of them do exactly what I'm looking for, which is very simple:

Given some 2-D tensor

z = tensor([[1., 1., 1., 1.],
            [1., 1., 1., 1.],
            [1., 1., 1., 1.]])

And a list (or tensor?) of 2-d indexes:

inds = tensor([[0, 0],
               [1, 1],
               [1, 2]])

I want to add a scalar to z at those indexes (and do it efficiently):

znew = z.something_add(inds, 3)
->
znew = tensor([[4., 1., 1., 1.],
               [1., 4., 4., 1.],
               [1., 1., 1., 1.]])

If I have to I can make that scalar a tensor of whatever shape (where all elements = 3), but I'd rather not...


I am trying to use OpenNMT-py with python 2.7. OpenNMT-py requires torchtext, so I installed it but now when I am running my program, I am getting the following error message.

Traceback (most recent call last):
  File "examples/StackPointerParser.py", line 23, in <module>
    from neuronlp2.io import get_logger, conllx_stacked_data
  File "./neuronlp2/__init__.py", line 7, in <module>
    from . import models
  File "./neuronlp2/models/__init__.py", line 4, in <module>
    from .parsing import *
  File "./neuronlp2/models/parsing.py", line 15, in <module>
    from onmt.modules import LayerNorm, Transformer
  File "/home/wasiahmad/software/anaconda2/lib/python2.7/site-packages/onmt/__init__.py", line 1, in <module>
    import onmt.io
  File "/home/wasiahmad/software/anaconda2/lib/python2.7/site-packages/onmt/io/__init__.py", line 1, in <module>
    from onmt.io.IO import collect_feature_vocabs, make_features, \
  File "/home/wasiahmad/software/anaconda2/lib/python2.7/site-packages/onmt/io/IO.py", line 8, in <module>
    import torchtext.data
  File "/home/wasiahmad/software/anaconda2/lib/python2.7/site-packages/torchtext/__init__.py", line 1, in <module>
    from . import data
  File "/home/wasiahmad/software/anaconda2/lib/python2.7/site-packages/torchtext/data/__init__.py", line 4, in <module>
    from .field import RawField, Field, ReversibleField, SubwordField, NestedField, LabelField
  File "/home/wasiahmad/software/anaconda2/lib/python2.7/site-packages/torchtext/data/field.py", line 61, in <module>
    class Field(RawField):
  File "/home/wasiahmad/software/anaconda2/lib/python2.7/site-packages/torchtext/data/field.py", line 115, in Field
    torch.float32: float,
AttributeError: 'module' object has no attribute 'float32'

I tried to look for a solution to resolve this issue but couldn't find any. Any help would be appreciated.


title: Default dilation value in PyTorch

body:
As given in the documentation of PyTorch, the layer Conv2d uses a default dilation of 1. Does this mean that if I want to create a simple conv2d layer  I would have to write

nn.conv2d(in_channels = 3, out_channels = 64, kernel_size = 3, dilation = 0)

instead of simply writing

nn.conv2d(in_channels = 3, out_channels = 64, kernel_size = 3)

Or is it the case that in PyTorch dilation = 1 means same as dilation = 0 as given here in the Dilated Convolution section?


title: Pytorch BatchNorm2d RuntimeError: running_mean should contain 64 elements not 0

body:
I'm using Octave Convolutions and have set up a BatchNorm2d adaptation that for some reasen is giving me

RuntimeError: running_mean should contain 64 elements not 0

I've set up some debugging prints to check what was wrong with my Tensors' dimensions, but was unable to find it.
Here is my class:

class _BatchNorm2d(nn.Module):
  def __init__(self, num_features, alpha_in=0, alpha_out=0, eps=1e-5, momentum=0.1, affine=True,
               track_running_stats=True):
    super(_BatchNorm2d, self).__init__()
    hf_ch = int(num_features * (1 - alpha_out))
    lf_ch = num_features - hf_ch
    self.bnh = nn.BatchNorm2d(hf_ch)
    self.bnl = nn.BatchNorm2d(lf_ch)
  def forward(self, x):
    if isinstance(x, tuple):
        hf, lf = x
        print("IN ON BN: ",lf.shape if lf is not None else None) #DEBUGGING PRINT
        print(self.bnl)  #DEBUGGING PRINT
        hf = self.bnh(hf) if type(hf) == torch.Tensor else hf
        lf = self.bnh(lf) if type(lf) == torch.Tensor else lf #THIS IS THE LINE ACCUSING THE ERROR
        print("ENDED BN")
        return hf, lf
    else:
        return self.bnh(x)

Here is the printing error:

IN ON BN:  torch.Size([32, 64, 3, 3])
BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)

It seems to me the function should have worked, since x has 64 channels and bn expects 64 channels.

EDIT:
May also be important to mention the error only happens on alpha value of 1. However, I do not understand it, since the volumes are still the same.


title: using huggingface Trainer with distributed data parallel

body:
To speed up performace I looked into pytorches DistributedDataParallel and tried to apply it to transformer Trainer.
The pytorch examples for DDP states that this should at least be faster:

DataParallel is single-process, multi-thread, and only works on a single machine, while DistributedDataParallel is multi-process and works for both single- and multi- machine training. DataParallel is usually slower than DistributedDataParallel even on a single machine due to GIL contention across threads, per-iteration replicated model, and additional overhead introduced by scattering inputs and gathering outputs.

My DataParallel trainer looks like this:
import os
from datetime import datetime
import sys
import torch
from transformers import Trainer, TrainingArguments, BertConfig

training_args = TrainingArguments(
        output_dir=os.path.join(path_storage, 'results', "mlm"),  # output directory
        num_train_epochs=1,  # total # of training epochs
        gradient_accumulation_steps=2,  # for accumulation over multiple steps
        per_device_train_batch_size=4,  # batch size per device during training
        per_device_eval_batch_size=4,  # batch size for evaluation
        logging_dir=os.path.join(path_storage, 'logs', "mlm"),  # directory for storing logs
        evaluate_during_training=False,
        max_steps=20,
    )

mlm_train_dataset = ProteinBertMaskedLMDataset(
        path_vocab, os.path.join(path_storage, "data", "uniparc", "uniparc_train_sorted.h5"),
)

mlm_config = BertConfig(
        vocab_size=mlm_train_dataset.tokenizer.vocab_size,
        max_position_embeddings=mlm_train_dataset.input_size
)
mlm_model = ProteinBertForMaskedLM(mlm_config)
trainer = Trainer(
   model=mlm_model,  # the instantiated  Transformers model to be trained
   args=training_args,  # training arguments, defined above
   train_dataset=mlm_train_dataset,  # training dataset
   data_collator=mlm_train_dataset.collate_fn,
)
print("build trainer with on device:", training_args.device, "with n gpus:", training_args.n_gpu)
start = datetime.now()
trainer.train()
print(f"finished in {datetime.now() - start} seconds")

The output:
build trainer with on device: cuda:0 with n gpus: 4
finished in 0:02:47.537038 seconds

My DistributedDataParallel trainer is build like this:
def create_transformer_trainer(rank, world_size, train_dataset, model):
    os.environ['MASTER_ADDR'] = 'localhost'
    os.environ['MASTER_PORT'] = '12355'
    os.environ["RANK"] = str(rank)
    os.environ["WORLD_SIZE"] = str(world_size)

    training_args = TrainingArguments(
        output_dir=os.path.join(path_storage, 'results', "mlm"),  # output directory
        num_train_epochs=1,  # total # of training epochs
        gradient_accumulation_steps=2,  # for accumulation over multiple steps
        per_device_train_batch_size=4,  # batch size per device during training
        per_device_eval_batch_size=4,  # batch size for evaluation
        logging_dir=os.path.join(path_storage, 'logs', "mlm"),  # directory for storing logs
        local_rank=rank,
        max_steps=20,
    )

    trainer = Trainer(
        model=model,  # the instantiated  Transformers model to be trained
        args=training_args,  # training arguments, defined above
        train_dataset=train_dataset,  # training dataset
        data_collator=train_dataset.collate_fn,
    )
    print("build trainer with on device:", training_args.device, "with n gpus:", training_args.n_gpu)
    start = datetime.now()
    trainer.train()
    print(f"finished in {datetime.now() - start} seconds")

mlm_train_dataset = ProteinBertMaskedLMDataset(
    path_vocab, os.path.join(path_storage, "data", "uniparc", "uniparc_train_sorted.h5"))

mlm_config = BertConfig(
    vocab_size=mlm_train_dataset.tokenizer.vocab_size,
    max_position_embeddings=mlm_train_dataset.input_size
)
mlm_model = ProteinBertForMaskedLM(mlm_config)
torch.multiprocessing.spawn(create_transformer_trainer,
     args=(4, mlm_train_dataset, mlm_model),
     nprocs=4,
     join=True)

The output:
The current process just got forked. Disabling parallelism to avoid deadlocks...
To disable this warning, please explicitly set TOKENIZERS_PARALLELISM=(true | false)
The current process just got forked. Disabling parallelism to avoid deadlocks...
To disable this warning, please explicitly set TOKENIZERS_PARALLELISM=(true | false)
The current process just got forked. Disabling parallelism to avoid deadlocks...
To disable this warning, please explicitly set TOKENIZERS_PARALLELISM=(true | false)
The current process just got forked. Disabling parallelism to avoid deadlocks...
To disable this warning, please explicitly set TOKENIZERS_PARALLELISM=(true | false)
The current process just got forked. Disabling parallelism to avoid deadlocks...
To disable this warning, please explicitly set TOKENIZERS_PARALLELISM=(true | false)
build trainer with on device: cuda:1 with n gpus: 1
build trainer with on device: cuda:2 with n gpus: 1
build trainer with on device: cuda:3 with n gpus: 1
build trainer with on device: cuda:0 with n gpus: 1
finished in 0:04:15.937331 seconds
finished in 0:04:16.899411 seconds
finished in 0:04:16.938141 seconds
finished in 0:04:17.391887 seconds

About the inital forking warning: What is exaclty forked and is this expected?
And about the resulting time: Is the trainer incorrectly used since it seemed to be a lot slower than the DataParallel approach?


title: Is NVIDIA GeForce GT 635M suitable for Deep Learning?

body: My GPU model is NVIDIA GeForce GT 635M and on NVIDIA site it is said that this GPU  is CUDA-enabled. Can I use TensorFlow or PyTorch or any other kind of Deep Learning platforms with this GPU or it is not suitable?


title: Pytorch datasets.ImageFolder flag FileNotFoundError: [Errno 2] No such file or directory: '\u2068/

body:
Trying to load data from the local directory in Pytorch using dataset.ImageLoader but getting FileNotFoundError...

import torch
from torchvision import datasets, transforms

data_dir = '/Users/Desktop/Udacity/AI for Trading/deep-learning-v2-pytorch/intro-to-pytorch/data/Cat_Dog_data/train'

transform = transforms.Compose([transforms.Resize(255),
                                transforms.CenterCrop(224),
                                transforms.ToTensor()])

dataset = datasets.ImageFolder(root=data_dir, transform=transform)
dataloader = torch.utils.data.DataLoader(dataset, batch_size=32, shuffle=True)

FileNotFoundError: [Errno 2] No such file or directory: '\u2068/Users/Desktop/Udacity/AI for Trading/deep-learning-v2-pytorch/intro-to-pytorch/data\u2069/Cat_Dog_data\u2069/\u2068train\u2069'

Answer: Retyping the Image Dir Path manually instead of copy-pasting solved the problem.


title: "error while loading shared libraries: libnvinfer.so.7: cannot open shared object file: No such file or directory" when running TRTorch sample

body:
I'm trying to compile my pytorch model with TRTorch engine.
I've installed TRTorch according to this link.
When the sample code is run (with the below command from this link) the given error arise:
sudo bazel run //cpp/trtorchexec -- $(realpath /home/TRTorch/tests/modules/alexnet_scripted.jit.pt) "(1,3,227,227)"

error while loading shared libraries: libnvinfer.so.7: cannot open shared object file: No such file or directory

Also, the LD_LIBRARY_PATH is set correctly.

export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/home/TensorRT/TensorRT-7.0.0.11/lib

More info:

TRTorch: latest version (python package and binary)
TensorRT: 7.0.0.11
Pytorch: 1.5.1
CUDA: 10.2
Python: 3.6


title: Model returns a Nan value

body:
I was trying to build a neural network with 4 input nodes/ features and just one output feature(0/1). I wrote this code and it runs but while training the model returns NaN. I debugged too and weights and biases are fine until they go through the model.

From what I've searched so far, this could be a problem in the way I am passing the data.
My input data is : tensor([[0.0000e+00, 0.0000e+00, 0.0000e+00, 1.5340e+00],
        [1.5000e+01, 1.0000e-01, 2.4210e+00, 3.0000e+01],
        [3.0000e+00, 2.2000e-01, 2.2000e-01, 4.5000e+01],
        ...,
        [1.0000e+00, 2.0000e-02, 2.0000e-02, 1.5000e+01],
        [6.0000e+00, 2.0000e-01, 2.0000e-01, 1.5000e+01],
        [1.7000e+01, 5.2400e-01, 5.2400e-01, 2.0000e+00]], dtype=torch.float64)

import torch
from torchvision import datasets, transforms
import pandas as pd
import numpy as np
from torch.autograd import Variable
# Import tensor dataset & data loader
from torch.utils.data import TensorDataset, DataLoader
from torch import nn, optim
import torch.nn.functional as F

file = pd.read_csv('ks-projects-201801.csv')
array = np.array(file.values)
result = np.empty(len(array))
input_data = np.empty((len(array), 4))
for i in range(len(array)):
    input_data[i] = np.array([array[i][10], array[i][12]/1000, array[i][13]/1000, array[i][14]/1000])
    if array[i][9] == 'successful':
        result[i] = 1
    else:
        result[i] = 0

input_node = Variable(torch.from_numpy(input_data))
output = torch.from_numpy(result)
print(input_node)
print(output)

train_ds = TensorDataset(input_node.squeeze(), output.squeeze())

batch_size = 5
train_dl = DataLoader(train_ds, batch_size, shuffle=True)

This is the actual model and training

model = nn.Linear(4, 1)
print(model.weight)
print(model.bias)
criterion = nn.MSELoss()
optimizer = torch.optim.SGD(model.parameters(), lr=0.003)

epochs = 5
model = model.double()
for e in range(epochs):
    running_loss = 0
    for xb, yb in train_dl:
        optimizer.zero_grad()
        res = model(xb)
        loss = criterion(res, yb)
        loss.backward()
        optimizer.step()

        running_loss += loss.item()
    else:
        print(f"model : {loss}")

This prints out model: nan for every epoch and terminates. I am very new to pytorch and I'm not sure how to handle this problem.


title: Pytorch broadcasting product of two tensors

body:
I want to multiply two tensors, here is what I have got:

A tensor of shape (20, 96, 110)
B tensor of shape (20, 16, 110)

The first index is for batch size.
What I want to do is essentially take each tensor from B - (20, 1, 110), for example, and with that, I want to multiply each A tensor (20, n, 110).
So the product will be at the end: tensor AB which shape is (20, 96 * 16, 110).

So I want to multiply each tensor from A by broadcasting with B.
Is there a method in PyTorch that does it?


title: epsilon parameter in Adam opitmizer

body:
Using pyTorch and tensorflow (TF), I was wandering how the Adam optimizer is implemented for curiosity. And I do not know if I am wrong or not but it seems to me that the two implementations differ and the pyTorch one is the original one from https://arxiv.org/pdf/1412.6980.pdf.

My problem comes from the eps-parameter. Using the TF implentation seems to lead to a time-and-b2 dependance of this parameter, namely

q(t+1) = q(t) - \gamma * sqrt[(1-b2^t)]/(1-b1^t) * m(t)/[sqrt[v(t)]+eps]

which in the original algorithm notation can be reformulate as

 q(t+1) = q(t) - \gamma * mhat(t)/[sqrt[vhat(t)]+ eps/sqrt[(1-b2^t)]]

and this point out the variation of the eps-parameter which is not the case neither in the original algorithm neither in the pyTorch implementation.

Am I wrong? or it is well known?
Thanks for your help.


title: LSTM Sequence length

body:
Greetings to everyone!

I want to ask if there is an optimal sequence length of a LSTM network in general, or in terms of time series prediction problems?

I read about vanishing gradient or exploding gradient problems that very long RNN networks had and LSTM tried to solve and succeeded to a certain extent.

I also heard about techniques to handle very large sequences with LSTM’s and RNN’s in general like: truncating sequences, summarizing sequences, truncating backpropagation through time or even using an Encoder-Decoder architecture.

I asked this question because I didn’t find a research paper about this, only this blog post that stated an optimal sequence length between 10-30.

Thanks in advance!

Have a nice day!


title: torch.nn.DataParallel not working in pytorch

body:
I have used DataParallel in my script for multiple Gpu.The torch.nn.DataParallel is only using the gpu with id 0 and not utilizing the gpu with id 1.So the utilization is very low.Here is the full code:

from __future__ import print_function

from miscc.utils import mkdir_p
from miscc.utils import build_super_images
from miscc.losses import sent_loss, words_loss
from miscc.config import cfg, cfg_from_file

from datasets import TextDataset
from datasets import prepare_data

from model import RNN_ENCODER, CNN_ENCODER

import os
import sys
import time
import random
import pprint
import datetime
import dateutil.tz
import argparse
import numpy as np
from PIL import Image

import torch
import torch.nn as nn
import torch.optim as optim
from torch.autograd import Variable
import torch.backends.cudnn as cudnn
import torchvision.transforms as transforms

dir_path = (os.path.abspath(os.path.join(os.path.realpath(__file__), './.')))
sys.path.append(dir_path)

UPDATE_INTERVAL = 200
def parse_args():
    parser = argparse.ArgumentParser(description='Train a DAMSM network')
    parser.add_argument('--cfg', dest='cfg_file',
                        help='optional config file',
                        default='cfg/bird.yml', type=str)
    parser.add_argument('--gpu', dest='gpu_id', type=int, default=0)
    parser.add_argument('--data_dir', dest='data_dir', type=str, default='')
    parser.add_argument('--manualSeed', type=int, help='manual seed')
    args = parser.parse_args()
    return args

def train(dataloader, cnn_model, rnn_model, batch_size, labels, optimizer, epoch, ixtoword, image_dir):
    torch.nn.DataParallel(cnn_model.train(),device_ids=[0,1],dim=1).cuda()
    torch.nn.DataParallel(rnn_model.train(),device_ids=[0,1],dim=1).cuda()
    s_total_loss0 = 0
    s_total_loss1 = 0
    w_total_loss0 = 0
    w_total_loss1 = 0
    count = (epoch + 1) * len(dataloader)
    start_time = time.time()
    for step, data in enumerate(dataloader, 0):
        # print('step', step)
        rnn_model.zero_grad()
        cnn_model.zero_grad()

        imgs, captions, cap_lens, \
            class_ids, keys = prepare_data(data)

        # words_features: batch_size x nef x 17 x 17
        # sent_code: batch_size x nef
        words_features, sent_code = cnn_model(imgs[-1])
        # --> batch_size x nef x 17*17
        nef, att_sze = words_features.size(1), words_features.size(2)
        # words_features = words_features.view(batch_size, nef, -1)

        hidden = rnn_model.init_hidden(batch_size)
        # words_emb: batch_size x nef x seq_len
        # sent_emb: batch_size x nef
        words_emb, sent_emb = rnn_model(captions, cap_lens, hidden)

        w_loss0, w_loss1, attn_maps = words_loss(words_features, words_emb, labels,
                                                 cap_lens, class_ids, batch_size)
        w_total_loss0 += w_loss0.data
        w_total_loss1 += w_loss1.data
        loss = w_loss0 + w_loss1

        s_loss0, s_loss1 = \
            sent_loss(sent_code, sent_emb, labels, class_ids, batch_size)
        loss += s_loss0 + s_loss1
        s_total_loss0 += s_loss0.data
        s_total_loss1 += s_loss1.data
        #
        loss.backward()
        #
        # `clip_grad_norm` helps prevent
        # the exploding gradient problem in RNNs / LSTMs.
        torch.nn.utils.clip_grad_norm(rnn_model.parameters(),
                                      cfg.TRAIN.RNN_GRAD_CLIP)
        optimizer.step()

        if step % UPDATE_INTERVAL == 0:
            count = epoch * len(dataloader) + step

            s_cur_loss0 = s_total_loss0[0] / UPDATE_INTERVAL
            s_cur_loss1 = s_total_loss1[0] / UPDATE_INTERVAL

            w_cur_loss0 = w_total_loss0[0] / UPDATE_INTERVAL
            w_cur_loss1 = w_total_loss1[0] / UPDATE_INTERVAL

            elapsed = time.time() - start_time
            print('| epoch {:3d} | {:5d}/{:5d} batches | ms/batch {:5.2f} | '
                  's_loss {:5.2f} {:5.2f} | '
                  'w_loss {:5.2f} {:5.2f}'
                  .format(epoch, step, len(dataloader),
                          elapsed * 1000. / UPDATE_INTERVAL,
                          s_cur_loss0, s_cur_loss1,
                          w_cur_loss0, w_cur_loss1))
            s_total_loss0 = 0
            s_total_loss1 = 0
            w_total_loss0 = 0
            w_total_loss1 = 0
            start_time = time.time()
            # attention Maps
            img_set, _ = \
                build_super_images(imgs[-1].cpu(), captions,
                                   ixtoword, attn_maps, att_sze)
            if img_set is not None:
                im = Image.fromarray(img_set)
                fullpath = '%s/attention_maps%d.png' % (image_dir, step)
                im.save(fullpath)
    return count

def evaluate(dataloader, cnn_model, rnn_model, batch_size):
    cnn_model.eval().cuda()
    rnn_model.eval().cuda()
    s_total_loss = 0
    w_total_loss = 0
    for step, data in enumerate(dataloader, 0):
        real_imgs, captions, cap_lens, \
                class_ids, keys = prepare_data(data)

        words_features, sent_code = cnn_model(real_imgs[-1])
        # nef = words_features.size(1)
        # words_features = words_features.view(batch_size, nef, -1)

        hidden = rnn_model.init_hidden(batch_size)
        words_emb, sent_emb = rnn_model(captions, cap_lens, hidden)

        w_loss0, w_loss1, attn = words_loss(words_features, words_emb, labels,
                                            cap_lens, class_ids, batch_size)
        w_total_loss += (w_loss0 + w_loss1).data

        s_loss0, s_loss1 = \
            sent_loss(sent_code, sent_emb, labels, class_ids, batch_size)
        s_total_loss += (s_loss0 + s_loss1).data

        if step == 50:
            break

    s_cur_loss = s_total_loss[0] / step
    w_cur_loss = w_total_loss[0] / step

    return s_cur_loss, w_cur_loss

def build_models():
    # build model ############################################################
    text_encoder = RNN_ENCODER(dataset.n_words, nhidden=cfg.TEXT.EMBEDDING_DIM)
    image_encoder = CNN_ENCODER(cfg.TEXT.EMBEDDING_DIM)
    labels = Variable(torch.LongTensor(range(batch_size)))
    start_epoch = 0
    if cfg.TRAIN.NET_E != '':
        state_dict = torch.load(cfg.TRAIN.NET_E)
        text_encoder.load_state_dict(state_dict)
        print('Load ', cfg.TRAIN.NET_E)
        #
        name = cfg.TRAIN.NET_E.replace('text_encoder', 'image_encoder')
        state_dict = torch.load(name)
        image_encoder.load_state_dict(state_dict)
        print('Load ', name)

        istart = cfg.TRAIN.NET_E.rfind('_') + 8
        iend = cfg.TRAIN.NET_E.rfind('.')
        start_epoch = cfg.TRAIN.NET_E[istart:iend]
        start_epoch = int(start_epoch) + 1
        print('start_epoch', start_epoch)
    if cfg.CUDA:
        text_encoder = text_encoder.cuda()#torch.nn.DataParallel(text_encoder,device_ids=[0,1],dim=1)
        image_encoder = image_encoder.cuda()#torch.nn.DataParallel(image_encoder,device_ids=[0,1],dim=1)
        labels = labels.cuda()#torch.nn.DataParallel(labels,device_ids=[0,1],dim=1)

    return text_encoder, image_encoder, labels, start_epoch

if __name__ == "__main__":
    args = parse_args()
    if args.cfg_file is not None:
        cfg_from_file(args.cfg_file)

    if args.gpu_id == -1:
        cfg.CUDA = False
    else:
        cfg.GPU_ID =args.gpu_id

    if args.data_dir != '':
        cfg.DATA_DIR = args.data_dir
    print('Using config:')
    pprint.pprint(cfg)

    if not cfg.TRAIN.FLAG:
        args.manualSeed = 100
    elif args.manualSeed is None:
        args.manualSeed = random.randint(1, 10000)
    random.seed(args.manualSeed)
    np.random.seed(args.manualSeed)
    torch.manual_seed(args.manualSeed)
    if cfg.CUDA:
        torch.cuda.manual_seed_all(args.manualSeed)

    ##########################################################################
    now = datetime.datetime.now(dateutil.tz.tzlocal())
    timestamp = now.strftime('%Y_%m_%d_%H_%M_%S')
    output_dir = '../output/%s_%s_%s' % \
        (cfg.DATASET_NAME, cfg.CONFIG_NAME, timestamp)

    model_dir = os.path.join(output_dir, 'Model')
    image_dir = os.path.join(output_dir, 'Image')
    mkdir_p(model_dir)
    mkdir_p(image_dir)

    #torch.cuda.set_device()
    cudnn.benchmark = True

    # Get data loader ##################################################
    imsize = cfg.TREE.BASE_SIZE * (2 ** (cfg.TREE.BRANCH_NUM-1))
    batch_size = cfg.TRAIN.BATCH_SIZE
    image_transform = transforms.Compose([
        transforms.Scale(int(imsize * 76 / 64)),
        transforms.RandomCrop(imsize),
        transforms.RandomHorizontalFlip()])
    dataset = TextDataset(cfg.DATA_DIR, 'train',
                          base_size=cfg.TREE.BASE_SIZE,
                          transform=image_transform)

    print(dataset.n_words, dataset.embeddings_num)
    assert dataset
    dataloader = torch.utils.data.DataLoader(
        dataset, batch_size=batch_size, drop_last=True,
        shuffle=True, num_workers=int(cfg.WORKERS))

    # # validation data #
    dataset_val = TextDataset(cfg.DATA_DIR, 'test',
                              base_size=cfg.TREE.BASE_SIZE,
                              transform=image_transform)
    dataloader_val = torch.utils.data.DataLoader(
        dataset_val, batch_size=batch_size, drop_last=True,
        shuffle=True, num_workers=int(cfg.WORKERS))

    # Train ##############################################################
    text_encoder, image_encoder, labels, start_epoch = build_models()
    para = list(text_encoder.parameters())
    for v in image_encoder.parameters():
        if v.requires_grad:
            para.append(v)
    # optimizer = optim.Adam(para, lr=cfg.TRAIN.ENCODER_LR, betas=(0.5, 0.999))
    # At any point you can hit Ctrl + C to break out of training early.
    try:
        lr = cfg.TRAIN.ENCODER_LR
        for epoch in range(start_epoch, cfg.TRAIN.MAX_EPOCH):
            optimizer = optim.Adam(para, lr=lr, betas=(0.5, 0.999))
            epoch_start_time = time.time()
            count = torch.nn.DataParallel(train(dataloader, image_encoder, text_encoder,
                          batch_size, labels, optimizer, epoch,
                          dataset.ixtoword, image_dir),device_ids=[0,1],dim=1).cuda()
            print('-' * 89)
            if len(dataloader_val) > 0:
                s_loss, w_loss = torch.nn.DataParallel(evaluate(dataloader_val, image_encoder,
                                          text_encoder, batch_size)).cuda()
                print('| end epoch {:3d} | valid loss '
                      '{:5.2f} {:5.2f} | lr {:.5f}|'
                      .format(epoch, s_loss, w_loss, lr))
            print('-' * 89)
            if lr > cfg.TRAIN.ENCODER_LR/10.:
                lr *= 0.98

            if (epoch % cfg.TRAIN.SNAPSHOT_INTERVAL == 0 or
                epoch == cfg.TRAIN.MAX_EPOCH):
                torch.save(image_encoder.state_dict(),
                           '%s/image_encoder%d.pth' % (model_dir, epoch))
                torch.save(text_encoder.state_dict(),
                           '%s/text_encoder%d.pth' % (model_dir, epoch))
                print('Save G/Ds models.')
    except KeyboardInterrupt:
        print('-' * 89)
        print('Exiting from training early')

I have read various articles regarding DataParallel and according to them this should have worked.Can Somebody help me in finding the solution to the problem?


I want to calculate AUROC using dataset with all instance zero. However, the following error occurred:

ValueError: Only one class present in y_true. ROC AUC score is not defined in that case

def computeAUROC (dataGT, dataPRED, classCount):

    outAUROC = []

    datanpGT = dataGT.cpu().numpy()
    datanpPRED = dataPRED.cpu().numpy()

    for i in range(classCount):
        outAUROC.append(roc_auc_score(datanpGT[:, i], datanpPRED[:, i]))

    return outAUROC


title: How can I access layers in a pytorch module by index?

body:
I am trying to write a pytorch module with multiple layers. Since I need the intermediate outputs I cannot put them all in a Sequantial as usual. On the other hand, since there are many layers, what I have in mind is to put the layers in a list and access them by index in a loop. Below describe what I am trying to achieve:
import torch
import torch.nn as nn
import torch.optim as optim

class MyModel(nn.Module):
    def __init__(self):
        super(MyModel, self).__init__()

        self.layer_list = []

        self.layer_list.append(nn.Linear(2,3))
        self.layer_list.append(nn.Linear(3,4))
        self.layer_list.append(nn.Linear(4,5))

    def forward(self, x):
        res_list = [x]
        for i in range(len(self.layer_list)):
            res_list.append(self.layer_list[i](res_list[-1]))
        return res_list

model = MyModel()
x = torch.randn(4,2)
y = model(x)

print(y)

optimizer = optim.Adam(model.parameters())

The forward method works fine, but when I want to set an optimizer the program says
ValueError: optimizer got an empty parameter list

It appears that the layers in the list are not registered here. What can I do?


title: PyTorch GAN RuntimeError: Caught RuntimeError in DataLoader worker process 0

body:
My data contains 120 JPEG images. When I want to train my model an error occurs. But when I use one of torchvision.datasets instead of my It works well.
batchSize = 64
imageSize = 64

transform=transforms.Compose([transforms.Resize(imageSize),transforms.ToTensor(),transforms.Normalize((0.5,0.5,0.5), (0.5, 0.5, 0.5)),])

dataset = dset.ImageFolder(root = './input/',  transform = transform)
dataloader = torch.utils.data.DataLoader(dataset, batch_size = batchSize,shuffle=True, num_workers = 1)

Error:
RuntimeError: Caught RuntimeError in DataLoader worker process 0.
Original Traceback (most recent call last):
  File "C:\Users\Szymon\Anaconda3\lib\site-packages\torch\utils\data\_utils\worker.py", line 178, in _worker_loop
    data = fetcher.fetch(index)
  File "C:\Users\Szymon\Anaconda3\lib\site-packages\torch\utils\data\_utils\fetch.py", line 47, in fetch
    return self.collate_fn(data)
  File "C:\Users\Szymon\Anaconda3\lib\site-packages\torch\utils\data\_utils\collate.py", line 79, in default_collate
    return [default_collate(samples) for samples in transposed]
  File "C:\Users\Szymon\Anaconda3\lib\site-packages\torch\utils\data\_utils\collate.py", line 79, in <listcomp>
    return [default_collate(samples) for samples in transposed]
  File "C:\Users\Szymon\Anaconda3\lib\site-packages\torch\utils\data\_utils\collate.py", line 55, in default_collate
    return torch.stack(batch, 0, out=out)
RuntimeError: invalid argument 0: Sizes of tensors must match except in dimension 0. Got 64 and 65 in dimension 2 at C:\w\1\s\tmp_conda_3.7_100118\conda\conda-bld\pytorch_1579082551706\work\aten\src\TH/generic/THTensor.cpp:612


title: How to install pytorch on Power 8 or PPC64 machine?

body: I am trying to install pytorch using conda on power 8 IBM machine. Although, I read articles from IBM blog, I couldn't install with success because I got stuck with compiling magma.


title: Can not insert a Tensor that requires grad as a constant

body:
I have made a model using pyro on top of pytorch. However, I get the following error:

h_0_contig = self.h_0.expand(1,size,self.rnn.hidden_size)

  TypeError: expand(): argument 'size' must be tuple of ints, but found element of type Tensor at pos 2

I thought it was because I was giving a tensor as an input. And then I explicitly gave it as an integer and then it gave me the following error:

  RuntimeError: Cannot insert a Tensor that requires grad as a constant. Consider making it a parameter or input, or detaching the gradient

  Tensor:

  (1,.,.) =
   Columns 1 to 26  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0

I have shown my code below in which I get the error (function guide). mini_batch, mini_batch_reversed, mini_batch_mask, and annealing factor, they all are tensors. I am using the code from the following:

https://github.com/pyro-ppl/pyro/blob/dev/examples/dmm/dmm.py

I don't know what is causing this error. Insights would be appreciated.

    def guide(self,mini_batch,mini_batch_reversed,mini_batch_mask,annealing_factor = torch.tensor(1.0)):

        #T_max = mini_batch.size(1)
        T_max = 247

        pyro.module('DMM',self)

        #size = mini_batch.size(0)
        size = 5
        print('type of size : ',type(size))

        print('rnn hidden size : ',self.rnn.hidden_size)

        h_0_contig = self.h_0.expand(1,size,self.rnn.hidden_size)

        rnn_output,_ = self.rnn(mini_batch_reversed,h_0_contig)

        rnn_output = dp.pad_and_reverse(rnn_output, mini_batch.size(1))

        z_prev = self.z_q_0.expand(mini_batch.size(0),self.z_q_0.size(0))

        with pyro.plate('z_minibatch',len(mini_batch)):
            for t in range(1,T_max+1):
                z_loc,z_scale = self.combiner(z_prev,rnn_output[:,t-1,:])

                z_dist = dist.Normal(z_loc,z_scale)

                with pyro.poutine.scale(None,annealing_factor):
                    z_t = pyro.sample("z_%d" % t,z_dist.mask(mini_batch_mask[:, t - 1:t]).to_event(1))

                z_prev = z_t


My code is

a=torch.randn(1,80,100,requires_grad=True)
torch.onnx.export(waveglow,a, "waveglow.onnx")

I am trying to export a PyTorch model to ONNX format so i can use it in TensorRT. while testing my model in PyTorch the input tensor dimension is (1,80,x) where x varies depending on the input text length(the model i am using is TTS model named waveglow)

when i try to run the above code for exporting the model to onnx, I am always getting this error

RuntimeError: Only tuples, lists and Variables supported as JIT inputs, but got NoneType

Please help


I'm having this error when executing my file. My system configurations are listed below.

OS: CentOS Linux 7
PyTorch 1.1.0
TensorFlow version: 1.2.0
Python version: 3.6.8
CUDA/cuDNN version: 8.0/7.0.5
GPU: Nvidia GPU GeForce GTX 1080

/pytorch/aten/src/ATen/native/cuda/IndexKernel.cu:53: lambda [](int)->auto::operator()(int)->auto: block: [1426,0,0], thread: [96,0,0] Assertion `index >= -sizes[i] && index < sizes[i] && "index out of bounds"` failed.
...
/pytorch/aten/src/ATen/native/cuda/IndexKernel.cu:53: lambda [](int)->auto::operator()(int)->auto: block: [1426,0,0], thread: [122,0,0] Assertion `index >= -sizes[i] && index < sizes[i] && "index out of bounds"` failed.
/pytorch/aten/src/ATen/native/cuda/IndexKernel.cu:53: lambda [](int)->auto::operator()(int)->auto: block: [1426,0,0], thread: [123,0,0] Assertion `index >= -sizes[i] && index < sizes[i] && "index out of bounds"` failed.
/pytorch/aten/src/ATen/native/cuda/IndexKernel.cu:53: lambda [](int)->auto::operator()(int)->auto: block: [1426,0,0], thread: [124,0,0] Assertion `index >= -sizes[i] && index < sizes[i] && "index out of bounds"` failed.
/pytorch/aten/src/ATen/native/cuda/IndexKernel.cu:53: lambda [](int)->auto::operator()(int)->auto: block: [1426,0,0], thread: [125,0,0] Assertion `index >= -sizes[i] && index < sizes[i] && "index out of bounds"` failed.
/pytorch/aten/src/ATen/native/cuda/IndexKernel.cu:53: lambda [](int)->auto::operator()(int)->auto: block: [1426,0,0], thread: [126,0,0] Assertion `index >= -sizes[i] && index < sizes[i] && "index out of bounds"` failed.
/pytorch/aten/src/ATen/native/cuda/IndexKernel.cu:53: lambda [](int)->auto::operator()(int)->auto: block: [1426,0,0], thread: [127,0,0] Assertion `index >= -sizes[i] && index < sizes[i] && "index out of bounds"` failed.
Traceback (most recent call last):
  File "./train.py", line 115, in <module>
    trainer.train()
  File "../../tasks/semantic/modules/trainer.py", line 239, in train
    show_scans=self.ARCH["train"]["show_scans"])
  File "../../tasks/semantic/modules/trainer.py", line 320, in train_epoch
    output = model(in_vol, proj_mask)
  File "/home/media-server/.pyenv/versions/anaconda3-5.0.0/envs/rangenet++/lib/python3.6/site-packages/torch/nn/modules/module.py", line 493, in __call__
    result = self.forward(*input, **kwargs)
  File "../../tasks/semantic/modules/segmentator.py", line 149, in forward
    y, skips = self.backbone(x)
  File "/home/media-server/.pyenv/versions/anaconda3-5.0.0/envs/rangenet++/lib/python3.6/site-packages/torch/nn/modules/module.py", line 493, in __call__
    result = self.forward(*input, **kwargs)
  File "../..//backbones/darknet.py", line 171, in forward
    x, skips, os = self.run_layer(x, self.conv1, skips, os)
  File "../..//backbones/darknet.py", line 154, in run_layer
    y = layer(x)
  File "/home/media-server/.pyenv/versions/anaconda3-5.0.0/envs/rangenet++/lib/python3.6/site-packages/torch/nn/modules/module.py", line 493, in __call__
    result = self.forward(*input, **kwargs)
  File "/home/media-server/.pyenv/versions/anaconda3-5.0.0/envs/rangenet++/lib/python3.6/site-packages/torch/nn/modules/conv.py", line 338, in forward
    self.padding, self.dilation, self.groups)
RuntimeError: cuDNN error: CUDNN_STATUS_INTERNAL_ERROR

I tried entering sudo rm -rf ~/.nv and rebooting, but it didn't work. RuntimeError: CUDNN_STATUS_INTERNAL_ERROR


title: How to get backward gradients from sparse matrices in PyTorch networks?

body:
I am following the PyTorch tutorial example:

https://pytorch.org/tutorials/beginner/pytorch_with_examples.html

The example runs without any problems, but as soon as I switched it to my own dataset (which is a sparse tensor, because it is too big to be used as a dense tensor), I run into this error:

RuntimeError                              Traceback (most recent call last)
<ipython-input-127-8b4999644085> in <module>()
 41     # Backward pass: compute gradient of the loss with respect to model
 42     # parameters
---> 43     loss.backward()
 44
 45     # Calling the step function on an Optimizer makes an update to its

~/miniconda3/envs/py3/lib/python3.6/site-packages/torch/tensor.py in backward(self, gradient, retain_graph, create_graph)
 91                 products. Defaults to ``False``.
 92         """
---> 93         torch.autograd.backward(self, gradient, retain_graph, create_graph)
 94
 95     def register_hook(self, hook):

~/miniconda3/envs/py3/lib/python3.6/site-packages/torch/autograd/__init__.py in backward(tensors, grad_tensors, retain_graph, create_graph, grad_variables)
 87     Variable._execution_engine.run_backward(
 88         tensors, grad_tensors, retain_graph, create_graph,
---> 89         allow_unreachable=True)  # allow_unreachable flag
 90
 91

RuntimeError: Expected object of type torch.FloatTensor but found type torch.sparse.FloatTensor for argument #2 'mat2'

I tried switching the optimizer (Adagrad, Adam), but it doesn't seem to work.

Edit: Added more of the error output. The error occurs on backward()


title: RuntimeError: DataLoader worker is killed by signal: Illegal instruction.

body:
I am using the following code. I run this on a GPU machine with below config:

nvcc: NVIDIA (R) Cuda compiler driver
Copyright (c) 2005-2017 NVIDIA Corporation
Built on Fri_Sep__1_21:08:03_CDT_2017
Cuda compilation tools, release 9.0, V9.0.176

I am using pytorch version 1.0.0.dev20181123.

I face this error: RuntimeError: DataLoader worker (pid 23646) is killed by signal: Illegal instruction.

Thinking that its a shared memory problem, I tried the solution here at: https://www.lucidarme.me/increase-shared-memory-limit/

It didn't help. Any pointers?

import argparse
import json
import os
import torch

#=====START: ADDED FOR DISTRIBUTED======
from distributed import init_distributed, apply_gradient_allreduce, reduce_tensor
from torch.utils.data.distributed import DistributedSampler
#=====END:   ADDED FOR DISTRIBUTED======

from torch.utils.data import DataLoader
from glow import WaveGlow, WaveGlowLoss
from mel2samp import Mel2Samp

def load_checkpoint(checkpoint_path, model, optimizer):
    assert os.path.isfile(checkpoint_path)
    checkpoint_dict = torch.load(checkpoint_path, map_location='cpu')
    iteration = checkpoint_dict['iteration']
    optimizer.load_state_dict(checkpoint_dict['optimizer'])
    model_for_loading = checkpoint_dict['model']
    model.load_state_dict(model_for_loading.state_dict())
    print("Loaded checkpoint '{}' (iteration {})" .format(
          checkpoint_path, iteration))
    return model, optimizer, iteration

def save_checkpoint(model, optimizer, learning_rate, iteration, filepath):
    print("Saving model and optimizer state at iteration {} to {}".format(
          iteration, filepath))
    model_for_saving = WaveGlow(**waveglow_config).cuda()
    model_for_saving.load_state_dict(model.state_dict())
    torch.save({'model': model_for_saving,
                'iteration': iteration,
                'optimizer': optimizer.state_dict(),
                'learning_rate': learning_rate}, filepath)

def train(num_gpus, rank, group_name, output_directory, epochs, learning_rate,
          sigma, iters_per_checkpoint, batch_size, seed, checkpoint_path):
    torch.manual_seed(seed)
    torch.cuda.manual_seed(seed)
    #=====START: ADDED FOR DISTRIBUTED======
    if num_gpus > 1:
        init_distributed(rank, num_gpus, group_name, **dist_config)
    #=====END:   ADDED FOR DISTRIBUTED======

    criterion = WaveGlowLoss(sigma)
    model = WaveGlow(**waveglow_config).cuda()

    #=====START: ADDED FOR DISTRIBUTED======
    if num_gpus > 1:
        model = apply_gradient_allreduce(model)
    #=====END:   ADDED FOR DISTRIBUTED======

    optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)

    # Load checkpoint if one exists
    iteration = 0
    if checkpoint_path != "":
        model, optimizer, iteration = load_checkpoint(checkpoint_path, model,
                                                      optimizer)
        iteration += 1  # next iteration is iteration + 1

    trainset = Mel2Samp(**data_config)
    # =====START: ADDED FOR DISTRIBUTED======
    train_sampler = DistributedSampler(trainset) if num_gpus > 1 else None
    # =====END:   ADDED FOR DISTRIBUTED======
    train_loader = DataLoader(trainset, num_workers=1, shuffle=False,
                              sampler=train_sampler,
                              batch_size=batch_size,
                              pin_memory=False,
                              drop_last=True)

    # Get shared output_directory ready
    if rank == 0:
        if not os.path.isdir(output_directory):
            os.makedirs(output_directory)
            os.chmod(output_directory, 0o775)
        print("output directory", output_directory)

    model.train()
    epoch_offset = max(0, int(iteration / len(train_loader)))
    # ================ MAIN TRAINNIG LOOP! ===================
    for epoch in range(epoch_offset, epochs):
        print("Epoch: {}".format(epoch))
        for i, batch in enumerate(train_loader):
            model.zero_grad()

            mel, audio = batch
            mel = torch.autograd.Variable(mel.cuda())
            audio = torch.autograd.Variable(audio.cuda())
            outputs = model((mel, audio))

            loss = criterion(outputs)
            if num_gpus > 1:
                reduced_loss = reduce_tensor(loss.data, num_gpus).item()
            else:
                reduced_loss = loss.item()
            loss.backward()
            optimizer.step()

            print("{}:\t{:.9f}".format(iteration, reduced_loss))

            if (iteration % iters_per_checkpoint == 0):
                if rank == 0:
                    checkpoint_path = "{}/waveglow_{}".format(
                        output_directory, iteration)
                    save_checkpoint(model, optimizer, learning_rate, iteration,
                                    checkpoint_path)

            iteration += 1

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument('-c', '--config', type=str,
                        help='JSON file for configuration')
    parser.add_argument('-r', '--rank', type=int, default=0,
                        help='rank of process for distributed')
    parser.add_argument('-g', '--group_name', type=str, default='',
                        help='name of group for distributed')
    args = parser.parse_args()

    # Parse configs.  Globals nicer in this case
    with open(args.config) as f:
        data = f.read()
    config = json.loads(data)
    train_config = config["train_config"]
    global data_config
    data_config = config["data_config"]
    global dist_config
    dist_config = config["dist_config"]
    global waveglow_config
    waveglow_config = config["waveglow_config"]

    num_gpus = torch.cuda.device_count()
    if num_gpus > 1:
        if args.group_name == '':
            print("WARNING: Multiple GPUs detected but no distributed group set")
            print("Only running 1 GPU.  Use distributed.py for multiple GPUs")
            num_gpus = 1

    if num_gpus == 1 and args.rank != 0:
        raise Exception("Doing single GPU training on rank > 0")

    torch.backends.cudnn.enabled = True
    torch.backends.cudnn.benchmark = False
    train(num_gpus, args.rank, args.group_name, **train_config)


title: pytorch equivalent tf.gather

body:
I'm having some trouble porting some code over from tensorflow to pytorch.

So I have a matrix with dimensions 10x30 representing 10 examples each with 30 features. Then I have another matrix with dimensions 10x5 containing indices of the the 5 closest examples for each examples in the first matrix. I want to 'gather' using the indices contained in the second matrix the 5 closet examples for each example in the first matrix leaving me with a 3d tensor of shape 10x5x30.

In tensorflow this is done with tf.gather(matrix1, matrix2). Does anyone know how i could do this in pytorch?


title: PyTorch - unable to use batchnorm1d with Linear

body:
Using PyTorch's BatchNorm1D on a 1-D tensor gives the error:

  RuntimeError: running_mean should contain 1 elements not 2304

Any suggestions on what might be wrong?

My Code:

self.net_common = nn.Sequential (
    nn.Linear(64*64, 48*48),
    nn.BatchNorm1d(48*48),
    nn.Tanh(),
    nn.Dropout(p=0.25),
    nn.Linear(48*48, 32*32),
    nn.BatchNorm1d(32*32),
    nn.Tanh(),
)


title: Pytorch: ValueError: Expected input batch_size (32) to match target batch_size (64)

body:
Tried to run the CNN examples on MNIST dataset, batch size=64, channel =1, n_h=28, n_w=28, n_iters = 1000. The program runs for first 500 interation and then gives the above mentioned error.
There are same topics already being discussed on the forum such as : topic 1
and topic 2, but none of them could help me identify the mistake in the following code:

class CNN_MNIST(nn.Module):
def __init__(self):
    super(CNN_MNIST,self).__init__()

    # convolution layer 1
    self.cnn1 = nn.Conv2d(in_channels=1, out_channels= 32, kernel_size=5,
                          stride=1,padding=2)

    # ReLU activation
    self.relu1 = nn.ReLU()

    # maxpool 1
    self.maxpool1 = nn.MaxPool2d(kernel_size=2,stride=2)

    # convolution 2
    self.cnn2 = nn.Conv2d(in_channels=32, out_channels=64, kernel_size=5,
                          stride=1,padding=2)

    # ReLU activation
    self.relu2 = nn.ReLU()

    # maxpool 2
    self.maxpool2 = nn.MaxPool2d(kernel_size=2,stride=2)

    # fully connected 1
    self.fc1 = nn.Linear(7*7*64,1000)
    # fully connected 2
    self.fc2 = nn.Linear(1000,10)

def forward(self,x):

    # convolution 1
    out = self.cnn1(x)
    # activation function
    out = self.relu1(out)
    # maxpool 1
    out = self.maxpool1(out)

    # convolution 2
    out = self.cnn2(out)
    # activation function
    out = self.relu2(out)
    # maxpool 2
    out = self.maxpool2(out)

    # flatten the output
    out = out.view(out.size(0),-1)

    # fully connected layers
    out = self.fc1(out)
    out = self.fc2(out)

    return out
# model trainning
count = 0
loss_list = []
iteration_list = []
accuracy_list = []

for epoch in range(int(n_epochs)):
    for i, (image,labels) in enumerate(train_loader):

    train = Variable(image)
    labels = Variable(labels)

    # clear gradient
    optimizer.zero_grad()

    # forward propagation
    output = cnn_model(train)

    # calculate softmax and cross entropy loss
    loss = error(output,label)

    # calculate gradients
    loss.backward()

    # update the optimizer
    optimizer.step()

    count += 1

    if count % 50 ==0:
        # calculate the accuracy
        correct = 0
        total = 0

        # iterate through the test data
        for image, labels in test_loader:

            test = Variable(image)

            # forward propagation
            output = cnn_model(test)

            # get prediction
            predict = torch.max(output.data,1)[1]

            # total number of labels
            total += len(labels)

            # correct prediction
            correct += (predict==labels).sum()

        # accuracy
        accuracy = 100*correct/float(total)

        # store loss, number of iteration, and accuracy
        loss_list.append(loss.data)
        iteration_list.append(count)
        accuracy_list.append(accuracy)

        # print loss and accurcay as the algorithm progresses
        if count % 500 ==0:
            print('Iteration :{}    Loss :{}    Accuracy :

{}'.format(count,loss.item(),accuracy))

The error is as follows:

    ---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-19-9e93a242961b> in <module>
     18
     19         # calculate softmax and cross entropy loss
---> 20         loss = error(output,label)
     21
     22         # calculate gradients

~\Anaconda3\lib\site-packages\torch\nn\modules\module.py in __call__(self, *input, **kwargs)
    545             result = self._slow_forward(*input, **kwargs)
    546         else:
--> 547             result = self.forward(*input, **kwargs)
    548         for hook in self._forward_hooks.values():
    549             hook_result = hook(self, input, result)

~\Anaconda3\lib\site-packages\torch\nn\modules\loss.py in forward(self, input, target)
    914     def forward(self, input, target):
    915         return F.cross_entropy(input, target, weight=self.weight,
--> 916                                ignore_index=self.ignore_index, reduction=self.reduction)
    917
    918

~\Anaconda3\lib\site-packages\torch\nn\functional.py in cross_entropy(input, target, weight, size_average, ignore_index, reduce, reduction)
   1993     if size_average is not None or reduce is not None:
   1994         reduction = _Reduction.legacy_get_string(size_average, reduce)
-> 1995     return nll_loss(log_softmax(input, 1), target, weight, None, ignore_index, None, reduction)
   1996
   1997

~\Anaconda3\lib\site-packages\torch\nn\functional.py in nll_loss(input, target, weight, size_average, ignore_index, reduce, reduction)
   1820     if input.size(0) != target.size(0):
   1821         raise ValueError('Expected input batch_size ({}) to match target batch_size ({}).'
-> 1822                          .format(input.size(0), target.size(0)))
   1823     if dim == 2:
   1824         ret = torch._C._nn.nll_loss(input, target, weight, _Reduction.get_enum(reduction), ignore_index)

ValueError: Expected input batch_size (32) to match target batch_size (64).


title: Validation dataset in PyTorch using DataLoaders

body:
I want to load MNIST dataset in PyTorch and Torchvision, dividing it into train, validation and test parts. So far I have:
def load_dataset():
    train_loader = torch.utils.data.DataLoader(
        torchvision.datasets.MNIST(
            '/data/', train=True, download=True,
            transform=torchvision.transforms.Compose([
                torchvision.transforms.ToTensor()])),
        batch_size=batch_size_train, shuffle=True)

    test_loader = torch.utils.data.DataLoader(
        torchvision.datasets.MNIST(
            '/data/', train=False, download=True,
            transform=torchvision.transforms.Compose([
                torchvision.transforms.ToTensor()])),
        batch_size=batch_size_test, shuffle=True)

How can I divide the training dataset into training and validation if it's in the DataLoader? I want to use last 10000 examples from the training dataset as a validation dataset (I know that I should do CV for more accurate results, I just want a quick validation here).


title: Why do we pass nn.Module as an argument to class definition for neural nets?

body:
I want to understand why we pass torch.nn.Module as a argument when we define the class for a neural network like GAN's

import torch
import torch.nn as nn

class Generator(nn.Module):
    def __init__(self, input_size, hidden_size, output_size, f):
        super(Generator, self).__init__()
        self.map1 = nn.Linear(input_size, hidden_size)
        self.map2 = nn.Linear(hidden_size, hidden_size)
        self.map3 = nn.Linear(hidden_size, output_size)
        self.f = f


title: Saving vocabulary object from pytorch's torchtext library

body:
Building a text classification model using pytorch's torchtext . The vocabulary object is in the data.field :

def create_tabularDataset_object(self,csv_path):
   self.TEXT = data.Field(tokenize=self.tokenizer,batch_first=True,include_lengths=True)
   self.LABEL = data.LabelField(dtype = torch.float,batch_first=True)

def get_vocab_with_glov(self,data):
   # initialize glove embeddings
   self.TEXT.build_vocab(data,min_freq=100,vectors = "glove.6B.100d")

After training , when serving the model in production how do i hold the TEXT object ? at prediction time  i need it for indexing the words tokens

[TEXT.vocab.stoi[t] for t in tokenizedׁ_sentence]

am i missing something and it is not necessary to hold  that object ? Do i need any other file other then the model weights ?


title: AttributeError: module 'torch' has no attribute 'hub'

body:
import torch
model = torch.hub.list('pytorch/vision')

My pytorch version is 1.0.0, but I can't load the hub, why is this?


title: Taking a norm of matrix rows/cols in pytorch

body:
The norm of a vector can be taken by

torch.norm(vec)

However, how to take a norm of a set of vectors grouped as a matrix (either as rows or columns)?

For example, if a matrix size is (5,8), then the rows norms should return a vector of norms of size (5).


title: Huggingface GPT2 and T5 model APIs for sentence classification?

body:
I've successfully used the Huggingface Transformers BERT model to do sentence classification  using the BERTForSequenceClassification class and API. I've used it for both 1-sentence sentiment analysis and 2-sentence NLI.
I can see that other models have analogous classes, e.g. XLNetForSequenceClassification and RobertaForSequenceClassification. This type of sentence classification usually involves placing a classifier layer on top of a dense vector representing the entirety of the sentence.
Now I'm trying to use the GPT2 and T5 models. However, when I look at the available classes and API for each one, there is no equivalent "ForSequenceClassification" class. For example, for GPT2 there are GPT2Model, GPT2LMHeadModel, and GPT2DoubleHeadsModel classes. Perhaps I'm not familiar enough with the research for GPT2 and T5, but I'm certain that both models are capable of sentence classification.
So my questions are:

What Huggingface classes for GPT2 and T5 should I use for 1-sentence classification?

What classes should I use for 2-sentence (sentence pair) classification (like natural language inference)?

Thank you for any help.


While training the resnet50 model through pytorch I got this error:

  RuntimeError: invalid argument 0: Sizes of tensors must match except
  in dimension 0. Got 3 and 1 in dimension 1 at
  /pytorch/aten/src/TH/generic/THTensorMoreMath.cpp:1333

I'm using this: http://github.com/Helias/Car-Model-Recognition/

with this dataset http://vmmrdb.cecsresearch.org/


title: Rearranging a 3-D array using indices from sorting?

body:
I have a 3-D array of random numbers of size [channels = 3, height = 10, width = 10].

Then I sorted it using sort command from pytorch along the columns and obtained the indices as well.

The corresponding index is shown below:

Now, I would like to return to the original matrix using these indices. I currently use for loops to do this (without considering the batches). The code is:

import torch
torch.manual_seed(1)
ch = 3
h = 10
w = 10
inp_unf = torch.randn(ch,h,w)
inp_sort, indices  = torch.sort(inp_unf,1)
resort = torch.zeros(inp_sort.shape)

for i in range(ch):
    for j in range(inp_sort.shape[1]):
        for k in range (inp_sort.shape[2]):
            temp = inp_sort[i,j,k]
            resort[i,indices[i,j,k],k] = temp

I would like it to be vectorized considering batches as well i.e.input size is [batch, channel, height, width].


title: Saving and Loading Pytorch Model Checkpoint for inference not working

body:
I have a trained model using LSTM. The model is trained on GPU (On Google COLABORATORY).
I have to save the model for inference; which I will run on CPU.
Once trained, I saved the model checkpoint as follows:

torch.save({'model_state_dict': model.state_dict()},'lstmmodelgpu.tar')

And, for inference, I loaded the model as :

# model definition
vocab_size = len(vocab_to_int)+1
output_size = 1
embedding_dim = 300
hidden_dim = 256
n_layers = 2

model = SentimentLSTM(vocab_size, output_size, embedding_dim, hidden_dim, n_layers)

# loading model
device = torch.device('cpu')
checkpoint = torch.load('lstmmodelgpu.tar', map_location=device)
model.load_state_dict(checkpoint['model_state_dict'])
model.eval()

But, it is raising the following error:

model.load_state_dict(checkpoint['model_state_dict'])
  File "workspace/envs/envdeeplearning/lib/python3.5/site-packages/torch/nn/modules/module.py", line 719, in load_state_dict
    self.__class__.__name__, "\n\t".join(error_msgs)))
RuntimeError: Error(s) in loading state_dict for SentimentLSTM:
    Missing key(s) in state_dict: "embedding.weight".
    Unexpected key(s) in state_dict: "encoder.weight".

Is there anything I missed while saving the checkpoint?


title: Suppress use of Softmax in CrossEntropyLoss for PyTorch Neural Net

body:
I know theres no need to use a nn.Softmax() Function in the output layer for a neural net when using nn.CrossEntropyLoss as a loss function.

However I need to do so, is there a way to suppress the implemented use of softmax in nn.CrossEntropyLoss and instead use nn.Softmax() on my output layer of the neural network itself?

Motivation: I am using shap package to analyze the features influences afterwards, where I can only feed my trained model as an input. The outputs however don't make any sense then because I am looking at unbound values instead of probabilites.

Example: Instead of -69.36 as an output value for one class of my model, I want something between 0 and 1, summing up to 1 for all classes. As I can't alter it afterwards, the outputs need to be like this already during training.


title: Lack of Sparse Solution with L1 Regularization in Pytorch

body:
I am trying to implement L1 regularization onto the first layer of a simple neural network (1 hidden layer). I looked into some other posts on StackOverflow that apply l1 regularization using Pytorch to figure out how it should be done (references: Adding L1/L2 regularization in PyTorch?, In Pytorch, how to add L1 regularizer to activations?). No matter how high I increase lambda (the l1 regularization strength parameter) I do not get true zeros in the first weight matrix. Why would this be? (Code is below)

import torch
import torch.nn as nn
import torch.nn.functional as F
import numpy as np

class Network(nn.Module):
    def __init__(self,nf,nh,nc):
        super(Network,self).__init__()
        self.lin1=nn.Linear(nf,nh)
        self.lin2=nn.Linear(nh,nc)

    def forward(self,x):
        l1out=F.relu(self.lin1(x))
        out=F.softmax(self.lin2(l1out))
        return out, l1out

def l1loss(layer):
    return torch.norm(layer.weight.data, p=1)

nf=10
nc=2
nh=6
learningrate=0.02
lmbda=10.
batchsize=50

net=Network(nf,nh,nc)

crit=nn.MSELoss()
optimizer=torch.optim.Adagrad(net.parameters(),lr=learningrate)

xtr=torch.Tensor(xtr)
ytr=torch.Tensor(ytr)
#ytr=torch.LongTensor(ytr)
xte=torch.Tensor(xte)
yte=torch.LongTensor(yte)
#cyte=torch.Tensor(yte)

it=200
for epoch in range(it):
    per=torch.randperm(len(xtr))
    for i in range(0,len(xtr),batchsize):
        ind=per[i:i+batchsize]
        bx,by=xtr[ind],ytr[ind]
        optimizer.zero_grad()
        output, l1out=net(bx)
#        l1reg=l1loss(net.lin1)
        loss=crit(output,by)+lmbda*l1loss(net.lin1)
        loss.backward()
        optimizer.step()
    print('Epoch [%i/%i], Loss: %.4f' %(epoch+1,it, np.float32(loss.data.numpy())))

corr=0
tot=0
for x,y in list(zip(xte,yte)):
    output,_=net(x)
    _,pred=torch.max(output,-1)
    tot+=1 #y.size(0)
    corr+=(pred==y).sum()
print(corr)

Note: The data has 10 features (2 classes and 800 training samples) and only the first 2 are relevant (by design) so one would assume true zeros should be easy enough to learn.


title: PyTorch Tensors of Inputs and Labels in LSTM

body:
I am new to PyTorch, and I'm working on a simple project to generate text, in order to get my hands on pytorch. I am using the concept of this code and converting it to PyTorch: https://machinelearningmastery.com/text-generation-lstm-recurrent-neural-networks-python-keras/
I have 10 timesteps and 990 samples. For each of these 990 samples, there are 10 values, corresponding to the indexes (scaled) of the sequence. My output samples are the rest of the letters (not including the first letter). For example if my sample is 'Hello Worl" my corresponding output is "ello World". My input size (features) is 1 since I want to feed in one letter at a time. Therefore, my final input shape is (990, 10, 1). I then convert the output tensor into one hot vector, so the final shape is (9900, 42), where 42 are the number of elements in the one hot vector. When I run the output, I can get a shape of (9900, 42), so this is the output of all my timesteps, and each one of them contains the corresponding one-hot vector. But when I calculate the loss, then there is an error:

  multi-target not supported

Can I understand what I did wrong? Thanks . Below is my code

#The file contains 163780 characters
#The file contains 1000 characters
#There are 42 unique characters

char2int = {char:value for (value,char) in enumerate (unique)}
int2char = {value:char for (value,char) in enumerate (unique)}

learning_rate = 0.01
num_epochs = 5
input_size = 1              #The number of input neurons (features) to our RNN
units = 100
num_layers = 2
num_classes = len(unique)   #The number of output neurons

timesteps = 10
datax = []
datay = []
for index in range(0, len(file) - timesteps, 1):
    prev_letters = file[index:index + timesteps]
    output = file[index + 1: index + timesteps + 1]
    #Convert the 10 previous characters to their integers and put in a list. Append that list to the dataset
    datax.append([char2int[c] for c in prev_letters])
    datay.append([char2int[c] for c in output])
print('There are {} Sequences in the dataset'.format(len(datax)))
#There are 990 Sequences in the dataset

x = np.array(datax)
x = x / float(len(unique))
x = torch.FloatTensor(x)
x = x.view(x.size(0), timesteps, input_size)
print(x.shape)   #torch.Size([990, 10, 1])

y = torch.LongTensor(datay)
print(y.shape)   #torch.Size([990, 10])
y_one_hot = torch.zeros(y.shape[0] * y.shape[1], num_classes)
index = y.long()
index = index.view(-1,1)          #The expected shape for the scatter function
y_one_hot.scatter_(1,index,1)    #(dim (1 for along rows and 0 for along cols), index, number to insert)
y_one_hot = y_one_hot.view(-1, num_classes)    # Make the tensor of shape(rows, cols)
y_one_hot = y_one_hot.long()
print(y_one_hot.shape)
#torch.Size([9900, 42])

inputs = Variable(x)
labels = Variable(y_one_hot)

class TextGenerator(nn.Module):
    def __init__(self,input_size,units,num_layers,num_classes,timesteps):
        super(TextGenerator,self).__init__()
        self.units = units
        self.num_layers = num_layers
        self.timesteps = timesteps
        self.input_size = input_size
        # When batch_first=true, inputs are of shape (batch_size/samples, sequence_length, input_dimension)
        self.lstm = nn.LSTM(input_size = input_size, hidden_size = units, num_layers = num_layers, batch_first = True)
        #The output layer
        self.fc = nn.Linear(units, num_classes)
    def forward(self,x):
        #Initialize the hidden state
        h0 = Variable(torch.zeros(self.num_layers, x.size(0), self.units))
        #Initialize the cell state
        c0 = Variable(torch.zeros(self.num_layers, x.size(0), self.units))
        out,_ = self.lstm(x, (h0,c0))
        #Reshape the outout from (samples,timesteps,output_features) to a shape appropriate for the FC layer
        out = out.contiguous().view(-1, self.units)
        out = self.fc(out)
        return out

net = TextGenerator(input_size,units,num_layers,num_classes,timesteps)
loss_fn = torch.nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(net.parameters(), lr=learning_rate)
out = net(inputs)
out.shape   #(9900, 42)
loss_fn(out,labels)


title: RuntimeError: Error(s) in loading state_dict for Generator: size mismatch for weights and biases using Pytorch

body:
I'm training a 3D-GAN to generate MRI volumes. I defined my model as follows:

###### Definition of the generator ######

class Generator(nn.Module):
  def __init__(self, ngpu):
    #super() makes Generator a subclass of nn.Module, so that it inherites all the methods of nn.Module
    super(Generator, self).__init__()
    self.ngpu = ngpu
    #we can use Sequential() since the output of one layer is the input of the next one
    self.main = nn.Sequential(
        # input is latent vector z, going into a convolution
        nn.ConvTranspose3d(nz, ngf * 8, 4, stride=2, padding=0, bias=True), # try to put kernel = (batch_size,4,4,4,512)
        nn.BatchNorm3d(ngf * 8),
        nn.ReLU(True), #True means that it does the operation inplace, default is False

        nn.ConvTranspose3d(ngf * 8, ngf * 4, 4, stride=2, padding=1, bias=True), # try to put kernel = (batch_size,8,8,8,256)
        nn.BatchNorm3d(ngf * 4),
        nn.ReLU(True),

        nn.ConvTranspose3d(ngf * 4, ngf * 2, 4, stride=2, padding=1, bias=True), # try to put kernel = (batch_size,16,16,16,128)
        nn.BatchNorm3d(ngf * 2),
        nn.ReLU(True),

        nn.ConvTranspose3d( ngf * 2, ngf, 4, stride=2, padding=1, bias=True), # try to put kernel = (batch_size,32,32,32,64)
        nn.BatchNorm3d(ngf),
        nn.ReLU(True),

        nn.ConvTranspose3d(ngf, nc, 4, stride=2, padding=1, bias=True), # try to put kernel = (batch_size,64,64,64,1)
        nn.Sigmoid()

        )

  def forward(self, x):
    return self.main(x)

###### Definition of the Discriminator ######

class Discriminator(nn.Module):
    def __init__(self, ngpu):
        super(Discriminator, self).__init__()
        self.ngpu = ngpu
        self.main = nn.Sequential(
            nn.Conv3d(nc, ndf, 4, stride=2, padding=1, bias=True),
            nn.BatchNorm3d(ndf),
            nn.LeakyReLU(leak_value, inplace=True),

            nn.Conv3d(ndf, ndf * 2, 4, stride=2, padding=1, bias=True),
            nn.BatchNorm3d(ndf * 2),
            nn.LeakyReLU(leak_value, inplace=True),

            nn.Conv3d(ndf * 2, ndf * 4, 4, stride=2, padding=1, bias=True),
            nn.BatchNorm3d(ndf * 4),
            nn.LeakyReLU(leak_value, inplace=True),

            nn.Conv3d(ndf * 4, ndf * 8, 4, stride=2, padding=1, bias=True),
            nn.BatchNorm3d(ndf * 8),
            nn.LeakyReLU(leak_value, inplace=True),

            nn.Conv3d(ndf * 8, nc, 4, stride=1, padding=0, bias=True),
            nn.Sigmoid()
        )

    def forward(self, x):
        return self.main(x)

I then train the model and save it. When loading the model for evaluation and testing I get the following error:

  RuntimeError: Error(s) in loading state_dict for Generator:
      size mismatch for main.0.weight: copying a param with shape torch.Size([64, 1, 4, 4, 4]) from checkpoint, the shape in current model is torch.Size([200, 512, 4, 4, 4]).
      size mismatch for main.0.bias: copying a param with shape torch.Size([64]) from checkpoint, the shape in current model is torch.Size([512]).
      size mismatch for main.1.weight: copying a param with shape torch.Size([64]) from checkpoint, the shape in current model is torch.Size([512]).
      size mismatch for main.1.running_mean: copying a param with shape torch.Size([64]) from checkpoint, the shape in current model is torch.Size([512]).
      size mismatch for main.1.bias: copying a param with shape torch.Size([64]) from checkpoint, the shape in current model is torch.Size([512]).
      size mismatch for main.1.running_var: copying a param with shape torch.Size([64]) from checkpoint, the shape in current model is torch.Size([512]).
      size mismatch for main.3.weight: copying a param with shape torch.Size([128, 64, 4, 4, 4]) from checkpoint, the shape in current model is torch.Size([512, 256, 4, 4, 4]).
      size mismatch for main.3.bias: copying a param with shape torch.Size([128]) from checkpoint, the shape in current model is torch.Size([256]).
      size mismatch for main.4.weight: copying a param with shape torch.Size([128]) from checkpoint, the shape in current model is torch.Size([256]).
      size mismatch for main.4.running_mean: copying a param with shape torch.Size([128]) from checkpoint, the shape in current model is torch.Size([256]).
      size mismatch for main.4.bias: copying a param with shape torch.Size([128]) from checkpoint, the shape in current model is torch.Size([256]).
      size mismatch for main.4.running_var: copying a param with shape torch.Size([128]) from checkpoint, the shape in current model is torch.Size([256]).
      size mismatch for main.6.bias: copying a param with shape torch.Size([256]) from checkpoint, the shape in current model is torch.Size([128]).
      size mismatch for main.7.weight: copying a param with shape torch.Size([256]) from checkpoint, the shape in current model is torch.Size([128]).
      size mismatch for main.7.running_mean: copying a param with shape torch.Size([256]) from checkpoint, the shape in current model is torch.Size([128]).
      size mismatch for main.7.bias: copying a param with shape torch.Size([256]) from checkpoint, the shape in current model is torch.Size([128]).
      size mismatch for main.7.running_var: copying a param with shape torch.Size([256]) from checkpoint, the shape in current model is torch.Size([128]).
      size mismatch for main.9.weight: copying a param with shape torch.Size([512, 256, 4, 4, 4]) from checkpoint, the shape in current model is torch.Size([128, 64, 4, 4, 4]).
      size mismatch for main.9.bias: copying a param with shape torch.Size([512]) from checkpoint, the shape in current model is torch.Size([64]).
      size mismatch for main.10.weight: copying a param with shape torch.Size([512]) from checkpoint, the shape in current model is torch.Size([64]).
      size mismatch for main.10.running_mean: copying a param with shape torch.Size([512]) from checkpoint, the shape in current model is torch.Size([64]).
      size mismatch for main.10.bias: copying a param with shape torch.Size([512]) from checkpoint, the shape in current model is torch.Size([64]).
      size mismatch for main.10.running_var: copying a param with shape torch.Size([512]) from checkpoint, the shape in current model is torch.Size([64]).
      size mismatch for main.12.weight: copying a param with shape torch.Size([1, 512, 4, 4, 4]) from checkpoint, the shape in current model is torch.Size([64, 1, 4, 4, 4]).

What I'm I doing wrong?

Thanks in advance!


title: How can I count the number of 1's and 0's in the second dimension of a PyTorch Tensor?

body:
I have a tensor with size: torch.Size([64, 2941]), which is 64 batches of 2941 elements.

Across all 64 batches, I want to count the total number of 1s and 0s in the second dimension of the tensor, all the way to the 2941th so that I will have those counts as a tensor of size torch.Size([2941])

How do I do that?


title: Add white noise on image batch data when training

body:
I am trying out a de-noise model, the goal is to print out clean/ add_noise/ model_output of each batch.

I am using PyTorch DataLoader. Each image has shape = (256, 128), and the set
batch_size = 10, therefore each batch has size (10, 256, 128). I would like to print out the first data of each batch, i.e. batch_data[0]

Each image has shape = (256, 128)

I've written a function to add noise as below:

def add_noise(data, bs, target_snr, noise_type):

    if noise_type == 'white':
        noise = acoustics.generator.white(bs*256*128).reshape(bs, 256, 128)

    if noise_type == 'pink':
        noise = acoustics.generator.pink(bs*256*128).reshape(bs, 256, 128)

    print ('data shape = ', data.shape)

    average = np.mean(data)
    std = np.std(noise)
    current_snr = average/std

    noise = noise * (current_snr/ target_snr)
    data = data + noise

    return data

However, it keeps return error message as below:

TypeError: mean() missing 3 required positional argument: "dim", "keepdim", "dtype"

How should I deal with this?


title: Why PyTorch model takes multiple image size inside the model?

body:
I am using a simple object detection model in PyTorch and using a Pytoch Model for Inferencing.
When I am using a simple iterator over the code
for k, image_path in enumerate(image_list):
    image = imgproc.loadImage(image_path)
    print(image.shape)
    with torch.no_grad():
        y, feature = net(x)
    result = image.cuda()

It prints our variable sized images such as
torch.Size([1, 3, 384, 320])

torch.Size([1, 3, 704, 1024])

torch.Size([1, 3, 1280, 1280])

So When I am using Batch Inferencing using a DataLoader applying the same transformation the code is not running.
However, when I am resizing all the images as 600.600 the batch processing runs successfully.
I am having Two Doubts,
First why Pytorch is capable of inputting dynamically sized inputs in Deep Learning Model and Why dynamic sized input is failing in Batch Processing.


title: Groups in Convolutional Neural Network / CNN

body:
I came across this PyTorch example for depthwise separable convolutions using the groups parameter:

class depthwise_separable_conv(nn.Module):
    def __init__(self, nin, nout):
        super(depthwise_separable_conv, self).__init__()
        self.depthwise = nn.Conv2d(nin, nin, kernel_size=3, padding=1, groups=nin)
        self.pointwise = nn.Conv2d(nin, nout, kernel_size=1)

    def forward(self, x):
        out = self.depthwise(x)
        out = self.pointwise(out)
        return out

I haven't seen any usage of groups in CNNs before. The documentation is also a bit sparse as far as that is concerned:

  groups controls the connections between inputs and outputs.
  in_channels and out_channels must both be divisible by groups.

So my questions are:

What are groups in CNNs?
In which cases do I need to use groups?

(I guess this is more a general, not PyTorch specific.)


title: Why does setting backward(retain_graph=True) use up lot GPU memory?

body:
I need to backpropagate through my neural network multiple times, so I set backward(retain_graph=True).

However, this is causing

  RuntimeError: CUDA out of memory

I don't understand why this is.

Are the number of variables or weights doubling? Shouldn't the amount of memory used remain the same regardless of how many times backward() is called?


title: Using torch.nn.Embedding for GloVe: should we fine-tune the embeddings or just use them as they are?

body:
while transfer learning / fine-tuning recent language models, such as BERT and XLNET, is by far a very common practice, how is this for GloVe?

Basically, I see two options when using GloVe to get dense vector representations that can be used by downstream NNs.

1) Fine-tune GloVe embeddings (in pytorch terms, gradient enabled)

2) Just use the embeddings without gradient.

For instance, given GloVe's embeddings matrix, I do

embed = nn.Embedding.from_pretrained(torch.tensor(embedding_matrix, dtype=torch.float))
...
dense = nn.Linear(...)

Is it best practice to solely use GloVe to get vector representation (and only train the dense layer and potentially other layers) or would one fine-tune the embeddings matrix, too?


title: Fail to load a .pth file (pre-trained neural network) using torch.load() on google colab

body:
My google drive is linked to my google colab notebook. Using the pytorch library torch.load($PATH)  fails to load this 219 Mo file (pre-trained neural network) (https://drive.google.com/drive/folders/1-9m4aVg8Hze0IsZRyxvm5gLybuRLJHv-) which is in my google drive. However it works fine when I do it locally on my computer. The error i get on google collab is: (settings: Python 3.6, pytorch 1.3.1):

state_dict = torch.load(model_path)['state_dict']
File "/usr/local/lib/python3.6/dist-packages/torch/serialization.py", line 303, in load
return _load(f, map_location, pickle_module)
File "/usr/local/lib/python3.6/dist-packages/torch/serialization.py", line 454, in _load
return legacy_load(f)
File "/usr/local/lib/python3.6/dist-packages/torch/serialization.py", line 380, in legacy_load
with closing(tarfile.open(fileobj=f, mode='r:', format=tarfile.PAX_FORMAT)) as tar,
File "/usr/lib/python3.6/tarfile.py", line 1589, in open
return func(name, filemode, fileobj, **kwargs)
File "/usr/lib/python3.6/tarfile.py", line 1619, in taropen
return cls(name, mode, fileobj, **kwargs)
File "/usr/lib/python3.6/tarfile.py", line 1482, in init
self.firstmember = self.next()
File "/usr/lib/python3.6/tarfile.py", line 2297, in next
tarinfo = self.tarinfo.fromtarfile(self)
File "/usr/lib/python3.6/tarfile.py", line 1092, in fromtarfile
buf = tarfile.fileobj.read(BLOCKSIZE)
OSError: [Errno 5] Input/output error```

Any help would be much appreciated!


title: How does the MSE loss function work in pytorch?

body:
In the following code (extracted from SentEval), a neural network structure is defined which maps 1024 real numbers to 5 output predictions. The problem is to assess the relatedness between two sentences (each represented with 512 features). The relatedness is a number in [1,5]. I think if the training relatedness numbers were in {1,2,3,4,5}, the cross entropy was a better loss function, but since in the training set we have real relatedness numbers in [1,5], the MSE is used as the loss function.

Question: Since for each input, the network outputs 5 probability numbers, how the MSE is calculated between a real number and 5 probability numbers?

from torch import nn

inputdim = 1024
nclasses = 5
model = nn.Sequential(
            nn.Linear(inputdim, nclasses),
            nn.Softmax(dim=-1),
        )
loss_fn = nn.MSELoss()