title: Pytorch: The size of tensor a (24) must match the size of tensor b (48) at non-singleton dimension 3

body:
Below code works fine and generate proper results.
import torch
import torch.nn as nn
import torch.nn.functional as F

from modules import ConvLSTMCell, Sign

class EncoderCell(nn.Module):
    def __init__(self):
        super(EncoderCell, self).__init__()

        self.conv = nn.Conv2d(
            3, 64, kernel_size=3, stride=2, padding=1, bias=False)
        self.rnn1 = ConvLSTMCell(
            64,
            256,
            kernel_size=3,
            stride=2,
            padding=1,
            hidden_kernel_size=1,
            bias=False)
        self.rnn2 = ConvLSTMCell(
            256,
            512,
            kernel_size=3,
            stride=2,
            padding=1,
            hidden_kernel_size=1,
            bias=False)
        self.rnn3 = ConvLSTMCell(
            512,
            512,
            kernel_size=3,
            stride=2,
            padding=1,
            hidden_kernel_size=1,
            bias=False)

    def forward(self, input, hidden1, hidden2, hidden3):
        x = self.conv(input)

        hidden1 = self.rnn1(x, hidden1)
        x = hidden1[0]

        hidden2 = self.rnn2(x, hidden2)
        x = hidden2[0]

        hidden3 = self.rnn3(x, hidden3)
        x = hidden3[0]

        return x, hidden1, hidden2, hidden3

class Binarizer(nn.Module):
    def __init__(self):
        super(Binarizer, self).__init__()
        self.conv = nn.Conv2d(512, 32, kernel_size=1, bias=False)
        self.sign = Sign()

    def forward(self, input):
        feat = self.conv(input)
        x = F.tanh(feat)
        return self.sign(x)

class DecoderCell(nn.Module):
    def __init__(self):
        super(DecoderCell, self).__init__()

        self.conv1 = nn.Conv2d(
            32, 512, kernel_size=1, stride=1, padding=0, bias=False)
        self.rnn1 = ConvLSTMCell(
            512,
            512,
            kernel_size=3,
            stride=1,
            padding=1,
            hidden_kernel_size=1,
            bias=False)
        self.rnn2 = ConvLSTMCell(
            128,
            512,
            kernel_size=3,
            stride=1,
            padding=1,
            hidden_kernel_size=1,
            bias=False)
        self.rnn3 = ConvLSTMCell(
            128,
            256,
            kernel_size=3,
            stride=1,
            padding=1,
            hidden_kernel_size=3,
            bias=False)
        self.rnn4 = ConvLSTMCell(
            64,
            128,
            kernel_size=3,
            stride=1,
            padding=1,
            hidden_kernel_size=3,
            bias=False)
        self.conv2 = nn.Conv2d(
            32, 3, kernel_size=1, stride=1, padding=0, bias=False)

    def forward(self, input, hidden1, hidden2, hidden3, hidden4):
        x = self.conv1(input)

        hidden1 = self.rnn1(x, hidden1)
        x = hidden1[0]
        x = F.pixel_shuffle(x, 2)

        hidden2 = self.rnn2(x, hidden2)
        x = hidden2[0]
        x = F.pixel_shuffle(x, 2)

        hidden3 = self.rnn3(x, hidden3)
        x = hidden3[0]
        x = F.pixel_shuffle(x, 2)

        hidden4 = self.rnn4(x, hidden4)
        x = hidden4[0]
        x = F.pixel_shuffle(x, 2)

        x = F.tanh(self.conv2(x)) / 2
        return x, hidden1, hidden2, hidden3, hidden4

Now i have changed in self.con and add pretrained resent with layer. Now it shows tensor mismatched error after training. All things are same just add this line in code. I put ** in those line
import torch
import torch.nn as nn
import torch.nn.functional as F
import torchvision.models as models

from modules import ConvLSTMCell, Sign

class EncoderCell(nn.Module):
    def __init__(self):
        super(EncoderCell, self).__init__()

        #self.conv = nn.Conv2d(3, 64, kernel_size=3, stride=2, padding=1, bias=False)

        **resConv = models.resnet50(pretrained=True)
        resConv.layer4 = nn.Conv2d(3, 64, kernel_size=3, stride=2, padding=1, bias=False)
        self.conv = resConv.layer4**

        self.rnn1 = ConvLSTMCell(
            64,
            256,
            kernel_size=3,
            stride=2,
            padding=1,
            hidden_kernel_size=1,
            bias=False)
        self.rnn2 = ConvLSTMCell(
            256,
            512,
            kernel_size=3,
            stride=2,
            padding=1,
            hidden_kernel_size=1,
            bias=False)
        self.rnn3 = ConvLSTMCell(
            512,
            512,
            kernel_size=3,
            stride=2,
            padding=1,
            hidden_kernel_size=1,
            bias=False)

    def forward(self, input, hidden1, hidden2, hidden3):

        x = self.conv(input)

        hidden1 = self.rnn1(x, hidden1)
        x = hidden1[0]

        hidden2 = self.rnn2(x, hidden2)
        x = hidden2[0]

        hidden3 = self.rnn3(x, hidden3)
        x = hidden3[0]

        return x, hidden1, hidden2, hidden3

class Binarizer(nn.Module):
    def __init__(self):
        super(Binarizer, self).__init__()
        self.conv = nn.Conv2d(512, 32, kernel_size=1, bias=False)
        self.sign = Sign()

    def forward(self, input):
        feat = self.conv(input)
        x = F.tanh(feat)
        return self.sign(x)

class DecoderCell(nn.Module):
    def __init__(self):
        super(DecoderCell, self).__init__()

        **resConv = models.resnet50(pretrained=True)
        resConv.layer4 = nn.Conv2d(32, 512, kernel_size=3, stride=2, padding=1, bias=False)
        self.conv1 = resConv.layer4**

        self.rnn1 = ConvLSTMCell(
            512,
            512,
            kernel_size=3,
            stride=1,
            padding=1,
            hidden_kernel_size=1,
            bias=False)
        self.rnn2 = ConvLSTMCell(
            128,
            512,
            kernel_size=3,
            stride=1,
            padding=1,
            hidden_kernel_size=1,
            bias=False)
        self.rnn3 = ConvLSTMCell(
            128,
            256,
            kernel_size=3,
            stride=1,
            padding=1,
            hidden_kernel_size=3,
            bias=False)
        self.rnn4 = ConvLSTMCell(
            64,
            128,
            kernel_size=3,
            stride=1,
            padding=1,
            hidden_kernel_size=3,
            bias=False)

        **resConv2 = models.resnet50(pretrained=True)
        resConv2.layer4 = nn.Conv2d(32, 3, kernel_size=1, stride=1, padding=0, bias=False)
        self.conv2 = resConv2.layer4**

    def forward(self, input, hidden1, hidden2, hidden3, hidden4):
        x = self.conv1(input)

        hidden1 = self.rnn1(x, hidden1)
        x = hidden1[0]
        x = F.pixel_shuffle(x, 2)

        hidden2 = self.rnn2(x, hidden2)
        x = hidden2[0]
        x = F.pixel_shuffle(x, 2)

        hidden3 = self.rnn3(x, hidden3)
        x = hidden3[0]
        x = F.pixel_shuffle(x, 2)

        hidden4 = self.rnn4(x, hidden4)
        x = hidden4[0]
        x = F.pixel_shuffle(x, 2)

        x = F.tanh(self.conv2(x)) / 2
        return x, hidden1, hidden2, hidden3, hidden4


title: Can we fuse Linear with BatchNorm parameters

body:
I would like to remove the BatchNorm from my model. So, I thought about fusing it with Linear. My model structure is like:

Linear -> ReLU -> BatchNorm -> Dropout -> Linear

I tried fusing BatchNorm -> Linear and I couldn't fuse with my code available. Is there any way to fuse the BatchNorm with any of the above layers.


title: Loss does not decrease for binary classification

body:
I am trying to implement binary classification. I have 100K (3 channel, 224 x 224px pre-resized) image dataset that I am trying to train the model for if picture is safe for work or not. I am data engineer with statistician background so I am working on the model like last 5-10 days. I have tried to implement the solutions based on suggestions but unfortunately loss didn’t decrease.
Here is the class implemented by using PyTorch Lightning,
from .dataset import CloudDataset
from .split import DatasetSplit
from pytorch_lightning import LightningModule
from pytorch_lightning.metrics import Accuracy
from torch import stack
from torch.nn import BCEWithLogitsLoss, Conv2d, Dropout, Linear, MaxPool2d, ReLU
from torch.optim import Adam
from torch.utils.data import DataLoader
from torch.utils.data.dataloader import default_collate
from torchvision.transforms import ToTensor
from util import logger
from util.config import config

class ClassifyModel(LightningModule):
    def __init__(self):
        super(ClassifyModel, self).__init__()

        # custom dataset split class
        ds = DatasetSplit(config.s3.bucket, config.train.ratio)

        # split records for train, validation and test
        self._train_itr, self._valid_itr, self._test_itr = ds.split()

        self.conv1 = Conv2d(3, 32, 3, padding=1)
        self.conv2 = Conv2d(32, 64, 3, padding=1)
        self.conv3 = Conv2d(64, 64, 3, padding=1)

        self.pool = MaxPool2d(2, 2)

        self.fc1 = Linear(7 * 28 * 64, 512)
        self.fc2 = Linear(512, 16)
        self.fc3 = Linear(16, 4)
        self.fc4 = Linear(4, 1)

        self.dropout = Dropout(0.25)

        self.relu = ReLU(inplace=True)

        self.accuracy = Accuracy()

    def forward(self, x):
        # comments are shape before execution
        # [32, 3, 224, 224]
        x = self.pool(self.relu(self.conv1(x)))
        # [32, 32, 112, 112]
        x = self.pool(self.relu(self.conv2(x)))
        # [32, 64, 56, 56]
        x = self.pool(self.relu(self.conv3(x)))
        # [32, 64, 28, 28]
        x = self.pool(self.relu(self.conv3(x)))
        # [32, 64, 14, 14]
        x = self.dropout(x)

        # [32, 64, 14, 14]
        x = x.view(-1, 7 * 28 * 64)

        # [32, 12544]
        x = self.relu(self.fc1(x))
        # [32, 512]
        x = self.relu(self.fc2(x))
        # [32, 16]
        x = self.relu(self.fc3(x))
        # [32, 4]
        x = self.dropout(self.fc4(x))

        # [32, 1]
        x = x.squeeze(1)
        # [32]
        return x

    def configure_optimizers(self):
        return Adam(self.parameters(), lr=0.001)

    def training_step(self, batch, batch_idx):
        image, target = batch
        target = target.float()

        output = self.forward(image)

        loss = BCEWithLogitsLoss()
        output = loss(output, target)

        logits = self(image)
        self.accuracy(logits, target)

        return {'loss': output}

    def validation_step(self, batch, batch_idx):
        image, target = batch
        target = target.float()

        output = self.forward(image)

        loss = BCEWithLogitsLoss()
        output = loss(output, target)

        return {'val_loss': output}

    def collate_fn(self, batch):
        batch = list(filter(lambda x: x is not None, batch))
        return default_collate(batch)

    def train_dataloader(self):
        transform = ToTensor()
        workers = 0 if config.train.test else config.train.workers

        # custom data set class that read files from s3
        cds = CloudDataset(config.s3.bucket, self._train_itr, transform)

        return DataLoader(
            dataset=cds,
            batch_size=32,
            shuffle=True,
            num_workers=workers,
            collate_fn=self.collate_fn,
        )

    def val_dataloader(self):
        transform = ToTensor()
        workers = 0 if config.train.test else config.train.workers

        # custom data set class that read files from s3
        cds = CloudDataset(config.s3.bucket, self._valid_itr, transform)

        return DataLoader(
            dataset=cds,
            batch_size=32,
            num_workers=workers,
            collate_fn=self.collate_fn,
        )

    def test_dataloader(self):
        transform = ToTensor()
        workers = 0 if config.train.test else config.train.workers

        # custom data set class that read files from s3
        cds = CloudDataset(config.s3.bucket, self._test_itr, transform)

        return DataLoader(
            dataset=cds,
            batch_size=32,
            shuffle=True,
            num_workers=workers,
            collate_fn=self.collate_fn,
        )

    def validation_epoch_end(self, outputs):
        avg_loss = stack([x['val_loss'] for x in outputs]).mean()

        logger.info(f'Validation loss is {avg_loss}')

    def training_epoch_end(self, outs):
        accuracy = self.accuracy.compute()

        logger.info(f'Training accuracy is {accuracy}')

Here is the custom log output,
epoch 0
Validation loss is 0.5988735556602478
Training accuracy is 0.4441356360912323

epoch 1
Validation loss is 0.6406065225601196
Training accuracy is 0.4441356360912323

epoch 2
Validation loss is 0.621654748916626
Training accuracy is 0.443579763174057

epoch 3
Validation loss is 0.5089989304542542
Training accuracy is 0.4580322504043579

epoch 4
Validation loss is 0.5484663248062134
Training accuracy is 0.4886047840118408

epoch 5
Validation loss is 0.5552918314933777
Training accuracy is 0.6142301559448242

epoch 6
Validation loss is 0.661466121673584
Training accuracy is 0.625903308391571

The problem may related to optimizer or loss function but I couldn't figure out.


title: Confused Regarding PyTorch GRU Docs

body:
This may be too basic of a question, but what do the docs mean by the input to the GRU needs to be 3 dimensional? The GRU docs for PyTorch state:

input of shape (seq_len, batch, input_size): tensor containing the
features of the input sequence. The input can also be a packed
variable length sequence. See
torch.nn.utils.rnn.pack_padded_sequence() for details.

https://pytorch.org/docs/stable/generated/torch.nn.GRU.html
Let us say I am trying to predict the next # in a sequence and have the following dataset:
n, label
1, 2
2, 3
3, 6
4, 9
...

If I window the data using the prior 2 inputs for consideration when guessing the next, the dataset becomes:
t-2, t-1, t, label
na, na, 1, 2
na, 1, 2, 3
1, 2, 3, 6
2, 3, 4, 10
...

where t-x just represents using an input value from a prior time step.
So, when creating a sequential loader, it should create the following tensor for the line 1,2,3,6:
inputs: tensor([[1,2,3]]) #shape(1,3)
labels: tensor([[6]])     #shape(1,1)

I currently understand the input shape as (# batches, # features per batch)
and the output shape as (# batches, # output features per batch)
My question is, should that input tensor actually look like:
tensor([[[1],[2],[3]]])

Which represents (# batches, #prior inputs to consider, #features per input)
I guess I am better trying to understand why the input to a GRU has 3 dimensions in PyTorch. What does that 3rd dimension fundamentally represent? And if I have a transformed dataset like above, how to properly pass it to the model.
Edit:
So the pattern present is:
1 + 1 = 2
2 + 1 = 3
3 + 2 + 1 = 6
4+ 3 + 2 + 1 = 10

I want it where t-2, t-1, and t represent the features at each time step used to help guess. For example, at every point in time there could be 2 features. The dimensions would be (1 batch size, 3 timesteps, 2 features).
My question is wether the GRU takes a flattened input:
(1 batch size, 3 time steps * 2 features per time step)

or the unflattened input:
(1 batch size, 3 time steps, 2 features per timestep)

I am currently under the impression that it is the 2nd input, but would like to check my understanding.


title: Sorting a list of tensors by their length in Pytorch

body:
I have a list of tensors in the form list = [tensor([1,2]), tensor([3, 4, 5])] and would like to order it in descending order based on the length of the tensors. This means the sorted list should look like list = [tensor([3, 4, 5]), tensor([1, 2])].
Using .sort(key=length) does not work, and have also tried using .sort(key= lambda x: len(x)) unsuccessfully.


title: Problem in Seq2Seq translation model when using fixed embeddings

body:
I have already trained a model using this code https://github.com/spro/practical-pytorch/blob/master/seq2seq-translation/seq2seq-translation.ipynb from PyTorch tutorials.

The next thing I am trying to do is train my model with fixed embedding.
I have an array of embeddings (length 16) for every word in the training set. For example:

looks   0.007668 -0.011884 -0.009672 0.011174 0.030010 0.023078 -0.028925 -0.003394 -0.028641 0.010982 -0.021261 0.009686 0.020191 -0.004228 -0.009142 -0.022826
like    0.007500 0.024508 -0.019484 -0.003581 0.007626 0.012839 0.003500 0.023110 0.008789 0.020518 -0.020819 -0.012393 0.011967 0.028991 0.010099 0.004167
that    0.004982 0.007892 0.020845 0.025090 -0.015644 -0.015740 -0.010126 -0.016822 0.007736 -0.023931 -0.014181 -0.031085 0.013599 0.027267 -0.013781 -0.028441

etc.

I used the from_pretrained method in order to provide my own embeddings. I changed the Encoder as:

class EncoderRNN(nn.Module):
    def __init__(self, input_size, hidden_size):
        super(EncoderRNN, self).__init__()
        self.hidden_size = hidden_size

        #self.embedding = nn.Embedding(input_size, hidden_size)
        embedds = np.load('embedds.dat')
        embedds = torch.from_numpy(embedds)
        embedds = embedds.type(torch.FloatTensor)     #x = x.type(torch.cuda.FloatTensor)
        self.embedding = nn.Embedding.from_pretrained(embedds)

        self.gru = nn.GRU(hidden_size, hidden_size)

    def forward(self, input, hidden):
        embedded = self.embedding(input).view(1, 1, -1)
        #print(embedded.size())
        #print(embedded)
        output = embedded
        output, hidden = self.gru(output, hidden)
        return output, hidden

    def initHidden(self):
        return torch.zeros(1, 1, self.hidden_size, device=device)

But when I train an error occurs:

  RuntimeError: index out of range at
  c:\programdata\miniconda3\conda-bld\pytorch_1533090623466\work\aten\src\th\generic/THTensorMath.cpp:352

Can anyone provide any help on that?
Thanks in advance


title: Compute maxima and minima of a 4D tensor in PyTorch

body:
Suppose that we have a 4-dimensional tensor, for instance

import torch
X = torch.rand(2, 3, 4, 4)
tensor([[[[-0.9951,  1.6668,  1.3140,  1.4274],
          [ 0.2614,  2.6442, -0.3041,  0.7337],
          [-1.2690,  0.0125, -0.3885,  0.0535],
          [ 1.5270, -0.1186, -0.4458,  0.1389]],

         [[ 0.9125, -1.2998, -0.4277, -0.2688],
          [-1.6917, -0.8855, -0.2784, -0.6717],
          [ 1.1417,  0.4574,  0.4803, -1.6637],
          [ 0.7322,  0.2654, -0.1525,  1.7285]],

         [[ 1.8310, -1.5765,  0.1392,  1.3431],
          [-0.6641, -1.5090, -0.4893, -1.4110],
          [ 0.5875,  0.7528, -0.6482, -0.2547],
          [-2.3133,  0.3888,  2.1428,  0.2331]]]])

I want to compute the maximum and the minimum values of X over the dimensions 2 and 3, that is, to compute two tensors of size (2,3,1,1), one for the maximum and one for the minimum values of the 4x4 blocks.

I started by trying to do that with torch.max() and torch.min(), but I had no luck. I would expect the dim argument of the above functions to be able to take tuple values, but it can take only an integer. So I don't know how to proceed.

However, specifically for the maximum values, I decided to use torch.nn.MaxPool2d() with kernel_size=4 and stride=4. This indeed did the job:

max_pool = nn.MaxPool2d(kernel_size=4, stride=4)
X_max = max_pool(X)
tensor([[[[2.6442]],
         [[1.7285]],
         [[2.1428]]]])

But, afaik, there's no similar layer for "min"-pooling. Could you please help me on how to compute the minima similarly to the maxima?

Thank you.


title: PyTorch batch Size suddenly reduced after n epochs

body:
I have a pytorch nn model running on win 10 cpu.
batch size is 42
After 67 iterations, a strange thing happens: batch size is suddenly reduced to 28, and I get

RuntimeError: Expected hidden[0] size (1, 28, 256), got (1, 42, 256)


title: How to implement LadderNet (2 U-Nets) in Keras? (With available PyTorch script as reference)

body:
I am trying to implement the architecture of LadderNet (https://arxiv.org/abs/1810.07810) in Keras, with only the PyTorch version available as reference. The architecture in the paper is comprised of 2 U-Nets:

The codes for the PyTorch implementation of LadderNet's architecture (obtained from https://github.com/juntang-zhuang/LadderNet/blob/master/src/LadderNetv65.py) and Keras' implementation of U-Net (obtained from https://github.com/zhixuhao/unet/blob/master/model.py) are respectively:

drop = 0.25

def conv3x3(in_planes, out_planes, stride=1):
    """3x3 convolution with padding"""
    return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride,
                     padding=1, bias=True)

class BasicBlock(nn.Module):
    expansion = 1

    def __init__(self, inplanes, planes, stride=1, downsample=None):
        super(BasicBlock, self).__init__()
        if inplanes!= planes:
            self.conv0 = conv3x3(inplanes,planes)

        self.inplanes = inplanes
        self.planes = planes

        self.conv1 = conv3x3(planes, planes, stride)
        #self.bn1 = nn.BatchNorm2d(planes)
        self.relu = nn.ReLU(inplace=True)
        #self.conv2 = conv3x3(planes, planes)
        #self.bn2 = nn.BatchNorm2d(planes)
        self.downsample = downsample
        self.stride = stride
        self.drop = nn.Dropout2d(p=drop)

    def forward(self, x):
        if self.inplanes != self.planes:
            x = self.conv0(x)
            x = F.relu(x)

        out = self.conv1(x)
        #out = self.bn1(out)
        out = self.relu(out)

        out = self.drop(out)

        out1 = self.conv1(out)
        #out1 = self.relu(out1)

        out2 = out1 + x

        return F.relu(out2)

class Bottleneck(nn.Module):
    expansion = 4

    def __init__(self, inplanes, planes, stride=1, downsample=None):
        super(Bottleneck, self).__init__()
        self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False)
        self.bn1 = nn.BatchNorm2d(planes)
        self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride,
                               padding=1, bias=False)
        self.bn2 = nn.BatchNorm2d(planes)
        self.conv3 = nn.Conv2d(planes, planes * self.expansion, kernel_size=1, bias=False)
        self.bn3 = nn.BatchNorm2d(planes * self.expansion)
        self.relu = nn.ReLU(inplace=True)
        self.downsample = downsample
        self.stride = stride

    def forward(self, x):
        residual = x

        out = self.conv1(x)
        out = self.bn1(out)
        out = self.relu(out)

        out = self.conv2(out)
        out = self.bn2(out)
        out = self.relu(out)

        out = self.conv3(out)
        out = self.bn3(out)

        if self.downsample is not None:
            residual = self.downsample(x)

        out += residual
        out = self.relu(out)

        return out

class Initial_LadderBlock(nn.Module):

    def __init__(self,planes,layers,kernel=3,block=BasicBlock,inplanes = 3):
        super().__init__()
        self.planes = planes
        self.layers = layers
        self.kernel = kernel

        self.padding = int((kernel-1)/2)
        self.inconv = nn.Conv2d(in_channels=inplanes,out_channels=planes,
                                kernel_size=3,stride=1,padding=1,bias=True)

        # create module list for down branch
        self.down_module_list = nn.ModuleList()
        for i in range(0,layers):
            self.down_module_list.append(block(planes*(2**i),planes*(2**i)))

        # use strided conv instead of poooling
        self.down_conv_list = nn.ModuleList()
        for i in range(0,layers):
            self.down_conv_list.append(nn.Conv2d(planes*2**i,planes*2**(i+1),stride=2,kernel_size=kernel,padding=self.padding))

        # create module for bottom block
        self.bottom = block(planes*(2**layers),planes*(2**layers))

        # create module list for up branch
        self.up_conv_list = nn.ModuleList()
        self.up_dense_list = nn.ModuleList()
        for i in range(0, layers):
            self.up_conv_list.append(nn.ConvTranspose2d(in_channels=planes*2**(layers-i), out_channels=planes*2**max(0,layers-i-1), kernel_size=3,
                                                        stride=2,padding=1,output_padding=1,bias=True))
            self.up_dense_list.append(block(planes*2**max(0,layers-i-1),planes*2**max(0,layers-i-1)))

    def forward(self, x):
        out = self.inconv(x)
        out = F.relu(out)

        down_out = []
        # down branch
        for i in range(0,self.layers):
            out = self.down_module_list[i](out)
            down_out.append(out)
            out = self.down_conv_list[i](out)
            out = F.relu(out)

        # bottom branch
        out = self.bottom(out)
        bottom = out

        # up branch
        up_out = []
        up_out.append(bottom)

        for j in range(0,self.layers):
            out = self.up_conv_list[j](out) + down_out[self.layers-j-1]
            #out = F.relu(out)
            out = self.up_dense_list[j](out)
            up_out.append(out)

        return up_out

class LadderBlock(nn.Module):

    def __init__(self,planes,layers,kernel=3,block=BasicBlock,inplanes = 3):
        super().__init__()
        self.planes = planes
        self.layers = layers
        self.kernel = kernel

        self.padding = int((kernel-1)/2)
        self.inconv = block(planes,planes)

        # create module list for down branch
        self.down_module_list = nn.ModuleList()
        for i in range(0,layers):
            self.down_module_list.append(block(planes*(2**i),planes*(2**i)))

        # use strided conv instead of poooling
        self.down_conv_list = nn.ModuleList()
        for i in range(0,layers):
            self.down_conv_list.append(nn.Conv2d(planes*2**i,planes*2**(i+1),stride=2,kernel_size=kernel,padding=self.padding))

        # create module for bottom block
        self.bottom = block(planes*(2**layers),planes*(2**layers))

        # create module list for up branch
        self.up_conv_list = nn.ModuleList()
        self.up_dense_list = nn.ModuleList()
        for i in range(0, layers):
            self.up_conv_list.append(nn.ConvTranspose2d(planes*2**(layers-i), planes*2**max(0,layers-i-1), kernel_size=3,
                                                        stride=2,padding=1,output_padding=1,bias=True))
            self.up_dense_list.append(block(planes*2**max(0,layers-i-1),planes*2**max(0,layers-i-1)))

    def forward(self, x):
        out = self.inconv(x[-1])

        down_out = []
        # down branch
        for i in range(0,self.layers):
            out = out + x[-i-1]
            out = self.down_module_list[i](out)
            down_out.append(out)

            out = self.down_conv_list[i](out)
            out = F.relu(out)

        # bottom branch
        out = self.bottom(out)
        bottom = out

        # up branch
        up_out = []
        up_out.append(bottom)

        for j in range(0,self.layers):
            out = self.up_conv_list[j](out) + down_out[self.layers-j-1]
            #out = F.relu(out)
            out = self.up_dense_list[j](out)
            up_out.append(out)

        return up_out

class Final_LadderBlock(nn.Module):

    def __init__(self,planes,layers,kernel=3,block=BasicBlock,inplanes = 3):
        super().__init__()
        self.block = LadderBlock(planes,layers,kernel=kernel,block=block)

    def forward(self, x):
        out = self.block(x)
        return out[-1]

class LadderNetv6(nn.Module):
    def __init__(self,layers=3,filters=16,num_classes=2,inplanes=3):
        super().__init__()
        self.initial_block = Initial_LadderBlock(planes=filters,layers=layers,inplanes=inplanes)
        #self.middle_block = LadderBlock(planes=filters,layers=layers)
        self.final_block = Final_LadderBlock(planes=filters,layers=layers)
        self.final = nn.Conv2d(in_channels=filters,out_channels=num_classes,kernel_size=1)

    def forward(self,x):
        out = self.initial_block(x)
        #out = self.middle_block(out)
        out = self.final_block(out)
        out = self.final(out)
        #out = F.relu(out)
        out = F.log_softmax(out,dim=1)
        return out

and

def unet(pretrained_weights = None,input_size = (256,256,1)):
    inputs = Input(input_size)
    conv1 = Conv2D(64, 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(inputs)
    conv1 = Conv2D(64, 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(conv1)
    pool1 = MaxPooling2D(pool_size=(2, 2))(conv1)
    conv2 = Conv2D(128, 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(pool1)
    conv2 = Conv2D(128, 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(conv2)
    pool2 = MaxPooling2D(pool_size=(2, 2))(conv2)
    conv3 = Conv2D(256, 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(pool2)
    conv3 = Conv2D(256, 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(conv3)
    pool3 = MaxPooling2D(pool_size=(2, 2))(conv3)
    conv4 = Conv2D(512, 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(pool3)
    conv4 = Conv2D(512, 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(conv4)
    drop4 = Dropout(0.5)(conv4)
    pool4 = MaxPooling2D(pool_size=(2, 2))(drop4)

    conv5 = Conv2D(1024, 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(pool4)
    conv5 = Conv2D(1024, 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(conv5)
    drop5 = Dropout(0.5)(conv5)

    up6 = Conv2D(512, 2, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(UpSampling2D(size = (2,2))(drop5))
    merge6 = concatenate([drop4,up6], axis = 3)
    conv6 = Conv2D(512, 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(merge6)
    conv6 = Conv2D(512, 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(conv6)

    up7 = Conv2D(256, 2, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(UpSampling2D(size = (2,2))(conv6))
    merge7 = concatenate([conv3,up7], axis = 3)
    conv7 = Conv2D(256, 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(merge7)
    conv7 = Conv2D(256, 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(conv7)

    up8 = Conv2D(128, 2, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(UpSampling2D(size = (2,2))(conv7))
    merge8 = concatenate([conv2,up8], axis = 3)
    conv8 = Conv2D(128, 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(merge8)
    conv8 = Conv2D(128, 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(conv8)

    up9 = Conv2D(64, 2, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(UpSampling2D(size = (2,2))(conv8))
    merge9 = concatenate([conv1,up9], axis = 3)
    conv9 = Conv2D(64, 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(merge9)
    conv9 = Conv2D(64, 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(conv9)
    conv9 = Conv2D(2, 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(conv9)
    conv10 = Conv2D(1, 1, activation = 'sigmoid')(conv9)

    model = Model(input = inputs, output = conv10)

    model.compile(optimizer = Adam(lr = 1e-4), loss = 'binary_crossentropy', metrics = ['accuracy'])

    #model.summary()

    if(pretrained_weights):
        model.load_weights(pretrained_weights)

    return model

I'm very new to PyTorch, and I am still familiarizing myself with the transition between Keras and PyTorch, and I'm also hoping that the above can help in this transition of mine.

With regards to the implementation in Keras for LadderNet, if I understood the paper correctly, is it simply just 2 U-Nets superimposed side-by-side (named LaddderNetKeras) as follows:

def LadderNetKeras(pretrained_weights = None,input_size = (256,256,1)):
    inputs = Input(input_size)
    conv1 = Conv2D(64, 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(inputs)
    conv1 = Conv2D(64, 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(conv1)
    pool1 = MaxPooling2D(pool_size=(2, 2))(conv1)
    conv2 = Conv2D(128, 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(pool1)
    conv2 = Conv2D(128, 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(conv2)
    pool2 = MaxPooling2D(pool_size=(2, 2))(conv2)
    conv3 = Conv2D(256, 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(pool2)
    conv3 = Conv2D(256, 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(conv3)
    pool3 = MaxPooling2D(pool_size=(2, 2))(conv3)
    conv4 = Conv2D(512, 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(pool3)
    conv4 = Conv2D(512, 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(conv4)
    drop4 = Dropout(0.5)(conv4)
    pool4 = MaxPooling2D(pool_size=(2, 2))(drop4)

    conv5 = Conv2D(1024, 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(pool4)
    conv5 = Conv2D(1024, 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(conv5)
    drop5 = Dropout(0.5)(conv5)

    up6 = Conv2D(512, 2, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(UpSampling2D(size = (2,2))(drop5))
    merge6 = concatenate([drop4,up6], axis = 3)
    conv6 = Conv2D(512, 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(merge6)
    conv6 = Conv2D(512, 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(conv6)

    up7 = Conv2D(256, 2, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(UpSampling2D(size = (2,2))(conv6))
    merge7 = concatenate([conv3,up7], axis = 3)
    conv7 = Conv2D(256, 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(merge7)
    conv7 = Conv2D(256, 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(conv7)

    up8 = Conv2D(128, 2, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(UpSampling2D(size = (2,2))(conv7))
    merge8 = concatenate([conv2,up8], axis = 3)
    conv8 = Conv2D(128, 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(merge8)
    conv8 = Conv2D(128, 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(conv8)

    up9 = Conv2D(64, 2, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(UpSampling2D(size = (2,2))(conv8))
    merge9 = concatenate([conv1,up9], axis = 3)
    conv9 = Conv2D(64, 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(merge9)
    conv9 = Conv2D(64, 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(conv9)
    conv9 = Conv2D(2, 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(conv9)
    conv10 = Conv2D(1, 1, activation = 'sigmoid')(conv9)

    # SECOND U-NET
    conv1 = Conv2D(64, 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(conv10)
    conv1 = Conv2D(64, 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(conv1)
    pool1 = MaxPooling2D(pool_size=(2, 2))(conv1)
    conv2 = Conv2D(128, 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(pool1)
    conv2 = Conv2D(128, 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(conv2)
    pool2 = MaxPooling2D(pool_size=(2, 2))(conv2)
    conv3 = Conv2D(256, 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(pool2)
    conv3 = Conv2D(256, 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(conv3)
    pool3 = MaxPooling2D(pool_size=(2, 2))(conv3)
    conv4 = Conv2D(512, 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(pool3)
    conv4 = Conv2D(512, 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(conv4)
    drop4 = Dropout(0.5)(conv4)
    pool4 = MaxPooling2D(pool_size=(2, 2))(drop4)

    conv5 = Conv2D(1024, 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(pool4)
    conv5 = Conv2D(1024, 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(conv5)
    drop5 = Dropout(0.5)(conv5)

    up6 = Conv2D(512, 2, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(UpSampling2D(size = (2,2))(drop5))
    merge6 = concatenate([drop4,up6], axis = 3)
    conv6 = Conv2D(512, 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(merge6)
    conv6 = Conv2D(512, 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(conv6)

    up7 = Conv2D(256, 2, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(UpSampling2D(size = (2,2))(conv6))
    merge7 = concatenate([conv3,up7], axis = 3)
    conv7 = Conv2D(256, 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(merge7)
    conv7 = Conv2D(256, 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(conv7)

    up8 = Conv2D(128, 2, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(UpSampling2D(size = (2,2))(conv7))
    merge8 = concatenate([conv2,up8], axis = 3)
    conv8 = Conv2D(128, 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(merge8)
    conv8 = Conv2D(128, 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(conv8)

    up9 = Conv2D(64, 2, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(UpSampling2D(size = (2,2))(conv8))
    merge9 = concatenate([conv1,up9], axis = 3)
    conv9 = Conv2D(64, 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(merge9)
    conv9 = Conv2D(64, 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(conv9)
    conv9 = Conv2D(2, 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(conv9)
    conv10 = Conv2D(1, 1, activation = 'sigmoid')(conv9)

    model = Model(input = inputs, output = conv10)

    model.compile(optimizer = Adam(lr = 1e-4), loss = 'binary_crossentropy', metrics = ['accuracy'])

    #model.summary()

    if(pretrained_weights):
        model.load_weights(pretrained_weights)

    return model

Thank you and some insights will be deeply appreciated!


title: Big performance difference between Pytorch and Keras implementation in text classification

body:
I have implemented CNN in both Keras and Pytorch for a multi-label text classification task. Two implementations ended with two very different performances. CNN with Keras is noticeable outperform the CNN with Pytorch. Both used one layer CNN with kernel size 4.

The Pytorch version scores 0.023 for micro F1 and 0.47 for macro F1. The model is shown as below (more details in colab notebook):

class CNN_simple(nn.Module):
    def __init__(self, vocab_size, embedding_dim, n_filters, filter_sizes, output_dim,
                 dropout, pad_idx):
        super().__init__()
        self.embedding = nn.Embedding(vocab_size, embedding_dim, padding_idx = pad_idx)
        self.conv = nn.Conv1d(in_channels = embedding_dim,
                              out_channels = n_filters,
                              kernel_size = 4) # (N,C,L)

        self.fc = nn.Linear(n_filters, output_dim)
        self.dropout = nn.Dropout(dropout)

    def forward(self, text):
        embedded = self.embedding(text)
        embedded = embedded.permute(0, 2, 1)
        conved = F.relu(self.conv(embedded))
        pooled = F.max_pool1d(conved, conved.shape[2]).squeeze(2)
        dropout = self.dropout(pooled)
        return self.fc(dropout)

The Keras version scores 0.70 for micro F1 and 0.56 for macro F1. The model is shown as below (more details in colab notebook):

def get_model_cnn():
    inp = Input(shape=(MAX_TEXT_LENGTH, ))
    x = Embedding(MAX_VOCAB_SIZE, embed_size)(inp)

    x = Conv1D(embed_size, 4, activation="relu")(x)
    x = GlobalMaxPool1D()(x)
    x = Dense(6, activation="sigmoid")(x)

    model = Model(inputs=inp, outputs=x)
    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
    return model

I assume there is something wrong with my Pytorch implementation. Any comment is appreciated. I have created a colab notebook for the full implementations of Pytorch and Keras. Feel free to copy and run it. Please enlighten me anything that I did wrong in the Pytorch implementations. Thanks.


title: using dataloader to interface kafka data

body:
i use dataloader to inferface the data in kafka and it doesnt work

here is my code

class kfkdataset(Dataset):
def __init__(self,consumer,image_size):
    super(kfkdataset).__init__()
    self.image_size=image_size
    self.consumer = consumer
def __getitem__(self, index):
    info = json.loads(next(self.consumer).value)
    image_osspath = info['path']
    image = prep_image_batch(image_osspath,self.image_size)
    return image,image_osspath

def __len__(self):
    # You should change 0 to the total size of your dataset.
    return 9000000

consumer = KafkaConsumer('my-topic',bootstrap_servers=[])

prodataset = kfkdataset(consumer,image_size=608)#)
k = DataLoader(prodataset,
        batch_size=batch_size,
        num_workers=16)
for inputimage,osspath in k:

    inputimage = inputimage.to(device)
    detections,_ = model(inputimage)
detections = non_max_suppression(detections, 0.98, 0.4)

it works when num_workers is 1

when num_workers >1:
 errors came out

  File "batch_upload.py", line 80, in
      for inputimage,osspath in k:
         File "/usr/local/lib/python3.6/dist-packages/torch/utils/data/dataloader.py", line 801, in__next__
           return self._process_data(data)
         File "/usr/local/lib/python3.6/dist-packages/torch/utils/data/dataloader.py", line 846,in_process_data
           data.reraise()
        File "/usr/local/lib/python3.6/dist-packages/torch/_utils.py", line 369, in reraise
          raise self.exc_type(msg)
      FileExistsError: Caught FileExistsError in DataLoader worker process 1.
      Original Traceback (most recent call last):
        File "/usr/local/lib/python3.6/dist-packages/torch/utils/data/_utils/worker.py", line 178, in _worker_loop
          data = fetcher.fetch(index)
        File "/usr/local/lib/python3.6/dist-packages/torch/utils/data/_utils/fetch.py", line 44, in fetch
          data = [self.dataset[idx] for idx in possibly_batched_index]
        File "/usr/local/lib/python3.6/dist-packages/torch/utils/data/_utils/fetch.py", line 44, in
          data = [self.dataset[idx] for idx in possibly_batched_index]
        File "/appbatch/utils/utils.py", line 49, in getitem
          info = json.loads(next(self.consumer).value)
        File "/usr/local/lib/python3.6/dist-packages/kafka/consumer/group.py", line 1192, in next
          return self.next_v2()
        File "/usr/local/lib/python3.6/dist-packages/kafka/consumer/group.py", line 1200, in next_v2
          return next(self._iterator)
        File "/usr/local/lib/python3.6/dist-packages/kafka/consumer/group.py", line 1115, in _message_generator_v2
          record_map = self.poll(timeout_ms=timeout_ms, update_offsets=False)
        File "/usr/local/lib/python3.6/dist-packages/kafka/consumer/group.py", line 654, in poll
          records = self._poll_once(remaining, max_records, update_offsets=update_offsets)
        File "/usr/local/lib/python3.6/dist-packages/kafka/consumer/group.py", line 701, in _poll_once
          self._client.poll(timeout_ms=timeout_ms)
        File "/usr/local/lib/python3.6/dist-packages/kafka/client_async.py", line 600, in poll
          self._poll(timeout / 1000)
        File "/usr/local/lib/python3.6/dist-packages/kafka/client_async.py", line 629, in _poll
          self._register_send_sockets()
        File "/usr/local/lib/python3.6/dist-packages/kafka/client_async.py", line 619, in _register_send_sockets
          self._selector.modify(key.fileobj, events, key.data)
        File "/usr/lib/python3.6/selectors.py", line 261, in modify
          key = self.register(fileobj, events, data)
        File "/usr/lib/python3.6/selectors.py", line 412, in register
          self._epoll.register(key.fd, epoll_events)
        FileExistsError: [Errno 17] File exists

i want know how to make it works


title: Am I understanding PyTorch's add_ and mul_ correctly?

body:
In this notebook the author writes the following nesterov update:

def nesterov_update(w, dw, v, lr, weight_decay, momentum):
    dw.add_(weight_decay, w).mul_(-lr)
    v.mul_(momentum).add_(dw)
    w.add_(dw.add_(momentum, v))

As I understand it, a.add(b) in PyTorch implements a+b and a.add(b,c) implements a+(b*c), because b is in the slot of the alpha parameter. And lastly, add_ does the in-place version of add.

Q: Am I right so far?

Then, if I were to sketch the above nesterov update in an expanded form that illustrates the logic, I would write:

dw = -lr*(dw + weight_decay*w)
v = v*momentum + dw
w = w + dw + momentum*v

Q: is this correct?

I'm not planning to use the above expanded "code," I'm just writing it this way to try communicate what I'm understanding that it's doing, to check.


title: How can I zero out duplicate values in each row of a PyTorch tensor?

body:
I would like to write a function that achieves the behavior described in this question.
That is, I want to zero out duplicate values in each row of a matrix in PyTorch. For example, given a matrix
torch.Tensor(([1, 2, 3, 4, 3, 3, 4],
              [1, 6, 3, 5, 3, 5, 4]])

I would like to get
torch.Tensor(([1, 2, 3, 4, 0, 0, 0],
              [1, 6, 3, 5, 0, 0, 4]])

or
torch.Tensor(([1, 2, 3, 4, 0, 0, 0],
              [1, 6, 3, 5, 4, 0, 0]])

According to the linked question, torch.unique() alone is not sufficient. I want to know how to implement this function without a loop.


title: Pytorch: Fast create score tensor method

body:
I am new to Pytorch and am looking for a quick get score function. That, given a bunch of samples and a distribution, outputs a tensor consisting of the corresponding score for each individual sample. For instance, consider the following code:
norm = torch.distributions.multivariate_normal.MultivariateNormal(torch.zeros(2),torch.eye(2))
samples = norm.sample((1000,))
samples.requires_grad_(True)

Using samples I would like to create a score tensor of size [1000,2] where the ith component score[i] is the gradient of log p(samples[i]), where p is the density of the given distribution. The method I have come up with is the following:
def get_score(samples,distribution):
    log_probs = distribution.log_prob(samples)
    for i in range(log_probs.size()[0]):
        log_probs[i].backward(retain_graph = True)

The resulting score tensor is then samples.grad. The issue is that my method is quite slow for larger samples (e.g. for a sample of size [50000,2] it takes about 25-30 seconds on my CPU). Is this as fast as it can get?
The only alternative I can think of is to hard-code the score function for each distribution I will use, this doesn't seem like a good solution!

From experimentation, for 50000 samples, the following is about 50% quicker:
for i in range(50000):
    sample = norm.sample((1,))
    sample.requires_grad_(True)
    log_prob = norm.log_prob(a)
    log_prob.backward()

This indicates that there should be a better way!


title: torch.nn.CrossEntropyLoss().ignore_index is crashing when importing transfomers library

body:
I am using layoutlm github which require python 3.6, transformer 2.9.0. I created an conda env:
name: env_test

    channels:
    - defaults
    - conda-forge
    dependencies:
    - python=3.6
    - pip=20.3.3
    - pytorch=1.4.0
    - cudatoolkit=10.1
    - pip:
      - transformers==2.9.0

I have the following test.py code to reproduce the issue:
import sys

import torch
from torch.nn import CrossEntropyLoss

from transformers import (
    BertConfig,
    __version__
)

print (sys.version)
print(torch.__version__)
print(__version__)
CrossEntropyLoss().ignore_index

print("success!")

Importing transformers library results in segmentation fault (core dumped) a when calling CrossEntropyLoss().ignore_index:
$python test.py
3.6.12 |Anaconda, Inc.| (default, Sep  8 2020, 23:10:56)
[GCC 7.3.0]
1.4.0
2.9.0
Segmentation fault (core dumped)

I tried to investigate a bit but I don't really see from where the problem is coming from:
gdb python
GNU gdb (Ubuntu 8.1.1-0ubuntu1) 8.1.1
Copyright (C) 2018 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law.  Type "show copying"
and "show warranty" for details.
This GDB was configured as "x86_64-linux-gnu".
Type "show configuration" for configuration details.
For bug reporting instructions, please see:
<http://www.gnu.org/software/gdb/bugs/>.
Find the GDB manual and other documentation resources online at:
<http://www.gnu.org/software/gdb/documentation/>.
For help, type "help".
Type "apropos word" to search for commands related to "word"...
Reading symbols from python...done.
(gdb) r test.py
Starting program: /home/jupyter/.conda-env/env_test/bin/python test.py
warning: Error disabling address space randomization: Operation not permitted
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1".
3.6.12 |Anaconda, Inc.| (default, Sep  8 2020, 23:10:56)
[GCC 7.3.0]
1.4.0
2.9.0

Program received signal SIGSEGV, Segmentation fault.
0x00007f97000055fb in ?? ()
(gdb) where
#0  0x00007f97000055fb in ?? ()
#1  0x00007f97f4755729 in void pybind11::cpp_function::initialize<void (*&)(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&), void, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, pybind11::name, pybind11::scope, pybind11::sibling>(void (*&)(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&), void (*)(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&), pybind11::name const&, pybind11::scope const&, pybind11::sibling const&)::{lambda(pybind11::detail::function_call&)#3}::_FUN(pybind11::detail::function_call&) ()
   from /home/jupyter/.conda-env/env_test/lib/python3.6/site-packages/torch/lib/libtorch_python.so
#2  0x00007f97f436bca6 in pybind11::cpp_function::dispatcher(_object*, _object*, _object*) () from /home/jupyter/.conda-env/env_test/lib/python3.6/site-packages/torch/lib/libtorch_python.so
#3  0x000055fbadd73a14 in _PyCFunction_FastCallDict () at /tmp/build/80754af9/python_1599604603603/work/Objects/methodobject.c:231
#4  0x000055fbaddfba5c in call_function () at /tmp/build/80754af9/python_1599604603603/work/Python/ceval.c:4851
#5  0x000055fbade1e25a in _PyEval_EvalFrameDefault () at /tmp/build/80754af9/python_1599604603603/work/Python/ceval.c:3335
#6  0x000055fbaddf5c1b in _PyFunction_FastCall (globals=<optimized out>, nargs=1, args=<optimized out>, co=<optimized out>) at /tmp/build/80754af9/python_1599604603603/work/Python/ceval.c:4933
#7  fast_function () at /tmp/build/80754af9/python_1599604603603/work/Python/ceval.c:4968
#8  0x000055fbaddfbb35 in call_function () at /tmp/build/80754af9/python_1599604603603/work/Python/ceval.c:4872
#9  0x000055fbade1e25a in _PyEval_EvalFrameDefault () at /tmp/build/80754af9/python_1599604603603/work/Python/ceval.c:3335
#10 0x000055fbaddf5166 in _PyEval_EvalCodeWithName () at /tmp/build/80754af9/python_1599604603603/work/Python/ceval.c:4166
#11 0x000055fbaddf5e51 in fast_function () at /tmp/build/80754af9/python_1599604603603/work/Python/ceval.c:4992
#12 0x000055fbaddfbb35 in call_function () at /tmp/build/80754af9/python_1599604603603/work/Python/ceval.c:4872
#13 0x000055fbade1e25a in _PyEval_EvalFrameDefault () at /tmp/build/80754af9/python_1599604603603/work/Python/ceval.c:3335
#14 0x000055fbaddf5166 in _PyEval_EvalCodeWithName () at /tmp/build/80754af9/python_1599604603603/work/Python/ceval.c:4166
#15 0x000055fbaddf5e51 in fast_function () at /tmp/build/80754af9/python_1599604603603/work/Python/ceval.c:4992
#16 0x000055fbaddfbb35 in call_function () at /tmp/build/80754af9/python_1599604603603/work/Python/ceval.c:4872
#17 0x000055fbade1e25a in _PyEval_EvalFrameDefault () at /tmp/build/80754af9/python_1599604603603/work/Python/ceval.c:3335
#18 0x000055fbaddf5166 in _PyEval_EvalCodeWithName () at /tmp/build/80754af9/python_1599604603603/work/Python/ceval.c:4166
#19 0x000055fbaddf632c in _PyFunction_FastCallDict () at /tmp/build/80754af9/python_1599604603603/work/Python/ceval.c:5084
#20 0x000055fbadd73ddf in _PyObject_FastCallDict () at /tmp/build/80754af9/python_1599604603603/work/Objects/abstract.c:2310
#21 0x000055fbadd78873 in _PyObject_Call_Prepend () at /tmp/build/80754af9/python_1599604603603/work/Objects/abstract.c:2373
#22 0x000055fbadd7381e in PyObject_Call () at /tmp/build/80754af9/python_1599604603603/work/Objects/abstract.c:2261
#23 0x000055fbaddcc88b in slot_tp_init () at /tmp/build/80754af9/python_1599604603603/work/Objects/typeobject.c:6420
#24 0x000055fbaddfbd97 in type_call () at /tmp/build/80754af9/python_1599604603603/work/Objects/typeobject.c:915
#25 0x000055fbadd73bfb in _PyObject_FastCallDict () at /tmp/build/80754af9/python_1599604603603/work/Objects/abstract.c:2331
#26 0x000055fbaddfbbae in call_function () at /tmp/build/80754af9/python_1599604603603/work/Python/ceval.c:4875
#27 0x000055fbade1e25a in _PyEval_EvalFrameDefault () at /tmp/build/80754af9/python_1599604603603/work/Python/ceval.c:3335
#28 0x000055fbaddf6969 in _PyEval_EvalCodeWithName (qualname=0x0, name=<optimized out>, closure=0x0, kwdefs=0x0, defcount=0, defs=0x0, kwstep=2, kwcount=<optimized out>, kwargs=0x0, kwnames=0x0, argcount=0, args=0x0,
    locals=0x7f98035bf1f8, globals=0x7f98035bf1f8, _co=0x7f980357aae0) at /tmp/build/80754af9/python_1599604603603/work/Python/ceval.c:4166
#29 PyEval_EvalCodeEx () at /tmp/build/80754af9/python_1599604603603/work/Python/ceval.c:4187
#30 0x000055fbaddf770c in PyEval_EvalCode (co=co@entry=0x7f980357aae0, globals=globals@entry=0x7f98035bf1f8, locals=locals@entry=0x7f98035bf1f8) at /tmp/build/80754af9/python_1599604603603/work/Python/ceval.c:731
#31 0x000055fbade77574 in run_mod () at /tmp/build/80754af9/python_1599604603603/work/Python/pythonrun.c:1025
#32 0x000055fbade77971 in PyRun_FileExFlags () at /tmp/build/80754af9/python_1599604603603/work/Python/pythonrun.c:978
#33 0x000055fbade77b73 in PyRun_SimpleFileExFlags () at /tmp/build/80754af9/python_1599604603603/work/Python/pythonrun.c:419
#34 0x000055fbade77c7d in PyRun_AnyFileExFlags () at /tmp/build/80754af9/python_1599604603603/work/Python/pythonrun.c:81
#35 0x000055fbade7b663 in run_file (p_cf=0x7fff210dc16c, filename=0x55fbaefa6dc0 L"test.py", fp=0x55fbaefda800) at /tmp/build/80754af9/python_1599604603603/work/Modules/main.c:340
#36 Py_Main () at /tmp/build/80754af9/python_1599604603603/work/Modules/main.c:811
#37 0x000055fbadd4543e in main () at /tmp/build/80754af9/python_1599604603603/work/Programs/python.c:69
#38 0x00007f9803fd6bf7 in __libc_start_main (main=0x55fbadd45350 <main>, argc=2, argv=0x7fff210dc378, init=<optimized out>, fini=<optimized out>, rtld_fini=<optimized out>, stack_end=0x7fff210dc368) at ../csu/libc-start.c:310
#39 0x000055fbade24d0b in _start () at ../sysdeps/x86_64/elf/start.S:103
(gdb

I am the following list of packages:
_libgcc_mutex             0.1                        main    defaults
_pytorch_select           0.2                       gpu_0    defaults
blas                      1.0                         mkl    defaults
ca-certificates           2020.12.8            h06a4308_0    defaults
certifi                   2020.12.5        py36h06a4308_0    defaults
cffi                      1.14.4           py36h261ae71_0    defaults
chardet                   4.0.0                    pypi_0    pypi
click                     7.1.2                    pypi_0    pypi
cudatoolkit               10.1.243             h6bb024c_0    defaults
cudnn                     7.6.5                cuda10.1_0    defaults
dataclasses               0.8                      pypi_0    pypi
filelock                  3.0.12                   pypi_0    pypi
idna                      2.10                     pypi_0    pypi
intel-openmp              2020.2                      254    defaults
joblib                    1.0.0                    pypi_0    pypi
ld_impl_linux-64          2.33.1               h53a641e_7    defaults
libedit                   3.1.20191231         h14c3975_1    defaults
libffi                    3.3                  he6710b0_2    defaults
libgcc-ng                 9.1.0                hdf63c60_0    defaults
libstdcxx-ng              9.1.0                hdf63c60_0    defaults
mkl                       2020.2                      256    defaults
mkl-service               2.3.0            py36he8ac12f_0    defaults
mkl_fft                   1.2.0            py36h23d657b_0    defaults
mkl_random                1.1.1            py36h0573a6f_0    defaults
ncurses                   6.2                  he6710b0_1    defaults
ninja                     1.10.2           py36hff7bd54_0    defaults
numpy                     1.19.2           py36h54aff64_0    defaults
numpy-base                1.19.2           py36hfa32c7d_0    defaults
openssl                   1.1.1i               h27cfd23_0    defaults
pip                       20.3.3           py36h06a4308_0    defaults
pycparser                 2.20                       py_2    defaults
python                    3.6.12               hcff3b4d_2    defaults
pytorch                   1.4.0           cuda101py36h02f0884_0    defaults
readline                  8.0                  h7b6447c_0    defaults
regex                     2020.11.13               pypi_0    pypi
requests                  2.25.1                   pypi_0    pypi
sacremoses                0.0.43                   pypi_0    pypi
sentencepiece             0.1.94                   pypi_0    pypi
setuptools                51.0.0           py36h06a4308_2    defaults
six                       1.15.0           py36h06a4308_0    defaults
sqlite                    3.33.0               h62c20be_0    defaults
tk                        8.6.10               hbc83047_0    defaults
tokenizers                0.7.0                    pypi_0    pypi
tqdm                      4.55.1                   pypi_0    pypi
transformers              2.9.0                    pypi_0    pypi
urllib3                   1.26.2                   pypi_0    pypi
wheel                     0.36.2             pyhd3eb1b0_0    defaults
xz                        5.2.5                h7b6447c_0    defaults
zlib                      1.2.11               h7b6447c_3    defaults

What is responsible fo this core dump (I have a VM with 30 GB of memory) ? Seems to be related to transformers. Some dependency issues not catched by conda ? This piece of code seems to work with the latest version of transformers 4.1.1 but this is not compatible with layoutlm. Any suggestions?


title: pytorch previous version installation conflict in my local machine

body:
I am trying to install PyTorch and Torchvision with the following conda command in python 2.7 environment

conda install pytorch==1.1.0 torchvision==0.3.0 -c pytorch

But getting this error,
UnsatisfiableError: The following specifications were found
to be incompatible with the existing python installation in your environment:

Specifications:

  - torchvision==0.3.0 -> python[version='>=3.5,<3.6.0a0|>=3.6,<3.7.0a0|>=3.7,<3.8.0a0']

Your python: python=2.7

If python is on the left-most side of the chain, that's the version you've asked for.
When python appears to the right, that indicates that the thing on the left is somehow
not available for the python version you are constrained to. Note that conda will not
change your python version to a different minor version unless you explicitly specify
that.

The following specifications were found to be incompatible with each other:

Output in format: Requested package -> Available versions

Package vs2008_runtime conflicts for:
python=2.7 -> vc=9 -> vs2008_runtime[version='>=9.0.30729.1,<10.0a0']
pytorch==1.1.0 -> ninja -> vs2008_runtime
python=2.7 -> vs2008_runtime

Package zlib conflicts for:
python=2.7 -> sqlite[version='>=3.30.1,<4.0a0'] -> zlib[version='>=1.2.11,<1.3.0a0']
torchvision==0.3.0 -> pillow[version='>=4.1.1'] -> zlib[version='1.2.*|1.2.11|1.2.11.*|>=1.2.11,<1.3.0a0|1.2.8']

My conda list for the specific python 2.7 environment is
# Name                    Version                   Build  Channel
ca-certificates           2021.1.19            haa95532_0
certifi                   2020.6.20          pyhd3eb1b0_3
pip                       19.3.1                   py27_0
python                    2.7.18               hfb89ab9_0
setuptools                44.0.0                   py27_0
sqlite                    3.30.1               h0c8e037_0
vc                        9                    h7299396_1
vs2008_runtime            9.00.30729.1         hfaea7d5_1
wheel                     0.36.2             pyhd3eb1b0_0
wincertstore              0.2              py27hf04cefb_0

I tried to upgrade sqlite to 3.33.0 but it is showing

All requested packages already installed.

And when I tried to uninstall sqlite the whole python package for the environment is getting uninstalled.
Kindly someone helps me with this.


title: Pytorch tensor shape

body:
I have a simple question regarding the shapes of 2 different tensors - tensor_1 and tensor_2.

tensor_1.shape outputs torch.Size([784, 1]);
tensor_2.shape outputs torch.Size([784]).

I understand that the first one is rank-2 tensor, whereas the second is rank-1. What's hard for me is to conceptualize the difference between shape [784, 1] and [784].
Is it correct to think that tensor_1 has 784 rows and 1 column with a scalar inside each place? If so, why can't we call it simply a column vector (which is, in fact, rank-1 tensor), which also has values displayed vertically?
Similarly, can the shape of the second tensor ([784]) be imagined as 784 values inside a horizontal vector?


title: TypeError: img should be PIL Image. Got

body:
import torch
import torch.nn as nn

import torchvision.transforms.functional as TF

class DoubleConv(nn.Module):
    def __init__(self, in_channels, out_channels):
        super(DoubleConv, self).__init__()
        self.conv = nn.Sequential(
            nn.Conv2d(in_channels, out_channels, 3, 1, 1, bias=False),
            nn.BatchNorm2d(out_channels),
            nn.ReLU(inplace=True),
            nn.Conv2d(out_channels, out_channels, 3, 1, 1, bias=False),
            nn.BatchNorm2d(out_channels),
            nn.ReLU(inplace=True),
        )

    def forward(self, x):
        return self.conv(x)

class UNET(nn.Module):
    def __init__(
            self, in_channels=3, out_channels=1, features=[64, 128, 256, 512],
    ):
        super(UNET, self).__init__()
        self.ups = nn.ModuleList()
        self.downs = nn.ModuleList()
        self.pool = nn.MaxPool2d(kernel_size=2, stride=2)

        # Down part of UNET
        for feature in features:
            self.downs.append(DoubleConv(in_channels, feature))
            in_channels = feature

        # Up part of UNET
        for feature in reversed(features):
            self.ups.append(
                nn.ConvTranspose2d(
                    feature*2, feature, kernel_size=2, stride=2,
                )
            )
            self.ups.append(DoubleConv(feature*2, feature))

        self.bottleneck = DoubleConv(features[-1], features[-1]*2)
        self.final_conv = nn.Conv2d(features[0], out_channels, kernel_size=1)

    def forward(self, x):
        skip_connections = []

        for down in self.downs:
            x = down(x)
            skip_connections.append(x)
            x = self.pool(x)

        x = self.bottleneck(x)
        skip_connections = skip_connections[::-1]

        for idx in range(0, len(self.ups), 2):
            x = self.ups[idx](x)
            skip_connection = skip_connections[idx//2]

            if x.shape != skip_connection.shape:
                x = TF.resize(x, size=skip_connection.shape[2:])
            concat_skip = torch.cat((skip_connection, x), dim=1)
            x = self.ups[idx+1](concat_skip)

        return self.final_conv(x)

def test():
    x = torch.randn((3, 1, 161, 161))
    model = UNET(in_channels=1, out_channels=1)
    preds = model(x)
    #print(preds.shape)
    print(x.shape)
    assert preds.shape == x.shape

if __name__ == "__main__":
    test()

This is my code for UNet. The error massage I am getting is:
raise TypeError('img should be PIL Image. Got {}'.format(type(img)))

TypeError: img should be PIL Image. Got <class 'torch.Tensor'>

The error mentioning this line x = TF.resize(x, size=skip_connection.shape[2:])
What's wrong I am doing here?


title: Importing MNIST dataset from local directory in a closed system

body:
I am trying to run a tutorial based on MNIST data in a cluster and the node where training script runs don't have internet access so I am manually placing the MNIST dataset in the desired directory but I am getting Dataset not found error.
I am trying to run this tutorial on the cluster.
I have tried this answer but the answer doesn't resolve my problem.
Below is my code modifications -
import horovod.torch as hvd
train_dataset = \
datasets.MNIST('/scratch/netra/MNIST/processed/training.pt-%d' % hvd.rank(), train=True, download=True,
               transform=transforms.Compose([
                   transforms.ToTensor(),
                   transforms.Normalize((0.1307,), (0.3081,))
               ]))
test_dataset = \
datasets.MNIST('/scratch/netra/MNIST/processed/test.pt-%d' % hvd.rank(), train=False,
               transform=transforms.Compose([
                   transforms.ToTensor(),
                   transforms.Normalize((0.1307,), (0.3081,))
               ]))

How to resolve it?


title: DMN neural network with poor validation results -- only 50%

body:
I have this problem with my Neural Network. I'm trying to implement what's called a DMN (Dynamic Memory Network) for the babi data set. A paper about the DMN model can be found here: http://arxiv.org/abs/1506.07285 Another paper about DMNs can be found here: https://yerevann.github.io/2016/02/05/implementing-dynamic-memory-networks/

Here's my problem. btw I'm using PyTorch.

I split the training and testing data into parts for training, testing, and validation. I use 1000 parts for training, 500 parts for testing and 500 parts for validation. I run into a problem. I can train successfully but when I go to the validation step I never get a score above 50% accuracy. With the babi data set it is documented that you should be able to get 100% accuracy with the first test set. (There are 20 test sets in all). I can get 100% accuracy during training, but only 50% in validation. My question to you is what part of the program would be responsible for this kind of behavior? In other words, can you tell me why I'm always getting 50% ?? Thanks for your time. I'm limiting my experiments to the first babi test for now.

I thought I had this all figured out but my problem has cropped up again. I really don't have a clue what it is. Here is a link to the code. If you could take a look I would be most grateful. https://github.com/radiodee1/awesome-chatbot/blob/master/model/babi_iv.py

Some code is included below.

class WrapMemRNN(nn.Module):
    def __init__(self,vocab_size, embed_dim,  hidden_size, n_layers, dropout=0.3, do_babi=True, bad_token_lst=[], freeze_embedding=False, embedding=None, print_to_screen=False):
        super(WrapMemRNN, self).__init__()
        self.hidden_size = hidden_size
        self.n_layers = n_layers
        self.do_babi = do_babi
        self.print_to_screen = print_to_screen
        self.bad_token_lst = bad_token_lst
        self.embedding = embedding
        self.freeze_embedding = freeze_embedding
        self.teacher_forcing_ratio = hparams['teacher_forcing_ratio']

        gru_dropout = dropout * 0

        self.model_1_enc = Encoder(vocab_size, embed_dim, hidden_size, n_layers, dropout=dropout,embedding=embedding, bidirectional=False)
        self.model_2_enc = Encoder(vocab_size, embed_dim, hidden_size, n_layers, dropout=gru_dropout, embedding=embedding, bidirectional=False)
        self.model_3_mem_a = MemRNN(hidden_size, dropout=gru_dropout)
        self.model_3_mem_b = MemRNN(hidden_size, dropout=gru_dropout)
        self.model_4_att = EpisodicAttn(hidden_size, dropout=gru_dropout)
        self.model_5_ans = AnswerModule(vocab_size, hidden_size,dropout=dropout)

        self.input_var = None  # for input
        self.q_var = None  # for question
        self.answer_var = None  # for answer
        self.q_q = None  # extra question
        self.inp_c = None  # extra input
        self.inp_c_seq = None
        self.all_mem = None
        self.last_mem = None  # output of mem unit
        self.prediction = None  # final single word prediction
        self.memory_hops = hparams['babi_memory_hops']

        self.reset_parameters()

        if self.freeze_embedding or self.embedding is not None:
            self.new_freeze_embedding()
        #self.criterion = nn.CrossEntropyLoss()

        pass

    def reset_parameters(self):
        #print('reset')
        stdv = 1.0 / math.sqrt(self.hidden_size)
        for weight in self.parameters():
            #print('here...')
            weight.data.uniform_(-stdv, stdv)
            if len(weight.size()) > 1:
                init.xavier_normal_(weight)

    def forward(self, input_variable, question_variable, target_variable, criterion=None):

        self.new_input_module(input_variable, question_variable)
        self.new_episodic_module()
        outputs,  ans = self.new_answer_module_simple()

        return outputs, None, ans, None

    def new_freeze_embedding(self):
        self.model_1_enc.embed.weight.requires_grad = False
        self.model_2_enc.embed.weight.requires_grad = False
        print('freeze embedding')
        pass

    def new_input_module(self, input_variable, question_variable):

        prev_h1 = []

        for ii in input_variable:

            ii = self.prune_tensor(ii, 2)

            out1, hidden1 = self.model_1_enc(ii, None)

            prev_h1.append(hidden1)

        self.inp_c_seq = prev_h1
        self.inp_c = prev_h1[-1]

        prev_h2 = []

        for ii in question_variable:
            ii = self.prune_tensor(ii, 2)

            out2, hidden2 = self.model_2_enc(ii, None)
            prev_h2.append(hidden2)

        self.q_q = hidden2[:,-1,:]

        return

    def new_episodic_module(self):
        if True:

            mem_list = []

            sequences = self.inp_c_seq

            for i in range(len(sequences)):

                m_list = [self.q_q.clone()]

                #print(sequences[i].size(),'seq')

                for iter in range(self.memory_hops):

                    x = self.new_attention_step(sequences[i], None, m_list[iter], self.q_q)

                    if self.print_to_screen and not self.training:
                        print(x,'x -- after', len(x), sequences[i].size())

                    e, _ = self.new_episode_small_step(sequences[i], x.permute(1,0), None)

                    assert len(sequences[i].size()) == 3
                    #print(e.size(),'e')
                    ee = e[:, 0, -1]#.permute(2,1,0)

                    _, out = self.model_3_mem_a(ee.unsqueeze(0), self.prune_tensor(m_list[iter], 3))

                    m_list.append(out)

                mem_list.append(m_list[self.memory_hops])

            mm_list = torch.cat(mem_list, dim=1)

            self.last_mem = mm_list

            #print(self.last_mem.size(),'lm')

        return None

    def new_episode_small_step(self, ct, g, prev_h):

        assert len(ct.size()) == 3
        bat, sen, emb = ct.size()
        #print(ct.size(),'ct')
        #print(sen,'sen', g.size())
        last = [prev_h]

        ep = []
        for iii in range(sen):

            c = ct[0,iii,:].unsqueeze(0)

            if prev_h is not None:
                prev_h = self.prune_tensor(prev_h, 3)

            out, gru = self.model_3_mem_b(c, last[iii] )

            last.append(out)

            g = g.squeeze(0)
            gru = gru.squeeze(0).permute(1,0)

            #if not self.training: print(g.size(),'g', iii)
            #ggg = g[:, iii]
            ggg = g[iii]
            h = torch.mul(ggg , gru)#  + torch.mul((1 - g[iii]) , prev_h.permute(1,0))

            index = -1 #-1 # -2
            if last[iii + index] is not None:
                #print(last[iii].size(),'last -',ggg.size(), ggg, sen)
                if False: h = h + torch.mul((1 - ggg), last[iii + index])

            #print(h.size(),'hsize')
            if iii == sen - 1 : ep.append(h.unsqueeze(1))

        h = torch.cat(ep, dim=1)

        #print(h.size(),ep[0].size(),'h',sen, gru.size())

        return h, gru

    def new_attention_step(self, ct, prev_g, mem, q_q):

        q_q = self.prune_tensor(q_q,3)
        mem = self.prune_tensor(mem,3)

        assert len(ct.size()) == 3
        bat, sen, emb = ct.size()

        #print(sen,'len sen')

        att = []
        for iii in range(sen):
            c = ct[0,iii,:]

            concat_list = [
                c.unsqueeze(0),
                mem.squeeze(0),
                q_q.squeeze(0),
                (c * q_q).squeeze(0),
                (c * mem).squeeze(0),
                (torch.abs(c - q_q) ).squeeze(0),
                (torch.abs(c - mem) ).squeeze(0)
            ]
            #for ii in concat_list: print(ii.size())
            #print(sen,'sen')
            #exit()
            #z = F.sigmoid(z)
            concat_list = torch.cat(concat_list, dim=1)
            #print(concat_list.size(),'cl')
            att.append(concat_list)

        att = torch.cat(att, dim=0)
        #z = torch.cat(att, dim=0)
        z = self.model_4_att(att)
        z = F.sigmoid(z)
        #z =  F.softmax(z, dim=1) #F.sigmoid(z)
        #print(z.size(),'z')
        return z

    def prune_tensor(self, input, size):
        if len(input.size()) < size:
            input = input.unsqueeze(0)
        if len(input.size()) > size:
            input = input.squeeze(0)
        return input

    def new_answer_module_simple(self):
        #outputs

        ansx = self.model_5_ans(self.last_mem, None)

        #ansx = F.softmax(ansx, dim=0)

        if self.print_to_screen:
            print(ansx, 'ansx printed')
            print(ansx.size(), 'ansx')
            vocab, sen = ansx.size()
            aa = torch.argmax(ansx, dim=0)
            print(aa.size(),'aa')
            for i in range(sen):
                zz = aa[i]
                z = ansx[:, i]
                a = torch.argmax(z, dim=0)
                print(a.item(), zz.item())
            print('----')
        #ans = torch.argmax(ansx,dim=1)#[0]

        return [None], ansx

        pass


title: Where to apply Batch Normalisation

body:
I have input data and I am passing it through the relu-hidden layer. I wanted to ask where should I apply Batch Normalisation? Following is the model architecture:

def function(self, data):
    hidden_layer = F.relu((self.layer1(data)))
    pred = self.pred(hidden_layer)
    pred = self.softmax(pred)
    return pred

My best guess is that it should go on the hidden layer like self.batch_norm(hidden_layer). The input data is already scaled using X = StandardScaler().fit_transform(X)


title: What particular change of formula in target changes neural network from gradient descent into gradient ascent?

body: It was weird when I face it in reinforcement learning. A loss is MSE. Everything should be perfect to be gradient descent and now it is a gradient ascent. I wanna know the magic. I did numpy neural network. Change in a derivative lead to gradient ascent. What particular change in a derivative lead to gradient ascent? Is it that simple that autograd sees that it is concave or convex?


title: How to store switch locations during max-pooling layer

body: I am implementing the article https://arxiv.org/abs/1311.2901 by Zeiler and Fergus on visualizing and understanding convolutional Networks. To be able reflect hidden layers back to the image space we need deconvolution kernels, rectified linear functions and switch locations. I couldn't find how to store switch locations during max-pooling. I would be glad if you can explain how to do it in pytorch or tensorflow. Thanks in advance.


title: How to physically remove a neuron in PyTorch?

body: In the paper "Learning both Weights and Connections for Efficient Neural Networks", it shrink the size of a neural networks according to a criteria. However, I found that all of the implementation of this paper does not physically shrink the size of the model. They are just zeroing the weight of some neurons to get the same result. But physically the size of the model and the inference time do not change. I'm wondering how could I physically do this?


title: Pytorch pretrained model not recognizing my image

body:
I have an image that is a torch.Tensor format. I want to feed it directly into a pre-trained classifier, like Inception v3.

However, it's being predicted incorrectly (no error message, just output is wrong). I guess this is because I didn't normalize it (in accordance with: https://pytorch.org/docs/stable/torchvision/models.html), so that's what I'm trying to do.

The problem is, the normalization requires numpy input. However, to get numpy, I do this and it gives me an error:

----> 9 image = data.numpy()[0].transpose((1, 2, 0))  # [image_size, image_size, RGB]

RuntimeError: Can't call numpy() on Variable that requires grad. Use var.detach().numpy() instead.

I cannot call detach because I require gradients to flow through the image (which was generated by other functions). Is there a way to bypass converting this to a numpy. If not, how do I keep the gradient flow?


title: Pytorch data loader shuffles example inside a batch

body:
I have the following Dataloader that I am using for an autoencoder in Pytorch

class DataLoader(data.DataLoader):
    def __init__(self,  *args, **kwargs):
        super(DataLoader, self).__init__(*args, **kwargs)
        self.iterator = self.__iter__()

    def __next__(self):
        try:
            return next(self.iterator)
        except:
            self.iterator = self.__iter__()
            return next(self.iterator)

When running the loss function computation I want to display predictions as images for the first file in my dataset, but even if I only have one batch of 2 images for training, these images are being exchanged so my code:

class CrossEntropyLoss2d(nn.Module):
    def __init__(self, weight=None):
        super(CrossEntropyLoss2d, self).__init__()

    def forward(self, pred, target, weight=None):
        pred = pred.clamp(min=1e-16)

        plt.clf()
        predpic = (np.uint8(np.argmax(pred.data.cpu().numpy(), 1)))
        plt.imshow(predpic[0]) ### here I want only the first picture!
        plt.title('Example prediction')
        plt.savefig(os.path.join(results_path, "exampleprediction.png"))
        plt.show()

        ##### COMPUTE LOSS
        loss = -(pred.log() * target)
        loss = loss.sum(1)
        if weight is not None:
            loss = loss * weight
        loss = loss.mean()
        return loss

does not work.

How can I fix this? Thanks!


title: how to reduce the calculation time of optimizer.step() without loss.item() in Pytorch

body:
When I　test my imagenet model's calculation time, I found that if　tensor like loss or top1, top5 don't be transformed by function tensor.item() or tensor.cpu().data.numpy(), the calculation time of optimizer.step() will be greatly increased as shown below.

total_top1 = torch.tensor(0.).cuda()  # The first case
total_top5 = torch.tensor(0.).cuda()  # The first case
total_loss = torch.tensor(0.).cuda()  # The first case
# total_top1 = 0.0 # The second case
# total_top5 = 0.0 # The second case
# total_loss = 0.0 # The second case

outputs, loss = model(images, path, labels, criterion, gpu_nums)

st = time.time()
optimizer.step()
optimizer.zero_grad()
total_to += time.time() - st # step() time

top1, top5 = accuracy(outputs, labels, topk=(1, 5))

st = time.time()
total_counter += images.size(0)
# total_top1 += top1.cpu().data.numpy() #  or top1.item() # The second case
# total_top5 += top5.cpu().data.numpy() # The second case
# total_loss += loss.cpu().data.numpy() # The second case
total_top1 += top1 # The first case
total_top5 += top5 # The first case
total_loss += loss # The first case
total_ti += time.time() - st # data transform time

In first case, total_to = 18.0s, total_ti = 0.2s for 100 iterations calculation.But in second case, total_to = 8s, total_ti = 9s.

I tried to find the reason for this kind of difference.The most plausible answer is that item() prevents the newly calculated variable from entering the calculation graph, so the step()'s time is reduced.However, when I run the code below:

outputs, loss = model(images, path, labels, criterion, gpu_nums)

st = time.time()
optimizer.step()
optimizer.zero_grad()
total_to += time.time() - st # step() time

st = time.time()
total_counter += images.size(0)
# loss.item()
total_ti += time.time() - st # data transform time

total_to is still around 18s, and total_ti = 0s.When using loss.item(), total_to change to 8s, but total_ti changes to 9s.

So could you tell me what's the reason there in pytorch and how could I make total_ti be 0s and total_to be 8s? Thank you very much for your answers.


title: matrix multiplication using star operator

body:
I am curious about what the difference is between calling torch.mm(A, B) and A*B?

Looks like torch.mm gives us the desirable result, but A*B sometimes doesn't work.

It will be better if any documentation is provided.

Thank you!


title: any similar function like df.mask for tensor in pytorch?

body:
I want to replace all 0s in a 2-D tensor with -5.

With dataframe, i can easily do this:

df = df.mask(df=0, -5)

but this does not work for tensors. I have tried:

 y = torch.where(y = 0, -5, y)


title: How to collect the predictions for each observation in pytorch?

body:
Im trying to make a RNN myself using a loop rather than the nn.rnn module. But I would still like to use pytorch's backward for backpropogation.

Each observation makes 1 prediction which is then used for subsequent predictions. But how should each prediction be stored such that we can calculate the loss and get back propagation?

can we just create a normal list and append to it with each prediction? Or will that not allow back propogation?


title: LSTM hidden_state PyTorch almost identicals which leads to a negative KLDivLoss

body:
I've been puzzling over this issue and I cannot figure out what am I doing wrong...
I have trained an autoencoder (LSTM-LSTM) and I am now trying to use the encoded features for another task using the KLDivLoss. However, it turns out that the encoded features are almost always identical (see example below when setting the print precision to 10):

# last hidden state of the encoder
tensor([[[ 0.1086065620, -0.0446619801, -0.0530930459,  ...,
          -0.0573375113,  0.1083261892,  0.0037083717],
         [ 0.1086065620, -0.0446619801, -0.0530930459,  ...,
          -0.0573375151,  0.1083261892,  0.0037083712],
         [ 0.1086065620, -0.0446619801, -0.0530930459,  ...,
          -0.0573375188,  0.1083262041,  0.0037083719],
         ...,
         [ 0.1086065620, -0.0446619801, -0.0530930422,  ...,
          -0.0573375151,  0.1083262041,  0.0037083724],
         [ 0.1086065620, -0.0446619801, -0.0530930385,  ...,
          -0.0573375151,  0.1083262041,  0.0037083712],
         [ 0.1086065620, -0.0446619801, -0.0530930385,  ...,
          -0.0573375188,  0.1083261892,  0.0037083707]]],
       grad_fn=<StackBackward>)

Do you think there can be an explanation for such a behavior?
I am working with high-dimensional time series and my aim is to implement an unsupervised clustering method by taking inspiration from this paper. If I feed the raw output of the encoder into the KLDivLoss I obtain a negative loss... However, if I rescale the output of the encoder (using sklearn.preprocessing.StandardScaler) I get the desired behavior: a positive loss value.

I already ensured that the first term in the KLDivLoss is log-probabilities and the second proabilities...

Code for the encoder (which contains an attention mechanism to identify the relevant driving series):

class Encoder(nn.Module):
    def __init__(self, config, input_size: int):
        super(Encoder, self).__init__()
        self.input_size = input_size
        self.hidden_size = config['hidden_size_encoder']
        self.seq_len = config['seq_len']

        self.lstm = nn.LSTM(
            input_size=self.input_size,
            hidden_size=self.hidden_size,
            num_layers=1
        )
        self.attn = nn.Linear(
            in_features=2 * self.hidden_size + self.seq_len,
            out_features=1
        )
        self.dropout = nn.Dropout(p=0.5)
        self.softmax = nn.Softmax(dim=1)

    def forward(self, input_data):
        h_t, c_t = (init_hidden(input_data, self.hidden_size), init_hidden(input_data, self.hidden_size))
        input_weighted = Variable(torch.zeros(input_data.size(0), self.seq_len, self.input_size))

        for t in range(self.seq_len):
            x = torch.cat((h_t.repeat(self.input_size, 1, 1).permute(1, 0, 2),
                           c_t.repeat(self.input_size, 1, 1).permute(1, 0, 2),
                           input_data.permute(0, 2, 1).to(device)), dim=2).to(
                device)

            e_t = self.attn(x.view(-1, self.hidden_size * 2 + self.seq_len))
            a_t = self.dropout(self.softmax(e_t.view(-1, self.input_size)))

            weighted_input = torch.mul(a_t, input_data[:, t, :].to(device))
            self.lstm.flatten_parameters()
            _, (h_t, c_t) = self.lstm(weighted_input.unsqueeze(0), (h_t, c_t))

            input_weighted[:, t, :] = weighted_input

        return input_weighted[:, -1:, :], h_t, c_t

Code for clusterizing

class Clusterizer(nn.Module):
    def __init__(self, n_clusters, hidden_dim, encode, alpha=1.0):
        super(Clusterizer, self).__init__()
        self.n_clusters = n_clusters
        self.hidden_dim = hidden_dim
        self.encoder = encoder
        self.alpha = alpha
        self.centroids = None

    def init_centroids(self, encoded_x):
        '''initialize clusters center using KMeans'''
        kmeans = KMeans(n_clusters=self.n_clusters, random_state=0, n_init=10).fit(encoded_x)
        centroids = torch.tensor(kmeans.cluster_centers_, dtype=torch.float)
        self.centroids = nn.Parameter(centroids, requires_grad=True)

    def target_distribution(self, q_):
        weight = (q_ ** 2) / torch.sum(q_, 0)
        return (weight.t() / torch.sum(weight, 1)).t()

    def forward(self, encoded_x):
        num = ((1 + torch.norm(encoded_x.unsqueeze(1) - self.centroids, dim=2)) / self.alpha) ** (-(self.alpha + 1) / 2)
        den = torch.sum(num, dim=1, keepdim=True)
        return num / den

Final part of the code (where the problem occurs), here encoded_data is the stacked output of the encoder of shape (nb_observation, hidden_size).

criterion = nn.KLDivLoss(size_average=False)

...

clusterizer.init_centroids(encoded_data)
output = clusterizer(encoded_data.to(device))

target_distrib = clusterizer.target_distribution(output)
loss = criterion(output.log(), target_distrib)

I'm sorry that's a lot of code but I hope it helps to identify the source of the problem.


title: How to split a fine-tuned huggingface transformer model into two networks?

body:
I´m working on a framework where I have multiple text classification tasks (as shown in the example for binary text classification). For these tasks I´m using the powerful huggingface-transformer library. My code looks actually like this for the binary classification task.
class BertForBinaryDocumentClassification(BertPreTrainedModel):

def __init__(self, config):
    super().__init__(config)
    self.bert = BertModel.from_pretrained('bert-base-uncased')
    self.dropout = nn.Dropout(0.1)
    self.classifier = nn.Linear(in_features=config.hidden_size, out_features=config.num_labels)
    self.init_weights()

def forward(self, input_ids, token_type_ids=None, attention_mask=None,head_mask=None, labels=None):
    outputs = self.bert(input_ids, token_type_ids=token_type_ids, attention_mask=attention_mask, head_mask=head_mask)
    pooled_output = outputs[0]
    pooled_output = self.dropout(pooled_output)
    logits = self.classifier(pooled_output)

    outputs = (logits,) + outputs[2:]  # add hidden states and attention if they are here

    loss_fct = BCELoss()
    loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1))
    outputs = (loss,) + outputs

    return outputs  # (loss), logits, (hidden_states), (attentions)

Now I´m takle performance problems because for each classifiction task I use the whole BERT model with the proper classification head (binary and multiclass). One model is about 400-500Mb.
Now I´m wondering if I could split these networks in the BERT model, which produces the sentence embedding and send these encoded sentence embeddings to standalone classification networks with the final linear layer as shown in the code below.
The problem is now that I´m not really sure how to tackle this problem. I was reading of freezing the last layer and send the intermediate embedding to the classification network. One thing is that I wonder if it still represents the whole information of the BERT layers from 1 to 11?
Another thing is that I´m not really sure how the standalone classification layer should look like. I think it should look like this way.
class BinaryClassifier(nn.Module):

def __init__(self, vocab_size, embedding_dim, context_size):
    super(BinaryClassifier, self).__init__()
    self.classifier = nn.Linear(in_features=config.hidden_size, out_features=config.num_labels)
    self.init_weights()

def forward(self, inputs):
    embeds = self.embeddings(inputs).view((1, -1))
    logits = self.classifier(embeds)
    outputs = (logits,) + outputs[2:]  # add hidden states and attention if they are here

    loss_fct = BCELoss()
    loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1))
    outputs = (loss,) + outputs

    return outputs  # (loss), logits, (hidden_states), (attentions)

Can anybody tell me if my approach is correct and if not how to tackle this task? Thanks in advance.


title: Attribute Error : Can't pickle local object

body:
I am making a chatbot using pytorch. I developed this chatbot on a linux system which was working flawlessly. But when i am running the same model in windows 10. There is an error :-

Traceback (most recent call last):
return self.wsgi_app(environ, start_response)
return super(_SocketIOMiddleware, self).call(environ,
return self.wsgi_app(environ, start_response)
response = self.handle_exception(e)
return cors_after_request(app.make_response(f(*args, **kwargs)))
reraise(exc_type, exc_value, tb)
raise value
response = self.full_dispatch_request()
rv = self.handle_user_exception(e)
return cors_after_request(app.make_response(f(*args, **kwargs)))
reraise(exc_type, exc_value, tb)
raise value
rv = self.dispatch_request()
return self.view_functionsrule.endpoint                      > load(candidateSkillset.get(name))
File
for (words, labels_net) in train_loader:
return _MultiProcessingDataLoaderIter(self)
w.start()
self._popen = self._Popen(self)
return _default_context.get_context().Process._Popen(process_obj)
return Popen(process_obj)
reduction.dump(process_obj, to_child)
ForkingPickler(file, protocol).dump(obj)
AttributeError: Can't pickle local object 'load..ChatDataset'
exitcode = _main(fd, parent_sentinel)
self = reduction.pickle.load(from_parent)
EOFError: Ran out of input

I am calling the function with the dataset i want it to train on... Here is my code :-
def load(inp):

    global words,labels,doc_x,doc_y,questionsP1,questionsP2,questionsP3,questionsP4,model,questionTag, all_words, tags, xy, questions_list

    all_words=[]
    tags=[]
    xy=[]
    questions_list = []

    words=[]
    labels=[]
    docs_x=[]
    docs_y=[]
    questionsP1=[]
    questionsP2=[]
    questionsP3=[]
    questionsP4=[]
    questionTag={}

    print( dataFileNames.get(inp) )
    with open(dataFileNames.get(inp)) as file:
        data = json.load(file)

    for intent in data["intents"]:
        for proficiency in intent["proficiency"]:
            for questions in proficiency["questions"]:
                questions_list.append(questions["question"])
                for responses in questions["responses"]:
                    wrds = tokenize(responses)
                    all_words.extend(wrds)
                    xy.append((wrds, questions["question"]))

                if questions["tag"] in tags:
                    print(questions["tag"])

                if questions["tag"] not in tags:
                    tags.append(questions["tag"])

                if proficiency["level"] == "P1":
                    questionsP1.append(questions["question"])
                    questionTag[questions["question"]]=questions["tag"]

                if proficiency["level"] == "P2":
                    questionsP2.append(questions["question"])
                    questionTag[questions["question"]]=questions["tag"]

                if proficiency["level"] == "P3":
                    questionsP3.append(questions["question"])
                    questionTag[questions["question"]]=questions["tag"]

                if proficiency["level"] == "P4":
                    questionsP4.append(questions["question"])
                    questionTag[questions["question"]]=questions["tag"]

    #PyTorch Implementation

    ignore_words = ['?', '!', '.', ',']
    all_words = [stem(x) for x in all_words if x not in ignore_words]
    all_words = sorted(set(all_words))
    tags = sorted(set(tags))

    X_train = []
    y_train = []

    for tokenized_response, question in xy:
        bag = bag_of_words(tokenized_response, all_words)
        X_train.append(bag)
        label = questions_list.index( question)
        y_train.append(label)

    X_train = np.array(X_train)
    y_train = np.array(y_train)

    class ChatDataset(Dataset):
        def __init__(self):
            self.n_samples = len(X_train)
            self.x_data = X_train
            self.y_data = y_train

        def __getitem__(self, index):
            return self.x_data[index], self.y_data[index]

        def __len__(self):
            return self.n_samples

    #HyperParameters
    batch_size = 8
    hidden_size = 8
    output_size = len(tags)
    input_size = len(X_train[0])
    learning_rate = 0.001
    num_epochs = 1000

    dataset = ChatDataset()
    train_loader = DataLoader(dataset = dataset, batch_size=batch_size, shuffle = True, num_workers = 2)

    device = 'cpu'
    # print(device)
    global model_main
    model_main = NeuralNet(input_size, hidden_size, output_size).to(device)

    try:
        print("Inside Try")
        data = torch.load(modelNames.get(inp))
        input_size = data["input_size"]
        hidden_size = data["hidden_size"]
        output_size = data["output_size"]
        all_words = data["all_words"]
        questions_list = data["questions_list"]
        model_state = data["model_state"]

        model_main = NeuralNet(input_size, hidden_size, output_size).to(device)
        model_main.load_state_dict(model_state)
        model_main.eval()
    except:
        #loss and optimizer
        print("Inside Except")
        criterion = nn.CrossEntropyLoss()
        optimizer = torch.optim.Adam(model_main.parameters(), lr = learning_rate)
        for epoch in range(num_epochs):
            for (words, labels_net) in train_loader:
                words = words.to(device)
                labels_net = labels_net.to(device)

                #Forward
                outputs = model_main(words)
                loss = criterion(outputs, labels_net)
                # print(loss)
                #backward and optimizer step
                optimizer.zero_grad()
                loss.backward()
                optimizer.step()
                if (epoch + 1) % 100 == 0:
                    print(f'epoch {epoch + 1}/ {num_epochs}, loss={loss.item(): .4f}')

        print(f'final loss, loss={loss.item(): .4f}')

        ###############  Accuracy Calculation  ##############

        correct = 0
        total = 0
        # print(device)
        with torch.no_grad():
            for words, labels_net in train_loader:
                outputs = model_main(words)
                _, predicted = torch.max(outputs.data, 1)
                total += labels_net.size(0)
                correct += (predicted == labels_net).sum().item()

        print('Accuracy : %d %%' % ( 100 * correct / total ))

        #Saving the model

        data = {
            "model_state": model_main.state_dict(),
            "input_size": input_size,
            "output_size": output_size,
            "hidden_size": hidden_size,
            "all_words": all_words,
            "questions_list": questions_list
        }

        FILE = modelNames.get(inp)
        torch.save(data, FILE)


trying to export
https://github.com/minivision-ai/photo2cartoon
to onnx using the steps from
https://pytorch.org/tutorials/advanced/super_resolution_with_onnxruntime.html
args = parse_args()

torch_model = UgatitSadalinHourglass(args)
torch_model.build_model()

random_input = torch.randn(1, 3, 64, 64, dtype=torch.float32)

input_names = ["image"]
output_names = ["image"]

torch.onnx.export(torch_model, random_input, './model.onnx', verbose=False,
input_names=input_names, output_names=output_names,
opset_version=11)

AttributeError Traceback (most recent call last)
in ()
8 torch.onnx.export(torch_model, random_input, './model.onnx', verbose=False,
9 input_names=input_names, output_names=output_names,
---> 10 opset_version=11)

4 frames
/usr/local/lib/python3.6/dist-packages/torch/onnx/utils.py in select_model_mode_for_export(model, mode)
34 def select_model_mode_for_export(model, mode):
35 if not isinstance(model, torch.jit.ScriptFunction):
---> 36 is_originally_training = model.training
37
38 if mode is None:

AttributeError: 'UgatitSadalinHourglass' object has no attribute 'training'


title: Using pytorch Dataloader in pytorch kmeans

body: I'm Trying to perform a Kmeans clusterization using the Pytorch-kmeans (github). I have aprox. 27M array with 512 elements each, the aprox. size of the numpy array is 51GB which is bigger than my GPU RAM (32GB). Is there a way I can batch the arrays using the pytorch DataLoader class?