title: Adapting Pytorch "NLP from Scratch" for bidirectional GRU

body:
I have taken the code from the tutorial and attempted to modify it to include bi-directionality and any arbitrary numbers of layers for GRU.

Link to the tutorial which uses uni-directional, single layer GRU:
https://pytorch.org/tutorials/intermediate/seq2seq_translation_tutorial.html

The model works fine, but when i use set bidirectional=True, i get the a dimension mismatch error (shown below). Any thoughts why this is?

Encoder:

import torch.nn.init as init
class EncoderRNN(nn.Module):
    def __init__(self, input_size, hidden_size, n_layers=1, bidirectional=False):
        super(EncoderRNN, self).__init__()
        self.hidden_size = hidden_size
        self.hidden_var = hidden_size//2 if bidirectional else hidden_size
        self.n_layers = n_layers
        self.bidirectional = bidirectional
        self.n_directions = 2 if bidirectional else 1

        self.embedding = nn.Embedding(input_size, hidden_size)
        self.gru = nn.GRU(hidden_size,
                          self.hidden_var,
                          num_layers=self.n_layers,
                          bidirectional=self.bidirectional)

    def forward(self, input, hidden):
        embedded = self.embedding(input).view(1, 1, -1)
        output = embedded
        output, hidden = self.gru(output, hidden)
        #output = (output[:, :, :self.hidden_size] +
        #        output[:, :, self.hidden_size:])
        return output, hidden

    def initHidden(self):
        return torch.zeros(self.n_layers*self.n_directions, 1, self.hidden_var, device=device)

AttnDecoder:

class AttnDecoderRNN(nn.Module):
    def __init__(self, hidden_size, output_size, n_layers=1, dropout_p=0.1, max_length=MAX_LENGTH):
        super(AttnDecoderRNN, self).__init__()
        self.hidden_size = hidden_size
        self.output_size = output_size
        self.dropout_p = dropout_p
        self.max_length = max_length
        self.n_layers = n_layers

        self.embedding = nn.Embedding(self.output_size, self.hidden_size)
        self.attn = nn.Linear(self.hidden_size * 2, self.max_length)
        self.attn_combine = nn.Linear(self.hidden_size * 2, self.hidden_size)
        self.dropout = nn.Dropout(self.dropout_p)

        self.gru = nn.GRU(self.hidden_size,
                          self.hidden_size,
                          num_layers = self.n_layers)

        self.out = nn.Linear(self.hidden_size, self.output_size)

    def forward(self, input, hidden, encoder_outputs):
        embedded = self.embedding(input).view(1, 1, -1)
        embedded = self.dropout(embedded)

        attn_weights = F.softmax(
            self.attn(torch.cat((embedded[0], hidden[0]), 1)), dim=1)
        attn_applied = torch.bmm(attn_weights.unsqueeze(0),
                                 encoder_outputs.unsqueeze(0))

        output = torch.cat((embedded[0], attn_applied[0]), 1)
        output = self.attn_combine(output).unsqueeze(0)

        output = F.relu(output)
        output, hidden = self.gru(output, hidden)

        output = F.log_softmax(self.out(output[0]), dim=1)

        return output, hidden, attn_weights

    def initHidden(self):
        return torch.zeros(1*self.n_layers, 1, self.hidden_size, device=device)

Everything else from the tutorial is kept exactly the same apart from this code block ( to account for the new parameters):

n_layers=1
bidirectional = True
hidden_size = 256
encoder1 = EncoderRNN(input_lang.n_words, hidden_size, n_layers=n_layers, bidirectional=bidirectional).to(device)
attn_decoder1 = AttnDecoderRNN(hidden_size, output_lang.n_words, dropout_p=0.1, n_layers=n_layers).to(device)
trainIters(encoder1, attn_decoder1, 75000, print_every=5000)

Error:

---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-133-37084c93a197> in <module>
      5 attn_decoder1 = AttnDecoderRNN(hidden_size, output_lang.n_words, dropout_p=0.1, n_layers=n_layers).to(device)
      6
----> 7 trainIters(encoder1, attn_decoder1, 75000, print_every=5000)

<ipython-input-131-774ce8edefa6> in trainIters(encoder, decoder, n_iters, print_every, plot_every, learning_rate)
     16
     17         loss = train(input_tensor, target_tensor, encoder,
---> 18                      decoder, encoder_optimizer, decoder_optimizer, criterion)
     19         print_loss_total += loss
     20         plot_loss_total += loss

<ipython-input-130-67be7e8c2a58> in train(input_tensor, target_tensor, encoder, decoder, encoder_optimizer, decoder_optimizer, criterion, max_length)
     39         for di in range(target_length):
     40             decoder_output, decoder_hidden, decoder_attention = decoder(
---> 41                 decoder_input, decoder_hidden, encoder_outputs)
     42             topv, topi = decoder_output.topk(1)
     43             decoder_input = topi.squeeze().detach()  # detach from history as input

~/miniconda3/envs/pytorch/lib/python3.7/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    545             result = self._slow_forward(*input, **kwargs)
    546         else:
--> 547             result = self.forward(*input, **kwargs)
    548         for hook in self._forward_hooks.values():
    549             hook_result = hook(self, input, result)

<ipython-input-129-6dd1d30fe28f> in forward(self, input, hidden, encoder_outputs)
     24
     25         attn_weights = F.softmax(
---> 26             self.attn(torch.cat((embedded[0], hidden[0]), 1)), dim=1)
     27         attn_applied = torch.bmm(attn_weights.unsqueeze(0),
     28                                  encoder_outputs.unsqueeze(0))

~/miniconda3/envs/pytorch/lib/python3.7/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    545             result = self._slow_forward(*input, **kwargs)
    546         else:
--> 547             result = self.forward(*input, **kwargs)
    548         for hook in self._forward_hooks.values():
    549             hook_result = hook(self, input, result)

~/miniconda3/envs/pytorch/lib/python3.7/site-packages/torch/nn/modules/linear.py in forward(self, input)
     85
     86     def forward(self, input):
---> 87         return F.linear(input, self.weight, self.bias)
     88
     89     def extra_repr(self):

~/miniconda3/envs/pytorch/lib/python3.7/site-packages/torch/nn/functional.py in linear(input, weight, bias)
   1367     if input.dim() == 2 and bias is not None:
   1368         # fused op is marginally faster
-> 1369         ret = torch.addmm(bias, input, weight.t())
   1370     else:
   1371         output = input.matmul(weight.t())

RuntimeError: size mismatch, m1: [1 x 384], m2: [512 x 10] at /tmp/pip-req-build-58y_cjjl/aten/src/TH/generic/THTensorMath.cpp:752

Any help would be appreciated!

Update based on user3923920 comment (encoder-decoder also includes LSTM option & now works with bidirectionality)

New working and adapted Encoder

class EncoderRNN(nn.Module):
        def __init__(self, input_size, hidden_size, n_layers=1, bidirectional=False, method='GRU'):
            super(EncoderRNN, self).__init__()
            self.hidden_size = hidden_size
            self.hidden_var = hidden_size // 2 if bidirectional else hidden_size
            self.n_layers = n_layers
            self.bidirectional = bidirectional
            self.n_directions = 2 if bidirectional else 1
            self.method = method

            self.embedding = nn.Embedding(input_size, hidden_size)
            if self.method == 'GRU':
                self.net = nn.GRU(hidden_size,
                                  self.hidden_var,
                                  num_layers=self.n_layers,
                                  bidirectional=self.bidirectional)
            elif self.method == 'LSTM':
                self.net = nn.LSTM(hidden_size,
                                   self.hidden_var,
                                   num_layers=self.n_layers,
                                   bidirectional=self.bidirectional)

        def forward(self, input, hidden):
            embedded = self.embedding(input).view(1, 1, -1)
            output = embedded
            output, hidden = self.net(output, hidden)
            # output = (output[:, :, :self.hidden_size] +
            #        output[:, :, self.hidden_size:])
            return output, hidden, embedded

        def initHidden(self):
            if self.method == 'GRU':
                return torch.zeros(self.n_layers * self.n_directions, 1, self.hidden_var, device=device)
            elif self.method == 'LSTM':
                h_state = torch.zeros(self.n_layers * self.n_directions, 1, self.hidden_var)
                c_state = torch.zeros(self.n_layers * self.n_directions, 1, self.hidden_var)
                hidden = (h_state, c_state)
                return hidden

New working and adapted Decoder

class AttnDecoderRNN(nn.Module):
        def __init__(self, hidden_size, output_size, n_layers=1, dropout_p=0.1,
                     max_length=MAX_LENGTH, method='GRU', bidirectional=False):

            super(AttnDecoderRNN, self).__init__()
            self.hidden_size = hidden_size
            self.output_size = output_size
            self.dropout_p = dropout_p
            self.max_length = max_length
            self.n_layers = n_layers
            self.method = method
            self.bidirectional = bidirectional

            self.embedding = nn.Embedding(self.output_size, self.hidden_size)
            self.attn = nn.Linear(self.hidden_size * 2, self.max_length)
            self.attn_combine = nn.Linear(self.hidden_size * 2, self.hidden_size)
            self.dropout = nn.Dropout(self.dropout_p)

            if self.method == 'GRU':
                self.net = nn.GRU(self.hidden_size,
                                  self.hidden_size,
                                  num_layers=self.n_layers)
            elif self.method == 'LSTM':
                self.net = nn.LSTM(self.hidden_size,
                                   self.hidden_size,
                                   num_layers=self.n_layers)

            self.out = nn.Linear(self.hidden_size, self.output_size)

        def forward(self, input, hidden, encoder_outputs):

            # Embed
            embedded = self.embedding(input).view(1, 1, -1)
            embedded = self.dropout(embedded)
            self.hidden = hidden

            # Concatenate all of the layers
            hidden_h_rows = ()
            hidden_c_rows = ()

            if self.method == 'LSTM':
                # hidden is a tuple of h_state and c_state
                decoder_h, decoder_c = hidden
                print(decoder_h.shape)
                hidden_shape = decoder_h.shape[0]

                # h_state
                for x in range(0, hidden_shape):
                    hidden_h_rows += (decoder_h[x],)

                # c_state
                for x in range(0, hidden_shape):
                    hidden_c_rows += (decoder_c[x],)

            elif self.method == "GRU":

                # hidden is not a tuple (GRU)
                decoder_h = hidden
                hidden_shape = decoder_h.shape[0]

                # h_state
                for x in range(0, hidden_shape):
                    hidden_h_rows += (decoder_h[x],)

            if self.bidirectional:
                decoder_h_cat = torch.cat(hidden_h_rows, 1)
                # Make sure the h_dim size is compatible with num_layers with concatenation.
                decoder_h = decoder_h_cat.view((self.n_layers, 1, self.hidden_size))  # hidden_size=256

                if self.method == "LSTM":
                    decoder_c_cat = torch.cat(hidden_c_rows, 1)
                    decoder_c = decoder_c_cat.view((self.n_layers, 1, self.hidden_size))  # hidden_size=256
                    hidden_lstm = (decoder_h, decoder_c)

                elif self.method == "GRU":
                    hidden_gru = decoder_h

            # Attention Block
            attn_weights = F.softmax(
                self.attn(torch.cat((embedded[0], hidden_lstm[0][0] if self.method == "LSTM" else \
                    hidden_gru[0]), 1)), dim=1)
            attn_applied = torch.bmm(attn_weights.unsqueeze(0), encoder_outputs.unsqueeze(0))
            output = torch.cat((embedded[0], attn_applied[0]), 1)
            output = self.attn_combine(output).unsqueeze(0)

            output = F.relu(output)
            output, hidden = self.net(output,
                                      hidden_lstm if self.method == "LSTM" else hidden_gru)  # I am not sure about this!
            output = F.log_softmax(self.out(output[0]), dim=1)
            return output, hidden, attn_weights

        def initHidden(self):

            if self.method == 'GRU':
                return torch.zeros(self.n_layers * 1, 1, self.hidden_var, device=device)
            elif self.method == 'LSTM':
                h_state = torch.zeros(self.n_layers * 1, 1, self.hidden_var)
                c_state = torch.zeros(self.n_layers * 1, 1, self.hidden_var)
                hidden = (h_state, c_state)
                return hidden


title: In pytorch, the meaning of y.backward([0.1, 1.0, 0.0001])

body:
In pytorch
what is the meaning of y.backward([0.1, 1.0, 0.0001])?

I understand that y.backward() means doing backpropagation.
But what is the meaning of [0.1, 1.0, 0.0001] in y.backward([0.1, 1.0, 0.0001])?


title: Encountering Import Error DLL load failed constantly

body:
I have been trying to intall scikit-learn and pytorch using their respective commands given in the docs:
The commands for installing PyTorch are:
1) pip3 install https://download.pytorch.org/whl/cpu/torch-1.0.1-cp37-cp37m-win_amd64.whl
2) pip3 install torchvision

The command for installing scikit-learn is:
pip install -U scikit-learn

Some background:
I am using Windows 8.1, Python 3.7.2. My pip is updated. I have also installed Anaconda for solving this using conda, but had zero luck!(Also, here I am running into 'conda' unrecognized error which is another story). Here are the paths my PATH variable holds.
PATH

C:\Users\satya\Anaconda3;
C:\Users\satya\Anaconda3\Library\mingw-w64\bin;
C:\Users\satya\Anaconda3\Library\usr\bin;
C:\Users\satya\Anaconda3\Library\bin;
C:\Users\satya\Anaconda3\Scripts;
C:\Users\satya\AppData\Local\Programs\Python\Python37\Scripts\;
C:\Users\satya\AppData\Local\Programs\Python\Python37\; C:\Users\satya\AppData\Roaming\Microsoft\Windows\Start Menu\Programs\Python 3.7

The Actual Problem:
The same commands for installation given above work perfectly fine on my other Windows 10, but, for my Windows 8.1 it gives this error which has become a real PITA

Import Error: DLL load failed The specified module could not be found

When I import sklearn or import torch I get the exact same error. All the time.
Back Story:
I have searched almost all the related questions I could find on Stackoverflow and Github for 6+ hours to help me solve this problem. But, none of the answers have helped till now and some haven't had an "understandable" answer. Maybe, its just a small fix, but now, I am choosing to post a question on SO.
My Question Again:
Can someone please help out and try to explain what I am missing out here? I really want to fix this error for good(and want to be in a position to fix it if I encounter it again). An elaborate answer would really help understand easily.
Thank You!


title: PyTorch equivalent of a Tensorflow linear layer

body:
I was trying to reimplement a Tensorflow code using PyTorch framework. Below I have included the TF sample code and my PyT interpretation.

TensorFlow implementation:

W1 = tf.Variable(xavier_init([135, 128]))
b1 = tf.Variable(tf.zeros(shape=[128]))

def fcn(x):
    z = tf.reshape(x, (-1, 135))
    out1 = leaky_relu( tf.matmul(z, W1) + b1 )

    return out1

PyTorch implementation:

class decoder(nn.Module):
    def __init__(self):
        super(decoder, self).__init__()

        self.layer_10 = nn.Linear(135, 128, bias=True)
        self.leaky = nn.LeakyReLU(0.2, inplace=False)
        init.xavier_uniform(self.layer_10.weight)

    def forward(self, x):
        z = x.view(-1, 135)
        h30 = self.leaky(self.layer_10(z))

        return h30

I was wondering what is the proper way to implement the matmul part, given that the weights in pytorch are not explicitly defined as they are in the TF (or correct me if I'm wrong).


title: Pytorch equivalent features in tensorflow?

body: I recently was reading a Pytorch code and came across loss.backward() and optimizer.step() functions, are there any equivalent of these using tensorflow/keras?


title: How to derive weight and bias in a neural network?

body:
This is the Network:

    class Net(torch.nn.Module):
        def __init__(self, n_feature, n_hidden, n_output):
            super(Net, self).__init__()
            self.hidden = torch.nn.Linear(n_feature, n_hidden)
            self.predict = torch.nn.Linear(n_hidden, n_output)   # output layer
        def forward(self, x):
            x = F.relu(self.hidden(x))      # activation function for hidden layer
            x = self.predict(x)             # linear output
            return x

    net = Net(n_feature=1, n_hidden=10, n_output=1)
    pytorch_total_params = sum(p.numel() for p in net.parameters())
    print(pytorch_total_params)

    w = list(net.parameters())
    print(w)

This is the runnung result:
31
[Parameter containing:
tensor([[ 0.9534],
        [-0.0309],
        [-0.9570],
        [-0.4179],
        [-0.3757],
        [-0.4227],
        [-0.8866],
        [ 0.2107],
        [ 0.0222],
        [ 0.2531]], requires_grad=True), Parameter containing:
tensor([-0.0358, -0.2533,  0.2979,  0.9777,  0.9606,  0.9460,  0.9059,  0.7582,
        -0.5286,  0.3367], requires_grad=True), Parameter containing:
tensor([[-0.2863, -0.3157,  0.2086, -0.0011, -0.0415, -0.2574, -0.0683, -0.0788,
         -0.0339, -0.0195]], requires_grad=True), Parameter containing:
tensor([0.2031], requires_grad=True)]

I don't know why is the number of parameters be 31?
And also don't understand the numbers printed above.(whether it is weight or bias

Because I thought in Relu function, there will only be (2 parameters*10) which is weight and bias multiply 10 hidden layers.


title: RuntimeError: cuda runtime error (8) in pytorch

body:
when I try to run a pytorch faster rcnn code (from https://github.com/rowanz/neural-motifs), I get the issuse as follow

THCudaCheck FAIL file=/opt/conda/conda-bld/pytorch_1518238409320/work/torch/lib/THC/generic/THCTensorMathPairwise.cu
line=21 error=8 : invalid device function

It arises when operate
    keep.append(keep_im + s)
where keep is a list, s is a int number and keep_im  is a torch.cuda.Longtensor

It's strange that, when I modify the code as

try:
    keep.append(keep_im + s)
except BaseException:
    keep.append(keep_im + s)

It return error in the try and then operates again in the except, and successes....

Anyone know what happen here?
I use python 2.7 + pytorch 0.3 + Cuda 8 + cudnn7.1, Titan XP in Ubuntu 16. Thanks


title: Why "ValueError: optimizer got an empty parameter list" is happened in this case?

body:
I already studied the questions on this topic and I saw only recommendations to use ModuleList instead of usual list. But I don't understand why this error occurs in the case when I use nn.Sequential?
I tried to build AlexNet like in official implementation here: https://github.com/pytorch/vision/blob/master/torchvision/models/alexnet.py
but got "ValueError: optimizer got an empty parameter list"

class AlexNet(nn.Module):

def __init__(self, input_channels, n_classes=1000):
    super(AlexNet, self).__init__()

    self.features = nn.Sequential
    (
        nn.Conv2d(input_channels, 96, kernel_size=11, stride=4),
        nn.LocalResponseNorm(size=2, alpha=2e-5),
        nn.MaxPool2d(kernel_size=3, stride=2),
        nn.ReLU(inplace=True),

        nn.Conv2d(96, 256, kernel_size=5, stride=1),
        nn.LocalResponseNorm(size=2, alpha=2e-5),
        nn.MaxPool2d(kernel_size=3, stride=2),
        nn.ReLU(inplace=True),

        nn.Conv2d(256, 384, kernel_size=3, stride=1),
        nn.ReLU(inplace=True),

        nn.Conv2d(384, 384, kernel_size=3, stride=1),
        nn.ReLU(inplace=True),

        nn.Conv2d(384, 256, kernel_size=3, stride=1),
        nn.ReLU(inplace=True),

    )

    self.fully_connected = nn.Sequential
    (
        nn.Dropout2d(0.5),
        nn.Linear(256, 4096),
        nn.ReLU(inplace=True),
        nn.Linear(4096, 4096),
        nn.ReLU(inplace=True),
        nn.Linear(4096, n_classes)
    )

def forward(self, x):

    x = self.features(x)
    x = nn.Flatten(x)
    x = self.fully_connected(x)

    return x

model = AlexNet(input_channels=1, n_classes=10)
optimizer = optim.Adam(model.parameters() , lr=1e-3)


title: Cuda Out of Memory Error even after reducing the batch size

body:
I'm trying to train a deep learning model in Azure Notebook which uses GPU from the DSVM-Ubuntu 18.04 which consists of Standard NC6 (6 vcpus, 56 GiB memory) and is getting the following Error:

  RuntimeError: CUDA out of memory. Tried to allocate 64.00 MiB (GPU 0; 11.17 GiB total capacity; 10.76 GiB already allocated; 50.31 MiB free; 10.84 GiB reserved in total by PyTorch)

I've searched on this regard and couldn't find any solution in any of the questions on the web. And '10.84 GiB reserved in total by PyTorch' in the error message caught my attention, whether this can be configured to have a low memory value? I would like to receive any opinions in this regard. thank you.

This is my Code for Fine-tuning/training

for epoch in range(EPOCHS):

    for idx,article in tqdm_notebook(enumerate(article_loader)):

        article_tens = torch.tensor(tokenizer.encode(article[0], max_length=1024)).unsqueeze(0).to(device)

        outputs = model(article_tens, labels=article_tens)

        train_loss, prediction_scores = outputs[:2]
        train_loss.backward()
        train_sum_loss = train_sum_loss + train_loss.detach().data

        iteration_count=idx

        article_count = article_count + 1
        if article_count == BATCH_SIZE:
            article_count = 0
            batch_count += 1
            optimizer.step()
            scheduler.step()
            optimizer.zero_grad()
            model.zero_grad()

Whole Stack-trace of the error:

---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-11-2c74a22e42f7> in <module>
     20         article_tens = torch.tensor(tokenizer.encode(article[0], max_length=1024)).unsqueeze(0).to(device)
     21
---> 22         outputs = model(article_tens, labels=article_tens)
     23
     24         train_loss, prediction_scores = outputs[:2]

/anaconda/envs/py37_pytorch/lib/python3.7/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    548             result = self._slow_forward(*input, **kwargs)
    549         else:
--> 550             result = self.forward(*input, **kwargs)
    551         for hook in self._forward_hooks.values():
    552             hook_result = hook(self, input, result)

/anaconda/envs/py37_pytorch/lib/python3.7/site-packages/transformers/modeling_gpt2.py in forward(self, input_ids, past, attention_mask, token_type_ids, position_ids, head_mask, inputs_embeds, labels, use_cache)
    602             head_mask=head_mask,
    603             inputs_embeds=inputs_embeds,
--> 604             use_cache=use_cache,
    605         )
    606         hidden_states = transformer_outputs[0]

/anaconda/envs/py37_pytorch/lib/python3.7/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    548             result = self._slow_forward(*input, **kwargs)
    549         else:
--> 550             result = self.forward(*input, **kwargs)
    551         for hook in self._forward_hooks.values():
    552             hook_result = hook(self, input, result)

/anaconda/envs/py37_pytorch/lib/python3.7/site-packages/transformers/modeling_gpt2.py in forward(self, input_ids, past, attention_mask, token_type_ids, position_ids, head_mask, inputs_embeds, use_cache)
    486                 attention_mask=attention_mask,
    487                 head_mask=head_mask[i],
--> 488                 use_cache=use_cache,
    489             )
    490

/anaconda/envs/py37_pytorch/lib/python3.7/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    548             result = self._slow_forward(*input, **kwargs)
    549         else:
--> 550             result = self.forward(*input, **kwargs)
    551         for hook in self._forward_hooks.values():
    552             hook_result = hook(self, input, result)

/anaconda/envs/py37_pytorch/lib/python3.7/site-packages/transformers/modeling_gpt2.py in forward(self, x, layer_past, attention_mask, head_mask, use_cache)
    240
    241         x = x + a
--> 242         m = self.mlp(self.ln_2(x))
    243         x = x + m
    244

/anaconda/envs/py37_pytorch/lib/python3.7/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    548             result = self._slow_forward(*input, **kwargs)
    549         else:
--> 550             result = self.forward(*input, **kwargs)
    551         for hook in self._forward_hooks.values():
    552             hook_result = hook(self, input, result)

/anaconda/envs/py37_pytorch/lib/python3.7/site-packages/transformers/modeling_gpt2.py in forward(self, x)
    215
    216     def forward(self, x):
--> 217         h = self.act(self.c_fc(x))
    218         h2 = self.c_proj(h)
    219         return self.dropout(h2)

/anaconda/envs/py37_pytorch/lib/python3.7/site-packages/transformers/activations.py in gelu_new(x)
     27         Also see https://arxiv.org/abs/1606.08415
     28     """
---> 29     return 0.5 * x * (1.0 + torch.tanh(math.sqrt(2.0 / math.pi) * (x + 0.044715 * torch.pow(x, 3.0))))
     30
     31

RuntimeError: CUDA out of memory. Tried to allocate 16.00 MiB (GPU 0; 11.17 GiB total capacity; 10.74 GiB already allocated; 320.00 KiB free; 10.89 GiB reserved in total by PyTorch)


I am trying to train my first neural net in pyTorch (I'm not a programmer, just a confused chemist).
The net itself is supposed to take 1064 element vectors and rate them with a float number.
So far I have encountered a variety of errors, ranging from 'float instead of long' to 'Target 1 is out of bounds'. Thus I have redefined the dtypes, corrected the dimensions of the input vector, changed the loss function and now I am stuck in the situation when correcting the current error sets me back to the previous ones.
Which is:
ValueError: Target size (torch.Size([1000])) must be the same as input size (torch.Size([1000, 1]))

at the 'loss=loss_calc(outputs, target)' line.
I tried unsqueezing the label during the DataSet class definition, but this solution sets me back. When I tried to label = label.view(1,1), the resulting error changed to

Target size (torch.Size([1000, 1, 1])) must be the same as input size (torch.Size([1000, 1]))

Could anyone please help me figure this out?
import pandas as pd
import numpy as np
import rdkit
from rdkit import Chem
from rdkit.Chem import AllChem
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torch.utils.data import Dataset, DataLoader

class dataset(Dataset):
    def __init__(self, path, transform=None):
        self.data = pd.read_excel(path)
        self.transform = transform

    def __len__(self):
        return len(self.data)

    def __getitem__(self, index):
        smiles=self.data.at[index, 'smiles']
        mol=Chem.MolFromSmiles(smiles)
        morgan = torch.tensor(list(AllChem.GetMorganFingerprintAsBitVect(mol, 1, nBits=1064)), dtype=torch.float)

        label=torch.tensor(self.data.at[index, 'score'], dtype=torch.long)

        if self.transform is not None:
            morgan=self.transform(morgan)

        return morgan, label

class Net(nn.Module):

    def __init__(self):
        super(Net, self).__init__()
        self.fc1 = nn.Linear(1064, 1064)
        self.fc2 = nn.Linear(1064, 1)
        self.act = nn.Tanh()

    def forward(self, x):
        x = self.act(self.fc1(x))
        x = self.act(self.fc2(x))
        x = self.fc2(x)
        return x

trainSet=dataset(r'C:\Users\BajMic\Peptides\trainingSet.xlsx')
testSet=dataset(r'C:\Users\BajMic\Peptides\testSet.xlsx')

net = Net()
loss_calc = nn.CrossEntropyLoss()
optimizer = optim.SGD(net.parameters(), lr=0.001, momentum=0.9)

for epoch in range(2):
    running_loss=0.0
    for data in DataLoader(trainSet, batch_size=1000, shuffle=True):
        inputs, target = data
        optimizer.zero_grad()
        outputs = net(inputs)
        print(outputs)
        loss = loss_calc(outputs, target)
        loss.backward()
        optimizer.step()        # print statistics
        running_loss += loss.item()
        print('[%d, %5d] loss: %.3f' %(epoch + 1, i + 1, running_loss))
        running_loss = 0.0
print('Finished Training')


title: Pytorch Train & Eval Different Sample Sizes

body:
I am learning pytorch, and have the following (abbreviated) code to setup for modeling:

# define the model class for a neural net with 1 hidden layer
class myNN(nn.Module):
    def __init__(self, D_in, H, D_out):
        super(myNN, self).__init__()
        self.lin1 = nn.Linear(D_in,H)
        self.lin2 = nn.Linear(H,D_out)
    def forward(self,X):
        return torch.sigmoid(self.lin2(torch.sigmoid(self.lin1(x))))

# now make the datasets & dataloaders
batchSize = 5
# Create the data class
class Data(Dataset):
    def __init__(self, x, y):
        self.x = torch.FloatTensor(x)
        self.y = torch.Tensor(y.astype(int))
        self.len = self.x.shape[0]
        self.p = self.x.shape[1]
    def __getitem__(self, index):
        return self.x[index], self.y[index]
    def __len__(self):
        return self.len
trainData = Data(trnX, trnY)
trainLoad = DataLoader(dataset = trainData, batch_size = batchSize)
testData = Data(tstX, tstY)
testLoad = DataLoader(dataset = testData, batch_size = len(testData))

# define the modeling objects
hiddenLayers = 30
learningRate = 0.1
model = myNN(p,hiddenLayers,1)
print(model)
optimizer = torch.optim.SGD(model.parameters(), lr = learningRate)
loss = nn.BCELoss()

with trnX.shape=(70, 2), trnY.shape=(70,), tstX.shape=(30,2), and tstY.shape=(30,). The code for training is:

# train!
epochs = 1000
talkFreq = 0.2
trnLoss = [np.inf]*epochs
tstLoss = [np.inf]*epochs
for i in range(epochs):
    # train with minibatch gradient descent
    for x, y in trainLoad:
        # forward step
        yhat = model(x)
        # compute loss (not storing for now, will do after minibatching)
        l = loss(yhat, y)
        # backward step
        optimizer.zero_grad()
        l.backward()
        optimizer.step()
    # evaluate loss on training set
    yhat = model(trainData.x)
    trnLoss[i] = loss(yhat, trainData.y)
    # evaluate loss on testing set
    yhat = model(testData.x)
    tstLoss[i] = loss(yhat, testData.y)

The datasets trainData and testData have 70 and 30 observations, respectively. This is probably just a newbie problem, but when I run the training cell, it errors on the trnLoss[i] = loss(yhat, trainData.y) line with the error

ValueError: Target and input must have the same number of elements. target nelement (70) != input nelement (5)

When I inspect the output of the yhat=model(trainData.x) line, I see that yhat is a tensor with batchSize elements, despite the fact that trainData.x.shape = torch.Size([70, 2]).

How can I iteratively train the model with the minibatch gradient descent, then use the model to compute the loss & accuracy on the full training & testing sets? I tried setting model.train() just before the mini batch iteration, then model.eval() just before the evaluation code, to no avail.


title: Pytorch + Residual Network throws unexpected: TypeError: 'NoneType' object is not callable

body:
I am diving into the world of Deep Learning and as Python is my most prominent programming language I decided to start with the Pytorch framework. Following a tutorial I implemented a ResNet of 50 layers. Afterwards I tried to make an ResNet-18 with simpler blocks which looked like this:

class baseBlock(torch.nn.Module):
  expansion = 1
  def __init__(self,input_planes,planes,stride=1,dim_change=None):
    super(baseBlock,self).__init__()
    #Declare convolutional layers with batch norms
    self.conv1 = torch.nn.Conv2d(input_planes, planes, stride=stride, kernel_size=3, padding=1)
    self.bn1   = torch.nn.BatchNorm2d(planes)
    self.conv2 = torch.nn.Conv2d(planes, planes, stride=1, kernel_size=3, padding=1)
    self.bn2   = torch.nn.BatchNorm2d(planes)
    self.dim_change = dim_change

  def forward(self,x):
    #Save the residue
    res = x
    output = F.relu(self.bn1(self.conv1(x)))
    output = self.bn2(self.conv2(output))

    if self.dim_change is not None:
      res = self.dim_change(res)

    output += res
    output = F.relu(output)

    return output

I divided the code into the blocks and the actual Neural Network to make it more understandable. The rest of the code for the NN is:

class ResNet(torch.nn.Module):
  def __init__(self,block,num_layers,classes=10):
    super(ResNet,self).__init__()
    #according to research paper
    self.input_planes = 64
    self.conv1  = torch.nn.Conv2d(3,64,kernel_size =3,stride=1,padding=1)
    self.bn1    = torch.nn.BatchNorm2d(64)
    self.layer1 = self._layer(block, 64, num_layers[0], stride=1)
    self.layer2 = self._layer(block, 128, num_layers[1], stride=2)
    self.layer3 = self._layer(block, 256, num_layers[2], stride=2)
    self.layer4 = self._layer(block, 512, num_layers[3], stride=2)
    self.avaragePool = torch.nn.AvgPool2d(kernel_size=4,stride=1)
    self.fc = torch.nn.Linear(512*block.expansion, classes)

  def _layer(self,block,planes,num_layers,stride=1):
    dim_change = None
    if stride != 1 or planes != self.input_planes*block.expansion:
      dim_change = torch.nn.Sequential(torch.nn.Conv2d(self.input_planes, planes*block.expansion, kernel_size = 1, stride=stride),
                                                        torch.nn.BatchNorm2d(planes*block.expansion))
      netLayers = []
      netLayers.append(block(self.input_planes,planes,stride=stride, dim_change=dim_change))

      self.input_planes = planes*block.expansion

      for i in range(1,num_layers):
        netLayers.append(block(self.input_planes,planes))
        self.input_planes = planes * block.expansion

      return torch.nn.Sequential(*netLayers)

  def forward(self,x):
    x = F.relu(self.bn1(self.conv1(x)))

    #Problem at this layer
    x = self.layer1(x) <- NoneType object is not callable
    x = self.layer2(x)
    x = self.layer3(x)
    x = self.layer4(x)

    x = F.avg_pool2d(x,4)
    x = x.view(x.size(0),-1)
    x = self.fc(x)

    return x

The problem arises on the last forward function in the layer1 that is marked on the code. If I comment it the network keeps on computing (obviously it throws an error later because the dimensions are not right but it still does something). If I change the stride on this first layer the problem also resolves but still throws an error later for the same dimensional reasons. I don't seem to figure out what the problem is. I use a standard CIFAR10 dataset to train it and the training code is as standard as it gets.

If I can provided any additional information let me know. Thank you


title: Pytorch cross entropy input dimensions

body:
I'm trying to develop a binary classifier with Huggingface's BertModel and Pytorch.
The classifier module is something like this:

class SSTClassifierModel(nn.Module):

  def __init__(self, num_classes = 2, hidden_size = 768):
    super(SSTClassifierModel, self).__init__()
    self.number_of_classes = num_classes
    self.dropout = nn.Dropout(0.01)
    self.hidden_size = hidden_size
    self.bert = BertModel.from_pretrained('bert-base-uncased')
    self.classifier = nn.Linear(hidden_size, num_classes)

  def forward(self, input_ids, att_masks,token_type_ids,  labels):
    _, embedding = self.bert(input_ids, token_type_ids, att_masks)
    output = self.classifier(self.dropout(embedding))
    return output

The way I train the model is as follows:

loss_function = BCELoss()
model.train()
for epoch in range(NO_OF_EPOCHS):
  for step, batch in enumerate(train_dataloader):
        input_ids = batch[0].to(device)
        input_mask = batch[1].to(device)
        token_type_ids = batch[2].to(device)
        labels = batch[3].to(device)
        # assuming batch size = 3, labels is something like:
        # tensor([[0],[1],[1]])
        model.zero_grad()
        model_output = model(input_ids,
                             input_mask,
                             token_type_ids,
                             labels)
        # model output is something like: (with batch size = 3)
        # tensor([[ 0.3566, -0.0333],
                 #[ 0.1154,  0.2842],
                 #[-0.0016,  0.3767]], grad_fn=<AddmmBackward>)

        loss = loss_function(model_output.view(-1,2) , labels.view(-1))

I'm doing the .view()s because of the Huggingface's source code for BertForSequenceClassification here which uses the exact same way to compute the loss. But I get this error:

/usr/local/lib/python3.6/dist-packages/torch/nn/functional.py in binary_cross_entropy(input, target, weight, size_average, reduce, reduction)
   2068     if input.numel() != target.numel():
   2069         raise ValueError("Target and input must have the same number of elements. target nelement ({}) "
-> 2070                          "!= input nelement ({})".format(target.numel(), input.numel()))
   2071
   2072     if weight is not None:

ValueError: Target and input must have the same number of elements. target nelement (3) != input nelement (6)

Is there something wrong with my labels? or my model's output? I'm really stuck here. The documentation for Pytorch's BCELoss says:

Input: (N,∗) where ∗ means, any number of additional dimensions
Target: (N,∗), same shape as the input

How should I make my labels the same shape as the model output? I feel like there's something huge that I'm missing but I can't find it.


title: only one element tensors can be converted to python scalars

body:
def Log(A):
    '''
    theta = arccos((tr(A)-1)/2)
    K=1/(2sin(theta))(A-A^T)
    log(A)=theta K
    '''
    theta=torch.acos(torch.tensor((torch.trace(A)-1)/2))
    K=(1/(2*torch.sin(theta)))*(torch.add(A,-torch.transpose(A,0,1)))

    return theta*K

def tensor_Log(A):
    blah=[[Log(A[i,j]) for j in range(A.shape[1])] for i in range(A.shape[0])]
    new=torch.tensor(blah)

    return new

ValueError: only one element tensors can be converted to Python scalars

during training to get the outputs of my network, the above function is producing the following error, it is called inside a custom layer and I do not know what it is referencing, any thoughts?


title: Loading a single graph into a pytorch geometric data object for node classification

body:
I have one graph, defined by 4 matrices: x (node features), y (node labels), edge_index (edges list) and edge_attr (edge features). I want to create a dataset in Pytorch Geometric with this single graph and perform node-level classification. It seems that just wrapping these 4 matrices into a data object fails, for some reason.
I have created a dataset containing the attributes:
Data(edge_attr=[3339730, 1], edge_index=[2, 3339730], x=[6911, 50000], y=[6911, 1])

representing a graph. If I try to slice this graph, like:
train_dataset, test_dataset = dataset[:5000], dataset[5000:]

I get the error:
---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<ipython-input-11-feb278180c99> in <module>
      3 # train_dataset, test_dataset = torch.utils.data.random_split(dataset, [train_size, test_size])
      4
----> 5 train_dataset, test_dataset = dataset[:5000], dataset[5000:]
      6
      7 # Create dataloader for training and test dataset.

~/anaconda3/envs/py38/lib/python3.8/site-packages/torch_geometric/data/data.py in __getitem__(self, key)
     92     def __getitem__(self, key):
     93         r"""Gets the data of the attribute :obj:`key`."""
---> 94         return getattr(self, key, None)
     95
     96     def __setitem__(self, key, value):

TypeError: getattr(): attribute name must be string

What am I doing wrong in the data construction?


title: pix2pixHD error with own dataset

body:
I am trying to generate my own images using the pix2pixHD pre-trained model. Github repo found here

The images inside the dataset has to be in grayscale with no alpha channel. The images in the repo has a size of 16 bitPerSample and I have both images in size 8 and 16 bitsPerSample.

When I check both my images and the images in the repo using sips -g all. This is the outcome I get:

pixelWidth: 2048
pixelHeight: 1024
typeIdentifier: public.png
format: png
formatOptions: default
dpiWidth: 72.000
dpiHeight: 72.000
samplesPerPixel: 1
bitsPerSample: 16
hasAlpha: no
space: Gray

The strange thing is that it works with the images that has 8 bitPerSample.
This is the outcome I get:

Grayscale input

Converted label map

Final output

When I run test.py with 16 bitsPerSample images, it doesn't work.
This is the error it gives me:

model [Pix2PixHDModel] was created
Traceback (most recent call last):
  File "test.py", line 26, in <module>
    for i, data in enumerate(dataset):
  File "/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py", line 210, in __next__
    return self._process_next_batch(batch)
  File "/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py", line 230, in _process_next_batch
    raise batch.exc_type(batch.exc_msg)
TypeError: Traceback (most recent call last):
  File "/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py", line 42, in _worker_loop
    samples = collate_fn([dataset[i] for i in batch_indices])
  File "/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py", line 42, in <listcomp>
    samples = collate_fn([dataset[i] for i in batch_indices])
  File "/home/paperspace/Documents/pix2pixHD/data/aligned_dataset.py", line 41, in __getitem__
    label_tensor = transform_label(label) * 255.0
  File "/usr/local/lib/python3.5/dist-packages/torch/tensor.py", line 309, in __mul__
    return self.mul(other)
TypeError: mul received an invalid combination of arguments - got (float), but expected one of:
 * (int value)
      didn't match because some of the arguments have invalid types: (float)
 * (torch.IntTensor other)
      didn't match because some of the arguments have invalid types: (float)

I am new fairly to Tensorflow and I have never used pytorch before.

Any idea what this error mean and how can I resolve it?


title: Any reason to save a pretrained BERT tokenizer?

body:
Say I am using tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True), and all I am doing with that tokenizer during fine-tuning of a new model is the standard tokenizer.encode()
I have seen in most places that people save that tokenizer at the same time that they save their model, but I am unclear on why it's necessary to save since it seems like an out-of-the-box tokenizer that does not get modified in any way during training.


title: RuntimeError: view size is not compatible with input tensor's size and stride (at least one dimension spans across two contiguous subspaces)

body: I am using Pytorch. I got this RuntimeError while evaluating a model. Any idea how to solve this?


title: Incorporating dim parameter of torch.topk in tf.nn.top_k

body:
Pytorch provide torch.topk(input, k, dim=None, largest=True, sorted=True) function to calculate k largest elements of the given input tensor along a given dimension dim.

I have a tensor of shape (16, 512, 4096) and I am using torch.topk in the following manner-

# inputs.shape (16L, 512L, 4096L)
dist, idx = torch.topk(inputs, 64, dim=2, largest=False, sorted=False)
# dist.shape (16L, 512L, 64L), idx.shape (16L, 512L, 64L)

I found similar tensorflow implementaion as following - tf.nn.top_k(input, k=1, sorted=True, name=None).

My question is how to Incorporate dim=2 parameter in tf.nn.top_k so as to achieve the tensor of the same shape as calculated by pytorch?


title: Slightly adapt L1 loss to a weighted L1 loss in Pytorch, does gradient computation still work properly?

body:
I implemented a neural network in Pytorch and I would like to use a weighted L1 loss function to train the network.

The implementation with the regular L1 loss contains this code for each epoch:

optimiser.zero_grad()
net.train()
_,forecast = net(torch.tensor(feature, dtype=torch.float).to(DEVICE))
loss = F.l1_loss(forecast, torch.tensor(target,dtype=torch.float).to(DEVICE),reduction='mean')
loss.backward()
params.append(net.parameters())
optimiser.step()

Now I want to use a weighted L1 loss instead. So I thought to use the same standard Pytorch L1 function again and rescale the forecasts and targets with weights. Will the gradient computation still be done correctly?

optimiser.zero_grad()
net.train()
_,forecast = net(torch.tensor(feature, dtype=torch.float).to(DEVICE))
loss = F.l1_loss(torch.t(torch.mul(torch.t(forecast),
                                           torch.tensor(weight,dtype=torch.float).to(DEVICE))) ,
                         torch.t(torch.mul(torch.t(torch.tensor(target,dtype=torch.float).to(DEVICE)),
                                           torch.tensor(weight,dtype=torch.float).to(DEVICE))),reduction='mean')
loss.backward()
params.append(net.parameters())
optimiser.step()


title: CUDA Runtime Error: Which Cuda version is compatible to run NER task using BERT-NER

body:
I have setup all the requirement packages installed on my VM and i found no nvidia GPU driver  installed, In the requirements doesn't have nvidia GPU driver installation instructions, I want to know which cuda version and it compatible nvidia driver which needs too resolve the below error.

Github link: github

Error logs:

  File "run_ner.py", line 594, in <module>
    main()
  File "run_ner.py", line 489, in main
    loss = model(input_ids, segment_ids, input_mask, label_ids,valid_ids,l_mask)
  File "/home/pt3_gcp/BERT-NER/ber_ner/lib/python3.7/site-packages/torch/nn/modules/module.py", line 547, in __call__
    result = self.forward(*input, **kwargs)
  File "run_ner.py", line 35, in forward
    valid_output = torch.zeros(batch_size,max_len,feat_dim,dtype=torch.float32,device='cuda')
  File "/home/pt3_gcp/BERT-NER/ber_ner/lib/python3.7/site-packages/torch/cuda/__init__.py", line 178, in _lazy_init
    _check_driver()
  File "/home/pt3_gcp/BERT-NER/ber_ner/lib/python3.7/site-packages/torch/cuda/__init__.py", line 99, in _check_driver
    http://www.nvidia.com/Download/index.aspx""")
AssertionError:
**Found no NVIDIA driver on your system. Please check that you
have an NVIDIA GPU and installed a driver from
http://www.nvidia.com/Download/index.aspx
**

After installing latest cuda version from the following link,
cuda I got the following error,

06/04/2020 07:38:40 - INFO - __main__ -   ***** Running training *****
06/04/2020 07:38:40 - INFO - __main__ -     Num examples = 14041
06/04/2020 07:38:40 - INFO - __main__ -     Batch size = 32
06/04/2020 07:38:40 - INFO - __main__ -     Num steps = 2190
Epoch:   0%|                                                                                | 0/5 [00:00<?, ?it/sTHCudaCheck FAIL file=/pytorch/aten/src/THC/THCGeneral.cpp line=50 error=38 : no CUDA-capable device is detectedt/s]
Traceback (most recent call last):
  File "run_ner.py", line 594, in <module>
    main()
  File "run_ner.py", line 489, in main
    loss = model(input_ids, segment_ids, input_mask, label_ids,valid_ids,l_mask)
  File "/home/pt3_gcp/.local/lib/python3.7/site-packages/torch/nn/modules/module.py", line 547, in __call__
    result = self.forward(*input, **kwargs)
  File "run_ner.py", line 35, in forward
    valid_output = torch.zeros(batch_size,max_len,feat_dim,dtype=torch.float32,device='cuda')
  File "/home/pt3_gcp/.local/lib/python3.7/site-packages/torch/cuda/__init__.py", line 179, in _lazy_init
    torch._C._cuda_init()
RuntimeError: cuda runtime error (38) : no CUDA-capable device is detected at /pytorch/aten/src/THC/THCGeneral.cpp:50


title: Training on minibatches of varying size

body:
I'm trying to train a deep learning model in PyTorch on images that have been bucketed to particular dimensions. I'd like to train my model using mini-batches, but the mini-batch size does not neatly divide the number of examples in each bucket.

One solution I saw in a previous post was to pad the images with additional whitespace (either on the fly or all at once at the beginning of training), but I do not want to do this. Instead, I would like to allow the batch size to be flexible during training.

Specifically, if N is the number of images in a bucket and B is the batch size, then for that bucket I would like to get N // B batches if B divides N, and N // B + 1 batches otherwise. The last batch can have fewer than B examples.

As an example, suppose I have indexes [0, 1, ..., 19], inclusive and I'd like to use a batch size of 3.

The indexes [0, 9] correspond to images in bucket 0 (shape (C, W1, H1))
The indexes [10, 19] correspond to images in bucket 1 (shape (C, W2, H2))

(The channel depth is the same for all images). Then an acceptable partitioning of the indexes would be

batches = [
    [0, 1, 2],
    [3, 4, 5],
    [6, 7, 8],
    [9],
    [10, 11, 12],
    [13, 14, 15],
    [16, 17, 18],
    [19]
]

I would prefer to process the images indexed at 9 and 19 separately because they have different dimensions.

Looking through PyTorch's documentation, I found the BatchSampler class that generates lists of mini-batch indexes. I made a custom Sampler class that emulates the partitioning of indexes described above. If it helps, here's my implementation for this:

class CustomSampler(Sampler):

    def __init__(self, dataset, batch_size):
        self.batch_size = batch_size
        self.buckets = self._get_buckets(dataset)
        self.num_examples = len(dataset)

    def __iter__(self):
        batch = []
        # Process buckets in random order
        dims = random.sample(list(self.buckets), len(self.buckets))
        for dim in dims:
            # Process images in buckets in random order
            bucket = self.buckets[dim]
            bucket = random.sample(bucket, len(bucket))
            for idx in bucket:
                batch.append(idx)
                if len(batch) == self.batch_size:
                    yield batch
                    batch = []
            # Yield half-full batch before moving to next bucket
            if len(batch) > 0:
                yield batch
                batch = []

    def __len__(self):
        return self.num_examples

    def _get_buckets(self, dataset):
        buckets = defaultdict(list)
        for i in range(len(dataset)):
            img, _ = dataset[i]
            dims = img.shape
            buckets[dims].append(i)
        return buckets

However, when I use my custom Sampler class I generate the following error:

Traceback (most recent call last):
    File "sampler.py", line 143, in <module>
        for i, batch in enumerate(dataloader):
    File "/home/roflcakzorz/anaconda3/lib/python3.6/site-packages/torch/utils/data/dataloader.py", line 263, in __next__
        indices = next(self.sample_iter)  # may raise StopIteration
    File "/home/roflcakzorz/anaconda3/lib/python3.6/site-packages/torch/utils/data/sampler.py", line 139, in __iter__
        batch.append(int(idx))
TypeError: int() argument must be a string, a bytes-like object or a number, not 'list'

The DataLoader class seems to expect to be passed indexes, not list of indexes.

Should I not be using a custom Sampler class for this task? I also considered making a custom collate_fn to pass to the DataLoader, but with that approach I don't believe I can control which indexes are allowed to be in the same mini-batch. Any guidance would be greatly appreciated.


title: when training simple code of pytorch, cpu ratio increased. GPU is 0% approximately

body:
I'm doing tutorial of Pytorch.

Code is clearly completed. but i have one problem.

It is about my CPU use ratio.
If I enter into training, CPU usage ratio is increasıng up to 100%.
but GPU is roughly 0%.

I installed CUDA 9.2 and cudnn.
and I already checked massage about torch.cuda.is_available()==True.

is it OK, or my setup is wrong?


title: Torch multi-processing on single GPU is extremely slow

body:
I have a torch code in which I am trying to calculate the output of 10 layers that are independent of each other. It seems like a case where torch multi-processing fits perfectly. Here is the code for that replicates the situation in my case:

from torch.multiprocessing import Pool, set_start_method
import torch
from torch.autograd import Variable

from kernel_2_0 import *
from kernel_2_1 import *
from kernel_2_2 import *
from kernel_2_3 import *
from kernel_2_4 import *
from kernel_2_5 import *
from kernel_2_6 import *
from kernel_2_7 import *
from kernel_2_8 import *
from kernel_2_9 import *

def apply_fn(args):
    f = args[0]
    x = args[1]
    return f(x)

def forward_2(x):
    assert isinstance(x.data, torch.cuda.BoolTensor)
    try:
        set_start_method('spawn', force=True)
    except RuntimeError:
        print('Could not set spawn method')
    with Pool(12) as pool:
        mapped_out = pool.map(apply_fn,
                              [(kernel_2_0, x), (kernel_2_1, x), (kernel_2_2, x), (kernel_2_3, x), (kernel_2_4, x),
                               (kernel_2_5, x), (kernel_2_6, x), (kernel_2_7, x), (kernel_2_8, x), (kernel_2_9, x)])

out = Variable(torch.cuda.BoolTensor(x.size()[0], 10))
out[:, 0] = mapped_out[0]
out[:, 1] = mapped_out[1]
out[:, 2] = mapped_out[2]
out[:, 3] = mapped_out[3]
out[:, 4] = mapped_out[4]
out[:, 5] = mapped_out[5]
out[:, 6] = mapped_out[6]
out[:, 7] = mapped_out[7]
out[:, 8] = mapped_out[8]
out[:, 9] = mapped_out[9]

return out

However, when the code runs it is almost 100 times slower than the version where I do not do multi processing an instead call all the kernels in a sequential manner. FYI I am using torch==1.4.0 and cuda-10.0. I am not able to understand why this happens. Can someone please help?
Thanks in advance!


title: Should I transpose features or weights in Neural network?

body:
I am learning Neural network.
Here is the complete piece of code:
https://github.com/udacity/deep-learning-v2-pytorch/blob/master/intro-to-pytorch/Part%201%20-%20Tensors%20in%20PyTorch%20(Exercises).ipynb
When I transpose features, I get the following output:
import torch
def activation(x):
    return 1/(1+torch.exp(-x))

### Generate some data
torch.manual_seed(7) # Set the random seed so things are predictable

# Features are 5 random normal variables
features = torch.randn((1, 5))
# True weights for our data, random normal variables again
weights = torch.randn_like(features)
# and a true bias term
bias = torch.randn((1, 1))

product = features.t() * weights + bias
output = activation(product.sum())

tensor(0.9897)

However, if I transpose weights, I get a different output:
weights_prime = weights.view(5,1)
prod = torch.mm(features, weights_prime) + bias
y_hat = activation(prod.sum())

tensor(0.1595)

Why does this happen?

Update
I took a look at the solution:
https://github.com/udacity/deep-learning-v2-pytorch/blob/master/intro-to-pytorch/Part%201%20-%20Tensors%20in%20PyTorch%20(Solution).ipynb
And I saw this:
y = activation((features * weights).sum() + bias)
why can a matrix features(1,5) multiply another matrix weights(1,5) without transposing weights first?
Update 2
After read several posts, I realized that
matrixA * matrixB is different from torch.mm(matrixA,matrixB) and torch.matmul(matrixA,matrixB).
Could someone confirm my three understandings between?

So the * means element-wise multiplication, whereas torch.mm() and torch.matmul() are matrix-wise multiplication.

differences between torch.mm() and torch.matmul(): mm() is used specifically for 2 dimensions matrix, whereas matmul() can be used for more complicated cases.

In Neutral Network for this Udacity coding exercise mentioned in my above link, it needs element-wise multiplication.

Update 3
Just to bring in the Video screenshot for someone who has the same confusion:

And here is the video link: https://www.youtube.com/watch?time_continue=98&v=6Z7WntXays8&feature=emb_logo


I run my code in google colab, but I meet some error.

torch:1.1.0, torchtext:0.4.0, I update them, but it's useless

[2019-09-14 02:57:15,574 INFO]  * src vocab size = 49766
[2019-09-14 02:57:15,574 INFO]  * history vocab size = 49766
[2019-09-14 02:57:15,574 INFO]  * tgt vocab size = 49766
[2019-09-14 02:57:15,574 INFO] Building model...
Traceback (most recent call last):
  File "/content/drive/My Drive/ReDR/train.py", line 109, in <module>
    main(opt)
  File "/content/drive/My Drive/ReDR/train.py", line 41, in main
    single_main(opt, -1)
  File "/content/drive/My Drive/ReDR/onmt/train_single.py", line 86, in main
    model = build_model(model_opt, opt, fields, checkpoint)
  File "/content/drive/My Drive/ReDR/onmt/model_builder.py", line 235, in build_model
    vocab = torch.load(opt.drqa_vocab_path)
  File "/usr/local/lib/python3.6/dist-packages/torch/serialization.py", line 387, in load
    return _load(f, map_location, pickle_module, **pickle_load_args)
  File "/usr/local/lib/python3.6/dist-packages/torch/serialization.py", line 574, in _load
    result = unpickler.load()
AttributeError: Can't get attribute '_default_unk_index' on <module 'torchtext.vocab' from '/usr/local/lib/python3.6/dist-packages/torchtext/vocab.py'>

I will appreciate any help from you!


title: how does BatchNorm1d() method whithin the torch library work?

body: I'm learning pytorch, I don;t know if this question is stupid but I can't find the official web for explaining nn.batchnorm1d. I'm wondering how torch.nn.BatchNorm1d(d1) work? I know that batch norm is about making mean and variance of a batch of example to be 0 and 1 respectively. I'm wondering if there is nn.batchnorm2d, if so, what does it do? what is the d1 parameter ?


title: how to find shape of inputs tensor?

body:
import torchvision
from torch.utils.data import Dataset, DataLoader
from torch import from_numpy, tensor
import numpy as np
trans = transforms.Compose(
    [
     transforms.Resize(128),
     transforms.ToTensor()
    ])
class UBFCDataset(Dataset):

    def __init__(self):

        xy = np.loadtxt('/content/drive/My Drive/Subject3hr.csv')
        self.x_data = torch.from_numpy(xy[:])
        self.len = xy.shape[0]

    def __len__(self):
        return self.len

    def __getitem__(self, index):
        train_data_tensor = torchvision.datasets.ImageFolder("/content/drive/My Drive/Meta-rPPG-master/da", transform=trans)

        return train_data_tensor[index], self.x_data[index]

dataset = UBFCDataset()
dataset[0]
train_loader = DataLoader(dataset=dataset,batch_size=128,shuffle=True)

for epoch in range(2):
    for i, data in enumerate(train_loader, 0):
        # get the inputs
        inputs, labels = data
        print(inputs.shape)
        # Run your training process
        print(f'Epoch: {i} | Inputs {inputs} | Labels {labels}')

Error:

AttributeError                            Traceback (most recent call last)
<ipython-input-156-fecbaf453580> in <module>()
      4         inputs, labels = data
      5
----> 6         print(inputs.shape)
      7         # Run your training process
      8         print(f'Epoch: {i} | Inputs {inputs} | Labels {labels}')

AttributeError: 'list' object has no attribute 'shape'


title: Loss is not decreasing at all for RNN

body:
I have already tried to change the weights initialization parameters, learning rate and the batch size and the activation functions to ReLu
Still no decrease in the loss
This is the code:

import torch
import torchvision.datasets as dsets
import torchvision.transforms as transforms
from torch.autograd import Variable
import numpy as np

no_time_steps = 28
input_size = 28
hidden_size = 30
output_size = 10
batch_size = 100
num_epochs = 2
learning_rate = 0.01
dtype = torch.DoubleTensor

# MNIST Dataset
train_dataset = dsets.MNIST(root='./data/',
                        train=True,
                        transform=transforms.ToTensor(),
                        download=True)

test_dataset = dsets.MNIST(root='./data/',
                       train=False,
                       transform=transforms.ToTensor())

train_loader = torch.utils.data.DataLoader(dataset=train_dataset,
                                       batch_size=batch_size,
                                       shuffle=True)

test_loader = torch.utils.data.DataLoader(dataset=test_dataset,
                                      batch_size=batch_size,
                                      shuffle=False)

class RNN(torch.nn.Module):
def __init__(self,input_size,hidden_size,output_size,batch_size):
    super(RNN, self).__init__()
    self.input_size=input_size
    self.hidden_size=hidden_size
    self.output_size=output_size
    self.wxh=Variable(torch.randn(input_size,hidden_size).type(dtype)*0.1,requires_grad=True)
    self.whh=Variable(torch.randn(hidden_size,hidden_size).type(dtype)*0.1,requires_grad=True)
    self.why=Variable(torch.randn(hidden_size,output_size).type(dtype)*0.1,requires_grad=True)
    self.by=Variable(torch.Tensor(batch_size,output_size).type(dtype).zero_(),requires_grad=True)
    self.bh=Variable(torch.Tensor(batch_size,hidden_size).type(dtype).zero_(),requires_grad=True)

    self.mWxh= torch.zeros_like(self.wxh)
    self.mWhh= torch.zeros_like(self.whh)
    self.mWhy= torch.zeros_like(self.why)
    self.mbh= torch.zeros_like(self.bh)
    self.mby= torch.zeros_like(self.by)
    self.dwxh, self.dwhh, self.dwhy = torch.zeros_like(self.wxh), torch.zeros_like(self.whh), torch.zeros_like(self.why)
    self.dbh, self.dby = torch.zeros_like(self.bh), torch.zeros_like(self.by)

def hidden_init(self,batch_size):
    self.hidden={}
    self.hidden[0]=Variable(torch.Tensor(batch_size,hidden_size).type(dtype).zero_())

def tanh(self,value):
    return (torch.exp(value)-torch.exp(-value))/(torch.exp(value)+torch.exp(-value))

def parameter(self):
    self.params = torch.nn.ParameterList([torch.nn.Parameter(self.wxh.data),torch.nn.Parameter(self.whh.data),torch.nn.Parameter(self.why.data),torch.nn.Parameter(self.bh.data),torch.nn.Parameter(self.by.data)])
    return self.params

def grad_data(self):
    print(self.dwxh,self.dwhy)

def softmax(self,value):
    return torch.exp(value) / torch.sum(torch.exp(value))

def updatess(self,lr):
    for param, dparam, mem in zip([self.wxh, self.whh, self.why, self.bh, self.by],
                            [self.dwxh,self.dwhh,self.dwhy,self.dbh,self.dby],
                            [self.mWxh, self.mWhh, self.mWhy, self.mbh, self.mby]):
        mem.data += dparam.data * dparam.data
        param.data += -learning_rate * dparam.data / torch.sqrt(mem.data + 1e-8)

def forward(self,inputs,batch_size,no_time_steps,labels):
    self.hidden_init(batch_size)

    inputs=Variable(inputs.type(dtype))
    self.output=Variable(torch.Tensor(no_time_steps,batch_size,self.output_size).type(dtype))

    for t in xrange(no_time_steps):
        if t==0:
            self.hidden[t]=torch.matmul(self.hidden[0],self.whh)
            #print 'time  ',t#,"Inputs",inputs[:,t,:],"Weights",self.wxh
            #print "hidden MATRIX",inputs[:,t,:]
            self.hidden[t]+=torch.matmul(inputs[:,t,:],self.wxh)
            self.hidden[t]=self.tanh(self.hidden[t]+self.bh)
            #print 'time  ',t#,"Inputs",inputs[:,t,:],"Weights",self.wxh
            #print "HIDDEN MATRIX",self.hidden[t]
        else:
            self.hidden[t]=torch.matmul(self.hidden[t-1],self.whh)#+torch.matmul(self.hidden[t-1],self.whh)
            #print 'time  ',t#,"Inputs",inputs[:,t,:],"Weights",self.wxh
            self.hidden[t]+=torch.matmul(inputs[:,t,:],self.wxh)
            self.hidden[t]=self.tanh(self.hidden[t]+self.bh)
        #print 'time  ',t#,"Inputs",inputs[:,t,:],"Weights",self.wxh
        #print "############################################################################################"
        #print "hidden MATRIX",self.hidden[t]
        self.output[t]=self.softmax(torch.matmul(self.hidden[t],self.why)+self.by)
        #print "OUTPUT MATRIX",self.output[t]
    return self.output
def backward(self,loss,label,inputs):
    inputs=Variable(inputs.type(dtype))
    self.dhnext = torch.zeros_like(self.hidden[0])
    self.dy=self.output[27].clone()
        #print(self.dy.shape)
    self.dy[:,int(label[0])]=self.dy[:,int(label[0])]-1
        #print(self.dy.shape)
    self.dwhy += torch.matmul( self.hidden[27].t(),self.dy)
    self.dby += self.dy
    for t in reversed(xrange(no_time_steps)):
        self.dh = torch.matmul(self.dy,self.why.t()) + self.dhnext # backprop into h
        self.dhraw = (1 - self.hidden[t] * self.hidden[t]) * self.dh # backprop through tanh nonlinearity
        self.dbh += self.dhraw #derivative of hidden bias
        self.dwxh += torch.matmul(inputs[:,t,:].t(),self.dhraw) #derivative of input to hidden layer weight
        self.dwhh += torch.matmul( self.hidden[t-1].t(),self.dhraw) #derivative of hidden layer to hidden layer weight
        self.dhnext = torch.matmul(self.dhraw,self.whh.t())

rnn=RNN(input_size,hidden_size,output_size,batch_size)
def onehot(values,shape):
temp=torch.Tensor(shape).zero_()
for k,j in enumerate(labels):
    temp[k][int(j)]=1
return Variable(temp)

for epoch in range(5):
for i, (images, labels) in enumerate(train_loader):
    images = images.view(-1, no_time_steps, input_size)
    outputs = rnn(images,batch_size,no_time_steps,labels)
    labels = Variable(labels.double())
    output=outputs[27,:,:]
    labelss=onehot(labels,output.shape)
    #print output
    loss=-torch.mul(torch.log(output),labelss.double())
    #print loss
    loss=torch.sum(loss)
    #print(labels)
    rnn.backward(loss,labels,images)
    rnn.updatess(0.01)
    if i==1110:
        break
    if (i+1) % 100 == 0:
        print ('Epoch [%d/%d], Step [%d/%d], Loss: %.4f'
               %(epoch+1, num_epochs, i+1, len(train_dataset)//batch_size, loss.data[0]))

OUTPUT:

Epoch [1/2], Step [100/600], Loss: 714.8081

Epoch [1/2], Step [200/600], Loss: 692.7232

Epoch [1/2], Step [300/600], Loss: 700.1103

Epoch [1/2], Step [400/600], Loss: 698.5468

Epoch [1/2], Step [500/600], Loss: 702.1227

Epoch [1/2], Step [600/600], Loss: 705.9571


title: Errors converting PyTorch Unet ("tiramisu") into coreml, via onnx

body:
I'm trying to convert a pytorch "tiramisu" UNet (from: https://github.com/bfortuner/pytorch_tiramisu) to coreml, via onnx, and I'm getting this error in onnx-coreml's _operators.py:

TypeError: Error while converting op of type: Concat. Error message: Unsupported axis 1 in input of shape

Any thoughts about how I might work around this?
The layers file is here, for reference: https://github.com/bfortuner/pytorch_tiramisu/blob/master/models/layers.py

UPDATE 1: So, digging down further into this, I've noticed that some of my graph.shape_dicts contain keys with no values/shapes: e.g., '422': (1, 324, 2, 4), '423': (1, 324, 2, 4), '424': (1, 12, 2, 4), '425': (1, 48, 2, 4), '426': (), '427': (), '428': (), '429': (), '430': () [...]

I'm really not sure how this is happening, or whether its my original model, onnx, or onnx-coreml that's the problem. But with this in mind, I changed if node.inputs[0] in graph.shape_dict: to if len(graph.shape_dict[node.inputs[0]]) != 0:) in _operators.py. This allows the conversion to complete, though I won't know until I try it whether the model is actually converted correctly...

UPDATE 2: Okay, so the resulting mlmodel fails validation in Xcode with validator error: Layer '427' of type 320 has 1 inputs but expects at least 2. I'm guessing this is a consequence of (simplistically) ignoring the empty shape_dict values I found in "UPDATE 1" above. But clearly the model is incorrect. Could the error be originating in onnx?


title: Is my custom loss function correct? (Pytorch)

body:
I want to do word recognition using a CNN + Classifier, where the input is an image and the output a matrice 10x37. 10 is the maximum number of characters in a word and 37 is the number of letters in my example.

I wrote a custom loss function for this model but I'm not sure if it's correct since I can't get above 80% Test Accuracy.

I'm using Pytorch

class CustomLoss(nn.Module):
    def __init__(self):
        super().__init__()
        self.nllloss = nn.NLLLoss()

    def forward(self, output, labels):
        loss = 0
        for i in range(labels.shape[1]):
            loss += self.nllloss(output[:, i, :], labels[:, i])
        loss /= labels.shape[1]
        return loss

Infos:

output.shape = (batch_size, 10, 37)

labels.shape = (batch_size, 10)

Is the loss function correct?
And what my classification problem is called (Multiple Multi class classification) ?


title: Execute another model in parallel to a model's forward pass with PyTorch

body:
I am trying to make some changes to the ResNet-18 model in PyTorch to invoke the execution of another auxiliary trained model which takes in the ResNet intermediate layer output at the end of each ResNet block as an input and makes some auxiliary predictions during the inference phase.

I want to be able to do the auxiliary computation after the computation of a block in parallel to the computation of the next ResNet block so as to reduce the end-to-end latency of the entire pipeline execution on GPU.

I have a base code that works correctly from the functionality perspective, but the execution of the auxiliary model is serial to the computation of the ResNet block. I verified this in two ways -

By adding print statements and verifying the order of execution.
By instrumenting the running time of the original ResNet model (say time t1) and the auxiliary model (say time t2). My execution time is currently t1+t2.

The original ResNet block code (This is the BasicBlock since I am experimenting on ResNet-18). The entire code is available here

  class BasicBlock(nn.module):
  ...

      def forward(self, x):

        residual = x
        out = self.conv1(x)
        out = self.bn1(out)
        out = self.relu(out)
        out = self.conv2(out)
        out = self.bn2(out)

        if self.downsample is not None:
          residual = self.downsample(x)

        out += residual
        out = self.relu(out)
        return out

This is my modification which works in a serial fashion -

def forward(self, x):
    if len(x[0]) == self.auxiliary_prediction_size: # Got an Auxiliary prediction earlier
        return x

    # Do usual block computation
    residual = x
    out = self.conv1(x)
    out = self.bn1(out)
    out = self.relu(out)
    out = self.conv2(out)
    out = self.bn2(out)

    if self.downsample is not None:
        residual = self.downsample(x)

    out += residual
    out = self.relu(out)

    # Try to make an auxiliary prediction
    # First flatten the tensor (also assume for now that batch size is 1)
    batchSize = x.shape[0]
    intermediate_output = out.view(batchSize, -1)
    # Place the flattened on GPU
    device = torch.device("cuda:0")
    input = intermediate_output.to(device)
    # Make auxiliary prediction
    auxiliary_input = out.float()
    auxiliary_prediction = self.auxiliary_model(auxiliary_input)
    if auxiliary_prediction meets some condition:
      return auxiliary_prediction

    # If no auxiliary prediction, then return intermediate output
    return out

Understandably, the above code causes a data dependency between the execution of the auxiliary model and the next block and hence things to happen serially. The first solution I tried was to check if breaking this data dependency reduces latency. I tried doing so by allowing the auxiliary model to execute but not having the auxiliary_prediction return if the condition is met (Note that this would break functionality but this experiment was purely to understand the behavior). Essentially, what I did was -

    batchSize = x.shape[0]
    intermediate_output = out.view(batchSize, -1)
    # Place the flattened on GPU
    device = torch.device("cuda:0")
    input = intermediate_output.to(device)
    # Make auxiliary prediction
    auxiliary_input = out.float()
    auxiliary_prediction = self.auxiliary_model(auxiliary_input)
    if auxiliary_prediction meets some condition:
      # Comment out return to break data dependency
      #return auxiliary_prediction

    # If no auxiliary prediction, then return intermediate output
    return out

However, this did not work and upon researching further, I stumbled upon CUDA streams at this Stack Overflow link. I tried incorporating the idea of CUDA streams to solve my problem in the below way -

def forward(self, x):
    if len(x[0]) == self.auxiliary_prediction_size: # Got an Auxiliary prediction earlier
        return x

    s1 = torch.cuda.Stream()
    s2 = torch.cuda.Stream()

    with torch.cuda.Stream(s1):
      # Do usual block computation
      residual = x
      out = self.conv1(x)
      out = self.bn1(out)
      out = self.relu(out)
      out = self.conv2(out)
      out = self.bn2(out)

      if self.downsample is not None:
        residual = self.downsample(x)

      out += residual
      out = self.relu(out)

    with torch.cuda.Stream(s2):
      # Try to make an auxiliary prediction
      # First flatten the tensor (also assume for now that batch size is 1)
      out_detach = out.detach() # Detach from backprop flow and from computational graph dependency
      batchSize = x.shape[0]
      intermediate_output = out_detach.view(batchSize, -1)
      # Place the flattened on GPU
      device = torch.device("cuda:0")
      input = intermediate_output.to(device)
      # Make auxiliary prediction
      auxiliary_input = out_detach.float()
      auxiliary_prediction = self.auxiliary_model(auxiliary_input)
      if auxiliary_prediction meets some condition:
        return auxiliary_prediction

    # If no auxiliary prediction, then return intermediate output
    return out

However, the output from Nvidia Visual Profiler still indicates that all work is still being done on the default stream and still serialized. Note that I did verify with a small CUDA program that CUDA streams is supported by the CUDA version I am using.

My questions -

Why does breaking the data dependency not cause PyTorch to schedule the computations in parallel? I thought this was the point of the dynamic computation graphs in PyTorch.
Why does using CUDA streams not delegate the computation to non-default streams?
Are there alternative approaches to execute the auxiliary model asynchronously/parallelly to the ResNet block computation?


title: How to create a completely (uniformly) random dataset on PyTorch

body:
I need to run some experiments on custom datasets using pytorch. The question is, how can I create a dataset using torch.Dataloader?

I have two lists, one is called Values and has a datapoint tensor at every entry, and the other one is called Labels, that has the corresponding label. What I did is the following:

for i in range(samples):
dataset[i] = [values[i],labels[I]]

So I have a list with datapoint and respective label, and then tried the following:

dataset = torch.tensor(dataset).float()
dataset = torch.utils.data.TensorDataset(dataset)

data_loader = torch.utils.data.DataLoader(dataset=dataset, batch_size=100, shuffle=True, num_workers=4, pin_memory=True)

But, first of all, I get the error "Not a sequence" in the torch.tensor command, and second, I'm not sure this is the right way of creating one. Any suggestion?

Thank you very much!


I have been trying to implement MNIST dataset on a simple Neural Network(784,512,128,10) using cross-entropy loss. I am using Keras for getting the MNIST dataset. But I'm getting an error :
RuntimeError: 1D target tensor expected, multi-target not supported

When I have main model as:
for epoch in range(num_epochs):
  for x,y in train_data:
    x=Variable(x)
    y=Variable(y)
    print(x.shape)
    y_pred=model(x)
    optimizer.zero_grad()
    loss=criterion(y_pred,y)
    loss.backward()
    optimizer.step()

So, to remove that error I implemented that:
y=y[0][0:]
y_pred=y_pred[0][0:]
loss=criterion(y_pred,y)

But after that I'm getting this error:
IndexError: Dimension out of range (expected to be in range of [-1, 0], but got 1)

I read many articles regarding how to solve this error but none helped.
Is this error coming because of Keras dataset?
or Is something wrong in my code can someone please help find the error?
My Code:
import torch
import torch.nn as nn
import numpy as np
from sklearn.preprocessing import StandardScaler
import matplotlib.pyplot as plt
from torch.utils.data import DataLoader,Dataset
import keras
import torch.nn.functional as F
from torch.autograd import Variable

class Netz(nn.Module):
  def __init__(self,n_input_features):
    super(Netz,self).__init__()
    self.linear=nn.Linear(784,512,bias=True)
    self.l1=nn.Linear(512,128,bias=True)
    self.l2=nn.Linear(128,10,bias=True)
    self.relu=nn.ReLU()
    self.relu2=nn.ReLU()
    self.softmax=nn.Softmax(dim=-1)
  def forward(self,x):
    # x=x.view(-1,784)
    x=self.relu(self.linear(x))
    x=self.relu2(self.l1(x))
    x=self.softmax(self.l2(x))
    return x

model=Netz(784)

class Data(Dataset):
    def __init__(self):
        self.x=x_train
        self.y=y_train
        self.len=self.x.shape[0]
    def __getitem__(self,index):
        return self.x[index],self.y[index]

mnist = keras.datasets.mnist
#Copying data
(x_train, y_train),(x_test, y_test) = mnist.load_data()
#One-hot encoding the labels
y_train = keras.utils.to_categorical(y_train,10)
y_test = keras.utils.to_categorical(y_test,10)
#Flattening the images
x_train_reshaped = x_train.reshape((60000,784))
x_test_reshaped = x_test.reshape((10000,784))
#Normalizing the inputs
x_train = x_train_reshaped/255.0
x_test = x_test_reshaped/255.0

x_train=torch.from_numpy(x_train.astype(np.float32))
x_test=torch.from_numpy(x_test.astype(np.float32))
y_train=torch.from_numpy(y_train.astype(np.float32))
y_test=torch.from_numpy(y_test.astype(np.float32))

criterion=nn.CrossEntropyLoss()
print(criterion)
optimizer=torch.optim.SGD(model.parameters(),lr=0.05)
dataset=Data()
train_data=DataLoader(dataset=dataset,batch_size=1,shuffle=False)

num_epochs=5
for epoch in range(num_epochs):
  for x,y in train_data:
    x=Variable(x)
    y=Variable(y)
    y_pred=model(x)
    optimizer.zero_grad()
    y=y[0][0:]
    y_pred=y_pred[0][0:]
    loss=criterion(y_pred,y)
    loss.backward()
    optimizer.step()


It occured when I writing Python code with PyTorch. I am trying to construct a simple language model using CrossEntropyLoss, but this error came up. Actually, I wrote this piece of code according to this. Here is my code.

import numpy as np
import torch
from torch.autograd import Variable
import torch.nn as nn

data = '...'
words = list(set(data))
word2ind = {word: i for i, word in enumerate(words)}
ind2word = {i: word for i, word in enumerate(words)}

class RNN(nn.Module):
    def __init__(self, input_size, hidden_size, output_size):
        super(RNN, self).__init__()
        self.hidden_size = hidden_size
        self.in2h = nn.Linear(input_size-1+hidden_size, hidden_size)
        self.in2o = nn.Linear(input_size-1+hidden_size, output_size)
        self.o2o = nn.Linear(hidden_size+output_size, output_size)
        self.softmax = nn.LogSoftmax()

    def forward(self, inputs, hidden):
        input_combined = torch.cat((inputs.float(), hidden.float()), 1)
        print(type(input_combined.data))
        hidden = self.in2h(input_combined)
        output = self.in2o(input_combined)
        output_combined = torch.cat((hidden, output), 1)
        output = self.o2o(output_combined)
        output = self.softmax(output)
        print(output)
        return output, hidden

    def init_hidden(self):
        return Variable(torch.from_numpy(np.zeros((1, self.hidden_size))).type(torch.LongTensor))

def form_onehot(sent):
    one_hot = np.zeros((len(data), len(words)), dtype=np.int64)
    for i, word in enumerate(sent):
        one_hot[i, word2ind[word]] = 1
    return torch.LongTensor(one_hot)

def random_choice(vec):
    return np.random.choice(range(len(words)), p=vec)

def train(rnn, learning_rate, optimizer, criterion, input_tensor, target_tensor):
    hidden = rnn.init_hidden()
    optimizer.zero_grad()
    for i in range(input_tensor.size(1)):
        output, hidden = rnn(input_tensor[i, :].unsqueeze(0), hidden)
        loss = criterion(output, target_tensor[i])
        loss.backward()
        optimizer.step()
    return output, loss.data[0] / input_tensor.size()[0]

onehot_data = form_onehot(data)
rnn = RNN(len(words), 10, len(words))
learning_rate = 0.1
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.SGD(rnn.parameters(), lr=learning_rate)
input_tensor = Variable(onehot_data[:, :-1].type(torch.FloatTensor))
print(type(input_tensor.data))
target_tensor = Variable(onehot_data[:, 1:])
int_target_tensor = Variable(onehot_data[1:, :].type(torch.LongTensor))
output, loss = train(rnn, learning_rate, optimizer, criterion, input_tensor, int_target_tensor)

Here is the error:

    ---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-449-8abc91b616c7> in <module>()
----> 1 output, loss = train(rnn, learning_rate, optimizer, criterion, input_tensor, int_target_tensor)

<ipython-input-445-72363097fc21> in train(rnn, learning_rate, optimizer, criterion, input_tensor, target_tensor)
     52         output, hidden = rnn(input_tensor[i, :].unsqueeze(0), hidden)
     53         print(output.size(), target_tensor[i].size())
---> 54         loss = criterion(output, target_tensor[i])
     55         print('aaaaaaaaaaa')
     56         loss.backward()

D:\Anaconda3\lib\site-packages\torch\nn\modules\module.py in __call__(self, *input, **kwargs)
    204
    205     def __call__(self, *input, **kwargs):
--> 206         result = self.forward(*input, **kwargs)
    207         for hook in self._forward_hooks.values():
    208             hook_result = hook(self, input, result)

D:\Anaconda3\lib\site-packages\torch\nn\modules\loss.py in forward(self, input, target)
    319         _assert_no_grad(target)
    320         return F.cross_entropy(input, target,
--> 321                                self.weight, self.size_average)
    322
    323

D:\Anaconda3\lib\site-packages\torch\nn\functional.py in cross_entropy(input, target, weight, size_average)
    535                 for each minibatch.
    536     """
--> 537     return nll_loss(log_softmax(input), target, weight, size_average)
    538
    539

D:\Anaconda3\lib\site-packages\torch\nn\functional.py in nll_loss(input, target, weight, size_average)
    503     else:
    504         raise ValueError('Expected 2 or 4 dimensions (got {})'.format(dim))
--> 505     return f(input, target)
    506
    507

D:\Anaconda3\lib\site-packages\torch\nn\_functions\thnn\auto.py in forward(self, input, target)
     39         output = input.new(1)
     40         getattr(self._backend, update_output.name)(self._backend.library_state, input, target,
---> 41                                                    output, *self.additional_args)
     42         return output
     43

RuntimeError: Assertion `THIndexTensor_(size)(target, 0) == batch_size' failed.  at d:\downloads\pytorch-master-1\torch\lib\thnn\generic/ClassNLLCriterion.c:50

I am really stuck here now, so any reply will be appreciated.


title: Pytorch Loading two Images from Dataloader

body:
I'm trying to make a GAN which takes a lo-res image, and tries to create a hi-res image from it. To do this, I need to user a Dataloader which has both the hi-res and low-res training images stored in it.

     data_transform = transforms.Compose([transforms.Resize(imageSize),
                                         transforms.Grayscale(num_output_channels=1),
                                         transforms.ToTensor()])

    dataset_hi = "./hi-res-train"
    dataset_lo = "./low-res-train"

    img_data_hi = dset.ImageFolder(root=dataset_hi,transform=data_transform)
    img_data_lo = dset.ImageFolder(root=dataset_lo,transform=data_transform)

    dataloader_hi = torch.utils.data.DataLoader(img_data_hi, batch_size = batchSize, shuffle = True, num_workers = 2)
    dataloader_lo = torch.utils.data.DataLoader(img_data_lo, batch_size = batchSize, shuffle = True, num_workers = 2)

I've tried using two separate data loaders (shown above) but when they are shuffled, I cant enumerate through them both because the hi-res and low-res images are not matched up. How can I make it so I can enumerate and shuffle both with pytorch?


title: VSCode bug with PyTorch DataLoader?

body:
The following code example works in Python, but fails in VSCode in Linux (but not VSCode in Windows).  I am wondering if there is something wrong with my code, or if there is something wrong with VSCode under Linux?

#Test of PyTorch DataLoader and Visual Studio Code
from torch.utils.data import Dataset, DataLoader

class SimpleData(Dataset):
    """Very simple dataset"""
    def __init__(self):
        self.data = range(20)
    def __len__(self):
        return len(self.data)
    def __getitem__(self, idx):
        return self.data[idx]

if __name__ == '__main__':

    #Initialize DataLoader with above Dataset:
    dataloader = DataLoader(SimpleData(), batch_size=4, num_workers=1)

    print('Using DataLoader to show data in batches: ')
    for i, sample_batch in enumerate(dataloader):    #This fails in VSCode in Linux
        print('batch ', i, ':', sample_batch)

    print("--- Done ---")

The expected output is:

Using DataLoader to show data in batches:
batch  0 : tensor([0, 1, 2, 3])
batch  1 : tensor([4, 5, 6, 7])
batch  2 : tensor([ 8,  9, 10, 11])
batch  3 : tensor([12, 13, 14, 15])
batch  4 : tensor([16, 17, 18, 19])
--- Done ---

But with VSCode in Linux it hangs after printing the first line.


I am new to using Pytorch and I receive this error when running my code:

TypeError: indexing a tensor with an object of type torch.LongTensor. The only supported types are integers, slices, numpy scalars and torch.LongTensor or torch.ByteTensor as the only argument.

Can you please point me and the right direction and any help would be greatly appreciated.

if os.path.exists(CHECKPOINT_NAME):
print("=> loading checkpoint '{}'".format(CHECKPOINT_NAME))
checkpoint = torch.load(CHECKPOINT_NAME)
EPOCH = checkpoint['epoch']
BEST_LOSS = checkpoint['best_loss']
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
print("=> loaded checkpoint '{}' (epoch {})"
      .format(CHECKPOINT_NAME, checkpoint['epoch']))
else:
    print("=> no checkpoint found at '{}'. Starting from scratch".format(CHECKPOINT_NAME))

for epoch in range(EPOCH, NUM_EPOCHS):
    train(train_dataset_loader, model, loss_fn, optimizer, epoch + 1, val_dataset_loader)
    loss = validate(val_dataset_loader, model, loss_fn)

    if loss < BEST_LOSS:
        print('{} better than previous best loss of {}'.format(loss, BEST_LOSS))
        BEST_LOSS = loss
        is_best = True
    else:
        is_best = False

    save_checkpoint({
            'epoch': epoch + 1,
            'state_dict': model.state_dict(),
            'best_loss': BEST_LOSS,
            'optimizer' : optimizer.state_dict(),
        }, is_best
    )

ypeError                                 Traceback (most recent call last)
<ipython-input-16-4c3a0a33f81b> in <module>()
     12
     13 for epoch in range(EPOCH, NUM_EPOCHS):
---> 14     train(train_dataset_loader, model, loss_fn, optimizer, epoch + 1, val_dataset_loader)
     15     loss = validate(val_dataset_loader, model, loss_fn)
     16

<ipython-input-14-13120db09b49> in train(train_loader, model, criterion, optimizer, epoch, val_loader)
     65         # compute output
     66         model.zero_grad()
---> 67         log_probas, indices = model.forward(batch)
     68
     69         labels = Variable(batch['class'][indices])

<ipython-input-13-f9a47d332f53> in forward(self, batch)
     18         gene = batch['gene'][indices]
     19         variation = batch['variation'][indices]
---> 20         text_batch = torch.stack(batch['text'], 0)[:, indices]
     21
     22         # Wrap all tensors around a variable. Send to GPU if possible.


title: How to perform roll-out of an RNN In pytorch

body:
I trained an RNN (a GRU to be specific) to be a language model roughly as follows:

inputs = [<start>, tok1, tok2, tok3, . . .]

outputs = [tok1, tok2, tok3, . . .]

h0 = initial state of all-zeros

gru_outputs, hn = gru(inputs, h0)

cost = loss (gru_outputs, outputs)

Essentially during training there is a "guide" input, where I feed in the target token from the last-step to the current-step (teacher forcing?).

Now this RNN trains well, and converges. But I have no idea how to actually use it to generate a sequence from scratch. As the gru expect an explicit input argument, how do I tell it to use its own output from the last step without giving it an input?

Essentially I want

roll_out = gru(h0, step_number = 10)

Couldn't quite figure out how to do this. Do I need to use a different API and do manual roll-out in both training and using?

full-code in this gist if it is of any use: https://gist.github.com/evanthebouncy/b5039dc72d3d9fea66dad3306e479e6b

thanks !

--evan


title: PyTorch n-to-1 LSTM does not learn anything

body:
I am new to PyTorch and LSTMs and I am trying to train a classification model that takes a sentences where each word is encoded via word2vec (pre-trained vectors) and outputs one class after it saw the full sentence. I have four different classes. The sentences have variable length.

My code is running without errors, but it always predicts the same class, no matter how many epochs I train my model. So I think the gradients are not properly backpropagated. Here is my code:

class LSTM(nn.Module):
    def __init__(self, embedding_dim, hidden_dim, tagset_size):
        super(LSTM, self).__init__()
        self.hidden_dim = hidden_dim
        self.lstm = nn.LSTM(embedding_dim, hidden_dim)
        self.hidden2tag = nn.Linear(hidden_dim, tagset_size)
        self.hidden = self.init_hidden()

    def init_hidden(self):
        # The axes semantics are (num_layers, minibatch_size, hidden_dim)
        return (torch.zeros(1, 1, self.hidden_dim).to(device),
                torch.zeros(1, 1, self.hidden_dim).to(device))

    def forward(self, sentence):
        lstm_out, self.hidden = self.lstm(sentence.view(len(sentence), 1, -1), self.hidden)
        tag_space = self.hidden2tag(lstm_out.view(len(sentence), -1))
        tag_scores = F.log_softmax(tag_space, dim=1)
        return tag_scores

EMBEDDING_DIM = len(training_data[0][0][0])
HIDDEN_DIM = 256

model = LSTM(EMBEDDING_DIM, HIDDEN_DIM, 4)
model.to(device)
loss_function = nn.NLLLoss()
optimizer = optim.SGD(model.parameters(), lr=0.1)

for epoch in tqdm(range(n_epochs)):
    for sentence, tag in tqdm(training_data):
        model.zero_grad()

        model.hidden = model.init_hidden()

        sentence_in = torch.tensor(sentence, dtype=torch.float).to(device)
        targets = torch.tensor([label_to_idx[tag]], dtype=torch.long).to(device)

        tag_scores = model(sentence_in)

        res = torch.tensor(tag_scores[-1], dtype=torch.float).view(1,-1).to(device)
        # I THINK THIS IS WRONG???
        print(res)     # tensor([[-10.6328, -10.6783, -10.6667,  -0.0001]], device='cuda:0', grad_fn=<CopyBackwards>)
        print(targets) # tensor([3], device='cuda:0')

        loss = loss_function(res, targets)

        loss.backward()
        optimizer.step()

The code is largely inspired by https://pytorch.org/tutorials/beginner/nlp/sequence_models_tutorial.html
The difference is that they have a sequence-to-sequence model and I have a sequence-to-ONE model.

I am not sure what the problem is, but I guess that the scores returned by the model contain a score for each tag and my ground truth only contains the index of the correct class? How would this be handled correctly?

Or is the loss function maybe not the correct one for my use case? Also I am not sure if this is done correctly:

res = torch.tensor(tag_scores[-1], dtype=torch.float).view(1,-1).to(device)

By taking tag_scores[-1] I want to get the scores after the last word has been given to the network because tag_scores contains the scores after each step, if I understand correctly.

And this is how I evaluate:

with torch.no_grad():
    preds = []
    gts = []

    for sentence, tag in tqdm(test_data):
        inputs = torch.tensor(sentence, dtype=torch.float).to(device)

        tag_scores = model(inputs)

        # find index with max value (this is the class to be predicted)
        pred = [j for j,v in enumerate(tag_scores[-1]) if v == max(tag_scores[-1])][0]

        print(pred, idx_to_label[pred], tag)
        preds.append(pred)
        gts.append(label_to_idx[tag])

print(f1_score(gts, preds, average='micro'))
print(classification_report(gts, preds))

EDIT:

When shuffling the data before training it seems to work. But why?

EDIT 2:

I think the reason why shuffling is needed is that my training data contains samples for each class in groups. So when training them each after the other, the model will only see the same class in the last N iterations and therefore it will only predict this class. Another reason might also be that I am currently using mini-batches of only one sample because I haven't figured out yet how to use other sizes.


title: Simple way to evaluate Pytorch torchvision on a single image

body:
I have a pre-trained model on Pytorch v1.3, torchvision v0.4.2  as the following:

import PIL, torch, torchvision
# Load and normalize the image
img_file = "./robot_image.jpg"
img = PIL.Image.open(img_file)
img = torchvision.transforms.ToTensor()((img))
img = 0.5 + 0.5 * (img - img.mean()) / img.std()

# Load a pre-trained network and compute its prediction
alexnet = torchvision.models.alexnet(pretrained=True)

I want to test this single image, but I get an error:

alexnet(img)
RuntimeError: Expected 4-dimensional input for 4-dimensional weight 64 3 11 11, but got 3-dimensional input of size [3, 741, 435] instead

what is the most simple and idiomatic way of getting the model to evaluate a single data point?


title: Getting ModuleNotFoundError while the module is next to the calling file

body:
I am in a weird situation where in order for my package to be called just fine, one single file (models.py) needs to be next to the calling script and also inside the package folder where it is used.

To make it a bit more clear, this is how the package organization looks like :

-FV_dir
 ---__init__.py
 ---F_V.py
 ---models.py
 ---utils.py
 ---service_utils.py
 ---subModule1_dir
    ----__init__.py
    ----detector.py
    ----utils.py
    ----subdir1
        --- etc
    ----subdir2
        --- etc
    ----subdir3

This whole package is placed in the site-packages, so it's usable system-wide. And there is a user script that uses this package like this:
service_client.py:

from FV.service_utils import ServiceCore
from FV.utils import a_helper_function

def run():
  service = ServiceCore()
  service.run()

if __name__ == "__main__":
   run()

The ServiceCore itself uses F_V.py which is the main module here. The F_V module itself uses the models.py and utils.py next to it like  this :

F_V.py :

from FV.utils import func1, func2
from FV.models import model1, model2, model3
...

Now the problem is, if the models.py is not next to the client code (service_client.py) it just complains that the module is not found : here is an example error I get when this is the case:

           └─19146 /home/user1/anaconda3/bin/python3 /home/user1/Documents/service_client.py

Mar 17 19:22:54 ubuntu python3[19146]:     self.fv = FaceVerification(**cfg['Face_Verification']['ARGS'])
Mar 17 19:22:54 ubuntu python3[19146]:   File "/home/user1/anaconda3/lib/python3.7/site-packages/FV/F_V.py", line 58, in __init__
Mar 17 19:22:54 ubuntu python3[19146]:     self._init_model()
Mar 17 19:22:54 ubuntu python3[19146]:   File "/home/user1/anaconda3/lib/python3.7/site-packages/FV/F_V.py", line 80, in _init_model
Mar 17 19:22:54 ubuntu python3[19146]:     checkpoint = torch.load(self.model_checkpoint_path, map_location=torch.device('cpu'))
Mar 17 19:22:54 ubuntu python3[19146]:   File "/home/user1/anaconda3/lib/python3.7/site-packages/torch/serialization.py", line 529, in
Mar 17 19:22:54 ubuntu python3[19146]:     return _legacy_load(opened_file, map_location, pickle_module, **pickle_load_args)
Mar 17 19:22:54 ubuntu python3[19146]:   File "/home/user1/anaconda3/lib/python3.7/site-packages/torch/serialization.py", line 702, in
Mar 17 19:22:54 ubuntu python3[19146]:     result = unpickler.load()
Mar 17 19:22:54 ubuntu python3[19146]: ModuleNotFoundError: No module named 'models'

If I remove the models.py next to the F_V.py obviously F_V.py would complain, as it's directly using it :

           └─19216 /home/user1/anaconda3/bin/python3 /home/user1/Documents/fv_service_linux.py

Mar 17 19:27:33 ubuntu systemd[1532]: Started FV Service.
Mar 17 19:27:34 ubuntu python3[19216]: Traceback (most recent call last):
Mar 17 19:27:34 ubuntu python3[19216]:   File "/home/user1/Documents/fv_service_linux.py", line 83, in <module>
Mar 17 19:27:34 ubuntu python3[19216]:     from FV.service_utils import ServiceCore
Mar 17 19:27:34 ubuntu python3[19216]:   File "/home/user1/anaconda3/lib/python3.7/site-packages/FV/service_utils.py", line 21, in <mod
Mar 17 19:27:34 ubuntu python3[19216]:     from FV.F_V import FaceVerification
Mar 17 19:27:34 ubuntu python3[19216]:   File "/home/user1/anaconda3/lib/python3.7/site-packages/FV/F_V.py", line 17, in <module>
Mar 17 19:27:34 ubuntu python3[19216]:     from FV.models import resnet18, resnet50, resnet101
Mar 17 19:27:34 ubuntu python3[19216]: ModuleNotFoundError: No module named 'FV.models'

So the only way to get this to work is to have models.py next to the client code as well. I can't understand why this is happening, as the client code doesn't even directly interacts with models.py. What am I missing here?

Update:

In the F_V.py I use Pytorch and load a pretrained model. I thought this was unrelated and I was doing something wrong with regards to the packaging in Python, however, it turns out this was indeed the culprit. read the answer for more information.