Could you help me fix the above error? If I were to load the mnist dataset, there is no error popping up. The error has to do with the dimension of the other datasets, cifar10, fmnist and so on and cannot be run when applied to these sets. Any help appreciated.

# noinspection PyUnresolvedReferences
import os
# imports
# noinspection PyUnresolvedReferences
import pickle
from time import time

from torchvision import datasets, transforms
from torchvision.utils import save_image

import site
site.addsitedir('/content/gw_gan/model')
from loss import gwnorm_distance, loss_total_variation, loss_procrustes
from model_cnn import Generator, Adversary
from model_cnn import weights_init_generator, weights_init_adversary

# internal imports
from utils import *

# get arguments
args = get_args()

# system preferences
seed = np.random.randint(100)
torch.set_default_dtype(torch.double)
np.random.seed(seed)
torch.manual_seed(seed)

# settings
batch_size = 256
z_dim = 100
lr = 0.0002
ngen = 3
beta = args.beta
lam = 0.5
niter = 10
epsilon = 0.005
num_epochs = args.num_epochs
cuda = args.cuda
channels = args.n_channels
id1 = args.id

model = 'gwgan_{}_eps_{}_tv_{}_procrustes_{}_ngen_{}_channels_{}_{}' \
        .format(args.data, epsilon, lam, beta, ngen, channels, id1)
save_fig_path = 'out_' + model
if not os.path.exists(save_fig_path):
    os.makedirs(save_fig_path)

# data import
dataloader = torch.utils.data.DataLoader(
        datasets.CIFAR10('./data/cifar10', train=True, download=True,
                         transform=transforms.Compose([
                            transforms.ToTensor(),
                            transforms.Normalize((0.5, 0.5, 0.5),
                                                 (0.5, 0.5, 0.5))])),
        batch_size=batch_size, drop_last=True, shuffle=True)

# print example images
save_image(next(iter(dataloader))[0][:25],
           os.path.join(save_fig_path, 'real.pdf'), nrow=5, normalize=True)

# define networks and parameters
generator = Generator(output_dim=channels)
adversary = Adversary(input_dim=channels)

# weight initialisation
generator.apply(weights_init_generator)
adversary.apply(weights_init_adversary)

if cuda:
    generator = generator.cuda()
    adversary = adversary.cuda()

# create optimizer
g_optimizer = torch.optim.Adam(generator.parameters(), lr, betas=(0.5, 0.99))
# zero gradients
generator.zero_grad()

c_optimizer = torch.optim.Adam(adversary.parameters(), lr, betas=(0.5, 0.99))
# zero gradients
adversary.zero_grad()

# sample for plotting
num_test_samples = batch_size
z_ex = torch.randn(num_test_samples, z_dim)
if cuda:
    z_ex = z_ex.cuda()

loss_history = list()
loss_tv = list()
loss_orth = list()
loss_og = 0
is_hist = list()

for epoch in range(num_epochs):
    t0 = time()

    for it, (image, _) in enumerate(dataloader):
        train_c = ((it + 1) % (ngen + 1) == 0)

        x = image.double()
        if cuda:
            x = x.cuda()

        # sample random number z from Z
        z = torch.randn(image.shape[0], z_dim)

        if cuda:
            z = z.cuda()

        if train_c:
            for q in generator.parameters():
                q.requires_grad = False
            for p in adversary.parameters():
                p.requires_grad = True
        else:
            for q in generator.parameters():
                q.requires_grad = True
            for p in adversary.parameters():
                p.requires_grad = False

        # result generator
        g = generator.forward(z)

        # result adversary
        f_x = adversary.forward(x)
        f_g = adversary.forward(g)

        # compute inner distances
        D_g = get_inner_distances(f_g, metric='euclidean', concat=False)
        D_x = get_inner_distances(f_x, metric='euclidean', concat=False)

        # distance matrix normalisation
        D_x_norm = normalise_matrices(D_x)
        D_g_norm = normalise_matrices(D_g)

        # compute normalized gromov-wasserstein distance
        loss, T = gwnorm_distance((D_x, D_x_norm), (D_g, D_g_norm),
                                  epsilon, niter, loss_fun='square_loss',
                                  coupling=True, cuda=cuda)

        if train_c:
            # train adversary
            loss_og = loss_procrustes(f_x, x.view(x.shape[0], -1), cuda)
            loss_to = -loss + beta * loss_og
            loss_to.backward()

            # parameter updates
            c_optimizer.step()
            # zero gradients
            reset_grad(generator, adversary)

        else:
            # train generator
            loss_t = loss_total_variation(g)
            loss_to = loss + lam * loss_t
            loss_to.backward()

            # parameter updates
            g_optimizer.step()
            # zero gradients
            reset_grad(generator, adversary)

    # plotting
    # get generator example
    g_ex = generator.forward(z_ex)
    g_plot = g_ex.cpu().detach()

    # plot result
    save_image(g_plot.data[:25],
               os.path.join(save_fig_path, 'g_%d.pdf' % epoch),
               nrow=5, normalize=True)

    fig1, ax = plt.subplots(1, 3, figsize=(15, 5))
    ax0 = ax[0].imshow(T.cpu().detach().numpy(), cmap='RdBu_r')
    colorbar(ax0)
    ax1 = ax[1].imshow(D_x.cpu().detach().numpy(), cmap='Blues')
    colorbar(ax1)
    ax2 = ax[2].imshow(D_g.cpu().detach().numpy(), cmap='Blues')
    colorbar(ax2)
    ax[0].set_title(r'$T$')
    ax[1].set_title(r'inner distances of $D$')
    ax[2].set_title(r'inner distances of $G$')
    plt.tight_layout(h_pad=1)
    fig1.savefig(os.path.join(save_fig_path, '{}_ccc.pdf'.format(
            str(epoch).zfill(3))), bbox_inches='tight')

    loss_history.append(loss)
    loss_tv.append(loss_t)
    loss_orth.append(loss_og)
    plt.close('all')

# plot loss history
fig2 = plt.figure(figsize=(2.4, 2))
ax2 = fig2.add_subplot(111)
ax2.plot(loss_history, 'k.')
ax2.set_xlabel('Iterations')
ax2.set_ylabel(r'$\overline{GW}_\epsilon$ Loss')
plt.tight_layout()
plt.grid()
fig2.savefig(save_fig_path + '/loss_history.pdf')

fig3 = plt.figure(figsize=(2.4, 2))
ax3 = fig3.add_subplot(111)
ax3.plot(loss_tv, 'k.')
ax3.set_xlabel('Iterations')
ax3.set_ylabel(r'Total Variation Loss')
plt.tight_layout()
plt.grid()
fig3.savefig(save_fig_path + '/loss_tv.pdf')

fig4 = plt.figure(figsize=(2.4, 2))
ax4 = fig4.add_subplot(111)
ax4.plot(loss_orth, 'k.')
ax4.set_xlabel('Iterations')
ax4.set_ylabel(r'$R_\beta(f_\omega(X), X)$ Loss')
plt.tight_layout()
plt.grid()
fig4.savefig(save_fig_path + '/loss_orth.pdf')

The error displays:
Traceback (most recent call last):
  File "/content/gw_gan/main_gwgan_cnn.py", line 160, in <module>
    f_x = adversary.forward(x)
  File "/content/gw_gan/model/model_cnn.py", line 62, in forward
    x = self.conv(input)
  File "/usr/local/lib/python3.6/dist-packages/torch/nn/modules/module.py", line 727, in _call_impl
    result = self.forward(*input, **kwargs)
  File "/usr/local/lib/python3.6/dist-packages/torch/nn/modules/container.py", line 117, in forward
    input = module(input)
  File "/usr/local/lib/python3.6/dist-packages/torch/nn/modules/module.py", line 727, in _call_impl
    result = self.forward(*input, **kwargs)
  File "/usr/local/lib/python3.6/dist-packages/torch/nn/modules/conv.py", line 423, in forward
    return self._conv_forward(input, self.weight)
  File "/usr/local/lib/python3.6/dist-packages/torch/nn/modules/conv.py", line 420, in _conv_forward
    self.padding, self.dilation, self.groups)
RuntimeError: Given groups=1, weight of size [64, 1, 4, 4], expected input[256, 3, 32, 32] to have 1 channels, but got 3 channels instead

This is for an application of a generative model, where this is a CNN. The reference this is taken from is from main_gwgan_cnn at https://github.com/bunnech/gw_gan. A GAN is proposed to learn from incomparable spaces and produce results.


title: Display number of images per class using Pytorch

body:
I am using Pytorch with FashionMNIST dataset I would like to display 8 image sample from each of the 10 classes. However, I did not figure how to split the training test into train_labels since I need to loop on the labels(class) and print 8 of each class.
any idea how I can achieve this?
classes = ('T-shirt/top', 'Trouser', 'Pullover', 'Dress', 'Coat', 'Sandal', 'Shirt', 'Sneaker', 'Bag', 'Ankle boot')

# Define a transform to normalize the data
transform = transforms.Compose([transforms.ToTensor(),
                              #  transforms.Lambda(lambda x: x.repeat(3,1,1)),
                                transforms.Normalize((0.5, ), (0.5,))])
# Download and load the training data
trainset = datasets.FashionMNIST('~/.pytorch/F_MNIST_data/', download=True, train=True, transform=transform)
trainloader = torch.utils.data.DataLoader(trainset, batch_size=4, shuffle=True)
# Download and load the test data
testset = datasets.FashionMNIST('~/.pytorch/F_MNIST_data/', download=True, train=False, transform=transform)
testloader = torch.utils.data.DataLoader(testset, batch_size=4, shuffle=True)

print('Training set size:', len(trainset))
print('Test set size:',len(testset))


title: how to use MaxUnpool1d from pytorch library in convolution autoencoder?

body:
I want to build a Convolution AutoEncoder using Pytorch library in python.
All of the examples have no MaxUnpool1d.

The architecture I want to build should be like:

class autoencoder(nn.Module):
    def __init__(self):
            self.encoder = nn.Sequential(
                                 nn.Conv1d(in_channels=1, out_channels=64 ,kernel_size=16 , stride=3),
                                 nn.ReLU(True),
                                 nn.MaxPool1d(4),
                                 nn.Conv1d(64, 128 ,8, 3),
                                 nn.ReLU(True),
                                 nn.MaxPool1d(4),
                                 )

            self.decoder = nn.Sequential(
                                 nn.MaxUnpool1d(4),
                                 nn.ConvTranspose1d(in_channels =128, out_channels =64 , kernel_size =8, stride= 3),
                                 nn.ReLU(True),
                                 nn.MaxUnpool1d(4),
                                 nn.ConvTranspose1d(64, 1 ,16, 3),
                                 nn.Tanh()
                                 )

    def forward(self, x):
        x = self.encoder(x)
        x = self.decoder(x)
        return x

To make it work I should provide indices for the MaxUnpool1d layer.
How do I use it for the current architecture and also making the input size equals to the output?

Thanks.


title: Where is the `torch.empty` defined?

body:
(I am not a Python veteran so pardon me if this is a naive question.)

Following the PyTorch tutorial, below code works fine.

import torch
x= torch.empty(5,3) # <========= HERE
print(x)

But the Visual Studio Code complains that:

  E1101:Module 'torch' has no 'empty' member

Eclipse PyDev has a similar warning.

So I use below code to check what torch offers:

for (k, v) in torch.__dict__.items():
    print(k, v)

And I find that it is a built-in:

empty <built-in method empty of type object at 0x00007FFA380E0A80>

Since it is a method, I searched all the *.py file in the torch package folder for something like def empty and empty, no luck.

So where is this empty method defined? Some pyd/DLL?

How is it imported into the namespace?

Or put it another way, how to create a built-in function for my own package?

ADD 1 -- 10:38 AM 11/12/2018

Extending Python Built-ins with C/C++

https://docs.python.org/3/extending/extending.html

I guess it's the torch::empty method. I have found a lot of matches in the github PyTorch codebase. But I haven't nailed the definition yet.


title: Updating pre-trained Deep Learning model with respect to new data points

body:
Considering the example of Image classification on ImageNet, How to update the pre-trained model using the new data points.
I have loaded the pre-trained model. I have a new data point that is quite different from the distribution of the original data on which the model was previously trained. So, I would like to update/fine-tune the model with the help of new data point. How to go about doing it? Can anyone help me out in doing it? I am using pytorch 0.4.0 for implementation, running on GPU Tesla K40C.


title: how to calculate cross entropy in 3d image pytorch?

body:
See the figure here

left thing is (2, 480, 640) and it's softmax value

right thing is (2, 480, 640) and it's one-hot encoding value

how to get cross entropy loss in all element?


title: Why can't I install new version of Pytorch?

body:
My OS is CentOS 7, and I want to install PyTorch so I did the following:

(pt_gpu) [martin@A08-R32-I196-3-FZ2LTP2 mlm]$ conda -V
conda 4.6.2

(pt_gpu) [martin@A08-R32-I196-3-FZ2LTP2 mlm]$ conda install -c anaconda pytorch-gpu

What's strange is that the installation message shows that it is installing  a very old version of PyTorch:

Collecting package metadata: done
Solving environment: done

## Package Plan ##

  environment location: /home/martin/anaconda3/envs/pt_gpu

  added / updated specs:
    - pytorch-gpu

The following packages will be downloaded:

    package                    |            build
    ---------------------------|-----------------
    ca-certificates-2018.12.5  |                0         123 KB  anaconda
    certifi-2018.11.29         |           py36_0         146 KB  anaconda
    pytorch-gpu-0.1.12         |           py36_0        16.8 MB  anaconda
    ------------------------------------------------------------
                                           Total:        17.0 MB

The following packages will be UPDATED:

  openssl              pkgs/main::openssl-1.1.1a-h7b6447c_0 --> anaconda::openssl-1.1.1-h7b6447c_0

The following packages will be SUPERSEDED by a higher-priority channel:

  ca-certificates                                 pkgs/main --> anaconda
  certifi                                         pkgs/main --> anaconda
  mkl                    pkgs/main::mkl-2017.0.4-h4c4d0af_0 --> anaconda::mkl-2017.0.1-0
  pytorch-gpu                                     pkgs/free --> anaconda

Proceed ([y]/n)? y

Downloading and Extracting Packages
certifi-2018.11.29   | 146 KB    | ########################################################################################################################## | 100%
ca-certificates-2018 | 123 KB    | ########################################################################################################################## | 100%
pytorch-gpu-0.1.12   | 16.8 MB   | ########################################################################################################################## | 100%
Preparing transaction: done
Verifying transaction: done
Executing transaction: done

To verify what's installed, I did:

(pt_gpu) [martin@A08-R32-I196-3-FZ2LTP2 mlm]$ python -c "import torch; print(torch.__version__)"
0.1.12

Why is it that?


title: Pytorchs' nn.Linear returns only `nan` values under certain conditions

body:
I have a pytorch tensor with my normalized data that at some point pass through this layer on my model,

linear = nn.Linear(in_features=137, out_features=1)

The data can be seen here,

The issue is that I only get nans as a result

and the parameters of linear appear to also be nan,

In order to hopefully better understand what the issue was I saved the data at that exact moment into a csv file and loaded them in an interpreter,

performing the exact operation yields the expected result,

>>> linear(x)
tensor([[ 0.0594],
        [ 0.1051],
        [ 0.1000],
        ...,
        [ 0.1043],
        [-0.1210],
        [-0.0332]], grad_fn=<AddmmBackward>)

Where could this odd behaviour be attributed to, and how may I fix it?


title: How to reshape last layer of pytorch CNN model while doing transfer learning

body:
Actually i am trying to replicate keras structure to pytorch.(new in pytorch).
Here is keras architecture

base_model = InceptionV3(weights='imagenet', include_top=False)
x = base_model.output
x = Dense(512, activation='relu')(x)
predictions = Dense(49*6,activation='sigmoid')(x)
reshape=Reshape((49,6))(predictions)
model = Model(inputs=base_model.input, outputs=reshape)
for layer in base_model.layers:
    layer.trainable = False

I want to reshape last layer of my netwrok. I have implemented transfer learning.

model =  models.inception_v3(pretrained=True)

for param in model.parameters():
    param.requires_grad = False

num_ftrs = model.fc.in_features

I believe if i can attach last layer of resnet with my following architecture, problem can be solved. But i dont know how i can attach them

class CNN(nn.Module):
    def __init__(self):
        super(CNN, self).__init__()
        self.fc1 = nn.Linear(num_ftrs, 512)
        self.fc2 = nn.Linear(512, 49*6)

    def forward(self, x):
        print (x.shape)
        x = x.view(-1,num_ftrs)
        #print (x.shape)
        x = F.relu(self.fc1(x))
        x = self.fc2(x)
        x=torch.sigmoid(x.view(49,6))
        return x

Any idea, how this problem can be resolved


title: Why cleverhans pytorch tutorial using log_softmax instead of logits as output

body:
When generating adversarial examples, it is typically using logits as the output of the neural network, and then train the network with cross-entropy.

However, I found that the tutorial of cleverhans uses log softmax and then convert the pytorch model to a tensorflow model, and finally train the model.

https://github.com/tensorflow/cleverhans/blob/master/cleverhans_tutorials/mnist_tutorial_pytorch.py#L65

I am wondering if anyone has the idea about whether using logits instead of log_softmax will make any difference?


title: Is it possible to use LSTM predictions as inputs for next time steps?

body:
I am working with LSTM (in PyTorch) for multivariate time series prediction. Let’s imagine the situation: I have 2 time series, A and B, and I want to predict t-value of B using previous values of A and B (before t). Such prediction works fine, my model gets good results.

But what if (during testing, after training) I want to use predicted values of B as inputs for next time step instead of real values? For example: I predict first value of B, make a step, put predicted value instead of a real, and make prediction again. Then I use two predicted values instead of real two, and so on. In some steps only predicted values will be in time series B.

Are there any possibilities to do that?


title: How should I apply a variational autoencoder in a low-dimensional real value case?

body:
I am trying to applying VAE in a simple toy example to familiarize with its property. However, I get stuck in training the model. The total loss and the reconstruction error does not seem to decrease.
The toy example is listed below

random generate 5000 observation from a 2-dimensional multivariate normal distribution.
apply a transformation f, [x,y] --> [sin(x),sin(y)]
train a VAE with 1 hidden layer with 5 neuron in both encoder and decoder. The VAE has 2 latent variables.

In this example, I am not able to decrease the training loss to a sufficiently low level, and the reconstruction is also messy.
I have made several attempts

increase the hidden layers to 2 and 3 (this does not help)
--> I think it is not due to the complexity of the model
check the network on MNIST (the result is comparable with the example I found on other sources)
--> the model design is right
delete the KL divergence in the Loss function (the model can reconstruct well)
--> the model design is right
I try to balance the weight on KL divergence
--> when beta on KL divergence is low, it reconstruct well, but latent space is too far away from standard normal, when beta on KL divergence is high, it can not reconstruct well, but the latent space perform well.

I now suspect several potential reasons, but I can not distinguish which one could be the reason.

It seems that I need to find a balance between weight on KL divergence and reconstruction loss
Is it appropriate to use MSE loss + KL divergence as loss function?
In low-dimension, the VAE does not perform because the ELBO is not so tight?

Could any one help?
The code is attached.
This part defines the model
import torch
import torch.nn as nn
import pandas as pd
import numpy as np

class VAE_Encoder(nn.Module):

    def __init__(self,input_size,hidden_size_list,latent_size):

        """
        The class is the builder of the encoder part of VAE. It does not need to be directly called.

        :param input_size: int
        :param hidden_size_list: list(int)
        :param latent_size: int
        """

        super().__init__()

        encoder_size = [input_size]+hidden_size_list
        encoder_layers = []

        for in_size,out_size in zip(encoder_size[:-1],encoder_size[1:]):

            encoder_layers.append(nn.Linear(in_size,out_size))
            encoder_layers.append(nn.ReLU())

        self.encoder = nn.Sequential(*encoder_layers)

        self.encoder_mu = nn.Linear(encoder_size[-1],latent_size)
        self.encoder_logvar = nn.Linear(encoder_size[-1],latent_size)

    def encode(self,x):

        return self.encoder(x)

    def encode_gaussian_param(self,encode_x):

        return self.encoder_mu(encode_x),self.encoder_logvar(encode_x)

    def reparametrize(self,mu,logvar):

        std = torch.exp(0.5*logvar)
        eps = torch.randn_like(std)

        return mu+eps*std

    def forward(self,x):

        encode_x = self.encoder(x)
        mu,logvar = self.encode_gaussian_param(encode_x)
        z = self.reparametrize(mu,logvar)

        return z,mu,logvar

class VAE_Decoder(nn.Module):

    def __init__(self,input_size,hidden_size_list,latent_size):

        """
        The class is the builder of the decoder part of VAE. It does not need to be directly called.

        :param input_size: int
        :param hidden_size_list: list(int)
        :param latent_size: int

        """

        super().__init__()

        decoder_size = [latent_size] + hidden_size_list
        decoder_layers = []

        for in_size,out_size in zip(decoder_size[:-1],decoder_size[1:]):

            decoder_layers.append(nn.Linear(in_size,out_size))
            decoder_layers.append(nn.ReLU())

        decoder_layers.append(nn.Linear(decoder_size[-1],input_size))
        self.decoder = nn.Sequential(*decoder_layers)

    def forward(self,z):

        return self.decoder(z)

class VAE(nn.Module):

    def __init__(self,input_size,encoder_size,latent_size,decoder_size=None):

        """
        The class builds the whole VAE. It consists of a encoder model and a decoder model.
        The user has flexibility to choose the number of layers in each part of the model by
        setting the encoder size and decoder size.

        :param input_size: int
        :param encoder_size: list(int)
        :param latent_size: int
        :param decoder_size: list(int)

        """

        super().__init__()

        if decoder_size is None:

            decoder_size = encoder_size[::-1]

        self.encoder = VAE_Encoder(input_size,encoder_size,latent_size)
        self.decoder = VAE_Decoder(input_size,decoder_size,latent_size)

    def decode(self,z):

        return self.decoder(z)

    def forward(self,x):

        z,mu,logvar = self.encoder(x)
        x = self.decoder(z)

        return x,mu,logvar

def simple_vae_loss(real,recon,mu,logvar,penalty=1):

    MSE = nn.functional.mse_loss(recon,real,reduction="sum")
    KLD = -0.5 * torch.sum(1 + logvar - mu.pow(2) - logvar.exp())

    return MSE+penalty*KLD

This part generate the toy example

def generate_value(x,y):

    v1 = np.sin(x)
    v2 = np.sin(y)

    x = np.zeros(x.shape)
    y = np.zeros(y.shape)

    return (v1,v2)

rand_number = torch.randn((5000,2))

x = rand_number.numpy()[:,0]
y = rand_number.numpy()[:,1]

new_value = generate_value(x,y)

new_x = new_value[0].reshape((5000,1))
new_y = new_value[1].reshape((5000,1))
new_data = np.concatenate((new_x,new_y),axis=1)


title: torchvision.transforms.Normalize() slows down learning when adding to torchvision,transforms.Compose()

body:
when i use
train_transforms = torchvision.transforms.Compose([
  torchvision.transforms.ToTensor(),
  torchvision.transforms.Normalize((0.1307,), (0.3081,))
])

for loading MNIST dataset, it slows down learning even with mean = 0 and std = 1.


title: OpenNMT issue with PyTorch: .copy_ function not clear behavior

body:
I'm working with the PyTorch version OpenNMT and I'm trying to modify the Beam Search algorithm. I'm currently stuck in the  beam_update function (in OpenNMT-py/onmt/decoders/decoder.py file). When it is called:

sent_states.data.copy_(
                sent_states.data.index_select(1, positions))

according to the pythorch documentation of the .copy_ function it will

  Copies the elements from src into self tensor and returns self.

But, what is "self tensor" referring to? Can someone explain to me what this function do or point me to the source code, since I cannot find it...


title: Pytorch tensor equality not making sense

body:
tensor(1.0000, grad_fn=<SumBackward0>)

tensor([1.])

Why are these two tensors not equivalent in pytorch?


title: Understanding when to use python list in Pytorch

body:
Basically as this thread discusses here, you cannot use python list to wrap your sub-modules (for example your layers); otherwise, Pytorch is not going to update the parameters of the sub-modules inside the list. Instead you should use nn.ModuleList to wrap your sub-modules to make sure their parameters are going to be updated. Now I have also seen codes like following where the author uses python list to calculate the loss and then do loss.backward() to do the update (in reinforce algorithm of RL). Here is the code:

 policy_loss = []
    for log_prob in self.controller.log_probability_slected_action_list:
        policy_loss.append(- log_prob * (average_reward - b))
    self.optimizer.zero_grad()
    final_policy_loss = (torch.cat(policy_loss).sum()) * gamma
    final_policy_loss.backward()
    self.optimizer.step()

Why using the list in this format works for updating the parameters of modules but the first case does not work? I am very confused now. If I change in the previous code policy_loss = nn.ModuleList([]), it throws an exception saying that tensor float is not sub-module.


title: pytorch: Getting same result in each epoch

body:
I m try to train an rbf network... I used MNIST database. And pytorch framework...
The results are the same in each epoch...

The results....:

Epoch: 1
Accuracy: 0.785     Loss: 2.435     Recall: 0.386   Precision: 0.258

Epoch: 2
Accuracy: 0.785     Loss: 2.435     Recall: 0.386   Precision: 0.258

Epoch: 3
Accuracy: 0.785     Loss: 2.435     Recall: 0.386   Precision: 0.258

Epoch: 4
Accuracy: 0.785     Loss: 2.435     Recall: 0.386   Precision: 0.258

My Code... I think that the problem is somewhere in the linear layer. The model has no improve after the training epoch, maybe it's the linear layer. It seems like the weights no change...! But i don't know why...?

class RBF(nn.Module):
    def __init__(self, in_layers, centers, sigmas):
        super(RBF, self).__init__()
        self.in_layers = in_layers
        self.centers = nn.Parameter(centers)
        self.sigmas = nn.Parameter(torch.Tensor(self.centers.size(0)))
        torch.nn.init.constant_(self.sigmas, sigmas)

    def forward(self, x):
        x = x.view(-1, self.in_layers)
        size = [self.centers.size(0), x.size(0)]
        sigma = self.sigmas.view(-1).to(device)**2

        dists = torch.empty(size).to(device)

        for i,c in enumerate(self.centers):
            c = c.reshape(-1,c.size(0))
            temp = (x-c).pow(2).sum(-1).pow(0.5)
            dists[i] = temp

        dists = dists.permute(1,0)
        phi = torch.exp(-1*(dists/(2*sigma))) #gaussian
        return phi

class Net(nn.Module):
    def __init__(self, in_layers, centers, sigmas):
        super(Net, self).__init__()
        self.rbf_layers = nn.ModuleList()
        self.linear_layers = nn.ModuleList()
        for i in range(len(in_layers) - 1):
            self.rbf_layers.append(RBF(in_layers[i], centers, sigmas))
            self.linear_layers.append(nn.Linear(centers.size(0), in_layers[i+1], bias = True))

    def forward(self, x):
        out = x
        for i in range(len(self.rbf_layers)):
            out = self.rbf_layers[i](out)
            out = F.sigmoid( self.linear_layers[i](out.float()) )
        return out

def training(engine, batch, device, model, criterion, optimizer):
    inputs, labels = batch[0].to(device), batch[1].to(device)
    optimizer.zero_grad()
    outputs = model(inputs)
    loss = criterion(outputs, labels)
    loss.backward()
    optimizer.step()
    return outputs, labels

Ofcourse the code continuous but i think these are enough to solve the problem(if you want smthing extra 'i m here').... Do you have any ideas???

and the training part of code....

def training(engine, batch, device, model, criterion, optimizer):
    inputs, labels = batch[0].to(device), batch[1].to(device)
    optimizer.zero_grad()
    outputs = model(inputs)
    loss = criterion(outputs, labels)
    loss.backward()
    optimizer.step()
    return outputs, labels

def nn_run1(batch, classes, dim, learning_rate, epochs, clusters):
    # ---Load Model's Parameters---
    train_loader, test_loader = data_loading(batch, shuffle=False)
    kmeans_input = train_loader.dataset.train_data
    kmeans_input = torch.reshape(kmeans_input.double(), (kmeans_input.size(0), -1))
    _, centers = Kmeans(kmeans_input, clusters)
    centers = centers.to(device)
    sigma = Sigmas(centers)
    layers = in_layers(dim, len(classes), layers = 1)

    # ---Model Setup---
    model = Net(layers, centers, sigma)
    model.cuda()
    criterion = nn.CrossEntropyLoss()
    print(model.parameters)
    optimizer = torch.optim.SGD(model.parameters(), learning_rate)


title: How to custom fairseq model input?

body: I am using pytorch/fairseq for create a seq2seq translation model that use src_tokens and its postag as input to improve machine translation result in a way that describe in this paper. I'm already custom preprocess.py file to preprocess tag file but still confuse about how fairseq model receives input. Please explain me about how fairseq model receives input and give me some idea about how to make model receive postag and src tokens as tensor. Thanks


title: How do I convert int8 into its binary representation in pytorch?

body:
Given a pytorch tensor in dtype=int8:
tensor([[[-3, -6, -1],
         [-6, -10, -1,
         [9, 9, 6],
         [[-4, -7, -3],
          [-4, -6, -1],
          [14, 16, 8]],
         [[-4, -6, -2],
          [-6, -9, -2],
          [9, 10, 5]]], device='cuda:0', dtype=torch.int8)

How do I convert the above tensor into its binary representation?
I tried to convert to numpy to use np.unpackbits function but it only takes un-sign integer 8 as input.


title: How can I incorporate PReLU in a quantized model?

body:
I'm trying to quantize a model which uses PReLU. Replacing PReLU with ReLU is not possible as it drastically affects the network performance to the point its useless.
As far as I know, PReLU is not supported in Pytorch when it comes to quantization. So I tried to rewrite this module manually and implement the multiplication and additions using torch.FloatFunctional() to get around this limitation.
This is what I have come up so far:
class PReLU_Quantized(nn.Module):
    def __init__(self, prelu_object):
        super().__init__()
        self.weight = prelu_object.weight
        self.quantized_op = nn.quantized.FloatFunctional()
        self.quant = torch.quantization.QuantStub()
        self.dequant = torch.quantization.DeQuantStub()

    def forward(self, inputs):
        # inputs = torch.max(0, inputs) + self.weight * torch.min(0, inputs)
        self.weight = self.quant(self.weight)
        weight_min_res = self.quantized_op.mul(self.weight, torch.min(inputs)[0])
        inputs = self.quantized_op.add(torch.max(inputs)[0], weight_min_res).unsqueeze(0)
        self.weight = self.dequant(self.weight)
        return inputs

and for the replacement :
class model(nn.Module):
     def __init__(self)
         super().__init__()
         ....
        self.prelu = PReLU()
        self.prelu_q = PReLU_Quantized(self.prelu)
         ....

Basically, I read the learned parameter of the existing prelu module, and run the calculation myself in a new module. The module seems to be working in the sense, its not failing the whole application.
However, in order to assess whether my implementation is actually correct and yields the same result as the original module, I tried to test it.
Here is a counterpart for normal models (i.e. not quantized model):
For some reason, the error between the actual PReLU and my implementation is very large!
Here are sample diffs in different layers:
diff : 1.1562038660049438
diff : 0.02868632599711418
diff : 0.3653906583786011
diff : 1.6100226640701294
diff : 0.8999372720718384
diff : 0.03773299604654312
diff : -0.5090572834014893
diff : 0.1654307246208191
diff : 1.161868691444397
diff : 0.026089997962117195
diff : 0.4205571115016937
diff : 1.5337920188903809
diff : 0.8799554705619812
diff : 0.03827812895178795
diff : -0.40296515822410583
diff : 0.15618863701820374

and the diff is calculated like this in the forward pass:
def forward(self, x):
    residual = x
    out = self.bn0(x)
    out = self.conv1(out)
    out = self.bn1(out)

    out = self.prelu(out)
    out2 = self.prelu2(out)
    print(f'diff : {( out - out2).mean().item()}')

    out = self.conv2(out)
...

This is the normal implementation which I used on ordinary model (i.e. not quantized!) to asses whether it produces correct result and then move on to quantized version:
class PReLU_2(nn.Module):
    def __init__(self, prelu_object):
        super().__init__()
        self.prelu_weight = prelu_object.weight
        self.weight = self.prelu_weight

    def forward(self, inputs):
        x = self.weight
        tmin, _ = torch.min(inputs,dim=0)
        tmax, _ = torch.max(inputs,dim=0)
        weight_min_res = torch.mul(x, tmin)
        inputs = torch.add(tmax, weight_min_res)
        inputs = inputs.unsqueeze(0)
        return inputs

What am I missing here?


title: Does Pytorch-Lightning have a multiprocessing (or Joblib) module?

body:
I have been googling around but can't seem to find if there is a multiprocessing module available in Pytorch-Lightning, just like how Pytorch has a torch.multiprocessing module.
Does anyone know if Pytorch-Lightning has this (or a Joblib similar) module? I am looking for a Pytorch-Lightning module which allows me to parallelize over multiple GPUs
Many thanks in advance.
Edit: To be more specific, I am looking for a multiprocessing module in Pytorch-Lightning which allows me to parallelize over multiple GPUs on non-neural network computations, such as:
import numpy as np
import torch
from torch.multiprocessing import Pool

X = np.array([[1, 3, 2, 3], [2, 3, 5, 6], [1, 2, 3, 4]])
X = torch.DoubleTensor(X)

def X_power_func(j):
    X_power = X.cuda()**j
    return X_power

if __name__ == '__main__':
  with Pool(processes = 2) as p:   # Parallelizing over 2 GPUs
    results = p.map(X_power_func, range(4))

results


title: trying to build pytorch 1.0.0 cuda 10.2 with support for old gpu (3.0)

body:
I'm playing with a couple of projects that explicitly require pytorch == 1.0.0, but I have an old graphics card that only supports cuda 3.0 so I'm using the cpu, which is very slow, being the graphics card a dual gpu I decided to give a try and build pytorch from the sources with support for 3.0 (I have planned to update the pc but is not gonna happen anytime soon).
I am using docker to do the build, in particular I tried to modify an existing Dockerfile from build-pytorch, on the host system I am using debian/sid and there is cuda 10.2 cudnn 7.6 installed, I'm not sure if I can downgrade cuda, and I don't know if the versions in the container must be exactly the same as the host (like for nvidia drivers).
Gist of the modified Dockerfile
The first thing I noticed when updating the versions is that package cuda-cublas-dev-10-2 was not found, the latest version was 10-0,
CUBLAS packaging changed in CUDA 10.1 to be outside of the toolkit installation path
If I install cublas version 10-0 or if I don't install it obviously no header files are found (error below), if I install the recommended libcublas-dev version the build continues for a while, with some warnings (below) , but then it stops with the error below.
I searched for the error online but I did not find anything specific, if I understand correctly there is a function declared more than once and when it is called the choice is ambiguous, but I have not yet investigated looking at the sources.
I would like to know if anyone has run into this error before and knows how to fix it.
libcublas-dev installed error:
[ 67%] Building NVCC (Device) object caffe2/CMakeFiles/caffe2_gpu.dir/__/aten/src/ATen/native/sparse/cuda/caffe2_gpu_generated_SparseCUDABlas.cu.o
/pytorch/aten/src/ATen/native/sparse/cuda/SparseCUDABlas.cu(58): error: more than one instance of function "at::native::sparse::cuda::cusparseGetErrorString" matches the argument list:
            function "cusparseGetErrorString(cusparseStatus_t)"
            function "at::native::sparse::cuda::cusparseGetErrorString(cusparseStatus_t)"
            argument types are: (cusparseStatus_t)

1 error detected in the compilation of "/tmp/tmpxft_00004ccc_00000000-6_SparseCUDABlas.cpp1.ii".
CMake Error at caffe2_gpu_generated_SparseCUDABlas.cu.o.Release.cmake:279 (message):
  Error generating file
  /pytorch/build/caffe2/CMakeFiles/caffe2_gpu.dir/__/aten/src/ATen/native/sparse/cuda/./caffe2_gpu_generated_SparseCUDABlas.cu.o

caffe2/CMakeFiles/caffe2_gpu.dir/build.make:1260: recipe for target 'caffe2/CMakeFiles/caffe2_gpu.dir/__/aten/src/ATen/native/sparse/cuda/caffe2_gpu_generated_SparseCUDABlas.cu.o' failed

warnings:
ptxas warning : Too big maxrregcount value specified 96, will be ignored

missing header error:
Scanning dependencies of target caffe2_pybind11_state
[ 59%] Building CXX object caffe2/CMakeFiles/caffe2_pybind11_state.dir/python/pybind_state.cc.o
In file included from /pytorch/aten/src/THC/THC.h:4:0,
                 from /pytorch/torch/lib/THD/../THD/base/TensorDescriptor.h:6,
                 from /pytorch/torch/lib/THD/../THD/base/TensorDescriptor.hpp:6,
                 from /pytorch/torch/lib/THD/../THD/THD.h:14,
                 from /pytorch/torch/lib/THD/base/DataChannelRequest.h:3,
                 from /pytorch/torch/lib/THD/base/DataChannelRequest.hpp:6,
                 from /pytorch/torch/lib/THD/base/DataChannelRequest.cpp:1:
/pytorch/build/caffe2/aten/src/THC/THCGeneral.h:17:23: fatal error: cublas_v2.h: No such file or directory
compilation terminated.
make[2]: *** [caffe2/torch/lib/THD/CMakeFiles/THD.dir/base/DataChannelRequest.cpp.o] Error 1
make[2]: *** Waiting for unfinished jobs....


title: Pytorch cuda version can not install

body:
In the beginning, I checked my cuda version using nvcc --version command and it shows version as 10.2
So i started to install pytorch with cuda based on instruction in pytorch so I tried with bellow command in anaconda prompt with python 3.8.3 virtual environment
conda install pytorch torchvision torchaudio cudatoolkit=10.2 -c pytorch
in terminal it shows
    added / updated specs:
        - cudatoolkit=10.2
        - pytorch
        - torchaudio
        - torchvision

The following NEW packages will be INSTALLED:

    cudatoolkit: 10.2.89-h74a9793_1
    pytorch:     1.7.0-py3.8_cpu_0  pytorch [cpuonly]
    torchaudio:  0.7.0-py38         pytorch
    torchvision: 0.8.1-py38_cpu     pytorch [cpuonly]

there are no any errors in installing process and then I try with below codes in notebook by activating this installed environment
import torch
print(torch.__version__, torch.cuda.get_device_properties(0) if torch.cuda.is_available() else 'CPU')

output was 1.7.0 CPU
so I run conda list PyTorch command in anaconda prompt
it also shows below output
# Name                    Version                   Build  Channel
pytorch                   1.7.0               py3.8_cpu_0  [cpuonly]  pytorch

why it didn't install the GPU version and how can I fix this
Thank you


title: TypeError: object.__init__() takes no parameters trying to subclass torch.FloatTensor

body:
I'm trying to subclass torch.FloatTensor in Python 3 and am having trouble calling the superclass __init__. Minimal example:

import inspect
import torch

x = [[1, 2], [3, 4]]

print(torch.FloatTensor(x))
'''
 1  2
 3  4
[torch.FloatTensor of size 2x2]
'''

print(inspect.signature(torch.FloatTensor.__init__))
'''
(self, /, *args, **kwargs)
'''

class Image(torch.FloatTensor):
    def __init__(self, arg):
        print(inspect.signature(super().__init__))
        super().__init__(arg)

Image(x)
'''
(*args, **kwargs)
---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<ipython-input-369-4b1aee6414f6> in <module>()
     21         super().__init__(arg)
     22
---> 23 Image(x)

<ipython-input-369-4b1aee6414f6> in __init__(self, arg)
     19     def __init__(self, arg):
     20         print(inspect.signature(super().__init__))
---> 21         super().__init__(arg)
     22
     23 Image(x)

TypeError: object.__init__() takes no parameters
'''

I'm relatively new to Python so I might be missing something obvious. Why does it say object.__init__() takes no parameters when inspecting shows it can take (*args, **kwargs)?

Edit: torch.FloatTensor is defined here. It is derived from torch._TensorBase which is defined here.


title: MXNet using NDArray Scatter_nd on multi GPU

body:
When I use multi gpu to train on MXNet(CUDA8.0+cudnn7), I firstly initialize parameters on different context, then I perform scatter_nd on different contexts, the first time scatter nd would work perfectly, but when compute for the second gpu card, I got

F1217 23:53:01.012707  2619 stream_gpu-inl.h:62] Check failed: e == cudaSuccess CUDA: an illegal memory access was encountered


title: How to use run_classifer.py,an example of Pytorch implementation of Bert for classification Task?

body:
How to use the fine-tuned bert pytorch model for classification (CoLa) task?

I do not see the argument --do_predict, in /examples/run_classifier.py.

However, --do_predict exists in the original implementation of the Bert.

The fine-tuned model is getting saving in the BERT_OUTPUT_DIR as pytorch_model.bin, but is there a simple way to reuse it through the command line?

Using Pytorch implementation from: https://github.com/huggingface/pytorch-pretrained-BERT

The command which I am using to execute the code is:

python run_classifier.py \
  --task_name CoLA \
  --do_train \
  --do_eval \
  --do_lower_case \
  --data_dir ./split/ \
  --bert_model bert-base-uncased \
  --max_seq_length 128 \
  --train_batch_size 32 \
  --learning_rate 2e-5 \
  --num_train_epochs 3.0 \


title: Runtime error when load tensorflow and pytorch models at the same time

body:
Here is the setup

Project A
    |-- Module A
           |--->step1 - Loads and run tensorflow pretrained model to detect
                           texts
           |
           |--->step2 - import and Instantiates Module-B from Project B
                             --> Runtime Failure here.

Project B
    |-- Module B
           |--> step 1 - Loads pretrained Pytorch model
           |--> step 2 - Runs model to classify text.

Here is the actual runtime error

self.trmodel.load_state_dict(torch.load(self.opts.saved_model, map_location='cpu'))
File/root/anaconda3/envs/py3/lib/python3.6/site-packages/torch/nn/modules/module.py", line 777, in load_state_dict
self.__class__.__name__,\n\t".join(error_msgs)))
RuntimeError: Error(s) in loading state_dict for DataParallel:
Unexpected key(s) in state_dict:module.Transformation.LocalizationNetwork.conv.0.weight",module.Transformation.LocalizationNetwork.conv.1.weight",module.Transformation.LocalizationNetwork.conv.1.bias",module.Transformation.LocalizationNetwork.conv.1.running_mean",module.Transformation.LocalizationNetwork.conv.1.running_var",module.Transformation.LocalizationNetwork.conv.1.num_batches_tracked",module.Transformation.LocalizationNetwork.conv.4.weight",module.Transformation.LocalizationNetwork.conv.5.weight",module.Transformation.LocalizationNetwork.conv.5.bias",module.Transformation.LocalizationNetwork.conv.5.running_mean",module.Transformation.LocalizationNetwork.conv.5.running_var",module.Transformation.LocalizationNetwork.conv.5.num_batches_tracked",module.Transformation.LocalizationNetwork.conv.8.weight",module.Transformation.LocalizationNetwork.conv.9.weight",module.Transformation.LocalizationNetwork.conv.9.bias",module.Transformation.LocalizationNetwork.conv.9.running_mean",module.Transformation.LocalizationNetwork.conv.9.running_var",module.Transformation.LocalizationNetwork.conv.9.num_batches_tracked",module.Transformation.LocalizationNetwork.conv.12.weight",module.Transformation.LocalizationNetwork.conv.13.weight",module.Transformation.LocalizationNetwork.conv.13.bias",module.Transformation.LocalizationNetwork.conv.13.running_mean",module.Transformation.LocalizationNetwork.conv.13.running_var",module.Transformation.LocalizationNetwork.conv.13.num_batches_tracked",module.Transformation.LocalizationNetwork.localization_fc1.0.weight",module.Transformation.LocalizationNetwork.localization_fc1.0.bias",module.Transformation.LocalizationNetwork.localization_fc2.weight",module.Transformation.LocalizationNetwork.localization_fc2.bias",module.Transformation.GridGenerator.inv_delta_C",module.Transformation.GridGenerator.P_hat",module.FeatureExtraction.ConvNet.conv0_1.weight",module.FeatureExtraction.ConvNet.bn0_1.weight",module.FeatureExtraction.ConvNet.bn0_1.bias",module.FeatureExtraction.ConvNet.bn0_1.running_mean",module.FeatureExtraction.ConvNet.bn0_1.running_var",module.FeatureExtraction.ConvNet.bn0_1.num_batches_tracked",module.FeatureExtraction.ConvNet.conv0_2.weight",module.FeatureExtraction.ConvNet.bn0_2.weight",module.FeatureExtraction.ConvNet.bn0_2.bias",module.FeatureExtraction.ConvNet.bn0_2.running_mean",module.FeatureExtraction.ConvNet.bn0_2.running_var",module.FeatureExtraction.ConvNet.bn0_2.num_batches_tracked",module.FeatureExtraction.ConvNet.layer1.0.conv1.weight",module.FeatureExtraction.ConvNet.layer1.0.bn1.weight",module.FeatureExtraction.ConvNet.layer1.0.bn1.bias",module.FeatureExtraction.ConvNet.layer1.0.bn1.running_mean",module.FeatureExtraction.ConvNet.layer1.0.bn1.running_var",module.FeatureExtraction.ConvNet.layer1.0.bn1.num_batches_tracked",module.FeatureExtraction.ConvNet.layer1.0.conv2.weight",module.FeatureExtraction.ConvNet.layer1.0.bn2.weight",module.FeatureExtraction.ConvNet.layer1.0.bn2.bias",module.FeatureExtraction.ConvNet.layer1.0.bn2.running_mean",module.FeatureExtraction.ConvNet.layer1.0.bn2.running_var",module.FeatureExtraction.ConvNet.layer1.0.bn2.num_batches_tracked",module.FeatureExtraction.ConvNet.layer1.0.downsample.0.weight",module.FeatureExtraction.ConvNet.layer1.0.downsample.1.weight",module.FeatureExtraction.ConvNet.layer1.0.downsample.1.bias",module.FeatureExtraction.ConvNet.layer1.0.downsample.1.running_mean",module.FeatureExtraction.ConvNet.layer1.0.downsample.1.running_var",module.FeatureExtraction.ConvNet.layer1.0.downsample.1.num_batches_tracked",module.FeatureExtraction.ConvNet.conv1.weight",module.FeatureExtraction.ConvNet.bn1.weight",module.FeatureExtraction.ConvNet.bn1.bias",module.FeatureExtraction.ConvNet.bn1.running_mean",module.FeatureExtraction.ConvNet.bn1.running_var",module.FeatureExtraction.ConvNet.bn1.num_batches_tracked",module.FeatureExtraction.ConvNet.layer2.0.conv1.weight",module.FeatureExtraction.ConvNet.layer2.0.bn1.weight",module.FeatureExtraction.ConvNet.layer2.0.bn1.bias",module.FeatureExtraction.ConvNet.layer2.0.bn1.running_mean",module.FeatureExtraction.ConvNet.layer2.0.bn1.running_var",module.FeatureExtraction.ConvNet.layer2.0.bn1.num_batches_tracked",module.FeatureExtraction.ConvNet.layer2.0.conv2.weight",module.FeatureExtraction.ConvNet.layer2.0.bn2.weight",module.FeatureExtraction.ConvNet.layer2.0.bn2.bias",module.FeatureExtraction.ConvNet.layer2.0.bn2.running_mean",module.FeatureExtraction.ConvNet.layer2.0.bn2.running_var",module.FeatureExtraction.ConvNet.layer2.0.bn2.num_batches_tracked",module.FeatureExtraction.ConvNet.layer2.0.downsample.0.weight",module.FeatureExtraction.ConvNet.layer2.0.downsample.1.weight",module.FeatureExtraction.ConvNet.layer2.0.downsample.1.bias",module.FeatureExtraction.ConvNet.layer2.0.downsample.1.running_mean",module.FeatureExtraction.ConvNet.layer2.0.downsample.1.running_var",module.FeatureExtraction.ConvNet.layer2.0.downsample.1.num_batches_tracked",module.FeatureExtraction.ConvNet.layer2.1.conv1.weight",module.FeatureExtraction.ConvNet.layer2.1.bn1.weight",module.FeatureExtraction.ConvNet.layer2.1.bn1.bias",module.FeatureExtraction.ConvNet.layer2.1.bn1.running_mean",module.FeatureExtraction.ConvNet.layer2.1.bn1.running_var",module.FeatureExtraction.ConvNet.layer2.1.bn1.num_batches_tracked",module.FeatureExtraction.ConvNet.layer2.1.conv2.weight",module.FeatureExtraction.ConvNet.layer2.1.bn2.weight",module.FeatureExtraction.ConvNet.layer2.1.bn2.bias",module.FeatureExtraction.ConvNet.layer2.1.bn2.running_mean",module.FeatureExtraction.ConvNet.layer2.1.bn2.running_var",module.FeatureExtraction.ConvNet.layer2.1.bn2.num_batches_tracked",module.FeatureExtraction.ConvNet.conv2.weight",module.FeatureExtraction.ConvNet.bn2.weight",module.FeatureExtraction.ConvNet.bn2.bias",module.FeatureExtraction.ConvNet.bn2.running_mean",module.FeatureExtraction.ConvNet.bn2.running_var",module.FeatureExtraction.ConvNet.bn2.num_batches_tracked",module.FeatureExtraction.ConvNet.layer3.0.conv1.weight",module.FeatureExtraction.ConvNet.layer3.0.bn1.weight",module.FeatureExtraction.ConvNet.layer3.0.bn1.bias",module.FeatureExtraction.ConvNet.layer3.0.bn1.running_mean",module.FeatureExtraction.ConvNet.layer3.0.bn1.running_var",module.FeatureExtraction.ConvNet.layer3.0.bn1.num_batches_tracked",module.FeatureExtraction.ConvNet.layer3.0.conv2.weight",module.FeatureExtraction.ConvNet.layer3.0.bn2.weight",module.FeatureExtraction.ConvNet.layer3.0.bn2.bias",module.FeatureExtraction.ConvNet.layer3.0.bn2.running_mean",module.FeatureExtraction.ConvNet.layer3.0.bn2.running_var",module.FeatureExtraction.ConvNet.layer3.0.bn2.num_batches_tracked",module.FeatureExtraction.ConvNet.layer3.0.downsample.0.weight",module.FeatureExtraction.ConvNet.layer3.0.downsample.1.weight",module.FeatureExtraction.ConvNet.layer3.0.downsample.1.bias",module.FeatureExtraction.ConvNet.layer3.0.downsample.1.running_mean",module.FeatureExtraction.ConvNet.layer3.0.downsample.1.running_var",module.FeatureExtraction.ConvNet.layer3.0.downsample.1.num_batches_tracked",module.FeatureExtraction.ConvNet.layer3.1.conv1.weight",module.FeatureExtraction.ConvNet.layer3.1.bn1.weight",module.FeatureExtraction.ConvNet.layer3.1.bn1.bias",module.FeatureExtraction.ConvNet.layer3.1.bn1.running_mean",module.FeatureExtraction.ConvNet.layer3.1.bn1.running_var",module.FeatureExtraction.ConvNet.layer3.1.bn1.num_batches_tracked",module.FeatureExtraction.ConvNet.layer3.1.conv2.weight",module.FeatureExtraction.ConvNet.layer3.1.bn2.weight",module.FeatureExtraction.ConvNet.layer3.1.bn2.bias",module.FeatureExtraction.ConvNet.layer3.1.bn2.running_mean",module.FeatureExtraction.ConvNet.layer3.1.bn2.running_var",module.FeatureExtraction.ConvNet.layer3.1.bn2.num_batches_tracked",module.FeatureExtraction.ConvNet.layer3.2.conv1.weight",module.FeatureExtraction.ConvNet.layer3.2.bn1.weight",module.FeatureExtraction.ConvNet.layer3.2.bn1.bias",module.FeatureExtraction.ConvNet.layer3.2.bn1.running_mean",module.FeatureExtraction.ConvNet.layer3.2.bn1.running_var",module.FeatureExtraction.ConvNet.layer3.2.bn1.num_batches_tracked",module.FeatureExtraction.ConvNet.layer3.2.conv2.weight",module.FeatureExtraction.ConvNet.layer3.2.bn2.weight",module.FeatureExtraction.ConvNet.layer3.2.bn2.bias",module.FeatureExtraction.ConvNet.layer3.2.bn2.running_mean",module.FeatureExtraction.ConvNet.layer3.2.bn2.running_var",module.FeatureExtraction.ConvNet.layer3.2.bn2.num_batches_tracked",module.FeatureExtraction.ConvNet.layer3.3.conv1.weight",module.FeatureExtraction.ConvNet.layer3.3.bn1.weight",module.FeatureExtraction.ConvNet.layer3.3.bn1.bias",module.FeatureExtraction.ConvNet.layer3.3.bn1.running_mean",module.FeatureExtraction.ConvNet.layer3.3.bn1.running_var",module.FeatureExtraction.ConvNet.layer3.3.bn1.num_batches_tracked",module.FeatureExtraction.ConvNet.layer3.3.conv2.weight",module.FeatureExtraction.ConvNet.layer3.3.bn2.weight",module.FeatureExtraction.ConvNet.layer3.3.bn2.bias",module.FeatureExtraction.ConvNet.layer3.3.bn2.running_mean",module.FeatureExtraction.ConvNet.layer3.3.bn2.running_var",module.FeatureExtraction.ConvNet.layer3.3.bn2.num_batches_tracked",module.FeatureExtraction.ConvNet.layer3.4.conv1.weight",module.FeatureExtraction.ConvNet.layer3.4.bn1.weight",module.FeatureExtraction.ConvNet.layer3.4.bn1.bias",module.FeatureExtraction.ConvNet.layer3.4.bn1.running_mean",module.FeatureExtraction.ConvNet.layer3.4.bn1.running_var",module.FeatureExtraction.ConvNet.layer3.4.bn1.num_batches_tracked",module.FeatureExtraction.ConvNet.layer3.4.conv2.weight",module.FeatureExtraction.ConvNet.layer3.4.bn2.weight",module.FeatureExtraction.ConvNet.layer3.4.bn2.bias",module.FeatureExtraction.ConvNet.layer3.4.bn2.running_mean",module.FeatureExtraction.ConvNet.layer3.4.bn2.running_var",module.FeatureExtraction.ConvNet.layer3.4.bn2.num_batches_tracked",module.FeatureExtraction.ConvNet.conv3.weight",module.FeatureExtraction.ConvNet.bn3.weight",module.FeatureExtraction.ConvNet.bn3.bias",module.FeatureExtraction.ConvNet.bn3.running_mean",module.FeatureExtraction.ConvNet.bn3.running_var",module.FeatureExtraction.ConvNet.bn3.num_batches_tracked",module.FeatureExtraction.ConvNet.layer4.0.conv1.weight",module.FeatureExtraction.ConvNet.layer4.0.bn1.weight",module.FeatureExtraction.ConvNet.layer4.0.bn1.bias",module.FeatureExtraction.ConvNet.layer4.0.bn1.running_mean",module.FeatureExtraction.ConvNet.layer4.0.bn1.running_var",module.FeatureExtraction.ConvNet.layer4.0.bn1.num_batches_tracked",module.FeatureExtraction.ConvNet.layer4.0.conv2.weight",module.FeatureExtraction.ConvNet.layer4.0.bn2.weight",module.FeatureExtraction.ConvNet.layer4.0.bn2.bias",module.FeatureExtraction.ConvNet.layer4.0.bn2.running_mean",module.FeatureExtraction.ConvNet.layer4.0.bn2.running_var",module.FeatureExtraction.ConvNet.layer4.0.bn2.num_batches_tracked",module.FeatureExtraction.ConvNet.layer4.1.conv1.weight",module.FeatureExtraction.ConvNet.layer4.1.bn1.weight",module.FeatureExtraction.ConvNet.layer4.1.bn1.bias",module.FeatureExtraction.ConvNet.layer4.1.bn1.running_mean",module.FeatureExtraction.ConvNet.layer4.1.bn1.running_var",module.FeatureExtraction.ConvNet.layer4.1.bn1.num_batches_tracked",module.FeatureExtraction.ConvNet.layer4.1.conv2.weight",module.FeatureExtraction.ConvNet.layer4.1.bn2.weight",module.FeatureExtraction.ConvNet.layer4.1.bn2.bias",module.FeatureExtraction.ConvNet.layer4.1.bn2.running_mean",module.FeatureExtraction.ConvNet.layer4.1.bn2.running_var",module.FeatureExtraction.ConvNet.layer4.1.bn2.num_batches_tracked",module.FeatureExtraction.ConvNet.layer4.2.conv1.weight",module.FeatureExtraction.ConvNet.layer4.2.bn1.weight",module.FeatureExtraction.ConvNet.layer4.2.bn1.bias",module.FeatureExtraction.ConvNet.layer4.2.bn1.running_mean",module.FeatureExtraction.ConvNet.layer4.2.bn1.running_var",module.FeatureExtraction.ConvNet.layer4.2.bn1.num_batches_tracked",module.FeatureExtraction.ConvNet.layer4.2.conv2.weight",module.FeatureExtraction.ConvNet.layer4.2.bn2.weight",module.FeatureExtraction.ConvNet.layer4.2.bn2.bias",module.FeatureExtraction.ConvNet.layer4.2.bn2.running_mean",module.FeatureExtraction.ConvNet.layer4.2.bn2.running_var",module.FeatureExtraction.ConvNet.layer4.2.bn2.num_batches_tracked",module.FeatureExtraction.ConvNet.conv4_1.weight",module.FeatureExtraction.ConvNet.bn4_1.weight",module.FeatureExtraction.ConvNet.bn4_1.bias",module.FeatureExtraction.ConvNet.bn4_1.running_mean",module.FeatureExtraction.ConvNet.bn4_1.running_var",module.FeatureExtraction.ConvNet.bn4_1.num_batches_tracked",module.FeatureExtraction.ConvNet.conv4_2.weight",module.FeatureExtraction.ConvNet.bn4_2.weight",module.FeatureExtraction.ConvNet.bn4_2.bias",module.FeatureExtraction.ConvNet.bn4_2.running_mean",module.FeatureExtraction.ConvNet.bn4_2.running_var",module.FeatureExtraction.ConvNet.bn4_2.num_batches_tracked",module.SequenceModeling.0.rnn.weight_ih_l0",module.SequenceModeling.0.rnn.weight_hh_l0",module.SequenceModeling.0.rnn.bias_ih_l0",module.SequenceModeling.0.rnn.bias_hh_l0",module.SequenceModeling.0.rnn.weight_ih_l0_reverse",module.SequenceModeling.0.rnn.weight_hh_l0_reverse",module.SequenceModeling.0.rnn.bias_ih_l0_reverse",module.SequenceModeling.0.rnn.bias_hh_l0_reverse",module.SequenceModeling.0.linear.weight",module.SequenceModeling.0.linear.bias",module.SequenceModeling.1.rnn.weight_ih_l0",module.SequenceModeling.1.rnn.weight_hh_l0",module.SequenceModeling.1.rnn.bias_ih_l0",module.SequenceModeling.1.rnn.bias_hh_l0",module.SequenceModeling.1.rnn.weight_ih_l0_reverse",module.SequenceModeling.1.rnn.weight_hh_l0_reverse",module.SequenceModeling.1.rnn.bias_ih_l0_reverse",module.SequenceModeling.1.rnn.bias_hh_l0_reverse",module.SequenceModeling.1.linear.weight",module.SequenceModeling.1.linear.bias",module.Prediction.attention_cell.i2h.weight",module.Prediction.attention_cell.h2h.weight",module.Prediction.attention_cell.h2h.bias",module.Prediction.attention_cell.score.weight",module.Prediction.attention_cell.rnn.weight_ih",module.Prediction.attention_cell.rnn.weight_hh",module.Prediction.attention_cell.rnn.bias_ih",module.Prediction.attention_cell.rnn.bias_hh",module.Prediction.generator.weight",module.Prediction.generator.bias".

Here is the catch

Independently Both Module A and Module B runs without a problem in the same Py3 env
I used Python shell and imported Module B after loading Module A and it loads just fine.

Any clue on whats going on?

For reference

Module B (code where it fails)

    def pre_load_model(self):

            print("preloading the model with opts "+str(self.opts))

            self.trmodel = Model(self.opts)

            self.trmodel = torch.nn.DataParallel(self.trmodel)
            if torch.cuda.is_available():
                self.trmodel = self.trmodel.cuda()
                self.device = torch.device('cuda:0')
            else:
                self.device = torch.device('cpu')

            # load model
            print('loading pretrained model from %s' % self.opts.saved_model)
            if torch.cuda.is_available():
                self.trmodel.load_state_dict(torch.load(self.opts.saved_model))
            else:
                self.trmodel.load_state_dict(torch.load(self.opts.saved_model, map_location='cpu'))


title: Pytorch ValueError: Target and input must have the same number of elements after change Image size

body:
i have a working peace of code, which takes a Batchsize from 32 Image with the shape of 256*256 and i can train my neuronal network.

class Netz(nn.Module):
def __init__(self):
    super(Netz,self).__init__()
    self.conv1 = nn.Conv2d(3, 6, kernel_size=5)
    self.conv2 = nn.Conv2d(6, 12, kernel_size=3)
    self.conv3 = nn.Conv2d(12, 18, kernel_size=3)
    self.conv4 = nn.Conv2d(18, 24, kernel_size=3)
    self.fc1 = nn.Linear(4704, 1000)
    self.fc2 = nn.Linear(1000, 350)
    self.fc3 = nn.Linear(350,43)

def forward (self,x):
    x = F.relu(F.max_pool2d(self.conv1(x), 2))
    x = F.relu(F.max_pool2d(self.conv2(x), 2))
    x = F.relu(F.max_pool2d(self.conv3(x), 2))
    x = F.relu(F.max_pool2d(self.conv4(x), 2))
    x = x.view(-1,4704)
    x = F.relu(self.fc1(x))
    x = F.relu(self.fc2(x))
    x = self.fc3(x)
    return torch.sigmoid(x)

    # Traningsalgorithmus
optimizer = optim.Adam(model.parameters(), lr=0.001)
def train(epoch):
    model.train()
    batch_id = 0
    for data, target in train_data_set:
        data = Variable(data)
        target = torch.Tensor(target)
        target = Variable(target)
        optimizer.zero_grad()
        out = model(data)
        criterion = F.binary_cross_entropy
        loss = criterion(out,target)
        loss.backward()
        optimizer.step()
        print ('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(
            epoch, batch_id * len(data), len(train_data_set)*32,
                100. * batch_id / len(train_data_set), loss.item()))
        batch_id = batch_id + 1

When i change the size of the Image to 50*50 and i change the code Net like this:

class Netz(nn.Module):
def __init__(self):
    super(Netz,self).__init__()
    self.conv1 = nn.Conv2d(3, 6, kernel_size=5)
    self.conv2 = nn.Conv2d(6, 12, kernel_size=3)
    self.conv3 = nn.Conv2d(12, 18, kernel_size=3)
    self.conv4 = nn.Conv2d(18, 24, kernel_size=3)
    self.fc1 = nn.Linear(768, 1000)
    self.fc2 = nn.Linear(1000, 350)
    self.fc3 = nn.Linear(350,43)

def forward (self,x):
    x = F.relu(F.max_pool2d(self.conv1(x), 2))
    x = F.relu(F.max_pool2d(self.conv2(x), 2))
    x = F.relu(F.max_pool2d(self.conv3(x), 2))
    x = F.relu(F.max_pool2d(self.conv4(x), 2))
    x = x.view(-1,768)
    x = F.relu(self.fc1(x))
    x = F.relu(self.fc2(x))
    x = self.fc3(x)
    return torch.sigmoid(x)

I gonne get the error:
ValueError: Target and input must have the same number of elements. target nelement (1376) != input nelement (43)

So far i see it the problem comes after the x = x.view(-1,768) it returns an Tensor with the torch.Size ([1,768]). When i use the Image Size 256*256 it returns Tensor with torch.Size ([32,4704]) and i don't get the error.

Does someone know how i can fix me problem ?


title: Create a tensor view (no data copy) where one row/col is removed

body:
TL;DR: Is there a way to delete one row or column of a torch tensor's view without creating a copy of the underlying data?
Background:
Starting from a base tensor (samples, features) of shape (2000,10000) (about 90MB), I want to create a 2-tuple of new views for each feature that consist of the one-feature-training-target with shape (2000,1) and the training inputs with shape (2000,9999) containing the remaining features.
As I want to train one rather simple model for each feature (10k) and as many as possible in parallel, I would like to have this data available in parallel, but my memory does not allow for this unless the input tensors are views.
While slicing creates views and can easily be used for the targets, so far I haven't found a slicing or function that can be used for the inputs. Creating two slices and concatenating them again creates a copy. The same happens when I use a mask to remove the unwanted feature.


title: RNN Language Model in PyTorch predicting the same three words repeatedly

body:
I am attempting to create a word-level language model using an RNN in PyTorch. Whenever I am training the loss stays about the same for the whole training set and when I try to sample a new sentence the same three words are predicted in the same order. For example in my most recent attempt the RNN predicted 'the' then 'same' then 'of' and that sequence just kept repeating. I have tried changing the how I've set up the RNN including using LSTM's, GRU's, and different embeddings but so far nothing has worked.
The way that I am training the RNN is by taking a sentence of 50 words, and selecting a progressively larger part of the sentence with the next word being the target. At the end of the sentence, I have an EOS tag. I am using text from Republic by Plato as my training set and embedding it using a pytorch embedding layer. I am then feeding it into an LSTM and then a linear layer to get the right shape. I am not sure if the problem is in the RNN, the data, the training or what so any help would be greatly appreciated.
If anyone has any experience in nlp or in language modeling I would greatly appreciate any help you could offer for this fixing this problem. My end goal is simply to just be able to generate a sentence. Thanks in advance!
Here is my RNN
class LanguageModel(nn.Module):
  """
    Class that defines the reccurent neural network.

    Methods
    -------
    forward(input, h, c)
      Forward propogation through the RNN.
    initHidden()
      Initializes the hidden and cell states.
  """
  def __init__(self, vocabSize, seqLen = 51, embeddingDim = 30, hiddenSize = 32, numLayers = 1, bid = False):
    """
      Initializes the class

      Parameters
      ----------
      seqLen : int, optional
        The length of the input sequence.
      embeddingDim : int, optional
        The dimension that the embedding dimension for the encoder should be.
      vocabSize : int
        The length of the vocab dictionary.
      hiddenSize : int, optional
        The size that the hidden state should be.
      numLayers : int, optional
        The number of LSTM Layers.
      bid : bool, optional
        Whether the RNN should be bidirctional or not.
    """

    super(LanguageModel, self).__init__()
    self.hiddenSize = hiddenSize
    self.numLayers = numLayers

    # Set value of numDirections based on whether or not the RNN is bidirectional.
    if bid == True:
      self.numDirections = 2
    else:
      self.numDirections = 1

    self.encoder = nn.Embedding(vocabSize, embeddingDim)
    self.LSTM = nn.LSTM(input_size = embeddingDim, hidden_size = hiddenSize, num_layers = numLayers, bidirectional = bid)
    self.decoder = nn.Linear(seqLen * self.numDirections * hiddenSize, vocabSize)

  def forward(self, input, h, c):
    """
      Forward propogates through the RNN

      Parameters
      ----------
      input : torch.Tensor
        Input to RNN. Should be formatter using makeInput() and padSeq().
      h : torch.Tensor
        Hidden state.
      c : torch.Tensor
        Cell state.

      Returns
      -------
      torch.Tensor
        Log probabilities for the predicted word from the RNN.
    """

    emb = self.encoder(input)
    emb.unsqueeze_(1) # Add in the batch dimension so the shape is right for the LSTM

    out, (h, c) = self.LSTM(emb, (h, c))
    out = out.view(1, -1) # Reshaping to fit into the loss function.

    out = self.decoder(out)

    logProbs = F.log_softmax(out, dim = 1)

    return logProbs

  def initHidden(self):
    """
      Initializes the hidden and cell states.

      Returns
      -------
      torch.Tensor
        Tensor containing the initial hidden state.
      torch.Tensor
        Tensor containing the intial cell state.
    """
    h = torch.zeros(self.numLayers * self.numDirections, 1, self.hiddenSize)
    c = torch.zeros(self.numLayers * self.numDirections, 1, self.hiddenSize)

    return h, c

here is how I create my input and targets
def makeInput(sentence):
  """
    Prepares a sentence for input to the RNN.

    Parameters
    ----------
    sentence : list
      The sentence to be converted into input. Should be of form: [str]

    Returns
    -------
    torch.Tensor
      Tensor of the indices for each word in the input sentence.
  """

  sen = sentence[0].split() # Split the list into individual words
  sen.insert(0, 'START')

  input = [word2Idx[word] for word in sen] # Iterate over the words in sentence and convert to indices

  return torch.tensor(input)

def makeTarget(sentence):
  """
    Prepares a sentence to be a target.

    Parameters
    ----------
    sentence : str
      The sentence to be made into a target. Should be of form: [str]

    Returns
    -------
    torch.Tensor
      Tensor of the indices for the target phrase including the <EOS> tag.
  """

  sen = sentence[0].split() # Split the list into individual words
  sen.append('EOS')

  target = [word2Idx[word] for word in sen]
  target = torch.tensor(target, dtype = torch.long)

  return target.unsqueeze_(-1) # Removing dimension for loss function

def padSeq(seq, refSeq):
  """
    Pads a sequence to be the same shape as another sequence.

    Parameters
    ----------
    seq : torch.Tensor
      The sequence to pad.
    refSeq : torch.Tensor
      The reference sequence. seq will be padded to be the same shape as refSeq.

    Returns
    -------
    torch.Tensor
      Tensor containing the padded sequence.
  """

  padded = pad_sequence([refSeq, seq])
  tmp = torch.t(padded) # Transpose the padded sequence for easier indexing on return

  return tmp[1] # Return only the padded seq not both sequences

and here is my training loop
def train():
  """
    Trains the model.
  """

  start = time.time()
  for i, data in enumerate(trainLoader):
    inputTensor = makeInput(data)
    targetTensor = makeTarget(data)

    targetTensor = targetTensor.to(device)

    h, c = model.initHidden()
    h = h.to(device)
    c = c.to(device)

    optimizer.zero_grad()
    loss = 0

    for x in range(inputTensor.size(0)): # Iterate over all of the words in the input sentence
      """ Preparing input for the rnn """
      input = inputTensor[: x + 1] # We only want part of the input so the RNN can learn on predicting the next words
      input = padSeq(input, inputTensor)
      input = input.to(device)

      out = model(input, h, c)
      l = criterion(out, targetTensor[x])
      loss += l

    loss.backward()
    optimizer.step()

    if i % 250 == 0: # Print updates to the models loss every 10 iters.
      print('[{}] Epoch: {} -> {}'.format(timeSince(start), i, loss / inputTensor.size(0)))


title: Can you integrate opencv SIFT with a tensorflow model?

body:
I am trying to create a CNN, but using the SIFT algorithm instead of any pooling layers.
Problem is I can't seem to find any Python implementation of the algorithm in Tensorflow or PyTorch. The only implementation I have seen of it is with opencv.
Is it possible to use the opencv SIFT implementation as a layer in a Tensorflow CNN Model?
If so, how would you go about creating it?


title: Is the gradient of the sum equal to the sum of the gradients for a neural network in pytorch?

body:
Let's suppose I have the code below and I want to calculate the jacobian of L, which is the prediction made by a neural network in Pytorch, L is of size nx1 where n is the number of samples in a mini batch. In order to avoid a for loop for each entry of L (n entries) to calculate the jacobian for each sample in the mini batch some codes I found just sum the n predictions of the neural network (L) with respect with the inputs and then calculate the gradient of the sum. First I can't understand why is the gradient of the sum the same of the sum of the gradients for each sample in pytorch architecture. Second I tried both with the sum and with a for loop and the results diverge. Could it be due to numerical approximations or because the sum just doesn't make sense?
The code is below, where both functions belong to a nn.module:
def forward(self, x):
        with torch.set_grad_enabled(True):
            def function(x,t):
                 self.n = n = x.shape[1]//2

                 qqd = x.requires_grad_(True)
                 L = self._lagrangian(qqd).sum()
                 J = grad(L, qqd, create_graph=True)[0]

def _lagrangian(self, qqd):
    x = F.softplus(self.fc1(qqd))
    x = F.softplus(self.fc2(x))
    x = F.softplus(self.fc3(x))
    L = self.fc_last(x)
    return L


title: How to monitor GPU memory usage when training a DNN?

body: I give a result example. I want to ask how to get the data like this graph.


title: what is Pytorch's add_module()?

body:
I stumbled upon the method add_module() in a Pytorch model.
The doc only states

Adds a child module to the current module.
The module can be accessed as an attribute using the given name.

I don't understand what "adding a child module" means.
How is it different from just setting a pointer to the other module using self._other module = other_module?
What are the nuances?


title: PyTorch - AssertionError: Size mismatch between tensors

body:
I am trying to adapt a Pytorch script that was created for linear regression. It was originally written to take in a set of random values(created with np.random) as features and targets.
I have now created a dataframe of actual data for analysis:
df = pd.read_csv('file_name.csv')

The df looks like this:
      X1     X2     X3     X4    X5   X6   X7  X8    Y1     Y2
0    0.98  514.5  294.0  110.25  7.0   2  0.0   0  15.55  21.33
1    0.98  514.5  294.0  110.25  7.0   3  0.0   0  15.55  21.33
2    0.98  514.5  294.0  110.25  7.0   4  0.0   0  15.55  21.33
3    0.98  514.5  294.0  110.25  7.0   5  0.0   0  15.55  21.33
4    0.90  563.5  318.5  122.50  7.0   2  0.0   0  20.84  28.28

...and I am currently extracting just two columns(X1 and X2) as my features, and one column(Y1) as my targets, like this:
x = df[['X1', 'X2']]
y = df['Y1']

So features look like this:
      X1     X2
0    0.98  514.5
1    0.98  514.5
2    0.98  514.5
3    0.98  514.5
4    0.90  563.5

and targets look like this:
        Y1
0      15.55
1      15.55
2      15.55
3      15.55
4      20.84

However, when I attempt to convert the features (X1 and X1) and targets(Y1) to tensors, in order to feed them to the NN, the code fails at the line:
dataset = TensorDataset(x_tensor_flat, y_tensor_flat)
I get the error:
line 45, in <module> dataset = TensorDataset(x_tensor, y_tensor)
AssertionError: Size mismatch between tensors

There's clearly some shaping issue at play, but I can't work out what.
I have tried to flatten as well as transposing the tensors, but I get the same error.
Any help would be hugely appreciated.
Here's the full section of code that is causing the issue:
import pandas as pd
import torch
import torch.optim as optim
import torch.nn as nn
from torch.utils.data import Dataset, TensorDataset, DataLoader
from torch.utils.data.dataset import random_split

device = 'cuda' if torch.cuda.is_available() else 'cpu'

df = pd.read_csv('file_name.csv')
x = df[['X1', 'X2']]
y = df['Y1']

x_tensor = torch.from_numpy(np.array(x)).float()
y_tensor = torch.from_numpy(np.array(y)).float()

train_loader = DataLoader(dataset=train_dataset, batch_size=10)
val_loader = DataLoader(dataset=val_dataset, batch_size=10)

class ManualLinearRegression(nn.Module):
    def __init__(self):
        super().__init__()
        self.linear = nn.Linear(2, 1)

    def forward(self, x):
        return self.linear(x)

def make_train_step(model, loss_fn, optimizer):
    def train_step(x, y):
        model.train()
        yhat = model(x)
        loss = loss_fn(y, yhat)
        loss.backward()
        optimizer.step()
        optimizer.zero_grad()
        return loss.item()
    return train_step

torch.manual_seed(42)

model = ManualLinearRegression().to(device)
loss_fn = nn.MSELoss(reduction='mean')
optimizer = optim.SGD(model.parameters(), lr=1e-1)
train_step = make_train_step(model, loss_fn, optimizer)

n_epochs = 50
training_losses = []
validation_losses = []
print(model.state_dict())

for epoch in range(n_epochs):
    batch_losses = []
    for x_batch, y_batch in train_loader:
        x_batch = x_batch.to(device)
        y_batch = y_batch.to(device)
        loss = train_step(x_batch, y_batch)
        batch_losses.append(loss)
    training_loss = np.mean(batch_losses)
    training_losses.append(training_loss)

    with torch.no_grad():
        val_losses = []
        for x_val, y_val in val_loader:
            x_val = x_val.to(device)
            y_val = y_val.to(device)
            model.eval()
            yhat = model(x_val)
            val_loss = loss_fn(y_val, yhat).item()
            val_losses.append(val_loss)
        validation_loss = np.mean(val_losses)
        validation_losses.append(validation_loss)

    print(f"[{epoch+1}] Training loss: {training_loss:.3f}\t Validation loss: {validation_loss:.3f}")

print(model.state_dict())


title: Calling overridden parent class method in parent class

body:
I'm trying to override two parent class functions train and eval in a ChildClass. In the parent class, eval() basically calls train(). However, I realize that when I write my code as below, eval() in the parent class is trying to call the function train() in ChildClass - I would like eval() in the parent class to call train() in the parent class instead.

I'm just wondering if there is any clean solutions to make changes to ChildClass that would allow the parent class to call the parent train() function?

class ChildClass(nn.Module):
    def __init__(self):
        super(ChildClass, self).__init__()

    def train(self):
        super(ChildClass, self).train()

    def eval(self):
        super(ChildClass, self).eval()

Parent class is in a Python Package (pytorch), so no changes should be made:

class Module(object):
    #...

    def train(self, mode=True):
        # ...
        return self

    def eval(self):
        return self.train(False)


title: GPU out of memory when initializing network

body:
I am trying to initialize a CNN and then put it on my GPU for training. When I put it on GPU I get the error: (CUDA error: out of memory). I have run similar networks with no such problems. This is the only thing in cuda as I have not loaded any images as of yet. Any ideas as to what is going wrong?

I am using pytorch version 0.4.1 on a GTX 1070ti 8GB.

| NVIDIA-SMI 410.104      Driver Version: 410.104      CUDA Version: 10.0     |
|-------------------------------+----------------------+----------------------+
| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |
| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |
|===============================+======================+======================|
|   0  GeForce GTX 107...  Off  | 00000000:01:00.0  On |                  N/A |
|  0%   43C    P2    39W / 180W |   8024MiB /  8111MiB |      0%      Default |
+-------------------------------+----------------------+----------------------+

+-----------------------------------------------------------------------------+
| Processes:                                                       GPU Memory |
|  GPU       PID   Type   Process name                             Usage      |
|=============================================================================|
|    0      1129      G   /usr/lib/xorg/Xorg                            36MiB |
|    0      1164      G   /usr/bin/gnome-shell                          57MiB |
|    0      1415      G   /usr/lib/xorg/Xorg                           200MiB |
|    0      1548      G   /usr/bin/gnome-shell                          90MiB |
|    0      6323      C   /usr/bin/python3                             525MiB |
|    0      9521      C   /usr/bin/python3                            1827MiB |
|    0     18821      C   /usr/bin/python3                            4883MiB |
|    0     27137      G   ...uest-channel-token=16389326112703159917    45MiB |
|    0     29161      C   /usr/bin/python3                             355MiB |

I have tried reducing the size of the linear layers with no luck.

net = piccnn()
net.to(device)


title: How to fix error while using DataParallel in GRU's network

body:
I'm trying to do dataparallel into GRU's network as explained in the docs and I keep getting the same error

"""Defines the neural network, losss function and metrics"""

import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F

class Net(nn.Module):
    def __init__(self, params, anchor_is_phrase):
        """
        Simple LSTM, used to generate the LSTM for both the word and video
        embeddings.
        Args:
            params: (Params) contains vocab_size, embedding_dim, lstm_hidden_dim
            is_phrase: is word lstm or the vid lstm
        """
        super(Net, self).__init__()
        if anchor_is_phrase:
            self.lstm = nn.DataParallel(nn.GRU(params.word_embedding_dim, params.hidden_dim, 1)).cuda()#, batch_first=True)
        else:
            self.lstm = nn.DataParallel(nn.GRU(params.vid_embedding_dim, params.hidden_dim, 1)).cuda() #, batch_first=True)

    def forward(self, s, anchor_is_phrase = False):
        """
        Forward prop.
        """
        s, _ = self.lstm(s)
        s.data.contiguous()
        return s

The error happens at line  s, _ = self.lstm(s) in the previous code :

 here:  s, _ = self.lstm(s)
        s.data.contiguous()
        return s

I get the following error message :

    s, _ = self.lstm(s)
  File "/home/pavelameen/miniconda3/envs/TD2/lib/python3.6/site-packages/torch/nn/modules/module.py", line 493, in __call__
    result = self.forward(*input, **kwargs)
  File "/home/pavelameen/miniconda3/envs/TD2/lib/python3.6/site-packages/torch/nn/parallel/data_parallel.py", line 152, in forward
    outputs = self.parallel_apply(replicas, inputs, kwargs)
  File "/home/pavelameen/miniconda3/envs/TD2/lib/python3.6/site-packages/torch/nn/parallel/data_parallel.py", line 162, in parallel_apply
    return parallel_apply(replicas, inputs, kwargs, self.device_ids[:len(replicas)])
  File "/home/pavelameen/miniconda3/envs/TD2/lib/python3.6/site-packages/torch/nn/parallel/parallel_apply.py", line 83, in parallel_apply
    raise output
  File "/home/pavelameen/miniconda3/envs/TD2/lib/python3.6/site-packages/torch/nn/parallel/parallel_apply.py", line 59, in _worker
    output = module(*input, **kwargs)
  File "/home/pavelameen/miniconda3/envs/TD2/lib/python3.6/site-packages/torch/nn/modules/module.py", line 493, in __call__
    result = self.forward(*input, **kwargs)
  File "/home/pavelameen/miniconda3/envs/TD2/lib/python3.6/site-packages/torch/nn/modules/rnn.py", line 193, in forward
    max_batch_size = input.size(0) if self.batch_first else input.size(1)
AttributeError: 'tuple' object has no attribute 'size'

the interesting part is i try to output the type of s in line 27 and i get PackedSequence , why it convert to tuple in lstm forward method?


I'm experimenting with argumentation mining with BERT and try to identify argument components (BIO-classification task). I closely followed the code from this article, Named Entity Recognition With Bert, but adapted it to fit my data. I run the code on colab (hardware accelerator is set to GPU).

Does anybody know a solution to my problem? Thanks in advance!

epochs = 5
max_grad_norm = 1.0

for _ in trange(epochs, desc="Epoch"):
    # TRAIN loop
    model.train()
    tr_loss = 0
    nb_tr_examples, nb_tr_steps = 0, 0
    for step, batch in enumerate(train_dataloader):
        # add batch to gpu
        batch = tuple(t.to(device) for t in batch)
        b_input_ids, b_input_mask, b_labels = batch
        # forward pass
        loss = model(b_input_ids, token_type_ids=None, attention_mask=b_input_mask, labels=b_labels)
        # backward pass
        loss.backward()
        # track train loss
        tr_loss += loss.item()
        nb_tr_examples += b_input_ids.size(0)
        nb_tr_steps += 1
        # gradient clipping
        torch.nn.utils.clip_grad_norm_(parameters=model.parameters(), max_norm=max_grad_norm)
        # update parameters
        optimizer.step()
        model.zero_grad()
    # print train loss per epoch
    print("Train loss: {}".format(tr_loss/nb_tr_steps))
    # VALIDATION on validation set
    model.eval()
    eval_loss, eval_accuracy = 0, 0
    nb_eval_steps, nb_eval_examples = 0, 0
    predictions , true_labels = [], []
    for batch in valid_dataloader:
        batch = tuple(t.to(device) for t in batch)
        b_input_ids, b_input_mask, b_labels = batch

        with torch.no_grad():
            tmp_eval_loss = model(b_input_ids, token_type_ids=None,
                                  attention_mask=b_input_mask, labels=b_labels)
            logits = model(b_input_ids, token_type_ids=None,
                           attention_mask=b_input_mask)
        logits = logits.detach().cpu().numpy()
        label_ids = b_labels.to('cpu').numpy()
        predictions.extend([list(p) for p in np.argmax(logits, axis=2)])
        true_labels.append(label_ids)

        tmp_eval_accuracy = flat_accuracy(logits, label_ids)

        eval_loss += tmp_eval_loss.mean().item()
        eval_accuracy += tmp_eval_accuracy

        nb_eval_examples += b_input_ids.size(0)
        nb_eval_steps += 1
    eval_loss = eval_loss/nb_eval_steps
    print("Validation loss: {}".format(eval_loss))
    print("Validation Accuracy: {}".format(eval_accuracy/nb_eval_steps))
    pred_tags = [tags_vals[p_i] for p in predictions for p_i in p]
    valid_tags = [tags_vals[l_ii] for l in true_labels for l_i in l for l_ii in l_i]
    print("F1-Score: {}".format(f1_score(pred_tags, valid_tags)))

Although I closely followed the original code, it does not seem to work for me and I keep on getting this mistake...

RuntimeError                              Traceback (most recent call last)
<ipython-input-45-aa659bbf8fac> in <module>()
     12
     13         # forward pass
---> 14         loss = model(b_input_ids, token_type_ids=None, attention_mask=b_input_mask, labels=b_labels)
     15         # backward pass
     16         loss.backward()

8 frames
/usr/local/lib/python3.6/dist-packages/torch/nn/functional.py in embedding(input, weight, padding_idx, max_norm, norm_type, scale_grad_by_freq, sparse)
   1465         # remove once script supports set_grad_enabled
   1466         _no_grad_embedding_renorm_(weight, input, max_norm, norm_type)
-> 1467     return torch.embedding(weight, input, padding_idx, scale_grad_by_freq, sparse)
   1468
   1469

RuntimeError: Expected object of backend CPU but got backend CUDA for argument #3 'index'

SOLUTION:
setting model = model.to(device) plus reducing the batch size from 32 to 4 allowed the program to run without any runtimeerrors.


title: How to add 1x1 convolution as classification layer to VGG?

body:
I am trying to train a neural network using the first 5 layers of VGG16 as a feature extractor with Imagenet pre-trained weights. These layers contain 5 Max pooling layers which reduce images to 1/32th of original image size. For images with size 640x360, I have activation maps of 20x11 and 512 channels.

I want to classify each activation between 20 classes, taking as input the 512 channels by creating a 1x1 convolution created like this:

# Model definition
class BlockWiseCSRNet(nn.Module):
    """
    Implementation of paper Counting objects by clockwise classification

    Implementation of a method with CRSNet backend
    """

    def __init__(self, load_weights=False, count_levels=10):

        super(BlockWiseCSRNet, self).__init__()
        self.seen = 0
        self.backend_feat = [64, 64, 'M', 128, 128, 'M', 256, 256, 256, 'M', 512, 512, 512, 'M', 512, 512, 512, 'M']
        self.backend = make_layers(self.backend_feat)
        self.count_levels = count_levels

        self.classification = nn.Sequential(
                nn.Conv2d(in_channels=512, out_channels=count_levels, kernel_size=1),
                nn.Softmax(dim=1)
            )

        if not load_weights:
            mod = models.vgg16(pretrained=True)
            self._initialize_weights()
            # copy weights from VGG pretrained to frontend
            for k, v in self.backend.state_dict().items():
                if 'weight' in k:
                    self.backend.state_dict()[k].data[:] = mod.state_dict()['features.' + k][1].data[:]
                else:
                    self.backend.state_dict()[k].data = mod.state_dict()['features.' + k][1].data

    def freeze_backend(self):
        """
        Freeze backend parameters to make it not trainable
        :return:
        """
        for p in self.backend.parameters():
            p.requires_grad = False

    def unfreeze_classification(self):
        for p in self.classification.parameters():
            p.requires_grad = True

    def forward(self, x):
        x = self.backend(x)
        x = self.classification(x)
        return x

    def _initialize_weights(self):
        for m in self.modules():
            if isinstance(m, nn.Conv2d):
                nn.init.normal_(m.weight, std=0.01)
                if m.bias is not None:
                    nn.init.constant_(m.bias, 0)
            elif isinstance(m, nn.BatchNorm2d):
                nn.init.constant_(m.weight, 1)
                nn.init.constant_(m.bias, 0)

def make_layers(cfg, in_channels=3, batch_norm=False, dilation=False):
    if dilation:
        d_rate = 2
    else:
        d_rate = 1
    layers = []
    for v in cfg:
        if v == 'M':
            layers += [nn.MaxPool2d(kernel_size=2, stride=2)]
        else:
            conv2d = nn.Conv2d(in_channels, v, kernel_size=3, padding=d_rate, dilation=d_rate)
            if batch_norm:
                layers += [conv2d, nn.BatchNorm2d(v), nn.ReLU(inplace=True)]
            else:
                layers += [conv2d, nn.ReLU(inplace=True)]
            in_channels = v
    return nn.Sequential(*layers)

### Relevant train loop

model = BlockWiseCSRNet(count_levels=20)
criterion = torch.nn.CrossEntropyLoss()

optimizer = torch.optim.Adam(filter(lambda x: x.requires_grad, model.parameters()), args.learning_rate)

# train loop step
def train_step(epoch, device, data_loader, model, criterion, optimizer):
    print('train step!')
    running_loss = 0.0

    # set model to train mode
    model.train()
    # only train classification layers, keep weights of backend
    model.freeze_backend()
    model.unfreeze_classification()

    for mini_batch, (batch, labels, filenames) in enumerate(data_loader):
        # Transfer to GPU
        batch = batch.to(device)
        labels = labels.to(device)

        # zero the parameter gradients
        optimizer.zero_grad()

        # process batch
        outputs = model(batch)

        loss = criterion(outputs, labels)
        a = list(model.classification.parameters())[0].clone()
        loss.backward()
        optimizer.step()
        b = list(model.classification.parameters())[0].clone()
        print('weights updated? {}'.format(not torch.equal(a.data, b.data)))

        #print(torch.sum(model.classification[0].weight.sum()))

        # print statistics every print_stats_steps mini-batches
        running_loss += loss.item()

    print('[{}] total train loss: {}'.format(epoch, running_loss))

This solution is able to process images and give output activations with proper sizes.

However, I am not able to train the network. I am using CrossEntropyLoss and Adam optimizer but output maps are always constant (even after multiple training iterations)

# batch is an image

batch.shape -> torch.Size([1, 3, 360, 640])

output = model(batch)

output.shape -> torch.Size([1, 20, 11, 20])

criterion = torch.nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters(), args.learning_rate)

_, predicted_class_map = torch.max(output, dim=1)

# predicted class map is always 0

I have tried several learning rates and configurations, but the predicted class map is always 0. I debugged weight updating and updates are always 0 (before and after optimizer step weights are equal)

What I am doing wrong? I am not sure if the problem is at the architecture or in the training loop (which I took from PyTorch tutorials)

EDIT: After inspecting the content of net output, I think that the problem is that all channels in output have the same values for all image. So:

all_means = [ outputs[0, t, :, :].mean().item() for t in range(0, outputs.shape[1])]

has a vector with all positions equal.

Thanks


title: How can you specify docker flags when a step is ran on Google Cloud Build?

body:
I need to be able to specify --ipc=host to the docker command that runs a step. How do I do that?

A little more context.... When running the tests in one of my Google Cloud Build steps I am running into this error:

ERROR: Unexpected bus error encountered in worker. This might be caused by insufficient shared memory (shm).

This error is common when using pytorch, like I am, in docker but it is easily addressed with the flag --ipc=host. I don't know where to put this flag in my step's settings in the cloudbuild.yml.

Right now my step looks like this:

- id: 'run-tests'
  name: 'gcr.io/project/app:test-$BRANCH_NAME'
  entrypoint: 'bash'
  args:
    - '-c'
    - './bin/run-tests.sh'

I have tried adding the "ipc=host" to the args but those go to bash and not the docker command running the image.


title: Checking the contents of python dataloader

body:
This is probably a simple question, but how see how the contents of this standard data loader looks like:

from torchtext import datasets
import random

train_data, test_data = datasets.IMDB.splits(TEXT, LABEL)
train_data, valid_data = train_data.split(random_state = random.seed(SEED))

I can't use .head() and:

print(test_data)

just gives me:

<torchtext.datasets.imdb.IMDB object at 0x7f0b42e8c240>

I'm probably just missing .values or [0] or something similar...


title: Creating Pytorch Model from existing .PTH file

body:
As a result of a GAN training session, I've generated a midi generator and a midi discriminator as .PTH files and want to use the generator to create a midi using a pytorch model.
When looking at the PyTorch documentation I found this bit of code to describe how to load a model from that .PTH file.  this is what I found.
model = TheModelClass(*args, **kwargs)
model.load_state_dict(torch.load(PATH))
model.eval()

I recreated this with the code:
import torch

model = TheModelClass(*args,**kwargs)
model.load_state_dict(torch.load("C:\\Users\\ryand\\OneDrive\\Desktop\\c_rnn_gan_g.pth"))
model.eval()

this has generated the following error:
---------------------------------------------------------------------------
OSError                                   Traceback (most recent call last)
<ipython-input-12-8a77f4d4790c> in <module>
----> 1 import torch
      2
      3 model = TheModelClass(*args,**kwargs)
      4 model.load_state_dict(torch.load("C:\\Users\\ryand\\OneDrive\\Desktop\\c_rnn_gan_g.pth"))
      5 model.eval()

~\Anaconda3\lib\site-packages\torch\__init__.py in <module>
     79     dlls = glob.glob(os.path.join(th_dll_path, '*.dll'))
     80     for dll in dlls:
---> 81         ctypes.CDLL(dll)
     82
     83

~\Anaconda3\lib\ctypes\__init__.py in __init__(self, name, mode, handle, use_errno, use_last_error)
    362
    363         if handle is None:
--> 364             self._handle = _dlopen(self._name, mode)
    365         else:
    366             self._handle = handle

OSError: [WinError 126] The specified module could not be found

This is running inside an anaconda environment with PyTorch installed.  What do I have to do to generate that midi with that model?


title: How many weight convolution layer has?

body:
I have a simple convolution network:
import torch.nn as nn

class model(nn.Module):
def __init__(self, ks=1):
    super(model, self).__init__()
    self.conv1 = nn.Conv2d(in_channels=4, out_channels=32, kernel_size=ks, stride=1)

    self.fc1 = nn.Linear(8*8*32*ks, 64)
    self.fc2 = nn.Linear(64, 64)

def forward(self, x):
    x = F.relu(self.conv1(x))
    x = x.view(x.size(0), -1)
    x = F.relu(self.fc1(x))
    x = self.fc2(x)
    return x

cnn = model(1)

Since the kernel size is 1 and the output channel is 32, I assume that there should be 32*1*1 weights in this layer. But, when I ask pytorch about the shape of the weight matrix cnn.conv1.weight.shape, it returns torch.Size([32, 4, 1, 1]). Why the number of input channel should matter on the weight of a conv2d layer?
Am I missing something?


title: Loading pretrained model fast-bert

body:
This is my first time working on binary text classification, I'm working with CamemBert using  fast-bert library. After fine-tuning, I tried training but got this error :
Error(s) in loading state_dict for CamembertForSequenceClassification:
    size mismatch for classifier.out_proj.weight: copying a param with shape torch.Size([10, 768]) from checkpoint, the shape in current model is torch.Size([2, 768]).
    size mismatch for classifier.out_proj.bias: copying a param with shape torch.Size([10]) from checkpoint, the shape in current model is torch.Size([2]).

The weird thing is that I tried doing multi-label classification and it worked fine.
How can I fix this?
Here's the config.json file obtained after the fine-tuning:
{
  "architectures": [
    "CamembertForMultiLabelSequenceClassification"
  ],
  "attention_probs_dropout_prob": 0.1,
  "bos_token_id": 5,
  "eos_token_id": 6,
  "gradient_checkpointing": false,
  "hidden_act": "gelu",
  "hidden_dropout_prob": 0.1,
  "hidden_size": 768,
  "id2label": {
    "0": "LABEL_0",
    "1": "LABEL_1",
    "2": "LABEL_2",
    "3": "LABEL_3",
    "4": "LABEL_4",
    "5": "LABEL_5",
    "6": "LABEL_6",
    "7": "LABEL_7",
    "8": "LABEL_8",
    "9": "LABEL_9"
  },
  "initializer_range": 0.02,
  "intermediate_size": 3072,
  "label2id": {
    "LABEL_0": 0,
    "LABEL_1": 1,
    "LABEL_2": 2,
    "LABEL_3": 3,
    "LABEL_4": 4,
    "LABEL_5": 5,
    "LABEL_6": 6,
    "LABEL_7": 7,
    "LABEL_8": 8,
    "LABEL_9": 9
  },
  "layer_norm_eps": 1e-05,
  "max_position_embeddings": 514,
  "model_type": "camembert",
  "num_attention_heads": 12,
  "num_hidden_layers": 12,
  "output_past": true,
  "pad_token_id": 1,
  "type_vocab_size": 1,
  "vocab_size": 32005
}


title: How torch.distributed.launch assign data to each GPU?

body:
When our batch size is 1 or 2 and we have 8 GPUs, how torch.distributed.launch assign data to each GPUs? I converted my model to torch.nn.parallel.DistributedDataParallel,
model = DistributedDataParallel(model,
                                device_ids=[args.local_rank],
                                output_device=args.local_rank,
                                find_unused_parameters=True,
                                )

but it stated in the documentation that DistributedDataParallel:

parallelizes the application of the given module by
splitting the input across the specified devices by chunking in the
batch dimension.

My question is when batch size is smaller than the number of GPUs how it deal with it?