I'm following this tutorial.
I'm at the last part where we combine the models in a regression.
I'm coding this in jupyter as follows:
import shutil
import os
import time
from datetime import datetime
import argparse
import pandas
import numpy as np
from tqdm import tqdm
from tqdm import tqdm_notebook

import torch
import torch.nn as nn
import torch.optim as optim
from torch.autograd import Variable
from torchsample.transforms import RandomRotate, RandomTranslate, RandomFlip, ToTensor, Compose, RandomAffine
from torchvision import transforms
import torch.nn.functional as F
from tensorboardX import SummaryWriter

import dataloader
from dataloader import MRDataset
import model

from sklearn import metrics

def extract_predictions(task, plane, train=True):
    assert task in ['acl', 'meniscus', 'abnormal']
    assert plane in ['axial', 'coronal', 'sagittal']

    models = os.listdir('models/')

    model_name = list(filter(lambda name: task in name and plane in name, models))[0]
    model_path = f'models/{model_name}'

    mrnet = torch.load(model_path)
    _ = mrnet.eval()

    train_dataset = MRDataset('data/',
                              task,
                              plane,
                              transform=None,
                              train=train,
                              )

    train_loader = torch.utils.data.DataLoader(train_dataset,
                                               batch_size=1,
                                               shuffle=False,
                                               num_workers=10,
                                               drop_last=False)
    predictions = []
    labels = []
    with torch.no_grad():
        for image, label, _ in tqdm_notebook(train_loader):
            logit = mrnet(image.cuda())
            prediction = torch.sigmoid(logit)
            predictions.append(prediction.item())
            labels.append(label.item())

    return predictions, labels

task = 'acl'
results = {}

for plane in ['axial', 'coronal', 'sagittal']:
    predictions, labels = extract_predictions(task, plane)
    results['labels'] = labels
    results[plane] = predictions

X = np.zeros((len(predictions), 3))
X[:, 0] = results['axial']
X[:, 1] = results['coronal']
X[:, 2] = results['sagittal']

y = np.array(labels)

logreg = LogisticRegression(solver='lbfgs')
logreg.fit(X, y)

task = 'acl'
results_val = {}

for plane in ['axial', 'coronal', 'sagittal']:
    predictions, labels = extract_predictions(task, plane, train=False)
    results_val['labels'] = labels
    results_val[plane] = predictions

y_pred = logreg.predict_proba(X_val)[:, 1]
metrics.roc_auc_score(y_val, y_pred)

However I get this error:
ValueError                                Traceback (most recent call last)
<ipython-input-2-979acb314bc5> in <module>
      3
      4 for plane in ['axial', 'coronal', 'sagittal']:
----> 5     predictions, labels = extract_predictions(task, plane)
      6     results['labels'] = labels
      7     results[plane] = predictions

<ipython-input-1-647731b6b5c8> in extract_predictions(task, plane, train)
     54             logit = mrnet(image.cuda())
     55             prediction = torch.sigmoid(logit)
---> 56             predictions.append(prediction.item())
     57             labels.append(label.item())
     58

ValueError: only one element tensors can be converted to Python scalars

Here's the MRDataset code in case:
class MRDataset(data.Dataset):
    def __init__(self, root_dir, task, plane, train=True, transform=None, weights=None):
        super().__init__()
        self.task = task
        self.plane = plane
        self.root_dir = root_dir
        self.train = train
        if self.train:
            self.folder_path = self.root_dir + 'train/{0}/'.format(plane)
            self.records = pd.read_csv(
                self.root_dir + 'train-{0}.csv'.format(task), header=None, names=['id', 'label'])
        else:
            transform = None
            self.folder_path = self.root_dir + 'valid/{0}/'.format(plane)
            self.records = pd.read_csv(
                self.root_dir + 'valid-{0}.csv'.format(task), header=None, names=['id', 'label'])

        self.records['id'] = self.records['id'].map(
            lambda i: '0' * (4 - len(str(i))) + str(i))
        self.paths = [self.folder_path + filename +
                      '.npy' for filename in self.records['id'].tolist()]
        self.labels = self.records['label'].tolist()

        self.transform = transform
        if weights is None:
            pos = np.sum(self.labels)
            neg = len(self.labels) - pos
            self.weights = torch.FloatTensor([1, neg / pos])
        else:
            self.weights = torch.FloatTensor(weights)

    def __len__(self):
        return len(self.paths)

    def __getitem__(self, index):
        array = np.load(self.paths[index])
        label = self.labels[index]
        if label == 1:
            label = torch.FloatTensor([[0, 1]])
        elif label == 0:
            label = torch.FloatTensor([[1, 0]])

        if self.transform:
            array = self.transform(array)
        else:
            array = np.stack((array,)*3, axis=1)
            array = torch.FloatTensor(array)

        # if label.item() == 1:
        #     weight = np.array([self.weights[1]])
        #     weight = torch.FloatTensor(weight)
        # else:
        #     weight = np.array([self.weights[0]])
        #     weight = torch.FloatTensor(weight)

        return array, label, self.weights

I've only trained my models using 1 and 2 epochs for each plane of the MRI instead of 35 as in the tutorial, not sure if that has anything to do with it. Other than that I'm stranded as to what this could be? I also removed normalize=False in the options for train_dataset as it kept giving me an error and I read that it could be removed, but I'm not so sure?


title: When should I use nn.ModuleList and when should I use nn.Sequential?

body: I am new to Pytorch and one thing that I don't quite understand is the usage of nn.ModuleList and nn.Sequential. Can I know when I should use one over the other? Thanks.


title: BatchNorm momentum convention PyTorch

body: Is the batchnorm momentum convention (default=0.1) correct as in other libraries e.g. Tensorflow it seems to usually be 0.9 or 0.99 by default? Or maybe we are just using a different convention?


title: Adding batch normalization decreases the performance

body:
I'm using PyTorch to implement a classification network for skeleton-based action recognition. The model consists of three convolutional layers and two fully connected layers. This base model gave me an accuracy of around 70% in the NTU-RGB+D dataset. I wanted to learn more about batch normalization, so I added a batch normalization for all the layers except for the last one. To my surprise, the evaluation accuracy dropped to 60% rather than increasing But the training accuracy has increased from 80% to 90%. Can anyone say what am I doing wrong? or Adding batch normalization need not increase the accuracy?

The model with batch normalization

class BaseModelV0p2(nn.Module):

    def __init__(self, num_person, num_joint, num_class, num_coords):
        super().__init__()
        self.name = 'BaseModelV0p2'
        self.num_person = num_person
        self.num_joint = num_joint
        self.num_class = num_class
        self.channels = num_coords
        self.out_channel = [32, 64, 128]
        self.loss = loss
        self.metric = metric
        self.bn_momentum = 0.01

        self.bn_cv1 = nn.BatchNorm2d(self.out_channel[0], momentum=self.bn_momentum)
        self.conv1 = nn.Sequential(nn.Conv2d(in_channels=self.channels, out_channels=self.out_channel[0],
                                             kernel_size=3, stride=1, padding=1),
                                   self.bn_cv1,
                                    nn.ReLU(),
                                    nn.MaxPool2d(kernel_size=2, stride=2))

        self.bn_cv2 = nn.BatchNorm2d(self.out_channel[1], momentum=self.bn_momentum)
        self.conv2 = nn.Sequential(nn.Conv2d(in_channels=self.out_channel[0], out_channels=self.out_channel[1],
                                            kernel_size=3, stride=1, padding=1),
                                   self.bn_cv2,
                                nn.ReLU(),
                                nn.MaxPool2d(kernel_size=2, stride=2))

        self.bn_cv3 = nn.BatchNorm2d(self.out_channel[2], momentum=self.bn_momentum)
        self.conv3 = nn.Sequential(nn.Conv2d(in_channels=self.out_channel[1], out_channels=self.out_channel[2],
                                            kernel_size=3, stride=1, padding=1),
                                   self.bn_cv3,
                                  nn.ReLU(),
                                  nn.MaxPool2d(kernel_size=2, stride=2))

        self.bn_fc1 = nn.BatchNorm1d(256 * 2, momentum=self.bn_momentum)
        self.fc1 = nn.Sequential(nn.Linear(self.out_channel[2]*8*3, 256*2),
                                 self.bn_fc1,
                                 nn.ReLU(),
                                 nn.Dropout2d(p=0.5))  # TO check

        self.fc2 = nn.Sequential(nn.Linear(256*2, self.num_class))

    def forward(self, input):
        list_bn_layers = [self.bn_fc1, self.bn_cv3, self.bn_cv2, self.bn_cv1]
        # set the momentum  of the batch norm layers to given momentum value during trianing and 0 during evaluation
        # ref: https://discuss.pytorch.org/t/model-eval-gives-incorrect-loss-for-model-with-batchnorm-layers/7561
        # ref: https://github.com/pytorch/pytorch/issues/4741
        for bn_layer in list_bn_layers:
            if self.training:
                bn_layer.momentum = self.bn_momentum
            else:
                bn_layer.momentum = 0

        logits = []
        for i in range(self.num_person):
            out = self.conv1(input[:, :, :, :, i])

            out = self.conv2(out)

            out = self.conv3(out)

            logits.append(out)

        out = torch.max(logits[0], logits[1])
        out = out.view(out.size(0), -1)
        out = self.fc1(out)
        out = self.fc2(out)

        t = out

        assert not ((t != t).any())  # find out nan in tensor
        assert not (t.abs().sum() == 0)  # find out 0 tensor

        return out


title: The purpose of introducing nn.Parameter in pytorch

body:
I am new to Pytorch and I am confused about the difference between nn.Parameter  and autograd.Variable. I know that the former one is the subclass of Variable and has the gradient. But I really don't understand why we introduce Parameter and when we should use it?

SUMMARY:

Thanks for the explanation of iacolippo, i finally understand the difference between parameter and variable. In a summary, variable in pytorch is NOT same as in  the variable in tensorflow, the former one is not attach to the model's trainable parameters while the later one will. Attaching to the model means that using model.parameters() will return the certain parameter to you, which is useful in training phase to specify the variable needed to train. The 'variable' is more helpful as a cache in some network.


title: Applying convolution operation to image - PyTorch

body:
To render an image if shape 27x35 I use :

random_image = []
for x in range(1 , 946):
        random_image.append(random.randint(0 , 255))

random_image_arr = np.array(random_image)
matplotlib.pyplot.imshow(random_image_arr.reshape(27 , 35))

This generates :

I then try to apply a convolution to the image using the torch.nn.Conv2d :

conv2 = torch.nn.Conv2d(3, 18, kernel_size=3, stride=1, padding=1)

image_d = np.asarray(random_image_arr.reshape(27 , 35))

conv2(torch.from_numpy(image_d))

But this displays error :

~/.local/lib/python3.6/site-packages/torch/nn/modules/conv.py in forward(self, input)
    299     def forward(self, input):
    300         return F.conv2d(input, self.weight, self.bias, self.stride,
--> 301                         self.padding, self.dilation, self.groups)
    302
    303

RuntimeError: input has less dimensions than expected

The shape of the input image_d is (27, 35)

Should I change the parameters of Conv2d in order to apply the convolution to the image ?

Update. From @McLawrence answer I have :

random_image = []
for x in range(1 , 946):
        random_image.append(random.randint(0 , 255))

random_image_arr = np.array(random_image)
matplotlib.pyplot.imshow(random_image_arr.reshape(27 , 35))

This renders image :

Applying the convolution operation :

conv2 = torch.nn.Conv2d(1, 18, kernel_size=3, stride=1, padding=1)

image_d = torch.FloatTensor(np.asarray(random_image_arr.reshape(1, 1, 27 , 35))).numpy()

fc = conv2(torch.from_numpy(image_d))

matplotlib.pyplot.imshow(fc[0][0].data.numpy())

renders image :


title: can't import torch mac

body:
I'm trying to import torch and I'm getting the next problem:

Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/torch/__init__.py", line 66, in <module>
    import torch._dl as _dl_flags
ImportError: dlopen(/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/torch/_dl.so, 2): no suitable image found.  Did find:
    /Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/torch/_dl.so: mach-o, but wrong architecture
    /Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/torch/_dl.so: mach-o, but wrong architecture

someone knows how can I solve this?
Thanks:)


title: Why TensorBoard summary is not updating?

body:
I use tensorboard with pytorch1.1 to log loss values.

I use writer.add_scalar("loss", loss.item(), global_step) in every for- loop body.

However, the plotting graph does not update while the training is processing.

Every time I want to see the latest loss, I have to restart the tensorboard server.

The code is here

import torch
import torch.nn as nn
import torch.nn.functional as F
import torchvision
from torch.utils.tensorboard import SummaryWriter
from torchvision import datasets, transforms

# Writer will output to ./runs/ directory by default
writer = SummaryWriter()

transform = transforms.Compose(
    [transforms.ToTensor(), transforms.Normalize((0.5,), (0.5,))]
)
trainset = datasets.MNIST("mnist_train", train=True, download=True, transform=transform)
trainloader = torch.utils.data.DataLoader(trainset, batch_size=64, shuffle=True)
model = torchvision.models.resnet50(False)
# Have ResNet model take in grayscale rather than RGB
model.conv1 = nn.Conv2d(1, 64, kernel_size=7, stride=2, padding=3, bias=False)
model.fc = nn.Linear(2048, 10, True)

criterion = nn.CrossEntropyLoss()

epochs = 100

opt = torch.optim.Adam(model.parameters())

niter = 0

for epoch in range(epochs):
    for step, (x, y) in enumerate(trainloader):
        yp = model(x)
        loss = criterion(yp, y)
        opt.zero_grad()
        loss.backward()
        opt.step()
        writer.add_scalar("loss", loss.item(), niter)
        niter += 1
        print(loss.item())

grid = torchvision.utils.make_grid(images)
writer.add_image("images", grid, 0)
writer.add_graph(model, images)
writer.close()

The training is still going on, and the global steps has already been 3594. However, the tensorboard still shows around 1900.


I-m trying to run my python program it seems that it should run smoothly however I encounter an error that I haven't seen before it says:

free(): invalid pointer
Aborted (core dumped)

However I'm not sure how to try and fix error since it doesn't give me too much information about the problem itself.

At first I thought it should be a problem with the sizes of the tensor in my network however they are completely fine. I've google the problem a little and found that I can see that is a problem with allocating memory where I shouldn't, but I don't know how to fix this problem

My code is divided in two different files, and I use two libraries to be able to use Sinkhorn loss function and make sample randomly a mesh.

import argparse
import point_cloud_utils as pcu
import time

import numpy as np
import torch
import torch.nn as nn
from fml.nn import SinkhornLoss

import common
def main():
    # x is a tensor of shape [n, 3] containing the positions of the vertices that
    x = torch._C.from_numpy(common.loadpointcloud("sphere.txt"))
    # t is a tensor of shape [n, 3] containing a set of nicely distributed samples in the unit cube
    v, f = common.unit_cube()
    t = torch._C.sample_mesh_lloyd(pcu.lloyd(v,f,x.shape[0]).astype(np.float32)) # sample randomly a point cloud (cube for now?)

    # The model is a simple fully connected network mapping a 3D parameter point to 3D
    phi = common.MLP(in_dim=3, out_dim=3)

    # Eps is 1/lambda and max_iters is the maximum number of Sinkhorn iterations to do
    emd_loss_fun = SinkhornLoss(eps=1e-3, max_iters=20,
                                stop_thresh=1e-3, return_transport_matrix=True)

    mse_loss_fun = torch.nn.MSELoss()

    # Adam optimizer at first
    optimizer = torch.optim.Adam(phi.parameters(), lr= 10e-3)

    fit_start_time = time.time()

    for epoch in range(100):
        optimizer.zero_grad()

        # Do the forward pass of the neural net, evaluating the function at the parametric points
        y = phi(t)

        # Compute the Sinkhorn divergence between the reconstruction*(using the francis library) and the target
        # NOTE: The Sinkhorn function expects a batch of b point sets (i.e. tensors of shape [b, n, 3])
        # since we only have 1, we unsqueeze so x and y have dimension [1, n, 3]
        with torch.no_grad():
            _, P = emd_loss_fun(phi(t).unsqueeze(0), x.unsqueeze(0))

        # Project the transport matrix onto the space of permutation matrices and compute the L-2 loss
        # between the permuted points
        loss = mse_loss_fun(y[P.squeeze().max(0)[1], :], x)
        # loss = mse_loss_fun(P.squeeze() @ y,  x)  # Use the transport matrix directly

        # Take an optimizer step
        loss.backward()
        optimizer.step()

        print("Epoch %d, loss = %f" % (epoch, loss.item()))

    fit_end_time = time.time()

    print("Total time = %f" % (fit_end_time - fit_start_time))
    # Plot the ground truth, reconstructed points, and a mesh representing the fitted function, phi
    common.visualitation(x,t,phi)

if __name__ == "__main__":
    main()

The error message is:
free(): invalid pointer
Aborted (core dumped)

That again doesn't help me that much. I'll appreciate it a lot if someone has any idea what is happening or if you know more about this error.


title: How does one dynamically add new parameters to optimizers in Pytorch?

body:
I was going through this post in the pytorch forum, and I also wanted to do this. The original post removes and adds layers but I think my situation is not that different. I also want to add layers or more filters or word embeddings. My main motivation is that the AI agent does not know the whole vocabulary/dictionary in advance because its large. I prefer strongly (for the moment) to not do character by character RNNs.

So what will happen for me is when the agent starts a forward pass it might find new words it has never seen and will need to add them to the embedding table (or perhaps add new filters before it starts the forward pass).

So what I want to make sure is:

embeddings are added correctly (at the right time, when a new computation graph is made) so that they are updatable by the optimizer
no issues with stored info of past parameters e.g. if its using some sort of momentum

How does one do this? Any sample code that works?


title: LSTM in Pytorch

body:
I'm new to PyTorch. I came across some this GitHub repository (link to full code example) containing various different examples.

There is also an example about LSTMs, this is the Network class:

# RNN Model (Many-to-One)
class RNN(nn.Module):
    def __init__(self, input_size, hidden_size, num_layers, num_classes):
        super(RNN, self).__init__()
        self.hidden_size = hidden_size
        self.num_layers = num_layers
        self.lstm = nn.LSTM(input_size, hidden_size, num_layers, batch_first=True)
        self.fc = nn.Linear(hidden_size, num_classes)

    def forward(self, x):
        # Set initial states
        h0 = Variable(torch.zeros(self.num_layers, x.size(0), self.hidden_size))
        c0 = Variable(torch.zeros(self.num_layers, x.size(0), self.hidden_size))

        # Forward propagate RNN
        out, _ = self.lstm(x, (h0, c0))

        # Decode hidden state of last time step
        out = self.fc(out[:, -1, :])
        return out

So my question is about the following lines:

h0 = Variable(torch.zeros(self.num_layers, x.size(0), self.hidden_size))
c0 = Variable(torch.zeros(self.num_layers, x.size(0), self.hidden_size))

As far as I understand it, forward() is called for every training example. But this would mean, that the hidden state and cell state would be resettet i.e. replaced with a matrix of zeros on every training example.

The names h0 and c0 indicate that this is only the hidden/cell state at t=0, but why then are theses zeros matrices handed over to the lstm with every training example?

Even if they are just ignored after the first call, it would not be a very nice solution.

When testing the code it states an accuracy of 97% on the MNIST set, so it seems to work this way, but it doesn't make sense to me.

Hope someone can help me out with this.

Thanks in advance!


title: Pytorch, can't run backward() on even the most simple network without getting an error

body:
I am new to pytorch and I can't run backward() on even the most simple network without generating an error.
For example:

(Linear(6, 6)(Variable(torch.zeros([10, 6]))) - Variable(torch.zeros([10, 6]))).backward()

Throws the following error

{RuntimeError}element 0 of variables does not require grad and does not have a grad_fn

What have I done wrong in the code to create this issue?


title: Increasingly large, positive WGAN-GP loss

body:
I'm investigating the use of a Wasserstein GAN with gradient penalty in PyTorch, but consistently get large, positive generator losses that increase over epochs.
I'm heavily borrowing from Caogang's implementation, but am using the discriminator and generator losses used in this implementation because I get Invalid gradient at index 0 - expected shape[] but got [1] if I try to call .backward() with the one and mone args used in the Caogang implementation.

I'm training on a augmented WikiArt dataset (>400k 64x64 images) and CIFAR-10, and have gotten a normal WGAN (with weight clipping to work) [i.e. it produces passable images after 25 epochs], despite the fact that the D and G losses both hover around 3 [I calculate them using torch.mean(D_real) etc.] for all epochs. However, in the WGAN-GP version, the generator loss increases dramatically on both the WikiArt and CIFAR-10 datasets, and completely fails to generate anything other than noise on WikiArt.

Here's an example of the loss after 25 epochs on CIFAR-10:

I don't use any tricks like one-sided label smoothing, and I train with the default learning rate of 0.001, the Adam optimizer and I train the discriminator 5 times for every generator update. Why does this crazy loss behaviour happen, and why does the normal weight-clipping WGAN still 'work' on WikiArt but WGANGP completely fail?

This happens irrespective of the structure, whether both G and D are DCGANs or when using this modified DCGAN, the Creative Adversarial Network, which requires that D be able to classify images and G generate ambiguous images.

Below is the relevant part of my current trainmethod:

self.generator = Can64Generator(self.z_noise, self.channels, self.num_gen_filters).to(self.device)
self.discriminator =WCan64Discriminator(self.channels,self.y_dim, self.num_disc_filters).to(self.device)
style_criterion = nn.CrossEntropyLoss()

self.disc_optimizer = optim.Adam(self.discriminator.parameters(), lr=self.lr, betas=(self.beta1, 0.9))
self.gen_optimizer = optim.Adam(self.generator.parameters(), lr=self.lr, betas=(self.beta1, 0.9))

while i < len(dataloader):
            j = 0
            disc_loss_epoch = []
            gen_loss_epoch = []
            if self.type == "can":
                disc_class_loss_epoch = []
                gen_class_loss_epoch = []

            if self.gradient_penalty == False:
                # critic training methodology in official WGAN implementation
                if gen_iterations < 25 or (gen_iterations % 500 == 0):
                    disc_iters = 100
            else:
                disc_iters = self.disc_iterations

            while j < disc_iters and (i < len(dataloader)):
                # if using wgan with weight clipping
                if self.gradient_penalty == False:
                    # Train Discriminator
                    for param in self.discriminator.parameters():
                        param.data.clamp_(self.lower_clamp,self.upper_clamp)

                for param in self.discriminator.parameters():
                    param.requires_grad_(True)

                j+=1
                i+=1
                data = data_iterator.next()
                self.discriminator.zero_grad()
                real_images, image_labels = data
                # image labels are the the image's classes (e.g. Impressionism)
                real_images = real_images.to(self.device)
                batch_size = real_images.size(0)
                real_image_labels = torch.LongTensor(batch_size).to(self.device)
                real_image_labels.copy_(image_labels)

                labels = torch.full((batch_size,),real_label,device=self.device)

                if self.type == 'can':
                    predicted_output_real, predicted_styles_real = self.discriminator(real_images.detach())
                    predicted_styles_real = predicted_styles_real.to(self.device)
                    disc_class_loss = style_criterion(predicted_styles_real,real_image_labels)
                    disc_class_loss.backward(retain_graph=True)

                else:
                    predicted_output_real = self.discriminator(real_images.detach())

                disc_loss_real = -torch.mean(predicted_output_real)

                # fake

                noise = torch.randn(batch_size,self.z_noise,1,1,device=self.device)
                with torch.no_grad():
                    noise_g = noise.detach()
                fake_images = self.generator(noise_g)
                labels.fill_(fake_label)

                if self.type == 'can':
                    predicted_output_fake, predicted_styles_fake = self.discriminator(fake_images)

                else:
                    predicted_output_fake = self.discriminator(fake_images)

                disc_gen_z_1 = predicted_output_fake.mean().item()

                disc_loss_fake = torch.mean(predicted_output_fake)

                #via https://github.com/znxlwm/pytorch-generative-model-collections/blob/master/WGAN_GP.py
                if self.gradient_penalty:
                    # gradient penalty
                    alpha = torch.rand((real_images.size()[0], 1, 1, 1)).to(self.device)
                    x_hat = alpha * real_images.data + (1 - alpha) * fake_images.data
                    x_hat.requires_grad_(True)
                    if self.type == 'can':
                        pred_hat, _ = self.discriminator(x_hat)
                    else:
                        pred_hat = self.discriminator(x_hat)
                    gradients = grad(outputs=pred_hat, inputs=x_hat, grad_outputs=torch.ones(pred_hat.size()).to(self.device),
                                    create_graph=True, retain_graph=True, only_inputs=True)[0]

                    gradient_penalty = lambda_ * ((gradients.view(gradients.size()[0], -1).norm(2, 1) - 1) ** 2).mean()
                    disc_loss = disc_loss_fake + disc_loss_real + gradient_penalty
                else:
                    disc_loss  =  disc_loss_fake  + disc_loss_real

                if self.type == 'can':
                    disc_loss += disc_class_loss.mean()

                disc_x = disc_loss.mean().item()
                disc_loss.backward(retain_graph=True)
                self.disc_optimizer.step()

            # train generator
            for param in self.discriminator.parameters():
                param.requires_grad_(False)

            self.generator.zero_grad()
            labels.fill_(real_label)

            if self.type == 'can':
                predicted_output_fake, predicted_styles_fake = self.discriminator(fake_images)
                predicted_styles_fake = predicted_styles_fake.to(self.device)

            else:
                predicted_output_fake = self.discriminator(fake_images)

            gen_loss = -torch.mean(predicted_output_fake)
            disc_gen_z_2 = gen_loss.mean().item()

            if self.type == 'can':
                fake_batch_labels = 1.0/self.y_dim * torch.ones_like(predicted_styles_fake)
                fake_batch_labels = torch.mean(fake_batch_labels,1).long().to(self.device)
                gen_class_loss = style_criterion(predicted_styles_fake,fake_batch_labels)
                gen_class_loss.backward(retain_graph=True)
                gen_loss += gen_class_loss.mean()

            gen_loss.backward()
            gen_iterations += 1

This is the code for the (DCGAN) generator:

class Can64Generator(nn.Module):
def __init__(self, z_noise, channels, num_gen_filters):
    super(Can64Generator,self).__init__()
    self.ngpu = 1
    self.main = nn.Sequential(
    nn.ConvTranspose2d(z_noise, num_gen_filters * 16, 4, 1, 0, bias=False),
    nn.BatchNorm2d(num_gen_filters * 16),
    nn.ReLU(True),
    nn.ConvTranspose2d(num_gen_filters * 16, num_gen_filters * 4, 4, 2, 1, bias=False),
    nn.BatchNorm2d(num_gen_filters * 4),
    nn.ReLU(True),
    nn.ConvTranspose2d(num_gen_filters * 4, num_gen_filters * 2, 4, 2, 1, bias=False),
    nn.BatchNorm2d(num_gen_filters * 2),
    nn.ReLU(True),
    nn.ConvTranspose2d(num_gen_filters * 2, num_gen_filters, 4, 2, 1, bias=False),
    nn.BatchNorm2d(num_gen_filters),
    nn.ReLU(True),
    nn.ConvTranspose2d(num_gen_filters, 3, 4, 2, 1, bias=False),
    nn.Tanh()
    )
def forward(self, inp):
    output = self.main(inp)
    return output

And this is the (current) CAN discriminator, which has extra layers for
style (image class) classification):

class Can64Discriminator(nn.Module):

def __init__(self, channels,y_dim, num_disc_filters):
        super(Can64Discriminator, self).__init__()
        self.ngpu = 1
        self.conv = nn.Sequential(
                nn.Conv2d(channels, num_disc_filters // 2, 4, 2, 1, bias=False),
                nn.LeakyReLU(0.2, inplace=True),

                nn.Conv2d(num_disc_filters // 2, num_disc_filters, 4, 2, 1, bias=False),
                nn.BatchNorm2d(num_disc_filters),
                nn.LeakyReLU(0.2, inplace=True),

                nn.Conv2d(num_disc_filters, num_disc_filters * 2, 4, 2, 1, bias=False),
                nn.BatchNorm2d(num_disc_filters * 2),
                nn.LeakyReLU(0.2, inplace=True),

                nn.Conv2d(num_disc_filters * 2, num_disc_filters * 4, 4, 2, 1, bias=False),
                nn.BatchNorm2d(num_disc_filters * 4),
                nn.LeakyReLU(0.2, inplace=True),

                nn.Conv2d(num_disc_filters * 4, num_disc_filters * 8, 4, 1, 0, bias=False),
                nn.BatchNorm2d(num_disc_filters * 8),
                nn.LeakyReLU(0.2, inplace=True),

            )
        # was this
        #self.final_conv = nn.Conv2d(num_disc_filters * 8, num_disc_filters * 8, 4, 2, 1, bias=False)

        self.real_fake_head = nn.Linear(num_disc_filters * 8, 1)

        # no bn and lrelu needed
        self.sig = nn.Sigmoid()
        self.fc = nn.Sequential()
        self.fc.add_module("linear_layer{0}".format(num_disc_filters*16),nn.Linear(num_disc_filters*8,num_disc_filters*16))
        self.fc.add_module("linear_layer{0}".format(num_disc_filters*8),nn.Linear(num_disc_filters*16,num_disc_filters*8))
        self.fc.add_module("linear_layer{0}".format(num_disc_filters),nn.Linear(num_disc_filters*8,y_dim))
        self.fc.add_module('softmax',nn.Softmax(dim=1))

def forward(self, inp):
    x = self.conv(inp)
    x = x.view(x.size(0),-1)
    real_out = self.sig(self.real_fake_head(x))
    real_out = real_out.view(-1,1).squeeze(1)
    style = self.fc(x)
    #style = torch.mean(style,1) # CrossEntropyLoss requires input be (N,C)
    return real_out,style

The only differences between the WGANGP version and the WGAN version of my GAN is the WGAN version uses RMSprop with lr=0.00005 and clips the weights of the discriminator, as per the WGAN paper.

What could be causing this? I'd like to make as minimal change as possible, as I want to compare loss functions alone. The same problem is encountered even when using an unmodified DCGAN discriminator on CIFAR-10. Am I encountering this perhaps because I am only training currently for 25 epochs, or is there another reason? Interestingly, my GAN also completely fails to generate anything other than noise when using LSGAN (nn.MSELoss()).

Thanks in advance!


File "/home/username/anaconda3/lib/python3.6/site-packages/torch/nn/modules/loss.py", line 433, in forward     reduce=self.reduce)
File "/home/username/anaconda3/lib/python3.6/site-packages/torch/nn/functional.py", line 1483, in binary_cross_entropy
return torch._C._nn.binary_cross_entropy(input, target, weight, size_average, reduce)

RuntimeError: reduce failed to synchronize: device-side assert triggered


title: How to change certain values in a torch tensor based on an index in another torch tensor?

body:
This is an issue I'm running while convertinf DQN to Double DQN for the cartpole problem. I'm getting close to figuring it out.

tensor([0.1205, 0.1207, 0.1197, 0.1195, 0.1204, 0.1205, 0.1208, 0.1199, 0.1206,
        0.1199, 0.1204, 0.1205, 0.1199, 0.1204, 0.1204, 0.1203, 0.1198, 0.1198,
        0.1205, 0.1204, 0.1201, 0.1205, 0.1208, 0.1202, 0.1205, 0.1203, 0.1204,
        0.1205, 0.1206, 0.1206, 0.1205, 0.1204, 0.1201, 0.1206, 0.1206, 0.1199,
        0.1198, 0.1200, 0.1206, 0.1207, 0.1208, 0.1202, 0.1201, 0.1210, 0.1208,
        0.1205, 0.1205, 0.1201, 0.1193, 0.1201, 0.1205, 0.1207, 0.1207, 0.1195,
        0.1210, 0.1204, 0.1209, 0.1207, 0.1187, 0.1202, 0.1198, 0.1202])
tensor([ True,  True,  True,  True,  True,  True,  True,  True,  True,  True,
         True,  True,  True,  True,  True,  True, False,  True,  True,  True,
         True,  True,  True,  True,  True,  True,  True, False,  True,  True,
         True,  True,  True,  True,  True,  True,  True,  True,  True,  True,
         True,  True,  True,  True,  True,  True,  True,  True,  True,  True,
         True,  True,  True,  True,  True,  True,  True,  True,  True,  True,
         True,  True,  True,  True])

As you can see here two tensors.
The first has the q values I want
but,
some values need to be changed to zeros because of it an end state.
The second tensor shows where it will be zeros.

At the index where the Boolean value is false is the equivalent spot for where the upper tensor needs to be zeros.
I am not sure how to do that.


title: Can’t convert np.ndarray of type numpy.bool_

body:
I am wondering why I am getting this error when I used :

Y_train_class = torch.tensor(Y_train_class.values)

  TypeError: can't convert np.ndarray of type numpy.bool_. The only supported types are: double, float, float16, int64, int32, and uint8.

I have tried to convert my data to float but seems failed

X_train = pd.read_csv('c:/Data/x_train_set_yu.csv', header= None)
Y_train = pd.read_csv('c:/Data/y_train_set_yu.1.csv', header= None)

Y_train_class = (Y_train >= 550)
print (Y_train_class)
X_test = pd.read_csv('c:/Data/X_test.csv',header= None)

X_train = torch.tensor(X_train.values)

Y_train.astype(np.float32)
Y_train_class.astype(np.float32)

Y_train_class = torch.tensor(Y_train_class.values)

  TypeError: can't convert np.ndarray of type numpy.bool_. The only supported types are: double, float, float16, int64, int32, and uint8.


title: Indexing the max elements in a multidimensional tensor in PyTorch

body:
I'm trying to index the maximum elements along the last dimension in a multidimensional tensor. For example, say I have a tensor

A = torch.randn((5, 2, 3))
_, idx = torch.max(A, dim=2)

Here idx stores the maximum indices, which may look something like

>>>> A
tensor([[[ 1.0503,  0.4448,  1.8663],
     [ 0.8627,  0.0685,  1.4241]],

    [[ 1.2924,  0.2456,  0.1764],
     [ 1.3777,  0.9401,  1.4637]],

    [[ 0.5235,  0.4550,  0.2476],
     [ 0.7823,  0.3004,  0.7792]],

    [[ 1.9384,  0.3291,  0.7914],
     [ 0.5211,  0.1320,  0.6330]],

    [[ 0.3292,  0.9086,  0.0078],
     [ 1.3612,  0.0610,  0.4023]]])
>>>> idx
tensor([[ 2,  2],
    [ 0,  2],
    [ 0,  0],
    [ 0,  2],
    [ 1,  0]])

I want to be able to access these indices and assign to another tensor based on them. Meaning I want to be able to do

B = torch.new_zeros(A.size())
B[idx] = A[idx]

where B is 0 everywhere except where A is maximum along the last dimension. That is B should store

>>>>B
tensor([[[ 0,  0,  1.8663],
     [ 0,  0,  1.4241]],

    [[ 1.2924,  0,  0],
     [ 0,  0,  1.4637]],

    [[ 0.5235,  0,  0],
     [ 0.7823,  0,  0]],

    [[ 1.9384,  0,  0],
     [ 0,  0,  0.6330]],

    [[ 0,  0.9086,  0],
     [ 1.3612,  0,  0]]])

This is proving to be much more difficult than I expected, as the idx does not index the array A properly. Thus far I have been unable to find a vectorized solution to use idx to index A.

Is there a good vectorized way to do this?


title: Low GPU utilization issue in PyTorch

body:
I am training a large network like ResNet with very small batch size say 25. When I do that, I get a very low and oscillating GPU utilization. I have seen several posts regarding the low GPU utilization in PyTorch. However, they are suggesting either of the following:

“Increase the batchsize.”: But, this is not a computational choice and I want my batch size to be small.

“Increase the number of workers as dataloading might be the bottleneck.”: First of all dataloading is not the bottleneck as it takes much less time. Secondly, increasing the number of loaders increases the running time of my code. Third, low and oscillating GPU utilization persists even after increasing the number of loaders. Hence, this suggestion also does not apply.

“Set shuffle = False”: Again not a feasible solution as I have to shuffle my data somehow.

Do you have any other suggestion for more effective use of GPUs when we have small batchsize?


title: How to replace infs to avoid nan gradients in PyTorch

body:
I need to compute log(1 + exp(x)) and then use automatic differentiation on it. But for too large x, it outputs inf because of the exponentiation:

>>> x = torch.tensor([0., 1., 100.], requires_grad=True)
>>> x.exp().log1p()
tensor([0.6931, 1.3133,    inf], grad_fn=<Log1PBackward>)

Since log(1 + exp(x)) ≈ x for large x, I thought I could replace the infs with x using torch.where. But when doing this, I still get nan for the gradient of too large values. Do you know why this happens and if there is another way to make it work?

>>> exp = x.exp()
>>> y = x.where(torch.isinf(exp), exp.log1p())  # Replace infs with x
>>> y  # No infs
tensor([  0.6931,   1.3133, 100.0000], grad_fn=<SWhereBackward>)
>>> y.sum().backward()  # Automatic differentiation
>>> x.grad  # Why is there a nan and how can I get rid of it?
tensor([0.5000, 0.7311,    nan])


title: Display Pytorch tensor as image using Matplotlib

body:
I am trying to display an image stored as a pytorch tensor.
trainset = datasets.ImageFolder('data/Cat_Dog_data/train/', transform=transforms)
trainload = torch.utils.data.DataLoader(trainset, batch_size=32, shuffle=True)

images, labels = iter(trainload).next()
image = images[0]
image.shape

>>> torch.Size([3, 224, 224]) # pyplot doesn't like this, so reshape

image = image.reshape(224,224,3)
plt.imshow(image.numpy())

This method is displaying a 3 by 3 grid of the same image, always in greyscale. For example:

How do I fix this so that the single color image is displayed correctly?


title: How to predict a label in MultiClass classification model in pytorch?

body:
I am currently working on my  mini-project, where I predict movie genres based on their posters. So in the dataset that I have, each movie can have from 1 to 3 genres, therefore each instance can belong to multiple classes. I have total of 15 classes(15 genres). So now I am facing with the problem of how to do predictions  using pytorch for this particular problem.

In pytorch CIFAR-tutorial, where each instance can have only one class ( for example, if image is a car it should belong to class of cars) and there are 10 classes in total. So in this case, model prediction is defined in the following way(copying code snippet from pytorch website):

import torch.optim as optim

criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(net.parameters(), lr=0.001, momentum=0.9)

for epoch in range(2):  # loop over the dataset multiple times

running_loss = 0.0
for i, data in enumerate(trainloader, 0):
    # get the inputs
    inputs, labels = data

    # zero the parameter gradients
    optimizer.zero_grad()

    # forward + backward + optimize
    outputs = net(inputs)
    loss = criterion(outputs, labels)
    loss.backward()
    optimizer.step()

    # print statistics
    running_loss += loss.item()
    if i % 2000 == 1999:    # print every 2000 mini-batches
        print('[%d, %5d] loss: %.3f' %
              (epoch + 1, i + 1, running_loss / 2000))
        running_loss = 0.0

print('Finished Training')

Question 1(for training part). What could you suggest to use  as an activation function. I was thinking about BCEWithLogitsLoss() but I am not sure how good it will be.

and then the accuracy of prediction for testset is defined in the following way:
for the entire network:

correct = 0
total = 0
with torch.no_grad():
    for data in testloader:
        images, labels = data
        outputs = net(images)
        _, predicted = torch.max(outputs.data, 1)
        total += labels.size(0)
        correct += (predicted == labels).sum().item()

print('Accuracy of the network on the 10000 test images: %d %%' % (
    100 * correct / total))

and for each class:

class_correct = list(0. for i in range(10))
class_total = list(0. for i in range(10))
with torch.no_grad():
    for data in testloader:
        images, labels = data
        outputs = net(images)
        _, predicted = torch.max(outputs, 1)
        c = (predicted == labels).squeeze()
        for i in range(4):
            label = labels[i]
            class_correct[label] += c[i].item()
            class_total[label] += 1

for i in range(10):
    print('Accuracy of %5s : %2d %%' % (
        classes[i], 100 * class_correct[i] / class_total[i]))

where the output is as follows:

Accuracy of plane : 36 %
Accuracy of   car : 40 %
Accuracy of  bird : 30 %
Accuracy of   cat : 19 %
Accuracy of  deer : 28 %
Accuracy of   dog : 17 %
Accuracy of  frog : 34 %
Accuracy of horse : 43 %
Accuracy of  ship : 57 %
Accuracy of truck : 35 %

Now here is question 2:
How can I determine the accuracy so it would look in the following way:

For example:

The Matrix (1999)                ['Action: 91%', 'Drama: 25%', 'Adventure: 13%']
The Others (2001)                ['Drama: 76%', 'Horror: 65%', 'Action: 41%']
Alien: Resurrection (1997)       ['Horror: 67%', 'Action: 64%', 'Drama: 43%']
The Martian (2015)               ['Drama: 95%', 'Adventure: 81%']

Considering that every movie does not always have 3 genres, sometimes is 2 and sometimes is 1. So  as I see it, I should find 3 maximum values, 2 maximum values or 1 maximum value of my output list , which is list of 15 genres so, for example, if

my predicted genres are [Movie, Adventure] then

some_kind_of_function(outputs) should give me output of

[1 0 0 0 0 0 0 0 0 0 0 1 0 0 0] ,

which I can compare afterwards with ground_truth.
 I don't think torchmax will work in this case, cause it gives only one max value from [weigts array], so

What's the best way to implement it?

Thank you in advance, appreciate any help or suggestion:)


title: Pytorch - Getting gradient for intermediate variables / tensors

body:
As an exercice in pytorch framework (0.4.1) , I am trying to display the gradient of X (gX or dSdX) in a simple Linear layer (Z = X.W + B). To simplify my toy example, I backward() from a sum of Z (not a loss).

To sum up, I want gX(dSdX) of S=sum(XW+B).

The problem is that the gradient of Z (dSdZ) is None. As a result, gX is wrong too of course.

import torch
X = torch.tensor([[0.5, 0.3, 2.1], [0.2, 0.1, 1.1]], requires_grad=True)
W = torch.tensor([[2.1, 1.5], [-1.4, 0.5], [0.2, 1.1]])
B = torch.tensor([1.1, -0.3])
Z = torch.nn.functional.linear(X, weight=W.t(), bias=B)
S = torch.sum(Z)
S.backward()
print("Z:\n", Z)
print("gZ:\n", Z.grad)
print("gX:\n", X.grad)

Result:

Z:
 tensor([[2.1500, 2.9100],
        [1.6000, 1.2600]], grad_fn=<ThAddmmBackward>)
gZ:
 None
gX:
 tensor([[ 3.6000, -0.9000,  1.3000],
        [ 3.6000, -0.9000,  1.3000]])

I have exactly the same result if I use nn.Module as below:

class Net1Linear(torch.nn.Module):
    def __init__(self, wi, wo,W,B):
        super(Net1Linear, self).__init__()
        self.linear1 = torch.nn.Linear(wi, wo)
        self.linear1.weight = torch.nn.Parameter(W.t())
        self.linear1.bias = torch.nn.Parameter(B)
    def forward(self, x):
        return self.linear1(x)
net = Net1Linear(3,2,W,B)
Z = net(X)
S = torch.sum(Z)
S.backward()
print("Z:\n", Z)
print("gZ:\n", Z.grad)
print("gX:\n", X.grad)


title: PyTorch: What's the difference between state_dict and parameters()?

body:
In order to access a model's parameters in pytorch, I saw two methods:
using state_dict and using parameters()
I wonder what's the difference, or if one is good practice and the other is bad practice.
Thanks


title: when is a pytorch custom function needed (rather than only a module)?

body:
Pytorch beginner here! Consider the following custom Module:

class Testme(nn.Module):
    def __init__(self):
        super(Testme, self).__init__()

def forward(self, x):
    return x / t_.max(x).expand_as(x)

As far as I understand the documentation:
I believe this could also be implemented as a custom Function.  A subclass of Function requires a backward() method, but the
Module does not.  As well, in the doc example of a Linear Module, it depends on a Linear Function:

class Linear(nn.Module):
    def __init__(self, input_features, output_features, bias=True):
        ...
    def forward(self, input):
        return Linear()(input, self.weight, self.bias)

The question: I do not understand the relation between Module and Function. In the first listing above (module Testme), should it have an associated function? If not, then it is possible to implement this without a backward method by subclassing Module, so why does a Function always require a backward method?

Perhaps Functions are intended only for functions that are not composed out of existing torch functions? Say differently: maybe modules do not need the associated Function if their forward method is composed entirely from previously defined torch functions?


title: How to use Pytorch OneCycleLR in a training loop (and optimizer/scheduler interactions)?

body:
I'm training an NN and using RMSprop as an optimizer and OneCycleLR as a scheduler.  I've been running it like this (in slightly simplified code):

optimizer = torch.optim.RMSprop(model.parameters(), lr=0.00001,
                              alpha=0.99, eps=1e-08, weight_decay=0.0001, momentum=0.0001, centered=False)
scheduler = torch.optim.lr_scheduler.OneCycleLR(optimizer, max_lr=0.0005, epochs=epochs)

    for epoch in range(epochs):
        model.train()
        for counter, (images, targets) in enumerate(train_loader):

            # clear gradients from last run
            optimizer.zero_grad()

            # Run forward pass through the mini-batch
            outputs = model(images)

            # Calculate the losses
            loss = loss_fn(outputs, targets)

            # Calculate the gradients
            loss.backward()

            # Update parameters
            optimizer.step()   # Optimizer before scheduler????
            scheduler.step()

            # Check loss on training set
            test()

Note the optimizer and scheduler calls in each mini-batch.  This is working, though when I plot the learning rates through the training, the curve is very bumpy.  I checked the docs again, and this is the example shown for torch.optim.lr_scheduler.OneCycleLR

>>> data_loader = torch.utils.data.DataLoader(...)
>>> optimizer = torch.optim.SGD(model.parameters(), lr=0.1, momentum=0.9)
>>> scheduler = torch.optim.lr_scheduler.OneCycleLR(optimizer, max_lr=0.01, steps_per_epoch=len(data_loader), epochs=10)
>>> for epoch in range(10):
>>>     for batch in data_loader:
>>>         train_batch(...)
>>>         scheduler.step()

Here, they omit the optimizer.step() in the training loop. And I thought, that makes sense since the optimizer is provided to OneCycleLR in its initialization, so it must be taking care of that on the back end.  But doing so gets me the warning:

UserWarning: Detected call of `lr_scheduler.step()` before `optimizer.step()`. In PyTorch 1.1.0 and later, you should call them in the opposite order: `optimizer.step()` before `lr_scheduler.step()`.

Do I ignore that and trust the pseudocode in the docs?  Well, I did, and the model didn't do any learning, so the warning is correct and I put optimizer.step() back in.

This gets to the point that I don't really understand how the optimizer and scheduler interact (edit: how the Learning Rate in the optimizer interacts with the Learning Rate in the scheduler).  I see that generally the optimizer is run every mini-batch and the scheduler every epoch, though for OneCycleLR, they want you to run it every mini-batch too.

Any guidance (or a good tutorial article) would be appreciated!


I am getting this error only during the testing phase, but I do not face any problem in the training and validation phase.

IndexError: tensors used as indices must be long, byte or bool tensors

I get this error for the last line in the given code snippet.

The code snippet looks like the one below,

NumClass = 10
mask = torch.zeros(batch_size, self.mem_dim, 4, 4)
ones = torch.ones(1, 4, 4)
NumRows = self.mem_dim
Elements = NumRows//NumClass
for i in range(batch_size):
    lab = torch.arange(Elements * label[i], Elements*(label[i]+1), 1)
    mask[i,lab] = ones

The "lab" is a tensor value and prints out the range in such a way,

tensor([6, 7, 8])
tensor([ 9, 10, 11])
tensor([21, 22, 23])

(Note*: the length of this lab tensor can be of length 'n' based on the value of ElementsPerClass)


title: RuntimeError: size mismatch, m1: [192 x 68], m2: [1024 x 68] at /opt/conda/conda-bld/pytorch_/work/aten/src/THC/generic/THCTensorMathBlas.cu:268

body:
I'm getting a size mismatch error that I can't understand.

(Pdb) self.W_di
Linear(in_features=68, out_features=1024, bias=True)
(Pdb) indices.size()
torch.Size([32, 6, 68])
(Pdb) self.W_di(indices)
*** RuntimeError: size mismatch, m1: [192 x 68], m2: [1024 x 68] at /opt/conda/conda-bld/pytorch_1556653099582/work/aten/src/THC/generic/THCTensorMathBlas.cu:268

Why is there a mismatch?
Maybe because of the way I defined the weight in forward (instead of _init_)?

This is how I defined self.W_di:

def forward(self):

    if self.W_di is None:
        self.W_di_weight = nn.Parameter(torch.randn(mL_n * 2,1024).to(device))
        self.W_di_bias = nn.Parameter(torch.ones(1024).to(device))

    self.W_di = nn.Linear(mL_n * 2, 1024)
    self.W_di.weight = self.W_di_weight
    self.W_di.bias = self.W_di_bias

    result = self.W_di(indices)

Any pointer would be highly appreciated!


title: Pytorch model weight type conversion

body:
I'm trying to do inference on FlowNet2-C model loading from file.
However, I met some data type problem.  How can I resolve it?

Source code
FlowNet2-C pre-trained model

$ python main.py

Initializing Datasets
  [0.000s] Loading checkpoint '/notebooks/data/model/FlowNet2-C_checkpoint.pth.tar'
  [1.293s] Loaded checkpoint '/notebooks/data/model/FlowNet2-C_checkpoint.pth.tar' (at epoch 0)
(1L, 6L, 384L, 512L)
<class 'torch.autograd.variable.Variable'>
  [1.642s] Operation failed

Traceback (most recent call last):
  File "main.py", line 102, in <module>
    main()
  File "main.py", line 98, in main
    summary(input_size, model)
  File "main.py", line 61, in summary
    model(x)
  File "/usr/local/lib/python2.7/dist-packages/torch/nn/modules/module.py", line 357, in __call__
    result = self.forward(*input, **kwargs)
  File "/notebooks/data/vinet/FlowNetC.py", line 75, in forward
    out_conv1a = self.conv1(x1)
  File "/usr/local/lib/python2.7/dist-packages/torch/nn/modules/module.py", line 357, in __call__
    result = self.forward(*input, **kwargs)
  File "/usr/local/lib/python2.7/dist-packages/torch/nn/modules/container.py", line 67, in forward
    input = module(input)
  File "/usr/local/lib/python2.7/dist-packages/torch/nn/modules/module.py", line 357, in __call__
    result = self.forward(*input, **kwargs)
  File "/usr/local/lib/python2.7/dist-packages/torch/nn/modules/conv.py", line 282, in forward
    self.padding, self.dilation, self.groups)
  File "/usr/local/lib/python2.7/dist-packages/torch/nn/functional.py", line 90, in conv2d
    return f(input, weight, bias)
RuntimeError: Input type (CUDAFloatTensor) and weight type (CPUFloatTensor) should be the same


title: NameError: name 'transforms' is not defined, while using google colab

body:
train = datasets.MNIST("", train=True, download=True,transform = transforms.Compose([transforms.ToTensor()]))
test = datasets.MNIST("", train=False, download=True,transform = transforms.Compose([transforms.ToTensor()]))`

After executing this on colab notebook, I am getting this error:

Traceback (most recent call last)
<ipython-input-6-b81aa6cf1cbe> in <module>()
----> 1 train = datasets.MNIST("", train=True, download=True,transform = transforms.Compose([transforms.ToTensor()]))
      2 test = datasets.MNIST("", train=False, download=True,transform = transforms.Compose([transforms.ToTensor()]))

NameError: name 'transforms' is not defined*


I am doing the following operation,
energy.masked_fill(mask == 0, float("-1e20"))

my python traces are below,
    File "/usr/local/lib/python3.6/dist-packages/torch/nn/modules/module.py", line 722, in _call_impl
    result = self.forward(*input, **kwargs)
  File "seq_sum.py", line 418, in forward
    enc_src = self.encoder(src, src_mask)
  File "/usr/local/lib/python3.6/dist-packages/torch/nn/modules/module.py", line 722, in _call_impl
    result = self.forward(*input, **kwargs)
  File "seq_sum.py", line 71, in forward
    src = layer(src, src_mask)
  File "/usr/local/lib/python3.6/dist-packages/torch/nn/modules/module.py", line 722, in _call_impl
    result = self.forward(*input, **kwargs)
  File "seq_sum.py", line 110, in forward
    _src, _ = self.self_attention(src, src, src, src_mask)
  File "/usr/local/lib/python3.6/dist-packages/torch/nn/modules/module.py", line 722, in _call_impl
    result = self.forward(*input, **kwargs)
  File "seq_sum.py", line 191, in forward
    energy =  energy.masked_fill(mask == 0, float("-1e20"))
RuntimeError: The size of tensor a (1024) must match the size of tensor b (512) at non-singleton dimension 3

These are my attention layers code,
    Q = self.fc_q(query)
    K = self.fc_k(key)
    V = self.fc_v(value)

    #Q = [batch size, query len, hid dim]
    #K = [batch size, key len, hid dim]
    #V = [batch size, value len, hid dim]

    # Q = Q.view(batch_size, -1, self.n_heads, self.head_dim).permute(0, 2, 1, 3)
    # K = K.view(batch_size, -1, self.n_heads, self.head_dim).permute(0, 2, 1, 3)
    # V = V.view(batch_size, -1, self.n_heads, self.head_dim).permute(0, 2, 1, 3)

    Q = Q.view(batch_size, -1, self.n_heads, self.head_dim).view(-1, 1024)
    K = K.view(batch_size, -1, self.n_heads, self.head_dim).view(-1, 1024)
    V = V.view(batch_size, -1, self.n_heads, self.head_dim).view(-1, 1024)
    energy = torch.matmul(Q, K.transpose(1,0)) / self.scale

I am following below github code to do my seq to seq operation,seq2seq pytorch
actual testing code is available on the below location, code to test a seq of 1024 to 1024 output
2nd example tried here I have commented out pos_embedding due CUDA error with large index (RuntimeError: cuda runtime error (59)


title: How to implement my own ResNet with torch.nn.Sequential in Pytorch?

body:
I want to implement a ResNet network (or rather, residual blocks) but I really want it to be in the sequential network form.

What I mean by sequential network form is the following:

## mdl5, from cifar10 tutorial
mdl5 = nn.Sequential(OrderedDict([
    ('pool1', nn.MaxPool2d(2, 2)),
    ('relu1', nn.ReLU()),
    ('conv1', nn.Conv2d(3, 6, 5)),
    ('pool1', nn.MaxPool2d(2, 2)),
    ('relu2', nn.ReLU()),
    ('conv2', nn.Conv2d(6, 16, 5)),
    ('relu2', nn.ReLU()),
    ('Flatten', Flatten()),
    ('fc1', nn.Linear(1024, 120)), # figure out equation properly
    ('relu4', nn.ReLU()),
    ('fc2', nn.Linear(120, 84)),
    ('relu5', nn.ReLU()),
    ('fc3', nn.Linear(84, 10))
]))

but of course with the NN lego blocks being “ResNet”.

I know the equation is something like:

but I am not sure how to do it in Pytorch AND Sequential. Sequential is key for me!

Cross-posted:

https://discuss.pytorch.org/t/how-to-have-residual-network-using-only-sequential-blocks/51541
https://www.quora.com/unanswered/How-does-one-implement-my-own-ResNet-with-torch-nn-Sequential-in-Pytorch


title: PyTorch: How do the means and stds get calculated in the Transfer Learning tutorial?

body:
I'm going through the PyTorch Transfer Learning tutorial at: link

In the data augmentation stage, there is the following step to normalize images:

transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])

I can understand why it's doing this but I can't find how the mean and std values get calculated? I tried to calculate the mean on the train data set and the mean values are:

array([ 0.11727478,  0.04542569, -0.28624609], dtype=float32)


title: TypeError: conv2d(): argument ‘input’ (position 1) must be Tensor, not PackedSequence

body:
I'm trying to apply a conv2d layer to a batch of sequence data where the sequences don't have the same length. I tried passing a packed sequence but that doesn't work...

import torch
import torch.nn as nn
conv1 = nn.Conv2d(1, 16, 5)

x = torch.randn([16,1580,201]) # Where x has dimensions (batch*sequence*feats)
lengths = torch.tensor(
    [1580, 959, 896, 881, 881, 881, 881, 881, 881, 881, 881, 881, 881, 335, 254, 219]
)
# So only the first element in the batch has size 1580, the rest have been padded

x_pack = torch.nn.utils.rnn.pack_padded_sequence(x, lengths, batch_first=True)

x_pack = conv1(x_pack)
print(x_pack)

# ... then feed x_pack to an LSTM...

Does anyone know how to not apply the convolution on the padded values of the inputs? will I have to iterate though each element of the batch?

Thanks.


title: Transforms not applying to the dataset

body:
I'm new to pytorch and would like to understand something.

I am loading MNIST as follows:

transform_train = transforms.Compose(
    [transforms.ToTensor(),
     transforms.Resize(size, interpolation=2),
     # transforms.Grayscale(num_output_channels=1),
     transforms.RandomHorizontalFlip(p=0.5),
     transforms.Normalize((mean), (std))])

trainset = torchvision.datasets.MNIST(root='./data', train=True,
                                      download=True, transform=transform_train)
trainloader = torch.utils.data.DataLoader(trainset, batch_size=batch_size,
                                          shuffle=True, num_workers=2)

However, when I explore the dataset, i.e. trainloader.dataset.train_data[0], I am getting a tensor in range [0,255] with shape (28,28).

What am I missing? Is this because the transforms are not applied directly to the dataloader but only in runtime? How can I explore my data otherwise?


title: HTTP Error when trying to download MNIST data

body:
I am using Google Colab for training a LeNet-300-100 fully-connected neural network on MNIST using Python3 and PyTorch 1.8.
To apply the transformations and download the MNIST dataset, the following code is being used:
# MNIST dataset statistics:
# mean = tensor([0.1307]) & std dev = tensor([0.3081])
mean = np.array([0.1307])
std_dev = np.array([0.3081])

transforms_apply = transforms.Compose([
    transforms.ToTensor(),
    transforms.Normalize(mean = mean, std = std_dev)
    ])

which gives the error:

Downloading
http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz to
./data/MNIST/raw/train-images-idx3-ubyte.gz
--------------------------------------------------------------------------- HTTPError                                 Traceback (most recent call
last)  in ()
2 train_dataset = torchvision.datasets.MNIST(
3         root = './data', train = True,
----> 4         transform = transforms_apply, download = True
5         )
6
11 frames /usr/lib/python3.7/urllib/request.py in
http_error_default(self, req, fp, code, msg, hdrs)
647 class HTTPDefaultErrorHandler(BaseHandler):
648     def http_error_default(self, req, fp, code, msg, hdrs):
--> 649         raise HTTPError(req.full_url, code, msg, hdrs, fp)
650
651 class HTTPRedirectHandler(BaseHandler):
HTTPError: HTTP Error 503: Service Unavailable

What's wrong?


title: how to concatenate embedding layer in pytorch

body:
I am trying to concatenate embedding layer with other features. It doesn’t give me any error, but doesn’t do any training either. Is anything wrong with this model definition, how to debug this?

Note: The last column (feature) in my X is feature with word2ix (single word).
Note: The net works fine without the embedding feature/layer

originally posted on pytorch forum

class Net(torch.nn.Module):
    def __init__(self, n_features, h_sizes, num_words, embed_dim, out_size, dropout=None):
        super().__init__()

        self.num_layers = len(h_sizes)  # hidden + input

        self.embedding = torch.nn.Embedding(num_words, embed_dim)
        self.hidden = torch.nn.ModuleList()
        self.bnorm = torch.nn.ModuleList()
        if dropout is not None:
            self.dropout = torch.nn.ModuleList()
        else:
            self.dropout = None
        for k in range(len(h_sizes)):
            if k == 0:
                self.hidden.append(torch.nn.Linear(n_features, h_sizes[0]))
                self.bnorm.append(torch.nn.BatchNorm1d(h_sizes[0]))
                if self.dropout is not None:
                    self.dropout.append(torch.nn.Dropout(p=dropout))

            else:
                if k == 1:
                    input_dim = h_sizes[0] + embed_dim
                else:
                    input_dim = h_sizes[k-1]

                self.hidden.append(torch.nn.Linear(input_dim, h_sizes[k]))
                self.bnorm.append(torch.nn.BatchNorm1d(h_sizes[k]))
                if self.dropout is not None:
                    self.dropout.append(torch.nn.Dropout(p=dropout))

        # Output layer
        self.out = torch.nn.Linear(h_sizes[-1], out_size)

    def forward(self, inputs):

        # Feedforward

        for l in range(self.num_layers):
            if l == 0:
                x = self.hidden[l](inputs[:, :-1])
                x = self.bnorm[l](x)
                if self.dropout is not None:
                    x= self.dropout[l](x)

                embeds = self.embedding(inputs[:,-1])#.view((1, -1)
                x = torch.cat((embeds, x),dim=1)

            else:
                x = self.hidden[l](x)
                x = self.bnorm[l](x)
                if self.dropout is not None:
                    x = self.dropout[l](x)
            x = F.relu(x)
        output= self.out(x)

        return output


title: tensorboard colab tensorflow._api.v1.io.gfile' has no attribute 'get_filesystem

body:
I am trying to use tensorboard on colab. I manage to make it work, but not for all commands. add_graph and add_scalar works, but when I tried to run add_embedding I am getting the following error:

AttributeError: module 'tensorflow._api.v1.io.gfile' has no attribute 'get_filesystem'

This is the relevant code (I think);

import os
from torch.utils.tensorboard import SummaryWriter
writer = SummaryWriter(log_dir ="logs" )

images, labels = select_n_random(trainset.data, trainset.targets)
images = torch.from_numpy(images)
labels = torch.from_numpy(np.array(labels))
class_labels = [classes[lab] for lab in labels]

# log embeddings
features = images.reshape((-1,32*32*3))
writer.add_embedding(features,metadata=class_labels) #, label_img=images.unsqueeze(1))

The complete error is:

/tensorflow-1.15.0/python3.6/tensorflow_core/python/util/module_wrapper.py in __getattr__(self, name)
    191   def __getattr__(self, name):
    192     try:
--> 193       attr = getattr(self._tfmw_wrapped_module, name)
    194     except AttributeError:
    195       if not self._tfmw_public_apis:

AttributeError: module 'tensorflow._api.v1.io.gfile' has no attribute 'get_filesystem'

using

tensorflow-1.15.0 (tried to install 2.0 but had different issues)
Python 3.6.9
torch 1.4.0
tensorboard 2.1.1 (tried also with 1.15.0 but same issue)

I also tried using the "magic" command:

%load_ext tensorboard
%tensorboard --logdir logs

But I wasn't able to make it work that way (other issues).

Any suggestions how can I make it work?


I'm trying to run  this https://github.com/shariqiqbal2810/MAAC repository and it has a module called torch
import torch as McLawrence
from torch.autograd import Variable

I'm using python version 3.7.1 and I downgraded to 3.6.5 on win10
I tried to use
pip install torch
pip install pytorch
pip install torchvision

I change torch to pytorch
I enter their website https://pytorch.org/ and I tried all suggestion that uses pip such as
pip3 install https://download.pytorch.org/whl/cu80/torch-1.0.0-cp36-cp36m-win_amd64.whl
pip3 install torchvision

I read every comment relate to this issue in
https://github.com/pytorch/pytorch/issues/566
I read most of the answer here
https://stackoverflow.com/search?q=module+named+torch
But I still get the same message:
Collecting torch
  Using cached https://files.pythonhosted.org/packages/5f/e9/bac4204fe9cb1a002ec6140b47f51affda1655379fe302a1caef421f9846/torch-0.1.2.post1.tar.gz
    Complete output from command python setup.py egg_info:
    Traceback (most recent call last):
      File "<string>", line 1, in <module>
      File "C:\Users\iibra\AppData\Local\Temp\pip-install-yupbt_qk\torch\setup.py", line 11, in <module>
        raise RuntimeError(README)
    RuntimeError: PyTorch does not currently provide packages for PyPI (see status at https://github.com/pytorch/pytorch/issues/566).

    Please follow the instructions at http://pytorch.org/ to install with miniconda instead.

pip install pytorch
Collecting pytorch
  Using cached https://files.pythonhosted.org/packages/a9/41/4487bc23e3ac4d674943176f5aa309427b011e00607eb98899e9d951f67b/pytorch-0.1.2.tar.gz
Building wheels for collected packages: pytorch
  Running setup.py bdist_wheel for pytorch ... error
  Complete output from command c:\users\iibra\appdata\local\programs\python\python36\python.exe -u -c "import setuptools, tokenize;__file__='C:\\Users\\iibra\\AppData\\Local\\Temp\\pip-install-7qwe1571\\pytorch\\setup.py';f=getattr(tokenize, 'open', open)(__file__);code=f.read().replace('\r\n', '\n');f.close();exec(compile(code, __file__, 'exec'))" bdist_wheel -d C:\Users\iibra\AppData\Local\Temp\pip-wheel-_mjh_olk --python-tag cp36:
  Traceback (most recent call last):
    File "<string>", line 1, in <module>
    File "C:\Users\iibra\AppData\Local\Temp\pip-install-7qwe1571\pytorch\setup.py", line 17, in <module>
      raise Exception(message)
  Exception: You should install pytorch from http://pytorch.org

pip install torchvision
Collecting torchvision
  Using cached https://files.pythonhosted.org/packages/ca/0d/f00b2885711e08bd71242ebe7b96561e6f6d01fdb4b9dcf4d37e2e13c5e1/torchvision-0.2.1-py2.py3-none-any.whl
Requirement already satisfied: six in c:\users\iibra\appdata\local\programs\python\python36\lib\site-packages (from torchvision) (1.11.0)
Requirement already satisfied: pillow>=4.1.1 in c:\users\iibra\appdata\local\programs\python\python36\lib\site-packages (from torchvision) (5.1.0)
Collecting torch (from torchvision)
  Using cached https://files.pythonhosted.org/packages/5f/e9/bac4204fe9cb1a002ec6140b47f51affda1655379fe302a1caef421f9846/torch-0.1.2.post1.tar.gz
    Complete output from command python setup.py egg_info:
    Traceback (most recent call last):
      File "<string>", line 1, in <module>
      File "C:\Users\iibra\AppData\Local\Temp\pip-install-k8dl2vhz\torch\setup.py", line 11, in <module>
        raise RuntimeError(README)
    RuntimeError: PyTorch does not currently provide packages for PyPI (see status at https://github.com/pytorch/pytorch/issues/566).

    Please follow the instructions at http://pytorch.org/ to install with miniconda instead.


title: One-hot encoding in pytorch/torchtext

body:
I have a Bucketiterator from torchtext that I feed to a model in pytorch. An example of how the iterator is constructed:

train_iter, val_iter = BucketIterator.splits((train,val),
                                             batch_size=batch_size,
                                             sort_within_batch = True,
                                             device = device,
                                             shuffle=True,
                                             sort_key=lambda x: (len(x.src), len(x.trg)))

The data is then fed to a model like this, where I use the nn.Embedding layer.

class encoder(nn.Module):
    def __init__(self, input_dim, emb_dim, hid_dim, n_layers, dropout):
        super().__init__()

        self.input_dim = input_dim
        self.emb_dim = emb_dim
        self.hid_dim = hid_dim
        self.n_layers = n_layers
        self.dropout = dropout

        self.embedding = nn.Embedding(input_dim, emb_dim)

        self.rnn = nn.LSTM(emb_dim, hid_dim, n_layers, dropout = dropout)

        self.dropout = nn.Dropout(dropout)

    def forward(self, src):

        #src = [src sent len, batch size]

        embedded = self.dropout(self.embedding(src))

        #embedded = [src sent len, batch size, emb dim]
        hidden_enc = []
        outputs, hidden = self.rnn(embedded[0,:,:].unsqueeze(0))
        for i in range(1,len(embedded[:,1,1])):
            outputs, hidden = self.rnn(embedded[i,:,:].unsqueeze(0),hidden)
            hidden_cpu = []
            for k in range(len(hidden)):
                hidden_cpu.append(hidden[k])
                hidden_cpu[k] = hidden[k].cpu()
            hidden_enc.append(tuple(hidden_cpu))

        #outputs, hidden = self.rnn(embedded)

        #outputs = [src sent len, batch size, hid dim * n directions]
        #hidden = [n layers * n directions, batch size, hid dim]
        #cell = [n layers * n directions, batch size, hid dim]
        None
        #outputs are always from the top hidden layer

        return hidden, hidden_enc

But what if I wanted the embedding to be one-hot encoded? I work on formal languages and it would be nice to preserve orthogonality between tokens. It doesn't seem like pytorch or torchtext has any functionality for doing this.


title: How to visualise filters in a CNN with PyTorch

body:
I'm new to deep learning and Pytorch. I want to visual my filter in my CNN model so that I can iterate layer in the CNN model that I define. But I meet error like below.
error: 'CNN' object is not iterable
The CNN object is my model.
My iteration code is like below：
for index, layer in enumerate(self.model):
# Forward pass layer by layer
    x = layer(x)

my model code like below：
class CNN(nn.Module):
    def __init__(self):
        super(CNN,self).__init__()
        self.Conv1 = nn.Sequential( # input image size (1,28,20)
            nn.Conv2d(1, 16, 5, 1, 2), # outputize (16,28,20)
            nn.ReLU(),
            nn.MaxPool2d(2),           #outputize (16,14,10)
        )
        self.Conv2 = nn.Sequential( # input ize ? (16,,14,10)
            nn.Conv2d(16, 32, 5, 1, 2),   #output size(32,14,10)
            nn.ReLU(),
            nn.MaxPool2d(2),        #output size (32,7,5)
        )
        self.fc1 = nn.Linear(32 * 7 * 5, 800)
        self.fc2 = nn.Linear(800,500)
        self.fc3 = nn.Linear(500,10)
        #self.fc4 = nn.Linear(200,10)

    def forward(self,x):
        x = self.Conv1(x)
        x = self.Conv2(x)
        x = x.view(x.size(0), -1)
        x = self.fc1(x)
        x = F.dropout(x)
        x = F.relu(x)
        x = self.fc2(x)
        x = F.dropout(x)
        x = F.relu(x)
        x = self.fc3(x)
        #x = F.relu(x)
        #x = self.fc4(x)
        return x

So anyone can tell me how can I solve this problem.


title: Is there any way to convert pytorch tensor to tensorflow tensor

body:
https://github.com/taoshen58/BiBloSA/blob/ec67cbdc411278dd29e8888e9fd6451695efc26c/context_fusion/self_attn.py#L29

I need to use mulit_dimensional_attention from the above link which is implemented in TensorFlow but I am using PyTorch so can I Convert Pytorch Tensor to TensorFlow Tensor or I have to implement it in PyTorch.

code which I am trying to use here I have to pass 'rep_tensor' as TensorFlow tensor type but I have PyTorch tensor

def multi_dimensional_attention(rep_tensor, rep_mask=None, scope=None,
                                keep_prob=1., is_train=None, wd=0., activation='elu',
                                tensor_dict=None, name=None):

    # bs, sl, vec = tf.shape(rep_tensor)[0], tf.shape(rep_tensor)[1], tf.shape(rep_tensor)[2]

    ivec = rep_tensor.shape[2]
    with tf.variable_scope(scope or 'multi_dimensional_attention'):
        map1 = bn_dense_layer(rep_tensor, ivec, True, 0., 'bn_dense_map1', activation,
                              False, wd, keep_prob, is_train)
        map2 = bn_dense_layer(map1, ivec, True, 0., 'bn_dense_map2', 'linear',
                              False, wd, keep_prob, is_train)
        # map2_masked = exp_mask_for_high_rank(map2, rep_mask)

        soft = tf.nn.softmax(map2, 1)  # bs,sl,vec
        attn_output = tf.reduce_sum(soft * rep_tensor, 1)  # bs, vec

        # save attn
        if tensor_dict is not None and name is not None:
            tensor_dict[name] = soft

        return attn_output


title: Difference between nn.MaxPool2d vs.nn.functional.max_pool2d?

body:
Whats the difference between: nn.MaxPool2d(kernel_size, stride) and nn.functional.max_pool2d(t, kernel_size, stride)?
The first one I define in the module and the second in the forward function?
Thanks


title: how to save torchtext Dataset?

body:
I'm working with text and use torchtext.data.Dataset.
Creating the dataset takes a considerable amount of time.
For just running the program this is still acceptable. But I would like to debug the torch code for the neural network. And if python is started in debug mode, the dataset creation takes roughly 20 minutes (!!). That's just to get a working environment where I can debug-step through the neural network code.

I would like to save the Dataset, for example with pickle. This sample code is taken from here, but I removed everything that is not necessary for this example:

from torchtext import data
from fastai.nlp import *

PATH = 'data/aclImdb/'

TRN_PATH = 'train/all/'
VAL_PATH = 'test/all/'
TRN = f'{PATH}{TRN_PATH}'
VAL = f'{PATH}{VAL_PATH}'

TEXT = data.Field(lower=True, tokenize="spacy")

bs = 64;
bptt = 70

FILES = dict(train=TRN_PATH, validation=VAL_PATH, test=VAL_PATH)
md = LanguageModelData.from_text_files(PATH, TEXT, **FILES, bs=bs, bptt=bptt, min_freq=10)

with open("md.pkl", "wb") as file:
    pickle.dump(md, file)

To run the code, you need the aclImdb dataset, it can be downloaded from here. Extract it into a data/ folder next to this code snippet. The code produces an error in the last line, where pickle is used:

Traceback (most recent call last):
  File "/home/lhk/programming/fastai_sandbox/lesson4-imdb2.py", line 27, in <module>
    pickle.dump(md, file)
TypeError: 'generator' object is not callable

The samples from fastai often use dill instead of pickle. But that doesn't work for me either.


title: How a robust background removal is implemented?

body:
I found that a deep-learning-based method (e.g., 1) is much more robust than a non-deep-learning-based method (e.g., 2, using OpenCV).

https://www.remove.bg
How do I remove the background from this kind of image?

In the OpenCV example, Canny is used to detect the edges. But this step can be very sensitive to the image. The contour detection may end up with wrong contours. It is also difficult to determine wich contours should be kept.

How a robust deep-learning method is implemented? Is any good example code? Thanks.


title: How to convert numpy array into LongTensor type Variable in PyTorch?

body:
I am trying to convert numpy array into PyTorch LongTensor type Variable as follows:

import numpy as np
import torch as th

y = np.array([1., 1., 1.1478225, 1.1478225, 0.8521775, 0.8521775, 0.4434675])
yth = Variable(th.from_numpy(y)).type(torch.LongTensor)

However the result I am getting is a rounded off version:

tensor([ 1,  1,  1,  1,  0,  0,  0])

How can I keep the precision of numpy array while getting LongTensor variable?

Expected result should be:

tensor([1., 1., 1.1478225, 1.1478225, 0.8521775, 0.8521775, 0.4434675])


title: How to convert FloatTensor to ByteTensor with Pytorch?

body:
I'm new to Pytorch and neural network programming but I've an issue I encountered and I'm not able to solve it on my own. My data are numpy arrays of 1 and 0. But when I try to train my net, I get this error :

  RuntimeError: Expected object of type torch.ByteTensor but found type torch.FloatTensor for argument #2 'mat2'

the line where the error comes from is in the forward method of my net

x = self.fc1(x)

I've tried these to convert my tensors but I still get the error :

  x = x.type('torch.ByteTensor')

and

  x.byte()


title: How to upscale image in pytorch?

body:
how to upscale an image in Pytorch without defining height and width using transforms?
 ('--upscale_factor', type=int, required=True, help="super resolution upscale factor")


title: Beginner PyTorch - RuntimeError: shape '[16, 400]' is invalid for input of size 9600

body:
I'm trying to build a CNN but I get this error:

---> 52         x = x.view(x.size(0), 5 * 5 * 16)
RuntimeError: shape '[16, 400]' is invalid for input of size 9600

It's not clear for me what the inputs of the 'x.view' line should be. Also, I don't really understand how many times I should have this 'x.view' function in my code. Is it only once, after the 3 convolutional layers and 2 linear layers? Or is it 5 times, one after every layer?

Here's my code:

CNN

import torch.nn.functional as F

# Convolutional neural network
class ConvNet(nn.Module):

    def __init__(self, num_classes=10):
        super(ConvNet, self).__init__()

        self.conv1 = nn.Conv2d(
            in_channels=3,
            out_channels=16,
            kernel_size=3)

        self.conv2 = nn.Conv2d(
            in_channels=16,
            out_channels=24,
            kernel_size=4)

        self.conv3 = nn.Conv2d(
            in_channels=24,
            out_channels=32,
            kernel_size=4)

        self.dropout = nn.Dropout2d(p=0.3)

        self.pool = nn.MaxPool2d(2)

        self.fc1 = nn.Linear(16 * 5 * 5, 120)
        self.fc2 = nn.Linear(512, 10)

        self.final = nn.Softmax(dim=1)

    def forward(self, x):

        print('shape 0 ' + str(x.shape))

        x = F.max_pool2d(F.relu(self.conv1(x)), 2)
        x = self.dropout(x)

        print('shape 1 ' + str(x.shape))

        x = F.max_pool2d(F.relu(self.conv2(x)), 2)
        x = self.dropout(x)

        print('shape 2 ' + str(x.shape))

        # x = F.max_pool2d(F.relu(self.conv3(x)), 2)
        # x = self.dropout(x)

        x = F.interpolate(x, size=(5, 5))
        x = x.view(x.size(0), 5 * 5 * 16)

        x = self.fc1(x)

        return x

net = ConvNet()

Can someone help me understand the problem?

The output of 'x.shape' is:

shape 0 torch.Size([16, 3, 256, 256])

shape 1 torch.Size([16, 16, 127, 127])

shape 2 torch.Size([16, 24, 62, 62])

Thanks


title: Why does the pytorch Docker image not come with torch?

body:
I am trying to build a docker container that will enable me to run my code that requires the python torch module.

I have chosen to start my image from the pytorch/pytorch:latest base image and add a few required modules manually. The build, push, and pull to the remote server were successful (no error messages at least).

Currently my Dockerfile looks like this:

FROM pytorch/pytorch:latest

RUN apt-get update \
     && apt-get install -y \
        libgl1-mesa-glx \
        libx11-xcb1 \
     && apt-get clean all \
     && rm -r /var/lib/apt/lists/*

RUN /opt/conda/bin/conda install --yes \
    astropy \
    matplotlib \
    pandas \
    glob2 \
    PIL \
    scikit-learn \
    scikit-image \
    numpy

However, when running my python script within the container, I get ImportError: No module named torch. This strikes me as rather weird, as it leads me to assume that the pytorch base image does not include the torch module...?

I have nevertheless tried to add torch to the list of modules to install in the Dockerfile, but then the initial build will already fail with the error message PackagesNotFoundError: The following packages are not available from current channels:  - torch. Following the advice given here did unfortunately not help me.

I'd appreciate any explanation as to why the torch module could not be found within the built container, and of course any help to fix this problem! Thanks!


title: Iterating over Torchtext.data.BucketIterator object throws AttributeError 'Field' object has no attribute 'vocab'

body:
When I try to look into a batch, by printing the next iteration of the BucketIterator object, the AttributeError is thrown.

tv_datafields=[("Tweet",TEXT), ("Anger",LABEL), ("Fear",LABEL), ("Joy",LABEL), ("Sadness",LABEL)]
train, vld = data.TabularDataset.splits(path="./data/", train="train.csv",validation="test.csv",format="csv", fields=tv_datafields)

train_iter, val_iter = BucketIterator.splits(
(train, vld),
batch_sizes=(64, 64),
device=-1,
sort_key=lambda x: len(x.Tweet),
sort_within_batch=False,
repeat=False
)
print(next(iter(train_dl)))


title: what does padding_idx do in nn.embeddings()

body:
I'm learning pytorch and
I'm wondering what does the padding_idx attribute do in torch.nn.Embedding(n1, d1, padding_idx=0)?
I have looked everywhere and couldn't find something I can get.
Can you show example to illustrate this?


title: How to pre-load numpy data into a buffer like io.BytesIO to make it seekable?

body:
Following function basically returns numpy.ndarray

def getimage(id):
     img = self.coco.loadImgs(id)
     I = io.imread(img['coco_url'])
     return I #returns 'numpy.ndarray'

The getimage function being called from main:

x = load.getimage(id).
x = torch.load(x)

Error thrown:

'numpy.ndarray' object has no attribute 'seek'. You can only torch.load from a file that is seekable. Please pre-load the data into a buffer like io.BytesIO and try to load from it instead.


title: How To downgrade Torch Version for Google Colab

body: I would like to downgrade the Torch version used in my Google Colab notebooks. How could I do that?


title: Installing Pytorch on Windows 10

body:
I am trying to install Pytorch on Windows 10 anaconda environment with Python 3.6 using the following command:

conda install -c peterjc123 pytorch

But it gives the following error:

  UnsatisfiableError: The following specifications were found to be in conflict:
    - curl -> krb5=1.14 -> *[track_features=vc14]
    - curl -> libssh2=1.8 -> vc==14
    - pytorch
  Use "conda info " to see the dependencies for each package.

The same goes for the new updated December code

conda install -c peterjc123 pytorch cuda80

Is there a way which will help me install Pytorch on Windows ?


title: How to disable TOKENIZERS_PARALLELISM=(true | false) warning?

body:
I use pytorch to train huggingface-transformers model, but every epoch, always output the warning:
The current process just got forked. Disabling parallelism to avoid deadlocks... To disable this warning, please explicitly set TOKENIZERS_PARALLELISM=(true | false)

How to disable this warning?


title: Fixing the seed for torch random_split()

body: Is it possible to fix the seed for torch.utils.data.random_split() when splitting a dataset so that it is possible to reproduce the test results?


title: How to implement current pytorch activation functions with parameters?

body:
I am looking for a simple way to use an activation function which exist in the pytorch library, but using some sort of parameter. for example:

Tanh(x/10)

The only way I came up with looking for solution was implementing the custom function completely from scratch. Is there any better/more elegant way to do this?

edit:

I am looking for some way to append to my model the function Tanh(x/10) rather than plain Tanh(x). Here is the relevant code block:

    self.model = nn.Sequential()
    for i in range(len(self.layers)-1):
        self.model.add_module("linear_layer_" + str(i), nn.Linear(self.layers[i], self.layers[i + 1]))
        if activations == None:
            self.model.add_module("activation_" + str(i), nn.Tanh())
        else:
            if activations[i] == "T":
                self.model.add_module("activation_" + str(i), nn.Tanh())
            elif activations[i] == "R":
                self.model.add_module("activation_" + str(i), nn.ReLU())
            else:
                #no activation
                pass


title: Pytorch saving and reloading model

body:
I have the following structure for a VGG16 model:

<bound method Module.state_dict of VGG(
  (features): Sequential(
    (0): Conv2d(3, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (1): ReLU(inplace)
    (2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (3): ReLU(inplace)
    (4): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)
    (5): Conv2d(64, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (6): ReLU(inplace)
    (7): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (8): ReLU(inplace)
    (9): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)
    (10): Conv2d(128, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (11): ReLU(inplace)
    (12): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (13): ReLU(inplace)
    (14): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (15): ReLU(inplace)
    (16): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)
    (17): Conv2d(256, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (18): ReLU(inplace)
    (19): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (20): ReLU(inplace)
    (21): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (22): ReLU(inplace)
    (23): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)
    (24): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (25): ReLU(inplace)
    (26): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (27): ReLU(inplace)
    (28): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (29): ReLU(inplace)
    (30): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)
  )
  (classifier): Network(
    (hidden_layers): ModuleList(
      (0): Linear(in_features=25088, out_features=4096, bias=True)
    )
    (output): Linear(in_features=4096, out_features=102, bias=True)
    (dropout): Dropout(p=0.5)
  )

When saving the model with the following code:

checkpoint = {'input_size': 25088,
              'output_size': 102,
              'hidden_layers': [each.out_features for each in model.hidden_layers],
              'state_dict': model.state_dict()}

torch.save(checkpoint, 'checkpoint.pth')

I get the following error:

AttributeError                            Traceback (most recent call last)
<ipython-input-13-b4654570e6e8> in <module>()
      2 checkpoint = {'input_size': 25088,
      3               'output_size': 102,
----> 4               'hidden_layers': [each.out_features for each in model.hidden_layers],
      5               'state_dict': model.state_dict()}
      6

/opt/conda/lib/python3.6/site-packages/torch/nn/modules/module.py in __getattr__(self, name)
    530                 return modules[name]
    531         raise AttributeError("'{}' object has no attribute '{}'".format(
--> 532             type(self).__name__, name))
    533
    534     def __setattr__(self, name, value):

  AttributeError: 'VGG' object has no attribute 'hidden_layers'

However, VGG has hidden_layers.
How to save A VGG with learning transfer?


title: How Pytorch Tensor get the index of elements?

body:
I have 2 Tensors named x and list and their definitions are below:

x = torch.tensor(3)
list = torch.tensor([1,2,3,4,5])

Now I want to get the index of element x from list. The expected output is an Integer:

2

How can I do in an easy way?


title: Convert Dataloader Dictionary to Pytorch Tensor

body:
I am trying to apply an autoencoder to a custom dataset in Pytorch using the following tutorial as a guide.

https://pytorch.org/tutorials/beginner/data_loading_tutorial.html

My dataset consists of images and values associated with those images.

I run into issues when trying to train the model because the items in the dataset I created act like a dictionary and the model wants tensors.

dataloader type =  <class 'torch.utils.data.dataloader.DataLoader'>

For item in dataloader:

<class 'dict'>
{'image': tensor([[[[0.5270, 0.5667, 0.6228,  ..., 0.6588, 0.6748, 0.6787],
          [0.5551, 0.5591, 0.6049,  ..., 0.6441, 0.6659, 0.6718],
          [0.4706, 0.4230, 0.5051,  ..., 0.6439, 0.6608, 0.6627],
          ...,
          [0.6914, 0.6478, 0.6574,  ..., 0.5968, 0.5735, 0.5676],
          [0.6814, 0.6475, 0.6713,  ..., 0.5664, 0.5059, 0.4669],
          [0.6588, 0.6623, 0.6914,  ..., 0.4667, 0.3453, 0.3132]],

         [[0.5061, 0.5270, 0.6422,  ..., 0.6642, 0.6787, 0.6931],
          [0.5375, 0.5426, 0.6169,  ..., 0.6520, 0.6711, 0.6730],
          [0.3980, 0.3314, 0.4326,  ..., 0.6608, 0.6738, 0.6716],
          ...,
          [0.7196, 0.6326, 0.6333,  ..., 0.6000, 0.5814, 0.5801],
          [0.6877, 0.6005, 0.6537,  ..., 0.5368, 0.4382, 0.3944],
          [0.6828, 0.6755, 0.6983,  ..., 0.3880, 0.2973, 0.2806]],

         [[0.5456, 0.5431, 0.5968,  ..., 0.6657, 0.6650, 0.6735],
          [0.4939, 0.3998, 0.4319,  ..., 0.6650, 0.6765, 0.6676],
          [0.4272, 0.2904, 0.3010,  ..., 0.6618, 0.6745, 0.6583],
          ...,
          [0.7324, 0.6740, 0.6505,  ..., 0.5436, 0.4958, 0.4971],
          [0.6939, 0.6836, 0.6887,  ..., 0.4218, 0.3672, 0.3875],
          [0.6669, 0.6686, 0.6826,  ..., 0.3093, 0.3108, 0.3341]],

         [[0.9961, 0.9961, 0.9961,  ..., 0.9961, 0.9961, 0.9961],
          [0.9961, 0.9961, 0.9961,  ..., 0.9961, 0.9961, 0.9961],
          [0.9961, 0.9961, 0.9961,  ..., 0.9961, 0.9961, 0.9961],
          ...,
          [0.9961, 0.9961, 0.9961,  ..., 0.9961, 0.9961, 0.9961],
          [0.9961, 0.9961, 0.9961,  ..., 0.9961, 0.9961, 0.9961],
          [0.9961, 0.9961, 0.9961,  ..., 0.9961, 0.9961, 0.9961]]]],
       dtype=torch.float64), 'hardness_arr': tensor([[[ 54.1600,   8.1600],
         [  7.4000, -38.6000]]], dtype=torch.float64)}

Can I covert from a dataset dictionary to a tensor, or is there a better way of approaching the issue?

    autoencoder = AutoEncoder()

optimizer = torch.optim.Adam(autoencoder.parameters(), lr=LR)
loss_func = nn.MSELoss()

for epoch in range(EPOCH):
    for step, (dataloader['image'], dataloader['hardness_arr']) in enumerate(dataloader['image']):
        b_x = x['image'].view(-1, crop_size*crop_size)   # batch x, shape (batch, 28*28)
        b_y = x['image'].view(-1, crop_size*crop_size)   # batch y, shape (batch, 28*28)

        encoded, decoded = autoencoder(b_x)

        loss = loss_func(decoded, b_y)      # mean square error
        optimizer.zero_grad()               # clear gradients for this training step
        loss.backward()                     # backpropagation, compute gradients
        optimizer.step()                    # apply gradients

TypeError                                 Traceback (most recent call last)
<ipython-input-82-25093ba202c6> in <module>
      5
      6 for epoch in range(EPOCH):
----> 7     for step, (dataloader['image'], dataloader['hardness_arr']) in enumerate(dataloader['image']):
      8         b_x = x['image'].view(-1, crop_size*crop_size)   # batch x, shape (batch, 28*28)
      9         b_y = x['image'].view(-1, crop_size*crop_size)   # batch y, shape (batch, 28*28)

TypeError: 'DataLoader' object is not subscriptable

Alternatively, if I try to enumerate over the dataloader itself I receive the following error.

TypeError: 'DataLoader' object does not support item assignment


title: When to use layernorm/batch norm?

body: Where should you splice the normalization when designing a network? E.g. if you have a stacked Transformer or Attention network, does it make sense to normalize any time after you have a dense layer?


title: Install torch on python 3.8.1 windows 10

body: I have been reading this post How to install pytorch in windows? but no one answer work for me on the versio 3.8.1 of python. Anything else I can do?


title: cuDNN error: CUDNN_STATUS_EXECUTION_FAILED while using flair

body:
I have been using https://github.com/zalandoresearch/flair#example-usage

tried using flair to experiment flair but then I don't know why I am  not able to use the GPU.
and tried the following:

>>> from flair.data import Sentence
>>> from flair.models import SequenceTagger
>>> sentence = Sentence('I love Berlin .')
>>> tagger = SequenceTagger.load('ner')
2019-07-20 17:52:15,062 loading file /home/vz/.flair/models/en-ner-conll03-v0.4.pt
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/home/vz/miniconda3/envs/gp/lib/python3.6/site-packages/flair/nn.py", line 103, in load
    model = cls._init_model_with_state_dict(state)
  File "/home/vz/miniconda3/envs/gp/lib/python3.6/site-packages/flair/models/sequence_tagger_model.py", line 205, in _init_model_with_state_dict
    locked_dropout=use_locked_dropout,
  File "/home/vz/miniconda3/envs/gp/lib/python3.6/site-packages/flair/models/sequence_tagger_model.py", line 166, in __init__
    self.to(flair.device)
  File "/home/vz/miniconda3/envs/gp/lib/python3.6/site-packages/torch/nn/modules/module.py", line 386, in to
    return self._apply(convert)
  File "/home/vz/miniconda3/envs/gp/lib/python3.6/site-packages/torch/nn/modules/module.py", line 193, in _apply
    module._apply(fn)
  File "/home/vz/miniconda3/envs/gp/lib/python3.6/site-packages/torch/nn/modules/module.py", line 193, in _apply
    module._apply(fn)
  File "/home/vz/miniconda3/envs/gp/lib/python3.6/site-packages/torch/nn/modules/module.py", line 193, in _apply
    module._apply(fn)
  [Previous line repeated 1 more time]
  File "/home/vz/miniconda3/envs/gp/lib/python3.6/site-packages/torch/nn/modules/rnn.py", line 127, in _apply
    self.flatten_parameters()
  File "/home/vz/miniconda3/envs/gp/lib/python3.6/site-packages/torch/nn/modules/rnn.py", line 123, in flatten_parameters
    self.batch_first, bool(self.bidirectional))
RuntimeError: cuDNN error: CUDNN_STATUS_EXECUTION_FAILED

Can anyone please help me as to how to fix this error ?
Thanks in advance.


title: Performing Convolution (NOT cross-correlation) in pytorch

body:
I have a network that I am trying to implement in pytorch, and I cannot seem to figure out how to implement "pure" convolution. In tensorflow it could be accomplished like this:
def conv2d_flipkernel(x, k, name=None):
    return tf.nn.conv2d(x, flipkernel(k), name=name,
                        strides=(1, 1, 1, 1), padding='SAME')

With the flipkernel function being:
def flipkernel(kern):
      return kern[(slice(None, None, -1),) * 2 + (slice(None), slice(None))]

How can something similar be done in pytorch?


Here are the codes：

    from __future__ import print_function
    from itertools import count

    import torch
    import torch.autograd
    import torch.nn.functional as F

    POLY_DEGREE = 4
    W_target = torch.randn(POLY_DEGREE, 1) * 5
    b_target = torch.randn(1) * 5

    def make_features(x):
        x = x.unsqueeze(1)
        return torch.cat([x ** i for i in range(1, POLY_DEGREE+1)], 1)

    def f(x):
        return x.mm(W_target) + b_target.item()

This resulted in the following error message:

AttributeError: 'torch.FloatTensor' object has no attribute 'item'

How can I solve this please?


title: Is there a pytorch method to check the number of cpus?

body: I can use this torch.cuda.device_count() to check the number of GPUs. I was wondering if there was something equivalent to check the number of CPUs.


title: Find number of non-zero elements in a tensor along an aixs

body:
I want to find the number of non-zero elements in a tensor along a particular axis. Is there any PyTorch function which can do this?

I tried to use the nonzero() method in PyTorch.

torch.nonzero(losses).size(0)

Here, lossess is a tensor of shape 64 x 1. When I run the above statement, it gives me the following error.

TypeError: Type Variable doesn't implement stateless method nonzero

But if I run, torch.nonzero(losses.data).size(0), then it works fine. Any clue, why this is happening or what the error means?


title: Is this the way to create a PyTorch scalar?

body:
I'm new to PyTorch, and just wanted to kindly confirm if the following creates scalars with values, 1, 2, and 3, respectively?

import torch

a = torch.tensor(1)
b = torch.tensor(2)
c = torch.tensor(3)

Thanks.