title: An error happens while following pytorch tutorial

body:
I was following pytorch tutorial, training a classifier using CIFAR10 dataset. All I've done was copy and paste codes in tutorial page to Pycharm project, yet I faced unknown errors. And I also used Jupyter notebook to run same code, and met errors again. Below is the code and errors. What should I do?

import torch
import torchvision
import matplotlib.pyplot as plt
import numpy as np
import torchvision.transforms as transforms
import torchvision.models as models
import torch.autograd as Variable
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim

transform = transforms.Compose(
    [transforms.ToTensor(),
     transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])

trainset = torchvision.datasets.CIFAR10(root='./data', train=True,
                                    download=True, transform=transform)
trainloader = torch.utils.data.DataLoader(trainset, batch_size=4,
                                      shuffle=True, num_workers=2)

testset = torchvision.datasets.CIFAR10(root='./data', train=False,
                                   download=True, transform=transform)
testloader = torch.utils.data.DataLoader(testset, batch_size=4,
                                     shuffle=False, num_workers=2)

classes = ('plane', 'car', 'bird', 'cat',
       'deer', 'dog', 'frog', 'horse', 'ship', 'truck')

class Net(nn.Module):
  def __init__(self):
    super(Net, self).__init__()
    self.conv1 = nn.Conv2d(3, 6, 5)
    self.pool = nn.MaxPool2d(2, 2)
    self.conv2 = nn.Conv2d(6, 16, 5)
    self.fc1 = nn.Linear(16 * 5 * 5, 120)
    self.fc2 = nn.Linear(120, 84)
    self.fc3 = nn.Linear(84, 10)

  def forward(self, x):
    x = self.pool(F.relu(self.conv1(x)))
    x = self.pool(F.relu(self.conv2(x)))
    x = x.view(-1, 16 * 5 * 5)
    x = F.relu(self.fc1(x))
    x = F.relu(self.fc2(x))
    x = self.fc3(x)
    return x

net = Net()
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(net.parameters(), lr=0.001, momentum=0.9)

for epoch in range(2):

  running_loss = 0.0
    for i, data in enumerate(trainloader, 0):

      inputs, labels = data

      optimizer.zero_grad()

      outputs = net(inputs)
      loss = criterion(outputs, labels)
      loss.backward()
      optimizer.step()

      running_loss += loss.item()
      if i % 2000 == 1999:    # print every 2000 mini-batches
        print('[%d, %5d] loss: %.3f' %
              (epoch + 1, i + 1, running_loss / 2000))
        running_loss = 0.0

print('Finished Training')

Error in Pycharm:

Traceback (most recent call last):
File "<string>", line 1, in <module>
File "D:\ProgramData\Anaconda3\envs\pytorch\lib\multiprocessing\spawn.py", line 105, in spawn_main
exitcode = _main(fd)
File "D:\ProgramData\Anaconda3\envs\pytorch\lib\multiprocessing\spawn.py", line 114, in _main
prepare(preparation_data)
File "D:\ProgramData\Anaconda3\envs\pytorch\lib\multiprocessing\spawn.py", line 225, in prepare
_fixup_main_from_path(data['init_main_from_path'])
File "D:\ProgramData\Anaconda3\envs\pytorch\lib\multiprocessing\spawn.py", line 277, in _fixup_main_from_path
run_name="__mp_main__")
File "D:\ProgramData\Anaconda3\envs\pytorch\lib\runpy.py", line 263, in run_path
pkg_name=pkg_name, script_name=fname)
File "D:\ProgramData\Anaconda3\envs\pytorch\lib\runpy.py", line 96, in _run_module_code
mod_name, mod_spec, pkg_name, script_name)
File "D:\ProgramData\Anaconda3\envs\pytorch\lib\runpy.py", line 85, in _run_code
exec(code, run_globals)
File "D:\Users\VML1\PycharmProjects\untitled1\HY1.py", line 57, in <module>
for i, data in enumerate(trainloader, 0):
File "D:\ProgramData\Anaconda3\envs\pytorch\lib\site-packages\torch\utils\data\dataloader.py", line 417, in __iter__
return DataLoaderIter(self)
File "D:\ProgramData\Anaconda3\envs\pytorch\lib\site-packages\torch\utils\data\dataloader.py", line 234, in __init__
w.start()
File "D:\ProgramData\Anaconda3\envs\pytorch\lib\multiprocessing\process.py", line 105, in start
self._popen = self._Popen(self)
File "D:\ProgramData\Anaconda3\envs\pytorch\lib\multiprocessing\context.py", line 223, in _Popen
return _default_context.get_context().Process._Popen(process_obj)
File "D:\ProgramData\Anaconda3\envs\pytorch\lib\multiprocessing\context.py", line 322, in _Popen
return Popen(process_obj)
File "D:\ProgramData\Anaconda3\envs\pytorch\lib\multiprocessing\popen_spawn_win32.py", line 33, in __init__
prep_data = spawn.get_preparation_data(process_obj._name)
File "D:\ProgramData\Anaconda3\envs\pytorch\lib\multiprocessing\spawn.py", line 143, in get_preparation_data
_check_not_importing_main()
File "D:\ProgramData\Anaconda3\envs\pytorch\lib\multiprocessing\spawn.py", line 136, in _check_not_importing_main
is not going to be frozen to produce an executable.''')
RuntimeError:
    An attempt has been made to start a new process before the
    current process has finished its bootstrapping phase.

    This probably means that you are not using fork to start your
    child processes and you have forgotten to use the proper idiom
    in the main module:

        if __name__ == '__main__':
            freeze_support()
            ...

    The "freeze_support()" line can be omitted if the program
    is not going to be frozen to produce an executable.
Traceback (most recent call last):
File "D:/Users/VML1/PycharmProjects/untitled1/HY1.py", line 57, in <module>
for i, data in enumerate(trainloader, 0):
File "D:\ProgramData\Anaconda3\envs\pytorch\lib\site-packages\torch\utils\data\dataloader.py", line 417, in __iter__
return DataLoaderIter(self)
File "D:\ProgramData\Anaconda3\envs\pytorch\lib\site-packages\torch\utils\data\dataloader.py", line 234, in __init__
w.start()
File "D:\ProgramData\Anaconda3\envs\pytorch\lib\multiprocessing\process.py", line 105, in start
self._popen = self._Popen(self)
File "D:\ProgramData\Anaconda3\envs\pytorch\lib\multiprocessing\context.py", line 223, in _Popen
return _default_context.get_context().Process._Popen(process_obj)
File "D:\ProgramData\Anaconda3\envs\pytorch\lib\multiprocessing\context.py", line 322, in _Popen
return Popen(process_obj)
File "D:\ProgramData\Anaconda3\envs\pytorch\lib\multiprocessing\popen_spawn_win32.py", line 65, in __init__
reduction.dump(process_obj, to_child)
  File "D:\ProgramData\Anaconda3\envs\pytorch\lib\multiprocessing\reduction.py", line 60, in dump
    ForkingPickler(file, protocol).dump(obj)
BrokenPipeError: [Errno 32] Broken pipe

Error in Jupyter:

TypeError
Traceback (most recent call last)
<ipython-input-2-1dd1ecde814e> in <module>
---> 45         outputs = net(inputs)
     46         loss = criterion(outputs, labels)
     47         loss.backward()

c:\users\vml1\anaconda3\envs\hypytorch\lib\site-packages\torch\nn\modules\module.py in __call__(self, *input, **kwargs)
    355             result = self._slow_forward(*input, **kwargs)
    356         else:
--> 357             result = self.forward(*input, **kwargs)
    358         for hook in self._forward_hooks.values():
    359             hook_result = hook(self, input, result)

c:\users\vml1\anaconda3\envs\hypytorch\lib\site-packages\torchvision\models\resnet.py in forward(self, x)
    137
    138     def forward(self, x):
--> 139         x = self.conv1(x)
    140         x = self.bn1(x)
    141         x = self.relu(x)

c:\users\vml1\anaconda3\envs\hypytorch\lib\site-
packages\torch\nn\modules\module.py in __call__(self, *input, **kwargs)
    355             result = self._slow_forward(*input, **kwargs)
    356         else:
--> 357             result = self.forward(*input, **kwargs)
    358         for hook in self._forward_hooks.values():
    359             hook_result = hook(self, input, result)

    c:\users\vml1\anaconda3\envs\hypytorch\lib\site-packages\torch\nn\modules\conv.py in forward(self, input)
    280     def forward(self, input):
    281         return F.conv2d(input, self.weight, self.bias, self.stride,
--> 282                         self.padding, self.dilation, self.groups)
    283
    284

c:\users\vml1\anaconda3\envs\hypytorch\lib\site-
packages\torch\nn\functional.py in conv2d(input, weight, bias, stride,
padding, dilation, groups)
     88                 _pair(0), groups, torch.backends.cudnn.benchmark,
     89                 torch.backends.cudnn.deterministic,
torch.backends.cudnn.enabled)
---> 90     return f(input, weight, bias)
     91
     92

TypeError: argument 0 is not a Variable


title: Memory keep accumulating while training by PyTorch

body:
I am training a deep learning model using PyTorch. Due to unknown reasons, memory keeps accumulating, which leads to session killed under 30 epochs and underfitting.

Some thoughts here:

Wondering if it's caused by matplotlib so I added plt.close('all'); didn't work
Added gc.collect(); didn't work
Wondering if it's caused by cv2.imwrite(), but don't know how to inspect this. Any suggestions?
PyTorch issues?
others...

model.train()
for epo in range(epoch):
    for i, data in enumerate(trainloader, 0):
        inputs = data
        inputs = Variable(inputs)
        optimizer.zero_grad()

        top = model.upward(inputs + white(inputs))
        outputs = model.downward(top, shortcut = True)

        loss = criterion(inputs, outputs)
        loss.backward()
        optimizer.step()

        # Print generated pictures every 100 iters
        if i % 100 == 0:
            inn = inputs[0].view(128, 128).detach().numpy() * 255
            cv2.imwrite("/home/tk/Documents/recover/" + str(epo) + "_" + str(i) + ".png", inn)

            out = outputs[0].view(128, 128).detach().numpy() * 255
            cv2.imwrite("/home/tk/Documents/recover/" + str(epo) + "_" + str(i) + "_re.png", out)

        # Print loss every 50 iters
        if i % 50 == 0:
            print ('[%d, %5d] loss: %.3f' % (epo, i, loss.item()))

    gc.collect()
    plt.close("all")

===================================================================

20181222 Update

Datasets & DalaLoader

class MSourceDataSet(Dataset):

    def __init__(self, clean_dir):

        for i in cleanfolder:
            with open(clean_dir + '{}'.format(i)) as f:
                clean_list.append(torch.Tensor(json.load(f)))

        cleanblock = torch.cat(clean_list, 0)
        self.spec = cleanblock

    def __len__(self):
        return self.spec.shape[0]

    def __getitem__(self, index):

        spec = self.spec[index]
        return spec

trainset = MSourceDataSet(clean_dir)
trainloader = torch.utils.data.DataLoader(dataset = trainset,
                                            batch_size = 4,
                                            shuffle = True)

The model is really complicated and long...plus the memory accumulation issue didn't happen before (using the same model), so I will not post it here...


title: loading pretrained model in pytorch

body:
First, I'd like to apologize this question may sound stupid but I'm new to deep learning. Can anybody explain to me the following lines of code which was used to load the pre-trained model in PyTorch?

# Retrieving model parameters from checkpoint.
vocab_size = checkpoint["model"]["_word_embedding.weight"].size(0)
embedding_dim = checkpoint["model"]['_word_embedding.weight'].size(1)
hidden_size = checkpoint["model"]["_projection.0.weight"].size(0)
num_classes = checkpoint["model"]["_classification.4.weight"].size(0)

I can't understand the projection, weight, classification, size(0), size(1) in the above text.


title: Dimension mismatch while using Pytorch LSTM module

body:
I have a pytorch pretrained model from which I am generating features/embedding for some input sentences. The features are essentially torch object. For example a sample input_embedding (a list of torch objects) for one sentence looks like below

[tensor([-0.8264,  0.2524], device='cuda:0', grad_fn=<SelectBackward>)]

Now, I want to pass this embedding through a custom model which is fundamentally a bi-directional LSTM:

def custom_model(input_embedding):
    #initialize BiLSTM
    bilstm = torch.nn.LSTM(input_size=1, hidden_size=1, num_layers=1, batch_first=False, bidirectional=True)
    #feed input to bilstm object
    bi_output, bi_hidden = bilstm(input_embedding)
    # more code ....
    return F.softmax(x)

I wanted to pass my input_embedding to this custom model to get the prediction output like below:

for item in input_embedding:
    y_pred = biLSTM_single_sentence_student_model(item)

But it is throwing error on the bi_output, bi_hidden = bilstm(input_embedding) line saying:

IndexError: Dimension out of range (expected to be in range of [-1, 0], but got 1)

Most likely I am not defining the bilstm object properly due to my lack of understanding of Pytorch nn.LSTM input.

Please suggest.


title: I am trying to classify flowers with a pretrained network, but for some reason it does not train

body:
I am currently trying to classify flowers from this dataset, using Pytorch.

First of all I started to transfrom my data for the training, validation and testing set.

data_dir = 'flowers'
train_dir = data_dir + '/train'
valid_dir = data_dir + '/valid'
test_dir = data_dir + '/test'

train_transforms = transforms.Compose([transforms.RandomRotation(30),
                                       transforms.RandomResizedCrop(224),
                                       transforms.RandomHorizontalFlip(),
                                       transforms.ToTensor(),
                                       transforms.Normalize([0.485, 0.456, 0.406],
                                                            [0.229, 0.224, 0.225])])

test_transforms = transforms.Compose([transforms.Resize(224),
                                      transforms.CenterCrop(224),
                                      transforms.ToTensor(),
                                      transforms.Normalize([0.485, 0.456, 0.406],
                                                           [0.229, 0.224, 0.225])])

Afterwards I loaded the data with ImageFolder:

trainset = datasets.ImageFolder(train_dir, transform=train_transforms)
testset = datasets.ImageFolder(test_dir, transform=test_transforms)
validationset = datasets.ImageFolder(valid_dir, transform=test_transforms)

Then I defined my DataLoaders:

trainloader = torch.utils.data.DataLoader(trainset, batch_size = 64, shuffle = True)
testloader = torch.utils.data.DataLoader(testset, batch_size = 32)
validationloader = torch.utils.data.DataLoader(validationset, batch_size = 32)

I choose vgg to be my pretrained model:

model = models.vgg16(pretrained = True)

And defined a new classifier:

for param in model.parameters():
    param.requires_grad = False

classifier = nn.Sequential(OrderedDict([
    ('fc1', nn.Linear(25088, 4096)),
    ('relu', nn.ReLU()),
    ('fc2', nn.Linear(4096, 4096)),
    ('relu', nn.ReLU()),
    ('fc3', nn.Linear(4096, 102)),
    ('output', nn.Softmax(dim = 1))

]))

model.classifier = classifier

This is the code to actually train my NN (on the GPU):

criterion = nn.NLLLoss()
optimizer = optim.Adam(model.classifier.parameters(), lr = 0.005)

epochs = 9
print_every = 10
steps = 0

model.to('cuda')

for e in range(epochs):
    running_loss = 0

    for ii, (inputs, labels) in enumerate(trainloader):
        steps += 1

        inputs, labels = inputs.to('cuda'), labels.to('cuda')

        optimizer.zero_grad()

        # Forward and backward
        outputs = model.forward(inputs)
        loss = criterion(outputs, labels)

        loss.backward()
        optimizer.step()

        running_loss += loss.item()

        if steps % print_every == 0:
            print("Epoch: {}/{}... ".format(e+1, epochs),
                  "Loss: {:.4f}".format(running_loss/print_every))

            running_loss = 0

But when I run my model, the loss is random and I am not sure why.

Thank you for any kind of help in advance and Greetings from Germany!


title: Pytorch: How to create an update rule that doesn't come from derivatives?

body:
I want to implement the following algorithm, taken from this book, section 13.6:

I don't understand how to implement the update rule in pytorch (the rule for w is quite similar to that of theta).

As far as I know, torch requires a loss for loss.backwward().

This form does not seem to apply for the quoted algorithm.

I'm still certain there is a correct way of implementing such update rules in pytorch.

Would greatly appreciate a code snippet of how the w weights should be updated, given that V(s,w) is the output of the neural net, parameterized by w.

EDIT: Chris Holland suggested a way to implement, and I implemented it. It does not converge on Cartpole, and I wonder if I did something wrong.

The critic does converge on the solution to the function gamma*f(n)=f(n)-1 which happens to be the sum of the series gamma+gamma^2+...+gamma^inf
meaning, gamma=1 diverges. gamma=0.99 converges on 100, gamma=0.5 converges on 2 and so on. Regardless of the actor or policy.

The code:

def _update_grads_with_eligibility(self, is_critic, delta, discount, ep_t):
    gamma = self.args.gamma
    if is_critic:
        params = list(self.critic_nn.parameters())
        lamb = self.critic_lambda
        eligibilities = self.critic_eligibilities
    else:
        params = list(self.actor_nn.parameters())
        lamb = self.actor_lambda
        eligibilities = self.actor_eligibilities

    is_episode_just_started = (ep_t == 0)
    if is_episode_just_started:
        eligibilities.clear()
        for i, p in enumerate(params):
            if not p.requires_grad:
                continue
            eligibilities.append(torch.zeros_like(p.grad, requires_grad=False))

    # eligibility traces
    for i, p in enumerate(params):

        if not p.requires_grad:
            continue
        eligibilities[i][:] = (gamma * lamb * eligibilities[i]) + (discount * p.grad)
        p.grad[:] = delta.squeeze() * eligibilities[i]

and

expected_reward_from_t = self.critic_nn(s_t)
probs_t = self.actor_nn(s_t)
expected_reward_from_t1 = torch.tensor([[0]], dtype=torch.float)
if s_t1 is not None:  # s_t is not a terminal state, s_t1 exists.
    expected_reward_from_t1 = self.critic_nn(s_t1)

delta = r_t + gamma * expected_reward_from_t1.data - expected_reward_from_t.data

negative_expected_reward_from_t = -expected_reward_from_t
self.critic_optimizer.zero_grad()
negative_expected_reward_from_t.backward()
self._update_grads_with_eligibility(is_critic=True,
                                    delta=delta,
                                    discount=discount,
                                    ep_t=ep_t)
self.critic_optimizer.step()

EDIT 2:
Chris Holland's solution works. The problem originated from a bug in my code that caused the line

if s_t1 is not None:
    expected_reward_from_t1 = self.critic_nn(s_t1)

to always get called, thus expected_reward_from_t1 was never zero, and thus no stopping condition was specified for the bellman equation recursion.

With no reward engineering, gamma=1, lambda=0.6, and a single hidden layer of size 128 for both actor and critic, this converged on a rather stable optimal policy within 500 episodes.

Even faster with gamma=0.99, as the graph shows (best discounted episode reward is about 86.6).

BIG thank you to @Chris Holland, who "gave this a try"


title: LSTM returning only zeros (PyTorch)

body:
After succesfully implementing a LSTM "from scratch" based on linear layers, I decided to start using the existing LSTM class to make things easier and gain in performance.
But somehow when I try it, it only returns tensors full of zeros.
Here is the model :

class pytorchLSTM(nn.Module):
    def __init__(self,input_size,hidden_size):
        super().__init__()
        self.input_size = input_size
        self.hidden_size = hidden_size
        self.lstm = nn.LSTM(input_size, hidden_size)
        self.softmax = nn.LogSoftmax(dim = 1)

    def forward(self, input):
        out, hidden = self.lstm(input)
        out = self.softmax(out)
        return out, hidden

the input is a (1,1,60) tensor representing a one-hot encoded letter :

tensor([[[0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
          0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
          0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
          0., 0., 0., 0., 0., 0., 0., 1., 0.]]])

and the models returns, invariably (I tried modifying the values inside of the input, but the result is always the same) :

tensor([[[0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
           0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
           0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
           0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
           0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
           0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.]]],
        grad_fn=<LogSoftmaxBackward>)

Any idea where my mistake is and what I understood wrong about the LSTM class ?


title: PyTorch: Sharing a model between threads, do I need to lock it myself?

body:
I'm sharing a PyTorch neural network model between a main thread which trains the model and a number of worker threads which eval the model to generate training samples (à la AlphaGo).

My question is, do I need to create a separate mutex to lock and unlock when accessing the model in different threads?

I did not see any mutex in the PyTorch Hogwild example, however the difference there is that all threads are in train mode.

Apart from the official tutorial, I'm unable to find even basic information (like this question) online.

I'm using the torch.multiprocessing module for this and I'm using model.share_memory() and torch.multiprocessing.set_start_method('spawn', force=True).


title: turn off or ignore neurons in pytorch in neural networks

body:
I want to build a feed forward-neural network and train it (to overfitt) on a small portion of the input features. For this I used dropout regularization as it follows the logic on training on small portion and then test model on the whole features (Turning off during training and turn on test).

But since I am interested on overfitting, I don't think dropout is a good solution for me. So how can I turn off  some input nodes during training in the same way as used in Dropout regularization, but this time I don't want randomly turn them off, but chose which featured will be ignored during training ?


title: How to prune neurons in neural network

body:
Context:

Suppose we have a simple 3-layer feed-forward network. The hidden size of the first linear layer is 100000 -- W1[input_size, 100000] in which input_size is a number much smaller than 100000. Some of neurons won't be learning any thing. I want to select and shutdown these neurons using pruning.

Expected outcomes

After pruning the selected neurons, we will have a smaller network with the less neurons in the first layer, say reduced to 500. And this smaller network turns out to have the same predicting capacity to the large one.

My implementation:

According to some criterion (some metrics applied to check weight similarities after each backpropagation update), I have cheery picked the indices of neurons I want to shut down, e.g., [1,7,8 ...].

Zero out the weights represented by the indices in W1, W1[:, 1, 7, 8 ...] = 0. So that no information will be passed forward via these neurons to the next layer.

Will that be enough? Should I be manually intervening the backpropagation as well? Zero out neurons stop only computations passing forward, but for learning/updating on weights, backpropgation matters more. Since I am using pytorch, it will be great if illustrations are provided in pytorch, other frameworks like tensorflow, Keras are also fine.


title: What defines legitimate input dimensions of a pytorch nn?

body:
From the tutorial / example github, the network to be trained on the MNIST dataset is defined as:

class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        self.conv1 = nn.Conv2d(1, 20, 5, 1)
        self.conv2 = nn.Conv2d(20, 50, 5, 1)
        self.fc1 = nn.Linear(4*4*50, 500)
        self.fc2 = nn.Linear(500, 10)

    def forward(self, x):
        x = F.relu(self.conv1(x))
        x = F.max_pool2d(x, 2, 2)
        x = F.relu(self.conv2(x))
        x = F.max_pool2d(x, 2, 2)
        x = x.view(-1, 4*4*50)
        x = F.relu(self.fc1(x))
        x = self.fc2(x)
        return F.log_softmax(x, dim=1)

The image sizes this network is trained with are 28x28 pixels.

Testing images of different sizes creates errors. To be precise: I tested images of size 27x27 and 32x32 for whom it does break with an error; an input size of 29x29 does create none.

Where is the size of 28x28 actually defined? What formula can I use to discern parameters for different input sizes for different tasks? Can one use images of different sizes as input? How?


title: Constant Training Loss and Validation Loss

body:
I am running a RNN model with Pytorch library to do sentiment analysis on movie review, but somehow the training loss and validation loss remained constant throughout the training. I have looked up different online sources but still stuck.

Can someone please help and take a look at my code?

Some parameters are specified by the assignment:

embedding_dim = 64

n_layers = 1

n_hidden = 128

dropout = 0.5

batch_size = 32

My main code

txt_field = data.Field(tokenize=word_tokenize, lower=True, include_lengths=True, batch_first=True)
label_field = data.Field(sequential=False, use_vocab=False, batch_first=True)

train = data.TabularDataset(path=part2_filepath+"train_Copy.csv", format='csv',
                            fields=[('label', label_field), ('text', txt_field)], skip_header=True)
validation = data.TabularDataset(path=part2_filepath+"validation_Copy.csv", format='csv',
                            fields=[('label', label_field), ('text', txt_field)], skip_header=True)

txt_field.build_vocab(train, min_freq=5)
label_field.build_vocab(train, min_freq=2)

device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
train_iter, valid_iter, test_iter = data.BucketIterator.splits(
    (train, validation, test),
    batch_size=32,
    sort_key=lambda x: len(x.text),
    sort_within_batch=True,
    device=device)

n_vocab = len(txt_field.vocab)
embedding_dim = 64
n_hidden = 128
n_layers = 1
dropout = 0.5

model = Text_RNN(n_vocab, embedding_dim, n_hidden, n_layers, dropout)

optimizer = torch.optim.Adam(model.parameters(), lr=0.0001)
criterion = torch.nn.BCELoss().to(device)

N_EPOCHS = 15
best_valid_loss = float('inf')

for epoch in range(N_EPOCHS):
    train_loss, train_acc = RNN_train(model, train_iter, optimizer, criterion)
    valid_loss, valid_acc = evaluate(model, valid_iter, criterion)

My Model

class Text_RNN(nn.Module):
    def __init__(self, n_vocab, embedding_dim, n_hidden, n_layers, dropout):
        super(Text_RNN, self).__init__()
        self.n_layers = n_layers
        self.n_hidden = n_hidden
        self.emb = nn.Embedding(n_vocab, embedding_dim)
        self.rnn = nn.RNN(
            input_size=embedding_dim,
            hidden_size=n_hidden,
            num_layers=n_layers,
            dropout=dropout,
            batch_first=True
        )
        self.sigmoid = nn.Sigmoid()
        self.linear = nn.Linear(n_hidden, 2)

    def forward(self, sent, sent_len):
        sent_emb = self.emb(sent)
        outputs, hidden = self.rnn(sent_emb)
        prob = self.sigmoid(self.linear(hidden.squeeze(0)))

        return prob

The training function

def RNN_train(model, iterator, optimizer, criterion):
    epoch_loss = 0
    epoch_acc = 0
    model.train()
    for batch in iterator:
        text, text_lengths = batch.text
        predictions = model(text, text_lengths)
        batch.label = batch.label.type(torch.FloatTensor).squeeze()
        predictions = torch.max(predictions.data, 1).indices.type(torch.FloatTensor)
        loss = criterion(predictions, batch.label)
        loss.requires_grad = True
        acc = binary_accuracy(predictions, batch.label)
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()
        epoch_loss += loss.item()
        epoch_acc += acc.item()

    return epoch_loss / len(iterator), epoch_acc / len(iterator)

The output I run on 10 testing reviews + 5 validation reviews

Epoch [1/15]:   Train Loss: 15.351 | Train Acc: 44.44%  Val. Loss: 11.052 |  Val. Acc: 60.00%
Epoch [2/15]:   Train Loss: 15.351 | Train Acc: 44.44%  Val. Loss: 11.052 |  Val. Acc: 60.00%
Epoch [3/15]:   Train Loss: 15.351 | Train Acc: 44.44%  Val. Loss: 11.052 |  Val. Acc: 60.00%
Epoch [4/15]:   Train Loss: 15.351 | Train Acc: 44.44%  Val. Loss: 11.052 |  Val. Acc: 60.00%
...

Appreciate if someone can point me to the right direction, I believe is something with the training code, since for most parts I follow this article:
https://www.analyticsvidhya.com/blog/2020/01/first-text-classification-in-pytorch/


title: How can I train Super-Resolution Generative Adversarial Network (SRGAN) with high-frequency grayscale images?

body:
This question is almost a duplicate of the post from Cross Validated, but none has replied to that and I hope it is okay I ask almost the same question here.

I have been reading and looking at implementations of the SRGAN, from Photo-realistic Single Image Super Resolution with Generative Adversarial Networks. I implemented the PyTorch implementation of SRGAN for 3 channel images and it makes some decent super resolution images. However, when I try it out with 1 channel images it fails to generate plausible images.

The grayscale images I use is from the public release MSTAR of high resolution Synthetic Aparture Radar (SAR) data from sdms. The dataset contains of 2774 images and has 10 classes. Some samples are shown below:

Since SRGAN use VGG as one of the networks I had to convert my grayscale images to RGB. I copied the first channel to the other two channels and created a 3 channel image by the following approach:

    w, h = hr_image.shape
    ret = np.empty((w, h, 3), dtype=np.uint8)
    ret[:, :, :] = hr_image[:, :, np.newaxis]
    hr_image = ret

The following are the output after 34 epochs (this continues) and is low resolution, high resolution, super resolution.

What I notice is that the Discriminators loss quickly goes to 0, Generators loss to 0.08, generators score to 0 and discriminators score to 1. I assume this means that it is too easy for the discriminator to distinguish between the real and fake image. This presumable causes the generator to not learn anything new and just stops learning.

I tried to isolate one class in the MSTAR dataset, but that did not change anything. I noticed that others Super-resolution SAR Image Reconstruction via Generative Adversarial Network did use SRGAN for SAR images and it seems to work, but their paper does not explain how they implemented it.

I am wondering if I am using the wrong approach and need to change the loss functions. SRGAN is using MSE, TVLoss and perceptual loss. MSE is by itself not the best loss functions and is explained really well on here. But it is probably good to keep the images inside the MSE-hypersphere. But I ask myself whether it make sense to use a network that is trained on low frequency images and use it on high frequency images? As I understand it the loss functions are designed to work really well on low frequency images (something our eyes like to look at) and not that much on high frequency images. My questions are therefore:

Should I change the loss functions? Should I pre train my own network with high frequency grayscale images so the network are more suitable for these images?
How come the generator crashing after few epochs? Why is it so hard for the generator to make plausible transformations? Are the images too "noisy"?
Should I change something in the structure of the generator and discriminator? And should I try to use another pretrained network that is more suitable for grayscale images, and if so, which one?

Should I use a pretrained network that is trained on high frequency images?

UPDATED TEXT 11 JUNE 2020 BECAUSE OF NEW RESULTS

Instead of converting all my images to 3 channels I only did this before the VGG network in line 22 by the following command:

    out_images = out_images.repeat(1,3, 1,1)
    target_images = target_images.repeat(1, 3, 1,1)

I also made some minor changes in my code and this changed the output of the model after 250:

However as can be seen it did not improve in making the super resolution image. The discriminators quickly learns to distinguish between real and fake images as can be seen on the loss plot below:

Does anyone has any suggestions on how I can make the generator stronger? I tried to add more layers in the generator and removed some of the discriminator but with no success.


title: Multiple Inputs / Outputs for Dataset class in PyTorch

body: I've created a Dataset class for PyTorch that can read/generate batches for problems with Nin images as inputs and Nout images as outputs. Now I have to pass this class to the DataLoader, but I don't know how to format such inputs / outputs in order to be supported by the DataLoader. For example I don't think it has to be BatchSize x Nin x Channels x Height x Width because it could not support images with different channels (or dimensions in general).


title: optimizer got an empty parameter list (skorch)

body:
So, I am used to use PyTorch and now decided to give Skorch a shot.

Here they define the network as

class ClassifierModule(nn.Module):
    def __init__(
            self,
            num_units=10,
            nonlin=F.relu,
            dropout=0.5,
    ):
        super(ClassifierModule, self).__init__()
        self.num_units = num_units
        self.nonlin = nonlin
        self.dropout = dropout

        self.dense0 = nn.Linear(20, num_units)
        self.nonlin = nonlin
        self.dropout = nn.Dropout(dropout)
        self.dense1 = nn.Linear(num_units, 10)
        self.output = nn.Linear(10, 2)

    def forward(self, X, **kwargs):
        X = self.nonlin(self.dense0(X))
        X = self.dropout(X)
        X = F.relu(self.dense1(X))
        X = F.softmax(self.output(X), dim=-1)
        return X

I prefer inputting lists of neurons in each layer i.e num_units=[30,15,5,2] would have 2 hidden layers with 15 and 5 neurons. Furthermore we have 30 features and 2 classes, thus re-writing it to something like this

class Net(nn.Module):
    def __init__(
            self,
            num_units=[30,15,5,2],
            nonlin=[F.relu,F.relu,F.relu],
            dropout=[0.5,0.5,0.5],
            ):
        super(Net, self).__init__()

        self.layer_units = layer_units
        self.nonlin = nonlin #Activation function
        self.dropout = dropout #Drop-out rates in each layer
        self.layers = [nn.Linear(i,p) for i,p in zip(layer_units,layer_units[1:])] #Dense layers

    def forward(self, X, **kwargs):
        print("Forwards")
        for layer,func,drop in zip(self.layers[:-1],self.nonlin,self.dropout):
            print(layer,func,drop)
            X=drop(func(layer(X)))

        X = F.softmax(X, dim=-1)
        return X

should do the trick. The problem is that when calling

net = NeuralNetClassifier(Net,max_epochs=20,lr=0.1,device="cuda")
net.fit(X,y)

I get the error "ValueError: optimizer got an empty parameter list". I have narrowed it down to the removal of self.output = nn.Linear(10, 2) simply makes the net not enter forward i.e it seems like output is some kind of "trigger" variable. Is that really the case the network need a variable called output (being a layer) at the end, and that we are not free to define the variable-names ourself ?


title: How to find the mean and Std of a 3 channel image

body:
from numpy import asarray
from PIL import Image

image = Image.open('../input/chest-xray-pneumonia/chest_xray/train/NORMAL/IM-0115-0001.jpeg')
pixels = asarray(image)

pixels = pixels.astype('float32')
means = pixels.mean(axis=(0,1), dtype='float64')
stds = pixels.std(axis=(0,1), dtype='float64')
print('Means: %s, Stds: %s' % (means, stds))
pixels = (pixels - means) / stds
means = pixels.mean(axis=(0,1), dtype='float64')
stds = pixels.std(axis=(0,1), dtype='float64')
print('Means: %s, Stds: %s' % (means, stds))'''

output>> Means: 128.90747832983968, Stds: 62.30103035552067
         Means: 1.2235509834827096e-07, Stds: 1.0000000181304383

The problem is while putting 3 channel image ,only got two values each


title: PyTorch : cuda out of memory but enough memory left (add error message)

body:
I'm trying to classify cat vs dog with GoogleNet(Pytorch).
Each class contains 4000 images to train and 1000 images to test, which's size is 300*300.
My computer has 32GB RAM and RTX 2080 Super graphic card.
And this error occurs when it just start to train. Below is my code the part of GoogleNet :
class Inception(nn.Module) :
    def __init__(self, in_dim, out_dim1, mid_dim3, out_dim3, mid_dim5, out_dim5, pool):
       super(Inception, self).__init__()
        self.lay1 = nn.Sequential(nn.Conv2d(in_dim, out_dim1, kernel_size= 1), nn.BatchNorm2d(out_dim1), nn.ReLU())
        self.lay2 = nn.Sequential(nn.Conv2d(in_dim, mid_dim3, kernel_size = 1), nn.BatchNorm2d(mid_dim3), nn.ReLU(), nn.Conv2d(mid_dim3, out_dim3, kernel_size = 3, padding = 1), nn.BatchNorm2d(out_dim3), nn.ReLU())
        self.lay3 = nn.Sequential(nn.Conv2d(in_dim, mid_dim5, kernel_size = 1), nn.BatchNorm2d(mid_dim5), nn.ReLU(), nn.Conv2d(mid_dim5, out_dim5, kernel_size = 3, padding = 1), nn.BatchNorm2d(out_dim5), nn.ReLU(), nn.Conv2d(out_dim5, out_dim5, kernel_size = 3, padding = 1), nn.BatchNorm2d(out_dim5), nn.ReLU())
        self.lay4 = nn.Sequential(nn.MaxPool2d(3, 1, 1), nn.Conv2d(in_dim, pool, kernel_size = 1), nn.BatchNorm2d(pool), nn.ReLU())

    def forward(self, x):
        y1 = self.lay1(x)
        y2 = self.lay2(x)
        y3 = self.lay3(x)
        y4 = self.lay4(x)

        return torch.cat([y1, y2, y3, y4], 1)

class Google(nn.Module):
    def __init__(self):
        super(Google, self).__init__()
        self.pre_lay = nn.Sequential(nn.Conv2d(1, 48, 3, padding = 1), nn.BatchNorm2d(48), nn.ReLU())
        self.glay1 = Inception(48, 16, 24, 32, 4, 8, 8)
        self.glay2 = Inception(64, 32, 32, 48, 8, 24, 16) # input channel : prev output channel sum(torch.cat)
        self.maxpool = nn.MaxPool2d(3, stride = 2, padding = 1)
        self.glay3 = Inception(120, 48, 24, 52, 4, 12, 16)
        self.glay4 = Inception(128, 40, 28, 56, 6, 16, 16)
        self.glay5 = Inception(128, 32, 32, 64, 6, 16, 16)
        self.glay6 = Inception(128, 28, 36, 72, 8, 16, 16)
        self.glay7 = Inception(132, 64, 40, 80, 8, 32, 32)
        self.glay8 = Inception(208, 64, 40, 80, 8, 32, 32)
        self.glay9 = Inception(208, 96, 48, 96, 12, 32, 32)

        self.avgpool = nn.AvgPool2d(8, stride = 1)
        self.linear = nn.Linear(47872, 2)
        gc.collect()
        torch.cuda.empty_cache()

    def forward(self, x):
        gc.collect()
        torch.cuda.empty_cache()
        # with torch.no_grad() : //
            out = self.pre_lay(x) # CUDA out of memory Occurs!!
            out = self.glay1(out)
            out = self.glay2(out)
            out = self.maxpool(out)
            out = self.glay3(out)
            out = self.glay4(out)
            out = self.glay5(out)
            out = self.glay6(out)
            out = self.glay7(out)
            out = self.maxpool(out)
            out = self.glay8(out)
            out = self.glay9(out)
            out = self.avgpool(out)
            out = out.view(out.size(0), -1)
            print("Out size : ", out.size())
            out = self.linear(out)

        return out

As I wrote, at the first step of GoogleNet the error occurs. Prior line of the error-occurs, someone suggested to add with torch.no_grad() : at model's forward function if there's cuda memory error but memory wasn't the issue. But then element 0 of tensors does not require grad and does not have a grad_fn error was out.
I tried empty_cache on every step I suspect that using GPU memory a lot, but still not working.
If anyone who've been similar error or know the reason, your advice must be very thankful.
Add full error message
cuda memory error
This is what I got with memory error.
grad error
Below error was after I add torch.no_grad() at forward()


title: How exactly does torch / np einsum work internally

body:
This is a query regarding the internal working of torch.einsum in the GPU. I know how to use einsum. Does it perform all possible matrix multiplications, and just pick out the relevant ones, or does it perform only the required computation?
For example, consider two tensors a and b, of shape (N,P), and I wish to find the dot product of each corresponding tensor ni, of shape (1,P).
Using einsum, the code is:
torch.einsum('ij,ij->i',a,b)

Without using einsum, another way to obtain the output is :
torch.diag(a @ b.t())

Now, the second code is supposed to perform significantly more computations than the first one (eg if N = 2000, it performs 2000 times more computation). However, when I try to time the two operations, they take roughly the same amount of time to complete, which begs the question. Does einsum perform all combinations (like the second code), and picks out the relevant values?
Sample Code to test:
import time
import torch
for i in range(100):
  a = torch.rand(50000, 256).cuda()
  b = torch.rand(50000, 256).cuda()

  t1 = time.time()
  val = torch.diag(a @ b.t())
  t2 = time.time()
  val2 = torch.einsum('ij,ij->i',a,b)
  t3 = time.time()
  print(t2-t1,t3-t2, torch.allclose(val,val2))


title: Using PyTorch nn.Sequential() to define a network in a flexible way but with results beyond expectation

body:
I tried to define a network in a more flexible way using nn.Sequential so that I can define its number of layers according to layernum:
seed = 0
torch.manual_seed(seed)
# ====== net_a =====
layers = [ nn.Linear(7, 64), nn.Tanh()]
for i in range(layernum-1): # layernum = 3
    layers.append(nn.Linear(64, 64))
    layers.append(nn.Tanh())
layers.append(nn.Linear(64, 8))
net_x = nn.Sequential(*layers)
net_y = nn.Sequential(*layers)
net_z = nn.Sequential(*layers)

# ====== net_b =====
net_x = nn.Sequential(
    nn.Linear(7, 64),
    nn.Tanh(),
    nn.Linear(64, 64),
    nn.Tanh(),
    nn.Linear(64, 64),
    nn.Tanh(),
    nn.Linear(64, 8),
)
net_y = nn.Sequential(
    #... same as net_x
)
net_z = nn.Sequential(
    #... same as net_x
)

# print(net_x)
# print(net_x[0].weight)

I use both of them individually, i.e. they are in the same .py file, but I use either of them and make the other as comments. Both of them consist of 3 networks with respect to 3 dimentions (x,y, and z).
I expected them to be the same network with same training performance.
The structure seems to be the same according to print(net_x):
    # Sequential(
    #   (0): Linear(in_features=7, out_features=64, bias=True)
    #   (1): Tanh()
    #   (2): Linear(in_features=64, out_features=64, bias=True)
    #   (3): Tanh()
    #   (4): Linear(in_features=64, out_features=64, bias=True)
    #   (5): Tanh()
    #   (6): Linear(in_features=64, out_features=8, bias=True)
    # )

But their initial weights are different according to print(net_x[0].weight):
print(net_x[0].weight) # net_a
    # tensor([[-0.0028,  0.2028, -0.3111, -0.2782, -0.1456,  0.1014, -0.0075],
    #         [ 0.2997, -0.0335,  0.1000, -0.1142, -0.0743, -0.3611, -0.2503],
    #         ......

print(net_x[0].weight) # net_b
    # tensor([[ 0.2813,  0.2968,  0.0078,  0.1518,  0.3776, -0.3247,  0.0071],
    #         [ 0.3448, -0.0988, -0.2798,  0.3347,  0.3581,  0.2229,  0.2841],
    #         ......

======ADDED=====
I trained the network like this:
def train_on_batch(x, y, net, stepsize=innerstepsize):
    x = totorch(x)
    y = totorch(y)
    if(use_cuda):
        x,y = x.cuda(),y.cuda()
    net.zero_grad()
    ypred = net(x)
    loss = (ypred - y).pow(2).mean()
    loss.backward()
    for param in net.parameters():
        param.data -= stepsize * param.grad.data

iteration = 100
for iter in range(iteration):

    # TRAIN
    PrepareSample() # get in_support
    for i in range(tnum_support):
        out_x = trajectory_support_x[i,1:9]
        out_y = trajectory_support_y[i,1:9]
        out_z = trajectory_support_z[i,1:9]
        # Do SGD on this task
        for _ in range(innerepochs): # SGD 1 times
            train_on_batch(in_support[i], out_x, net_x)
            train_on_batch(in_support[i], out_y, net_y)
            train_on_batch(in_support[i], out_z, net_z)

    # TEST
    if iter==0 or (iter+1) % 10 == 0:
        ind = [0,1,2,3,4,5,6,7,8,9]
        loss = [0,0,0,0,0,0]
        for i in range(tnum_test):
            inputs = in_test[i]
            outputs_x = trajectory_test_x[i].tolist()
            x_test = trajectory_test_x[i,[0,9]]
            y_test = trajectory_test_x[i,1:9]
            pred_x = np.hstack((x_test[0],predict(inputs, net_x),x_test[1]))
            loss[i] = np.square(predict(inputs, net_x) - y_test).mean() # mse

            inputs = in_test[i]
            outputs_y = trajectory_test_y[i].tolist()
            x_test = trajectory_test_y[i,[0,9]]
            y_test = trajectory_test_y[i,1:9]
            pred_y = np.hstack((x_test[0],predict(inputs, net_y),x_test[1]))
            loss[i+2] = np.square(predict(inputs, net_y) - y_test).mean() # mse

            inputs = in_test[i]
            outputs_z = trajectory_test_z[i].tolist()
            x_test = trajectory_test_z[i,[0,9]]
            y_test = trajectory_test_z[i,1:9]
            pred_z = np.hstack((x_test[0],predict(inputs, net_z),x_test[1]))
            loss[i+4] = np.square(predict(inputs, net_z) - y_test).mean() # mse

        iterNum.append(iter+1)
        avgloss.append(np.mean(loss))

both of them are trained with exactly the same data (they are in the same .py file and of course use the same data).
=====This is avgloss of net_a:

=====This is avgloss of net_a with torch.manual_seed(seed) before every network definition:

=====This is avgloss of net_b:

=====This is avgloss of net_b with torch.manual_seed(seed) before every network definition:

The training of net_a is weird, the MSE is high at initial time, and didn't reduce. On the contrary, the training of net_b seems common, the MSE is relatively low at first, and reduce to a smaller value after 100 iterations.
Is anyone know how to fix this?  I would like to go through different layer number, layer size, activation functions of the network. I don't want to write every network of specified hyper-parameters.


title: Torch C++: API to check NAN

body:
I am using libtorch C++. In python version we can easily check the value of a tensor by calling its numpy value, and in numpy we have np.isnan(). I was wondering if there is a built in function in libtorch C++ to check whether a tensor has any NAN value?

Thanks,
Afshin


title: When does ONNX decide to include a Gather op?

body:
Obscure question, I suppose, but I have a PyTorch model that I've just built, and it's failing to convert to CoreML because ONNX has added Gather ops. The complete model is actually an amalgamation of two separate models, intended to improve performance by keeping the processing on the GPU/Metal for as long as possible.

Building this "composite" model required me to create a couple of slices, of the form x = y[:, 0], and I'm wondering if these might be the reason for the Gather ops?

I do realize I can create a custom layer, but I've just been through a horrible fiasco with custom layers in CoreML, that wasted many, many hours, and got me nowhere, so I'm trying to find another way around the problem.

If finding a way around those slices would prevent ONNX from adding Gather I'd be willing to search for a solution.

Any thoughts appreciated.


title: How to train PyTorch transfer learning tutorial with more then 1 GPU

body:
I am currently following the PyTorch transfer learning tutorial in: https://pytorch.org/tutorials/beginner/transfer_learning_tutorial.html

I have been able to complete tutorial and train on both a CPU and 1 GPU.

I am utilising Google Cloud Platform Notebook Instances and using 4 NVIDIA Tesla k80 x 4 GPU. It is here that I run into a Server Connection Error (invalid response: 504) error when I train the network on more than 1 GPU

model_ft = models.resnet18(pretrained=True)
num_ftrs = model_ft.fc.in_features
model_ft.fc = nn.Linear(num_ftrs, 2)

device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")

## Using 4 GPUs
if torch.cuda.device_count() > 1:
    model_ft = nn.DataParallel(model_ft)
model_ft = model_ft.to(device)

criterion = nn.CrossEntropyLoss()

optimizer_ft = optim.SGD(model_ft.parameters(), lr=0.001, momentum=0.9)

exp_lr_scheduler = lr_scheduler.StepLR(optimizer_ft, step_size=7, gamma=0.1)

model_ft = train_model(model_ft, criterion, optimizer_ft, exp_lr_scheduler, num_epochs=25)

The idea was to utilise the data parallelism function to utilise all available GPU (which is 4) to train the network.

Am I missing something in the implementation, please advice.

Thanks


title: Unexpected model output running Onnx model in Unity using Barracuda

body:
Context
I am trying to use a pre-trained model in ONNX format to do inference on image data in Unity. The model is linked to the executing component in Unity as an asset called modelAsset. I am using Barracuda version 1.0.0 for this and executing the model as follows:
// Initialisation
this.model = ModelLoader.Load(this.modelAsset);
this.worker = WorkerFactory.CreateWorker(WorkerFactory.Type.CSharpBurst, model);

// Loop
Tensor tensor = new Tensor(1, IMAGE_H, IMAGE_W, 3, data);
worker.Execute(tensor);
Tensor modelOutput = worker.PeekOutput(OUTPUT_NAME);

The data going into the input tensor (of which the model has only 1) is image data of h * w with 3 channels for RGB values between -0.5 and 0.5. The model has multiple outputs which I retrieve in the last line shown above.
Expected behavior
Using the same input data, the PyTorch model and converted ONNX model produce the same output data in Python (ONNXRuntime and PyTorch) as in Barracuda in Unity.
Problem
In python both the ONNX and PyTorch model produce the same output. However, the same ONNX model running in Barracuda produces a different output. The difference is mainly that we expect a heatmap but Barracuda consistently produces values somewhere between 0.001 and -0.0004 in these patterns:

This makes it almost seem like the model weights are not properly loaded.
What we found
When converting to ONNX as per the Barracuda manual we found that if we did not set the model to inference mode in the PyTorch net before conversion (link), these same, incorrect, results were generated by ONNXRuntime in Python. In other words, it looks like this inference mode is saved in the ONNX model and is recognized by ONNXRuntime in Python but not in Barracuda.
Our question
In general:

How do we get this model in Barracuda in Unity to produce the same results as ONNXRuntime/PyTorch in Python?

And potentially:

How does the inference mode get embedded into the ONNX file and how is it used in ONNXRuntime vs Barracuda?


title: How to convert a list of Torch tensor with grad to tensor

body:
I have a variable called pts which is shaped [batch, ch, h, w]. This is a heatmap and I want to convert it to 2nd co-ordinates. The goal is, pts_o = heatmap_to_pts(pts) where pts_o will be [batch, ch, 2]. I have wrote this function so far,
def heatmap_to_pts(self, pts):  <- pts [batch, 68, 128, 128]

    pt_num = []

    for i in range(len(pts)):

        pt = pts[i]
        if type(pt) == torch.Tensor:

            d = torch.tensor(128)                                                   * get the
            m = pt.view(68, -1).argmax(1)                                           * indices
            indices = torch.cat(((m / d).view(-1, 1), (m % d).view(-1, 1)), dim=1)  * from heatmaps

            pt_num.append(indices.type(torch.DoubleTensor) )   <- store the indices in a list

    b = torch.Tensor(68, 2)                   * trying to convert
    c = torch.cat(pt_num, out=b) *error*      * a list of tensors with grad
    c = c.reshape(68,2)                       * to a tensor like [batch, 68, 2]

    return c

The error says "cat(): functions with out=... arguments don't support automatic differentiation, but one of the arguments requires grad.". It's unable to do the operations because tensors in pt_num requires grad".
How can I convert that list to a tensor?


title: What is the upsampling method called 'area' used for?

body:
The PyTorch function torch.nn.functional.interpolate contains several modes for upsampling, such as: nearest, linear, bilinear, bicubic, trilinear, area.
What is the area upsampling modes used for?


title: How to convert a yolo darknet format into .csv file

body:
I have a few annotations that is originally in YOLO format. I need to convert it into yolo csv format in order to train with my transformers model.
Sample .csv file I need:
Sample annotation file in CSV format
The csv attributes include: image_id, width, height and coordinates of the image's bounding box.
Any help would be appreciated!


title: How to convert keras LSTM to pytorch LSTM?

body:
I have a very simple LSTM example written in Keras that I am trying to port to pytorch. But it does not seem to be able to learn at all. I am an absolute beginning so any advice is appreciated.

KERAS:

X_train_lmse has shape (1691, 1, 1), I am essentially running X(t) with X(t-1) as single feature

lstm_model = Sequential()
lstm_model.add(LSTM(7, input_shape=(1, X_train_lmse.shape[1]), activation='relu', kernel_initializer='lecun_uniform', return_sequences=False))
lstm_model.add(Dense(1))
lstm_model.compile(loss='mean_squared_error', optimizer='adam')
early_stop = EarlyStopping(monitor='loss', patience=2, verbose=1)
history_lstm_model = lstm_model.fit(X_train_lmse, y_train, epochs=100, batch_size=1, verbose=1, shuffle=False, callbacks=[early_stop])

Output:

Epoch 1/100
1691/1691 [==============================] - 10s 6ms/step - loss: 0.0236
Epoch 2/100
1691/1691 [==============================] - 9s 5ms/step - loss: 0.0076
Epoch 3/100
...

PYTORCH:

X_train_tensor has same shape as in keras (1691, 1, 1). I am specifying batch_first to be true below so I think it should be ok.

class LSTM_model(nn.Module):

    def __init__(self):
        super(LSTM_model, self).__init__()
        self.lstm = nn.LSTM(input_size=1, hidden_size=7, num_layers=1, batch_first=True)
        self.dense = nn.Linear(7, 1)

    def forward(self, x):
        out, states = self.lstm(x)
        out = self.dense(out)
        return out

lstm_model = LSTM_model()
loss_function = nn.MSELoss()
optimizer = optim.Adam(lstm_model.parameters())

for t in range(100):
    y_pred = lstm_model(X_train_tensor)
    loss = loss_function(y_pred, Y_train_tensor)
    optimizer.zero_grad()
    loss.backward()
    optimizer.step()
    print('Train Epoch ', t, ' Loss = ', loss)

Output:

Train Epoch  0  Loss =  tensor(0.2834, grad_fn=<MseLossBackward>)
Train Epoch  1  Loss =  tensor(0.2812, grad_fn=<MseLossBackward>)
Train Epoch  2  Loss =  tensor(0.2790, grad_fn=<MseLossBackward>)
Train Epoch  3  Loss =  tensor(0.2768, grad_fn=<MseLossBackward>)
Train Epoch  4  Loss =  tensor(0.2746, grad_fn=<MseLossBackward>)
Train Epoch  5  Loss =  tensor(0.2725, grad_fn=<MseLossBackward>)
Train Epoch  6  Loss =  tensor(0.2704, grad_fn=<MseLossBackward>)
Train Epoch  7  Loss =  tensor(0.2683, grad_fn=<MseLossBackward>)
...

As you can see, the error barely moves in Pytorch. Also each epoch runs much much faster than keras.

I must be doing something stupid. I checked the input data and they look identical in both implementations. Thanks!


title: Getting the output metrics from Detectron2's COCOEvaluator

body:
The metrics from standard out are much more useful than the outputs written in the output folder.

Standard out

metrics.json and coco_instances_results.json

def test(instance, cfg, trainer, test_instance):
  cfg.MODEL.WEIGHTS = os.path.join(cfg.OUTPUT_DIR, "model_final.pth")
  cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.7   # TODO What is this
  cfg.DATASETS.TEST = (test_instance, )
  predictor = DefaultPredictor(cfg)
  evaluator = COCOEvaluator(test_instance, cfg, False, output_dir=cfg.OUTPUT_DIR)
  val_loader = build_detection_test_loader(cfg, test_instance)
  result = inference_on_dataset(trainer.model, val_loader, evaluator)

How can I get the details like the Average Percision for ioU=0.5 or per-category bbox AP?


title: Pytorch won't run on deep-learning-vm "Found no NVIDIA driver on your system"

body:
I did create a new conda environment to run my code from. But I get the following error:
AssertionError:
Found no NVIDIA driver on your system. Please check that you
have an NVIDIA GPU and installed a driver from
http://www.nvidia.com/Download/index.aspx
> /opt/conda/envs/ml/lib/python3.8/site-packages/torch/cuda/__init__.py(51)_check_driver()
     49         if torch._C._cuda_getDriverVersion() == 0:
     50             # found no NVIDIA driver on the system
---> 51             raise AssertionError("""
     52 Found no NVIDIA driver on your system. Please check that you
     53 have an NVIDIA GPU and installed a driver from

Any help very much appreciated!


title: Why does importing Pytorch in Sublime Text cause a ModuleNotFoundError?

body:
The command 'import torch' causes the build error: 'ModuleNotFoundError: No module named 'torch''.

But when I run the command in the terminal, pytorch works just fine (Python 3.7.X).


title: Libtorch/Pytorch Dilemma when combined static library into one STATIC library

body:
I have around 26 static libraries such as liba.a, libb.a, libc.a, ..., libz.a. There are two catches here:

1) circular dependencies between for example liba.a and libb.a;

2) some lib*.a has static global registration code which is unreferenced, but shall NOT be stripped away.

Thanks to stackoverflow, I managed to solve above two problems with the ld option  -Wl,--whole-archive  -la -lb -lc -ld -le ...(omitted)... -lz -Wl,--no-whole-archive  -lpthread -lm -ldl -lrt -fopenmp , and the executable binary works! It is also explained in ld linker question: the --whole-archive option

Now I need to combine all 26 lib*.a into one static library liball.a. Also thanks to stackoverflow, the below MRI script.mri successuflly produces liball.a through the command ar -M < script.mri

create liball.a
addlib liba.a
addlib libb.a
addlib libc.a
addlib libd.a
... //omitted
addlib libz.a
save
end

However, here comes the issue when linking with newly combined static library liball.a:

1) with the option -Wl,--whole-archive  -lall  -Wl,--no-whole-archive  -lpthread -lm -ldl -lrt -fopenmp FAILS!! It generates thousands of multiple defined symbols error or undefined symbols error!!

2) without the option -Wl,--whole-archive, the link command with-lall -lpthread -lm -ldl -lrt -fopenmp successfully produced the executable binary. However, the binary fails to execute , complaining some Device Registration code error. I understand that this is caused by some CPU initialization code getting pruned away when linking. Below is the detailed error

 p INTERNAL ASSERT FAILED at ../c10/core/impl/DeviceGuardImplInterface.h:132, please report a bug to PyTorch. DeviceGuardImpl for cpu is not available (getDeviceGuardImpl at ../c10/core/impl/DeviceGuardImplInterface.h:132)

This isssue is also mentioned here https://github.com/pytorch/pytorch/issues/14367

One way to solve it is to use selective registration. Could anyone share more details on this?

This question is NOT a duplicate of How to merge two "ar" static libraries into one?
The MRI script method is from highest vote answer of the above link, and it is not working.
Please remove the duplicate mark so that people can contribute. Thanks.


title: Error: symeig_cpu: the algorithm failed to converge: 6 off-diagonal elements of an intermediate tridiagonal form did not converge to zero

body:
I am trying to use https://github.com/Michaelvll/DeepCCA

After 20-40 iterations, It gives the following error:

RuntimeError: symeig_cpu: the algorithm failed to converge; 6 off-diagonal elements of an intermediate tridiagonal form did not converge to zero.

Error is generated from https://github.com/Michaelvll/DeepCCA/blob/master/objectives.py#L46

[D1, V1] = torch.symeig(SigmaHat11, eigenvectors=True)

System Configuration:

Windows 10.
Python 3.7
Pytorch 1.2.0

How can I debug this?


title: How to Create Class Label for Mosaic Augmentation in Image Classification?

body:
To create a class label in CutMix or MixUp type augmentation, we can use beta such as np.random.beta or scipy.stats.beta and do as follows for two labels:
label = label_one*beta + (1-beta)*label_two

But what if we've more than two images? In YoLo4, they've tried an interesting augmentation called Mosaic Augmentation for object detection problems. Unlike CutMix or MixUp, this augmentation creates augmented samples with 4 images. In object detection cases, we can compute the shift of each instance co-ords and thus possible to get the proper ground truth, here. But for only image classification cases, how can we do that?
Here is a starter.
import tensorflow as tf
import matplotlib.pyplot as plt
import random

(train_images, train_labels), (test_images, test_labels) = \
tf.keras.datasets.cifar10.load_data()
train_images = train_images[:10,:,:]
train_labels = train_labels[:10]
train_images.shape, train_labels.shape

((10, 32, 32, 3), (10, 1))

Here is a function we've written for this augmentation; ( too ugly with an `inner-outer loop! Please suggest if we can do it efficiently.)
def mosaicmix(image, label, DIM, minfrac=0.25, maxfrac=0.75):
    '''image, label: batches of samples
    '''
    xc, yc  = np.random.randint(DIM * minfrac, DIM * maxfrac, (2,))
    indices = np.random.permutation(int(image.shape[0]))
    mosaic_image = np.zeros((DIM, DIM, 3), dtype=np.float32)
    final_imgs, final_lbs = [], []

    # Iterate over the full indices
    for j in range(len(indices)):
        # Take 4 sample for to create a mosaic sample randomly
        rand4indices = [j] + random.sample(list(indices), 3)

        # Make mosaic with 4 samples
        for i in range(len(rand4indices)):
            if i == 0:    # top left
                x1a, y1a, x2a, y2a =  0,  0, xc, yc
                x1b, y1b, x2b, y2b = DIM - xc, DIM - yc, DIM, DIM # from bottom right
            elif i == 1:  # top right
                x1a, y1a, x2a, y2a = xc, 0, DIM , yc
                x1b, y1b, x2b, y2b = 0, DIM - yc, DIM - xc, DIM # from bottom left
            elif i == 2:  # bottom left
                x1a, y1a, x2a, y2a = 0, yc, xc, DIM
                x1b, y1b, x2b, y2b = DIM - xc, 0, DIM, DIM-yc   # from top right
            elif i == 3:  # bottom right
                x1a, y1a, x2a, y2a = xc, yc,  DIM, DIM
                x1b, y1b, x2b, y2b = 0, 0, DIM-xc, DIM-yc    # from top left

            # Copy-Paste
            mosaic_image[y1a:y2a, x1a:x2a] = image[i,][y1b:y2b, x1b:x2b]

        # Append the Mosiac samples
        final_imgs.append(mosaic_image)

    return final_imgs, label

The augmented samples, currently with the wrong labels.
data, label = mosaicmix(train_images, train_labels, 32)
plt.imshow(data[5]/255)

However, here are some more examples to motivate you. Data is from the Cassava Leaf competition.

However, to get the proper label from this augmented sample, we tried something like this, let's say for each interaction over the batches inside the outer loop and inner loop if we can compute the distribution of the 4 samples, like how each of them covers the area within mosaic_image, so that we can multiply each with distribution probability of a.
    # Iterate over the full indices
    for j in range(len(indices)):

        b = tf.random.uniform([],0,1) # this is beta dist with alpha=1.0
        P = tf.cast( tf.random.uniform([],0,1)<=1.0, tf.int32)

        for i in range(len(rand4indices)):
            ....
            WIDTH = tf.cast( DIM * tf.math.sqrt(1-b),tf.int32) * P
            a = tf.cast(WIDTH*WIDTH/DIM/DIM,tf.float32)


title: Package Cuda 8 not found for PyTorch

body:
I am trying to follow the tutorial for chatbots at facebook (https://github.com/facebookresearch/end-to-end-negotiator) and I am stuck here :
conda install pytorch torchvision cuda80 -c soumith
 I managed to get all packages except for cuda80 which is not found. I also tried
conda install magma-cuda80 -c soumith
conda install cuda80 -c soumith
conda install magma-cuda80 -c pytorch

with no success.
PackagesNotFoundError: The following packages are not available from current channels:
  - magma-cuda80

Where can I find the package ? I am on Windows10.


title: Change array shape / pytorch preprocessing / python

body:
I'm experimenting with pytorch and trying to run It on GPU and have such error

ValueError: Target size (torch.Size([4, 256, 1, 320])) must be the same as input size (torch.Size([4, 1, 256, 320]))

These are how I reshaping the array.

def __getitem__(self, idx):
    img_filename = os.path.join(
        self.images_dir, self.images_name[idx] + '.jpg')
    img = np.array(Image.open(img_filename))
    img = cv2.resize(img, (320, 256))

    if self.target_dir:
        mask_filename = os.path.join(
            self.target_dir, self.images_name[idx] + '.png')
        mask = np.array(Image.open(mask_filename))
        mask = np.resize(mask, (320, 256))
        mask = np.reshape(mask, (1,) + mask.shape)
    else:
        mask = []

    if self.transforms:
        img = self.transforms(img)
        if mask != []:
            mask = transforms.ToTensor()(mask)

    return {'img': img, 'mask': mask}

and these are a full log of an Error
Any suggestion?

------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-14-d3dc126e6038> in <module>()
      6         optimizer.zero_grad()
      7         output = unet(batch['img'].cuda())
----> 8         loss = criterion(output, batch['mask'])
      9         loss.backward()
     10         optimizer.step()

~\Anaconda3\lib\site-packages\torch\nn\modules\module.py in __call__(self,
*input, **kwargs)
    475             result = self._slow_forward(*input, **kwargs)
    476         else:
--> 477             result = self.forward(*input, **kwargs)
    478         for hook in self._forward_hooks.values():
    479             hook_result = hook(self, input, result)

~\Anaconda3\lib\site-packages\torch\nn\modules\loss.py in forward(self,
input, target)
    571                                                   self.weight,
    572
pos_weight=self.pos_weight,
--> 573
reduction=self.reduction)
    574
    575

~\Anaconda3\lib\site-packages\torch\nn\functional.py in
binary_cross_entropy_with_logits(input, target, weight, size_average,
reduce, reduction, pos_weight)
   1644         reduction = _Reduction.legacy_get_string(size_average, reduce)
   1645     if not (target.size() == input.size()):
-> 1646         raise ValueError("Target size ({}) must be the same as input
size ({})".format(target.size(), input.size()))
   1647
   1648     max_val = (-input).clamp(min=0)

ValueError: Target size (torch.Size([4, 256, 1, 320])) must be the same as
input size (torch.Size([4, 1, 256, 320]))

update:
these are how reshaping works and all looks fine. Just don't understand why I got such an error.

mask = np.array(Image.open('data/train_mask/1.png'))
mask = np.resize(mask, (320, 240))
mask = np.reshape(mask, mask.shape + (1,))
img = np.array(Image.open('data/train/1.jpg'))
print(np.shape(mask), np.shape(img))

(320, 240, 1) (320, 240, 3)


title: Convolution Neural Network for regression using pytorch

body:
I am trying to do create CNN for regression purpose. Input is image data.
For learning purpose , i have 10 image of shape (10,3,448,448), where 10 are images, 3 are channel and 448 are hieght and width.
Output lables are (10,245).
Here is my architecture

class CNN(nn.Module):
    def __init__(self):
        super(CNN, self).__init__()
        self.conv1 = nn.Conv2d(3, 32, kernel_size=5)
        self.conv2 = nn.Conv2d(32, 32, kernel_size=5)
        self.conv3 = nn.Conv2d(32,64, kernel_size=5)
        self.fc1 = nn.Linear(3*3*64, 256)
        self.fc2 = nn.Linear(256, 245)

    def forward(self, x):
        x = F.relu(self.conv1(x))
        #x = F.dropout(x, p=0.5, training=self.training)
        x = F.relu(F.max_pool2d(self.conv2(x), 2))
        x = F.dropout(x, p=0.5, training=self.training)
        x = F.relu(F.max_pool2d(self.conv3(x),2))
        x = F.dropout(x, p=0.5, training=self.training)
        x = x.view(-1,3*3*64 )
        x = F.relu(self.fc1(x))
        x = F.dropout(x, training=self.training)
        x = self.fc2(x)
        return x

cnn = CNN()
print(cnn)

it = iter(train_loader)
X_batch, y_batch = next(it)
print(cnn.forward(X_batch).shape)

Using batch size 2 i am expecting data shape produced by model is (2,245). But it is producing data of shape (2592, 245)


title: list of tensors into tensors

body:
I have a list like

xs = [tensor1, tensor2, tensor3....]

I want to change it into a tensor so that I can feed xs and ys into the Pytorch DataLoader.
I tried this,

xs = torch.Tensor(xs)

but it doesn't work as the individual elements are not items but tensors.


title: PyTorch - TypeError: forward() takes 1 positional argument but 2 were given

body:
I do not understand where this error comes from, the number of arguments to the model seems correct, below is my model:
class MancalaModel(nn.Module):

    def __init__(self, n_inputs=16, n_outputs=16):
        super().__init__()

        n_neurons = 256

        def create_block(n_in, n_out):
            block = nn.ModuleList()
            block.append(nn.Linear(n_in, n_out))
            block.append(nn.ReLU())
            return block

        self.blocks = nn.ModuleList()
        self.blocks.append(create_block(n_inputs, n_neurons))
        for _ in range(6):
            self.blocks.append(create_block(n_neurons, n_neurons))

        self.actor_block = nn.ModuleList()
        self.critic_block = nn.ModuleList()
        for _ in range(2):
            self.actor_block.append(create_block(n_neurons, n_neurons))
            self.critic_block.append(create_block(n_neurons, n_neurons))

        self.actor_block.append(create_block(n_neurons, n_outputs))
        self.critic_block.append(create_block(n_neurons, 1))

        self.apply(init_weights)

    def forward(self, x):
        x = self.blocks(x)
        actor = F.softmax(self.actor_block(x))
        critics = self.critic_block(x)
        return actor, critics

Then I create an instance and make a forward pass with random number
model = MancalaModel()
x = model(torch.rand(1, 16))

Then I got the TypeError saying the number of arguments is not correct:
      2 model = MancalaModel()
----> 3 x = model(torch.rand(1, 16))
      4 # summary(model, (16,), device='cpu')
      5

d:\environments\python\lib\site-packages\torch\nn\modules\module.py in _call_impl(self, *input, **kwargs)
    725             result = self._slow_forward(*input, **kwargs)
    726         else:
--> 727             result = self.forward(*input, **kwargs)
    728         for hook in itertools.chain(
    729                 _global_forward_hooks.values(),

D:\UOM\Year3\AI & Games\KalahPlayer\agents\model_agent.py in forward(self, x)
     54
     55     def forward(self, x):
---> 56         x = self.blocks(x)
     57         actor = F.softmax(self.actor_block(x))
     58         critics = self.critic_block(x)

d:\environments\python\lib\site-packages\torch\nn\modules\module.py in _call_impl(self, *input, **kwargs)
    725             result = self._slow_forward(*input, **kwargs)
    726         else:
--> 727             result = self.forward(*input, **kwargs)
    728         for hook in itertools.chain(
    729                 _global_forward_hooks.values(),

TypeError: forward() takes 1 positional argument but 2 were given

Any help is appreciated, thanks!


title: How to convert custom Pytorch model to torchscript (pth to pt model)?

body:
I trained a custom model with pytorch using colab environment. I was able to successfully save the trained model to Google Drive with the name model_final.pth . I want to convert model_final.pth to model_final.pt so that it can be used on mobile devices.
The code I use to train the model is as follows;

cfg = get_cfg()
cfg.merge_from_file(model_zoo.get_config_file("COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml"))
cfg.DATASETS.TRAIN = ("mouse_train",)
cfg.DATASETS.TEST = ()
cfg.DATALOADER.NUM_WORKERS = 2
cfg.MODEL.WEIGHTS = model_zoo.get_checkpoint_url("COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml")
cfg.SOLVER.IMS_PER_BATCH = 2
cfg.SOLVER.BASE_LR = 0.00025
cfg.SOLVER.MAX_ITER = 1000
cfg.SOLVER.STEPS = []
cfg.MODEL.ROI_HEADS.BATCH_SIZE_PER_IMAGE = 512
cfg.MODEL.ROI_HEADS.NUM_CLASSES = 1
cfg.OUTPUT_DIR="drive/Detectron2/"

os.makedirs(cfg.OUTPUT_DIR, exist_ok=True)
trainer = DefaultTrainer(cfg)
trainer.resume_or_load(resume=False)
trainer.train()

The code I used to convert the model as follows;
import torch
import torchvision

print("cfg.MODEL.WEIGHTS: ",cfg.MODEL.WEIGHTS)   ## RETURNS : cfg.MODEL.WEIGHTS:  drive/Detectron2/model_final.pth
model = build_model(cfg)
model.eval()
example = torch.rand(1, 3, 224, 224)
traced_script_module = torch.jit.trace(model, example)
traced_script_module.save("drive/Detectron2/model-final.pt")

But I am getting this error ;
/usr/local/lib/python3.6/dist-packages/torch/tensor.py:593: RuntimeWarning: Iterating over a tensor might cause the trace to be incorrect. Passing a tensor of different shape won't change the number of iterations executed (and might lead to errors or silently give incorrect results).
  'incorrect results).', category=RuntimeWarning)
---------------------------------------------------------------------------
IndexError                                Traceback (most recent call last)
<ipython-input-17-8e544c0f39c8> in <module>()
      7 model.eval()
      8 example = torch.rand(1, 3, 224, 224)
----> 9 traced_script_module = torch.jit.trace(model, example)
     10 traced_script_module.save("drive/Detectron2/model_final.pt")

7 frames
/usr/local/lib/python3.6/dist-packages/detectron2/modeling/meta_arch/rcnn.py in <listcomp>(.0)
    219         Normalize, pad and batch the input images.
    220         """
--> 221         images = [x["image"].to(self.device) for x in batched_inputs]
    222         images = [(x - self.pixel_mean) / self.pixel_std for x in images]
    223         images = ImageList.from_tensors(images, self.backbone.size_divisibility)

IndexError: too many indices for tensor of dimension 3


title: How to perform inference on tensorRT pytorch GAN model

body:
I have implemented my Pix2Pix GAN model in tensorrt using onnx format.
But I do not know how to perform inference on tensorRT model, because input to the model in (3, 512, 512 ) image and output is also (3, 512, 512) image. The samples do not clearly show how to input and output image from tensorRT engine.

This is my code :

   def get_image():
    img = _load_and_resize("/home/abdullah/Desktop/Profile.png")
    img = _shuffle_and_normalize(img)
    return img

def _load_and_resize(input_image_path):

    image_raw = Image.open(input_image_path)
    # convention (width, height) in PIL:

    image_resized = image_raw.resize(
        (512, 512), resample=Image.BICUBIC)
    image_resized = np.array(image_resized, dtype=np.float32, order='C')
    return image_resized

def _shuffle_and_normalize(image):

    image /= 255.0
    # HWC to CHW format:
    image = np.transpose(image, [2, 0, 1])
    # CHW to NCHW format
    image = np.expand_dims(image, axis=0)
    # Convert the image to row-major order, also known as "C order":
    image = np.array(image, dtype=np.float32, order='C')
    return image
def _reshape_output(output):

    return np.reshape(output, (512, 512, 3))
def main():
    """Create inference for generator for eye."""

    onnx_file_path = 'generator.onnx'
    engine_file_path = "generator.trt"

    output_shapes = [(1, 3,512, 512)]
    image =  get_image()
    print('IMage shape is ', image.shape)
    # Do inference with TensorRT

    with get_engine(onnx_file_path, engine_file_path) as engine, engine.create_execution_context() as context:

        inputs, outputs, bindings, stream = common.allocate_buffers(engine)
        print('Running inference on image {}...'.format(1))
        inputs[0].host = image
        print(inputs)
        trt_outputs = common.do_inference(context, bindings=bindings, inputs=inputs, outputs=outputs, stream=stream)

    # Before doing post-processing, we need to reshape the outputs as the common.do_inference will give us flat arrays.
    trt_outputs = [output.reshape(shape) for output, shape in zip(trt_outputs, output_shapes)]

    image = _reshape_output(trt_outputs)
    image = np.array(image, dtype = 'float32')
    plt.imshow(image)
    plt.show()
    print('Output shape is ', trt_outputs.shape)

But the output is not as it is expected. Please tell me, what is the best way to input image into tensorRT engine and how to get proper output from it.
Thanks


title: RoBERTa classification RuntimeError: shape '[-1, 9]' is invalid for input of size 8

body:
m = MultiLabelBinarizer()

    X = pd.read_csv('data/data.csv', sep=None, engine='python')
    X = X.dropna()

    Y_train = m.fit_transform(X['labels'])
    Y_train2 = [list(i) for i in Y_train]

    data = pd.DataFrame({'text': pd.Series(X[text_col]), 'labels': Y_train2})
    data = data.dropna()

    train_df, eval_df = train_test_split(data, test_size=0.2)

    numLabels = len(pd.unique(X['labels])) # count of the labels

    model = MultiLabelClassificationModel('roberta', 'roberta-base', num_labels=numLabels, use_cuda=False)

    model.train_model(pd.DataFrame(train_df))

My data-structure for the label column is: [[0,1,0,0,0,1,0,0], [0,1,1,0,0,1,0,0], [0,0,0,0,0,0,0,1]....] for every row there is one label-list like [0,1,0,0,0,1,0,0] in the label-column

And for the texts there is one text (newspaper article) per row.

(got it from that source: https://github.com/ThilinaRajapakse/simpletransformers#minimal-start-for-multilabel-classification)

the model can be trained if i train it with only 4 entries. But when i want to train it with the whole dataset it gives me that: RuntimeError: shape '[-1, 9]' is invalid for input of size 8:

 File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/simpletransformers/classification/multi_label_classification_model.py", line 121, in train_model
    return super().train_model(train_df, multi_label=multi_label, eval_df=eval_df, output_dir=output_dir, show_running_loss=show_running_loss, args=args)
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/simpletransformers/classification/classification_model.py", line 208, in train_model
    global_step, tr_loss = self.train(train_dataset, output_dir, multi_label=multi_label, show_running_loss=show_running_loss, eval_df=eval_df, **kwargs)
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/simpletransformers/classification/classification_model.py", line 306, in train
    outputs = model(**inputs)
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/torch/nn/modules/module.py", line 541, in __call__
    result = self.forward(*input, **kwargs)
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/simpletransformers/custom_models/models.py", line 117, in forward
    loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1, self.num_labels))
RuntimeError: shape '[-1, 9]' is invalid for input of size 8

I have no idea where that size 8 comes from and what to do now since it works with very few entries.
Can anyone help?


title: getting unicode decode error while trying to load pre-trained model using torch.load(PATH)

body:
Trying to load a ResNet 18 pre-trained model using the torch.load(PATH) but getting Unicode decode error please help.

Traceback (most recent call last):
  File "main.py", line 312, in <module>
    main()
  File "main.py", line 138, in main
    checkpoint = torch.load(args.resume)
  File "F:\InsSoft\Anaconda\lib\site-packages\torch\serialization.py", line 593, in load
    return _legacy_load(opened_file, map_location, pickle_module, **pickle_load_args)
  File "F:\InsSoft\Anaconda\lib\site-packages\torch\serialization.py", line 773, in _legacy_load
    result = unpickler.load()
UnicodeDecodeError: 'utf-8' codec can't decode byte 0xbe in position 2: invalid start byte


title: Why the CUDA memory is not release with torch.cuda.empty_cache()

body:
On my Windows 10, if I directly create a GPU tensor, I can successfully release its memory.
import torch
a = torch.zeros(300000000, dtype=torch.int8, device='cuda')
del a
torch.cuda.empty_cache()

But if I create a normal tensor and convert it to GPU tensor, I can no longer release its memory.
import torch
a = torch.zeros(300000000, dtype=torch.int8)
a.cuda()
del a
torch.cuda.empty_cache()

Why this is happening.


title: what is the tensorflow equivalent for pytorch probability function: torch.bernoulli?

body:
In Pytorch, you can do following:
x = torch.bernoulli(my_data)

Any similar functionality in tensorflow? Can the input be 2-D tensor, such as (batch, len)?
I tried tensorflow.contrib.distributions.Bernoulli:
import numpy as np
tmp_x1 = np.random.rand(20,5)
new_data_2 = tf.convert_to_tensor(tmp_x1)
from tensorflow.contrib.distributions import  Bernoulli
tmp2_x1 = Bernoulli(probs=new_data_2)

got error:
return math_ops.log(probs) - math_ops.log1p(-1. * probs), probs
TypeError: unsupported operand type(s) for *: 'float' and 'Tensor'


title: torch.nn.functional.conv2d for several channels/batches

body:
I have an image with I want to pad (to maintain the same shape) and then perform a convolution with a given kernel. It works ok if I have only one channel and one image in the batch. But how to properly rewrite it for several batches & channels? I suppose, for batches I can just duplicate the kernel along dimension 0. But what about channels? What is the proper way to do it in torch? See the toy example below.

import torch.nn.functional as f

x = torch.zeros((1,1,16,16))
x[...,6:10,6:10] = 1.
ker = torch.ones(3,3)
ker[1,1] = -4
padding = (ker.shape[1] // 2, ker.shape[1] // 2,
           ker.shape[0] // 2, ker.shape[0] // 2)
x = f.pad(x, padding, mode='replicate')
kernel = ker.reshape((1, 1, ker.shape[0], -1))
result = f.conv2d(x, ker, groups=1)

How to rewrite this piece of code to deal with multiple channels and batches? I read documentation but to be honest it did not seem very detailed to me.


title: pix2pixHD shows error after changing datasets

body:
I was trying out the pix2pixHD code from the link below.
https://github.com/NVIDIA/pix2pixHD

The train.py worked with default images (in datasets/cityscapes). However, after changing images in the dataset, it shows the error below.

    model [Pix2PixHDModel] was created
    create web directory ./checkpoints/label2city/web...
    Traceback (most recent call last):
      File "/home/shimada/venv/py2.7/projects/Hiwi/pix2pixHD/train.py", line 58, in <module>
        Variable(data['image']), Variable(data['feat']), infer=save_fake)
      File "/home/shimada/venv/py2.7/local/lib/python2.7/site-packages/torch/nn/modules/module.py", line 325, in __call__
        result = self.forward(*input, **kwargs)
      File "/home/shimada/venv/py2.7/local/lib/python2.7/site-packages/torch/nn/parallel/data_parallel.py", line 66, in forward
        return self.module(*inputs[0], **kwargs[0])
      File "/home/shimada/venv/py2.7/local/lib/python2.7/site-packages/torch/nn/modules/module.py", line 325, in __call__
        result = self.forward(*input, **kwargs)
      File "/home/shimada/venv/py2.7/projects/Hiwi/pix2pixHD/models/pix2pixHD_model.py", line 141, in forward
        fake_image = self.netG.forward(input_concat)
      File "/home/shimada/venv/py2.7/projects/Hiwi/pix2pixHD/models/networks.py", line 213, in forward
        return self.model(input)
      File "/home/shimada/venv/py2.7/local/lib/python2.7/site-packages/torch/nn/modules/module.py", line 325, in __call__
        result = self.forward(*input, **kwargs)
      File "/home/shimada/venv/py2.7/local/lib/python2.7/site-packages/torch/nn/modules/container.py", line 67, in forward
        input = module(input)
      File "/home/shimada/venv/py2.7/local/lib/python2.7/site-packages/torch/nn/modules/module.py", line 325, in __call__
        result = self.forward(*input, **kwargs)
      File "/home/shimada/venv/py2.7/local/lib/python2.7/site-packages/torch/nn/modules/conv.py", line 277, in forward
        self.padding, self.dilation, self.groups)
      File "/home/shimada/venv/py2.7/local/lib/python2.7/site-packages/torch/nn/functional.py", line 90, in conv2d
        return f(input, weight, bias)
    RuntimeError: Given groups=1, weight[64, 36, 7, 7], so expected input[1, 39, 518, 1030] to have 36 channels, but got 39 channels instead
    THCudaCheck FAIL file=/pytorch/torch/lib/THC/generic/THCStorage.c line=184 error=59 : device-side assert triggered
    terminate called after throwing an instance of 'std::runtime_error'
      what():  cuda runtime error (59) : device-side assert triggered at /pytorch/torch/lib/THC/generic/THCStorage.c:184
    bash: line 1: 10965 Aborted                 (core dumped) env "PYCHARM_HOSTED"="1" "PYTHONUNBUFFERED"="1" "PYTHONIOENCODING"="UTF-8" "PYCHARM_MATPLOTLIB_PORT"="42188" "JETBRAINS_REMOTE_RUN"="1" "PYTHONPATH"="/home/shimada/.pycharm_helpers/pycharm_matplotlib_backend:/home/shimada/venv/py2.7/projects/Hiwi/pix2pixHD" /home/shimada/venv/py2.7/bin/python -u /home/shimada/venv/py2.7/projects/Hiwi/pix2pixHD/train.py

I changed the images with same size (width 2048, hight 1024), same extension (.png) and gave the same names. Why doesn't it work?


title: Fail to train large data in pytorch

body:
I tried to construct two fully-connected layers in pytorch to embed features like [x1,x2,...,xn] into multiple targets [y1,y2,y3,y4,y5]. I post my code below:

class FullConnect(nn.Module):
    def __init__(self):
        super(FullConnect, self).__init__()
        self.fc = nn.Sequential(
            nn.Linear(195, 100),
            nn.Linear(100, 5)
        )
    def forward(self, x):
        out = self.fc(x)
        return out

class LossFunc(nn.Module):
    def __init__(self):
        super(LossFunc, self).__init__()
    def forward(self,x,y):
        loss=torch.div(torch.sum(torch.pow(torch.log(torch.div(x+1,y+1)),2)),5)
        return loss

small_data=np.random.randn(100, 200)
small_data[small_data<0]=0
model = FullConnect()
optimizer = optim.SGD(model.parameters(), lr=0.01, momentum=0.3)
criterion = LossFunc()
for epoch in range(5):
    acc=0
    for i in range(small_data.shape[0]):
        x = Variable(torch.FloatTensor(small_data[i][5:]))
        y = Variable(torch.FloatTensor(small_data[i][:5]))
        output=model(x)
        loss=criterion(output,y)
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()
        acc+=loss
    print("epoch:",epoch)
    print("Loss:",acc)

This code works fine when I fed small size training set into it, returning:

epoch: 0
Loss: Variable containing:
 15.7719
[torch.FloatTensor of size 1]

epoch: 1
Loss: Variable containing:
 12.0258
[torch.FloatTensor of size 1]

epoch: 2
Loss: Variable containing:
 9.9758
[torch.FloatTensor of size 1]

epoch: 3
Loss: Variable containing:
 8.5442
[torch.FloatTensor of size 1]

epoch: 4
Loss: Variable containing:
 7.4562
[torch.FloatTensor of size 1]

But when I replaced small_data with a large training set like:

large_data=np.random.randn(60000, 200)
large_data[large_data<0]=0

Jupyter notebook returned me an error The kernel appears to have died. It will restart automatically. I suppose this error related to the size of the input.

My cuda9.1 is available, however cudnn is not acceptable in torch.
Now I'm searching ways to improve my code and make this training procedure works. I appreciate any advice might help me out.


title: Parallel training of ensembled neural networks using torch.multiprocessing

body:
I'm trying to implement an ensemble model using pytorch, and as there're some independent models, I want to traing the models in parallel using torch.multiprocessing, However, I always get Too many open files error.

Here is a minimal example that reproduce the error:

import torch
import torch.nn as nn
from   torch.multiprocessing import Pool

class MyModel:
    def __init__(self):
        self.nn = nn.Sequential(
                nn.Linear(10, 10),
                nn.ReLU(),
                nn.Linear(10, 10),
                nn.ReLU(),
                nn.Linear(10, 10),
                nn.ReLU(),
                nn.Linear(10, 10),
                nn.ReLU(),
                nn.Linear(10, 10),
                nn.ReLU(),
                nn.Linear(10, 10),
                nn.ReLU(),
                nn.Linear(10, 10),
                nn.ReLU(),
                nn.Linear(10, 10),
                nn.ReLU()
                )

    def train(self):
        pass

class EnsembleModel:
    def __init__(self, K):
        self.K      = K;
        self.models = [MyModel() for i in range(self.K)]

    def f(self, i):
        return i

    def train(self):

        pool = Pool(processes = 3)
        ret  = pool.map(self.f, range(self.K))
        print(ret)

md = EnsembleModel(15);
md.train()

And this is the error message:

/home/alaya/anaconda3/lib/python3.6/multiprocessing/reduction.py:153: RuntimeWarning: received malformed or improperly-truncated ancillary data
  msg, ancdata, flags, addr = sock.recvmsg(1, socket.CMSG_LEN(bytes_size))
Process ForkPoolWorker-3:
Traceback (most recent call last):
  File "/home/alaya/anaconda3/lib/python3.6/multiprocessing/process.py", line 258, in _bootstrap
    self.run()
  File "/home/alaya/anaconda3/lib/python3.6/multiprocessing/process.py", line 93, in run
    self._target(*self._args, **self._kwargs)
  File "/home/alaya/anaconda3/lib/python3.6/multiprocessing/pool.py", line 108, in worker
    task = get()
  File "/home/alaya/anaconda3/lib/python3.6/multiprocessing/queues.py", line 337, in get
    return _ForkingPickler.loads(res)
  File "/home/alaya/anaconda3/lib/python3.6/site-packages/torch/multiprocessing/reductions.py", line 151, in rebuild_storage_fd
    fd = df.detach()
  File "/home/alaya/anaconda3/lib/python3.6/multiprocessing/resource_sharer.py", line 58, in detach
    return reduction.recv_handle(conn)
  File "/home/alaya/anaconda3/lib/python3.6/multiprocessing/reduction.py", line 182, in recv_handle
    return recvfds(s, 1)[0]
  File "/home/alaya/anaconda3/lib/python3.6/multiprocessing/reduction.py", line 172, in recvfds
    raise RuntimeError('Invalid data received')
RuntimeError: Invalid data received
Traceback (most recent call last):
  File "/home/alaya/anaconda3/lib/python3.6/multiprocessing/resource_sharer.py", line 149, in _serve
  File "/home/alaya/anaconda3/lib/python3.6/multiprocessing/resource_sharer.py", line 50, in send
  File "/home/alaya/anaconda3/lib/python3.6/multiprocessing/reduction.py", line 176, in send_handle
  File "/home/alaya/anaconda3/lib/python3.6/socket.py", line 460, in fromfd
OSError: [Errno 24] Too many open files
Traceback (most recent call last):
  File "/home/alaya/anaconda3/lib/python3.6/multiprocessing/resource_sharer.py", line 142, in _serve
  File "/home/alaya/anaconda3/lib/python3.6/multiprocessing/connection.py", line 453, in accept
  File "/home/alaya/anaconda3/lib/python3.6/multiprocessing/connection.py", line 593, in accept
  File "/home/alaya/anaconda3/lib/python3.6/socket.py", line 205, in accept
OSError: [Errno 24] Too many open files
Traceback (most recent call last):
  File "/home/alaya/anaconda3/lib/python3.6/multiprocessing/resource_sharer.py", line 142, in _serve
  File "/home/alaya/anaconda3/lib/python3.6/multiprocessing/connection.py", line 453, in accept
  File "/home/alaya/anaconda3/lib/python3.6/multiprocessing/connection.py", line 593, in accept
  File "/home/alaya/anaconda3/lib/python3.6/socket.py", line 205, in accept
OSError: [Errno 24] Too many open files
Exception in thread Thread-1:
Traceback (most recent call last):
  File "/home/alaya/anaconda3/lib/python3.6/threading.py", line 916, in _bootstrap_inner
    self.run()
  File "/home/alaya/anaconda3/lib/python3.6/threading.py", line 864, in run
    self._target(*self._args, **self._kwargs)
  File "/home/alaya/anaconda3/lib/python3.6/multiprocessing/pool.py", line 405, in _handle_workers
    pool._maintain_pool()
  File "/home/alaya/anaconda3/lib/python3.6/multiprocessing/pool.py", line 246, in _maintain_pool
    self._repopulate_pool()
  File "/home/alaya/anaconda3/lib/python3.6/multiprocessing/pool.py", line 239, in _repopulate_pool
    w.start()
  File "/home/alaya/anaconda3/lib/python3.6/multiprocessing/process.py", line 105, in start
    self._popen = self._Popen(self)
  File "/home/alaya/anaconda3/lib/python3.6/multiprocessing/context.py", line 277, in _Popen
    return Popen(process_obj)
  File "/home/alaya/anaconda3/lib/python3.6/multiprocessing/popen_fork.py", line 19, in __init__
    self._launch(process_obj)
  File "/home/alaya/anaconda3/lib/python3.6/multiprocessing/popen_fork.py", line 65, in _launch
    parent_r, child_w = os.pipe()
OSError: [Errno 24] Too many open files

```


title: How to extract patches from an image in pytorch?

body:
I want to extract image patches from an image with patch size 128 and stride 32, so I have this code, but it gives me an error :

from PIL import Image
img = Image.open("cat.jpg")
x = transforms.ToTensor()(img)

x = x.unsqueeze(0)

size = 128 # patch size
stride = 32 # patch stride
patches = x.unfold(1, size, stride).unfold(2, size, stride).unfold(3, size, stride)
print(patches.shape)

and the error I get is :

RuntimeError: maximum size for tensor at dimension 1 is 3 but size is 128

This is the only method I've found so far. but it gives me this error


title: Pytorch: encode categorical feature by using nn.Embedding

body: I am new to Pytorch. I learnt some tutorials about how to build a simple NN model by using pytorch, e.g. (this one). I find that they prefer to nn.Embedding to encode categorical features. My questions are: (1) why not use one hot encoding? (2) nn.Embedding is initialized randomly, and I think the numerical values of the embeddings will not be trained because they are treated as input. If the nn.Embedding initializes the categorical features randomly, would it have a negative impact to the training?


title: Input size mismatch error when using pre-trained inceptionV3 model for image classification

body:
I'm facing trouble when training a model using pre-trained inceptionV3 for my own image data set.

I'm loading images using data.Dataset loader and 'transforms' for image transformation.

Here's my inceptionV3 model

inceptionV3 = torchvision.models.inception_v3(pretrained=True)
pretrained_model = nn.Sequential(*list(inceptionV3.children()))
pretrained_model = nn.Sequential(*list(pretrained_features.children())[:-1])
for param in pretrained_model.parameters(): param.requires_grad = False

Here's my transforms code

data_transforms = transforms.Compose([
transforms.Scale((299,299)),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
])

I'm getting this error at the line 'out_features = pretrained_model(inputs)'

Traceback (most recent call last): File "build_models.py", line 42, in <module> use_gpu=use_gpu)
File "/home/ubuntu/apparel-styles/ml_src/classifiers.py", line 488, in create_attributes_model batch_size, num_workers, num_epochs, use_gpu=use_gpu)
File "/home/ubuntu/apparel-styles/ml_src/classifiers.py", line 276, in train_model flatten_pretrained_out=flatten_pretrained_out)
File "/home/ubuntu/apparel-styles/ml_src/classifiers.py", line 181, in train_attribute_model out_features = pretrained_model(inputs)
File "/home/ubuntu/apparel-styles/env/venv/lib/python3.6/site-packages/torch/nn/modules/module.py", line 325, in __call__ result = self.forward(*input, **kwargs)
File "/home/ubuntu/apparel-styles/env/venv/lib/python3.6/site-packages/torch/nn/parallel/data_parallel.py", line 68, in forward outputs = self.parallel_apply(replicas, inputs, kwargs)
File "/home/ubuntu/apparel-styles/env/venv/lib/python3.6/site-packages/torch/nn/parallel/data_parallel.py", line 78, in parallel_apply return parallel_apply(replicas, inputs, kwargs, self.device_ids[:len(replicas)])
File "/home/ubuntu/apparel-styles/env/venv/lib/python3.6/site-packages/torch/nn/parallel/parallel_apply.py", line 67, in parallel_apply raise output
File "/home/ubuntu/apparel-styles/env/venv/lib/python3.6/site-packages/torch/nn/parallel/parallel_apply.py", line 42, in _worker output = module(*input, **kwargs)
File "/home/ubuntu/apparel-styles/env/venv/lib/python3.6/site-packages/torch/nn/modules/module.py", line 325, in __call__ result = self.forward(*input, **kwargs)
File "/home/ubuntu/apparel-styles/env/venv/lib/python3.6/site-packages/torch/nn/modules/container.py", line 67, in forward input = module(input)
File "/home/ubuntu/apparel-styles/env/venv/lib/python3.6/site-packages/torch/nn/modules/module.py", line 325, in __call__ result = self.forward(*input, **kwargs)
File "/home/ubuntu/apparel-styles/env/venv/lib/python3.6/site-packages/torchvision/models/inception.py", line 312, in forward x = self.fc(x)
File "/home/ubuntu/apparel-styles/env/venv/lib/python3.6/site-packages/torch/nn/modules/module.py", line 325, in __call__ result = self.forward(*input, **kwargs)
File "/home/ubuntu/apparel-styles/env/venv/lib/python3.6/site-packages/torch/nn/modules/linear.py", line 55, in forward return F.linear(input, self.weight, self.bias)
File "/home/ubuntu/apparel-styles/env/venv/lib/python3.6/site-packages/torch/nn/functional.py", line 835, in linear return torch.addmm(bias, input, weight.t()) RuntimeError: size mismatch at /pytorch/torch/lib/THC/generic/THCTensorMathBlas.cu:243

After getting this error, I tried to print the input size. It's value is

print(inputs.shape)

torch.Size([256, 3, 299, 299])


title: How to specify os-specific wheel files in the Pipfile

body:
I'm trying to use the pipenv and pytorch.

To install pytorch in windows, I have to write following codes into the Pipfile:

[packages]
torch = {file = "http://download.pytorch.org/whl/cpu/torch-0.4.1-cp37-cp37m-win_amd64.whl"}

However, the wheel file is different for the linux.

[packages]
torch = {file = "http://download.pytorch.org/whl/cpu/torch-0.4.1.post2-cp37-cp37m-linux_x86_64.whl "}

How to specify both of them in Pipfile?