title: Tensor type mismatch when moving to GPU

body:
I'm getting the following error when trying to move my network and tensors to GPU. I've checked that the network parameters are moved to the GPU and check each batch's tensor and move them if they're not already on the GPU. But I'm still getting this issue say that there's a mismatch in the tensor types - one is a torch.cuda.FloatTensor and the other is a torch.FloatTensor? Could someone tell me what I'm doing wrong? Thanks.

My code:

class Train():
  def __init__(self, network, training, address):
    self.network    = network
    self.address    = address
    self.batch_size = training['batch_size']
    self.iterations = training['iterations']
    self.samples    = training['samples']
    self.data       = training['data']
    self.lr         = training['lr']
    self.noisy_lr   = training['nlr']
    self.cuda       = training['cuda']
    self.save       = training['save']
    self.scale      = training['scale']
    self.limit      = training['limit']
    self.replace    = training['strategy']
    self.optimizer  = torch.optim.Adam(self.network.parameters(), lr=self.lr)

  def tensor_to_Variable(self, t):
    if next(self.network.parameters()).is_cuda and not t.is_cuda:
        t = t.cuda()

    return Variable(t)

  def train(self):
    if self.cuda:
        self.network.cuda()
    dh = DataHandler(self.data)
    loss_fn = torch.nn.MSELoss()
    losses    = []
    validate  = []
    val_size  = 100
    val_diff  = 1
    total_val = float(val_size * self.batch_size)
    hypos     = []
    labels    = []

    # training loop
    for i in range(self.iterations):
        x, y = dh.get_batch(self.batch_size)
        x = self.tensor_to_Variable(x)
        y = self.tensor_to_Variable(y)

        self.optimizer.zero_grad()
        hypo = self.network(x)
        loss = loss_fn(hypo, y)
        loss.backward()
        self.optimizer.step()

class Feedforward(nn.Module):
   def __init__(self, topology):
    super(Feedforward, self).__init__()
    self.input_dim     = topology['features']
    self.num_hidden    = topology['hidden_layers']
    self.hidden_dim    = topology['hidden_dim']
    self.output_dim    = topology['output_dim']
    self.input_layer   = nn.Linear(self.input_dim, self.hidden_dim)
    self.hidden_layer  = nn.Linear(self.hidden_dim, self.hidden_dim)
    self.output_layer  = nn.Linear(self.hidden_dim, self.output_dim)
    self.dropout_layer = nn.Dropout(p=0.2)

def forward(self, x):
    batch_size = x.size()[0]
    feat_size  = x.size()[1]
    input_size = batch_size * feat_size

    self.input_layer = nn.Linear(input_size, self.hidden_dim)
    hidden = self.input_layer(x.view(1, input_size)).clamp(min=0)

    for _ in range(self.num_hidden):
        hidden = self.dropout_layer(F.relu(self.hidden_layer(hidden)))

    output_size = batch_size * self.output_dim
    self.output_layer = nn.Linear(self.hidden_dim, output_size)
    return self.output_layer(hidden).view(output_size)

The error:

Traceback (most recent call last):
  File "/media/project/train.py", line 78, in train
    hypo = self.network(x)

 * (torch.cuda.FloatTensor mat1, torch.cuda.FloatTensor mat2)
 * (torch.cuda.sparse.FloatTensor mat1, torch.cuda.FloatTensor mat2)
 * (float beta, torch.cuda.FloatTensor mat1, torch.cuda.FloatTensor mat2)
 * (float alpha, torch.cuda.FloatTensor mat1, torch.cuda.FloatTensor mat2)
 * (float beta, torch.cuda.sparse.FloatTensor mat1, torch.cuda.FloatTensor mat2)
 * (float alpha, torch.cuda.sparse.FloatTensor mat1, torch.cuda.FloatTensor mat2)
 * (float beta, float alpha, torch.cuda.FloatTensor mat1, torch.cuda.FloatTensor mat2)
      didn't match because some of the arguments have invalid types: (int, int, torch.cuda.FloatTensor, torch.FloatTensor)
 * (float beta, float alpha, torch.cuda.sparse.FloatTensor mat1, torch.cuda.FloatTensor mat2)
      didn't match because some of the arguments have invalid types: (int, int, torch.cuda.FloatTensor, torch.FloatTensor)

Stacktrace:

Traceback (most recent call last):
File "smpl.py", line 90, in <module>
main()
File "smpl.py", line 80, in main
trainer.train()
File "/media/mpl/temp/train.py", line 82, in train
hypo = self.network(x)
File "/usr/local/lib/python2.7/dist-packages/torch/nn/modules/module.py", line 206, in call
result = self.forward(input, **kwargs)
File "model/network.py", line 35, in forward
hidden = self.input_layer(x.view(1, input_size)).clamp(min=0)
File "/usr/local/lib/python2.7/dist-packages/torch/nn/modules/module.py", line 206, in call
result = self.forward(input, *kwargs)
File "/usr/local/lib/python2.7/dist-packages/torch/nn/modules/linear.py", line 54, in forward
return self.backend.Linear()(input, self.weight, self.bias)
File "/usr/local/lib/python2.7/dist-packages/torch/nn/_functions/linear.py", line 10, in forward
output.addmm(0, 1, input, weight.t())
TypeError: addmm_ received an invalid combination of arguments - got (int, int, torch.cuda.FloatTensor, torch.FloatTensor), but expected one of: (torch.cuda.FloatTensor mat1, torch.cuda.FloatTensor mat2)
(torch.cuda.sparse.FloatTensor mat1, torch.cuda.FloatTensor mat2) (float beta, torch.cuda.FloatTensor mat1, torch.cuda.FloatTensor mat2)
(float alpha, torch.cuda.FloatTensor mat1, torch.cuda.FloatTensor mat2) (float beta, torch.cuda.sparse.FloatTensor mat1, torch.cuda.FloatTensor mat2)
(float alpha, torch.cuda.sparse.FloatTensor mat1, torch.cuda.FloatTensor mat2) (float beta, float alpha, torch.cuda.FloatTensor mat1, torch.cuda.FloatTensor mat2)
didn't match because some of the arguments have invalid types: (int, int, torch.cuda.FloatTensor, torch.FloatTensor)
* (float beta, float alpha, torch.cuda.sparse.FloatTensor mat1, torch.cuda.FloatTensor mat2)
didn't match because some of the arguments have invalid types: (int, int, torch.cuda.FloatTensor, torch.FloatTensor


title: Pytorch beginner : tensor.new method

body:
everyone, I have a small question.

What is the purpose of the method tensor.new(..) in Pytorch, I didn't find anything in the documentation. It looks like it creates a new Tensor (like the name suggests), but why we don't just use torch.Tensor constructors instead of using this new method that requires an existing tensor.

Thank you in advance.


title: What does the "greater than" operator ">" mean for PyTorch tensors?

body:
I have a tensor it defined as:

import torch
it = torch.tensor([0,  0,  0,  0,  0,  0,  0,  0,  0,  0], device='cuda:0')

Given this definition, what does it > 0 then mean?


I am testing a ResNet-34 trained_model using Pytorch and fastai on a linux system with the latest anaconda3. To run it as a batch job, I commented out the gui related lines. It started to run for a few hrs, then stopped in the Validation step, the error message is as below.

...
^M100%|█████████▉| 452/453 [1:07:07<00:08,  8.75s/it,
loss=1.23]^[[A^[[A^[[A

^MValidation:   0%|          | 0/40 [00:00<?, ?it/s]^[[A^[[A^[[ATraceback
(most recent call last):
  File "./resnet34_pretrained_PNG_nogui_2.py", line 279, in <module>
    learner.fit(lr,1,callbacks=[f1_callback])
  File "/project/6000192/jemmyhu/resnet_png/fastai/learner.py", line 302,
in fit
    return self.fit_gen(self.model, self.data, layer_opt, n_cycle,
**kwargs)
  File "/project/6000192/jemmyhu/resnet_png/fastai/learner.py", line 249,
in fit_gen
    swa_eval_freq=swa_eval_freq, **kwargs)
  File "/project/6000192/jemmyhu/resnet_png/fastai/model.py", line 162, in
fit
    vals = validate(model_stepper, cur_data.val_dl, metrics, epoch,
seq_first=seq_first, validate_skip = validate_skip)
  File "/project/6000192/jemmyhu/resnet_png/fastai/model.py", line 241, in
validate
    res.append([to_np(f(datafy(preds), datafy(y))) for f in metrics])
  File "/project/6000192/jemmyhu/resnet_png/fastai/model.py", line 241, in
<listcomp>
    res.append([to_np(f(datafy(preds), datafy(y))) for f in metrics])
  File "./resnet34_pretrained_PNG_nogui_2.py", line 237, in __call__
    self.TP += (preds*targs).float().sum(dim=0)
TypeError: add(): argument 'other' (position 1) must be Tensor, not
numpy.ndarray

The link for the original code is
https://www.kaggle.com/iafoss/pretrained-resnet34-with-rgby-0-460-public-lb

lines 279 and 237 in my copy are shown below:

226 class F1:
227     __name__ = 'F1 macro'
228     def __init__(self,n=28):
229         self.n = n
230         self.TP = np.zeros(self.n)
231         self.FP = np.zeros(self.n)
232         self.FN = np.zeros(self.n)
233
234     def __call__(self,preds,targs,th=0.0):
235         preds = (preds > th).int()
236         targs = targs.int()
237         self.TP += (preds*targs).float().sum(dim=0)
238         self.FP += (preds > targs).float().sum(dim=0)
239         self.FN += (preds < targs).float().sum(dim=0)
240         score = (2.0*self.TP/(2.0*self.TP + self.FP + self.FN + 1e-6)).mean()
241         return score

276 lr = 0.5e-2
277 with warnings.catch_warnings():
278     warnings.simplefilter("ignore")
279     learner.fit(lr,1,callbacks=[f1_callback])

Could anyone have a clue for the issue?

Many thanks,
Jemmy


title: PyTorch equivalence for softmax_cross_entropy_with_logits

body: I was wondering is there an equivalent PyTorch loss function for TensorFlow's softmax_cross_entropy_with_logits?


title: What does layout = torch.strided mean?

body:
As I was going through pytorch documentation I came across a term layout = torch.strided in many of the functions. Can anyone help me in understanding where is it used and how. The description says it's the the desired layout of returned Tensor. What does layout mean and how many types of layout are there ?

torch.rand(*sizes, out=None, dtype=None, layout=torch.strided, device=None, requires_grad=False)


title: Pytorch AttributeError: module 'torch' has no attribute 'as_tensor'

body:
$ python main.py --hetero
Created directory results/ACMRaw_2020-01-13_01-20-26
Traceback (most recent call last):
  File "main.py", line 101, in <module>
    main(args)
  File "main.py", line 30, in main
    val_mask, test_mask = load_data(args['dataset'])
  File "/home/cnudi1/wook/dgl/examples/pytorch/han/utils.py", line 225, in load_data
    return load_acm_raw(remove_self_loop)
  File "/home/cnudi1/wook/dgl/examples/pytorch/han/utils.py", line 189, in load_acm_raw
    pa = dgl.bipartite(p_vs_a, 'paper', 'pa', 'author')
  File "/home/cnudi1/.conda/envs/lcr_env/lib/python3.6/site-packages/dgl-0.4-py3.6-linux-ppc64le.egg/dgl/convert.py", line 260, in bipartite
    return create_from_scipy(data, utype, etype, vtype)
  File "/home/cnudi1/.conda/envs/lcr_env/lib/python3.6/site-packages/dgl-0.4-py3.6-linux-ppc64le.egg/dgl/convert.py", line 823, in create_from_scipy
    indptr = utils.toindex(spmat.indptr)
  File "/home/cnudi1/.conda/envs/lcr_env/lib/python3.6/site-packages/dgl-0.4-py3.6-linux-ppc64le.egg/dgl/utils.py", line 242, in toindex
    return data if isinstance(data, Index) else Index(data)
  File "/home/cnudi1/.conda/envs/lcr_env/lib/python3.6/site-packages/dgl-0.4-py3.6-linux-ppc64le.egg/dgl/utils.py", line 15, in __init__
    self._initialize_data(data)
  File "/home/cnudi1/.conda/envs/lcr_env/lib/python3.6/site-packages/dgl-0.4-py3.6-linux-ppc64le.egg/dgl/utils.py", line 22, in _initialize_data
    self._dispatch(data)
  File "/home/cnudi1/.conda/envs/lcr_env/lib/python3.6/site-packages/dgl-0.4-py3.6-linux-ppc64le.egg/dgl/utils.py", line 75, in _dispatch
    self._user_tensor_data[F.cpu()] = F.zerocopy_from_numpy(self._pydata)
  File "/home/cnudi1/.conda/envs/lcr_env/lib/python3.6/site-packages/dgl-0.4-py3.6-linux-ppc64le.egg/dgl/backend/pytorch/tensor.py", line 276, in zerocopy_from_numpy
    return th.as_tensor(np_array)
AttributeError: module 'torch' has no attribute 'as_tensor'

I got an error when I try to run the code (https://github.com/dmlc/dgl/blob/master/examples/pytorch/han/main.py)

from the DGL
(https://github.com/dmlc/dgl)

It requires CUDA and Pytorch so I managed to install it.

But I got an error and couldn't find the solution with Google/Stackoverflow search

My environment is

Linux minsky 3.10.0-957.5.1.el7.ppc64le
CentOS
Python 3.6.9
Conda 4.5.11
CUDA 10.1
NVCC 10.1
Pytorch 0.4.0
Torchvision 0.2.1

Pytorch works fine in Python

>>> import torch
>>> print (torch.__version__)
0.4.0
>>> import torchvision
>>> print (torchvision.__version__)
0.2.1

Please could you help me out?

* DGL is installed from the source code

** Pytorch is installed with conda from channel:engility(How to install pytorch on Power 8 or PPC64 machine?)

conda install -c engility pytorch

because other ways(default conda, pip, install from the source code) never works for ppc64le


title: PyTorch ValueError: Target and input must have the same number of elements

body:
I am kinda new to PyTorch, but I am trying to understand how the sizes of target and input work in torch.nn.BCELoss() when computing loss function.

import torch
import torch.nn as nn
from torch.autograd import Variable

time_steps = 15
batch_size = 3
embeddings_size = 100
num_classes = 2

model = nn.LSTM(embeddings_size, num_classes)
input_seq = Variable(torch.randn(time_steps, batch_size, embeddings_size))
lstm_out, _ = model(input_seq)
last_out = lstm_out[-1]
print(last_out)

loss = nn.BCELoss()
target = Variable(torch.LongTensor(batch_size).random_(0, num_classes))
print(target)
err = loss(last_out.long(), target)
err.backward()

I received the following error:

Warning (from warnings module):
File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/torch/nn/functional.py", line 767
"Please ensure they have the same size.".format(target.size(), input.size()))

File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/torch/nn/functional.py", line 770, in binary_cross_entropy
"!= input nelement ({})".format(target.nelement(), input.nelement()))
ValueError: Target and input must have the same number of elements. target nelement (3) != input nelement (6)

This error definitely comes from the different sizes of last_out (size 3x2) and target (size 3). So my question is how can I convert last_out into something like target (of size 3 and containing just 0s and 1s) to compute loss function?


title: PyTorch: DecoderRNN: RuntimeError: input must have 3 dimensions, got 2

body:
I am building a DecoderRNN using PyTorch (This is an image-caption decoder):

class DecoderRNN(nn.Module):
    def __init__(self, embed_size, hidden_size, vocab_size):

        super(DecoderRNN, self).__init__()
        self.hidden_size = hidden_size
        self.gru = nn.GRU(embed_size, hidden_size, hidden_size)
        self.softmax = nn.LogSoftmax(dim=1)

    def forward(self, features, captions):

        print (features.shape)
        print (captions.shape)
        output, hidden = self.gru(features, captions)
        output = self.softmax(self.out(output[0]))
        return output, hidden

The data have the following shapes:

torch.Size([10, 200])  <- features.shape (10 for batch size)
torch.Size([10, 12])   <- captions.shape (10 for batch size)

Then I got the following errors. Any ideas what I missed here? Thanks!

---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-2-76e05ba08b1d> in <module>()
     44         # Pass the inputs through the CNN-RNN model.
     45         features = encoder(images)
---> 46         outputs = decoder(features, captions)
     47
     48         # Calculate the batch loss.

/opt/conda/lib/python3.6/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    323         for hook in self._forward_pre_hooks.values():
    324             hook(self, input)
--> 325         result = self.forward(*input, **kwargs)
    326         for hook in self._forward_hooks.values():
    327             hook_result = hook(self, input, result)

/home/workspace/model.py in forward(self, features, captions)
     37         print (captions.shape)
     38         # features = features.unsqueeze(1)
---> 39         output, hidden = self.gru(features, captions)
     40         output = self.softmax(self.out(output[0]))
     41         return output, hidden

/opt/conda/lib/python3.6/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    323         for hook in self._forward_pre_hooks.values():
    324             hook(self, input)
--> 325         result = self.forward(*input, **kwargs)
    326         for hook in self._forward_hooks.values():
    327             hook_result = hook(self, input, result)

/opt/conda/lib/python3.6/site-packages/torch/nn/modules/rnn.py in forward(self, input, hx)
    167             flat_weight=flat_weight
    168         )
--> 169         output, hidden = func(input, self.all_weights, hx)
    170         if is_packed:
    171             output = PackedSequence(output, batch_sizes)

/opt/conda/lib/python3.6/site-packages/torch/nn/_functions/rnn.py in forward(input, *fargs, **fkwargs)
    383             return hack_onnx_rnn((input,) + fargs, output, args, kwargs)
    384         else:
--> 385             return func(input, *fargs, **fkwargs)
    386
    387     return forward

/opt/conda/lib/python3.6/site-packages/torch/autograd/function.py in _do_forward(self, *input)
    326         self._nested_input = input
    327         flat_input = tuple(_iter_variables(input))
--> 328         flat_output = super(NestedIOFunction, self)._do_forward(*flat_input)
    329         nested_output = self._nested_output
    330         nested_variables = _unflatten(flat_output, self._nested_output)

/opt/conda/lib/python3.6/site-packages/torch/autograd/function.py in forward(self, *args)
    348     def forward(self, *args):
    349         nested_tensors = _map_variable_tensor(self._nested_input)
--> 350         result = self.forward_extended(*nested_tensors)
    351         del self._nested_input
    352         self._nested_output = result

/opt/conda/lib/python3.6/site-packages/torch/nn/_functions/rnn.py in forward_extended(self, input, weight, hx)
    292             hy = tuple(h.new() for h in hx)
    293
--> 294         cudnn.rnn.forward(self, input, hx, weight, output, hy)
    295
    296         self.save_for_backward(input, hx, weight, output)

/opt/conda/lib/python3.6/site-packages/torch/backends/cudnn/rnn.py in forward(fn, input, hx, weight, output, hy)
    206         if (not is_input_packed and input.dim() != 3) or (is_input_packed and input.dim() != 2):
    207             raise RuntimeError(
--> 208                 'input must have 3 dimensions, got {}'.format(input.dim()))
    209         if fn.input_size != input.size(-1):
    210             raise RuntimeError('input.size(-1) must be equal to input_size. Expected {}, got {}'.format(

RuntimeError: input must have 3 dimensions, got 2


title: "RuntimeError: Found 0 files in subfolders of ".. Error about subfolder in Pytorch

body:
I'm based on Window 10, Jupyter Notebook, Pytorch 1.0, Python 3.6.x currently.

At first I confirm to the correct path of files using this code : print(os.listdir('./Dataset/images/')).

and I could check that this path is correct.

but I met Error :

  RuntimeError: Found 0 files in subfolders of: ./Dataset/images/
  Supported extensions are: .jpg,.jpeg,.png,.ppm,.bmp,.pgm,.tif"

What is the matter?
Could you suggest a solution?

I tried to ./dataset/1/images like this method. but the result was same....

img_dir = './Dataset/images/'
img_data = torchvision.datasets.ImageFolder(os.path.join(img_dir), transforms.Compose([
            transforms.Scale(256),
            transforms.RandomResizedCrop(224),
            transforms.RandomHorizontalFlip(),
            transforms.ToTensor(),
            ]))
img_batch = data.DataLoader(img_data, batch_size=batch_size,
                               shuffle = True, drop_last=True)


title: Force GPU memory limit in PyTorch

body: Is there a way to force a maximum value for the amount of GPU memory that I want to be available for a particular Pytorch instance? For example, my GPU may have 12Gb available, but I'd like to assign 4Gb max to a particular process.


I am loading my model using the following code.

def load_model(checkpoint_path):
  '''
  Function that loads a checkpoint and rebuilds the model
  '''

  checkpoint = torch.load(checkpoint_path, map_location = 'cpu')

  if checkpoint['architecture'] == 'resnet18':
    model = models.resnet18(pretrained=True)

  # Freezing the parameters
    for param in model.parameters():
        param.requires_grad = False

  else:
    print('Wrong Architecture!')
    return None

  model.class_to_idx = checkpoint['class_to_idx']

  classifier = nn.Sequential(OrderedDict([
                            ('fc1', nn.Linear(512, 1024)),
                            ('relu1', nn.ReLU()),
                            ('dropout', nn.Dropout(0.2)),
                            ('fc2', nn.Linear(1024, 102))
                            ]))

  model.fc = classifier

  model.load_state_dict(checkpoint['state_dict'])

  return model

And while running

# Load your model to this variable
model = load_model('checkpoint.pt')

I get the following error,

  RuntimeError                              Traceback (most recent call
  last)
       in ()
            1 # Load your model to this variable
      ----> 2 model = load_model('checkpoint.pt')
            3
            4 # If you used something other than 224x224 cropped images, set the
      correct size here
            5 image_size = 224

<ipython-input-11-81aef50793cb> in load_model(checkpoint_path)
     30   model.fc = classifier
     31
---> 32   model.load_state_dict(checkpoint['state_dict'])
     33
     34   return model

/opt/conda/lib/python3.6/site-packages/torch/nn/modules/module.py in
load_state_dict(self, state_dict, strict)
    719         if len(error_msgs) > 0:
    720             raise RuntimeError('Error(s) in loading state_dict for
{}:\n\t{}'.format(
--> 721                                self.__class__.__name__,
"\n\t".join(error_msgs)))
    722
    723     def parameters(self):

RuntimeError: Error(s) in loading state_dict for ResNet:
  Unexpected key(s) in state_dict: "bn1.num_batches_tracked",
"layer1.0.bn1.num_batches_tracked", "layer1.0.bn2.num_batches_tracked",
"layer1.1.bn1.num_batches_tracked", "layer1.1.bn2.num_batches_tracked",
"layer2.0.bn1.num_batches_tracked", "layer2.0.bn2.num_batches_tracked",
"layer2.0.downsample.1.num_batches_tracked",
"layer2.1.bn1.num_batches_tracked", "layer2.1.bn2.num_batches_tracked",
"layer3.0.bn1.num_batches_tracked", "layer3.0.bn2.num_batches_tracked",
"layer3.0.downsample.1.num_batches_tracked",
"layer3.1.bn1.num_batches_tracked", "layer3.1.bn2.num_batches_tracked",
"layer4.0.bn1.num_batches_tracked", "layer4.0.bn2.num_batches_tracked",
"layer4.0.downsample.1.num_batches_tracked",
"layer4.1.bn1.num_batches_tracked", "layer4.1.bn2.num_batches_tracked".


title: Runtime error 999 when trying to use cuda with pytorch

body:
I installed Cuda 10.1 and the latest Nvidia Driver for my Geforce 2080 ti. I try to run a basic script to test if pytorch is working and I get the following error:

RuntimeError: cuda runtime error (999) : unknown error at ..\aten\src\THC\THCGeneral.cpp:50

Below is the code im trying to run:

import torch
torch.cuda.current_device()
torch.cuda.is_available()
torch.cuda.get_device_name(0)


title: PyTorch RuntimeError: Assertion `cur_target >= 0 && cur_target < n_classes' failed

body:
I’m trying to create a basic binary classifier in Pytorch that classifies whether my player plays on the right or the left side in the game Pong. The input is an 1x42x42 image and the label is my player's side (right = 1 or left = 2). The code:

class Net(nn.Module):
    def __init__(self, input_size, hidden_size, num_classes):
        super(Net, self).__init__()
        self.fc1 = nn.Linear(input_size, hidden_size)
        self.relu = nn.ReLU()
        self.fc2 = nn.Linear(hidden_size, num_classes)

    def forward(self, x):
        out = self.fc1(x)
        out = self.relu(out)
        out = self.fc2(out)
        return out

net = Net(42 * 42, 100, 2)

# Loss and Optimizer
criterion = nn.CrossEntropyLoss()
optimizer_net = torch.optim.Adam(net.parameters(), 0.001)
net.train()

while True:
    state = get_game_img()
    state = torch.from_numpy(state)

    # right = 1, left = 2
    current_side = get_player_side()
    target = torch.LongTensor(current_side)
    x = Variable(state.view(-1, 42 * 42))
    y = Variable(target)
    optimizer_net.zero_grad()
    y_pred = net(x)
    loss = criterion(y_pred, y)
    loss.backward()
    optimizer.step()

The error I get:

  File "train.py", line 109, in train
    loss = criterion(y_pred, y)
  File "/home/shani/anaconda2/lib/python2.7/site-packages/torch/nn/modules/module.py", line 206, in __call__
    result = self.forward(*input, **kwargs)
  File "/home/shani/anaconda2/lib/python2.7/site-packages/torch/nn/modules/loss.py", line 321, in forward
    self.weight, self.size_average)
  File "/home/shani/anaconda2/lib/python2.7/site-packages/torch/nn/functional.py", line 533, in cross_entropy
    return nll_loss(log_softmax(input), target, weight, size_average)
  File "/home/shani/anaconda2/lib/python2.7/site-packages/torch/nn/functional.py", line 501, in nll_loss
    return f(input, target)
  File "/home/shani/anaconda2/lib/python2.7/site-packages/torch/nn/_functions/thnn/auto.py", line 41, in forward
    output, *self.additional_args)
RuntimeError: Assertion `cur_target >= 0 && cur_target < n_classes' failed.  at /py/conda-bld/pytorch_1493676237139/work/torch/lib/THNN/generic/ClassNLLCriterion.c:57


title: PyTorch: manually setting weight parameters with numpy array for GRU / LSTM

body:
I'm trying to fill up GRU/LSTM with manually defined parameters in pytorch.

I have numpy arrays for parameters with shapes as defined in their documentation (https://pytorch.org/docs/stable/nn.html#torch.nn.GRU).

It seems to work but I'm not sure whether the returned values are correct.

Is this a right way to fill up GRU/LSTM with numpy parameters?

gru = nn.GRU(input_size, hidden_size, num_layers,
              bias=True, batch_first=False, dropout=dropout, bidirectional=bidirectional)

def set_nn_wih(layer, parameter_name, w, l0=True):
    param = getattr(layer, parameter_name)
    if l0:
        for i in range(3*hidden_size):
            param.data[i] = w[i*input_size:(i+1)*input_size]
    else:
        for i in range(3*hidden_size):
            param.data[i] = w[i*num_directions*hidden_size:(i+1)*num_directions*hidden_size]

def set_nn_whh(layer, parameter_name, w):
    param = getattr(layer, parameter_name)
    for i in range(3*hidden_size):
        param.data[i] = w[i*hidden_size:(i+1)*hidden_size]

l0=True

for i in range(num_directions):
    for j in range(num_layers):
        if j == 0:
            wih = w0[i, :, :3*input_size]
            whh = w0[i, :, 3*input_size:]  # check
            l0=True
        else:
            wih = w[j-1, i, :, :num_directions*3*hidden_size]
            whh = w[j-1, i, :, num_directions*3*hidden_size:]
            l0=False

        if i == 0:
            set_nn_wih(
                gru, "weight_ih_l{}".format(j), torch.from_numpy(wih.flatten()),l0)
            set_nn_whh(
                gru, "weight_hh_l{}".format(j), torch.from_numpy(whh.flatten()))
        else:
            set_nn_wih(
                gru, "weight_ih_l{}_reverse".format(j), torch.from_numpy(wih.flatten()),l0)
            set_nn_whh(
                gru, "weight_hh_l{}_reverse".format(j), torch.from_numpy(whh.flatten()))

y, hn = gru(x_t, h_t)

numpy arrays are defined as following:

rng = np.random.RandomState(313)
w0 = rng.randn(num_directions, hidden_size, 3*(input_size +
               hidden_size)).astype(np.float32)
w = rng.randn(max(1, num_layers-1), num_directions, hidden_size,
              3*(num_directions*hidden_size + hidden_size)).astype(np.float32)


title: How does pytorch calculate matrix pairwise distance? Why isn't 'self' distance not zero?

body:
If this is a naive question, please forgive me, my test code like this:

import torch
from torch.nn.modules.distance import PairwiseDistance

list_1 = [[1., 1.,],[1., 1.]]
list_2 = [[1., 1.,],[2., 1.]]

mtrxA=torch.tensor(list_1)
mtrxB=torch.tensor(list_2)

print "A-B distance     :",PairwiseDistance(2).forward(mtrxA, mtrxB)
print "A 'self' distance:",PairwiseDistance(2).forward(mtrxA, mtrxA)
print "B 'self' distance:",PairwiseDistance(2).forward(mtrxB, mtrxB)

Result:

A-B distance     : tensor([1.4142e-06, 1.0000e+00])
A 'self' distance: tensor([1.4142e-06, 1.4142e-06])
B 'self' distance: tensor([1.4142e-06, 1.4142e-06])

Questions are:

How does pytorch calculate pairwise distance? Is it to calculate row vectors distance?
Why isn't 'self' distance 0?

Update

After changing list_1 and list_2 to this:

list_1 = [[1., 1.,1.,],[1., 1.,1.,]]
list_2 = [[1., 1.,1.,],[2., 1.,1.,]]

Result becomes:

A-B distance     : tensor([1.7321e-06, 1.0000e+00])
A 'self' distance: tensor([1.7321e-06, 1.7321e-06])
B 'self' distance: tensor([1.7321e-06, 1.7321e-06])


title: Concat tensors in PyTorch

body:
I have a tensor called data of the shape [128, 4, 150, 150] where 128 is the batch size, 4 is the number of channels, and the last 2 dimensions are height and width. I have another tensor called fake of the shape [128, 1, 150, 150].

I want to drop the last list/array from the 2nd dimension of data; the shape of data would now be [128, 3, 150, 150]; and concatenate it with fake giving the output dimension of the concatenation as [128, 4, 150, 150].

Basically, in other words, I want to concatenate the first 3 dimensions of data with fake to give a 4-dimensional tensor.

I am using PyTorch and came across the functions torch.cat() and torch.stack()

Here is a sample code I've written:

fake_combined = []
        for j in range(batch_size):
            fake_combined.append(torch.stack((data[j][0].to(device), data[j][1].to(device), data[j][2].to(device), fake[j][0].to(device))))
fake_combined = torch.tensor(fake_combined, dtype=torch.float32)
fake_combined = fake_combined.to(device)

But I am getting an error in the line:

fake_combined = torch.tensor(fake_combined, dtype=torch.float32)

The error is:

ValueError: only one element tensors can be converted to Python scalars

Also, if I print the shape of fake_combined, I get the output as [128,] instead of [128, 4, 150, 150]

And when I print the shape of fake_combined[0], I get the output as [4, 150, 150], which is as expected.

So my question is, why am I not able to convert the list to tensor using torch.tensor(). Am I missing something? Is there any better way to do what I intend to do?

Any help will be appreciated! Thanks!


i am trying to train a very basic CNN on CIFAR10 data set and getting the following error :
AttributeError: 'CrossEntropyLoss' object has no attribute 'backward'

criterion =nn.CrossEntropyLoss
optimizer=optim.SGD(net.parameters(),lr=0.001,momentum=0.9)

for epoch in range(2):  # loop over the dataset multiple times
        running_loss = 0.0
        for i, data in enumerate(trainloader, 0):
            # get the inputs
            inputs, labels = data
             # wrap them in Variable
            inputs, labels = Variable(inputs), Variable(labels)

            # zero the parameter gradients
            optimizer.zero_grad()
            # forward + backward + optimize
            outputs = net(inputs)
            loss = criterion(outputs, labels)
            loss.backward()
            optimizer.step()
            # print statistics
            running_loss += loss.data[0]
            if i % 2000 == 1999:    # print every 2000 mini-batches
                print('[%d, %5d] loss: %.3f' %
                      (epoch + 1, i + 1, running_loss / 2000))
                running_loss = 0.0


title: What exactly is the definition of a 'Module' in PyTorch?

body:
Please excuse the novice question, but is Module just the same as saying model?

That's what it sounds like, when the documentation says:

  Whenever you want a model more complex than a simple sequence of existing Modules you will need to define your model (as a custom Module subclass).

Or... when they mention Module, are they referring to something more formal and computer-sciency, like a protocol / interface type thing?


title: Stacking copies of an array/ a torch tensor efficiently?

body:
I'm a Python/Pytorch user. First, in numpy, let's say I have an array M of size LxL, and i want to have the following
array: A=(M,...,M) of size, say, NxLxL, is there a more elegant/memory efficient way of doing it than :

A=np.array([M]*N) ?

Same question with torch tensor !
Cause, Now, if M is a Variable(torch.tensor), i have to do:

A=torch.autograd.Variable(torch.tensor(np.array([M]*N)))

which is ugly !


I am following this tutorial: http://nlp.seas.harvard.edu/2018/04/03/attention.html
to implement the Transformer model from the "Attention Is All You Need" paper.

However I am getting the following error :
RuntimeError: "exp" not implemented for 'torch.LongTensor'

This is the line, in the PositionalEnconding class, that is causing the error:

div_term = torch.exp(torch.arange(0, d_model, 2) * -(math.log(10000.0) / d_model))

When it is being constructed here:

pe = PositionalEncoding(20, 0)

Any ideas?? I've already tried converting this to perhaps a Tensor Float type, but this has not worked.

I've even downloaded the whole notebook with accompanying files and the error seems to persist in the original tutorial.

Any ideas what may be causing this error?

Thanks!


title: Pytorch module error in Jupyter Notebook

body:
I installed pytorch using conda command when the virtual env was activated.

But, there are some problems when I import torch modules in Jupyter Notebook.

I checked the sys.path both in the prompt and in Jupyter Notebook.

Well.. in the prompt, the result of sys.path is

['', '/home/usrname/anaconda3/lib/python36.zip',
     '/home/usrname/anaconda3/lib/python3.6',
     '/home/usrname/anaconda3/lib/python3.6/lib-dynload',
     '/home/usrname/anaconda3/lib/python3.6/site-packages']

and there are no errors when I import torch modules.

But, in the jupyter notebook(executed in chrome), the sys.path is

['',
 '/home/usrname/anaconda3/lib/python36.zip',
 '/home/usrname/anaconda3/lib/python3.6',
 '/home/usrname/anaconda3/lib/python3.6/lib-dynload',
 '/home/usrname/anaconda3/lib/python3.6/site-packages',
 '/home/usrname/anaconda3/lib/python3.6/site-packages/IPython/extensions',
 '/home/usrname/.ipython']

and I see an error: No module named 'torch'

I can't solve this problem...


title: What does data.norm() < 1000 do in PyTorch?

body:
I am following the PyTorch tutorial here.
It says that

x = torch.randn(3, requires_grad=True)

y = x * 2
while y.data.norm() < 1000:
    y = y * 2

print(y)

Out:
tensor([-590.4467,   97.6760,  921.0221])

Could someone explain what data.norm() does here?
When I change .randn to .ones its output is  tensor([ 1024.,  1024.,  1024.]).


title: how tf.space_to_depth() works in tensorflow?

body:
I am a pytorch user. I have got a pretrained model in tensorflow and I would like to transfer it into pytorch. In one part of model architecture, I mean in tensorflow-defined model, there is a function tf.space_to_depth which transfers an input size of (None, 38,38,64) to (None, 19,19, 256). (https://www.tensorflow.org/api_docs/python/tf/space_to_depth) is the doc of this function. But I could not understand what this function actually do. Could you please provide some numpy codes to illustrate it for me?

Actually I would like to make an exact similar layer in pytorch.

Some codes in tensorflow reveals another secret:
Here is some codes:

import numpy as np
import tensorflow as tf

norm = tf.random_normal([1, 2, 2, 1], mean=0, stddev=1)
trans = tf.space_to_depth(norm,2)

with tf.Session() as s:
    norm = s.run(norm)
    trans = s.run(trans)

print("Norm")
print(norm.shape)
for index,value in np.ndenumerate(norm):
    print(value)

print("Trans")
print(trans.shape)
for index,value in np.ndenumerate(trans):
    print(value)

And here is the output:

Norm
(1, 2, 2, 1)
0.695261
0.455764
1.04699
-0.237587
Trans
(1, 1, 1, 4)
1.01139
0.898777
0.210135
2.36742

As you can see above, In Addition to data reshaping, the tensor values has changed!


title: Pytorch: IndexError: index out of range in self. How to solve?

body:
This training code is based on the run_glue.py script found here:
# Set the seed value all over the place to make this reproducible.
seed_val = 42

random.seed(seed_val)
np.random.seed(seed_val)
torch.manual_seed(seed_val)
torch.cuda.manual_seed_all(seed_val)

# Store the average loss after each epoch so we can plot them.
loss_values = []

# For each epoch...
for epoch_i in range(0, epochs):

    # ========================================
    #               Training
    # ========================================

    # Perform one full pass over the training set.

    print("")
    print('======== Epoch {:} / {:} ========'.format(epoch_i + 1, epochs))
    print('Training...')

    # Measure how long the training epoch takes.
    t0 = time.time()

    # Reset the total loss for this epoch.
    total_loss = 0

    # Put the model into training mode. Don't be mislead--the call to
    # `train` just changes the *mode*, it doesn't *perform* the training.
    # `dropout` and `batchnorm` layers behave differently during training
    # vs. test (source: https://stackoverflow.com/questions/51433378/what-does-model-train-do-in-pytorch)
    model.train()

    # For each batch of training data...
    for step, batch in enumerate(train_dataloader):

        # Progress update every 100 batches.
        if step % 100 == 0 and not step == 0:
            # Calculate elapsed time in minutes.
            elapsed = format_time(time.time() - t0)

            # Report progress.
            print('  Batch {:>5,}  of  {:>5,}.    Elapsed: {:}.'.format(step, len(train_dataloader), elapsed))

        # Unpack this training batch from our dataloader.
        #
        # As we unpack the batch, we'll also copy each tensor to the GPU using the
        # `to` method.
        #
        # `batch` contains three pytorch tensors:
        #   [0]: input ids
        #   [1]: attention masks
        #   [2]: labels
        b_input_ids = batch[0].to(device)
        b_input_mask = batch[1].to(device)
        b_labels = batch[2].to(device)

        # Always clear any previously calculated gradients before performing a
        # backward pass. PyTorch doesn't do this automatically because
        # accumulating the gradients is "convenient while training RNNs".
        # (source: https://stackoverflow.com/questions/48001598/why-do-we-need-to-call-zero-grad-in-pytorch)
        model.zero_grad()

        # Perform a forward pass (evaluate the model on this training batch).
        # This will return the loss (rather than the model output) because we
        # have provided the `labels`.
        # The documentation for this `model` function is here:
        # https://huggingface.co/transformers/v2.2.0/model_doc/bert.html#transformers.BertForSequenceClassification
        outputs = model(b_input_ids,
                    token_type_ids=None,
                    attention_mask=b_input_mask,
                    labels=b_labels)

        # The call to `model` always returns a tuple, so we need to pull the
        # loss value out of the tuple.
        loss = outputs[0]

        # Accumulate the training loss over all of the batches so that we can
        # calculate the average loss at the end. `loss` is a Tensor containing a
        # single value; the `.item()` function just returns the Python value
        # from the tensor.
        total_loss += loss.item()

        # Perform a backward pass to calculate the gradients.
        loss.backward()

        # Clip the norm of the gradients to 1.0.
        # This is to help prevent the "exploding gradients" problem.
        torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)

        # Update parameters and take a step using the computed gradient.
        # The optimizer dictates the "update rule"--how the parameters are
        # modified based on their gradients, the learning rate, etc.
        optimizer.step()

        # Update the learning rate.
        scheduler.step()

    # Calculate the average loss over the training data.
    avg_train_loss = total_loss / len(train_dataloader)

    # Store the loss value for plotting the learning curve.
    loss_values.append(avg_train_loss)

    print("")
    print("  Average training loss: {0:.2f}".format(avg_train_loss))
    print("  Training epcoh took: {:}".format(format_time(time.time() - t0)))

    # ========================================
    #               Validation
    # ========================================
    # After the completion of each training epoch, measure our performance on
    # our validation set.

    print("")
    print("Running Validation...")

    t0 = time.time()

    # Put the model in evaluation mode--the dropout layers behave differently
    # during evaluation.
    model.eval()

    # Tracking variables
    eval_loss, eval_accuracy = 0, 0
    nb_eval_steps, nb_eval_examples = 0, 0

    # Evaluate data for one epoch
    for batch in validation_dataloader:

        # Add batch to GPU
        batch = tuple(t.to(device) for t in batch)

        # Unpack the inputs from our dataloader
        b_input_ids, b_input_mask, b_labels = batch

        # Telling the model not to compute or store gradients, saving memory and
        # speeding up validation
        with torch.no_grad():

            # Forward pass, calculate logit predictions.
            # This will return the logits rather than the loss because we have
            # not provided labels.
            # token_type_ids is the same as the "segment ids", which
            # differentiates sentence 1 and 2 in 2-sentence tasks.
            # The documentation for this `model` function is here:
            # https://huggingface.co/transformers/v2.2.0/model_doc/bert.html#transformers.BertForSequenceClassification
            outputs = model(b_input_ids,
                            token_type_ids=None,
                            attention_mask=b_input_mask)

        # Get the "logits" output by the model. The "logits" are the output
        # values prior to applying an activation function like the softmax.
        logits = outputs[0]

        # Move logits and labels to CPU
        logits = logits.detach().cpu().numpy()
        label_ids = b_labels.to('cpu').numpy()

        # Calculate the accuracy for this batch of test sentences.
        tmp_eval_accuracy = flat_accuracy(logits, label_ids)

        # Accumulate the total accuracy.
        eval_accuracy += tmp_eval_accuracy

        # Track the number of batches
        nb_eval_steps += 1

    # Report the final accuracy for this validation run.
    print("  Accuracy: {0:.2f}".format(eval_accuracy/nb_eval_steps))
    print("  Validation took: {:}".format(format_time(time.time() - t0)))

print("")
print("Training complete!")

The error is as follows, while running the training for text classification using bert models came across the follow.
    ~/anaconda3/lib/python3.7/site-packages/torch/nn/modules/sparse.py in forward(self, input)
    112         return F.embedding(
    113             input, self.weight, self.padding_idx, self.max_norm,
--> 114             self.norm_type, self.scale_grad_by_freq, self.sparse)
    115
    116     def extra_repr(self):

~/anaconda3/lib/python3.7/site-packages/torch/nn/functional.py in embedding(input, weight, padding_idx, max_norm, norm_type, scale_grad_by_freq, sparse)
   1722         # remove once script supports set_grad_enabled
   1723         _no_grad_embedding_renorm_(weight, input, max_norm, norm_type)
-> 1724     return torch.embedding(weight, input, padding_idx, scale_grad_by_freq, sparse)
   1725
   1726

IndexError: index out of range in self

How can I fix it?


title: Converting Pytorch model .pth into onnx model

body:
I have one pre-trained model into format of .pth extension. I want to convert that into Tensorflow protobuf. But I am not finding any way to do that. I have seen onnx can convert models from pytorch into onnx and then from onnx to Tensorflow. But with that approach I got following error in the first stage of conversion.

from torch.autograd import Variable
import torch.onnx
import torchvision
import torch

dummy_input = Variable(torch.randn(1, 3, 256, 256))
model = torch.load('./my_model.pth')
torch.onnx.export(model, dummy_input, "moment-in-time.onnx")`

It gives error like this.

File "t.py", line 9, in <module>
    torch.onnx.export(model, dummy_input, "moment-in-time.onnx")
  File "/usr/local/lib/python3.5/dist-packages/torch/onnx/__init__.py", line 75, in export
    _export(model, args, f, export_params, verbose, training)
  File "/usr/local/lib/python3.5/dist-packages/torch/onnx/__init__.py", line 108, in _export
    orig_state_dict_keys = model.state_dict().keys()
AttributeError: 'dict' object has no attribute 'state_dict'

What is possible solution ?


title: Is .data still useful in pytorch?

body: I'm new to pytorch. I read much pytorch code which heavily uses tensor's .data member. But I search .data in the official document and Google, finding little. I guess .data contains the data in the tensor, but I don't know when we need it and when not?


title: How to convert torch tensor to pandas dataframe?

body:
I'd like to convert a torch tensor to pandas dataframe but by using pd.DataFrame I'm getting a dataframe filled with tensors instead of numeric values.

import torch
import pandas as  pd
x = torch.rand(4,4)
px = pd.DataFrame(x)

Here's what I get when clicking on px in the variable explorer:

0   1   2   3
tensor(0.3880)  tensor(0.4598)  tensor(0.4239)  tensor(0.7376)
tensor(0.4174)  tensor(0.9581)  tensor(0.0987)  tensor(0.6359)
tensor(0.6199)  tensor(0.8235)  tensor(0.9947)  tensor(0.9679)
tensor(0.7164)  tensor(0.9270)  tensor(0.7853)  tensor(0.6921)


title: Trying to load a custom dataset in Pytorch

body:
I'm just starting out with PyTorch and am, unfortunately, a bit confused when it comes to using my own training/testing image dataset for a custom algorithm. For starters, I am making a small "hello world"-esque convolutional shirt/sock/pants classifying network. I've only loaded a few images and am just making sure that PyTorch can load them and transform them down properly to 32x32 usable images. My ImageFolder is set up like so:

  imgs/socks/sockimages.jpeg
  imgs/pants/pantsimages.jpeg
  imgs/shirt/shirtimages.jpeg

and a similar setup for my testing images folder. According to my current knowledge, the image loader built into PyTorch should read the labels from the subfolder names within the training/test images. However, I'm getting a TypeError complaining that my iterator is not iterable. Here's my code and the error:

import torch
import torchvision
import torchvision.datasets as dset
import torchvision.transforms as transforms

transform = transforms.Compose(
[transforms.ToTensor(),
 transforms.Scale((32,32)),
 transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])

trainset = dset.ImageFolder(root="imgs",transform=transform)
trainloader = torch.utils.data.DataLoader(trainset, batch_size=4,shuffle=True,         num_workers=2)

testset = dset.ImageFolder(root='tests',transform=transform)
testloader = torch.utils.data.DataLoader(testset, batch_size=4,shuffle=True,     num_workers=2)

classes=('shirt','pants','sock')

import matplotlib.pyplot as plt
import numpy as np

# functions to show an image
def imshow(img):
    img = img / 2 + 0.5     # unnormalize
    npimg = img.numpy()
    plt.imshow(np.transpose(npimg, (1, 2, 0)))

# get some random training images
dataiter = iter(trainloader)
images, labels = dataiter.next()

# show images
imshow(torchvision.utils.make_grid(images))
# print labels
print(' '.join('%5s' % classes[labels[j]] for j in range(4)))

Error:

TypeError: 'builtin_function_or_method' object is not iterable

It says that it is in reference to the line containing dataiter.next(), meaning that the compiler believes that I cannot iterate dataiter?

Please help! Thanks in advance,

-David Sillman, new to PyTorch


title: PyTorch runtime error : invalid argument 0: Sizes of tensors must match except in dimension 1

body:
I have a PyTorch model and I'm trying to test it by performing a forward pass. Here is the code:

class ResBlock(nn.Module):
    def __init__(self, inplanes, planes, stride=1):
        super(ResBlock, self).__init__()
        self.conv1x1 = nn.Conv2d(inplanes, planes, kernel_size=1, stride=1, bias=False)
        self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=3, stride=stride, padding=1, bias=False)
        #batch normalization
        self.bn1 = nn.BatchNorm2d(planes)
        self.relu = nn.ReLU()
        self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride, padding=1, bias=False)
        self.bn2 = nn.BatchNorm2d(planes)
        self.stride = stride

    def forward(self, x):
        residual = self.conv1x1(x)

        out = self.conv1(x)
        out = self.bn1(out)
        out = self.relu(out)

        out = self.conv2(out)
        out = self.bn2(out)

        #adding the skip connection
        out += residual
        out = self.relu(out)

        return out

class ResUnet (nn.Module):

    def __init__(self, in_shape,  num_classes):
        super(ResUnet, self).__init__()
        in_channels, height, width = in_shape
        #
        #self.L1 = IncResBlock(in_channels,64)
        self.e1 = nn.Sequential(
            nn.Conv2d(in_channels, 64, kernel_size=4, stride=2,padding=1),
            ResBlock(64,64))

        self.e2 = nn.Sequential(
            nn.LeakyReLU(0.2,),
            nn.Conv2d(64, 128, kernel_size=4, stride=2,padding=1),
            nn.BatchNorm2d(128),
            ResBlock(128,128))
        #
        self.e2add = nn.Sequential(
            nn.Conv2d(128, 128, kernel_size=3, stride=1,padding=1),
            nn.BatchNorm2d(128))
        #
        ##
        self.e3 = nn.Sequential(
            nn.LeakyReLU(0.2,inplace=True),
            nn.Conv2d(128, 128, kernel_size=3, stride=1,padding=1),
            nn.BatchNorm2d(128),
            nn.LeakyReLU(0.2,),
            nn.Conv2d(128,256, kernel_size=4, stride=2,padding=1),
            nn.BatchNorm2d(256),
            ResBlock(256,256))

        self.e4 = nn.Sequential(
            nn.LeakyReLU(0.2,),
            nn.Conv2d(256,512, kernel_size=4, stride=2,padding=1),
            nn.BatchNorm2d(512),
            ResBlock(512,512))
        #
        self.e4add = nn.Sequential(
            nn.Conv2d(512,512, kernel_size=3, stride=1,padding=1),
            nn.BatchNorm2d(512))
        #
        self.e5 = nn.Sequential(
            nn.LeakyReLU(0.2,inplace=True),
            nn.Conv2d(512,512, kernel_size=3, stride=1,padding=1),
            nn.BatchNorm2d(512),
            nn.LeakyReLU(0.2,),
            nn.Conv2d(512,512, kernel_size=4, stride=2,padding=1),
            nn.BatchNorm2d(512),
            ResBlock(512,512))
        #
        #
        self.e6 = nn.Sequential(
            nn.LeakyReLU(0.2,),
            nn.Conv2d(512,512, kernel_size=4, stride=2,padding=1),
            nn.BatchNorm2d(512),
            ResBlock(512,512))
        #
        self.e6add = nn.Sequential(
            nn.Conv2d(512,512, kernel_size=3, stride=1,padding=1),
            nn.BatchNorm2d(512))
        #
        self.e7 = nn.Sequential(
            nn.LeakyReLU(0.2,inplace=True),
            nn.Conv2d(512,512, kernel_size=3, stride=1,padding=1),
            nn.BatchNorm2d(512),
            nn.LeakyReLU(0.2,),
            nn.Conv2d(512,512, kernel_size=4, stride=2,padding=1),
            nn.BatchNorm2d(512),
            ResBlock(512,512))
        #
        self.e8 = nn.Sequential(
            nn.LeakyReLU(0.2,),
            nn.Conv2d(512,512, kernel_size=4, stride=2,padding=1))
            #nn.BatchNorm2d(512))

        self.d1 = nn.Sequential(
            nn.ReLU(),
            nn.ConvTranspose2d(512, 512, kernel_size=4, stride=2,padding=1),
            nn.BatchNorm2d(512),
            nn.Dropout(p=0.5),
            ResBlock(512,512))
        #
        self.d2 = nn.Sequential(
            nn.ReLU(),
            nn.ConvTranspose2d(1024, 512, kernel_size=4, stride=2,padding=1),
            nn.BatchNorm2d(512),
            nn.Dropout(p=0.5),
            ResBlock(512,512))
        #
        self.d3 = nn.Sequential(
            nn.ReLU(),
            nn.ConvTranspose2d(1024, 512, kernel_size=4, stride=2,padding=1),
            nn.BatchNorm2d(512),
            nn.Dropout(p=0.5),
            ResBlock(512,512))
        #
        self.d4 = nn.Sequential(
            nn.ReLU(),
            nn.ConvTranspose2d(1024, 512, kernel_size=4, stride=2,padding=1),
            nn.BatchNorm2d(512),
            ResBlock(512,512))

        #
        self.d5 = nn.Sequential(
            nn.ReLU(),
            nn.ConvTranspose2d(1024, 256, kernel_size=4, stride=2,padding=1),
            nn.BatchNorm2d(256),
            ResBlock(256,256))
        #
        self.d6 = nn.Sequential(
            nn.ReLU(),
            nn.ConvTranspose2d(512, 128, kernel_size=4, stride=2,padding=1),
            nn.BatchNorm2d(128),
            ResBlock(128,128))
        #
        self.d7 = nn.Sequential(
            nn.ReLU(),
            nn.ConvTranspose2d(256, 64, kernel_size=4, stride=2,padding=1),
            nn.BatchNorm2d(64),
            ResBlock(64,64))
        #
        self.d8 = nn.Sequential(
            nn.ReLU(),
            nn.ConvTranspose2d(128, 64, kernel_size=4, stride=2,padding=1))
            #nn.BatchNorm2d(64),
            #nn.ReLU())

        self.out_l = nn.Sequential(
            nn.Conv2d(64,num_classes,kernel_size=1,stride=1))
            #nn.ReLU())

    def forward(self, x):

        #Image Encoder

        #### Encoder #####

        en1 = self.e1(x)

        en2 = self.e2(en1)
        en2add = self.e2add(en2)

        en3 = self.e3(en2add)

        en4 = self.e4(en3)
        en4add = self.e4add(en4)

        en5 = self.e5(en4add)

        en6 = self.e6(en5)
        en6add = self.e6add(en6)

        en7 = self.e7(en6add)

        en8 = self.e8(en7)

        #### Decoder ####
        de1_ = self.d1(en8)
        de1 = torch.cat([en7,de1_],1)

        de2_ = self.d2(de1)
        de2 = torch.cat([en6add,de2_],1)

        de3_ = self.d3(de2)
        de3 = torch.cat([en5,de3_],1)

        de4_ = self.d4(de3)
        de4 = torch.cat([en4add,de4_],1)

        de5_ = self.d5(de4)
        de5 = torch.cat([en3,de5_],1)

        de6_ = self.d6(de5)
        de6 = torch.cat([en2add,de6_],1)

        de7_ = self.d7(de6)
        de7 = torch.cat([en1,de7_],1)
        de8 = self.d8(de7)

        out_l_mask = self.out_l(de8)

        return out_l_mask

Here is how I attempt to test it:

modl = ResUnet((1,512,512), 1)
x = torch.rand(1, 1, 512, 512)
modl(x)

This works fine, as does for any size that are multiples of 64.

If I try:

modl = ResUnet((1,320,320), 1)
x = torch.rand(1, 1, 320, 320)
modl(x)

It throws an error

---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-46-4ddc821c365b> in <module>
----> 1 modl(x)

~/.conda/envs/torch0.4/lib/python3.6/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    475             result = self._slow_forward(*input, **kwargs)
    476         else:
--> 477             result = self.forward(*input, **kwargs)
    478         for hook in self._forward_hooks.values():
    479             hook_result = hook(self, input, result)

<ipython-input-36-f9eeefa3c0b8> in forward(self, x)
    221         de2_ = self.d2(de1)
    222         #print de2_.size()
--> 223         de2 = torch.cat([en6add,de2_],1)
    224         #print de2.size()
    225

RuntimeError: invalid argument 0: Sizes of tensors must match except in dimension 1. Got 5 and 4 in dimension 2 at /opt/conda/conda-bld/pytorch_1535491974311/work/aten/src/TH/generic/THTensorMath.cpp:3616

I figure the problem is caused by the input size not being a power of 2 but I am not sure how to rectify it for the given input dimenstions (320, 320).


while trying to finetune inception_V3 for my own dataset by changing the last fc layer like

last_layer =nn.Linear(n_inputs, len(classes))
inception_v3.fc = last_layer

after that when I train it got this error on this position

    # on training loop
    output = inception_v3(data)
    # calculate the batch loss
    loss = criterion(output, target)

Error is

 AttributeError: 'tuple' object has no attribute 'log_softmax'


title: If I'm not specifying to use CPU/GPU, which one is my script using?

body:
In pytorch, if I'm not writing anything about using CPU/GPU, and my machine supports CUDA (torch.cuda.is_available() == True):

What is my script using, CPU or GPU?
If CPU, what should I do to make it run on GPU? Do I need to rewrite everything?
If GPU, will this script crash if torch.cuda.is_available() == False?
Does this do anything about making the training faster?
I'm aware of Porting PyTorch code from CPU to GPU but this is old. Does this situation change in v0.4 or the upcoming v1.0?


In a pytorch model training process I get this error:

  RuntimeError: one of the variables needed for gradient computation has been modified by an inplace operation: [torch.cuda.LongTensor [128, 1]] is at version 8; expected version 7 instead. Hint: the backtrace further above shows the operation that failed to compute its gradient. The variable in question was changed in there or anywhere later. Good luck!

with stack trace

   sys:1: RuntimeWarning: Traceback of forward call that caused the error:
  File "/home/arash/anaconda2/envs/mzh27/lib/python2.7/runpy.py", line 174, in _run_module_as_main
    "__main__", fname, loader, pkg_name)
  File "/home/arash/anaconda2/envs/mzh27/lib/python2.7/runpy.py", line 72, in _run_code
    exec code in run_globals
  File "/home/arash/anaconda2/envs/mzh27/lib/python2.7/site-packages/ipykernel_launcher.py", line 16, in <module>
    app.launch_new_instance()
  File "/home/arash/anaconda2/envs/mzh27/lib/python2.7/site-packages/traitlets/config/application.py", line 658, in launch_instance
    app.start()
  File "/home/arash/anaconda2/envs/mzh27/lib/python2.7/site-packages/ipykernel/kernelapp.py", line 499, in start
    self.io_loop.start()
  File "/home/arash/anaconda2/envs/mzh27/lib/python2.7/site-packages/tornado/ioloop.py", line 1073, in start
    handler_func(fd_obj, events)
  File "/home/arash/anaconda2/envs/mzh27/lib/python2.7/site-packages/tornado/stack_context.py", line 300, in null_wrapper
    return fn(*args, **kwargs)
  File "/home/arash/.local/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 456, in _handle_events
    self._handle_recv()
  File "/home/arash/.local/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 486, in _handle_recv
    self._run_callback(callback, msg)
  File "/home/arash/.local/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 438, in _run_callback
    callback(*args, **kwargs)
  File "/home/arash/anaconda2/envs/mzh27/lib/python2.7/site-packages/tornado/stack_context.py", line 300, in null_wrapper
    return fn(*args, **kwargs)
  File "/home/arash/anaconda2/envs/mzh27/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 283, in dispatcher
    return self.dispatch_shell(stream, msg)
  File "/home/arash/anaconda2/envs/mzh27/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 233, in dispatch_shell
    handler(stream, idents, msg)
  File "/home/arash/anaconda2/envs/mzh27/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 399, in execute_request
    user_expressions, allow_stdin)
  File "/home/arash/anaconda2/envs/mzh27/lib/python2.7/site-packages/ipykernel/ipkernel.py", line 208, in do_execute
    res = shell.run_cell(code, store_history=store_history, silent=silent)
  File "/home/arash/anaconda2/envs/mzh27/lib/python2.7/site-packages/ipykernel/zmqshell.py", line 537, in run_cell
    return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)
  File "/home/arash/anaconda2/envs/mzh27/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2714, in run_cell
    interactivity=interactivity, compiler=compiler, result=result)
  File "/home/arash/anaconda2/envs/mzh27/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2818, in run_ast_nodes
    if self.run_code(code, result):
  File "/home/arash/anaconda2/envs/mzh27/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2878, in run_code
    exec(code_obj, self.user_global_ns, self.user_ns)
  File "<ipython-input-71-a5b255596e11>", line 33, in <module>
    sampled_captions, sampled_log_probs=predict_captions(out_enc,hid_enc,enc_pp,'sample')
  File "<ipython-input-70-a6ea511f0678>", line 18, in predict_captions
    out_dec, hid_dec, word_logits = dec.forward(r, last_enc, img_features)
  File "<ipython-input-21-0601dad4805f>", line 21, in forward
    emb = self.embedding(input)
  File "/home/arash/.local/lib/python2.7/site-packages/torch/nn/modules/module.py", line 493, in __call__
    result = self.forward(*input, **kwargs)
  File "/home/arash/.local/lib/python2.7/site-packages/torch/nn/modules/sparse.py", line 117, in forward
    self.norm_type, self.scale_grad_by_freq, self.sparse)
  File "/home/arash/.local/lib/python2.7/site-packages/torch/nn/functional.py", line 1506, in embedding
    return torch.embedding(weight, input, padding_idx, scale_grad_by_freq, sparse)

---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-71-a5b255596e11> in <module>()
     54                  torch.stack(sampled_log_probs).reshape(-1)).mean()
     55         torch.autograd.set_detect_anomaly(True)
---> 56         loss.backward()
     57         clip_grad_value_(enc_optim.param_groups[0]['params'], 5.0)
     58         clip_grad_value_(dec_optim.param_groups[0]['params'], 5.0)

/home/arash/.local/lib/python2.7/site-packages/torch/tensor.pyc in backward(self, gradient, retain_graph, create_graph)
    105                 products. Defaults to ``False``.
    106         """
--> 107         torch.autograd.backward(self, gradient, retain_graph, create_graph)
    108
    109     def register_hook(self, hook):

/home/arash/.local/lib/python2.7/site-packages/torch/autograd/__init__.pyc in backward(tensors, grad_tensors, retain_graph, create_graph, grad_variables)
     91     Variable._execution_engine.run_backward(
     92         tensors, grad_tensors, retain_graph, create_graph,
---> 93         allow_unreachable=True)  # allow_unreachable flag
     94
     95

But I'm unable to locate in-place operation that causes the error.
Here is my code:

for epoch in xrange(0, 13):
    print ("Starting New Epoch: %d" % epoch)
    rewards = {
        'sample_cider': [],
        'sample_context': [],
        'sample_reward': [],  # actual reward, controlled by beta
        'greedy_cider': [],
        'greedy_context': [],
        'greedy_reward': []
    }

    order = np.arange(enc_padded_text.shape[0])
    np.random.shuffle(order)
    enc_padded_text = enc_padded_text[order]
    input_text=[input_text[i] for i in order]
    dec_text_tensor.data = dec_text_tensor.data[order]

    for i in xrange(num_batches):
        s = i * BATCH_SIZE
        e = (i+1) * BATCH_SIZE

        _, enc_pp, dec_pp, enc_lengths = make_packpadded_s2s(s, e, enc_padded_text, dec_text_tensor)

        enc.zero_grad()
        dec.zero_grad()

        hid = enc.initHidden(BATCH_SIZE)

        out_enc, hid_enc = enc.forward(enc_pp, hid, enc_lengths)

        hid_enc = torch.cat([hid_enc[0,:, :], hid_enc[1,:,:]], dim=1).unsqueeze(0)
        gt_dict = dict(zip(_,input_text[s:e]))
        sampled_captions, sampled_log_probs=predict_captions(out_enc,hid_enc,enc_pp,'sample')
        sampled_dict = dict(zip(_, sampled_captions))
        with torch.no_grad():
            greedy_captions = predict_captions(out_enc,hid_enc,enc_pp, 'greedy')
            greedy_dict = dict(zip(_, greedy_captions))

        sample_cider_score, sample_context_score, sample_reward = get_scores(
            dec_pp[:,1:], sampled_captions, gt_dict, sampled_dict)
        greedy_cider_score, greedy_context_score, greedy_reward = get_scores(
            dec_pp[:,1:], greedy_captions, gt_dict, greedy_dict)

        # self-critical: score from sampling - score from test time
        advantages = torch.Tensor((sample_cider_score - greedy_cider_score).reshape(-1))

        # normalize advantages
        advantages = ((advantages - advantages.mean()) /
                      advantages.std() + 1e-9)
        if cuda:
            advantages = advantages.cuda()
        loss = -(advantages *
                 torch.stack(sampled_log_probs).reshape(-1)).mean()
        torch.autograd.set_detect_anomaly(True)
        loss.backward()
        clip_grad_value_(enc_optim.param_groups[0]['params'], 5.0)
        clip_grad_value_(dec_optim.param_groups[0]['params'], 5.0)
        enc_optim.step()
        dec_optim.step()

        rewards['sample_cider'].extend(sample_cider_score)
        rewards['sample_context'].extend(sample_context_score)
        rewards['sample_reward'].extend(sample_reward)
        rewards['greedy_cider'].extend(greedy_cider_score)
        rewards['greedy_context'].extend(greedy_context_score)
        rewards['greedy_reward'].extend(greedy_reward)

        if (b + 1) % 100 == 0:
            print('\t[Batch {} running metrics] - R train {:.2f} - R train (greedy): {:.2f}'.format(
                b + 1, np.mean(rewards['sample_reward']), np.mean(rewards['greedy_reward'])))

predict_captions function:

def predict_captions(img_features,hid_enc,enc_pp, mode='sample', constrain=False,L=22):
    dec_tensor = torch.ones((enc_pp.shape[0]), L+1, dtype=torch.long) * Toks.SOS
    global cuda
    if cuda:
        dec_tensor = dec_tensor.cuda(device=device)
    last_enc = hid_enc
    if mode == 'beam_search':
        return self.beam_search(img_features, state, lstm_states)

    predictions = []
    log_probs = []

    # this should store the index of the first occurrence of <EOS>
    # for each sample in the batch
    EOS_tracker = np.full(img_features.shape[0], None)
    for i in range(L):
        r=dec_tensor[:,i].unsqueeze(1)
        out_dec, hid_dec, word_logits = dec.forward(r, last_enc, img_features)
        out_dec[:, 0, Toks.UNK] = -np.inf # ignore unknowns
        l=out_dec[:,0]
        chosen = torch.argmax(l,dim=1)
        dec_tensor[:, i+1] = chosen
        last_enc = hid_dec
        # decoding stuff
        probs = F.softmax(word_logits, dim=2)
        probs=probs.reshape(128,20004)

        if constrain:
            # enforce constraint that the same word can't be predicted
            # twice in a row. zero-out the probability of previous words
            for p, prev_idx in zip(probs, state['prev_word_indeces']):
                p[prev_idx] = 0

        if mode == 'sample':
            idxs = torch.multinomial(probs, 1)
        else:
            idxs = torch.argmax(probs, dim=1)
        if cuda:
            idxs = idxs.cpu()
        words = [dec_idx_to_word[index] for index in idxs]
        predictions.append(np.array(words).reshape(-1))

        # get the respective log probability of chosen word
        # for each sample in the batch
        log_probs.append([lp[i] for (lp, i)
                         in zip(torch.log(probs), idxs)])

        # inefficient but this should be fast enough anyway... ? :(
        eos_idxs = (np.array(words)==dec_idx_to_word[2]).nonzero()[0]
        for idx in eos_idxs:
            if EOS_tracker[idx] is None:
                EOS_tracker[idx] = i + 1

        # finish loop if they're all done
        if len(EOS_tracker[EOS_tracker == None])==0:
            break

    # build the actual sentences, up until the first occurrence of <EOS>
    captions = [
        [' '.join(w[:eos_idx])] for (w, eos_idx) in
        zip(np.array(predictions).T, EOS_tracker)
    ]
    print captions
    # do this only when training. not needed otherwise.
    if mode == 'sample':
        log_probs = [lp[:eos_idx].sum() for (lp, eos_idx) in zip(np.array(log_probs).T, EOS_tracker)]
        return captions, log_probs

    return captions

models:

class Encoder_s2s(nn.Module):
    def __init__(self, input_size, hidden_size):
        super(Encoder_s2s, self).__init__()
        assert hidden_size % 2 == 0

        self.hidden_size = hidden_size
        self.input_size = input_size

        self.hidden_init_tensor = torch.zeros(2, 1, self.hidden_size/2, requires_grad=True)
        nn.init.normal_(self.hidden_init_tensor, mean=0, std=0.05)
        self.hidden_init = torch.nn.Parameter(self.hidden_init_tensor, requires_grad=True)

        self.embedding = nn.Embedding(input_size, hidden_size)
        self.emb_drop = nn.Dropout(0.2)
        self.gru = nn.GRU(hidden_size, hidden_size/2, batch_first=True, bidirectional=True)
        self.gru_out_drop = nn.Dropout(0.2)
        self.gru_hid_drop = nn.Dropout(0.3)

    def forward(self, input, hidden, lengths):
        emb = self.emb_drop(self.embedding(input))
        #emb = embedded_dropout(self.embedding, input, dropout=0.2 if self.training else 0)
        pp = torch.nn.utils.rnn.pack_padded_sequence(emb, lengths, batch_first=True)
        out, hidden = self.gru(pp, hidden)
        out = torch.nn.utils.rnn.pad_packed_sequence(out, batch_first=True)[0]
        out = self.gru_out_drop(out)
        hidden = self.gru_hid_drop(hidden)
        return out, hidden

    def initHidden(self, bs):
        return self.hidden_init.expand(2, bs, self.hidden_size/2).contiguous()

class DecoderAttn(nn.Module):
    def __init__(self, input_size, hidden_size, output_size, out_bias):
        super(DecoderAttn, self).__init__()
        self.hidden_size = hidden_size
        self.input_size = input_size

        self.embedding = nn.Embedding(input_size, hidden_size)
        self.emb_drop = nn.Dropout(0.2)
        self.gru = nn.GRU(hidden_size, hidden_size, batch_first=True)
        self.gru_drop = nn.Dropout(0.2)
        self.mlp = nn.Linear(hidden_size*2, output_size)
        if out_bias is not None:
            out_bias_tensor = torch.tensor(out_bias, requires_grad=False)
            self.mlp.bias.data[:] = out_bias_tensor
        self.logsoftmax = nn.LogSoftmax(dim=2)

        self.att_mlp = nn.Linear(hidden_size, hidden_size, bias=False)
        self.attn_softmax = nn.Softmax(dim=2)

    def forward(self, input, hidden, encoder_outs):
        emb = self.embedding(input)
        emb=self.emb_drop(emb)
        out, hidden = self.gru(emb, hidden)

        out_proj = self.att_mlp(out)
        enc_out_perm = encoder_outs.permute(0, 2, 1)
        e_exp = torch.bmm(out_proj, enc_out_perm)
        attn = self.attn_softmax(e_exp)

        ctx = torch.bmm(attn, encoder_outs)

        full_ctx = torch.cat([self.gru_drop(out), ctx], dim=2)

        out = self.mlp(full_ctx)
        out1 = self.logsoftmax(out)
        return out1, hidden, out


title: Skorch GridSearchCV: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to nan

body:
I have data x of dimension (n_samples, time_steps, n_features) for the features and (n_samples, 1, n_labels) for the labels y.

From this I create a train, development and test pytorch datasets.

I want to use GridSearchCV to do a grid search on the hyperparameters. That is what I wrote:

'Define the network'
sampling_interval = 0.1
net = ConvNet(time_window, ny)
net.float()

'Split test training set'
# trainin test. In this case we take some experiements as test and some as trainint
train_set_split = 0.9
dev_set_split = 0.05
test_set_split = 0.05

# Creating data indices for training and validation splits:
dataset_size = x.shape[0]
indices = list(range(dataset_size))
np.random.shuffle(indices)
split1 = int(np.floor(train_set_split * dataset_size))
split2 = int(np.floor(dev_set_split * dataset_size))
split3 = int(np.floor(test_set_split * dataset_size))

train_indices, dev_indices, test_indices = indices[:split1], indices[split1:split1 + split2], indices[split1 + split2:]

'Create a dataset'
train_dataset = MyDataset(x[train_indices, :, :], y[train_indices], net.device, net.dtype)
dev_dataset = MyDataset(x[dev_indices, :, :], y[dev_indices], net.device, net.dtype)
test_dataset = MyDataset(x[test_indices, :, :], y[test_indices], net.device, net.dtype)

'Define the optimizer'
optimizer = torch.optim.Adam(net.parameters(), lr=learning_rate)

'Define the loss function'
loss_func = torch.nn.MSELoss()

net_regr = NeuralNetRegressor(
    module=ConvNet,
    module__ny=ny,
    module__time_window=time_window,
    max_epochs=100,
    lr=0.1,
    train_split=predefined_split(dev_dataset),
    criterion=mean_squared_error,
    batch_size=batch_size,
)

params = {
    'lr': [0.01, 0.05, 0.1],
    'max_epochs': [100, 200, 300],
}

X_sl = SliceDataset(train_dataset, idx=0)  # idx=0 is the default
y_sl = SliceDataset(train_dataset, idx=1)

gs = GridSearchCV(net_regr, params, refit=False, verbose=4)

gs.fit(X_sl, y_sl)
print(gs.best_score_, gs.best_params_)

But I get this error

 FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to nan. Details:
ValueError: The target data shouldn't be 1-dimensional but instead have 2 dimensions, with the second dimension having the same size as the number of regression targets (usually 1). Please reshape your target data to be 2-dimensional (e.g. y = y.reshape(-1, 1).

but the shape of the target is 2 dimensional

>>> y_sl[0].shape

torch.Size([1, 4])

where 4 is the number of targets (n_labels).

So I don't understand where this error comes from


title: Indexing second dimension of Tensor using indices

body:
I selected element in my tensor using a tensor of indices. Here the code below I use list of indices 0, 3, 2, 1 to select 11, 15, 2, 5

>>> import torch
>>> a = torch.Tensor([5,2,11, 15])
>>> torch.randperm(4)

 0
 3
 2
 1
[torch.LongTensor of size 4]

>>> i = torch.randperm(4)
>>> a[i]

 11
 15
  2
  5
[torch.FloatTensor of size 4]

Now, I have

>>> b = torch.Tensor([[5, 2, 11, 15],[5, 2, 11, 15], [5, 2, 11, 15]])
>>> b

  5   2  11  15
  5   2  11  15
  5   2  11  15
[torch.FloatTensor of size 3x4]

Now, I want to use indices to select column 0, 3, 2, 1. In others word, I want a tensor like this

>>> b

 11  15   2   5
 11  15   2   5
 11  15   2   5
[torch.FloatTensor of size 3x4]


title: How to assign a new value to a pytorch Variable without breaking backpropagation?

body:
I have a pytorch variable that is used as a trainable input for a model. At some point I need to manually reassign all values in this variable.
How can I do that without breaking the connections with the loss function?
Suppose the current values are [1.2, 3.2, 43.2] and I simply want them to become [1,2,3].

Edit
At the time I asked this question, I hadn't realized that PyTorch doesn't have a static graph as Tensorflow or Keras do.
In PyTorch, the training loop is made manually and you need to call everything in each training step. (There isn't the notion of placeholder + static graph for later feeding data).
Consequently, we can't "break the graph", since we will use the new variable to perform all the further computations again. I was worried about a problem that happens in Keras, not in PyTorch.


title: How do you alter the size of a Pytorch Dataset?

body:
Say I am loading MNIST from torchvision.datasets.MNIST, but I only want to load in 10000 images total, how would I slice the data to limit it to only some number of data points?  I understand that the DataLoader is a generator yielding data in the size of the specified batch size, but how do you slice datasets?

tr = datasets.MNIST('../data', train=True, download=True, transform=transform)
te = datasets.MNIST('../data', train=False, transform=transform)
train_loader = DataLoader(tr, batch_size=args.batch_size, shuffle=True, num_workers=4, **kwargs)
test_loader = DataLoader(te, batch_size=args.batch_size, shuffle=True, num_workers=4, **kwargs)


title: pytorch custom layer "is not a Module subclass"

body:
I am new to PyTorch, trying it out after using a different toolkit for a while.

I would like understand how to program custom layers and functions. And as a simple test, I wrote this:

class Testme(nn.Module):         ## it _is_ a sublcass of module ##
    def __init__(self):
        super(Testme, self).__init__()

    def forward(self, x):
        return x / t_.max(x)

which is intended to cause the data passing through it to sum to 1. Not actually useful, just at test.

Then I plug it to the example code from the PyTorch Playground:

def make_layers(cfg, batch_norm=False):
    layers = []
    in_channels = 3
    for i, v in enumerate(cfg):
        if v == 'M':
            layers += [nn.MaxPool2d(kernel_size=2, stride=2)]
        else:
            padding = v[1] if isinstance(v, tuple) else 1
            out_channels = v[0] if isinstance(v, tuple) else v
            conv2d = nn.Conv2d(in_channels, out_channels, kernel_size=3, padding=padding)
            if batch_norm:
                layers += [conv2d, nn.BatchNorm2d(out_channels, affine=False), nn.ReLU()]
            else:
                layers += [conv2d, nn.ReLU()]
            layers += [Testme]                           # here <------------------
            in_channels = out_channels
    return nn.Sequential(*layers)

The result is an error!

TypeError: model.Testme is not a Module subclass

Maybe this needs to be a Function rather than a Module? Also not clear what the difference is between Function, Module.

For example, why does a Function need a backward(), even if it is constructed entirely from standard pytorch primitive, whereas a Module does not need this?


title: Should I use softmax as output when using cross entropy loss in pytorch?

body:
I have a problem with classifying fully connected deep neural net with 2 hidden layers for MNIST dataset in pytorch.
I want to use tanh as activations in both hidden layers, but in the end, I should use softmax.
For the loss, I am choosing nn.CrossEntropyLoss() in PyTOrch, which (as I have found out) does not want to take one-hot encoded labels as true labels, but takes LongTensor of classes instead.
My model is nn.Sequential() and when I am using softmax in the end, it gives me worse results in terms of accuracy on testing data. Why?
import torch
from torch import nn

inputs, n_hidden0, n_hidden1, out = 784, 128, 64, 10
n_epochs = 500
model = nn.Sequential(
    nn.Linear(inputs, n_hidden0, bias=True),
    nn.Tanh(),
    nn.Linear(n_hidden0, n_hidden1, bias=True),
    nn.Tanh(),
    nn.Linear(n_hidden1, out, bias=True),
    nn.Softmax()  # SHOULD THIS BE THERE?
)

criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.SGD(model.parameters(), lr=0.1, momentum=0.5)

for epoch in range(n_epochs):
    y_pred = model(X_train)
    loss = criterion(y_pred, Y_train)
    print('epoch: ', epoch+1,' loss: ', loss.item())
    optimizer.zero_grad()
    loss.backward()
    optimizer.step()


title: RuntimeError: size mismatch m1: [a x b], m2: [c x d]

body:
Can anyone help me in this.? I am getting below error. I use Google Colab. How to Solve this error.?

  size mismatch, m1: [64 x 100], m2: [784 x 128] at /pytorch/aten/src/TH/generic/THTensorMath.cpp:2070

Below Code I am trying to Run.

    import torch
    from torch import nn
    import torch.nn.functional as F
    from torchvision import datasets, transforms

    # Define a transform to normalize the data
    transform =
    transforms.Compose([transforms.CenterCrop(10),transforms.ToTensor(),])
    # Download the load the training data
    trainset = datasets.MNIST('~/.pytorch/MNIST_data/', download=True,
    train=True, transform=transform)
    trainloader = torch.utils.data.DataLoader(trainset, batch_size=64,
    shuffle=True)

    # Build a feed-forward network
    model = nn.Sequential(nn.Linear(784, 128),nn.ReLU(),nn.Linear(128,
    64),nn.ReLU(),nn.Linear(64, 10))

    # Define the loss
    criterion = nn.CrossEntropyLoss()

   # Get our data
   images, labels = next(iter(trainloader))
   # Faltten images
   images = images.view(images.shape[0], -1)

   # Forward pass, get our logits
   logits = model(images)
   # Calculate the loss with the logits and the labels
   loss = criterion(logits, labels)
   print(loss)


title: How does the __getitem__'s idx work within PyTorch's DataLoader?

body:
I'm currently trying to use PyTorch's DataLoader to process data to feed into my deep learning model, but am facing some difficulty.

The data that I need is of shape (minibatch_size=32, rows=100, columns=41). The __getitem__ coad that I have within the custom Dataset class that I wrote looks something like this:

def __getitem__(self, idx):
    x = np.array(self.train.iloc[idx:100, :])
    return x

The reason I wrote it like that is because I want the DataLoader to handle input instances of shape (100, 41) at a time, and we have 32 of these single instances.

However, I noticed that contrary to my initial belief the idx argument the DataLoader passes to the function is not sequential (this is crucial because my data is time series data). For example, printing the values gave me something like this:

idx = 206000
idx = 113814
idx = 80597
idx = 3836
idx = 156187
idx = 54990
idx = 8694
idx = 190555
idx = 84418
idx = 161773
idx = 177725
idx = 178351
idx = 89217
idx = 11048
idx = 135994
idx = 15067

Is this normal behavior? I'm posting this question because the data  batches that are being returned are not what I initially wanted them to be.

The original logic that I used to preprocess the data before using the DataLoader was:

Read data in from either txt or csv file.
Calculate how many batches are in the data and slice the data accordingly. For example, since one input instance is of shape (100, 41) and 32 of these form one minibatch, we usually end up with around 100 or so batches and reshape the data accordingly.
One input is of shape (32, 100, 41).

I'm not sure how else I should be handling the DataLoader hook methods. Any tips or advice are greatly appreciated. Thanks in advance.


title: Creating a Simple 1D CNN in PyTorch with Multiple Channels

body:
The dimensionality of the PyTorch inputs are not what the model expects, and I am not sure why.

To my understanding...

in_channels is first the number of 1D inputs we would like to pass to the model, and is the previous out_channel for all subsequent layers.

out_channels is the desired number of kernels (filters).

kernel_size is the number of parameters per filter.

Therefore, we would expect, as data passed to forward, a dataset with 7 1D channels (i.e. a 2D input).

However, the following code throws an error that is not consistent with what I expect, where this code:

import numpy
import torch

X = numpy.random.uniform(-10, 10, 70).reshape(-1, 7)
# Y = np.random.randint(0, 9, 10).reshape(-1, 1)

class Simple1DCNN(torch.nn.Module):
    def __init__(self):
        super(Simple1DCNN, self).__init__()
        self.layer1 = torch.nn.Conv1d(in_channels=7, out_channels=20, kernel_size=5, stride=2)
        self.act1 = torch.nn.ReLU()
        self.layer2 = torch.nn.Conv1d(in_channels=20, out_channels=10, kernel_size=1)
    def forward(self, x):
        x = self.layer1(x)
        x = self.act1(x)
        x = self.layer2(x)

        log_probs = torch.nn.functional.log_softmax(x, dim=1)

        return log_probs

model = Simple1DCNN()
print(model(torch.tensor(X)).size)

Throws the following error:

---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-5-eca5856a2314> in <module>()
     21
     22 model = Simple1DCNN()
---> 23 print(model(torch.tensor(X)).size)

~/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    487             result = self._slow_forward(*input, **kwargs)
    488         else:
--> 489             result = self.forward(*input, **kwargs)
    490         for hook in self._forward_hooks.values():
    491             hook_result = hook(self, input, result)

<ipython-input-5-eca5856a2314> in forward(self, x)
     12         self.layer2 = torch.nn.Conv1d(in_channels=20, out_channels=10, kernel_size=1)
     13     def forward(self, x):
---> 14         x = self.layer1(x)
     15         x = self.act1(x)
     16         x = self.layer2(x)

~/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    487             result = self._slow_forward(*input, **kwargs)
    488         else:
--> 489             result = self.forward(*input, **kwargs)
    490         for hook in self._forward_hooks.values():
    491             hook_result = hook(self, input, result)

~/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages/torch/nn/modules/conv.py in forward(self, input)
    185     def forward(self, input):
    186         return F.conv1d(input, self.weight, self.bias, self.stride,
--> 187                         self.padding, self.dilation, self.groups)
    188
    189

RuntimeError: Expected 3-dimensional input for 3-dimensional weight [20, 7, 5], but got 2-dimensional input of size [10, 7] instead

Edit: See below for solution, motivated by Shai.

import numpy
import torch

X = numpy.random.uniform(-10, 10, 70).reshape(1, 7, -1)
# Y = np.random.randint(0, 9, 10).reshape(1, 1, -1)

class Simple1DCNN(torch.nn.Module):
    def __init__(self):
        super(Simple1DCNN, self).__init__()
        self.layer1 = torch.nn.Conv1d(in_channels=7, out_channels=20, kernel_size=5, stride=2)
        self.act1 = torch.nn.ReLU()
        self.layer2 = torch.nn.Conv1d(in_channels=20, out_channels=10, kernel_size=1)
    def forward(self, x):
        x = self.layer1(x)
        x = self.act1(x)
        x = self.layer2(x)

        log_probs = torch.nn.functional.log_softmax(x, dim=1)

        return log_probs

model = Simple1DCNN().double()
print(model(torch.tensor(X)).shape)


title: How to repeat tensor in a specific new dimension in PyTorch

body:
If I have a tensor A which has shape [M, N],
I want to repeat the tensor K times so that the result B has shape [M, K, N]
and each slice B[:, k, :] should has the same data as A.
Which is the best practice without a for loop.
K might be in other dimension.

torch.repeat_interleave() and tensor.repeat() does not seem to work. Or I am using it in a wrong way.


title: Using a pytorch model for inference

body:
I am using the fastai library (fast.ai) to train an image classifier. The model created by fastai is actually a pytorch model.

type(model)
<class 'torch.nn.modules.container.Sequential'>

Now, I want to use this model from pytorch for inference. Here is my code so far:

torch.save(model,"./torch_model_v1")
the_model = torch.load("./torch_model_v1")
the_model.eval() # shows the entire network architecture

Based on the example shown here: http://pytorch.org/tutorials/beginner/data_loading_tutorial.html#sphx-glr-beginner-data-loading-tutorial-py, I understand that I need to write my own data loading class which will override some of the functions in the Dataset class. But what is not clear to me is the transformations that I need to apply at test time? In particular, how do I normalize the images at test time?

Another question: is my approach of saving and loading the model in pytorch fine? I read in the tutorial here: http://pytorch.org/docs/master/notes/serialization.html that the approach that I have used is not recommended. The reason is not clear though.


title: What does next() and iter() do in PyTorch's DataLoader()

body:
I have the following code:
import torch
import numpy as np
import pandas as pd
from torch.utils.data import TensorDataset, DataLoader

# Load dataset
df = pd.read_csv(r'../iris.csv')

# Extract features and target
data = df.drop('target',axis=1).values
labels = df['target'].values

# Create tensor dataset
iris = TensorDataset(torch.FloatTensor(data),torch.LongTensor(labels))

# Create random batches
iris_loader = DataLoader(iris, batch_size=105, shuffle=True)

next(iter(iris_loader))

What does next() and iter() do in the above code? I have went through PyTorch's documentation and still can quite understand what is next() and iter() doing here. Can anyone help in explaining this? Many thanks in advance.


title: Which part of Pytorch tensor represents channels?

body:
Surprisingly I have not found an answer to this question after looking around the internet. I am specifically interested in a 3d tensor. From doing my own experiments, I have found that when I create a tensor:

h=torch.randn(5,12,5)

And then put a convolutional layer on it defined as follows:

conv=torch.nn.Conv1d(12,48,3,padding=1)

The output is a (5,48,5) tensor. So, am I correct in assuming that for a 3d tensor in pytorch the middle number represents the number of channels?

Edit: It seems that when running a conv2d, the input dimension is the first entry in the tensor, and I need to make it a 4d tensor (1,48,5,5) for example. Now I am very confused...

Any help is much appreciated!


title: Invalid device Ordinal , CUDA / TORCH

body:
I am getting this error on running the script in ubuntu 16.04 . Please bear with me , i am new to python ,
I have checked the already available options on internet but i couldnt fix it.

 RuntimeError: cuda runtime error (10) : invalid device ordinal at torch/csrc/cuda/Module.cpp:32

I am currently running this file .

from __future__ import print_function
from models import LipRead
import torch
import toml
from training import Trainer
from validation import Validator

print("Loading options...")
with open('options.toml', 'r') as optionsFile:
options = toml.loads(optionsFile.read())

if(options["general"]["usecudnnbenchmark"] and options["general"]    ["usecudnn"]):
print("Running cudnn benchmark...")
torch.backends.cudnn.benchmark = True

#Create the model.
model = LipRead(options)

if(options["general"]["loadpretrainedmodel"]):
model.load_state_dict(torch.load(options["general"]    ["pretrainedmodelpath"]))

#Move the model to the GPU.
if(options["general"]["usecudnn"]):
model = model.cuda(options["general"]["gpuid"])

trainer = Trainer(options)
validator = Validator(options)

for epoch in range(options["training"]["startepoch"], options["training"]["epochs"]):

if(options["training"]["train"]):
    trainer.epoch(model, epoch)

if(options["validation"]["validate"]):
    validator.epoch(model)

And  I doubt this file has something to do with the error popped

Title = "TOML Example"

[general]
usecudnn = true
usecudnnbenchmark = true
gpuid = 0
loadpretrainedmodel = true
pretrainedmodelpath = "trainedmodel.pt"
savemodel = true
modelsavepath = "savedmodel.pt"

[input]
batchsize = 18
numworkers = 18
shuffle = true

[model]
type = "LSTM"
inputdim = 256
hiddendim = 256
numclasses = 500
numlstms = 2

[training]
train = true
epochs = 15
startepoch = 10
statsfrequency = 1000
dataset = "/udisk/pszts-ssd/AV-ASR-data/BBC_Oxford/lipread_mp4"
learningrate = 0.003
momentum = 0.9
weightdecay = 0.0001

[validation]
validate = true
dataset = "/udisk/pszts-ssd/AV-ASR-data/BBC_Oxford/lipread_mp4"
saveaccuracy = true
accuracyfilelocation = "accuracy.txt"

The error is mostly in the gpuid line as i have finally reached.


title: AttributeError: module 'torch' has no attribute "device"

body:
---> 13 device = torch.device({"cuda"} if torch.cuda.is_available() else {"cpu"})
     14
     15

AttributeError: module 'torch' has no attribute 'device'

I'm 99% sure this is because I didn't upgrade pytorch from 0.31 to 0.4 however I can't upgrade pytorch for now.

I need to translate .device (0.4) to something that works in 0.31.

I check the migration document  however it doesn't provide how I can convert torch.device in retrospect. Please help!


title: How can I downgrade the version pytorch from 0.4 to 0.31 with anaconda?

body:
I'm using anaconda now and due to some code error I need to downgrade the pytorch from version 0.4 to version 0.31.
However, as I use anaconda as my python package management tool, I used the instruction below to downgrade it and encountered the following error:
 conda install pytorch=0.31 cuda80 -c soumith

PackagesNotFoundError: The following packages are not available from current channels:

pytorch=0.31

And I also have tried some normal method to revert the package into previous verion in anaconda, but they failed either.
Could anyone tell me how to downgrade it? Thank you so much!!


title: PyTorch Model Training: RuntimeError: cuDNN error: CUDNN_STATUS_INTERNAL_ERROR

body:
After training a PyTorch model on a GPU for several hours, the program fails with the error

  RuntimeError: cuDNN error: CUDNN_STATUS_INTERNAL_ERROR

Training Conditions

Neural Network: PyTorch 4-layer nn.LSTM with nn.Linear output
Deep Q Network Agent (Vanilla DQN with Replay Memory)
state passed into forward() has the shape (32, 20, 15), where 32 is the batch size
50 seconds per episode
Error occurs after about 583 episodes (8 hours) or 1,150,000 steps, where each step involves a forward pass through the LSTM model.

My code also has the following values set before the training began

torch.manual_seed(0)
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
np.random.seed(0)

How can we troubleshoot this problem? Since this occurred 8 hours into the training, some educated guess will be very helpful here!

Thanks!

Update:

Commenting out the 2 torch.backends.cudnn... lines did not work. CUDNN_STATUS_INTERNAL_ERROR still occurs, but much earlier at around Episode 300 (585,000 steps).

torch.manual_seed(0)
#torch.backends.cudnn.deterministic = True
#torch.backends.cudnn.benchmark = False
np.random.seed(0)

System

PyTorch 1.6.0.dev20200525
CUDA 10.2
cuDNN 7604
Python 3.8
Windows 10
nVidia 1080 GPU

Error Traceback

RuntimeError                              Traceback (most recent call last)
<ipython-input-18-f5bbb4fdfda5> in <module>
     57
     58     while not done:
---> 59         action = agent.choose_action(state)
     60         state_, reward, done, info = env.step(action)
     61         score += reward

<ipython-input-11-5ad4dd57b5ad> in choose_action(self, state)
     58         if np.random.random() > self.epsilon:
     59             state = T.tensor([state], dtype=T.float).to(self.q_eval.device)
---> 60             actions = self.q_eval.forward(state)
     61             action = T.argmax(actions).item()
     62         else:

<ipython-input-10-94271a92f66e> in forward(self, state)
     20
     21     def forward(self, state):
---> 22         lstm, hidden = self.lstm(state)
     23         actions = self.fc1(lstm[:,-1:].squeeze(1))
     24         return actions

~\AppData\Local\Continuum\anaconda3\envs\rl\lib\site-packages\torch\nn\modules\module.py in __call__(self, *input, **kwargs)
    575             result = self._slow_forward(*input, **kwargs)
    576         else:
--> 577             result = self.forward(*input, **kwargs)
    578         for hook in self._forward_hooks.values():
    579             hook_result = hook(self, input, result)

~\AppData\Local\Continuum\anaconda3\envs\rl\lib\site-packages\torch\nn\modules\rnn.py in forward(self, input, hx)
    571         self.check_forward_args(input, hx, batch_sizes)
    572         if batch_sizes is None:
--> 573             result = _VF.lstm(input, hx, self._flat_weights, self.bias, self.num_layers,
    574                               self.dropout, self.training, self.bidirectional, self.batch_first)
    575         else:

RuntimeError: cuDNN error: CUDNN_STATUS_INTERNAL_ERROR

Update: Tried try... except on my code where this error occurs at, and in addition to RuntimeError: cuDNN error: CUDNN_STATUS_INTERNAL_ERROR, we also get a second traceback for the error RuntimeError: CUDA error: unspecified launch failure

During handling of the above exception, another exception occurred:

RuntimeError                              Traceback (most recent call last)
<ipython-input-4-e8f15cc8cf4f> in <module>
     61
     62     while not done:
---> 63         action = agent.choose_action(state)
     64         state_, reward, done, info = env.step(action)
     65         score += reward

<ipython-input-3-1aae79080e99> in choose_action(self, state)
     58         if np.random.random() > self.epsilon:
     59             state = T.tensor([state], dtype=T.float).to(self.q_eval.device)
---> 60             actions = self.q_eval.forward(state)
     61             action = T.argmax(actions).item()
     62         else:

<ipython-input-2-6d22bb632c4c> in forward(self, state)
     25         except Exception as e:
     26             print('error in forward() with state:', state.shape, 'exception:', e)
---> 27             print('state:', state)
     28         actions = self.fc1(lstm[:,-1:].squeeze(1))
     29         return actions

~\AppData\Local\Continuum\anaconda3\envs\rl\lib\site-packages\torch\tensor.py in __repr__(self)
    152     def __repr__(self):
    153         # All strings are unicode in Python 3.
--> 154         return torch._tensor_str._str(self)
    155
    156     def backward(self, gradient=None, retain_graph=None, create_graph=False):

~\AppData\Local\Continuum\anaconda3\envs\rl\lib\site-packages\torch\_tensor_str.py in _str(self)
    331                 tensor_str = _tensor_str(self.to_dense(), indent)
    332             else:
--> 333                 tensor_str = _tensor_str(self, indent)
    334
    335     if self.layout != torch.strided:

~\AppData\Local\Continuum\anaconda3\envs\rl\lib\site-packages\torch\_tensor_str.py in _tensor_str(self, indent)
    227     if self.dtype is torch.float16 or self.dtype is torch.bfloat16:
    228         self = self.float()
--> 229     formatter = _Formatter(get_summarized_data(self) if summarize else self)
    230     return _tensor_str_with_formatter(self, indent, formatter, summarize)
    231

~\AppData\Local\Continuum\anaconda3\envs\rl\lib\site-packages\torch\_tensor_str.py in __init__(self, tensor)
     99
    100         else:
--> 101             nonzero_finite_vals = torch.masked_select(tensor_view, torch.isfinite(tensor_view) & tensor_view.ne(0))
    102
    103             if nonzero_finite_vals.numel() == 0:

RuntimeError: CUDA error: unspecified launch failure