title: Attaching a loss to PyTorch's computational graph

body:
I have a model and inputs:
import torch
from torch.autograd import grad
import torch.nn as nn
import torch.optim as optim

class net_x(nn.Module):
        def __init__(self):
            super(net_x, self).__init__()
            self.fc1=nn.Linear(1, 1)
            self.out=nn.Linear(1, 1)

        def forward(self, x):
            x=self.fc1(x)
            x=self.out(x)
            return x

nx = net_x()
#inputs
t = torch.tensor([1.0, 2.0, 3.2], requires_grad = True) #input vector
t = torch.reshape(t, (3,1)) #reshape for batch

I'm trying to create a custom loss based on 2 lists (will make more sense in a minute):
pred_lst = []
goal_lst = list(range(10))

My training loop is:
for epoch in range(10):
    optimizer.zero_grad()
    y = nx(t)
    if torch.sum(y) > 5:
        lst.append(epoch)
    else:
        pass
    loss = len(set(pred_lst).symmetric_difference(set(goal_lst)))
    loss = torch.tensor(float(loss), requires_grad = True)
    print('loss: ', loss)
    loss.backward()

print('\nafter training')
for p in nx.parameters():
    print(p)

Essentially in this example, if the model's output is >5, pred_lst saves the epoch number. At the end, the loss is simply the total number of epochs that the epoch number wasn't saved.
This doesn't work as from my understanding the loss is not connected to the computational graph and hence the parameters are not being updated:
>>>
before training
Parameter containing:
tensor([[-0.7613]], requires_grad=True)
Parameter containing:
tensor([-0.3810], requires_grad=True)
Parameter containing:
tensor([[-0.9269]], requires_grad=True)
Parameter containing:
tensor([-0.8938], requires_grad=True)

loss:  tensor(10., requires_grad=True)
loss:  tensor(10., requires_grad=True)
loss:  tensor(10., requires_grad=True)
loss:  tensor(10., requires_grad=True)
loss:  tensor(10., requires_grad=True)
loss:  tensor(10., requires_grad=True)
loss:  tensor(10., requires_grad=True)
loss:  tensor(10., requires_grad=True)
loss:  tensor(10., requires_grad=True)
loss:  tensor(10., requires_grad=True)

after training
Parameter containing:
tensor([[-0.7613]], requires_grad=True)
Parameter containing:
tensor([-0.3810], requires_grad=True)
Parameter containing:
tensor([[-0.9269]], requires_grad=True)
Parameter containing:
tensor([-0.8938], requires_grad=True)

How can I "attach"/"connect" this loss to the computational graph nevertheless?


title: Conv-Variational-Autoencoder Loss is NaN

body:
I am training a Variational Autoencoder. Suddenly my Loss Explodes and then becomes NaN
and I dont know why.When printing the output of the Encoder the encoder already outputs NaN values after some batches! So it doesnt happen in the sampling part. May there be anything wrong with my loss function?
I am using an Adam optimizer btw with a learning rate of 0.002!

class VAE(nn.Module):
   def __init__(self, input_shape, z_dim):
       super().__init__()
       self.z_dim = z_dim
       self.input_shape = input_shape

       # encoder
       self.encoder_conv = nn.Sequential(
           nn.Conv2d(1, 32, 3, stride=2, padding=1),
           nn.BatchNorm2d(32),
           nn.LeakyReLU(),
           nn.Conv2d(32, 64, 3, stride=2, padding=1),
           nn.BatchNorm2d(64),
           nn.LeakyReLU(),
           nn.Conv2d(64, 64, 3, stride=2, padding=1),
           nn.BatchNorm2d(64),
           nn.LeakyReLU(),
           nn.Conv2d(64, 64, 3, stride=2, padding=1),
           nn.BatchNorm2d(64),
           nn.LeakyReLU()
       )
       self.conv_out_size = self._get_conv_out_size(input_shape)
       self.mu = nn.Sequential(
           nn.Linear(self.conv_out_size, z_dim),
           nn.LeakyReLU(),
           nn.Dropout(0.2)
       )
       self.log_var = nn.Sequential(
           nn.Linear(self.conv_out_size, z_dim),
           nn.LeakyReLU(),
           nn.Dropout(0.2)
       )

       # decoder
       self.decoder_linear = nn.Sequential(
           nn.Linear(z_dim, self.conv_out_size),
           nn.LeakyReLU(),
           nn.Dropout(0.2)
       )

       self.decoder_conv = nn.Sequential(
           nn.UpsamplingNearest2d(scale_factor=2),
           nn.ConvTranspose2d(64, 64, 3, stride=1, padding=1),
           nn.BatchNorm2d(64),
           nn.LeakyReLU(),
           nn.UpsamplingNearest2d(scale_factor=2),
           nn.ConvTranspose2d(64, 64, 3, stride=1, padding=1),
           nn.BatchNorm2d(64),
           nn.LeakyReLU(),
           nn.UpsamplingNearest2d(scale_factor=2),
           nn.ConvTranspose2d(64, 32, 3, stride=1, padding=1),
           nn.BatchNorm2d(32),
           nn.LeakyReLU(),
           nn.UpsamplingNearest2d(scale_factor=2),
           nn.ConvTranspose2d(32, 1, 3, stride=1, padding=(5,3)),
           nn.Sigmoid()
       )

   def sampling(self, mu, log_var):
       epsilon = torch.Tensor(np.random.normal(size=(self.z_dim), scale=1.0)).cuda()
       return mu + epsilon * torch.exp(log_var / 2)

   def forward_encoder(self, x):
       x = self.encoder_conv(x)
       x = x.view(x.size()[0], -1)
       mu_p = self.mu(x)
       log_var_p = self.log_var(x)
       return [mu_p, log_var_p]

   def forward_decoder(self, x):
       x = self.decoder_linear(x)
       x = x.view(x.size()[0], *self.conv_out_shape[1:])
       x = self.decoder_conv(x)
       return x

   def forward(self, x):
       mu_p, log_var_p = self.forward_encoder(x)
       x = self.sampling(mu_p, log_var_p)
       images_p = self.forward_decoder(x)
       return [mu_p, log_var_p, images_p]

   def _get_conv_out_size(self, shape):
       out = self.encoder_conv(torch.zeros(1, *shape))
       self.conv_out_shape = out.size()
       return int(np.prod(self.conv_out_shape))

   def forward_no_epsilon(self, x):
       mu_p, log_var_p = self.forward_encoder(x)
       x = mu_p
       images_p = self.forward_decoder(x)
       return images_p

Loss:
def kl_loss(mu, log_var):
    return -0.5 * torch.mean(1 + log_var - mu.pow(2) - torch.exp(log_var))

def r_loss(y_train, y_pred):
    r_loss = torch.mean((y_train - y_pred) ** 2)
    return r_loss

train:
mu_v, log_var_v, images_out_v = vae(images_v)
r_loss_v = r_loss(images_out_v, labels_v)
kl_loss_v = kl_loss(mu_v, log_var_v)
loss = kl_loss_v + r_loss_v * 10000.0
loss.backward()
optimizer.step()

Losses:


title: Do we have lower performance and accuracy than when not using `pytorch.nn.Sequnetial` and if yes, why?

body:
I was checking out this video where Phil points out to this fact that using torch.nn.Sequential is faster than not using it. I did quick google and came across this post which is not answered satisfactorily, so I am replicating it here.
Here is the code from the post with Sequential:
class net2(nn.Module):
    def __init__(self):
        super(net2, self).__init__()
        self.conv1 = nn.Sequential(
            nn.Conv2d(
                in_channels=1,
                out_channels=16,
                kernel_size=5,
                stride=1,
                padding=2,
            ),
            nn.ReLU(),
            nn.MaxPool2d(kernel_size=2),
        )
        self.conv2 = nn.Sequential(
            nn.Conv2d(16, 32, 5, 1, 2),
            nn.ReLU(),
            nn.MaxPool2d(2),
        )
        self.out = nn.Linear(32 * 7 * 7, 10)

    def forward(self, x):
        x = self.conv1(x)
        x = self.conv2(x)
        x = x.view(x.size(0), -1)
        output = self.out(x)
        return output

And here is the net without Sequential:
class net1(nn.Module):
    def __init__(self):
        super(net1, self).__init__()
        self.conv1 = nn.Conv2d(1,16,5,1,2)
        self.conv2 = nn.Conv2d(16,32,5,1,2)
        self.pool1 = nn.MaxPool2d(2)
        self.pool2 = nn.MaxPool2d(2)
        self.fc1 = nn.Linear(32*7*7,10)

    def forward(self, x):
        out = F.relu(self.conv1(x))
        out = self.pool1(out)
        out = F.relu(self.conv2(out))
        out = self.pool2(out)

        out = out.view(-1,32*7*7)
        out = F.relu(self.fc1(out))
        return out

In the comment, the author also states that the accuracy is better with Sequential than without it. One of the comment states that speed is more with Sequential because loop unrolling results in faster execution. So here are my related questions:

Is loop unrolling only reason why Sequntial implementation faster than the other one?

Can someone explain how non-Sequential does not result in loop unrolling and Sequntial results in loop-unrolling, perhaps by pointing me to pytorch source lines on github

Does Sequential really result in more accuracy, if yes why?


title: Normalising images using mean and std of a dataset

body:
I used the following snippet to compute the mean and std of the images in the cityscapes dataset to normalise them:
def compute_mean_std(dataloader):

    pop_mean = []

    pop_std = []

    for i, (img,mask, rgb_mask) in enumerate(dataloader):

        numpy_image = img.cpu().numpy()

        batch_mean = np.mean(numpy_image,axis=(0,2,3))
        pop_mean.append(batch_mean)

        #print(batch_mean.shape)

        batch_std = np.mean(numpy_image, axis=(0,2,3))
        pop_std.append(batch_std)

        #print(batch_std.shape)

    pop_mean = np.array(pop_mean).mean(axis=0)
    pop_std = np.array(pop_std).std(axis=0)

    print(pop_mean.shape)
    print(pop_std.shape)

    return(pop_mean, pop_std)

This code gave me the following mean and std:

MEAN = [0.28660315, 0.32426634, 0.28302112]
STD = [0.00310452, 0.00292714, 0.00296411]

but when I computed the mean and std of images after normalisation using these mean and std, they are not close to 0 and 1.
Is this approach correct to compute mean and std over the whole dataset and normalising images?


title: Multiply by the factor the number of times that the position appears in the array (Python)

body:
I'm trying to get a "pythonic" way or using a library (anything except "for" loops) for doing this.
my_array = np.array([2,4,6])
my_array[[0,2,0]] = my_array[[0,2,0]] * 2

Expected result:
[8,4,12]

However, I get:
[4,4,12]

This doesn't work for me as I need to multiply twice the factor (2) for the position 0 if, through the index, the 0 position repeats n amount of times.
"For" loops can't be used as there are billions of data points


Does torch.manual_seed include the operation of torch.cuda.manual_seed_all?
If yes, we can just use torch.manual_seed to set the seed. Otherwise we should call both functions.


title: Print the validation loss in each epoch in PyTorch

body:
I want to print the model's validation loss in each epoch, what is the right way to get and print the validation loss?
Is it like this:
criterion = nn.CrossEntropyLoss(reduction='mean')
for x, y in validation_loader:
 optimizer.zero_grad()
 out = model(x)
 loss = criterion(out, y)
 loss.backward()
 optimizer.step()
 losses += loss

display_loss = losses/len(validation_loader)
print(display_loss)

or like this
criterion = nn.CrossEntropyLoss(reduction='mean')
for x, y in validation_loader:
     optimizer.zero_grad()
     out = model(x)
     loss = criterion(out, y)
     loss.backward()
     optimizer.step()
     losses += loss

display_loss = losses/len(validation_loader.dataset)
print(display_loss)

or something else? Thank you.


title: How to fix an error with the quickstart tutorial for pytorch?

body:
I am trying to follow the tutorial on pytorch HERE, but there seems to be a problem. I have created a custom dataloader named training_data that returns an object as required HERE which is a dictionary
{"image": image, "label": label}

where image is a tensor and label is a string. I then follow the tutorial and create a DataLoader as follows:
train_dataloader = DataLoader(training_data, batch_size=batch_size)

and use that DataLoader in the method train:
def train(dataloader, model, loss_fn, optimizer):
    size = len(dataloader)
    for batch, (X, y) in enumerate(dataloader):
        X, y = X.to(device), y.to(device)

        # Compute prediction error
        pred = model(X)
        loss = loss_fn(pred, y)

        # Backpropagation
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

        if batch % 100 == 0:
            loss, current = loss.item(), batch * len(X)
            print(f"loss: {loss:>7f}  [{current:>5d}/{size:>5d}]")

However, when I call the training method for a batch
train(train_dataloader, model, loss_fn, optimizer)

I get an error
Traceback (most recent call last):
  File "train_network.py", line 110, in <module>
    train(train_dataloader, model, loss_fn, optimizer)
  File "train_network.py", line 76, in train
    X, y = X.to(device), y.to(device)
AttributeError: 'str' object has no attribute 'to'

as y is a string with the content label. What am I doing wrong?


title: Python colon and comma indexing

body:
I'm looking at some deep learning code in pytorch and I see this kind of indexing out[:, -1, :]
What does it mean?


When I try to run my program I get an error saying:
RuntimeError: CUDA error: invalid device ordinal
The full error is below.
I don't have much experience with this sort of thing; furthermore it's neither my own program, nor my own machine.
Based on this question on github, I tested the following things:
Python 3.8.8 | packaged by conda-forge | (default, Feb 20 2021, 16:22:27)
[GCC 9.3.0] on linux
Type "help", "copyright", "credits" or "license" for more information.
>>> import torch
>>> torch.__version__
'1.8.1+cu102'
>>> torch.cuda.is_available()
True
>>> torch.cuda.device_count()
1
>>> torch.cuda.get_device_name()
'GeForce RTX 2080 Ti'
>>>

Unlike in that question, the machine I'm using only seems to have access to a single GPU. A colleague suggested it might have something to do with self.device yielding the wrong value?
Of course, any help is very much appreciated!
(rlpyt) hbp@aklma-MS-7B24:~$ cd Documents/Bing/Mathieu/learning_to_be_taught/experiments/vmpo_replay_ratio/(rlpyt) hbp@aklma-MS-7B24:~/Documents/Bing/Mathieu/learning_to_be_taught/experiments/vmpo_replay_ratio$ python vmpo_replay_ratio.py
/home/hbp/anaconda3/envs/rlpyt/lib/python3.8/site-packages/gym/logger.py:30: UserWarning: WARN: Box bound precision lowered by casting to float32
  warnings.warn(colorize('%s: %s'%('WARN', msg % args), 'yellow'))
training started with parameters: Namespace(epochs=None, log_dir=None, log_dir_positional=None, name='run', run_id=None, serial_mode=False, slot_affinity_code=None, snapshot_file=None)
exp_dir: /home/hbp/Documents/Bing/Mathieu/learning_to_be_taught/experiments/vmpo_replay_ratio/logs/run_6
using seed 5986
2021-05-27 14:11:40.546471  | run_6 Running 1520 sampler iterations.
2021-05-27 14:11:40.600944  | run_6 Optimizer master CPU affinity: [0].
2021-05-27 14:11:40.626970  | run_6 Initialized async CPU agent model.
2021-05-27 14:11:40.627073  | run_6 WARNING: unequal number of envs per process, from batch_B 6400 and n_worker 7 (possible suboptimal speed).
2021-05-27 14:11:40.627223  | run_6 Total parallel evaluation envs: 21.
2021-05-27 14:11:40.657946  | run_6 Optimizer master Torch threads: 1.
using seed 5987
using seed 5986
using seed 5988
using seed 5989
using seed 5990
using seed 5991
Traceback (most recent call last):
  File "vmpo_replay_ratio.py", line 213, in <module>
    build_and_train(slot_affinity_code=args.slot_affinity_code,
  File "vmpo_replay_ratio.py", line 135, in build_and_train
    runner.train()
  File "/home/hbp/Documents/Bing/Mathieu/rlpyt/rlpyt/runners/async_rl.py", line 87, in train
    throttle_itr, delta_throttle_itr = self.startup()
  File "/home/hbp/Documents/Bing/Mathieu/rlpyt/rlpyt/runners/async_rl.py", line 161, in startup
    throttle_itr, delta_throttle_itr = self.optim_startup()
  File "/home/hbp/Documents/Bing/Mathieu/rlpyt/rlpyt/runners/async_rl.py", line 177, in optim_startup
    self.agent.to_device(main_affinity.get("cuda_idx", None))
  File "/home/hbp/Documents/Bing/Mathieu/rlpyt/rlpyt/agents/base.py", line 115, in to_device
using seed 5992
    self.model.to(self.device)
  File "/home/hbp/anaconda3/envs/rlpyt/lib/python3.8/site-packages/torch/nn/modules/module.py", line 673, in to
    return self._apply(convert)
  File "/home/hbp/anaconda3/envs/rlpyt/lib/python3.8/site-packages/torch/nn/modules/module.py", line 387, in _apply
    module._apply(fn)
  File "/home/hbp/anaconda3/envs/rlpyt/lib/python3.8/site-packages/torch/nn/modules/module.py", line 387, in _apply
    module._apply(fn)
  File "/home/hbp/anaconda3/envs/rlpyt/lib/python3.8/site-packages/torch/nn/modules/module.py", line 387, in _apply
    module._apply(fn)
  File "/home/hbp/anaconda3/envs/rlpyt/lib/python3.8/site-packages/torch/nn/modules/module.py", line 409, in _apply
    param_applied = fn(param)
  File "/home/hbp/anaconda3/envs/rlpyt/lib/python3.8/site-packages/torch/nn/modules/module.py", line 671, in convert
    return t.to(device, dtype if t.is_floating_point() or t.is_complex() else None, non_blocking)
RuntimeError: CUDA error: invalid device ordinal
2021-05-27 14:11:40.987723  | run_6 Sampler rank 1 initialized, CPU affinity [2], Torch threads 1, Seed 5987
2021-05-27 14:11:40.987714  | run_6 Sampler rank 0 initialized, CPU affinity [1], Torch threads 1, Seed 5986
2021-05-27 14:11:40.988088  | run_6 Sampler rank 2 initialized, CPU affinity [3], Torch threads 1, Seed 5988
2021-05-27 14:11:40.989922  | run_6 Sampler rank 3 initialized, CPU affinity [4], Torch threads 1, Seed 5989
2021-05-27 14:11:40.992058  | run_6 Sampler rank 4 initialized, CPU affinity [5], Torch threads 1, Seed 5990
2021-05-27 14:11:40.995587  | run_6 Sampler rank 5 initialized, CPU affinity [6], Torch threads 1, Seed 5991
2021-05-27 14:11:40.996119  | run_6 Sampler rank 6 initialized, CPU affinity [7], Torch threads 1, Seed 5992


title: How to update pytorch model with supervision provided by caffe model?

body:
I want to do something like this:

# pytorch_model to train, caffe_model freezed
torch_out = pytorch_model(input)
caffe_out = caffe_model(torch_out)
loss = criterion(caffe_out, label)
loss.backward() # or something like torch_out.backward()

I can easily get the gradient of torch_out provided by caffe_model.backward(), but how can I update pytorch_model with that?


title: Why does not using retain_graph=True result in error?

body:
If I need to backpropagate through a neural network twice and I don't use retain_graph=True, I get an error.

Why? I realize it is nice to keep the intermediate variables used for the first backpropagation to be reused for the second backpropagation. However, why aren't they simply recalculated, like they were originally calculated in the first backpropagation?


title: How to encode my data to use it with an Pytorch-LSTM

body:
I am trying to set up a Pytorch-LSTM for an affect-analyis-problem. I am currently stuck on how I should correctly encode my data and what kind of data the LSTM expects.

I have a dataset of roughly 12000 sentences, all tagges with a specific emotion. My data is encoded as follows:

I created a seperate dataset that consists of the Part-of-Speech-Tags for every word in the sentences. The POS-Tags are one-hot-encoded:

0           1               83
[1,0,0..0], [0,1,0..0], ... [0,0,0..0]
[0,0,0..1], [0,0,1..0], ... [1,0,0..0]
...

Every row is one sentence. All rows have a length of 84 (shorter sentences are padded with a zero-vector). The lists in each cell correspond to a specific pos-tags, they have a length of 17.

The target-dataset has the same number of rows as the dataset above, but every row only contains one list, encoding one of four emotions:

0
[1,0,0,0]
[0,0,0,1]
...

I am now stuck on how to instert this data (especially the pos-dataset) into the lstm. The docs say, the LSTM expects an input-dimension and output-dimension parameter. What is exactly is my input dimension here? Every sequence has a length of 84, but every item in this sequence is a list of length 17.

I've read the docs and some tutorials, however none of these seem to use a similar data structure as I do. Any help would be greatly appreciated.


title: Why did the loss of CNN model change little throughout the whole epoch？

body:
I am running a deep learning project about visual speech recognition tasks, and I found a strange phenomenon.
In the first several epochs, the loss can decrease at a normal speed. I mean, in a epoch, the loss would decrease with the increase of the number of iterations. While in the later epochs, the loss almost unchanged in the whole epoch, but would decrease at the start of the next epoch.

Sometimes I interrupted the running code after an epoch finishing and restart from the trained weights. The loss would decrease at the start of the next epoch, too.

This is the training code:

    for epoch in range(283,args.epochs):

        model.train()
        running_loss, running_corrects, running_all, cer = 0., 0., 0., 0.

        for batch_idx, sample_batched in enumerate(dset_loaders['train']):
            optimizer.zero_grad()
            inputs,targets,lengths,y_lengths,idx = sample_batched
            inputs = inputs.float()
            inputs, targets = inputs.to(device) , targets.to(device)
            outputs = model(inputs)
            loss = criterion(F.log_softmax(outputs,dim=-1),targets,lengths,y_lengths)
            loss.backward()
            optimizer.step()

            decoded = decoder.decode_greedy(outputs,lengths)
            cursor, gt = 0, []
            for b in range(inputs.size(0)):
                y_str = ''.join([vocabularies[ch] for ch in targets[cursor: cursor + y_lengths[b]]])
                gt.append(y_str)
                cursor += y_lengths[b]
            CER = decoder.cer_batch(decoded,gt)
            cer += CER
            cer_mean = cer/(batch_idx+1)

            running_loss += loss.data * inputs.size(0)
            running_all += len(inputs)
            if batch_idx == 0:
                since = time.time()
            else (batch_idx+1) % args.interval == 0 or (batch_idx == len(dset_loaders['train'])-1):
                print('Process: [{:5.0f}/{:5.0f} ({:.0f}%)]\tLoss: {:.4f}\tcer:{:.4f}\tCost time:{:5.0f}s\tEstimated time:{:5.0f}s\t'.format(
                    running_all,
                    len(dset_loaders['train'].dataset),
                    100. * batch_idx / (len(dset_loaders['train'])-1),
                    running_loss / running_all,
                    cer_mean,
                    time.time()-since,
                    (time.time()-since)*(len(dset_loaders['train'])-1) / batch_idx - (time.time()-since)))
        print('{} Epoch:\t{:2}\tLoss: {:.4f}\tcer:{:.4f}\t'.format(
            'pretrain',
            epoch,
            running_loss / len(dset_loaders['train'].dataset),
            cer_mean)+'\n')
        torch.save(model.state_dict(), save_path+'/'+args.mode+'_'+str(epoch+1)+'.pt')

I get very confused about this phenomenon. I think if the Loss hasn't changed in the whole epoch, the Loss in the next epoch shouldn't have changed either. Why the Loss still change at the beginning of the next epoch after unchanged in the whole epoch?
Can someone help me solve this problem? Thanks!


title: Why am I getting a Pytorch Runtime Error on Test Set

body:
I have a model that is a binary image classification model with the resnext model.  I keep getting  a run time error when it gets to the test set.  Error message is
RuntimeError: Expected object of backend CPU but got backend CUDA for argument #2 'weight'

I am sending my test set tensors to my GPU like my train model.  I've looked at the following and I'm doing what was suggested here as stated above.

Here is my model code:

resnext = models.resnext50_32x4d(pretrained=True)
resnext = resnext.to(device)
for param in resnext.parameters():
    param.requires_grad = True
resnext.classifier = nn.Sequential(nn.Linear(2048, 1000),
                                 nn.ReLU(),
                                 nn.Dropout(0.4),
                                 nn.Linear(1000, 2),
                                 nn.Softmax(dim = 1))
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(resnext.classifier.parameters(), lr=0.001)
import time
start_time = time.time()

epochs = 1

max_trn_batch = 5
max_tst_batch = 156

y_val_list = []
policy_list = []

train_losses = []
test_losses = []
train_correct = []
test_correct = []

for i in range(epochs):
    for i in tqdm(range(0, max_trn_batch)):
        trn_corr = 0
        tst_corr = 0

        # Run the training batches
        for b, (X_train, y_train, policy) in enumerate(train_loader):
            #print(y_train, policy)
            X_train = X_train.to(device)
            y_train = y_train.to(device)
            if b == max_trn_batch:
                break
            b+=1

            # Apply the model
            y_pred = resnext(X_train)
            loss = criterion(y_pred, y_train)

            # Tally the number of correct predictions
            predicted = torch.max(y_pred.data, 1)[1]
            batch_corr = (predicted == y_train).sum()
            trn_corr += batch_corr
            # Update parameters
            optimizer.zero_grad()
            loss.backward()
            optimizer.step()

            # Print interim results
            if b%1 == 0:
                print(f'epoch: {i:2}  batch: {b:4} [{100*b:6}/63610]  loss: {loss.item():10.8f}  \
    accuracy: {trn_corr.item()/(100*b):7.3f}%')

        train_losses.append(loss)
        train_correct.append(trn_corr)

        # Run the testing batches
        with torch.no_grad():
            for b, (X_test, y_test, policy) in enumerate(test_loader):
                policy_list.append(policy)
                X_test.to(device)
                y_test.to(device)
                if b == max_tst_batch:
                    break

                # Apply the model
                y_val = resnext(X_test)
                y_val_list.append(y_val.data)
                # Tally the number of correct predictions
                predicted = torch.max(y_val.data, 1)[1]
                tst_corr += (predicted == y_test).sum()

        loss = criterion(y_val, y_test)
        test_losses.append(loss)
        test_correct.append(tst_corr)

    print(f'\nDuration: {time.time() - start_time:.0f} seconds') # print the time elapsed

Here is the full traceback:

---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-84-48bce2e8d4fa> in <module>
     60
     61                 # Apply the model
---> 62                 y_val = resnext(X_test)
     63                 y_val_list.append(y_val.data)
     64                 # Tally the number of correct predictions

C:\ProgramData\Anaconda3\lib\site-packages\torch\nn\modules\module.py in __call__(self, *input, **kwargs)
    545             result = self._slow_forward(*input, **kwargs)
    546         else:
--> 547             result = self.forward(*input, **kwargs)
    548         for hook in self._forward_hooks.values():
    549             hook_result = hook(self, input, result)

C:\ProgramData\Anaconda3\lib\site-packages\torchvision\models\resnet.py in forward(self, x)
    194
    195     def forward(self, x):
--> 196         x = self.conv1(x)
    197         x = self.bn1(x)
    198         x = self.relu(x)

C:\ProgramData\Anaconda3\lib\site-packages\torch\nn\modules\module.py in __call__(self, *input, **kwargs)
    545             result = self._slow_forward(*input, **kwargs)
    546         else:
--> 547             result = self.forward(*input, **kwargs)
    548         for hook in self._forward_hooks.values():
    549             hook_result = hook(self, input, result)

C:\ProgramData\Anaconda3\lib\site-packages\torch\nn\modules\conv.py in forward(self, input)
    341
    342     def forward(self, input):
--> 343         return self.conv2d_forward(input, self.weight)
    344
    345 class Conv3d(_ConvNd):

C:\ProgramData\Anaconda3\lib\site-packages\torch\nn\modules\conv.py in conv2d_forward(self, input, weight)
    338                             _pair(0), self.dilation, self.groups)
    339         return F.conv2d(input, weight, self.bias, self.stride,
--> 340                         self.padding, self.dilation, self.groups)
    341
    342     def forward(self, input):

RuntimeError: Expected object of backend CPU but got backend CUDA for argument #2 'weight'

Again, my tensors and the model are sent to the GPU so I'm not sure what is going on. Does anyone see my mistake?


title: Implementing backprop in numpy

body:
I a trying to implement backprop in numpy by defining a function that performs some kind operation given an input, weight matrix and bias, and returns the output with the backward function, which can be used to update weights.

Currently this is my code , however I think there are some bugs in the derivation, as the gradients for the W1 matrix are too large. Here is a pytorch implementation for the same thing as a reference torch.

Any help is appreciated.


title: How to process dataset from loader in the same order at every training

body:
I would like to know how to process the indices that I have in my data loader always at the same order for every training. I want to do this because it takes a lot of time for my code to save visual outputs of the results on validation set, so I decided to save only the N first examples seen on validation.

However, when I pass all my batches in my validation loop:

for t, (x, y, indices) in enumerate(dataset['loader_val'])

  x = x.to(device=device, dtype=dtype)  # move to device, e.g. GPU
  y = y.to(device=device, dtype=dtype)  # move to device, e.g. GPU

  # Obtaining the scores ... -------------------------------------------------------------------
  scores = model(x)

  ......

My values for indices are not always the same at the first iteration for instance. I am sure I have always the same indices for the validation set since I checked that at dataset['loader_val'].sampler.indices is always the same array and ordered in the same way. Is there a way for enumerate() to take always the indices inside dataset['loader_val'] in the same way?


title: Reading vector information instead when trying to read filename to create custom dataset

body:
Using python v 3.7.3, pytorch v 0.4.1, windows 10, running code on Jupyter

I am pretty new to programming and deep learning, and creating a new dataset by combining 2 existing datasets.  I am trying to read the filenames of the images but instead it is returning the vector information.  Here is a snippet of the output (imgname is the path, names should just be the filename eg vinegar_41):

imgname: C:\Users\User\Documents\Dataset\train\vinegar\vinegar_41.png

 names:  [[[128 128 128]
  [128 128 128]
  [128 128 128]
  ...
  [128 128 128]
  [128 128 128]
  [128 128 128]]

 [[128 128 128]
  [128 128 128]
  [128 128 128]
  ...
  [128 128 128]
  [128 128 128]
  [128 128 128]]

Here is my code thus far:

__all__ = ['MyDataset']

class MyDataset(Dataset):

    def __init__(self, root, transform=None, target_transform=None):
        root = train_set
        print(root)
        root_list = os.listdir(root)
        print(root_list)
        for f in root_list:
            print('checkpoint')
            imgs = []
            for img in os.listdir(f):
                #print('\n image: ', img)
                imgname = (root + '\\' + f + '\\' + img)
                print('\n imgname:', imgname)
                open(imgname, 'r')
                if img.endswith('.png'):
                    names = cv2.imread(imgname)
                    imgs.append(names)
                    #print('\n imgs: ', imgs)
                    print('\n names: ', names)

                #Image.close()
        self.imgs = imgs
        self.transform = transform
        self.target_transform = target_transform

    def __getitem__(self, index):
        fn, label = self.imgs[index]
        img = PIL.Image.open(fn).convert('RGB')
        if self.transform is not None:
            img = self.transform(img)
        if self.target_transform is not None:
            label = self.transform(label)
        return img, label

    def __len__(self):
        return len(self.imgs)

I'm in over my head, but my project is due in a couple days, so any advice on how to make this work would be greatly appreciated.


title: CMake cannot find libraries and package when building using Android toolchain

body:
I am working on an algorithm library that provides pre-compiled .so file for an Android application. We want to use PyTorch C++ API in our library so I put some effort on it. However I encountered some problems when trying to build with CMake.

Building works well without using android.cmake.toolchain provided in the NDK, and the compiled program behaves as expected. Things go wrong when I try to use android.cmake.toolchain to build library, the following two lines in TorchConfig.cmake(this file is provided by Torch) went wrong weirdly:

find_package(Caffe2 REQUIRED PATHS ${CMAKE_CURRENT_LIST_DIR}/../Caffe2) [Line 40]
find_library(C10_LIBRARY c10 PATHS "${TORCH_INSTALL_PREFIX}/lib") [Line 47]

Although those commands used PATHS to specify absolute path, CMake insists that those package and library cannot be found. I tried use message to print out those path and check, the files CMaKe should be looking for (Caffe2Config.cmake and libc10.so) do exist in the path specified.

I am aware that android toolchain will change the compilers to be used (to bundled clang), but I cannot see the reason why cmake behaves oddly in my case.


title: Image Classification - pytorch

body:
I am trying to predict features using a pretrained model.
I am getting the output as following, however, how can I use torch.max() to get the classes of interest. Code whhich I tried:

... loading model
input = transformation_sequence(sample).unsqueeze(0)
outputs = model(input)
_, predicted = torch.max(outputs,1) #this line returns error

#print of `outputs` variable
[tensor([[ 3.0654, -3.0650]]), tensor([[ 1.5634, -1.5672]]), tensor([[ 1.2867, -1.2888]]), tensor([[ 1.2974, -1.2928]]), tensor([[ 6.4537, -6.4487]]), tensor([[ 2.4851, -2.4710]]), tensor([[ 0.9855, -0.9809]]), tensor([[ 0.3995, -0.4033]]), tensor([[ 0.6301, -0.6276]]), tensor([[ 5.7082, -5.6931]]), tensor([[ 1.9354, -1.9365]]), tensor([[ 0.6091, -0.6074]]), tensor([[ 5.4509, -5.4417]]), tensor([[ 3.7231, -3.7115]]), tensor([[ 4.4494, -4.4361]]), tensor([[ 0.8867, -0.8902]]), tensor([[ 2.7410, -2.7402]]), tensor([[ 5.4919, -5.4909]]), tensor([[ 2.2687, -2.2744]]), tensor([[-0.9695,  0.9723]]), tensor([[ 1.5100, -1.5114]]), tensor([[-2.7077,  2.7140]]), tensor([[ 4.4661, -4.4734]]), tensor([[ 0.4846, -0.4821]]), tensor([[-2.9743,  2.9643]]), tensor([[ 1.3900, -1.3874]]), tensor([[ 7.6764, -7.6742]]), tensor([[ 0.5173, -0.5118]]), tensor([[ 1.3513, -1.3503]]), tensor([[ 2.5381, -2.5356]]), tensor([[ 4.9850, -5.0074]]), tensor([[-2.8397,  2.8484]]), tensor([[ 3.1010, -3.1137]]), tensor([[-0.2374,  0.2406]]), tensor([[ 0.5338, -0.5358]]), tensor([[ 3.4912, -3.4979]]), tensor([[ 1.1957, -1.1876]]), tensor([[ 1.1189, -1.1163]]), tensor([[ 3.6400, -3.6365]]), tensor([[-1.3123,  1.3132]])]

#list of error:

  _, predicted = torch.max(outputs,1)
TypeError: max() received an invalid combination of arguments - got (list, int), but expected one of:
 * (Tensor input)
 * (Tensor input, Tensor other, Tensor out)
 * (Tensor input, int dim, bool keepdim, tuple of Tensors out)


title: How do I get the image file to be properly labeled when trying to produce topk predictions on an image?

body:
I am trying to produce a graph of the top 5 predictions from a image classifier for any given image.  My files are organized as such:

Test/ --> class_name/ --> image_number
for example,
Test/ --> chocolate/ --> 0015254.jpg

When I use the following code, I can't get the class name to redirect to print, it gives an error that results in every class being outputted (shown below).

class_to_idx:

classes = os.listdir(train_folder)
classes.sort()
#print(classes)
label_mapping = {k: v for v, k in enumerate(classes)}

class_to_idx = {classes[i]: i for i in range(len(classes))}

Here is the code to generate an image and print its top5 predictions:

def sanity_check(path):

  imagepath = test_folder + path
  image = process_image(imagepath)

  plot = imshow(image, ax = plt)
  plot.axis('off')
  plot.title(class_to_idx[str(classes)])
  plot.show()

  axes = predict(imagepath, model)

  yaxis = [class_to_idx[str(i)] for i in np.array(axes[1][0].cpu())]
  y_pos = np.arange(len(yaxis))
  xaxis = np.array(axes[0][0].cpu().numpy())

  plt.barh(y_pos, xaxis)
  plt.xlabel('probability')
  plt.yticks(y_pos, yaxis)
  plt.title('probability of {} classification'.format(name))

  plt.show()

path = '/chocolate_mousse/1379570.jpg'
sanity_check(path)

Here is the error:

KeyError                                  Traceback (most recent call last)
<ipython-input-64-5747e27aee19> in <module>()
      1 path = '/chocolate_mousse/1379570.jpg'
----> 2 sanity_check(path, name = 'Chocolate Mousse')

<ipython-input-62-295700419574> in sanity_check(path, name)
      7   plot = imshow(image, ax = plt)
      8   plot.axis('off')
----> 9   plot.title(class_to_idx[str(classes)])
     10   plot.show()
     11

KeyError: "['apple_pie', 'baby_back_ribs', 'baklava', 'beef_carpaccio', 'beef_tartare', 'beet_salad', 'beignets', .... etc


title: How to set the next n elements of a Numpy / PyTorch array to some value?

body:
If I have a variable side_inputs, with .shape of [48, 161]. I want to set the last n values to some other value.

My other variable, side_input has a .shape of [45, 161].

I want to set the last 45 elements of side_inputs to side_input with:

side_inputs[:45,:] = side_input

But I get an error:

RuntimeError: The expanded size of the tensor (48) must match the existing size (45) at non-singleton dimension 0.  Target sizes: [48, 161].  Tensor sizes: [45, 161]


title: Up-to-date TPU implementation of DCGAN Examples

body:
Where can I find an up-to-date TPU implementation of a DCGAN?

I tried to implement this myself but I keep running into errors so I'm hoping some sample code could help me out.

There are plenty of examples that use Tensorflow 1.0 but I am looking for an example that uses Tensorflow 2.0/Keras because it is easier to implement a TPU in.

If there is a simple way to convert a pytorch DCGAN to run with a TPU that too would be helpful.


title: Pytorch, cant' run CNN on GPU. Input type (torch.FloatTensor) and weight type (torch.cuda.FloatTensor) should be the same

body:
I'm building a simple image recognition convolutional neural network and trying to run this on my GPU but I haven't done something important apparently.

I checked If GPU is available at the beginning and in train, set batches to the device (cuda:0).

import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim

# Checks if GPU is available otherwise uses CPU
if torch.cuda.is_available():
    device = torch.device("cuda:0")
    print("Running on the GPU!")
else:
    device = torch.device("cpu")
    print("Running on the CPU!")

REBUILD_DATA = False

# Data clean up and format
class DogsVsCats():
    IMG_SIZE = 50
    CATS = "PetImages/Cat"
    DOGS = "PetImages/Dog"

    LABELS = {CATS: 0, DOGS: 1}

    training_data = []

    catcount =  0
    dogcount = 0

    def make_training_data(self):
        for label in self.LABELS:
            print(label)
            for f in tqdm(os.listdir(label)):
                try:
                    path = os.path.join(label, f)
                    img = cv2.imread(path, cv2.IMREAD_GRAYSCALE)
                    img = cv2.resize(img, (self.IMG_SIZE, self.IMG_SIZE))

                    self.training_data.append([np.array(img), np.eye(2)[self.LABELS[label]] ])

                    if label == self.CATS:
                        self.catcount += 1
                    elif label == self.DOGS:
                        self.dogcount += 1
                except Exception as e:
                    pass

        np.random.shuffle(self.training_data)
        np.save("training_data.npy", self.training_data)

        print("Cats: ", self.catcount)
        print("Dogs: ", self.dogcount)

if REBUILD_DATA:
    dogsvcats = DogsVsCats()
    dogsvcats.make_training_data()

training_data = np.load("training_data.npy", allow_pickle=True)

# print(len("training_data.npy"))

class Net(nn.Module):
    def __init__(self):
        super().__init__()
        self.conv1 = nn.Conv2d(1, 32, 5)
        self.conv2 = nn.Conv2d(32, 64, 5)
        self.conv3 = nn.Conv2d(64, 128, 5)

        x = torch.randn(50,50).view(-1,1,50,50)

        self._to_linear = None
        self.convs(x)

        self.fc1 = nn.Linear(self._to_linear, 512)
        self.fc2 = nn.Linear(512, 2)

    def convs(self, x):
        x = F.max_pool2d(F.relu(self.conv1(x)), (2,2))
        x = F.max_pool2d(F.relu(self.conv2(x)), (2,2))
        x = F.max_pool2d(F.relu(self.conv3(x)), (2,2))

        print(x[0].shape)

        if self._to_linear is None:
            self._to_linear = x[0].shape[0]*x[0].shape[1]*x[0].shape[2]
        return x

    def forward(self, x):
        x = self.convs(x)
        x = x.view(-1, self._to_linear)
        x = F.relu(self.fc1(x))
        x = self.fc2(x)
        return F.softmax(x, dim = 1)

net = Net().to(device)

optimizer = optim.Adam(net.parameters(), lr = 0.001)
loss_function =  nn.MSELoss()

X = torch.Tensor([i[0] for i in training_data]).view(-1, 50, 50)
X = X/255.0
y = torch.Tensor([i[1] for i in training_data])

VAL_PCT = 0.1
val_size = int(len(X)*VAL_PCT)

print(val_size)

train_X = X[:-val_size]
train_y = y[:-val_size]

test_X = X[-val_size:]
test_y = y[-val_size:]

BATCH_SIZE = 100

EPOCHS = 1

def train(net):
    for epoch in range(EPOCHS):
        for i in tqdm(range(0, len(train_X), BATCH_SIZE)):
            # print(i, i+BATCH_SIZE)
            batch_X = train_X[i:i+BATCH_SIZE].view(-1,1,50,50).to(device)
            batch_y = train_y[i:i+BATCH_SIZE].to(device)

            net.zero_grad()

            outputs = net(batch_X)
            loss = loss_function(outputs, batch_y)
            loss.backward()
            optimizer.step()

        print(loss)

correct = 0
total = 0

with torch.no_grad():
    for i in tqdm(range(len(test_X))):
        real_class = torch.argmax(test_y[i])
        net_out = net(test_X[i].view(-1, 1, 50, 50))[0]
        predicted_class = torch.argmax(net_out)
        if predicted_class == real_class:
            correct += 1
        total += 1

print("Accuracy: ", round(correct/total,3))

train(net)

Sorry if it's too simple of a question. Thank you in advance!


title: tensorflow make_template in PyTorch

body:
I am transferring something from tensorflow to PyTorch. In tensorflow code the make_template method was used. I am wondering what is the synonym for that in PyTorch.

Code example:

  self.var = tf.make_template('var_name', SomeClass(param1, param2))

Thank you!


title: GAN diverges after 3rd warm restart

body:
I've been working with a ClusterGAN, originally based on the one here: https://github.com/zhampel/clusterGAN
While trying to stabilize training on my data I integrated a bunch of current strategies, like making it fully convolutional, including spectral norm, using warm restarts, and adding instance noise. Though I was able to solve problems I was having with mode collapse, and I can get useful output images, it will eventually diverge and I'm wondering what might be going on (see chart below). To be clear, the output of the model is actually quite usable around epoch 69 (in this run), right before the last restart, but I'm still curious what might be wrong, partly for my own understanding, but also to inform future exploration. Is this overfitting, perhaps?
Worth noting, perhaps, is that a larger version of the model—just using more filters—did tend to diverge later in the training, but would still diverge eventually (and didn't necessarily produce clearly "better" images). I'm running on mobile, so I'd like to make the model as small as possible, both to improve inference speed and keep app size down.

UPDATE: Increasing the initial learning rate stabilized training for this number of epochs (though I'm curious whether it would still diverge if I ran it through another warm restart cycle).


title: erros with torch.load when doing distributed training

body:
when i use torch.load to load a dataset, error occurs(i use 4 gpus and one machine for distributed training),
Traceback (most recent call last):
  File "Run.py", line 437, in <module>
    ready_train()
  File "Run.py", line 288, in ready_train
    train_ae(aemodel)
  File "Run.py", line 125, in train_ae
    'nlp_chinese_corpus'])
  File "Run.py", line 51, in read_json
    samples = torch.load(slpath + dir + ".pkl")  # 加载一个数据集下的数据
  File "/home_ex/tianhongtao/SW/anaconda3/envs/Hisense/lib/python3.7/site-packages/torch/serialization.py", line 526, in load
    if _is_zipfile(opened_file):
  File "/home_ex/tianhongtao/SW/anaconda3/envs/Hisense/lib/python3.7/site-packages/torch/serialization.py", line 76, in _is_zipfile
    if ord(magic_byte) != ord(read_byte):
TypeError: ord() expected a character, but string of length 0 found

how would i do?
Hi, Guillem. Thanks for the suggestion, I slightly modified the way of loading data and now the error has not occured. Thanks again.


title: what is correct way to pass class labels to confusion matrix?

body:
I'm trying to plot confusion matrix from prediction, final prediction tensor is concatenation of predictions appended during each validation run, predlist = torch.cat([predlist, predicted.view(-1)]).
but when I plot the class label axes go bad, i'm guessing since i'm appending labels lbllist = torch.cat([lbllist, labels.view(-1)]) as well they show up on plot during each run ?

Is there any way to pass class labels correctly in this case to plot correctly?
my class labels are class:0 class:1 class:2
so it should should look like this:
tensor(0)   1        0          3
tensor(1)   0        0          1
tensor(2)   0        2          1
         tensor(0) tensor(1) tensor(2)

Code:
    with torch.no_grad():
        correct = 0
        total = 0
        for testdata in dataloader:
            images = testdata['t1']['data']).to('cuda:0')
            labels =   testdata['t1']['mylabel'].to('cuda:0')
            print(labels)

            outputs = model(images)

            _, predicted = torch.max(outputs.data, 1)
        # Append batch prediction results
        predlist = torch.cat([predlist, predicted.view(-1)])
        lbllist = torch.cat([lbllist, labels.view(-1)])

conf_mat=confusion_matrix(lbllist.cpu().numpy(), predlist.cpu().numpy())

plt.figure(figsize=(10,10))
plot_confusion_matrix(conf_mat, lbllist.cpu())
plt.show()


title: Python print file name from a Data class

body:
I'm trying to print a filename from a class. The class is defined in here in the below code. The code runs without any issues but it doesn't print the filename, my guess is that i have an issue with calling the class.
I am trying to call the class in another file to print the file name into a CSV during training, and using this code to do this:
filename=CellsDataset(args.data_dir,transform=ToTensor(),return_filenames=True)
print(filename)


title: loss.backward() is giving error due to packed_sequence

body:
I am trying to train an encoder-decoder model for image captioning. Input images are of size 3x224x224, and captions are a list of tensors of variable size, each element(index) of which corresponds to a word in the dictionary. For image feature extraction, I have used pre-trained vgg16 and replace its classifier layer by the Linear layer. The decoder converts input captions into packed_sequence because each caption length is different, then the packed_sequence is passed in the embedding layer and then rnn. The output of rnn is passed through a fully connected layer, transforming the length of output into vocab size. But, I am getting error while back propagation. Kindly, someone help. My guess is, packed_sequence is creating problem, but i am not sure why?
Here are my encoder and decoder classes.
class EncoderCNN(nn.Module):
def __init__(self, hidden_size):
    super(EncoderCNN, self).__init__()
    vgg = torch.hub.load('pytorch/vision:v0.6.0', 'vgg16', pretrained=True)
    vgg.eval()

    num_features = vgg.classifier[0].in_features
    modules = list(vgg.classifier.children())[:-7] # Remove layer
    modules.extend([nn.Linear(in_features=num_features, out_features=hidden_size)])
    vgg.classifier = nn.Sequential(*modules)
    self.vgg = vgg

def forward(self, images):
    features = self.vgg(images)
    return features

class DecoderRNN(nn.Module):
    def __init__(self, embed_size, hidden_size, vocab_size, num_layers=1):
        super(DecoderRNN, self).__init__()

        # define the properties
        self.embed_size = embed_size
        self.hidden_size = hidden_size
        self.vocab_size = vocab_size
        self.encoder    = EncoderCNN(hidden_size)

        # embedding layer
        self.embed = nn.Embedding(num_embeddings=self.vocab_size, embedding_dim=self.embed_size)

        # lstm cell
        self.rnn = nn.RNN(input_size=embed_size, hidden_size=hidden_size, nonlinearity = 'relu',batch_first = True)

        # output fully connected layer
        self.fc_out = nn.Linear(in_features=self.hidden_size, out_features=self.vocab_size)

        # activations
        self.softmax = nn.Softmax(dim=1)

    def forward(self, images, packed_captions):

        features = self.vgg(images)             #[num_images_in_batch,hidden_size]
        print(features.shape)
        batch_size = features.size(0)           # batch size
        features = features.view(1,batch_size,self.hidden_size)

        embed_packed_captions = torch.nn.utils.rnn.PackedSequence(self.embed(packed_captions.data), packed_captions.batch_sizes,packed_captions.sorted_indices, packed_captions.unsorted_indices)

        outputs,h_n = self.rnn(embed_packed_captions, features)

        outputs = self.fc_out(outputs.data)
        print(outputs.data.shape,h_n.shape)
        return outputs

Here is the train_model() function,
def train_model(model, loss_function, optimizer, x_train,y_train, MODEL_PATH):

num_epochs = 2
loss_threshold = 5
batch_size = 8
num_data = len(y_train)

for epoch in range(num_epochs):
    epoch_loss = 0
    acc_val = 0
    j = 1
    for i in range(0,num_data,batch_size):
        batch_x = x_train[i:i+batch_size]
        batch_y = y_train[i:i+batch_size]
        batch_y = pack_sequence(batch_y,False)

        ##### farward propogation
        y_pred = model(batch_x,batch_y)

        ##### LOSS FUNCTION
        loss = loss_function(y_pred.data, batch_y.data)
        epoch_loss += loss
        print(loss)

        ##### BACK PROPAGATION
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()
        if(j%50 == 0):
            print(j,epoch_loss/j)
        j+=1

    avg_loss = (epoch_loss/num_data)

    print(epoch, avg_loss)
    epoch = epoch + 1
print("training done!")

return model

Calling train_model():
decoder = DecoderRNN(embed_size, hidden_size, vocab_size)

# Define the loss function.
loss_function = nn.NLLLoss()

# Define the optimizer.
optimizer = torch.optim.Adam(decoder.parameters(), lr = learning_rate)

train_model(decoder, loss_function, optimizer, x_train, y_train[0], MODEL_PATH)

ERROR:
    <ipython-input-30-c67353f7c26a> in train_model(model, loss_function, optimizer, x_train, y_train, MODEL_PATH)
     27             ##### BACK PROPAGATION
     28             optimizer.zero_grad()
---> 29             loss.backward()
     30             optimizer.step()
     31             if(j%50 == 0):

/usr/local/lib/python3.6/dist-packages/torch/tensor.py in backward(self, gradient, retain_graph, create_graph)
    196                 products. Defaults to ``False``.
    197         """
--> 198         torch.autograd.backward(self, gradient, retain_graph, create_graph)
    199
    200     def register_hook(self, hook):

/usr/local/lib/python3.6/dist-packages/torch/autograd/__init__.py in backward(tensors, grad_tensors, retain_graph, create_graph, grad_variables)
     98     Variable._execution_engine.run_backward(
     99         tensors, grad_tensors, retain_graph, create_graph,
--> 100         allow_unreachable=True)  # allow_unreachable flag
    101
    102

RuntimeError: element 0 of tensors does not require grad and does not have a grad_fn


title: Can the SSH client affect how long it takes to run some code?

body:
I'm running a CNN algorithm on an interpreter which is running on a remote machine. The code is pretty simple and resembles this: https://pytorch.org/tutorials/beginner/blitz/cifar10_tutorial.html
But I added a TensorBoard writer to draw the training and validation loss as well as 3 images per epoch.
I used both PyCharm and MobaXterm to access the remote server (an Amazon AWS EC2 instance). But I've noticed significant performance differences whether I run the code from PyCharm or from MobaXterm with a command line.
The results averaged on 10 iterations.
Pycharm: Each iteration takes 55s.
MobaXterm: Each iteration takes 95s.
I've made sure that both are using the same conda environment and the same Python to run (Python 3.7).
Maybe I'm missing a key comprehension of MobaXterm and working with a remote interpreter?


title: Pytorch in Jetson

body:
I have installed PyTorch in Nvidia TX2. I tried to import pytorch as follows,
import torch

However, I'm getting the following error,

ImportError: /home/nvidia/archiconda3/envs/pyto/bin/...libstdc++.so.6:
version GLIBCXX_3.4.26 not found. (required by
/home/nvidia/archiconda3/envs/pyto/lib/python3.7/site-packages/torch/lib/libtorch_python.so)

What am I missing?


title: Constant loss through epochs

body:
I code this neural network to make a gaussian regression but I don't understand why my loss doesn't change through epochs. I set the learning rate to 1 to see the loss decreases but it does not. I chose to take 2000 poitns to train my Neural network. I watched several algorithms on this website and I don't really understand why my algorithm do not achieve what I expect.
I have already imported all libraries needed.
Thank you for your help

def f(x):
    return x * np.sin(x) # function to predict

m =2000
X_bis = np.zeros((1,m),dtype = float)
X_bis=np.random.random(m)*10

        ## Create my training,validation and test set

X_train = X_bis[0:600]
X_val = X_bis[600:800]
X_test = X_bis[800:]

y_train = f(X_train)
y_val = f(X_val)
y_test = f(X_test)

mean_X_train = np.mean(X_train)
std_X_train = np.std(X_train)

mean_y_train = np.mean(y_train)
std_y_train =np.std(y_train)

class MyDataset(data.Dataset):

    def __init__(self, data_feature, data_target):
        self.data_feature = data_feature
        self.data_target = data_target

    def __len__(self):
        return len(self.data_feature)

    def __getitem__(self, index):
        X_train_normalized = (self.data_feature[index] - mean_X_train) / std_X_train
        y_train_normalized = (self.data_target[index] - mean_y_train) / std_y_train
        return torch.from_numpy(np.array(X_train_normalized,ndmin=1)).float(), torch.from_numpy(np.array(y_train_normalized, ndmin = 1)).float()

training_set  = MyDataset(X_train,y_train)
train_loading = torch.utils.data.DataLoader(training_set, batch_size= 100)

val_set = MyDataset(X_val, y_val)
val_loading = torch.utils.data.DataLoader(val_set, batch_size= 10)

test_set  = MyDataset(X_test,y_test)
test_loading = torch.utils.data.DataLoader(test_set, batch_size= 100)

class Net(nn.Module):
  def __init__(self):
    super(Net, self).__init__()
    self.FC1 = nn.Linear(1,10)
    self.FC2 = nn.Linear(10, 1)
  def forward(self, x):
    x = F.relu(self.FC1(x))
    x = self.FC2(x)
    return x

model = Net()

criterion = nn.MSELoss()
optimizer = torch.optim.SGD(model.parameters(),
lr=1, weight_decay= 0.01, momentum = 0.9)

def train(net, train_loader, optimizer, epoch):
    net.train()
    total_loss=0
    for idx,(data, target) in enumerate(train_loader, 0):
        outputs = net(data)
        loss = criterion(outputs,target)
        total_loss +=loss.cpu().item()
        optimizer.step()
    print('Epoch:', epoch , 'average training loss ', total_loss/ len(train_loader))

def test(net,test_loader):
    net.eval()
    total_loss = 0
    for idx,(data, target) in enumerate(test_loader,0):
        outputs = net(data)
        outputs = outputs * std_X_train + mean_X_train
        target = target * std_y_train + mean_y_train
        loss = criterion(outputs,target)
        total_loss += sqrt(loss.cpu().item())
    print('average testing loss', total_loss/len(test_loader))

for epoch in range(50):
    train(model,train_loading,optimizer,epoch)
    test(model,val_loading)

'''


title: MPI4PY does not stop running

body:
I am working on a python code that uses MPI to send data to processors to do augmentation then return the data. However when I run with, it just keeps running. From reading online, I guess the issue is due to communication not complete. I've tried putting print everywhere but still couldn't figure out where the issue happens. Any help is appreciated thank you!!
mpirun -n 3 python Project_1.py

This is the code(not complete):
# Initialise MPI
comm = MPI.COMM_WORLD
size = comm.Get_size()
rank = comm.Get_rank()
name = MPI.Get_processor_name()
status = MPI.Status()

class Ori_Data(Dataset):
    def __init__(self, data, transform = None):
        self.data = data[0]
        self.label = data[1]
        self.transform = transform

    def __getitem__(self, index):
        img, label = self.data[index], self.label[index]
        img = Image.fromarray(img) # Return an image object
        if self.transform is not None:
            img = self.transform(img)

        return img, label

    def __len__(self):
        return len(self.data)

class DistData(Dataset):
    def __init__(self, data, transform = None):
        self.data = data
        self.transform = transform

    def __getitem__(self, index):
        return self.send_loader(index)

    def __len__(self):
        return len(self.data[0])

    def send_loader(self, index):
        if rank == 0 :
            while True:
                image_data = comm.recv(source=MPI.ANY_SOURCE, tag=MPI.ANY_TAG, status=status)
                source = status.Get_source()

                tag = status.Get_tag()

                if tag == tags.READY:
                    comm.send(index, dest=source, tag=tags.START)

                elif tag == tags.DONE:
                    break
            return image_data

        else :
            print('I am in send loader with rank ', rank)
            while True:
                comm.send(None , dest = 0, tag = tags.READY)
                receive_task = comm.recv(source=0, tag=MPI.ANY_SOURCE, status=status)
                tag = status.Get_tag()
                if tag == tags.START:
                    result = Ori_Data(self.data, self.transform)
                    result = result[receive_task]
                    comm.send(result, dest=0, tag=tags.DONE)

                if tag == tags.EXIT:
                    break

def enum(*sequential, **named):
    enums = dict(zip(sequential, range(len(sequential))), **named)
    return type('Enum', (), enums)

# Define MPI message tags
tags = enum('READY', 'DONE', 'EXIT', 'START')

def main():

    train_aug = transforms.Compose([
            transforms.RandomCrop((32, 32)),
            transforms.RandomHorizontalFlip(),
            transforms.RandomRotation(15),
            transforms.ToTensor(),
            transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
            ])

    test_aug = transforms.Compose([
            transforms.ToTensor(),
            transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
            ])

    # LD.train_loader is cifar10 dataset from another file
    training_set = DistData(LD.train_loader,transform = train_aug)
    trainloader = DataLoader(training_set, batch_size = 2, shuffle = True)

    for i, data in enumerate(trainloader, 0):
        print('----------------------------')
        inputs, labels = data
        print (labels)
        break


title: pytorch: calculating loss function for multi label text classification

body:
I am using pytorch with transformers to use a trained bert model. while I am calculating the loss function. I am getting a error regarding different sizes and it put me off for a while now. Here is the whole Notebook. Here is the error:
ValueError: Target size (torch.Size([32, 14])) must be the same as input size (torch.Size([24576, 14]))

please make sure you scroll through the notebook if you have any question.


title: opencv Code keep crashing and didn't execute afterward

body:
I was working on a project that was running fine but now when I try to setup the whole code again for the client. my opencv code execute and crashes the first time I try to run it.afterwards the code doesn't even execute until I restart my system again.then code execute and crash again then it doesn't even execute. the terminal shows the msg(attached image) and nothing happen afterwards
os: ubuntu 18.04
python version: python3
opencv version 4


title: Loss from linear transformed output and ground truth for training

body:
I have a prediction model in pytorch that takes inputs and generates outputs in a specific coordinate system. In my process I transform the output and ground truth into a different coordinate system (2-dimensional translation and rotation). I can now calculate the loss in both coordinate systems, which have the same values (RMSE and NLL loss).
Does it matter which loss I use for the training to run loss.backward() on?


title: how to I load a pre-batched dataset on pytorch

body:
I have a huge dataset that cannot be stored in memory so I prebatched it several files how do I make my dataset and data loader class such that load one bath at a time.

All the files have the same base name and a unique batch number an
Example file would be called o3_batch_1.hdf5 or o3_batch_2.hdf5 the
Largest batch number is  o3_batch_102.hdf5

here is what I have tried so far:
would it work?
length would be the total length of the data.
batchNum would be the non-unique number at the end of the file.
base is the common name shared by the file.
class Data(Dataset):
# Constructor
def __init__(self, base, batchNum, length):
    name = base + str(batchNum)
    with h5py.File(name, "r") as f:
        puzz = np.array(f.get('puzzle'))
        sol = np.array(f.get('Sol'))
    self.puzz = torch.from_numpy(puzz)
    self.sol = torch.from_numpy(sol)
    self.len = length

# Getter
def __getitem__(self, batchNum, index):
    return self.puzz[index], self.sol[index]

# Get length
def __len__(self):
    return self.len


title: calculate median of overlapped pixels pytorch

body: I am able to perform the mean operation on the overlapped pixels using nn.fold operation. how to perform the median operation without looping


title: Is PyTorch DistributedDataParallel with different GPU speeds syncing weights?

body:
In the following scenario, there are 2 GPUs, each with a significantly different speed: GPU0 is faster than GPU1 by around 40%. I want to train the the model for 100k steps, which ordinarily if the speeds are the same, would be reached in the equivalent of 50k steps.
However, since the GPUs are of different speeds, when GPU0 hits 50K steps, GPU1 reaches only 30K steps. Effectively, the model has been trained for 80k steps.
In practice, will PyTorch's DistributedDataParallel work with GPUs of different speed? Currently, the script [A] runs such that GPU0 proceeds at its original speed without waiting for GPU1, so I was wondering how any syncing would work. I printed the parameters of the model in each GPU at the same step, and they are indeed different significantly. If so, where does the syncing work?
In the original source code [B] for DDP, it does seem that syncing is made before each forward pass of the model. But I don't know why if this is the case, the sum of the parameters in each GPU is off by around 1-2% of the total value.
The function to get the parameters value is simply this:
def get_params_sum(net):
    total = 0
    for param in net.parameters():
        total = total + torch.sum(param)

    return total

Is there a way to automatically let GPU0 take over some "leftover" training of GPU1 when it's done?
[A] A running script can be found here:
https://github.com/yangkky/distributed_tutorial/blob/master/src/mnist-distributed.py
[B] https://github.com/pytorch/pytorch/blob/master/torch/nn/parallel/distributed.py#L707


title: Plotting a graph using pickle file saved on gpu using pytorch

body:
import pickle
import matplotlib.pyplot as plt
plt.figure()
plt.title('train loss')
plt.xlabel('')
plt.ylabel('')
plt.plot(torch.load('./t_accli2.pickle', map_location = 'cpu'))
plt.savefig('./train_acc')
plt.show()

I am trying to plot the above graph. and get the following error.
My model is trained on a GPU and the plotting is also being done on the GPU. Still I receive the following error. I have saved the file using pickle.dump. It is a pickle file of accuracies and not the saved model.
plt.plot(torch.load('./t_accli2.pickle', map_location = 'cpu'))
  File "/home/enterprise.internal.city.ac.uk/adbb083/.local/lib/python3.5/site-packages/torch/serialization.py", line 529, in load
    return _legacy_load(opened_file, map_location, pickle_module, **pickle_load_args)
  File "/home/enterprise.internal.city.ac.uk/adbb083/.local/lib/python3.5/site-packages/torch/serialization.py", line 692, in _legacy_load
    magic_number = pickle_module.load(f, **pickle_load_args)
  File "/home/enterprise.internal.city.ac.uk/adbb083/.local/lib/python3.5/site-packages/torch/storage.py", line 134, in _load_from_bytes
    return torch.load(io.BytesIO(b))
  File "/home/enterprise.internal.city.ac.uk/adbb083/.local/lib/python3.5/site-packages/torch/serialization.py", line 529, in load
    return _legacy_load(opened_file, map_location, pickle_module, **pickle_load_args)
  File "/home/enterprise.internal.city.ac.uk/adbb083/.local/lib/python3.5/site-packages/torch/serialization.py", line 702, in _legacy_load
    result = unpickler.load()
  File "/home/enterprise.internal.city.ac.uk/adbb083/.local/lib/python3.5/site-packages/torch/serialization.py", line 665, in persistent_load
    deserialized_objects[root_key] = restore_location(obj, location)
  File "/home/enterprise.internal.city.ac.uk/adbb083/.local/lib/python3.5/site-packages/torch/serialization.py", line 156, in default_restore_location
    result = fn(storage, location)
  File "/home/enterprise.internal.city.ac.uk/adbb083/.local/lib/python3.5/site-packages/torch/serialization.py", line 132, in _cuda_deserialize
    device = validate_cuda_device(location)
  File "/home/enterprise.internal.city.ac.uk/adbb083/.local/lib/python3.5/site-packages/torch/serialization.py", line 116, in validate_cuda_device
    raise RuntimeError('Attempting to deserialize object on a CUDA '
RuntimeError: Attempting to deserialize object on a CUDA device but torch.cuda.is_available() is False. If you are running on a CPU-only machine, please use torch.load with map_location=torch.device('cpu') to map your storages to the CPU.


title: PyTorch: Adding with index

body:
Given a tensor T of size [n,n,d], an index I of size [m,2], and  a tensor of features F of size [m,d], for each i in [0,m] I want to compute T[I[i],:] += F[i].
How can I implement this efficiently in PyTorch? I thought it could be somehow achieved with scatter_add but did not get very far.


title: Asynchrony loss function over an array of 1D signals

body:
So I have an array of N 1D-signals (e.g. time series) with same number of samples per signal (all in equal resolution) and I want to define a differentiable loss function to penalize asynchrony among them and therefore be zero if all N 1D signals will be equal to each other. I've been searching the literature to find something but haven't had luck yet.
Few remarks:
1 - since N (number of signals) could be quite large I can not afford to calculate Mean squared loss between every single pair  which could grow combinatorialy large. also I'm not quite sure whether it would be optimal in any mathematical sense for the goal to achieve.
There are two naive loss functions that I could think of :
a) Total variation loss for each time sample across all signals (to force to reach ideally zero variation). the problem is here the weight needs to be very large to yield zero varion. masking any other loss term that is going to be added and also there is no inherent order among the N signals, which doesnt make it suitable to TV loss to begin with.
b) minimizing the sum of variance at each time point among all signals. however, choice of the reference of variance (aka mean) could be crucial I believe as just using the sample mean might not really yield the desired result, not quite sure.


title: Extracting ExtraFileMap information form PyTorch jit model in Java

body:
I can run a Pytorch model trained in Python natively in Java using the PyTorch Java Bindings library. This is fantastic, however, I am unable to extract any of the metadata associated with model?
When saving a trained model, extra information can be added in Python or C++ using an ExtraFilesMap object. In this case the extra info is the type of audio data the model was trained on; this is vital for pre-conditioning audio data before it's passed to a model for classification. In Python this extra metadata can be extracted from a saved model via...
extra_files = torch._C.ExtraFilesMap()
model = torch.jit.load(filepath_model, _extra_files=extra_files)
dataOptsStr = extra_files['dataOpts']

In Java a trained model can be loaded using...
Module mod = Module.load(modelPath);

However, there does not appear to be any method to extract the ExtraFileMap information. I've spent a long time trying to find a solution with no luck. Am missing something here? It seems like important functionality for sharing models between different languages etc.?
Full code example code can be found in the BatDL.java here.
If there is indeed no way of doing this using the current Pytorch Java library, does anyone have any suggestions on how else it might be done within Java?
Thanks in advance for any help.


title: How to do sanity test for PyTorchJob in Kubeflow properly?

body:
I need to run sanity test to make sure that a PyTorchJob can be created and run till completion in Kubeflow. The job is supposed to be simple and minimal.
Here is the sample I followed for creating the job: https://github.com/kubeflow/pytorch-operator/tree/master/examples/mnist. With information from other sources, I slightly modified the sample yaml. The job could be successfully created, but somehow the containers always encounter CrashLoopBackOff error. I couldn't investigate the logs since the container didn't really start.
Any suggestions for a proper, working and minimal PyTorchJob sample?
Here is my yaml:
apiVersion: "kubeflow.org/v1"
kind: PyTorchJob
metadata:
  name: pytorch-dist-mnist-gloo
spec:
  pytorchReplicaSpecs:
    Master:
      replicas: 1
      restartPolicy: OnFailure
      template:
        metadata:
          annotations:
            sidecar.istio.io/inject: "false"
        spec:
          containers:
            - name: pytorch
              image: gcr.io/kubeflow-ci/pytorch-dist-mnist-test:v1.0
              imagePullPolicy: Always
              command:
                - "python"
                - "/var/mnist.py"
    Worker:
      replicas: 1
      restartPolicy: OnFailure
      template:
        metadata:
          annotations:
            sidecar.istio.io/inject: "false"
        spec:
          containers:
            - name: pytorch
              image: gcr.io/kubeflow-ci/pytorch-dist-mnist-test:v1.0
              imagePullPolicy: Always
              command:
                - "python"
                - "/var/mnist.py"


title: How can I vectorize the operation where all non zero values in a PyTorch Tensor are copied to the left most position without any gaps?

body:
I have a PyTorch tensor with 0 and non 0 values, and I want to copy the non zero values in a new tensor such that it is as compact as possible. ie its shape is [num_rows,max([number of non0 values in each row])]. What is the a vectorised way to do this without putting a for loop?
The following image should illustrate(black = non zero values):


title: One-hot mask a range of tensors given a label

body:
I am wondering how can I make use of torch.scatter_() or any other built-in masking function for this one-hot masking task-
I have two tensors-
X = [batch, 100], label = [batch], and num_classes = 10
so each label has 10 tensors out of those 100 tensors in 'X'.
For instance X of shape [1x100],
X = ([ 0.0468, -1.7434, -1.0217, -0.0724, -0.5169, -1.7318, -0.1207, -0.8377,
        -0.8055,  0.7438,  0.1139,  1.2162, -1.7950,  1.7416, -1.2031, -1.4833,
        -0.5454,  0.2466, -1.2303, -0.4257,  0.9873, -1.5905, -1.3950,  0.4013,
        -1.0523,  1.4450,  0.6574,  1.5239, -0.3503, -0.1114,  1.8192, -1.7425,
         0.4678,  0.4074,  1.7606, -1.0502,  0.0724,  0.1721,  0.1108,  0.4453,
         0.2278, -1.5352, -0.1232,  1.1052,  0.2496,  1.2898, -0.4167, -0.8211,
         0.2340, -0.3829, -0.1328,  0.1033,  2.8693, -0.8802, -0.0433,  0.5335,
         0.0662,  0.4250,  0.2353, -0.1590,  0.0865,  0.6519, -0.2242,  1.5300,
         1.7021, -0.9451,  0.5845, -0.7309,  0.7124,  0.6544, -1.4426, -0.1859,
        -1.5313, -1.5391, -0.2138, -1.0203,  0.6678,  1.3445, -1.3453,  0.5222,
         0.9510,  0.0969, -0.5437, -0.2727, -0.6090, -2.9624,  0.4578,  0.5257,
        -0.2866,  0.0818, -1.2454,  1.6511,  0.1634,  1.3720, -0.4222,  0.5347,
         0.3586, -0.3506,  2.6866,  0.5084])

label = [3]

I would like to do one-hot masking of "1" to tensors 30-40 and rest all the tensors as "0" on the tensor 'X'.
so for,
label = 1 -> mask (0 to 10) as ‘1’ and rest as ‘0’
label = 2 -> mask (10 to 20) as ‘1’ and rest as ‘0’
… so on.


title: Time is running backwards in pytorch

body:
Executing some pytorch code I get this message by stdout: "Time is running backwards!". What does it mean?
The program is running well, and appearently this message doesn't cause any trouble.


title: Chained numerical comparisons in PyTorch

body:
In PyTorch we can do comparisons between elements in tensors like so:
import torch

a = torch.tensor([[1,2], [2,3], [3,4]])
b = torch.tensor([[3,4], [1,2], [2,3]])

print(a.size())
# torch.Size([3, 2])
print(b.size())
# torch.Size([3, 2])

c = a[:, 0] < b[:, 0]

print(c)
# tensor([ True, False, False])

However, when we try to add a condition, the snippet fails:
c = a[:, 0] < b[:, 1] < b[:, 0]

The expected output is
 tensor([ False, False,  False])

So, for each element in a, compare its first element with the second element of the corresponding item in b, and compare that element with the first element of the same item in b.
Traceback (most recent call last):
File "scratch_12.py", line 9, in
c = a[:, 0] < b[:, 1] < b[:, 0]
RuntimeError: bool value of Tensor with more than one value is ambiguous
Why is that, and how can we solve it?


title: How to properly reshape a 3D tensor to a 2D forward linear layer then reshape new 3D tensor's fibers corresponding to the old 3D

body:
I have a 3D tensor of names that comes out of an LSTM that's of shape (batch size x name length x embedding size)
I've been reshaping it to a 2D to put it through a linear layer, because linear layer requires (batch size, linear dimension size) by using the following
y0 = output.contiguous().view(-1, output.size(-1))

this converts outputs to (batchsize, name length * number of possible characters)
then I put y0 through a linear layer and then reshape it back to a 3D using
y = y0.contiguous().view(output.size(0), -1, y0.size(-1))

But I'm not really sure if the fibers of y are correlated properly with the cells of output and I worry this is messing up my learning, because batch size of 1 is actually generating proper names and any larger batch size is generating nonsense.
So what I mean exactly is
output = (batch size * name length, embed size)
y =  (batch size * name length, number of possible characters)
I need to make sure y[i,j,:] is the linear transformed version of output[i,j,:]


title: Issue running backward function after calculating loss

body:
import torch
import torch.nn as nn
import torch.nn.functional as F
import torchvision
from torchvision import transforms, datasets

#neural network class
class Net(nn.Module):
  #intialize class
  def __init__(self):
    super().__init__()
    #feedforward neural network passes data from input layer to output layer
    #fully connected layer with input shape of 28*28 pixels (flatten image to one row) and output feature is size 64. Linear means flat layer
    self.fc1 = nn.Linear(28*28,64)
    #feed in data from fc1 to fc2
    self.fc2 = nn.Linear(64,64)
    self.fc3 = nn.Linear(64,64)
    #output layer has input 64 and output of size 10 to represent 10 classes in MNIST
    self.fc4 = nn.Linear(64,10)
  #forward pass through the data
  def forward(self, x):
    #relu is activation function and performs operation on input data
    #input and output dimenson of relu are the same
    x = F.relu(self.fc1(x))
    x = F.relu(self.fc2(x))
    x = F.relu(self.fc3(x))
    #softmax function gets probability distribution for each class adding up to 1 for output layer
    x = F.log_softmax(self.fc4(x), dim=1)
    return x

#declare a model
net = Net()
#print(net)
#passing in random data
x = torch.rand(28,28)
#resize to represent input shape (batch size, input x, input y)
x = x.view(-1,28,28)
#print(x)

#optmizer adjusts neural network based on error calculation
import torch.optim as optim
#net.parameters() means all the adjustable parts of the neural network, learning rate is amount of change (we don't want model to swerve based on one train case)
optimizer = optim.Adam(net.parameters(),lr=0.001)

#get datasets using torchvision.datasets transforms are application applied to data (transforms conversion to tensors)
train = torchvision.datasets.MNIST("", train=True, download=True,transform=transforms.Compose([transforms.ToTensor()]))
test = torchvision.datasets.MNIST("", train=False, download=True,transform=transforms.Compose([transforms.ToTensor()]))
#store in data loader, batch size is how many samples is passed through the model at once (in GPU memory), best batch size is between 8-64
#shuffling avoids feeding too much of one kind of image and leads to more generalization
trainset = torch.utils.data.DataLoader(train,batch_size=10,shuffle=True)
testset = torch.utils.data.DataLoader(test,batch_size=10,shuffle=True)

#full pass through data is epoch
EPOCHS = 3
for epoch in range(EPOCHS):
  #data is a batch of data in the training set
  for data in trainset:
    #split into features and labels
    features, labels = data
    #print(features, labels)
    #reset the gradient for next passes to avoid convoluting the results of multiple backpropogations
    net.zero_grad()
    #pass data into network (make sure input shape matches)
    output = net(features.view(-1,28*28))
    #compute error (output,expected)
    loss = F.nll_loss(output,labels)
    print("loss is the ", loss)
    #backpropogate loss through trainiable parameters of model
    loss.backward()
    #adjust neural network
    optimizer.step()

I am using Pytorch on Google colab. The error message says that the gradient isn't there.
I am unsure where the error stems from? I used this tutorial: https://www.youtube.com/watch?v=9j-_dOze4IM&list=PLQVvvaa0QuDdeMyHEYc0gxFpYwHY2Qfdh&index=4
Error message from Google colab
Error message


title: Dice score changes for the same reshaped inputs

body:
I'm calculating the Dice score to evaluate my model for a binary image segmentation problem.
The function I wrote in PyTorch is:
def dice_score_reduced_over_batch(x, y, smooth=1):
    assert x.ndim == y.ndim
    # reduction over all axes except 0 i.e. batch
    axes = tuple([i for i in range(1, x.ndim)])

    intersection = torch.abs((x * y).sum(dim=axes))
    union = torch.abs(x.sum(dim=axes)) + torch.abs(y.sum(dim=axes))
    dice = torch.mean(2. * (intersection + smooth) / (union + smooth), dim=0)
    return dice

The input tensors x and y have the shape [batch_size, nChannel, height, width] where nChannel=1 since ground truth is a 2d binary mask. The standard way to calculate the dice score is to compute it along the batch axis and taking the mean value at the end. I found that the score is affected by the way inputs are flattened.
+-------------------+------------------+--------+
| input tensor      | flattened tensor | dice   |
+-------------------+------------------+--------+
| [64, 1, 128, 128] | -                | 0.2754 |
+-------------------+------------------+--------+
| [64, 1, 128, 128] | [64, 16384]      | 0.2754 |
+-------------------+------------------+--------+
| [64, 1, 128, 128] | [1, 1048576]     | 0.3121 |
+-------------------+------------------+--------+

My best guess was this difference is due to the way values are being averaged but it's not the case. The code must return the exact same answer irrespective of the arrangement/shape of the input data. How this behavior can be explained? What's the best way to avoid it?


title: generating segment labels for a Tensor given a value indicating segment boundaries

body:
Does anyone know of a way to generate a 'segment label' for a Tensor, given a unique value that represents segment boundaries within the Tensor?
For example, given a 1D input tensor where the value 1 represents a segment boundary,
x = torch.Tensor([5, 4, 1, 3, 6, 2])
the resulting segment label Tensor should have the same shape with values representing the two segments:
segment_label = torch.Tensor([1, 1, 1, 2, 2, 2])
Likewise, for a batch of inputs, e.g. batch size = 3,
x = torch.Tensor([
    [5, 4, 1, 3, 6, 2],
    [9, 4, 5, 1, 8, 10],
    [10, 1, 5, 4, 8, 9]
    ])

the resulting segment label Tensor (using 1 as the segment separator) should look something like this:
segment_label = torch.Tensor([
    [1, 1, 1, 2, 2, 2],
    [1, 1, 1, 1, 2, 2],
    [1, 1, 2, 2, 2, 2]
    ])

Context: I'm currently working with Fairseq's Transformer implementation in PyTorch for a seq2seq NLP task. I am looking for a way to incorporate BERT-like segment embeddings in Transformer during the encoder's forward pass, rather than modifying an exisiting dataset used for translation tasks such as language_pair_dataset.
Thanks in advance!


title: How to solve the failure of getting a python file from cpp extension of pytorch using setuptools?

body:
I wanted to try a github project named deformable kernels, and followed the steps described in the README.md file:
conda env create -f environment.yml

cd deformable_kernels/ops/deform_kernel;
pip install -e .;

The structure of deformable_kernel/ops/deform_kernel is showed here:
.
csrc
    filter_sample_depthwise_cuda.cpp
    filter_sample_depthwise_cuda.h
    filter_sample_depthwise_cuda_kernel.cu
    nd_linear_sample_cuda.cpp
    nd_linear_sample_cuda.h
    nd_linear_sample_cuda_kernel.cu
functions
    filter_sample_depthwise.py
    __init__.py
    nd_linear_sample.py
__init__.py
modules
    filter_sample_depthwise.py
    __init__.py
setup.py

And the content of file setup.py is showed here:
from setuptools import setup
from torch.utils.cpp_extension import BuildExtension, CUDAExtension

setup(
    name='filter_sample_depthwise',
    ext_modules=[
        CUDAExtension(
            'filter_sample_depthwise_cuda',
            [
                'csrc/filter_sample_depthwise_cuda.cpp',
                'csrc/filter_sample_depthwise_cuda_kernel.cu',
            ]
        ),
    ],
    cmdclass={'build_ext': BuildExtension}
)

setup(
    name="nd_linear_sample",
    ext_modules=[
        CUDAExtension(
            "nd_linear_sample_cuda",
            [
                "csrc/nd_linear_sample_cuda.cpp",
                "csrc/nd_linear_sample_cuda_kernel.cu",
            ],
        )
    ],
    cmdclass={"build_ext": BuildExtension},
)

When I install this directory using command pip install -e ., it failed and the result is:
Obtaining file:///home/xxx/Downloads/deformable_kernels/deformable_kernels/ops/deform_kernel
ERROR: More than one .egg-info directory found in /tmp/pip-pip-egg-info-pta6z__q

So I tried to separate the 2 setup()s in different setup.py files. It worked but I didn't get a python file. Instead a .so file was generated.
Does anyone know how to solve a problem like this?


title: pytorch MNIST neural network produces several non-zero outputs

body:
I tried to do a neural network that operates on MNIST data set. I was mostly following the pytorch.nn tutorial. As a result, i got a model that learns, but there's something wrong with the process or with the model itself. Instead of one active neuron at the output, i recieve multiple ones.
Here's the model itself:
model = nn.Sequential(
    nn.Linear(784, 64),
    nn.ReLU(),
    nn.Linear(64, 10),
    nn.ReLU(),
)

And here's the training process:
loss_func = nn.CrossEntropyLoss()
opt = optim.SGD(model.parameters(), lr=lr)

for epoch in range(epochs):
    model.train()
    for xbt, ybt in train_dl:
        pred = model(xbt)
        loss = loss_func(pred, ybt)
        opt.zero_grad()
        loss.backward()
        opt.step()

    model.eval()
    # Validation
    if epoch % 10 == 0:
        with torch.no_grad():
            losses, nums = zip(
                *[(loss_func(model(xbv), ybv), len(xbv)) for xbv, ybv in valid_dl]
            )
        val_loss = np.sum(np.multiply(losses, nums)) / np.sum(nums)

        print(epoch, val_loss)

Here's average loss each 10th epoch:
0 0.13384412774592638
10 0.0900113809091039
20 0.09795805384699234
30 0.10341344920364791
40 0.10804545368137551

And thats how result of applying the model to the validation set looks like:
[[ 0.         0.         0.        ... 28.436266   0.         5.001435 ]
 [ 7.3331523 12.666427  31.898096  ...  0.         0.         0.       ]
 [ 0.        18.116354   8.049953  ...  4.330721   0.         0.       ]
 ...
 [ 8.504517   0.         6.302228  ...  0.         0.         0.       ]
 [ 1.7339934  0.         0.        ...  0.         2.1565871  0.       ]
 [45.750134   0.         6.2685804 ...  2.247082   0.         0.       ]]
 Shape: (9984, 10)

I tried changing learning speed, model layers, amount of epochs, but nothing seems to work.


title: Pytorch Installation Issue: The procedure entry point OPENSSL_sk_new_reserve could not be located in the dynamic link libcrypto-1_1 -x64.dll

body:
I am trying in install PyTorch, But I am getting error message.

"{ pytorch The procedure entry point OPENSSL_sk_new_reserve could not be located in the dynamic
link libcrypto-1_1 -x64.dll.}"

I am on Windows 7 with Python 3.7 (Anaconda)
Steps I followed:

Tried creating virtual environment. Virtual environment created and activated (through pip in CMD), but when i am trying pip install PyTorch then I am getting error message.
I tried creating virtual env in Anaconda Navigator. but its giving me the error.
I tried to manually install PyTorch on Anaconda, but this didn't work either. I was unable to install PyTorch on my my Jupyter notebook.


title: How to assign a subset of a tensor but still keep the original tensor?

body:
I want to zero the main diagonal of the tensor exp_scaled_pcs, and by doing so I don't want the original tensor to change but I want to receive the output of the assignment into a new tensor zero_sim_exp_scaled_pcs.
By doing:
zero_sim_exp_scaled_pcs = torch.diagonal(exp_scaled_pcs, 0).zero_()

I noted that I change my original tensor also. How can I keep it?


title: self-defined stacked LSTM has different output than nn.LSTM

body:
I implemented multi layer LSTM, however the result is different from nn.LSTM if init_state is not None. I loaded weights from an LSTM model to both my self defined model and pytorch's nn.LSTM model. I suspect I may do something wrong in forward function. Any help will be greatly appreciated. Many thanks!

class StackedLSTMs(nn.Module):
    def __init__(self, input_sz:int, hidden_sz: int, num_layers: int):
        super().__init__()
        self.num_layers = num_layers
        self.hidden_sz = hidden_sz
        self.LSTMs = nn.ModuleList()
        for layer in range(num_layers):
            if layer == 0:
                self.LSTMs.append(nn.LSTMCell(input_sz, hidden_sz))
                #self.LSTMs.append(NaiveCustomLSTMCell(input_sz, hidden_sz))
            else:
                self.LSTMs.append(nn.LSTMCell(hidden_sz, hidden_sz))
                #self.LSTMs.append(NaiveCustomLSTMCell(hidden_sz, hidden_sz))

    def forward(self, x, h: Optional[Tuple[torch.Tensor, torch.Tensor]] = None):
        print('hidden', h)
        seq_size, bs, _ = x.size()
        outputs = []
        if h is None:
            hn = torch.zeros(self.num_layers, bs, self.hidden_sz)
            cn = torch.zeros(self.num_layers, bs, self.hidden_sz)
        else:
            (hn, cn) = h

        for t in range(seq_size):
            for layer, lstm in enumerate(self.LSTMs):
                if layer == 0:
                    hn[layer, :, :], cn[layer, :, :] = lstm(x[t, :, :], (hn[layer, :, :], cn[layer, :, :]))
                else:
                    hn[layer, :, :], cn[layer, :, :] = lstm(hn[layer-1, :, :], (hn[layer, :, :], cn[layer, :, :]))
            temp = hn[self.num_layers - 1, :, :].detach().clone()
            outputs.append(temp)
        outputs = torch.stack(outputs, dim=0)
        h = (hn, cn)
        #outputs = outputs.transpose(0, 1).contiguous()
        return outputs, h

torch.manual_seed(999)
lstms = nn.LSTM(320, 320, 2)
stackedlstms = StackedLSTMs(320, 320, 2)

stackedlstms.LSTMs[0].weight_ih = oldmodel.prediction.dec_rnn.lstm.weight_ih_l0
stackedlstms.LSTMs[0].weight_hh = oldmodel.prediction.dec_rnn.lstm.weight_hh_l0
stackedlstms.LSTMs[0].bias_ih = oldmodel.prediction.dec_rnn.lstm.bias_ih_l0
stackedlstms.LSTMs[0].bias_hh = oldmodel.prediction.dec_rnn.lstm.bias_hh_l0

stackedlstms.LSTMs[1].weight_ih = oldmodel.prediction.dec_rnn.lstm.weight_ih_l1
stackedlstms.LSTMs[1].weight_hh = oldmodel.prediction.dec_rnn.lstm.weight_hh_l1
stackedlstms.LSTMs[1].bias_ih = oldmodel.prediction.dec_rnn.lstm.bias_ih_l1
stackedlstms.LSTMs[1].bias_hh = oldmodel.prediction.dec_rnn.lstm.bias_ih_l1

lstms.weight_ih_l0 = oldmodel.prediction.dec_rnn.lstm.weight_ih_l0
lstms.weight_hh_l0 = oldmodel.prediction.dec_rnn.lstm.weight_hh_l0
lstms.bias_ih_l0 = oldmodel.prediction.dec_rnn.lstm.bias_ih_l0
lstms.bias_hh_l0 = oldmodel.prediction.dec_rnn.lstm.bias_ih_l0

lstms.weight_ih_l1 = oldmodel.prediction.dec_rnn.lstm.weight_ih_l1
lstms.weight_hh_l1 = oldmodel.prediction.dec_rnn.lstm.weight_hh_l1
lstms.bias_ih_l1 = oldmodel.prediction.dec_rnn.lstm.bias_ih_l1
lstms.bias_hh_l1 = oldmodel.prediction.dec_rnn.lstm.bias_ih_l1

hidden = torch.load('hidden.pt')
newembedt = torch.load('newembed_t.pt')

lstms_res = lstms(newembedt, hidden)
stackedlstms_res = stackedlstms(newembedt, hidden)

print(torch.sum(abs(lstms_res[0]-stackedlstms_res[0])))
print(torch.sum(abs(lstms_res[1][0]-stackedlstms_res[1][0])))


title: Reshaping function in numpy

body:
I am trying to reshape data for image classification purpose. I want to convert shape (32,32,3) to (1,3,32,32). I have used two ways for the reshaping purpose and got different results. The first one is numpy reshape method. The other code is written by me.
def res(t):
    n = np.zeros((3,32,32))

    for j in range(3):
        for k in range(32):
            for l in range(32):
                n[j][k][l]=t[k][l][j]

    n=n.reshape(1,3,32,32)
    return n

I am not able to understand what is the difference between both approaches.


title: How to separate torch dataset items at the collate function phase?

body:
At times, my data set will emit output that contains two examples instead of two, owing to the length of the item.
I am wondering if, when two examples are emitted, that I can split them and count them as two batches instead of one at the collate_fn phase of the data loader, or if I have to go back to how the data was engineered and alter that?
So, for example, data that looks like this might come across:
batch['input_ids'] = [[[101, 2014, 2353, 2201, 1010, 1000, 102],
  [101, 111, 222, 333, 444, 555, 102]]]

When normally the data looks like this:
batch['input_ids'] = [[101, 109, 209, 909, 434, 543, 0]]

I have tried to play with a collate function which works independently, but not when you mix batches of different sizes, such as below:
# check output
for idx, batch in enumerate(test_ds):
    if idx == 0: break
batch['input_ids']

#[[[101, 2014, 2353, 2201, 1010, 1000, 102],
#  [101, 111, 222, 333, 444, 555, 102]]]

out = torch.stack([torch.as_tensor(item) for item in batch['input_ids']]).squeeze(0)
out.shape  # [2, 7]

# collate fn
def collate(batch):
    embeddings = torch.stack([torch.as_tensor(item) for item in batch['input_ids']]).squeeze(0)
    return collate

# create dataloader
test_dl = DataLoader(test_ds,
                     batch_size=2,
                     collate_fn=collate,
                     drop_last=False)

# check output
for idx, batch in enumerate(test_dl):
    if idx == 0: break

# Error: list indices must be integers or slices, not str

Here is dummy code of the phenomenon:
# packages
import torch
from torch.utils.data import DataLoader, Dataset

# prepare torch data set
class torchds(torch.utils.data.Dataset):
    '''
    dummy test set
    '''
    def __init__(self):
        self.inputs = [[[[101, 2014, 2353, 2201, 1010, 1000, 102],
                     [101, 111, 222, 333, 444, 555, 102]]],
                     [[101, 109, 209, 909, 434, 543, 000]]]
        return None

    # get len
    def __len__(self):
        return len(self.inputs)

    # pull a sample of data
    def __getitem__(self, idx):
        if torch.is_tensor(idx):
            idx = idx.tolist()
        return {'input_ids': self.inputs[idx]}

# instantiate ds
test_ds = torchds()

# check output
for idx, batch in enumerate(test_ds):
    if idx == 0: break
batch['input_ids']

out = torch.stack([torch.as_tensor(item) for item in batch['input_ids']]).squeeze(0)
out.shape

# collate fn
def collate(batch):
    embeddings = torch.stack([torch.as_tensor(item) for item in batch['input_ids']]).squeeze(0)
    return collate

# create dataloader
test_dl = DataLoader(test_ds,
                     batch_size=2,
                     collate_fn=collate,
                     drop_last=False)

# check output
for idx, batch in enumerate(test_dl):
    if idx == 0: break
batch


title: Intermediate output should be in [-1, 1] but tanh activation saturates - solution?

body:
I am prediction a set of coordinates on a grid at an intermediate layer of my neural network. This coordinates should be in range [-1, 1]. My idea was to use a tanh activation function to achieve that - but unfortunately, by doing so after a few steps the output is always -1 or 1. I read, that this can be prevented by using a batch normalization before the tanh. This does indeed work, but at a cost - it forces all predicted coordinates towards the centre, which is a huge and a wrong bias, as I am predicting body landmarks, which are not necessarily in the middle of the image.
Does anyone have an idea, what I can do? The way I predict the coordinates is the following:
class Encoder(nn.Module):
    def __init__(self, n_parts, n_features, residual_dim, reconstr_dim, depth_s, depth_a, device, p_dropout=0.2, c=16, dim=64):
        super(Encoder, self).__init__()
        self.preprocessor = PreProcessor(residual_dim, reconstr_dim)
        self.k = n_parts
        self.dropout = nn.Dropout(p_dropout)
        self.tanh = nn.Tanh()
        self.relu = nn.ReLU()
        self.relu = nn.LeakyReLU()
        self.bn1 = nn.BatchNorm2d(c * self.k)
        self.bn2 = nn.BatchNorm2d(2 * n_parts)
        self.hg_shape = Hourglass(depth_s, residual_dim)
        self.to_channels = Conv(residual_dim, c * n_parts, kernel_size=3, stride=1, bn=False, relu=False)
        self.mu = nn.Conv2d(in_channels=c * n_parts, out_channels=2 * n_parts, kernel_size=dim, groups=n_parts)

    def forward(self, img):
        bn = img.shape[0]
        img_preprocessed = self.preprocessor(img) # shape [bn, 256, 64, 64]

        # Get Shape Representation
        img_shape = self.hg_shape(img_preprocessed)
        img_shape = self.dropout(img_shape)
        img_c = self.to_channels(img_shape)
        img_c = self.relu(self.bn1(img_c))
        mu = self.mu(img_c)
        mu = self.tanh(mu).reshape(bn, self.k, 2)

        return mu