title: How to fix "RuntimeError: Function CudnnConvolutionBackward returned an invalid gradient at index 1"

body:
I encountered this problem when I tried to remove some filters and then retrain the network though I have moved both model and data to cuda. It appears that the error is at the backward stage when I called loss.backward(); here is my code:
criterion = nn.NLLLoss()

optimizer = optim.SGD(model.parameters(), lr = 0.01, momentum=0.9,
                          weight_decay=0.0005)

def train(model, train_loader, criterion, optimizer, device, epoch):
    model.train()
    for idx, (data, target) in enumerate(train_loader):
        data, target = data.to(device), target.to(device)
        optimizer.zero_grad()
        output = model(data)
        loss = criterion(output,target)
        loss.backward() #The problem is here
        optimizer.step()
        if idx % 5 == 0:
            print("Epoch {} [{}/{} ({:.2f}%)]\tLoss: {:.6f}"
                  .format(epoch,idx*len(data),len(train_loader.dataset),
                   100.0*idx/len(train_loader),loss.item()))

def test(model,test_loader,criterion,device):
    model.eval()
    test_loss = 0
    correct = 0
    with torch.no_grad():
        for data,target in test_loader:
            data, target = data.to(device), target.to(device)
            output = model(data)
            loss = criterion(output,target)
            test_loss += loss.item()
            pred = output.argmax(dim=1, keepdim=True)
            correct += pred.eq(target.view_as(pred)).sum().item()
        test_loss /= len(test_loader.dataset)
        print('\nTest set: Average loss: {:.4f}, Accuracy: {}/{}
              ({:.0f}%)\n'
              .format(test_loss, correct, len(test_loader.dataset),100.
              * correct / len(test_loader.dataset)))

#This function returns the index of the filter to be pruned
def index_pruned(model, layer):
    l = []
    for i in range(model.features[layer].weight.size(0)):
        l.append(abs(model.features[layer].weight[i,:,:,:]).sum())
    return l.index(min(l))

def prune_filter(model, layer, next_layer, indexes):
    #Making a new layer, with one less filter
    new_conv =
    torch.nn.Conv2d(in_channels=model.features[layer].in_channels,

                    out_channels=model.features[layer].out_channels-1,

                    kernel_size=model.features[layer].kernel_size,
                    stride=model.features[layer].stride,
                    padding=model.features[layer].padding)

    #Pass the old weights to the new layer except in the pruned filter
    new_conv.weight[0:indexes,:,:,:] =
           model.features[layer].weight[0:indexes,:,:,:]
    new_conv.weight[indexes:,:,:,:] =
           model.features[layer].weight[indexes+1,:,:,:]
    new_conv.bias[0:indexes] = model.features[layer].bias[0:indexes]
    new_conv.bias[indexes:] = model.features[layer].bias[indexes+1:]
    #replace the layer
    model.features[layer] = new_conv

    #When not pruning the last layer
    if layer != 10:
        #Reduce the 'thickness' of the consecutive layer
        #as the previous one is pruned
        next_new_conv = torch.nn.Conv2d(
                 in_channels=model.features[next_layer].in_channels-1,

                 out_channels=model.features[next_layer].out_channels,

                 kernel_size=model.features[next_layer].kernel_size,

                 stride=model.features[next_layer].stride,

                 padding=model.features[next_layer].padding)
                 next_new_conv.weight[:,0:indexes,:,:] =
                 model.features[next_layer].weight[:,0:indexes,:,:]
        next_new_conv.weight[:,indexes:,:,:] =
            model.features[next_layer].weight[:,indexes+1:,:,:]
        model.features[next_layer] = next_new_conv

    #Pruning the last layer affects the linear layer
    elif layer == 10:
        params = int(model.classifier[0].in_features /
                        (model.features[10].out_channels+1))
        new_fc1 = torch.nn.Linear(
             in_features=int(model.classifier[0].in_features-params),

             out_features=int(model.classifier[0].out_features))
        new_fc1.weight[:,0:indexes*params] =
                 model.classifier[0].weight[:,0:indexes*params]
        new_fc1.weight[:,:params*indexes] =
                 model.classifier[0].weight[:,:(indexes+1)*params]
        new_fc1.bias = model.classifier[0].bias
        model.classifier[0]=new_fc1
    return model

def main(model, train_loader,test_loader,criterion,optimizer,
         pretrained=False,prune=False,save=False,pruneFilter=False):
    device = 'cuda'
    if pretrained == True:
        model.load_state_dict(torch.load('AlexNet_pruned.pt'))
        for params in model.parameters():
            params.requires_grad = True
    model.to(device)
    if pruneFilter == True:
        #conv0:
        for num_filters_pruned in range(16):
            model=prune_filter(model=model, layer=0, next_layer=3, indexes=index_pruned(model,0))
            if num_filters_pruned %4 == 0:
                model=model.cuda()
                train(model, train_loader, criterion, optimizer, device, 1)
                test(model,test_loader,criterion,device)

        torch.save(model.state_dict(),'AlexNet_filers_pruned.pt')

    if save == True:
        torch.save(model.state_dict(),'AlexNet.pt')
    if prune == True:
        torch.save(model.state_dict(),'AlexNet_pruned.pt')

And the error is at loss.backward(). I tried to check if every param is in cuda
conv0:
for num_filters_pruned in range(16):
    model=prune_filter(model=model, layer=0, next_layer=3, indexes=index_pruned(model,0))
    if num_filters_pruned %4 == 0:
        model=model.cuda()
        for name, param in model.named_parameters():
            print(name,':',param.device)
        train(model, train_loader, criterion, optimizer, device, 1)
        test(model,test_loader,criterion,device)

And this is the result

features.0.weight : cuda:0
features.0.bias : cuda:0
features.3.weight : cuda:0
features.3.bias : cuda:0
features.6.weight : cuda:0
features.6.bias : cuda:0
features.8.weight : cuda:0
features.8.bias : cuda:0
features.10.weight : cuda:0
features.10.bias : cuda:0
classifier.fc1.weight : cuda:0
classifier.fc1.bias : cuda:0
classifier.fc2.weight : cuda:0
classifier.fc2.bias : cuda:0
--------------------------------------------------------------------------- RuntimeError                              Traceback (most recent call last) <ipython-input-12-46a65917686e> in <module>
----> 1 main(model,train_loader,test_loader,criterion,optimizer,pretrained=True,prune=False,save=False,pruneFilter=True)

<ipython-input-11-8c11e06a650b> in main(model, train_loader, test_loader, criterion, optimizer, pretrained, prune, save, pruneFilter)
     33                     print(name,':',param.device)
     34
---> 35                 train(model, train_loader, criterion, optimizer, device, 1)
     36                 test(model,test_loader,criterion,device)
     37         #conv2:

<ipython-input-8-f4fd4c83eff2> in train(model, train_loader, criterion, optimizer, device, epoch)
      9         loss = criterion(output,target)
     10         loss = loss.cuda()
---> 11         loss.backward()
     12         optimizer.step()
     13         if idx % 5 == 0:

/usr/local/lib/python3.5/dist-packages/torch/tensor.py in backward(self, gradient, retain_graph, create_graph)
    100                 products. Defaults to ``False``.
    101         """
--> 102         torch.autograd.backward(self, gradient, retain_graph, create_graph)
    103
    104     def register_hook(self, hook):

/usr/local/lib/python3.5/dist-packages/torch/autograd/__init__.py in backward(tensors, grad_tensors, retain_graph, create_graph, grad_variables)
     88     Variable._execution_engine.run_backward(
     89         tensors, grad_tensors, retain_graph, create_graph,
---> 90         allow_unreachable=True)  # allow_unreachable flag
     91
     92

RuntimeError: Function CudnnConvolutionBackward returned an invalid gradient at index 1 - expected type torch.FloatTensor but got torch.cuda.FloatTensor


title: pytorch simple custom recurrent layer extremely slow

body:
I implemented a very simple custom recurrent layer in pytorch using PackedSequence. The layer slows down my network in the order of x20. I read about slow down on custom layers without using JIT, but in the order of x1.7, which is something I could live with.

I am simply indexing the packed sequences per sequence and performing a recursion.
I have the suspicion some of the code is not executed on the GPU?

I'm also grateful for any other tips how to implement this type of RNN (essentially not having a dense layer, without any mixing between features).

import torch
import torch.nn as nn
from torch.nn.utils.rnn import PackedSequence

def getPackedSequenceIndices(batch_sizes):
    """input: batch_sizes from PackedSequence object
    requires length-sorted sequences!
    """
    nBatches = batch_sizes[0]
    seqIdx = []
    for ii in range(nBatches):
        seqLen = torch.sum((batch_sizes - ii) > 0).item()
        idx = torch.LongTensor(seqLen)
        idx[0] = ii
        idx[1:] = batch_sizes[0:seqLen-1]
        seqIdx.append( torch.cumsum(idx, dim=0) )
    return seqIdx

class LinearRecursionLayer(nn.Module):
    """Linear recursive smoothing layer with trainable smoothing constants."""

    def __init__(self, feat_dim, alpha_smooth=0.5):
        super(LinearRecursionLayer, self).__init__()
        self.feat_dim = feat_dim
        # trainable parameters
        self.alpha_smooth = nn.Parameter(alpha_smooth*torch.ones(self.feat_dim))
        self.wx = nn.Parameter(torch.ones(self.feat_dim))
        self.activ = nn.Tanh

    def forward(self, x):
        if isinstance(x, PackedSequence):
            seqIdx = getPackedSequenceIndices(x.batch_sizes)
            ydata = torch.zeros_like(x.data)
            for idx in seqIdx:
                y_frame = x.data[idx[0]] # init with first frame
                # iterate over sequence
                for nn in idx:
                    x_frame = x.data[nn]
                    y_frame = self.alpha_smooth*y_frame + (1-self.alpha_smooth)*x_frame # smoothing recurrence
                    ydata[nn,:] = self.activ(self.wx*(y_frame))
            y = PackedSequence(ydata, x.batch_sizes) # pack

        else:  # tensor
            raise ValueError('not implemented')
        return y


title: How to save custom embedding matrix to .txt file format?

body:
I have made a dictionary which contains word and its corresponding word vector in the following format:

{'word1': array([ 4.530e-02, -1.170e-02, -1.201e-01,  2.439e-01,  4.670e-02d], type=float32),
'word2': array([ 4.530e-02, -1.170e-02, -1.201e-01,  2.439e-01,  4.670e-02d], type=float32)}

I would like to save this dictionary to custom_embeddings.txt file in the following format:

The format of your custom_embeddings.txt file needs to be the token followed by the values of each of the dimensions for the embedding, all separated by a single space, e.g. here's two tokens with 5 dimensional embeddings:

word1 4.530e-02 -1.170e-02 -1.201e-01  2.439e-01  4.670e-02d
word2 4.530e-02 -1.170e-02 -1.201e-01  2.439e-01  4.670e-02d

It will be really helpful if you could tell me how to achieve this result?

Thanks in advance


title: How can I improve my PyTorch implementation of ResNet for CIFAR-10 classification?

body:
I have recently started to study the neural networks. I've got good results on MNIST with MLP and decided to write a classifier for CIFAR-10 dataset using CNN. I've chosen ResNet architecture to implement and tried to follow the wellknown article "Deep Residual Learning for Image Recognition": it is here .
But the accuracy I get with my implementation is about 84% - 85% with no augmentation for test data and about 88% with augmentation for test data which is absolutely far away from the results shown in the article - about 91% - 92% with no augmentation.

Here is my implementation of the ResNet:

class Block_of_net(nn.Module):
  def __init__(self, in_channels, out_channels, kernel_size, stride, padding):
    super(Block_of_net, self).__init__()
    self.conv1 = nn.Conv2d(in_channels, out_channels, kernel_size, stride, padding)
    self.bn = nn.BatchNorm2d(out_channels)
    self.conv2 = nn.Conv2d(out_channels, out_channels, kernel_size, 1, padding)

    for m in self.modules():
        if isinstance(m, nn.Conv2d):
            nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')

  def forward(self, x):
    x = F.relu(self.bn(self.conv1(x)))
    x = self.conv2(x)
    return x

# 3 blocks -- 6 convs -- of one map size with shortcuts after each block
# shorcut before non-linearity
class ResBlock(nn.Module):
  def __init__(self, in_channels, out_channels):
    super(ResBlock, self).__init__()
    self.in_ch = in_channels
    self.out_ch = out_channels
    self.proj = nn.Conv2d(in_channels, out_channels, 1, 2)
    self.bn_proj = nn.BatchNorm2d(out_channels)
    if (in_channels == out_channels):
      self.bl1 = Block_of_net(in_channels, out_channels, 3, 1, 1)
    else:
      self.bl1 = Block_of_net(in_channels, out_channels, 3, 2, 1)
    self.bn1 = nn.BatchNorm2d(out_channels)
    self.bl2 = Block_of_net(out_channels, out_channels, 3, 1, 1)
    self.bn2 = nn.BatchNorm2d(out_channels)
    self.bl3 = Block_of_net(out_channels, out_channels, 3, 1, 1)
    self.bn3 = nn.BatchNorm2d(out_channels)

  def forward(self, x):
    out = x

    if (self.in_ch == self.out_ch):
      shortcut1 = out.clone()
    else:
      shortcut1 = self.proj(out)
    out = self.bl1(out)
    out += shortcut1
    out = self.bn1(out)
    out = F.relu(out)

    shortcut2 = out.clone()
    out = self.bl2(out)
    out += shortcut2
    out = self.bn2(out)
    out = F.relu(out)

    shortcut3 = out.clone()
    out = self.bl3(out)
    out += shortcut3
    out = self.bn3(out)

    return out

class ResNet(nn.Module):
  def __init__(self):
    super(ResNet, self).__init__()
    self.conv1 = nn.Conv2d(3, 16, 3, 1, 1)
    self.bn0 = nn.BatchNorm2d(16)
    self.res_block1 = ResBlock(16, 16)
    self.res_block2 = ResBlock(16, 32)
    self.res_block3 = ResBlock(32, 64)
    self.agP = nn.AvgPool2d(8)
    self.fc = nn.Linear(64, 10)

  def forward(self, x):
    out = F.relu(self.bn0(self.conv1(x)))

    out = F.relu(self.res_block1(out))
    out = F.relu(self.res_block2(out))
    out = F.relu(self.res_block3(out))

    out = self.agP(out)
    out = out.view(-1, 64)
    out = self.fc(out)

    return out

I use Adam as the optimizer with learning_rate = 0.0001, batch_size = 128, weight_decay = 0.0001 and CrossEntropyLoss.
The net learns for 300 epochs.

Here are the learning code and graphics of train_loss and val_acc:

epoch_loss = []
acc = []
train_time = []

for i in range(epochs):
  start_time = time.time()
  ep = 0
  net.train()
  for X_b, y_b in train_loader:
    optimizer.zero_grad()
    X_b = X_b.to(device)
    y_b = y_b.to(device)

    net_out = net(X_b)

    loss = loss_fn(net_out, y_b)
    loss.backward()
    optimizer.step()
    with torch.no_grad():
        ep += loss.item()

  epoch_loss.append(round(ep, 3))

  correct = 0
  total = 0
  net.eval()
  with torch.no_grad():
    for X_b, y_b in test_loader2:
        X_b = X_b.to(device)
        y_b = y_b.to(device)

        net_out = net(X_b)
        loss = loss_fn(net_out, y_b)
        probs = F.softmax(net_out, dim=1)
        label = torch.argmax(probs, dim=1)
        correct += torch.sum(label==y_b).item()
        total = y_b.shape[0]
        break
    acc.append(round(correct/total,3))
  print("Epoch {0}: train_loss=={1} -- val_acc=={2}".format(i+1, ep, acc[-1]))

Curves of training loss and validation accuracy

For training I use simple augmentation: : 4 pixels are padded on each side,
and a 32×32 crop is randomly sampled from the padded image or its horizontal flip, also I apply randomly one of RandomRotation or RandomVerticalFlip.

So I need advice on how I can improve my implementation and where are the problems.


title: Optuna Pytorch: returned value from the objective function cannot be cast to float

body:
def autotune(trial):

      cfg= { 'device' : "cuda" if torch.cuda.is_available() else "cpu",
         #   'train_batch_size' : 64,
         #   'test_batch_size' : 1000,
         #   'n_epochs' : 1,
         #   'seed' : 0,
         #   'log_interval' : 100,
         #   'save_model' : False,
         #   'dropout_rate' : trial.suggest_uniform('dropout_rate',0,1.0),
            'lr' : trial.suggest_loguniform('lr', 1e-3, 1e-2),
            'momentum' : trial.suggest_uniform('momentum', 0.4, 0.99),
            'optimizer': trial.suggest_categorical('optimizer',[torch.optim.Adam,torch.optim.SGD, torch.optim.RMSprop, torch.optim.$
            'activation': F.tanh}
      optimizer = cfg['optimizer'](model.parameters(), lr=cfg['lr'])
      #optimizer = torch.optim.Adam(model.parameters(),lr=0.001

As u can see above , I am trying to run Optuna trials to search for the most optimal hyper parameters for my CNN model.
# Train the model
# use small epoch for large dataset
# An epoch is 1 run through all the training data
# losses = [] # use this array for plotting losses
      for _ in range(epochs):
    # using data_loader
         for i, (data, labels) in enumerate(trainloader):
        # Forward and get a prediction
        # x is the training data which is X_train
            if name.lower() == "rnn":
                model.hidden = (torch.zeros(1,1,model.hidden_sz),
                    torch.zeros(1,1,model.hidden_sz))

            y_pred = model.forward(data)

        # compute loss/error by comparing predicted out vs acutal labels
            loss = criterion(y_pred, labels)
        #losses.append(loss)

            if i%10==0:  # print out loss at every 10 epoch
                 print(f'epoch {i} and loss is: {loss}')

        #Backpropagation
            optimizer.zero_grad()
            loss.backward()
            optimizer.step()

study = optuna.create_study(sampler=optuna.samplers.TPESampler(), direction='minimize',pruner=optuna.pruners.SuccessiveHalvingPrune$
study.optimize(autotune, n_trials=1)

BUT , when I run the above code to tune and find out my most optimal parameters , the follow error occured, seems like the trial has failed even though I still get epoch losses and values. Please advise thanks!
[W 2020-11-11 13:59:48,000] Trial 0 failed, because the returned value from the objective function cannot be cast to float. Returned value is: None
Traceback (most recent call last):
  File "autotune2", line 481, in <module>
    n_instances, n_features, scores = run_analysis()
  File "autotune2", line 350, in run_analysis
    print(study.best_params)
  File "/home/shar/anaconda3/lib/python3.7/site-packages/optuna/study.py", line 67, in best_params
    return self.best_trial.params
  File "/home/shar/anaconda3/lib/python3.7/site-packages/optuna/study.py", line 92, in best_trial
    return copy.deepcopy(self._storage.get_best_trial(self._study_id))
  File "/home/shar/anaconda3/lib/python3.7/site-packages/optuna/storages/_in_memory.py", line 287, in get_best_trial
    raise ValueError("No trials are completed yet.")
ValueError: No trials are completed yet.


title: PyTorch - modifications of autograd variables

body:
In my PyTorch program, I have a matrix which is continuously updated over runtime.

I wonder how to perform this update. I tried using something like this:

matrix[0, index] = hidden[0]

Both matrix and hidden are autograd Variables. When using the example above I'm getting the following error message:

RuntimeError: one of the variables needed for gradient computation has been modified by an inplace operation

I wonder how to get around this and perform the update without using in-place operations.

Probably slicing the matrix and creating a new matrix with torch.cat would work, but this doesn't seem to be a very nice solution.

Is there a better way doing this?

Thanks in advance!


title: Pytorch loading images and ground truth images for for training neural network

body:
I am trying to use custom image data for training and testing a pytorch neural network. I have a folder of images as data and another folder images as ground truth. Files and ground truth images have same name but they are in a separate folder. I want to use them as image and ground truth as pair, for example, 1.jpg feed to network and result of htis picture should compare with other 1.jpg(its ground truth). I want to my neural network output to be image not classification(like image to image translation or pix2pix).

I have tried pytorch dataloader and some other docs but they was not what I needed. Some docs I tried:

https://www.kaggle.com/leighplt/pytorch-tutorial-dataset-data-preparetion-stage

https://towardsdatascience.com/building-efficient-custom-datasets-in-pytorch-2563b946fd9f

https://pytorch.org/tutorials/beginner/data_loading_tutorial.html


title: How to make Pytorch nn module work with celery?

body:
Working with someone else's code here. I got a neural network model from him saved in the form nn-classify-cpu_0.pkl

I am trying to deploy the model via celery. To that end, I am defining a custom class in my main as

class NN(nn.Module):##Neural network
    def __init__(self):
        super(NN, self).__init__()
        n_layers = len(hidden_layer_sizes)
        .....

Then loading the model as

torch.load(f='nn-classify-cpu_0.pkl')

The problem is, when I want to start a celery server using the above NN class, it throws an error saying

Can't get attribute 'NN' on <module 'celery.bin.celery' from '/home/modeladmin/.local/lib/python3.6/site-packages/celery/bin/celery.py'>

Why is the model loading perfectly when I am running a standalone function, but failing when I apply the celery decorator? Is it something like I have to make the class NN visible to the celery worker in some special way?

I am using celery 4.3.0 on python 3.6


Questions and Help
I'm trying to finetune a transformer model for question answering, and am using the BCEWithLogitsLoss function.
However when I try calculating loss i get this error:

RuntimeError: _th_exp_out not supported on CUDAType for Long

I'm inputting matrices with the shape and dtype of [16, 2] and Long
Bud sadly I've no idea to solve this. I tried using another dtype (int32, float32, double), which didn't work.
Here is the code:
def loss_fn(self, preds, labels):
    return torch.nn.BCEWithLogitsLoss()(preds, labels)

def train_fn(self, dataloader, model, optimizer, device):
    # Some other stuff here
    pred = model(
            token_ids = token_ids,
            attention_mask = attention_mask,
            token_type_ids = token_type_ids)
    start_scores = torch.argmax(pred[0], dim=1)
    end_scores = torch.argmax(pred[1], dim=1)

    pred = torch.tensor(list(zip(start_scores, end_scores)))
    pred = pred.to(device, dtype=torch.long)

    batch_loss = self.loss_fn(pred, label)


title: What is the numpy equivalent of expand in pytorch?

body:
Suppose I have a numpy array x of shape [1,5]. I want to expand it along axis 0 such that the resulting array y has shape [10,5] and y[i:i+1,:] is equal to x for each i.
If x were a pytorch tensor I could simply do
y = x.expand(10,-1)

But there is no expand in numpy and the ones that look like it (expand_dims and repeat) don't seem to behave like it.

Example:
>>> import torch
>>> x = torch.randn(1,5)
>>> print(x)
tensor([[ 1.3306,  0.0627,  0.5585, -1.3128, -1.4724]])
>>> print(x.expand(10,-1))
tensor([[ 1.3306,  0.0627,  0.5585, -1.3128, -1.4724],
        [ 1.3306,  0.0627,  0.5585, -1.3128, -1.4724],
        [ 1.3306,  0.0627,  0.5585, -1.3128, -1.4724],
        [ 1.3306,  0.0627,  0.5585, -1.3128, -1.4724],
        [ 1.3306,  0.0627,  0.5585, -1.3128, -1.4724],
        [ 1.3306,  0.0627,  0.5585, -1.3128, -1.4724],
        [ 1.3306,  0.0627,  0.5585, -1.3128, -1.4724],
        [ 1.3306,  0.0627,  0.5585, -1.3128, -1.4724],
        [ 1.3306,  0.0627,  0.5585, -1.3128, -1.4724],
        [ 1.3306,  0.0627,  0.5585, -1.3128, -1.4724]])


title: How to read a .csv file for "training category" with pytorch if all images are in one folder?

body:
In this tutorial, they use the following code to read the datasets:

data_dir = 'data/hymenoptera_data'
image_datasets = {x: datasets.ImageFolder(os.path.join(data_dir, x),
                                          data_transforms[x])
                  for x in ['train', 'val']}
dataloaders = {x: torch.utils.data.DataLoader(image_datasets[x],
                                              batch_size=4,
                                              shuffle=True,
                                              num_workers=4)
               for x in ['train', 'val']}

In that code, the png dataset is separated in different folders.

The name of each folder means the category of the png file in the folder.

I am in a situation where:

all png files are in a single folder;
the category of each png file is recorded in a train.csv file.

How can I implement the code with the above function datasets.ImageFolder?


title: PyTorch, PPO with LSTM doesn't train with episode trajectory

body:
I'm studying reinforcement learning.
this code causes some problem.
When it runs train() function, its policy and value function outputs are all Nan.
Here's my code which run an simple environment, Cartpole.

It's import setting.

import gym
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torch.distributions import Categorical

It's hyper parameter.

learning_rate = 0.0005
gamma = 0.98
lmbda = 0.95
K = 3
epsilon = 0.1

It's agent code which is used PPO and LSTM.

class PPO_lstm(nn.Module):
  def __init__(self):
    super(PPO_lstm, self).__init__()
    self.log_list = []

    self.fc1 = nn.Linear(4, 32) # state encoder, value and policy function use it
    self.lstm = nn.LSTM(32, 64) # LSTM, value and policy function use it
    self.fc2_pi = nn.Linear(64, 2) # policy function
    self.fc2_v = nn.Linear(64, 1) # value function
    self.optimizer = optim.Adam(self.parameters(), lr=learning_rate)

  def pi(self, x, hidden_state):  # get policy
    x = F.relu(self.fc1(x)) # x size : [32] or [batch, 32]
    x = x.view(1, -1, 32) # x size : [1, batch, 32]
    x, new_hidden = self.lstm(x, hidden_state) # x size : [1, batch, 64]
    x = x.squeeze(0) # x size : [batch, 64]
    pi = F.log_softmax(self.fc2_pi(x), dim=1)
    return pi, new_hidden # pi size : [batch, 2], new_hidden size : [1, batch, 64]

  def v(self, x, hidden_state):  # get state value
    x = F.relu(self.fc1(x)) # x size : [32] or [batch, 32]
    x = x.view(1, -1, 32) # x size : [1, batch, 32]
    x, new_hidden = self.lstm(x, hidden_state) # x size : [1, batch, 64]
    x = x.squeeze(0) # x size : [batch, 64]
    v = self.fc2_v(x) # v size : [batch, 1]
    return v # v size : [batch, 1]

  def add_log(self, log):  # add transition which used train function
    self.log_list.append(log) # s, a, r, s', action_prob, done, hidden(h, c), hidden_prime(h, c)

  def make_batch(self): # make batch file using log_list for training
    s_lst, a_lst, r_lst, s_prime_lst, prob_a_lst, done_lst, h_h_lst, h_c_lst, h_prime_h_lst, h_prime_c_lst = [], [], [], [], [], [], [], [], [], []
    for i in self.log_list:
      s, a, r, s_prime, done, action_prob, h, h_prime = i

      s_lst.append(s) # s_lst size : [batch, 4]
      a_lst.append([a]) # a_lst size : [batch, 1]
      r_lst.append([r]) # r_lst size : [batch, 1]
      s_prime_lst.append(s_prime) # s_prime_lst size : [batch, 4]
      prob_a_lst.append([action_prob]) # prob_a_lst size : [batch, 1]
      done_lst.append([0 if done else 1]) # done_lst size : [batch, 1]
      h_h, h_c = h # hidden_h, hidden_c size : [1, 1, 64]
      h_h_lst.append(h_h.detach().numpy()) # hidden1_c_lst size : [batch] inside : [1, 1, 64]
      h_c_lst.append(h_c.detach().numpy()) # hidden1_h_lst size : [batch] inside : [1, 1, 64]
      h_prime_h, h_prime_c = h_prime
      h_prime_h_lst.append(h_prime_h.detach().numpy()) # hidden2_c_lst size : [batch] inside : [1, 1, 64]
      h_prime_c_lst.append(h_prime_c.detach().numpy()) # hidden2_h_lst size : [batch] inside : [1, 1, 64]

    s_lst = torch.tensor(s_lst, dtype=torch.float)
    a_lst = torch.tensor(a_lst)
    r_lst = torch.tensor(r_lst)
    s_prime_lst = torch.tensor(s_prime_lst, dtype=torch.float)
    prob_a_lst = torch.tensor(prob_a_lst, dtype=torch.float)
    done_lst = torch.tensor(done_lst)
    h_h_lst = torch.tensor(h_h_lst, dtype=torch.float).squeeze().unsqueeze(0)
    h_c_lst = torch.tensor(h_c_lst, dtype=torch.float).squeeze().unsqueeze(0)
    h_prime_h_lst = torch.tensor(h_prime_h_lst, dtype=torch.float).squeeze().unsqueeze(0)
    h_prime_c_lst = torch.tensor(h_prime_c_lst, dtype=torch.float).squeeze().unsqueeze(0) # size : [1, batch, 64]

    self.log_list = []
    return s_lst, a_lst, r_lst, s_prime_lst, prob_a_lst, done_lst, h_h_lst, h_c_lst, h_prime_h_lst, h_prime_c_lst

it's train function which used batch (s, a, r, s', done, hidden, hidden')

  def train(self):
    s, a, r, s_prime, prob_a, done_mask, h_h, h_c, h_prime_h, h_prime_c = self.make_batch()

    h = (h_h.detach(), h_c.detach()) # h size : ([1, batch, 64], [1, batch, 64])
    h_prime = (h_prime_h.detach(), h_prime_c.detach()) # h_prime size : ([1, batch, 64], [1, batch, 64])

    for k in range(K): # PPO Algo. K iteration
      td_target = r + gamma * self.v(s_prime, h_prime) * done_mask # td_target size : [batch, 1]
      delta = td_target - self.v(s, h) # 기존 Advantage : Q(s, a) - V(s) == delta : r + gamma * V(s') - V(s)
      delta = delta.detach().numpy()

      advantage_lst = [] # GAE
      advantage = 0.0
      for delta_t in delta[::-1]:
        advantage = gamma * lmbda * advantage + delta_t[0]
        advantage_lst.append([advantage])
      advantage_lst.reverse() # advantage_lst size : [batch, 1]
      advantage = torch.tensor(advantage_lst, dtype=torch.float)

      curr_pi, _ = self.pi(s, h) # curr_pi size : [batch, 2]
      curr_pi_a = curr_pi.gather(1, a) # curr_pi_a size : [batch, 2].gather(1, [batch, 1]) = [batch, 1]
      past_pi = prob_a # past_pi size : [batch, 1]
      ratio = torch.exp(torch.log(curr_pi_a) - torch.log(past_pi))  # a/b == exp(log(a)-log(b))
      # ratio size : [batch, 1]

      surr_1 = ratio * advantage
      surr_2 = torch.clamp(ratio, 1 - epsilon, 1 + epsilon) * advantage
      loss = -torch.min(surr_1, surr_2) + F.smooth_l1_loss(self.v(s, h), td_target.detach())
      # actor loss + critic loss

      self.optimizer.zero_grad()
      loss.mean().backward()
      self.optimizer.step()

and it's main function which run openAI Gym cartpole environment.

def main():
  env = gym.make('CartPole-v1')
  agent = PPO_lstm()
  score = 0.0
  print_interval = 20

  for episode in range(10000):
    hidden = (torch.zeros([1, 1, 64], dtype=torch.float), torch.zeros([1, 1, 64], dtype=torch.float)) # lstm init hidden state
    s = env.reset()
    done = False
    while not done:
      for step in range(20):
        pi, hidden_prime = agent.pi(torch.from_numpy(s).float(), hidden) # pi size : [1, 2]
        pi = pi.squeeze() # pi size : [2]
        m = Categorical(pi)
        action = m.sample().item()
        s_prime, reward, done, info = env.step(action)
        agent.add_log((s, action, reward / 100.0, s_prime, pi[action].item(), done, hidden, hidden_prime))
        # hidden dim : ([1, 1, 64], [1, 1, 64])
        hidden = hidden_prime
        s = s_prime
        score += reward
        if done:
          break
      agent.train()
    if episode % print_interval == 0 and episode != 0:
      print("episode {}'s avg score : {}".format(episode, score/print_interval))
      score = 0.0

  env.close()

if __name__ == '__main__':
  main()

When I try this code without LSTM(using MLP policy and value function), it runs well.
Can you tell me there are any problem using LSTM or hidden state?


title: Using pretrained models in Pytorch for Semantic Segmentation, then training only the fully connected layers with our own dataset

body:
I am learning Pytorch and trying to understand how the library works for semantic segmentation.
What I've understood so far is that we can use a pre-trained model in pytorch. I've found an article which was using this model in the .eval() mode but I have not been able to find any tutorial on using such a model for training on our own dataset. I have a very small dataset and I need transfer learning to get results. My goal is to only train the FC layers with my own data. How is that achievable in Pytorch without complicating the code with OOP or so many .py files. I have been having a hard time figuring out such repos in github as I am not the most proficient person when it comes to OOP. I have been using Keras for Deep Learning until recently and there everything is easy and straightforward. Do I have the same options in Pycharm?
I appreciate any guidance on this. I need to run a piece of code that does the semantic segmentation and I am really confused about many of the steps I need to take.


title: How can I use 'torch.utils.mobile_optimizer' in PyTorch 1.6.0?

body:
According to https://pytorch.org/docs/stable/mobile_optimizer.html (currently for 1.6.0)

Torch mobile supports torch.mobile_optimizer.optimize_for_mobile utility to run a list of optimization pass with modules in eval mode. The method takes the following parameters: a torch.jit.ScriptModule object, a blacklisting optimization set and a preserved method list

The page title and

torch.utils.mobile_optimizer.optimize_for_mobile(script_module, optimization_blacklist: Set[torch._C.MobileOptimizerType] = None)

at the end suggest it should be torch.utils.mobile_optimizer instead. However, neither work for me:
In [25]: module1 = torch.utils.mobile_optimizer.optimize_for_mobile(module)
---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
<ipython-input-25-79e42fe53cdf> in <module>
----> 1 module1 = torch.utils.mobile_optimizer.optimize_for_mobile(module)

AttributeError: module 'torch.utils' has no attribute 'mobile_optimizer'

In [26]: module1 = torch.mobile_optimizer.optimize_for_mobile(module)
---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
<ipython-input-26-469dd450df3d> in <module>
----> 1 module1 = torch.mobile_optimizer.optimize_for_mobile(module)

AttributeError: module 'torch' has no attribute 'mobile_optimizer'

In [27]: print(torch.__version__)
1.6.0

Am I missing anything? E.g. do I have to install some package other than torch itself?


title: Pytorch model doesn't learn identity function?

body:
I wrote some models in pytorch which was not able to learn anything even after many epochs. In order to debug the problem I made a simple model which models identity function of an input. The difficulty is this model also doesn't learn nothing despite training for 50k epochs,
import torch
import torch.nn as nn

torch.manual_seed(1)

class Net(nn.Module):
    def __init__(self):
        super().__init__()
        self.input = nn.Linear(2,4)
        self.hidden = nn.Linear(4,4)
        self.output = nn.Linear(4,2)
        self.relu = nn.ReLU()
        self.softmax = nn.Softmax(dim=1)
        self.dropout = nn.Dropout(0.5)
    def forward(self,x):
        x = self.input(x)
        x = self.dropout(x)
        x = self.relu(x)
        x = self.hidden(x)
        x = self.dropout(x)
        x = self.relu(x)
        x = self.output(x)
        x = self.softmax(x)
        return x

X = torch.tensor([[1,0],[1,0],[0,1],[0,1]],dtype=torch.float)

net = Net()

criterion = nn.CrossEntropyLoss()

opt = torch.optim.Adam(net.parameters(), lr=0.001)

for i in range(100000):
    opt.zero_grad()
    y = net(X)
    loss = criterion(y,torch.argmax(X,dim=1))
    loss.backward()
    if i%500 ==0:
        print("Epoch: ",i)
        print(torch.argmax(y,dim=1).detach().numpy().tolist())
        print("Loss: ",loss.item())
        print()

Output
Epoch:  52500
[0, 0, 1, 0]
Loss:  0.6554909944534302

Epoch:  53000
[0, 0, 0, 0]
Loss:  0.7004914283752441

Epoch:  53500
[0, 0, 0, 0]
Loss:  0.7156486511230469

Epoch:  54000
[0, 0, 0, 0]
Loss:  0.7171240448951721

Epoch:  54500
[0, 0, 0, 0]
Loss:  0.691678524017334

Epoch:  55000
[0, 0, 0, 0]
Loss:  0.7301554679870605

Epoch:  55500
[0, 0, 0, 0]
Loss:  0.728650689125061

What is wrong with my implementation?


title: Loss is “nan” when fine-tuning HuggingFace NLI model (both RoBERTa/BART)

body:
I'm using HuggingFace's Transformer's library and I’m trying to fine-tune a pre-trained NLI model (ynie/roberta-large-snli_mnli_fever_anli_R1_R2_R3-nli) on a dataset of around 276.000 hypothesis-premise pairs. I’m following the instructions from the docs here and here. I have the impression that the fine-tuning works (it does the training and saves the checkpoints), but trainer.train() and trainer.evaluate() return "nan" for the loss.
What I've tried:

I tried using both ynie/roberta-large-snli_mnli_fever_anli_R1_R2_R3-nli and facebook/bart-large-mnli to make sure that it's not linked to specific model, but I get the issue for both models
I tried following the advice in this related github issue, but adding num_labels=3 to the config file does not solve the issue. (I think my issue is different because the models are already fine-tuned on NLI in my case)
I tried many different ways of changing my input data because I suspected that there could be an issue with my input data, but I also couldn't solve it that way.
The probable source of the issue: I inspected the prediction output from the model during training and the weird thing is the prediction value always seems to be "0" (entailment) in 100% of cases (see printed output in the code below). This is clearly an error. I think the source for this is that the logits the model seems to return during training are torch.tensor([[np.nan, np.nan, np.nan]]) and when you apply .argmax(-1) to this, you get torch.tensor(0). The big mystery for me is why the logits would become "nan", because the model does not do that when I use the same input data only outside of the trainer.
=> Does anyone know where this issues comes from? See my code below.

Thanks a lot in advance for any suggestion!
Here is my code:
### load model & tokenize
from transformers import AutoTokenizer, AutoModelForSequenceClassification
import torch

max_length = 256
hg_model_hub_name = "ynie/roberta-large-snli_mnli_fever_anli_R1_R2_R3-nli"
# also tried: hg_model_hub_name = "facebook/bart-large-mnli"
tokenizer = AutoTokenizer.from_pretrained(hg_model_hub_name)
model = AutoModelForSequenceClassification.from_pretrained(hg_model_hub_name)
model.config

device = "cuda" if torch.cuda.is_available() else "cpu"
print(f"Device: {device}")
if device == "cuda":
  model = model.half()
model.to(device)
model.train();

#... some data preprocessing

encodings_train = tokenizer(premise_train, hypothesis_train, return_tensors="pt", max_length=max_length,
                            return_token_type_ids=True, truncation=False, padding=True)
encodings_val = tokenizer(premise_val, hypothesis_val, return_tensors="pt", max_length=max_length,
                          return_token_type_ids=True, truncation=False, padding=True)
encodings_test = tokenizer(premise_test, hypothesis_test, return_tensors="pt", max_length=max_length,
                           return_token_type_ids=True, truncation=False, padding=True)

### create pytorch dataset object
class XDataset(torch.utils.data.Dataset):
    def __init__(self, encodings, labels):
        self.encodings = encodings
        self.labels = labels
    def __getitem__(self, idx):
        item = {key: torch.as_tensor(val[idx]) for key, val in self.encodings.items()}
        #item = {key: torch.as_tensor(val[idx]).to(device) for key, val in self.encodings.items()}
        item['labels'] = torch.as_tensor(self.labels[idx])
        #item['labels'] = self.labels[idx]
        return item
    def __len__(self):
        return len(self.labels)

dataset_train = XDataset(encodings_train, label_train)
dataset_val = XDataset(encodings_val, label_val)
dataset_test = XDataset(encodings_test, label_test)

# compute metrics with trainer
from sklearn.metrics import accuracy_score, precision_recall_fscore_support
def compute_metrics(pred):
    labels = pred.label_ids
    print(labels)
    preds = pred.predictions.argmax(-1)
    print(preds)
    precision, recall, f1, _ = precision_recall_fscore_support(labels, preds, average='binary', pos_label=0)
    acc = accuracy_score(labels, preds)
    return {
        'accuracy': acc,
        'f1': f1,
        'precision': precision,
        'recall': recall
    }

## training
from transformers import Trainer, TrainingArguments

# https://huggingface.co/transformers/main_classes/trainer.html#transformers.TrainingArguments
training_args = TrainingArguments(
    output_dir='./results',          # output directory
    num_train_epochs=1,              # total number of training epochs
    per_device_train_batch_size=8,  # batch size per device during training
    per_device_eval_batch_size=8,   # batch size for evaluation
    warmup_steps=500,                # number of warmup steps for learning rate scheduler
    weight_decay=0.01,               # strength of weight decay
    logging_dir='./logs',            # directory for storing logs
    logging_steps=100,
)

trainer = Trainer(
    model=model,                         # the instantiated  Transformers model to be trained
    args=training_args,                  # training arguments, defined above
    train_dataset=dataset_train,         # training dataset
    eval_dataset=dataset_val             # evaluation dataset
)

trainer.train()
# output: TrainOutput(global_step=181, training_loss=nan)
trainer.evaluate()
# output:
[2 2 2 0 0 2 2 2 0 2 0 0 2 2 2 2 0 2 0 2 2 2 2 0 2 0 2 0 0 2 0 0 2 0 0 0 2
 0 2 0 0 0 0 0 2 0 0 2 2 2 0 2 2 2 2 2 0 0 0 0 2 0 0 0 2 2 0 0 0 2 0 0 0 2
 2 0 2 0 0 2 2 2 0 2 2 0 0 0 0 0 0 0 2 0 0 0 0 2 0 2 2 0 2 0 0 2 2 2 2 2 2
 2 0 0 0 0 2 0 0 2 0 0 0 0 2 2 2 0 0 0 0 0 2 0 0 2 0 2 0 2 0 2 0 0 2 2 0 0
 2 2 2 2 2 2 0 0 2 2 2 2 0 2 0 0 2 2 2 0 0 2 0 2 0 2 0 0 0 0 0 0 2 0 0 2 2
 0 2 2 2 0 2 2 0 2 2 2 2 2 2 0 0 2 0 0 2 2 0 0 0 2 0 2 2 2 0 0 0 0 0 0 0 0
 2 0 2 2 2 0 2 0 0 2 0 2 2 0 0 0 0 2 2 2 0 0 0 2 2 2 2 0 2 0 2 2 2]
[0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]

{'epoch': 1.0,
 'eval_accuracy': 0.5137254901960784,
 'eval_f1': 0.6787564766839378,
 'eval_loss': nan,
 'eval_precision': 0.5137254901960784,
 'eval_recall': 1.0}

Edit:  I've also opened a github issue with a but more detailed description of the issue here: https://github.com/huggingface/transformers/issues/9160


title: How to stick a list of block tensors to form a larger tensor in PyTorch

body:
Assume that there is a list of small tensors (say 16 blocks), and it is desired to stick these small tensors along horizontally and vertically to create a larger 2D image.

torch.split() is available for partition the tensor to smaller blocks, is there any operation for the opposite case?

Thanks


title: Transform flair language model tensors for viewing in TensorBoard Projector

body:
I want to convert "vectors,"

vectors = [token.embedding for token in sentence]
print(type(vectors))
<class 'list'>

print(vectors)
[tensor([ 0.0077, -0.0227, -0.0004,  ...,  0.1377, -0.0003,  0.0028]),
...
tensor([ 0.0003, -0.0461,  0.0043,  ..., -0.0126, -0.0004,  0.0142])]

to

0.0077 -0.0227 -0.0004 ... 0.1377 -0.0003 0.0028
...
0.0003 -0.0461 0.0043 ... -0.0126 -0.0004 0.0142

and write that to a TSV.

Aside: those embeddings are from flair (https://github.com/zalandoresearch/flair): how can I get the full output, not the -0.0004 ... 0.1377 abbreviated output?


title: Creating LSTM model with pytorch

body:
I'm quite new to using LSTM in Pytorch, I'm trying to create a model that gets a tensor of size 42 and a sequence of 62.(so 62 tensor a of size 42 each).
Which means that I have 62 tensors in a sequence. Each tensor is of size 42.(shape is [62,42]. Call this input tensor.

and I want to predict a tensor of 1 with a sequence of 8 (so size 1 tensor and 8 sequences) using this.
Which means that there are 8 tensors in a sequence of size 1 each. Call this label tensor.

The connection between those tensors is this:
Input tensor is made of columns:
A1 A2 A3 ...... A42
While label tensor if more like:
A3

What I’m trying to show is that if needed label tensor can be padded with zero in all places instead of the value of A3, so it can reach a length of 42.

How can I do this? since from what I'm reading from the Pytorch documentation I can only predict in the same ratio(1 point predict 1), while I want to predict from tensor of 42 with a sequence of 62 a tensor of 1 and sequence of 8.
Is it doable? Do I need to pad the predicted tensor to size 42 from 1?
Thanks!

a good solution will be using seq2seq for example


title: pytorch: compute vector-Jacobian product for vector function

body:
Good day!
I am trying to grasp torch.autograd basics. In particular I want to test this statement from https://pytorch.org/tutorials/beginner/blitz/autograd_tutorial.html#sphx-glr-beginner-blitz-autograd-tutorial-py

So my idea is to construct a vector function, say:
(y_1; y_2; y_3) = (x_1*x_1 + x_2; x_2 * x_2 + x_3; x_3 * x_3)
Then count Jacobian matrix at point (1,1,1) and multiply it on vector (3, 5, 7).
Jacobian = (2x_1; 1.   ; 0.  )
(0.  ; 2x_2 ; 1.  )
(0.  ; 0.   ; 2x_3)
I am expecting result Jacobian(x=(1,1,1)) * v = (6+5, 10 + 7, 2 * 7) = (11, 17, 14).
Now below is my attempt to do it in pytorch:
import torch

x = torch.ones(3, requires_grad=True)
print(x)

y = torch.tensor([x[0]**2 + x [1], x[1]**2 + x[2], x[2]**2], requires_grad=True)
print(y)

v = torch.tensor([3, 5, 7])

y.backward(v)
x.grad

which gives not expected result (2., 2., 1.). I think I define tensor y in a wrong way. If I simply do y = x * 2, then gradient will work, but what about creating more complex tensor like in this case?
Thank you.


title: torch.cuda.is_available() True in (base), but False in other conda env

body:
I run this code in Anaconda prompt, and it returns True.
(base) C:\User
torch.cuda.is_available()
True

But when I run other conda environment, it just doesn't work.
(pytorch_project) C:\User
torch.cuda.is_available()
False

The problem seems to be different results of torch.version.cuda.
(base) torch.version.cuda = 10.1
(pytorch_project) torch.version.cuda = 10.2
But I don't know why they are different...
How do I make 10.2 down to 10.1 and make is_available() == True ?

Here is my info.
 Windows 10 / nvidia-smi=425.31 / CUDA ver=10.1 / pytorch=1.4.0 / torchvision=0.5.0

conda list


title: How to assign a name for a pytorch layer?

body:
Following a previous question, I want to plot weights, biases, activations and gradients to achieve a similar result to this.
Using
for name, param in model.named_parameters():
    summary_writer.add_histogram(f'{name}.grad', param.grad, step_index)

as was suggested in the previous question gives sub-optimal results, since layer names come out similar to '_decoder._decoder.4.weight', which is hard to follow, especially since the architecture is changing due to research. 4 in this run won't be the same in the next, and is really meaningless.
Thus, I wanted to give my own string names to each layer.

I found this Pytorch forum discussion, but no single best practice was agreed upon.
What is the recommended way to assign names to Pytorch layers?
Namely, layers defined in various ways:

Sequential:

self._seq = nn.Sequential(nn.Linear(1, 2), nn.Linear(3, 4),)

Dynamic:

self._dynamic = nn.ModuleList()
    for _ in range(self._n_features):
        self._last_layer.append(nn.Conv1d(in_channels=5, out_channels=6, kernel_size=3, stride=1, padding=1,),)

Direct:

self._direct = nn.Linear(7, 8)

Other ways I didn't think about

I would like to be able to give a string name to each layer, defined in each of the above ways.


title: How to leverage the world-size parameter for DistributedDataParallel in Pytorch example for multiple GPUs?

body:
I am running this Pytorch example on a g2.2xlarge AWS machine. So, when I run time python imageNet.py ImageNet2, it runs well with the following timing:

real    3m16.253s
user    1m50.376s
sys 1m0.872s

However, when I add the world-size parameter, it gets stuck and does not execute anything. The command is as follows: time python imageNet.py --world-size 2 ImageNet2

So, how do I leverage the DistributedDataParallel functionality with the world-size parameter in this script. The world-size parameter is nothing but number of distributed processes.

Do I spin up another similar instance for this purpose? If yes, then how do the script recognize the instance? Do I need to add some parameters like the instance's IP or something?


title: add_graph working and printing result on Verbose but not saving on log file

body:
When using the tensorboard in utils with the new Pytorch 1.2, the add_graph function, the graph is not added to the tensorboard log file. If I log an scalar before calling the add_graph it gets logged, but anything I add after doesn't.

I won't be able to provide the whole model here, it uses LSTMs and Attention but nothing too fancy. The snippet where I call add_graph:

    from torch.utils.tensorboard import SummaryWriter

    w = SummaryWriter('./runs/jointmodel')

    dummy_input = torch.Tensor(torch.rand(2, 10, 1024))
    w.add_scalar('log', 10, 1)
    w.add_graph(model, dummy_input, True)
    w.add_scalar('log', 10, 2)

The first scalar gets logged, the second one doesn't. The add_graph prints the whole model graph in console. The file generated is empty with just the first scalar logged. The forward pass works normally for the model. The verbose of add_graph prints 510 elements. Any help will be appreciated.

Pytorch version: 1.2
Tensorboard version: 1.15.0a20190806


title: Difference between PyTorch, PyTorchModel in sagemaker.pytorch

body:
I am trying to create a model using pytorch in sagemaker. I tried deploying using - PyTorch module in sagemaker.pytorch [from sagemaker.pytorch import PyTorch].

But, I want to understand what is PyTorchModel in sagemaker.pytorch [from sagemaker.pytorch import PyTorchModel]. They both have deploy() . And I followed the link https://sagemaker.readthedocs.io/en/stable/using_pytorch.html to create and deploy the model . Where I don't see using "PyTorchModel" anywhere. I would like to know the difference and when to use what.

I tried the following so far.

Step 1 : I called pytorch estimator

pytorch_model = PyTorch(entry_point='entry_v1.py',
                        train_instance_type='ml.m5.4xlarge',
                        role = role,
                        train_instance_count=1,
                        output_path = "s3://model-output-bucket/test",
                        framework_version='1.1',
                        hyperparameters = {'epochs': 10,'learning-rate': 0.01})

Step2  : I called fit method

pytorch_model.fit({'train': 's3://training-data/train_data.csv',
                  'test':'s3://testing-data/test_data.csv'})

Step3: I called deploy method.

predictor = pytorch_model.deploy(instance_type='ml.m4.xlarge', initial_instance_count=1)

I would like to know when to call create_model() here.

I got some understanding over here. We use [from sagemaker.pytorch import PyTorch] for end to end process where we train a model with .fit() and then we can deploy the model with .deploy()

But, with [from sagemaker.pytorch import PyTorchModel] we can use the model which is already trained.

Step1:

pytorch_model = PyTorchModel(model_data='s3://model-output-bucket/sagemaker-pytorch-2019-08-20-16-54-32-500/output/model.tar.gz', role=role,entry_point=entry_v1.py,sagemaker_session=sagemaker_session)

Step2:

predictor = pytorch_model.deploy(instance_type='ml.c4.xlarge', initial_instance_count=1)

Also, .create_model() of PyTorch Estimator will return an object of PyTorchModel.

Please correct me if i am wrong anywhere.


title: Is there a similar function in tensorflow like load_state_dict() in Pytorch?

body:
Like it has been described, I am wondering is there a similar function in tensorflow for load_state_dict() like the one does in Pytorch. To demonstrate a scenario, please refer to the code following:

# Suppose we have two correctly initialized neural networks: net2 and net1
# Using Pytorch
net2.load_state_dict(net1.state_dict())

Does anyone have any idea?


title: Moving to numerically stable log-sum-exp leads to extremely large loss values

body:
I am working on a network that uses a LSTM along with MDNs to predict some distributions. The loss function I use for these MDNs involve trying to fit my target data to the predicted distributions. I am trying to compute the log-sum-exp for the log_probs of these target data to compute the loss. When I use standard log-sum-exp, I get reasonable initial loss values (around 50-70) even though later it encounters some NaNs and breaks. Based on what I have read online, a numerically stable version of log-sum-exp is required to avoid this problem. However as soon I use the stable version, my loss values shoot up to the order of 15-20k. They do come down upon training but eventually they also lead to NaNs.

NOTE : I did not use the logsumexp function in PyTorch, since I needed to have a weighted summation based on my mixture components.

def log_sum_exp(self,value, weights, dim=None):
        eps = 1e-20
        m, idx = torch.max(value, dim=dim, keepdim=True)
        return m.squeeze(dim) + torch.log(torch.sum(torch.exp(value-m)*(weights.unsqueeze(2)),
                                       dim=dim) + eps)

def mdn_loss(self, pi, sigma, mu, target):
        eps = 1e-20
        target = target.unsqueeze(1)
        m = torch.distributions.Normal(loc=mu, scale=sigma)
        probs = m.log_prob(target)
        # Size of probs is batch_size x num_mixtures x num_out_features
        # Size of pi is batch_size x num_mixtures
        loss = -self.log_sum_exp(probs, pi, dim=1)
        return loss.mean()

Upon adding anomaly_detection, the NaNs seem to occur at :
        probs = m.log_prob(target)

Seeing these huge initial loss values just by moving to the numerically stable version have led me to believe I have some bug in my current implementation. Any help please.


title: Famous neural networks for regression

body: I have come across many neural network architecture for classification problems. AlexNet, ResNet, VGGNet, GoogLeNet etc... Is there similar networks for regression problems which can be used for transfer learning?


title: Unable to load quantized pytorch mobile model on android

body:
I am struggling to get my quantized pytorch mobile model (custom MobilenetV3) running on android. I have followed this tutorial https://pytorch.org/tutorials/advanced/static_quantization_tutorial.html and I managed to quantize my model without any problems. However, when I try to load the model via. module = Module.load(assetFilePath(this, MODEL_NAME));
I get the following Exception:

Unknown builtin op: quantized::linear_unpack_fp16.
Could not find any similar ops to quantized::linear_unpack_fp16. This
op may not exist or may not be currently supported in TorchScript.

Why are there even float16 values in the quantized model, I thought quantization would replace all float32 vlaues with qint8/quint8? Any ideas on how this can be fixed?
This is how the pytorch model was quantized and saved:
torch.quantization.get_default_qconfig(backend='qnnpack')
model.qconfig = torch.quantization.default_qconfig
torch.quantization.prepare(model, inplace=True)
torch.quantization.convert(model, inplace=True)
traced_script_module = torch.jit.trace(model,dummyInput)
traced_script_module.save("model/modelQuantized.pt")


title: lstm:Input batch size 100 doesn't match hidden[0] batch size 1

body:
I am trying to add an lstm layer to my previous working AI model.
On adding the model I am getting this error when training the batch to my AI.
earlier without the LSTM the error was not there and it worked fine.
Input batch size 100 doesn't match hidden[0] batch size 1.
I am using nn.LSTMCell
can anyone please help a check if I am missing some parameter to init my lstmcell so it can take batch inputs as well.
below is my code...
import os
import time
import random
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
from sklearn.preprocessing import StandardScaler
import torch
import torch.nn as nn
import torch.nn.functional as F
from random import random as rndm
from torch.autograd import Variable
from collections import deque

os.chdir("C:\\Users\\granthjain\\Desktop\\startup_code")

torch.set_default_tensor_type('torch.DoubleTensor')

class ReplayBuffer(object):

  def __init__(self, max_size=1e6):
    self.storage = []
    self.max_size = max_size
    self.ptr = 0

  def add(self, transition):
    if len(self.storage) == self.max_size:
      self.storage[int(self.ptr)] = transition
    else:
      self.storage.append(transition)
    self.ptr = (self.ptr + 1) % self.max_size

  def sample(self, batch_size):

    ind = np.random.randint(0, self.ptr, size=batch_size)
    batch_states, batch_next_states, batch_actions, batch_rewards, batch_dones = [], [], [], [], []
    for i in ind:
      state, next_state, action, reward, done = self.storage[i]

      if state is None:
          continue
      elif next_state is None:
          continue
      elif action is None:
          continue
      elif reward is None:
          continue
      elif done is None:
          continue

      batch_states.append(np.array(state, copy=False))
      batch_next_states.append(np.array(next_state, copy=False))
      batch_actions.append(np.array(action, copy=False))
      batch_rewards.append(np.array(reward, copy=False))
      batch_dones.append(np.array(done, copy=False))

    return np.array(batch_states,dtype=object).astype(float), np.array(batch_next_states,dtype=object).astype(float), np.array(batch_actions,dtype=object).astype(float), np.array(batch_rewards,dtype=object).astype(float), np.array(batch_dones,dtype=object).astype(float)

class Actor(nn.Module):

  def __init__(self, state_dim, action_dim, max_action):
    super(Actor, self).__init__()
    self.lstm = nn.LSTMCell(state_dim, 256)
    self.layer_1 = nn.Linear(256, 400)
    self.layer_2 = nn.Linear(400, 300)
    self.layer_3 = nn.Linear(300, action_dim)
    self.hx = torch.zeros(1,256)
    self.cx = torch.zeros(1,256)
    self.max_action = max_action

  def forward(self, x):
    self.hx, self.cx = self.lstm(x, (self.hx, self.cx))
    x = F.relu(self.layer_1(self.hx))
    x = F.relu(self.layer_2(x))
    x = self.max_action * torch.tanh(self.layer_3(x))
    return x

class Critic(nn.Module):

  def __init__(self, state_dim, action_dim):
    super(Critic, self).__init__()
    # Defining the first Critic neural network
    self.lstm1 = nn.LSTMCell(state_dim + action_dim, 256)
    self.layer_1 = nn.Linear(256, 400)
    self.layer_2 = nn.Linear(400, 300)
    self.layer_3 = nn.Linear(300, 1)
    # Defining the second Critic neural network
    self.lstm2 = nn.LSTMCell(state_dim + action_dim, 256)
    self.layer_4 = nn.Linear(256, 400)
    self.layer_5 = nn.Linear(400, 300)
    self.layer_6 = nn.Linear(300, 1)
    self.hx1 = torch.zeros(1,256)
    self.cx1 = torch.zeros(1,256)
    self.hx2 = torch.zeros(1,256)
    self.cx2 = torch.zeros(1,256)

  def forward(self, x, u):
    xu = torch.cat([x, u], 1)
    # Forward-Propagation on the first Critic Neural Network
    self.hx1,self.cx1 = self.lstm(xu, (self.hx1, self.cx1))
    x1 = F.relu(self.layer_1(self.hx1))
    x1 = F.relu(self.layer_2(x1))
    x1 = self.layer_3(x1)
    # Forward-Propagation on the second Critic Neural Network
    self.hx2,self.cx2 = self.lstm(xu, (self.hx2, self.cx2))
    x2 = F.relu(self.layer_4(self.hx2))
    x2 = F.relu(self.layer_5(x2))
    x2 = self.layer_6(x2)
    return x1, x2

  def Q1(self, x, u):
    xu = torch.cat([x, u], 1)
    self.hx1,self.cx1 = self.lstm(xu, (self.hx1, self.cx1))
    x1 = F.relu(self.layer_1(self.hx1))
    x1 = F.relu(self.layer_2(x1))
    x1 = self.layer_3(x1)
    return x1

# Selecting the device (CPU or GPU)
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

# Building the whole Training Process into a class

class TD3(object):

  def __init__(self, state_dim, action_dim, max_action):
    self.actor = Actor(state_dim, action_dim, max_action).to(device)
    self.actor_target = Actor(state_dim, action_dim, max_action).to(device)
    self.actor_target.load_state_dict(self.actor.state_dict())
    self.actor_optimizer = torch.optim.Adam(self.actor.parameters())
    self.critic = Critic(state_dim, action_dim).to(device)
    self.critic_target = Critic(state_dim, action_dim).to(device)
    self.critic_target.load_state_dict(self.critic.state_dict())
    self.critic_optimizer = torch.optim.Adam(self.critic.parameters())
    self.max_action = max_action

  def reset_hxcx(self):
    self.actor.cx = torch.zeros(1,256)
    self.actor.hx = torch.zeros(1,256)
    self.actor_target.cx = torch.zeros(1,256)
    self.actor_target.hx = torch.zeros(1,256)
    self.critic.cx1 = torch.zeros(1,256)
    self.critic.cx2 = torch.zeros(1,256)
    self.critic.hx1 = torch.zeros(1,256)
    self.critic.hx2 = torch.zeros(1,256)
    self.critic_target.cx1 = torch.zeros(1,256)
    self.critic_target.cx2 = torch.zeros(1,256)
    self.critic_target.hx1 = torch.zeros(1,256)
    self.critic_target.hx2 = torch.zeros(1,256)

  def select_action(self, state):
    print("state =", type(state))
    return self.actor(state).cpu().data.numpy().flatten()

  def train(self, replay_buffer, iterations, batch_size=50, discount=0.99, tau=0.005, policy_noise=0.2, noise_clip=0.5, policy_freq=2):

    for it in range(iterations):

      # Step 4: We sample a batch of transitions (s, s’, a, r) from the memory
      batch_states, batch_next_states, batch_actions, batch_rewards, batch_dones = replay_buffer.sample(batch_size)

      batch_states=batch_states.astype(float)
      batch_next_states=batch_next_states.astype(float)
      batch_actions=batch_actions.astype(float)
      batch_rewards=batch_rewards.astype(float)
      batch_dones=batch_dones.astype(float)

      state = torch.from_numpy(batch_states)
      next_state = torch.from_numpy(batch_next_states)
      action = torch.from_numpy(batch_actions)
      reward = torch.from_numpy(batch_rewards)
      done = torch.from_numpy(batch_dones)

#      print("actor cx:",self.actor.cx)
#      print("actor hx:",self.actor.hx)
#      print("actor_target cx:",self.actor_target.cx)
#      print("actor_target cx:",self.actor_target.cx)
#      print("self.critic.cx1:",self.critic.cx1)
#      print("self.critic.cx2",self.critic.cx2)
#      print("self.critic.hx1:",self.critic.hx1)
#      print("self.critic.hx2:",self.critic.hx2)
#      print("self.critic_target.cx1:",self.critic_target.cx1)
#      print("self.critic_target.hx1",self.critic_target.hx1)
#      print("self.critic_target.cx2:",self.critic_target.cx2)
#      print("self.critic_target.hx2:",self.critic_target.hx2)

      # Step 5: From the next state s’, the Actor target plays the next action a’
      next_action = self.actor_target(next_state)

      # Step 6: We add Gaussian noise to this next action a’ and we clamp it in a range of values supported by the environment
      noise = torch.Tensor(batch_actions).data.normal_(0, policy_noise).to(device)
      noise = noise.clamp(-noise_clip, noise_clip)
      next_action = (next_action + noise).clamp(-self.max_action, self.max_action)

      # Step 7: The two Critic targets take each the couple (s’, a’) as input and return two Q-values Qt1(s’,a’) and Qt2(s’,a’) as outputs
      target_Q1, target_Q2 = self.critic_target(next_state, next_action)

      # Step 8: We keep the minimum of these two Q-values: min(Qt1, Qt2)
      target_Q = torch.min(target_Q1, target_Q2).double()

      # Step 9: We get the final target of the two Critic models, which is: Qt = r + γ * min(Qt1, Qt2), where γ is the discount factor

      done = done.resize_((done.shape[0],1))

      reward = reward.resize_((reward.shape[0],1))

      target_Q = reward + ((1 - done) * discount * target_Q).detach()

      # Step 10: The two Critic models take each the couple (s, a) as input and return two Q-values Q1(s,a) and Q2(s,a) as outputs
      current_Q1, current_Q2 = self.critic(state, action)

      # Step 11: We compute the loss coming from the two Critic models: Critic Loss = MSE_Loss(Q1(s,a), Qt) + MSE_Loss(Q2(s,a), Qt)
      critic_loss = F.mse_loss(current_Q1, target_Q) + F.mse_loss(current_Q2, target_Q)

      # Step 12: We backpropagate this Critic loss and update the parameters of the two Critic models with a SGD optimizer
      self.critic_optimizer.zero_grad()
      critic_loss.backward()
      self.critic_optimizer.step()

      # Step 13: Once every two iterations, we update our Actor model by performing gradient ascent on the output of the first Critic model
      if it % policy_freq == 0:
        actor_loss = -self.critic.Q1(state, self.actor(state)).mean()
        self.actor_optimizer.zero_grad()
        actor_loss.backward()
        self.actor_optimizer.step()

        # Step 14: Still once every two iterations, we update the weights of the Actor target by polyak averaging
        for param, target_param in zip(self.actor.parameters(), self.actor_target.parameters()):
          target_param.data.copy_(tau * param.data + (1 - tau) * target_param.data)

        # Step 15: Still once every two iterations, we update the weights of the Critic target by polyak averaging
        for param, target_param in zip(self.critic.parameters(), self.critic_target.parameters()):
          target_param.data.copy_(tau * param.data + (1 - tau) * target_param.data)

  # Making a save method to save a trained model
  def save(self, filename, directory):
    torch.save(self.actor.state_dict(), '%s/%s_actor.pth' % (directory, filename))
    torch.save(self.critic.state_dict(), '%s/%s_critic.pth' % (directory, filename))

  # Making a load method to load a pre-trained model
  def load(self, filename, directory):
    self.actor.load_state_dict(torch.load('%s/%s_actor.pth' % (directory, filename)))
    self.critic.load_state_dict(torch.load('%s/%s_critic.pth' % (directory, filename)))

#set the parameters

start_timesteps = 1e3 # Number of iterations/timesteps before which the model randomly chooses an action, and after which it starts to use the policy network
eval_freq = 5e1 # How often the evaluation step is performed (after how many timesteps)
max_timesteps = 5e3 # Total number of iterations/timesteps
save_models = True # Boolean checker whether or not to save the pre-trained model
expl_noise = 0.1 # Exploration noise - STD value of exploration Gaussian noise
batch_size = 100 # Size of the batch
discount = 0.99 # Discount factor gamma, used in the calculation of the total discounted reward
tau = 0.005 # Target network update rate
policy_noise = 0.2 # STD of Gaussian noise added to the actions for the exploration purposes
noise_clip = 0.5 # Maximum value of the Gaussian noise added to the actions (policy)
policy_freq = 2 # Number of iterations to wait before the policy network (Actor model) is updated

state_dim = 3
action_dim = 3
max_action = 1
idx = 0

class env1:

    def __init__(self,state_dim,action_dim,data):
        self.state_dim = state_dim
        self.state = torch.zeros(self.state_dim)
        self.state[state_dim-1]=1000.0
        self.next_state = torch.zeros(self.state_dim)
        self.next_state[state_dim-1] = 1000.0
        self.action_dim = action_dim
        self.data = data
        self.idx = 0
        self.count = 0
        self._max_episode_steps = 200
        self.state[1] = self.data[self.idx]
        self.next_state[1] = self.data[self.idx]

    def reset(self):
        self.next_state = torch.zeros(self.state_dim)
        self.next_state[state_dim-1]=1000.0
        self.state = torch.zeros(self.state_dim)
        self.state[state_dim-1]=1000.0
        self.state[1] = self.data[self.idx]
        self.next_state[1] = self.data[self.idx]
        self.count = 0
        ch = self.state[0]
        cp = self.state[1]
        cc = self.state[2]
        st = torch.tensor([ch,cp,cc])
        return st

    def step(self,action):
        done = False
        act_t = torch.argmax(action)
        self.idx += 1
        if(act_t==0):
            num_s = int(self.state[2]/self.state[1])

            self.next_state[0] += num_s
            self.next_state[2] = self.state[2]%self.state[1]

            self.next_state[1] = self.data[self.idx]
        elif(act_t==1):
            self.next_state[1] = self.data[self.idx]
        elif(act_t==2):
            self.next_state[2] = self.state[2]+ self.state[1]*self.state[0]
            self.next_state[0] = 0
            self.next_state[1] = self.data[self.idx]

        reward = self.next_state[2] - self.state[2] + self.next_state[1]*self.next_state[0] - self.state[1]*self.state[0] -1

        self.state[0] = self.next_state[0]
        self.state[1] = self.next_state[1]
        self.state[2] = self.next_state[2]

        ch = self.state[0]
        cp = self.state[1]
        cc = self.state[2]

        st = torch.tensor([ch,cp,cc])

        self.count = (self.count + 1)%100
        if(self.count==0):
            done = True

        return st, reward, done
policy = TD3(state_dim, action_dim, max_action)

#Create the environment
data = pd.read_csv('PAGEIND.csv')
data = data['Close']
data = np.array(data).reshape(-1,1)

max_timesteps = data.shape[0]
sc = StandardScaler()
data = sc.fit_transform(data)
data = torch.DoubleTensor(data)
env = env1(state_dim,action_dim,data)

replay_buffer = ReplayBuffer()

#init training variables

total_timesteps = 0
timesteps_since_eval = 0
episode_num = 0
done = True
t0 = time.time()

# We start the main loop over 500,000 timesteps
while total_timesteps < max_timesteps:

  # If the episode is done
  if done:

    # If we are not at the very beginning, we start the training process of the model
    if total_timesteps != 0:
      print("Total Timesteps: {} Episode Num: {} Reward: {}".format(total_timesteps, episode_num, episode_reward))
      policy.train(replay_buffer, episode_timesteps, batch_size, discount, tau, policy_noise, noise_clip, policy_freq)

    # When the training step is done, we reset the state of the environment
    obs = env.reset()
    policy.reset_hxcx()

    # Set the Done to False
    done = False

    # Set rewards and episode timesteps to zero
    episode_reward = 0
    episode_timesteps = 0
    episode_num += 1

  # Before 1000 timesteps, we play random actions
  if total_timesteps < 0.8*max_timesteps:
#random action
      actn = torch.randn(action_dim)
      action = torch.zeros(action_dim)
      action[torch.argmax(actn)] = 1

  else: # After 1000 timesteps, we switch to the model
    action = policy.select_action(torch.tensor(obs))
    # If the explore_noise parameter is not 0, we add noise to the action and we clip it
    if expl_noise != 0:
      print("policy action:",action)
      actn = (action + torch.randn(action_dim))
      action = torch.zeros(action_dim)
      action[torch.argmax(actn)] = 1

  # The agent performs the action in the environment, then reaches the next state and receives the reward
  new_obs, reward, done = env.step(action)

  # We check if the episode is done
  done_bool = 0 if episode_timesteps + 1 == env._max_episode_steps else float(done)

  # We increase the total reward
  episode_reward += reward

  # We store the new transition into the Experience Replay memory (ReplayBuffer)
  replay_buffer.add((obs, new_obs, action, reward, done_bool))

  # We update the state, the episode timestep, the total timesteps, and the timesteps since the evaluation of the policy
  obs = new_obs
  episode_timesteps += 1
  total_timesteps += 1
  timesteps_since_eval += 1

and below is the error msg:
    policy.train(replay_buffer, episode_timesteps, batch_size, discount, tau, policy_noise, noise_clip, policy_freq)

  File "C:/Users/granthjain/Desktop/startup_code/td3_lstm_try.py", line 196, in train
    next_action = self.actor_target(next_state)

  File "C:\Users\granthjain\Anaconda3_1\lib\site-packages\torch\nn\modules\module.py", line 477, in __call__
    result = self.forward(*input, **kwargs)

  File "C:/Users/granthjain/Desktop/startup_code/td3_lstm_try.py", line 79, in forward
    self.hx, self.cx = self.lstm(x, (self.hx, self.cx))

  File "C:\Users\granthjain\Anaconda3_1\lib\site-packages\torch\nn\modules\module.py", line 477, in __call__
    result = self.forward(*input, **kwargs)

  File "C:\Users\granthjain\Anaconda3_1\lib\site-packages\torch\nn\modules\rnn.py", line 708, in forward
    self.check_forward_hidden(input, hx[0], '[0]')

  File "C:\Users\granthjain\Anaconda3_1\lib\site-packages\torch\nn\modules\rnn.py", line 532, in check_forward_hidden
    input.size(0), hidden_label, hx.size(0)))

RuntimeError: Input batch size 100 doesn't match hidden[0] batch size 1


title: Issue with running multiple models in PyTorch Lightning

body:
I am developing a system which needs to train dozens of individual models (>50) using Lightning, each with their own TensorBoard plots and logs. My current implementation has one Trainer object per model and it seems like I'm running into this error when I go over ~90 Trainer objects. Interestingly, the error only appears when I run the .test() method, not during .fit():
Traceback (most recent call last):
  File "lightning/main_2.py", line 193, in <module>
    main()
  File "lightning/main_2.py", line 174, in main
    new_trainer.test(model=new_model, test_dataloaders=te_loader)
  File "\Anaconda3\envs\pyenv\lib\site-packages\pytorch_lightning\trainer\trainer.py", line 1279, in test
    results = self.__test_given_model(model, test_dataloaders)
  File "\Anaconda3\envs\pyenv\lib\site-packages\pytorch_lightning\trainer\trainer.py", line 1343, in __test_given_model
    self.set_random_port(force=True)
  File "\Anaconda3\envs\pyenv\lib\site-packages\pytorch_lightning\trainer\distrib_data_parallel.py", line 398, in set_random_port
    default_port = RANDOM_PORTS[-1]
IndexError: index -1 is out of bounds for axis 0 with size 0

As I just started with Lightning, I am not sure if having one Trainer/model is the best approach. However, I require individual plots from each model, and it seems that if I use a single trainer for multiple models the results get overridden.
For reference, I'm defining different lists of trainers as such:
for i in range(args["num_users"]):
    trainer_list_0.append(Trainer(max_epochs=args["epochs"], gpus=1, default_root_dir=args["save_path"],
                                          fast_dev_run=args["fast_dev_run"], weights_summary=None))
    trainer_list_1.append(Trainer(max_epochs=args["epochs"], gpus=1, default_root_dir=args["save_path"],
                                            fast_dev_run=args["fast_dev_run"], weights_summary=None))
    trainer_list_2.append(Trainer(max_epochs=args["epochs"], gpus=1, default_root_dir=args["save_path"],
                                            fast_dev_run=args["fast_dev_run"], weights_summary=None))

As for training:
for i in range(args["num_users"]):
    trainer_list_0[i].fit(model_list_0[i], train_dataloader=dataloader_list[i],
                                      val_dataloaders=val_loader)
    trainer_list_1[i].fit(model_list_1[i], train_dataloader=dataloader_list[i],
                                        val_dataloaders=val_loader)
    trainer_list_2[i].fit(model_list_2[i], train_dataloader=dataloader_list[i],
                                        val_dataloaders=val_loader)

And testing:
for i in range(args["num_users"]):
    trainer_list_0[i].test(test_dataloaders=te_loader)
    trainer_list_1[i].test(test_dataloaders=te_loader)
    trainer_list_2[i].test(test_dataloaders=te_loader)

Thanks!


title: Torch.cuda.empty_cache() very very slow performance

body:
I have a very slow performance problem when I execute an inference batch loop on a single GPU.
This slow behavior appears after the first batch has been processed -
that is when the GPU is already almost full and its memory needs to be recycled to accept the next batch.
At a pristine GPU state - the performance is super fast (as expected).
I hope both the following code snippet and the output illustrate the problem in a nutshell.
(I've removed the print and time measurements from the snippet for brevity)
predictions = None

for i, batch in enumerate(self.test_dataloader):

    # if this line is active - the bottleneck after the first batch moves here, rather than below
    # i.e. when i > 0
    # torch.cuda.empty_cache()

    # HUGE PERFORMANCE HIT HAPPENS HERE - after the first batch
    # i.e. when i > 0
    # obviously tensor.to(device) uses torch.cuda.empty_cache() internally when needed
    # and it is inexplicably SLOW
    batch = tuple(t.to(device) for t in batch)  # to GPU (or CPU) when gpu

    b_input_ids, b_input_mask, b_labels = batch

    with torch.no_grad():
        outputs = model(b_input_ids, token_type_ids=None, attention_mask=b_input_mask)

    logits = outputs[0]
    logits = logits.detach()

    # that doesn't help alleviate the issue
    del outputs

    predictions = logits if predictions is None else torch.cat((predictions, logits), 0)

    # nor do all of the below - freeing references doesn't help speeding up
    del logits
    del b_input_ids
    del b_input_mask
    del b_labels
    for o in batch:
        del o
    del batch

output
start empty cache... 0.00082
end empty cache... 1.9e-05
start to device... 3e-06
end to device... 0.001179 - HERE - time is super fast (as expected)
start outputs... 8e-06
end outputs... 0.334536
logits... 6e-06
start detach... 1.7e-05
end detach... 0.004036

start empty cache... 0.335932
end empty cache... 4e-06
start to device... 3e-06
end to device... 16.553849 - HERE - time is ridiculously high - it's 16 seconds to move tensor to GPU
start outputs... 2.3e-05
end outputs... 0.020878
logits... 7e-06
start detach... 1.4e-05
end detach... 0.00036

start empty cache... 0.00082
end empty cache... 6e-06
start to device... 4e-06
end to device... 17.385204 - HERE - time is ridiculously high
start outputs... 2.9e-05
end outputs... 0.021351
logits... 4e-06
start detach... 1.3e-05
end detach... 1.1e-05

...

Have I missed something obvious or is this the expected GPU behavior?
I am posting this question before engaging in complex coding, juggling between a couple of GPUs and CPU available on my server.
Thanks in advance,
Albert
EDIT
RESOLVED The issue was: in DataLoader constructor - I changed the pin_memory to False (True was causing the issue). That cut the .to(device) time by 350%-400%
self.test_dataloader = DataLoader(
            test_dataset,
            sampler=SequentialSampler(test_dataset),
            # batch_size=len(test_dataset)  # AKA - single batch - nope! no mem for that
            batch_size=BATCH_SIZE_AKA_MAX_ROWS_PER_GUESS_TO_FIT_GPU_MEM,
            # tests
            num_workers=8,
            # maybe this is the culprit as suggested by user12750353 in stackoverflow
            # pin_memory=True
            pin_memory=False
        )


title: Pip list shows only one version of `torch`, yet conda list shows two

body:
pip list

conda list

I once install torch 0.1.12, then I remove it, and reinstall it.
How can I remove the 0.1.12 version?


title: How to initialize decoder's state when it has multiple layers?

body:
I'm using PyTorch/Tensorflow to implement a seq2seq model.

When the encoder has m layers while the decoder has n layers, should one use only the state of the last encoder layer to initialize the state of the first decoder layer and keep all the states of layers above zeros?

Or, when m == n, should one use all states of the encoder to initialize the states of the corresponding layer of the decoder?

If the encoder(decoder) is bidirectional while the decoder(encoder) is not, how to initialize the hidden state of the decoder in this situation?


title: The loss of semantic segmentation of binary classes with inbalanced data dose not converge

body:
I want to segment objects which just occupy a little part of the whole dataset(e.g. the ICDAR 2015 or the person in CamVid). The ratio between positive and negtive sample in pixel-level is about 1:200.

My network, whose backbone is pre-trained VGG16 or ResNet50, could work well in the CamVid dataset with 12 classes. However, when I fine-tuned this network to segment only person in CamVid, the loss will stay around 0.31 and will never decrease even with a truly small learning-rate (like 1e-5). What's more, when training a new model on another dataset (ICDAR 2015) to get the area of text in images, I encountered the same issue.

I built my model with pytorch, and the loss function used by me is Weighted Cross-Entropy Loss.

Could anybody tell me where the problem is? Is that about the loss function? Because I think there is no problem in the model architecture. Really appreciate for any suggestions.


title: pytorch: How does loss behave when coming from two networks?

body:
I am trying to implement the following algorithm in this book, section 13.5, in pytorch.

This would require two separate neural networks, (in this question, model1 and model2). One's loss is dependent only on its own output [via delta] (parameterized by w), the other (parameterized by theta), dependent both on its own output [via ln(pi)], and on the other's output [again, via delta].

I want to update each one separately

Assume the following models implement nn.Module:

model1 = Mynet1()
model2 = Mynet2()

val1 = model1(input1)
val2 = model2(input2)

self.optimizer1 = optim.Adam(model1.parameters(), lr1)
self.optimizer2 = optim.Adam(model2.parameters(), lr2)

loss1 = f(val1)
loss2 = f(val1, val2)#THIS IS THE INTERESTING PART

optim1.zero_grad()
loss1.backward
optim1.step()

optim2.zero_grad()
loss2.backward
optim2.step()

I understand that applying backward on loss1, then stepping its optimizer would update model1's parameters.

My question is what happens when activating the same on loss2, model2, optimizer2, where loss 2 is dependant on outputs both from model1 and model2?

How can I make the loss2 update not affect model1 parameters?


title: Stretch the values of a pytorch tensor

body:
I wish to 'stretch' the last two dimensions of a pytorch tensor to increase the spatial resolution of a (batch, channels, y, x) tensor.

Minimal example (I need 'new_function')

a = torch.tensor([[1, 2], [3, 4]])
b = new_function(a, (2, 3))
print(b)
tensor([[1, 1, 1, 2, 2, 2],
        [1, 1, 1, 2, 2, 2],
        [3, 3, 3, 4, 4, 4],
        [3, 3, 3, 4, 4, 4]])

One way of doing this (for the real problem):

a = torch.ones((2, 256, 2, 2)) # my original data.
b = torch.zeros((2, 256, 80, 96)) # The output I need
b[:, :, :40, :48] = a[:, :, 0, 0]
b[:, :, 40:, :48] = a[:, :, 1, 0]
b[:, :, :40, 48:] = a[:, :, 0, 1]
b[:, :, 40:, 48:] = a[:, :, 1, 1]


title: Pytorch-Implement the same model in pytorch and keras but got different results

body:
I am learning pytorch and want to practice it with an keras example (https://keras.io/examples/lstm_seq2seq/), this is a seq2seq 101 example which translate eng to fra on char-level features (no embedding).

Keras code is below:

from keras.models import Model
from keras.layers import Input, LSTM, Dense
import numpy as np

batch_size = 64  # Batch size for training.
epochs = 100  # Number of epochs to train for.
latent_dim = 256  # Latent dimensionality of the encoding space.
num_samples = 10000  # Number of samples to train on.
# Path to the data txt file on disk.
data_path = 'fra-eng/fra.txt'

# Vectorize the data.
input_texts = []
target_texts = []
input_characters = set()
target_characters = set()
with open(data_path, 'r', encoding='utf-8') as f:
    lines = f.read().split('\n')
for line in lines[: min(num_samples, len(lines) - 1)]:
    input_text, target_text = line.split('\t')
    # We use "tab" as the "start sequence" character
    # for the targets, and "\n" as "end sequence" character.
    target_text = '\t' + target_text + '\n'
    input_texts.append(input_text)
    target_texts.append(target_text)
    for char in input_text:
        if char not in input_characters:
            input_characters.add(char)
    for char in target_text:
        if char not in target_characters:
            target_characters.add(char)

input_characters = sorted(list(input_characters))
target_characters = sorted(list(target_characters))
num_encoder_tokens = len(input_characters)
num_decoder_tokens = len(target_characters)
max_encoder_seq_length = max([len(txt) for txt in input_texts])
max_decoder_seq_length = max([len(txt) for txt in target_texts])

print('Number of samples:', len(input_texts))
print('Number of unique input tokens:', num_encoder_tokens)
print('Number of unique output tokens:', num_decoder_tokens)
print('Max sequence length for inputs:', max_encoder_seq_length)
print('Max sequence length for outputs:', max_decoder_seq_length)

input_token_index = dict(
    [(char, i) for i, char in enumerate(input_characters)])
target_token_index = dict(
    [(char, i) for i, char in enumerate(target_characters)])

encoder_input_data = np.zeros(
    (len(input_texts), max_encoder_seq_length, num_encoder_tokens),
    dtype='float32')
decoder_input_data = np.zeros(
    (len(input_texts), max_decoder_seq_length, num_decoder_tokens),
    dtype='float32')
decoder_target_data = np.zeros(
    (len(input_texts), max_decoder_seq_length, num_decoder_tokens),
    dtype='float32')

for i, (input_text, target_text) in enumerate(zip(input_texts, target_texts)):
    for t, char in enumerate(input_text):
        encoder_input_data[i, t, input_token_index[char]] = 1.
    for t, char in enumerate(target_text):
        # decoder_target_data is ahead of decoder_input_data by one timestep
        decoder_input_data[i, t, target_token_index[char]] = 1.
        if t > 0:
            # decoder_target_data will be ahead by one timestep
            # and will not include the start character.
            decoder_target_data[i, t - 1, target_token_index[char]] = 1.

# Define an input sequence and process it.
encoder_inputs = Input(shape=(None, num_encoder_tokens))
encoder = LSTM(latent_dim, return_state=True)
encoder_outputs, state_h, state_c = encoder(encoder_inputs)
# We discard `encoder_outputs` and only keep the states.
encoder_states = [state_h, state_c]

# Set up the decoder, using `encoder_states` as initial state.
decoder_inputs = Input(shape=(None, num_decoder_tokens))
# We set up our decoder to return full output sequences,
# and to return internal states as well. We don't use the
# return states in the training model, but we will use them in inference.
decoder_lstm = LSTM(latent_dim, return_sequences=True, return_state=True)
decoder_outputs, _, _ = decoder_lstm(decoder_inputs,
                                     initial_state=encoder_states)
decoder_dense = Dense(num_decoder_tokens, activation='softmax')
decoder_outputs = decoder_dense(decoder_outputs)

# Define the model that will turn
# `encoder_input_data` & `decoder_input_data` into `decoder_target_data`
model = Model([encoder_inputs, decoder_inputs], decoder_outputs)

# Run training
model.compile(optimizer='rmsprop', loss='categorical_crossentropy')
model.fit([encoder_input_data, decoder_input_data], decoder_target_data,
          batch_size=batch_size,
          epochs=epochs,
          validation_split=0.2)
# Save model
model.save('s2s.h5')

# Next: inference mode (sampling).
# Here's the drill:
# 1) encode input and retrieve initial decoder state
# 2) run one step of decoder with this initial state
# and a "start of sequence" token as target.
# Output will be the next target token
# 3) Repeat with the current target token and current states

# Define sampling models
encoder_model = Model(encoder_inputs, encoder_states)

decoder_state_input_h = Input(shape=(latent_dim,))
decoder_state_input_c = Input(shape=(latent_dim,))
decoder_states_inputs = [decoder_state_input_h, decoder_state_input_c]
decoder_outputs, state_h, state_c = decoder_lstm(
    decoder_inputs, initial_state=decoder_states_inputs)
decoder_states = [state_h, state_c]
decoder_outputs = decoder_dense(decoder_outputs)
decoder_model = Model(
    [decoder_inputs] + decoder_states_inputs,
    [decoder_outputs] + decoder_states)

# Reverse-lookup token index to decode sequences back to
# something readable.
reverse_input_char_index = dict(
    (i, char) for char, i in input_token_index.items())
reverse_target_char_index = dict(
    (i, char) for char, i in target_token_index.items())

def decode_sequence(input_seq):
    # Encode the input as state vectors.
    states_value = encoder_model.predict(input_seq)

    # Generate empty target sequence of length 1.
    target_seq = np.zeros((1, 1, num_decoder_tokens))
    # Populate the first character of target sequence with the start character.
    target_seq[0, 0, target_token_index['\t']] = 1.

    # Sampling loop for a batch of sequences
    # (to simplify, here we assume a batch of size 1).
    stop_condition = False
    decoded_sentence = ''
    while not stop_condition:
        output_tokens, h, c = decoder_model.predict(
            [target_seq] + states_value)

        # Sample a token
        sampled_token_index = np.argmax(output_tokens[0, -1, :])
        sampled_char = reverse_target_char_index[sampled_token_index]
        decoded_sentence += sampled_char

        # Exit condition: either hit max length
        # or find stop character.
        if (sampled_char == '\n' or
           len(decoded_sentence) > max_decoder_seq_length):
            stop_condition = True

        # Update the target sequence (of length 1).
        target_seq = np.zeros((1, 1, num_decoder_tokens))
        target_seq[0, 0, sampled_token_index] = 1.

        # Update states
        states_value = [h, c]

    return decoded_sentence

for seq_index in range(100):
    # Take one sequence (part of the training set)
    # for trying out decoding.
    input_seq = encoder_input_data[seq_index: seq_index + 1]
    decoded_sentence = decode_sequence(input_seq)
    print('-')
    print('Input sentence:', input_texts[seq_index])
    print('Decoded sentence:', decoded_sentence)

I want to implement this exact same model using pytorch, below is my code:

from __future__ import unicode_literals, print_function, division
from io import open
import unicodedata
import string
import re
import random
import numpy as np
import torch
import torch.nn as nn
from torch import optim
import torch.nn.functional as F

device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
data_path = './eng_fra.txt'

# Vectorize the data.
input_texts = []
target_texts = []
input_characters = set()
target_characters = set()
with open(data_path, 'r', encoding='utf-8') as f:
    lines = f.read().split('\n')
for line in lines[: min(num_samples, len(lines) - 1)]:
    #print('line:',line)
    input_text, target_text = line.split('\t')
    # We use "tab" as the "start sequence" character
    # for the targets, and "\n" as "end sequence" character.
    target_text = '\t' + target_text + '\n' # why?
   # print('input_text and target_text:',input_text, target_text)
    input_texts.append(input_text)
    target_texts.append(target_text)
    for char in input_text:
        if char not in input_characters:
            input_characters.add(char)
    for char in target_text:
        if char not in target_characters:
            target_characters.add(char)

input_characters = sorted(list(input_characters))
target_characters = sorted(list(target_characters))
num_encoder_tokens = len(input_characters)
print('input_characters',input_characters)
num_decoder_tokens = len(target_characters)
print('target_characters',target_characters)
max_encoder_seq_length = max([len(txt) for txt in input_texts])
max_decoder_seq_length = max([len(txt) for txt in target_texts])
print('max_encoder_seq_length and max_decoder_seq_length',max_encoder_seq_length,max_decoder_seq_length)

input_token_index = dict(
    [(char, i) for i, char in enumerate(input_characters)])
target_token_index = dict(
    [(char, i) for i, char in enumerate(target_characters)])

# define the shapes
encoder_input_data = np.zeros(
    (len(input_texts), max_encoder_seq_length, num_encoder_tokens),
    dtype='float32')
decoder_input_data = np.zeros(
    (len(input_texts), max_decoder_seq_length, num_decoder_tokens),
    dtype='float32')
decoder_target_data = np.zeros(
    (len(input_texts), max_decoder_seq_length, num_decoder_tokens),
    dtype='float32')

# one hot encoding for each word in each sentence
for i, (input_text, target_text) in enumerate(zip(input_texts, target_texts)):
    for t, char in enumerate(input_text):
        encoder_input_data[i, t, input_token_index[char]] = 1.
    for t, char in enumerate(target_text):
        # decoder_target_data is ahead of decoder_input_data by one timestep
        decoder_input_data[i, t, target_token_index[char]] = 1.
        if t > 0:
            # decoder_target_data will be ahead by one timestep
            # and will not include the start character.
            decoder_target_data[i, t - 1, target_token_index[char]] = 1.
encoder_input_data=torch.Tensor(encoder_input_data).to(device)
decoder_input_data=torch.Tensor(decoder_input_data).to(device)
decoder_target_data=torch.Tensor(decoder_target_data).to(device)

class encoder(nn.Module):
  def __init__(self):
    super(encoder,self).__init__()
    self.LSTM=nn.LSTM(input_size=num_encoder_tokens,hidden_size=256,batch_first=True)
  def forward(self,x):
    out,(h,c)=self.LSTM(x)
    return h,c

class decoder(nn.Module):
  def __init__(self):
    super(decoder,self).__init__()
    self.LSTM=nn.LSTM(input_size=num_decoder_tokens,hidden_size=256,batch_first=True)
    self.FC=nn.Linear(256,num_decoder_tokens)

  def forward(self,x, hidden):
    out,(h,c)=self.LSTM(x,hidden)
    out=self.FC(out)
    return out,(h,c)

class seq2seq(nn.Module):
  def __init__(self,encoder,decoder):
    super(seq2seq,self).__init__()
    self.encoder=encoder
    self.decoder=decoder

  def forward(self,encode_input_data,decode_input_data):
    hidden, cell = self.encoder(encode_input_data)
    output, (hidden, cell) = self.decoder(decode_input_data, (hidden, cell))
    return output

encoder=encoder().to(device)
# encoder_loss = nn.CrossEntropyLoss() # CrossEntropyLoss compute softmax internally in pytorch
# encoder_optimizer = torch.optim.Adam(encoder.parameters(), lr=0.001)

decoder=decoder().to(device)
# decoder_loss = nn.CrossEntropyLoss() # CrossEntropyLoss compute softmax internally in pytorch
# decoder_optimizer = torch.optim.Adam(decoder.parameters(), lr=0.001)

model=seq2seq(encoder,decoder).to(device)
optimizer = optim.RMSprop(model.parameters(),lr=0.01)
loss_fun=nn.CrossEntropyLoss()

# model.train()

num_epochs=50
batches=np.array_split(range(decoder_target_data.shape[0]),100)
total_step=len(batches)
for epoch in range(num_epochs):

  for i,batch_ids in enumerate(batches):
    encoder_input=encoder_input_data[batch_ids]
    decoder_input=decoder_input_data[batch_ids]
    decoder_target=decoder_target_data[batch_ids]

    output = model(encoder_input, decoder_input)
    loss=loss_fun(output.view(-1,93).to(device),decoder_target.view(-1,93).max(dim=1)[1].to(device))

    # Backward and optimize
    optimizer.zero_grad()
    loss.backward()
    optimizer.step()

    if (i+1) % 20 == 0:
        print ('Epoch [{}/{}], Step [{}/{}], Loss: {:.4f}'
               .format(epoch+1, num_epochs, i+1, total_step, loss.item()))

# Reverse-lookup token index to decode sequences back to
# something readable.
reverse_input_char_index = dict(
    (i, char) for char, i in input_token_index.items())
reverse_target_char_index = dict(
    (i, char) for char, i in target_token_index.items())

def decode_sequence(input_seq):
    # Encode the input as state vectors.
    h,c=model.encoder(input_seq)

    # Generate empty target sequence of length 1.
    # Populate the first character of target sequence with the start character.
    target_seq = torch.zeros((1, 1, num_decoder_tokens)).to(device)
    target_seq[0, 0, target_token_index['\t']] = 1.

    # Sampling loop for a batch of sequences
    # (to simplify, here we assume a batch of size 1).
    stop_condition = False
    decoded_sentence = ''
    while not stop_condition:
        output_tokens, (h_t, c_t) = model.decoder(target_seq,(h,c))

        # Sample a token
        sampled_token_index = output_tokens.view(-1,93).squeeze(0).max(dim=0)[1].item()
        sampled_char = reverse_target_char_index[sampled_token_index]
        decoded_sentence += sampled_char

        # Exit condition: either hit max length
        # or find stop character.
        if (sampled_char == '\n' or
           len(decoded_sentence) > max_decoder_seq_length):
            stop_condition = True

        # Update the target sequence (of length 1).
        target_seq = torch.zeros((1, 1, num_decoder_tokens)).to(device)
        target_seq[0, 0, sampled_token_index] = 1.

        # Update states
        h,c=h_t,c_t

    return decoded_sentence

for seq_index in range(100):
    # Take one sequence (part of the training set)
    # for trying out decoding.
    input_seq = encoder_input_data[seq_index: seq_index + 1]
    decoded_sentence = decode_sequence(input_seq)
    print('-')
    print('Input sentence:', input_texts[seq_index])
    print('Decoded sentence:', decoded_sentence)

As you can see, I used exactly the same data processing and model structure. My pytorch version can run without error, but the performance seems worse than the original keras version by comparing the translation results.

One thing might cause error is the loss function (cross_entropy). In pytorch, cross_entropy loss function seems does not support one-hot labels directly, which I need to change the label to integer. However I don't think this should make big difference.

If you want to run the models, the data can be downloaded from:
https://github.com/jinfagang/pytorch_chatbot/blob/master/datasets/eng-fra.txt

Did I do something wrong in my code? Many thanks


title: Can the coordinates of a 2 dimensional pytorch tensor be remapped?

body: I'm trying to remap a 2d image to new coordinates. Previously I was using OpenCV's cv.remap method. I'd like to use CUDA with compiling C directly. Is there a Pytorch version of remap that will interpolate values?