title: PyTorch - `THCudaCheck FAIL file=..\aten\src\THC\THCCachingHostAllocator.cpp line=278 error=719 : unspecified launch failure` after a few epochs?

body:
Getting THCudaCheck FAIL file=..\aten\src\THC\THCCachingHostAllocator.cpp line=278 error=719 : unspecified launch failure error during the execution of epochs of my LSTM model after a couple of epochs. The error stacktrace points the line out, hidden = self.rnn(x, hidden) in the forward function as the reason for error.

Here is my network model:

import torch
from torch.utils.data import TensorDataset, DataLoader
import torch.nn as nn

import numpy as np
import pandas as pd
from time import time

class SignalNet(nn.Module):
    def __init__(self, input_size, output_size, hidden_dim, num_layers):
        super(SignalNet, self).__init__()

        self.hidden_dim = hidden_dim
        self.num_layers = num_layers

        self.rnn = nn.LSTM(input_size, hidden_dim, num_layers, batch_first=True)
        self.fc = nn.Linear(hidden_dim, output_size)

    def forward(self, x):
        batch_size = x.size(0)

        hidden = self.init_hidden(batch_size)

        out, hidden = self.rnn(x, hidden)

        out = out.contiguous().view(-1, self.hidden_dim)
        out = self.fc(out)

        return out, hidden

    def init_hidden(self, batch_size):
        return (torch.zeros(self.num_layers, batch_size, self.hidden_dim).to(device),
                torch.zeros(self.num_layers, batch_size, self.hidden_dim).to(device))

def main():
    global device
    is_cuda = torch.cuda.is_available()

    if is_cuda:
        device = torch.device("cuda")
    else:
        device = torch.device("cpu")

    input_dim = 8
    batch_size = 1024  # was 32
    output_dim = 1
    num_layers = 5
    hidden_dim = 10
    learning_rate = 0.1
    num_epochs = 5

    model = SignalNet(input_size=input_dim, output_size=output_dim, hidden_dim=hidden_dim, num_layers=num_layers)
    model.to(device)

    loss_fn = torch.nn.MSELoss(reduction='sum')
    optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)

    df = pd.read_csv('data\\emg2.csv', sep=',')

    split_frac = 0.8  # 80% train, 20% test
    split_id = int(split_frac * len(df))

    train_data, train_labels = df.iloc[:split_id, :-1], df.iloc[:split_id, -1]
    test_data, test_labels = df.iloc[split_id:len(df) * 9 // 10, :-1], df.iloc[split_id:len(df) * 9 // 10, -1]
    val_data, val_labels = df.iloc[len(df) * 9 // 10:, :-1], df.iloc[len(df) * 9 // 10:, -1]

    # LSTM starts HERE
    train_dataset = TensorDataset(torch.from_numpy(train_data.values).float(),
                                  torch.from_numpy(train_labels.values).float())
    val_dataset = TensorDataset(torch.from_numpy(val_data.values).float(),
                                torch.from_numpy(val_labels.values).float())
    test_dataset = TensorDataset(torch.from_numpy(test_data.values).float(),
                                 torch.from_numpy(test_labels.values).float())

    train_loader = DataLoader(train_dataset, shuffle=True, batch_size=batch_size, num_workers=6)
    val_loader = DataLoader(val_dataset, shuffle=True, batch_size=batch_size, num_workers=6)
    test_loader = DataLoader(test_dataset, shuffle=True, batch_size=batch_size, num_workers=6)

    # Training Run
    model.train()
    for epoch in range(1, num_epochs + 1):
        epoch_loss = 0
        index = 0
        for batch_idx, (X_train, y_train) in enumerate(train_loader):
            optimizer.zero_grad()  # Clears existing gradients from previous epoch
            X_train, y_train = X_train.to(device), y_train.to(device)
            # model.init_hidden(batch_size)
            output, hidden = model(X_train[np.newaxis, ...])
            # loss = loss_fn(output, y_train.view(-1).long())
            loss = loss_fn(output.view(-1), y_train.view(-1))
            epoch_loss += loss.item()
            loss.backward()  # Does backpropagation and calculates gradients
            optimizer.step()  # Updates the weights accordingly

            if batch_idx % 50 == 0:
                print(f'Epoch: {epoch}/{num_epochs} Batch #{batch_idx + 1}/{len(train_loader)}.............', end=' ')
                print("Loss: {:.4f}".format(loss.item()))
            index = batch_idx

        print(f'Epoch #{epoch}: Avg. loss: {epoch_loss / index + 1}')

    # TEST
    test_losses = []
    num_correct = 0
    h = model.init_hidden(batch_size)

    model.eval()
    for inputs, labels in test_loader:
        h = tuple([each.data for each in h])
        inputs, labels = inputs.to(device), labels.to(device)
        output, h = model(inputs[np.newaxis, ...])
        test_loss = loss_fn(output.squeeze(), labels.float())
        test_losses.append(test_loss.item())
        pred = torch.round(output.squeeze())  # Rounds the output to 0/1
        correct_tensor = pred.eq(labels.float().view_as(pred))
        correct = np.squeeze(correct_tensor.cpu().numpy())
        num_correct += np.sum(correct)

    print("Test loss: {:.3f}".format(np.mean(test_losses)))
    test_acc = num_correct / len(test_loader.dataset)
    print("Test accuracy: {:.3f}%".format(test_acc * 100))

if __name__ == '__main__':
    main()

Here is error stacktrace:

Epoch: 1/5 Batch #1/235............. Loss: 5569.8594
Epoch: 1/5 Batch #51/235............. Loss: 3521.5000
Epoch: 1/5 Batch #101/235............. Loss: 3525.8384
Epoch: 1/5 Batch #151/235............. Loss: 3854.3398
Epoch: 1/5 Batch #201/235............. Loss: 3402.1775
Epoch #1: Avg. loss: 3645.545551528279
Epoch: 2/5 Batch #1/235............. Loss: 3649.2971
Traceback (most recent call last):
  File "C:\Program Files\JetBrains\PyCharm 2018.1.2\helpers\pydev\pydevd.py", line 1415, in _exec
    pydev_imports.execfile(file, globals, locals)  # execute the script
  File "C:\Program Files\JetBrains\PyCharm 2018.1.2\helpers\pydev\_pydev_imps\_pydev_execfile.py", line 18, in execfile
    exec(compile(contents+"\n", file, 'exec'), glob, loc)
  File "D:/PyCharmProjects/signal_processing/run_5.py", line 149, in <module>
    main()
  File "D:/PyCharmProjects/signal_processing/run_5.py", line 111, in main
    output, hidden = model(X_train[np.newaxis, ...])
  File "D:\.virtualenvs\signal_processing-0P6HK7Nb\lib\site-packages\torch\nn\modules\module.py", line 541, in __call__
    result = self.forward(*input, **kwargs)
  File "D:/PyCharmProjects/signal_processing/run_5.py", line 32, in forward
    out, hidden = self.rnn(x, hidden)
  File "D:\.virtualenvs\signal_processing-0P6HK7Nb\lib\site-packages\torch\nn\modules\module.py", line 541, in __call__
    result = self.forward(*input, **kwargs)
  File "D:\.virtualenvs\signal_processing-0P6HK7Nb\lib\site-packages\torch\nn\modules\rnn.py", line 564, in forward
    return self.forward_tensor(input, hx)
  File "D:\.virtualenvs\signal_processing-0P6HK7Nb\lib\site-packages\torch\nn\modules\rnn.py", line 543, in forward_tensor
    output, hidden = self.forward_impl(input, hx, batch_sizes, max_batch_size, sorted_indices)
  File "D:\.virtualenvs\signal_processing-0P6HK7Nb\lib\site-packages\torch\nn\modules\rnn.py", line 526, in forward_impl
    self.dropout, self.training, self.bidirectional, self.batch_first)
RuntimeError: cuDNN error: CUDNN_STATUS_INTERNAL_ERROR
THCudaCheck FAIL file=..\aten\src\THC\THCCachingHostAllocator.cpp line=278 error=719 : unspecified launch failure


title: I want to use the GPU instead of CPU while performing computations using PyTorch

body:
I'm trying to switch the stress from CPU to GPU as my trusty RTX2070 can do it better than the CPU but I keep running into this problem and I'm quite new to AI so if you are kind enough to share some insights with me regarding any potential solution, it would be highly appreciated, thank you.

**I'm using PyTorch

Here's the code that I'm using :

# to measure run-time

# for csv dataset
import os
# to shuffle data
import random
# to get the alphabet
import string

# import statements for iterating over csv file
import cv2
# for plotting
import matplotlib.pyplot as plt
import numpy as np
# pytorch stuff
import torch
import torch.nn as nn
from PIL import Image

# generate the targets
# the targets are one hot encoding vectors
# print(torch.cuda.is_available())
nvcc_args = [
    '-gencode', 'arch=compute_30,code=sm_30',
    '-gencode', 'arch=compute_35,code=sm_35',
    '-gencode', 'arch=compute_37,code=sm_37',
    '-gencode', 'arch=compute_50,code=sm_50',
    '-gencode', 'arch=compute_52,code=sm_52',
    '-gencode', 'arch=compute_60,code=sm_60',
    '-gencode', 'arch=compute_61,code=sm_61',
    '-gencode', 'arch=compute_70,code=sm_70',
    '-gencode', 'arch=compute_75,code=sm_75'
]

alphabet = list(string.ascii_lowercase)
target = {}

# Initalize a target dict that has letters as its keys and empty one-hot encoding vectors of size 37 as its values
for letter in alphabet:
    target[letter] = [0] * 37

# Do the one-hot encoding for each letter now
curr_pos = 0
for curr_letter in target.keys():
    target[curr_letter][curr_pos] = 1
    curr_pos += 1

# extra symbols
symbols = ["space", "number", "period", "comma", "colon", "apostrophe", "hyphen", "semicolon", "question",
           "exclamation", "capitalize"]

# create vectors
for curr_symbol in symbols:
    target[curr_symbol] = [0] * 37

# create one-hot encoding vectors
for curr_symbol in symbols:
    target[curr_symbol][curr_pos] = 1
    curr_pos += 1

# collect all data from the csv file
data = []

for tgt in os.listdir("dataset"):
    if not tgt == ".DS_Store":
        for folder in os.listdir("dataset/" + tgt + "/Uploaded"):
            if not folder == ".DS_Store":
                for filename in os.listdir("dataset/" + tgt + "/Uploaded/" + folder):
                    if not filename == ".DS_Store":
                        # store the image and label
                        picture = []
                        curr_target = target[tgt]
                        image = Image.open("dataset/" + tgt + "/Uploaded/" + folder + "/" + filename)
                        image = image.convert('RGB')
                        # f.show()
                        image = np.array(image)
                        # resize image to 28x28x3
                        image = cv2.resize(image, (28, 28))
                        # normalize to 0-1
                        image = image.astype(np.float32) / 255.0
                        image = torch.from_numpy(image)
                        picture.append(image)
                        # convert the target to a long tensor
                        curr_target = torch.Tensor([curr_target])
                        picture.append(curr_target)
                        # append the current image & target
                        data.append(picture)

# create a dictionary of all the characters
characters = alphabet + symbols

index2char = {}
number = 0
for char in characters:
    index2char[number] = char
    number += 1

# find the number of each character in a dataset
def num_chars(dataset, index2char):
    chars = {}
    for _, label in dataset:
        char = index2char[int(torch.argmax(label))]
        # update
        if char in chars:
            chars[char] += 1
        # initialize
        else:
            chars[char] = 1
    return chars

# Create dataloader objects

# shuffle all the data
random.shuffle(data)

# batch sizes for train, test, and validation
batch_size_train = 30
batch_size_test = 30
batch_size_validation = 30

# splitting data to get training, test, and validation sets
# change once get more data
# 1600 for train
train_dataset = data[:22000]
# test has 212
test_dataset = data[22000:24400]
# validation has 212
validation_dataset = data[24400:]

# create the dataloader objects
train_loader = torch.utils.data.DataLoader(dataset=train_dataset, batch_size=batch_size_train, shuffle=True)
test_loader = torch.utils.data.DataLoader(dataset=test_dataset, batch_size=batch_size_test, shuffle=False)
validation_loader = torch.utils.data.DataLoader(dataset=validation_dataset, batch_size=batch_size_validation,
                                                shuffle=True)

# to check if a dataset is missing a char
test_chars = num_chars(test_dataset, index2char)

num = 0
for char in characters:
    if char in test_chars:
        num += 1
    else:
        break
print(num)

class CNN(nn.Module):
    def __init__(self):
        self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
        super(CNN, self).__init__()
        self.block1 = nn.Sequential(
            # 3x28x28
            nn.Conv2d(in_channels=3,
                      out_channels=16,
                      kernel_size=5,
                      stride=1,
                      padding=2),
            # batch normalization
            # nn.BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True),
            # 16x28x28
            nn.MaxPool2d(kernel_size=2),
            # 16x14x14
            nn.LeakyReLU()
        )
        # 16x14x14
        self.block2 = nn.Sequential(
            nn.Conv2d(in_channels=16,
                      out_channels=32,
                      kernel_size=5,
                      stride=1,
                      padding=2),
            # batch normalization
            # nn.BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True),
            # 32x14x14
            nn.MaxPool2d(kernel_size=2),
            # 32x7x7
            nn.LeakyReLU()
        )
        # linearly
        self.block3 = nn.Sequential(
            nn.Linear(32 * 7 * 7, 100),
            # batch normalization
            # nn.BatchNorm1d(100),
            nn.LeakyReLU(),
            nn.Linear(100, 37)
        )
        # 1x37

    def forward(self, x):
        out = self.block1(x)
        out = self.block2(out)
        # flatten the dataset
        out = out.view(-1, 32 * 7 * 7)
        out = self.block3(out)
        return out

# convolutional neural network model
model = CNN()
model.cuda()

# print summary of the neural network model to check if everything is fine.
print(model)
print("# parameter: ", sum([param.nelement() for param in model.parameters()]))

# setting the learning rate
learning_rate = 1e-4

# Using a variable to store the cross entropy method
criterion = nn.CrossEntropyLoss()

# Using a variable to store the optimizer
optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)

# list of all train_losses
train_losses = []

# list of all validation losses
validation_losses = []

# for loop that iterates over all the epochs
num_epochs = 20
for epoch in range(num_epochs):

    # variables to store/keep track of the loss and number of iterations
    train_loss = 0
    num_iter_train = 0

    # train the model
    model.train()

    # Iterate over train_loader
    for i, (images, labels) in enumerate(train_loader):
        # need to permute so that the images are of size 3x28x28
        # essential to be able to feed images into the model
        images = images.permute(0, 3, 1, 2)

        # Zero the gradient buffer
        # resets the gradient after each epoch so that the gradients don't add up
        optimizer.zero_grad()

        # Forward, get output
        outputs = model(images)

        # convert the labels from one hot encoding vectors into integer values
        labels = labels.view(-1, 37)
        y_true = torch.argmax(labels, 1)

        # calculate training loss
        loss = criterion(outputs, y_true)

        # Backward (computes all the gradients)
        loss.backward()

        # Optimize
        # loops through all parameters and updates weights by using the gradients
        # takes steps backwards to optimize (to reach the minimum weight)
        optimizer.step()
        # update the training loss and number of iterations
        train_loss += loss.data
        num_iter_train += 1

    print('Epoch: {}'.format(epoch + 1))
    print('Training Loss: {:.4f}'.format(train_loss / num_iter_train))
    # append training loss over all the epochs
    train_losses.append(train_loss / num_iter_train)

    # evaluate the model
    model.eval()

    # variables to store/keep track of the loss and number of iterations
    validation_loss = 0
    num_iter_validation = 0

    # Iterate over validation_loader
    for i, (images, labels) in enumerate(validation_loader):
        # need to permute so that the images are of size 3x28x28
        # essential to be able to feed images into the model
        images = images.permute(0, 3, 1, 2)

        # Forward, get output
        outputs = model(images)

        # convert the labels from one hot encoding vectors to integer values
        labels = labels.view(-1, 37)
        y_true = torch.argmax(labels, 1)

        # calculate the validation loss
        loss = criterion(outputs, y_true)

        # update the training loss and number of iterations
        validation_loss += loss.data
        num_iter_validation += 1

    print('Validation Loss: {:.4f}'.format(validation_loss / num_iter_validation))
    # append all validation_losses over all the epochs
    validation_losses.append(validation_loss / num_iter_validation)

    num_iter_test = 0
    correct = 0

    # Iterate over test_loader
    for images, labels in test_loader:
        # need to permute so that the images are of size 3x28x28
        # essential to be able to feed images into the model
        images = images.permute(0, 3, 1, 2)

        # Forward
        outputs = model(images)

        # convert the labels from one hot encoding vectors into integer values
        labels = labels.view(-1, 37)
        y_true = torch.argmax(labels, 1)

        # find the index of the prediction
        y_pred = torch.argmax(outputs, 1).type('torch.FloatTensor')

        # convert to FloatTensor
        y_true = y_true.type('torch.FloatTensor')

        # find the mean difference of the comparisons
        correct += torch.sum(torch.eq(y_true, y_pred).type('torch.FloatTensor'))

    print('Accuracy on the test set: {:.4f}%'.format(correct / len(test_dataset) * 100))
    print()

# learning curve function
def plot_learning_curve(train_losses, validation_losses):
    # plot the training and validation losses
    plt.ylabel('Loss')
    plt.xlabel('Number of Epochs')
    plt.plot(train_losses, label="training")
    plt.plot(validation_losses, label="validation")
    plt.legend(loc=1)

# plot the learning curve
plt.title("Learning Curve (Loss vs Number of Epochs)")
plot_learning_curve(train_losses, validation_losses)

torch.save(model.state_dict(), "model1.pth")


title: What is the state of the art way of doing regression with probability in pytorch

body:
All regression examples I find are examples where you predict a real number and unlike with classification you dont the the confidence the model had when predicting that number. I have done in reinforcement learning another way the output is instead the mean and std and then you sample from that distribution. Then you know how confident the model is at predicting every value. Now I cant find how to do this using supervised learning in pytorch. The problem is that I dont understand how to perform sample from the distribution the get the actual value while training or what sort of loss function I should use, not sure how for example MSE or L1Smooth would work.

Is there any example ot there where this is done in pytorch in a robust and state of the art way?


title: How to set PyTorch shared memory size?

body:
I want to put the elements in a list to shared memory. The code (main.py) is as follows.

import torch
if __name__=='__main__':
    mylist = []
    list_len = 10000
    for i in range(list_len):
        element = torch.randn(10)
        mylist.append(element)

    for element in mylist:
        element.share_memory_()

The problem is that, when I run "python main.py" from command line, it outputs the following error

Traceback (most recent call last):
  File "main.py", line 9, in <module>
  File "/home/rozyang/anaconda3/lib/python3.6/site-packages/torch/tensor.py", line 217, in share_memory_
  File "/home/rozyang/anaconda3/lib/python3.6/site-packages/torch/storage.py", line 104, in share_memory_
RuntimeError: unable to open shared memory object </torch_1543_4219316339> in read-write mode at /opt/conda/conda-bld/pytorch_1524590031827/work/aten/src/TH/THAllocator.c:342

However, no error appears when I run the same code in PyCharm. I have checked that PyCharm is using the same python interpreter and same PyTorch.

If I change "list_len=10000" to "list_len=100", the command line "python main.py" also works.

So I am wondering if there is somewhere I can set the shared memory size (i.e. the number of shared variables). Many thanks to any hints to solve this!

P.S. I am using anaconda python 3.6 and pytorch 0.4.0


title: how to build a multidimensional autoencoder with pytorch

body:
I followed this great answer for sequence autoencoder,

LSTM autoencoder always returns the average of the input sequence.

but I met some problem when I try to change the code:

question one:
Your explanation is so professional, but the problem is a little bit different from mine, I attached some code I changed from your example. My input features are 2 dimensional, and my output is same with the input.
for example:

input_x = torch.Tensor([[0.0,0.0], [0.1,0.1], [0.2,0.2], [0.3,0.3], [0.4,0.4]])
output_y = torch.Tensor([[0.0,0.0], [0.1,0.1], [0.2,0.2], [0.3,0.3], [0.4,0.4]])
the input_x and output_y are same, 5-timesteps, 2-dimensional feature.

        import torch
        import torch.nn as nn
        import torch.optim as optim

        class LSTM(nn.Module):
            def __init__(self, input_dim, latent_dim, num_layers):
                super(LSTM, self).__init__()
               self.input_dim = input_dim
                self.latent_dim = latent_dim
                self.num_layers = num_layers
                self.encoder = nn.LSTM(self.input_dim, self.latent_dim, self.num_layers)

                # I changed here, to 40 dimesion, I think there is some problem
                # self.decoder = nn.LSTM(self.latent_dim, self.input_dim, self.num_layers)
                self.decoder = nn.LSTM(40, self.input_dim, self.num_layers)

            def forward(self, input):
                # Encode
                _, (last_hidden, _) = self.encoder(input)
                # It is way more general that way
                encoded = last_hidden.repeat(input.shape)
                # Decode
                y, _ = self.decoder(encoded)
               return torch.squeeze(y)

        model = LSTM(input_dim=2, latent_dim=20, num_layers=1)
        loss_function = nn.MSELoss()
        optimizer = optim.Adam(model.parameters())
        y = torch.Tensor([[0.0,0.0], [0.1,0.1], [0.2,0.2], [0.3,0.3], [0.4,0.4]])
        x = y.view(len(y), -1, 2)   # I changed here

        while True:
            y_pred = model(x)
            optimizer.zero_grad()
            loss = loss_function(y_pred, y)
            loss.backward()
            optimizer.step()
            print(y_pred)

The above code can learn very well, can you help review the code and give some instructions.

When I input 2 examples as the input to the model, the model cannot work:

for example, change the code:

y = torch.Tensor([[0.0,0.0], [0.1,0.1], [0.2,0.2], [0.3,0.3], [0.4,0.4]])

to:

y = torch.Tensor([[[0.0,0.0],[0.5,0.5]], [[0.1,0.1], [0.6,0.6]], [[0.2,0.2],[0.7,0.7]], [[0.3,0.3],[0.8,0.8]], [[0.4,0.4],[0.9,0.9]]])

When I compute the loss function, it complain some errors? can anyone help have a look

question two:
my training samples are with different length:
for example:

x1 = [[0.0,0.0], [0.1,0.1], [0.2,0.2], [0.3,0.3], [0.4,0.4]]   #with 5 timesteps
x2 = [[0.5,0.5], [0.6,0.6], [0.7,0.7]] #with only 3 timesteps

How can I input these two training sample into the model at the same time for a batch training.


title: Index tensor must have the same number of dimensions as input tensor error encountered when using torch.gather()

body:
I'm very new to PyTorch, and I have encountered the "Index tensor must have the same number of dimensions as input tensor" error when running my neural network. It happens with I call an instance of torch.gather(). Could someone help me understand torch.gather() and explain the cause of this error?
Here is the code where the error occurs:
  def learn(batch, optim, net, target_net, gamma, global_step, target_update):
      my_loss = []
      optim.zero_grad()

      state, action, next_state, reward, done, next_action = batch
      qval = net(state.float())

      loss_a = torch.gather(qval, 3, action.view(-1,1,1,1)).squeeze() #Error happens here!

      loss_b = reward + gamma * torch.max(target_net(next_state.float()).cuda(), dim=3).values * (1 - done.int())
      loss_val = torch.sum(( torch.abs(loss_a-loss_b) ))
      loss_val /= 128
      my_loss.append(loss_val.item())
      loss_val.backward()
      optim.step()
      if global_step % target_update == 0:
          target_network.load_state_dict(q_network.state_dict())

In case it is helpful, here is the batch function that creates the batch that the action comes from:
def sample_batch(memory,batch_size):

    indices = np.random.randint(0,len(memory), (batch_size,))

    state = torch.stack([memory[i][0] for i in indices])
    action = torch.tensor([memory[i][1] for i in indices], dtype = torch.long)
    next_state = torch.stack([memory[i][2] for i in indices])
    reward = torch.tensor([memory[i][3] for i in indices], dtype = torch.float)
    done = torch.tensor([memory[i][4] for i in indices], dtype = torch.float)
    next_action = torch.tensor([memory[i][5] for i in indices], dtype = torch.long)

    return state,action,next_state,reward,done,next_action

When I print out the different shapes of 'qvals', 'action', and 'action.view(-1,1,1,1)' this is the output:
qval torch.Size([10, 225])
act view torch.Size([10, 1, 1, 1])
action shape  torch.Size([10])

Any explanation as to what is causing this error is appreciated! I want to understand more what is going on in the code as well as how to fix the problem. Thanks!


title: How can I optimize gradient flow in LSTM with Pytorch?

body:
I'm working in lstm with time-series data and I've observed a problem in the gradients of my network. I've one layer of 121 lstm cells. For each cell I've one input value and I get one output value. I work with a batch size of 121 values and I define lstm cell with batch_first = True, so my outputs are [batch,timestep,features].

Once I've the outputs (tensor of size [121,121,1]), I calculate the loss using MSELoss() and I backpropagate it. And here appears the problem. Looking the gradients of each cell, I notice that the gradients of first 100 cells (more or less) are null.

In theory, if I'm not wrong, when I backpropagate the error I calculate a gradient for each output, so I have a gradient for each cell. If that is true, I can't understand why in the first cells they are zero.

Does somebody knows what is happening?

Thank you!

PS.: I show you the gradient flow of the last cells:

Update:
As I tried to ask before, I still have a question about LSTM backpropagation. As you can see from the image below, in one cell, apart from the gradients that come from other cells, I think there’s also another gradient form itself.

For example, let’s look at the cell 1. I get the output y1 and I calculate the loss E1. I do the same with other cells. So, when I backpropagate in cell 1, I get dE2/dy2 * dy2/dh1 * dh1/dw1 + ... which are the gradients related to following cells in the network (BPTT) as @kmario23 and @DavidNg explained. And I also have the gradient related to E1 (dE1/dy1 * dy1/dw1). The first gradients can vanish during the flow, but this one not.

So, to sum up, although having a long layer of lstm cells, to my understand I have a gradient related only to each cell, therefore I don’t understand why I have gradients equal to zero. What does it happens with the error related to E1? Why is only bptt calculated?


title: CUDA for pytorch: CUDA C++ stream and state

body:
I am trying to follow this tutorial and make a simple c++ extension with CUDA backend.
My CPU implementation seems to work fine.

I am having trouble finding examples and documentation (it seems like things are constantly changing).

Specifically,

I see pytorch cuda functions getting THCState *state argument - where does this argument come from? How can I get a state for my function as well?
For instance, in cuda implementation of tensor.cat:

void THCTensor_(cat)(THCState *state, THCTensor *result, THCTensor *ta, THCTensor *tb, int dimension)

However, when calling tensor.cat() from python one does not provide any state argument, pytorch provides it "behind the scene". How pytorch provides this information and how can I get it?
state is then converted to cudaStream_t stream = THCState_getCurrentStream(state);
For some reason, THCState_getCurrentStream is no longer defined? How can I get the stream from my state?

I also tried asking on pytorch forum - so far to no avail.


When I use PyTorch to import weak_script_method, the code throws an error:

  from torch._jit_internal import weak_script_method ImportError: cannot
  import name 'weak_script_method'

The code is:

from torch._jit_internal import weak_script_method

Does anyone have an idea about this? How to fix this error?


title: I have installed pytorch, but I still meet bug below

body:
Traceback (most recent call last):
  File "tools/create_dictionary.py", line 7, in <module>
    from dataset import Dictionary
  File "/home/supermicro/xswork/xsvqa/dataset.py", line 6, in <module>
    import utils
  File "/home/supermicro/xswork/xsvqa/utils.py", line 7, in <module>
    import torch
ImportError: No module named torch
Traceback (most recent call last):
  File "tools/compute_softscore.py", line 10, in <module>
    from dataset import Dictionary
  File "/home/supermicro/xswork/xsvqa/dataset.py", line 6, in <module>
    import utils
  File "/home/supermicro/xswork/xsvqa/utils.py", line 7, in <module>
    import torch
ImportError: No module named torch
/home/supermicro/xswork/ENV/local/lib/python2.7/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.
  from ._conv import register_converters as _register_converters
Traceback (most recent call last):
  File "tools/detection_features_converter.py", line 22, in <module>
    import utils
  File "/home/supermicro/xswork/xsvqa/utils.py", line 7, in <module>
    import torch
ImportError: No module named torch


title: PyTorch LSTM for multiclass classification: TypeError: '<' not supported between instances of 'Example' and 'Example'

body:
I am trying to modify the code in this Tutorial to adapt it to a multiclass data (I have 55 distinct classes). An error is triggered and I am uncertain of the root cause. The changes I made to this tutorial have been annotated in same-line comments.

One of two solutions would satisfy this questions:

(A) Help identifying the root cause of the error, OR

(B) A boilerplate script for multiclass classification using PyTorch LSTM

import spacy
import torchtext
from torchtext import data
import re

TEXT = data.Field(tokenize = 'spacy', include_lengths = True)
LABEL = data.LabelField(dtype = torch.float)
fields = [(None,None),('text', TEXT), ('wage_label', LABEL)]

train_torch, test_torch = data.TabularDataset.splits(path='/Users/jdmoore7/Desktop/Python Projects/560_capstone/',
                                            format='csv',
                                            train='train_text_target.csv',
                                            test='test_text_target.csv',
                                            fields=fields,
                                            skip_header=True)

import random
train_data, valid_data = train_torch.split(random_state = random.seed(SEED))

MAX_VOCAB_SIZE = 25_000

TEXT.build_vocab(train_data,
                 max_size = MAX_VOCAB_SIZE,
                 vectors = "glove.6B.100d",
                 unk_init = torch.Tensor.normal_)

LABEL.build_vocab(train_data)

BATCH_SIZE = 64

device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

train_iterator, valid_iterator, test_iterator = data.BucketIterator.splits(
    (train_data, valid_data, test_torch),
    batch_size = BATCH_SIZE,
    sort_within_batch = True,
    device = device)

import torch.nn as nn

class RNN(nn.Module):
    def __init__(self, vocab_size, embedding_dim, hidden_dim, output_dim, n_layers,
                 bidirectional, dropout, pad_idx):

        super().__init__()

        self.embedding = nn.Embedding(vocab_size, embedding_dim, padding_idx = pad_idx)

        self.rnn = nn.LSTM(embedding_dim,
                           hidden_dim,
                           num_layers=n_layers,
                           bidirectional=bidirectional,
                           dropout=dropout)

        self.fc = nn.Linear(hidden_dim * 2, output_dim)

        self.dropout = nn.Dropout(dropout)

    def forward(self, text, text_lengths):

        #text = [sent len, batch size]

        embedded = self.dropout(self.embedding(text))

        #embedded = [sent len, batch size, emb dim]

        #pack sequence
        packed_embedded = nn.utils.rnn.pack_padded_sequence(embedded, text_lengths)

        packed_output, (hidden, cell) = self.rnn(packed_embedded)

        #unpack sequence
        output, output_lengths = nn.utils.rnn.pad_packed_sequence(packed_output)

        #output = [sent len, batch size, hid dim * num directions]
        #output over padding tokens are zero tensors

        #hidden = [num layers * num directions, batch size, hid dim]
        #cell = [num layers * num directions, batch size, hid dim]

        #concat the final forward (hidden[-2,:,:]) and backward (hidden[-1,:,:]) hidden layers
        #and apply dropout

        hidden = self.dropout(torch.cat((hidden[-2,:,:], hidden[-1,:,:]), dim = 1))

        #hidden = [batch size, hid dim * num directions]

        return self.fc(hidden)

INPUT_DIM = len(TEXT.vocab)
EMBEDDING_DIM = 100
HIDDEN_DIM = 256
OUTPUT_DIM = len(LABEL.vocab) ### changed from previous value (1)
N_LAYERS = 2
BIDIRECTIONAL = True
DROPOUT = 0.5
PAD_IDX = TEXT.vocab.stoi[TEXT.pad_token]

model = RNN(INPUT_DIM,
            EMBEDDING_DIM,
            HIDDEN_DIM,
            OUTPUT_DIM,
            N_LAYERS,
            BIDIRECTIONAL,
            DROPOUT,
            PAD_IDX)

import torch.optim as optim
optimizer = optim.Adam(model.parameters())

criterion = nn.CrossEntropyLoss() # Previously: criterion = nn.BCEWithLogitsLoss()
model = model.to(device)
criterion = criterion.to(device)

def binary_accuracy(preds, y):
    """
    Returns accuracy per batch, i.e. if you get 8/10 right, this returns 0.8, NOT 8
    """

    #round predictions to the closest integer
    rounded_preds = torch.round(torch.sigmoid(preds))
    correct = (rounded_preds == y).float() #convert into float for division
    acc = correct.sum() / len(correct)
    return acc
def train(model, iterator, optimizer, criterion):

    epoch_loss = 0
    epoch_acc = 0

    model.train()

    for batch in iterator:

        optimizer.zero_grad()

        text, text_lengths = batch.text

        predictions = model(text, text_lengths).squeeze(1)

        loss = criterion(predictions, batch.label)

        acc = binary_accuracy(predictions, batch.label)

        loss.backward()

        optimizer.step()

        epoch_loss += loss.item()
        epoch_acc += acc.item()

    return epoch_loss / len(iterator), epoch_acc / len(iterator)

def evaluate(model, iterator, criterion):

    epoch_loss = 0
    epoch_acc = 0

    model.eval()

    with torch.no_grad():

        for batch in iterator:

            text, text_lengths = batch.text

            predictions = model(text, text_lengths).squeeze(1)

            loss = criterion(predictions, batch.label)

            acc = binary_accuracy(predictions, batch.label)

            epoch_loss += loss.item()
            epoch_acc += acc.item()

    return epoch_loss / len(iterator), epoch_acc / len(iterator)

import time

def epoch_time(start_time, end_time):
    elapsed_time = end_time - start_time
    elapsed_mins = int(elapsed_time / 60)
    elapsed_secs = int(elapsed_time - (elapsed_mins * 60))
    return elapsed_mins, elapsed_secs

All the above ran smoothly, it's the next code block which triggers the error:

N_EPOCHS = 5

best_valid_loss = float('inf')

for epoch in range(N_EPOCHS):

    start_time = time.time()

    train_loss, train_acc = train(model, train_iterator, optimizer, criterion)
    valid_loss, valid_acc = evaluate(model, valid_iterator, criterion)

    end_time = time.time()

    epoch_mins, epoch_secs = epoch_time(start_time, end_time)

    if valid_loss < best_valid_loss:
        best_valid_loss = valid_loss
        torch.save(model.state_dict(), 'tut2-model.pt')

    print(f'Epoch: {epoch+1:02} | Epoch Time: {epoch_mins}m {epoch_secs}s')
    print(f'\tTrain Loss: {train_loss:.3f} | Train Acc: {train_acc*100:.2f}%')
    print(f'\t Val. Loss: {valid_loss:.3f} |  Val. Acc: {valid_acc*100:.2f}%')

---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<ipython-input-888-c1b298b1eeea> in <module>
      7     start_time = time.time()
      8
----> 9     train_loss, train_acc = train(model, train_iterator, optimizer, criterion)
     10     valid_loss, valid_acc = evaluate(model, valid_iterator, criterion)
     11

<ipython-input-885-9a57198441ec> in train(model, iterator, optimizer, criterion)
      6     model.train()
      7
----> 8     for batch in iterator:
      9
     10         optimizer.zero_grad()

~/opt/anaconda3/lib/python3.7/site-packages/torchtext/data/iterator.py in __iter__(self)
    140         while True:
    141             self.init_epoch()
--> 142             for idx, minibatch in enumerate(self.batches):
    143                 # fast-forward if loaded from state
    144                 if self._iterations_this_epoch > idx:

~/opt/anaconda3/lib/python3.7/site-packages/torchtext/data/iterator.py in pool(data, batch_size, key, batch_size_fn, random_shuffler, shuffle, sort_within_batch)
    284     for p in batch(data, batch_size * 100, batch_size_fn):
    285         p_batch = batch(sorted(p, key=key), batch_size, batch_size_fn) \
--> 286             if sort_within_batch \
    287             else batch(p, batch_size, batch_size_fn)
    288         if shuffle:

TypeError: '<' not supported between instances of 'Example' and 'Example'

Lastly, the PyTorch forum has an issue opened for this error, however, the code that produced it is not similar so I understand that to be a separate issue.


title: How do I add LSTM, GRU or other recurrent layers to a Sequential in PyTorch

body:
I like using torch.nn.Sequential as in

self.conv_layer = torch.nn.Sequential(
    torch.nn.Conv1d(196, 196, kernel_size=15, stride=4),
    torch.nn.Dropout()
)

But when I want to add a recurrent layer such as torch.nn.GRU it won't work because the output of recurrent layers in PyTorch is a tuple and you need to choose which part of the output you want to further process.

So is there any way to get

self.rec_layer = nn.Sequential(
    torch.nn.GRU(input_size=2, hidden_size=256),
    torch.nn.Linear(in_features=256, out_features=1)
)

to work? For this example, let's say I want to feed torch.nn.GRU(input_size=2, hidden_size=20)(x)[1][-1] (the last hidden state of the last layer) into the following Linear layer.


title: Stride in Conv2d vs. ConvTranspose2d

body:
I am trying to learn an autoencoder on CIFAR10. I have written the code with sequential in the code you can see below and it worked fine:
class Autoencoder(nn.Module):
def __init__(self):
    super(Autoencoder, self).__init__()
    self.Flatten = Flatten()

    # Input size: [batch, 3, 32, 32]
    # Output size: [batch, 3, 32, 32]
    self.encoder = nn.Sequential(
        nn.Conv2d(3, 12, 4, stride=2, padding=1),            # [batch, 12, 16, 16]
        nn.ReLU(),
        nn.Conv2d(12, 24, 4, stride=2, padding=1),           # [batch, 24, 8, 8]
        nn.ReLU(),
        nn.Conv2d(24, 48, 4, stride=2, padding=1),           # [batch, 48, 4, 4]
        ReLU(),

    )
    self.decoder = nn.Sequential(

        nn.ConvTranspose2d(48, 24, 4, stride=2, padding=1),  # [batch, 24, 8, 8]
        nn.ReLU(),
        nn.ConvTranspose2d(24, 12, 4, stride=2, padding=1),  # [batch, 12, 16, 16]
        nn.ReLU(),
        nn.ConvTranspose2d(12, 3, 4, stride=2, padding=1),   # [batch, 3, 32, 32]
        nn.Sigmoid(),
    )

def forward(self, x):
    encoded = self.encoder(x)

    decoded = self.decoder(encoded)
    return encoded, decoded

I decided to add a linear layer to reduce the output dimensionality. For this, I rewrote the above code as you can see below (this time I avoided using sequential):
class Autoencoder(nn.Module):

def __init__(self):
    super(Autoencoder, self).__init__()

    # Encoder architecture
    self.conv_1 = nn.Conv2d(3, 12, kernel_size=3, stride=2, padding=1)
    self.conv_2 = nn.Conv2d(12, 24, kernel_size=3, stride=2, padding=1)
    self.conv_3 = nn.Conv2d(24, 48, kernel_size=3, stride=2, padding=1)
    #Decodere architecture: convolutional transpose layers or deconvolutional layers + upsampling pooling layers
    self.de_conv_3 = nn.ConvTranspose2d(48, 24, kernel_size=3, padding=1, stride=2)
    self.de_conv_2 = nn.ConvTranspose2d(24, 12, kernel_size=3, padding=1, stride=2)
    self.de_conv_1 = nn.ConvTranspose2d(12, 3, kernel_size=3, padding=1, stride=2)

    self.linear1 = nn.Linear(48*4*4, 10)
    self.linear2 = nn.Linear(10, 48*4*4)
    self.sigmoid = nn.Sigmoid()

def forward(self, images):
    code = self.encode(images)
    out = self.decode(code)
    return code, out

def encode(self, images):
    #print('input shape', images.shape)
    code = self.conv_1(images)
    code = F.relu(code)
    print("conv1 output", code.shape)

    code = self.conv_2(code)
    code = F.relu(code)
    #print('batch batch_norm shape', code.shape)
    print("conv2 output", code.shape)

    code = self.conv_3(code)
    code = F.relu(code)
    print("conv3 output", code.shape)

    code = code.view(code.size(0), 48 * 4 * 4)
    print('code after view', code.shape)
    #add a linear layer that flattens and also gives a 10
    code = F.relu(self.linear1(code))
    print('code after linear1', code.shape)
    return code#[100,10]

def decode(self, code):
    code = self.linear2(code)
    print('code after linear2', code.shape)
    #reshape to conv
    code = code.view(100, 48, 4, 4)
    print('code after view', code.shape)
    #upsample, followed by a conv layer, with relu activation function
    #this function is called `upsample` in some PyTorch versions

    code = F.relu(self.de_conv_3(code))
    print('deconv3 output', code.shape)
    code = F.relu(self.de_conv_2(code))
    print('deconv2 output', code.shape)

    code = self.de_conv_1(code)
    print("deconv1 output", code.shape)
    out = self.sigmoid(code)
    #print('final output', out.shape)

    return out

But I get an error and when I print the output of each layer, I see that the decoder does not construct the same shape as the encoder outputs. I am confused why I am not getting the same results as the first model written with sequential. Is it because maybe stride=2 in convtranspose layer works differently? but why not when written in sequential.
These are the results of printing the shape of the decoder layers:
conv1 output torch.Size([100, 12, 16, 16])
conv2 output torch.Size([100, 24, 8, 8])
conv3 output torch.Size([100, 48, 4, 4])
code after view torch.Size([100, 768])
code after linear1 torch.Size([100, 10])
code after linear2 torch.Size([100, 768])
code after view torch.Size([100, 48, 4, 4])
deconv3 output torch.Size([100, 24, 7, 7])
deconv2 output torch.Size([100, 12, 13, 13])
deconv1 output torch.Size([100, 3, 25, 25])


title: Assign torch.cuda.FloatTensor

body:
I'd like to know how can I do the following code, but now using pytorch,
where dtype = torch.cuda.FloatTensor. There's the code straight python (using numpy):

import numpy as np
import random as rand
xmax, xmin = 5, -5
pop = 30
x = (xmax-xmin)*rand.random(pop,1)
y = x**2
[minz, indexmin] = np.amin(y), np.argmin(y)
best = x[indexmin]

This is my attempt to do it:

import torch
dtype = torch.cuda.FloatTensor
def fit (position):
    return  position**2
def main():
    pop = 30
    xmax, xmin = 5, -5
    x= (xmax-xmin)*torch.rand(pop, 1).type(dtype)+xmin
    y = fit(x)
    [miny, indexmin] = torch.min(y,0)
    best = x[indexmin]
    print(best)

The last part where I define the variable best as the value of x with index equal to indexmin it doesn't work. What am I doing wrong here.

The following messenge appears:

RuntimeError: expecting vector of indices at /opt/conda/conda-bld/pytorch_1501971235237/work/pytorch-0.1.12/torch/lib/THC/generic/THCTensorIndex.cu:405


title: Un-normalizing PyTorch data

body:
Below code :

ux = torch.tensor(np.array([[255,1,255],[255,1,255]])).float()

print(ux)

ux = F.normalize(ux, p=2, dim=1)

print(ux)

prints :

tensor([[ 255.,    1.,  255.],
        [ 255.,    1.,  255.]])
tensor([[ 0.7071,  0.0028,  0.7071],
        [ 0.7071,  0.0028,  0.7071]])

How can I un-normalize the ux in order to return to values

tensor([[ 255.,    1.,  255.],
        [ 255.,    1.,  255.]])

from

tensor([[ 0.7071,  0.0028,  0.7071],
        [ 0.7071,  0.0028,  0.7071]])

There are various resources that detail this process such as https://discuss.pytorch.org/t/simple-way-to-inverse-normalize-a-batch-of-input-variable/12385/3 but do not detail unnormalizing result of F.normalize


title: setuptools: installing pytorch from download link: 403 Forbidden

body:
I am trying to include pytorch in the requirements list for setuptools:

install_requires=[
      'torch'
      ],
dependency_links=[
      'http://download.pytorch.org/whl/cpu/torch-0.3.0.post4-cp27-cp27mu-linux_x86_64.whl' '@develop#egg=torch'
      ],

But after running python setup.py develop I receive:

error: Can't download http://download.pytorch.org/whl/cpu/torch-0.3.0.post4-cp27-cp27mu-linux_x86_64.whl@develop#egg=torch: 403 Forbidden

I also tried using the source install link rather than the one for pip:

install_requires=[
      'torch'
      ],
  dependency_links=[
      'https://github.com/pytorch/pytorch#from-source' '@develop#egg=torch'
      ],

But then:

RuntimeError: PyTorch does not currently provide packages for PyPI (see status at https://github.com/pytorch/pytorch/issues/566).

What is the correct way to install pytorch through setuptools? (Note: using anaconda is not an option in my case)

Thank you.

EDIT:

As suggested in the answer, I also tried:

dependency_links=[
      'https://github.com/pytorch/pytorch'
      ],

dependency_links=[
      'https://github.com/pytorch/pytorch#egg=torch'
      ],

dependency_links=[
      'http://download.pytorch.org/whl/cpu/torch-0.3.0.post4-cp27-cp27mu-linux_x86_64.whl'
      ],

And received:

RuntimeError: PyTorch does not currently provide packages for PyPI (see status at https://github.com/pytorch/pytorch/issues/566).


title: Pytorch: Storage has wrong size while loading state_dict from GPU on CPU

body:
I've trained a model on a GPU (server) and saved the state_dict. Now I want to do some tests locally on my CPU, so I downloaded the state_dict, create a model and want to load that state_dict. After some research, I found the map_location-attribute of the torch.load()-function. So finally I'm trying to load the state_dict in the following way:

device = torch.device('cpu')
model = BertForTokenClassification.from_pretrained('bert-base-uncased', num_labels=4)
model.load_state_dict(torch.load('state_dict.pth', map_location=device))

But I get the following RuntimeError:

RuntimeError: storage has wrong size: expected 1005829391 got 393216

Does anyone know, what to do for loading the state_dict on my CPU?


title: PyTorch ImageFolder: how to sample images belonging to particular classes

body:
I have a pytorch ImageFolder. I see that the __getitem__(index) method of ImageFolder can get both the tensor and the class of an index. But what if I want to use ImageFolder to randomly sample a certain number of images from a class (or multiple classes) of my choosing, and then get their indices for the __getitem__ method later? (Sort of like operating in reverse, i.e. choose class -> get index rather than choose index -> get class)

How would this be done?


title: How to install pytorch 1.2.0 with CUDA support on Windows 10?

body:
I'm trying to install pytorch 1.2.0 with CUDA support on Windows 10 with Python 3.9 and it keeps erroring out.
For example, I do pip install pytorch==1.2.0 torchvision==0.4.0 cudatoolkit=10.0 -c pytorch
and get:
ERROR: Could not open requirements file: [Errno 2] No such file or directory: 'pytorch'
Any ideas? Thank you.


title: Is it possible to drop_last in reverse with PyTorch's DataLoader?

body:
I'm currently trying to make a custom PyTorch DataLoader.

I'm aware that setting drop_last=True when first declaring the DataLoader object tells the object to drop the last incomplete batch if the size is not appropriate. However, I was wondering if that could be done in reverse, where the DataLoader computes the number of batches and counts from the back.

The reason I'm asking this is because the data that I'm currently using is time series data, and I want to use the most recent sample and therefore it would be ideal if the "leftover" samples were dropped from the oldest portion of the data.

I've thought of ways like reversing the data to start with, then creating the DataLoader object and reversing it back to the way it was, or to first reverse the data and create the object and then feed in the idx's in reverse order when running __getitem__, but this seems troublesome and prone to making errors so I was wondering if PyTorch offers this behavior.

Thanks in advance.


title: Bert model giving CUDA out of memory error on google colab

body:
I am using the following tutorial here to train and test a Bertsequenceclassifier model on a dataset of documents of varying lengths (small(0-280), medium(280-10000), large(10000 plus)) on the google collab platform using a GPU. I have been able to successfully train and test the small dataset using a maximum sequence length of 380 on 140,000 entries. When I try to train the medium dataset on the maximum sequence length which is 512 for only 1 epoch and only on 10000 entries I get a CUDA out of memory error. When I change the maximum sequence length to 400 it is able to train the model though.

I guess that obviously the length of the documents is causing this but can anyone explain exactly why this is happening and if there is any way to fix this. Thank you

RuntimeError: CUDA out of memory. Tried to allocate 20.00 MiB (GPU 0; 15.90 GiB total capacity; 15.15 GiB already allocated; 7.88 MiB free; 44.38 MiB cached)


title: Pytorch how to increase batch size

body: I currently have a tensor of torch.Size([1, 3, 256, 224]) but I need it to be input shape [32, 3, 256, 224].  I am capturing data in real-time so dataloader doesn't seem to be a good option.  Is there any easy way to take 32 of size torch.Size([1, 3, 256, 224]) and combine them to create 1 tensor of size [32, 3, 256, 224]?


title: Error in ret = torch._C._nn.nll_loss2d(input, target, weight, _Reduction.get_enum(reduction), ignore_index)

body:
The input image size is 512*512,in order to suit to the input of resnet.input image
I used

_img = Image.open(self.images[index]).convert('RGB')

in dataloader.
I used resnet50 as my network's backbone without fc.The output shape is

[4,2048,16,16]

then used two (conv bn relu) and a interpolate

    def forward(self, input):
        x=self.backbone(input)
        x = self.conv1(x)
        x= self.bn1(x)
        x = self.relu(x)
        x = self.conv2(x)
        x= self.bn2(x)
        x = self.relu(x)
        x = F.interpolate(x, size=[512,512], mode='bilinear', align_corners=True)
        return x

The part of training

    self.criterion=nn.CrossEntropyLoss()
    if self.args.cuda:
        image, target = image.cuda(), target.cuda()
    self.scheduler(self.optimizer, i, epoch, self.best_pred)
    self.optimizer.zero_grad()
    output = self.model(image)
    loss = self.criterion(output, target.long())
    loss.backward()

But the Error occurs

File "E:/python_workspace/1006/train.py", line 135, in training
loss = self.criterion(output, target.long())
File "E:\python_workspace\1006\utils\loss.py", line 28, in CrossEntropyLoss
loss = criterion(logit, target.long())
File "E:\anaconda3\lib\site-packages\torch\nn\modules\module.py", line 547, in __call__
result = self.forward(*input, **kwargs)
File "E:\anaconda3\lib\site-packages\torch\nn\modules\loss.py", line 916, in forward
ignore_index=self.ignore_index, reduction=self.reduction)
File "E:\anaconda3\lib\site-packages\torch\nn\functional.py", line 1995, in cross_entropy
return nll_loss(log_softmax(input, 1), target, weight, None, ignore_index, None, reduction)
File "E:\anaconda3\lib\site-packages\torch\nn\functional.py", line 1826, in nll_loss
ret = torch._C._nn.nll_loss2d(input, target, weight, _Reduction.get_enum(reduction), ignore_index)
RuntimeError: Assertion `cur_target >= 0 && cur_target < n_classes' failed.  at C:\w\1\s\tmp_conda_3.6_045031\conda\conda-bld\pytorch_1565412750030\work\aten\src\THNN/generic/SpatialClassNLLCriterion.c:111

image.shape is [4, 3, 512, 512],dtype is torch.float32
target.shape is [4, 512, 512],dtype is torch.float32
output.shape is [4, 3, 512, 512],dtype is torch.float32

target image
The target images all only have three different colors.so I set output to 3 channel.And there's Image mode is P
Where may have problems in my code?


title: OpenNMT issues with Pytorch: cPickle.UnpicklingError: invalid load key, ''

body:
I am trying to run the OpenNMT project using the instruction from the link: http://forum.opennmt.net/t/text-summarization-on-gigaword-and-rouge-scoring/85/6

I am using Python 2.7 and installed pytorch from the github repository.
I am trying to run the program using the prebuild model of the OpenNMT, which I have downloaded from the following: http://opennmt.net/Models/

I tried the command:

python translate.py -model textsum_epoch7_14.69_release.t7 -src data/Giga/input.txt

Got the following error:

Traceback (most recent call last):
  File "translate.py", line 151, in <module>
    main()
  File "translate.py", line 70, in main
    translator = onmt.Translator(opt)
  File "/home/ubuntu/opennmt/onmt/Translator.py", line 21, in __init__
    checkpoint = torch.load(opt.model)
  File "/usr/local/lib/python2.7/dist-packages/torch/serialization.py", line 229, in load
    return _load(f, map_location, pickle_module)
  File "/usr/local/lib/python2.7/dist-packages/torch/serialization.py", line 367, in _load
    magic_number = pickle_module.load(f)
cPickle.UnpicklingError: invalid load key, ''.

Kindly let me know what I need to do so that I can use the model and check the library OpenNMT.


title: How to dump confusion matrix using TensorBoard logger in pytorch-lightning?

body:
The official doc only states
>>> from pytorch_lightning.metrics import ConfusionMatrix
>>> target = torch.tensor([1, 1, 0, 0])
>>> preds = torch.tensor([0, 1, 0, 0])
>>> confmat = ConfusionMatrix(num_classes=2)
>>> confmat(preds, target)

This doesn't show how to use the metric with the framework.
My attempt (methods are not complete and only show relevant parts):
def __init__(...):
    self.val_confusion = pl.metrics.classification.ConfusionMatrix(num_classes=self._config.n_clusters)

def validation_step(self, batch, batch_index):
    ...
    log_probs = self.forward(orig_batch)
    loss = self._criterion(log_probs, label_batch)

    self.val_confusion.update(log_probs, label_batch)
    self.log('validation_confusion_step', self.val_confusion, on_step=True, on_epoch=False)

def validation_step_end(self, outputs):
    return outputs

def validation_epoch_end(self, outs):
    self.log('validation_confusion_epoch', self.val_confusion.compute())

After the 0th epoch, this gives
    Traceback (most recent call last):
      File "C:\code\EPMD\Kodex\Templates\Testing\venv\lib\site-packages\pytorch_lightning\trainer\trainer.py", line 521, in train
        self.train_loop.run_training_epoch()
      File "C:\code\EPMD\Kodex\Templates\Testing\venv\lib\site-packages\pytorch_lightning\trainer\training_loop.py", line 588, in run_training_epoch
        self.trainer.run_evaluation(test_mode=False)
      File "C:\code\EPMD\Kodex\Templates\Testing\venv\lib\site-packages\pytorch_lightning\trainer\trainer.py", line 613, in run_evaluation
        self.evaluation_loop.log_evaluation_step_metrics(output, batch_idx)
      File "C:\code\EPMD\Kodex\Templates\Testing\venv\lib\site-packages\pytorch_lightning\trainer\evaluation_loop.py", line 346, in log_evaluation_step_metrics
        self.__log_result_step_metrics(step_log_metrics, step_pbar_metrics, batch_idx)
      File "C:\code\EPMD\Kodex\Templates\Testing\venv\lib\site-packages\pytorch_lightning\trainer\evaluation_loop.py", line 350, in __log_result_step_metrics
        cached_batch_pbar_metrics, cached_batch_log_metrics = cached_results.update_logger_connector()
      File "C:\code\EPMD\Kodex\Templates\Testing\venv\lib\site-packages\pytorch_lightning\trainer\connectors\logger_connector\epoch_result_store.py", line 378, in update_logger_connector
        batch_log_metrics = self.get_latest_batch_log_metrics()
      File "C:\code\EPMD\Kodex\Templates\Testing\venv\lib\site-packages\pytorch_lightning\trainer\connectors\logger_connector\epoch_result_store.py", line 418, in get_latest_batch_log_metrics
        batch_log_metrics = self.run_batch_from_func_name("get_batch_log_metrics")
      File "C:\code\EPMD\Kodex\Templates\Testing\venv\lib\site-packages\pytorch_lightning\trainer\connectors\logger_connector\epoch_result_store.py", line 414, in run_batch_from_func_name
        results = [func(include_forked_originals=False) for func in results]
      File "C:\code\EPMD\Kodex\Templates\Testing\venv\lib\site-packages\pytorch_lightning\trainer\connectors\logger_connector\epoch_result_store.py", line 414, in <listcomp>
        results = [func(include_forked_originals=False) for func in results]
      File "C:\code\EPMD\Kodex\Templates\Testing\venv\lib\site-packages\pytorch_lightning\trainer\connectors\logger_connector\epoch_result_store.py", line 122, in get_batch_log_metrics
        return self.run_latest_batch_metrics_with_func_name("get_batch_log_metrics",
*args, **kwargs)
      File "C:\code\EPMD\Kodex\Templates\Testing\venv\lib\site-packages\pytorch_lightning\trainer\connectors\logger_connector\epoch_result_store.py", line 115, in run_latest_batch_metrics_with_func_name
        for dl_idx in range(self.num_dataloaders)
      File "C:\code\EPMD\Kodex\Templates\Testing\venv\lib\site-packages\pytorch_lightning\trainer\connectors\logger_connector\epoch_result_store.py", line 115, in <listcomp>
        for dl_idx in range(self.num_dataloaders)
      File "C:\code\EPMD\Kodex\Templates\Testing\venv\lib\site-packages\pytorch_lightning\trainer\connectors\logger_connector\epoch_result_store.py", line 100, in get_latest_from_func_name
        results.update(func(*args, add_dataloader_idx=add_dataloader_idx, **kwargs))
      File "C:\code\EPMD\Kodex\Templates\Testing\venv\lib\site-packages\pytorch_lightning\core\step_result.py", line 298, in get_batch_log_metrics
        result[dl_key] = self[k]._forward_cache.detach()
    AttributeError: 'NoneType' object has no attribute 'detach'

It does pass the sanity validation check before training.
The failure happens on the return in validation_step_end. Makes little sense to me.
The exact same method of using mertics works fine with accuracy.
How to get a correct confusion matrix?


title: Fastai learner not loading

body:
So I'm trying to load a model using:

learn = create_cnn(data, models.resnet50, lin_ftrs=[2048], metrics=accuracy)
learn.clip_grad();
learn.load(f'{name}-stage-2.1')

But I get the following error

RuntimeError: Error(s) in loading state_dict for Sequential:
size mismatch for 1.8.weight: copying a param with shape torch.Size([5004, 2048]) from checkpoint, the shape in current model is torch.Size([4542, 2048]).
size mismatch for 1.8.bias: copying a param with shape torch.Size([5004]) from checkpoint, the shape in current model is torch.Size([4542]).

The only thing that is different thing is that I added a random validation split that wasn't there in the stage-2.1 model, when I remove the split and have no validation set as the stage-2.1 was trained all goes well.

Whats happening?


title: FastAI v1 PyTorch Custom Model

body:
i have been trying to use fastai with a custom torch model. My code is as follow:

X_train = np.load(dirpath + 'X_train.npy')
X_valid = np.load(dirpath + 'X_valid.npy')
Y_train = np.load(dirpath + 'Y_train.npy')
Y_valid = np.load(dirpath + 'Y_valid.npy')

X_train's shape is : (240, 122, 96),
and Y_train's shape is : (240,1)

Then i convert these to torch tensors ,

# Converting data to torch tensors
def to_torch_data(x,np_type,tch_type):
    return torch.from_numpy(x.astype(np_type)).to(tch_type)

X_train = to_torch_data(X_train,float,torch.float32)
X_valid = to_torch_data(X_valid,float,torch.float32)
Y_train = to_torch_data(Y_train,float,torch.float32)
Y_valid = to_torch_data(Y_valid,float,torch.float32)

Creating TensorDataSets for fastai DataBunch wrapper,

# Creating torch tensor datasets so that data can be used
# on ImageDataBunch function for fastai
train_ds = tdatautils.TensorDataset(X_train,Y_train)
valid_ds = tdatautils.TensorDataset(X_valid,Y_valid)

# Creating DataBunch object to be used as data in fastai methods.
batch_size = 24
my_data_bunch = DataBunch.create(train_ds,valid_ds,bs=batch_size)

And this is my custom torch model :

# Creating corresponding torch model
import torch.nn.functional as F
class Net(nn.Module):
    def __init__(self,droprate=0,activationF=None):
        super(Net, self).__init__()
        self.lstm_0 = nn.LSTM(96, 720)
        self.activation_0 = nn.ELU()
        self.dropout_0 = nn.Dropout(p=droprate)
        self.lstm_1 = nn.LSTM(720,480)
        self.activation_1 = nn.ELU()
        self.batch_norm_1 = nn.BatchNorm1d(122)
        self.fc_2 = nn.Linear(480,128)
        self.dropout_2 = nn.Dropout(p=droprate)
        self.last = nn.Linear(128,1)
        self.last_act = nn.ReLU()

    def forward(self, x):
        out,hid1 = self.lstm_0(x)
        out = self.dropout_0(self.activation_0(out))
        out,hid2 = self.lstm_1(out)
        out = out[:,-1,:]
        out = self.batch_norm_1(self.activation_1(out))
        out = self.dropout_2(self.fc_2(out))
        out = self.last_act(self.last(out))
        return out

#create instance of model
net = Net(droprate=train_droprate,activationF=train_activation) #.cuda()
print(net)

After all these, i run the learn for lr_find method. And i get this error :

Empty                                     Traceback (most recent call last)
C:\Anaconda3\envs\fastai\lib\site-packages\torch\utils\data\dataloader.py in _try_get_batch(self, timeout)
    510         try:
--> 511             data = self.data_queue.get(timeout=timeout)
    512             return (True, data)

C:\Anaconda3\envs\fastai\lib\queue.py in get(self, block, timeout)
    171                     if remaining <= 0.0:
--> 172                         raise Empty
    173                     self.not_empty.wait(remaining)

Empty:

During handling of the above exception, another exception occurred:

RuntimeError                              Traceback (most recent call last)
<ipython-input-35-e4b7603c0a82> in <module>
----> 1 my_learner.lr_find()

~\Desktop\fastai\fastai\fastai\train.py in lr_find(learn, start_lr, end_lr, num_it, stop_div, wd)
     30     cb = LRFinder(learn, start_lr, end_lr, num_it, stop_div)
     31     epochs = int(np.ceil(num_it/len(learn.data.train_dl)))
---> 32     learn.fit(epochs, start_lr, callbacks=[cb], wd=wd)
     33
     34 def to_fp16(learn:Learner, loss_scale:float=None, max_noskip:int=1000, dynamic:bool=True, clip:float=None,

~\Desktop\fastai\fastai\fastai\basic_train.py in fit(self, epochs, lr, wd, callbacks)
    197         callbacks = [cb(self) for cb in self.callback_fns + listify(defaults.extra_callback_fns)] + listify(callbacks)
    198         if defaults.extra_callbacks is not None: callbacks += defaults.extra_callbacks
--> 199         fit(epochs, self, metrics=self.metrics, callbacks=self.callbacks+callbacks)
    200
    201     def create_opt(self, lr:Floats, wd:Floats=0.)->None:

~\Desktop\fastai\fastai\fastai\basic_train.py in fit(epochs, learn, callbacks, metrics)
     97             cb_handler.set_dl(learn.data.train_dl)
     98             cb_handler.on_epoch_begin()
---> 99             for xb,yb in progress_bar(learn.data.train_dl, parent=pbar):
    100                 xb, yb = cb_handler.on_batch_begin(xb, yb)
    101                 loss = loss_batch(learn.model, xb, yb, learn.loss_func, learn.opt, cb_handler)

C:\Anaconda3\envs\fastai\lib\site-packages\fastprogress\fastprogress.py in __iter__(self)
     70         self.update(0)
     71         try:
---> 72             for i,o in enumerate(self._gen):
     73                 if i >= self.total: break
     74                 yield o

~\Desktop\fastai\fastai\fastai\basic_data.py in __iter__(self)
     73     def __iter__(self):
     74         "Process and returns items from `DataLoader`."
---> 75         for b in self.dl: yield self.proc_batch(b)
     76
     77     @classmethod

C:\Anaconda3\envs\fastai\lib\site-packages\torch\utils\data\dataloader.py in __next__(self)
    574         while True:
    575             assert (not self.shutdown and self.batches_outstanding > 0)
--> 576             idx, batch = self._get_batch()
    577             self.batches_outstanding -= 1
    578             if idx != self.rcvd_idx:

C:\Anaconda3\envs\fastai\lib\site-packages\torch\utils\data\dataloader.py in _get_batch(self)
    541         elif self.pin_memory:
    542             while self.pin_memory_thread.is_alive():
--> 543                 success, data = self._try_get_batch()
    544                 if success:
    545                     return data

C:\Anaconda3\envs\fastai\lib\site-packages\torch\utils\data\dataloader.py in _try_get_batch(self, timeout)
    517             if not all(w.is_alive() for w in self.workers):
    518                 pids_str = ', '.join(str(w.pid) for w in self.workers if not w.is_alive())
--> 519                 raise RuntimeError('DataLoader worker (pid(s) {}) exited unexpectedly'.format(pids_str))
    520             if isinstance(e, queue.Empty):
    521                 return (False, None)

RuntimeError: DataLoader worker (pid(s) 9584, 7236, 5108, 932, 13228, 13992, 4576, 13204) exited unexpectedly

I have researched about DataLoader but couldn't find anything useful.


title: How can I do the centercrop of 3D volumes inside the network model with pytorch

body:
In keras, there is Cropping3D layer for centercropping tensors of 3D volumnes inside the neural network. However, I failed to find out anything similar in pytorch, though they have torchvision.transforms.CenterCrop(size) for 2D images.

How can I do the cropping inside the network? Otherwise I need to do it in preprocessing which is the last thing I want to do for specific reasons.

Do I need to write a custom layer like slicing the input tensors along each axices? Hope to get some inspiration for this


title: (pytorch / mse) How can I change the shape of tensor?

body:
Problem definition:

I have to use MSELoss function to define the loss to classification problem. Therefore it keeps saying the error message regarding the shape of tensor.

Entire error message:

  torch.Size([32, 10]) torch.Size([32])
  --------------------------------------------------------------------------- RuntimeError                              Traceback (most recent call
  last)  in
       53     output = model.forward(images)
       54     print(output.shape, labels.shape)
  ---> 55     loss = criterion(output, labels)
       56     loss.backward()
       57     optimizer.step()

  /opt/conda/lib/python3.7/site-packages/torch/nn/modules/module.py in
  call(self, *input, **kwargs)
      530             result = self._slow_forward(*input, **kwargs)
      531         else:
  --> 532             result = self.forward(*input, **kwargs)
      533         for hook in self._forward_hooks.values():
      534             hook_result = hook(self, input, result)

  /opt/conda/lib/python3.7/site-packages/torch/nn/modules/loss.py in
  forward(self, input, target)
      429
      430     def forward(self, input, target):
  --> 431         return F.mse_loss(input, target, reduction=self.reduction)
      432
      433

  /opt/conda/lib/python3.7/site-packages/torch/nn/functional.py in
  mse_loss(input, target, size_average, reduce, reduction)    2213
  ret = torch.mean(ret) if reduction == 'mean' else torch.sum(ret)
  2214     else:
  -> 2215         expanded_input, expanded_target = torch.broadcast_tensors(input, target)    2216         ret =
  torch._C._nn.mse_loss(expanded_input, expanded_target,
  _Reduction.get_enum(reduction))    2217     return ret

  /opt/conda/lib/python3.7/site-packages/torch/functional.py in
  broadcast_tensors(*tensors)
       50                 [0, 1, 2]])
       51     """
  ---> 52     return torch._C._VariableFunctions.broadcast_tensors(tensors)
       53
       54

  > RuntimeError: The size of tensor a (10) must match the size of tensor
  b (32) at non-singleton dimension 1

How can I reshape the tensor, and which tensor (output or labels) should I change to calculate the loss?

Entire code is attached below.

import numpy as np
import torch

# Loading the Fashion-MNIST dataset
from torchvision import datasets, transforms

# Get GPU Device
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")

transform = transforms.Compose([transforms.ToTensor(),
                                    transforms.Normalize((0.5,), (0.5,))])
# Download and load the training data
trainset = datasets.FashionMNIST('MNIST_data/', download = True, train = True, transform = transform)
testset = datasets.FashionMNIST('MNIST_data/', download = True, train = False, transform = transform)
trainloader = torch.utils.data.DataLoader(trainset, batch_size = 32, shuffle = True, num_workers=4)
testloader = torch.utils.data.DataLoader(testset, batch_size = 32, shuffle = True, num_workers=4)

# Examine a sample
dataiter = iter(trainloader)
images, labels = dataiter.next()

# Define the network architecture
from torch import nn, optim
import torch.nn.functional as F

model = nn.Sequential(nn.Linear(784, 128),
                      nn.ReLU(),
                      nn.Linear(128, 10),
                      nn.LogSoftmax(dim = 1))
model.to(device)

# Define the loss
criterion = nn.MSELoss()

# Define the optimizer
optimizer = optim.Adam(model.parameters(), lr = 0.001)

# Define the epochs
epochs = 5

train_losses, test_losses = [], []

for e in range(epochs):
  running_loss = 0
  for images, labels in trainloader:
    # Flatten Fashion-MNIST images into a 784 long vector
    images = images.to(device)
    labels = labels.to(device)
    images = images.view(images.shape[0], -1)

    # Training pass
    optimizer.zero_grad()
    output = model.forward(images)
    print(output.shape, labels.shape)
    loss = criterion(output, labels)
    loss.backward()
    optimizer.step()

    running_loss += loss.item()
  else:
    test_loss = 0
    accuracy = 0

    # Turn off gradients for validation, saves memory and computation
    with torch.no_grad():
      # Set the model to evaluation mode
      model.eval()

      # Validation pass
      for images, labels in testloader:
        images = images.to(device)
        labels = labels.to(device)
        images = images.view(images.shape[0], -1)
        ps = model(images)
        test_loss += criterion(ps, labels)
        top_p, top_class = ps.topk(1, dim = 1)
        equals = top_class == labels.view(*top_class.shape)
        accuracy += torch.mean(equals.type(torch.FloatTensor))

    model.train()

    print("Epoch: {}/{}..".format(e+1, epochs),
          "Training loss: {:.3f}..".format(running_loss/len(trainloader)),
          "Test loss: {:.3f}..".format(test_loss/len(testloader)),
          "Test Accuracy: {:.3f}".format(accuracy/len(testloader)))


title: Re-weight the input to a neural network

body:
For example, I feed a set of images into a CNN. And the default weight of these images is 1. How can I re-weight some of these images so that they have different weights? Can 'DataLoader' achieve this goal in pytorch?

I learned two other possibilities:

Defining a custom loss function, providing weights for each sample as I require.
Repeating samples in the training set, which will result in more frequent samples having a higher weight in the final loss.

Is there any other way, we can achieve that? Any suggestion would be appreciated.


title: How to make prediction from train Pytorch and PytorchText model?

body:
General speaking, after I have successfully trained a text RNN model with Pytorch, using PytorchText to leverage data loading on an origin source, I would like to test with other data sets (a sort of blink test) that are from different sources but the same text format.

First I defined a class to handle the data loading.

class Dataset(object):
    def __init__(self, config):
        # init what I need

    def load_data(self, df: pd.DataFrame, *args):
        # implementation below
        # Data format like `(LABEL, TEXT)`

    def load_data_but_error(self, df: pd.DataFrame):
        # implementation below
        # Data format like `(TEXT)`

Here is the detail of load_data which I load data that trained successfully.

TEXT = data.Field(sequential=True, tokenize=tokenizer, lower=True, fix_length=self.config.max_sen_len)
LABEL = data.Field(sequential=False, use_vocab=False)

datafields = [(label_col, LABEL), (data_col, TEXT)]

# split my data to train/test
train_df, test_df = train_test_split(df, test_size=0.33, random_state=random_state)

train_examples = [data.Example.fromlist(i, datafields) for i in train_df.values.tolist()]
train_data = data.Dataset(train_examples, datafields)

# split train to train/val
train_data, val_data = train_data.split(split_ratio=0.8)

# build vocab
TEXT.build_vocab(train_data, vectors=Vectors(w2v_file))
self.word_embeddings = TEXT.vocab.vectors
self.vocab = TEXT.vocab

test_examples = [data.Example.fromlist(i, datafields) for i in test_df.values.tolist()]
test_data = data.Dataset(test_examples, datafields)

self.train_iterator = data.BucketIterator(
            (train_data),
            batch_size=self.config.batch_size,
            sort_key=lambda x: len(x.title),
            repeat=False,
            shuffle=True)

self.val_iterator, self.test_iterator = data.BucketIterator.splits(
    (val_data, test_data),
    batch_size=self.config.batch_size,
    sort_key=lambda x: len(x.title),
    repeat=False,
    shuffle=False)

Next is my code (load_data_but_error) to load others source but causing error

TEXT = data.Field(sequential=True, tokenize=tokenizer, lower=True, fix_length=self.config.max_sen_len)
datafields = [('title', TEXT)]

examples = [data.Example.fromlist(i, datafields) for i in df.values.tolist()]
blink_test = data.Dataset(examples, datafields)

self.blink_test = data.BucketIterator(
    (blink_test),
    batch_size=self.config.batch_size,
    sort_key=lambda x: len(x.title),
    repeat=False,
    shuffle=True)

When I was executing code, I had an error AttributeError: 'Field' object has no attribute 'vocab' which has a question at here but it doesn't like my situation as here I had vocab from load_data and I want to use it for blink tests.

My question is what the correct way to load and feed new data with a trained PyTorch model for testing current model is?


title: Is there any way that can convert a data format of .pb file from NCHW into NHWC?

body:
I have a CNN model which was trained in Pytorch based on the data format N(batch) x C(channel) x H(height) x W(width). I saved the pre-trained model as model.pth. Afterward, I converted the pre-trained model from model.pth -> model.onnx by using existing function:

torch.onnx.export(model, dummy_input, "model.onnx")

And then, I converted this model.onnx -> model.pb by the module below:

import onnx
from onnx_tf.backend import prepare

model_onnx = onnx.load('model.onnx')
tf_rep = prepare(model_onnx)
tf_rep.export_graph('model.pb')

The problem is: I want to utilize this model.pb on a CPU device, which needs a NHWC data format. However, my model is based on NCHW data format. Is there any method that can convert the data format of this model.pb from NCHW into NHWC?


title: Pytorch :RuntimeError: Cannot initialize CUDA without ATen_cuda library

body:
I am new to model training in python I have installed pytorch using :

pip install torch==1.4.0+cpu torchvision==0.5.0+cpu -f https://download.pytorch.org/whl/torch_stable.html

Now my model is giving me this error :
RuntimeError: Cannot initialize CUDA without ATen_cuda library. PyTorch splits its backend into two shared libraries: a CPU library and a CUDA library; this error has occurred because you are trying to use some CUDA functionality, but the CUDA library has not been loaded by the dynamic linker for some reason.  The CUDA library MUST be loaded, EVEN IF you don't directly use any symbols from the CUDA library! One common culprit is a lack of -Wl,--no-as-needed in your link arguments; many dynamic linkers will delete dynamic library dependencies if you don't depend on any of their symbols.  You can check if this has occurred by using ldd on your binary to see if there is a dependency on *_cuda.so library.

I know its cuda problem but I have defined in my model if cuda is not available then cpu.My code is

# import the usual resources
import matplotlib.pyplot as plt
import numpy as np

# import utilities to keep workspaces alive during model training
#from workspace_utils import active_session

# watch for any changes in model.py, if it changes, re-load it automatically
#%load_ext autoreload
#%autoreload 2

## TODO: Define the Net in models.py

import torch
import torch.nn as nn
import torch.nn.functional as F
import datetime

## TODO: Once you've define the network, you can instantiate it
# one example conv layer has been provided for you
from torch.utils.data import Dataset, DataLoader
import torch
from torchvision import transforms, utils
from models import Net
#-------------------------------------------------------------------------------
# device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
if torch.cuda.is_available():
    device = torch.device("cuda:0")
else:
    device = torch.device("cpu")
#-------------------------------------------------------------------------------
print(device)
print(torch.version)
net = Net().to(device)
print(net)

from torch.utils.data import Dataset, DataLoader
from torchvision import transforms, utils

# the dataset we created in Notebook 1 is copied in the helper file `data_load.py`
from data_load import FacialKeypointsDataset
# the transforms we defined in Notebook 1 are in the helper file `data_load.py`
from data_load import Rescale, RandomCrop, Normalize, ToTensor

## TODO: define the data_transform using transforms.Compose([all tx's, . , .])
# order matters! i.e. rescaling should come before a smaller crop

# testing that you've defined a transform

data_transform = transforms.Compose(
    [Rescale(250), RandomCrop(224),
     Normalize(), ToTensor()])
assert (data_transform is not None), 'Define a data_transform'

# create the transformed dataset
transformed_dataset = FacialKeypointsDataset(
    csv_file=
    "person_keypoints_train2017_foot_v1\\NEWOUT.csv",
    root_dir=
    "person_keypoints_train2017_foot_v1\\out\\",
    transform=data_transform)

print('Number of images: ', len(transformed_dataset))

# iterate through the transformed dataset and print some stats about the first few samples
for i in range(4):
    sample = transformed_dataset[i]
    print(i, sample['image'].size(), sample['keypoints'].size())

# load training data in batches
batch_size = 32

train_loader = DataLoader(
    transformed_dataset, batch_size=batch_size, shuffle=True, num_workers=0)

# load in the test data, using the dataset class
# AND apply the data_transform you defined above

# create the test dataset
test_dataset = FacialKeypointsDataset(
    csv_file=
    "person_keypoints_val2017_foot_v1\\NEWOUT.csv",
    root_dir=
    "person_keypoints_val2017_foot_v1\\ready_val_out\\",
    transform=data_transform)

test_loader = DataLoader(
    test_dataset, batch_size=batch_size, shuffle=True, num_workers=0)

# test the model on a batch of test images

def net_sample_output():

    # iterate through the test dataset
    for i, sample in enumerate(test_loader):

        # get sample data: images and ground truth keypoints
        images = sample['image']
        key_pts = sample['keypoints']

        # convert images to FloatTensors
        images = images.type(torch.cuda.FloatTensor)

        # forward pass to get net output
        output_pts = net(images)

        # reshape to batch_size x 68 x 2 pts ##CHANGED
        output_pts = output_pts.view(output_pts.size()[0], 6, -1)

        # break after first image is tested
        if i == 0:
            return images, output_pts, key_pts

test_images, test_outputs, gt_pts = net_sample_output()

# print out the dimensions of the data to see if they make sense
print(test_images.data.size())
print(test_outputs.data.size())
print(gt_pts.size())

def show_all_keypoints(image, predicted_key_pts, gt_pts=None):
    """Show image with predicted keypoints"""
    # image is grayscale
    plt.imshow(image, cmap='gray')
    plt.scatter(
        predicted_key_pts[:, 0],
        predicted_key_pts[:, 1],
        s=20,
        marker='.',
        c='m')
    # plot ground truth points as green pts
    if gt_pts is not None:
        plt.scatter(gt_pts[:, 0], gt_pts[:, 1], s=20, marker='.', c='g')

# visualize the output
# by default this shows a batch of 10 images
def visualize_output(test_images, test_outputs, gt_pts=None, batch_size=4):
    fig = plt.figure()
    iimg = 1
    sp = 1
    for i in range(batch_size):

        iimg += 1
        if i % 2 == 0:
            sp += 1
            iimg = 1
        fig.add_subplot(sp, batch_size, iimg)

        # un-transform the image data
        image = test_images.cpu(
        )[i].data  # get the image from it's Variable wrapper
        image = image.numpy()  # convert to numpy array from a Tensor
        image = np.transpose(
            image, (1, 2, 0))  # transpose to go from torch to numpy image

        # un-transform the predicted key_pts data
        predicted_key_pts = test_outputs.cpu()[i].data
        predicted_key_pts = predicted_key_pts.numpy()
        # undo normalization of keypoints
        predicted_key_pts = predicted_key_pts * 50.0 + 100

        # plot ground truth points for comparison, if they exist
        ground_truth_pts = None
        if gt_pts is not None:
            ground_truth_pts = gt_pts[i]
            ground_truth_pts = ground_truth_pts * 50.0 + 100

        # call show_all_keypoints
        show_all_keypoints(
            np.squeeze(image), predicted_key_pts, ground_truth_pts)

        plt.axis('off')

    plt.show()

# call it
#visualize_output(test_images, test_outputs, gt_pts)

## TODO: Define the loss and optimization
import torch.optim as optim

#criterion = nn.CrossEntropyLoss()
criterion = nn.MSELoss()

#optimizer = optim.SGD(net.parameters(), lr=0.001, momentum=0.9)
optimizer = optim.Adam(params=net.parameters(), lr=0.001)

def train_net(n_epochs):
    print("Training...")

    # prepare the net for training
    net.train()

    for epoch in range(n_epochs):  # loop over the dataset multiple times

        running_loss = 0.0

        # train on batches of data, assumes you already have train_loader
        for batch_i, data in enumerate(train_loader):
            # get the input images and their corresponding labels
            images = data['image']
            key_pts = data['keypoints']

            # flatten pts
            key_pts = key_pts.view(key_pts.size(0), -1)

            # convert variables to floats for regression loss
            key_pts = key_pts.type(torch.cuda.FloatTensor)
            images = images.type(torch.cuda.FloatTensor)

            # forward pass to get outputs
            output_pts = net(images)
            #output_pts = output_pts.type(torch.cuda.FloatTensor)
            #print(output_pts.type)
            #print(key_pts.type)
            # calculate the loss between predicted and target keypoints
            loss = criterion(output_pts, key_pts)

            # zero the parameter (weight) gradients
            optimizer.zero_grad()

            # backward pass to calculate the weight gradients
            loss.backward()

            # update the weights
            optimizer.step()

            # print loss statistics
            running_loss += loss.item()
            if batch_i % 32 == 31:  # print every 10 batches
                print(datetime.datetime.now())
                print('Epoch: {}, Batch: {}, Avg. Loss: {}'.format(
                    epoch + 1, batch_i + 1, running_loss / 1000))
                running_loss = 0.0

    print('Finished Training')

# train your network
n_epochs = 500  # start small, and increase when you've decided on your model structure and hyperparams

# this is a Workspaces-specific context manager to keep the connection
# alive while training your model, not part of pytorch
#with active_session():

#train_net(n_epochs)

## TODO: change the name to something uniqe for each new model
model_dir = 'D:\\Users\\Tsvetan\\FootDataset\\'
model_name = 'keypoints.pt'

cpnt = torch.load(model_dir + "500" + model_name)
print()
print()
for key in cpnt.keys():
    print(key + " {}".format(cpnt[key]))
print()
print()
net.load_state_dict(torch.load(model_dir + "500" + model_name))

train_net(10)

net.eval()

# after training, save your model parameters in the dir 'saved_models'
#torch.save(net.state_dict(), model_dir + model_name)

# Get the weights in the first conv layer, "conv1"
# if necessary, change this to reflect the name of your first conv layer
weights1 = net.conv1.weight.data.cpu()

w = weights1.numpy()

filter_index = 0

print(w[filter_index][0])
print(w[filter_index][0].shape)

# display the filter weights
#plt.imshow(w[filter_index][0], cmap='gray')

####
test_images, test_outputs, gt_pts = net_sample_output()
visualize_output(test_images, test_outputs, gt_pts)

I have re-write the condition but its still giving me error.


title: What is the PyTorch alternative for Keras input_shape, output_shape, get_weights, get_config and summary

body:
In Keras, after creating a model, we can see its input and output shapes using model.input_shape, model.output_shape. For weights and config  we can use model.get_weights() and model.get_config(), respectively.

What are the similar alternatives for PyTorch? Also is there any other functions we need to know for inspecting a PyTorch model?

To get the summary in PyTorch, we print model print(model) but this gives lesser information than model.summary(). Is there a better summary for PyTorch?


title: Modifying the forward function in vgg model

body:
I need to modify the existing forward method in VGG16 so that it can  pass through the two classifiers and return the value

I tried creating the custom forward method manually and overriding the existing method but I get the following error

vgg.forward = forward

forward() missing 1 required positional argument: 'x'

My custom forward function

def forward(self,x):
    x = self.features(x)
    x = self.avgpool(x)
    x = x.view(x.size(0), -1)
    x = self.classifier(x)
    y = self.classifier_2(x)
    return x,y

I have modified the default vgg16_bn with one additional classifier as

vgg = models.vgg16_bn()
final_in_features = vgg.classifier[6].in_features
mod_classifier = list(vgg.classifier.children())[:-1]
mod_classifier.extend([nn.Linear(final_in_features, 10)])
vgg.add_module('classifier_2',vgg.classifier)

My model looks like this after the addition of above classifier

(classifier): Sequential(
    (0): Linear(in_features=25088, out_features=4096, bias=True)
    (1): ReLU(inplace)
    (2): Dropout(p=0.5)
    (3): Linear(in_features=4096, out_features=4096, bias=True)
    (4): ReLU(inplace)
    (5): Dropout(p=0.5)
    (6): Linear(in_features=4096, out_features=10, bias=True)
  )
  (classifier_2): Sequential(
    (0): Linear(in_features=25088, out_features=4096, bias=True)
    (1): ReLU(inplace)
    (2): Dropout(p=0.5)
    (3): Linear(in_features=4096, out_features=4096, bias=True)
    (4): ReLU(inplace)
    (5): Dropout(p=0.5)
    (6): Linear(in_features=4096, out_features=10, bias=True)
  )

My convolutional layers results are supposed to be passed through two separate FFN layers. So how do i modify my forward pass


title: How can I cross-validate by Pytorch and Optuna

body:
I want to use cross-validation against the official Optuna and pytorch-based sample code (https://github.com/optuna/optuna/blob/master/examples/pytorch_simple.py).
I thought about splitting the data for cross-validation and trying parameter tuning for each fold, but it seems that the average accuracy of each parameter cannot be obtained because the parameters that can be checked in study.trials_dataframe() are different each time.


title: Load huge image dataset and some data augmentation using Pytorch

body:
I have a question. I have about 2 million images (place365-standard dataset) and I want to do some data augmentation like transforming, cropping etc. Also, I have to make my own target image (y) based on some color model algorithms (CMYK) for example.

So Actually, my preprocessing step includes augmentation and making terget image (y). And then I should feed these images to a deep network. When should I do this based on Dataset class? Should I do my processing step in __getitem__()? If yes, would it be parallel and fast?

Here is my template of Dataset(data.Dataset) class:

import torch
from torch.utils import data

class Dataset(data.Dataset):
    """
    Return Dataset class representing our data set
    """
    def __int__(self, list_IDs, labels):
        """
        Initialize data set as a list of IDs corresponding to each item of data set and labels of each data

        Args:
            list_IDs: a list of IDs for each data point in data set
            labels: label of an item in data set with respect to the ID
        """

        self.labels = labels
        self.list_IDs = list_IDs

    def __len__(self):
        """
        Return the length of data set using list of IDs

        :return: number of samples in data set
        """
        return len(self.list_IDs)

    def __getitem__(self, item):
        """
        Generate one item of data set. Here we apply our preprocessing things like halftone styles and subtractive color process using CMYK color model etc. (See the paper for operations)

        :param item: index of item in IDs list

        :return: a sample of data
        """
        ID = self.list_IDs[item]

        # Code to load data
        X = None #

        # code to apply your custom function to make y image (time consuming task - some algorithms)
        y = None #

        return X, y

Thanks for any advice

Best regards


title: How to use TPUs with PyTorch?

body:
I am trying to use TPU using pytorch_xla, but it shows import error in _XLAC.
!curl https://raw.githubusercontent.com/pytorch/xla/master/contrib/scripts/env-setup.py -o pytorch-xla-env-setup.py
!python pytorch-xla-env-setup.py --version $VERSION

import torch_xla
import torch_xla.core.xla_model as xm

ImportError                               Traceback (most recent call last)
<ipython-input-60-6a19e980152f> in <module>()
----> 1 import torch_xla
      2 import torch_xla.core.xla_model as xm

/usr/local/lib/python3.6/dist-packages/torch_xla/__init__.py in <module>()
     39 import torch
     40 from .version import __version__
---> 41 import _XLAC
     42
     43 _XLAC._initialize_aten_bindings()

ImportError: /usr/local/lib/python3.6/dist-packages/_XLAC.cpython-36m-x86_64-linux-gnu.so: undefined symbol: _ZN2at6native6einsumENSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEN3c108ArrayRefINS_6TensorEEE


title: training a RNN in Pytorch

body:
I want to have an RNN model and teach it to learn generating "ihello" from "hihell". I am new in Pytorch and following the instruction in a video to write the code.
I have written two python files named train.py and model.py.
this is model.py:

#----------------- model for teach rnn hihell to ihello
#-----------------  OUR MODEL ---------------------
import torch
import torch.nn as nn
from torch import autograd

class Model(nn.Module):
    def __init__(self):
        super(Model,self).__init__()
        self.rnn=nn.RNN(input_size=input_size,hidden_size=hidden_size,batch_first=True)
    def forward(self,x,hidden):
        #Reshape input in (batch_size,sequence_length,input_size)
        x=x.view(batch_size,sequence_length,input_size)
        #Propagate input through RNN
        #Input:(batch,seq+len,input_size)
        out,hidden=self.rnn(x,hidden)
        out=out.view(-1,num_classes)
        return hidden,out
    def init_hidden(self):
        #Initialize hidden and cell states
        #(num_layers*num_directions,batch,hidden_size)
        return autograd.Variable(torch.zeros(num_layers,batch_size,hidden_size))

and this is train.py:

"""----------------------train for teach rnn to hihell to ihello--------------------------"""
#-----------------  DATA PREPARATION ---------------------
#Import
import torch
import torch.nn as nn
from torch import autograd
from model import Model
import sys

idx2char=['h','i','e','l','o']
#Teach hihell->ihello
x_data=[0,1,0,2,3,3]#hihell
y_data=[1,0,2,3,3,4]#ihello
one_hot_lookup=[[1,0,0,0,0],#0
                [0,1,0,0,0],#1
                [0,0,1,0,0],#2
                [0,0,0,1,0],#3
                [0,0,0,0,1]]#4
x_one_hot=[one_hot_lookup[x] for x in x_data]
inputs=autograd.Variable(torch.Tensor(x_one_hot))
labels=autograd.Variable(torch.LongTensor(y_data))
""" ----------- Parameters Initialization------------"""
num_classes = 5
input_size = 5  # one hot size
hidden_size = 5  # output from LSTM to directly predict onr-hot
batch_size = 1  # one sequence
sequence_length = 1  # let's do one by one
num_layers = 1  # one layer RNN
"""-----------------  LOSS AND TRAINING ---------------------"""
#Instantiate RNN model
model=Model()
#Set loss and optimizer function
#CrossEntropyLoss=LogSoftmax+NLLLOSS
criterion=torch.nn.CrossEntropyLoss()
optimizer=torch.optim.Adam(model.parameters(),lr=0.1)

"""----------------Train the model-------------------"""
for epoch in range(100):
    optimizer.zero_grad()
    loss=0
    hidden=model.init_hidden()
    sys.stdout.write("Predicted String:")
    for input,label in zip(inputs,labels):
        #print(input.size(),label.size())
        hidden,output=model(input,hidden)
        val,idx=output.max(1)
        sys.stdout.write(idx2char[idx.data[0]])
        loss+=criterion(output,label)
    print(",epoch:%d,loss:%1.3f"%(epoch+1,loss.data[0]))
    loss.backward()
    optimizer.step()

when I run train.py, I receive this error:

  self.rnn=nn.RNN(input_size=input_size,hidden_size=hidden_size,batch_first=True)
      NameError: name 'input_size' is not defined

I don't know why I receive this error because I have input_size=5 in the above lines of my code. could anybody help me? thanks.


title: Creating custom dataset in PyTorch

body:
Problem

In PyTorch, I am trying to write a class that could return the entire data and label separately using syntax like dataset.data and dataset.label. The code skeleton looks like:

class MyDataset(object):
  data = _get_data()
  label = _get_label()
  def __init__(self, dir, transforms):
    self.img_list = ... # all image paths loaded from dir
    # do something

  def __getitem__(self):
    # do something
    return data, label

  def __len__(self):
    return len(self.img_list)

  def _get_data():
    # do something

  def _get_label():
    # do something

However, when I use dataset.data and dataset.label to access the corresponding variables, nothing is returned.

I am wondering why this is the case and how I can fix this.

Edit

Thank you for all of your attention.

I have solved this problem by myself. The solution is pretty straightforward, which just utilizes the property of class variables.

class FaceDataset(object):
    # class variable
    data = None
    label = None

    def __init__(self, root, transforms=None):
        # read img_list from root
        img_list = ...
        self.transforms = ...
        FaceDataset.data = FaceDataset._get_data(self.img_list, self.transforms)
        FaceDataset.label = FaceDataset._get_label(self.img_list)

    @classmethod
    def _get_data(cls, img_list, transforms):
        data_list = []
        for img_path in img_list:
            data_list.append(transforms(Image.open(img_path)).unsqueeze(0))
        return torch.stack(data_list, dim=0)

    @classmethod
    def _get_label(cls, img_list):
        label = torch.zeros(len(img_list))
        for i, img_path in enumerate(img_list):
            label[i] = ...
        return label

    def __getitem__(self, index):
        img_path = self.img_list[index]
        label = ...

        # read image from file
        data = Image.open(img_path)
        # apply transform defined in __init__
        data = self.transforms(data)

        return data, label

    def __len__(self):
        return len(self.img_list)