title: Saving/Loading PyTorch Models Wrapped in sklearn Compatable Estimators

body:
I have written an Autoencoder using PyTorch and I have rolled it into a custom sklearn BaseEstimator. I normally train the estimator on a machine with a GPU and save them for later evaluation using pickle. If I try to load an estimator on a machine where the model was stored on the GPU, I get the following error:

  RuntimeError: Attempting to deserialize object on a CUDA device but torch.cuda.is_available() is False. If you are running on a CPU-only machine, please use torch.load with map_location='cpu' to map your storages to the CPU.

Is there a way to force the PyTorch model to be moved to the CPU before pickling the estimator without an explicit call?
Is there a way to unpickle an estimator that was saved while the model was on the GPU?

The following is an example of my PyTorch model and sklearn comptatable estimator, along with an example of how I am trying to save and load my models.

PyTorch Model

import torch.nn as nn
class _AutoEncoder(nn.Module):
    def __init__(self, input_dim, output_dim, encoder_dim=4):
        super(_AutoEncoder, self).__init__()
        hidden_dim = int( (input_dim+encoder_dim)/2 )
        layers = []
        layers.append( nn.Linear(input_dim, hidden_dim) )
        layers.append( nn.Linear(hidden_dim, encoder_dim) )
        self.encoder = nn.Sequential(*layers)
        layers = []
        layers.append( nn.Linear(encoder_dim, hidden_dim) )
        layers.append( nn.Linear(hidden_dim, output_dim) )
        self.decoder = nn.Sequential(*layers)

    def forward(self, X):
        return self.decoder( self.encoder( X ) )

sklearn Comptable Estimator

import warnings
import inspect
import numpy as np
from sklearn.base import BaseEstimator, TransformerMixin
from sklearn import utils as sk_utils

import torch
import torch.nn as nn
import torch.optim as optim
import torch.utils.data

class AutoEncoder(BaseEstimator, TransformerMixin):
    def __init__(
        self,
        encoder_dim=4,
        n_epochs=200,
        batch_size=None,
        shuffle=True,
        use_cuda=False
    ):
        super(AutoEncoder, self).__init__()
        args, _, _, values = inspect.getargvalues(inspect.currentframe())
        values.pop("self")
        for arg, val in values.items():
            setattr(self, arg, val)

        if use_cuda:
            if torch.cuda.is_available():
                self.device = torch.device("cuda")
            else:
                self.device = torch.device("cpu")
                warnings.warn("cuda not avaliable", UserWarning)
        else:
            self.device = torch.device("cpu")

    def fit(self, X, y=None):
        # X, y = sk_utils.check_X_y(X, y, ensure_2d=False, allow_nd=True)
        self._model = self._train_classifier(X, y)
        return self

    def transform(self, X):
        sk_utils.validation.check_is_fitted(self, ['_model'])
        X = sk_utils.check_array(X)
        X = torch.from_numpy(X.astype(np.float32)).to(self.device)
        with torch.no_grad():
            self._model.eval()
            output = self._model.forward( X )
            return output.cpu().numpy()

    def encode(self, X):
        sk_utils.validation.check_is_fitted(self, ['_model'])
        X = sk_utils.check_array(X)
        X = torch.from_numpy(X.astype(np.float32)).to(self.device)
        with torch.no_grad():
            self._model.eval()
            output = self._model.encoder( X )
            return output.cpu().numpy()

    def decode(self, X):
        sk_utils.validation.check_is_fitted(self, ['_model'])
        X = sk_utils.check_array(X)
        X = torch.from_numpy(X.astype(np.float32)).to(self.device)
        with torch.no_grad():
            self._model.eval()
            output = self._model.decoder( X )
            return output.cpu().numpy()

    def _train_classifier(self, x_train, y_train):
        x_train = torch.from_numpy(x_train.astype(np.float32)).to(self.device)
        y_train = torch.from_numpy(y_train.astype(np.float32)).to(self.device)
        input_dim = x_train.shape[-1]
        output_dim = y_train.shape[-1]
        model = _AutoEncoder(input_dim, output_dim, encoder_dim=self.encoder_dim).to(self.device)
        loss_function = nn.MSELoss()
        optimizer = optim.Adam(model.parameters())
        print model
        if self.batch_size is None:
            return self._batch_train_simple_classifier(x_train, y_train, model, loss_function, optimizer)
        else:
            return self._minibatch_train_simple_classifier(x_train, y_train, model, loss_function, optimizer)

    def _batch_train_simple_classifier(self, x_train, y_train, model, loss_function, optimizer):
        for epoch in range(1, self.n_epochs+1):
            model.train()
            optimizer.zero_grad()
            outputs = model.forward(x_train)
            loss = loss_function(outputs, y_train)
            loss.backward()
            optimizer.step()
            if epoch % 10 == 0 or epoch == self.n_epochs:
                message = "Train Epoch: {:5d},    Loss: {:15.6f}".format(
                    epoch,
                    loss.item()
                )
                print message
        return model

    def _minibatch_train_simple_classifier(self, x_train, y_train, model, loss_function, optimizer):
        train_data = torch.utils.data.TensorDataset(x_train, y_train)
        train_loader = torch.utils.data.DataLoader(train_data, batch_size=self.batch_size, shuffle=self.shuffle)
        for epoch in range(1, self.n_epochs+1):
            for data, target in train_loader:
                model.train()
                optimizer.zero_grad()
                outputs = model.forward(data)
                loss = loss_function(outputs, target)
                loss.backward()
                optimizer.step()
            if epoch % 10 == 0 or epoch == self.n_epochs:
                model.eval()
                outputs = model.forward(x_train)
                loss = loss_function(outputs, y_train)
                message = "Train Epoch: {:5d},    Loss: {:15.6f}".format(
                    epoch,
                    loss.item()
                )
                print message
        return model

Training

This is normally done on a machine with a GPU.

from sklearn import datasets as sk_datasets
digits = sk_datasets.load_digits(n_class=10, return_X_y=False)
data = digits.data

ae = AutoEncoder(
    encoder_dim=2,
    n_epochs=100,
    batch_size=128,
    shuffle=True,
    use_cuda=True
)

data_fitted = ae.fit_transform(data, data)

Saving the Estimator

I'd like to find a way to have the PyTorch model moded to the CPU before it is saved without having an explicit call.  Maybe a function as part of the AutoEncoder class that gets called as it is pickled?

with open("autoencoder.pkl", "wb") as fp:
    # ae._model needs to be moved to the CPU here.
    # I don't want to have to call ae._model.cpu() explicitly
    pickle.dump(ae, fp)

Loading

I can not figure out how to load the estimator on a machine without a GPU if it was saved while the PyTorch model was still on the GPU.

# This gives an error if the model was saved while on the GPU,
# and a GPU is not avaiable when loading.
with open("autoencoder.pkl", "rb") as fp:
    model = pickle.load(fp)

# This also a similar error. I also would not expect this to
# work since the pickle file contains an sklearn estimator
# wrapping a PyTorch model.
with open("autoencoder.pkl", "rb") as fp:
    touch.load(fp, map_location="cpu")


title: Pytorch - Convert a list of numpy array to a list of tensor

body: I have a list of numpy array. Is there a quick way to convert them into tensor in Pytorch? I know I can do it simply using a for loop. But are there any other ways to do so?


title: Fastest way to read an image from huge uncompressed tar file in __getitem__ of PyTorch custom dataset

body:
I have a huge dataset (2 million) of jpg images in one uncompressed TAR file. I also have a txt file each line is the name of the image in TAR file in order.

img_0000001.jpg
img_0000002.jpg
img_0000003.jpg
...

and images in tar file are exactly the same.
I searched alot and find out tarfile module is the best one, but when I tried to read images from tar file using name, it takes too long. And the reason is, everytime I call getmemeber(name) method, it calls getmembers() method which scan whole tar file, then return a Namespace of all names, then start finding in this Namespace.

if it helps, my dataset size is 20GB single tar file.

I don't know it is better idea to first extract all then use extracted folders in my CustomDataset or reading directly from archive.

Here is the code I am using to read a single file from tar file:

        with tarfile.open('data.tar') as tf:
            tarinfo = tf.getmember('img_000001.jpg')
            image = tf.extractfile(tarinfo)
            image = image.read()
            image = Image.open(io.BytesIO(image))

I used this code in my __getitem__ method of CustomDataset class that loops over all names in filelist.txt

Thanks for any advice


title: Replicate subtensors in PyTorch

body:
I have a tensor “image_features” having shape torch.Size([100, 1024, 14, 14]). I need to replicate each subtensor (1024, 14, 14) 10 times, obtaining a tensor having shape torch.Size([1000, 1024, 14, 14]).

Basically, the first ten rows of the resulting tensor should correspond to the first row of the original one, the following ten rows of the resulting tensor should correspond to the second row of the original one, and so on. If possible, I don’t want to create a copy (each replicated subtensor can share the memory with the tensor it is replicated from), but it is ok to create a copy if there isn’t any other way.

How can I do it?

Thank you very much.


title: How to get weights shape for each layer?

body:
There is a good question how to get model summary in pytorch
Model summary in pytorch but it doesn't output shape of weights.

Is it possible also to output shape of weights for each layer?


title: Loss Function Not Decreasing in CNN

body:
I am new to Pytorch and I'm training a model for binary classification of images. The images are currently stored as .npy files and I am loading them and training my model in batches. When I implement this, the loss function does not decrease. When I test the model on the training and test set again, the accuracy is constant at 50%. The data set is balanced.

I tried making the dataset smaller (around 125 for each class) and I still have the same problem. I expect the model to overfit the training set but this does not occur.

Please see my code below

class Network(nn.Module):
    def __init__(self):
        super(Network,self).__init__()
        self.conv1=nn.Conv2d(in_channels=2, out_channels=32, kernel_size=3)
        self.conv2=nn.Conv2d(in_channels=32, out_channels=64, kernel_size=3)
        self.conv3=nn.Conv2d(in_channels=64, out_channels=128, kernel_size=3)
        self.fc1=nn.Linear(in_features=128*6*6, out_features=1000)
        self.fc2=nn.Linear(in_features=1000, out_features=100)
        self.out=nn.Linear(in_features=100, out_features=2)
    def forward(self,t):
        POOL_stride=2
        #Conv1
        t=F.relu(self.conv1(t))
        t=F.max_pool2d(t, kernel_size=2, stride=POOL_stride)
        #Conv2
        t=F.relu(self.conv2(t))
        t=F.max_pool2d(t, kernel_size=2, stride=POOL_stride)
        #Conv3
        t=F.relu(self.conv3(t))
        t=F.max_pool2d(t, kernel_size=2, stride=POOL_stride)
        # dense 1
        t=t.reshape(-1, 128*6*6)
        t=self.fc1(t)
        t=F.relu(t)
        #dense 2
        t=self.fc2(t)
        t=F.relu(t)
        t=self.out(t)
        return t

def npy_loader(path):
    sample = torch.from_numpy(np.load(path))
    return sample

criterion=nn.CrossEntropyLoss()
optimizer = optim.Adam(self.model.parameters(), lr=0.003)
model = Network()
trainset = datasets.DatasetFolder(
            root=train_dir,
            loader=npy_loader,
            extensions=['.npy']
        )
train_loader = torch.utils.data.DataLoader(
            trainset,
            batch_size=batch_size,
            shuffle=True,
        )
for epoch in range(epochs):
            running_loss = 0
            batches = 0
            for inputs, labels in train_loader:
                batches = batches+1
                inputs, labels = inputs.to(device), labels.to(device)
                optimizer.zero_grad()
                output = model(inputs)
                loss = criterion(output.squeeze(), labels.squeeze())
                loss.backward()
                optimizer.step()
                running_loss += loss.item()
            print('Loss :{:.4f} Epoch[{}/{}]'.format(running_loss/batches, epoch, epochs))
'''


title: Obtain torch.tensor from string of floats

body:
We can convert 1 dimensional array of floats, stored as a space separated numbers in text file, in to a numpy array or a  torch tensor as follows.

line = "1 5 3 7 4"
np_array = np.fromstring(line, dtype='int', sep=" ")

np_array
>> array([1, 5, 3, 7, 4])

And to convert above numpy array to a torch tensor, we can do following :

torch_tensor = torch.tensor(np_array)
torch_tensor
>>tensor([1, 5, 3, 7, 4])

How can I convert a string of numbers separated by space in to a torch.Tensor directly without
converting them to a numpy array? We can also do this by fist splitting the string at a space, mapping them to int or float, and then feeding it to torch.tensor. But like numpy's fromstring, is there any such method in pytorch?


title: Fine-Tuning DistilBertForSequenceClassification: Is not learning, why is loss not changing? Weights not updated?

body:
I am relatively new to PyTorch and Huggingface-transformers and experimented with DistillBertForSequenceClassification on this Kaggle-Dataset.
from transformers import DistilBertForSequenceClassification
import torch.optim as optim
import torch.nn as nn
from transformers import get_linear_schedule_with_warmup

n_epochs = 5 # or whatever
batch_size = 32 # or whatever

bert_distil = DistilBertForSequenceClassification.from_pretrained('distilbert-base-uncased')
#bert_distil.classifier = nn.Sequential(nn.Linear(in_features=768, out_features=1), nn.Sigmoid())
tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased')
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(bert_distil.parameters(), lr=0.1)

X_train = []
Y_train = []

for row in train_df.iterrows():
    seq = tokenizer.encode(preprocess_text(row[1]['text']),  add_special_tokens=True, pad_to_max_length=True)
    X_train.append(torch.tensor(seq).unsqueeze(0))
    Y_train.append(torch.tensor([row[1]['target']]).unsqueeze(0))
X_train = torch.cat(X_train)
Y_train = torch.cat(Y_train)

running_loss = 0.0
bert_distil.cuda()
bert_distil.train(True)
for epoch in range(n_epochs):
    permutation = torch.randperm(len(X_train))
    j = 0
    for i in range(0,len(X_train), batch_size):
        optimizer.zero_grad()
        indices = permutation[i:i+batch_size]
        batch_x, batch_y = X_train[indices], Y_train[indices]
        batch_x.cuda()
        batch_y.cuda()
        outputs = bert_distil.forward(batch_x.cuda())
        loss = criterion(outputs[0],batch_y.squeeze().cuda())
        loss.requires_grad = True

        loss.backward()
        optimizer.step()

        running_loss += loss.item()
        j+=1
        if j == 20:
            #print(outputs[0])
            print('[%d, %5d] running loss: %.3f loss: %.3f ' %
              (epoch + 1, i*1, running_loss / 20, loss.item()))
            running_loss = 0.0
            j = 0

[1,   608] running loss: 0.689 loss: 0.687
[1,  1248] running loss: 0.693 loss: 0.694
[1,  1888] running loss: 0.693 loss: 0.683
[1,  2528] running loss: 0.689 loss: 0.701
[1,  3168] running loss: 0.690 loss: 0.684
[1,  3808] running loss: 0.689 loss: 0.688
[1,  4448] running loss: 0.689 loss: 0.692  etc...

Regardless on what I tried, loss did never decrease, or even increase, nor did the prediction get better. It seems to me that I forgot something so that weights are actually not updated. Someone has an idea?
O
what I tried

Different loss functions

BCE
CrossEntropy
even MSE-loss

One-Hot Encoding vs A single neuron output
Different learning rates, and optimizers
I even changed all the targets to only one single label, but even then, the network did'nt converge.


title: PyTorch dll issues for Caffe2

body:
I am using a Windows 10 Machine and after re-installing Anaconda and all of the packages I had previously, including torchvision, torch and necessary dependencies, I am still getting this error:
OSError: [WinError 127] The specified procedure could not be found. Error loading "C:\Users\XXX\Anaconda3\envs\XXX\lib\site-packages\torch\lib\caffe2.dll" or one of its dependencies.
I am using python 3.7.9 and:

torchaudio=0.6.0=py37
torchvision=0.7.0=py37_cpu
tornado=6.0.4=py37he774522_1
traitlets=5.0.5=py_0

I've looked into it quite a bit but feel like this should be an easy solve...
I do not have CUDA and have used this:
conda install pytorch torchvision torchaudio cpuonly -c pytorch
as per instructed on:
https://pytorch.org/


title: Initialize weights and bias in torch

body:
What is the equivalent command for the below keras code in Pytorch

Dense(64, kernel_initializer='he_normal', bias_initializer='zeros', name='uhat_digitcaps')(d5)

How to I initialize weights and bias?

Thanks!


title: Appending a recurrent layer to PyTorch LSTM model with different hidden size

body:
I'm developing a BI-LSTM model for sequence analysis using PyTorch. For which I am using torch.nn.LSTM. Using that module, you can have several layers with just passing a parameter num_layers to be the number of layers (e.g., num_layers=2). However all of them will have the same hidden_size which is partially fine for me, I just want to have all of them the same hidden_size but the last layer  with a different size. Basic example follows:

rnn = nn.LSTM(input_size=10, hidden_size=20, num_layers=2)
inp = torch.randn(5, 3, 10)
h0 = torch.randn(2, 3, 20)
c0 = torch.randn(2, 3, 20)
output, (hn, cn) = rnn(inp, (h0, c0))

The output dim is (5, 3,20)

One solution (But unfavorable to me) is implementing extra model that outputs the dimension I need and takes the input from the first model, e.g.,:

rnn_two = nn.LSTM(input_size=20, hidden_size=2)
output2, _ = rnn_two(output)

However, I do not wanna do this because I parallelize the model using DataParallel, so I need all to be one package. I was hoping to find something similar to keras, e.g.,:

rnn.add(LSTM, hidden_size=2)

I have checked the LSTM source code but couldn't find what I need.

Any suggestions?


title: How to train identical torch.nn.Modules with different parameters in parallel in GPU

body:
I have two instances obj1 and obj2 of a pytorch nn.Module like this one

class Embedding(nn.Module):

    def __init__(self, dropout=0, emb_dim):
        self.lin = nn.Linear(1, emb_dim)
        init.xavier_uniform(self.lin1.weight, gain=np.sqrt(2.0))

    def forward(self, values)
        return F.relu(self.lin1(values))

And I have two tensors of data t1, t2
I would like to do

obj1.forward(t1)
obj2.forward(t2)

in parallel in the GPU, and intuitively this would have to be possible to do, since I am executing the same operations but with different data.
Is there any way I can use pytorch so I can do this?


title: Why doesn't the learning rate (LR) go below 1e-08 in pytorch?

body:
I am training a model. To overcome overfitting I have done optimization, data augmentation etc etc. I have an updated LR (I tried for both SGD and Adam), and when there is a plateu (also tried step), the learning rate is decreased by a factor until it reaches LR 1e-08 but won't go below than that and my model's validation gets stuck after this point. I tried passing the epsilon parameter to Adam to suggest a smaller value, but it still got stuck at LR 1e-08. I also pass a weight decay, but it doesn't change the situation. Neither did setting the amsgrad to true.

I did some research and people suggest that Adam optimizer has inherent problems but nothing is mentioned about the learning rate - and every discussion added that with SGD, there is no problem.

Why is this? Is it a bug or is it designed so because authors think it is meaninglessly a small value after that? It seems like it would really help to have a smaller learning rate for my dataset because all seems well up until learning rate is down to LR 1e-08.


title: How to convert numpy.array(dtype=object) to tensor?

body:
I have imported all the required libraries for PyTorch. Here are the numpy arrays obtained from my image. And the size of my images are 150x150 grayscale images
array([[array([[ 89, 117,  59, ...,  39,  48,  38],
       [118,  85,  96, ...,  41,  42,  31],
       [171, 118,  70, ...,  33,  42,  32],
       ...,
       [ 32,  49,  32, ..., 153, 181, 180],
       [ 30,  25,  28, ..., 150, 180, 185],
       [ 48,  33,  34, ..., 159, 168, 184]], dtype=uint8),
        array([0., 1.])],
       [array([[195, 192, 202, ..., 255, 254, 255],
       [206, 190, 200, ..., 255, 255, 255],
       [221, 186, 192, ..., 254, 254, 255],
       ...,
       [254, 255, 255, ..., 254, 253, 255],
       [255, 255, 255, ..., 255, 255, 255],
       [255, 255, 255, ..., 255, 255, 255]], dtype=uint8),
        array([0., 1.])],
       [array([[60, 58, 66, ..., 10, 11, 10],
       [61, 59, 56, ..., 11, 11, 11],
       [58, 53, 63, ..., 10, 11, 10],
       ...,
       [23, 45, 56, ..., 46, 38, 19],
       [71, 46, 91, ..., 34, 40, 36],
       [69, 76, 32, ..., 65, 26, 51]], dtype=uint8),
        array([1., 0.])],
       ...,
       [array([[247, 234, 231, ..., 112, 112, 114],
       [243, 229, 230, ..., 113, 114, 114],
       [236, 228, 231, ..., 118, 117, 115],
       ...,
       [ 22,  24,  24, ...,  57,  61,  61],
       [ 28,  26,  25, ...,  88,  87,  86],
       [ 30,  28,  29, ...,  92,  91,  91]], dtype=uint8),
        array([0., 1.])],
       [array([[207, 194, 195, ..., 254, 254, 255],
       [209, 199, 200, ..., 254, 255, 255],
       [199, 207, 201, ..., 254, 254, 255],
       ...,
       [255, 255, 255, ..., 255, 255, 255],
       [255, 255, 255, ..., 255, 255, 255],
       [255, 255, 255, ..., 255, 255, 255]], dtype=uint8),
        array([1., 0.])],
       [array([[ 65,  62,  72, ...,  54,  69,  53],
       [ 80,  44,  35, ...,  60,  37,  37],
       [ 75,  45,  93, ...,  43,  52,  54],
       ...,
       [ 39,  41, 106, ...,  21,  19,  18],
       [ 49,  34, 106, ...,  19,  18,  20],
       [ 42,  45,  88, ...,  19,  15,  17]], dtype=uint8),
        array([1., 0.])]], dtype=object)

The datatype is object and I am getting this error:
TypeError: can't convert np.ndarray of type numpy.object_. The only supported types are: float64, float32, float16, int64, int32, int16, int8, uint8, and bool.
How can I convert this object dtype to supported types?


title: Pytorch how use a linear activation function

body:
In Keras, I can create any network layer with a linear activation function as follows (for example, a fully-connected layer is taken):
model.add(keras.layers.Dense(outs, input_shape=(160,), activation='linear'))

But I can't find the linear activation function in the PyTorch documentation. ReLU is not suitable, because there are negative values in my sample. How do I create a layer with a linear activation function in PyTorch?


title: PyTorch - Do overwritten variables stay in the graph?

body:
I am wondering if PyTorch Tensors where the Python Variables are overwritten are still being kept in the computational graph of PyTorch.

So here is a small example, where I have an RNN Model where hidden states (and some other variables) are reset after every iteration, backward() is called later.

Example:

for i in range(5):
   output = rnn_model(inputs[i])
   loss += criterion(output, target[i])
   ## hidden states are overwritten with a zero vector
   rnn_model.reset_hidden_states()
loss.backward()

So my question is:

Is there a problem in overwriting the hidden states before calling backward()?
Or does the computational graph keep the necessary information of the hidden states of previous iterations in memory to compute the gradients?
Edit: It would be great to have a statement of an official source for this. e.g. stating that all variables relevant for the CG are kept - no matter if there are still other python references for this variables. I assume that there is a reference in the graph itself preventing the garbage collector from deleting it. But I'd like to know if this is truly the case.

Thanks in advance!


title: When using Pytorch-GPU in Anaconda, is it not necessary to install CUDA?

body:
I found that after installing Pytorch 0.4 GPU version in Anaconda, you don't need to install CUDA locally to call gpu acceleration. When running code, the GPU core can be used at more than 90%.

Edit:I used it in Windows 10. Don't know if it works in Linux.


title: pytorch , changing learning rate during training

body:
x=np.linspace(0,20,100)

g=1+0.2*np.exp(-0.1*(x-7)**2)
y=np.sin(g*x)

plt.plot(x,y)

plt.show()

x=torch.from_numpy(x)

y=torch.from_numpy(y)

x=x.reshape((100,1))
y=y.reshape((100,1))

MM=nn.Sequential()
MM.add_module('L1',nn.Linear(1,128))
MM.add_module('R1',nn.ReLU())
MM.add_module('L2',nn.Linear(128,128))
MM.add_module('R2',nn.ReLU())
MM.add_module('L3',nn.Linear(128,128))
MM.add_module('R3',nn.ReLU())
MM.add_module('L4',nn.Linear(128,128))
MM.add_module('R5',nn.ReLU())
MM.add_module('L5',nn.Linear(128,1))
MM.double()
L=nn.MSELoss()

lr=3e-05           ######
opt=torch.optim.Adam(MM.parameters(),lr)     #########
Epo=[]
COST=[]

for epoch in range(8000):

  opt.zero_grad()
  err=L(torch.sin(MM(x)),y)
  Epo.append(epoch)
  COST.append(err)
  err.backward()
  if epoch%100==0:
    print(err)
  opt.step()

Epo=np.array(Epo)/1000.
COST=np.array(COST)
pred=torch.sin(MM(x)).detach().numpy()
Trans=MM(x).detach().numpy()
x=x.reshape((100))
pred=pred.reshape((100))
Trans=Trans.reshape((100))

fig = plt.figure(figsize=(10,10))
#ax = fig.gca(projection='3d')
ax = fig.add_subplot(2,2,1)
surf = ax.plot(x,y,'r')

    #ax.plot_surface(x_dat,y_dat,z_pred)
    #ax.plot_wireframe(x_dat,y_dat,z_pred,linewidth=0.1)
fig.tight_layout()
    #plt.show()
ax = fig.add_subplot(2,2,2)
surf = ax.plot(x,pred,'g')
fig.tight_layout()

ax = fig.add_subplot(2,2,3)
surff=ax.plot(Epo,COST,'y+')
plt.ylim(0,1100)

ax = fig.add_subplot(2,2,4)
surf = ax.plot(x,Trans,'b')
fig.tight_layout()

plt.show()

This is the original code 1.
For changing learning rate during training, I tried to move the position of 'opt' as
Epo=[]
COST=[]

for epoch in range(8000):
  lr=3e-05           ######
  opt=torch.optim.Adam(MM.parameters(),lr)     #########
  opt.zero_grad()
  err=L(torch.sin(MM(x)),y)
  Epo.append(epoch)
  COST.append(err)
  err.backward()
  if epoch%100==0:
    print(err)
  opt.step()

This is code 2.
The code 2 also operate, but the result is quite different with code 1.
What is the difference and for changing learning rate during training(like lr=(1-epoch/10000 *0.99), what should I do?


title: Pytorch autograd fails with "RuntimeError: differentiated input is unreachable" after collecting inputs

body:
Pytorch version 0.3.1

EDIT: I am rewriting this question to be simpler, as I've narrowed down the bug.

I have some variables:

x = ag.Variable(torch.ones(1, 1), requires_grad = True)
y = ag.Variable(torch.ones(1, 1), requires_grad = True)
z = ag.Variable(torch.ones(1, 1), requires_grad = True)

I then create a variable representing their concatenation:

w = torch.cat([x, y, z])
f = x + y + z

Then I try to take derivatives:

ag.grad(f, x, retain_graph=True, create_graph=True)

This is fine and returns 1, as expected. Same for y and z.

However,

ag.grad(f, w, retain_graph=True, create_graph=True)

Returns an error: RuntimeError: differentiated input is unreachable

Of course that makes sense - w is not explicitly used in the declaration of f. However, I’d like a behavior where one line of code can generate something like [1; 1; 1] as output.

Let’s say I wanted to conveniently batch my variables together, and then take the gradient of the whole shebang at once, rather than processing variables independently (which can make bookkeeping a nightmare). Is there any way to get the outcome I desire?


title: Can I install pytorch on nvidia jetson tx1?

body: Because Tensorflow does not support ARM chips, I would like to install pytorch on Jetson TX1 to do some experiments. But I have not found any related resources about building pytorch into Jetson TX, is this possible?


title: PyTorch DataLoader adding extra dimension for TorchVision MNIST

body:
I am fairly new to PyTorch and have been experimenting with the DataLoader class.
When I attempt to load the MNIST dataset, the DataLoader appears to add an additional dimension after the batch dimension. I am not sure what is causing this to occur.

import torch
from torchvision.datasets import MNIST
from torchvision import transforms

if __name__ == '__main__':
    mnist_train = MNIST(root='./data', train=True, download=True, transform=transforms.Compose([transforms.ToTensor()]))
    first_x = mnist_train.data[0]
    print(first_x.shape)  # expect to see [28, 28], actual [28, 28]

    train_loader = torch.utils.data.DataLoader(mnist_train, batch_size=200)
    batch_x, batch_y = next(iter(train_loader))  # get first batch
    print(batch_x.shape)  # expect to see [200, 28, 28], actual [200, 1, 28, 28]
    # Where is the extra dimension of 1 from?

Can anyone shed some light on the issue?


title: PyTorch error - 'numpy.ndarray' object has no attribute 'relu'

body:
I am testing my CNN model, but keep on getting error "AttributeError: 'numpy.ndarray' object has no attribute 'relu'".

my dataset is extracted by below code:

import torch
from torch.utils.data import Dataset, DataLoader
from torch.autograd import Variable
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F
import numpy as np

class MyDataset(Dataset):
    def __init__(self, data, target, transform=None):
        self.data = torch.from_numpy(data).float()
        self.target = torch.from_numpy(target).long()
        self.transform = transform

    def __getitem__(self, index):
        x = self.data[index]
        y = self.target[index]

        if self.transform:
            x = self.transform(x)

        return x, y

    def __len__(self):
        return len(self.data)

numpy_data = np.random.randn(100,3,224,224) # 10 samples, image size = 224 x 224 x 3
numpy_target = np.random.randint(0,5,size=(100))

dataset = MyDataset(numpy_data, numpy_target)

my model is very simple as below:

class Network(nn.Module):
    def __init__(self):
        super().__init__()
        self.conv1 = nn.Conv2d(in_channels = 3, out_channels = 36, kernel_size = 100)

    def forward(self, t):

        t = self.conv1(t)
        print(t.shape)
        print(type(t))
        t = F.relu(t)
        print(t.shape)

        return t

I test model using below:

sample, target = next(iter(dataset))
network=Network()
pred = network(sample.unsqueeze(0))

I got below result and error:

torch.Size([1, 6, 125, 125])
<class 'torch.Tensor'>

---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
<ipython-input-40-37e58cfe971f> in <module>
----> 1 pred = network(sample.unsqueeze(0))

C:\Miniconda\envs\py37_default\lib\site-packages\torch\nn\modules\module.py in __call__(self, *input, **kwargs)
    548             result = self._slow_forward(*input, **kwargs)
    549         else:
--> 550             result = self.forward(*input, **kwargs)
    551         for hook in self._forward_hooks.values():
    552             hook_result = hook(self, input, result)

<ipython-input-30-0d0592ad705d> in forward(self, t)
     22         print(t.shape)
     23         print(type(t))
---> 24         t = F.relu(t)
     25         print(t.shape)
     26         #t = F.max_pool2d(t, kernel_size=2, stride=2)

AttributeError: 'numpy.ndarray' object has no attribute 'relu'

I could not figure out why, type(t) does output as , why error is showing it is numpy.ndarray?


title: how to add BatchNormalization with SWA:stochastic weights average?

body:
I am a beginner in Deepleaning and Pytorch.

I don't understand how to use BatchNormalization in using SWA.

pytorch.org says in https://pytorch.org/blog/stochastic-weight-averaging-in-pytorch/:

  Note that the SWA averages of the weights are never used to make
  predictions during training, and so the batch normalization layers do
  not have the activation statistics computed after you reset the
  weights of your model with opt.swap_swa_sgd()

This means it's suitable for adding BatchNormalization layer after using SWA?

# it means, in my idea

#for example

opt = torchcontrib.optim.SWA(base_opt)
for i in range(100):
    opt.zero_grad()
    loss_fn(model(input), target).backward()
    opt.step()
    if i > 10 and i % 5 == 0:
        opt.update_swa()
opt.swap_swa_sgd()

#save model once
torch.save(model,"swa_model.pt")

#model_load
saved_model=torch.load("swa_model.pt")

#it means adding BatchNormalization layer??
model2=saved_model
model2.add_module("Batch1",nn.BatchNorm1d(10))

# decay learning_rate more
learning_rate=0.005
optimizer = torch.optim.SGD(model2.parameters(), lr=learning_rate)

# train model again
for epoch in range(num_epochs):
    loss = train(train_loader)
    val_loss, val_acc = valid(test_loader)

I appreciate your replying.

following your advise,

I try to make example model adding  optimizer.bn_update()

# add  optimizer.bn_update() to model

criterion = nn.CrossEntropyLoss()
learning_rate=0.01

base_opt = torch.optim.SGD(model.parameters(), lr=0.1)
optimizer = SWA(base_opt, swa_start=10, swa_freq=5, swa_lr=0.05)

def train(train_loader):
    #mode:train
    model.train()
    running_loss = 0
    for batch_idx, (images, labels) in enumerate(train_loader):
        optimizer.zero_grad()
        outputs = model(images)
        #loss
        loss = criterion(outputs, labels)
        running_loss += loss.item()
        loss.backward()
        optimizer.step()

    optimizer.swap_swa_sgd()
    train_loss = running_loss / len(train_loader)

    return train_loss

def valid(test_loader):

    model.eval()
    running_loss = 0
    correct = 0
    total = 0
    #torch.no_grad
    with torch.no_grad():
        for batch_idx, (images, labels) in enumerate(test_loader):
            outputs = model(images)

            loss = criterion(outputs, labels)
            running_loss += loss.item()

            _, predicted = torch.max(outputs, 1)
            correct += (predicted == labels).sum().item()
            total += labels.size(0)

    val_loss = running_loss / len(test_loader)
    val_acc = float(correct) / total

    return val_loss, val_acc

num_epochs=30

loss_list = []
val_loss_list = []
val_acc_list = []
for epoch in range(num_epochs):
    loss = train(train_loader)
    val_loss, val_acc = valid(test_loader)
    optimizer.bn_update(train_loader, model)
    print('epoch %d, loss: %.4f val_loss: %.4f val_acc: %.4f'
          % (epoch, loss, val_loss, val_acc))

    # logging
    loss_list.append(loss)
    val_loss_list.append(val_loss)
    val_acc_list.append(val_acc)

# optimizer.bn_updata()
optimizer.bn_update(train_loader, model)

# go on evaluating model,,,


title: Keeping gradients while rearranging data in a tensor, with pytorch

body:
I have a scheme where I store a matrix with zeros on the diagonals as a vector. I want to later on optimize over that vector, so I require gradient tracking.
My challenge is to reshape between the two.

I want - for domain specific reasons - keep the order of data in the matrix so that transposed elements of the W matrix next to each other in the vector form.

The size of the W matrix is subject to change, so I start with enumering items in the top-left part of the matrix, and continue outwards.

I have come up with two ways to do this. See code snippet.

import torch
import torch.sparse

w = torch.tensor([10,11,12,13,14,15],requires_grad=True,dtype=torch.float)
i = torch.LongTensor([
    [0, 1,0],
    [1, 0,1],
    [0, 2,2],
    [2, 0,3],
    [1, 2,4],
    [2, 1,5],
])
v = torch.FloatTensor([1,      1,      1 ,1,1,1   ])
reshaper = torch.sparse.FloatTensor(i.t(), v, torch.Size([3,3,6])).to_dense()
W_mat_with_reshaper = reshaper @ w
W_mat_directly = torch.tensor([
  [0,    w[0],  w[2],],
  [w[1],    0,  w[4],],
  [w[3], w[5],     0,],
])
print(W_mat_with_reshaper)
print(W_mat_directly)

and this gives output

tensor([[ 0., 10., 12.],
        [11.,  0., 14.],
        [13., 15.,  0.]], grad_fn=<UnsafeViewBackward>)
tensor([[ 0., 10., 12.],
        [11.,  0., 14.],
        [13., 15.,  0.]])

As you can see, the direct way to reshape the vector into a matrix does not have a grad function, but the multiply-with-a-reshaper-tensor does. Creating the reshaper-tensor seems like it will be a hassle, but on the other hand, manually writing the matrix for is also infeasible.

Is there a way to do arbitrary reshapes in pytorch that keeps grack of gradients?


title: Solving Linear Equations on the GPU with NumPy and PyTorch

body:
I am trying to solve a lot of linear equations as fast as possible. To find out the fastest way I benchmarked NumPy and PyTorch, each on the CPU and on my GeForce 1080 GPU (using Numba for NumPy). The results really confused me.

This is the code I used with Python 3.8:

import timeit

import torch
import numpy
from numba import njit

def solve_numpy_cpu(dim: int = 5):
    a = numpy.random.rand(dim, dim)
    b = numpy.random.rand(dim)

    for _ in range(1000):
        numpy.linalg.solve(a, b)

def solve_numpy_njit_a(dim: int = 5):
    njit(solve_numpy_cpu, dim=dim)

@njit
def solve_numpy_njit_b(dim: int = 5):
    a = numpy.random.rand(dim, dim)
    b = numpy.random.rand(dim)

    for _ in range(1000):
        numpy.linalg.solve(a, b)

def solve_torch_cpu(dim: int = 5):
    a = torch.rand(dim, dim)
    b = torch.rand(dim, 1)

    for _ in range(1000):
        torch.solve(b, a)

def solve_torch_gpu(dim: int = 5):
    torch.set_default_tensor_type("torch.cuda.FloatTensor")
    solve_torch_cpu(dim=dim)

def main():
    for f in (solve_numpy_cpu, solve_torch_cpu, solve_torch_gpu, solve_numpy_njit_a, solve_numpy_njit_b):
        time = timeit.timeit(f, number=1)
        print(f"{f.__name__:<20s}: {time:f}")

if __name__ == "__main__":
    main()

And these are the results:

solve_numpy_cpu     : 0.007275
solve_torch_cpu     : 0.012244
solve_torch_gpu     : 5.239126
solve_numpy_njit_a  : 0.000158
solve_numpy_njit_b  : 1.273660

The slowest is CUDA accelerated PyTorch. I verified that PyTorch is using my GPU with

import torch
torch.cuda.is_available()
torch.cuda.get_device_name(0)

returning

True
'GeForce GTX 1080'

I can get behind that, on the CPU, PyTorch is slower than NumPy. What I cannot understand is why PyTorch on the GPU is so much slower. Not that important but actually even more confusing is that Numba's njit decorator makes performance orders of magnitude slower, until you don't use the @ decorator syntax anymore.

Is it my setup? Occasionally I get a weird message about the windows page / swap file not being big enough. In case I've taken a completely obscure path to solving linear equations on the GPU, I'd be happy to be directed into another direction.

Edit

So, I focussed on Numba and changed my benchmarking a bit. As suggested by @max9111 I rewrote the functions to receive input and produce output because, in the end, that's what anyone would want to use them for. Now, I also perform a first compile run for the Numba accelerated function so the subsequent timing is fairer. Finally, I checked the performance against matrix size and plotted the results.

TL/DR: Up to matrix sizes of 500x500, Numba acceleration doesn't really make a difference for numpy.linalg.solve.

Here is the code:

import time
from typing import Tuple

import numpy
from matplotlib import pyplot
from numba import jit

@jit(nopython=True)
def solve_numpy_njit(a: numpy.ndarray, b: numpy.ndarray) -> numpy.ndarray:
    parameters = numpy.linalg.solve(a, b)
    return parameters

def solve_numpy(a: numpy.ndarray, b: numpy.ndarray) -> numpy.ndarray:
    parameters = numpy.linalg.solve(a, b)
    return parameters

def get_data(dim: int) -> Tuple[numpy.ndarray, numpy.ndarray]:
    a = numpy.random.random((dim, dim))
    b = numpy.random.random(dim)
    return a, b

def main():
    a, b = get_data(10)
    # compile numba function
    p = solve_numpy_njit(a, b)

    matrix_size = [(x + 1) * 10 for x in range(50)]
    non_accelerated = []
    accelerated = []
    results = non_accelerated, accelerated

    for j, each_matrix_size in enumerate(matrix_size):
        for m, f in enumerate((solve_numpy, solve_numpy_njit)):
            average_time = -1.
            for k in range(5):
                time_start = time.time()
                for i in range(100):
                    a, b = get_data(each_matrix_size)
                    p = f(a, b)
                d_t = time.time() - time_start
                print(f"{each_matrix_size:d} {f.__name__:<30s}: {d_t:f}")
                average_time = (average_time * k + d_t) / (k + 1)
            results[m].append(average_time)

    pyplot.plot(matrix_size, non_accelerated, label="not numba")
    pyplot.plot(matrix_size, accelerated, label="numba")
    pyplot.legend()
    pyplot.show()

if __name__ == "__main__":
    main()

And these are the results (runtime against matrix edge length):

Edit 2

Seeing that Numba doesn't make much of a difference in my case, I came back to benchmarking PyTorch. And indeed, it appears to be roughly 4x faster than Numpy without even using a CUDA device.

Here is the code I used:

import time
from typing import Tuple

import numpy
import torch
from matplotlib import pyplot

def solve_numpy(a: numpy.ndarray, b: numpy.ndarray) -> numpy.ndarray:
    parameters = numpy.linalg.solve(a, b)
    return parameters

def get_data(dim: int) -> Tuple[numpy.ndarray, numpy.ndarray]:
    a = numpy.random.random((dim, dim))
    b = numpy.random.random(dim)
    return a, b

def get_data_torch(dim: int) -> Tuple[torch.tensor, torch.tensor]:
    a = torch.rand(dim, dim)
    b = torch.rand(dim, 1)
    return a, b

def solve_torch(a: torch.tensor, b: torch.tensor) -> torch.tensor:
    parameters, _ = torch.solve(b, a)
    return parameters

def experiment_numpy(matrix_size: int, repetitions: int = 100):
    for i in range(repetitions):
        a, b = get_data(matrix_size)
        p = solve_numpy(a, b)

def experiment_pytorch(matrix_size: int, repetitions: int = 100):
    for i in range(repetitions):
        a, b = get_data_torch(matrix_size)
        p = solve_torch(a, b)

def main():
    matrix_size = [x for x in range(5, 505, 5)]
    experiments = experiment_numpy, experiment_pytorch
    results = tuple([] for _ in experiments)

    for i, each_experiment in enumerate(experiments):
        for j, each_matrix_size in enumerate(matrix_size):
            time_start = time.time()
            each_experiment(each_matrix_size, repetitions=100)
            d_t = time.time() - time_start
            print(f"{each_matrix_size:d} {each_experiment.__name__:<30s}: {d_t:f}")
            results[i].append(d_t)

    for each_experiment, each_result in zip(experiments, results):
        pyplot.plot(matrix_size, each_result, label=each_experiment.__name__)

    pyplot.legend()
    pyplot.show()

if __name__ == "__main__":
    main()

And here's the result (runtime against matrix edge length):

So for now, I'll be sticking with torch.solve. However, the original question remains:

How can I exploit my GPU to solve linear equations even faster?


title: How to correctly use Cross Entropy Loss vs Softmax for classification?

body:
I want to train a multi class classifier using Pytorch.
Following the official Pytorch doc shows how to use a nn.CrossEntropyLoss() after a last layer of type nn.Linear(84, 10).
However, I remember this is what Softmax does.
This leaves me confused.

How to train a "standard" classification network in the best way?
If the network has a final linear layer, how to infer the probabilities per class?
If the network has a final softmax layer, how to train the network (which loss, and how)?

I found this thread on the Pytorch forum, which likely answers all that, but I couldn't compile it into working and readable Pytorch code.

My assumed answers:

Like the doc says.
Exponentiation of the outputs of the linear layer, which are really logits (log probalbilities).
I don't understand.


I apologize in advance in this was asked before. I genuinely did not understand the solution.
MAX_LEN = 160
BATCH_SIZE = 16
EPOCHS = 10

class GPReviewDataset(data.Dataset):
    def __init__(self, review, target, tokenizer, max_len):
        self.review = review
        self.target = target
        self.tokenizer = tokenizer
        self.max_len = max_len

    def __len__(self):
        return len(self.review)

    def __getitem__(self, item):
        review = str(self.review[item])

        encoding = tokenizer.encode_plus(text=review,
                                         max_length=self.max_len,
                                         add_special_tokens=True, padding='max_length',
                                         return_attention_mask=True,
                                         return_token_type_ids=False, return_tensors='pt')

        return {'review': review,
                'input_ids': encoding['input_ids'].flatten(),
                'attention_mask': encoding['attention_mask'].flatten(),
                'targets': torch.tensor(self.target[item], dtype=torch.long)}

free_df_train, free_df_test = train_test_split(free_df, test_size=0.2)
free_df_val, free_df_test = train_test_split(free_df_test, test_size=0.5)

def create_data_loader(df, tokenizer, max_len, batch_size):
    ds = GPReviewDataset(review=df.content.to_numpy(),
                         target=df['score'].to_numpy(),
                         tokenizer=tokenizer,
                         max_len=max_len)

    return data.DataLoader(ds, batch_size=batch_size,
                           num_workers=0)

train_data_loader = create_data_loader(free_df_train, tokenizer, MAX_LEN, BATCH_SIZE)
val_data_loader = create_data_loader(free_df_val, tokenizer, MAX_LEN, BATCH_SIZE)
test_data_loader = create_data_loader(free_df_test, tokenizer, MAX_LEN, BATCH_SIZE)

data = next(iter(train_data_loader))

I wrote a function later on to take in train_data_loader when training the data, but it was giving me the Runtime error. It seems like the proper solution is to use some sort of collate_fn; however I am confused on how exactly to apply that function.
My error below:
---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<timed exec> in <module>

<ipython-input-26-8ba1e19dd195> in train_epoch(model, data_loader, loss_fn, optimizer, device, scheduler, n_examples)
      4     correct_predictions = 0
      5
----> 6     for i in data_loader:
      7         input_ids = i['input_ids'].to(device)
      8         attention_mask = i['attention_mask'].to(device)

~\Anaconda3\lib\site-packages\torch\utils\data\dataloader.py in __next__(self)
    361
    362     def __next__(self):
--> 363         data = self._next_data()
    364         self._num_yielded += 1
    365         if self._dataset_kind == _DatasetKind.Iterable and \

~\Anaconda3\lib\site-packages\torch\utils\data\dataloader.py in _next_data(self)
    401     def _next_data(self):
    402         index = self._next_index()  # may raise StopIteration
--> 403         data = self._dataset_fetcher.fetch(index)  # may raise StopIteration
    404         if self._pin_memory:
    405             data = _utils.pin_memory.pin_memory(data)

~\Anaconda3\lib\site-packages\torch\utils\data\_utils\fetch.py in fetch(self, possibly_batched_index)
     45         else:
     46             data = self.dataset[possibly_batched_index]
---> 47         return self.collate_fn(data)

~\Anaconda3\lib\site-packages\torch\utils\data\_utils\collate.py in default_collate(batch)
     72         return batch
     73     elif isinstance(elem, container_abcs.Mapping):
---> 74         return {key: default_collate([d[key] for d in batch]) for key in elem}
     75     elif isinstance(elem, tuple) and hasattr(elem, '_fields'):  # namedtuple
     76         return elem_type(*(default_collate(samples) for samples in zip(*batch)))

~\Anaconda3\lib\site-packages\torch\utils\data\_utils\collate.py in <dictcomp>(.0)
     72         return batch
     73     elif isinstance(elem, container_abcs.Mapping):
---> 74         return {key: default_collate([d[key] for d in batch]) for key in elem}
     75     elif isinstance(elem, tuple) and hasattr(elem, '_fields'):  # namedtuple
     76         return elem_type(*(default_collate(samples) for samples in zip(*batch)))

~\Anaconda3\lib\site-packages\torch\utils\data\_utils\collate.py in default_collate(batch)
     53             storage = elem.storage()._new_shared(numel)
     54             out = elem.new(storage)
---> 55         return torch.stack(batch, 0, out=out)
     56     elif elem_type.__module__ == 'numpy' and elem_type.__name__ != 'str_' \
     57             and elem_type.__name__ != 'string_':

RuntimeError: stack expects each tensor to be equal size, but got [160] at entry 0 and [376] at entry 5


title: Pytorch: How to get all model's parameters that require grad?

body:
I have some model in pytorch, whose updatable weights I want to access and change manually.

How would that be done correctly?

Ideally, I would like a tensor of those weights.

It seems to me that

for parameter in model.parameters():
    do_something_to_parameter(parameter)

wouldn't be the right way to go, because

It doesn't utilize GPU, and is not able to
It doesn't even utilize low level implementation

What is the correct way of accessing a model's weights manually (not through loss.backward and optimizer.step)?


title: Get each sequence's last item from packed sequence

body:
I am trying to put a packed and padded sequence through a GRU, and retrieve the output of the last item of each sequence. Of course I don't mean the -1 item, but the actual last, not-padded item. We know the lengths of the sequences in advance, so it should be as easy as to extract for each sequence the length-1 item.

I tried the following

import torch
from torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence

# Data
input = torch.Tensor([[[0., 0., 0.],
                       [1., 0., 1.],
                       [1., 1., 0.],
                       [1., 0., 1.],
                       [1., 0., 1.],
                       [1., 1., 0.]],

                      [[1., 1., 0.],
                       [0., 1., 0.],
                       [0., 0., 0.],
                       [0., 1., 0.],
                       [0., 0., 0.],
                       [0., 0., 0.]],

                      [[0., 0., 0.],
                       [1., 0., 0.],
                       [1., 1., 1.],
                       [0., 0., 0.],
                       [0., 0., 0.],
                       [0., 0., 0.]],

                      [[1., 1., 0.],
                       [0., 0., 0.],
                       [0., 0., 0.],
                       [0., 0., 0.],
                       [0., 0., 0.],
                       [0., 0., 0.]]])

lengths = [6, 4, 3, 1]
p = pack_padded_sequence(input, lengths, batch_first=True)

# Forward
gru = torch.nn.GRU(3, 12, batch_first=True)
packed_output, gru_h = gru(p)

# Unpack
output, input_sizes = pad_packed_sequence(packed_output, batch_first=True)

last_seq_idxs = torch.LongTensor([x-1 for x in input_sizes])

last_seq_items = torch.index_select(output, 1, last_seq_idxs)

print(last_seq_items.size())
# torch.Size([4, 4, 12])

But the shape is not what I expect. I had expected to get 4x12, i.e. last item of each individual sequence x hidden.`

I could loop through the whole thing, and build a new tensor containing the items I need, but I was hoping for a built-in approach that took advantage of some smart math. I fear that manually looping and building, will result in very poor performance.


title: Getting Torch to recognize GPU

body:
How do you get Torch to recognize CUDA on your video card?

I have a Nvidia GeForce GT 1030 running under Ubuntu 18.04, and it claims to support CUDA, yet when I first tested Torch with it by running:

virtualenv -p python3.7 .env
. .env/bin/activate
pip install torch
python -c "import torch; print(torch.cuda.is_available())"

it returned False, along with the warning:

    The NVIDIA driver on your system is too old (found version 9010).
Please update your GPU driver by downloading and installing a new
version from the URL: http://www.nvidia.com/Download/index.aspx
Alternatively, go to: https://pytorch.org to install
a PyTorch version that has been compiled with your version
of the CUDA driver.

So I ran all system updates and used Ubuntu's proprietary driver installer to install the most recent Nvidia-435 driver for my card.

However, torch.cuda.is_available() still returns false, but now it doesn't give me any warning.

Have I mis-configured Torch or does my GPU just not support CUDA?


title: How to compute Hessian of the loss w.r.t. the parameters in PyTorch using autograd.grad

body:
I know there is quite a bit of content out there about "computing the Hessian" in pytorch, but as far as I've seen I haven't found anything working for me. So to try to be most precise, the Hessian that I want is the Jacobian of the gradient of the loss with respect to the network parameters. Also called the matrix of second-order derivatives with respect to the parameters.
I found some code that works in an intuitive way, although shouldn't be fast. It clearly just computes the gradient of the gradient of the loss w.r.t. the params w.r.t the params, and it does it one element (of the gradient) at a time. I think the logic is definitely right but I am getting an error, having to do with requires_grad. I'm a pytorch beginner so maybe its a simple thing, but the error seems to be saying that it can't take the gradient of the env_grads variable, which is the output from the previous grad function call.
Any help with this would be greatly appreciated. Here is the code followed by the error message. I also printed out the env_grads[0] variable so we can see that it is in fact a tensor, which is the correct output from the previous grad call.
env_loss = loss_fn(env_outputs, env_targets)
total_loss += env_loss
env_grads = torch.autograd.grad(env_loss, params,retain_graph=True)

print( env_grads[0] )
hess_params = torch.zeros_like(env_grads[0])
for i in range(env_grads[0].size(0)):
    for j in range(env_grads[0].size(1)):
        hess_params[i, j] = torch.autograd.grad(env_grads[0][i][j], params, retain_graph=True)[0][i, j] #  <--- error here
print( hess_params )
exit()

Output:
tensor([[-6.4064e-03, -3.1738e-03,  1.7128e-02,  8.0391e-03],
        [ 7.1698e-03, -2.4640e-03, -2.2769e-03, -1.0687e-03],
        [-3.0390e-04, -2.4273e-03, -4.0799e-02, -1.9149e-02],
        ...,
        [ 1.1258e-02, -2.5911e-05, -9.8133e-02, -4.6059e-02],
        [ 8.1502e-04, -2.5814e-03,  4.1772e-02,  1.9606e-02],
        [-1.0075e-02,  6.6072e-03,  8.3118e-04,  3.9011e-04]], device='cuda:0')

Error:
Traceback (most recent call last):
  File "/home/jefferythewind/anaconda3/envs/rapids3/lib/python3.7/runpy.py", line 193, in _run_module_as_main
    "__main__", mod_spec)
  File "/home/jefferythewind/anaconda3/envs/rapids3/lib/python3.7/runpy.py", line 85, in _run_code
    exec(code, run_globals)
  File "/home/jefferythewind/Projects/Irina/learning-explanations-hard-to-vary/and_mask/run_synthetic.py", line 258, in <module>
    main(args)
  File "/home/jefferythewind/Projects/Irina/learning-explanations-hard-to-vary/and_mask/run_synthetic.py", line 245, in main
    deep_mask=args.deep_mask
  File "/home/jefferythewind/Projects/Irina/learning-explanations-hard-to-vary/and_mask/run_synthetic.py", line 103, in train
    scale_grad_inverse_sparsity=scale_grad_inverse_sparsity
  File "/home/jefferythewind/Projects/Irina/learning-explanations-hard-to-vary/and_mask/and_mask_utils.py", line 154, in get_grads_deep
    hess_params[i, j] = torch.autograd.grad(env_grads[0][i][j], params, retain_graph=True)[0][i, j]
  File "/home/jefferythewind/anaconda3/envs/rapids3/lib/python3.7/site-packages/torch/autograd/__init__.py", line 157, in grad
    inputs, allow_unused)
RuntimeError: element 0 of tensors does not require grad and does not have a grad_fn


title: Pytorch LSTM - Training for Q&A classification

body:
I'm trying to train a model to classify if an answer answers the question given using this dataset.

I'm training in batches and using GloVe word embeddings. I train in batches of 1000 except the last one. The method I'm trying to use is to first giving the first sentence (question), and then the second sentence (answer) to LSTM and have it give me a number between 0 and 1 by using sigmoid function.

The problem is, loss always repeats itself after epoch 1. It never converges to the correct result which is if the answer belongs to the question, 1, 0 otherwise.

My code is as below:

class QandA(nn.Module):
    def __init__(self, input_size, hidden_size):
        super(QandA, self).__init__()

        self.hidden_size = hidden_size
        self.num_layers = 1
        self.bidirectional = True

        self.lstm = nn.LSTM(input_size, self.hidden_size, num_layers = self.num_layers, bidirectional = self.bidirectional)
        self.lstm.to(device)
        self.hidden2class = nn.Linear(self.hidden_size * 2, 1)
        self.hidden2class.to(device)

    def forward(self, glove_vec, glove_vec2):
        # glove_vec.shape = (sentence_len, batch_size, 300)
        output, hidden = self.lstm(glove_vec)
        output, _ = self.lstm(glove_vec2, hidden)
        # output.shape = (sentence_len, batch_size, hidden_size * 2)
        output = self.hidden2class(output[-1,:,:])
        # output.shape = (batch_size, 1)
        return F.sigmoid(output)
model = QandA(300, 60).to(device)
loss_function = nn.BCELoss()
optimizer = optim.Adam(model.parameters(), lr=0.1)

Is my approach so wrong that it can't work in practice? Or is there any other problem that I'm overseeing?

edit: Extra code regarding the training;

batch_size = 1000
# load_dataset loads the data from the file.
questions, answers, outputs = load_dataset()
N = len(outputs)
losses = []
for epoch in range(10):
    for batch in range(math.ceil(N / batch_size)):
        model.zero_grad()

        # get_data gets the data from the dataset (size batch_size, sequence batch)
        input1, input2, targets = get_data(batch, batch_size)

        class_pred = model(input1, input2)
        loss = loss_function(class_pred, targets)
        loss.backward()
        optimizer.step()


title: How does Module.parameters() find the parameters?

body:
I noticed that whenever you create a new net extending torch.nn.Module, you can immediately call net.parameters() to find the parameters relevant for backpropagation.

import torch

class MyNet(torch.nn.Module):
    def __init__(self):
        super(MyNet, self).__init__()
        self.fc = torch.nn.Linear(5, 5)

    def forward(self, x):
        return self.fc(x)

net = MyNet()
print(list(net.parameters()))

But then I wondered, how is this even possible? I just assigned this Linear layer object to a member variable but it is not recorded anywhere else (or is it?). Somehow MyNet must be able to keep track of the parameters used but how?


title: How to check the root cause of CUDA out of memory issue in the middle of training?

body: I'm running roberta on huggingface language_modeling.py. After doing 400 steps I suddenly get a CUDA out of memory issue. Don't know how to deal with it. Can you please help? Thanks


title: Pytorch runtime error: Cuda Out of memory. Works fine with jupyter notebook but doesn't as a script

body:
I have a special kind of problem. I am able to run the code in jupyter notebook perfectly fine with no OOM error. However when i run the same code as a script in linux it gives me the OOM error. Has anyone have the same issue. I tried gc.collect() and torch.cuda.empty_cache() inside the code and nothing helps.

It always gives me this error.
RuntimeError: CUDA out of memory. Tried to allocate 1.30 GiB (GPU 0; 7.79 GiB total capacity; 4.80 GiB already allocated; 922.69 MiB free; 6.12 GiB reserved in total by PyTorch

The code:

def lemmatize(phrase):
    """Return lematized words"""
    spa = spacy.load("en_core_web_sm")
    return " ".join([word.lemma_ for word in spa(phrase)])

def reading_csv(path_to_csv):
    """Return text column in csv"""
    data = pd.read_csv(path_to_csv)
    ctx_paragraph = []
    for txt in data['text']:
        if not pd.isna(txt):
            ctx_paragraph.append(txt)
    return ctx_paragraph

def processing_question(ques, paragraphs, domain_lemma_cache, domain_pickle):
    """Return answer"""
    #Lemmatizing whole csv text column
    lemma_cache = domain_lemma_cache
    if not os.path.isfile(lemma_cache):
        lemmas = [lemmatize(par) for par in tqdm(paragraphs)]
        df = pd.DataFrame(data={'context': paragraphs, 'lemmas': lemmas})
        df.to_feather(lemma_cache)
    df = pd.read_feather(lemma_cache)
    paragraphs = df.context
    lemmas = df.lemmas
    #Vectorizor cache
    if not os.path.isfile(VEC_PICKLE_LOC):
        vectorizer = TfidfVectorizer(
            stop_words='english', min_df=5, max_df=.5, ngram_range=(1, 3))
        vectorizer.fit_transform(lemmas)
        pickle.dump(vectorizer, open(VEC_PICKLE_LOC, "wb"))
    #Vectorized lemmas cache cache
    if not os.path.isfile(domain_pickle):
        tfidf = vectorizer.fit_transform(lemmas)
        pickle.dump(tfidf, open(domain_pickle, "wb"))
    vectorizer = pickle.load(open(VEC_PICKLE_LOC, "rb"))
    tfidf = pickle.load(open(domain_pickle, "rb"))
    question = ques
    query = vectorizer.transform([lemmatize(question)])
    (query > 0).sum(), vectorizer.inverse_transform(query)
    scores = (tfidf * query.T).toarray()
    results = (np.flip(np.argsort(scores, axis=0)))
    qapipe = pipeline('question-answering',
                      model='distilbert-base-uncased-distilled-squad',
                      tokenizer='bert-base-uncased',
                      device=0)
    candidate_idxs = [(i, scores[i]) for i in results[0:10, 0]]
    contexts = [(paragraphs[i], s) for (i, s) in candidate_idxs if s > 0.01]
    question_df = pd.DataFrame.from_records([{
        'question': question,
        'context':  ctx
    } for (ctx, s) in contexts])
    preds = qapipe(question_df.to_dict(orient="records"))
    answer_df = pd.DataFrame.from_records(preds)
    answer_df["context"] = question_df["context"]
    answer_df = answer_df.sort_values(by="score", ascending=False)
    return answer_df


title: pytorch conv2d with weights

body:
I have these 2 variables:
result- tensor 1 X 251 X 20
kernel - tensor 1 X 10 X 10

when I run the command:
from torch.nn import functional as F
result = F.conv2d(result, kernel)

I get the error:
RuntimeError: expected stride to be a single integer value or a list of 1 values to match the convolution dimensions, but got stride=[1, 1]

I am not giving any stride, what am I doing wrong?


title: Which function in Tensorflow is similar to expand_as in Pytorch

body:
I am trying to adapt some code from Pytorch to Tensorflow, though I'm not so familiar with the latter.

I use expand_as in my old code. Is there any similar function which can be used in Tensorflow?


title: How can I add bias using pytorch to a neural network?

body:
If I have this neural network (NN):

 class NeuralNet(nn.Module):
    def __init__(self, input_size, hidden_size, num_classes, p=dropout):
        super(NeuralNet, self).__init__()
        self.fc1 = nn.Linear(input_size, hidden_size)
        self.fc2 = nn.Linear(hidden_size, hidden_size)
        self.fc3 = nn.Linear(hidden_size, num_classes)
        self.dropout = nn.Dropout(p=p)

    def forward(self, x):
        out = F.relu(self.fc1(x))
        out = F.relu(  self.dropout(self.fc2(out)))
        out = self.fc3(out)
        return out

How do you add bias to it? For instance, on the hidden layer. I can't find anything online.


title: List index out of range error in object detection using YoloV3 in Pytorch

body:
I am trying to implement Object Detection using YOLOV3 AND Pytorch. I am training the model on my custom Dataset, which contains 200 images of one type only and has only one object (which is labelled, for ref. check the image below). I have annotated the Images and saved them into .txt format.

Dataset

--------images
         img1.jpg
         img2.jpg

--------labels
         img1.txt
         img2.txt

test.txt
valid.txt

My model training part is working fine. I save weights, after every epoch(Checkpoints). I run the program(training part) for 20 Epoch values and take the last saved checkpoint weight for the object detection part.
There is only 1 class (IRA), indexed (0)
My Yolo.cfg file is also correct. [class = 1, filters = 18] (updated in three parts of the file)

But the problem is:-
On running the object detection part of the program, I am getting this error

This problem is specific to this only. I tested on other images dataset, it worked fine but I don't know why it is not giving any label in this case?

Here is my training model part

 from __future__ import division

from models import *
from utils import *
from datasets import *
from parse_config import *

import os
import sys
import time
import datetime
import argparse

import torch
from torch.utils.data import DataLoader
from torchvision import datasets
from torchvision import transforms
from torch.autograd import Variable
import torch.optim as optim

parser = argparse.ArgumentParser()
parser.add_argument("-f")
parser.add_argument("--epochs", type=int, default=20, help="number of epochs")
parser.add_argument("--image_folder", type=str, default="/content/drive/My Drive/FINAL_PYTORCH_DATA/images", help="path to dataset")
parser.add_argument("--batch_size", type=int, default=16, help="size of each image batch")
parser.add_argument("--model_config_path", type=str, default="/content/drive/My Drive/config/yolov3.cfg", help="path to model config file")
parser.add_argument("--data_config_path", type=str, default="/content/drive/My Drive/config/coco.data", help="path to data config file")
parser.add_argument("--weights_path", type=str, default="/content/drive/My Drive/config/yolov3.weights", help="path to weights file")
parser.add_argument("--class_path", type=str, default="/content/drive/My Drive/config/coco.names", help="path to class label file")
parser.add_argument("--conf_thres", type=float, default=0.8, help="object confidence threshold")
parser.add_argument("--nms_thres", type=float, default=0.4, help="iou thresshold for non-maximum suppression")
parser.add_argument("--n_cpu", type=int, default=0, help="number of cpu threads to use during batch generation")
parser.add_argument("--img_size", type=int, default=416, help="size of each image dimension")
parser.add_argument("--checkpoint_interval", type=int, default=1, help="interval between saving model weights")
parser.add_argument("--checkpoint_dir", type=str, default="/content/checkpoints", help="directory where model checkpoints are saved")
parser.add_argument("--use_cuda", type=bool, default=True, help="whether to use cuda if available")
opt = parser.parse_args()
print(opt)

cuda = torch.cuda.is_available() and opt.use_cuda

os.makedirs("checkpoints", exist_ok=True)

classes = load_classes(opt.class_path)

# Get data configuration
data_config = parse_data_config(opt.data_config_path)
train_path = data_config["train"]

# Get hyper parameters
hyperparams = parse_model_config(opt.model_config_path)[0]
learning_rate = float(hyperparams["learning_rate"])
momentum = float(hyperparams["momentum"])
decay = float(hyperparams["decay"])
burn_in = int(hyperparams["burn_in"])

# Initiate model
model = Darknet(opt.model_config_path)
model.load_weights(opt.weights_path)
#model.apply(weights_init_normal)

if cuda:
    model = model.cuda()

model.train()

# Get dataloader
dataloader = torch.utils.data.DataLoader(
    ListDataset(train_path), batch_size=opt.batch_size, shuffle=False, num_workers=opt.n_cpu
)

Tensor = torch.cuda.FloatTensor if cuda else torch.FloatTensor

optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, model.parameters()))

for epoch in range(opt.epochs):
    for batch_i, (_, imgs, targets) in enumerate(dataloader):
        imgs = Variable(imgs.type(Tensor))
        targets = Variable(targets.type(Tensor), requires_grad=False)

        optimizer.zero_grad()

        loss = model(imgs, targets)

        loss.backward()
        optimizer.step()

        print(
            "[Epoch %d/%d, Batch %d/%d] [Losses: x %f, y %f, w %f, h %f, conf %f, cls %f, total %f, recall: %.5f, precision: %.5f]"
            % (
                epoch,
                opt.epochs,
                batch_i,
                len(dataloader),
                model.losses["x"],
                model.losses["y"],
                model.losses["w"],
                model.losses["h"],
                model.losses["conf"],
                model.losses["cls"],
                loss.item(),
                model.losses["recall"],
                model.losses["precision"],
            )
        )

        model.seen += imgs.size(0)

    if epoch % opt.checkpoint_interval == 0:
        model.save_weights("%s%d.weights" % (opt.checkpoint_dir, epoch))

Here is my model loading part:-

config_path='/content/drive/My Drive/config/yolov3.cfg'
weights_path='/content/checkpoints19.weights'
class_path='/content/drive/My Drive/config/coco.names'
img_size=416
conf_thres=0.8
nms_thres=0.4

# Load model and weights
model = Darknet(config_path, img_size=img_size)
model.load_weights(weights_path)
model.cuda()
model.eval()
classes = utils.load_classes(class_path)
Tensor = torch.cuda.FloatTensor

Here is my object detection part:-

def detect_image(img):
    # scale and pad image
    ratio = min(img_size/img.size[0], img_size/img.size[1])
    imw = round(img.size[0] * ratio)
    imh = round(img.size[1] * ratio)
    img_transforms = transforms.Compose([ transforms.Resize((imh, imw)),
         transforms.Pad((max(int((imh-imw)/2),0), max(int((imw-imh)/2),0), max(int((imh-imw)/2),0), max(int((imw-imh)/2),0)),
                        (128,128,128)),
         transforms.ToTensor(),
         ])
    # convert image to Tensor
    image_tensor = img_transforms(img).float()
    image_tensor = image_tensor.unsqueeze_(0)
    input_img = Variable(image_tensor.type(Tensor))
    # run inference on the model and get detections
    with torch.no_grad():
        detections = model(input_img)
        detections = utils.non_max_suppression(detections, 80, conf_thres, nms_thres)
    return detections[0]

# load image and get detections
img_path = "/content/image1 - Copy (2).jpg"
prev_time = time.time()
img = Image.open(img_path)
detections = detect_image(img)
inference_time = datetime.timedelta(seconds=time.time() - prev_time)
print ('Inference Time: %s' % (inference_time))

# Get bounding-box colors
cmap = plt.get_cmap('tab20b')
colors = [cmap(i) for i in np.linspace(0, 1, 20)]

img = np.array(img)
plt.figure()
fig, ax = plt.subplots(1, figsize=(12,9))
ax.imshow(img)

pad_x = max(img.shape[0] - img.shape[1], 0) * (img_size / max(img.shape))
pad_y = max(img.shape[1] - img.shape[0], 0) * (img_size / max(img.shape))
unpad_h = img_size - pad_y
unpad_w = img_size - pad_x
k=[]
if detections is not None:
    unique_labels = detections[:, -1].cpu().unique()
    n_cls_preds = len(unique_labels)
    bbox_colors = random.sample(colors, n_cls_preds)
    # browse detections and draw bounding boxes
    for x1, y1, x2, y2, conf, cls_conf, cls_pred in detections:
        box_h = ((y2 - y1) / unpad_h) * img.shape[0]
        box_w = ((x2 - x1) / unpad_w) * img.shape[1]
        y1 = ((y1 - pad_y // 2) / unpad_h) * img.shape[0]
        x1 = ((x1 - pad_x // 2) / unpad_w) * img.shape[1]
        color = bbox_colors[int(np.where(unique_labels == int(cls_pred))[0])]
        bbox = patches.Rectangle((x1, y1), box_w, box_h, linewidth=2, edgecolor=color, facecolor='none')
        ax.add_patch(bbox)
        k=classes[int(cls_pred)]
        plt.text(x1, y1, s=classes[int(cls_pred)], color='white', verticalalignment='top',
                bbox={'color': color, 'pad': 0})
plt.axis('off')
# save image
plt.savefig(img_path.replace(".jpg", "-det.jpg"), bbox_inches='tight', pad_inches=0.0)
plt.show()
print(k)

If anyone can help me understand, what causes this error and how to solve this?


title: PyTorch code stops with message "Killed". What killed it?

body:
I train a network on GPU with Pytorch. However, after at most 3 epochs, code stops with a message :
Killed
No other error message is given.
I monitored the memory and gpu usage, there were still space during the run. I reviewed the /var/sys/dmesg to find a detailed message regarding to this, however no message with "kill" was entered. What might be the problem?
Cuda version: 9.0
Pytorch version: 1.1.0


title: pyspark and pytorch: unable to retrieve gradients on the driver

body:
I'm new to pytorch and I'm trying to explore the feasibility of its usage with spark (for now I'm working in spark standalone).

As for now I'm struggling on a very specific topic.
Let's start with a very simple model:

# linmodel.py
import torch
import torch.nn as nn
import numpy as np

def standardize(x):
    return (x - np.mean(x)) / np.std(x)

def add_noise(y):
    rnd = np.random.randn(y.shape[0])
    return y + rnd

def cost(target, predicted):
    cost = torch.sum((torch.t(target) - predicted) ** 2)
    return cost

class LinModel(nn.Module):
    def __init__(self, in_size, out_size):
        super(LinModel, self).__init__()  # always call parent's init
        self.linear = nn.Linear(in_size, out_size, bias=False)  # layer parameters

    def forward(self, x):
        return self.linear(x)

Which instantiates a basic linear model, along with some utility functions.
The goal is to approximate a target matrix, and to keep track of how the
gradients behave.

I'm trying to achieve the following:

create my target matrix
split the inputs on the workers
instantiate models and optimizer on the workers
compute the approximation on subsets of input
retrieve the gradients for further analysis

And everything works fine until point 5.
Here's the code:

#test.py
import torch
import torch.nn as nn
import numpy as np
import torch.optim
from torch.autograd import Variable
from pyspark import SparkContext
import linmodel

def prepare_input(nsamples=400):
   Xold = np.linspace(0, 1000, nsamples).reshape([nsamples, 1])
   X = linmodel.standardize(Xold)
   W = np.random.randint(1, 10, size=(5, 1))
   Y = W.dot(X.T)  # target
   for i in range(Y.shape[1]):
      Y[:, i] = linmodel.add_noise(Y[:, i])
   x = Variable(torch.from_numpy(X), requires_grad=False).type(torch.FloatTensor)
   y = Variable(torch.from_numpy(Y), requires_grad=False).type(torch.FloatTensor)
   print("created torch variables {} {}".format(x.size(), y.size()))
   return x, y, W

def initialize(tup):
    x, y = tup[0]   # data
    m, o = tup[1]   # model and optimizer
    model, optimizer = torch_step(x, y, m, o)
    # here we have the gradients
    print('gradient: {}'.format([param.grad.data for param in model.parameters()]))
    return (x, y), (model, optimizer)

def create_model():
    model = linmodel.LinModel(1, 5)
    optimizer = torch.optim.Adam(model.parameters(), lr=1e-2)
    return model, optimizer

def torch_step(x, y, model, optimizer):
    prediction = model(x)
    loss = linmodel.cost(y, prediction)
    optimizer.zero_grad()
    loss.backward()
    optimizer.step()
    return model, optimizer

def main(sc, num_partitions=4):
    x, y, W = prepare_input()
    parts_x = list(torch.split(x, int(x.size()[0] / num_partitions)))
    parts_y = list(torch.split(y, int(x.size()[0] / num_partitions), 1))
    rdd_models = sc.parallelize([create_model() for _ in range(num_partitions)]).repartition(num_partitions)

    rdd_x = sc.parallelize(parts_x).repartition(num_partitions)
    rdd_y = sc.parallelize(parts_y).repartition(num_partitions)
    parts = rdd_x.zip(rdd_y)  # [((100x1), (5x100)), ...]
    full = parts.zip(rdd_models).map(initialize).cache()

    models_out = full.map(lambda x: x[1][0]).collect()
    test_model = models_out[0]
    print(type(test_model))
    print('gradient: {}'.format([param.grad.data for param in test_model.parameters()]))

if __name__ == '__main__':
    sc = SparkContext(appName='test')
    main(sc)

As you can see in the comments , when the function initialize is mapped on the full rdd, if you inspect the logs of the executors you'll find the gradients to be computed.
When I collect the result and try to access the very same attribute on the driver I receive a AttributeError: 'NoneType' object has no attribute 'data'
meaning that all the model.grad attribute are set to None.

I'm sure I'm missing something big here, but I cannot see it.

Any hint is appreciated.

Thanks a lot.


title: The strange loss fluctuation when loading previous trained model

body:
I am using PyTorch for deep learning now.

I trained a model before and had the parameters saved. Loss values before the end of the training were about 0.003~0.006.

However, when I load the same model with the same training data, loss values at first fluctuate to around 0.5.

The loss values then decrease very quickly to around 0.01 in ~10 iterations and now decreasing slowly.

Does anyone know why this situation keep happens? Since I am loading the same model/ training data. I was expecting the loss values would start at a similar level as the end of last training.


title: How can Pytorch share memory among several processes?

body: According to this, 'processes have separate memory'. But Pytorch can somehow share memory among several processes, according to this link: 'Once the tensor/storage is moved to shared_memory (see share_memory_()), it will be possible to send it to other processes without making any copies.' Why is it possible to share memory among separate memory? Doesn't it sound like a paradox?


title: I define a loss function but backward present error to me could someone tell me how to fix it

body:
class loss(Function):
    @staticmethod
    def forward(ctx,x,INPUT):

        batch_size = x.shape[0]
        X = x.detach().numpy()
        input = INPUT.detach().numpy()
        Loss = 0
        for i in range(batch_size):
            t_R_r = input[i,0:4]
            R_r = t_R_r[np.newaxis,:]
            t_R_i = input[i,4:8]
            R_i = t_R_i[np.newaxis,:]
            t_H_r = input[i,8:12]
            H_r = t_H_r[np.newaxis,:]
            t_H_i = input[i,12:16]
            H_i = t_H_i[np.newaxis,:]

            t_T_r = input[i, 16:32]
            T_r = t_T_r.reshape(4,4)
            t_T_i = input[i, 32:48]
            T_i = t_T_i.reshape(4,4)

            R = np.concatenate((R_r, R_i), axis=1)
            H = np.concatenate((H_r, H_i), axis=1)

            temp_t1 = np.concatenate((T_r,T_i),axis=1)
            temp_t2 = np.concatenate((-T_i,T_r),axis=1)
            T = np.concatenate((temp_t1,temp_t2),axis=0)
            phi_r = np.zeros((4,4))
            row, col = np.diag_indices(4)
            phi_r[row,col] = X[i,0:4]
            phi_i = np.zeros((4, 4))
            row, col = np.diag_indices(4)
            phi_i[row, col] = 1 - np.power(X[i, 0:4],2)

            temp_phi1 = np.concatenate((phi_r,phi_i),axis=1)
            temp_phi2 = np.concatenate((-phi_i, phi_r), axis=1)
            phi = np.concatenate((temp_phi1,temp_phi2),axis=0)

            temp1 = np.matmul(R,phi)

            temp2 = np.matmul(temp1,T)  # error
            H_hat = H + temp2

            t_Q_r = np.zeros((4,4))
            t_Q_r[np.triu_indices(4,1)] = X[i,4:10]
            Q_r = t_Q_r + t_Q_r.T
            row,col = np.diag_indices(4)
            Q_r[row,col] = X[i,10:14]
            Q_i = np.zeros((4,4))
            Q_i[np.triu_indices(4,1)] = X[i,14:20]
            Q_i = Q_i - Q_i.T

            temp_Q1 = np.concatenate((Q_r,Q_i),axis=1)
            temp_Q2 = np.concatenate((-Q_i,Q_r),axis=1)
            Q = np.concatenate((temp_Q1,temp_Q2),axis=0)

            t_H_hat_r = H_hat[0,0:4]
            H_hat_r = t_H_hat_r[np.newaxis,:]
            t_H_hat_i= H_hat[0,4:8]
            H_hat_i = t_H_hat_i[np.newaxis,:]

            temp_H1 = np.concatenate((-H_hat_i.T,H_hat_r.T),axis=0)
            H_hat_H = np.concatenate((H_hat.T,temp_H1),axis=1)
            temp_result1 = np.matmul(H_hat,Q)
            temp_result2 = np.matmul(temp_result1,H_hat_H)

            Loss += np.log10(1+temp_result2[0][0])
        Loss = t.from_numpy(np.array(Loss / batch_size))
        return Loss
    @staticmethod
    def backward(ctx,grad_output):
        print('gradient')
        return grad_output
def criterion(output,input):
    return loss.apply(output,input)

This is my loss function. But it present the error:
Traceback (most recent call last):

  File "/Users/mrfang/channel_capacity/training.py", line 24, in

      loss.backward()   File "/Users/mrfang/anaconda3/lib/python3.6/site-packages/torch/tensor.py",
  line 150, in backward
      torch.autograd.backward(self, gradient, retain_graph, create_graph)   File
  "/Users/mrfang/anaconda3/lib/python3.6/site-packages/torch/autograd/init.py",
  line 99, in backward
      allow_unreachable=True)  # allow_unreachable flag RuntimeError: function lossBackward returned an incorrect number of gradients
  (expected 2, got 1)

How could I fix it. Thanks very much


title: "TorchScript requires source access in order to carry out compilation" error after converting script to exe

body:
I am trying to convert the script to exe using pyinstaller
the script is using the inception resnet v1 model from facenet_pytorch by tim esler found here

after running the converted script exe the following error appears:

  Traceback (most recent call last):

  File "site-packages\torch_utils_internal.py", line 46, in
  get_source_lines_and_file

  File "inspect.py", line 955, in getsourcelines

  File "inspect.py", line 786, in findsource

  OSError: could not get source code

  During handling of the above exception, another exception occurred:

  Traceback (most recent call last):

  File "Fcenet-Pytorch\Test Rec2.py", line 1, in

  File "", line 983, in _find_and_load

  File "", line 967, in
  _find_and_load_unlocked

  File "", line 677, in _load_unlocked

  File
  "c:\users\jorda\appdata\local\programs\python\python37\lib\site-packages\PyInstaller\loader\pyimod03_importers.py",
  line 623, in exec_module
      exec(bytecode, module.dict)

  File "site-packages\facenet_pytorch__init__.py", line 1, in

# -*- coding: utf-8 -*-

  File "", line 983, in _find_and_load

  File "", line 967, in
  _find_and_load_unlocked

  File "", line 677, in _load_unlocked

  File
  "c:\users\jorda\appdata\local\programs\python\python37\lib\site-packages\PyInstaller\loader\pyimod03_importers.py",
  line 623, in exec_module
      exec(bytecode, module.dict)

  File "site-packages\facenet_pytorch\models\inception_resnet_v1.py",
  line 1, in

  File "", line 983, in _find_and_load

  File "", line 967, in
  _find_and_load_unlocked

  File "", line 677, in _load_unlocked

  File
  "c:\users\jorda\appdata\local\programs\python\python37\lib\site-packages\PyInstaller\loader\pyimod03_importers.py",
  line 623, in exec_module
      exec(bytecode, module.dict)

  File "site-packages\torch__init__.py", line 367, in

  File "", line 983, in _find_and_load

  File "", line 967, in
  _find_and_load_unlocked

  File "", line 677, in _load_unlocked

  File
  "c:\users\jorda\appdata\local\programs\python\python37\lib\site-packages\PyInstaller\loader\pyimod03_importers.py",
  line 623, in exec_module
      exec(bytecode, module.dict)

  File "site-packages\torch\distributions__init__.py", line 112, in

  File "", line 983, in _find_and_load

  File "", line 967, in
  _find_and_load_unlocked

  File "", line 677, in _load_unlocked

  File
  "c:\users\jorda\appdata\local\programs\python\python37\lib\site-
  packages\PyInstaller\loader\pyimod03_importers.py", line 623, in
  exec_module
      exec(bytecode, module.dict)

  File "site-packages\torch\distributions\von_mises.py", line 54, in

  File "site-packages\torch\jit__init__.py", line 1287, in script

  File "site-packages\torch\jit\frontend.py", line 164, in get_jit_def

  File "site-packages\torch_utils_internal.py", line 53, in
  get_source_lines_and_file

  OSError: Can't get source for . TorchScript requires  source access in order to
  carry out compilation, make sure original .py files are available.
  Original  error: could not get source code

  [4348] Failed to execute script Test Rec2

some solution I've found online is to downgrade torchvision to 0.2.2.post3
but that would prevent the original script from running

I've tried all kinds of hiddenimports the same error still occurs

any idea how to fix it?

System:
Windows 10

Python 3.7

torch 1.5.0

torchvision 0.6.0

pyinstaller 3.6

facenet-pytorch 2.2.9


title: How to sort a dataset in pytorch

body:
I would like to sort my dataset by the numerical values in the labels.

Is there a function from pytorch to handle this efficiently?

my dataset type() is in this from:

 <class 'torchvision.datasets.mnist.MNIST'>


title: How to define a loss function in pytorch with dependency to partial derivatives of the model w.r.t input?

body:
After reading about how to solve an ODE with neural networks following the paper Neural Ordinary Differential Equations and the blog that uses the library JAX I tried to do the same thing with "plain" Pytorch but found a point rather "obscure": How to properly use the partial derivative of a function (in this case the model) w.r.t one of the input parameters.

To resume the problem at hand as shown in 2 it is intended to solve the ODE y' = -2*x*y with the condition y(x=0) = 1 in the domain -2 <= x <= 2. Instead of using finite differences the solution is replaced by a NN as y(x) = NN(x) with a single layer with 10 nodes.

I managed to (more or less) replicate the blog with the following code

import torch
import torch.nn as nn
from torch import optim
import matplotlib.pyplot as plt
import numpy as np

# Define the NN model to solve the problem
class Model(nn.Module):
    def __init__(self):
        super(Model, self).__init__()
        self.lin1 = nn.Linear(1,10)
        self.lin2 = nn.Linear(10,1)

    def forward(self, x):
        x = torch.sigmoid(self.lin1(x))
        x = torch.sigmoid(self.lin2(x))
        return x

model = Model()

# Define loss_function from the Ordinary differential equation to solve
def ODE(x,y):
    dydx, = torch.autograd.grad(y, x,
    grad_outputs=y.data.new(y.shape).fill_(1),
    create_graph=True, retain_graph=True)

    eq = dydx + 2.* x * y # y' = - 2x*y
    ic = model(torch.tensor([0.])) - 1.    # y(x=0) = 1
    return torch.mean(eq**2) + ic**2

loss_func = ODE

# Define the optimization
# opt = optim.SGD(model.parameters(), lr=0.1, momentum=0.99,nesterov=True) # Equivalent to blog
opt = optim.Adam(model.parameters(),lr=0.1,amsgrad=True) # Got faster convergence with Adam using amsgrad

# Define reference grid
x_data = torch.linspace(-2.0,2.0,401,requires_grad=True)
x_data = x_data.view(401,1) # reshaping the tensor

# Iterative learning
epochs = 1000
for epoch in range(epochs):
    opt.zero_grad()
    y_trial = model(x_data)
    loss = loss_func(x_data, y_trial)

    loss.backward()
    opt.step()

    if epoch % 100 == 0:
        print('epoch {}, loss {}'.format(epoch, loss.item()))

# Plot Results
plt.plot(x_data.data.numpy(), np.exp(-x_data.data.numpy()**2), label='exact')
plt.plot(x_data.data.numpy(), y_data.data.numpy(), label='approx')
plt.legend()
plt.show()

From here I manage to get the results as shown in the fig.
enter image description here

The problems is that at the definition of the ODE functional, instead of passing (x,y) I would rather prefer to pass something like (x,fun) (where fun is my model) such that the partial derivative and specific evaluations of the model can be done with a call . So, something like

def ODE(x,fun):
    dydx, = "grad of fun w.r.t x as a function"

    eq = dydx(x) + 2.* x * fun(x)        # y' = - 2x*y
    ic = fun( torch.tensor([0.]) ) - 1.  # y(x=0) = 1
    return torch.mean(eq**2) + ic**2

Any ideas? Thanks in advance

EDIT:

After some trials I found a way to pass the model as an input but found another strange behavior... The new problem is to solve the ODE y'' = -2 with the BC y(x=-2) = -1 and y(x=2) = 1, for which the analytical solution is y(x) = -x^2+x/2+4

Let's modify a bit the previous code as:

import torch
import torch.nn as nn
from torch import optim
import matplotlib.pyplot as plt
import numpy as np

# Define the NN model to solve the equation
class Model(nn.Module):
    def __init__(self):
        super(Model, self).__init__()
        self.lin1 = nn.Linear(1,10)
        self.lin2 = nn.Linear(10,1)

    def forward(self, x):
        y = torch.sigmoid(self.lin1(x))
        z = torch.sigmoid(self.lin2(y))
        return z

model = Model()

# Define loss_function from the Ordinary differential equation to solve
def ODE(x,fun):
    y = fun(x)

    dydx = torch.autograd.grad(y, x,
    grad_outputs=y.data.new(y.shape).fill_(1),
    create_graph=True, retain_graph=True)[0]

    d2ydx2 = torch.autograd.grad(dydx, x,
    grad_outputs=dydx.data.new(dydx.shape).fill_(1),
    create_graph=True, retain_graph=True)[0]

    eq  = d2ydx2 + torch.tensor([ 2.])             # y'' = - 2
    bc1 =  fun(torch.tensor([-2.])) - torch.tensor([-1.]) # y(x=-2) = -1
    bc2 =  fun(torch.tensor([ 2.])) - torch.tensor([ 1.]) # y(x= 2) =  1
    return torch.mean(eq**2) + bc1**2 + bc2**2

loss_func = ODE

So, here I passed the model as argument and managed to derive twice... so far so good. BUT, using the sigmoid function for this case is not only not necessary but also gives a result that is far from the analytical one.

If I change the NN for:

class Model(nn.Module):
    def __init__(self):
        super(Model, self).__init__()
        self.lin1 = nn.Linear(1,1)
        self.lin2 = nn.Linear(1,1)

    def forward(self, x):
        y = self.lin1(x)
        z = self.lin2(y)
        return z

In which case I would expect to optimize a double pass through two linear functions that would retrieve a 2nd order function ... I get the error:

  RuntimeError: One of the differentiated Tensors appears to not have been used in the graph. Set allow_unused=True if this is the desired behavior.

Adding the option to the definition of dydx doesn't solve the problem, and adding it to d2ydx2 gives a NoneType definition.

Is there something wrong with the layers as they are?


title: Combine Pytorch ImageFolder dataset with custom Pytorch dataset

body:
I am loading data from multiple datasets using Pytorch. I have some images stored in properly labeled folders (e.g., \0 and \1), and in those cases I can use torch.utils.data.ConcatDataset after loading the lists, for example (where trans is a set of pre-defined Pytorch transformations):

l = []
l.append(datasets.ImageFolder(file_path, trans))
l.append(datasets.ImageFolder(file_path2, trans))
image_datasets = torch.utils.data.ConcatDataset(l)

img_datasets = dict()
img_datasets['train'], img_datasets['val'] = torch.utils.data.random_split(image_datasets, (round(0.8*len(image_datasets)), round(0.2*len(image_datasets)) ))

However, I am also loading images from other disparate locations using a csv file. The process here looks like this:

class MyData(Dataset):
  def __init__(self, df):
      self.df = df

  def __len__(self):
      return self.df.shape[0]

  def __getitem__(self, index):
      image = trans(PIL.Image.open(self.df.file_path[index]))
      label = self.df.label[index]

      return image, label

df = pd.read_csv(image_file_paths), names=["file_path", "label"])
mydata = MyData(df)

my_datasets = dict()
my_datasets['train'], my_datasets['val'] = torch.utils.data.random_split(mydata, (round(0.8*len(mydata)), round(0.2*len(mydata))))

So I'd like to be able to combine these datasets into a single dataloader. Any ideas for how I should go about doing this? Thanks!


title: How to solve PyTorch UnicodeEncodeError error when using torch.from_numpy?

body:
I am using PyTorch with python3.
I tried the following while in ipdb mode:

regions = np.zeros([107,4], dtype='uint8')
torch.from_numpy(regions)

This prints the tensor.
However when trying:

regions = np.zeros([107,107,4], dtype='uint8')
torch.from_numpy(regions)

I get the following error:

*** UnicodeEncodeError: 'ascii' codec can't encode character '\u22ee' in position 72: ordinal not in range(128)

I'm am using:

numpy==1.11.3
torch==0.2.0.post4
torchvision==0.1.9

and python3.5.3


title: Finding euclidean distance given an index array and a pytorch tensor

body:
I got a pytorch tensor:

Z = np.random.rand(100,2)
tZ = autograd.Variable(torch.cuda.FloatTensor(Z), requires_grad=True)

and an index array:

idx = (np.array([0, 0, 0, 4, 3, 8], dtype="int64"),
       np.array([0, 1, 2, 3, 7, 4], dtype="int64"))

I need to find the distances of all the pairs of points in my tZ tensor using the idx array as indexes.

Right now I am doing it using numpy, but it would be nice if it could all be done using torch

dist = np.linalg.norm(tZ.cpu().data.numpy()[idx[0]]-tZ.cpu().data.numpy()[idx[1]], axis=1)

If anyone knows a way of doing it utilizing pytorch, to speed it up, it would be a great help!