title: How do I fix this RuntimeError: size mismatch, m1: [64 x 103], m2: [550 x 50]

body:
I am running a GAN model however it looks like in the generator forward area, I am getting a size mismatch. I am unsure on how to fix this.
The entire error message is below.
---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-25-5498a3d56294> in <module>
    190         print('{}. {}: {:.4f}'.format(idx+1, ccle_features.index[top], pval[top]))
    191
--> 192 run_test_ccle(X_drug[:,ccle_selected],y_drug)

<ipython-input-25-5498a3d56294> in run_test_ccle(X, Y)
    182         # now run test
    183         #pval.append(GCIT(x,Y,z))
--> 184         pval.append(EBGCIT(x,Y,z))
    185
    186     ccle_features = features[ccle_selected]

<ipython-input-24-d7ac34907be1> in EBGCIT(x, y, z, n_iter)
     66
     67             G_loss=0
---> 68             d_input=torch.cat((G[j](g_input),z),axis=1)
     69             Gen_Dis=D(d_input)
     70             G_loss=loss_criteria(Gen_Dis.squeeze(),real)

~/anaconda3/lib/python3.7/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
    720             result = self._slow_forward(*input, **kwargs)
    721         else:
--> 722             result = self.forward(*input, **kwargs)
    723         for hook in itertools.chain(
    724                 _global_forward_hooks.values(),

<ipython-input-23-6f461c71bb16> in forward(self, x)
     32         # Linear function
     33         #print(x.shape)
---> 34         out = self.fc1(x)
     35         out = self.tanh(out)
     36         # out = self.fc2(out)

~/anaconda3/lib/python3.7/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
    720             result = self._slow_forward(*input, **kwargs)
    721         else:
--> 722             result = self.forward(*input, **kwargs)
    723         for hook in itertools.chain(
    724                 _global_forward_hooks.values(),

~/anaconda3/lib/python3.7/site-packages/torch/nn/modules/linear.py in forward(self, input)
     89
     90     def forward(self, input: Tensor) -> Tensor:
---> 91         return F.linear(input, self.weight, self.bias)
     92
     93     def extra_repr(self) -> str:

~/anaconda3/lib/python3.7/site-packages/torch/nn/functional.py in linear(input, weight, bias)
   1672     if input.dim() == 2 and bias is not None:
   1673         # fused op is marginally faster
-> 1674         ret = torch.addmm(bias, input, weight.t())
   1675     else:
   1676         output = input.matmul(weight.t())

RuntimeError: size mismatch, m1: [64 x 103], m2: [550 x 50] at /Users/distiller/project/conda/conda-bld/pytorch_1595629430416/work/aten/src/TH/generic/THTensorMath.cpp:41

This is a portion of my code for my EBGCIT function.
def EBGCIT(x, y, z, n_iter=1000):
    J_g=10
    J_d=1

    # Generator and Discriminator
    G=[None]*J_g
    lr_g=0.005

    noise_G=[None]*J_g

    G_optimizer=[None]*J_g

    for i in range(J_g):
        G[i]=Generator(z_dim+v_dim,h_dim, x_dim)
        G_optimizer[i]=torch.optim.SGD(G[i].parameters(),lr=lr_g)

    D=Discriminator(x_dim+z_dim,h_dim, x_dim)
    D_optimizer=torch.optim.SGD(D.parameters(),lr=lr_g)

    m=0
    for p in G[0].parameters():
        m=m+1

    M=[]
    for j in range(J_g):
        M.append([None]*m)

    for j in range(J_g):
        m=0
        for par in G[0].parameters():
            M[j][m]=torch.zeros(par.size())
            m=m+1

    n = len(z[:, 0])
    # define training and testing subsets, training for learning the sampler and
    # testing for computing test statistic. Set 2/3 and 1/3 as default
    x_train, y_train, z_train = x[:int(2*n/3),], y[:int(2*n/3),], z[:int(2*n/3),]
    x_test, y_test, z_test = x[int(2*n/3):,], y[int(2*n/3):,], z[int(2*n/3):,]

    n = len(z_train[:, 0])

    # 2. # of confounders

    z_train=torch.Tensor(z_train)
    x_train=torch.Tensor(x_train)

    z_test=torch.Tensor(z_test)
    check=x_test
    x_test=torch.Tensor(x_test)

    for iter in range(1000):

        # Train Generator
        for j in range(J_g):

            v=np.sqrt(1./3)*torch.randn(mini_batch,v_dim)
            perm = torch.randperm(n)
            idx=perm[:mini_batch]
            z=z_train[idx]
            g_input=torch.cat((z,v),axis=1)

            G_loss=0
            d_input=torch.cat((G[j](g_input),z),axis=1)
            Gen_Dis=D(d_input)
            G_loss=loss_criteria(Gen_Dis.squeeze(),real)

            l2=0.0
            for p in G[j].parameters():
                l2=l2+(p**2).sum()/200

            G_loss+=l2/n

            G_optimizer[j].zero_grad()

            G_loss.backward()
            G_optimizer[j].step()

            m=0
            for par in G[j].parameters():
                par.data.sub_(a*M[j][m]*lr_g/n)
                m=m+1

            with torch.no_grad():
                for param in G[j].parameters():
                    param.add_(torch.randn(param.size()) * np.sqrt(2*lr_g/n))

            m=0
            for par in G[j].parameters():
                M[j][m]*=beta_1
                M[j][m]+=(1-beta_1)*par.grad.data*n
                m=m+1

And this is the code for my generator and discriminator classes.
J_g=10
np.random.seed(1)
torch.manual_seed(1)

class Generator(nn.Module):
    def __init__(self, input_dim, hidden_dim, output_dim):
        super(Generator, self).__init__()
        # Linear function
        self.fc1 = nn.Linear(input_dim, hidden_dim)
        nn.init.xavier_normal_(self.fc1.weight)
        self.tanh = nn.Tanh()

        self.fc3 = nn.Linear(hidden_dim, output_dim)
        nn.init.xavier_normal_(self.fc3.weight)

    def forward(self, x):
        # Linear function
        #print(x.shape)
        out = self.fc1(x)
        out = self.tanh(out)
        out = self.fc3(out)
        return out

class Discriminator(nn.Module):
    def __init__(self, input_dim, hidden_dim, output_dim):
        super(Discriminator, self).__init__()
        # Linear function
        self.fc1 = nn.Linear(input_dim, hidden_dim)
        nn.init.xavier_normal_(self.fc1.weight)
        self.relu = nn.ReLU()

        self.fc3 = nn.Linear(hidden_dim, output_dim)
        nn.init.xavier_normal_(self.fc3.weight)

    def forward(self, x):
        # Linear function
        out = self.fc1(x)
        out = self.relu(out)
        out = self.fc3(out)
        return out

mini_batch=64
loss_criteria=nn.BCEWithLogitsLoss()

real=torch.ones(mini_batch)
fake=torch.zeros(mini_batch)

a=1
beta_1=0.9

A=1
B=1000
sig=nn.Sigmoid()

Any help would be much appreciated. Thanks!!


title: Can not make an exe file with tkinter and pytorch model

body:
I changed .spec file as follows:
# -*- mode: python ; coding: utf-8 -*-
block_cipher = None
excluded_modules = ['torch.distributions'] # add this
a = Analysis(['sample.py'],
             pathex=['path'],
             binaries=[],
             datas=[],
             hiddenimports=['pkg_resources.py2_warn'], # add this
             hookspath=[],
             runtime_hooks=[],
             excludes=excluded_modules, # add this
             win_no_prefer_redirects=False,
             win_private_assemblies=False,
             cipher=block_cipher,
             noarchive=False)
pyz = PYZ(a.pure, a.zipped_data,
             cipher=block_cipher)

Then run:
pyinstaller -F sample.spec

then i got this error

Note :

I load the pytorch model from a local .pth file


title: How to quickly cut slices from cuda tensor wrt to another tensors values

body:
I have a torch cuda tensor A of shape (100, 80000, 4), and another cuda tensor B of shape (1, 80000, 1). What I want to do, is for each element i in the second dimension (from 0 to 79999), take the value of tensor B (which will be from 0 to 99 and will point out to which value in the first dimension of A to take.
An additional problem is that for this element of B (B[0, i, 0]), I want to take a slice from A that is A[lower_bound:upper_bound, i:i+1, :].
To sum it up, my tensor B has the indices of the centers of slices that I would like to cut from A. I am wondering, whether there is a way, to do what I am doing with the below code faster (eg. using cuda)
A  # tensor of shape (100, 80000, 4)
B  # tensor of shape (1, 80000, 1) which has
k = 3  # width of the slice to take (or half of it to be exact)

A_list = []
for i in range(80000):
    lower_bound = max(0, B[0, i, 0]-k)
    upper_bound = min(100, B[0, i, 0]+k+1)
    A_mean = A[lower_bound:upper_bound, i:i+1, :].mean(0, keepdim=True)
    A_list.append(A_mean)

A = torch.cat(A_list , dim=1)


title: MKLDNN support for bidirectional LSTM in PyTorch

body:
I have PyTorch 1.7.1 and looking at this: https://github.com/pytorch/pytorch/blob/master/torch/utils/mkldnn.py I don’t see support for MKLDNN for bidirectional LSTMs on CPUs. Also, when I run my code in verbose, I find calls only for MKL not for MKLDNN. I checked my pytorch configuration which shows both MKL and MKLDNN are ON.
Am I missing something or there is no support of MKLDNN for bidirectional LSTMs on CPUs in Pytorch?
I am doing a comparison of LSTMs on Tensorflow/Keras and PyTorch so just want to be sure that I am not missing anything.


title: Implement Louvain algorithm in PyTorch

body: I am trying to represent a large graph in PyTorch by placing each element in it's own layer and then connecting layers. I would then like to find communities in that graph using the Louvain algorithm and would appreciate any suggestions or tips in regards to doing that. Thanks!


title: Does pytorch broadcast consume less memory than expand?

body:
Does pytorch operations using broadcast consume less memory than expand? For example, are the following two programs different in memory usage?
import torch
x = torch.randn(20,1)
y = torch.randn(1,20)
z = x*y

import torch
x = torch.randn(20,1).expand(-1,20)
y = torch.randn(1,20).expand(20,-1)
z = x*y


title: How pytorch implements back propagation from the output layer to the input layer

body:
I am having difficulty implementing the following functions.
Assuming that we have trained a network model, I want to backpropagate from the output layer to the input layer (not the first layer) to obtain a new input data. I want to know if there is a function in pytorch or other existing functions that can achieve this function, I did not find the relevant function in the pytorch tutorials.


title: Neural network graph visualization

body:
I would like to generate visualization of my neural network (PyTorch or ONNX model) similar to this using Graphcore Poplar.
I have looked in the documentation but I cannot find where this visualization feature is.
How can I achieve such a task ? Is there any other existing library ?


title: My DC-GAN on grayscale face images is not training well

body:
So I trained by python/pytorch DC-GAN (deep convolutional GAN) for 30 epochs on grayscale faces, and my GAN pretty much failed. I added batch normalization and leaky relu's to my generator and discriminator (I heard those are ways to make the GAN converge), and the Adam optimizer. My GAN still only putting out random grayscale pixels (nothing even remotely related to faces.) I have no problem with the discriminator, my discriminator works very well. I then implemented weight decay of 0.01 on my discriminator to make my GAN train better (since my discriminator was doing better than my generator) but to no avail. Finally, I tried training the GAN for more epochs, 60 epochs. My GAN still generates just random pixels, sometimes outputting completely black.
The GAN training method I used worked for the MNIST dataset (but I used a way simpler GAN architecture for that.)
import torch.nn as nn
import torch.nn.functional as F

class Discriminator(nn.Module):
    def __init__(self):
        super().__init__()
        self.conv1 = nn.Conv2d(1, 4, 3)
        self.conv2 = nn.Conv2d(4, 8, 3)
        self.bnorm1 = nn.BatchNorm2d(8)

        self.conv3 = nn.Conv2d(8, 16, 3)
        self.conv4 = nn.Conv2d(16, 32, 3)
        self.bnorm2 = nn.BatchNorm2d(32)

        self.conv5 = nn.Conv2d(32, 4, 3)

        self.fc1 = nn.Linear(5776, 1024)
        self.fc2 = nn.Linear(1024, 512)
        self.fc3 = nn.Linear(512, 1)
    def forward(self, x):
        pred = F.leaky_relu(self.conv1(x.reshape(-1,1,48,48)))
        pred = F.leaky_relu(self.bnorm1(self.conv2(pred)))
        pred = F.leaky_relu(self.conv3(pred))
        pred = F.leaky_relu(self.bnorm2(self.conv4(pred)))
        pred = F.leaky_relu(self.conv5(pred))

        pred = pred.reshape(-1, 5776)

        pred = F.leaky_relu(self.fc1(pred))
        pred = F.leaky_relu(self.fc2(pred))
        pred = torch.sigmoid(self.fc3(pred))

        return pred

class Generator(nn.Module):
    def __init__(self):
        super().__init__()
        self.fc1 = nn.Linear(512, 1024)
        self.fc2 = nn.Linear(1024, 2048)
        self.fc3 = nn.Linear(2048, 5776)

        self.convT1 = nn.ConvTranspose2d(4, 32, 3)
        self.convT2 = nn.ConvTranspose2d(32, 16, 3)
        self.bnorm1 = nn.BatchNorm2d(16)
        self.convT3 = nn.ConvTranspose2d(16, 8, 3)
        self.convT4 = nn.ConvTranspose2d(8, 4, 3)
        self.bnorm2 = nn.BatchNorm2d(4)
        self.convT5 = nn.ConvTranspose2d(4, 1, 3)

    def forward(self, x):
        pred = F.leaky_relu(self.fc1(x))
        pred = F.leaky_relu(self.fc2(pred))
        pred = F.leaky_relu(self.fc3(pred))

        pred = pred.reshape(-1, 4, 38, 38)

        pred = F.leaky_relu(self.convT1(pred))
        pred = F.leaky_relu(self.bnorm1(self.convT2(pred)))
        pred = F.leaky_relu(self.convT3(pred))
        pred = F.leaky_relu(self.bnorm2(self.convT4(pred)))
        pred = torch.sigmoid(self.convT5(pred))

        return pred

import torch.optim as optim

discriminator = discriminator.to("cuda")
generator = generator.to("cuda")

discriminator_losses = []
generator_losses = []

for epoch in range(30):
    for data,label in tensor_dataset:
        data = data.to("cuda")
        label = label.to("cuda")

        batch_size = data.size(0)
        real_labels = torch.ones(batch_size, 1).to("cuda")
        fake_labels = torch.zeros(batch_size, 1).to("cuda")

        noise = torch.randn(batch_size, 512).to("cuda")

        D_real = discriminator(data)
        D_fake = discriminator(generator(noise))

        D_real_loss = F.binary_cross_entropy(D_real, real_labels)
        D_fake_loss = F.binary_cross_entropy(D_fake, fake_labels)

        D_loss = D_real_loss+D_fake_loss

        d_optim.zero_grad()
        D_loss.backward()
        d_optim.step()

        noise = torch.randn(batch_size, 512).to("cuda")
        D_fake = discriminator(generator(noise))
        G_loss = F.binary_cross_entropy(D_fake, real_labels)

        g_optim.zero_grad()

        G_loss.backward()
        g_optim.step()

        discriminator_losses.append(D_loss)
        generator_losses.append(G_loss)

    print(epoch)


title: Is using Beam Search decoding in an OCR model effective?

body: I am training an Arabic OCR model, my question is, is using beam search in the inference process effective? because I am not sure if it makes sense to use such a search technique.


title: Multiclass classification: Dimension out of range (expected to be in range of [-1, 0], but got 1)

body:
When I use cross-entropy loss function for multiclass text classification, I get the error
Dimension out of range (expected to be in range of [-1, 0], but got 1)

This is my code:
def train(model, iterator):

    ...
    for batch in iterator:
        text, text_lengths = batch.Turn

        optimizer.zero_grad()
        predictions = model(text, text_lengths).squeeze(1)
        loss = criterion(predictions, batch.label)
        acc = categorical_accuracy(predictions, batch.label)

       ...

Dataset:
TEXT = data.Field(tokenize = 'spacy', include_lengths = True)
LABEL = data.LabelField(dtype = torch.long)

Forward:
 def forward(self, text, text_lengths):

        embedded = self.embedding(text)

        packed_embedded = nn.utils.rnn.pack_padded_sequence(embedded, text_lengths)

        packed_output, (hidden, cell) = self.rnn(packed_embedded)

        hidden = self.dropout(torch.cat((hidden[-2,:,:], hidden[-1,:,:]), dim = 1))
        output = self.fc1(hidden)
        output = self.dropout(self.fc2(output))

        return output


title: I got NotEmplemented Error on making model from CNN module in line output = model(input)

body:
here is my code :
import cv2
import torch
import torch.nn as nn
import torchvision.transforms as transforms
import torchvision
import torchvision.datasets as datasets
from torch.autograd import Variable
import matplotlib.pyplot as plt
from PIL import Image
import numpy as np

#Transformation for image
transform_ori = transforms.Compose([transforms.RandomResizedCrop(64),   #create 64x64 image
                                    transforms.RandomHorizontalFlip(),    #flipping the image horizontally
                                    transforms.ToTensor(),                 #convert the image to a Tensor
                                    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])])  #normalize the image

#Load our dataset
train_dataset = datasets.ImageFolder(root = '/content//Data',
                                     transform = transform_ori)

test_dataset = datasets.ImageFolder(root = '/content/Data',
                                    transform = transform_ori)

#Make the dataset iterable
batch_size = 100
train_load = torch.utils.data.DataLoader(dataset = train_dataset,
                                         batch_size = batch_size,
                                         shuffle = True)

batch_size = 100
train_load = torch.utils.data.DataLoader(dataset = train_dataset,
                                         batch_size = batch_size,
                                         shuffle = True)      #Shuffle to create a mixed batches of 100 of cat & dog images

test_load = torch.utils.data.DataLoader(dataset = test_dataset,
                                         batch_size = batch_size,
                                         shuffle = False)

#Show a batch of images
def imshow(img):
    img = img / 2 + 0.5     # unnormalize
    npimg = img.numpy()
    plt.figure(figsize=(20,20))
    plt.imshow(np.transpose(npimg, (1, 2, 0)))

# get some random training images
dataiter = iter(train_load)
images, labels = dataiter.next()

# show images
imshow(torchvision.utils.make_grid(images))

print("There are {} images in train set".format(len(train_dataset)))
print("There are {} images in train loader".format(len(train_load)))
print("There are {} images in test set".format(len(test_dataset)))
print("There are {} images in test loader".format(len(test_load)))

class CNN(nn.Module):
    def __init__(self):
        super(CNN,self).__init__()

        self.cnn1 = nn.Conv2d(in_channels=3, out_channels=8, kernel_size=3,stride=1, padding=1)
        self.batchnorm1 = nn.BatchNorm2d(8)        #Batch normalization
        self.relu = nn.ReLU()                 #RELU Activation
        self.maxpool1 = nn.MaxPool2d(kernel_size=2)   #Maxpooling reduces the size by kernel size. 64/2 = 32

        self.cnn2 = nn.Conv2d(in_channels=8, out_channels=32, kernel_size=5, stride=1, padding=2)
        self.batchnorm2 = nn.BatchNorm2d(32)
        self.maxpool2 = nn.MaxPool2d(kernel_size=2)    #Size now is 32/2 = 16

        #Flatten the feature maps. You have 32 feature mapsfrom cnn2. Each of the feature is of size 16x16 --> 32*16*16 = 8192
        self.fc1 = nn.Linear(in_features=8192, out_features=4000)   #Flattened image is fed into linear NN and reduced to half size
        self.droput = nn.Dropout(p=0.5)                    #Dropout used to reduce overfitting
        self.fc2 = nn.Linear(in_features=4000, out_features=2000)
        self.droput = nn.Dropout(p=0.5)
        self.fc3 = nn.Linear(in_features=2000, out_features=500)
        self.droput = nn.Dropout(p=0.5)
        self.fc4 = nn.Linear(in_features=500, out_features=50)
        self.droput = nn.Dropout(p=0.5)
        self.fc5 = nn.Linear(in_features=50, out_features=2)    #Since there were so many features, I decided to use 45 layers to get output layers. You can increase the kernels in Maxpooling to reduce image further and reduce number of hidden linear layers.

def forward(self,x):
        out = self.cnn1(x)
        out = self.batchnorm1(out)
        out = self.relu(out)
        out = self.maxpool1(out)
        out = self.cnn2(out)
        out = self.batchnorm2(out)
        out = self.relu(out)
        out = self.maxpool2(out)
        #Flattening is done here with .view() -> (batch_size, 32*16*16) = (100, 8192)
        out = out.view(-1,8192)   #-1 will automatically update the batchsize as 100; 8192 flattens 32,16,16
        #Then we forward through our fully connected layer
        out = self.fc1(out)
        out = self.relu(out)
        out = self.droput(out)
        out = self.fc2(out)
        out = self.relu(out)
        out = self.droput(out)
        out = self.fc3(out)
        out = self.relu(out)
        out = self.droput(out)
        out = self.fc4(out)
        out = self.relu(out)
        out = self.droput(out)
        out = self.fc5(out)
        return out

model = CNN()
CUDA = torch.cuda.is_available()
if CUDA:
    model = model.cuda()
loss_fn = nn.CrossEntropyLoss()
optimizer = torch.optim.SGD(model.parameters(), lr = 0.01)

import time

num_epochs = 50

#Define the lists to store the results of loss and accuracy
train_loss = []
test_loss = []
train_accuracy = []
test_accuracy = []

#Training
for epoch in range(num_epochs):
    #Reset these below variables to 0 at the begining of every epoch
    start = time.time()
    correct = 0
    iterations = 0
    iter_loss = 0.0

    model.train()                   # Put the network into training mode

    for i, (inputs, labels) in enumerate(train_load):

      # Convert torch tensor to Variable
      Inputs = Variable(inputs)
      Labels = Variable(labels)

      # If we have GPU, shift the data to GPU
      CUDA = torch.cuda.is_available()
      if CUDA:
        print("Cuda is Available!")
        Inputs = Inputs.cuda()
        Labels = Labels.cuda()

      optimizer.zero_grad()
      outputs = model(inputs)

      loss = loss_fn(outputs, labels)
      iter_loss += loss.data[0]       # Accumulate the loss
      loss.backward()                 # Backpropagation
      optimizer.step()                # Update the weights

      # Record the correct predictions for training data
      _, predicted = torch.max(outputs, 1)
      correct += (predicted == labels).sum()
      iterations += 1

    # Record the training loss
    train_loss.append(iter_loss/iterations)
    # Record the training accuracy
    train_accuracy.append((100 * correct / len(train_dataset)))

    #Testing
    loss = 0.0
    correct = 0
    iterations = 0

    model.eval()                    # Put the network into evaluation mode

    for i, (inputs, labels) in enumerate(test_load):

        # Convert torch tensor to Variable
        inputs = Variable(inputs)
        labels = Variable(labels)

        CUDA = torch.cuda.is_available()
        if CUDA:
            inputs = inputs.cuda()
            labels = labels.cuda()

        outputs = model(inputs)
        loss = loss_fn(outputs, labels) # Calculate the loss
        loss += loss.data[0]
        # Record the correct predictions for training data
        _, predicted = torch.max(outputs, 1)
        correct += (predicted == labels).sum()

        iterations += 1

    # Record the Testing loss
    test_loss.append(loss/iterations)
    # Record the Testing accuracy
    test_accuracy.append((100 * correct / len(test_dataset)))
    stop = time.time()

    print ('Epoch {}/{}, Training Loss: {:.3f}, Training Accuracy: {:.3f}, Testing Loss: {:.3f}, Testing Acc: {:.3f}, Time: {}s'
           .format(epoch+1, num_epochs, train_loss[-1], train_accuracy[-1], test_loss[-1], test_accuracy[-1], stop-start))

# Loss
f = plt.figure(figsize=(10, 10))
plt.plot(train_loss, label='Training Loss')
plt.plot(test_loss, label='Testing Loss')
plt.legend()
plt.show()

# Accuracy
f = plt.figure(figsize=(10, 10))
plt.plot(train_accuracy, label='Training Accuracy')
plt.plot(test_accuracy, label='Testing Accuracy')
plt.legend()
plt.show()

#Run this if you want to save the model
torch.save(model.state_dict(),'Cats-Dogs.pth')

#Run this if you want to load the model
model.load_state_dict(torch.load('Cats-Dogs.pth'))

def predict(img_name,model):
    image = cv2.imread(img_name)   #Read the image
    img = Image.fromarray(image)      #Convert the image to an array
    img = transforms_photo(img)     #Apply the transformations
    img = img.view(1,3,64,64)       #Add batch size
    img = Variable(img)
    #Wrap the tensor to a variable

    model.eval()

    if torch.cuda.is_available():
        model = model.cuda()
        img = img.cuda()

    output = model(img)
    print(output)
    print(output.data)
    _, predicted = torch.max(output,1)
    if predicted.item()==0:
        p = 'Cat'
    else:
        p = 'Dog'
    cv2.imshow('Original',image)
    return  p

pred = predict('dataset/single_prediction/cat_or_dog_1.jpg', model)
print("The Predicted Label is {}".format(pred))

#Load the DenseNet
model_conv = torchvision.models.densenet201(pretrained=True)

#Freeze all layers in the network
for param in model_conv.parameters():
    param.requires_grad = False

#Get the number of inputs of the last layer (or number of neurons in the layer preceeding the last layer)
num_ftrs = model_conv.classifier.in_features

#Reconstruct the last layer (output layer) to have only two classes
model_conv.classifier = nn.Linear(num_ftrs, 2)

#Initiate the model on GPU
if torch.cuda.is_available():
    model_conv = model_conv.cuda()

#Loss function and optimizer
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(model_conv.parameters(), lr=0.001, momentum=0.9) #Try Adam optimizer for better accuracy: optim.Adam(model_conv.parameters(), lr=0.001)

#Decay LR by a factor of 0.1 every 7 epochs
exp_lr_scheduler = lr_scheduler.StepLR(optimizer, step_size=7, gamma=0.1)

train_loss = []
test_loss = []
train_accuracy = []
test_accuracy = []

#Training
import time
num_epochs = 20
for epoch in range (num_epochs):
    start = time.time()
    exp_lr_scheduler.step()
    #Reset the correct to 0 after passing through all the dataset
    correct = 0
    for images,labels in dataloaders['training_set']:
        images = Variable(images)
        labels = Variable(labels)
        if torch.cuda.is_available():
            images = images.cuda()
            labels = labels.cuda()

        optimizer.zero_grad()
        outputs = model_conv(images)
        loss = criterion(outputs, labels)
        loss.backward()
        optimizer.step()
        _, predicted = torch.max(outputs, 1)
        correct += (predicted == labels).sum()

    train_acc = 100 * correct / dataset_sizes['training_set']
    stop = time.time()
    print ('Epoch [{}/{}], Loss: {:.4f}, Train Accuracy: {}%, Time: {:.2f}s'
            .format(epoch+1, num_epochs, loss.item(), train_acc, stop-start))

   #Testing
   model_conv.eval()
with torch.no_grad():
    correct = 0
    total = 0
    start = time.time()
    for (images, labels) in dataloaders['test_set']:

        images = Variable(images)
        labels = Variable(labels)
        if torch.cuda.is_available():
            images = images.cuda()
            labels = labels.cuda()

        outputs = model_conv(images)
        _, predicted = torch.max(outputs.data, 1)
        total += labels.size(0)
        correct += (predicted == labels).sum().item()
    stop = time.time()

    print('Test Accuracy: {:.3f} %, Time: {:.2f}s'.format(100 * correct / total,stop-start))

#Import your trial images and check against the model
import matplotlib.pyplot as plt
from PIL import Image
import numpy as np
#Predict your own image
def predict(img_name,model):
    image = cv2.imread(img_name)   #Read the image
    #ret, thresholded = cv2.threshold(image,127,255,cv2.THRESH_BINARY)   #Threshold the image
    img = Image.fromarray(image)      #Convert the image to an array
    img = transforms_photo(img)     #Apply the transformations
    img = img.view(1,3,224,224)       #Add batch size
    img = Variable(img)
   #Wrap the tensor to a variable

    model.eval()

    if torch.cuda.is_available():
        model = model.cuda()
        img = img.cuda()

    output = model(img)
    print(output)
    print(output.data)
    _, predicted = torch.max(output,1)
    if predicted.item()==0:
        p = 'Cat'
    else:
        p = 'Dog'
    cv2.imshow('Original',image)
    return  p

and i got this :
/usr/local/lib/python3.6/dist-packages/torch/nn/modules/module.py in _forward_unimplemented(self, *input)
registered hooks while the latter silently ignores them.
    raise NotImplementedError

I test some part of my code but it seems everything is ok and it's because of modules in torch library.anybody can help?


title: More explicit indexing

body:
import torch
x = torch.tensor([[10, 11], [12, 13]])
idx = torch.tensor([[0, 1, 0], [1, 1, 0]])
print(x[idx[0], idx[1]])

It outputs tensor([11, 13, 10]) and it's correct but is there a way to make the last row more explicit? I want something like x[*idx] because my tensor has many dimensions.


title: FER2013 training result is wrong - python

body:
I practiced following this on GitHub.
https://github.com/shangeth/Facial-Emotion-Recognition-PyTorch-ONNX
I have modified a few things in "train.py" to get it running.
Indentation of if statements, otherfile's file path...
The training was successful, but the test result was only one facial expression.
This changed every time the model was saved.
How can the training model be made to recognize various facial expressions?
Here is the "train.py" code I used.
import torch
import torch.nn as nn
from model import *
from dataset import *
import matplotlib.pyplot as plt

def train_model(model, trainloader, validloader, epochs=100, visualize_learning_curve=True):
    criterion = nn.NLLLoss()
    optimizer = torch.optim.Adam(model.parameters(), lr=1e-3)
    valid_loss_min = np.Inf
    train_losses, test_losses = [], []
    for e in range(epochs):
        model.train()
        running_loss = 0
        tr_accuracy = 0
        for images, labels in trainloader:
            images = images.cuda()
            labels = labels.long().cuda()
            optimizer.zero_grad()

            log_ps = model(images)
            loss = criterion(log_ps, labels)
            loss.backward()
            optimizer.step()

            running_loss += loss.item()

            ps = torch.exp(log_ps)
            top_p, top_class = ps.topk(1, dim=1)
            equals = top_class == labels.view(*top_class.shape)
            tr_accuracy += torch.mean(equals.type(torch.FloatTensor))
        else:
            test_loss = 0
            accuracy = 0
            with torch.no_grad():
                model.eval()
                for images, labels in validloader:
                    images = images.cuda()
                    labels = labels.long().cuda()
                    log_ps = model(images)
                    test_loss += criterion(log_ps, labels)

                    ps = torch.exp(log_ps)
                    top_p, top_class = ps.topk(1, dim=1)
                    equals = top_class == labels.view(*top_class.shape)
                    accuracy += torch.mean(equals.type(torch.FloatTensor))

            train_losses.append(running_loss/len(trainloader))
            test_losses.append(test_loss/len(validloader))

            print("Epoch: {}/{} ".format(e+1, epochs),
                "Training Loss: {:.3f} ".format(train_losses[-1]),
                "Training Acc: {:.3f} ".format(tr_accuracy/len(trainloader)),
                "Val Loss: {:.3f} ".format(test_losses[-1]),
                "Val Acc: {:.3f}".format(accuracy/len(validloader)))
            if test_loss/len(validloader) <= valid_loss_min:
                print('Validation loss decreased ({:.6f} --> {:.6f}).  Saving model ...'.format(
                valid_loss_min,
                test_loss/len(validloader)))
                torch.save(model.state_dict(), 'best_model.pt')
                valid_loss_min = test_loss/len(validloader)

    if visualize_learning_curve:
        plt.plot(train_losses, 'b', label='Training Loss')
        plt.plot(test_losses, 'r', label='Validation Loss')
        plt.show()
    return model

def main():
    trainloader, validloader = get_dataloaders()
    print('Data Preprocessed and got DataLoaders...')

    model = Face_Emotion_CNN()
    if torch.cuda.is_available():
        model.cuda()
        print('GPU Found!!!, Moving Model to CUDA.')
    else:
        print('GPU not found!!, using model with CPU.')

    print('Starting Training loop...\n')
    model = train_model(model, trainloader, validloader, epochs=50)

if __name__ == '__main__':
    main()


title: Unable to import pytorch in spyder after installing pytorch via conda prompt

body: The specified module could not be found. Error loading "C:\ProgramData\Anaconda3\lib\site-packages\torch\lib\asmjit.dll" or one of its dependencies.


title: MT5ForConditionalGeneration with Pytorch-lightning gives attribute_error

body:
Can't handle that problem for several days, as I'm new to NLP and actual solution can be really simple
class QAModel(pl.LightningDataModule):

  def __init__(self):
    super().__init__()
    self.model = MT5ForConditionalGeneration.from_pretrained(MODEL_NAME, return_dict=True)

  def forward(self, input_ids, attention_mask, labels=None):
    output = model(
        input_ids=input_ids,
        attention_mask=attention_mask,
        labels=labels
    )

    return output.loss, output.logits

  def training_step(self, batch, batch_idx):
    input_ids = batch['input_ids']
    attention_mask = batch['attention_mask']
    labels = batch['labels']
    loss, outputs = self(input_ids, attention_mask, labels)
    self.log('train_loss', loss, prog_bar=True, logger=True)
    return loss

  def validation_step(self, batch, batch_idx):
    input_ids = batch['input_ids']
    attention_mask = batch['attention_mask']
    labels = batch['labels']
    loss, outputs = self(input_ids, attention_mask, labels)
    self.log('val_loss', loss, prog_bar=True, logger=True)
    return loss

  def test_step(self, batch, batch_idx):
    input_ids = batch['input_ids']
    attention_mask = batch['attention_mask']
    labels = batch['labels']
    loss, outputs = self(input_ids, attention_mask, labels)
    self.log('test_loss', loss, prog_bar=True, logger=True)
    return loss

  def configure_optimizers(self):
    return AdamW(self.parameters(), lr=0.0001)
model = QAModel()
from pytorch_lightning.callbacks import ModelCheckpoint

checkpoint_callback = ModelCheckpoint(
    dirpath='/content/checkpoints',
    filename='best-checkpoint',
    save_top_k=1,
    verbose=True,
    monitor='val_loss',
    mode='min'
)
trainer = pl.Trainer(
    checkpoint_callback=checkpoint_callback,
    max_epochs=N_EPOCHS,
    gpus=1,
    progress_bar_refresh_rate=30
)
trainer.fit(model, data_module)

Running this code gives me
AttributeError: 'QAModel' object has no attribute 'automatic_optimization'
after fit() function
Probably, the problem is in MT5ForConditionalGeneration, as after passing it to funtion() we've got the same error


title: Why does Python's cProfile report a different elapsed time than using time.time() deltas when using PyTorch?

body:
I am profiling some code using PyTorch. I am aware the CUDA normally has some asynchronous execution (see PyTorch docs), but I believe that transferring from GPU to CPU will generally force synchronization.
For this reason, I decided to naively profile using cProfile, but I notice that the time reported by Profile.enable() ... Profile.disable() was different than the time recorded across time.time() (as a delta).
Here's what the code looks like at a high-level:
gpu = torch.device("cuda")
cpu = torch.device("cpu")

setup = Setup()
net = make_fcn_resnet50(num_classes=setup.D)
net.eval().to(gpu)

rgb_tensor = setup.sample(device=cpu)

pr = profile.Profile()
pr.enable()
t_start = time.time()

rgb_tensor = rgb_tensor.to(gpu)
y = net(rgb_tensor)
dd_tensor = y["out"]
dd_mean = torch.mean(dd_tensor[[0]]).to(cpu).numpy()
assert dd_mean is not None

dt = time.time() - t_start
pr.disable()
stats = pstats.Stats(pr)

stats.print_stats(5)
print(f"dt: {dt:.4f}s")

Here's the discrepancy I see:
2925 function calls (2734 primitive calls) in 0.009 seconds
...
dt: 0.0355s

I would've expected cProfile to report about 35ms (same as dt), but instead it reports about 10ms.
Why does this happen?
Full code + repro is here:
https://github.com/EricCousineau-TRI/repro/tree/bdef8a14/python/cprofile_with_torch


title: mlflow on Windows10 and desktop.ini

body:
it's the first time for me to leave a question here.
I'm currently using PyTorch on my research and trying to organize results with MLFlow.
I know that many problems when using MLflow on Windows10 but since there are no options for this... I'm trying to get used to it.
Errors that I encounter here are "Metrics 'desktop.ini' is malformed ...'
This error is nagging me when -

Using mlflow ui to see experiment results from the past mlflow ui error
When trying to use mlflow.pytorch.log_model(model, ...) pytorch.log_model error

These two are the main concerns for me. My question here is

Is there other result organizing tools that I can use except tensorboard?
Is there any method that can save model.pth from pytorch to mlflow? + if it's impossible, are there  any other formats that we use to save the configuration (such as YAML, other hierarchical languages like XML..)

Thank you


title: 10 fold cross validation evaluation

body:
I have the following model for classification. I have train set and test set. I trained it on train set ,the Input is 3400 vector and output it is a class between 3 classes (0,1,2). I saved the model as the following code . Now I want to apply 10 fold cross validation to evaluate the saved model on test set. Could you please tell how i can do it since I never used 10 cross validation before.
training_set = Dataset("train_data.txt","train_target.txt")
training_generator = torch.utils.data.DataLoader(training_set, **params)
testing_set = Dataset("test_data.txt","testtarget.txt")
testing_generator = torch.utils.data.DataLoader(testing_set, **params)
    for i, (seq_batch, stat_batch) in enumerate(training_generator):
        seq_batch, stat_batch = seq_batch.to(device), stat_batch.to(device)
        optimizer.zero_grad()
        #print(seq_batch.shape,stat_batch.shape)
        # Model computation
        seq_batch = seq_batch.unsqueeze(-1)
        outputs = model(seq_batch)
        if CUDA:
            loss = criterion(outputs, stat_batch)
        loss.backward()
        optimizer.step()
        # print statistics
        running_loss += loss.item()
        epoch_loss += loss.item()*outputs.shape[0]
        if i % 2000 == 1999:  # print every 2000 mini-batches
            print('[%d, %5d] loss: %.3f' %
                  (epoch + 1, i + 1, running_loss / 2000),"acc",(outputs.argmax(1) == stat_batch).float().mean())
            running_loss = 0.0
        sum_acc += (outputs.argmax(1) == stat_batch).float().sum()

    print("epoch" , epoch+1, "acc", sum_acc/len(training_set),"loss", epoch_loss/len(training_set))
    loss_values.append(epoch_loss/len(training_set))
    if epoch % 20 == 0:
        torch.save(model.state_dict(), path + name_file + "model_epoch_i_" + str(epoch) + ".cnn")


title: Facing Broken Pipe error when trying to run the next(iter(train_data_loader)). I'm running the code in local jupyter notebook

body:
I'm facing BrokenPipeError when I'm trying to run sentiment analysis with hugging face. It's returning [Error No] 32 Broken Pipe. Is there any way to rewrite the next(iter(train_data_loader)) code?
Link with total code 'https://colab.research.google.com/drive/1wBXKa-gkbSPPk-o7XdwixcGk7gSHRMas?usp=sharing'
The code is
def create_data_loader(df, tokenizer, max_len, batch_size):
  ds = GPReviewDataset(
    reviews=df.content.to_numpy(),
    targets=df.sentiment.to_numpy(),
    tokenizer=tokenizer,
    max_len=max_len
  )
  return DataLoader(
    ds,
    batch_size=batch_size,
    num_workers=4
  )

Followed by below code
BATCH_SIZE = 16
train_data_loader = create_data_loader(df_train, tokenizer, MAX_LEN, BATCH_SIZE)
val_data_loader = create_data_loader(df_val, tokenizer, MAX_LEN, BATCH_SIZE)
test_data_loader = create_data_loader(df_test, tokenizer, MAX_LEN, BATCH_SIZE)

Followed by
data = next(iter(train_data_loader))
data.keys()

I'm facing error with this 'data = next(iter(train_data_loader))' code
Error is BrokenPipeError: [Errno 32] Broken pipe


title: len() vs .size(0) when looping through DataLoader samples

body:
I came across this on github (snippet from here):
(...)
for epoch in range(round):
        for i, data in enumerate(dataloader, 0):
            ############################
            # (1) Update D network: maximize log(D(x)) + log(1 - D(G(z)))
            ###########################
            # train with real
            netD.zero_grad()
            real_cpu = data[0].to(device)
            batch_size = real_cpu.size(0)
            label = torch.full((batch_size,), real_label, device=device)
            (...)

Would replacing batch_size = real_cpu.size(0) with batch_size = len(data[0]) give the same effect? (or maybe at least with batch_size = len(real_cpu)?) Reason why I'm asking is that iirc the official PyTorch tutorial incorporated len(X) when displaying training progress during the loop for (X, y) in dataloader: etc. so I was wondering if the two methods are equivalent for displaying the number of 'samples' in the 'current' batch.


title: How load torch model for cpu inference

body:
Can't figure out what the problem is. Even if I point the download to cpu, torch tries to use cuda to work:
model = nn.Sequential(
    nn.Conv2d(3,16,5),
    nn.ReLU(),
    nn.MaxPool2d(kernel_size=2, stride=2),
    nn.Conv2d(16,16,5),
    nn.ReLU(),
    nn.MaxPool2d(kernel_size=2, stride=2),
    nn.Conv2d(16,32,5),
    nn.ReLU(),
    nn.MaxPool2d(kernel_size=2, stride=2),
    nn.Conv2d(32,64,5),
    nn.ReLU(),
    nn.MaxPool2d(kernel_size=2, stride=2),
    nn.Conv2d(64,128,5),
    nn.ReLU(),
    nn.MaxPool2d(kernel_size=2, stride=2),
    nn.Flatten(),
    torch.nn.Linear(1152, 256),
    torch.nn.ELU(), torch.nn.Linear(256,4)
            )
model.load_state_dict(torch.load('bbox.pth',map_location=torch.device('cpu')))
ds_trans = transforms.Compose([transforms.ToTensor(),normalize,transforms.Resize((224,224))])
batch = ds_trans(img).unsqueeze(0)
ans = model(batch)

result of run
AssertionError: Torch not compiled with CUDA enabled

Model was learned on GPU, and GPU inference is working well, but I need cpu inference on another pc.


title: Sparse Matrix for real world datasets

body: Can someone help me understand the process of creating Sparse Matrices for Recommender Systems in Python for real world datasets. All I see online are the datasets having customer ID or Item ID starting with 0 which is not valid in real world datasets.


title: brodcasting 2D index select in pytorch?

body:
I have a tensor of shape P.shape=[N,k] and an index tensor of shape ind.shape=[L,N], where ind[i,j]  is a column in P[j] (ind[i,j] < k always).
I wish to create a new tensor of dims [L,n], where the functionality can be generated using a for loop in the following manner:
    new= []
    num_points = P.shape[-1]
    for experiment in range(ind.shape[0]):
        new.append(P[torch.arange(num_points),ind[exp]])
    new= torch.stack(new)

But as L is really big, the code is extremely slow.
Using repeat I managed to replicate the functionality
new = P.unsqueeze(1).repeat(1,L,1,1).reshape(-1,*P.shape[1:])
new = new.gather(2,ind.unsqueeze(2)).squeeze(2)

But as L is really big I have an OOM exception on the .repeat(1,L,1,1) line.
I was wondering if I can accomplish something similar using broadcasting?


title: Huggingface error: AttributeError: 'ByteLevelBPETokenizer' object has no attribute 'pad_token_id'

body:
I am trying to tokenize some numerical strings using a WordLevel/BPE tokenizer, create a data collator and eventually use it in a PyTorch DataLoader to train a new model from scratch.
However, I am getting an error

AttributeError: 'ByteLevelBPETokenizer' object has no attribute 'pad_token_id'

when running the following code
from transformers import DataCollatorForLanguageModeling
from tokenizers import ByteLevelBPETokenizer
from tokenizers.pre_tokenizers import Whitespace
from torch.utils.data import DataLoader, TensorDataset

data = ['4814 4832 4761 4523 4999 4860 4699 5024 4788 <unk>']

# Tokenizer
tokenizer = ByteLevelBPETokenizer()
tokenizer.pre_tokenizer = Whitespace()
tokenizer.train_from_iterator(data, vocab_size=1000, min_frequency=1,
    special_tokens=[
        "<s>",
        "</s>",
        "<unk>",
        "<mask>",
    ])

# Data Collator
data_collator = DataCollatorForLanguageModeling(
    tokenizer=tokenizer, mlm=False
)

train_dataset = TensorDataset(torch.tensor(tokenizer(data, ......)))

# DataLoader
train_dataloader = DataLoader(
    train_dataset,
    collate_fn=data_collator
)

Is this error due to not having configured the pad_token_id for the tokenizer? If so, how can we do this?
Thanks!
Error trace:
AttributeError: Caught AttributeError in DataLoader worker process 0.
Original Traceback (most recent call last):
  File "/opt/anaconda3/envs/x/lib/python3.8/site-packages/torch/utils/data/_utils/worker.py", line 198, in _worker_loop
    data = fetcher.fetch(index)
  File "/opt/anaconda3/envs/x/lib/python3.8/site-packages/torch/utils/data/_utils/fetch.py", line 47, in fetch
    return self.collate_fn(data)
  File "/opt/anaconda3/envs/x/lib/python3.8/site-packages/transformers/data/data_collator.py", line 351, in __call__
    if self.tokenizer.pad_token_id is not None:
AttributeError: 'ByteLevelBPETokenizer' object has no attribute 'pad_token_id'

Conda packages
pytorch                   1.7.0           py3.8_cuda10.2.89_cudnn7.6.5_0    pytorch
pytorch-lightning         1.2.5              pyhd8ed1ab_0    conda-forge
tokenizers                0.10.1                   pypi_0    pypi
transformers              4.4.2                    pypi_0    pypi


title: Understanding the logic behind the forget gate in LSTM

body: I'm having trouble understanding the logic of the forget gate. I understand the math, however I'm thinking of extreme examples to figure out the intuition. From my understanding, it sounds like when the result of the sigmoid function (i.e. the forget state) is 0, then the point-wise multiplication with the cell state will cause the cell state to "forget all information". This currently doesn't make sense to me. What if you have very useful information in a sentence and then the next word is blank or something that doesn't fit; why should this affect all the information you had previously? I understand there are other gates here, but I'm not seeing the logic of the forget gate, yet. Any help would be appreciated.


title: Model.eval duration vs model.train

body:
We are experiencing strange behaviour in training and evaluating our model.
Since a while we are trying to optimize our performance, but we encountered as in our mind a strange behaviour.

To train one epoch we need roughly 70min with 5400 inputs. To

eval the model takes around 55min on 4600 inputs .

Does this even makes sense? We have a batch_size of 32. each batch takes around 1.5s to load into memory.
Following our train and eval loop:
I removed logging and timing from code to make it more compact:
Main.py
for epoch in range(1, config.general['epochs'] + 1):
    closs = runner.train(train_loader=train_loader,
                         epoch=epoch,
                         log_interval=config.general['log_interval'],
                         batch_size=config.data['batch_size'])

    vloss = runner.test(test_loader=test_loader, log_interval=config.general['log_interval'])
    scheduler.step()

Runner.py
def train(self, train_loader, epoch, log_interval, batch_size):
    self.model.train()
    closs = 0
    for batch_idx, (data, target) in enumerate(train_loader):
        self.optimizer.zero_grad()
        data = data.to(self.device)
        target = target.to(self.device)
        output = self.model(data)['normalized']
        loss = self.loss_function(output, target)
        loss.backward()
        closs = closs + loss.detach().item()
        self.optimizer.step()
    return closs

def test(self, test_loader, log_interval):
    self.model.eval()
    with torch.no_grad():
        correct = 0
        vloss = 0
        for batch_idx, (data, target) in enumerate(test_loader):
            data = data.to(self.device)
            target = target.to(self.device)
            output = self.model(data)
            loss = self.loss_function(output['normalized'], target)
            vloss = vloss + loss.detach().item()
            pred = output['x'].max(1)[1]  # get the index of the max log-probability
            correct += pred.eq(target).cpu().sum().item()

        accr = 100. * correct / len(test_loader.dataset) accr))
        return vloss

Performance measurements:
~7s for those 3 lines in train()
        output = self.model(data)['normalized']
        loss = self.loss_function(output, target)
        loss.backward()

~20s for those 2 lines test()
            loss = self.loss_function(output['normalized'], target)
            vloss = vloss + loss.detach().item()

Strange thing also even tough we are using torch.no_grad() our GPU-Memory usage is around 90%.
We are now totally not sure if we should invest more time in optimizing our performance or if this is just normal behaviour. Until up now I'm thinking that evaluation should be much faster than training even tough we have roughly the same size as training data(we use only 20% of whole train_set) and test data.
EDIT: Is it normal that evaluation takes as much time as training? My guess no. But i cannot explain the described behaviour
EDIT2: added measures. Those measurments seem to be extremely weird isn't it?


title: Architecture of the network used in Pyro normalizing flows

body:
I am reading the Pyro tutorial on normalizing flows (https://pyro.ai/examples/normalizing_flows_i.html) and I would like to understand better how the examples work under the hood.
For instance, I am referring to the architecture of the network used to obtain the marginal distributions of the concentric circles example.
In the example the base distribution (in the latent space) is normal and the flow is a rational  spline:
base_dist = dist.Normal(torch.zeros(2), torch.ones(2))
spline_transform = T.Spline(2, count_bins=16)
flow_dist = dist.TransformedDistribution(base_dist, [spline_transform])

According to the tutorial, the knots (of the spline) and their derivatives are parameters that can be learnt e.g., through stochastic gradient descent on a maximum likelihood objective. The authors proceed by showing how to do that :
%%time
steps = 1 if smoke_test else 1001
dataset = torch.tensor(X, dtype=torch.float)
optimizer = torch.optim.Adam(spline_transform.parameters(), lr=1e-2)
for step in range(steps):
    optimizer.zero_grad()
    loss = -flow_dist.log_prob(dataset).mean()
    loss.backward()
    optimizer.step()
    flow_dist.clear_cache()

    if step % 200 == 0:
        print('step: {}, loss: {}'.format(step, loss.item()))

Finally, the author indicates how to sample from the learned distribution in order to obtain a new sample :
X_flow = flow_dist.sample(torch.Size([1000,])).detach().numpy()

I would like to know what is the architecture of the NN used to learn those parameters and if is there a (possibly simple) way to modify this architecture.
More generally, I would like to adapt these simple examples to the univariate case of learning the density of time series data.


title: Computing gradients only for the front-end network in Pytorch

body:
I have a very simple question.
Let's say that I have two networks to train (i.e., net1, net2).
The output of net1 will be fed into net2 while training.
In my case, I would like to only update net1:
optimizer=Optimizer(net1.parameters(), **kwargs)
loss=net2(net1(x))
loss.backward()
optimizer.step()

Although this will achieve what I'm aiming for, it takes up too much redundant memory since this will compute the gradients for net2 (causes OOM error).
Therefore I have tried out several attempts to solve this issue:

torch.no_grad:

z=net1(x)
with torch.no_grad():
    loss=net2(z)

Didn't raise OOM but removed all the gradients including the ones from net1.

requires_grad=False:

net2.requires_grad=False
loss=net2(net1(x))

Raised OOM.

detach():

z=net1(x)
loss=net2(z).detach()

Didn't raise OOM but removed all the gradients including the ones from net1.

eval():

net2.eval()
loss=net2(net1(x))

Raised OOM.
Is there any way to compute the gradients only for the front-end network (net1) for memory efficiency?
Any suggestions would be appreciated.


title: whats the equivalent of torch.nn.Parameter() in TF?

body: Recently I want to convert the codes using pytorch to that using tensorflow, but l dont know how to replace the torch.nn.Parameter() with TF function.


title: Having problems with linear pytorch model initialization

body:
I am trying to figure out what is wrong with my initialization of the neural network model. I have already set a pdb trace to see that the defining neural network part is the source of error. Also, I get yellow marks on the defining neural network code because the module is expected to be returned but if I return the module, it causes a recursion error. It is a linear model that has to have an input dimension of the batch size * 81 and an output dimension of the batch size * 1. I am relatively new at pytorch and defining deep neural networks so this may not be a good question. My syntax may also be very bad. Any help is appreciated. The code below is the defining of the neural network and training of the pytorch model.
def get_nnet_model(module_list=nn.ModuleList(), input_dim: int = 8100, layer_dim: int = 100) -> nn.Module:
    """ Get the neural network model
    @return: neural network model
    """
    device = torch.device('cpu')

    module_list.append(nn.Linear(input_dim, layer_dim))
    module_list[-1].weight.data.normal_(0, 0.1)
    module_list[-1].bias.data.zero_()

def train_nnet(nnet: nn.Module, states_nnet: np.ndarray, outputs: np.ndarray, batch_size: int = 100, num_itrs: int = 10000, train_itr: int = 10000, device: torch.device, lr=0.01, lr_d=1):
    nnet.train()

    criterion = nn.MSELoss()
    optimizer = optim.Adam(nnet.parameters(), lr=lr)

    while train_itr < num_itrs:
        optimizer.zero_grad()

        lr_itr = lr + (lr_d ** train_itr)
        for param_group in optimizer.param_groups:
            param_group['lr'] = lr_itr

        data = pickle.load(open("data/data.pkl", "rb"))
        nnet_inputs_np, nnet_targets_np = data
        nnet_inputs_np = nnet_inputs_np.astype(np.float32)

        nnet_inputs = torch.tensor(nnet_inputs_np, device=device)
        nnet_targets = torch.tensor(nnet_targets_np, device=device)

        nnet_inputs = nnet_inputs.float()
        nnet_outputs = nnet(nnet_inputs)

        loss = criterion(nnet_outputs, nnet_targets)
        loss.backward()
        optimizer.step()


title: How to create a data preprocessing pipeline in pytorch outside the Dataloader class?

body:
I am trying to make a model for data with 40 features which have to classified into 10 classes. I am new to PyTorch and this is my first project in it.
I am given a custom Dataset class (which I am not allowed to change) which is as follows:
class MyData(Dataset):
    def _init_(self, mode):
        with open(mode+'.pkl', 'rb') as handle:
            data = pickle.load(handle)
            self.X = data['x'].astype('float')
            self.y = data['y'].astype('long')

    def _len_(self):
        return len(self.X)

    def _getitem_(self, idx):
        if torch.is_tensor(idx):
            idx = idx.tolist()

        sample = (self.X[idx], self.y[idx])

        return sample

I have done some preprocessing on the data like normalization and then trained and saved the model. As I wasn't allowed to change the dataset class, I made the changes outside of it and then used the DataLoader method. The preprocessing is as follows :
train_data=MyData("train")
features, labels = train_data[:]

df = pd.DataFrame(features)

x = df.values
min_max_scaler = preprocessing.MinMaxScaler()
x_scaled = min_max_scaler.fit_transform(x)

input_array = x_scaled
output_array = labels

inputs = torch.Tensor(input_array)
targets = torch.Tensor(output_array).type(torch.LongTensor)
dataset = TensorDataset(inputs, targets)
train_ds, val_ds = random_split(dataset, [3300, 300])

batch_size = 300
n_epochs = 200
log_interval = 10
train_losses = []
train_counter = []
test_losses = []

train_loader = DataLoader(train_ds, batch_size, shuffle=True)
val_loader = DataLoader(val_ds, batch_size)
test_counter = [i*len(train_loader.dataset) for i in range(n_epochs + 1)]

After this I define the training and testing functions ( and remove the print statements as the autograder will not be able to grade my assignment if I do so) as follows:
def train(epoch):
  model.train()
  for batch_idx, (data, target) in enumerate(train_loader):
    optimizer.zero_grad()
    output = model(data.double())
    loss = criterion(output, target)
    loss.backward()
    optimizer.step()
    if batch_idx % log_interval == 0:
      train_losses.append(loss.item())
      train_counter.append(
        (batch_idx*32) + ((epoch-1)*len(train_loader.dataset)))
    save_model(model)

def test():
  model.eval()
  test_loss = 0
  correct = 0
  with torch.no_grad():
    for data, target in val_loader:
      output = model(data.double())
      test_loss += criterion(output, target).item()
      pred = output.data.max(1, keepdim=True)[1]
      correct += pred.eq(target.data.view_as(pred)).sum()
  test_loss /= len(val_loader.dataset)
  test_losses.append(test_loss)

test()
for epoch in range(1, n_epochs + 1):
  train(epoch)
  test()

Even after doing that, the autograder is still not able to grade my code. I mainly think it's because maybe I am making an error with how I input the data to the model but I am not able to narrow down to what exactly is the problem and how do I correct it. As I'm new to pytorch, I was looking at how to do the preprocessing but all of them involved the Dataset Class so I'm not sure how to go about it.
My model is as follows:
class MyModel(nn.Module):
    def __init__(self):
        super(MyModel, self).__init__()
        #self.flatten=nn.Flatten()

        self.net_stack=nn.Sequential(
            nn.Conv1d(in_channels=40, out_channels=256, kernel_size=1, stride=2), #applying batch norm
            nn.ReLU(),
            nn.MaxPool1d(kernel_size=1),
            nn.Dropout(p=0.1),
            nn.BatchNorm1d(256, affine=True),
            nn.Conv1d(in_channels=256, out_channels=128, kernel_size=1, stride=2), #applying batch norm
            nn.ReLU(),
            nn.MaxPool1d(kernel_size=1),
            nn.Dropout(p=0.1),
            nn.BatchNorm1d(128, affine=True),
            nn.Conv1d(in_channels=128, out_channels=64, kernel_size=1, stride=2), #applying batch norm
            nn.ReLU(),
            nn.MaxPool1d(kernel_size=1),
            nn.Dropout(p=0.1),
            nn.BatchNorm1d(64, affine=True),
            nn.Conv1d(in_channels=64, out_channels=32, kernel_size=1, stride=2), #applying batch norm
            nn.ReLU(),
            nn.MaxPool1d(kernel_size=1),
            nn.Dropout(p=0.1),
            nn.BatchNorm1d(32, affine=True),
            nn.Flatten(),
            nn.Linear(32, 10),
            nn.Softmax(dim=1)).double()

    def forward(self,x):
        # result=self.net_stack(x[None])
        x=x.double()
        result=self.net_stack(x[:, :, None]).double()
        print(result.size())
        return result

One instruction I've got is that they've written:
# Please make sure we can load your model with:
# model = MyModel()
# This means you must give default values to all parameters you may wish to set, such as output size.


title: How to calculate mean on second axis until row-specific column in Pytorch?

body:
I am looking for a fast way to calculate the mean for each row of a 2d matrix but only until a specific column. The remaining values of each row can be ignored. The column is different for each row.
In Numpy, it could be coded like this. However, I am hoping to find a solution without a for loop which also does not break the gradients.
import numpy as np

arr = np.linspace(0, 10, 15).reshape(3,5)
cols = [2,0,4]

for row, col in enumerate(cols):
    arr[row, col+1:] = np.nan

result = np.nanmean(arr, axis=1)

Any suggestions?
Edit: Best solution I have found so far:
result = torch.stack([arr[i, 0:cols[i]+1].mean() for i in range(len(arr))])

But I would still like to avoid the for loop.


title: reshape tensor in a determined way

body:
I have some tensor x3. I got it in the following way:
x = torch.tensor([0, 0, 0, 0, 1, 0, 0, 0, 0])
x2 = torch.stack(5 * [x], 0)
x2 = x2.reshape(-1)
x3 = torch.stack(4 * [x2], 0)
x3 = torch.stack(6 * [x3], -1)
x3 = torch.stack(7 * [x3], -1)

In short it means that
x[0, :9, 0, 0] = [0, 0, 0, 0, 1, 0, 0, 0, 0]
x[0, 9:18, 0, 0] = [0, 0, 0, 0, 1, 0, 0, 0, 0]

and so on.
Then I want to reshape it that every nine values of the 1st dimension go to the new dimension. In other words, I want x3[0, 0, 0, 0, :] to give me tensor([0, 0, 0, 0, 1, 0, 0, 0, 0])
I tried to do:
x3.reshape(4, 5, 6, 7, 9)[0, 0, 0, 0, :]
tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])
x3.reshape(4, 9, 6, 7, 5).transpose(1, -1)[0, 0, 0, 0, :]
tensor([0, 0, 0, 0, 0, 0, 0, 0, 1])

As you see, none of it gives me the right answer
UPD: added x3 = torch.stack(7 * [x3], -1)


title: loss defined on repeated evaluation of the same model

body:
I have a model denoted by f(),
Suppose the target is t, f(x1) = y1 and f(x2) = y2 and my loss is defined as
loss = mse(y1,y2) + mse(y2,t)
Since both y1 and y2 reguires grad, I have received error such as

one of the variables needed for gradient computation has been modified by an inplace operation

My understanding is that, suppose I evaluate y1 first, the graph has been changed upon my evaluation of y2. Should I fix some tensor such as ,e.g., y1_no_grad = y1.detach().numpy() and then use
loss = mse(y1_no_grad,y2) + mse(y2,t)?
However, I still receive error Can't call numpy() on Tensor that requires grad. Use tensor.detach().numpy(), which I am not sure if it is because y1_no_grad is a numpy array while y2 is a tensor.
Update:
I realized my problem afterwards. It was due to that I created multiple loss tensors and that I backwarded one loss tensor first which changed the parameters in-place. This caused error when I wanted to backward another loss tensor.
E.g.
f(x1) = y1
f(x2) = y2
f(x3) = y3
...
f(xn) = yn

f(x) = y

for i in range(n):
    optimizer.zero_grad()
    loss = mse(y,yi) + mse(yi,t)
    loss.backward()
    optimizer.step()

To me the solutions are either:
1.accumulate the loss tensors before doing backward , i.e.
for i in range(n):
    loss = mse(y,yi) + mse(yi,t)
    loss.backward()
optimizer.step()

2.Evaluate again before each backward, i.e.:
for i in range(n):
    optimizer.zero_grad()
    y = f(x)
    yi = f(xi)
    loss = mse(y,yi) + mse(yi,t)
    loss.backward()
    optimizer.step()


title: Using Pytorch to adjust Highlights and Shadows

body:
Is there a way to adjust the image's highlights and shadows like in photoshop?

I have used torchvision.transforms.functional to adjust brightness and contrast. I want to know if they're a similar way to adjust highlights and shadows as well.


title: Backward Pass of Gradient Penalty Loss in WGANGP

body:
The loss term for the critic in WGANGP is:

L = D(x) - D(G(z)) + λ * ( norm(gradient(x')) -1) ^2

where
x = real image,
G(z) = generated image &
x' = ε * real_image  + (1 - ε) * generated image
My code in Pytorch:
def get_gradient(crit, real, fake, epsilon):
        '''
        Return the gradient of the critic's scores with respect to mixes of real and fake images.
        Parameters:
            crit: the critic model
            real: a batch of real images
            fake: a batch of fake images
            epsilon: a vector of the uniformly random proportions of real/fake per mixed image
        Returns:
            gradient: the gradient of the critic's scores, with respect to the mixed image
        '''
        # Mix the images together
        mixed_images = real * epsilon + fake * (1 - epsilon)

        # Calculate the critic's scores on the mixed images
        mixed_scores = crit(mixed_images)

        # Take the gradient of the scores with respect to the images
        gradient = torch.autograd.grad(
            inputs=mixed_images,
            outputs=mixed_scores,
            grad_outputs=torch.ones_like(mixed_scores),
            create_graph=True,
            retain_graph=True,
        )[0]
        return gradient

    def gradient_penalty(gradient):
            '''
            Return the gradient penalty, given a gradient.
            Given a batch of image gradients,calculate the magnitude of each image's gradient
            and penalize the mean quadratic distance of each magnitude to 1.
            Parameters:
            gradient: the gradient of the critic's scores, with respect to the mixed image
            Returns:
            penalty: the gradient penalty
            '''
            # Flatten the gradients so that each row captures one image
            gradient = gradient.view(len(gradient), -1)

            # Calculate the magnitude of every row
            gradient_norm = gradient.norm(2, dim=1)

            # Penalize the mean squared distance of the gradient norms from 1
            penalty = torch.mean((gradient_norm - 1)**2)
            return penalty

        #this is how critic will be updated in each epoch
        crit_opt.zero_grad()
        fake_noise = get_noise(cur_batch_size, z_dim, device=device)
        fake = gen(fake_noise)
        crit_fake_pred = crit(fake.detach())
        crit_real_pred = crit(real)

        epsilon = torch.rand(len(real), 1, 1, 1, device=device, requires_grad=True)
        gradient = get_gradient(crit, real, fake.detach(), epsilon)
        gp = gradient_penalty(gradient)
        crit_loss = get_crit_loss(crit_fake_pred, crit_real_pred, gp, c_lambda)

        # Keep track of the average critic loss in this batch
        mean_iteration_critic_loss += crit_loss.item() / crit_repeats
        # Update gradients
        crit_loss.backward(retain_graph=True)
        # Update optimizer
        crit_opt.step()
        critic_losses += [mean_iteration_critic_loss]

I'm trying to understand what Pytorch computes in the backward pass for the Gradient Penalty term. Since the loss term contains norm(gradient(x')), how is the double gradient computed in the backward pass & how is it used to calculate the gradient of the critic's neural network & epsilon?


title: PyTorch loads old data when using tensorboard

body: In using tensorboard I have cleared my data directory and trained a new model but I am seeing images from an old model. Why is tensorboard loading old data, where is it being stored, and how do I remove it?


title: Indexing whole Tensor along specific dimension and specific channels

body:
Let say we have a Tensor A with the dimension dim(A)=[i, j, k=6, u, v]. Now we are interested to get the whole tensor at dimension k with channels=[0:3]. I know we can get it this way:
B = A[:, :, 0:3, :, :]

Now I would like to know if there is any better "pythonic" way to achieve the same result without doing this suboptimal indexing. I mean something like.
B = subset(A, dim=2, index=[0, 1, 2])

No matter in which framework, i.e. pytorch, tensorflow, numpy, etc.
Thanks a lot


title: Pytorch install fails on conda with error as -CondaHTTPError: HTTP 403 FORBIDDEN for url

body:
I have installed anaconda 2019 version from repository as the latest version was throwing error on windows 10.
After installing anaconda(conda 4.10.1), i am unable to install Pytorch using command 'conda install pytorch-cpu -c pytorch' on anaconda prompt. It throws below  error. I believe it is trying to look for file 'win-64/pytorch-cpu-1.1.0-py3.7_cpu_1.tar.bz2' and strangely when i check on https://anaconda.org/pytorch/pytorch/files i can't locate this file. Please help.
Error message :-
CondaHTTPError: HTTP 403 FORBIDDEN for url https://conda.anaconda.org/pytorch/win-64/pytorch-cpu-1.1.0-py3.7_cpu_1.tar.bz2
Elapsed: 01:00.706682
An HTTP error occurred when trying to retrieve this URL.
HTTP errors are often intermittent, and a simple retry will get you on your way.


title: Predict multiple pictures and save prediction in array

body:
I'm currently looking for a way to predict multiple picture and save the predicts in an array. My current method is now save all image tensor in an array, iterate through the array and predict every images and save the inputs in array. Is there any way more efficient?
My code for 1 picture prediction
def predict_image(path):
    print("Prediction in progress")
    image = Image.open(path)

    transformation = transforms.Compose([
        transforms.CenterCrop(224),
        transforms.ToTensor(),
        transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
    ])

    img_tensor = transformation(image).float()
    img_tensor = img_tensor.unsqueeze_(0)

    if torch.cuda.is_available():
        img_tensor.cuda()

    input = Variable(img_tensor)
    output = model(input)
    index = output.data.numpy().argmax()
    return index


title: How to append values to a tensor in a Keras Deep Network

body:
I'm more familiar with Pytorch, however, a current project requires Keras.  My base model is Densenet121.  I have attached a Global Average Pooling layer and a softmax layer for the output.
What I want to do is attach the outputs from another Network to the GlobalAveragePooling2D() of the current batch while training, as a little experiment.  In Pytorch, I would just append the values in the Forward function, however being new to Keras, I'm a bit stuck.  Below is how I basically set up the model.  Any help would be appreciated.
x = base_model.output
x_pool = GlobalAveragePooling2D()(x)
predictions = Dense(1, activation="sigmoid")(x_pool)
model = Model(inputs=base_model.input, outputs=predictions)
model.compile(optimizer='adam',
              loss='categorical_crossentropy')
history = model.fit_generator(train_generator,
                              validation_data=valid_generator,
                              steps_per_epoch=100,
                              validation_steps=25,
                              epochs = 3)

In Pytorch, I would do something like this (this is very rough but hopefully you get the idea).
def forward(self, x, value):
     .
     .
     .
     x = self.conv_layer(x)
     x = self.global_pool(x)
     x = torch.cat((x, value), dim = 3) #tensor is of shape (batch_size, 1, x.shape[2]+1)
     x = x.view(-1, 1)
     x = self.sigmoid(x)
     return x


title: How to add training end callback to AllenNLP config file?

body:
Currently training models using AllenNLP 1.2 and the commands api:
allennlp train -f --include-package custom-exp /usr/training_config/mock_model_config.jsonnet -s test-mock-out
I'm trying to execute a forward pass on a test dataset after training is completed. I know how to add an epoch_callback, but am not sure about the syntax for the end_callback.
In my config.json, I have the following:
{
...
"trainer": {
...
"epoch_callbacks": [{"type": 'log_metrics_to_wandb',},]
}
...
}

I've tried:
              "end_callback": [{"type": 'my_custom_function',},]

but got an illegal argument error. Also, I am not sure how I would accurately specify the exact custom function and communicate it to the trainer.


title: Pytorch: forward() takes 1 positional argument but two were given

body:
This is my 3rd post, please be lenient with me.
I have created some PyTorch code to classify cats and dog from a .csv file. That part works. Getting to the meat of the matter, when running the model as a test for further development, conda spat out a error as followed:
Traceback (most recent call last):
  File "c:\Users\bala006\OneDrive - St John's Anglican College\Desktop\Personal\Dorime\CatsDogs\cats_dogs.py", line 70, in <module>
    run(loss_func, dataloader, model)
  File "c:\Users\bala006\OneDrive - St John's Anglican College\Desktop\Personal\Dorime\CatsDogs\cats_dogs.py", line 55, in run
    Pred = model(imagepre)
  File "C:\Users\bala006\Miniconda3.8\lib\site-packages\torch\nn\modules\module.py", line 889, in _call_impl
    result = self.forward(*input, **kwargs)
TypeError: forward() takes 1 positional argument but 2 were given

following that, the code is here.
class NN (nn.Module):
    def __init__(self, input_size, output_size):
        super().__init__()
        self.flatten = nn.Flatten()
        self.stack = nn.Sequential(
            nn.Linear(input_size, 300),
            nn.ReLU(),
            nn.Linear(300, 300),
            nn.ReLU(),
            nn.Linear(300, 256),
            nn.ReLU(),
            nn.Linear(256, 256),
            nn.ReLU(),
            nn.Linear(256, 150),
            nn.ReLU(),
            nn.Linear(150,100),
            nn.ReLU(),
            nn.Linear(100,75),
            nn.ReLU(),
            nn.Linear(75, output_size),
            nn.ReLU()
        )
    def forward (x):
        x = self.flatten(x)
        logits = self.stack(x)
        return logits

model = NN(256,2)
y_label, imagepre = dataset.__getitem__(1)
print (y_label)
print (imagepre)

loss_func = nn.CrossEntropyLoss()
optimiser = optim.SGD(model.parameters(), lr = 0.001)
epochs = 2

def run (loss_fn, dataloader, model):
    size = dataset.__len__()
    for y_label in enumerate(dataloader):
        Pred = model(imagepre)
        print (Pred)

        # Loss
        loss = loss_fn(pred, y_label)

        # Backprop
        optimiser.zero_grad()
        loss.backward()
        optimiser.step()

        print (loss)

for t in range(epochs):
    print ("Epoch #######################")
    run(loss_func, dataloader, model)

print ("Done")

Upon reflection, there seems to be some errors but the python error message for this one isn't too forgiving. Also, I just started learning PyTorch and and starting to understand what's happening. A side question that doesn't have to be answered, is this even succinct/efficient code to begin with. I know the model structure is a little Lengthy but without the function, the code seems to run fine. Please help me - a newbie to this exciting field.
P.S. Sorry for any confusion and thanks for any help!


title: How to add cosine similarity as score function in sentence similarity using BERT Transformers

body:
I've been working on a project where I want to calculate the similarity between 2 sentences as input to my model (using BERT by HuggingFace Transformers library and Qoura sentence pair dataset from kaggle). I was trying to use my scoring function as torch cosine similarity but for every input I get a value around 0.98, but as far as cosine similarity output goes, it values comes between -1 to 1. I have included the required code below :
class SentencePairSimilarityBert(nn.Module):
    def __init__(self):
        super(SentencePairSimilarityBert, self).__init__()
        self.bert1 = transformers.BertModel.from_pretrained('bert-base-uncased')
        self.bert2 = transformers.BertModel.from_pretrained('bert-base-uncased')
        self.cosine_similarity = nn.CosineSimilarity(dim=1, eps=1e-6)

    def forward(self, q1_input_ids, q1_attention_mask, q2_input_ids, q2_attention_mask):
        _, q1_pooled_output = self.bert1(input_ids=q1_input_ids, attention_mask=q1_attention_mask, return_dict=False)
        _, q2_pooled_output = self.bert2(input_ids=q2_input_ids, attention_mask=q2_attention_mask, return_dict=False)
        cosine_distance = self.cosine_similarity(q1_pooled_output, q2_pooled_output)
        return cosine_distance

I'm using loss function as Mean Squared Error loss (MSELoss) because I want to calculate how far I am from my target/real value.
The cosine_distance for a batch of 4 comes out to be : [0.982, 0.976, 0.974, 0.948] and targets are : [0, 0, 1, 0]
0 means both sentence are dissimilar and 1 means they are similar.
I'm trying the following training approach but I don't think so I'm going correct because the cosine_distance should be between -1 to 1 and my targets are 0 and 1, so how would I map my targets to this range of cosine_distance in order to calculate the loss and back-propagate.
loss_function = nn.MSELoss()
This is my train function below :
model = model.train()

losses = []

for dictionary in data_loader:
    q1_input_ids = dictionary['q1_input_ids'].to(device)
    q1_attention_mask = dictionary['q1_attention_mask'].to(device)
    q2_input_ids = dictionary['q2_input_ids'].to(device)
    q2_attention_mask = dictionary['q2_attention_mask'].to(device)
    targets = dictionary['targets'].to(device)

results = model(
     q1_input_ids=q1_input_ids,
     q1_attention_mask=q1_attention_mask,
     q2_input_ids=q2_input_ids,
     q2_attention_mask=q2_attention_mask,
)

loss = loss_function(results, targets)
losses.append(loss.item())
loss.backward()
optimizer.step()
scheduler.step()
optimizer.zero_grad()

Any help would be appreciated.
Thanks in advance !


title: TypeError when using torch.autograd.profiler.profile

body:
I am trying to analyze memory consumption for my model as described here:
https://pytorch.org/tutorials/recipes/recipes/profiler_recipe.html
using these lines:
with profiler.profile(profile_memory=True, record_shapes=True) as prof:
    tubes, _, _ = zip(*model(imgs, img_metas, return_loss=False))
print(prof.key_averages().table(sort_by="self_cpu_memory_usage", row_limit=10))

but I get this error

TypeError: init() got an unexpected keyword argument
'profile_memory'

So, what would be the reason and/or the solution for this error?


title: Converted tflite model not working with tflite_runtime

body:
I have a simple pytorch model which I transformed into ONNX and eventually to tflite.
When I load the model and do inference with TF.lite, all goes well.
However when I try using tflite_runtime to load the model and do inference, I get the following error:
RunTimeError: external/org_tensorflow/tensorflow/lite/kernels/add.cc:385 Type INT64 is unsupported by op Add.Node number 70 (ADD) failed to invoke.

Here is the conversion code I'm currently using with TF2.6:
converter = tf.lite.TFLiteConverter.from_saved_model(path)
converter.optimizations = [tf.lite.Optimize.DEFAULT]
converter.target_spec.supported_ops = [tf.lite.OpsSet.TFLITE_BUILTINS]
converter.allow_custom_ops=True  # if omitted, conversion fails
tflite_rep = converter.convert()
open('exports/deep_snore.tflite', 'wb').write(tflite_rep)

I have checked many possible tf blogs but I can't figure out where the issue is.
The solution I can think of is to rewrite the model with TF, retrain it and transform it to tflite.


title: How you do efficiently parameterize a batch of covariance matrices in PyTorch

body:
I want to output a batch of multivaraible Gaussians (For likelihood based learning). But I don't know how I can do that. I have something like this
.
.
.
covar=self.covar(o_t) #returns batch_size*(obs_dim*obs_dim)
mean=self.mean(o_t)  #batch_size*obs_dim

Now I would like to create batch_size number of multivaraible Gaussians, for that I also need to make the covariance function positive definite. For N*N Tensors I know torch.matmul(a,a.T) would work, but I don't know how to do it here.
Alternatively I know I can parameterize using scale_tril, but I have the same problem, how do you ensure it's lower triangular + positive diagonals. I can imagine using some complicated masking, and then adding up. But I am sure there's an easier way.
Again for context, what I want to do from here, is compute the likelihood, using observed data, and learn using a version of sgd/adam.
Thanks
Edits : cov=((torch.tril(torch.ones(self.obs,self.obs),diagonal=-1)*sigma)+torch.diag_embed(F.softplus(diag))).to(device)
seems to work where diag is batch_sizeobs_dim diagonals, and this gives a batch of lower triangular matrices with postive diagonals, and sigma is of shape batch_sizeobs_dim*obs_dim


title: How does "*" work in these lines of python pytorch code?

body:
Recently I am reading some pytorch code as below. I want to know how does * work in these lines of code? What does it mean here?
def transpose_for_scores(self, x):
        new_x_shape = x.size()[:-1] + (self.num_attention_heads, self.attention_head_size)
        x = x.view(*new_x_shape)
        return x.permute(0, 2, 1, 3)


title: PyTorch Boolean - Stop Backpropagation?

body:
I need to create a Neural Network where I use binary gates to zero-out certain tensors, which are the output of disabled circuits.
To improve runtime speed, I was looking forward to use torch.bool binary gates to stop backpropagation along disabled circuits in the network. However, I created a small experiment using the official PyTorch example for the CIFAR-10 dataset, and the runtime speed is exactly the same for any values for gate_A and gate_B: (this means that the idea is not working)
class Net(nn.Module):
    def __init__(self):
        super().__init__()
        self.pool = nn.MaxPool2d(2, 2)
        self.conv1a = nn.Conv2d(3, 6, 5)
        self.conv2a = nn.Conv2d(6, 16, 5)
        self.conv1b = nn.Conv2d(3, 6, 5)
        self.conv2b = nn.Conv2d(6, 16, 5)
        self.fc1 = nn.Linear(32 * 5 * 5, 120)
        self.fc2 = nn.Linear(120, 84)
        self.fc3 = nn.Linear(84, 10)

    def forward(self, x):
        # Only one gate is supposed to be enabled at random
        # However, for the experiment, I fixed the values to [1,0] and [1,1]
        choice  =  randint(0,1)
        gate_A  =  torch.tensor(choice   ,dtype = torch.bool)
        gate_B  =  torch.tensor(1-choice ,dtype = torch.bool)

        a = self.pool(F.relu(self.conv1a(x)))
        a = self.pool(F.relu(self.conv2a(a)))

        b = self.pool(F.relu(self.conv1b(x)))
        b = self.pool(F.relu(self.conv2b(b)))

        a *= gate_A
        b *= gate_B
        x  = torch.cat( [a,b], dim = 1 )

        x = torch.flatten(x, 1) # flatten all dimensions except batch
        x = F.relu(self.fc1(x))
        x = F.relu(self.fc2(x))
        x = self.fc3(x)
        return x

How can i define gate_A and gate_B in such a way that backpropagation effectively stops if they are zero?
PS. Changing concatenation dynamically at runtime would also change which weights are assigned to every module. (for example, the weights associated to a could be assigned to b in another pass, disrupting how the network operates).


title: How to Implement Backward (Top Down) Connections

body:
Artificial neural nets often use residual (or skip) connections. They look something like this:
l1 = layer1(input)
l2 = layer2(l1)
l3 = layer3(l2) + l1

Is there a way to reverse this? Here's what I mean:
l1 = layer1(input) + l3
l2 = layer2(l1)
l3 = layer3(l2)

Biological neural nets have such feedback loops. It would be interesting to implement something like this and test it to see how it performs. Any help would be greatly appreciated.


title: train() takes 2 positional arguments but 4 were given - Pytorch

body:
for epoch in range(epochs):
    train_loss = Engine.train(
        train_loader,
        model,
        optimizer,
        device

    )

This is where I get the error:  train() takes 2 positional arguments but 4 were given
And this is the func:
class Engine:
    @staticmethod
    def train(
       data_loader,
       model,
       optimizer,
       device,
       scheduler=None,
       accumulation_steps=1,
       use_tpu=False,
       fp16=False,
    )

Please help, thanks


title: RuntimeError: CUDA out of memory with pre-trained model

body:
I am using a pre-trained model for image improvement.
[https://github.com/swz30/MIRNet.][1]
I created a demo.py (code below) file in order to test my set of images for the pre-trained template provided. With my first set of images, all of them with very high resolutions I always got the same error:
RuntimeError: CUDA out of memory. Tried to allocate 5.38 GiB (GPU 0; 3.95 GiB total capacity; 379.90 MiB already allocated; 2.89 GiB free; 16.10 MiB cached)

When I tested for only one image with a lower resolution, the error persisted but in a strange way:
RuntimeError: CUDA out of memory. Tried to allocate 1014.00 MiB (GPU 0; 3.95 GiB total capacity; 2.61 GiB already allocated; 527.44 MiB free; 23.25 MiB cached)

I made the necessary changes to the demo.py file present in the other repository in order to test MIRNet on my image set. During the process I had to make some configurations related to graphics compatibility, but everything was resolved.
Do you have any suggestions to solve my problem? I am using the pre-trained model provided, in a linux environment, with all the correct specifications with anaconda and graphics -> NVIDIA GEFORCE GTX 960m 4gb
import torch
import torch.nn as nn
import torch.nn.functional as F
import torchvision.transforms.functional as TF
from PIL import Image
import os
from runpy import run_path
from skimage import img_as_ubyte
from collections import OrderedDict
from natsort import natsorted
from glob import glob
import cv2
import argparse

parser = argparse.ArgumentParser(description='Demo MIRNet')
parser.add_argument('--input_dir', default='./samples/', type=str, help='Input images')
parser.add_argument('--result_dir', default='./samples/output/', type=str, help='Directory for results')
parser.add_argument('--task', required=True, type=str, help='Task to run',
                    choices=['fivek', 'Denoising', 'SR_x3'])

args = parser.parse_args()

def save_img(filepath, img):
    cv2.imwrite(filepath, cv2.cvtColor(img, cv2.COLOR_RGB2BGR))

def load_checkpoint(model, weights):
    checkpoint = torch.load(weights)
    try:
        model.load_state_dict(checkpoint["state_dict"])
    except:
        state_dict = checkpoint["state_dict"]
        new_state_dict = OrderedDict()
        for k, v in state_dict.items():
            name = k[7:]  # remove `module.`
            new_state_dict[name] = v
        model.load_state_dict(new_state_dict)

task = args.task
inp_dir = args.input_dir
out_dir = args.result_dir

os.makedirs(out_dir, exist_ok=True)

files = natsorted(glob(os.path.join(inp_dir, '*.jpg'))
                  + glob(os.path.join(inp_dir, '*.JPG'))
                  + glob(os.path.join(inp_dir, '*.png'))
                  + glob(os.path.join(inp_dir, '*.PNG')))

if len(files) == 0:
    raise Exception(f"No files found at {inp_dir}")

# Load corresponding model architecture and weights
load_file = run_path(os.path.join("networks", "MIRNet_model.py"))
model = load_file['MIRNet']()
model.cuda()

weights = os.path.join("pretrained_models/denoising", "model_" + task.lower() + ".pth")
load_checkpoint(model, weights)
model.eval()

img_multiple_of = 8

for file_ in files:
    img = Image.open(file_).convert('RGB')
    input_ = TF.to_tensor(img).unsqueeze(0).cuda()

    # Pad the input if not_multiple_of 8
    h, w = input_.shape[2], input_.shape[3]
    H, W = ((h + img_multiple_of) // img_multiple_of) * img_multiple_of, (
                (w + img_multiple_of) // img_multiple_of) * img_multiple_of
    padh = H - h if h % img_multiple_of != 0 else 0
    padw = W - w if w % img_multiple_of != 0 else 0
    input_ = F.pad(input_, (0, padw, 0, padh), 'reflect')

    with torch.no_grad():
        restored = model(input_)
    restored = restored[0]
    restored = torch.clamp(restored, 0, 1)

    # Unpad the output
    restored = restored[:, :, :h, :w]

    restored = restored.permute(0, 2, 3, 1).cpu().detach().numpy()
    restored = img_as_ubyte(restored[0])

    f = os.path.splitext(os.path.split(file_)[-1])[0]
    save_img((os.path.join(out_dir, f + '.png')), restored)

print(f"Files saved at {out_dir}")

  [1]: https://github.com/swz30/MIRNet.


title: Learnable scalar weight in PyTorch

body:
I have two neural networks running in parallel. Each gives a features map of same size say Nx1. Now I want weighted average of these embedding like this w1 * embed1 + w2 * embed2. I have tried these 1 2.But the weights are not updating. Any help would be appreciated. Here is how I am trying to do it:
class LinearWeightedAvg(nn.Module):
      def __init__(self, n_inputs):
        super(LinearWeightedAvg, self).__init__()
        self.weight1 = Variable(torch.randn(1), requires_grad=True).cuda()
        self.weight2 = Variable(torch.randn(1), requires_grad=True).cuda()

    def forward(self, inp_embed):
        return self.weight1 * inp_embed[0] + self.weight2 * inp_embed[1]

class EmbedBranch(nn.Module):
    def __init__(self, feat_dim, embedding_dim):
        super(EmbedBranch, self).__init__()
        fc_layer1 = fc_layer
    def forward(self, x):
        x = self.fc_layer1(x)
        return x

class EmbeddingNetwork(nn.Module):
    def __init__(self, args, N):
        super(EmbeddingNetwork, self).__init__()
        embedding_dim = N

        self.embed1 = EmbedBranch(N, N)
        self.embed2 = EmbedBranch(N, N)
        self.comb_branch = LinearWeightedAvg(metric_dim)

        self.args = args
        if args.cuda:
            self.cuda()

    def forward(self, emb1, emb2):
        embeds1 = self.text_branch(emb1)
        embeds2 = self.image_branch(emb2)
        combined = self.comb_branch([embeds1, embeds2])
        return combined

    def train_forward(self, embed1, embed2):

        combined = self(embed1, embed2)

embeds = model.train_forward(embed1, embed2)
loss = loss_func(embeds, labels)
running_loss.update(loss.data.item())
optimizer.zero_grad()
loss.backward()

Also I want the weight to be within 0-1 range.
Thanks,


title: Pytorch Tensor::data_ptr() not working on Linux

body:
I cannot link my program to pytorch under Linux, get the following error:
/tmp/ccbgkLx2.o: In function `long long* at::Tensor::data<long long>() const':
test.cpp:(.text._ZNK2at6Tensor4dataIxEEPT_v[_ZNK2at6Tensor4dataIxEEPT_v]+0x14): undefined reference to `long long* at::Tensor::data_ptr<long long>() const'

I am building a very simple minimal example:
#include "torch/script.h"
#include <iostream>

int main() {
    auto options = torch::TensorOptions().dtype(torch::kInt64);
    torch::NoGradGuard no_grad;
    auto T = torch::zeros(20, options).view({ 10, 2 });
    long long *data = (long long *)T.data<long long>();
    data[0] = 1;
    return 0;
}

The command used to build it:
g++ -w -std=c++17 -o test-torch test.cpp -D_GLIBCXX_USE_CXX11_ABI=1 -Wl,--whole-archive -ldl -lpthread -Wl,--no-whole-archive -I../libtorch/include -L../libtorch/lib -ltorch -ltorch_cpu -lc10 -Wl,-rpath,../libtorch/lib

Pytorch has been downloaded from the link https://download.pytorch.org/libtorch/cpu/libtorch-cxx11-abi-shared-with-deps-1.7.0%2Bcpu.zip and unzipped (so I have the libtorch folder next to the folder with test.cpp).
Any ideas how to solve this problem? Same program works just fine under Visual C++.
P.S. I know pytorch is kind of designed for cmake, but I have zero experience with cmake and no desire to write a cmake-based build system for my app. Also, the examples they give are seemingly supposed to only work if pytorch is "installed" in the system. So I cannot just download the .zip with libs? And if I "install" it (e.g. from sources or in whatever other way) on an AVX512 system, will the binary I link to it and distribute to end-users work on non-AVX512? The documentation is completely incomprehensible for newbies.
UPDATE: I tried to do this via CMake following the tutorial https://pytorch.org/cppdocs/installing.html and got exactly the same error. Specifically, I renamed my directory to example-app and the source file to example-app.cpp. Then I created CMakeLists.txt in this directory with the following contents:
cmake_minimum_required(VERSION 3.0 FATAL_ERROR)
project(example-app)

find_package(Torch REQUIRED)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${TORCH_CXX_FLAGS}")

add_executable(example-app example-app.cpp)
target_link_libraries(example-app "${TORCH_LIBRARIES}")
set_property(TARGET example-app PROPERTY CXX_STANDARD 14)

Then
mkdir build
cd build
cmake -DCMAKE_PREFIX_PATH=../../libtorch ..
cmake --build . --config Release

And here's the output:
CMakeFiles/example-app.dir/example-app.cpp.o: In function `long long* at::Tensor::data<long long>() const':
example-app.cpp:(.text._ZNK2at6Tensor4dataIxEEPT_v[_ZNK2at6Tensor4dataIxEEPT_v]+0x14): undefined reference to `long long* at::Tensor::data_ptr<long long>() const'

Makes me think, maybe I forgot to include some header or define some variable?
Oh, this is all on Mint 19.2 (equivalent to Ubuntu 18.04), g++ version is 7.5.0, glibc is 2.27. Compiling with g++-8 gives the same result.


title: Pytorch: File-specific action for each image in the batch

body:
I have a dataset of images each of them having an additional attribute "channel_no". Each image should be processed with the nn layer according to its channel_no:
 images with channel_no=1 have to be processed with layer1
 images with channel_no=2 have to be processed with layer2
 images with channel_no=3 have to be processed with layer3
etc...

The problem is that when the batch contains more than one image, forward() function gets a torch tensor with the batch of images as input, and each of the images has different channel_no. So it is not clear how to process each image separately.
Here is the code for the case when the batch has 1 image only:
class Net(nn.Module):
    def __init__ (self, weight):
        super(Net, self).__init__()

        self.layer1 = nn.Linear(hidden_sizes[0], hidden_sizes[1])
        self.layer2 = nn.Linear(hidden_sizes[0], hidden_sizes[1])
        self.layer3 = nn.Linear(hidden_sizes[0], hidden_sizes[1])

        self.outp = nn.Linear(hidden_sizes[1], output_size)

    def forward(self, x, channel_no):
        channel_no = channel_no[0] #extract channel_no from the batch list

        x = x.view(-1,hidden_sizes[0])

        if channel_no == 1: x = F.relu(self.layer1(x))
        if channel_no == 2: x = F.relu(self.layer2(x))
        if channel_no == 3: x = F.relu(self.layer3(x))

        x = torch.sigmoid(self.outp(x))

        return x

Is it possible to process each image separately using batch size > 1 ?


title: How to train a neural network to predict ranking of output?

body:
I currently have a question about training a neural network which cares about the ranking of its outputs not the value of its outputs.
At first, I use the nn.MSELoss() between the output values and the target values to train my network. The only thing I care about is the correlation between outputs and targets. However, I found that the output distribution of network is different from the distribution of targets, which is shown in the picture. The blue one is the distribution of outputs and the orange one is targets'.
What should I do to improve the performance of my network? Is there any loss function directly related to the correlation between targets and outputs, or the ranking of outputs?
Different Distribution between Targets and Outputs


title: Create dataset out of x_train and y_train

body:
How to put the x_train and y_train into a model for training?
The x_train is a tensor of size (3000, 13).
The y_train is of size (3000, 1)
That is for each element of x_train (1, 13), the respective y label is one digit from y_train.
if I do:
train_data = (train_feat, train_labels)
print(train_data[0].shape)
print(train_data[1].shape)

torch.Size([3082092, 13])
torch.Size([3082092, 1])
train_loader = data.DataLoader(dataset=train_data,
                                batch_size= 7,
                                shuffle=True)

The dataloader  does not return the batch size, but returns the whole dataset instead


title: Setting constraints for parameters in pytorch

body:
I am trying to set some constraints for weight parameters in PyTorch, e.g. the sum of every row of the weight matrix be exactly one for a fully connected layer:
class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        self.layer1 = nn.Linear(28*28, 10*10)
        self.layer2 = nn.Linear(10*10, 5*5)
        self.layer3 = nn.Linear(5*5, 10)

    def forward(self, x):
        x=torch.sigmoid(self.layer1(x))
        x=torch.sigmoid(self.layer2(x))
        x=torch.sigmoid(self.layer3(x))

model=Net()

The constraint for this example network would be:
torch.sum(model.linear1.weight,0)==1
torch.sum(model.linear2.weight,0)==1
torch.sum(model.linear3.weight,0)==1

A commonly used method to set a constraint, clamp, is used to set constraints for every element, but in this case, I would be setting a constraint for every row, instead of any particular element of the weight matrix. Are there any ways to implement this kind of constraint?


title: How to write a fast PyTorch training loop?

body:
I've encountered a bad regression in performance after porting a machine learning algorithm from Keras to PyTorch.
The following construction in Keras:
model.compile(loss="binary_crossentropy", optimizer=SGD(learning_rate=0.01))
.
.
.
model.fit(states, actions)

is ~15x faster than the following verbose version in Pytorch:
def train_network(model, optimizer, train_loader,
                  num_epochs=16, pbar_update_interval=200, print_logs=True):

    criterion = nn.BCELoss()
    model.train()

    for i in range(num_epochs):
        for k, batch_data in enumerate(train_loader):
            optimizer.zero_grad()

            batch_x = batch_data[:, :-1]
            batch_y = batch_data[:, -1]

            y_pred = model(batch_x)
            loss = criterion(y_pred, batch_y.unsqueeze(1))
            loss.backward()
            optimizer.step()

optimizer = torch.optim.SGD(model.parameters(), lr=LEARNING_RATE)
train_data = torch.from_numpy(np.column_stack((states, actions)))
train_data = train_data.to(torch.float)
train_loader = torch.utils.data.DataLoader(train_data, shuffle=True, batch_size=32)
train_network(model, optimizer, train_loader)

Am I writing the PyTorch code wrong?
The variable model in the PyTorch code is an nn.Module, and states and actions are NumPy arrays. The code is running on 4 CPU cores.