title: IndexError: index 39092 is out of bounds for axis 0 with size 39092; I'm trying to train a multi label classification

body:
I think the problem is caused by the way I load the data from the csv file, but I don't know how to fix it
Here is a small portion of my train csv file:
train_datasets
(I use 15 labels from column 1 to the end)
The error looks like this:
IndexError
IndexError
Code==>
import csv
import os
import pandas as pd
import numpy as np
from PIL import Image
import torch

class FashionData(torch.utils.data.Dataset):
    def __init__(self, csv_file, mode='train', transform=None):
        self.mode=mode
        #label(img1) = [0, 0, 0, 1], label(img3) = [1, 0, 1, 0], ...),
        self.transform = transform

        self.data_info = pd.read_csv(csv_file, header=None)
        #print(self.data_info)

        # First column contains the image paths
        self.image_arr = np.asarray(self.data_info.iloc[1:, 0])
        if mode !='test':
            self.label_arr = np.asarray(self.data_info.iloc[1:, 2:]) # columns 1 to N
            self.label_arr=self.label_arr.astype('float32')

        # Calculate len
        self.data_len = len(self.data_info.index)
    def __getitem__(self, index):
        # Get image name from the pandas df
        single_image_name = self.image_arr[index]
        # Open image
        img_as_img = Image.open(single_image_name)

        if self.transform is not None:
            img_as_img = self.transform(img_as_img)
        if self.mode=='test':
            return img_as_img

        # Get label(class) of the image based on the cropped pandas column
        single_image_label = self.label_arr[index]
        #single_image_label = torch.from_numpy(self.label_arr[index]).float()
        #img = torch.from_numpy(img).float().to(device)
        #label = torch.tensor(int(self.labels[index]))

        return (img_as_img, single_image_label)

    def __len__(self):
        return self.data_len

transforms_train = transforms.Compose([transforms.Resize((224,224)),transforms.RandomHorizontalFlip(),transforms.ToTensor(),transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])])
transforms_test = transforms.Compose([transforms.Resize((224,224)),transforms.ToTensor(),transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])])
dataset_train = FashionData('./deep_fashion/train.csv', mode='train', transform=transforms_train)
dataset_val = FashionData('./deep_fashion/val.csv', mode='val', transform=transforms_test)
dataset_test = FashionData('./deep_fashion/test.csv', mode='test', transform=transforms_test)

from torch.utils.data import DataLoader

train_loader = DataLoader(dataset_train, batch_size=128, shuffle=True)
val_loader = DataLoader(dataset_val, batch_size=128, shuffle=False)
test_loader = DataLoader(dataset_test, batch_size=128, shuffle=False)

model=models.resnet50(pretrained=True)
for params in model.parameters():
  params.requires_grad=False
model.fc=nn.Sequential(
    nn.Linear(2048,15),
    nn.Sigmoid()
    )

device=torch.device('cuda' if torch.cuda.is_available() else 'cpu')
model=model.to(device)
print(model)

criterion=nn.BCELoss()
#criterion=nn.BCEWithLogitsLoss()
optimizer=optim.Adam(model.parameters(),lr=0.001)
criterion=criterion.to(device)

def train(train_loader,model,criterion,optimizer):
  model.train()
  loss_list=[]
  total_count=0
  acc_count=0

  for x,y in train_loader:

    x=x.to(device)
    y=y.to(device)

    optimizer.zero_grad()
    output=model(x)

    loss=criterion(output,y)
    loss.backward()
    optimizer.step()

    #_,predicted=torch.max(output,1)
    predicted=(output>0.5).float()
    total_count+=y.size(0)
    acc_count+=(predicted==y).sum().item()

    loss_list.append(loss.item())
  acc=acc_count/total_count
  loss=sum(loss_list)/len(loss_list)
  return acc, loss

def val(valid_loader,model,criterion):
  model.eval()
  loss_list=[]
  total_count=0
  acc_count=0
  with torch.no_grad():
    for x,y in valid_loader:
      x=x.to(device)

      y=y.to(device)

      output=model(x)
      loss=criterion(output,y)
      #_,predicted=torch.max(output,1)
      predicted=(output>0.5).float()

      total_count+=y.size(0)
      acc_count+=(predicted==y).sum().item()
      loss_list.append(loss.item())

  acc=acc_count/total_count
  loss=sum(loss_list)/len(loss_list)
  return acc, loss

train_acc_list = []
train_loss_list = []
val_acc_list = []
val_loss_list = []

for epoch in range(10):
    train_acc, train_loss = train(train_loader, model, criterion, optimizer)
    val_acc, val_loss=val(val_loader, model, criterion)
    train_acc_list.append(train_acc)
    train_loss_list.append(train_loss)
    val_acc_list.append(val_acc)
    val_loss_list.append(val_loss)
    print('epoch',epoch)
    print('Train Acc: {:.6f} Train Loss: {:.6f}'.format(train_acc, train_loss))
    print('  Val Acc: {:.6f}   Val Loss: {:.6f}'.format(val_acc, val_loss))

Did I do wrong in the loading data part? Or it's the other problem?

11111111111111111
11111111111111111
11111111111111111


title: Debugging pytorch code in pycharm (Feasibility)

body:
I am trying to run a code in written in python (pytorch code) which when passed as an arguments options trains the Neural network.
if __name__ == "__main__":
    args = docopt(__doc__)
    myparams = args["options"]
    ....
    /* do work */

Now if we have to run this code, I need to call it from console. python3 train.py --option1 123 etc. But in that case the debug points won't work in pycharm. Can anybody clarify how to debug in this scenario? (If you know the way it would be great if you let me know).


title: Defining matrix functions on higher dimensional tensors? Specifically a matrix logarithm

body:
Pytorch does not have a native matrix logarithm. I would like to create an efficient logarithm using 'scipy.linalg.logm'. The problem is that 'logm' can only be applied to square tensors. I would like to compute the log of a tensor shape (64,22,3,3), in which the log is being applied to the (3,3) matrix component. Is there a fast and efficient way to apply logm to the entire tensor?
I currently am resorting to list comprehension which is incredibly slow:
x=torch.rand(64,22,3,3)
Log_x=torch.stack([logm(x[i,j]) for i in range(x.shape[0]) for j in range(x.shape[1])])


title: calculate two losses in a model and backpropagate twice

body:
I'm creating a model using BertModel to identify answer span (without using BertForQA).
I have an indepent linear layer for determining start and end token respectively. In init():
self.start_linear = nn.Linear(h, output_dim)

self.end_linear = nn.Linear(h, output_dim)

In forward(), I output a predicted start layer and predicted end layer:
 def forward(self, input_ids, attention_mask):

 outputs = self.bert(input_ids, attention_mask) # input = bert tokenizer encoding

 lhs = outputs.last_hidden_state # (batch_size, sequence_length, hidden_size)

 out = lhs[:, -1, :] # (batch_size, hidden_dim)

 st = self.start_linear(out)

 end = self.end_linear(out)

 predict_start = self.softmax(st)

 predict_end = self.softmax(end)

 return predict_start, predict_end

Then in train_epoch(), I tried to backpropagate the losses separately:
def train_epoch(model, train_loader, optimizer):

 model.train()

 total = 0

 st_loss, st_correct, st_total_loss = 0, 0, 0

 end_loss, end_correct, end_total_loss = 0, 0, 0

 for batch in train_loader:

   optimizer.zero_grad()

   input_ids = batch['input_ids'].to(device)

   attention_mask = batch['attention_mask'].to(device)

   start_idx = batch['start'].to(device)

   end_idx = batch['end'].to(device)

   start, end = model(input_ids=input_ids, attention_mask=attention_mask)

   st_loss = model.compute_loss(start, start_idx)

   end_loss = model.compute_loss(end, end_idx)

   st_total_loss += st_loss.item()

   end_total_loss += end_loss.item()

 # perform backward propagation to compute the gradients

   st_loss.backward()

   end_loss.backward()

 # update the weights

   optimizer.step()

But then I got on the line of end_loss.backward():
Trying to backward through the graph a second time, but the saved intermediate results have already been freed. Specify retain_graph=True when calling backward the first time.

Am I supposed to do the backward pass separately? Or should I do it in another way? Thank you!


title: PyTorch Model's gradients are converging to zero

body:
I'm currently working on a personal implementation of the Transformer architecture. The code I've written as here.
The problem that I'm facing is that I believe my model isn't training properly and I'm not sure what kind of measures I should take to fix that. I've come to this conclusion after using Weights & Biases to visualize the model's gradient histograms and they look something like this:

The gradients seem to quickly converge to zero. There is a portion of code that contains a feedforward neural network that uses ReLU activation, and I changed this to Leaky ReLU under the suspicion that dying ReLU's may be the problem. However, using Leaky ReLU's doesn't help and just prolongs the zero-convergence.
Any feedback on what else I may try is appreciated.


title: How to handle this case in pytorch?

body:
I have the following kinds of two corelated tensors data and mask. The size of data is 1x2x24x2 and the size of mask is 1x2x24. In mask, the True means the corresponding data in data is efficient and should be gradient back-propagated.
data:
tensor([[[[ 1.0663e+03,  5.5993e+02],
          [ 1.0612e+03,  7.2023e+02],
          [ 1.0831e+03,  7.2179e+02],
          [ 1.0945e+03,  5.6083e+02],
          [-1.0000e+10,  1.0000e+10],
          [-1.0000e+10,  1.0000e+10],
          [-1.0000e+10,  1.0000e+10],
          [-1.0000e+10,  1.0000e+10],
          [-1.0000e+10,  1.0000e+10],
          [-1.0000e+10,  1.0000e+10],
          [-1.0000e+10,  1.0000e+10],
          [-1.0000e+10,  1.0000e+10],
          [-1.0000e+10,  1.0000e+10],
          [-1.0000e+10,  1.0000e+10],
          [-1.0000e+10,  1.0000e+10],
          [-1.0000e+10,  1.0000e+10],
          [-1.0000e+10,  1.0000e+10],
          [-1.0000e+10,  1.0000e+10],
          [-1.0000e+10,  1.0000e+10],
          [-1.0000e+10,  1.0000e+10],
          [-1.0000e+10,  1.0000e+10],
          [-1.0000e+10,  1.0000e+10],
          [-1.0000e+10,  1.0000e+10],
          [-1.0000e+10,  1.0000e+10]],

         [[ 6.9314e+02,  1.9700e+02],
          [ 6.3300e+02,  2.6924e+02],
          [ 6.3300e+02,  3.4165e+02],
          [ 7.7515e+02,  4.6000e+02],
          [ 8.2805e+02,  4.6000e+02],
          [ 9.0900e+02,  3.6276e+02],
          [ 9.0900e+02,  2.9035e+02],
          [ 7.9688e+02,  1.9700e+02],
          [-1.0000e+10,  1.0000e+10],
          [-1.0000e+10,  1.0000e+10],
          [-1.0000e+10,  1.0000e+10],
          [-1.0000e+10,  1.0000e+10],
          [-1.0000e+10,  1.0000e+10],
          [-1.0000e+10,  1.0000e+10],
          [-1.0000e+10,  1.0000e+10],
          [-1.0000e+10,  1.0000e+10],
          [-1.0000e+10,  1.0000e+10],
          [-1.0000e+10,  1.0000e+10],
          [-1.0000e+10,  1.0000e+10],
          [-1.0000e+10,  1.0000e+10],
          [-1.0000e+10,  1.0000e+10],
          [-1.0000e+10,  1.0000e+10],
          [-1.0000e+10,  1.0000e+10],
          [-1.0000e+10,  1.0000e+10]]]])
torch.Size([1, 2, 24, 2])

mask:
tensor([[[ True,  True,  True,  True, False, False, False, False, False, False,
          False, False, False, False, False, False, False, False, False, False,
          False, False, False, False],
         [ True,  True,  True,  True,  True,  True,  True,  True, False, False,
          False, False, False, False, False, False, False, False, False, False,
          False, False, False, False]]])
torch.Size([1, 2, 24])

Based on the data and mask, I need further processing. For example, I need to do this kind of for loop:
B, N = data.shape[0:2]
result = torch.zeros((B, N), dtype=torch.float32)
for b in range(B):
    for n in range(N):
        tmp_data = data[b,n]  # 24 x 2
        tmp_mask = mask[b,n]  # 24
        selected = tmp_data[tmp_mask]
        selected = torch.cat([selected, selected[0][None]], dim=0)
        total = selected[0:-1, 0] * selected[1:, 1] - selected[0:-1, 1] * selected[1:, 0]   # do cross product
        total = torch.sum(total, dim=0)
        result[b,n] = torch.abs(total) / 2

Then, the result tensor is fed into subsequent processing, e.g. loss computation. My question is, is there any speed up method to get rid of the for-loop? The main difficulty lies in that for each b and n, the count of True in mask is not equal, sometimes it is 3, sometimes it is 5, but the maximum count of True in each sample is 8. So, in the for-loop, it is necessary to handle each case one by one. Is there any vectorized way for this?
Another difficulty lies in the gradient back-propagation, which I need further investigation to check if this routine can work correctly for gradient back-propagation.
I'm looking forward to an elegant way to handle this kind of tensor processing in pytorch. Thanks in advance!
UPDATED:
The data and mask were sorted by the previous routine, for example:
sorted_inds = ...  # B x N x 24

data = torch.gather(data_original, dim=2, index=sorted_indices.unsqueeze(-1).repeat(1,1,1,2))  # B x N x 24 x 2
mask= torch.gather(mask_original, dim=2, index=sorted_indices)  # B x N x 24

# then the data and mask are obtained
# continue processing in this post
...


title: PyTorch - Sharing gradients between different iterations doesn't improve performance

body:
I need to find the largest eigenvalue of the NN, so I'm implementing the power method. The main function is a hessian-vector product (HVP), based on PyHessian implementation.
The main idea is the following:

We first compute gradients of the NN.
Then we find HVP with vector v by additionally differentiating the gradients: Hv = torch.autograd.grad(grads, params, grad_outputs=v, only_inputs=True, retain_graph=False)

Gradients are computed using passes over the data (so they are pretty expensive). Initially, they were computed during HVP computation, so I tried to move their computation outside.
My question. After the change above, the average time for HVP changed from 0.7 sec to 0.5 sec. Why there is only a minor performance improvement even though I got rid of the gradient computation at each iteration, which I suspected to be the most time-consuming part (since it requires passes over the data)?
My guess is that it's the nature of the computation graphs: even though I don't explicitly recompute them each time, they still trace back all the computations at each iteration. I.e. while it may be possible to optimize them (by tracing differentials directly to leaves), they don't do that. So I guess optimizing them could help somehow.
Code.
Some auxiliary functions:
get_params returns all learnable parameters of the NN:
def get_params(model):
  return [param for param in model.parameters() if param.requires_grad]

get_grads returns all gradients of the NN. Note + 0 to force creating a new tensor (so that gradients are not lost during model.zero_grad()):
def get_grads(model):
  return [param.grad + 0 for param in model.parameters() if param.requires_grad]

First I compute gradients of the NN. The function iterates over the data, computes average loss over the data (each batch is weighted with its length), and then do backprop.
def compute_grads(model, data):
  data_len = 0.0
  loss = torch.tensor([0.0], device=device)
  model.zero_grad()

  for inputs, labels in data:
      outputs = model(inputs.to(device))
      cur_len = len(labels)
      loss += cur_len * criterion(outputs, labels.to(device))
      data_len += cur_len

  loss /= data_len
  loss.backward(create_graph=True)
  grads = get_grads(model)
  model.zero_grad()
  return grads

HVP is performed as follows:
def hv_product(model, params, grads, v):
  model.zero_grad()
  Hv = torch.autograd.grad(grads, params, grad_outputs=v, only_inputs=True, retain_graph=True)
  # Hv = torch.autograd.grad(grads, params, grad_outputs=v, only_inputs=True, retain_graph=False)
  return Hv

And the power method:
def power_method(model, data, iters):
  grads = compute_grads(model, data)

  params = get_params(model)
  v = [torch.randn(p.shape, device=device) for p in params]
  for i in range(iters):
    v = hv_product(model, params, grads, v)
    # v = hv_product(model, params, compute_grads(model, data), v)
    normalize(v)

The current version computes gradients only once (see the first line of power_method).
To compute gradients at every iteration, uncomment the commented lines (and comment lines similar to them).
P.S. There are a lot of things I don't understand (to the point that I'm not sure that the implementation is correct) (e.g. I don't understand if I should call model.zero_grad() in hv_product), so if you have any comments about the code, they are welcome.


title: How to Increase the Scope of Images a Neural Network Can Recognize?

body:
I am working on an image recognition neural network with Pytorch. My goal is to take pictures of handwritten math equations, process them, and use the neural network to recognize each element. I've reached the point where I am able to separate every variable, number, or symbol from the equation, and everything is ready to be sent through the neural network. I've trained my network to recognize numbers quite well (this part was quite easy), but now I want to expand the scope of the neural network to recognizing letters as well as numbers. I loaded handwritten letters along with the numbers into tensors, shuffled the elements, and put them into batches. No matter how I vary my learning rate, my architecture (hidden layers and the number of neurons per layer), or my batch size I cannot get the neural network to recognize letters.
Here is my network architecture and the feed-forward function (you can see I experimented with the number of hidden layers):
class NeuralNetwork(nn.Module):

def __init__(self):

    super().__init__()
    inputNeurons, hiddenNeurons, outputNeurons = 784, 700, 36

    # Create tensors for the weights
    self.layerOne = nn.Linear(inputNeurons, hiddenNeurons)
    self.layerTwo = nn.Linear(hiddenNeurons, hiddenNeurons)
    self.layerThree = nn.Linear(hiddenNeurons, outputNeurons)
    #self.layerFour = nn.Linear(hiddenNeurons, outputNeurons)
    #self.layerFive = nn.Linear(hiddenNeurons, outputNeurons)

# Create function for Forward propagation
def Forward(self, input):

    # Begin Forward propagation
    input = torch.sigmoid(self.layerOne(torch.sigmoid(input)))
    input = torch.sigmoid(self.layerTwo(input))
    input = torch.sigmoid(self.layerThree(input))
    #input = torch.sigmoid(self.layerFour(input))
    #input = torch.sigmoid(self.layerFive(input))

    return input

And this is the training code block (the data is shuffled in a dataloader, the ground truths are shuffled in the same order, batch size is 10, total number letter and number data points is 244800):
neuralNet = NeuralNetwork()
params = list(neuralNet.parameters())
criterion = nn.MSELoss()
print(neuralNet)

dataSet = next(iter(imageDataLoader))
groundTruth = next(iter(groundTruthsDataLoader))

for i in range(15):

    for k in range(24480):

        neuralNet.zero_grad()

        prediction = neuralNet.Forward(dataSet)
        loss = criterion(prediction, groundTruth)
        loss.backward()

        for layer in range(len(params)):

            # Updating the weights of the neural network
            params[layer].data.sub_(params[layer].grad.data * learningRate)

Thanks for the help in advance!


title: Functionality of `detach()` in PyTorch for this specific case

body: I know that detach() is used for detaching a variable from the computational graph. In that context, are the following expressions x = x - torch.mean(x, dim=0).detach() and x = x - torch.mean(x, dim=0) equivalent? I just want to subtract the mean out, don't want to pass gradients through the average calculation.


title: Feeding Classifier data from LSTM Autoencoder

body:
Goal:
I have built a LSTM autoencoder for the purpose of feature reduction. My plan is to encode some input and feed it to a classifier in the future. The encoder takes data of the shape [batch_size, timesteps, features_of_timesteps However in the output layer of the encoder portion I am returning just the last hidden state in the form [1, timesteps, features_of_timesteps].
class Encoder(nn.Module):
    def __init__(self, input_size, first_layer, second_layer, n_layers):
        super(Encoder, self).__init__()
        self.n_layers = n_layers
        self.encode = nn.Sequential(nn.LSTM(input_size, first_layer, batch_first=True),
                                    getSequence(),
                                    nn.ReLU(True),
                                    nn.LSTM(first_layer, second_layer),
                                    getLast())
        self.decode = nn.Sequential(nn.LSTM(second_layer, first_layer, batch_first=True),
                                    getSequence(),
                                    nn.ReLU(True),
                                    nn.LSTM(first_layer, input_size),
                                    getSequence())

    def forward(self, x):
        x = x.float()
        x = self.encode(x)
        x = x.repeat(batch_size, 1, 1)
        x = self.decode(x)
        return x

Worry:
I am afraid that the last hidden state of the my second LSTM layer in the encoding portion of the model is summarizing the entire batch along with decreasing the feature dimensionality. This feels wrong because I am trying to reduce a single timeseries into a smaller vector, not an entire batch of timeseries into one vector. Am I correct in my worries?


title: pytorch OSError: [Errno 28] No space left on device

body:
I'm using an ubuntu 18 docker container.
$cat /etc/lsb-release
DISTRIB_ID=Ubuntu
DISTRIB_RELEASE=18.04
DISTRIB_CODENAME=bionic
DISTRIB_DESCRIPTION="Ubuntu 18.04.4 LTS"

When I try training a resnext101 model from torchvision, I get the following error.
Downloading: "https://download.pytorch.org/models/resnext101_32x8d-8ba56ff5.pth" to /home/vmuser/.cache/torch/hub/checkpoints/resnext101_32x8d-8ba56ff5.pth
  0%|                                                          | 0.00/340M [00:00<?, ?B/s]
Traceback (most recent call last):
  File "train_attn_best_config.py", line 377, in <module>
    tabct = TabCT(cnn = model, fc_dim = fd, attn_filters = af, n_attn_layers = nal).to(gpu)
  File "train_attn_best_config.py", line 219, in __init__
    self.ct_cnn = cnn_dict[cnn](pretrained = True)
  File "/home/vmuser/anaconda3/envs/pulmo/lib/python3.7/site-packages/torchvision/models/resnet.py", line 317, in resnext101_32x8d
    pretrained, progress, **kwargs)
  File "/home/vmuser/anaconda3/envs/pulmo/lib/python3.7/site-packages/torchvision/models/resnet.py", line 227, in _resnet
    progress=progress)
  File "/home/vmuser/anaconda3/envs/pulmo/lib/python3.7/site-packages/torch/hub.py", line 481, in load_state_dict_from_url
    download_url_to_file(url, cached_file, hash_prefix, progress=progress)
  File "/home/vmuser/anaconda3/envs/pulmo/lib/python3.7/site-packages/torch/hub.py", line 404, in download_url_to_file
    f.write(buffer)
  File "/home/vmuser/anaconda3/envs/pulmo/lib/python3.7/tempfile.py", line 481, in func_wrapper
    return func(*args, **kwargs)
OSError: [Errno 28] No space left on device

When I run df, I get this, one of my tmpfs is only 65 mb. I tried running export TMPDIR=/var/tmp and export TMPDIR=~/Data/tmp
$df
Filesystem      1K-blocks       Used Available Use% Mounted on
overlay        1797272568 1705953392         0 100% /
tmpfs               65536          0     65536   0% /dev
tmpfs            98346264          0  98346264   0% /sys/fs/cgroup
/dev/sda6      1797272568 1705953392         0 100% /etc/hosts
shm                 65536          0     65536   0% /dev/shm
/dev/sdb1      1845816492 1362932848 389098592  78% /home/vmuser/Data
tmpfs            98346264         12  98346252   1% /proc/driver/nvidia
tmpfs            19669256      93256  19576000   1% /run/nvidia-persistenced/socket
udev             98318592          0  98318592   0% /dev/nvidia1
tmpfs            98346264          0  98346264   0% /proc/acpi
tmpfs            98346264          0  98346264   0% /proc/scsi
tmpfs            98346264          0  98346264   0% /sys/firmware

But the error is still there.


title: Unable to backpropagate through torch.rfft

body:
I have two losses: one the usual L1 loss and second one involving torch.rfft()
def dft_amp(img):
    fft_im = torch.rfft( img, signal_ndim=2, onesided=False )
    fft_amp = fft_im[:,:,:,:,0]**2 + fft_im[:,:,:,:,1]**2
    return torch.sqrt(fft_amp)

l1_loss = torch.nn.L1Loss()

loss  = l1_loss(pred,gt) + l1_loss(dft_amp(pred),dft(gt_amp))

loss.backward()

This runs for the 1st iteration with both losses not bein being nan but the loss from 2nd iteration onwards becomes nan.
If however only the simple L1 loss is kept and l1_loss(dft_amp(pred),dft(gt_amp)) is omitted, the training proceeds normally.
Does torch.rfft() supports backpropagation? I am using pytorch 1.4.0
Any suggestions would be appreciated


title: Customized loss function in PyTorch which uses DNN outputs and additional variables

body:
(I am sorry if my English is not good)
I can create my own loss function in PyTorch if the function requires only DNN output vector(predicted) and DNN output vector(ground truth).
I want to use additional variables to calculate the loss.
I make my training and test data like below;
DNN input:

Data_A -> processing 1 -> Data_X

DNN output:

Data_A -> processing 1 -> Data_X
Data_B -> processing 1 -> Data_P
Data_X , Data_P -> processing 2 -> Data_Y

and I divide Data_X and Data_Y into train data and test data.
x_train, x_test, y_train, y_test = train_test_split(Data_X,Data_Y,test_size=0.2, random_state=0)
I want to use Data_A, Data_B, Data_Y(predicted), and Data_Y(ground truth) to calculate the loss.
I saw many examples for customized loss function which only use Data_Y(predicted) and Data_Y(ground truth). I could use such a customized loss function before.
However, I don't know what to do when I want to use another additional variables.
Is there a good way? Thank you for your help!


title: GAN Generator Loss Not Going Down

body:
I am applying deep learning to a voice conversion task (i.e. making the source speaker sound like the target speaker). Before touching GANs, I had a baseline (i.e. only generator) that was getting decent results, training on MSELoss between the predicted mel-spectrogram and the target mel-spectrogram.
In an effort to help improve the naturalness of the predictions, I am trying to add a GAN discriminator. I am using a PatchGAN-inspired discriminator and following the training paradigm of this repo ("Jointly Adversarial Enhancement Training for Robust End-to-End Speech Recognition").
However, my following loss curves seem off for me. It seems like the GAN is at a convergence failure where the generator's adversarial loss goes down and then back up and converged — and where the discriminator's loss goes low to almost zero.

Note: mse is MSELoss between predicted/target mel-spectrogram. gan is the MSE adversarial loss. total is = mse + 2 * gan. discrim_gen is the discriminator's MSELoss on the predicted sample and discrim_real is the discriminator's MSELoss on the real sample.
As a result, the quality of the predictions seem like the same as the baseline. Does anyone have any insights on how I could have the adversarial loss remain going down and closer to the discrim_real loss? Any insights would be much appreciated.
Edit: I have tried experimenting with having the discriminator's initial learning rate be a magnitude lower than the initial generator's initial learning rate. And I have tried start training the discriminator after the generator has first trained on some beginning epochs. Both approaches seems to lower the gan loss, but it goes back up again over time.


title: How train the pytorch model with another freeze model?

body:
I train the pytorch model(name = A) with pre-trained model(name = B).
I want to use two models in series like this: The input is used so that the output comes out through two models, and this output and the target label are compared. Model2 freezes and learns the learning parameters of model1.
https://arxiv.org/ftp/arxiv/papers/1905/1905.01898.pdf figure 1 (b).
And I wrote the code like this:
Class A(nn.Module):
      # MY CODE

Class B(nn.Module):
      # MY CODE

train_model = A()
freeze_model = B()

freeze_model.load_state_dict(torch.load("bestmodel.pth"))
for param in freeze_model.parameters():
    param.requires_grad = False

criterion=nn.MSELoss()
optimizer = torch.optim.Adam(train_model.parameters(), lr=learning_rate)

for epoch in range(100):
    ... iteration ...
    out1 = train_model(input1)
    out2 = freeze_model(out1,input2)
    loss = criterion(target,out2)
    ... optimizer.zero_grad, loss backward, ...
    optimizer.step()

However, The first iteration comes out with a real value, After optimizer_Step, model weigth and loss are changed to nan..
I think this is for an optimizer or some other reason. Is there a solution?


title: Using Pytorch how to define a tensor with indices and corresponding values

body:
Problem
I have a list of indices and a list of values like so:
i = torch.tensor([[2, 2, 1], [2, 0, 2]])
v = torch.tensor([1, 2, 3])

I want to define a (3x3 for the example) matrix which contains the values v at the indices i (1 at position (2,2), 2 at position (2, 0) and 3 at position (1,2)):
tensor([[0, 0, 0],
        [0, 0, 3],
        [2, 0, 1]])

What I have tried
I can do it using a trick, with torch.sparse and .to_dense() but I feel that it's not the "pytorchic" way to do that nor the most efficient:
f = torch.sparse.FloatTensor(indices, values, torch.Size([3, 3]))
print(f.to_dense())

Any idea for a better solution ?
Ideally I would appreciate a solution at least as fast than the one provided above.
Of course this was just an example, no particular structure in tensors i and v are assumed (neither for the dimension).


title: Partial Backwards in PyTorch Graph

body:
I have a medium sized tensor x. On this mediums sized tensor an computationally expensive function (both forward and backwards) q is applied to obtain another medium sized tensor y.
Using y I evaluate many functions f producing a scalar, they are not particularly computationally expensive however use large internal states causing a large computational graph for torch that.
Now I want to compute the gradient on x in the following way
y = q(x)

for f in functions
    res += f(y)

res.backward()

The proble with this implementation is that the graphs of all the functions f are retained. That causes the total memory usage to explode.
The other possibility would be to compute
y = q(x)

for f in functions
    partial = f(y)
    partial.backward(retain_graph = True)

The advantage is that after every function evaluation f the result goes out of scope and the graph is freed, saving massive amounts of memory. However, in this situation the function q(x) is evaluated backwards multiple times which is very time expensive.
In the ideal situation I would want to first compute the gradient for y using code similar to the second example and then only once evaluate q backwards to get the gradient for x. What is the proper way to do that with PyTorch?


title: Error while converting pytorch model to Coreml. Layer has 1 inputs but expects at least 2

body:
My goal is to convert my Pytorch model to Coreml. I have no problem with doing an inference with pytorch. However, after I trace my model and try to convert it
trace = torch.jit.trace(traceable_model, data)
mlmodel = ct.convert(
            trace,
            inputs=[ct.TensorType(name="Image", shape=data.shape)])

I get the following error Error compiling model: "Error reading protobuf spec. validator error: Layer 'cur_layer_input.1' of type 925 has 1 inputs but expects at least 2."
I have a ConvLSTM layer in my model with a cur_layer. And here is whats inside.
class ConvLSTM(nn.Module):

    def __init__(self, input_size, input_dim, hidden_dim, kernel_size, num_layers,
                #I cut out some of the init
        for i in range(0, self.num_layers):
            cur_input_dim = self.input_dim if i == 0 else self.hidden_dim[i - 1]
            cell_list.append(ConvLSTMCell(input_size=(self.height, self.width),
                                          input_dim=cur_input_dim,
                                          hidden_dim=self.hidden_dim[i],
                                          kernel_size=self.kernel_size[i],
                                          bias=self.bias))
        self.cell_list = nn.ModuleList(cell_list)
    def forward(self, input_tensor, hidden_state=None):
        if not self.batch_first:
            # (t, b, c, h, w) -> (b, t, c, h, w)
            input_tensor=input_tensor.permute(1, 0, 2, 3, 4)
        # Implement stateful ConvLSTM
        if hidden_state is not None:
            raise NotImplementedError()
        else:
            hidden_state = self._init_hidden(batch_size=input_tensor.size(0))

        layer_output_list = []
        last_state_list = []
        seq_len = input_tensor.size(1)
        cur_layer_input = input_tensor
        for layer_idx in range(self.num_layers):
            h, c = hidden_state[layer_idx]
            output_inner = []
            for t in range(seq_len):
                h, c = self.cell_list[layer_idx](input_tensor=cur_layer_input[:, t, :, :, :],
                                                 cur_state=[h, c])
                output_inner.append(h)

            layer_output = torch.stack(output_inner, dim=1)
            cur_layer_input = layer_output

            layer_output = layer_output.permute(1, 0, 2, 3, 4)

            layer_output_list.append(layer_output)
            last_state_list.append([h, c])

        if not self.return_all_layers:
            layer_output_list = layer_output_list[-1:]
            last_state_list = last_state_list[-1:]

        return layer_output_list, last_state_list

I do not quite understand where is the input that needs to be 2.


I have been running SRGAN to produce super resolution images. I received this error when I tried to add images with different sizes to the training and testing folders. Here's the error:
The size of tensor a (296) must match the size of tensor b (298) at non-singleton dimension 2

I have done internet researches and found some people suggesting, that the "dimensions in tensors" do not match, so I have to make "adjustments".
As much as that sound like an appropriate piece of advice, I still can't figure out what exactly I can do to "adjust the dimensions of the tensors".
Here's the code that led to the error message.
def first_train(epoch):
G.train()

G_loss=0

for batch_idx,(data_lr,data_hr)in enumerate(train_loader):
    if data_lr.size()[0]!=batch_size:
        break
    if cuda:
        data_lr=data_lr.cuda()
        data_hr=data_hr.cuda()
    fake_image=G(data_lr)
    G.zero_grad()

    G_loss=MSE_Loss(fake_image,data_hr)
    G_loss.backward()
    G_optimizer.step()
    G_loss+=G_loss.data

    G_loss/=len(train_loader)

    g_image=fake_image.data.cpu()
    hr_image=data_hr.data.cpu()
    HR_image=torch.cat((hr_image,g_image),0)
    save_image(HR_image,"save_image/epoch_{}.png".format(epoch))
    print("save_image")

    return G_loss

num_epoch=100

for epoch in range(1,num_epoch+1):
    if epoch==1:
        print("trainning start!!")
    g_loss_=first_train(epoch)

    if epoch%1==0:
        torch.save(G.state_dict(),"asset/G_first_epoch{}.pth".format(epoch))

"G_loss=MSE_Loss(fake_image,data_hr)" was marked red (indicating that the error derives from this specific code)"
I'd appreciate any advice.
Thanks a million in advance!


title: regarding the trick of using 1*1 convolution

body:
I once read the following statement on using 1*1 convolution, which can help connect the input and output with different dimensions:

For example, to reduce the activation dimensions (HxW) by a factor of 2, you can use a 1x1 convolution with a stride of 2.

How to understand this example?


title: Pytorch: model weights cannot be Updated”

body:
Here is a simple example:
class Net(nn.Module):
  def __init__(self):
    super(Net,self).__init__()
    A = Variable(torch.randn((1)), requires_grad=True)
    self.A = nn.Parameter(A)
    self.B = nn.Linear(2,2)
    self.register_parameter("Ablah",self.A)

  def forward(self,x):
    self.B.weight.data = self.B.weight * self.A
    return self.B(x)

net = Net()
input = torch.ones(2,2)
input.requires_grad_(True)
output = net(input)
optim = torch.optim.Adam(net.parameters(), lr = 0.01)
loss = torch.norm(output)
print(net.A)
loss.backward()
optim.step()
print(net.A)

I am trying to modify a PyTorch model's parameter self.B with another parameter self.A by using the code self.B.weight.data = self.B.weight * self.A, but it seems that A has no grad and it can not be updated.
It has confused me for a long time. Is there anything wrong?
Please give me some advice. Thank you!


title: Unable to rescale an image using transforms.Resize

body:
I am trying to rescale an image of size 255, 255 having 5 channels to something more manageable for my CNN, i.e 5, 40, 40
Here is a minimally reproducible example -
import torch
from torch import nn
import torchvision as TV
from torchvision import transforms
import numpy as np
x = np.random.rand(5,255,255)
transform = TV.transforms.Resize([transforms.Scale(48)])
transform(torch.from_numpy(x))

But I get the following error -
 File "C:\Users\thoma\anaconda3\envs\tf-latest\lib\site-packages\torchvision\transforms\transforms.py", line 186, in __init__                                                                                assert isinstance(size, int) or (isinstance(size, Iterable) and len(size) == 2)                     AssertionError


title: Getting specific items from binary files with an index in python

body:
I'm using pytorch but getting trouble with reading large datasets. Considering the size of data, it is necessary to save them as files.
Rather than preprocessing data into batches and save them as files respectively, I'm seeking for a method that can read specific parts of only one or a few binary files, with an index file (if necessary) indicating the corresponding file and positions for seeking.
For example, supposing a dataset file containing a lot of images (maybe with different size), and an index file indicating the start position and the end position of each image. A dict-like or list-like container  data is initialized with the index file. When I want to read the third image, I can simply use data[2] and it can jump to the specific position fast and return what I want.
Is there an existing method that can handle it well? As far as I know, SQL or pandas might be a possible solution for fixed field length data, but I don't know if they can handle varient filed length and binary data.


title: How do I pass an array of tensors into the criterion/loss function in PyTorch?

body:
My loss function gives an error:
self.loss_fn = nn.MSELoss()

#### -- Snip ####

loss = self.loss_fn(predictions, targets) # Error here: 'list' object has no attribute 'size'
loss.backward()

My predictions are an array of tensors as follows:
predictions = []
for _ in range(100):
   prediction = MyNeuralNet(inputs)
   predictions.append(prediction)

How can I pass an array of tensors into my loss criterion function without getting the above error?


title: How to use neural nets for classification with a large number of classes?

body:
Are there techniques I could use to classify data (I'm using pytorch) using a large number of classes?
I noticed that if I try to build a multi layer perceptron network I run out of memory on the GPU given the last layer must have too many neurons, even though my GPU has 24Gb of memory. I have about 3000 classes.
Is there a way or technique do handle this type of scenario?
Note that I am NOT asking for an opinion on which technique is better. I am asking for an objective list of techniques that could be used in this scenario. This can be answered in fact-based fashion and include citations, etc if needed.


title: Running multiple inferences in parallel with PyTorch

body:
I'm trying to implement Double DQN (not to be confused with DQN with a slightly delayed Q-target network) in PyTorch to train an agent to play an Atari OpenAI Gym game. Here I discuss the implementation of the following formula:
Update of Q-network, formula taken from Sutton & Barto.
My first implementation is:
Q_pred = self.Q_1.forward(s_now)[T.arange(batch_size), actions.long()]

Q_next_all = self.Q_1.forward(s_next)
maxA_id = T.argmax(Q_next_all, dim=1)
Q_pred2 = self.Q_2.forward(s_next)[T.arange(batch_size), maxA_id]

Q_target = (rewards + (~dones) * self.GAMMA * Q_pred2).detach()

self.Q_1.optimizer.zero_grad()
self.Q_1.loss(Q_target, Q_pred).backward()
self.Q_1.optimizer.step()

(Q_1 and Q_2 are nn.Module classes, and all of the variables involved here are already torch tensors lying in the GPU.)
I noticed that my program ran much slower than a previous implementation which used plain DQN.
I realized that I can combine the batches entering Q_1, so there will be one combined batch being forwarded in the neural network, instead of two batches in sequence. The code becomes:
s_combined = T.cat((s_now, s_next))
Q_combined = self.Q_1.forward(s_combined)

Q_pred = Q_combined[T.arange(batch_size), actions.long()]
Q_next_all = Q_combined[batch_size:]

Q_pred2_all = self.Q_2.forward(s_next)

maxA_id = T.argmax(Q_next_all, dim=1)
Q_pred2 = Q_pred2_all[T.arange(batch_size), maxA_id]
Q_target = (rewards + (~dones) * self.GAMMA * Q_pred2).detach()

self.Q_1.optimizer.zero_grad()
self.Q_1.loss(Q_target, Q_pred).backward()
self.Q_1.optimizer.step()

(This proves that I understand how to do batch training in PyTorch, so don't mark this as a duplicate of this question.)
Furthermore, I realized that Q_1 and Q_2 can process their batches in parallel. So I looked up how to do multiprocessing in PyTorch. Unfortunately, I couldn't find a good example. I tried to adapt a code that looks similar to my scenario, and my code becomes:
def spawned():
    s_combined = T.cat((s_now, s_next))
    Q_combined = self.Q_1.forward(s_combined)

    Q_pred = Q_combined[T.arange(batch_size), actions.long()]
    Q_next_all = Q_combined[batch_size:]

mp.set_start_method('spawn', force=True)
p = mp.Process(target=spawned)
p.start()
Q_pred2_all = self.Q_2.forward(s_next)
p.join()

maxA_id = T.argmax(Q_next_all, dim=1)
Q_pred2 = Q_pred2_all[T.arange(batch_size), maxA_id]
Q_target = (rewards + (~dones) * self.GAMMA * Q_pred2).detach()

self.Q_1.optimizer.zero_grad()
self.Q_1.loss(Q_target, Q_pred).backward()
self.Q_1.optimizer.step()

This crashes with the error message:
AttributeError: Can't pickle local object 'Agent.learn.<locals>.spawned'

So how do I make this work?
(Achieving this in CUDA programming is trivial. One simply launches two device kernels using a sequential host code, and the two kernels are automatically computed in parallel in the GPU.)


title: How should you store multi-dimensional arrays?

body:
Couldn't find how Numpy and PyTorch store arrays (tensors), so I'm asking here.
If you need to store multidimensional arrays like those two, should you store all values in a single-contionuous array and store the shape of the array to determine how you parse it (which is also beneficial when reshaping the array), like the following int arr[];, or should you store the multidimensional array as is, so for example storing a 2d integer array like so in C for example: int arr[][];.
Which should be the way for maximum performance and memory efficiency?
Also, any reference on how to go on developing such a thing is appreciated.
P.S That's for a school exam, so no libraries.


title: Use of the GPU on Colab with pytorch

body:
I am trying to use the GPU on Colab, but it takes about 400 seconds to train my model when I use GPU and 200 seconds to train it when I select CPU, so I think there is a problem somewhere..
Where did I made a mistake ?
import torch
from torch import Tensor
import dlc_practical_prologue as prologue
import time
import matplotlib.pyplot as plt

dev = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
print(dev)

#%% Exercise1

def sigma(x):
    return torch.tanh(x)

def dsigma(x):
    return 1-torch.pow(torch.tanh(x), 2)

#%% Exercise2

def loss(v, t):
    return torch.sum(torch.pow(t-v, 2))

def dloss(v, t):
    #print(v.size(), t.size())
    return -2*(t-v)

#%% Exercise3

def forward_pass(w1, b1, w2, b2, x):
    s1 = w1.mv(x) + b1
    x1 = sigma(s1)
    #print(b1.size())
    s2 = w2.mv(x1) + b2
    x2 = sigma(s2)
    #print(s2.size(), x2.size())
    return x, s1 , x1, s2, x2

def backward_pass(w1, b1, w2, b2, t, x, s1, x1, s2, x2, dl_dw1, dl_db1, dl_dw2, dl_db2):
    #print(dl_db2.size(), (dsigma(s2) * dloss(x2, t)).size())
    dl_db2_temp = dsigma(s2) * dloss(x2, t)
    dl_dw2_temp = dl_db2_temp.unsqueeze(0).t() * x1.unsqueeze(0)
    dl_db1_temp = w2.t().mv(dl_db2_temp)
    dl_dw1_temp = dl_db1_temp.unsqueeze(0).t() * x.unsqueeze(0)
    return dl_dw1 + dl_dw1_temp, dl_db1 + dl_db1_temp, dl_dw2 + dl_dw2_temp, dl_db2 + dl_db2_temp

#%% Exercise4

train_data, train_target, test_data, test_target = prologue.load_data(cifar = False, one_hot_labels = True, normalize = True, flatten = True)

train_data = train_data.to(dev)
train_target = train_target.to(dev)
test_data = test_data.to(dev)
test_target = test_target.to(dev)

train_target = train_target * 0.9
test_target = test_target * 0.9
print(train_target.is_cuda)

epsilon = 1e-6
w1 = torch.empty((50, 784), device=dev).normal_(0, epsilon)
b1 = torch.empty((50), device=dev).normal_(0, epsilon)
w2 = torch.empty((10, 50), device=dev).normal_(0, epsilon)
b2 = torch.empty((10), device=dev).normal_(0, epsilon)
print(w1.is_cuda)

dl_dw1 = torch.zeros((50, 784), device=dev)
dl_db1 = torch.zeros((50), device=dev)
dl_dw2 = torch.zeros((10, 50), device=dev)
dl_db2 = torch.zeros((10), device=dev)

eta = 0.1 / train_target.size(0)
nb_simul = 1000
error_train = torch.zeros(nb_simul, device=dev)
error_test = torch.zeros(nb_simul, device=dev)

t1 = time.time()

for i in range(nb_simul):
    dl_dw1 = torch.zeros((50, 784), device=dev)
    dl_db1 = torch.zeros((50), device=dev)
    dl_dw2 = torch.zeros((10, 50), device=dev)
    dl_db2 = torch.zeros((10), device=dev)

    for j in range(train_target.size(0)):
        x, s1 , x1, s2, x2 = forward_pass(w1, b1, w2, b2, train_data[j])
        #print(x.is_cuda)
        #print(x2.is_cuda)
        dl_dw1, dl_db1, dl_dw2, dl_db2 = backward_pass(w1, b1, w2, b2, train_target[j], x, s1, x1, s2, x2, dl_dw1, dl_db1, dl_dw2, dl_db2)
        #print(dl_dw1.is_cuda)

    w1 = w1 - eta * dl_dw1
    w2 = w2 - eta * dl_dw2
    b1 = b1 - eta * dl_db1
    b2 = b2 - eta * dl_db2
    #print(w1.is_cuda)

    for j in range(train_target.size(0)):
        x_train, s1_train, x1_train, s2_train, x2_train = forward_pass(w1, b1, w2, b2, train_data[j])
        _, predict = x2_train.max(0)
        _, value = train_target[j].max(0)
        if predict != value:
            error_train[i] += 1

    #print('error on training: ', error_train)

    for j in range(test_target.size(0)):
        x_test, s1_test, x1_test, s2_test, x2_test = forward_pass(w1, b1, w2, b2, test_data[j])
        _, predict = x2_test.max(0)
        _, value = test_target[j].max(0)
        if predict != value:
            error_test[i] += 1

    #print('error on test: ', error_test)

t2 = time.time()
print(t2-t1)

#%% Plot

abs_x = torch.arange(1, nb_simul+1, 1)
plt.plot(abs_x, error_train)
plt.plot(abs_x, error_test)
print(error_train[-1])
print(error_test[-1])


title: PyTorch using LR-Scheduler with param groups of different LR's

body:
I have defined the following optimizer with different learning rates for each parameter group:
  optimizer = optim.SGD([
          {'params': param_groups[0], 'lr': CFG.lr, 'weight_decay': CFG.weight_decay},
          {'params': param_groups[1], 'lr': 2*CFG.lr, 'weight_decay': 0},
          {'params': param_groups[2], 'lr': 10*CFG.lr, 'weight_decay': CFG.weight_decay},
          {'params': param_groups[3], 'lr': 20*CFG.lr, 'weight_decay': 0},
      ], lr=CFG.lr, momentum=0.9, weight_decay=CFG.weight_decay, nesterov=CFG.nesterov)

Now I want to use a LR-Scheduler to update all the learning rates and not only the first one, because by deafult, a scheduler would only update the param_groups[0]?
scheduler = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizer, T_0=5, T_mult=2, eta_min=CFG.min_lr, last_epoch=-1, verbose=True)

Giving me:
Parameter Group 0
    dampening: 0
    initial_lr: 0.001
    lr: 0.0009999603905218616
    momentum: 0.9
    nesterov: True
    weight_decay: 0.0001

Parameter Group 1
    dampening: 0
    initial_lr: 0.002
    lr: 0.002
    momentum: 0.9
    nesterov: True
    weight_decay: 0

Parameter Group 2
    dampening: 0
    initial_lr: 0.01
    lr: 0.01
    momentum: 0.9
    nesterov: True
    weight_decay: 0.0001

Parameter Group 3
    dampening: 0
    initial_lr: 0.02
    lr: 0.02
    momentum: 0.9
    nesterov: True
    weight_decay: 0
)

after one update.
Any idea how to update all the learning rates with a scheduler?


title: How to convert a full string output into torch tensor

body:
Having defined an API based on a transformer model which I send request to, I got as output an object of class string supposed to represent a torch tensor.
import requests
import json
import torch

response = requests.post("http://127.0.0.1:5000/predict", json="This is a test.")
data = response.json()
print(data)

tensor([[-5.5169e-01,  5.1988e-01, -1.4731e-01,  4.6096e-02, -5.5753e-02,
         -6.9530e-01,  7.3424e-01,  3.0014e-01,  4.0528e-01, -1.7587e-01,
          1.2586e-01,  5.6712e-01, -4.0757e-01,  1.5796e-01,  9.4700e-01,
          6.2967e-01,  3.1027e-01,  1.0169e-02,  3.1380e-02,  1.2585e-01,
          2.3633e-01,  6.0813e-01, -1.0548e+00, -1.8581e-01, -5.9870e-02,
          1.2952e-01, -3.8818e-01,  2.3425e-01]])

I checked in the requests module documentation, but couldn't find any way to convert this fully strings into a torch tensor?


title: How can I solve this? ..... Error: element 0 of tensors does not require grad and does not have a grad_fn

body:
class PairwiseRegistration(_PairwiseRegistration):
    def __init__(self, verbose=True):
        super(PairwiseRegistration, self).__init__(verbose=verbose)

    def _closure(self):
        self._optimizer.zero_grad()
        displacement = self._transformation()
        loss = self._image_loss
        # compute the image loss
        lossList = []
        loss_names = []
        for image_loss in self._image_loss:
             lossList.append(image_loss(displacement))
             loss_names.append(image_loss.name)
        # compute the regularisation loss on the displacement
        for reg_disp in self._regulariser_displacement:
            lossList.append(reg_disp(displacement))
            loss_names.append(reg_disp.name)
        # compute the regularisation loss on the parameter
        for reg_param in self._regulariser_parameter:
            lossList.append(reg_param(self._transformation.named_parameters()))
            loss_names.append(reg_param.name)

        if self._verbose:
            for loss_value, loss_name in zip(lossList, loss_names):
                print(str(loss_name) + ": " + str(loss_value.data.item()) + " ", end='', flush=True)
            print("")
        # sum up all loss terms
        loss = sum(lossList)

        loss.backward(create_graph=True)

        return loss

Error:
File "examples/9Read_data_POPI_optim_adahessian.py", line 204, in main
    registration.start()
  File "/content/drive/My Drive/curveball-reg/airlab/registration/registration.py", line 142, in start
    loss = self._optimizer.step(self._closure)
  File "/usr/local/lib/python3.7/dist-packages/torch/optim/optimizer.py", line 89, in wrapper
    return func(*args, **kwargs)
  File "/usr/local/lib/python3.7/dist-packages/torch/autograd/grad_mode.py", line 27, in decorate_context
    return func(*args, **kwargs)
  File "/content/drive/My Drive/curveball-reg/examples/ada_hessian.py", line 98, in step
    loss = closure()
  File "/content/drive/My Drive/curveball-reg/airlab/registration/registration.py", line 124, in _closure
    loss.backward(create_graph=True)
  File "/usr/local/lib/python3.7/dist-packages/torch/tensor.py", line 245, in backward
    torch.autograd.backward(self, gradient, retain_graph, create_graph, inputs=inputs)
  File "/usr/local/lib/python3.7/dist-packages/torch/autograd/__init__.py", line 147, in backward
    allow_unreachable=False, accumulate_grad=False)  # allow_unreachable flag
RuntimeError: element 0 of tensors does not require grad and does not have a grad_fn


title: Pytorch: can't control garbage collector and model deallocation

body:
I am trying to control the allocation and deallocation of a Pytorch model (BART from the transformers library). I would like to deallocate the model when it's done. This is what i tried:
import transformers as t
import tracemalloc
import gc
import torch

class BartSummarize:
    def __init__(self):
        self.device = 'cuda:0' if torch.cuda.is_available() else 'cpu'
        self._model = t.BartForConditionalGeneration.from_pretrained('facebook/bart-large-cnn').to(self.device)
        self._tokenizer = t.BartTokenizer.from_pretrained('facebook/bart-large-cnn')

    def __del__(self):
        print('I am deleted')

    def run(self):
        ARTICLE_TO_SUMMARIZE = "My friends are cool but they eat too many carbs."
        inputs = self._tokenizer([ARTICLE_TO_SUMMARIZE], max_length=1024, return_tensors='pt').to(self.device)
        summary_ids = self._model.generate(inputs['input_ids'], num_beams=4, max_length=5, early_stopping=True)
        return [self._tokenizer.decode(g, skip_special_tokens=True, clean_up_tokenization_spaces=False) for g in summary_ids]

if __name__ == '__main__':
    tracemalloc.start()
    BARTModel = BartSummarize()
    data = BARTModel.run()
    current, peak = tracemalloc.get_traced_memory()
    print(f"Current memory BEFORE is {current / 10**6}MB; Peak was {peak / 10**6}MB")
    del BARTModel
    gc.collect()
    current, peak = tracemalloc.get_traced_memory()
    print(f"Current memory AFTER usage is {current / 10**6}MB; Peak was {peak / 10**6}MB")

This is what I get when I run everything:
Current memory BEFORE is 90.008281MB; Peak was 90.010476MB
Current memory AFTER usage is 89.883413MB; Peak was 90.010476MB
I am deleted

I assumed that when the garbage collector is called, BartModel gets deleted from memory and I am deleted is printed, then finally I get the last measure of memory allocation. Clearly, the order of execution is different, why is that? Why I am deleted comes as last, and not second? Is there a way to control the deallocation?


title: Why is cyclic learning rate scheduler resulting in worse performance?

body:
Using SGD without Cyclic Learning Rate:
optimizer = torch.optim.SGD(params, lr=1e-3, momentum=0.9, weight_decay=1e-1)

And using SGD with Cyclic Learning Rate:
optimizer = torch.optim.SGD(params, lr=1e-3, momentum=0.9, weight_decay=1e-1)
scheduler = torch.optim.lr_scheduler.CyclicLR(optimizer, base_lr=1e-3, max_lr=1e-2)

I am doing optim.step() before scheduler.step().
Using the LR Scheduler, accuracy and loss seem to oscillate around. Is this because I am using lr scheduler wrong or is it just hyperparameters? If the latter, what is a good range to use?


title: Getting predict.proba from BERT classififer

body:
I have a classifier on top of BERT, and I would like to see the predict probability for creating the ROC curve. How do I get the predict proba?. The predicted probas will be used to calculate the TPR FPR and threshold for ROC curve.
here is the code
class BertBinaryClassifier(nn.Module):
    def __init__(self, dropout=0.1):
        super(BertBinaryClassifier, self).__init__()
        self.bert = BertModel.from_pretrained('bert-base-uncased')
        self.dropout = nn.Dropout(dropout)
        self.linear = nn.Linear(768, 1)
        self.sigmoid = nn.Sigmoid()

    def forward(self, tokens, masks=None):
        _, pooled_output = self.bert(tokens, attention_mask=masks, output_all_encoded_layers=False)
        dropout_output = self.dropout(pooled_output)
        linear_output = self.linear(dropout_output)
        prediction = self.sigmoid(linear_output)
        return prediction
# Config setting
BATCH_SIZE = 4
EPOCHS = 5
# Making dataloaders
train_dataset =  torch.utils.data.TensorDataset(train_tokens_tensor, train_masks_tensor, train_y_tensor)
train_sampler =  torch.utils.data.RandomSampler(train_dataset)
train_dataloader =  torch.utils.data.DataLoader(train_dataset, sampler=train_sampler, batch_size=BATCH_SIZE)
test_dataset =  torch.utils.data.TensorDataset(test_tokens_tensor, test_masks_tensor, test_y_tensor)
test_sampler =  torch.utils.data.SequentialSampler(test_dataset)
test_dataloader =  torch.utils.data.DataLoader(test_dataset, sampler=test_sampler, batch_size=BATCH_SIZE)

bert_clf = BertBinaryClassifier()
bert_clf = bert_clf.cuda()
#wandb.watch(bert_clf)
optimizer = torch.optim.Adam(bert_clf.parameters(), lr=3e-6)

# training
for epoch_num in range(EPOCHS):
    bert_clf.train()
    train_loss = 0
    for step_num, batch_data in enumerate(train_dataloader):
        token_ids, masks, labels = tuple(t for t in batch_data)
        token_ids, masks, labels = token_ids.to(device), masks.to(device), labels.to(device)
        preds = bert_clf(token_ids, masks)
        loss_func = nn.BCELoss()
        batch_loss = loss_func(preds, labels)
        train_loss += batch_loss.item()
        bert_clf.zero_grad()
        batch_loss.backward()
        optimizer.step()
        #wandb.log({"Training loss": train_loss})
        print('Epoch: ', epoch_num + 1)
        print("\r" + "{0}/{1} loss: {2} ".format(step_num, len(train_data) / BATCH_SIZE, train_loss / (step_num + 1)))

# evaluating on test
bert_clf.eval()
bert_predicted = []
all_logits = []
probs=[]
with torch.no_grad():
    test_loss = 0
    for step_num, batch_data in enumerate(test_dataloader):
        token_ids, masks, labels = tuple(t for t in batch_data)
        token_ids, masks, labels = token_ids.to(device), masks.to(device), labels.to(device)
        logits = bert_clf(token_ids, masks)
        pr=logits.ravel()
        probs+=pr
        loss_func = nn.BCELoss()
        loss = loss_func(logits, labels)
        test_loss += loss.item()
        numpy_logits = logits.cpu().detach().numpy()
        #print(numpy_logits)
        #wandb.log({"Testing loss": test_loss})
        bert_predicted += list(numpy_logits[:, 0] > 0.5)
        all_logits += list(numpy_logits[:, 0])

I am able to get the prediction score to calculate the accuracy or f1 score. But not the probability for creating ROC curve.
Thanks


title: Where I can find an intuitive explanation of PyTorch's Tensor.unfold() being used to get image patches?

body:
Recently I came across some code that extracted (sliding-window style) a number of square patches from an RGB image (or set of them) of shape N x B x H x W. They did this as follows:
patch_width = 3
patches = image.permute(0,2,3,1).unfold(dim = 1, size = patch_width, stride = patch_width) \
        .unfold(dim = 2, size = patch_width, stride = patch_width)

I understand that the unfold() method "returns all all slices of size size from self tensor in the dimension dim," from reading the documentation, but try as I might, I just can't get a good intuition for why stacking two .unfold() calls produces square patches. I get what happens when you use unfold() once on a tensor. I don't get what happens when you call it twice successively along two different dimensions.
I've seen this approach used multiple times, always without a good explanation as to why it works (1, 2), and it's driving me bonkers. Why are the spatial dimensions H and W permuted to be dims 1 and 2, while the channel dim is set to 3? Why does unfolding the same way on dim 1, then on dim 2 result in square patch_width by patch_width patches?
Any insight would be hugely appreciated, even if it's just a link to an article I missed. I've been Googling for well over an hour now and have met very little success. Thank you!
[1]PyTorch forum post
[2]Another forum post doing the same thing


title: PyTorch - Efficient way to apply different functions to different 'row/column' of a tensor

body:
Let's say I have a 2-d tensor:
x = torch.Tensor([[1, 2], [3, 4]])

Is there an efficient way to apply one function to the first 'row' [1, 2] and apply a second different function to the second row [3, 4]? (Doesn't have to be a row, could be across any dimension)
At the moment, I use the following code: Say I have my two functions, f and g, for example,
def f(z):
    return 2 * z
def g(z):
    return 0.5 * z

Then, to apply them to seperate rows I would do:
torch.cat([f(x[0]).unsqueeze(0), g(x[1]).unsqueeze(0)], dim = 0)

which gives the desired tensor [[2, 4], [1.5, 2]].
Obviously, in this 2-d example this solution is fine, but it seems a bit clunky. Is there a better way of doing this? Particularly in higher dimensions or when there are a large number of elements in the chosen dimension


title: UNET with CrossEntropy Loss Function

body:
I was trying to train UNET with input size as [3,128,128] and the corresponding mask is [1,128,128] which contains classes directly(instead of pixels it will contain class numbers - 1,2). I am trying for a two-class problem hence my mask contains 1,2 as labels. Now I send my images to the model and the dimension of the predicted masks are [2,128,128]. Now to train a model I choose 16 as batch size. So, now I have input as [16,3,128,128] so the predicted dimension is [16,2,128,128]. But I have ground-truth masks as [16,1,128,128]. Now how can I apply Cross entropy loss in Pytorch? I have tried as follows and getting the following error. Could you please help? Thanks in advance.
lr = 0.1  # 0.1
criterion = nn.CrossEntropyLoss() #nn.L1Loss()
optimizer = optim.SGD(model.parameters(), lr=lr, momentum=0.9, nesterov=True, weight_decay=0.0001)
is_train = True
is_pretrain = False
acc_best = 0
total_epoch = 30
if is_train is True:
    # Training
    for epoch in range(total_epoch):
        model.train()
        tims = time.time()
        for i, (images, labels) in enumerate(train_loader):
            images = Variable(images.permute(0,3,1,2).float().cuda())
            labels = Variable(labels.type(torch.LongTensor).cuda())
            # Forward + Backward + Optimize
            optimizer.zero_grad()
            outputs = model(images)
            outputs = outputs.type(torch.float)
            print('predictedLabelsType:',outputs[0].type())
            print('ActualLabelsType:',labels[0].type())
            print('shape of predicted outputs:',outputs.shape)
            print('shape of groundtruth masks:',labels.shape)
            loss = criterion(outputs, labels)
            loss.backward()
            optimizer.step()

My output is as follows:
predictedLabelsType: torch.cuda.FloatTensor
ActualLabelsType: torch.cuda.LongTensor
shape of predicted outputs: torch.Size([16, 2, 128, 128])
shape of groundtruth masks: torch.Size([16, 1, 128, 128])
---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-8-b692a8d536a9> in <module>()
     52             print('shape of predicted outputs:',outputs.shape)
     53             print('shape of groundtruth masks:',labels.shape)
---> 54             loss = criterion(outputs, labels)
     55             loss.backward()
     56             optimizer.step()

3 frames
/usr/local/lib/python3.7/dist-packages/torch/nn/functional.py in nll_loss(input, target, weight, size_average, ignore_index, reduce, reduction)
   2385         ret = torch._C._nn.nll_loss(input, target, weight, _Reduction.get_enum(reduction), ignore_index)
   2386     elif dim == 4:
-> 2387         ret = torch._C._nn.nll_loss2d(input, target, weight, _Reduction.get_enum(reduction), ignore_index)
   2388     else:
   2389         # dim == 3 or dim > 4

RuntimeError: 1only batches of spatial targets supported (3D tensors) but got targets of size: : [16, 1, 128, 128]

Could you please suggest where is the mistake and how this CrossEntropyLoss() will work in Pytorch for image segmentation? What I am missing here? I have tried reshaping the target size as [16,128,128] which is leads to another error. Thanks a lot!


title: How to find input that maximizes output of a neural network using pytorch

body:
I have a pytorch network, that have been trained and weights are updated (complete training).
class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        self.fc1 = nn.Linear(1, H)
        self.fc2 = nn.Linear(1, H)
        self.fc3 = nn.Linear(H, 1)

    def forward(self, x, y):
        h1 = F.relu(self.fc1(x)+self.fc2(y))
        h2 = self.fc3(h1)
        return h2

After training, I want to maximize the output of the network with respect to input. In other words, I want to optimize the input to maximize the neural network output, without changing weights. How can I achieve that.
My trial, but it doesn't make sense:
in     = torch.autograd.Variable(x)
out    = Net(in)
grad   = torch.autograd.grad(out, input)


I face this error .. How fix in LSTM this error RuntimeError: input must have 3 dimensions, got 4?

      self.check_forward_args(input, hx, batch_sizes)
  File "/home/user/.conda/envs/renv/lib/python3.9/site-packages/torch/nn/modules/rnn.py", line 605, in check_forward_args
    self.check_input(input, batch_sizes)
  File "/home/user/.conda/envs/renv/lib/python3.9/site-packages/torch/nn/modules/rnn.py", line 198, in check_input
    raise RuntimeError(
RuntimeError: input must have 3 dimensions, got 4

lines of code
 st_gcn(256, 256, kernel_size, 1, **kwargs),
    ))

    # initialize parameters for edge importance weighting
    if edge_importance_weighting:
        self.edge_importance = nn.ParameterList([
            nn.Parameter(torch.ones(self.A.size()))
            for i in self.st_gcn_networks
        ])
    else:
        self.edge_importance = [1] * len(self.st_gcn_networks)

    self.lstm = nn.LSTM(input_size=256, hidden_size=128, num_layers=32, batch_first=True) #lstm

    # fcn for prediction
    self.fcn = nn.Conv2d(256, num_class, kernel_size=1)

here the error .. by the way I use GPU
when I comment these lines the error disappears but I have to apply forwad
        # forwad
        for gcn, importance in zip(self.st_gcn_networks, self.edge_importance):
            x, _ = gcn(x, self.A * importance)

        # global pooling
        x = F.avg_pool2d(x, x.size()[2:])
        x = x.view(N, M, -1, 1, 1).mean(dim=1)

        h_0 = torch.randn(32, 5, 128)
        c_0 = torch.randn(32, 5, 128)
        output, (hn, cn) = self.lstm(x, (h_0, c_0)) #lstm with input, hidden, and internal state
        #hn = hn.view(-1, self.hidden_size) #reshaping the data for Dense layer next
        # prediction
        x = self.fcn(hn)


title: How to build CUDA custom C++ extension for PyTorch without CUDA?

body:
I was tasked with creating a CI workflow for building a PyTorch CUDA extension for this application. Up until now, the application was deployed by creating the target AWS VM with a CUDA GPU, pushing all the sources there and running setup.py, but instead I want to do the build in our CI system and deploy pre-built binaries to the production environment.
When running setup.py in the CI system, I get the error "No CUDA GPUs are available" - which is true, there are no CUDA GPUs in the CI system. Is there a way to just build the CUDA extension without a CUDA GPU available?
This is the error message:
gcc -pthread -shared -B /usr/local/miniconda/envs/build/compiler_compat -L/usr/local/miniconda/envs/build/lib -Wl,-rpath=/usr/local/miniconda/envs/build/lib -Wl,--no-as-needed -Wl,--sysroot=/ /app/my-app/build/temp.linux-x86_64-3.6/my-extension/my-module.o -L/usr/local/miniconda/envs/build/lib/python3.6/site-packages/torch/lib -lc10 -ltorch -ltorch_cpu -ltorch_python -o build/lib.linux-x86_64-3.6/my-extension/my-module.cpython-36m-x86_64-linux-gnu.so
building 'my-extension.my-module._cuda_ext' extension
creating /app/my-app/build/temp.linux-x86_64-3.6/my-extension/src
Traceback (most recent call last):
  File "setup.py", line 128, in <module>
    'build_ext': BuildExtension
  File "/usr/local/miniconda/envs/build/lib/python3.6/site-packages/setuptools/__init__.py", line 153, in setup
    return distutils.core.setup(**attrs)
  File "/usr/local/miniconda/envs/build/lib/python3.6/distutils/core.py", line 148, in setup
    dist.run_commands()
  File "/usr/local/miniconda/envs/build/lib/python3.6/distutils/dist.py", line 955, in run_commands
    self.run_command(cmd)
  File "/usr/local/miniconda/envs/build/lib/python3.6/distutils/dist.py", line 974, in run_command
    cmd_obj.run()
  File "/usr/local/miniconda/envs/build/lib/python3.6/site-packages/setuptools/command/build_ext.py", line 79, in run
    _build_ext.run(self)
  File "/usr/local/miniconda/envs/build/lib/python3.6/distutils/command/build_ext.py", line 339, in run
    self.build_extensions()
  File "/usr/local/miniconda/envs/build/lib/python3.6/site-packages/torch/utils/cpp_extension.py", line 653, in build_extensions
    build_ext.build_extensions(self)
  File "/usr/local/miniconda/envs/build/lib/python3.6/distutils/command/build_ext.py", line 448, in build_extensions
    self._build_extensions_serial()
  File "/usr/local/miniconda/envs/build/lib/python3.6/distutils/command/build_ext.py", line 473, in _build_extensions_serial
    self.build_extension(ext)
  File "/usr/local/miniconda/envs/build/lib/python3.6/site-packages/setuptools/command/build_ext.py", line 196, in build_extension
    _build_ext.build_extension(self, ext)
  File "/usr/local/miniconda/envs/build/lib/python3.6/distutils/command/build_ext.py", line 533, in build_extension
    depends=ext.depends)
  File "/usr/local/miniconda/envs/build/lib/python3.6/site-packages/torch/utils/cpp_extension.py", line 468, in unix_wrap_ninja_compile
    cuda_post_cflags = unix_cuda_flags(cuda_post_cflags)
  File "/usr/local/miniconda/envs/build/lib/python3.6/site-packages/torch/utils/cpp_extension.py", line 377, in unix_cuda_flags
    cflags + _get_cuda_arch_flags(cflags) +
  File "/usr/local/miniconda/envs/build/lib/python3.6/site-packages/torch/utils/cpp_extension.py", line 1407, in _get_cuda_arch_flags
    capability = torch.cuda.get_device_capability()
  File "/usr/local/miniconda/envs/build/lib/python3.6/site-packages/torch/cuda/__init__.py", line 291, in get_device_capability
    prop = get_device_properties(device)
  File "/usr/local/miniconda/envs/build/lib/python3.6/site-packages/torch/cuda/__init__.py", line 296, in get_device_properties
    _lazy_init()  # will define _get_device_properties
  File "/usr/local/miniconda/envs/build/lib/python3.6/site-packages/torch/cuda/__init__.py", line 172, in _lazy_init
    torch._C._cuda_init()
RuntimeError: No CUDA GPUs are available

I'm not very familiar with CUDA and only half proficient in Python (I'm here as the "ops" part of "devops").


title: How can I more efficiently multiply every element in a batch of tensors with every other batch element, except for itself?

body:
So, I have this code that multiplies every element in a batch of tensors with every other element, except for itself. The code works, but it becomes painfully slow with larger batch sizes (Ideally I want to be able to use it with batch sizes of up to 1000 or more, but even a couple hundred is okay). It basically freezes when using the PyTorch autograd system and large batch sizes (like 50 or greater).
I need help making the code faster and more efficient, while still getting the same output. Any help would be appreciated!
import torch

tensor = torch.randn(50, 512, 512)

batch_size = tensor.size(0)
list1 = []
for i in range(batch_size):
    list2 = []
    for j in range(batch_size):
        if j != i:
            x_out = (tensor[i] * tensor[j]).sum()
            list2.append(x_out )
    list1.append(sum(list2))

out = sum(list1)

I thought that torch.prod might be able to be used, but it doesn't seem to result in the same output as the code above. NumPy answers are acceptable as long as they can be recreated in PyTorch.


title: How to create n-dimensional sparse tensor? (pytorch)

body:
I want to initialize tensor to sparse tensor.
When tensor's dimensional is 2, I can use torch.nn.init.sparse(tensor, sparsity=0.1)
import torch
dim = torch.Size([3,2])
w = torch.Tensor(dim)
torch.nn.init.sparse_(w, sparsity=0.1)

Result
tensor([[ 0.0000,  0.0147],
        [-0.0190,  0.0004],
        [-0.0004,  0.0000]])

But when tensor dimensions > 2, this function isn't work.
v = torch.Tensor(torch.Size([5,5,30,2]))
torch.nn.init.sparse_(v, sparsity=0.1)

Result
ValueError: Only tensors with 2 dimensions are supported

I need this because I want to use it to initialize the convolution weights.
torch.nn.init.sparse_() function's def is below
def sparse_(tensor, sparsity, std=0.01):
    r"""Fills the 2D input `Tensor` as a sparse matrix, where the
    non-zero elements will be drawn from the normal distribution
    :math:`\mathcal{N}(0, 0.01)`, as described in `Deep learning via
    Hessian-free optimization` - Martens, J. (2010).

    Args:
        tensor: an n-dimensional `torch.Tensor`
        sparsity: The fraction of elements in each column to be set to zero
        std: the standard deviation of the normal distribution used to generate
            the non-zero values

    Examples:
        >>> w = torch.empty(3, 5)
        >>> nn.init.sparse_(w, sparsity=0.1)
    """
    if tensor.ndimension() != 2:
        raise ValueError("Only tensors with 2 dimensions are supported")

    rows, cols = tensor.shape
    num_zeros = int(math.ceil(sparsity * rows))

    with torch.no_grad():
        tensor.normal_(0, std)
        for col_idx in range(cols):
            row_indices = torch.randperm(rows)
            zero_indices = row_indices[:num_zeros]
            tensor[zero_indices, col_idx] = 0
    return tensor

How could I make n-dimensional sparse tensor?
Is there a way in pytorch to create this kind of tensor?
Or can I make it another way?


title: how can I load pretrained model by pytorch? ( mmfashion)

body:
import io

import torch
import torch.nn as nn
from torchvision import models
from PIL import Image
import torchvision.transforms as transforms

checkpoint_path = 'C:/venvs/ai/aiproduct/latest.pth'
pretrained_weights = torch.load(checkpoint_path, map_location='cpu', strict=False)

model = models.resnet50(pretrained=True)
model.load_state_dict(pretrained_weights)

this bring
TypeError: 'strict' is an invalid keyword argument for load()

and
import io

import torch
import torch.nn as nn
from torchvision import models
from PIL import Image
import torchvision.transforms as transforms

checkpoint_path = 'C:/venvs/ai/aiproduct/latest.pth'
pretrained_weights = torch.load(checkpoint_path, map_location='cpu')

model = models.resnet50(pretrained=True)
model.load_state_dict(pretrained_weights)
# model.eval()
print(model)
# model.summary()

if I get rid of "strict"
Traceback (most recent call last):
  File "c:\venvs\ai\aiproduct\test.py", line 13, in <module>
    model.load_state_dict(pretrained_weights)
  File "C:\Python39\lib\site-packages\torch\nn\modules\module.py", line 1223, in load_state_dict
    raise RuntimeError('Error(s) in loading state_dict for {}:\n\t{}'.format(
RuntimeError: Error(s) in loading state_dict for ResNet:
        Missing key(s) in state_dict: "conv1.weight", "bn1.weight", "bn1.bias", "bn1.running_mean", "bn1.running_var", "layer1.0.conv1.weight", "layer1.0.bn1.weight", "layer1.0.bn1.bias", "layer1.0.bn1.running_mean", "layer1.0.bn1.running_var", "layer1.0.conv2.weight", "layer1.0.bn2.weight", "layer1.0.bn2.bias", "layer1.0.bn2.running_mean", "layer1.0.bn2.running_var", "layer1.0.conv3.weight", "layer1.0.bn3.weight", "layer1.0.bn3.bias", "layer1.0.bn3.running_mean", "layer1.0.bn3.running_var", "layer1.0.downsample.0.weight", "layer1.0.downsample.1.weight", "layer1.0.downsample.1.bias", "layer1.0.downsample.1.running_mean", "layer1.0.downsample.1.running_var", "layer1.1.conv1.weight", "layer1.1.bn1.weight", "layer1.1.bn1.bias", "layer1.1.bn1.running_mean", "layer1.1.bn1.running_var", "layer1.1.conv2.weight", "layer1.1.bn2.weight", "layer1.1.bn2.bias", "layer1.1.bn2.running_mean",
"layer1.1.bn2.running_var", "layer1.1.conv3.weight", "layer1.1.bn3.weight", "layer1.1.bn3.bias", "layer1.1.bn3.running_mean", "layer1.1.bn3.running_var", "layer1.2.conv1.weight", "layer1.2.bn1.weight", "layer1.2.bn1.bias", "layer1.2.bn1.running_mean", "layer1.2.bn1.running_var", "layer1.2.conv2.weight", "layer1.2.bn2.weight", "layer1.2.bn2.bias", "layer1.2.bn2.running_mean", "layer1.2.bn2.running_var", "layer1.2.conv3.weight", "layer1.2.bn3.weight", "layer1.2.bn3.bias", "layer1.2.bn3.running_mean", "layer1.2.bn3.running_var", "layer2.0.conv1.weight", "layer2.0.bn1.weight", "layer2.0.bn1.bias", "layer2.0.bn1.running_mean", "layer2.0.bn1.running_var", "layer2.0.conv2.weight", "layer2.0.bn2.weight", "layer2.0.bn2.bias", "layer2.0.bn2.running_mean", "layer2.0.bn2.running_var", "layer2.0.conv3.weight", "layer2.0.bn3.weight", "layer2.0.bn3.bias", "layer2.0.bn3.running_mean", "layer2.0.bn3.running_var", "layer2.0.downsample.0.weight", "layer2.0.downsample.1.weight", "layer2.0.downsample.1.bias", "layer2.0.downsample.1.running_mean", "layer2.0.downsample.1.running_var", "layer2.1.conv1.weight", "layer2.1.bn1.weight", "layer2.1.bn1.bias", "layer2.1.bn1.running_mean", "layer2.1.bn1.running_var", "layer2.1.conv2.weight", "layer2.1.bn2.weight", "layer2.1.bn2.bias", "layer2.1.bn2.running_mean", "layer2.1.bn2.running_var", "layer2.1.conv3.weight", "layer2.1.bn3.weight", "layer2.1.bn3.bias", "layer2.1.bn3.running_mean", "layer2.1.bn3.running_var", "layer2.2.conv1.weight", "layer2.2.bn1.weight", "layer2.2.bn1.bias", "layer2.2.bn1.running_mean", "layer2.2.bn1.running_var", "layer2.2.conv2.weight", "layer2.2.bn2.weight", "layer2.2.bn2.bias", "layer2.2.bn2.running_mean", "layer2.2.bn2.running_var", "layer2.2.conv3.weight", "layer2.2.bn3.weight", "layer2.2.bn3.bias", "layer2.2.bn3.running_mean", "layer2.2.bn3.running_var", "layer2.3.conv1.weight", "layer2.3.bn1.weight", "layer2.3.bn1.bias", "layer2.3.bn1.running_mean", "layer2.3.bn1.running_var", "layer2.3.conv2.weight", "layer2.3.bn2.weight", "layer2.3.bn2.bias", "layer2.3.bn2.running_mean", "layer2.3.bn2.running_var", "layer2.3.conv3.weight", "layer2.3.bn3.weight", "layer2.3.bn3.bias", "layer2.3.bn3.running_mean", "layer2.3.bn3.running_var", "layer3.0.conv1.weight", "layer3.0.bn1.weight", "layer3.0.bn1.bias", "layer3.0.bn1.running_mean", "layer3.0.bn1.running_var", "layer3.0.conv2.weight", "layer3.0.bn2.weight", "layer3.0.bn2.bias", "layer3.0.bn2.running_mean", "layer3.0.bn2.running_var", "layer3.0.conv3.weight", "layer3.0.bn3.weight", "layer3.0.bn3.bias", "layer3.0.bn3.running_mean", "layer3.0.bn3.running_var", "layer3.0.downsample.0.weight", "layer3.0.downsample.1.weight", "layer3.0.downsample.1.bias", "layer3.0.downsample.1.running_mean", "layer3.0.downsample.1.running_var", "layer3.1.conv1.weight", "layer3.1.bn1.weight", "layer3.1.bn1.bias", "layer3.1.bn1.running_mean", "layer3.1.bn1.running_var", "layer3.1.conv2.weight", "layer3.1.bn2.weight", "layer3.1.bn2.bias", "layer3.1.bn2.running_mean", "layer3.1.bn2.running_var", "layer3.1.conv3.weight", "layer3.1.bn3.weight", "layer3.1.bn3.bias", "layer3.1.bn3.running_mean", "layer3.1.bn3.running_var", "layer3.2.conv1.weight", "layer3.2.bn1.weight", "layer3.2.bn1.bias", "layer3.2.bn1.running_mean", "layer3.2.bn1.running_var", "layer3.2.conv2.weight", "layer3.2.bn2.weight", "layer3.2.bn2.bias", "layer3.2.bn2.running_mean", "layer3.2.bn2.running_var", "layer3.2.conv3.weight", "layer3.2.bn3.weight", "layer3.2.bn3.bias", "layer3.2.bn3.running_mean", "layer3.2.bn3.running_var", "layer3.3.conv1.weight", "layer3.3.bn1.weight", "layer3.3.bn1.bias", "layer3.3.bn1.running_mean", "layer3.3.bn1.running_var", "layer3.3.conv2.weight", "layer3.3.bn2.weight", "layer3.3.bn2.bias", "layer3.3.bn2.running_mean", "layer3.3.bn2.running_var", "layer3.3.conv3.weight", "layer3.3.bn3.weight", "layer3.3.bn3.bias", "layer3.3.bn3.running_mean", "layer3.3.bn3.running_var", "layer3.4.conv1.weight", "layer3.4.bn1.weight", "layer3.4.bn1.bias", "layer3.4.bn1.running_mean", "layer3.4.bn1.running_var", "layer3.4.conv2.weight", "layer3.4.bn2.weight", "layer3.4.bn2.bias", "layer3.4.bn2.running_mean", "layer3.4.bn2.running_var", "layer3.4.conv3.weight", "layer3.4.bn3.weight", "layer3.4.bn3.bias", "layer3.4.bn3.running_mean", "layer3.4.bn3.running_var",
"layer3.5.conv1.weight", "layer3.5.bn1.weight", "layer3.5.bn1.bias", "layer3.5.bn1.running_mean", "layer3.5.bn1.running_var", "layer3.5.conv2.weight", "layer3.5.bn2.weight", "layer3.5.bn2.bias", "layer3.5.bn2.running_mean", "layer3.5.bn2.running_var", "layer3.5.conv3.weight", "layer3.5.bn3.weight", "layer3.5.bn3.bias", "layer3.5.bn3.running_mean", "layer3.5.bn3.running_var", "layer4.0.conv1.weight", "layer4.0.bn1.weight", "layer4.0.bn1.bias", "layer4.0.bn1.running_mean", "layer4.0.bn1.running_var", "layer4.0.conv2.weight", "layer4.0.bn2.weight", "layer4.0.bn2.bias", "layer4.0.bn2.running_mean", "layer4.0.bn2.running_var", "layer4.0.conv3.weight", "layer4.0.bn3.weight", "layer4.0.bn3.bias", "layer4.0.bn3.running_mean", "layer4.0.bn3.running_var", "layer4.0.downsample.0.weight", "layer4.0.downsample.1.weight", "layer4.0.downsample.1.bias", "layer4.0.downsample.1.running_mean", "layer4.0.downsample.1.running_var", "layer4.1.conv1.weight", "layer4.1.bn1.weight", "layer4.1.bn1.bias", "layer4.1.bn1.running_mean", "layer4.1.bn1.running_var", "layer4.1.conv2.weight", "layer4.1.bn2.weight", "layer4.1.bn2.bias", "layer4.1.bn2.running_mean", "layer4.1.bn2.running_var", "layer4.1.conv3.weight", "layer4.1.bn3.weight", "layer4.1.bn3.bias",
"layer4.1.bn3.running_mean", "layer4.1.bn3.running_var", "layer4.2.conv1.weight", "layer4.2.bn1.weight", "layer4.2.bn1.bias", "layer4.2.bn1.running_mean", "layer4.2.bn1.running_var", "layer4.2.conv2.weight", "layer4.2.bn2.weight", "layer4.2.bn2.bias", "layer4.2.bn2.running_mean", "layer4.2.bn2.running_var", "layer4.2.conv3.weight", "layer4.2.bn3.weight", "layer4.2.bn3.bias", "layer4.2.bn3.running_mean", "layer4.2.bn3.running_var", "fc.weight", "fc.bias".
        Unexpected key(s) in state_dict: "meta", "state_dict", "optimizer".

what should I do?
I just want to make some cloth attribute predicting web app with pretrained model (mmfashion https://github.com/open-mmlab/mmfashion/blob/master/docs/MODEL_ZOO.md)
but I fail to use pretrained model.


title: Different python versions in the same environment of Anaconda

body:
this is what the cmd shows:

C:\Users\74713\Documents>activate root

(root) C:\Users\74713\Documents>python
Python 3.6.2 |Anaconda custom (64-bit)| (default, Jul 20 2017, 12:30:02) [MSC v.1900 64 bit (AMD64)] on win32
Type "help", "copyright", "credits" or "license" for more information.
>>>

this is what my IDE shows:

Python 3.6.3 |Anaconda custom (64-bit)| (default, Oct 15 2017, 03:27:45) [MSC v.1900 64 bit (AMD64)]
Type "copyright", "credits" or "license" for more information.

IPython 6.1.0 -- An enhanced Interactive Python.

the first one's version is 3.6.2 while the second one is 3.6.3

and because of that, I meet trouble in import torchtext, which works well in the cmd(3.6.2) while fails in my IDE(3.6.3)


title: Constrain kernel filters to only have entries of -1, 0 or 1?

body:
My autoencoder model learns the most common patterns in images with pixel values -1, 0 or 1. While my model runs well, the decoding falls into a local minima with float values between -1 and 1.

Is there anyway in Pytorch I can constrain the kernel filters to have entries -1, 0 or 1 to observe better learning capabilities?


title: Error while configuring pytorch-kaldi on Linux Mint 19.1 cinnamon

body:
I am trying to configure Pytorch-Kaldi on a Linux Mint cinnamon 19.1.

I can not solve this error :

ZOE ERROR (from hmm-info): error opening HMM file ((null)/HMM/models)
ZOE library version 2013-02-16


title: Transform List to Tensor more accurat

body:
I want to return in the Dataloader a list.
But to return it, its need to be a tensor right?
So I transform it but in this process information is lost, is there another way to do this?

pt_tensor_from_list = torch.tensor(pose_transform)
pt_tensor_from_list = torch.FloatTensor(pose_transform)

I excpect the output:

  ([[-0.0003000000142492354, -0.0008999999845400453,
  0.00039999998989515007, 0], [0.0010000000474974513, -0.00019999999494757503, 0.0003000000142492354, 0], [0.00019999999494757503, -0.0005000000237487257,
  -0.0008999999845400453, 0], [5.484399795532227, -24.28619956970215, 117.5000991821289, 1])

But it is:

  ([[ -0.0003, -0.0009, 0.0004, 0.0000],
      [ 0.0010, -0.0002, 0.0003, 0.0000],
      [ 0.0002, -0.0005, -0.0009, 0.0000],
      [ 5.4844, -24.2862, 117.5001, 1.0000]])


title: Why do I converge only using a parameter wrapper in PyTorch?

body:
I'm writing an implementation of Bayesian Dropout, and the following code worked after several hours of debugging:

class DropoutLayer(nn.Module):

    def __init__(self, n_in, n_out, dropout_rate):
        super(DropoutLayer,self).__init__()
        self.M = Parameter(torch.normal(torch.zeros(n_in,n_out),0.01))
        self.m = Parameter(torch.rand(n_out))

        # 1-p is used to be consistent with original dropout definition
        self.z = Bernoulli(torch.tensor([1-dropout_rate]))

        self.W = Parameter(torch.mm(
        torch.diagflat(self.z.sample(sample_shape=(n_in,))),
        self.M))

    def forward(self,x,act):
        activation = torch.mm(x,self.W) + self.m

        out = act(activation)
        return out

However, I do not seem to understand, why the matrix multiplication in self.W needs a parameter wrapping. I would assume once I nominate self.M to be a parameter in autograd, I would not need to do it for any values which use it further down the line. Why do I need it here?

After adding this wrapper, the neural net converges without a problem.
Could you tell me why is it the case and if there is a simpler substitute for this solution which I don't like?


title: Is there a function to efficiently subtract two matrices and return a tensor?

body:
I want to efficiently reimplement the OpenCV BruteForce Matcher in pytorch. This means I get two matrices containing feature descriptors and I have to compute the distances between them.
Currently I do it like this:

m, n, d = 10000, 1000, 2
x = torch.Tensor(size=(m, d))
y = torch.Tensor(size=(n, d))
dist = torch.sum(torch.stack([x-y_i for y_i in y]), (2,)) ##slow
dist = dist.numpy()
matches = np.argsort(dist)[:,:2]
good_matches = []
for i, (a, b) in enumerate(matches):
    if dist[i,b] > ratio*dist[i, a]:
        good_matches.append([i,a])
return good_matches

Is there any operation to speed this up? Is pytorch multiprocessing a good way to go?


title: Assign Keras/TF/PyTorch layer to hardware type

body:
Suppose we have the following architecture:

Multiple CNN layers
RNN layer
(Time-distributed) Dense classification layer

We want to train this architecture now. Our fancy GPU is very fast at solving the CNN layers. Although using a lower clockrate, it can perform many convolutions in parallel, thus the speed. Our fancy CPU however is faster for the (very long) resulting time series, because the time steps cannot be parallelized, and the processing profits from the higher CPU clockrate. So the (supposedly) smart idea for execution would look like this:

Multiple CNN layers (run on GPU)
RNN layer (run on CPU)
(Time-distributed) Dense classification layer (run on GPU/CPU)

This lead me to two important questions:

Is it possible, with any of the frameworks mentioned in the title, to distribute certain layers to certain hardware, and how?

If it is possible, would the overhead for the additional memory operations, e.g. tranferring between GPU-/CPU-RAM, render the whole idea useless?


title: Replicating Semantic Analysis Model in Demo

body:
Good day, I am a student that is interested in NLP. I have come across the demo on  AllenNLP's homepage, which stated that:

  The model is a simple LSTM using GloVe embeddings that is trained on the binary classification setting of the Stanford Sentiment Treebank. It achieves about 87% accuracy on the test set.

Is there any reference to the sample code or any tutorial that I can follow to replicate this result, so that I can learn more about this subject? I am trying to obtain a Regression Output (Instead of classification).

I hope that someone can point me in the right direction.. Any help is much appreciated. Thank you!


title: Test Loss looks weird when plotted

body:
I am using LSTM in-order to perform binary-classification, when I plot the test-loss it is not reducing over time.It is rather fluctuating a lot over time and looks extremely weird.The training loss on the other hand looks normal and is decreasing over time.
 Here's a Picture of how it looks

This is my code of the model definition and configuration.

        # Create LSTM Model
class LSTMModel(nn.Module):
  def __init__(self, input_dim, hidden_dim, layer_dim, output_dim):
    super(LSTMModel, self).__init__()
    # Number of hidden dimensions
    self.hidden_dim = hidden_dim

    # Number of hidden layers
    self.layer_dim = layer_dim

    # LSTM
    self.lstm = nn.LSTM(input_dim, hidden_dim, layer_dim, batch_first=True, dropout=0.2)

    # Readout layer
    self.f1 = nn.Linear(hidden_dim, output_dim)
    self.softmax = nn.Sigmoid()

  def forward(self, x):
    # Initialize hidden state with zeros
    h0 = Variable(torch.zeros(self.layer_dim, x.size(0), self.hidden_dim).type(torch.FloatTensor).cuda())

    # Initialize cell state
    c0 = Variable(torch.zeros(self.layer_dim, x.size(0), self.hidden_dim).type(torch.FloatTensor).cuda())
    out, (hn, cn) = self.lstm(x, (h0,c0))
    out = self.f1(hn[-1])
    out = self.softmax(out)
    return out

#LSTM Configuration
batch_size = 10000
num_epochs = 200
learning_rate = 0.001#Try lowering the rate

# Create LSTM
input_dim = 1       # input dimension
hidden_dim = 50  # hidden layer dimension
layer_dim =2 # number of hidden layers
output_dim = 1      # output dimension

model = LSTMModel(input_dim, hidden_dim, layer_dim, output_dim)

model.cuda()
error = nn.BCELoss()
optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)

This is my code for training and testing

from tensorboardcolab import TensorBoardColab
globaliter = 0
globaliter2=0
tb = TensorBoardColab()
for epoch in tqdm(range(num_epochs)):
      # Train
    model.train()
    for i, (inputs, targets) in enumerate(train_loader):
        train  = Variable(inputs.type(torch.FloatTensor).cuda())
        targets = Variable(targets.type(torch.FloatTensor).cuda())

        optimizer.zero_grad()
        outputs = model(train)
        loss = error(outputs, targets)
        loss_list_train.append(loss.item())
        loss.backward()
        optimizer.step()
        tb.save_value('Train Loss', 'train_loss', globaliter, loss.item())
        globaliter += 1
        tb.flush_line('train_loss')

      # Test
      model.eval()
      for inputs, targets in test_loader:

        inputs = Variable(inputs.type(torch.FloatTensor).cuda())
        targets = Variable(targets.type(torch.FloatTensor).cuda())
        outputs = model(inputs)
        loss_test = error(outputs, targets)
        loss_list_test.append(loss_test.item())
        tb.save_value('Test Loss', 'test_loss', globaliter2, loss_test.item())
        globaliter2 += 1
        tb.flush_line('test_loss')

I'd really be grateful if someone helped me figure this out, or offered suggestions or advice


title: FCN with patches creates boundary

body:
I am trying to train a Unet model to do per pixel regression predictions on images. To do this, I separate my large image (1000x1000) to 200x200 pixel squares. Then use that to train an FCN model with a linear final layer. The loss function is MSE loss. In the prediction stage, I extract the same boxes but stitch it together and obtain a final output image. When I do that, the problem I am getting is that there is discontinuities between the boundary of boxes. (I can clearly see the boxes)

I've tried to deal with this by feeding 250x250 boxes to my FCN and calculating the loss for the 200x200 centre region. I do the same process for the prediction state. Extract 250x250 patches crop the 200x200 centre region and stitch the image back together. Please see some code below:

Loss Function:

criterion = nn.MSELoss()
optimizer = optim.Adam(self.model.parameters(), lr=LR)
for inputs, labels in train_loader:
    inputs, labels = inputs.to(device), labels.to(device)
    optimizer.zero_grad()
    output = model(inputs)
    output = output.squeeze()
     _, dimx, dimy = output.shape
     loss = criterion(output[:,25:dimx-25, 25:dimy-25], labels[:,25:dimx-25, 25:dimy-25])
     loss.backward()
     optimizer.step()

My code for predictions is as follows:

pred = np.zeros((height, width))
for i in range(25, height, 200):
    for j in range(25, width, 200):
        patch = img[:, i-25:i+225, j-25:j+225]
        patch = torch.from_numpy(patch)
        patch = patch.unsqueeze(dim=0).to(device)
        out = model(patch)
        out = out[0,0,25:225, 25:225]
        pred[i:i+200, j:j+200] = out.cpu().numpy()

I'm not sure if my problem makes complete sense. I can provide more clarification if necessary but I have been stuck on this for a while now.


title: Pytorch custom dataset not returning tabular data

body:
i’m trying to develop an image model that takes categorical embeddings and continous data to help inform the classification.
I added to my image data class to help pull in the other data but it is now working.

class image_Data_Dataset(Dataset):
    '''
    image class data set

    '''
    def __init__(self, data, cont, cat, transform = None):
        '''
        Args:
        ------------------------------------------------------------
            data = dataframe
            image = column in dataframe with absolute path to the image
            cont = list of continuous data columns
            cat = list of categorical data columns
            policy = ID variable

        '''
        #data frame
        self.image_frame = data

        #transform
        self.transform = transform

        #categorical data
        self.categorical = np.stack(pd.get_dummies(self.image_frame, columns=cat, drop_first=True).iloc[:, 15:].values)

        #numerical data
        self.features = self.image_frame[cont]
        scaler = StandardScaler().fit(self.features.values)
        self.numerical = scaler.transform(self.features.values)

    def __len__(self):
        return len(self.image_frame)

    def __getitem__(self, idx):
        if torch.is_tensor(idx):
            idx = idx.tolist()

        label = self.image_frame.iloc[idx, 15]
        pic = Path(self.image_frame.iloc[idx,18])
        img = Image.open(pic)
        policy = self.image_frame.iloc[idx, 0]
        categorical_data = self.categorical_data[idx]
        numerical_data = self.numerical_data[idx]
        sample = {'image': img, 'policy': policy, 'label':label, 'categorical_data': categorical_data
                  ,'numerical_data': numerical_data}

        if self.transform:
            image = self.transform(img)

        #return image, label, policy
        return categorical_data, numerical_data, label

Here is the error:

TypeError Traceback (most recent call last)
in
1 roof = image_Data_Dataset(train_test_split, cont = [‘age’, ‘aoh’,‘lmt01’, ‘totl_sq_ft_cnt’]
----> 2 , cat = [‘liablmt’, ‘deductible’, ‘dw_style_class’, ‘fnclrspn’], transform = None)
in init(self, data, cont, cat, transform)
22
23 #categorical data
—> 24 self.categorical = np.stack(pd.get_dummies(self.image_frame, columns=cat, drop_first=True).iloc[:, 15:].values)
25
26 #numerical data
C:\ProgramData\Anaconda3\lib\site-packages\pandas\core\reshape\reshape.py in get_dummies(data, prefix, prefix_sep, dummy_na, columns, sparse, drop_first, dtype)
864 sparse=sparse,
865 drop_first=drop_first,
–> 866 dtype=dtype)
867 return result
868
C:\ProgramData\Anaconda3\lib\site-packages\pandas\core\reshape\reshape.py in _get_dummies_1d(data, prefix, prefix_sep, dummy_na, sparse, drop_first, dtype)
899 # if dummy_na, we just fake a nan level. drop_first will drop it again
900 if drop_first and len(levels) == 1:
–> 901 return get_empty_frame(data)
902
903 number_of_cols = len(levels)
C:\ProgramData\Anaconda3\lib\site-packages\pandas\core\reshape\reshape.py in get_empty_frame(data)
885 index = data.index
886 else:
–> 887 index = np.arange(len(data))
888 return DataFrame(index=index)
889
TypeError: object of type ‘function’ has no len()

This custom data class is baffling me. Does anyone see what i’m doing wrong?


title: PyTorch CNN never converges (implementation issue suspected)

body:
I am having trouble getting this network to work as desired. I have tried so many iterations of this model and yet cannot get a reasonable error (it never fits, can’t even get it to overfit).

Where have I gone wrong? Any help would be greatly appreciated

For reference, there are 12 input ‘images’ (they’re actually water surface elevation at 9 stations in an estuary) of shape 49,9 and 12 labels of shape 1,9.

Full examples with data can be found at https://gitlab.com/jb4earth/effonn/

  net = []
  class Net(torch.nn.Module):
      def __init__(self, kernel_size):
          super(Net, self).__init__()
          mid_size = (49*49*9)
          self.predict = torch.nn.Sequential(
              nn.Conv2d(
                          in_channels=1,
                          out_channels=mid_size,
                          kernel_size=kernel_size,
                          stride=1,
                          padding=(0, 0)
                      ),
              nn.ReLU(),
              nn.MaxPool2d(1),
              nn.ReLU(),
              nn.Conv2d(
                          in_channels=mid_size,
                          out_channels=1,
                          kernel_size=kernel_size,
                          stride=1,
                          padding=(0, 0)
                      ),
              nn.ReLU()
          )

      def forward(self, x):
          x = self.predict(x)
          return x

  def train_network(x,y,optimizer,loss_func):
      prediction = net(x)
      loss = loss_func(prediction, y.squeeze())
      optimizer.zero_grad()
      loss.backward()
      optimizer.step()
      return prediction, loss

  net = Net((1,1))
  optimizer = torch.optim.Adam(net.parameters(), lr=0.01)
  loss_func = torch.nn.MSELoss()
  cnt = 0
  t = True
  while t == True:
      # get_xy in place of DataLoader
      (x,y) = get_xy(input_data,output_data,cnt)
      # x.shape is 1,1,49,9
      # y.shape is 1,1,1,9

      # train and predict
      (prediction,loss) = train_network(x,y,optimizer,loss_func)

      # prediction shape different than desired so averaging all results
      prediction_ = torch.mean(prediction)

      # only 12 IO's so loop through
      cnt += 1
      if cnt > 11:
          cnt = 0