title: Best input size of a the first layer 1D CNN

body:
I'm trying to build a 1D CNN with time series. The input is of length 500. There are (only) 2 labels. The architecture which I built so far is the following: there are 3 convolution layers each, of them followed by an activation layer. The first convolution layer takes 50 channels as input.

import torch
import torch.nn as nn
import numpy as np
import random

class Simple1DCNN3(torch.nn.Module):
    def __init__(self):
        super(Simple1DCNN5, self).__init__()
        self.sequence = nn.Sequential(
            torch.nn.Conv1d(in_channels=50,
                                          out_channels=64,
                                          kernel_size=5,
                                          stride=2),
            torch.nn.ReLU(),
            torch.nn.Conv1d(in_channels=64,
                                          out_channels=128,
                                          kernel_size=3),
            torch.nn.ReLU(),
            torch.nn.Conv1d(in_channels=128,
                                          out_channels=256,
                                          kernel_size=1),
            torch.nn.ReLU(),
        )
        self.fc1 = nn.Linear(256, 2)

    def forward(self, x):
        x = x.view(1, 50,-1)

        for layer in self.sequence:
            x = layer(x)
            print(x.size())
        x = x.view(1,-1)
        #print(x.size())
        x = self.fc1(x)
        #print(x.size())
        return x

net = Simple1DCNN3()

input_try = np.random.uniform(-10, 10, 500)
input_try = torch.from_numpy(input_try).float()
net(input_try)
print("input successfull passed to net")
input_try_modif = input_try.view(1, 50,-1)
print(input_try.shape)
print(input_try_modif.shape)

As far as I understood, that forced me to segment the input in 10 segments of 50 timepoints. Am I understanding it wrong ? Wouldn't it be wiser to construct the first layer with 500 channels as inputs and have a sliding window kernel? I tried it in the following other script but got the following error message

import torch
import torch.nn as nn
import numpy as np
import random

class Simple1DCNN4(torch.nn.Module):
    def __init__(self):
        super(Simple1DCNN5, self).__init__()
        self.sequence = nn.Sequential(
            torch.nn.Conv1d(in_channels=500,
                                          out_channels=64,
                                          kernel_size=5,
                                          stride=2),
            torch.nn.ReLU(),
            torch.nn.Conv1d(in_channels=64,
                                          out_channels=128,
                                          kernel_size=3),
            torch.nn.ReLU(),
            torch.nn.Conv1d(in_channels=128,
                                          out_channels=256,
                                          kernel_size=1),
            torch.nn.ReLU(),
        )
        self.fc1 = nn.Linear(256, 2)

    def forward(self, x):
        x = x.view(1, 50,-1)

        for layer in self.sequence:
            x = layer(x)
            print(x.size())
        x = x.view(1,-1)
        #print(x.size())
        x = self.fc1(x)
        #print(x.size())
        return x

net = Simple1DCNN4()

input_try = np.random.uniform(-10, 10, 500)
input_try = torch.from_numpy(input_try).float()
net(input_try)
print("input successfull passed to net")
input_try_modif = input_try.view(1, 50,-1)
print(input_try.shape)
print(input_try_modif.shape)

Error message:
RuntimeError: Given groups=1, weight of size [64, 500, 5], expected input[1, 50, 10] to have 500 channels, but got 50 channels instead

EDIT
Thanks to the answer of @ghchoi, here is the code of the working kernel. For this, I also had to change the kernel size to all the convolutional layers to 1.
class Simple1DCNN5(torch.nn.Module):
    def __init__(self):
        super(Simple1DCNN5, self).__init__()
        self.sequence = nn.Sequential(
            torch.nn.Conv1d(in_channels=500,
                                          out_channels=64,
                                          kernel_size=1,
                                          stride=2),
            torch.nn.ReLU(),
            torch.nn.Conv1d(in_channels=64,
                                          out_channels=128,
                                          kernel_size=1),
            torch.nn.ReLU(),
            torch.nn.Conv1d(in_channels=128,
                                          out_channels=256,
                                          kernel_size=1),
            torch.nn.ReLU(),
        )
        self.fc1 = nn.Linear(256, 2)

    def forward(self, x):
        x = x.view(1, 500,-1)

        for layer in self.sequence:
            x = layer(x)
            #print(x.size())
        x = x.view(1,-1)
        #print(x.size())
        x = self.fc1(x)
        #print(x.size())
        return x

The kind of data I have is mono-derivation ECG (electrocardiogram) signal of 2 seconds. This is a recording of the electrical signal of the hear. Here is the idea of what a sample could look like (ploted on a 2D graphs), where you have time on the x-axis and voltage/amplitude on the y-axis


title: Speeding-up the computation of second-order derivatives in Pytorch

body:
I'd like to compute the first- and second-order derivatives of a neural network with respect to its input. As a toy model, I defined a custom layer with a method called "derivative" that does exactly this:
import torch
from torch import nn
from torch.autograd import grad
import numpy as np

class Exponential(nn.Module):

    def __init__(self):
        super().__init__()
        self.beta = nn.Parameter(nn.init.uniform_(torch.empty(1), 0.1, 2.0), requires_grad=True)

    def forward(self, x):
        return torch.exp(-self.beta*x**2)

    def derivative(self, X):
        # define zero-vectors for first and second order derivatives
        grads_1, grads_2 = torch.zeros_like(X), torch.zeros_like(X)
        # loop over elements of the input vector
        for i in np.arange(X.shape[0]):
            x = X[i].requires_grad_(True)
            y = self.forward(x).requires_grad_(True)
            # compute first-order derivative
            grads_1[i] = grad(y, x, create_graph=True, retain_graph=True)[0]
            # compute second-order derivative
            grads_2[i] = grad(grads_1[i], x, create_graph=True, retain_graph=True)[0]
        return grads_1, grads_2

I compared against finite differences calculations of the derivatives and the results seem correct. However, computation is very slow! I wonder how would one go about speeding up the calculations (maybe by vectorising the loop?). Such vectorization is possible using "backward" method, e.g. see the answer to this question, however, I could not generlise this method for second-order derivatives. Thanks.


title: Number of images after image augmentation in pytorch

body: I have 100 images, I applied horizontal and vertical augmentation with a probability of 1 using Pytorch function RandomHorizontalFlip and RandomVerticalFlip. After this would my total number of images will be 300 or something else?


title: I am trying to train my network on iris data but having the following error. Although I changed the type as well, but it still pops up the same error

body:
'num_epochs = 500
train_loss = []
test_loss = []
train_accuracy = []
test_accuracy = []

for epoch in range(num_epochs):

    train_correct = 0
    train_total = 0

    for i, (items, classes) in enumerate(train_loader):

        items = Variable(items)
        classes = Variable(classes)

        net. train()           # Put the network into training mode

        optimizer.zero_grad() # Clear off the gradients from any past operation
        outputs = net(items)  # Do the forward pass
        loss = criterion(outputs, classes) # Calculate the loss
        loss.backward()       # Calculate the gradients with help of back propagation
        optimizer.step()      # Ask the optimizer to adjust the parameters based on the gradients

        train_total += classes.size(0)
        _, predicted = torch.max(outputs.data, 1)
        train_correct += (predicted == classes.data).sum()

        print ('Epoch %d/%d, Iteration %d/%d, Loss: %.4f'
           %(epoch+1, num_epochs, i+1, len(train_ds)//batch_size, loss.data[0]))

    net.eval()                 # Put the network into evaluation mode

    train_loss.append(loss.data[0])

    train_accuracy.append((100 * train_correct / train_total))

    test_items = torch.FloatTensor(test_ds.data.values[:, 0:4])
    test_classes = torch.LongTensor(test_ds.data.values[:, 4])

    outputs = net(Variable(test_items))
    loss = criterion(outputs, Variable(test_classes))
    test_loss.append(loss.data[0])
    _, predicted = torch.max(outputs.data, 1)
    total = test_classes.size(0)
    correct = (predicted == test_classes).sum()
    test_accuracy.append((100 * correct / total))`'

And the error I get is
RuntimeError
       Traceback (most recent call last)
        <ipython-input-21-d25f2c091906> in <module>
        21         optimizer.zero_grad() # Clear off the gradients from any past operation
        22         outputs = net(items)  # Do the forward pass
        ---> 23         loss = criterion(outputs, classes) # Calculate the loss
        24         loss.backward()       # Calculate the gradients with help of back propagation
        25         optimizer.step()      # Ask the optimizer to adjust the parameters based on the gradients

RuntimeError: expected scalar type Long but found Int


title: How to implement FocalLoss in Pytorch?

body:
Focal Loss is a loss aimed at addressing class imbalance for a classification task.
Here is my attempt
class FocalLoss(nn.Module):
    def __init__(
            self,
            weight=None,
            gamma=2.,
            reduction='none'
    ):
        nn.Module.__init__(self)
        self.weight = weight
        self.gamma = gamma
        self.reduction = reduction

    def forward(self, input_tensor, target_tensor):
        log_prob = F.log_softmax(input_tensor, dim=-1)
        prob = torch.exp(log_prob)
        return F.nll_loss(
                ((1 - prob) ** self.gamma) * log_prob,
                target_tensor,
                weight=self.weight,
                reduction=self.reduction
        )

Upon loss.backward() this gives
raise RuntimeError("grad can be implicitly created only for scalar outputs")
RuntimeError: grad can be implicitly created only for scalar outputs

This is the call to the loss function:
loss = self._criterion(log_probs, label_batch)

When self._criterion = nn.CrossEntropyLoss() it works, and when self._criterion = FocalLoss() it gives the error.
How do I make this loss behave like CrossEntropyLoss API-wise?


title: Bound optimization using pytorch

body:
How to include bounds when using optimization method in pytorch. I have a tensor of variables, each variable has different bound.
upper_bound = torch.tensor([1,5,10], requires_grad=False)
lower_bound = torch.tensor([-1,-5,-10], requires_grad=False)
X           = torch.tensor([10, -60, 105], require_grad=True)
for _ in range(100):
    optimizer.zero_grad()
    loss = ..
    loss.backward()
    optimizer.step()
    X[:] = X.clamp(lower_bound, upper_bound)

But, clamp only uses a single number. Since each variable is bounded differently, I need to include the upper and lower bounds tensors.


title: When and Whether should we normalize the ground-truth labels in the multi-task regression models?

body:
I am trying a multi-task regression model. However, the ground-truth labels of different tasks are on different scales. Therefore, I wonder whether it is necessary to normalize the targets. Otherwise, the MSE of some large-scale tasks will be extremely bigger. The figure below is part of my overall targets. You can certainly find that columns like ASA_m2_c have much higher values than some others.

First, I have already tried some weighted loss techniques to balance the concentration of my model when it does gradient backpropagation. The result shows it didn't perform well.
Secondly, I have seen tremendous discussions regarding normalizing the input data, but hardly discovered any particular talking about normalizing the labels. It's partly because most of the people's problems are classification type and a single task. I do know pytorch provides a convenient approach to normalize the vision dataset by transform.normalize, which is still operated on the input rather than the labels.
Similar questions: https://forums.fast.ai/t/normalizing-your-dataset/49799
https://discuss.pytorch.org/t/ground-truth-label-normalization/26981/19
PyTorch - How should you normalize individual instances
Moreover, I think it might be helpful to provide some details of my model architecture. The input is first fed into a feature extractor and then several generators use the shared output representation from that extractor to predict different targets.


title: Bubble up non null rows of a 3-D Tensor PyTorch

body:
I am having some problem trying to convert the following function into a function only manipulating tensors.
def valid_sequence_output(sequence_output, valid_mask):
    bs, max_len, feat_dim = sequence_output.shape
    valid_output = torch.zeros(bs, max_len, feat_dim, dtype=torch.float32)
    for i in range(bs):
        jj = -1
        for j in range(max_len):
            if valid_mask[i][j].item() == 1:
                jj += 1
                valid_output[i][jj] = sequence_output[i][j]
    return valid_output

where the input tensors can be created as follow:
size = ((2,5,2))
sequence_output = torch.randint(0, 250, size=size)
valid_mask = torch.randint(0, 2, size=size[:2])

I basically aim at “bubbling up” the non null rows of sequence_output. As an example having sequence_output equal to:
tensor([[[  0,   0],
         [ 15,  47],
         [124, 230],
         [ 0,  0],
         [  65,   31]],
        [[  0,   0],
         [  0,   0],
         [  0,   0],
         [139, 228],
         [224, 205]]])

I am trying to obtain the following tensor:
tensor([[[ 15,  47],
         [124, 230],
         [ 65,  31],
         [  0,   0],
         [  0,   0]],
        [[139, 228],
         [224, 205],
         [  0,   0],
         [  0,   0],
         [  0,   0]]])

If somebody has a suggestion on how to do this, I would really appreciate :D


title: How to incoporate mask into negative likelihood loss (torch.nn.functional.nll_loss)

body:
Hello I am implementing a lstm for language modelling
for homework and I am at the loss implementation phase. Our instructor told us to use F.nll_loss but the sequences are padded and we have to take into account a mask that is given which tells us when the sequences stop.
input:

log_probas (batch_size, sequence_length(padded), vocabulary size)
targets (batch_size, sequence_length(padded))
mask (batch_size, sequence_length(padded)

naive implementation which works without taking into account the mask:
import torch.nn.functional as F
loss = F.nll_loss(log_probas.transpose(1, 2), targets)

I've been crawling the internet and banging my head but can't seem to find an answer on how to incorporate a mask into the averaging scheme of the loss.


title: I can iterate through my first iteration of PyTorch successfully, but am unable to do a second iteration. Please see code and error I get

body:
I can iterate through my first iteration of PyTorch successfully, but am unable to do a second iteration.  Please see my code and error I get if I were to run my code a second time through.

current_grad = 0
l_rate = 10**-4
x=torch.tensor([[1.0,2.,4.],[2.,3.,2.]])
y=torch.tensor([1.0,0.])
w=torch.tensor([.5,2.,1.], requires_grad=True)

# forward propagate
output = x @ w.T
y_pred = 1/(1+ math.e**-(output))

# objective function
loss = sum(y*(y_pred**.5) + ((1-y) * (1-y_pred)**.5)) / len(y_pred)

# now, get gradient over these layers
x.requires_grad =True
y.requires_grad =True
w.requires_grad =True
loss.backward()

# update only 1 set of weights here.
with torch.no_grad():
    w = w + (w.grad * l_rate)

I get an error at my loss.backward line:
TypeError: unsupported operand type(s) for *: 'NoneType' and 'float'
How can I fix this so that my w.grad is not NoneType on the second time around?


title: Validation of the dataset in GCN

body:
I am using PyTorch I am trying to do the validation on my dataset to obtain optimal number of channels in my neural network. I have the following code:
def train_during_validation():
    for epoch in range (1, 201):
        model.train()
        optimizer.zero_grad()
        out = model(data.x, data.edge_index)
        loss = criterion(out[data.val_mask], data.y[data.val_mask])
        loss.backward()
        optimizer.step()
    return loss

def validation():
    loss_val = np.zeros(50, dtype = float)
    model = GCN(hidden_channels = 1)
    loss_val = train_during_validation()
    print(loss_val)

validation()

In the code above I train previously defined model with 16 channels and I obtain a loss of 0.33. But as soon as I start doing validation on hidden_channel (see code below), my loss does not go down (it remains on 1.95). I do not understand why. Can somebody explain?
def train_during_validation(model):
    print(f'Model:{model}')
    for epoch in range (1, 201):
        model.train()
        optimizer.zero_grad()
        out = model(data.x, data.edge_index)
        loss = criterion(out[data.val_mask], data.y[data.val_mask])
        loss.backward()
        optimizer.step()
    return loss

def validation():
    loss_val = np.zeros(50, dtype = float)
    model = GCN(hidden_channels = 1)
    for i in range (50):
        model = GCN(hidden_channels = i)
        #print(model)
        loss_val[i] = train_during_validation(model)
        print(loss_val[i])

validation()


title: New tensor based on old tensor and 2d indices

body:
I previously asked:
PyTorch tensors: new tensor based on old tensor and indices
I have the same problem now but need to use a 2d index tensor.
I have a tensor col of size [batch_size, k] with values between 0 and k-1:
idx = tensor([[0,1,2,0],
        [0,3,2,2],
        ...])

and the following matrix:
x = tensor([[[0, 9],
 [1, 8],
 [2, 3],
 [4, 9]],
 [[0, 0],
 [1, 2],
 [3, 4],
 [5, 6]]])

I want to create a new tensor which contains the rows specified in index, in that order. So I want:
tensor([[[0, 9],
 [1, 8],
 [2, 3],
 [0, 9]],
 [[0, 0],
 [5, 6],
 [3, 4],
 [3, 4]]])

Currently I'm doing it like this:
for i, batch in enumerate(t):
    t[i] = batch[col[i]]

How can I do it more efficiently?


title: Can't import torchsummary in jupyter notebook

body:
I pip install torchsummary successfully inside the notebook, but then when I run
from torchsummary import summary
I get a ModuleNotFoundError.
Did anyone managed to successfully run 'torchsummary' on jupyter notebook and can help me do the same?


title: How to use AllenNLP to implement a Decoder in seq2seq generation task?

body:
I`m trying to realize a seq2seq generation style experiment, but i cannot find any instructions for implement a 'Decoder'.
I'm using the Newest Allennlp V2.1. So,I will appreciate it if you could give me some advice!


title: Getting the Prediction of a Time Series or Sequence Model in Pytorch

body:
For simplicity, we have a linear regression model created using Pytorch instead of more suited for sequence modeling like LSTM. We want to feed a sequence of inputs with n_features features into the network and train it to predict one of the feature in the next time step.
What is the difference between the 2 methods below? Or are both methods equivalent?
Method 1:
Linear layer is set to output only 1 feature
import torch
import torch.nn as nn

class LR(nn.Module):
    def __init__(self, n_features):
        super().__init__()
        self.linear = nn.Linear(in_features=n_features, out_features=1)

    def forward(self, x):
        return self.linear()

seq_length = 20
batch_size = 10
n_features = 5
model = LR(n_features)
# train the model, then...
preds = model(torch.randn(seq_length, batch_size, n_features))
preds = preds[-1, :, :]
# torch.Size([10, 1])

Method 2:
Model outputs all features first, the we select only the first feature (assuming this is the feature we are trying to predict).
import torch
import torch.nn as nn

class LR(nn.Module):
    def __init__(self, n_features):
        super().__init__()
        self.linear = nn.Linear(in_features=n_features, out_features=n_features)

    def forward(self, x):
        return self.linear()

seq_length = 20
batch_size = 10
n_features = 5
model = LR(n_features)
# train the model, then...
preds = model(torch.randn(seq_length, batch_size, n_features))
preds = preds[-1, :, 0].unsqueeze(1)  # assume we are trying to predict the first feature
# torch.Size([10, 1])


title: Failing to understand data types and their indexing from a pyTorch tutorial

body:
Context
I am tryng to understand the basics of pyTorch, with particular care on how to deal with Tensors. I am following the tutorials, but, in one of them, I found a little detail which is not clear to me

Issue description
I am struggling to understand why, in the Quickstart tutorial, there are two different ways in which the argmax() function is applied. The problem is not the function per se, but actually the way it is applied on the input data: the variables are being indexed in different ways, and I do not understand either. This is probably due to the fact that I do not understand which shape the input data is, and I could not figure it out even if trying to experiment in the Colab notebook. Here is the first portion of code in which argmax() is used:
def test(dataloader, model):
    size = len(dataloader.dataset)
    model.eval()
    test_loss, correct = 0, 0
    with torch.no_grad():
        for X, y in dataloader:
            X, y = X.to(device), y.to(device)
            pred = model(X)
            test_loss += loss_fn(pred, y).item()
            correct += (pred.argmax(1) == y).type(torch.float).sum().item()
    test_loss /= size
    correct /= size
    print(f"Test Error: \n Accuracy: {(100*correct):>0.1f}%, Avg loss: {test_loss:>8f} \n")

the line of interest here is the one which computes the number of correct guesses. In this case, it seems to me as the right hand side of the equation wants to pick the maximum of the argument (i.e. the argmax) of the second element of the Tensor, which is the predicted label for the input element. But here print(pred.shape) returns torch.Size([64, 10]), so I am already lost.

Also, in the very last piece of code in the page, which is supposed to show a single sample of the dataset along with its prediction, argmax() is used like this:
classes = [
    "T-shirt/top",
    "Trouser",
    "Pullover",
    "Dress",
    "Coat",
    "Sandal",
    "Shirt",
    "Sneaker",
    "Bag",
    "Ankle boot",
]

model.eval()
x, y = test_data[0][0], test_data[0][1]
with torch.no_grad():
    pred = model(x)
    predicted, actual = classes[pred[0].argmax(0)], classes[y]
    print(f'Predicted: "{predicted}", Actual: "{actual}"')

Here I simply do not get why we write pred[0].argmax(0) in this specific way. I am lost in trying to wrap my head around data types and Tensor dimensions.

What I want
An explanation of the syntaxes reported above. Maybe providing me with some examples/way of thinking relative to Tensors and other relevant data types relevant to this question would also help me a lot too

What I have tried so far
I have tried experimenting in the Colab notebook, adding the following lines at the end of the last code block:
    print(pred)
    print(pred[0])
    print(type(test_data))
    print(type(pred))
    print(pred.shape)
    print(test_data[0][0].shape)
    print(type(test_data[0]))
    print(type(test_data[0][0]))

which yielded:
tensor([[0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 1.3538, 0.0000, 0.0000, 1.2093, 2.4238]])
tensor([0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 1.3538, 0.0000, 0.0000, 1.2093, 2.4238])
<class 'torchvision.datasets.mnist.FashionMNIST'>
<class 'torch.Tensor'>
torch.Size([1, 10])
torch.Size([1, 28, 28])
<class 'tuple'>
<class 'torch.Tensor'>

Why I am asking this
As I had many pleasant experiences on Stackoverflow so far, I prefer to first ask here rather than creating a new account and directly seeking help on pyTorch forums

Thank you!


title: How to efficiently repeat tensor element variable of time in pytorch?

body:
For example, if I have a tensor A = [[1,1,1], [2,2,2], [3,3,3]], and B = [1,2,3]. How do I get C = [[1,1,1], [2,2,2], [2,2,2], [3,3,3], [3,3,3], [3,3,3]], and doing this batch-wise?
My current element-wise solution btw (takes forever...):
        def get_char_context(valid_embeds, words_lens):
            chars_contexts = []
            for ve, wl in zip(valid_embeds, words_lens):
                for idx, (e, l) in enumerate(zip(ve, wl)):
                    if idx ==0:
                        chars_context = e.view(1,-1).repeat(l, 1)
                    else:
                        chars_context = torch.cat((chars_context, e.view(1,-1).repeat(l, 1)),0)
                chars_contexts.append(chars_context)
            return chars_contexts

I'm doing this to add bert word embedding to a char level seq2seq task...


title: CUDA driver version is higher than the CUDA runtime version？

body:
The terminal shows the error:
RuntimeError: cuda runtime error (35) : CUDA driver version is insufficient for CUDA runtime version at torch/csrc/cuda/Module.cpp:51

But my driver version (440.118.02) is sufficient for cuda9.0
Some info about my machine: cat /proc/driver/nvidia/version NVRM version: NVIDIA UNIX x86_64 Kernel Module 440.118.02 Thu Sep 3 09:54:46 UTC 2020
cat /usr/local/cuda/version.txt CUDA Version 9.0.176


title: Semantic Segmentation metric - making confusion matrix

body:
I want to get meanIoU and pixel Accuracy, so I customize get metric function from Deeplabv3+ github Evaluator class: https://github.com/jfzhang95/pytorch-deeplab-xception.
Currently I am having difficulty writing the confusion matrix of the two images.
Here is my first code block:
for i, row in df.iterrows():    # dataframe have filenames of ground-truth and predicted image
    print(" --- Iterrows start --- ")
    print("Target Image: \n{}\n{}\n".format(row['Yt'], row['Yp']))
    # gt image : aaa.png | target(predicted image) : segmented_aaa.jpg

    # Read gt segmap imagefile
    gt_image = Image.open(row['Yt']).convert('L')
    gt_image = np.array(gt_image)
    gt_image = cv2.resize(gt_image, (513, 513))

    # Read target(predicted) segmap imagefile
    target_image = Image.open(row['Yp']).convert('L')
    target_image = np.array(target_image)
    target_image = cv2.resize(target_image, (513, 513))

    print(gt_image.shape)        # (513, 513)
    print(target_image.shape)    # (513, 513)
    print("gt_image:", "\n", gt_image)
    print("target_image:", "\n", target_image)
    print("gt_image type:", "\n", type(gt_image))
    print("target_image type:", "\n", type(target_image))
    print("gt_image unique:", "\n", np.unique(gt_image))
    print("target_image unique:", "\n", np.unique(target_image))
    print(gt_image.shape, target_image.shape)

    # mIoU = get_iou(target_image, gt_image, num_classes)
    # print(mIoU)

    evaluator.add_batch(gt_image, target_image)
    print(" --- Iterrows end --- ")

Result of first code block in console log:
(513, 513)
(513, 513)
gt_image:
 [[0 0 0 ... 0 0 0]
 [0 0 0 ... 0 0 0]
 [0 0 0 ... 0 0 0]
 ...
 [0 0 0 ... 0 0 0]
 [0 0 0 ... 0 0 0]
 [0 0 0 ... 0 0 0]]
target_image:
 [[0 0 0 ... 0 0 0]
 [0 0 0 ... 0 0 0]
 [0 0 0 ... 0 0 0]
 ...
 [0 0 0 ... 0 0 0]
 [0 0 0 ... 0 0 0]
 [0 0 0 ... 0 0 0]]
gt_image type:
 <class 'numpy.ndarray'>
target_image type:
 <class 'numpy.ndarray'>
gt_image unique:
 [  0   1   2   3   4   5   6   7   8   9  10  11  12  13  14  15  16  17
  18  19  20  21  22  23  24  25  26  27  28  29  30  31  32  33  34  35
  36  37  38  39  40  41  42  43  44  45  46  47  48  49  50  51  52  53
  54  55  56  57  59  60  61  62  63  64  65  66  67  68  69  70  71  72
  73  74  75  77  78  79  80  81  83  84  85  86  87  88  89  90  91  92
  93  94  95  96  97  98  99 100 101 102 103 104 105 106 107 108 109 110
 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128
 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146
 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164
 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182
 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200
 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218
 219 220]
target_image unique:
 [  0   1   2   3   4   5   6   7   8   9  10  11  12  13  14  15  16  17
  18  19  20  21  22  23  24  25  26  27  28  29  30  31  32  33  34  35
  36  37  38  39  40  41  42  43  44  45  46  47  48  49  50  51  52  53
  54  55  56  57  58  59  60  61  62  63  64  65  66  67  68  69  70  71
  72  73  74  75  76  77  78  79  80  81  82  83  84  85  86  87  88  89
  90  91  92  93  94  95  96  97  98  99 100 101 102 103 104 105 106 107
 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125
 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143
 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 160]

and Evaluator(second) code block is:
    def _generate_matrix(self, gt_image, pre_image):
        print(np.unique(gt_image))
        mask = (gt_image >= 0) & (gt_image < self.num_class)
        print("\nMASK INFO")
        print(mask, mask.shape)
        print(np.unique(mask))
        print()

        print("\nGT_IMAGE INFO")
        print(gt_image[mask])
        print(np.unique(gt_image[mask]))
        print(gt_image[mask].astype('int'))
        print(set(gt_image[mask].astype('int').tolist()))
        print(sorted(set(self.num_class * gt_image[mask].astype('int'))))
        print(pre_image[mask], set(pre_image[mask].tolist()))
        print()

        label = self.num_class * gt_image[mask].astype('int') + pre_image[mask]
        # label[10] = 449
        print("\nLABEL INFO")
        print(label, label.shape, len(label.tolist()))
        print(set(label.tolist()))
        print(len(set(label.tolist())))
        print()

        count = np.bincount(label, minlength=self.num_class**2)
        print("\nCOUNT INFO")
        print(count.shape)
        print()

        # !!!!!!!!!! ERROR POINT !!!!!!!!!!
        confusion_matrix = count.reshape(self.num_class, self.num_class)
        return confusion_matrix

    def add_batch(self, gt_image, pre_image):
        if gt_image.shape != pre_image.shape:
            print("GT_Image's shape is different with PRE_IMAGE's shape!")
            exit(0)
        # assert gt_image.shape == pre_image.shape
        self.confusion_matrix += self._generate_matrix(gt_image, pre_image)

Result of second code block in console log:
(513, 513) (513, 513)
[  0   1   2   3   4   5   6   7   8   9  10  11  12  13  14  15  16  17
  18  19  20  21  22  23  24  25  26  27  28  29  30  31  32  33  34  35
  36  37  38  39  40  41  42  43  44  45  46  47  48  49  50  51  52  53
  54  55  56  57  59  60  61  62  63  64  65  66  67  68  69  70  71  72
  73  74  75  77  78  79  80  81  83  84  85  86  87  88  89  90  91  92
  93  94  95  96  97  98  99 100 101 102 103 104 105 106 107 108 109 110
 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128
 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146
 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164
 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182
 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200
 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218
 219 220]

MASK INFO
[[ True  True  True ...  True  True  True]
 [ True  True  True ...  True  True  True]
 [ True  True  True ...  True  True  True]
 ...
 [ True  True  True ...  True  True  True]
 [ True  True  True ...  True  True  True]
 [ True  True  True ...  True  True  True]] (513, 513)

GT_IMAGE INFO
[0 0 0 ... 0 0 0]
{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20}
[0, 21, 42, 63, 84, 105, 126, 147, 168, 189, 210, 231, 252, 273, 294, 315, 336, 357, 378, 399, 420]
[0 0 0 ... 0 0 0] {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 142, 145, 146}

LABEL INFO
[0 0 0 ... 0 0 0] (175389,) 175389
{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 142, 145, 146, 147, 148, 153, 157, 162, 168, 173, 189, 210, 218, 231, 240, 252, 253, 254, 273, 277, 294, 295, 315, 317, 336, 337, 357, 361, 364, 377, 378, 384, 395, 399, 407, 410, 420, 421, 424, 425, 432, 496}
183

COUNT INFO
(497,)

cannot reshape array of size 497 into shape (21,21)

In my opinion, if there is a value like 480 that is greater than 21*21(=441), it will cause an error.
21 is the number of classes in the Pascal VOC dataset.
The ground-truth and predicted image used as input is this image:
 [ground-truth image]
 [predicted image]


title: Accessing PyTorch modules - ResNet18

body:
I am using a ResNet-18 coded as follows:
class ResidualBlock(nn.Module):
    '''
    Residual Block within a ResNet CNN model
    '''
    def __init__(self, input_channels, num_channels,
                 use_1x1_conv = False, strides = 1):
        # super(ResidualBlock, self).__init__()
        super().__init__()

        self.conv1 = nn.Conv2d(
            in_channels = input_channels, out_channels = num_channels,
            kernel_size = 3, padding = 1, stride = strides,
            bias = False
            )
        self.bn1 = nn.BatchNorm2d(num_features = num_channels)

        self.conv2 = nn.Conv2d(
            in_channels = num_channels, out_channels = num_channels,
            kernel_size = 3, padding = 1, stride = 1,
            bias = False
            )
        self.bn2 = nn.BatchNorm2d(num_features = num_channels)

        if use_1x1_conv:
            self.conv3 = nn.Conv2d(
                in_channels = input_channels, out_channels = num_channels,
                kernel_size = 1, stride = strides
                )
            self.bn3 = nn.BatchNorm2d(num_features = num_channels)
        else:
            self.conv3 = None

        self.relu = nn.ReLU(inplace = True)

        self.initialize_weights()

    def forward(self, X):
        Y = F.relu(self.bn1(self.conv1(X)))
        Y = self.bn2(self.conv2(Y))

        if self.conv3:
            X = self.bn3(self.conv3(X))
            # print(f"X.shape due to 1x1: {X.shape} & Y.shape = {Y.shape}")
        else:
            # print(f"X.shape without 1x1: {X.shape} & Y.shape = {Y.shape}")
            pass

        Y += X
        return F.relu(Y)

    def shape_computation(self, X):
        Y = self.conv1(X)
        print(f"self.conv1(X).shape: {Y.shape}")
        Y = self.conv2(Y)
        print(f"self.conv2(X).shape: {Y.shape}")

        if self.conv3:
            h = self.conv3(X)
            print(f"self.conv3(X).shape: {h.shape}")

    def initialize_weights(self):
        for m in self.modules():
            # print(m)
            if isinstance(m, nn.Conv2d):
                nn.init.kaiming_uniform_(m.weight)

                '''
                # Do not initialize bias (due to batchnorm)-
                if m.bias is not None:
                    nn.init.constant_(m.bias, 0)
                '''

            elif isinstance(m, nn.BatchNorm2d):
                # Standard initialization for batch normalization-
                nn.init.constant_(m.weight, 1)
                nn.init.constant_(m.bias, 0)

            elif isinstance(m, nn.Linear):
                nn.init.kaiming_normal_(m.weight)
                nn.init.constant_(m.bias, 0)

b0 = nn.Sequential(
    nn.Conv2d(in_channels = 3, out_channels = 64, kernel_size = 3, stride = 1, padding = 1),
    nn.BatchNorm2d(num_features = 64),
    nn.ReLU())

def create_resnet_block(input_filters, output_filters, num_residuals, first_block = False):
    # Python list to hold the created ResNet blocks-
    resnet_blk = []

    for i in range(num_residuals):
        if i == 0 and first_block:
            resnet_blk.append(ResidualBlock(input_channels = input_filters, num_channels = output_filters, use_1x1_conv = True, strides = 2))
        else:
            resnet_blk.append(ResidualBlock(input_channels = output_filters, num_channels = output_filters, use_1x1_conv = False, strides = 1))

    return resnet_blk

b1 = nn.Sequential(*create_resnet_block(input_filters = 64, output_filters = 64, num_residuals = 2, first_block = True))

b2 = nn.Sequential(*create_resnet_block(input_filters = 64, output_filters = 128, num_residuals = 2, first_block = True))

b3 = nn.Sequential(*create_resnet_block(input_filters = 128, output_filters = 256, num_residuals = 2, first_block = True))

b4 = nn.Sequential(*create_resnet_block(input_filters = 256, output_filters = 512, num_residuals = 2, first_block = True))

# Initialize a ResNet-18 CNN model-
model = nn.Sequential(
    b0, b1, b2, b3, b4,
    nn.AdaptiveAvgPool2d(output_size = (1, 1)),
    nn.Flatten(),
    nn.Linear(in_features = 512, out_features = 10))

The layer names are now as follows:
for layer_name, param in trained_model.named_parameters():
    print(f"layer name: {layer_name} has {param.shape}")

Result:
> layer name: 0.0.weight has torch.Size([64, 3, 3, 3])

> layer name: 0.0.bias has torch.Size([64])

> layer name: 0.1.weight has torch.Size([64])

> layer name: 0.1.bias has torch.Size([64])

> layer name: 1.0.conv1.weight has torch.Size([64, 64, 3, 3])

> layer name: 1.0.bn1.weight has torch.Size([64])

> layer name: 1.0.bn1.bias has torch.Size([64])

> layer name: 1.0.conv2.weight has torch.Size([64, 64, 3, 3])

> layer name: 1.0.bn2.weight has torch.Size([64])

> layer name: 1.0.bn2.bias has torch.Size([64])

> layer name: 1.0.conv3.weight has torch.Size([64, 64, 1, 1])

> layer name: 1.0.conv3.bias has torch.Size([64])

> layer name: 1.0.bn3.weight has torch.Size([64])

> layer name: 1.0.bn3.bias has torch.Size([64])

> layer name: 1.1.conv1.weight has torch.Size([64, 64, 3, 3])

> layer name: 1.1.bn1.weight has torch.Size([64])

> layer name: 1.1.bn1.bias has torch.Size([64])

> layer name: 1.1.conv2.weight has torch.Size([64, 64, 3, 3])

> layer name: 1.1.bn2.weight has torch.Size([64])

> layer name: 1.1.bn2.bias has torch.Size([64])

> layer name: 2.0.conv1.weight has torch.Size([128, 64, 3, 3])

> layer name: 2.0.bn1.weight has torch.Size([128])

> layer name: 2.0.bn1.bias has torch.Size([128])

> layer name: 2.0.conv2.weight has torch.Size([128, 128, 3, 3])

> layer name: 2.0.bn2.weight has torch.Size([128])

> layer name: 2.0.bn2.bias has torch.Size([128])

> layer name: 2.0.conv3.weight has torch.Size([128, 64, 1, 1])

> layer name: 2.0.conv3.bias has torch.Size([128])

> layer name: 2.0.bn3.weight has torch.Size([128])

> layer name: 2.0.bn3.bias has torch.Size([128])

> layer name: 2.1.conv1.weight has torch.Size([128, 128, 3, 3])

> layer name: 2.1.bn1.weight has torch.Size([128])

> layer name: 2.1.bn1.bias has torch.Size([128])

> layer name: 2.1.conv2.weight has torch.Size([128, 128, 3, 3])

> layer name: 2.1.bn2.weight has torch.Size([128])

> layer name: 2.1.bn2.bias has torch.Size([128])

> layer name: 3.0.conv1.weight has torch.Size([256, 128, 3, 3])

> layer name: 3.0.bn1.weight has torch.Size([256])

> layer name: 3.0.bn1.bias has torch.Size([256])

> layer name: 3.0.conv2.weight has torch.Size([256, 256, 3, 3])

> layer name: 3.0.bn2.weight has torch.Size([256])

> layer name: 3.0.bn2.bias has torch.Size([256])

> layer name: 3.0.conv3.weight has torch.Size([256, 128, 1, 1])

> layer name: 3.0.conv3.bias has torch.Size([256])

> layer name: 3.0.bn3.weight has torch.Size([256])

> layer name: 3.0.bn3.bias has torch.Size([256])

> layer name: 3.1.conv1.weight has torch.Size([256, 256, 3, 3])

> layer name: 3.1.bn1.weight has torch.Size([256])

> layer name: 3.1.bn1.bias has torch.Size([256])

> layer name: 3.1.conv2.weight has torch.Size([256, 256, 3, 3])

> layer name: 3.1.bn2.weight has torch.Size([256])

> layer name: 3.1.bn2.bias has torch.Size([256])

> layer name: 4.0.conv1.weight has torch.Size([512, 256, 3, 3])

> layer name: 4.0.bn1.weight has torch.Size([512])

> layer name: 4.0.bn1.bias has torch.Size([512])

> layer name: 4.0.conv2.weight has torch.Size([512, 512, 3, 3])

> layer name: 4.0.bn2.weight has torch.Size([512])

> layer name: 4.0.bn2.bias has torch.Size([512])

> layer name: 4.0.conv3.weight has torch.Size([512, 256, 1, 1])

> layer name: 4.0.conv3.bias has torch.Size([512])

> layer name: 4.0.bn3.weight has torch.Size([512])

> layer name: 4.0.bn3.bias has torch.Size([512])

> layer name: 4.1.conv1.weight has torch.Size([512, 512, 3, 3])

> layer name: 4.1.bn1.weight has torch.Size([512])

> layer name: 4.1.bn1.bias has torch.Size([512])

> layer name: 4.1.conv2.weight has torch.Size([512, 512, 3, 3])

> layer name: 4.1.bn2.weight has torch.Size([512])

> layer name: 4.1.bn2.bias has torch.Size([512])

> layer name: 7.weight has torch.Size([10, 512])

> layer name: 7.bias has torch.Size([10])

In order to prune this model, I am referring to PyTorch pruning tutorial. It's mentioned here that to prune a module/layer, use the following code:
parameters_to_prune = (
    (model.conv1, 'weight'),
    (model.conv2, 'weight'),
    (model.fc1, 'weight'),
    (model.fc2, 'weight'),
    (model.fc3, 'weight'),
)

But for the code above, the modules/layers no longer have this naming convention. For example, to prune the first conv layer of this model:
> layer name: 0.0.weight has torch.Size([64, 3, 3, 3])

on trying the following code:
prune.random_unstructured(model.0.0, name = 'weight', amount = 0.3)

It gives me the error:
prune.random_unstructured(trained_model.0.0, name = 'weight', amount = 0.3)
                                           ^ SyntaxError: invalid syntax

How do I handle this?


title: Is it possible that the output of the deep learning shuffled randomly in same batch?

body:
I'm working with the VAE of 1-dimensional signal : which is kind of noise added sinusoidal signal. To check the shape of the reconstructed output, I draw some graphs with the original signal and VAE reconstructed signal. However, the signals with same index do not seem very resemble but not related signal of index shows the similar shape. I suspect that the output of the deep network is arranged randomly with in the batch range. I used the basic structure from the online examples.
Dataset Load :
train_data = signalDataset(train_dataset)
train_set = DataLoader(dataset=train_data, batch_size=opt.batch_size, shuffle=True, num_workers=opt.workers)

for batch_idx, data in enumerate(train_set):
    data = data.to(device)

Dataset :
class signalDataset:

def __init__(self, dataset_name, normalize=True):
    self.dataset_name = dataset_name
    t_data = pd.read_csv(dataset_name, header=None)
    data = torch.from_numpy(np.expand_dims(np.array([t_data.iloc[i] for i in range(0, len(t_data))]), -1)).float()
    self.data = self.normalize(data) if normalize else data
    self.seq_len = data.size(1)

    # Estimates distribution parameters of deltas (Gaussian) from normalized data
    original_deltas = data[:, -1] - data[:, 0]
    self.original_deltas = original_deltas
    self.or_delta_max, self.or_delta_min = original_deltas.max(), original_deltas.min()
    deltas = self.data[:, -1] - self.data[:, 0]
    self.deltas = deltas
    self.delta_mean, self.delta_std = deltas.mean(), deltas.std()
    self.delta_max, self.delta_min = deltas.max(), deltas.min()

def __len__(self):
    return len(self.data)

def __getitem__(self, idx):

    return self.data[idx]

def normalize(self, x_list):
    x_norm = torch.empty(0, dtype=torch.float)
    for x in x_list :
        self.max = x.max()
        self.min = x.min()
        # self.max = abs(x).max()
        # self.min = (-1) * abs(x).max()
        # x = (2 * (x - self.min) / (self.max - self.min) - 1) # -1~1
        x = (x - self.min) / (self.max - self.min) # 0~1
        x = torch.unsqueeze(x, 0)
        x_norm = torch.cat([x_norm, x], dim=0)
        # x = torch.sigmoid(x)

    return x_norm

def denormalize(self, x):
    if not hasattr(self, 'max') or not hasattr(self, 'min'):
        raise Exception("You are calling denormalize, but the input was not normalized")
    return 0.5 * (x * self.max - x * self.min + self.max + self.min)

def sample_deltas(self, number):
    return (torch.randn(number, 1) + self.delta_mean) * self.delta_std

def normalize_deltas(self, x):
    return ((self.delta_max - self.delta_min) * (x - self.or_delta_min) / (
            self.or_delta_max - self.or_delta_min) + self.delta_min)

LSTM VAE :
class LSTM_VAE(nn.Module):
def __init__(self,
             input_size=60000,
             hidden=[1024, 512, 256, 128, 64],
             latent_size=1024,
             num_layers=2,
             bidirectional=True):
    super().__init__()

    self.input_size = input_size
    self.hidden = hidden
    self.latent_size = latent_size
    self.num_layers = num_layers
    self.bidirectional = bidirectional

    self.relu = nn.ReLU()

    self.encode = nn.LSTM(input_size=self.input_size,
                          hidden_size=self.hidden[0],
                          num_layers=2,
                          batch_first=True,
                          bidirectional=True)

    self.decode = nn.LSTM(input_size=self.latent_size,
                          hidden_size=self.hidden[2],
                          num_layers=2,
                          batch_first=True,
                          bidirectional=True)

    self.fc1 = nn.Linear(self.hidden[0]*2, self.hidden[1])
    self.fc2 = nn.Linear(self.hidden[1], self.hidden[2])
    self.fc31 = nn.Linear(self.hidden[2], self.latent_size)
    self.fc32 = nn.Linear(self.hidden[2], self.latent_size)
    self.bn1 = nn.BatchNorm1d(1)
    self.bn2 = nn.BatchNorm1d(1)
    self.bn3 = nn.BatchNorm1d(1)

    self.fc4 = nn.Linear(self.hidden[2]*2, self.hidden[1])
    self.fc5 = nn.Linear(self.hidden[1], self.hidden[0])
    self.fc6 = nn.Linear(self.hidden[0], self.input_size)

def encoder(self, x):
    x = torch.unsqueeze(x, 1)
    x, _ = self.encode(x)
    x = self.fc1(x)
    x = self.fc2(x)

    mu = self.fc31(x)
    log_var = self.fc32(x)

    return mu, log_var

def decoder(self, z):
    z, _ = self.decode(z)
    z = self.fc4(z)
    z = self.fc5(z)
    z = self.fc6(z)
    z = torch.sigmoid(z)

    return torch.squeeze(z)

def sampling(self, mu, log_var):
    std = torch.exp(0.5 * log_var)
    eps = torch.randn_like(std)

    return mu + eps * std

def forward(self, x):
    mu, log_var = self.encoder(x.view(-1, self.input_size))
    z = self.sampling(mu, log_var)
    z = self.decoder(z)

    return z, mu, log_var

I searched for batch data disturbing and data arrangement but nothing helpful came up. I wonder is the output shuffle in batch possible? Or just my code have something wrong in it?
Any opinion or suggestion would be very helpful to me.


title: Inferencing YOLOv5 (PyTorch) with multiple processes in parallel on the CPU

body:
I want to run (PyTorch-based) YOLOv5 inference on a large list of images, so I would like to process them in parallel. On that machine I don't have a GPU, so I want to run multple processes on the CPU. I am doing so via a normal Python multiprocessing.Pool. Nothing fancy here, the processing function for the process pool is quite simple:
def pool_proc(frame_num):
    img = Image.open(os.path.join(frames_folder, f"frame-{frame_num:05d}.jpg"))
    results = model(img, size=img_size, augment=True)
    preds = results.pandas().xyxy[0]
    return (frame_num, preds)

The logic works, however, running with multiple processes is quite slow. In fact it is even a bit slower than running only one single (i.e. the main) process on the CPU. So it seems even when running with multiple processes they are still not inferencing in parallel but it's as if they are waiting for each other. I am pretty sure though the pool logic works, because when I comment out the call to model() and replace it with a time.sleep(1.0) then the whole programm is running super fast and in parallel.
Any ideas what I might be missing here?


title: How to prevent "EOFError: Ran out of input" when saving & loading model's parameters?

body:
I'm using Google Colab to run my PyTorch model. I'm saving my model's state dict using this method every 100 epochs, and I believe the reason I'm getting this error is that Google Colab is sometimes shutting down the session in the middle of the automatic state dict saving. That is, because I only get the error sometimes after the shutdown when I try to reload the state dict file.
The full error:
EOFError                                  Traceback (most recent call last)
<ipython-input-5-01fb93bcf161> in <module>()
----> 1 get_ipython().run_cell_magic(...)
4 frames
<decorator-gen-53> in time(self, line, cell, local_ns)

<timed exec> in <module>()

<timed exec> in train(self, n_epochs)

<timed exec> in load(self)

/usr/local/lib/python3.7/dist-packages/torch/serialization.py in _legacy_load(f, map_location, pickle_module, **pickle_load_args)
    760             "functionality.")
    761
--> 762     magic_number = pickle_module.load(f, **pickle_load_args)
    763     if magic_number != MAGIC_NUMBER:
    764         raise RuntimeError("Invalid magic number; corrupt file?")

EOFError: Ran out of input


title: One liner tensor with conditions

body:
I have two tensors:
a = torch.nn.Parameter(torch.rand(7, requires_grad=True))
b = torch.randint(0,60, (20,))

Is there a one liner (or a quick & short way) that can create a tensor (call it x) of size 20 (similar to "b") with conditions?
i.e.
[b<4 use a[0], 4 <=b<12 use a[1], 12<=b<22 use a[2], <28, <38, <50, >50] for every b

So if:
b = [12, 93, 54, 0...]

I want my new tensor "x" to be:
x = [a[2],a[6], a[6]...]

I'm going to use this "x" tensor to train and need the values the be backproped and learnable
i.e.
loss = torch.rand(20) * x

loss.backward() ...

So if one of the a's is not in x I want it to not change.


title: pytorch (cpu only) on osx fails with symbol not found

body:
I am trying to get started with PyTorch - on a mac osx computer. However, basic steps fail:
from torch_sparse import coalesce, SparseTensor

---------------------------------------------------------------------------
OSError                                   Traceback (most recent call last)
<ipython-input-1-dad8246d5249> in <module>
----> 1 from torch_sparse import coalesce, SparseTensor

/usr/local/Caskroom/miniconda/base/envs/my_conda_env/lib/python3.8/site-packages/torch_sparse/__init__.py in <module>
     10         '_saint', '_padding'
     11 ]:
---> 12     torch.ops.load_library(importlib.machinery.PathFinder().find_spec(
     13         library, [osp.dirname(__file__)]).origin)
     14

/usr/local/Caskroom/miniconda/base/envs/my_conda_env/lib/python3.8/site-packages/torch/_ops.py in load_library(self, path)
    102             # static (global) initialization code in order to register custom
    103             # operators with the JIT.
--> 104             ctypes.CDLL(path)
    105         self.loaded_libraries.add(path)
    106

/usr/local/Caskroom/miniconda/base/envs/my_conda_env/lib/python3.8/ctypes/__init__.py in __init__(self, name, mode, handle, use_errno, use_last_error, winmode)
    371
    372         if handle is None:
--> 373             self._handle = _dlopen(self._name, mode)
    374         else:
    375             self._handle = handle

OSError: dlopen(/usr/local/Caskroom/miniconda/base/envs/my_conda_env/lib/python3.8/site-packages/torch_sparse/_version.so, 6): Symbol not found: __ZN3c105ErrorC1ENS_14SourceLocationERKNSt3__112basic_stringIcNS2_11char_traitsIcEENS2_9allocatorIcEEEE
  Referenced from: /usr/local/Caskroom/miniconda/base/envs/my_conda_env/lib/python3.8/site-packages/torch_sparse/_version.so
  Expected in: flat namespace
 in /usr/local/Caskroom/miniconda/base/envs/my_conda_env/lib/python3.8/site-packages/torch_sparse/_version.so

I am using a conda environment of:
name: my_conda_env
channels:
  - pytorch
  - conda-forge
  - defaults
dependencies:
  - python>=3.8
  - pytorch
  - pytorch_geometric

and instantiated it using:
conda env create --force -f environment.yml


title: Python pytorch function consumes memory excessively quickly

body:
I am writing a function with pytorch that feeds inputs through a transformer model, then condenses the last embedding layer by computing the average along a specific axis (using a subset of indices defined by a mask). Since the output of the model is very very large, I need to process the inputs in batches.
My question does not concern the logic of this function as I believe I have the correct implementation. My issue is that the function I wrote consumes memory excessively quickly and practically makes it unusable.
Here is my function:
def get_chunk_embeddings(encoded_dataset, batch_size):
  chunk_embeddings = torch.empty([0,768])
  for i in range(len(encoded_dataset['input_ids'])//batch_size):
    input_ids = encoded_dataset['input_ids'][i*batch_size:i*batch_size + batch_size]
    attention_mask = encoded_dataset['attention_mask'][i*batch_size:i*batch_size + batch_size]
    embeddings = model.forward(input_ids=input_ids, attention_mask=attention_mask)['last_hidden_state']
    embeddings = embeddings * attention_mask[:,:,None]
    embeddings = embeddings.sum(dim=1)/attention_mask.sum(dim=1)[:,None]
    chunk_embeddings = torch.cat([chunk_embeddings, embeddings],0)
  return chunk_embeddings

Now let's talk memory (the numbers below assume that I pass a batch_size of 8):

I am using google colab and I have ~25 GB of RAM available
The model is a BERT model and consumes 413 MB
encoded_dataset consumes 0.48 GB
input_ids consumes 0.413 MB
attention_mask consumes 4.096 KB
embeddings at its peak consumption consume 12.6 MB
chunk_embeddings adds 0.024576 MB with each iteration

So from my understanding, I should be able to allow chunk_embeddings to grow up to: 25GB - 413MB - 0.48GB - 0.413MB - 4.096KB - 12.6MB ~= 24 GB. Enough for almost 1 million iterations..
Here I will walk through an example of what I am experiencing:

Before running my function, google colab tells me that I have plenty of memory

Now, for the sake of example, I will run the function (for only 3 iterations)
To be explicit, I put this at the end of my for loop:
if (i == 2):return chunk_embeddings

Now I run the code val = get_chunk_embeddings(train_encoded_dataset, 8)
So even with just 3 iterations, somehow I consume almost 5.5 GB of RAM.

Why is this happening? Also after I have returned from the function all of the local variables should be deleted and there is no way that val is so large.
Could someone tell me what it is I am doing wrong or am not understanding? Please let me know if further information is needed.


title: IndexError: Dimension out of range (expected to be in range of [-1, 0], but got -2) with torch_geometry

body:
I have some trouble with IndexError in my GCN code.
The error occurred on the GCNConv layer.
I cannot understand why self.node_dim is included in the size function which is shown in the error code.
How can I solve this problem?
data.x is node_feature in which the number of feature for each node is just one and the number of node is 2058.
Thanks in advance.
'''
class GCNEncoder(torch.nn.Module):

    def __init__(self, in_channels, out_channels):
        super(GCNEncoder, self).__init__()
        self.conv1 = GCNConv(in_channels, out_channels, cached=True)
        self.conv2 = GCNConv(out_channels, out_channels, cached=True)

    def forward(self, x, edge_index):
        x = self.conv1(x, edge_index).relu()
        return self.conv2(x, edge_index)

class OurGAE(torch.nn.Module):

    def __init__(self, num_features, out_chennels, node):
        super(OurGAE, self).__init__()
        self.encoder = GCNEncoder(num_features, out_channels)

        self.linear = nn.Linear(out_channels, len(node))

    def encode(self, node_feature, edge_lst):
        z = self.encoder(node_feature, edge_lst)
        return z

    def predict(self, z):
        predicted_node_features = self.linear(z)
        return predicted_node_features

############ train model #######

dataset = OurDataset(one_node_feature, super_edge_lst)

out_channels = 16
num_features = dataset.num_features

model = OurGAE(num_features, out_channels, node)

device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
model = model.to(device)

optimizer = torch.optim.Adam(model.parameters(), lr=1e-3)
epochs = 100

criterion = nn.MSELoss()

#model.train()

for epoch in range(1, epochs + 1):

    loss_ep = 0
    for data in dataset:

        optimizer.zero_grad()

        x = data.x.to(device)
        edge_lst = data.edge_index.to(device)

        z = model.encode(x, edge_lst)

        super_z = z[-1]

        predicted_x = model.predict(super_z)

        loss = criterion(x.reshape(-1), predicted_x)

        loss.backward()
        optimizer.step()

        loss_ep += loss.cpu().detach().data.numpy()

    if epoch % 1 == 0:
        print(f"epoch: {epoch}, loss : {loss_ep}") '''

---------------------------------------------------------------------------
IndexError                                Traceback (most recent call last)
<ipython-input-103-9dd243b81c8e> in <module>
     71         edge_lst = data.edge_index.to(device)
     72
---> 73         z = model.encode(x, edge_lst)
     74
     75         super_z = z[-1]

<ipython-input-103-9dd243b81c8e> in encode(self, node_feature, edge_lst)
     36
     37     def encode(self, node_feature, edge_lst):
---> 38         z = self.encoder(node_feature, edge_lst)
     39         return z
     40

~/anaconda3/lib/python3.8/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
    725             result = self._slow_forward(*input, **kwargs)
    726         else:
--> 727             result = self.forward(*input, **kwargs)
    728         for hook in itertools.chain(
    729                 _global_forward_hooks.values(),

<ipython-input-103-9dd243b81c8e> in forward(self, x, edge_index)
     15
     16     def forward(self, x, edge_index):
---> 17         x = self.conv1(x, edge_index).relu()
     18         return self.conv2(x, edge_index)
     19

~/anaconda3/lib/python3.8/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
    725             result = self._slow_forward(*input, **kwargs)
    726         else:
--> 727             result = self.forward(*input, **kwargs)
    728         for hook in itertools.chain(
    729                 _global_forward_hooks.values(),

~/anaconda3/lib/python3.8/site-packages/torch_geometric/nn/conv/gcn_conv.py in forward(self, x, edge_index, edge_weight)
    159                 if cache is None:
    160                     edge_index, edge_weight = gcn_norm(  # yapf: disable
--> 161                         edge_index, edge_weight, x.size(self.node_dim),
    162                         self.improved, self.add_self_loops)
    163                     if self.cached:

IndexError: Dimension out of range (expected to be in range of [-1, 0], but got -2)


title: Can I use a convolution filter instead of a dense layer for clasification?

body:
I was reading a decent paper S-DCNet and I fell upon a section (page3,table1,classifier) where a convolution layer has been used to to produce a binary classification output as part of an internal process. Since I am a noob and when someone talks to me about classification I automatically make a synapse relating to FCs combined with softmax, I started wondering ... Is this a possible thing to do? Can indeed a convolutional layer be used to classify a binary outcome? The whole concept triggered my imagination so much that I insist on getting answers...
Honestly, how does this actually work? What is the difference between using a convolution filter instead of a fully connected layer for classification purposes?


title: "Package _libgcc_mutex conflicts for:" when using conda install pytorch=0.3.0 torchvision=0.2.0 cuda90

body:
I'm following this git repo and trying to install the requirements as they asked. I'm running the command:
conda install pytorch=0.3.0 torchvision=0.2.0 cuda90 -c pytorch and getting the error:
Package _libgcc_mutex conflicts for:
pytorch=0.3.0 -> libgcc-ng -> _libgcc_mutex=[build=main]
python=3.6 -> libgcc-ng[version='>=7.3.0'] -> _libgcc_mutex=[build=main]

It also says:
UnsatisfiableError: The following specifications were found
to be incompatible with the existing python installation in your environment:

Specifications:

  - pytorch=0.3.0 -> python[version='>=3.7,<3.8.0a0|>=3.8,<3.9.0a0|>=3.9,<3.10.0a0']
  - torchvision=0.2.0 -> python[version='>=3.7,<3.8.0a0|>=3.8,<3.9.0a0|>=3.9,<3.10.0a0']

Your python: python=3.6

I tried changing my python to be 3.7 but that didn't work either-- got the following error:
The following specifications were found to be incompatible with each other:

Output in format: Requested package -> Available versions

Package _libgcc_mutex conflicts for:
pytorch=0.3.0 -> libgcc-ng -> _libgcc_mutex=[build=main]
python=3.7 -> libgcc-ng[version='>=7.3.0'] -> _libgcc_mutex=[build=main]

The git repo clearly says "Install python3.6 and pytorch 3" using conda install pytorch=0.3.0 torchvision=0.2.0 cuda90 -c pytorch. So I'm not sure where is the issue.


title: Python: BERT Tokenizer cannot be loaded

body:
I am working on the bert-base-mutilingual-uncased model but when I try to set the TOKENIZER in the config it throws an OSError.
Model Config
class config:
    DEVICE = "cuda:0"
    MAX_LEN = 256
    TRAIN_BATCH_SIZE = 8
    VALID_BATCH_SIZE = 4
    EPOCHS = 1

    BERT_PATH = {"bert-base-multilingual-uncased": "workspace/data/jigsaw-multilingual/input/bert-base-multilingual-uncased"}
    MODEL_PATH = "workspace/data/jigsaw-multilingual/model.bin"

    TOKENIZER = transformers.BertTokenizer.from_pretrained(
            BERT_PATH["bert-base-multilingual-uncased"],
            do_lower_case=True)

Error
    ---------------------------------------------------------------------------
    OSError                                   Traceback (most recent call last)
    <ipython-input-33-83880b6b788e> in <module>
    ----> 1 class config:
          2 #     def __init__(self):
          3
          4         DEVICE = "cuda:0"
          5         MAX_LEN = 256

    <ipython-input-33-83880b6b788e> in config()
         11         TOKENIZER = transformers.BertTokenizer.from_pretrained(
         12             BERT_PATH["bert-base-multilingual-uncased"],
    ---> 13             do_lower_case=True)

    /opt/conda/lib/python3.6/site-packages/transformers/tokenization_utils_base.py in from_pretrained(cls, *inputs, **kwargs)
       1138
       1139         """
    -> 1140         return cls._from_pretrained(*inputs, **kwargs)
       1141
       1142     @classmethod

    /opt/conda/lib/python3.6/site-packages/transformers/tokenization_utils_base.py in _from_pretrained(cls, pretrained_model_name_or_path, *init_inputs, **kwargs)
       1244                     ", ".join(s3_models),
       1245                     pretrained_model_name_or_path,
    -> 1246                     list(cls.vocab_files_names.values()),
       1247                 )
       1248             )

    OSError: Model name 'workspace/data/jigsaw-multilingual/input/bert-base-multilingual-uncased' was not
 found in tokenizers model name list (bert-base-uncased, bert-large-uncased, bert-base-cased, bert-large-cased, bert-base-multilingual-uncased, bert-base-multilingual-cased, bert-base-chinese, bert-base-german-cased, bert-large-uncased-whole-word-masking, bert-large-cased-whole-word-masking,
bert-large-uncased-whole-word-masking-finetuned-squad, bert-large-cased-whole-word-masking-finetuned-squad, bert-base-cased-finetuned-mrpc,
bert-base-german-dbmdz-cased, bert-base-german-dbmdz-uncased, TurkuNLP/bert-base-finnish-cased-v1, TurkuNLP/bert-base-finnish-uncased-v1,
wietsedv/bert-base-dutch-cased).

We assumed 'workspace/data/jigsaw-multilingual/input/bert-base-multilingual-uncased' was a path, a model   identifier, or url to a directory containing vocabulary files named ['vocab.txt'] but couldn't find such
 vocabulary files at this path or url.

As I can interpret the error, it says that the vocab.txt file was not found at the given location but actually its present.
Following are the files available in the bert-base-multilingual-uncased folder:

config.json
pytorch_model.bin
vocab.txt

I am new to working with bert, so I am not sure if there is a different way to define the tokenizer.


title: Python image types, shapes, and channels for segmentation

body:
I am using this tutorial for instance segmentation in PyTorch. The test data the tutorial uses includes images and accompanying image masks from a dataset available here. I have an example of one of the image masks from that data set here (example data for this question). That mask looks like this by default in the dataset:

The tutorial uses this code:
mask.putpalette([
    0, 0, 0, # black background
    255, 0, 0, # index 1 is red
    255, 255, 0, # index 2 is yellow
    255, 153, 0, # index 3 is orange
])

as explanatory step for the mask to make it look like this:

but that code is not requisite in the segmentation process itself. It's just used to show what the mask contains.
I am trying to use my own image data. I created masks for the images in G.I.M.P. This is one of the masks I made. It looks like this by default.

As I try to run the tutorial code, I have problems with the masks. This code chunk creates a class that creates PyTorch datasets.
import os
import numpy as np
import torch
import torch.utils.data
from PIL import Image

class PennFudanDataset(torch.utils.data.Dataset):
    def __init__(self, root, transforms=None):
        self.root = root
        self.transforms = transforms
        # load all image files, sorting them to
        # ensure that they are aligned
        self.imgs = list(sorted(os.listdir(os.path.join(root, "PNGImages"))))
        self.masks = list(sorted(os.listdir(os.path.join(root, "PedMasks"))))

    def __getitem__(self, idx):
        # load images ad masks
        img_path = os.path.join(self.root, "PNGImages", self.imgs[idx])
        mask_path = os.path.join(self.root, "PedMasks", self.masks[idx])
        img = Image.open(img_path).convert("RGB")
        # note that we haven't converted the mask to RGB,
        # because each color corresponds to a different instance
        # with 0 being background
        mask = Image.open(mask_path)

        mask = np.array(mask)
        # instances are encoded as different colors
        obj_ids = np.unique(mask)
        # first id is the background, so remove it
        obj_ids = obj_ids[1:]

        # split the color-encoded mask into a set
        # of binary masks
        masks = mask == obj_ids[:, None, None]

        # get bounding box coordinates for each mask
        num_objs = len(obj_ids)
        boxes = []
        for i in range(num_objs):
            pos = np.where(masks[i])
            xmin = np.min(pos[1])
            xmax = np.max(pos[1])
            ymin = np.min(pos[0])
            ymax = np.max(pos[0])
            boxes.append([xmin, ymin, xmax, ymax])

        boxes = torch.as_tensor(boxes, dtype=torch.float32)
        # there is only one class
        labels = torch.ones((num_objs,), dtype=torch.int64)
        masks = torch.as_tensor(masks, dtype=torch.uint8)

        image_id = torch.tensor([idx])
        area = (boxes[:, 3] - boxes[:, 1]) * (boxes[:, 2] - boxes[:, 0])
        # suppose all instances are not crowd
        iscrowd = torch.zeros((num_objs,), dtype=torch.int64)

        target = {}
        target["boxes"] = boxes
        target["labels"] = labels
        target["masks"] = masks
        target["image_id"] = image_id
        target["area"] = area
        target["iscrowd"] = iscrowd

        if self.transforms is not None:
            img, target = self.transforms(img, target)

        return img, target

    def __len__(self):
        return len(self.imgs)

dataset = PennFudanDataset('PennFudanPed/')
dataset[0]

The last line returns:
(<PIL.Image.Image image mode=RGB size=559x536 at 0x7FCB4267C390>,
 {'area': tensor([35358., 36225.]), 'boxes': tensor([[159., 181., 301., 430.],
          [419., 170., 534., 485.]]), 'image_id': tensor([0]), 'iscrowd': tensor([0, 0]), 'labels': tensor([1, 1]), 'masks': tensor([[[0, 0, 0,  ..., 0, 0, 0],
           [0, 0, 0,  ..., 0, 0, 0],
           [0, 0, 0,  ..., 0, 0, 0],
           ...,
           [0, 0, 0,  ..., 0, 0, 0],
           [0, 0, 0,  ..., 0, 0, 0],
           [0, 0, 0,  ..., 0, 0, 0]],

          [[0, 0, 0,  ..., 0, 0, 0],
           [0, 0, 0,  ..., 0, 0, 0],
           [0, 0, 0,  ..., 0, 0, 0],
           ...,
           [0, 0, 0,  ..., 0, 0, 0],
           [0, 0, 0,  ..., 0, 0, 0],
           [0, 0, 0,  ..., 0, 0, 0]]], dtype=torch.uint8)})

When I run this code with my data,
...
dataset = four_chs('drive/MyDrive/chambers/')
dataset[0]

I get this error:
/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:38: DeprecationWarning: elementwise comparison failed; this will raise an error in the future.

---------------------------------------------------------------------------

TypeError                                 Traceback (most recent call last)

<ipython-input-17-12074ae9ab35> in <module>()
      1 len(dataset)
----> 2 dataset[0]

<ipython-input-1-99ab92a46ebe> in __getitem__(self, idx)
     42         boxes = []
     43         for i in range(num_objs):
---> 44             pos = np.where(masks[i])
     45             xmin = np.min(pos[1])
     46             xmax = np.max(pos[1])

TypeError: 'bool' object is not subscriptable

I'm not sure exactly what is going on, but there is a difference between my mask and the mask that is in the test data. They are both PNG file types, but it seems that mine has the red, blue, green channels split out with another channel or something, but I don't know what it is, based on the shape of the object in Python. This comes from one of the masks I made:
mask2 = np.array(mask1)
mask2.shape
(5312, 2988, 4)

For one of the test data masks:
mask2 = np.array(mask)
mask2.shape
(536, 559)

There appears to be only one channel? So with their different shapes, I guess that why I get the error from (this is an excerpt from the code I pasted earlier)
...
        mask_path = os.path.join(self.root, "masks", self.masks[idx])
        # note that we haven't converted the mask to RGB,
        # because each color corresponds to a different instance
        # with 0 being background
        mask = Image.open(mask_path)

        mask = np.array(mask)
        # instances are encoded as different colors
        obj_ids = np.unique(mask)
        # first id is the background, so remove it
        obj_ids = obj_ids[1:]

        # split the color-encoded mask into a set
        # of binary masks
        masks = mask == obj_ids[:, None, None]

        # get bounding box coordinates for each mask
        num_objs = len(obj_ids)
for i in range(num_objs):
             pos = np.where(masks[i])
...

How do I get my masks' shapes to match those of the masks in the test data so that I can use the rest of segmentation code create my PyTorch compatible dataset that will work with the segmentation algorithm? I'm not trying to get the same height and width, but change the number of channels/layers, but I don't think I want it to be grayscale.


title: Cannot export Yolov5 model using export

body:
I've trained Yolov5s model in colab.research env. After training I've moved best.pt to main yolov5 directory and renamed file to yolov5s.pt.
After calling export.py i got error listed below
!python models/export.py

Namespace(batch_size=1, img_size=[640, 640], weights='./yolov5s.pt')
Traceback (most recent call last):
  File "models/export.py", line 33, in <module>
    model = attempt_load(opt.weights, map_location=torch.device('cpu'))  # load FP32 model
  File "./models/experimental.py", line 137, in attempt_load
    model.append(torch.load(w, map_location=map_location)['model'].float().fuse().eval())  # load FP32 model
  File "/usr/local/lib/python3.7/dist-packages/torch/serialization.py", line 587, in load
    with _open_zipfile_reader(opened_file) as opened_zipfile:
  File "/usr/local/lib/python3.7/dist-packages/torch/serialization.py", line 242, in __init__
    super(_open_zipfile_reader, self).__init__(torch._C.PyTorchFileReader(name_or_buffer))
RuntimeError: [enforce fail at inline_container.cc:145] . PytorchStreamReader failed reading zip archive: failed finding central directory

I'm trying to export this model to use it in Android App. If I use best.pt I've got other error on loading network:
E/AndroidRuntime: FATAL EXCEPTION: main
    Process: org.pytorch.demo.objectdetection, PID: 6935
    java.lang.RuntimeException: Unable to start activity ComponentInfo{org.pytorch.demo.objectdetection/org.pytorch.demo.objectdetection.MainActivity}: com.facebook.jni.CppException: [enforce fail at inline_container.cc:222] . file not found: archive/constants.pkl
    (no backtrace available)
        at android.app.ActivityThread.performLaunchActivity(ActivityThread.java:3449)
        at android.app.ActivityThread.handleLaunchActivity(ActivityThread.java:3601)
        at android.app.servertransaction.LaunchActivityItem.execute(LaunchActivityItem.java:85)
        at android.app.servertransaction.TransactionExecutor.executeCallbacks(TransactionExecutor.java:135)
        at android.app.servertransaction.TransactionExecutor.execute(TransactionExecutor.java:95)
        at android.app.ActivityThread$H.handleMessage(ActivityThread.java:2066)
        at android.os.Handler.dispatchMessage(Handler.java:106)
        at android.os.Looper.loop(Looper.java:223)
        at android.app.ActivityThread.main(ActivityThread.java:7656)
        at java.lang.reflect.Method.invoke(Native Method)
        at com.android.internal.os.RuntimeInit$MethodAndArgsCaller.run(RuntimeInit.java:592)
        at com.android.internal.os.ZygoteInit.main(ZygoteInit.java:947)
     Caused by: com.facebook.jni.CppException: [enforce fail at inline_container.cc:222] . file not found: archive/constants.pkl
    (no backtrace available)
        at org.pytorch.NativePeer.initHybridAndroidAsset(Native Method)
        at org.pytorch.NativePeer.<init>(NativePeer.java:27)
        at org.pytorch.PyTorchAndroid.loadModuleFromAsset(PyTorchAndroid.java:31)
        at org.pytorch.demo.objectdetection.MainActivity.onCreate(MainActivity.java:165)
        at android.app.Activity.performCreate(Activity.java:8000)
        at android.app.Activity.performCreate(Activity.java:7984)
        at android.app.Instrumentation.callActivityOnCreate(Instrumentation.java:1309)
        at android.app.ActivityThread.performLaunchActivity(ActivityThread.java:3422)
            ... 11 more
I/Process: Sending signal. PID: 6935 SIG: 9

I haven't found any solutions on my own. Do you have any idea how can I fix this error?
Edit:
Fixed - I could export model if I linked to directory:
!python models/export.py --weights ./runs/train/exp7/weights/best.pt --img 640 --batch 1


title: Class weight implementation different for pytorch and tensorflow (which one is correct?)

body:
The implementation of class weights for categorical losses is different for PyTorch and Tensorflow. I will show this below. Pytorch uses a weighted mean of the losses whereas tensorflow uses a simple mean of the product of the class weights and losses. In backpropagation the difference is a multiplicative factor on the gradient vector. This factor is not important for large batch size (law of large numbers applies and the difference is absorbed into the learning rate) but for small batch sizes this will be important (as the law of large numbers is not applicable).
So my question is whether there is a correct way of doing class weights?
Details
We begin with pytorch
import torch
import numpy as np
from torch.nn import functional as F

N=3
C=5

np.random.seed(42)
input=np.random.randn(N,C)
target=np.random.randint(0,C,(N,))
weight=np.random.rand(C)

input_t=torch.tensor(input)
target_t = torch.tensor(target)
weight_t = torch.tensor(weight)

The library version of categorical cross entropy using class weights is given by
F.cross_entropy(input_t,target_t,weight=weight_t)

and the manual version is
(-torch.nn.LogSoftmax(1)(input_t)[range(3),target_t]*weight[target_t]).sum()/weight[target_t].sum()

both of which give tensor(2.2850, dtype=torch.float64).
Thus, we see that the pytorch version is a weighted average of the losses

Now we look at tensorflow. This one is a bit convoluted. I couldn't find any function in tensorflow or keras that admits class weights but keras.losses.CategoricalCrossentropy() does take sample_weights. On the other hand model.fit() does take class weights. To use that I implement a model that is the identity operator and use it to compute the loss
x=tf.nn.softmax(input,1)
y=tf.one_hot(target,C)

m=keras.Sequential([
                    keras.layers.Input(C,),
                    keras.layers.Lambda(lambda x: x)
                    ])
m.compile(loss=keras.losses.CategoricalCrossentropy())
m.fit(x=x,y=y,class_weight=dict(zip(range(C),weight)))

This gives 1/1 [==============================] - 0s 144ms/step - loss: 0.7173. Now lets use sample weights in the following way
keras.losses.CategoricalCrossentropy()(y,x,sample_weight=weight[target])

which gives <tf.Tensor: shape=(), dtype=float64, numpy=0.7173250317573547> which are the same. Now we can use pytorch again to see what these functions are doing.
(-torch.nn.LogSoftmax(1)(input_t)[range(3),target_t]*weight[target_t]).mean()

which also gives tensor(0.7173, dtype=torch.float64).
So we see that the way tensorflow class weights is implemented as a simple mean of product of the loss and the weight.


title: PyTorch: TypeError: 'int' object is not subscriptable

body:
I'm new to pytorch and I am creating a one hot encoding function for multi-layer perceptron but I'm having some issues. Here is the code:
def one_hot_encoding(label):
    for idx, val in enumerate(label):
        one_hot_outputs = [0]*len(label)
        idx_n = idx[val]
        one_hot_outputs[idx_n] = 1
return one_hot_outputs

I'm having a type error saying:
in one_hot_encoding(label)
  2     for idx, val in enumerate(label):
  3         one_hot_outputs = [0]*len(label)
> 4         idx_n = idx[val]
  5         one_hot_outputs[idx_n] = 1
  6     return one_hot_outputs
TypeError: 'int' object is not subscriptable

Any help?


title: Will pytorch performs correctly with python calculate codes in net?

body:
Take the fake codes below as an example:

class():
    def forward(input):
        x = some_torch_layers(input)
        x = some_torch_layers(x)
        ...
        x = sum(x) # or numpy or other operations
        x = some_torch_layers(x)
        return x

Will the pytorch net operates well? Especially, while the sum(x) performs well in the backward process.


title: A strange problem encountered during training

body:
A hyperlink to the paper   https://arxiv.org/abs/1811.11431
A hyperlink to the code  https://github.com/sacmehta/ESPNetv2

I redesigned the network architecture by using the paper ESPNetv2's EESP module, and used it's open source code method to train the network, but there was a strange phenomenon. The miou value of the training set is constantly increasing, but the miou value of the verification set is like a Random value, the value will remain unchanged at the end of the network training and is very small !

Parameters: 317052

Epoch   Loss(Tr)    Loss(val)   mIOU (tr)   mIOU (val

0       0.9102      2.3089      0.2807      0.0208

1       0.7575      3.2041      0.3170      0.0135

2       0.6914      2.9142      0.3383      0.0273

3       0.6619      3.5181      0.3469      0.0130

4       0.6431      3.2931      0.3555      0.0160

5       0.6274      3.8645      0.3601      0.0237

6       0.6175      3.1952      0.3643      0.0266


title: i want to convert the code below(neural network) from keras to pytorch

body:
i want to use pytorch instead of keras but i failed to do it my self

Keras

def _model(self):
    model = Sequential()
    model.add(Dense(units=64, input_dim=self.state_size,
    activation="relu"))
    model.add(Dense(units=32, activation="relu"))
    model.add(Dense(units=8, activation="relu"))
    model.add(Dense(self.action_size, activation="linear"))
    model.compile(loss="mse", optimizer=Adam(lr=0.001))

    return model

Pytorch

class Model(nn.Module):
    def __init__(self, input_dim):
        super(Model, self).__init__()
        self.fc1 = nn.ReLU(input_dim, 64)
        self.fc2 = nn.ReLU(64,32)
        self.fc3 = nn.Relu(32, 8)
        self.fc4 = nn.Linear(8, 3)
model = Model()
criterion = nn.MSELoss()
optimizer = torch.optim.Adam(model.parameters(), lr=0.001)


title: why tensor.numpy() outputs different value?

body:
I used tensor.numpy() on a feature matrix and a row of this matrix for comparison. I found the value is not the same for the row output and the corresponding row of the matrix. Anyone knows why?

def consine_distance_byrow(f1, f2):
    feat1 = f1.view(-1, 1)
    dists = torch.mm(f2, feat1)
    dists = dists.numpy()
    return dists

def consine_distance(f1, f2):
    feat1 = torch.transpose(f1, 0, 1)
    dists = torch.mm(f2, feat1)
    dists = dists.numpy()
    return dists

# load trainingset data
feature_path = '../../cnn/pytorch_result_train_val_' + str(self.feature_nums) + '.mat'
result = scipy.io.loadmat(feature_path)
train_feature = torch.FloatTensor(result['train_f']) # M x 512d
dists = consine_distance(train_feature, train_feature) # way1
for i in range(len(train_label)):
    distsT = consine_distance_byrow(train_feature[i], train_feature) # way2
    # here, dists[:,i] is slightly different with distsT

e.g.

dists[:0] = [ 1.0000001  -0.15086517 -0.08085391 ... -0.05950543 -0.03058994
  0.10164267]

while i=0, distT = [[ 0.99999964]
 [-0.15086518]
 [-0.0808539 ]
 ...
 [-0.05950541]
 [-0.03058996]
 [ 0.1016427 ]]

apparently, there is a slightly difference.

and this is caused after tensor.numpy(), before this step the two tensors are the same


title: Shrink or grow a tensor away from the origin in pytorch?

body:
I have a batch x ndim tensor.

I want to add noise to each observation that pushes it either toward or away from the origin without flipping any of the signs.

Can anyone recommend a good approach?


title: Replicating in pytorch https://www.d2l.ai/chapter_linear-networks/linear-regression-scratch.html

body:
I am trying to replicate the code in pytorch. However I am having some problems with the autograd function. I am having the following runtime error.

RuntimeError: Trying to backward through the graph a second time

The code is the following:

for epoch in range(num_epochs):
    # Assuming the number of examples can be divided by the batch size, all
    # the examples in the training data set are used once in one epoch
    # iteration. The features and tags of mini-batch examples are given by X
    # and y respectively
    for X, y in data_iter(batch_size, features, labels):
        print (X)
        print (y)
        l = loss(net(X,w,b) , y)
        print (l)
        l.backward(retain_graph=True)
        print (w.grad)
        print (b.grad)

        with torch.no_grad():
          w -= w.grad * 1e-5/batch_size
          b -= b.grad * 1e-5/batch_size
          w.grad.zero_()
          b.grad.zero_()

Can someone explain how the autograd works in python? If someone can recommend me a good resource on learning pytorch that will be great.


title: How to add a layer which is implemented by TensorFlow into a PyTorch neural model?

body:
I’d like to add some layer into a Pytorch based neural model. Basically I am trying to combine to codes together.

But I notice that the layer I want to add is the implemented by Tensorflow. I’d like to know if there is an easy way to integrate a TensorFlow layer into a Pytorch neural model… ?

The error is shown as:

module ‘torch.nn’ has no attribute ‘tensorflow_layer’


title: using Fasterrcnn for regression or for only finding the bounding box in an image

body:
Is it possible to use FasterRcnn in pytorch just for finding the bounding box without considering the classification part ?
And is it possible to change the loss of the classification part ( categorical cross entropy ) to a regression loss ( MSE ) ?


title: can we pass images for which height!=width through our CNN for training in pytorch?

body:
can we pass images for which height!=width through our CNN in Pytorch?

In CNN, I have convolution, batch-norm, max-pool, relu, and fully connected layers.

My network

self.conv_seqn = nn.Sequential(
nn.Conv2d(in_channels=3, out_channels=32, kernel_size=3, padding=1),
nn.BatchNorm2d(32),
nn.ReLU(inplace=True),
nn.MaxPool2d(kernel_size=2, stride=2),
nn.Conv2d(in_channels=32, out_channels=64, kernel_size=3, padding=1),
nn.ReLU(inplace=True),
nn.MaxPool2d(kernel_size=4, stride=4),
nn.Conv2d(in_channels=64, out_channels=128, kernel_size=3, padding=1),
nn.ReLU(inplace=True),
nn.MaxPool2d(kernel_size=4, stride=4),
)
self.fc_seqn = nn.Sequential(
nn.Linear(1843200, 256),
nn.ReLU(inplace=True),
nn.Linear(256, total_configs)
)

my forward function

forward()
{
 x = self.conv_seqn(x)
 x = x.view(x.size(0), -1)
 x = self.fc_seqn(x)
 return x
}

If input image of size 3840*1920*3 after applying conv_seqn() it should be of size [1, 128, 120, 60] but I getting the size of [1,128,120,120] (batch size =1 here)

any suggestion will be highly helpful.


title: The Batch Size of batch normalisation and gradient descent

body:
We need to choose a batch size for GD as well as the normalization, they both called batch size, but in actual implementation, do they need to be the same?
Or otherwise how the framework handle them?

In Pytorch for example, one batch size is defined in dataloader, e.g.

torch.utils.data.DataLoader(image_datasets[x],
batch_size=16,
shuffle=True,
num_workers=4)

and if use BN

self.bn1 = nn.BatchNorm2d(16)

Does it has to be the same (16) or it can be different ? If different, is there any preferred relationship between the two 'batch size'
Thanks


title: Why does my torch convolutional NN return same outputs?

body:
I am working with tutorial for convolutional neural nets on pytorch and using MNIST dataset as example. But I was getting same outputs for all labels, net always returns same tensor as result. I've decided to reduce complexity of net and decrease num of epoch, so here is my code:

from __future__ import print_function, division
import os
import torch
import pandas as pd
from skimage import io, transform
import numpy as np
import matplotlib.pyplot as plt
from torch.utils.data import Dataset, DataLoader
from torchvision import transforms, utils
# Ignore warnings
import warnings
warnings.filterwarnings("ignore")

plt.ion()   # interactive mode

class DigitsDataset(Dataset):

    def __init__(self, csv_file, transform=None):
        """
        Args:
            csv_file (string): Path to the csv file.

            transform (callable, optional): Optional transform to be applied
                on a sample.
        """
        self.digits_frame = pd.read_csv(csv_file)

        self.transform = transform

    def __len__(self):
        return len(self.digits_frame)

    def __getitem__(self, idx):
        if torch.is_tensor(idx):
            idx = idx.tolist()

        label = digits_df.iloc[idx, 0]
        digit_pixels = digits_df.iloc[idx, 1:]
        digit_pixels = np.asarray(digit_pixels)
        digit_pixels = digit_pixels.astype('float').reshape(28, 28)
        sample = {'label' : label, 'image' : digit_pixels}
        if self.transform:
            sample['image'] = self.transform(sample['image'])

        return sample

class GrayScaleTransform:
    ''' Scale intensity from [0,255] to [0,1]'''
    def __init__(self, new_min, new_max):
        self.new_min = new_min
        self.new_max = new_max

    def __call__(self, x):
        return (x) * (self.new_max - self.new_min) / (255) + self.new_min

min_max_transform = GrayScaleTransform(new_min = 0, new_max = 1)

train_dataset = DigitsDataset(csv_file='data/train.csv', transform = min_max_transform)
test_dataset = DigitsDataset(csv_file='data/test.csv', transform = min_max_transform)

train_loader = DataLoader(train_dataset)
test_loader = DataLoader(test_dataset)

learning_rate = 0.1
num_epochs = 2

from torch import nn
class ConvNet(nn.Module):
    def __init__(self):
        super(ConvNet, self).__init__()
        self.layer1 = nn.Sequential( nn.Conv2d(1, 4, kernel_size=5, stride=1, padding=2),
        nn.ReLU(), nn.MaxPool2d(kernel_size=2, stride=2))
        self.layer2 = nn.Sequential( nn.Conv2d(4, 8, kernel_size=5, stride=1, padding=2),
        nn.ReLU(), nn.MaxPool2d(kernel_size=2, stride=2))
        self.drop_out = nn.Dropout()
        self.fc1 = nn.Linear(7 * 7 * 8, 10)
    def forward(self, x):
        out = self.layer1(x)
        out = self.layer2(out)
        out = out.reshape(out.size(0), -1)
        out = self.drop_out(out)
        out = self.fc1(out)
        return out

device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
model = ConvNet()
model.to(device)
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)

total_step = len(train_loader)
loss_list = []
acc_list = []
for epoch in range(num_epochs):
    for i, sample in enumerate(train_loader):
        # Прямой запуск
        img = sample['image'].view(-1, 1, 28, 28).float().to(device)
        label = sample['label'].to(device)
        output = model(img)
        loss = criterion(output, label)
        loss_list.append(loss.item())

        # Обратное распространение и оптимизатор
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

But I am still getting the same tensor as output. What am I doing wrong? Why am I always getting the same output?


title: Can't install Pytorch with pip by the commands provided by official website ("Could not find version ..." "No matching distribution ...")

body:
I try to follow the instruction on Pytorch official website and install Pytorch with pip. Just installed Python 3.8.2 yesterday. My laptop is Acer Aspire 5, so I choose None for CUDA. Then it reports Error as "Could not find a version that satisfies the requirement..." and "No matching distribution for..."

What did I do wrong? How can I install the latest Pytorch?


title: Difficulty in Implementing a simple single-layer RNN using Pytorch's base class “nn.Linear” class

body:
While working on making a simple RNN using Pytorch nn.linear function. So firstly I initialized my weights as

self.W_x = nn.Linear(self.input_dim, self.hidden_dim, bias=True)
self.W_h = nn.Linear(self.hidden_dim, self.hidden_dim, bias=True)

Now in the main step where I am getting the result of the current state by using the previous state and the values of the weights using this code statement

h_t = np.tanh((inp * self.W_x) + (prev_h * self.W_h))

So here I am getting the python error as shown below

TypeError: mul(): argument 'other' (position 1) must be Tensor, not Linear

Can anyone help me with his regards...


title: RuntimeError: polygamma(n,x) is not implemented for n>=2 error

body:
I am experimenting with different loss functions for a model. When I try to use loggamma and digamma functions, it gives me "RuntimeError: polygamma(n,x) is not implemented for n>=2, but was 2" error. Here is a sample code reproducing the error:

import torch

init_var = torch.randn([1,2], requires_grad = True)

x = torch.randn([1,2])

y = (init_var*x)

gt = torch.randn([1,2])

log_gamma_res = torch.digamma(y) #y

loss = (gt - log_gamma_res).sum()

grad = torch.autograd.grad(loss,init_var, create_graph = True)[0]

loss2 = (y - grad).sum()

loss2.backward()

Error message:

RuntimeError: polygamma(n,x) is not implemented for n>=2, but was 2 (polygamma_kernel at /opt/conda/conda-bld/pytorch_1587428398394/work/build/aten/src/ATen/native/cpu/UnaryOpsKernel.cpp.AVX2.cpp:217)
frame #0: c10::Error::Error(c10::SourceLocation, std::string const&) + 0x4e (0x7f2a8c4b0b5e in /home/deep/anaconda3/lib/python3.7/site-packages/torch/lib/libc10.so)
frame #1: <unknown function> + 0x113eff0 (0x7f2a6a7f2ff0 in /home/deep/anaconda3/lib/python3.7/site-packages/torch/lib/libtorch_cpu.so)
frame #2: <unknown function> + 0xbed703 (0x7f2a6a2a1703 in /home/deep/anaconda3/lib/python3.7/site-packages/torch/lib/libtorch_cpu.so)
frame #3: at::native::polygamma_out(at::Tensor&, long, at::Tensor const&) + 0x5b (0x7f2a6a294d1b in /home/deep/anaconda3/lib/python3.7/site-packages/torch/lib/libtorch_cpu.so)
frame #4: <unknown function> + 0xe9bf88 (0x7f2a6a54ff88 in /home/deep/anaconda3/lib/python3.7/site-packages/torch/lib/libtorch_cpu.so)
frame #5: at::native::polygamma(long, at::Tensor const&) + 0x11a (0x7f2a6a29bf2a in /home/deep/anaconda3/lib/python3.7/site-packages/torch/lib/libtorch_cpu.so)
frame #6: <unknown function> + 0xe9c028 (0x7f2a6a550028 in /home/deep/anaconda3/lib/python3.7/site-packages/torch/lib/libtorch_cpu.so)
frame #7: <unknown function> + 0xee7e30 (0x7f2a6a59be30 in /home/deep/anaconda3/lib/python3.7/site-packages/torch/lib/libtorch_cpu.so)
frame #8: <unknown function> + 0x2955ef5 (0x7f2a6c009ef5 in /home/deep/anaconda3/lib/python3.7/site-packages/torch/lib/libtorch_cpu.so)
frame #9: <unknown function> + 0xee7e30 (0x7f2a6a59be30 in /home/deep/anaconda3/lib/python3.7/site-packages/torch/lib/libtorch_cpu.so)
frame #10: torch::autograd::generated::PolygammaBackward::apply(std::vector<at::Tensor, std::allocator<at::Tensor> >&&) + 0x189 (0x7f2a6bcb6629 in /home/deep/anaconda3/lib/python3.7/site-packages/torch/lib/libtorch_cpu.so)
frame #11: <unknown function> + 0x2ae8215 (0x7f2a6c19c215 in /home/deep/anaconda3/lib/python3.7/site-packages/torch/lib/libtorch_cpu.so)
frame #12: torch::autograd::Engine::evaluate_function(std::shared_ptr<torch::autograd::GraphTask>&, torch::autograd::Node*, torch::autograd::InputBuffer&) + 0x16f3 (0x7f2a6c199513 in /home/deep/anaconda3/lib/python3.7/site-packages/torch/lib/libtorch_cpu.so)
frame #13: torch::autograd::Engine::thread_main(std::shared_ptr<torch::autograd::GraphTask> const&, bool) + 0x3d2 (0x7f2a6c19a2f2 in /home/deep/anaconda3/lib/python3.7/site-packages/torch/lib/libtorch_cpu.so)
frame #14: torch::autograd::Engine::thread_init(int) + 0x39 (0x7f2a6c192969 in /home/deep/anaconda3/lib/python3.7/site-packages/torch/lib/libtorch_cpu.so)
frame #15: torch::autograd::python::PythonEngine::thread_init(int) + 0x38 (0x7f2a6f2d7558 in /home/deep/anaconda3/lib/python3.7/site-packages/torch/lib/libtorch_python.so)
frame #16: <unknown function> + 0xc819d (0x7f2ab33d619d in /home/deep/anaconda3/lib/python3.7/site-packages/matplotlib/../../../libstdc++.so.6)
frame #17: <unknown function> + 0x76db (0x7f2ac7ef96db in /lib/x86_64-linux-gnu/libpthread.so.0)
frame #18: clone + 0x3f (0x7f2ac7c2288f in /lib/x86_64-linux-gnu/libc.so.6)


title: Pytorch vsion size mismatch, m1

body:
im trying to run a simple linear regression but i have error when i try to train.

The size of images is the shapes of data train print(dataset_train[0][0].shape) shows me torch.Size([3, 227, 227])

size_of_image=3*227*227

class linearRegression(nn.Module):
    def __init__(self, inputSize, outputSize):
        super(linearRegression, self).__init__()
        self.linear = nn.Linear(inputSize, outputSize)

    def forward(self, x):
        out = self.linear(x)
        return out

model = linearRegression(size_of_image, 1)
optimizer = torch.optim.SGD(model.parameters(), lr=0.1)
criterion = torch.nn.CrossEntropyLoss()
trainloader = DataLoader(dataset = dataset_train, batch_size = 1000)
for epoch in range(5):
    for x, y in trainloader:
        yhat = model(x)
        loss = criterion(yhat, y)
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

I tried to unserstand what its the mean of error but i dont found a solution, can anyone help me?

---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-44-6f00f9272a22> in <module>
      1 for epoch in range(5):
      2     for x, y in trainloader:
----> 3         yhat = model(x)
      4         loss = criterion(yhat, y)
      5         optimizer.zero_grad()

~/PycharmProjects/estudios/venv/lib/python3.8/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    548             result = self._slow_forward(*input, **kwargs)
    549         else:
--> 550             result = self.forward(*input, **kwargs)
    551         for hook in self._forward_hooks.values():
    552             hook_result = hook(self, input, result)

<ipython-input-21-d20eb6e0c349> in forward(self, x)
      5
      6     def forward(self, x):
----> 7         out = self.linear(x)
      8         return out

~/PycharmProjects/estudios/venv/lib/python3.8/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    548             result = self._slow_forward(*input, **kwargs)
    549         else:
--> 550             result = self.forward(*input, **kwargs)
    551         for hook in self._forward_hooks.values():
    552             hook_result = hook(self, input, result)

~/PycharmProjects/estudios/venv/lib/python3.8/site-packages/torch/nn/modules/linear.py in forward(self, input)
     85
     86     def forward(self, input):
---> 87         return F.linear(input, self.weight, self.bias)
     88
     89     def extra_repr(self):

~/PycharmProjects/estudios/venv/lib/python3.8/site-packages/torch/nn/functional.py in linear(input, weight, bias)
   1610         ret = torch.addmm(bias, input, weight.t())
   1611     else:
-> 1612         output = input.matmul(weight.t())
   1613         if bias is not None:
   1614             output += bias

Im RuntimeError: size mismatch, m1: [681000 x 227], m2: [154587 x 1] at /pytorch/aten/src/TH/generic/THTensorMath.cpp:41