title: Failed to deploy pytorch text classifier model on Google AI Platform, Error: Can't get attribute on

body:
I followed this article, try to deploy a PyTorch text classifier on Google AI platform, got below error.

Create Version failed. Bad model detected with error: "Failed to load
model: Unexpected error when loading the model: Can't get attribute
'TextPreprocessor' on <module 'main' from
'prediction_server_beta.py'> (Error code: 0)"

I deployed the model through GUI, here is the setting
!gcloud ai-platform versions create {v17} \
    --model {pytorch_text_classfier} \
    --origin=gs://pytorch_text_classfier_package \
    --python-version=3.7 \
    --runtime-version={1.15} \
    --framework ={"Custom prediction routine (BETA)"} \
    --package-uris=gs://pytorch_model_distribution_package_v3/my_package-1.7.tar.gz\
    --machine-type=mls1-c4-m4 \
    --prediction-class=model_prediction.CustomModelPrediction

I uploaded five files to the bucket, model_prediction.py, preprocess.py,  processor_state.pkl, setup.py, torch_model.py, torch_saved_model.pt. The tar.gz file is in the other bucket.
Is there anyone who knows how to fix this error?
Below is the model_prediction.py, preprocess.py, torch_model.py, setup.py code.
model_prediction.py
import os
import pickle
import numpy as np
import torch
import torch.nn as nn
from torch.autograd import Variable
from tensorflow.python.keras.preprocessing import sequence
from tensorflow.keras.preprocessing import text
from preprocess import TextPreprocessor
from torch_model import TorchTextClassifier

device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')

class CustomModelPrediction(object):
 def __init__(self, model, processor):
   self._model = model
   self._processor = processor

 def _postprocess(self, predictions):
   labels = ['no confusion', 'confusion']
   label_indexes = [np.argmax(prediction) for prediction in predictions.detach().numpy()]
   return [labels[label_index] for label_index in label_indexes]

 def predict(self, instances, **kwargs):
   preprocessed_data = self._processor.transform(instances)
   predictions =  self._model(Variable(torch.Tensor(preprocessed_data).long()))
   labels = self._postprocess(predictions)
   return labels

 @classmethod
 def from_path(cls, model_dir):

  import torch
  import torch_model

  state_dict = torch.load(os.path.join(model_dir,'torch_saved_model.pt'), map_location= device)
  print(f'Model loaded from <== {model_dir}')
  model = TorchTextClassifier().to(device)
  model.load_state_dict(state_dict['model_state_dict'])
  model.eval()
  print('set to eval mode')
  with open(os.path.join(model_dir, 'processor_state.pkl'), 'rb') as f:
      processor = pickle.load(f)
  print('loaded processor')
  return cls(model, processor)

proprocess.py
from tensorflow.python.keras.preprocessing import sequence
from tensorflow.keras.preprocessing import text

class TextPreprocessor(object):
 def __init__(self, vocab_size, max_sequence_length):
   self._vocab_size = vocab_size
   self._max_sequence_length = max_sequence_length
   self._tokenizer = None

 def fit(self, text_list):
   # Create vocabulary from input corpus.
   tokenizer = text.Tokenizer(num_words=self._vocab_size)
   tokenizer.fit_on_texts(text_list)
   self._tokenizer = tokenizer

 def transform(self, text_list):
   # Transform text to sequence of integers
   text_sequence = self._tokenizer.texts_to_sequences(text_list)

   # Fix sequence length to max value. Sequences shorter than the length are
   # padded in the beginning and sequences longer are truncated
   # at the beginning.
   padded_text_sequence = sequence.pad_sequences(text_sequence, maxlen=self._max_sequence_length, padding='post')
   return padded_text_sequence

setup.py
from setuptools import setup

setup(
 name="my_package",
 version="1.6",
 include_package_data=True,
 scripts=["preprocess.py", "model_prediction.py", "torch_model.py"],
 install_requires=['torch @ https://download.pytorch.org/whl/cpu/torch-1.7.0%2Bcpu-cp37-cp37m-linux_x86_64.whl']
)

torch_model.py
import torch
import torch.nn as nn
from torch.autograd import Variable
from tensorflow.python.keras.preprocessing import sequence
from tensorflow.keras.preprocessing import text

text_field_vocab_length = 15832  # this is set based on training data

class TorchTextClassifier(nn.Module):

    def __init__(self, dimension=128):
        super(TorchTextClassifier, self).__init__()

        self.embedding = nn.Embedding(text_field_vocab_length, 300)
        self.dimension = dimension
        self.lstm = nn.LSTM(input_size=300,
                            hidden_size=dimension,
                            num_layers=1,
                            batch_first=True,
                            bidirectional=True)
        self.drop = nn.Dropout(p=0.5)

        self.fc = nn.Linear(2*dimension, 2)

    def forward(self, text, text_len):

        text_emb = self.embedding(text)

        packed_input = pack_padded_sequence(text_emb, text_len, batch_first=True, enforce_sorted=False)
        packed_output, _ = self.lstm(packed_input)
        output, _ = pad_packed_sequence(packed_output, batch_first=True)

        out_forward = output[range(len(output)), text_len - 1, :self.dimension]
        out_reverse = output[:, 0, self.dimension:]
        out_reduced = torch.cat((out_forward, out_reverse), 1)
        text_fea = self.drop(out_reduced)

        text_fea = self.fc(text_fea)
        text_fea = torch.squeeze(text_fea, 1)
        text_out = torch.softmax(text_fea, dim=-1)

        return text_out


title: Is there an automatic way to delete all pytorch objects in GPU?

body:
Deleting a specific pytorch object is as simple as
del VGG19 #model
del X_train #tensor

followed by
torch.cuda.empty_cache()

To release the memory allocated.
But is there a way to wipe the GPU clean, without specifying each and every object?


title: How to initialize columns in hybrid sparse tensor

body:
How initialize in pytorch hybrid tensor torch.sparse_coo_tensor (one dimension is sparse and other is not), which have the following dense representation?
array([[1, 0, 5, 0],
       [2, 0, 6, 0],
       [3, 0, 7, 0],
       [4, 0, 8, 0]])

What should I put into the indices argument?


title: How to load a trained model to inference the predicted data

body:
I trained and saved CNN model for 1000 epochs and now want to retrieve validation data (predicted images). In code below test_pred and test_real outputs predicted and real images in validation set. Should I load and run the saved model for another 1 epoch to retrieve predicted images (this will end in CUDA out of memory since data are huge)? Or there are other ways? You can see part of my code below:
for epoch in range(epochs):
    mse_train_losses= []
    mae_train_losses = []
    N_train = []
    mse_val_losses = []
    mae_val_losses = []
    N_test = []

    if save_model:
        if epoch % 50 ==0:
            checkpoint = {'state_dict' : model.state_dict(),'optimizer' : optimizer.state_dict()}
            save_checkpoint(checkpoint)

    model.train()
    for data in train_loader:

        x_train_batch, y_train_batch = data[0].to(device,
            dtype=torch.float), data[1].to(device, dtype=torch.float)
        y_train_pred = model(x_train_batch)            # 1) Forward pass
        mse_train_loss = criterion(y_train_batch, y_train_pred, x_train_batch, mse)
        mae_train_loss = criterion(y_train_batch, y_train_pred, x_train_batch, l1loss)

        optimizer.zero_grad()
        mse_train_loss.backward()
        optimizer.step()

        mse_train_losses.append(mse_train_loss.item())
        mae_train_losses.append(mae_train_loss.item())
        N_train.append(len(x_train_batch))

    test_pred=[]
    test_real=[]
    model.eval()
    with torch.no_grad():
        for data in test_loader:
            x_test_batch, y_test_batch = data[0].to(device,
                dtype=torch.float), data[1].to(device, dtype=torch.float)

            y_test_pred = model(x_test_batch)
            mse_val_loss = criterion(y_test_batch, y_test_pred, x_test_batch, mse)
            mae_val_loss = criterion(y_test_batch, y_test_pred, x_test_batch, l1loss)

            mse_val_losses.append(mse_val_loss.item())
            mae_val_losses.append(mae_val_loss.item())
            N_test.append(len(x_test_batch))

            test_pred.append(y_test_pred)
            test_real.append(y_test_batch)


title: How to convert torch::conv2d return value(tensor) to cv::mat

body:
I run the following program:
read a picture of 3 channel input torch::nn::conv2d(3,3,3).pad(1).stride(1),then I got the results:
enter image description here
//code:
cv::Mat img = cv::imread("babyx2.png", 1);
torch::Tensor img_tensor = torch::from_blob(img.data, { img.rows, img.cols, 3 }, torch::kByte);
img_tensor = img_tensor.permute({ 2, 0, 1 });
img_tensor = img_tensor.unsqueeze(0);
img_tensor = img_tensor.to(kFloat32);
torch::Tensor result = C1(img_tensor);       //C1: torch::nn::Conv2d(torch::nn::Conv2dOptions(3, 3, 5).padding(1))
//.....then get the result use following method
auto ToCvImage(at::Tensor tensor)
{
    int width = tensor.sizes()[0];
    int height = tensor.sizes()[1];
    //auto sizes = tensor.sizes();
    try
    {
        cv::Mat output_mat(cv::Size{ height, width }, CV_8UC3, tensor.data_ptr<uchar>());

        return output_mat.clone();
    }
    catch (const c10::Error& e)
    {
        std::cout << "an error has occured : " << e.msg() << std::endl;
    }
    return cv::Mat(height, width, CV_8UC3);
}

what happen???I don’t think the result picture is right,how to do…help
I input picture following:
enter image description here
then torch::nn::Conv2D(3,3,3).padding.stride
why the return value of torch::nn::Conv2D like that??? How to show it in the right way？？


title: An error ocurred while starting the kernel Spyder when restarting with Pytorch

body:
Whenever I restart kernel while using Pytorch I got this error. How to resolve this. I tried to restart again, create new console but the error persist until I restart the spyder by closing and opening it again
Traceback (most recent call last):
File "C:\Anaconda3\lib\runpy.py", line 193, in _run_module_as_main
"__main__", mod_spec)
File "C:\Anaconda3\lib\runpy.py", line 85, in _run_code
exec(code, run_globals)
File "C:\Anaconda3\lib\site‑packages\spyder_kernels\console\__main__.py", line 11, in
start.main()
File "C:\Anaconda3\lib\site‑packages\spyder_kernels\console\start.py", line 287, in main
import_spydercustomize()
File "C:\Anaconda3\lib\site‑packages\spyder_kernels\console\start.py", line 39, in import_spydercustomize
import spydercustomize
File "C:\Anaconda3\lib\site‑packages\spyder_kernels\customize\spydercustomize.py", line 17, in
import pdb
File "C:\Anaconda3\lib\pdb.py", line 76, in
import code
File "D:\Datasets\Image dataset\Xray\code.py", line 77, in
train_loss=fit_train(batch)
File "D:\Datasets\Image dataset\Xray\code.py", line 66, in fit_train
img1_emb,img2_emb=model(img1.to(device)),model(img2.to(device))
File "C:\Anaconda3\lib\site‑packages\torch\nn\modules\module.py", line 727, in _call_impl
result = self.forward(*input, **kwargs)
File "D:\Datasets\Image dataset\Xray\cnn_model.py", line 37, in forward
x=self.relu(self.conv1_2(x))
File "C:\Anaconda3\lib\site‑packages\torch\nn\modules\module.py", line 727, in _call_impl
result = self.forward(*input, **kwargs)
File "C:\Anaconda3\lib\site‑packages\torch\nn\modules\conv.py", line 423, in forward
return self._conv_forward(input, self.weight)
File "C:\Anaconda3\lib\site‑packages\torch\nn\modules\conv.py", line 420, in _conv_forward
self.padding, self.dilation, self.groups)
RuntimeError: CUDA out of memory. Tried to allocate 294.00 MiB (GPU 0; 6.00 GiB total capacity; 4.37 GiB already allocated; 72.82 MiB free; 4.42 GiB rese

rved in total by PyTorch)


title: Supplying weights to nn.functional.conv2d in PyTorch

body:
I am trying to learn the weights of a 3x3 conv2d layer accepting 3 channels and outputting 3 channels. For this discussion consider bias=0 in each case. However, the weights of the conv layer are learned indirectly. I have a 2 layered Multi layer perception having 9 nodes in first layer and 9 in the second. The weights of the 2d conv layer are then precisely the weights learned using this MLP i.e. nn.Linear(9,9). I understand in this case I will have to use nn.functional.conv2d(input,weight). But how exactly to extract the weights from MLP and use it for convolution is not clear and can think of the following.
class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        m=nn.Linear(9,9)
    def forward(self, x):
        # some operations involving MLP `m`
        return nn.Functional.conv2d(x,m.weight)

Can some one provide a short, dummy code in PyTorch to achieve this training configuration allowing backpropagation?


title: The Transforms in the Torchvision isn't working

body:
This is my first post, please cut me some slack...
Anyway having installed torch and torchvision, I was greeted with

ImportError                               Traceback (most recent call last)
 in
----> 1 from torchvision import models, transform
2 import torch
ImportError: cannot import name 'transform' from 'torchvision' (C:\Users\bala006\Anaconda3\lib\site-packages\torchvision_init_.py)
Yes, this is in Jupyter, but for some reason, the models and the datasets worked but the transforms don't work...
Please help.


title: Get 2d indices for max and min in pytorch

body:
lets say I have a tensor of shape [batchsize , 7 , 7]
I wish to get for each sample in the batch the maximum two values with 2d indices + the minimum two values with 2d indices
I have tried something around :
torch.nonzero ((a == a.max (dim = 1, keepdim = True)[0]))
Example : lets say we have a tensor of shape [2 , 3  , 3]

tensor = [[[1,2,3] , [4,5,6] , [6,7,8]] ,
         [[-9,-8,-7],[-6,-5,-4],[-3,-2,-1]]]

the maximum is : [(2,2) , (2,1)] , [(2,2) , (2,1)] for the two tensors,
the minimum is : [(0 , 0) , (0,1)] , [(0,0) , (0,1)] for the two tensors as well.

but it works over rows/columns not on the whole tensor
thanks


title: Torch: How to inspect weights after training?

body:
I am wondering what I am doing wrong when looking to see how the weights changed during training.
My loss goes down considerably but it appears that the initialized weights are the same as trained weights. Am I looking in the wrong location? I would appreciate any insight that you might have!
import torch
import numpy as np
from torchvision import datasets, transforms
from torch.utils.data import DataLoader
import torch.nn.functional as F

# setup GPU/CPU processing
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")

# initialize model
class mlp1(torch.nn.Module):
    def __init__(self, num_features, num_hidden, num_classes):
        super(mlp1, self).__init__()
        self.num_classes = num_classes
        self.input_layer = torch.nn.Linear(num_features, num_hidden)
        self.out_layer = torch.nn.Linear(num_hidden, num_classes)

    def forward(self, x):
        x = self.input_layer(x)
        x = torch.sigmoid(x)
        logits = self.out_layer(x)
        probas = torch.softmax(logits, dim=1)
        return logits, probas

# instantiate model
model = mlp1(num_features=28*28, num_hidden=100, num_classes=10).to(device)

# check initial weights
weight_check_pre = model.state_dict()['input_layer.weight'][0][0:25]

# optim
optimizer = torch.optim.SGD(model.parameters(), lr=0.1)

# download data
train_dataset = datasets.MNIST(root='data',
                               train=True,
                               transform=transforms.ToTensor(),
                               download=True)
# data loader
train_dataloader = DataLoader(dataset=train_dataset,
                          batch_size=100,
                          shuffle=True)

# train
NUM_EPOCHS = 1
for epoch in range(NUM_EPOCHS):
    model.train()
    for batch_idx, (features, targets) in enumerate(train_dataloader):
        # send data to device
        features = features.view(-1, 28*28).to(device)
        targets = targets.to(device)
        # forward
        logits, probas = model(features)
        # loss
        loss = F.cross_entropy(logits, targets)
        optimizer.zero_grad()
        loss.backward()
        # now update weights
        optimizer.step()
        ### LOGGING
        if not batch_idx % 50:
            print ('Epoch: %03d/%03d | Batch %03d/%03d | Loss: %.4f'
                   %(epoch+1, NUM_EPOCHS, batch_idx,
                     len(train_dataloader), loss))

# check post training
weight_check_post = model.state_dict()['input_layer.weight'][0][0:25]

# compare
weight_check_pre == weight_check_post  # all equal


title: Question on what to feed into the loss function

body:
def fit(num_epochs, model, loss_fn, opt, train_dl):
    for epoch in range(num_epochs):
        for xb,yb in train_dl:
            pred = model(xb)
            loss = loss_fn(pred, yb)

train_dl is just a dataset split up by DataLoader into suitable batches. Model() is the activation function.  Anyways, why is the loss calculated using yb instead of xb? This is a tutorial code for calculating loss, and I was thinking that the code should be looking something like this instead.
def fit(num_epochs, model, loss_fn, opt, train_dl):
    for epoch in range(num_epochs):
        for xb in train_dl:
            pred = model(xb)
            loss = loss_fn(pred, xb)

The tutorial is from this website. https://jovian.ai/aakashns/02-linear-regression


title: Autoencoder NOT for images

body:
I have a dataset which I am trying to autoencode with Pytorch (I have been told a convolutional autoencoder is the way to go). Each 'point' in this dataset is a vector of 1024 bits, and I am trying to encode these into vectors with maybe 10 values.
I have been looking at examples but everything that I can find refers to image encoding, so I am struggling to work out how to apply this to my dataset.
For example, for this bit of code making an autoencoder for MNIST dataset:
import torch.nn as nn
import torch.nn.functional as F

# define the NN architecture
class ConvAutoencoder(nn.Module):
    def __init__(self):
        super(ConvAutoencoder, self).__init__()
        ## encoder layers ##
        # conv layer (depth from 1 --> 16), 3x3 kernels
        self.conv1 = nn.Conv2d(1, 16, 3, padding=1)
        # conv layer (depth from 16 --> 4), 3x3 kernels
        self.conv2 = nn.Conv2d(16, 4, 3, padding=1)
        # pooling layer to reduce x-y dims by two; kernel and stride of 2
        self.pool = nn.MaxPool2d(2, 2)

        ## decoder layers ##
        ## a kernel of 2 and a stride of 2 will increase the spatial dims by 2
        self.t_conv1 = nn.ConvTranspose2d(4, 16, 2, stride=2)
        self.t_conv2 = nn.ConvTranspose2d(16, 1, 2, stride=2)

    def forward(self, x):
        ## encode ##
        # add hidden layers with relu activation function
        # and maxpooling after
        x = F.relu(self.conv1(x))
        x = self.pool(x)
        # add second hidden layer
        x = F.relu(self.conv2(x))
        x = self.pool(x)  # compressed representation

        ## decode ##
        # add transpose conv layers, with relu activation function
        x = F.relu(self.t_conv1(x))
        # output layer (with sigmoid for scaling from 0 to 1)
        x = F.sigmoid(self.t_conv2(x))

        return x

# initialize the NN
model = ConvAutoencoder()
print(model)

# specify loss function
criterion = nn.MSELoss()

# specify loss function
optimizer = torch.optim.Adam(model.parameters(), lr=0.001)

# number of epochs to train the model
n_epochs = 30

for epoch in range(1, n_epochs+1):
    # monitor training loss
    train_loss = 0.0

    ###################
    # train the model #
    ###################
    for data in train_loader:
        # _ stands in for labels, here
        # no need to flatten images
        images, _ = data
        # clear the gradients of all optimized variables
        optimizer.zero_grad()
        # forward pass: compute predicted outputs by passing inputs to the model
        outputs = model(images)
        # calculate the loss
        loss = criterion(outputs, images)
        # backward pass: compute gradient of the loss with respect to model parameters
        loss.backward()
        # perform a single optimization step (parameter update)
        optimizer.step()
        # update running training loss
        train_loss += loss.item()*images.size(0)

    # print avg training statistics
    train_loss = train_loss/len(train_loader)
    print('Epoch: {} \tTraining Loss: {:.6f}'.format(
        epoch,
        train_loss
        ))

What would I need to change in something like this to make it suitable for my data? Is there another type of autoencoder that would be more suitable?
Any help or guidance really appreciated!


title: NLP - log analysis Scenario

body:
I am trying to solve a problem related to log file analysis related to multi class classification.
In each log file. we  have multiple lines.
we can recieve error at any line.based on the error we need to assign the error to a particular team like Data issues to Data Quality team, DBA team for database issues, application team for java realated ..  so 2 challanges.
1st one:
how to identify if there is a error in a line in log file ?do we need to hardcode a pattern for it or can we use ML?
2nd one:
if we identify the line. now we need to assign it to a team. this is a multi class classification.
can you  help me..how to approach these. if you give me clue. i will search and leaen from them...thanks in advance.


title: How to make a list of pytorch tensors in python?

body:
How do I make a list of tensors in Pytorch.
The list should look like this:
mylist = [tensor1, tensor2, tensor3]

where all the tensors have different shapes


title: PyTorchLightning returning CUDA error: printing somehow fixes this. Why?

body:
I'm using PyTorch Lightning to write a simple trainer, but when I try to run the trainer, for some reason, 9 out of 10 times it returns "CUDA error: device-side assert."   Simply printing a newline before it somehow seems to make it work. Any ideas?
My code:
class Elementwise(nn.ModuleList):
    """
    A simple network container.
    Parameters are a list of modules.
    Inputs are a 3d Tensor whose last dimension is the same length
    as the list.
    Outputs are the result of applying modules to inputs elementwise.
    An optional merge parameter allows the outputs to be reduced to a
    single Tensor.
    """

    def __init__(self, merge=None, *args):
        assert merge in [None, 'first', 'concat', 'sum', 'mlp']
        self.merge = merge
        super(Elementwise, self).__init__(*args)

    def forward(self, inputs):
        inputs_ = [feat.squeeze(1) for feat in inputs.split(1, dim=1)]
        for i, j in enumerate(inputs_):
            inp = torch.tensor(j).to(device).long()
            inputs_[i] = inp

        # this does not work
        outputs = [f(x) for i, (f, x) in enumerate(zip(self, inputs_))]

        if self.merge == 'first':
            return outputs[0]
        elif self.merge == 'concat' or self.merge == 'mlp':
            return torch.cat(outputs, 1)
        elif self.merge == 'sum':
            return sum(outputs)
        else:
            return outputs

but somehow magically this works:
class Elementwise(nn.ModuleList):
    """
    A simple network container.
    Parameters are a list of modules.
    Inputs are a 3d Tensor whose last dimension is the same length
    as the list.
    Outputs are the result of applying modules to inputs elementwise.
    An optional merge parameter allows the outputs to be reduced to a
    single Tensor.
    """

    def __init__(self, merge=None, *args):
        assert merge in [None, 'first', 'concat', 'sum', 'mlp']
        self.merge = merge
        super(Elementwise, self).__init__(*args)

    def forward(self, inputs):
        inputs_ = [feat.squeeze(1) for feat in inputs.split(1, dim=1)]
        for i, j in enumerate(inputs_):
            inp = torch.tensor(j).to(device).long()
            inputs_[i] = inp

        print("")
        outputs = [f(x) for i, (f, x) in enumerate(zip(self, inputs_))]

        if self.merge == 'first':
            return outputs[0]
        elif self.merge == 'concat' or self.merge == 'mlp':
            return torch.cat(outputs, 1)
        elif self.merge == 'sum':
            return sum(outputs)
        else:
            return outputs

Any idea as to how this error gets fixed by simply printing to output?
Edit: This error is only raised when using PyTorch Lightning for training abstraction, using plain PyTorch makes it work fine.


title: understanding final layer of output for classification problem BCELoss vs CrossEntropyLoss

body:
I'm making a binary image classifier. I'm just using a pretrained model to start and change the last fully connected layer to predict between 2 classes, which I'm told requires the last layer to be the number of features and then the number of classes.
    model = models.resnet18(pretrained=True, progress=True)
    num_ftrs = model.fc.in_features
    model.fc = nn.Linear(num_ftrs, 2)

I'm using a batch size of 6 so my model predicted probability of one batch is like so
tensor([[-0.4717, -0.2232],
        [-0.6481, -0.2630],
        [-0.2007, -0.1596],
        [ 0.0277, -0.0759],
        [-0.3314, -0.1211],
        [-0.1722, -0.5304]],

and my ground truth labels are torch.tensor([0, 0, 0, 0, 0, 0])
For BCELoss I get an error that says
Using a target size (torch.Size([6])) that is different to the input size (torch.Size([6, 2])) is deprecated. Please ensure they have the same size.

However, for CrossEntropyLoss this works just fine.
The same happens with the example below
outputs = torch.randn(3, 2, 1)
target = torch.empty(3, 1, dtype=torch.long).random_(2)

criterion = nn.CrossEntropyLoss(reduction='mean')
print(outputs)
print(target)
loss = criterion(outputs, target)
print(loss)

outputs = torch.randn(3, 2, 1)
target = torch.empty(3, 1, dtype=torch.long).random_(2)

criterion = torch.nn.BCELoss()
print(outputs)
print(target)
loss = criterion(outputs, target)
print(loss)


title: ImportError: DLL load failed: A dynamic link library (DLL) initialization routine failed. How to import Pytorch

body:
I want to use Pytorch, but when i import it, this error appears:
below the code that i used to import pytorch
import pandas as pd
import torch
import seaborn as sns
from sklearn.metrics import f1_score, accuracy_score, recall_score, precision_score
from sklearn.model_selection import train_test_split
from simpletransformers.classification import ClassificationModel, ClassificationArgs

ImportError                              Traceback (most recent call last)
<ipython-input-9-2359d130c1f0> in <module>
      1 import numpy as np
      2 import pandas as pd
----> 3 import torch
      4 import seaborn as sns
      5 from sklearn.metrics import f1_score, accuracy_score, recall_score, precision_score

~\.conda\envs\tensorflow\lib\site-packages\torch\__init__.py in <module>
     79 del _dl_flags
     80
---> 81 from torch._C import *
     82
     83 __all__ += [name for name in dir(_C)

ImportError: DLL load failed: A dynamic link library (DLL) initialization routine failed.```


title: Python Pytorch getting Index error in nn.Embedding

body:
I am facing a problem with Embeddings: IndexError: index out of range in self as shown below:
 y = self.ensemble(x, mask)
  File "/usr/local/lib/python3.8/dist-packages/torch/nn/modules/module.py", line 889, in _call_impl
    result = self.forward(*input, **kwargs)
  File "/home/lucasgongora/Documents/KTH/Courses/Thesis/tracab-masters-ball-from-context/model/networks.py", line 33, in forward
    src = self.encoder(src) * math.sqrt(self.ninp)
  File "/usr/local/lib/python3.8/dist-packages/torch/nn/modules/module.py", line 889, in _call_impl
    result = self.forward(*input, **kwargs)
  File "/usr/local/lib/python3.8/dist-packages/torch/nn/modules/sparse.py", line 145, in forward
    return F.embedding(
  File "/usr/local/lib/python3.8/dist-packages/torch/nn/functional.py", line 1913, in embedding
    return torch.embedding(weight, input, padding_idx, scale_grad_by_freq, sparse)
IndexError: index out of range in self

For context, I am running a database x with dimensions (batch_size x 50 x 48). And I need to input with this configuration. I am training it on The transformer network
class TransformerModel(nn.Module):

    def __init__(self, ntoken, ninp, nhead, nhid, nlayers, dropout=0.5):
        super(TransformerModel, self).__init__()
        from torch.nn import TransformerEncoder, TransformerEncoderLayer
        self.model_type = 'Transformer'
        self.pos_encoder = PositionalEncoding(ninp, dropout)
        encoder_layers = TransformerEncoderLayer(ninp, nhead, nhid, dropout)
        self.transformer_encoder = TransformerEncoder(encoder_layers, nlayers)
        self.encoder = nn.Embedding(ntoken, ninp)
        self.ninp = ninp
        self.decoder = nn.Linear(ninp, ntoken)

        self.init_weights()

    def generate_square_subsequent_mask(self, sz):
        mask = (torch.triu(torch.ones(sz, sz)) == 1).transpose(0, 1)
        mask = mask.float().masked_fill(mask == 0, float('-inf')).masked_fill(mask == 1, float(0.0))
        return mask

    def init_weights(self):
        initrange = 0.1
        self.encoder.weight.data.uniform_(-initrange, initrange)
        self.decoder.bias.data.zero_()
        self.decoder.weight.data.uniform_(-initrange, initrange)

    def forward(self, src, src_mask):
        src = self.encoder(src) * math.sqrt(self.ninp)
        src = self.pos_encoder(src)
        output = self.transformer_encoder(src, src_mask)
        output = self.decoder(output)
        return output

class PositionalEncoding(nn.Module):

    def __init__(self, d_model, dropout=0.1, max_len=5000):
        super(PositionalEncoding, self).__init__()
        self.dropout = nn.Dropout(p=dropout)

        pe = torch.zeros(max_len, d_model)
        position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1)
        div_term = torch.exp(torch.arange(0, d_model, 2).float() * (-math.log(10000.0) / d_model))
        pe[:, 0::2] = torch.sin(position * div_term)
        pe[:, 1::2] = torch.cos(position * div_term)
        pe = pe.unsqueeze(0).transpose(0, 1)
        self.register_buffer('pe', pe)

    def forward(self, x):
        x = x + self.pe[:x.size(0), :]
        return self.dropout(x)

class Ensemble_Networks(nn.Module):
    def __init__(self, num_output_target, num_in_players, num_output_players, dim_output_players, num_features,
                 num_inds_players=64, dim_hidden_players=128, num_heads_players=4):
        super(Ensemble_Networks, self).__init__()

        self.ensemble = TransformerModel(ntoken=48, ninp=50, nhead=1, nhid=200, nlayers=2, dropout=0.5)

    def forward(self, x, mask):
        y = self.ensemble(x, mask)
        return y

believe the error is at: self.ensemble = TransformerModel(ntoken=48, ninp=50, nhead=1, nhid=200, nlayers=2, dropout=0.5) where i cannot find the proper dimensions of the variables of the function.
I am running something like:
mask = self.generate_square_subsequent_mask(50)
        for Epochs in trange(self.n_epochs, desc='Epoch'):
                for step in range(iter_size):
                    x, y = data_import.create_batches(x_train, y_train)
                    y_pred = self.model(x, mask)


title: Predictions doesn't equal number of images

body:
My validation data are 150 images, but when i try to use my model to predict them my predictions are of length 22 I don't understand why?
total_v=0
correct_v=0

with torch.no_grad():
    model.eval()
    for data_v, target_v in (validloader):
        if SK:
            target_v = torch.tensor(np.where(target_v.numpy() == 2, 1, 0).astype(np.longlong))
        else:
            target_v = torch.tensor(np.where(target_v.numpy() == 0, 1, 0).astype(np.longlong))

        data_v, target_v = data_v.to(device), target_v.to(device)

        outputs_v = model(data_v)
        loss_v = criterion(outputs_v, target_v)
        batch_loss += loss_v.item()
        _,pred_v = torch.max(outputs_v, dim=1)
        correct_v += torch.sum(pred_v==target_v).item()
        total_v += target_v.size(0)
    val_acc.append(100 * correct_v/total_v)
    val_loss.append(batch_loss/len(validloader))
    network_learned = batch_loss < valid_loss_min
    print(f'validation loss: {np.mean(val_loss):.4f}, validation acc: {(100 *
          correct_v/total_v):.4f}\n')

this is my model
model = models.resnet50(pretrained = True)
num_ftrs = model.fc.in_features

model.fc = nn.Linear(num_ftrs, 2)

model.to(device)

criterion = nn.CrossEntropyLoss()
optimizer = optim.Adagrad(model.parameters())


title: Equivalence of Tensor.storage() for Numpy Array and Tensorflow.Tensor?

body:
I know in Pytorch we can check the underlying Tensor data storage by torch.Tensor.storage(). I think this is a practical method as sometimes the performance of Tensor depends on the underlying storage and with this method we can check it easily. I'm wondering if it also exists for Tensorflow Tensor or Numpy Array?
In [5]: t=torch.Tensor([[2,3,4],[5,6,7]])
In [7]: t.storage()
Out[7]:
 2.0
 3.0
 4.0
 5.0
 6.0
 7.0
[torch.FloatStorage of size 6]

https://pytorch.org/docs/stable/tensors.html#torch.Tensor.storage


title: How to enable error in pytorch on unintended GPU / CPU transfers?

body:
I have an unusual need: I want to get an error message.
I recently updated my pytorch version, and now I no longer get an error message that I relied on for debugging. Before the update, when I used a tensor as input to a model where the tensor was not on the same device as the model, I would get an error of the form "Runtime Error: expected both inputs to be on same device".
This error no longer occurs now. That's a problem, because I am debugging a large project and I know that the project contains several cases where this happens. Since it costs an expensive CPU/GPU transfer every time it happens, I would like to see the error so that I can find these problem spots and optimize them.
How do I re-enable this error message?


title: Loading a custom dataset instead of CIFAR10

body:
I'm trying to load a custom dataset of mine to perform "one pixel attack" as in this article:
https://arxiv.org/pdf/1710.08864v7.pdf
I'm using pytorch, so I tried to modify this code:
https://github.com/DebangLi/one-pixel-attack-pytorch
But it fails probably because my files are arranged differently and are different than the files were used originaly (I'm using a part of imagenet, called imagenette, while it was designed on CIFAR10)
Here is how I load my data:
# Data augmentation and normalization for training
# Just normalization for validation
data_transforms = {
    'train': transforms.Compose([
        transforms.RandomResizedCrop(224),
        transforms.RandomHorizontalFlip(),
        transforms.ToTensor(),
        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
    ]),
    'val': transforms.Compose([
        transforms.Resize(256),
        transforms.CenterCrop(224),
        transforms.ToTensor(),
        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
    ]),
}

data_dir = 'drive/MyDrive/Datasets/imagenette2'
image_datasets = {x: datasets.ImageFolder(os.path.join(data_dir, x),
                                          data_transforms[x])
                  for x in ['train', 'val']}
dataloaders = {x: torch.utils.data.DataLoader(image_datasets[x], batch_size=4,
                                             shuffle=True, num_workers=4)
              for x in ['train', 'val']}
dataset_sizes = {x: len(image_datasets[x]) for x in ['train', 'val']}
class_names = image_datasets['train'].classes

device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")

As advised in the pytorch transfer learning tutorial:
https://colab.research.google.com/github/pytorch/tutorials/blob/gh-pages/_downloads/62840b1eece760d5e42593187847261f/transfer_learning_tutorial.ipynb
And here is how the data is loaded in the "one pixel attack" code:
def main():

    print "==> Loading data and model..."
    tranfrom_test = transforms.Compose([
        transforms.ToTensor(),
        transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),
        ])
    test_set = torchvision.datasets.CIFAR10(root='./data', train=False, download=True, transform=tranfrom_test)
    testloader = torch.utils.data.DataLoader(test_set, batch_size=1, shuffle=True, num_workers=2)

    class_names = ['plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck']
    assert os.path.isdir('checkpoint'), 'Error: no checkpoint directory found!'
    checkpoint = torch.load('./checkpoint/%s.t7'%args.model)
    net = checkpoint['net']
    net.cuda()
    cudnn.benchmark = True

    print "==> Starting attck..."

    results = attack_all(net, testloader, pixels=args.pixels, targeted=args.targeted, maxiter=args.maxiter, popsize=args.popsize, verbose=args.verbose)
    print "Final success rate: %.4f"%results

Of course I also need to change the predicted classes over there to ones that fit my dataset as well.
Thanks for your help!


title: Save output image of CNN model

body:
I made a CNN model for edge detection task. It worked fine and I can see good results through the Feature maps. the problem is I'm using 4 filters in each convolutional layer, so the output is going to be an image with 4 channels but I need the image to be saved as a 3 channels to work with it later, here an example of the code I used:
#Feed image to model:
output = model(img) #input size (1, 3, 224, 224)
#convert output tensor to numpy
fimg = output.detach().cpu().numpy()
#get rid of the single batch dimension
f2img = fimg.squeeze(0)
#swap axes to (13, 13, 4)
f2img = fimg.transpose(1, 2, 0)

it's gonna be saved as (13,13,4) but I need it as (13, 13, 3). If anyone could help it would be appreciated.


title: Saving Torch Hub's SSD inferenced images in an output directory

body:
I'm using Pytorch SSD that loads a pre-trained model on the COCO dataset from Torch HUB. Modified the code in an API format to fetch some images and detect the objects in it.
Trying to save each inference image output in the /output using matplotlib's .savefig() method but getting an error:
import torch
import matplotlib.patches as patches
from matplotlib import pyplot as plt

class ObjectDetector:

    def __init__(self):
        self.precision = 'fp32'
        self.detect_model = torch.hub.load('NVIDIA/DeepLearningExamples:torchhub', 'nvidia_ssd', model_math=self.precision)
        self.utils = torch.hub.load('NVIDIA/DeepLearningExamples:torchhub', 'nvidia_ssd_processing_utils')

    def process(self):
        self.fetch_images()
        self.create_model()
        self.display_detections()

    def fetch_images(self):
        """To get the images from a website"""
        self.images = ['http://images.cocodataset.org/val2017/000000397133.jpg','http://images.cocodataset.org/val2017/000000037777.jpg','http://images.cocodataset.org/val2017/000000252219.jpg']

        return self.images

    def create_model(self):
        self.detect_model.to('cuda')
        self.detect_model.eval()

        self.inputs = [self.utils.prepare_input(uri) for uri in self.images]
        tensor = self.utils.prepare_tensor(self.inputs, self.precision == 'fp16')

        with torch.no_grad():
            detections_batch = self.detect_model(tensor)

        results_per_input = self.utils.decode_results(detections_batch)
        self.best_results_per_input = [self.utils.pick_best(results, 0.40) for results in results_per_input]
        self.classes_to_labels = self.utils.get_coco_object_dictionary()

        return self.best_results_per_input, self.classes_to_labels

    def display_detections(self):

        output_dir = "../data/vision/ssd/output"

        for image_idx in range(len(self.best_results_per_input)):
            fig, ax = plt.subplots(figsize=(20, 10))
            # Show original, denormalized image...
            image = self.inputs[image_idx] / 2 + 0.5
            ax.imshow(image)
            # ...with detections
            bboxes, classes, confidences = self.best_results_per_input[image_idx]
            for idx in range(len(bboxes)):
                left, bot, right, top = bboxes[idx]
                x, y, w, h = [val * 300 for val in [left, bot, right - left, top - bot]]
                rect = patches.Rectangle((x, y), w, h, linewidth=1, edgecolor='r', facecolor='none')
                ax.add_patch(rect)
                ax.text(x, y, "{} {:.0f}%".format(self.classes_to_labels[classes[idx] - 1], confidences[idx] * 100), bbox=dict(facecolor='white', alpha=0.5))

            plt.savefig(output_dir + str(image) + '.jpg')
        plt.show()

if __name__== '__main__':
    det = ObjectDetector()
    det.process()
    del det

The above code throws the following error:
---------------------------------------------------------------------------
FileNotFoundError                         Traceback (most recent call last)
<ipython-input-9-acbe775772c1> in <module>
     63 if __name__== '__main__':
     64     det = ObjectDetector()
---> 65     det.process()
     66
     67     del det

<ipython-input-9-acbe775772c1> in process(self)
     14         self.fetch_images()
     15         self.create_model()
---> 16         self.display_detections()
     17
     18

<ipython-input-9-acbe775772c1> in display_detections(self)
     57                 ax.text(x, y, "{} {:.0f}%".format(self.classes_to_labels[classes[idx] - 1], confidences[idx] * 100), bbox=dict(facecolor='white', alpha=0.5))
     58
---> 59             plt.savefig(output_dir + str(image) + '.jpg')
     60         plt.show()
     61

~/venv38/lib/python3.8/site-packages/matplotlib/pyplot.py in savefig(*args, **kwargs)
    721 def savefig(*args, **kwargs):
    722     fig = gcf()
--> 723     res = fig.savefig(*args, **kwargs)
    724     fig.canvas.draw_idle()   # need this if 'transparent=True' to reset colors
    725     return res

~/venv38/lib/python3.8/site-packages/matplotlib/figure.py in savefig(self, fname, transparent, **kwargs)
   2201             self.patch.set_visible(frameon)
   2202
-> 2203         self.canvas.print_figure(fname, **kwargs)
   2204
   2205         if frameon:

~/venv38/lib/python3.8/site-packages/matplotlib/backend_bases.py in print_figure(self, filename, dpi, facecolor, edgecolor, orientation, format, bbox_inches, **kwargs)
   2117
   2118             try:
-> 2119                 result = print_method(
   2120                     filename,
   2121                     dpi=dpi,

~/venv38/lib/python3.8/site-packages/matplotlib/cbook/deprecation.py in wrapper(*args, **kwargs)
    356                 f"%(removal)s.  If any parameter follows {name!r}, they "
    357                 f"should be pass as keyword, not positionally.")
--> 358         return func(*args, **kwargs)
    359
    360     return wrapper

~/venv38/lib/python3.8/site-packages/matplotlib/backends/backend_agg.py in print_jpg(self, filename_or_obj, dryrun, pil_kwargs, *args, **kwargs)
    597             pil_kwargs.setdefault("quality", rcParams["savefig.jpeg_quality"])
    598             pil_kwargs.setdefault("dpi", (self.figure.dpi, self.figure.dpi))
--> 599             return background.save(
    600                 filename_or_obj, format='jpeg', **pil_kwargs)
    601

~/venv38/lib/python3.8/site-packages/PIL/Image.py in save(self, fp, format, **params)
   2153                 fp = builtins.open(filename, "r+b")
   2154             else:
-> 2155                 fp = builtins.open(filename, "w+b")
   2156
   2157         try:

FileNotFoundError: [Errno 2] No such file or directory: '../data/vision/ssd/output[[[0.1050852  0.07895297 0.08367175]\n  [0.31462591 0.31466424 0.32513717]\n  [0.28277484 0.25506944 0.23508735]\n  ...\n  [0.42182888 0.27386384 0.07784647]\n  [0.67421166 0.57844825 0.39889071]\n  [0.554919   0.33316082 0.09618731]]\n\n [[0.05228582 0.03646781 0.0400054 ]\n  [0.06949542 0.06235639 0.05692344]\n  [0.25959795 0.18080175 0.18654409]\n  ...\n  [0.60428691 0.30419598 0.06168084]\n  [0.62523846 0.37480789 0.15464491]\n  [0.40595506 0.21335363 0.0789785 ]]\n\n [[0.10904118 0.11286539 0.09207947]\n  [0.0804173  0.04945466 0.03713621]\n  [0.24569849 0.12457102 0.1002835 ]\n  ...\n  [0.8473525  0.49805938 0.01584464]\n  [0.62128949 0.34659926 0.04259144]\n  [0.60784509 0.39757653 0.1146472 ]]\n\n ...\n\n [[0.54990582 0.37598903 0.20369267]\n  [0.5526588  0.38010985 0.19625383]\n  [0.56226779 0.38371096 0.20185737]\n  ...\n  [0.29863339 0.2165191  0.14226269]\n  [0.30894688 0.23059896 0.16393229]\n  [0.31879315 0.21973148 0.16671452]]\n\n [[0.54124921 0.37518263 0.19985079]\n  [0.54947818 0.38385507 0.19607851]\n  [0.54889008 0.37478852 0.18892228]\n  ...\n  [0.29478525 0.22002212 0.15326277]\n  [0.31478406 0.23243116 0.16062237]\n  [0.30818757 0.21890863 0.14786195]]\n\n [[0.53892612 0.37097071 0.1888549 ]\n  [0.54983966 0.38421659 0.19571689]\n  [0.55770917 0.38090676 0.18950984]\n  ...\n  [0.316164   0.24439232 0.16849774]\n  [0.32127783 0.23892493 0.16441515]\n  [0.30470566 0.21542674 0.14437993]]].jpg'


title: I want to modify the learning rate of the loaded model

body:
After loading the saved model, I want to modify the learning rate.
I want to set it to a value larger than the saved learning rate.
For now, the process has been 0.001 -> 0.0001 -> 0.00001, and the stored learning rate is 0.00001.
model_without_ddp.load_state_dict(checkpoint["model"])
optimizer.load_state_dict(checkpoint["optimizer"])
lr_scheduler.load_state_dict(checkpoint["lr_scheduler"])

The above code is the original code.
model_without_ddp.load_state_dict(checkpoint["model"])
params = [p for p in model_without_ddp.parameters() if p.requires_grad]
optimizer = torch.optim.SGD(
    params,
    lr=args.lr,
    momentum=args.momentum,
    weight_decay=args.weight_decay
)
lr_scheduler = MultiStepLR(optimizer, milestones=[100000, 500000], gamma=0.1)

It was modified like this.
However, if I get the validation loss of the model I loaded like this, the above code and the below code have different values.
What is my mistake?


title: Is there a way to modify weights, gradient outputs and activation outputs in PyTorch using hooks?

body:
I've been trying to create a way to modify activations, weights and gradients during training using PyTorch, as a way to simulate other data types and quantization models.
The idea is to use hooks to quantize or modify the value of the outputs and weights, and then reconvert the values to FP32 to continue training. This would "simulate" the network as if it was trained using lower bit width data types, or heuristically chopped bits that artificially lower the precision (as a means to test ASIC or FPGA training optimizations that reduce memory footprint and computation).
The issue is that PyTorch's documentation doesn't explicitly explain how to modify these values during training, and my only guess other than hooks would be to implement special versions of networks that allow for these types of modifications. However I would like for this to be a general tool that can be used with any type of network without it having to be modified, while at the same time being capable of tracing output/weight values for further analysis.
I know that hooks in PyTorch can return values, but every time I've tried to return something, it either gives me a data type error, or a not enough return values error.
Is there a way to modify gradient/activation outputs and weights with forward and backward hooks in PyTorch?
Under this Pastebin link (as the code was too long for stackoverflow) is the current code I'm trying to make work to trace gradients and modify gradient outputs:
https://pastebin.com/HUyRsEy7
for gradient in outputs:
    try:
        np.save(file_prefix+"out_"+str(subidx), gradient.cpu())
    except AttributeError:
        pass
    subidx += 1

temp_outputs = outputs
for gradient in temp_outputs:
    gradient *= 0.0
return module, inputs, temp_outputs

(Stack made me write part of the code to post the pastebin link)


title: cv2.imread() randomly make images become 'None'

body:
Description
The system is Ubuntu 20.04 LTS. The framework is PyTorch.
The problem is that cv2.imread() randomly makes 'None' images. It does not only return None, but also changed the images on the disk to be None, and cannot be read by cv2 next time.
I was training some end-to-end CNN models (ResNet/DenseNet). Everything was running well if I used a 'small' dataset(about 200-500 images, randomly sampled from a 'big' dataset(about 5000 images)).
However, if I used the 'big datasets', the error happened randomly, and the images on the disk will be destroyed permanently and became None type.
What I tried
I tried to write a function to avoid loading the destroyed images, which means that the training images would become fewer and fewer (about ten images per 100 epochs).
Another operation that I tried is to set the permission of the image file as read-only, but got errors.
I also tried different models but still got this problem randomly.
Another potential problem
I am not sure whether there is a relationship between fsck and the None image problem. every time I rebooted the system, I couldn't enter the ubuntu system but need to fsck manually.
Is there anyone who knows the solution or faces the same problem before? Could you please help me solve it? Thank you!


title: nn.DataParallel - Training doesn't seem to start

body:
I am having a lot of problems using nn.DistributedDataParallel, because I cannot find a good working example of how to specify GPU id's within a single node. For this reason, I want to start off by using nn.DataParallel, since it should be easier to implement. According to the documentation [https://pytorch.org/docs/stable/generated/torch.nn.DataParallel.html] the following should work:
device = torch.device('cuda:1' if torch.cuda.is_available() else 'cpu')
model = Model(arg).to(device)
model = torch.nn.DataParallel(model, device_ids=[1, 8, 9])
for step, (original, keypoints) in enumerate(train_loader):
                    original, keypoints = original.to(device), keypoints.to(device)
                    loss = model(original)
                    optimizer.zero_grad()
                    total_loss.backward()
                    optimizer.step()

However, when I start to process the model is distributed to all three GPU's, but the training doesn't start. The RAM of the GPU's remains almost empty (except for the memory used for the loading the model). This can be seen here (see GPU 1, 8, 9):

Can someone explain me why that's not working?
Thanks a lot!!


title: How to reshape a tensor where one dimension is reduced to 1?

body:
I'm trying to reshape a tensor using addition and squeeze/normalization from [30,50,32,64] to [30,50,32,1]. And when I use Tensorflow reshape, I'm getting the following error.
o = tf.reshape(o, shape=[30, 50, 32, 1])

ValueError: Cannot reshape a tensor with 3072000 elements to shape [30,50,32,1] (48000 elements) for 'discriminator/embedding/Reshape_4' (op: 'Reshape') with input shapes: [30,50,32,64], [4] and with input tensors computed as partial shapes: input[1] = [30,50,32,1].
Can someone suggest a way around it?


title: LSTM for image sequences- AttributeError: 'bool' object has no attribute 'float'

body:
Further to my previous post,
LSTM-CNN to classify sequences of images,
I got further and I am now  running the below code:
 loop += 1
        if loop % 50 == 0:
            # calculate Accuracy
            correct = 0
            total = 0
            # Iterate through the test dataset

            for j in range(iter):
                sequences = x_test[bs:bs + batch_size, :]
                labels_t = y_test[bs:bs + batch_size]
                test_images = dataset.load_images(sequences)
                bs += batch_size
                test_images = (torch.from_numpy(test_images)).view(-1, 4, 784)
                print('test_image_shape {} - iter: {}'.format(test_images.shape, j))
                labels_t = torch.from_numpy(labels_t)
                labels_t  = torch.argmax(labels_t, dim=1)
                test_outputs = model(test_images).float()
                pred_y = torch.max(test_outputs, 1)[1].data.numpy().squeeze()

                total += labels_t.size(0)
                correct += (pred_y == labels_t).float()
                accuracy = 100 * (correct / total)

                print('Iteration:{}. Loss:{}. Accuracy{}:'.format(loop, loss.item(), accuracy))

However it is giving me the following error:

correct += (pred_y == labels_t).float()
AttributeError: 'bool' object has no attribute 'float'

correct is always resulting in 0 and so is accuracy.
I am suspecting that it has to do with a certain package or requirement, because I searched and it seems that it should work. Any idea what could be missing here?


title: Concatenating two multi-dimensional numpy arrays at specific index

body:
I have two numpy arrays of sizes: [20,3,100,100] and [20,5,100,100].
They are 20 (100,100) images with 3 channels, and 5 channels, respectively. I want to concatenate the channels so that I have 20 (100,100) images with 8 channels.
I would like to concat them along (dim = 1) them together, without having to create a new numpy.zeros array of size [20,8,100,100]. Is this possible?


title: Why do PyTorch and Numpy's range indexing allow you to silently index out of bounds?

body:
I stumbled across a strange phenomenon today when debugging some PyTorch code:
I created a tensor like this
import torch
mat = torch.rand(3,5)

and then indexed it using a range like:
print(mat[2,4:7])

Lo and behold, it returned the element at mat[2,4] alone!, i.e. mat[2,4:6] == mat[2,4]. There was no indication that my range extended beyond the bounds of the tensor. However, running:
print(mat[2,5])

raised the expected IndexError: *** IndexError: index 5 is out of bounds for dimension 1 with size 5
Thinking I had stumbled across a PyTorch bug perhaps, I tested it on numpy arrays as well and found the same behavior to hold:
import numpy as np
mat = np.random.rand(3,5)
print(mat[2,4:7]) # prints mat[2,4]
print(mat[2,5]) # raises IndexError (*** IndexError: index 5 is out of bounds for axis 1 with size 5)

I would have expected this range indexing to at the very least raise a warning or perhaps even to loop around modulo the shape of the dimension so that it returns an array of the expected output shape. Frankly, I'm surprised it doesn't raise an IndexError altogether.
Why is the actual behavior to be expected?
(Note: I imagine it is based on a Numpy convention first)


title: PyTorch: apply mask with different shape

body:
I have a tensor of shape (60, 3, 32, 32) and a boolean mask of shape (60, 32, 32). I want to apply this mask to the tensor. The output tensor should have shape (60, 3, 32, 32), and values are kept if the mask is 1, else 0.
How can I do that fast?


title: How to fix Runtime error while running model for ultralytic yolo?

body:
I want to produce the profiling for the YOLO and I am using yolo.py of ultralytics/yolov5 they have everything pre-build to run the profiling that is getting the latency for each layer.
So I ran the following commands to get the output:
import torch
import torchvision

device = torch.device("cuda:0")

model = torch.hub.load('ultralytics/yolov5', 'yolov5s', force_reload=True)

img = torch.rand(8 if torch.cuda.is_available() else 1, 3, 320, 320).to(device)
y = model(img, profile=True)

So when I run this I get the following runtime error with some output
time (ms)     GFLOPS     params  module
      1.25       1.44       3520  models.common.Focus
      0.66       1.90      18560  models.common.Conv
      1.72       1.93      18816  models.common.C3
      0.46       1.89      73984  models.common.Conv
      1.95       4.02     156928  models.common.C3
      0.33       1.89     295424  models.common.Conv
      1.31       4.00     625152  models.common.C3
      0.39       1.89    1180672  models.common.Conv
      0.68       1.05     656896  models.common.SPP
      0.69       1.89    1182720  models.common.C3
      0.20       0.21     131584  models.common.Conv
      0.05       0.00          0  torch.nn.modules.upsampling.Upsample
      0.06       0.00          0  models.common.Concat
      0.70       2.32     361984  models.common.C3
      0.17       0.21      33024  models.common.Conv
      0.06       0.00          0  torch.nn.modules.upsampling.Upsample
      0.10       0.00          0  models.common.Concat
      0.80       2.33      90880  models.common.C3
      0.27       0.95     147712  models.common.Conv
      0.03       0.00          0  models.common.Concat
      0.68       1.90     296448  models.common.C3
      0.26       0.94     590336  models.common.Conv
      0.03       0.00          0  models.common.Concat
      0.60       1.89    1182720  models.common.C3
---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-7-f21874f58c27> in <module>()
      1 img = torch.rand(8 if torch.cuda.is_available() else 1, 3, 320, 320).to(device)
----> 2 y = model(img, profile=True)

10 frames
/usr/local/lib/python3.7/dist-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
    887             result = self._slow_forward(*input, **kwargs)
    888         else:
--> 889             result = self.forward(*input, **kwargs)
    890         for hook in itertools.chain(
    891                 _global_forward_hooks.values(),

/usr/local/lib/python3.7/dist-packages/torch/autograd/grad_mode.py in decorate_context(*args, **kwargs)
     25         def decorate_context(*args, **kwargs):
     26             with self.__class__():
---> 27                 return func(*args, **kwargs)
     28         return cast(F, decorate_context)
     29

/root/.cache/torch/hub/ultralytics_yolov5_master/models/common.py in forward(self, imgs, size, augment, profile)
    253         if isinstance(imgs, torch.Tensor):  # torch
    254             with amp.autocast(enabled=p.device.type != 'cpu'):
--> 255                 return self.model(imgs.to(p.device).type_as(p), augment, profile)  # inference
    256
    257         # Pre-process

/usr/local/lib/python3.7/dist-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
    887             result = self._slow_forward(*input, **kwargs)
    888         else:
--> 889             result = self.forward(*input, **kwargs)
    890         for hook in itertools.chain(
    891                 _global_forward_hooks.values(),

/root/.cache/torch/hub/ultralytics_yolov5_master/models/yolo.py in forward(self, x, augment, profile)
    122             return torch.cat(y, 1), None  # augmented inference, train
    123         else:
--> 124             return self.forward_once(x, profile)  # single-scale inference, train
    125
    126     def forward_once(self, x, profile=False):

/root/.cache/torch/hub/ultralytics_yolov5_master/models/yolo.py in forward_once(self, x, profile)
    134                 t = time_synchronized()
    135                 for _ in range(10):
--> 136                     _ = m(x)
    137                 dt.append((time_synchronized() - t) * 100)
    138                 if m == self.model[0]:

/usr/local/lib/python3.7/dist-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
    887             result = self._slow_forward(*input, **kwargs)
    888         else:
--> 889             result = self.forward(*input, **kwargs)
    890         for hook in itertools.chain(
    891                 _global_forward_hooks.values(),

/root/.cache/torch/hub/ultralytics_yolov5_master/models/yolo.py in forward(self, x)
     44         self.training |= self.export
     45         for i in range(self.nl):
---> 46             x[i] = self.m[i](x[i])  # conv
     47             bs, _, ny, nx = x[i].shape  # x(bs,255,20,20) to x(bs,3,20,20,85)
     48             x[i] = x[i].view(bs, self.na, self.no, ny, nx).permute(0, 1, 3, 4, 2).contiguous()

/usr/local/lib/python3.7/dist-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
    887             result = self._slow_forward(*input, **kwargs)
    888         else:
--> 889             result = self.forward(*input, **kwargs)
    890         for hook in itertools.chain(
    891                 _global_forward_hooks.values(),

/usr/local/lib/python3.7/dist-packages/torch/nn/modules/conv.py in forward(self, input)
    397
    398     def forward(self, input: Tensor) -> Tensor:
--> 399         return self._conv_forward(input, self.weight, self.bias)
    400
    401 class Conv3d(_ConvNd):

/usr/local/lib/python3.7/dist-packages/torch/nn/modules/conv.py in _conv_forward(self, input, weight, bias)
    394                             _pair(0), self.dilation, self.groups)
    395         return F.conv2d(input, weight, bias, self.stride,
--> 396                         self.padding, self.dilation, self.groups)
    397
    398     def forward(self, input: Tensor) -> Tensor:

RuntimeError: Expected 4-dimensional input for 4-dimensional weight [255, 128, 1, 1], but got 5-dimensional input of size [8, 3, 40, 40, 85] instead

Can you let me know that how I can get this output without errors?
Output to be received:
 time (ms)     GFLOPS     params  module
      0.93       1.44       3520  models.common.Focus
      0.52       1.90      18560  models.common.Conv
      1.31       1.93      18816  models.common.C3
      0.38       1.89      73984  models.common.Conv
      1.58       4.02     156928  models.common.C3
      0.37       1.89     295424  models.common.Conv
      1.48       4.00     625152  models.common.C3
      0.51       1.89    1180672  models.common.Conv
      0.79       1.05     656896  models.common.SPP
      0.98       1.89    1182720  models.common.C3
      0.18       0.21     131584  models.common.Conv
      0.03       0.00          0  torch.nn.modules.upsampling.Upsample
      0.05       0.00          0  models.common.Concat
      0.87       2.32     361984  models.common.C3
      0.16       0.21      33024  models.common.Conv
      0.05       0.00          0  torch.nn.modules.upsampling.Upsample
      0.09       0.00          0  models.common.Concat
      0.81       2.33      90880  models.common.C3
      0.35       0.95     147712  models.common.Conv
      0.03       0.00          0  models.common.Concat
      0.80       1.90     296448  models.common.C3
      0.35       0.94     590336  models.common.Conv
      0.03       0.00          0  models.common.Concat
      1.04       1.89    1182720  models.common.C3
      0.69       1.47     229245  Detect
14.4 ms total


title: PyTorch dataloader "data load" in GPU memory concept

body:
I am using pytorch dataloader to load my 7.5 GB of dataset. I am using batch size of 100 to load data. However I am not sure if I am loading whole dataset in one go or its loading data batch wise. As even with batch size as low as 1 its giving me CUDA out of memory error.
I wanted to know does default (as shown in pytorch documentation) dataloader uses batch wise loading or dump all data in one go? If it tries to dump 7.5 GB of data then this error can be due to large size of dataset.
For reference I am attaching code for the same-
class Image(Dataset):
    def __init__(self, setname):
        csv_path = osp.join(ROOT_PATH, setname + '.csv')
        lines = [x.strip() for x in open(csv_path, 'r').readlines()][1:]

        data = []
        label = []
        lb = -1

        self.wnids = []

        for l in lines:
            name, wnid = l.split(',')
            path = osp.join(ROOT_PATH, 'images', name)
            if wnid not in self.wnids:
                self.wnids.append(wnid)
                lb += 1
            data.append(path)
            label.append(lb)

        self.data = data
        self.label = label

        self.transform = transforms.Compose([
            transforms.Resize(140),
            transforms.CenterCrop(140),
            transforms.ToTensor(),
            transforms.Normalize(mean=[0.485, 0.456, 0.406],
                                 std=[0.229, 0.224, 0.225])
        ])

    def __len__(self):
        return len(self.data)

    def __getitem__(self, i):
        path, label = self.data[i], self.label[i]
        image = self.transform(Image.open(path).convert('RGB'))
        return image, label

Above is dataloader part. From main script it is called as below-
for epoch in range(1, args.max_epoch + 1):
    lr_scheduler.step()

    model.train()

    for i, batch in enumerate(train_loader, 1):
        data, _ = [_.cuda() for _ in batch]
        p = args.shot * args.train_way
        data_shot, data_query = data[:p], data[p:]

        proto = model(data_shot)

        label = torch.arange(args.train_way).repeat(args.query)
        label = label.type(torch.cuda.LongTensor)

        logits = euclidean_metric(model(data_query), proto)
        loss = F.cross_entropy(logits, label)
        tl.add(loss.item())
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()


title: Pixelwise regression. How to go from Nx1xHxW to Nx3xHxW?

body:
I have a Nx1xHxW feature maps. I need to add a second head that generates Nx3xHxW representing pixel wise regression with a triplet for each pixel .
The question is: how would you go from Nx1xHxW to Nx3xHxW? A fully connected layer would be too expensive in terms of introduced parameters.
That's what I am trying a 1x1x3 convolutional filter with stride 1 defined in PyTorch as nn.Conv2d(1, 3, (1, 1), stride=1, bias=True) but results does not seem encouraging. Any suggestion would be welcome.
Best


title: Weight of layer as a result of dot operation in PyTorch

body:
I’m trying to implement a network in which the weights of the layers are calculated as a result of a tensor operation. This is the code I have for a single layer and is repeated for all conv and fc layers:
class NOWANet(nn.Module):
    def __init__(self, V, Wstacked):
        super(NOWANet, self).__init__()

        # first conv layer
        # define layer
        self.conv1 = nn.Conv2d(in_channels=1, out_channels=6, kernel_size=5)

        # define tensors that will be used to calculate weights and biases
        self.conv1_W_V = nn.Parameter(V[0], requires_grad=True)
        self.conv1_B_V = nn.Parameter(V[1], requires_grad=True)
        self.conv1_W_stack = nn.Parameter(Wstacked[0], requires_grad=False)
        self.conv1_B_stack = nn.Parameter(Wstacked[1], requires_grad=False)

        # set layer weights and bias
        self.conv1.weight = nn.Parameter(torch.tensordot(self.conv1_W_V,
                                                         self.conv1_W_stack,
                                                         dims=1, out=None),
                                         requires_grad=True)
        self.conv1.bias = nn.Parameter(torch.tensordot(self.conv1_B_V,
                                                       self.conv1_B_stack,
                                                       dims=1, out=None),
                                       requires_grad=True)

The idea is to perform a tensordot between the V and W_stack tensors, which then should be used as weights and bias (one tensordot each). The catch is that I only want to optimize V so that W_stack remains the intact. The way it is currently written initializes the weights correctly, but backprop optimizes over the actual weights of the layer, there is no change to the V vector.
Any ideas or suggestions on how to do it?


title: How to use torch.utils.tensorboard.SummaryWriter from the last interrupted events

body:
As in this picture, if I want to add scalar from events.out.tfevents, but not create a new one.
How can I set the params int this code:
SummaryWriter(self, log_dir=None, comment='', purge_step=None, max_queue=10,
             flush_secs=120, filename_suffix='')


title: Why doesn't torch see сuda?

body:
i have problem with torch and cuda.
torch.version.__version__ gives "1.5.1+cu101"
torch.version.cuda gives "10.1"
torch.backends.cudnn.version() gives "7603"
But torch.cuda.is_avaliable() gives False. What can be wrong?
Parameters of my system:
Linux-x86_64 (Ubuntu 20.04)
GeForce GTX 1050
NVIDIA Driver Version: 460.67


title: Calling pytorch neural network forward() gives error "mat1 and mat2 shapes cannot be multiplied"

body:
I have following code which defines simple neural network:
class MlpNN(nn.Module):

    def __init__(self, in_dim=301, hidden_dims=[500,500,500,500,500,30]):

        super(MlpNN, self).__init__()

        self.net = nn.Sequential()
        self.net.add_module("lin_0", nn.Linear(in_dim, hidden_dims[0]))
        self.net.add_module("relu_0", nn.ReLU())

        layer_id = 1
        for hidden_dim in hidden_dims[1:]:
            self.net.add_module("lin_"+str(layer_id), nn.Linear(hidden_dim, hidden_dim))
            self.net.add_module("relu_"+str(layer_id), nn.ReLU())
            layer_id += 1

    def forward(self, x):
        return self.net(x)

The network formed is:
MlpNN(
  (net): Sequential(
    (lin_0): Linear(in_features=301, out_features=500, bias=True)
    (relu_0): ReLU()
    (lin_1): Linear(in_features=500, out_features=500, bias=True)
    (relu_1): ReLU()
    (lin_2): Linear(in_features=500, out_features=500, bias=True)
    (relu_2): ReLU()
    (lin_3): Linear(in_features=500, out_features=500, bias=True)
    (relu_3): ReLU()
    (lin_4): Linear(in_features=500, out_features=500, bias=True)
    (relu_4): ReLU()
    (lin_5): Linear(in_features=30, out_features=30, bias=True)
    (relu_5): ReLU()
  )
)

while doing forward(), it gives me following error:
mat1 and mat2 shapes cannot be multiplied (1x500 and 30x30)

Whats wrong I am doing here? I know it must be stupid basic given that I am quite new machine learning.


title: Changing learning rate after loading scheduler's state dict

body:
I have my model:
import torch
import torch.nn as nn
import torch.optim as optim

class net_x(nn.Module):
        def __init__(self):
            super(net_x, self).__init__()
            self.fc1=nn.Linear(2, 20)
            self.fc2=nn.Linear(20, 20)
            self.out=nn.Linear(20, 4)

        def forward(self, x):
            x=self.fc1(x)
            x=self.fc2(x)
            x=self.out(x)
            return x

nx = net_x()

r = torch.tensor([1.0,2.0])
optimizer = optim.Adam(nx.parameters(), lr = 0.1)
scheduler = torch.optim.lr_scheduler.CyclicLR(optimizer, base_lr=1e-2, max_lr=0.1, step_size_up=1, mode="triangular2", cycle_momentum=False)

path = 'opt.pt'
for epoch in range(10):
    optimizer.zero_grad()
    net_predictions = nx(r)
    loss = torch.sum(torch.randint(0,10,(4,)) - net_predictions)
    loss.backward()
    optimizer.step()
    scheduler.step()
    print('loss:' , loss)
    torch.save({    'epoch': epoch,
                    'net_x_state_dict': nx.state_dict(),
                    'optimizer_state_dict': optimizer.state_dict(),
                    'scheduler': scheduler.state_dict(),
                    }, path)

checkpoint = torch.load(path)
nx.load_state_dict(checkpoint['net_x_state_dict'])
optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
scheduler.load_state_dict(checkpoint['scheduler'])

It trains and loads just fine, but I'm interested in lowering the base_lr of the scheduler from 10^-2 to 10^-3 after training. Is there a way to do this without creating a completely new scheduler? From my understanding if I do that I will lose the parameters that the current scheduler is using and training will get worse/start from sort of scratch


title: Pytorch quickstart calls model.eval() but not model.train()

body:
In Pytorch quickstart tutorial the code uses model.eval() during evaluation/test but it does not call model.train() during training.
According to this and source, some modules like BatchNorm and Dropout need to know if the model is in train or evaluation mode. The model in the tutorial does not use any such module so it runs to convergence. Am I missing something or Pytorch's very first tutorial actually has a logical bug?
Training:
def train(dataloader, model, loss_fn, optimizer):
    size = len(dataloader.dataset)
    for batch, (X, y) in enumerate(dataloader):
        X, y = X.to(device), y.to(device)

        # Compute prediction error
        pred = model(X)
        loss = loss_fn(pred, y)

        # Backpropagation
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

        if batch % 100 == 0:
            loss, current = loss.item(), batch * len(X)
            print(f"loss: {loss:>7f}  [{current:>5d}/{size:>5d}]")

You can see there is no model.train() in the above code.
Testing:
def test(dataloader, model):
    size = len(dataloader.dataset)
    model.eval()
    test_loss, correct = 0, 0
    with torch.no_grad():
        for X, y in dataloader:
            X, y = X.to(device), y.to(device)
            pred = model(X)
            test_loss += loss_fn(pred, y).item()
            correct += (pred.argmax(1) == y).type(torch.float).sum().item()
    test_loss /= size
    correct /= size
    print(f"Test Error: \n Accuracy: {(100*correct):>0.1f}%, Avg loss: {test_loss:>8f} \n")

At the second line, there is a model.eval().
Training loop:
epochs = 5
for t in range(epochs):
    print(f"Epoch {t+1}\n-------------------------------")
    train(train_dataloader, model, loss_fn, optimizer)
    test(test_dataloader, model)
print("Done!")

This loop calls train() and test() methods without any call to model.train(). So after the first call of test(), the model is always in "evaluation" mode. If we add a BatchNorm to the model we'll be on our way to encounter a hard-to-find bug.
Main question:
Is it good practice to always call model.train() during training and model.eval() during evaluation/test?


title: Custom Pytorch layer to apply LSTM on each group

body:
I have a N × F tensor with features and a N × 1 tensor with group index. I want to design a custom pytorch layer which will apply LSTM on each group with sorted features. I have mentioned LSTM with sorted group features as an example, hypothetically it can be anything which supports variable length input or sequence. Please refer to the image below for visual interpretation of the problem.

The obvious approach would be calling a LSTM layer for each unique group but that would be inefficient. Is there any better way to do it?


title: Training Will Be Stop After a While in GRU Layer Pytorch

body:
I use my custom dataset class to convert audio files to mel-Spectrogram images. the shape will be padded to (128,1024). I have 10 classes. after a while of training in the first epoch, my network will be crashed inside the hidden layer in GRU shapes due to this error:
Current run is terminating due to exception: Expected hidden size (1, 7, 32), got [1, 16, 32]
Engine run is terminating due to exception: Expected hidden size (1, 7, 32), got [1, 16, 32]
Traceback (most recent call last):
  File "/home/omid/anaconda3/envs/pytorch/lib/python3.8/site-packages/IPython/core/interactiveshell.py", line 3418, in run_code
    exec(code_obj, self.user_global_ns, self.user_ns)
  File "<ipython-input-2-b8f3a45f8e35>", line 1, in <module>
    runfile('/home/omid/OMID/projects/python/mldl/NeuralMusicClassification/tools/train_net.py', wdir='/home/omid/OMID/projects/python/mldl/NeuralMusicClassification/tools')
  File "/home/omid/OMID/program/pycharm-professional-2020.2.4/pycharm-2020.2.4/plugins/python/helpers/pydev/_pydev_bundle/pydev_umd.py", line 197, in runfile
    pydev_imports.execfile(filename, global_vars, local_vars)  # execute the script
  File "/home/omid/OMID/program/pycharm-professional-2020.2.4/pycharm-2020.2.4/plugins/python/helpers/pydev/_pydev_imps/_pydev_execfile.py", line 18, in execfile
    exec(compile(contents+"\n", file, 'exec'), glob, loc)
  File "/home/omid/OMID/projects/python/mldl/NeuralMusicClassification/tools/train_net.py", line 60, in <module>
    main()
  File "/home/omid/OMID/projects/python/mldl/NeuralMusicClassification/tools/train_net.py", line 56, in main
    train(cfg)
  File "/home/omid/OMID/projects/python/mldl/NeuralMusicClassification/tools/train_net.py", line 35, in train
    do_train(
  File "/home/omid/OMID/projects/python/mldl/NeuralMusicClassification/engine/trainer.py", line 79, in do_train
    trainer.run(train_loader, max_epochs=epochs)
  File "/home/omid/anaconda3/envs/pytorch/lib/python3.8/site-packages/ignite/engine/engine.py", line 702, in run
    return self._internal_run()
  File "/home/omid/anaconda3/envs/pytorch/lib/python3.8/site-packages/ignite/engine/engine.py", line 775, in _internal_run
    self._handle_exception(e)
  File "/home/omid/anaconda3/envs/pytorch/lib/python3.8/site-packages/ignite/engine/engine.py", line 469, in _handle_exception
    raise e
  File "/home/omid/anaconda3/envs/pytorch/lib/python3.8/site-packages/ignite/engine/engine.py", line 745, in _internal_run
    time_taken = self._run_once_on_dataset()
  File "/home/omid/anaconda3/envs/pytorch/lib/python3.8/site-packages/ignite/engine/engine.py", line 850, in _run_once_on_dataset
    self._handle_exception(e)
  File "/home/omid/anaconda3/envs/pytorch/lib/python3.8/site-packages/ignite/engine/engine.py", line 469, in _handle_exception
    raise e
  File "/home/omid/anaconda3/envs/pytorch/lib/python3.8/site-packages/ignite/engine/engine.py", line 833, in _run_once_on_dataset
    self.state.output = self._process_function(self, self.state.batch)
  File "/home/omid/anaconda3/envs/pytorch/lib/python3.8/site-packages/ignite/engine/__init__.py", line 103, in _update
    y_pred = model(x)
  File "/home/omid/anaconda3/envs/pytorch/lib/python3.8/site-packages/torch/nn/modules/module.py", line 889, in _call_impl
    result = self.forward(*input, **kwargs)
  File "/home/omid/OMID/projects/python/mldl/NeuralMusicClassification/modeling/model.py", line 113, in forward
    x, h1 = self.gru1(x, h0)
  File "/home/omid/anaconda3/envs/pytorch/lib/python3.8/site-packages/torch/nn/modules/module.py", line 889, in _call_impl
    result = self.forward(*input, **kwargs)
  File "/home/omid/anaconda3/envs/pytorch/lib/python3.8/site-packages/torch/nn/modules/rnn.py", line 819, in forward
    self.check_forward_args(input, hx, batch_sizes)
  File "/home/omid/anaconda3/envs/pytorch/lib/python3.8/site-packages/torch/nn/modules/rnn.py", line 229, in check_forward_args
    self.check_hidden_size(hidden, expected_hidden_size)
  File "/home/omid/anaconda3/envs/pytorch/lib/python3.8/site-packages/torch/nn/modules/rnn.py", line 223, in check_hidden_size
    raise RuntimeError(msg.format(expected_hidden_size, list(hx.size())))
RuntimeError: Expected hidden size (1, 7, 32), got [1, 16, 32]

My Network Is:
import torch
import torch.nn as nn
import torch.nn.functional as F

print('cuda', torch.cuda.is_available())

class MusicClassification(nn.Module):
    def __init__(self, cfg):
        super(MusicClassification, self).__init__()
        device = cfg.MODEL.DEVICE
        num_class = cfg.MODEL.NUM_CLASSES

        self.np_layers = 4
        self.np_filters = [64, 128, 128, 128]
        self.kernel_size = (3, 3)

        self.pool_size = [(2, 2), (4, 2)]

        self.channel_axis = 1
        self.frequency_axis = 2
        self.time_axis = 3

        # self.h0 = torch.zeros((1, 16, 32)).to(device)

        self.bn0 = nn.BatchNorm2d(num_features=self.channel_axis)
        self.bn1 = nn.BatchNorm2d(num_features=self.np_filters[0])
        self.bn2 = nn.BatchNorm2d(num_features=self.np_filters[1])
        self.bn3 = nn.BatchNorm2d(num_features=self.np_filters[2])
        self.bn4 = nn.BatchNorm2d(num_features=self.np_filters[3])

        self.conv1 = nn.Conv2d(1, self.np_filters[0], kernel_size=self.kernel_size)
        self.conv2 = nn.Conv2d(self.np_filters[0], self.np_filters[1], kernel_size=self.kernel_size)
        self.conv3 = nn.Conv2d(self.np_filters[1], self.np_filters[2], kernel_size=self.kernel_size)
        self.conv4 = nn.Conv2d(self.np_filters[2], self.np_filters[3], kernel_size=self.kernel_size)

        self.max_pool_2_2 = nn.MaxPool2d(self.pool_size[0])
        self.max_pool_4_2 = nn.MaxPool2d(self.pool_size[1])

        self.drop_01 = nn.Dropout(0.1)
        self.drop_03 = nn.Dropout(0.3)

        self.gru1 = nn.GRU(input_size=128, hidden_size=32, batch_first=True)
        self.gru2 = nn.GRU(input_size=32, hidden_size=32, batch_first=True)

        self.activation = nn.ELU()

        self.dense = nn.Linear(32, num_class)
        self.softmax = nn.LogSoftmax(dim=1)

    def forward(self, x):
        # x [16, 1, 128,938]
        x = self.bn0(x)
        # x [16, 1, 128,938]
        x = F.pad(x, (0, 0, 2, 1))
        # x [16, 1, 131,938]
        x = self.conv1(x)
        # x [16, 64, 129,936]
        x = self.activation(x)
        # x [16, 64, 129,936]
        x = self.bn1(x)
        # x [16, 64, 129,936]
        x = self.max_pool_2_2(x)
        # x [16, 64, 64,468]
        x = self.drop_01(x)
        # x [16, 64, 64,468]
        x = F.pad(x, (0, 0, 2, 1))
        # x [16, 64, 67,468]
        x = self.conv2(x)
        # x [16, 128, 65,466]
        x = self.activation(x)
        # x [16, 128, 65,466]
        x = self.bn2(x)
        # x [16, 128, 65,455]
        x = self.max_pool_4_2(x)
        # x [16, 128, 16,233]
        x = self.drop_01(x)
        # x [16, 128, 16,233]
        x = F.pad(x, (0, 0, 2, 1))
        # x [16, 128, 19,233]
        x = self.conv3(x)
        # x [16, 128, 17,231]
        x = self.activation(x)
        # x [16, 128, 17,231]
        x = self.bn3(x)
        # x [16, 128, 17,231]
        x = self.max_pool_4_2(x)
        # x [16, 128, 4,115]
        x = self.drop_01(x)
        # x [16, 128, 4,115]
        x = F.pad(x, (0, 0, 2, 1))
        # x [16, 128, 7,115]
        x = self.conv4(x)
        # x [16, 128, 5,113]
        x = self.activation(x)
        # x [16, 128, 5,113]
        x = self.bn4(x)
        # x [16, 128, 5,113]
        x = self.max_pool_4_2(x)
        # x [16, 128, 1,56]
        x = self.drop_01(x)
        # x [16, 128, 1,56]

        x = x.permute(0, 3, 1, 2)
        # x [16, 56, 128,1]
        resize_shape = list(x.shape)[2] * list(x.shape)[3]
        # x [16, 128, 56,1], reshape size is 128
        x = torch.reshape(x, (list(x.shape)[0], list(x.shape)[1], resize_shape))
        # x [16, 56, 128]
        device = torch.device("cuda" if torch.cuda.is_available()
                              else "cpu")
        h0 = torch.zeros((1, 16, 32)).to(device)
        x, h1 = self.gru1(x, h0)
        # x [16, 56, 32]
        x, _ = self.gru2(x, h1)
        # x [16, 56, 32]
        x = x[:, -1, :]
        x = self.dense(x)
        # x [16,10]
        x = self.softmax(x)
        # x [16, 10]
        # x = torch.argmax(x, 1)
        return x

My Dataset is:

from __future__ import print_function, division

import os

import librosa
import matplotlib.pyplot as plt
import numpy as np
import torch
import torchaudio
from sklearn.preprocessing import OneHotEncoder, LabelEncoder
from torch.utils.data import Dataset
from utils.util import pad_along_axis

print(torch.__version__)
print(torchaudio.__version__)

# Ignore warnings
import warnings

warnings.filterwarnings("ignore")

plt.ion()

import pathlib

print(pathlib.Path().absolute())

class GTZANDataset(Dataset):
    def __init__(self,
                 genre_folder='/home/omid/OMID/projects/python/mldl/NeuralMusicClassification/data/dataset/genres_original',
                 one_hot_encoding=False,
                 sr=16000, n_mels=128,
                 n_fft=2048, hop_length=512,
                 transform=None):

        self.genre_folder = genre_folder
        self.one_hot_encoding = one_hot_encoding
        self.audio_address, self.labels = self.extract_address()
        self.sr = sr
        self.n_mels = n_mels
        self.n_fft = n_fft
        self.transform = transform
        self.le = LabelEncoder()
        self.hop_length = hop_length

    def __len__(self):
        return len(self.labels)

    def __getitem__(self, index):
        address = self.audio_address[index]
        y, sr = librosa.load(address, sr=self.sr)
        S = librosa.feature.melspectrogram(y, sr=sr,
                                           n_mels=self.n_mels,
                                           n_fft=self.n_fft,
                                           hop_length=self.hop_length)

        sample = librosa.amplitude_to_db(S, ref=1.0)
        sample = np.expand_dims(sample, axis=0)
        sample = pad_along_axis(sample, 1024, axis=2)
        # print(sample.shape)
        sample = torch.from_numpy(sample)

        label = self.labels[index]
        # label = torch.from_numpy(label)
        print(sample.shape,label)
        if self.transform:
            sample = self.transform(sample)
        return sample, label

    def extract_address(self):
        label_map = {
            'blues': 0,
            'classical': 1,
            'country': 2,
            'disco': 3,
            'hiphop': 4,
            'jazz': 5,
            'metal': 6,
            'pop': 7,
            'reggae': 8,
            'rock': 9
        }
        labels = []
        address = []
        # extract all genres' folders
        genres = [path for path in os.listdir(self.genre_folder)]
        for genre in genres:
            # e.g. ./data/generes_original/country
            genre_path = os.path.join(self.genre_folder, genre)
            # extract all sounds from genre_path
            songs = os.listdir(genre_path)

            for song in songs:
                song_path = os.path.join(genre_path, song)
                genre_id = label_map[genre]
                # one_hot_targets = torch.eye(10)[genre_id]
                labels.append(genre_id)
                address.append(song_path)

        samples = np.array(address)
        labels = np.array(labels)
        # convert labels to one-hot encoding
        # if self.one_hot_encoding:
        #     labels = OneHotEncoder(sparse=False).fit_transform(labels)
        # else:
        #     labels = LabelEncoder().fit_transform(labels)

        return samples, labels

and trainer :

# encoding: utf-8

import logging

from ignite.engine import Events, create_supervised_trainer, create_supervised_evaluator
from ignite.handlers import ModelCheckpoint, Timer
from ignite.metrics import Accuracy, Loss, RunningAverage

def do_train(
        cfg,
        model,
        train_loader,
        val_loader,
        optimizer,
        scheduler,
        loss_fn,
):
    log_period = cfg.SOLVER.LOG_PERIOD
    checkpoint_period = cfg.SOLVER.CHECKPOINT_PERIOD
    output_dir = cfg.OUTPUT_DIR
    device = cfg.MODEL.DEVICE
    epochs = cfg.SOLVER.MAX_EPOCHS

    model = model.to(device)

    logger = logging.getLogger("template_model.train")
    logger.info("Start training")
    trainer = create_supervised_trainer(model, optimizer, loss_fn, device=device)
    evaluator = create_supervised_evaluator(model, metrics={'accuracy': Accuracy(),
                                                            'ce_loss': Loss(loss_fn)}, device=device)
    checkpointer = ModelCheckpoint(output_dir, 'mnist', None, n_saved=10, require_empty=False)
    timer = Timer(average=True)

    trainer.add_event_handler(Events.EPOCH_COMPLETED, checkpointer, {'model': model.state_dict(),
                                                                     'optimizer': optimizer.state_dict()})
    timer.attach(trainer, start=Events.EPOCH_STARTED, resume=Events.ITERATION_STARTED,
                 pause=Events.ITERATION_COMPLETED, step=Events.ITERATION_COMPLETED)

    RunningAverage(output_transform=lambda x: x).attach(trainer, 'avg_loss')

    @trainer.on(Events.ITERATION_COMPLETED)
    def log_training_loss(engine):
        iter = (engine.state.iteration - 1) % len(train_loader) + 1

        if iter % log_period == 0:
            logger.info("Epoch[{}] Iteration[{}/{}] Loss: {:.2f}"
                        .format(engine.state.epoch, iter, len(train_loader), engine.state.metrics['avg_loss']))

    @trainer.on(Events.EPOCH_COMPLETED)
    def log_training_results(engine):
        evaluator.run(train_loader)
        metrics = evaluator.state.metrics
        avg_accuracy = metrics['accuracy']
        avg_loss = metrics['ce_loss']
        logger.info("Training Results - Epoch: {} Avg accuracy: {:.3f} Avg Loss: {:.3f}"
                    .format(engine.state.epoch, avg_accuracy, avg_loss))

    if val_loader is not None:
        @trainer.on(Events.EPOCH_COMPLETED)
        def log_validation_results(engine):
            evaluator.run(val_loader)
            metrics = evaluator.state.metrics
            avg_accuracy = metrics['accuracy']
            avg_loss = metrics['ce_loss']
            logger.info("Validation Results - Epoch: {} Avg accuracy: {:.3f} Avg Loss: {:.3f}"
                        .format(engine.state.epoch, avg_accuracy, avg_loss)
                        )

    # adding handlers using `trainer.on` decorator API
    @trainer.on(Events.EPOCH_COMPLETED)
    def print_times(engine):
        logger.info('Epoch {} done. Time per batch: {:.3f}[s] Speed: {:.1f}[samples/s]'
                    .format(engine.state.epoch, timer.value() * timer.step_count,
                            train_loader.batch_size / timer.value()))
        timer.reset()

    trainer.run(train_loader, max_epochs=epochs)


title: save and load unserialized pytorch pretrained model

body:
for normal saving and loading pytorch model is quite simple.
in python I can use torch.save(model, FilePath) and in c++torch::jit::load(FilePath). and the saved model and c++ load code can be placed in one directory. However, there is a limitation that binary file cannot be contained in the directory in the production mode (Please don't ask me why the binary file cannot be contained, I also wondering).
So, I want to know how to save the pytorch model from python without serializtion and load this model in c++. Is it possible?


I am getting this error "PipelineException: No mask_token ([MASK]) found on the input"
when I run this line.
fill_mask("Auto Car .")
I am running it on Colab.
My Code:
from transformers import BertTokenizer, BertForMaskedLM
from pathlib import Path
from tokenizers import ByteLevelBPETokenizer
from transformers import BertTokenizer, BertForMaskedLM

paths = [str(x) for x in Path(".").glob("**/*.txt")]
print(paths)

bert_tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')

from transformers import BertModel, BertConfig

configuration = BertConfig()
model = BertModel(configuration)
configuration = model.config
print(configuration)

model = BertForMaskedLM.from_pretrained("bert-base-uncased")

from transformers import LineByLineTextDataset
dataset = LineByLineTextDataset(
    tokenizer=bert_tokenizer,
    file_path="./kant.txt",
    block_size=128,
)

from transformers import DataCollatorForLanguageModeling
data_collator = DataCollatorForLanguageModeling(
    tokenizer=bert_tokenizer, mlm=True, mlm_probability=0.15
)

from transformers import Trainer, TrainingArguments

training_args = TrainingArguments(
    output_dir="./KantaiBERT",
    overwrite_output_dir=True,
    num_train_epochs=1,
    per_device_train_batch_size=64,
    save_steps=10_000,
    save_total_limit=2,
    )

trainer = Trainer(
    model=model,
    args=training_args,
    data_collator=data_collator,
    train_dataset=dataset,
)

trainer.train()

from transformers import pipeline

fill_mask = pipeline(
    "fill-mask",
    model=model,
    tokenizer=bert_tokenizer,
    device=0,
)

fill_mask("Auto Car <mask>.").     # This line is giving me the error...

The last line is giving me the error mentioned above. Please let me know what I am doing wrong or what I have to do in order to remove this error.
Complete error: "f"No mask_token ({self.tokenizer.mask_token}) found on the input","


title: Bert Tokenizer add_token function not working properly

body:
tokenizer add_tokens is not adding new tokens.
Here is my code:
  from transformers import BertTokenizer

  bert_tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')

  new_tokens = []
  text = open("parsed_data.txt", "r")
  for line in text:
     for word in line.split():
        new_tokens.append(word)

  print(len(new_tokens))      # 53966
  print(len(bert_tokenizer))  # 36369
  bert_tokenizer.add_tokens(new_tokens)
  print(len(bert_tokenizer))  # 36369


title: Are there any logistic error in my "focal loss" function?

body:
"focal loss" were born to addressed the difficult samples.I wrap my focal loss function with binary cross
entropy in pytorch:
class FocalLoss(nn.Module):

    def __init__(self, gamma=2):
        super(FocalLoss, self).__init__()
        self.gamma = gamma

    def forward(self, pred, label):
        # label is not the one-hot
        true = torch.zeros_like(pred, dtype=torch.float)
        for i, j in enumerate(label):
            true[i, j] = 1.0

        loss = nn.BCEWithLogitsLoss(pred, true)

        pred_prob = torch.sigmoid(pred)  # sigmoid
        p_t = true * pred_prob + (1 - true) * (1 - pred_prob)
        modulating_factor = (1.0 - p_t) ** self.gamma
        loss *= modulating_factor

        return loss.mean()

and I train my resnet18 in cifar10 dataset for classifiction task with it and nn.CrossEntropyLoss for
comparison of accuracy.

But the result is far away from my expectation: the accuracy in CrossEntropyLoss is higher than my focal loss about 5%! Can anyone find logistic errors in my focal loss code above?


title: Predict new samples with PyTorch model

body:
I am newbee in neural networks, i have teached my model and now i want to test it. I have wrote a code with google help, but it do not work.
The problem is that i do not understand from where i am getting the 4th dimension.
Code is the following:
import matplotlib.pyplot as plt
import numpy as np
import torch
import torchvision
import torchvision.transforms as transforms
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim

transform = transforms.Compose([
    transforms.Resize(32),
    transforms.CenterCrop(32),
    transforms.ToTensor(),
    transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
    ])

main_path = 'D:/RTU/dataset/ready_dataset_2classes'
train_data_path = main_path + '/train'
#test_data_path = main_path + '/test'
weigths_path = 'D:/RTU/dataset/weights_done/weights_noise_original037-97%.pt'

train_data = torchvision.datasets.ImageFolder(root=train_data_path, transform=transform)

class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        # convolutional layer (sees 32x32x3 image tensor)
        self.conv1 = nn.Conv2d(3, 16, 3, padding=1)
        # convolutional layer (sees 16x16x16 tensor)
        self.conv2 = nn.Conv2d(16, 32, 3, padding=1)
        # convolutional layer (sees 8x8x32 tensor)
        self.conv3 = nn.Conv2d(32, 64, 3, padding=1)
        # max pooling layer
        self.pool = nn.MaxPool2d(2, 2)
        # linear layer (64 * 4 * 4 -> 500)
        self.fc1 = nn.Linear(64 * 4 * 4, 500)
        # linear layer (500 -> 10)
        self.fc2 = nn.Linear(500, 250)
        self.fc3 = nn.Linear(250, 2)
        # dropout layer (p=0.25)
        self.dropout = nn.Dropout(0.25) #0.25

    def forward(self, x):
        # add sequence of convolutional and max pooling layers
        x = self.pool(F.relu(self.conv1(x)))
        x = self.pool(F.relu(self.conv2(x)))
        x = self.pool(F.relu(self.conv3(x)))
        # flatten image input
        x = x.view(-1, 64 * 4 * 4)
        # add dropout layer
        x = self.dropout(x)
        # add 1st hidden layer, with relu activation function
        x = F.relu(self.fc1(x))
        # add dropout layer
        x = self.dropout(x)
        # add 2nd hidden layer, with relu activation function
        x = F.relu(self.fc2(x))
        x = self.dropout(x)
        x = self.fc3(x)
        return x

# Disable grad
with torch.no_grad():

    # Retrieve item
    index = 1
    item = train_data[index]
    image = item[0]
    true_target = item[1]

    # Loading the saved model
    mlp = Net()
    optimizer = optim.SGD(mlp.parameters(), lr=0.01)
    epoch=5
    valid_loss_min = np.Inf
    checkpoint = torch.load(weigths_path , map_location=torch.device('cpu'))
    mlp.load_state_dict(checkpoint['model_state_dict'])
    optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
    epoch = checkpoint['epoch']
    valid_loss_min = checkpoint['valid_loss_min']
    mlp.eval()
    # Generate prediction
    prediction = mlp(image)

    # Predicted class value using argmax
    predicted_class = np.argmax(prediction)

    # Reshape image
    #image = image.reshape(28, 28, 1)

    # Show result
    plt.imshow(image, cmap='gray')
    plt.title(f'Prediction: {predicted_class} - Actual target: {true_target}')
    plt.show()

The code seems working till the "mlp.eval()" and then i am getting an error Expected 4-dimensional input for 4-dimensional weight [16, 3, 3, 3], but got 3-dimensional input of size [3, 32, 32] instead.
What i am doing wrong?
Error