title: Pytorch LSTM- VAE Sentence Generator: RuntimeError: one of the variables needed for gradient computation has been modified by an inplace operation

body:
I am trying to make a LSTM VAE as a learning stage for future work with pytorch.
I managed to get it to work on some small tester data but now that I want to run it on my actual data I am continuously getting this error:
RuntimeError: one of the variables needed for gradient computation has been modified by an inplace operation: [torch.FloatTensor [10, 40]], which is output 0 of TBackward, is at version 2; expected version 1 instead. Hint: the backtrace further above shows the operation that failed to compute its gradient. The variable in question was changed in there or anywhere later. Good luck!
Any help on how to solve this error and generally just make my code more efficient would be great!
Here is the appropriate section of the traceback:
File "<ipython-input-16-7fe0e9e30e5d>", line 190, in <module>
    rec_loss, kl_loss = train_batch(x,G_inp,epoch,train=True)
  File "<ipython-input-16-7fe0e9e30e5d>", line 166, in train_batch
    reconstruction, hidden, kld = model(x, G_inp, None, None)
  File "/usr/local/lib/python3.6/dist-packages/torch/nn/modules/module.py", line 727, in _call_impl
    result = self.forward(*input, **kwargs)
  File "<ipython-input-16-7fe0e9e30e5d>", line 93, in forward
    mu, logvar, z = self.encoder(x)
  File "/usr/local/lib/python3.6/dist-packages/torch/nn/modules/module.py", line 727, in _call_impl
    result = self.forward(*input, **kwargs)
  File "<ipython-input-16-7fe0e9e30e5d>", line 37, in forward
    out1, self.hidden  = self.lstm(x, self.hidden)
  File "/usr/local/lib/python3.6/dist-packages/torch/nn/modules/module.py", line 727, in _call_impl
    result = self.forward(*input, **kwargs)
  File "/usr/local/lib/python3.6/dist-packages/torch/nn/modules/rnn.py", line 582, in forward
    self.dropout, self.training, self.bidirectional, self.batch_first)
 (Triggered internally at  /pytorch/torch/csrc/autograd/python_anomaly_mode.cpp:104.)
  allow_unreachable=True)  # allow_unreachable flag

The code is below: (excuse all the .clone(), I read that this could be a solution so I was testing it out everywhere with no help)
import numpy as np
import pandas as pd
import torch
import torch.nn as nn
import torch.nn.parallel
import torch.optim as optim

from keras.preprocessing.text import Tokenizer
from keras import preprocessing
import torch.nn.functional as F
import math
import random

#encoder
class Encoder(nn.Module):
    def __init__(self,embedding_dim, vocab_size,  n_layers_E, n_hidden_E, dim_z, n_hidden_G, batch_size):
        super(Encoder,self).__init__()

        self.n_layers_E = n_layers_E
        self.n_hidden_E = n_hidden_E
        self.batch_size = batch_size
        self.dim_z = dim_z

        self.lstm = nn.LSTM(input_size = embedding_dim, hidden_size = n_hidden_E, num_layers = n_layers_E, batch_first=True, bidirectional = True)
        self.hidden_to_mu = nn.Linear(2*n_hidden_E,dim_z)
        self.hidden_to_logvar = nn.Linear(2*n_hidden_G, dim_z)

        self.hidden = (torch.zeros(2*n_layers_E, batch_size, n_hidden_E),torch.zeros(2*n_layers_E, batch_size, n_hidden_E))

    def forward(self,x):

        batch_size, n_seq, n_embed = x.size()
        #batch_size, n_seq = x.size()
        out1, self.hidden  = self.lstm(x, self.hidden)
        e_hidden = self.hidden[0].view(batch_size, 2 * self.n_hidden_E).clone()

        #e_hidden = torch.cat(list(hidden),dim = 0)

        mu = self.hidden_to_mu(e_hidden)
        logvar = self.hidden_to_logvar(e_hidden)
        epsilon = torch.randn([batch_size, self.dim_z])
        z = mu + torch.exp(logvar*0.5)*epsilon

        return mu, logvar, z

class Generator(nn.Module):
    def __init__(self,n_hidden_G,n_layers_G, embedding_dim, dim_z, vocab_size , batch_size):
        super(Generator,self).__init__()

        self.n_hidden_G =  n_hidden_G
        self.n_layers_G = n_layers_G
        self.n_z = dim_z
        self.batch_size = batch_size
        self.LSTM = nn.LSTM(input_size = embedding_dim + dim_z, hidden_size = n_hidden_G, num_layers = n_layers_G, batch_first = True)
        self.fc = nn.Linear(n_hidden_G, vocab_size)

        self.hidden = (torch.zeros(self.n_layers_G, batch_size, self.n_hidden_G)
                       ,torch.zeros(self.n_layers_G, batch_size, self.n_hidden_G))

    def forward(self,x,z, g_hidden = None):

        batch_size,n_seq, n_embed = x.size()
        #batch_size, n_seq= x.size()
        z = torch.cat([z]*n_seq,1).view(batch_size, n_seq, self.n_z)
        x = torch.cat([x,z], dim = 2)

        if g_hidden is None:                                        #if we are validating
            self.hidden = (torch.zeros(self.n_layers_G, batch_size, self.n_hidden_G)
                       ,torch.zeros(self.n_layers_G, batch_size, self.n_hidden_G))
        else:                                                       #if we are training
            self.hidden = g_hidden

        output, self.hidden = self.LSTM(x, self.hidden)
        output = self.fc(output)

        return output, self.hidden

class VAE(nn.Module):
    def __init__(self, embedding_dim, vocab_size,  n_layers_E, n_hidden_E, dim_z, n_hidden_G, batch_size,n_layers_G ):
        super(VAE, self).__init__()
        self.embedding = nn.Embedding(vocab_size, embedding_dim)
        self.encoder = Encoder(embedding_dim, vocab_size,  n_layers_E, n_hidden_E, dim_z, n_hidden_G, batch_size)
        self.generator = Generator(n_hidden_G,n_layers_G, embedding_dim, dim_z, vocab_size, batch_size )
        self.n_z = dim_z

    def forward(self, x, G_inp, z, G_hidden):
        if z is None:
            batch_size, n_seq = x.size()
            x = self.embedding(x)
            mu, logvar, z = self.encoder(x)

            kld = -0.5*torch.sum(logvar-mu.pow(2)-logvar.exp()+1).mean()
        else:
            kld = None
        G_inp = self.embedding(G_inp)

        logit, G_hidden = self.generator(G_inp,z, G_hidden)
        return logit, G_hidden, kld

train_df = pd.read_csv("train.csv", header =None)[0:500]
test_df = pd.read_csv("test.csv",header =None)[0:500]

train = train_df.iloc[:,0]

max_words = 2000
max_len = 25

tok = Tokenizer(num_words = max_words)
tok.fit_on_texts(train)
sequences = tok.texts_to_sequences(train)
sequences_matrix = preprocessing.sequence.pad_sequences(sequences, maxlen = max_len)

#tok.sequences_to_texts(sequences)

n_hidden_E = 10
n_layers_E = 1
embedding_dim = 10
vocab_size = max_words
n_hidden_G = 10
n_layers_G = 2
dim_z = 10
train_size = len(train)
batch_size = 100
rec_coef = 7

lr = 0.01
epochs = 100

def create_generator_input(x, train):
    G_inp = x[:, 0:max_len-1].clone()                       #input for generator should exclude last word of sequence
    # if train == False:
    #     return G_inp

    # r = np.random.rand(G_inp.size(0), G_inp.size(1))
    #                                                         #Perform word_dropout according to random values (r) generated for each word
    # for i in range(len(G_inp)):
    #     for j in range(1,G_inp.size(1)):
    #         if r[i, j] < opt.word_dropout and G_inp[i, j] not in [vocab.stoi[opt.pad_token], vocab.stoi[opt.end_token]]:
    #             G_inp[i, j] = vocab.stoi[opt.unk_token]

    return G_inp

def producebatches(x,batch_size):
    k = math.floor(x.shape[0]/batch_size)
    total = (k)*batch_size
    flatten = x[0:total].flatten()
    batches = flatten.reshape((k,batch_size,x.shape[1]))
    return batches

batches = producebatches(sequences_matrix, batch_size)

model = VAE(embedding_dim, vocab_size,  n_layers_E, n_hidden_E, dim_z, n_hidden_G, batch_size,n_layers_G)
optimizer = torch.optim.Adam(model.parameters(), lr = lr)
criterion = nn.BCELoss(reduction = 'sum')

model.train()

def train_batch(x,G_inp,step,train  =True):
    reconstruction, hidden, kld = model(x, G_inp, None, None)
    reconstruction2 = reconstruction.view(-1, vocab_size).clone()                       #converting into shape (batch_size*(n_seq-1), n_vocab) to facilitate performing F.cross_entropy()
    #y = x[:, 1:x.size(1)].clone()                                  #target for generator should exclude first word of sequence
    #y = y.contiguous().view(-1)
    G_inp2 = G_inp.contiguous().view(-1)                              #converting into shape (batch_size*(n_seq-1),1) to facilitate performing F.cross_entropy()
    rec_loss = F.cross_entropy(reconstruction2,G_inp2)
    kld_coef = (math.tanh((step - 15000)/1000) + 1) / 2
    #kld_coef = min(1,step/(200000.0))
    loss = rec_coef*rec_loss + kld_coef*kld
    if train == True:
        torch.autograd.set_detect_anomaly(True)                                 #skip below step if we are performing validation
        optimizer.zero_grad()
        loss.backward(retain_graph=True)
        optimizer.step()
    return rec_loss.item(), kld.item()

for epoch in range(epochs):
    train_rec_loss = []
    train_kl_loss = []

    for i in range(batches.shape[0]):
        x = torch.tensor(batches[i], dtype = torch.long)
        G_inp = create_generator_input(x, train = True)
        rec_loss, kl_loss = train_batch(x,G_inp,epoch,train=True)
        train_rec_loss.append(rec_loss)
        train_kl_loss.append(kl_loss)

    train_rec_loss = np.mean(train_rec_loss)
    train_kl_loss = np.mean(train_kl_loss)
    print("No.", epoch, "T_rec:", '%.2f'%rec_loss, "T_kld:", '%.2f'%kl_loss)

def generate_sentences(n):

    model.eval()
    sentences = []

    for i in range(n):
        z = torch.randn([1, dim_z])
        hidden = (torch.zeros(n_layers_G,1, n_hidden_G)
                       ,torch.zeros(n_layers_G, 1, n_hidden_G))
        G_inp = torch.LongTensor(1,1).fill_(1)
        str_ind = []
        while len(str_ind)<49:
            with torch.autograd.no_grad():
                logit, G_hidden, _ = model(None, G_inp, z, hidden)

            probs = F.softmax(logit[0],dim=1)
            G_inp = torch.multinomial(probs,1)
            str_ind.append(G_inp[0][0].item())
        sentences.append(str_ind)
    return sentences
t = generate_sentences(1)


title: Load pretrained pytorch model

body:
Haven't been able to find this solution elsewhere though the question may seem really simple.
I have a pytorch (.pt) file and I'm trying to load it. I know that I need to construct the model  first by doing
model = MyModel()

but my pytorch file buiilds a model (se_resnext101_32x4d) that I did not make a class for. As such when I try and do
model = se_resnext101_32x4d()

I get an error
name 'se_resnext101_32x4d' is not defined

I've tried doing
import pretrainedmodels

model = pretrainedmodels.__dict__[se_resnext101_32x4d]()

but the error persists.


title: Create a pyTorch testing Dataset (without labels)

body:
I have created a pyTorch dataset for my training data which consists of features and a label to be able to utilize the pyTorch DataLoader using this tutorial. This works well for my training data, but I get an error (KeyError: "['label'] not found in axis") when loading the testing csv file, which is identical other than there being no "label" column.
If it helps, the intended input csv file is MNIST data in csv file which has 28*28 feature columns.
import torch

class mnist(torch.utils.data.Dataset):

    def __init__(self, csv_file):
        self.train = pd.read_csv(csv_file)
        self.train_x = self.train.drop("label", axis=1)

    def __len__(self):
        return len(self.train)

    def __getitem__(self, idx):
        if torch.is_tensor(idx):
            idx = idx.tolist()
        if isinstance(idx, list):
            idx_len = len(idx)
        else:
            idx_len = 1

        X = np.asarray(self.train_x.iloc[idx], dtype=np.float32)
        X = np.reshape(X, (1,28,28))
        y = np.asarray(self.train.iloc[idx]['label'])

        sample = {'X': X, 'y':y}

        return torch.from_numpy(sample['X']), torch.from_numpy(sample['y'])


title: Huggingface error during training: AttributeError: 'str' object has no attribute 'size'

body:
While trying to finetune a Huggingface GPT2LMHeadModel model for casual language modeling (given a sequence of words, predict the next word) using Pytorch Lightning, I am getting an error during training:

AttributeError: 'str' object has no attribute 'size'

What went wrong with our training code? Is this due to the incorrect use of DataCollatorForLanguageModeling in a Pytorch DataLoader?
Reproducible Example:
import os
from pathlib import Path
import torch
import pytorch_lightning as pl
from transformers import (
    GPT2Config,
    GPT2LMHeadModel,
    GPT2Tokenizer,
    DataCollatorForLanguageModeling,
)
from transformers.optimization import AdamW
from tokenizers import ByteLevelBPETokenizer
from torch.utils.data import (
    DataLoader,
    Dataset,
)

TOKENIZER_DIRPATH = os.path.join("..", "data")

def tokenize_data():
    tokenizer = ByteLevelBPETokenizer()
    tokenizer.train(
        files=os.path.join(TOKENIZER_DIRPATH, "words.txt"),
        vocab_size=50000,
        min_frequency=2,
        special_tokens=["<s>", "</s>", "<unk>", "<mask>", "<pad>",],
    )
    tokenizer.save_model("../data")

class MyDataset(Dataset):
    def __init__(self):
        tokenizer = GPT2Tokenizer(
            os.path.join(TOKENIZER_DIRPATH, "vocab.json"),
            os.path.join(TOKENIZER_DIRPATH, "merges.txt"),
        )

        src_file = Path(os.path.join(TOKENIZER_DIRPATH, "words.txt"))
        lines = src_file.read_text(encoding="utf-8").splitlines()
        self.examples = [tokenizer.encode(line) for line in lines]

    def __len__(self):
        return len(self.examples)

    def __getitem__(self, i):
        return torch.tensor(self.examples[i])

class MyDataModule(pl.LightningDataModule):
    def __init__(self):
        super().__init__()
        self.tokenizer = GPT2Tokenizer(
            os.path.join(TOKENIZER_DIRPATH, "vocab.json"),
            os.path.join(TOKENIZER_DIRPATH, "merges.txt"),
        )

    def setup(self, stage):
        self.train_dataset = MyDataset()

    def train_dataloader(self):
        data_collator = DataCollatorForLanguageModeling(
            tokenizer=self.tokenizer, mlm=False
        )
        train_dataloader = DataLoader(self.train_dataset, collate_fn=data_collator)
        return train_dataloader

class MyModel(pl.LightningModule):
    def __init__(self, learning_rate, adam_beta1, adam_beta2, adam_epsilon):
        super().__init__()
        self.save_hyperparameters()
        config = GPT2Config()
        self.model = GPT2LMHeadModel(config)

    def forward(self, x):
        return self.model(x).logits

    def training_step(self, batch, batch_idx):
        input_ids, labels = batch
        loss = self.model(input_ids, labels=labels).loss
        self.log("train_loss", loss, on_epoch=True)
        return loss

    def configure_optimizers(self):
        optimizer = AdamW(
            self.parameters(),
            self.hparams.learning_rate,
            betas=(self.hparams.adam_beta1, self.hparams.adam_beta2),
            eps=self.hparams.adam_epsilon,
        )
        return optimizer

tokenize_data()
dm = MyDataModule()
model = MyModel(
    learning_rate=5e-5, adam_beta1=0.9, adam_beta2=0.999, adam_epsilon=1e-8,
)

trainer = pl.Trainer()
trainer.fit(model, dm)

Error Traceback:
Epoch 0:   0%|                                                                                | 0/9 [00:00<?, ?it/s]
Traceback (most recent call last):
  File "test_gpt.py", line 102, in <module>
    trainer.fit(model, dm)
  File "/opt/anaconda3/envs/test_huggingface/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 499, in fit
    self.dispatch()
  File "/opt/anaconda3/envs/test_huggingface/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 546, in dispatch
    self.accelerator.start_training(self)
  File "/opt/anaconda3/envs/test_huggingface/lib/python3.8/site-packages/pytorch_lightning/accelerators/accelerator.py", line 73, in start_training
    self.training_type_plugin.start_training(trainer)
  File "/opt/anaconda3/envs/test_huggingface/lib/python3.8/site-packages/pytorch_lightning/plugins/training_type/training_type_plugin.py", line 114, in start_training
    self._results = trainer.run_train()
  File "/opt/anaconda3/envs/test_huggingface/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 637, in run_train
    self.train_loop.run_training_epoch()
  File "/opt/anaconda3/envs/test_huggingface/lib/python3.8/site-packages/pytorch_lightning/trainer/training_loop.py", line 493, in run_training_epoch
    batch_output = self.run_training_batch(batch, batch_idx, dataloader_idx)
  File "/opt/anaconda3/envs/test_huggingface/lib/python3.8/site-packages/pytorch_lightning/trainer/training_loop.py", line 655, in run_training_batch
    self.optimizer_step(optimizer, opt_idx, batch_idx, train_step_and_backward_closure)
  File "/opt/anaconda3/envs/test_huggingface/lib/python3.8/site-packages/pytorch_lightning/trainer/training_loop.py", line 426, in optimizer_step
    model_ref.optimizer_step(
  File "/opt/anaconda3/envs/test_huggingface/lib/python3.8/site-packages/pytorch_lightning/core/lightning.py", line 1387, in optimizer_step
    optimizer.step(closure=optimizer_closure)
  File "/opt/anaconda3/envs/test_huggingface/lib/python3.8/site-packages/pytorch_lightning/core/optimizer.py", line 214, in step
    self.__optimizer_step(*args, closure=closure, profiler_name=profiler_name, **kwargs)
  File "/opt/anaconda3/envs/test_huggingface/lib/python3.8/site-packages/pytorch_lightning/core/optimizer.py", line 134, in __optimizer_step
    trainer.accelerator.optimizer_step(optimizer, self._optimizer_idx, lambda_closure=closure, **kwargs)
  File "/opt/anaconda3/envs/test_huggingface/lib/python3.8/site-packages/pytorch_lightning/accelerators/accelerator.py", line 277, in optimizer_step
    self.run_optimizer_step(optimizer, opt_idx, lambda_closure, **kwargs)
  File "/opt/anaconda3/envs/test_huggingface/lib/python3.8/site-packages/pytorch_lightning/accelerators/accelerator.py", line 282, in run_optimizer_step
    self.training_type_plugin.optimizer_step(optimizer, lambda_closure=lambda_closure, **kwargs)
  File "/opt/anaconda3/envs/test_huggingface/lib/python3.8/site-packages/pytorch_lightning/plugins/training_type/training_type_plugin.py", line 163, in optimizer_step
    optimizer.step(closure=lambda_closure, **kwargs)
  File "/opt/anaconda3/envs/test_huggingface/lib/python3.8/site-packages/transformers/optimization.py", line 318, in step
    loss = closure()
  File "/opt/anaconda3/envs/test_huggingface/lib/python3.8/site-packages/pytorch_lightning/trainer/training_loop.py", line 649, in train_step_and_backward_closure
    result = self.training_step_and_backward(
  File "/opt/anaconda3/envs/test_huggingface/lib/python3.8/site-packages/pytorch_lightning/trainer/training_loop.py", line 743, in training_step_and_backward
    result = self.training_step(split_batch, batch_idx, opt_idx, hiddens)
  File "/opt/anaconda3/envs/test_huggingface/lib/python3.8/site-packages/pytorch_lightning/trainer/training_loop.py", line 293, in training_step
    training_step_output = self.trainer.accelerator.training_step(args)
  File "/opt/anaconda3/envs/test_huggingface/lib/python3.8/site-packages/pytorch_lightning/accelerators/accelerator.py", line 156, in training_step
    return self.training_type_plugin.training_step(*args)
  File "/opt/anaconda3/envs/test_huggingface/lib/python3.8/site-packages/pytorch_lightning/plugins/training_type/training_type_plugin.py", line 125, in training_step
    return self.lightning_module.training_step(*args, **kwargs)
  File "test_gpt.py", line 81, in training_step
    loss = self.model(input_ids, labels=labels).loss
  File "/opt/anaconda3/envs/test_huggingface/lib/python3.8/site-packages/torch/nn/modules/module.py", line 727, in _call_impl
    result = self.forward(*input, **kwargs)
  File "/opt/anaconda3/envs/test_huggingface/lib/python3.8/site-packages/transformers/models/gpt2/modeling_gpt2.py", line 904, in forward
    transformer_outputs = self.transformer(
  File "/opt/anaconda3/envs/test_huggingface/lib/python3.8/site-packages/torch/nn/modules/module.py", line 727, in _call_impl
    result = self.forward(*input, **kwargs)
  File "/opt/anaconda3/envs/test_huggingface/lib/python3.8/site-packages/transformers/models/gpt2/modeling_gpt2.py", line 633, in forward
    input_shape = input_ids.size()
AttributeError: 'str' object has no attribute 'size'

Conda Packages:
pytorch                   1.7.0           py3.8_cuda10.2.89_cudnn7.6.5_0    pytorch
pytorch-lightning         1.2.5              pyhd8ed1ab_0    conda-forge
tokenizers                0.10.1                   pypi_0    pypi
transformers              4.4.2                    pypi_0    pypi


title: How to get grads in pytorch after matrix multiplication?

body:
I want to get the product of matrix multiplication in the latent space and optimize the weight matrix by the optimizer. I use different kinds of ways to do that. While, The value of 'pi_' in the below codes never changes. What should I do?

I've tried different functions to get the product, like torch.mm(), torch.matual() and @. The weight matrix 'pi_' never changed.

import torch
from torch.utils.data import DataLoader
from torch.utils.data import TensorDataset
#from torchvision import transforms
from torchvision.datasets import MNIST

def get_mnist(data_dir='./data/mnist/',batch_size=128):
    train=MNIST(root=data_dir,train=True,download=True)
    test=MNIST(root=data_dir,train=False,download=True)

    X=torch.cat([train.data.float().view(-1,784)/255.,test.data.float().view(-1,784)/255.],0)
    Y=torch.cat([train.targets,test.targets],0)

    dataset=dict()
    dataset['X']=X
    dataset['Y']=Y

    dataloader=DataLoader(TensorDataset(X,Y),batch_size=batch_size,shuffle=True)

    return dataloader

class tests(torch.nn.Module):
    def __init__(self):
        super(tests, self).__init__()

        self.pi_= torch.nn.Parameter(torch.FloatTensor(10, 1).fill_(1),requires_grad=True)
        self.linear0 = torch.nn.Linear(784,10)
        self.linear1 = torch.nn.Linear(1,784)

    def forward(self, data):
        data = torch.nn.functional.relu(self.linear0(data))
#        data = data.mm(self.pi_)
#        data = torch.mm(data, self.pi_)
#        data = data @ self.pi_
        data = torch.matmul(data, self.pi_)
        data = torch.nn.functional.relu(self.linear1(data))
        return data

if __name__ == '__main__':
    DL=get_mnist()
    t = tests().cuda()
    optimizer = torch.optim.Adam(t.parameters(), lr = 2e-3)

    for i in range(100):
        for inputs, classes in DL:
            inputs = inputs.cuda()

            res = t(inputs)
            loss = torch.nn.functional.mse_loss(res, inputs)

            optimizer.zero_grad()
            loss.backward()
            optimizer.step()

        print("Epoch:", i,"pi:",t.pi_)


title: Freezing all the layers but FCN head in FCN_ResNet101 Pytorch

body:
I want to finetune an FCN_ResNet101. I would like to change the last layer as my dataset has a different number of classes. Also, finetune only the FCN head.
For the former, is it enough to only change the num_classes argument when defining the model or I need to use something like this:

model = torchvision.models.segmentation.fcn_resnet101(pretrained=True)
model.classifier=nn.identity()
model.Conv2d = nn.Conv2d(
in_channels=256,
out_channels=nb_classes,
kernel_size=1,
stride=1
)

I took this piece of code from another thread. I am not sure if it is necessary to use nn.identity(). When I do, the last layer does not change but the last layer of the one to the last FCN!
And, how many layers must be changed so my FCN head is re_trianed?
I wrote it this way but I’m mostly confused about FCN_ResNet101 architecture.

model = torchvision.models.segmentation.fcn_resnet101(pretrained=True, progress=True, num_classes=?)
#model.classifier[4] = nn.Identity()

“”"
FCNHead(
(0): Conv2d(2048, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
(1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(2): ReLU()
(3): Dropout(p=0.1)
(4): Conv2d(512, 21, kernel_size=(1, 1), stride=(1, 1))
), FCNHead(
(0): Conv2d(1024, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
(1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(2): ReLU()
(3): Dropout(p=0.1)
(4): Conv2d(256, 21, kernel_size=(1, 1), stride=(1, 1))
)]
“”"

#setting our own number of classes

layer_list = list(model.children())[-5:]
model_small = nn.Sequential(*list(model.children()))[-5:]

for param in model_small.parameters():
param.requires_grad = False

model_small.Conv2d = nn.Conv2d( in_channels=1024,kernel_size=(3,3),stride=(1,1))
model_small.BatchNorm2d = nn.BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
model_small.ReLU = nn.ReLU()
model_small.Dropout = nn.Dropout(p=0.1)
model_small.Conv2d = nn.Conv2d(
in_channels=256,
out_channels=nb_classes,
kernel_size=1,
stride=1
)
model = model_small.to(device)

Any guidance is very much appreciated!


title: GPU out of memory with enwik8 reformer from huggingface

body:
I'm trying to run the pretrained model google/reformer-enwik8 from huggingface's transformers, but I'm getting CUDA out of memory errors unless I limit the sequences to one-fourth of the model capacity (~16k instead of the 65k).

This happens with a Titan Xp with 12GB RAM; I expected all the tricks of the Reformer to make the model with the original sequence size fit.

The code I'm running:

model = ReformerModelWithLMHead.from_pretrained('google/reformer-enwik8')
model.cuda()

config = model.config
max_len = config.max_position_embeddings
dataset = Enwik8Dataset(
    path, max_len, pad_id=config.pad_token_id,
    eos_id=config.eos_token_id)
loader = DataLoader(dataset, batch_size=1, shuffle=False)

acc_loss = 0
for batch in loader:
    with torch.no_grad():
        batch_loss = model(input_ids=batch, labels=batch)[0]
    acc_loss += batch_loss.mean().item()

acc_loss /= len(dataset)

The Enwik8Dataset inherits from Dataset and does the basic data preprocessing, I can post the code if necessary.


title: RuntimeError: while using pin_memory, num_workers (pytorch DataLoader) in pycharm

body:
When I'm using num_workers and pin_memory parameters in torch.utils.data.dataloader.Dataloader like the following:
train_data = DataLoader(train_ds, batch_size, shuffle = True, , num_workers = 4, pin_memory = True)

Is resulting in the following error:
RuntimeError:
        An attempt has been made to start a new process before the
        current process has finished its bootstrapping phase.

        This probably means that you are not using fork to start your
        child processes and you have forgotten to use the proper idiom
        in the main module:

            if __name__ == '__main__':
                freeze_support()
                ...

        The "freeze_support()" line can be omitted if the program
        is not going to be frozen to produce an executable.


title: pytorch view tensor and reduce one dimension

body:
So I have a 4d tensor with shape [4,1,128,678] and I would like to view/reshape it as [4,678,128].
I have to do this for multiple tensors where the last shape value 678 is not always know and could be different, so [4,1,128,575]should also go to [4,575,128]
Any idea on what is the optimal operation to transform the tensor? view/reshape? and how?
Thanks


title: IndexError: index out of range in self

body:
i got this error in runtime, can you please help me?
my vocab size is 76, as shown in below,
Some of my codes are below:
 class LSTMClassifier(nn.Module):
    def __init__(self, embed_size, hidden_size, vocab_size, num_layers, num_classes, batch_size):
        super(LSTMClassifier, self).__init__()

        self.embed_size = embed_size
        self.hidden_size = hidden_size
        self.batch_size = batch_size
        self.num_layers = num_layers

        self.embedding = nn.Embedding(vocab_size, embed_size) # a lookup table
        self.lstm = nn.LSTM(embed_size, hidden_size, num_layers, dropout=0.3, bidirectional=True)
        self.fc = nn.Sequential(
            nn.Linear(2*hidden_size, 100),
            nn.ReLU(),
            nn.Dropout(p=0.2),
            nn.Linear(100, num_classes)
        )
        self.hidden = self.init_hidden()

    def init_hidden(self):
        h = to_var(torch.zeros((2*self.num_layers, self.batch_size, self.hidden_size)))
        c = to_var(torch.zeros((2*self.num_layers, self.batch_size, self.hidden_size)))
        return h, c
    def forward(self, x):
        x = self.embedding(x)
        x, self.hidden = self.lstm(x, self.hidden)
        x = self.fc(x[-1])  # select the last output
        return x

 # LSTM parameters
embed_size =100
hidden_size = 256
num_layers = 1

# training parameters
lr = 0.001
num_epochs = 10

vocab_size = 2 + len([w for (w, c) in train_ds.vocab.word2count.items() if c >= min_count])
print(vocab_size)

76

model = LSTMClassifier(embed_size=embed_size,
                       hidden_size=hidden_size,
                       vocab_size=vocab_size,
                       num_layers=num_layers,
                       num_classes=train_ds.num_classes,
                       batch_size=batch_size)

if use_gpu:
    model = model.cuda()

hidden_size=hidden_size,

                       vocab_size=vocab_size,
                       num_layers=num_layers,
                       num_classes=train_ds.num_classes,
                       batch_size=batch_size)

if use_gpu:
    model = model.cuda()

criterion = nn.CrossEntropyLoss()
if use_gpu:
    criterion = criterion.cuda()

optimizer = optim.Adam(model.parameters(), lr=lr, betas=(0.7, 0.99))
scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=1, gamma=0.975)

hist = train(model, train_dl, valid_dl, criterion, optimizer, scheduler, num_epochs)

The error
> IndexError                                Traceback (most recent call
> last) <ipython-input-55-7e0f888e140e> in <module>
> ----> 1 hist = train(model, train_dl, valid_dl, criterion, optimizer, scheduler, num_epochs)
>
> ~\train_utils.py in train(model, train_dl, valid_dl, criterion,
> optimizer, scheduler, num_epochs)
>      82
>      83         ## perform one epoch of training and validation
> ---> 84         trn_loss, trn_acc = train_step(model, train_dl, criterion, optimizer, scheduler)
>      85         val_loss, val_acc = validate_step(model, valid_dl, criterion)
>      86
>
> ~\train_utils.py in train_step(model, train_dl, criterion, optimizer,
> scheduler)
>      26         model.hidden = detach(model.hidden)
>      27         model.zero_grad()
> ---> 28         output = model(train_inputs.t())
>      29
>      30         loss = criterion(output, train_labels)
>
> C:\ProgramData\Anaconda3\lib\site-packages\torch\nn\modules\module.py
> in _call_impl(self, *input, **kwargs)
>     720             result = self._slow_forward(*input, **kwargs)
>     721         else:
> --> 722             result = self.forward(*input, **kwargs)
>     723         for hook in itertools.chain(
>     724                 _global_forward_hooks.values(),
>
> <ipython-input-21-360ed93de0e5> in forward(self, x)
>      24
>      25     def forward(self, x):
> ---> 26         x = self.embedding(x)
>      27         x, self.hidden = self.lstm(x, self.hidden)
>      28         x = self.fc(x[-1])  # select the last output
>
> C:\ProgramData\Anaconda3\lib\site-packages\torch\nn\modules\module.py
> in _call_impl(self, *input, **kwargs)
>     720             result = self._slow_forward(*input, **kwargs)
>     721         else:
> --> 722             result = self.forward(*input, **kwargs)
>     723         for hook in itertools.chain(
>     724                 _global_forward_hooks.values(),
>
> C:\ProgramData\Anaconda3\lib\site-packages\torch\nn\modules\sparse.py
> in forward(self, input)
>     122
>     123     def forward(self, input: Tensor) -> Tensor:
> --> 124         return F.embedding(
>     125             input, self.weight, self.padding_idx, self.max_norm,
>     126             self.norm_type, self.scale_grad_by_freq, self.sparse)
>
> C:\ProgramData\Anaconda3\lib\site-packages\torch\nn\functional.py in
> embedding(input, weight, padding_idx, max_norm, norm_type,
> scale_grad_by_freq, sparse)    1812         # remove once script
> supports set_grad_enabled    1813
> _no_grad_embedding_renorm_(weight, input, max_norm, norm_type)
> -> 1814     return torch.embedding(weight, input, padding_idx, scale_grad_by_freq, sparse)    1815     1816
>
> IndexError: index out of range in self


title: Creating a train, test split for data nested in multiple folders

body:
I am preparing my data for training an image recognition model. I currently have one folder (the dataset) that contains multiple folders with the names of the labels and these folders have the images inside them.
I want to somehow split this dataset so that I have two main folders with the same subfolders, but the number of images inside these folders to be according to a preferred train/test split, so for instance 90% of the images in the train dataset and 10% in the test dataset.
I am struggling with finding the best way how to split my data. I have read a suggestion that pytorch torch.utils.Dataset class might be a way to do it but I can't seem to get it working as to preserve the folder hierarchy.


title: How can I do a seq2seq task with PyTorch Transformers if I am not trying to be autoregressive?

body:
I may be mistaken, but it seems that PyTorch Transformers are autoregressive, which is what masking is for. However, I've seen some implementations where people use just the Encoder and output that directly to a Linear layer.
In my case, I'm trying to convert a spectrogram (rows are frequencies and columns are timesteps) to another spectrogram of the same dimensions. I'm having an impossible time trying to figure out how to do this.
For my model, I have:
class TransformerReconstruct(nn.Module):
    def __init__(self, feature_size=250, num_layers=1, dropout=0.1, nhead=10, output_dim=1):
        super(TransformerReconstruct, self).__init__()
        self.model_type = 'Transformer'

        self.src_mask = None
        self.pos_encoder = PositionalEncoding(feature_size)
        self.encoder_layer = nn.TransformerEncoderLayer(d_model=feature_size, nhead=nhead, dropout=dropout)
        self.transformer_encoder = nn.TransformerEncoder(self.encoder_layer, num_layers=num_layers)
        self.decoder = nn.Linear(feature_size, output_dim)
        self.init_weights()

    def init_weights(self):
        initrange = 0.1
        self.decoder.bias.data.zero_()
        self.decoder.weight.data.uniform_(-initrange, initrange)

    def forward(self, src):
        if self.src_mask is None or self.src_mask.size(0) != len(src):
            device = src.device
            mask = self._generate_square_subsequent_mask(len(src)).to(device)
            self.src_mask = mask

        src = self.pos_encoder(src)
        output = self.transformer_encoder(src, self.src_mask)
        output = self.decoder(output)
        return output

    def _generate_square_subsequent_mask(self, sz):
        mask = (torch.triu(torch.ones(sz, sz)) == 1).transpose(0, 1)
        mask = mask.float().masked_fill(mask == 0, float('-inf')).masked_fill(mask == 1, float(0.0))
        return mask

And when training, I have:
model = TransformerReconstruct(feature_size=128, nhead=8, output_dim=128, num_layers=6).to(device)

This returns the right shape, but doesn't seem to learn.
My basic training loop looks like:
for i in range(0, len(data_source) - 1, input_window):
  data, target = get_batch(data_source, i, 1)
  output = recreate_model(data)

and I'm using an MSELoss and I'm trying to learn a very simple identity. Where the input and output are the same, however this is not learning. What could I be doing wrong? Thanks in advance.


title: How to generate the architectural figure of the network in pytorch when the type of the output of the network is list?

body:
I wonder how to use tochviz to generate network architecture, when the output is a list type?
the demo code is as follows:
 import torch
 import torch.nn as nn
 class ConvNet(nn.Module):
     def __init__(self):
         super(ConvNet, self).__init__()
         self.conv1 = nn.Sequential(
             nn.Conv2d(1, 16, 3, 1, 1),
             nn.ReLU(),
             nn.AvgPool2d(2, 2)
         )
         self.conv2 = nn.Sequential(
             nn.Conv2d(16, 32, 3, 1, 1),
             nn.ReLU(),
             nn.MaxPool2d(2, 2)
         )
         self.fc = nn.Sequential(
             nn.Linear(32 * 7 * 7, 128),
             nn.ReLU(),
             nn.Linear(128, 64),
             nn.ReLU()
         )
         self.out = nn.Linear(64, 10)
     def forward(self, x):
         x = self.conv1(x)
         x = self.conv2(x)
         x = x.view(x.size(0), -1)
         x = self.fc(x)
         output = []
         output.append(x)
         output.append(self.out(x))
         return output
 MyConvNet = ConvNet()

and I use torchviz to view this network's architecture like
 from torchviz import make_dot
 x = torch.randn(1, 1, 28, 28).requires_grad_(True)
 y = MyConvNet(x)
 MyConvNetVis = make_dot(y, params=dict(list(MyConvNet.named_parameters()) + [('x', x)]))
 MyConvNetVis.format = "png"
 MyConvNetVis.directory = "data"
 MyConvNetVis.view()

then, I was blocked with this problem
AttributeError                            Traceback (most recent call last)
<ipython-input-23-c8e3cd3a8b4e> in <module>
      2 x = torch.randn(1, 1, 28, 28).requires_grad_(True)
      3 y = MyConvNet(x)
----> 4 MyConvNetVis = make_dot(y, params=dict(list(MyConvNet.named_parameters()) + [('x', x)]))
      5 MyConvNetVis.format = "png"
      6 MyConvNetVis.directory = "data"

~/anaconda3/envs/torch1.3/lib/python3.6/site-packages/torchviz/dot.py in make_dot(var, params)
     35         return '(' + (', ').join(['%d' % v for v in size]) + ')'
     36
---> 37     output_nodes = (var.grad_fn,) if not isinstance(var, tuple) else tuple(v.grad_fn for v in var)
     38
     39     def add_nodes(var):

AttributeError: 'list' object has no attribute 'grad_fn'

Any advice will be appreciated.


title: Unable to make predictions due to incompatible matrix shape/size

body:
I am trying to make a model to predict insurance cost based on the individual. And this is the code for it.
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.utils.data import DataLoader, TensorDataset

import numpy as np
import pandas as pd

from LSR import ListSearchReplace as LSR

csv = pd.read_csv("main.csv")
partialInputs = csv[["age", "bmi", "children"]]
smoker, sex = list(csv["smoker"]), list(csv["sex"])

L1 = LSR(smoker)
L1.replace("yes", 1, True)
L1.replace("no", 0, True)

L2 = LSR(sex)
L2.replace("female", 1, True)
L2.replace("male", 0, True)

pdReadySmoker = pd.DataFrame({"smoker": smoker})
pdReadySex = pd.DataFrame({"sex": sex})

SmokerAndSex = pd.merge(pdReadySmoker, pdReadySex, how="outer", left_index=True, right_index=True)

INPUTS = pd.merge(partialInputs, SmokerAndSex, how="outer", left_index=True, right_index=True)
TARGETS = csv["charges"]

INPUTS = torch.from_numpy(np.array(INPUTS, dtype='float32'))
TARGETS = torch.from_numpy(np.array(TARGETS, dtype='float32'))

print(INPUTS.shape, TARGETS.shape)

loss_fn = F.mse_loss
model = nn.Linear(5, 3) # <-- changing this, changes the error message.
opt = torch.optim.SGD(model.parameters(), lr=1e-5)

trainDataset = TensorDataset(INPUTS, TARGETS)
BATCH_SIZE = 5
trainDataloader = DataLoader(trainDataset, BATCH_SIZE, shuffle=True)

def fit(numEpochs, model, loss_fn, opt, trainDataloader):
    for epochs in range(numEpochs):
        for inputBatch, targetBatch in trainDataloader:
            preds = model(inputBatch)
            loss = loss_fn(preds, targetBatch)
            loss.backward()

            opt.step()

            opt.zero_grad()

        e = epoch + 1
        if e % 10 == 0:
            print(f"Epoch: {e/numEpochs}, loss: {loss.item():.4f}")

fit(100, model, loss_fn, opt, trainDataloader) <-- error

Error produced:
<ipython-input-7-b7028a3d94fd>:5: UserWarning: Using a target size (torch.Size([5])) that is different to the input size (torch.Size([5, 3])). This will likely lead to incorrect results due to broadcasting. Please ensure they have the same size.
  loss = loss_fn(preds, targetBatch)

---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-20-d8f5bcdc847d> in <module>
----> 1 fit(100, model, loss_fn, opt, trainDataloader)

<ipython-input-7-b7028a3d94fd> in fit(numEpochs, model, loss_fn, opt, trainDataloader)
      3         for inputBatch, targetBatch in trainDataloader:
      4             preds = model(inputBatch)
----> 5             loss = loss_fn(preds, targetBatch)
      6             loss.backward()
      7

D:\coding\machine-learning\env-ml\lib\site-packages\torch\nn\functional.py in mse_loss(input, target, size_average, reduce, reduction)
   2657         reduction = _Reduction.legacy_get_string(size_average, reduce)
   2658
-> 2659     expanded_input, expanded_target = torch.broadcast_tensors(input, target)
   2660     return torch._C._nn.mse_loss(expanded_input, expanded_target, _Reduction.get_enum(reduction))
   2661

D:\coding\machine-learning\env-ml\lib\site-packages\torch\functional.py in broadcast_tensors(*tensors)
     69         if any(type(t) is not Tensor for t in tensors) and has_torch_function(tensors):
     70             return handle_torch_function(broadcast_tensors, tensors, *tensors)
---> 71     return _VF.broadcast_tensors(tensors)  # type: ignore
     72
     73

RuntimeError: The size of tensor a (3) must match the size of tensor b (5) at non-singleton dimension 1

I've tried changing the dimensions of the of model, and these are a few of the changes made and the associated errors:
model = nn.Linear(5, 1338)

Error:
RuntimeError: The size of tensor a (1338) must match the size of tensor b (5) at non-singleton dimension 1

model = nn.Linear(1338, 1338)

Error:
RuntimeError: mat1 and mat2 shapes cannot be multiplied (5x5 and 1338x1338)

Sometimes this error, will make me change the matrix to the correct shape, but that results in the previous error regarding non-singleton dimension


title: Having trouble installing PyTorch

body:
I'm having trouble installing PyTorch.
C:\Users\myself>pip install torch
Collecting torch
  Using cached torch-1.7.1-cp38-cp38-win_amd64.whl (184.0 MB)
Requirement already satisfied: numpy in c:\users\myself\appdata\local\packages\pythonsoftwarefoundation.python.3.8_qbz5n2kfra8p0\localcache\local-packages\python38\site-packages (from torch) (1.19.1)
Requirement already satisfied: typing-extensions in c:\users\myself\appdata\local\packages\pythonsoftwarefoundation.python.3.8_qbz5n2kfra8p0\localcache\local-packages\python38\site-packages (from torch) (3.7.4.3)
Installing collected packages: torch
ERROR: Could not install packages due to an EnvironmentError: [Errno 2] No such file or directory: 'C:\\Users\\myself\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.8_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python38\\site-packages\\caffe2\\python\\serialized_test\\data\\operator_test\\learning_rate_adaption_op_test.test_learning_rate_adaption_op_normalization.zip'

WARNING: You are using pip version 20.2.4; however, version 20.3.3 is available.
You should consider upgrading via the 'C:\Users\myself\AppData\Local\Microsoft\WindowsApps\PythonSoftwareFoundation.Python.3.8_qbz5n2kfra8p0\python.exe -m pip install --upgrade pip' command.

It says that I'm missing a file in caffe2, but I'm not sure how I can get that file back.


title: What is the LibTorch equivalent to PyTorch's torch.no_grad?

body:
When testing a network in PyTorch one can use with torch.no_grad():. What is the Libtorch (C++) equivalent?
Thanks!


title: Libtorch works with g++, but fails with Intel compiler

body:
I want to use a neural network developed in Python (PyTorch) in a Fortran program. My OS is Ubuntu 18.04.
What I am doing:

save it as torchscript: TurbNN.pt
call it from c++ program: call_ts.cpp, call_ts.h
call c++ program from Fortran program (using bind©): main.f90

I successfully compiled the codes using CMake (3.19.4) and g++ (7.5.0). However, I cannot compile them using Intel compilers (HPCKit 2021.1.0.2684):
# downloaded torchlib
export Torch_DIR=/home/aiskhak/nn/fortran_calls_torchscript3/build/libtorch/share/cmake/Torch/

# set environment for Intel compilers
. /opt/intel/oneapi/setvars.sh

# cmake
cmake \
-DCMAKE_CXX_COMPILER=icpc \
-DCMAKE_Fortran_COMPILER=ifort ..

# make
make

After “cmake” everything looks fine (just like for g++):
-- The CXX compiler identification is Intel 20.2.1.20201112
-- The Fortran compiler identification is Intel 20.2.1.20201112
-- Detecting CXX compiler ABI info
-- Detecting CXX compiler ABI info - done
-- Check for working CXX compiler: /opt/intel/oneapi/compiler/2021.1.1/linux/bin/intel64/icpc - skipped
-- Detecting CXX compile features
-- Detecting CXX compile features - done
-- Detecting Fortran compiler ABI info
-- Detecting Fortran compiler ABI info - done
-- Check for working Fortran compiler: /opt/intel/oneapi/compiler/2021.1.1/linux/bin/intel64/ifort - skipped
-- Checking whether /opt/intel/oneapi/compiler/2021.1.1/linux/bin/intel64/ifort supports Fortran 90
-- Checking whether /opt/intel/oneapi/compiler/2021.1.1/linux/bin/intel64/ifort supports Fortran 90 - yes
-- Looking for C++ include pthread.h
-- Looking for C++ include pthread.h - found
-- Performing Test CMAKE_HAVE_LIBC_PTHREAD
-- Performing Test CMAKE_HAVE_LIBC_PTHREAD - Failed
-- Looking for pthread_create in pthreads
-- Looking for pthread_create in pthreads - not found
-- Looking for pthread_create in pthread
-- Looking for pthread_create in pthread - found
-- Found Threads: TRUE
-- Found Torch: /home/aiskhak/nn/fortran_calls_torchscript3/build/libtorch/lib/libtorch.so
-- Performing Test COMPILER_HAS_HIDDEN_VISIBILITY
-- Performing Test COMPILER_HAS_HIDDEN_VISIBILITY - Success
-- Performing Test COMPILER_HAS_HIDDEN_INLINE_VISIBILITY
-- Performing Test COMPILER_HAS_HIDDEN_INLINE_VISIBILITY - Success
-- Performing Test COMPILER_HAS_DEPRECATED_ATTR
-- Performing Test COMPILER_HAS_DEPRECATED_ATTR - Success
-- Configuring done
-- Generating done
-- Build files have been written to: /home/aiskhak/nn/fortran_calls_torchscript3/build

However, after “make” I am getting:
Scanning dependencies of target call_ts_cpp
[ 25%] Building CXX object CMakeFiles/call_ts_cpp.dir/src/call_ts.cpp.o
[ 50%] Linking CXX shared library lib/libcall_ts_cpp.so
[ 50%] Built target call_ts_cpp
Scanning dependencies of target fortran_calls_ts.x
[ 75%] Building Fortran object CMakeFiles/fortran_calls_ts.x.dir/src/main.f90.o
[100%] Linking Fortran executable bin/fortran_calls_ts.x
lib/libcall_ts_cpp.so: undefined reference to `c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >)'
lib/libcall_ts_cpp.so: undefined reference to `torch::jit::load(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, c10::optional<c10::Device>, std::unordered_map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::hash<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::equal_to<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > >&)'
lib/libcall_ts_cpp.so: undefined reference to `c10::DeviceTypeName[abi:cxx11](c10::DeviceType, bool)'
lib/libcall_ts_cpp.so: undefined reference to `torch::jit::Object::find_method(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) const'
lib/libcall_ts_cpp.so: undefined reference to `torch::jit::Method::operator()(std::vector<c10::IValue, std::allocator<c10::IValue> >, std::unordered_map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, c10::IValue, std::hash<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::equal_to<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, c10::IValue> > > const&)'
CMakeFiles/fortran_calls_ts.x.dir/build.make:106: recipe for target 'bin/fortran_calls_ts.x' failed
make[2]: *** [bin/fortran_calls_ts.x] Error 1
CMakeFiles/Makefile2:123: recipe for target 'CMakeFiles/fortran_calls_ts.x.dir/all' failed
make[1]: *** [CMakeFiles/fortran_calls_ts.x.dir/all] Error 2
Makefile:148: recipe for target 'all' failed
make: *** [all] Error 2

What I am supposed to do with? I fixed similar problem when I had old g++, but my Intel compiler is new.
Below are my codes:
call_ts.cpp
#include "call_ts.h"
#include <torch/script.h>
#include <iostream>
#include <memory>

// c++ function invariant_nn
//
// takes inputs, reads a neural network TurbNN.pt, do a forward pass
// and returns outputs
//
// inputs: 5 tensor invariants I[0:4] (float)
// outputs: 10 tensor basis coefficients G[0:9] (float)

void invariant_nn(float I[], float G[])
{
    // deserialize scriptmodule from a .pt file
    torch::jit::script::Module module;
    const char *arg;
    arg = "../src/TurbNN.pt";
    module = torch::jit::load(arg);

    // create inputs
    std::vector<torch::jit::IValue> inputs;
    float data[] = {I[0], I[1], I[2], I[3], I[4]};
    inputs.push_back(torch::from_blob(data, {1, 5}));
    //std::cout << "inputs\n" << inputs;
    //std::cout << "\n";

    // do forward pass and turn its output into a tensor
    at::Tensor outputs = module.forward(inputs).toTensor();
    //std::cout << "outputs\n" << outputs;
    //std::cout << "\n";

    // return values
    for (int k = 0; k < 10; k++) {
        G[k] = outputs[0][k].item().to<float>();
        //std::cout << "G\n" << G[k];
    }

    return;
}

call_ts.h
#pragma once

/* export macros for library generated by CMake */
#ifndef CALL_TS_API
#include "call_ts_export.h"
#define CALL_TS_API CALL_TS_EXPORT
#endif

#ifdef __cplusplus
extern "C" {
#endif

CALL_TS_API
void invariant_nn(float I[], float G[]);

#ifdef __cplusplus
}
#endif

main. f90
! fortran program main
!
! calls c++ function invariant_nn, which calls a torchscript with
! a neural network
!

program main

    ! define interface to interact with c++
    use, intrinsic :: iso_c_binding, only: c_float
    implicit none
    interface invariant_nn
        subroutine invariant_nn(I, G) bind (c)
            import :: c_float
            real(c_float) :: I(5)
            real(c_float) :: G(10)
        end subroutine
    end interface

    ! fortran program
    real(4) I(5), G(10)

    ! invariants
    I(1) = 1.01
    I(2) = 1.01
    I(3) = 1.01
    I(4) = 1.01
    I(5) = 1.01
    print *, "Tensor invariants ", I

    ! tensor basis coefficients
    call invariant_nn(I, G)
    print *, "Tensor basis coefficients ", G

end program main

CMakeLists.txt
# stop configuration if cmake version is below 3.0
cmake_minimum_required(VERSION 3.0 FATAL_ERROR)

# project name and enabled languages
project(fortran_calls_ts CXX Fortran)

# find libtorch
find_package(Torch REQUIRED)

# if CMAKE_BUILD_TYPE undefined, set it to Release
if(NOT CMAKE_BUILD_TYPE)
    set(CMAKE_BUILD_TYPE "Release")
endif()

# compiler flags for release mode
set(CMAKE_CXX_FLAGS_RELEASE "-O3")
set(CMAKE_Fortran_FLAGS_RELEASE "-O3")

# set default build paths
set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/bin)
set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/lib)

# generated export header will be placed here
include_directories(${PROJECT_BINARY_DIR})

# c library
add_library(call_ts_cpp SHARED src/call_ts.cpp)

# fortran executable
add_executable(fortran_calls_ts.x src/main.f90)

# linked against c library
target_link_libraries(fortran_calls_ts.x call_ts_cpp)
target_link_libraries(call_ts_cpp "${TORCH_LIBRARIES}")

# we let cmake generate the export header
include(GenerateExportHeader)
generate_export_header(call_ts_cpp BASE_NAME call_ts)

install(TARGETS call_ts_cpp LIBRARY DESTINATION lib ARCHIVE DESTINATION lib)

install(FILES src/call_ts.h ${PROJECT_BINARY_DIR}/call_ts_export.h DESTINATION include)

set_property(TARGET fortran_calls_ts.x PROPERTY CXX_STANDARD 14)


title: I can't adapt my dataset to VGG-net, getting size mismatch

body:
I’m trying to implement the pre-trained VGG net to my script, in order to recognize faces from my dataset in RGB [256,256], but I’m getting a “size mismatch, m1: [1 x 2622], m2: [4096 x 2]” even if i'm resizing my images it doesn't work, as you can see my code work with resnet and alexnet.

I've tryed resizing the images with the function interpolate but the size mismatch persist.

def training(model_conv, learning_rate, wd, net):

criterion = nn.CrossEntropyLoss(weight= torch.FloatTensor([1,1]))
optimizer = torch.optim.Adam(model_conv.fc.parameters(),         lr=learning_rate, weight_decay = wd)
total_step = len(train_loader)
loss_list = []
acc_list = []
print("Inizio il training")

for epoch in range(num_epochs):
    for i, (im, labels) in enumerate(train_loader):

        images = torch.nn.functional.interpolate(im, 224, mode = 'bilinear')
        outputs = model_conv(images)
        loss = criterion(outputs, labels)
        loss_list.append(loss.item())
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()
        if (i + 1) % 100 == 0:
            print('Epoch [{}/{}], Step [{}/{}], Loss: {:.4f}'
                  .format(epoch + 1, num_epochs, i + 1, total_step, loss.item()))
torch.save(model_conv, 'TrainedModel.pt')
return images, labels

def main():

net = "vgg"
learning_rate = 10e-6
wd = 10e-4

if net == "vgg":
    print("Hai selezionato VGG")
    model_conv = VGG_FACE.vgg_face
    data = torch.load("VGG_FACE.pth")
    model_conv.load_state_dict(data)

    model_conv.fc = nn.Linear(4096, 2)
    model_conv[-1] = model_conv.fc

if __name__ == '__main__':
main()

For example this is another code where I used correctly my VGG with some random images

def test():
N=5
net = VGG_FACE.vgg_face
data = torch.load("VGG_FACE.pth")
net.load_state_dict(data)
net.eval()
names = open("names.txt").read().split()

with torch.no_grad():
    mean = np.array([93.5940, 104.7624, 129.1863])
    images = scipy.misc.imread("cooper2.jpg", mode="RGB")
    images = scipy.misc.imresize(images, [224, 224])
    images = images.astype(np.float32)
    images -= mean[np.newaxis, np.newaxis, :]
    images = np.transpose(images, (2, 0, 1))
    images = images[np.newaxis, ...]
    images = torch.tensor(images, dtype=torch.float32)

    y = net(images)
    y = torch.nn.functional.softmax(y, 1)
    rank = torch.topk(y[0, :], N)
    for i in range(N):
        index = rank[1][i].item()
        score = rank[0][i].item()
        print("{}) {} ({:.2f})".format(i + 1, names[index], score))
    print()

numero_classi = 2
net[-1] = torch.nn.Linear(4096, numero_classi)

if __name__ == "__main__":
test()

the error i'm gettin is

  File "/Users/danieleligato/PycharmProjects/parametral/VGGTEST.py", line 53, in training
outputs = model_conv(images)
RuntimeError: size mismatch, m1: [4 x 2622], m2: [4096 x 2] at /Users/soumith/code/builder/wheel/pytorch-src/aten/src/TH/generic/THTensorMath.cpp:2070

THIS IS THE VGG NET THAT I'M USING

class LambdaBase(nn.Sequential):
def __init__(self, fn, *args):
    super(LambdaBase, self).__init__(*args)
    self.lambda_func = fn

def forward_prepare(self, input):
    output = []
    for module in self._modules.values():
        output.append(module(input))
    return output if output else input

class Lambda(LambdaBase):
def forward(self, input):
    return self.lambda_func(self.forward_prepare(input))

class LambdaMap(LambdaBase):
def forward(self, input):
    return map(self.lambda_func,self.forward_prepare(input))

class LambdaReduce(LambdaBase):
def forward(self, input):
    return reduce(self.lambda_func,self.forward_prepare(input))

vgg_face = nn.Sequential( # Sequential,
nn.Conv2d(3,64,(3, 3),(1, 1),(1, 1)),
nn.ReLU(),
nn.Conv2d(64,64,(3, 3),(1, 1),(1, 1)),
nn.ReLU(),
nn.MaxPool2d((2, 2),(2, 2),(0, 0),ceil_mode=True),
nn.Conv2d(64,128,(3, 3),(1, 1),(1, 1)),
nn.ReLU(),
nn.Conv2d(128,128,(3, 3),(1, 1),(1, 1)),
nn.ReLU(),
nn.MaxPool2d((2, 2),(2, 2),(0, 0),ceil_mode=True),
nn.Conv2d(128,256,(3, 3),(1, 1),(1, 1)),
nn.ReLU(),
nn.Conv2d(256,256,(3, 3),(1, 1),(1, 1)),
nn.ReLU(),
nn.Conv2d(256,256,(3, 3),(1, 1),(1, 1)),
nn.ReLU(),
nn.MaxPool2d((2, 2),(2, 2),(0, 0),ceil_mode=True),
nn.Conv2d(256,512,(3, 3),(1, 1),(1, 1)),
nn.ReLU(),
nn.Conv2d(512,512,(3, 3),(1, 1),(1, 1)),
nn.ReLU(),
nn.Conv2d(512,512,(3, 3),(1, 1),(1, 1)),
nn.ReLU(),
nn.MaxPool2d((2, 2),(2, 2),(0, 0),ceil_mode=True),
nn.Conv2d(512,512,(3, 3),(1, 1),(1, 1)),
nn.ReLU(),
nn.Conv2d(512,512,(3, 3),(1, 1),(1, 1)),
nn.ReLU(),
nn.Conv2d(512,512,(3, 3),(1, 1),(1, 1)),
nn.ReLU(),
nn.MaxPool2d((2, 2),(2, 2),(0, 0),ceil_mode=True),
Lambda(lambda x: x.view(x.size(0),-1)), # View,
nn.Sequential(Lambda(lambda x: x.view(1,-1) if 1==len(x.size()) else x ),nn.Linear(25088,4096)), # Linear,
nn.ReLU(),
nn.Dropout(0.5),
nn.Sequential(Lambda(lambda x: x.view(1,-1) if 1==len(x.size()) else x ),nn.Linear(4096,4096)), # Linear,
nn.ReLU(),
nn.Dropout(0.5),
nn.Sequential(Lambda(lambda x: x.view(1,-1) if 1==len(x.size()) else x ),nn.Linear(4096,2622)), # Linear,
)


title: Poor fits for simple 2D Gaussian processes in `GPyTorch`

body:
I'm having a lot of difficulty fitting a simple 2-dimensional GP out-of-the box using GPyTorch. As you can see below, the fit is very poor, and does not improve much with either swapping out the RBF kernel for something like a Matern. The optimization does appear to converge, but not on anything sensible.

class GPRegressionModel(gpytorch.models.ExactGP):
    def __init__(self, train_x, train_y, likelihood):
        super(GPRegressionModel, self).__init__(train_x, train_y, likelihood)

        self.mean_module = gpytorch.means.ConstantMean()
        self.covar_module = gpytorch.kernels.ScaleKernel(
                gpytorch.kernels.RBFKernel(ard_num_dims=2),
            )

    def forward(self, x):
        mean_x = self.mean_module(x)
        covar_x = self.covar_module(x)
        return gpytorch.distributions.MultivariateNormal(mean_x, covar_x)

Does anyone have good tutorial examples beyond the ones included in the docs?


title: Multiprocessing chunks of a python list of sentences

body:
I'm trying to do some operations on a list of sentences using python Pool() to use all cores but it keeps running for a long time without any response.

here is the code structure I'm using :

import os
from multiprocessing import Pool
from flair.embeddings import WordEmbeddings
from flair.data import Sentence

def procces_sents(chunks):

    sentences = [Sentence(i) for i in chunks]
    # embed a sentence using glove.
    glove_embedding.embed(sentence)

    for sent in sentences:
        for token in sent:
            if token in my_dict:
                pass
            else:
                pass

# init embedding
glove_embedding = WordEmbeddings('glove')

my_dict={}
my_list["1.txt"]
n=32
filepath = "/some/path/"
for filename in my_list:
    with open(os.path.join(filepath,filename) ,"r") as m:

    all=[]
    data = m.readlines()
    all = [x.strip() for x in all_data]

    all_chunks = [all[i * n:(i + 1) * n] for i in range((len(all) + n - 1) // n )]

    p=Pool()
    p.map(procces_sents, all_chunks)
    p.close()
    p.join()

I expect that each core takes a 32 sized chunk of sentences and process them.

all is a list of all sentences

all_chunks is chunks of all list (each containing 32 sentences)
P.S. I'm using flair framework (https://github.com/zalandoresearch/flair)
Does anybody know what's wrong with my code?


title: Is torch.nn.LSTM equal to tf.nn.rnn_cell.BasicLSTMCell?

body:
One paper has released a tensorflow code for the classification task. It relies on tf.nn.rnn_cell.BasicLSTMCell with default setting.

I try to use PyTorch 1.0 to re-implement, and I use torch.nn.LSTM with default setting. But the testing accuracy is lower than the tensorflow codes.

I check the usage of these two lstm, and I think the default settings are the same. So I am confused...


title: Images change color after unfolding using pytorch

body:
I am new to pytorch, for one of my projects I am dividing a large image into smaller tiles/patches. I am using unfold to make this happen. My code is as follows

data = training_set[1][0].data.unfold(1, 64, 64).unfold(2, 64, 64).unfold(3, 64, 64)

After doing this I transpose the resultant matrix since the images are flipped, like this sample code

torch.t(data [0][0][0][0])

but the resultant images lose color, or get discolored for some reason, and I am worried that this might affect any calculations I do based on these patches.

The following is a screenshot of the problem

The top is the patch and the bottom one is the complete picture

Any help is appreciated, thanks


title: Getting F-Score of 0 when combining RoBERTa and BiLSTM

body:
I am trying to stack an LSTM on top of RoBERTa model for binary classification problem
I've tried to configurations:
    - Freeze RoBERTa embedding
    - Fine-tune embedding

In Freezing case I get around 57% F-score , this is relatively low compared to regular RoBERTa for sequence classification which got the same data around 81%

In fine-tune case I get 0% F-score and validation loss isn't converging

Most probably I am doing something wrong but I can't really spot it.

Here is the model part

class RoBERTaLSTMClassifier(nn.Module):
    def __init__(self, bert_config, num_classes, hidden_size=None, dropout=0.5):
        """
        bert: pretrained bert model
        num_classes: the number of num_classes
        hidden_size: the number of hiddens which will be used by LSTM layer
        dropout: dropout rate
        """
        super(RoBERTaLSTMClassifier, self).__init__()
        self.num_classes = num_classes
        self.model = RobertaModel(bert_config)
        if hidden_size is None: self.hidden_size = bert_config.hidden_size
        else: self.hidden_size = hidden_size
        self.lstm = nn.LSTM(bert_config.hidden_size, self.hidden_size, bidirectional=True,batch_first=True)
        self.dropout = nn.Dropout(dropout)
        self.classifier = nn.Linear(self.hidden_size * 2, 1)
        self.softmax = nn.Softmax()
        ## add sigmoid non linearity for binary classification
        self.sig           = nn.Sigmoid()

    def forward(self, input_ids, attention_mask, current_batch_size, hidden):
        """
        all_layers: whether or not to return all encoded_layers
        return: logits in the following format (batch_size, num_classes)
        """
        with torch.no_grad():
             ## freeze embedding from BERT
             outputs = self.model(input_ids=input_ids, attention_mask=attention_mask)
       # last hidden state is input to the LSTM
        output, (hidden_h, hidden_c) = self.lstm(outputs[0], hidden)

        output_hidden = torch.cat((hidden_h[0], hidden_h[1]), dim=1) #[B, H*2]

        logits = self.classifier(self.dropout(output_hidden)) #[B, C]
        sig_out = self.sig(logits).view(current_batch_size, -1)

        ## get the last batch output
        sig_out = sig_out[:, -1] # get last batch of labels
        hidden = (hidden_h, hidden_c)

        return sig_out, hidden

    def init_bilstm_hidden(self, batch_size):
        h0 = torch.zeros(2, batch_size, self.hidden_size).to(device) # 2 for bidirection
        c0 = torch.zeros(2, batch_size, self.hidden_size).to(device)

        return (h0, c0)

** Here is the Training Loop Part**

from sklearn.metrics import f1_score
from tqdm import tqdm, trange
import numpy as np
lr=0.0001
roberta_conf = RobertaConfig.from_pretrained('roberta-base')

num_classes = 2
hidden_size = 256

LSTMRoBERTaModel =  RoBERTaLSTMClassifier(roberta_conf, num_classes=num_classes,hidden_size= hidden_size,dropout=0.5)
criterion = nn.BCELoss() ## binary cross entropy
optimizer = torch.optim.Adam(LSTMRoBERTaModel.parameters(), lr=lr)

epochs = 5
counter = 0
max_grad_norm = 1.0

nb_tr_examples, nb_tr_steps = 0, 0
for _ in trange(epochs, desc="Epoch"):
    LSTMRoBERTaModel.cuda()
    LSTMRoBERTaModel.train()
    tr_loss = 0
    y_preds = []
    y_true = []
    hidden_init = LSTMRoBERTaModel.init_bilstm_hidden(batch_size=bs)
    h = hidden_init
    for step, batch in enumerate(train_dataloader):
        batch = tuple(t.to(device) for t in batch)
        b_input_ids, b_input_mask, b_labels = batch
        current_batch_size = b_input_ids.size()[0]
        ##

        h = tuple([each.data for each in h])

        ## forward pass
        preds, h = LSTMRoBERTaModel.forward(b_input_ids, b_input_mask, current_batch_size,h)

        loss = criterion(preds.squeeze(),b_labels.float())

        # track train loss
        tr_loss += loss.item()
        nb_tr_examples += b_input_ids.size(0)
        nb_tr_steps += 1

        # gradient clipping
        torch.nn.utils.clip_grad_norm_(parameters=LSTMRoBERTaModel.parameters(), max_norm=max_grad_norm)

        loss.backward()
        optimizer.step()
        LSTMRoBERTaModel.zero_grad()
    # print train loss per epoch
    print("\nTrain loss: {}".format(tr_loss/nb_tr_steps))

    LSTMRoBERTaModel.eval()
    eval_loss, eval_accuracy = 0, 0
    nb_eval_steps, nb_eval_examples = 0, 0

    val_h = LSTMRoBERTaModel.init_bilstm_hidden(bs)

    for batch in dev_dataloader:
        batch = tuple(t.to(device) for t in batch)
        b_input_ids, b_input_mask, b_labels = batch
        current_batch_size = b_input_ids.size()[0]
        with torch.no_grad():

             preds, val_h = LSTMRoBERTaModel.forward(b_input_ids, b_input_mask, current_batch_size, val_h)
             loss = criterion(preds.squeeze(),b_labels.float())

        eval_loss += loss
        y_preds.extend(np.round(preds.data.cpu()))
        y_true.extend(b_labels.data.cpu())
        #print(preds[2], b_labels[2] )
        #eval_accuracy += f1_score(torch.tensor.numpy(b_labels.float), toch.tensor.numpy(preds))
        nb_eval_examples += b_input_ids.size(0)
        nb_eval_steps += 1
    eval_loss = eval_loss/nb_eval_steps
    print("Validation loss: {}".format(eval_loss))
    print("F1 - Score: {}".format(f1_score(y_true,y_preds)))
    #print("F1- Score: {}".format(eval_accuracy/nb_eval_steps))


title: Loading pretrained FastAI models in Kaggle kernels without using internet

body:
I am trying to load a densenet121 model in Kaggle kernel without switching on the internet.
I have done the required steps such as adding the pre-trained weights to my input directory and moving it to '.cache/torch/checkpoints/'. It still would not work and throws a gaierror.

The following the is code SNIPPET:

!mkdir -p /tmp/.cache/torch/checkpoints
!cp ../input/fastai-pretrained-models/densenet121-a639ec97.pth /tmp/.cache/torch/checkpoints/densenet121-a639ec97.pth

learn_cd = create_cnn(data_cd, models.densenet121, metrics=[error_rate, accuracy],model_dir = Path('../kaggle/working/models'),path=Path('.'),).to_fp16()

I have been struggling with this for a long time. Any help would be immensely helpful


title: How can I trim a tensor based on a mask with PyTorch?

body:
I have a tensor inp, which has a size of: torch.Size([4, 122, 161]).

I also have a mask with a size of: torch.Size([4, 122]).

Each element in my mask looks something like:

tensor([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
        0., 0., 0., 0., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,
        1., 1., 1., 1., 1., 1., 1., 1., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],
       device='cuda:0', grad_fn=<SelectBackward>)

So I want to trim inp to be reduced along the dimension=1 to only exist where the mask has 1. In the case shown, there are 23 1s, so I want the size of inp to be: torch.Size([4, 23, 161])


title: AWS EC2 Deep Learning instance cuda 3.0

body:
I just launched (and paid for) the Deep Learning AMI (Ubuntu 18.04) Version 27.0 (ami-0dbb717f493016a1a) instance type g2.2xlarge.  I activated

for PyTorch with Python3 (CUDA 10.1 and Intel MKL) ____________source activate pytorch_p36

When I run my pytorch network I see a warning

/home/ubuntu/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages/torch/cuda/__init__.py:134: UserWarning:
    Found GPU0 GRID K520 which is of cuda capability 3.0.
    PyTorch no longer supports this GPU because it is too old.
    The minimum cuda capability that we support is 3.5.

Is this real?

This is my code to put my neural net on the gpu

if torch.cuda.is_available():
        device = torch.device("cuda:0")  # you can continue going on here, like cuda:1 cuda:2....etc.
        print("Running on the GPU")
    else:
        device = torch.device("cpu")
        print("Running on the CPU")

    net = Net(image_height, image_width)
    net.to(device)


title: How to properly train multiple models simultenously pytorch

body:
I am trying to implement the actor-critic algorithm in pytorch. Hence I want to have two models and train them separately, namely:

actor = Net1()
critic = Net2()

My code looks like this:

optim_a = optim.SGD(actor.paramaters(), lr=.1, momentum=.9)
optim_c = optim.SGD(critic.paramaters(), lr=.1, momentum=.9)
for t in range(hours):
    optim_a.zero_grad()
    optim_c.zero_grad()
    allocation = actor(solar[t], B[t], load_left)
    load = load.detach()
    value = critic(solar[t], B[t], load_left)
    environment_instance = environment(solar[t], B[t], load, G1[t], G2[t])
    R[t], C[t], B[t+1] = environment_instance.step()
    load_left = load_left - allocation
    value_f = critic(solar[t+1], B[t+1], load_left)
    delta = R[t]+gamma*value_f-value
    delta = delta.detach()
    loss_critic = delta*value
    loss_actor = delta*torch.log(allocation)
    loss = loss_actor+loss_critic
    loss.backward()
    optim_a.step()
    optim_c.step()

Let me explain briefly the background (if it helps). I am working on load_assigment for a power system that uses solar energy. There is a fixed load and variable load. the environment is a class that runs the environment, that is, I give it some parameters and it outputs the reward (R). I use that to define delta. Then, I define both losses, where delta acts as a constant (no gradient should be propagated, that is why I detach it)

That code runs but the optimizer is not updating the weights. Strangely enough, sometimes they do update either the critic or actor (I have no idea why) but never both. I think this is odd because I've seen other tutorials where things like this are done, e.g. GAN tutorial but it is not working for me.

I'd appreciate it if anyone can give me some guidance.


title: Extracting labels after applying softmax

body:
I have a multi class classification neural network. I apply softmax at the end to get probabilities for my classes. However, now I want to pick the maximum probability and get the corresponding label for it. I am able to extract the maximum probability but I'm confused how to get the label based on that. This is what I have:

labels = {'id1':0,'id2':2,'id3':1,'id4':3} ### labels

x_t = F.softmax(z,dim=-1)
#print(x_t)
y = torch.argmax(x_t, dim=1)
print(y.detach())

Can someone tell me how to get labels now. For example

y = tensor([3, 1, 3])

final_label = [id4,id3,id4]


title: Is there a way to get the total Flops in an inference pass for a given pytorch or tensorflow model?

body:
I have been studying the Nvidia WaveGlow model that's used by Nvidia in text-to-speech synthesis, but couldn't find any estimate of # of Flops in the inference for this model.

One can easily print out the model in pytorch using print(waveglow) and count the parameters. Similarly for Tensorflow one can do tf.trainable_variables() to count model-parameters.
But I couldn't see a way to at least get ball-park estimate of # of operations.

Is there a way to compute approximately total operations/Flops for a model for a given datatype, say fp32? For both Pytorch and Tensorflow?


title: Pytorch gridsample - Different Resolutions?

body:
Does Pytorch gridsample work for this particular case.
I have an image of size [B, 100, 200] that I want to map into a smaller [B, 50, 60] space. I have the pixel 1-1 mappings stored in a [B, 100, 200, 2] tensor where most of it i guess is 0?
Is this actually possible?


title: Equivalent of embeddings_regularizer in pyTorch

body: tf.keras.layers.Embedding has parameter embeddings_regularizer. What would be equivalent of this in pyTorch or nn.Embedding?


title: typeError: __init__() got an unexpected keyword argument 'text_len'

body:
I am trying to run tacotron model in colab...
def prepare_dataloaders(hparams):
    # Get data, data loaders and collate function ready

    trainset = TextMelLoader(hparams.training_files, hparams)
    valset = TextMelLoader(hparams.validation_files, hparams)
    collate_fn = TextMelCollate(1, text_len = 190, mel_len = 890)

    train_loader = DataLoader(trainset,
                          num_workers=2,
                          sampler=HkSampler(trainset),
                          batch_size=hparams.batch_size,
                          pin_memory=False,
                          drop_last=True,
                          collate_fn=collate_fn)
    return train_loader, valset, collate_fn

class TextMelCollate():
    """ Zero-pads model inputs and targets based on number of frames per setep
    """
    def __init__(self, n_frames_per_step, text_len=20, mel_len=100):
        # we only support one frame per step
        print('we are in TextMelCollage ==> ', text_len)
        assert (n_frames_per_step == 1)
        self.n_frames_per_step = n_frames_per_step
        print(text_len)
        self.text_len = text_len
        self.mel_len = mel_len

    def __call__(self, batch):
        """Collate's training batch from normalized text and mel-spectrogram
        PARAMS
        ------
        batch: [text_normalized, mel_normalized]
        """
        # Right zero-pad all one-hot text sequences to max input length
        text_len = max([len(x[0]) for x in batch]) // 5 * 5 + 5
        self.text_len = max(self.text_len, text_len)

        text_padded = torch.LongTensor(len(batch), self.text_len)
        text_padded.zero_()
        input_lengths = torch.LongTensor([len(x[0]) for x in batch])
        for i in range(len(batch)):
            text = batch[i][0]
            text_padded[i, :text.size(0)] = text

        # Right zero-pad mel-spec
        num_mels = batch[0][1].size(0)
        mel_len = max([x[1].size(1) for x in batch]) // 5 * 5 + 5
        self.mel_len = max(self.mel_len, mel_len)

        # include mel padded and gate padded
        mel_padded = torch.FloatTensor(len(batch), num_mels, self.mel_len)
        mel_padded.zero_()
        gate_padded = torch.FloatTensor(len(batch), self.mel_len)
        gate_padded.zero_()
        output_lengths = torch.LongTensor(len(batch))
        for i in range(len(batch)):
            mel = batch[i][1]
            mel_padded[i, :, :mel.size(1)] = mel
            gate_padded[i, mel.size(1) - 1:] = 1
            output_lengths[i] = mel.size(1)

        return text_padded, input_lengths, mel_padded, gate_padded, \
            output_lengths

but I got error like below
in init of TextMelCollate, text_len is int type
I don't catch up why type error happen.
you can run this code in colab
https://colab.research.google.com/drive/1XU4SWsybhuUYqooyLcFHXmyX3pOOWJON#scrollTo=UxGBm1FdJhlW&uniqifier=2

%tensorflow_versiononly switches the major version: 1.x or 2.x.<br> You set:1.15. This will be interpreted as: 1.x`.
TensorFlow 1.x selected.
tpu backend at grpc://10.92.95.162:8470
grpc://10.92.95.162:8470
Parsing command line hparams: batch_size=128
Traceback (most recent call last):
File "hk_train.py", line 220, in
args.warm_start, hparams)
File "hk_train.py", line 116, in train
train_loader, valset, collate_fn = prepare_dataloaders(hparams)
File "hk_train.py", line 56, in prepare_dataloaders
collate_fn = TextMelCollate(1, text_len = 190, mel_len = 890)

TypeError: init() got an unexpected keyword argument 'text_len'

enter image description here


title: Pytorch Model Summary

body:
I am trying to load a pytorch model using:
model = torch.load('/content/gdrive/model.pth.tar', map_location='cpu')

I want to check the model summary. When I try:
print(model)

I get the following output:
{'state_dict': {'model.conv1.weight': tensor([[[[ 2.0076e-02,  1.5264e-02, -1.2309e-02,  ..., -4.0222e-02,
           -4.0527e-02, -6.4458e-02],
          [ 6.3291e-03,  3.8393e-03,  1.2400e-02,  ..., -3.3926e-03,
           -2.1063e-02, -3.4743e-02],
          [ 1.9969e-02,  2.0064e-02,  1.4004e-02,  ...,  8.7359e-02,
            5.4801e-02,  4.8791e-02],
          ...,
          [ 2.5362e-02,  1.1433e-02, -7.6776e-02,  ..., -3.4798e-01,
           -2.7198e-01, -1.2066e-01],
          [ 8.0373e-02,  1.3095e-01,  1.4240e-01,  ..., -2.2933e-03,
           -1.0469e-01, -1.0922e-01],
          [-1.1147e-03,  7.4572e-02,  1.2814e-01,  ...,  1.6903e-01,
            1.0619e-01,  2.4744e-02]],
      'model.layer4.1.bn2.running_var': tensor([0.0271, 0.0155, 0.0199, 0.0198, 0.0132, 0.0148, 0.0182, 0.0170, 0.0134,
.
.
.

What does it even mean?
I also tried to use:
from torchsummary import summary
summary(model, input_size=(3, 224, 224))

But it gives me the following error:
---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
<ipython-input-31-ca828d30bd38> in <module>()
      1 from torchsummary import summary
----> 2 summary(model, input_size=(3, 224, 224))

/usr/local/lib/python3.7/dist-packages/torchsummary/torchsummary.py in summary(model, input_size, batch_size, device)
     66
     67     # register hook
---> 68     model.apply(register_hook)
     69
     70     # make a forward pass

AttributeError: 'dict' object has no attribute 'apply'

Please note that model is a custom model that I am trying to load.
How can I get a model summary in Pytorch?


title: PyTorch: RuntimeError: The size of tensor a (224) must match the size of tensor b (244) at non-singleton dimension 3

body:
I want to create and train AutoEncoder to extract features and use that features for the clustering algorithms. Right now I am getting errors while calculating the loss.
RuntimeError: The size of tensor a (224) must match the size of tensor b (244) at non-singleton dimension 3
and a warning
UserWarning: Using a target size (torch.Size([1, 3, 224, 244])) that is different to the input size (torch.Size([1, 3, 224, 224])). This will likely lead to incorrect results due to broadcasting. Please ensure they have the same size.

return F.mse_loss(input, target, reduction=self.reduction)*

I am using Pytorch.
Can anyone tell me what is wrong with this? In warning and error size of input and output is the same but it is saying it is different.
The summary sizes of input and output images are as follow
----------------------------------------------------------------
        Layer (type)               Output Shape         Param #
================================================================
            Conv2d-1         [-1, 16, 112, 112]             448
              ReLU-2         [-1, 16, 112, 112]               0
            Conv2d-3           [-1, 32, 56, 56]           4,640
              ReLU-4           [-1, 32, 56, 56]               0
            Conv2d-5           [-1, 64, 18, 18]         100,416
              ReLU-6           [-1, 64, 18, 18]               0
            Conv2d-7            [-1, 128, 3, 3]         401,536
              ReLU-8            [-1, 128, 3, 3]               0
            Conv2d-9            [-1, 256, 1, 1]         295,168
  ConvTranspose2d-10            [-1, 128, 3, 3]         295,040
             ReLU-11            [-1, 128, 3, 3]               0
  ConvTranspose2d-12           [-1, 64, 12, 12]         401,472
             ReLU-13           [-1, 64, 12, 12]               0
  ConvTranspose2d-14           [-1, 24, 28, 28]          75,288
             ReLU-15           [-1, 24, 28, 28]               0
  ConvTranspose2d-16           [-1, 16, 56, 56]           3,472
             ReLU-17           [-1, 16, 56, 56]               0
  ConvTranspose2d-18          [-1, 8, 111, 111]           1,160
             ReLU-19          [-1, 8, 111, 111]               0
  ConvTranspose2d-20          [-1, 3, 224, 224]             603
          Sigmoid-21          [-1, 3, 224, 224]               0
================================================================
Total params: 1,579,243
Trainable params: 1,579,243
Non-trainable params: 0
----------------------------------------------------------------
Input size (MB): 0.57
Forward/backward pass size (MB): 9.94
Params size (MB): 6.02
Estimated Total Size (MB): 16.54
----------------------------------------------------------------
Min Value of input Image =  tensor(0.0627)
Max Value of input Image =  tensor(0.5098)
Input Image shape =  torch.Size([1, 3, 224, 244])
Output Image shape =  torch.Size([1, 3, 224, 224])

My Autoencoder class is
class autoencoder(nn.Module):
    def __init__(self):
        super(autoencoder, self).__init__()
        self.encoder = nn.Sequential(

            nn.Conv2d(3, 16, 3, stride=2, padding=1),  # b, 16, 10, 10
            nn.ReLU(True),
            nn.Conv2d(16, 32, 3, stride=2, padding=1),  # b, 16, 10, 10
            nn.ReLU(True),
            nn.Conv2d(32, 64, 7, stride=3, padding=1),  # b, 16, 10, 10
            nn.ReLU(True),
            nn.Conv2d(64, 128, 7, stride=5, padding=1),  # b, 16, 10, 10
            nn.ReLU(True),
            nn.Conv2d(128, 256, 3, stride=5, padding=1)  # b, 16, 10, 10

        )
        self.decoder = nn.Sequential(
            nn.ConvTranspose2d(256, 128, 3),  # b, 16, 5, 5
            nn.ReLU(True),
            nn.ConvTranspose2d(128, 64, 7,stride=3, padding=1,output_padding=1),  # b, 16, 5, 5
            nn.ReLU(True),
            nn.ConvTranspose2d(64, 24, 7,stride=2, padding=1,output_padding=1),  # b, 16, 5, 5
            nn.ReLU(True),
            nn.ConvTranspose2d(24, 16, 3, stride=2, padding=1,output_padding=1),  # b, 8, 15, 15
            nn.ReLU(True),
            nn.ConvTranspose2d(16, 8, 3, stride=2, padding=1),  # b, 1, 28, 28
            nn.ReLU(True),
            nn.ConvTranspose2d(8,3, 5, stride=2, padding=1,output_padding=1),  # b, 1, 28, 28
            nn.Sigmoid()

        )

    def forward(self, x):
        x = self.encoder(x)
        x = self.decoder(x)
        return x

and training function is as follow
dataset = DatasetLoader('E:/DAL/Dataset/Images', get_transform(train=True))

torch.manual_seed(1)
indices = torch.randperm(len(dataset)).tolist()
dataset = torch.utils.data.Subset(dataset, indices[:-50])
dataset_test = torch.utils.data.Subset(dataset, indices[-50:])

data_loader = torch.utils.data.DataLoader(
    dataset, batch_size=1, shuffle=True, num_workers=0)

data_loader_test = torch.utils.data.DataLoader(
    dataset_test, batch_size=1, shuffle=False, num_workers=0)

model = autoencoder().cuda()
summary(model, (3, 224, 224))

criterion = nn.MSELoss()
optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate,weight_decay=1e-5)

total_loss = 0
for epoch in range(num_epochs):
    for data in data_loader:
        # print(data)
        img = data
        print("Min Value of input Image = ",torch.min(img))
        print("Max Value of input Image = ",torch.max(img))
        img = Variable(img).cuda()
        # ===================forward=====================
        output = model(img)
        print("Input Image shape = ",img.shape)
        print("Output Image shape = ",output.shape)
        loss = criterion(output, img)
        # ===================backward====================
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()
    # ===================log========================
    total_loss += loss.data
    print('epoch [{}/{}], loss:{:.4f}'
          .format(epoch+1, num_epochs, total_loss))
    if epoch % 10 == 0:
        pic = to_img(output.cpu().data)
        save_image(pic, './dc_img/image_{}.png'.format(epoch))

torch.save(model.state_dict(), './conv_autoencoder.pth')

Dataloader Class and transform function is as follow
def get_transform(train):
    transforms = []

    transforms.append(T.Resize((224,244)))
    if train:
        transforms.append(T.RandomHorizontalFlip(0.5))
        transforms.append(T.RandomVerticalFlip(0.5))
    transforms.append(T.ToTensor())
    return T.Compose(transforms)

class DatasetLoader(torch.utils.data.Dataset):
    def __init__(self, root, transforms=None):
        self.root = root
        self.transforms = transforms
        self.imgs = list(sorted(os.listdir(root)))

    def __getitem__(self, idx):
        img_path = os.path.join(self.root, self.imgs[idx])

        img = Image.open(img_path).convert("RGB")

        if self.transforms is not None:
            img = self.transforms(img)

        return img

    def __len__(self):
        return len(self.imgs)


title: torch: what are the differences between using net:cuda() and using cudnn.somemodules?

body:
I'm wondering what the differences are between:

using net:cuda() after making a neural net:

local net = nn.Sequential()
net:add(nn.SpatialConvolution(3, 8, 5, 5))
net:add(nn.View(8*20*20))
net:add(nn.Linear(8*20*20, 10))
net:add(nn.LogSoftMax())
net:cuda()

using cudnn.somemodules on the way completing a neural net:

local net = nn.Sequential()
net:add(cudnn.SpatialConvolution(3, 8, 5, 5))
net:add(cudnn.View(8*20*20))
net:add(cudnn.Linear(8*20*20, 10))
net:add(cudnn.LogSoftMax())


title: Does pytorch do eager pruning of its computational graph?

body:
This is a very simple example:

import torch

x = torch.tensor([1., 2., 3., 4., 5.], requires_grad=True)
y = torch.tensor([2., 2., 2., 2., 2.], requires_grad=True)
z = torch.tensor([1., 1., 0., 0., 0.], requires_grad=True)

s = torch.sum(x * y * z)
s.backward()

print(x.grad)

This will print,

tensor([2., 2., 0., 0., 0.]),

since, of course, ds/dx is zero for the entries where z is zero.

My question is: Is pytorch smart and stop the computations when it reaches a zero? Or does in fact do the calculation "2*5", only to later do "10 * 0 = 0"?

In this simple example it doesn't make a big difference, but in the (bigger) problem I am looking at, this will make a difference.

Thank you for any input.


title: How can I implement these bash commands in Google Colab

body:
I'm a beginner who is working on Neural Machine Translation, the transformer model. I want to implement fairseq Scaling Neural Machine Translation using Google Colab. I guess the commands shown in the README file is written in bash. I know that bash commands can be run in Google Colab by prefixing the command with !. Following commands are from the Github repository mentioned above.

TEXT=wmt16_en_de_bpe32k
mkdir -p $TEXT
tar -xzvf wmt16_en_de.tar.gz -C $TEXT

These commands throw errors when I add the ! as follows.


title: GPT2 input size with wiki articles

body: I'm trying to use a small pre-trained GPT2 model on wikipedia text. I try to use as much text as I can as the input for the gpt2 model. The model summarizes the text for me. How do I use the whole wiki article for input. As it is now I'm limited to something like 768 tokens. A typical wiki article is longer than that. Is there a trick to using text passages longer than 768 tokens?


title: how to visualize the Tensorflow model of Text-to-speech synthesis https://github.com/Rayhane-mamah/Tacotron-2

body:
I have been trying to visualize and see layers/parameters/Flops for this Tensorflow model of Text-to-speech synthesis https://github.com/Rayhane-mamah/Tacotron-2

Is there a way I can visualize or see graph of Tacotron-2 with all its RNN/LSTM layers using tensorflow?
Do I need train the model first before being able to print the model, or is there a way to simply see what ops are in each layer for the model without training?

I'm having a hard time figuring this out as I'm new to TF/pytorch frameworks. It seems to me one should be able to just print the model as the github has .py source, but I just don't know how this simple/basic things work with python and how to do it.


title: Calculate the standard deviation of a moving windows using 2d convolution

body:
I am doing a image processing project.
I want to calculate the standard deviation of a silding window using 2d convolution. I can now calculate the mean, but I cannot find a way to calculate the standard deviation. Here is my code:

import torch
import numpy as np
import cv2
import matplotlib.pyplot as plt
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
print(device)
torch.set_default_tensor_type('torch.cuda.FloatTensor')
testvideo=cv2.VideoCapture ("test.mp4")
if testvideo.isOpened():
    ret, frame = testvideo.read()
    if ret == True:
        gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
        imgarr=np.asarray(gray)
        #plt.imshow(imgarr,cmap="gray")
        imgtensor=torch.from_numpy(imgarr).float().to(device).unsqueeze(0).unsqueeze(0)
        print(imgtensor.device)
        #print(intensity.device)
        MeanConv=torch.nn.Conv2d(1, 1, kernel_size=100)
        kernel=(torch.ones(100,100)/10000).unsqueeze(0).unsqueeze(0)
        MeanConv.weight = torch.nn.Parameter(kernel)
        print(kernel)
        intensity=MeanConv(imgtensor)
        plt.imshow(intensity.squeeze(0).squeeze(0).detach().cpu().numpy(),cmap="gray")
        #for i in range(714):
            #for j in range(714):
                #intensity[i,j]=torch.mean(imgtensor[i:i+7,j:j+7])
                #sd[i,j]=torch.std(imgtensor[i:i+7,j:j+7])
        #x=(sd/intensity).cpu()
        #plt.imshow(x.numpy(),cmap="gray")
testvideo.release()


title: How to create new pandas dataframe column containing values of all other columns as a tensor?

body:
I'm trying to create a new feature column which is a tensor containing the values in the existing columns. So if col A's value is '1', col B's value is '0', and col C's value is 0, then the new feature column's value will be [1,0,0].
I tried the following code:
import numpy as np
import pandas as pd
import torch

df = pd.DataFrame({"A":[1,1,0], "B":[0,1,1], "Sentiment":[0,0,1]})

df["new_feature"] = [df["A"].values, df["B"].values, df["C"].values]

...but the result is not what I need. The result is getting the values down each column rather than the values across the row (multiple column values). For example, the new_feature column value for the first row should be [1,0,0] but its showing [1,1,0]
My ultimate aim is to get a dataframe column that I can use as a torch tensor to input into a neural net.


title: Why does a single Conv2d with 10x10x3 take up 850mb of gpu

body:
In Pytorch I am optimizing a model. If I run the following code, nvidia-smi shows that I am using  850MiB /  7979MiB of memory on my gpu. Why would this be the case?

  with torch.no_grad():
      A = nn.Conv2d(10,10,3).cuda()

I imagine there are some overhead or a default allocation size specified somewhere but I could not find such documentation. I do recall that tensorflow had a setting to limit the amount of memory allocated.

Related Git Issue


title: Why not accumulate query loss and then take derivative in MAML with Pytorch and Higher?

body:
when doing MAML (Model agnostic meta-learning) there are two ways to do the inner loop:

def inner_loop1():
    n_inner_iter = 5
    inner_opt = torch.optim.SGD(net.parameters(), lr=1e-1)

    qry_losses = []
    qry_accs = []
    meta_opt.zero_grad()
    for i in range(task_num):
        with higher.innerloop_ctx(
            net, inner_opt, copy_initial_weights=False
        ) as (fnet, diffopt):
            # Optimize the likelihood of the support set by taking
            # gradient steps w.r.t. the model's parameters.
            # This adapts the model's meta-parameters to the task.
            # higher is able to automatically keep copies of
            # your network's parameters as they are being updated.
            for _ in range(n_inner_iter):
                spt_logits = fnet(x_spt[i])
                spt_loss = F.cross_entropy(spt_logits, y_spt[i])
                diffopt.step(spt_loss)

            # The final set of adapted parameters will induce some
            # final loss and accuracy on the query dataset.
            # These will be used to update the model's meta-parameters.
            qry_logits = fnet(x_qry[i])
            qry_loss = F.cross_entropy(qry_logits, y_qry[i])
            qry_losses.append(qry_loss.detach())
            qry_acc = (qry_logits.argmax(
                dim=1) == y_qry[i]).sum().item() / querysz
            qry_accs.append(qry_acc)

            # Update the model's meta-parameters to optimize the query
            # losses across all of the tasks sampled in this batch.
            # This unrolls through the gradient steps.
            qry_loss.backward()

    meta_opt.step()
    qry_losses = sum(qry_losses) / task_num
    qry_accs = 100. * sum(qry_accs) / task_num
    i = epoch + float(batch_idx) / n_train_iter
    iter_time = time.time() - start_time

def inner_loop2():
    n_inner_iter = 5
    inner_opt = torch.optim.SGD(net.parameters(), lr=1e-1)

    qry_losses = []
    qry_accs = []
    meta_opt.zero_grad()
    meta_loss = 0
    for i in range(task_num):
        with higher.innerloop_ctx(
            net, inner_opt, copy_initial_weights=False
        ) as (fnet, diffopt):
            # Optimize the likelihood of the support set by taking
            # gradient steps w.r.t. the model's parameters.
            # This adapts the model's meta-parameters to the task.
            # higher is able to automatically keep copies of
            # your network's parameters as they are being updated.
            for _ in range(n_inner_iter):
                spt_logits = fnet(x_spt[i])
                spt_loss = F.cross_entropy(spt_logits, y_spt[i])
                diffopt.step(spt_loss)

            # The final set of adapted parameters will induce some
            # final loss and accuracy on the query dataset.
            # These will be used to update the model's meta-parameters.
            qry_logits = fnet(x_qry[i])
            qry_loss = F.cross_entropy(qry_logits, y_qry[i])
            qry_losses.append(qry_loss.detach())
            qry_acc = (qry_logits.argmax(
                dim=1) == y_qry[i]).sum().item() / querysz
            qry_accs.append(qry_acc)

            # Update the model's meta-parameters to optimize the query
            # losses across all of the tasks sampled in this batch.
            # This unrolls through the gradient steps.
            #qry_loss.backward()
            meta_loss += qry_loss

    meta_loss.backward()
    meta_opt.step()
    qry_accs = 100. * sum(qry_accs) / task_num
    i = epoch + float(batch_idx) / n_train_iter
    iter_time = time.time() - start_time

are they truly equivalent?

cross-posted:

git issue: https://github.com/facebookresearch/higher/issues/60


title: Is the difference between autoregressive transformer and non-autoregressive transformer is only the attention masks in decoder?

body:
The way i think is that,
The attention mask (not padding mask) for decoder of an autoregressive transformer model is a subsequent generated mask like as
[0, 1, 1, 1, 1]
[0, 0, 1, 1, 1]
[0, 0, 0, 1, 1]
[0, 0, 0, 0, 1]

While there's no attention mask being fed to the decoder of non-autoregressive transformer model, there's only a padding mask.
Am i correct?


title: torch.pow() results in Nan

body:
In my CNN model there is a MLP which has 4 fully connected layers with last one having just one node. This output I call alpha and has 2 dimensions as in this sample output of alpha [[0.1023]]. I make sure that minimum values never goes below 0.1 using alpha=torch.clamp(alpha,min=0.1,max=2)
I need now to raise some other Pytorch tensor x with alpha. Both are in torch.float type. The Pytorch tensor x has 4 dimensions (Batch, Channel, Height,Width).
So I have tried the following:-

x**alpha
x**alpha[0][0]
x.pow(alpha)
x.pow(alpha[0][0])

But just after the first iteration both x and alpha become nan. Why this happens?
However, if I do the following:-

x.pow(alpha.item())

The problem goes away. But I am not sure if backpropagation can happen through .item(). And even backpropagation happens, why this is alright and aforementioned techniques result in Nan just after the first iteration. I use a very small learning rate of 1e-4 with Adam Optimiser. I am using Pytorch 1.3.1.


title: Pytorch is really slow and uses a lot of GPU memory when used in Starlette with WEB_CONCURRENCY > 1

body:
I am trying to build an API that uses a Pytorch model. However, as soon as I increase WEB_CONCURRENCY to something above 1, it creates substantially more threads than expected and slows down by a lot, even when sending a single request.
Example code:
api.sh
export WEB_CONCURRENCY=2

python api.py

api.py
from starlette.applications import Starlette
from starlette.responses import UJSONResponse
from starlette.middleware.gzip import GZipMiddleware
from mymodel import Model

model = Model()
app = Starlette(debug=False)
app.add_middleware(GZipMiddleware, minimum_size=1000)

@app.route('/process', methods=['GET', 'POST', 'HEAD'])
async def add_styles(request):
    if request.method == 'GET':
        params = request.query_params
    elif request.method == 'POST':
        params = await request.json()
    elif request.method == 'HEAD':
        return UJSONResponse([], headers=response_header)

    print('===Request body===')
    print(params)

    model_output = model(params.get('data', [])) # It is very simplified. Inside there are
                                                 # many things that are happening, which
                                                 # involve file reading/writing
                                                 # and spawning processes with `popen` that
                                                 # do even more processing. But I don't
                                                 # think that should be an issue here.

    return model_output

if __name__ == '__main__':
    uvicorn.run('api:app', host='0.0.0.0', port=int(os.environ.get('PORT', 8080)))

When WEB_CONCURRENCY=1 in api.sh, there is only 1 python process seen when nvidia-smi is ran and model uses 1.2GB or VRAM. Request takes ~0.7s
When WEB_CONCURRENCY=2 in api.sh, there can be upwards of 8 python processes seen in nvidia-smi and they will use upwards of ~8GB of VRAM. Then one single request can take up to 3s, if you're lucky and don't get an out of memory error.
I am using Python3.8
Why isn't Pytorch using the expected VRAM of 2.4GB when WEB_CONCURRENCY=2? And why is it slowing down so much?