title: Batch normalization makes training worse

body:
I am trying to implement the batch normalization with Pytorch and use a simple fully connected neural network to approximate a given function.
The code is as follows. The result shows that the neural network without the batch normalization performs better than that with the batch normalization technique. This means that the batch normalization makes the training even worse. Could someone explain this result? Thanks!
import matplotlib.pyplot as plt
import numpy as np
import torch

class Net(torch.nn.Module):

    def __init__(self, num_inputs, num_outputs, hidden_size=256, is_bn=True):
        super(Net, self).__init__()
        self.num_inputs = num_inputs
        self.num_outputs = num_outputs
        self.is_bn = is_bn

        # no bias is needed if batch normalization
        if self.is_bn:
            self.linear1 = torch.nn.Linear(num_inputs, hidden_size, bias=False)
            self.linear2 = torch.nn.Linear(hidden_size, hidden_size, bias=False)
        else:
            self.linear1 = torch.nn.Linear(num_inputs, hidden_size)
            self.linear2 = torch.nn.Linear(hidden_size, hidden_size)

        self.linear3 = torch.nn.Linear(hidden_size, num_outputs)

        if self.is_bn:
            self.bn1 = torch.nn.BatchNorm1d(hidden_size)
            self.bn2 = torch.nn.BatchNorm1d(hidden_size)

        self.activation = torch.nn.ReLU()

    def forward(self, inputs):
        x = inputs
        if self.is_bn:
            x = self.activation(self.bn1(self.linear1(x)))
            x = self.activation(self.bn2(self.linear2(x)))
        else:
            x = self.activation(self.linear1(x))
            x = self.activation(self.linear2(x))
        out = self.linear3(x)
        return out

torch.manual_seed(0)    # reproducible

Nx = 100
x = torch.linspace(-1., 1., Nx)
x = torch.reshape(x, (Nx, 1))
y = torch.sin(3*x)

fcn_bn, fcn_no_bn = Net(num_inputs=1, num_outputs=1, is_bn=True), Net(num_inputs=1, num_outputs=1, is_bn=False)

criterion = torch.nn.MSELoss()
optimizer_bn = torch.optim.Adam(fcn_bn.parameters(), lr=0.001)
optimizer_no_bn = torch.optim.Adam(fcn_no_bn.parameters(), lr=0.001)

total_epoch = 5000

# record loss history
loss_history_bn = np.zeros(total_epoch)
loss_history_no_bn = np.zeros(total_epoch)

fcn_bn.train()
fcn_no_bn.train()
for epoch in range(total_epoch):

    optimizer_bn.zero_grad()
    loss = criterion(fcn_bn(x), y)
    loss_history_bn[epoch] = loss.item()
    loss.backward()
    optimizer_bn.step()

    optimizer_no_bn.zero_grad()
    loss = criterion(fcn_no_bn(x), y)
    loss_history_no_bn[epoch] = loss.item()
    loss.backward()
    optimizer_no_bn.step()

    if epoch%1000 == 0:
        print("epoch: %d; MSE (with bn): %.2e; MSE (without bn): %.2e"%(epoch, loss_history_bn[epoch], loss_history_no_bn[epoch]))

fcn_bn.eval()
fcn_no_bn.eval()

plt.figure()
plt.semilogy(np.arange(total_epoch), loss_history_bn, label='neural network (with bn)')
plt.semilogy(np.arange(total_epoch), loss_history_no_bn, label='neural network (without bn)')
plt.legend()

plt.figure()
plt.plot(x, y, '-', label='exact')
plt.plot(x, fcn_bn(x).detach(), 'o', markersize=2, label='neural network (with bn)')
plt.plot(x, fcn_no_bn(x).detach(), 'o', markersize=2, label='neural network (without bn)')
plt.legend()

plt.figure()
plt.plot(x, np.abs(fcn_bn(x).detach() - y), 'o', markersize=2, label='neural network (with bn)')
plt.plot(x, np.abs(fcn_no_bn(x).detach() - y), 'o', markersize=2, label='neural network (without bn)')
plt.legend()

plt.show()

The result is as follows:
epoch: 0; MSE (with bn): 3.99e-01; MSE (without bn): 4.84e-01
epoch: 1000; MSE (with bn): 4.70e-05; MSE (without bn): 1.27e-06
epoch: 2000; MSE (with bn): 1.81e-04; MSE (without bn): 7.93e-07
epoch: 3000; MSE (with bn): 2.73e-04; MSE (without bn): 7.45e-07
epoch: 4000; MSE (with bn): 4.04e-04; MSE (without bn): 5.68e-07


title: Unable to import torch

body:
I installed CPU version of pytorch using this:
conda install pytorch==1.6.0 torchvision==0.7.0 cpuonly -c pytorch

Unfortunately when I import using this:
import torchvision

It gives following error
OSError                                   Traceback (most recent call last)
<ipython-input-1-6dd351122000> in <module>
----> 1 import torchvision

c:\users\fatima.arshad\appdata\local\continuum\anaconda2\envs\web_scraping\lib\site-packages\torchvision\__init__.py in <module>
      4 from .extension import _HAS_OPS
      5
----> 6 from torchvision import models
      7 from torchvision import datasets
      8 from torchvision import ops

c:\users\fatima.arshad\appdata\local\continuum\anaconda2\envs\web_scraping\lib\site-packages\torchvision\models\__init__.py in <module>
----> 1 from .alexnet import *
      2 from .resnet import *
      3 from .vgg import *
      4 from .squeezenet import *
      5 from .inception import *

c:\users\fatima.arshad\appdata\local\continuum\anaconda2\envs\web_scraping\lib\site-packages\torchvision\models\alexnet.py in <module>
----> 1 import torch
      2 import torch.nn as nn
      3 from .utils import load_state_dict_from_url
      4
      5

c:\users\fatima.arshad\appdata\local\continuum\anaconda2\envs\web_scraping\lib\site-packages\torch\__init__.py in <module>
    115                 err = ctypes.WinError(last_error)
    116                 err.strerror += f' Error loading "{dll}" or one of its dependencies.'
--> 117                 raise err
    118             elif res is not None:
    119                 is_loaded = True

OSError: [WinError 127] The specified procedure could not be found. Error loading "c:\users\fatima.arshad\appdata\local\continuum\anaconda2\envs\web_scraping\lib\site-packages\torch\lib\caffe2.dll" or one of its dependencies.

I had anaconda2 installed after which I downloaded and installed anaconda3 and used the above statement. My python now is 3.8.


Runtime error while finetuning a pretrained GPT2-medium model using Huggingface library in SageMaker - ml.p3.8xlarge instance.
The finetuning_gpt2_script.py contains the below,
Libraries:
from transformers import Trainer, TrainingArguments
from transformers import EarlyStoppingCallback
from transformers import GPT2LMHeadModel, GPT2Tokenizer
from transformers import TextDataset,DataCollatorForLanguageModeling

Pretrained Models:
gpt2_model = GPT2LMHeadModel.from_pretrained("gpt2-medium")
gpt2_tokenizer = GPT2Tokenizer.from_pretrained("gpt2-medium")

Train and Test Data Construction:
train_dataset = TextDataset(
          tokenizer=gpt2_tokenizer,
          file_path=train_path,
          block_size=128)

test_dataset = TextDataset(
          tokenizer=gpt2_tokenizer,
          file_path=test_path,
          block_size=128)

data_collator = DataCollatorForLanguageModeling(
        tokenizer=gpt2_tokenizer, mlm=False,
    )

train_path & test_path are unstructured text data file of size 1.45 Million and 200K lines of data
Training arguments:
training_args = TrainingArguments(
        output_dir="./gpt2-finetuned-models", #The output directory
        overwrite_output_dir=True, #overwrite the content of the output directory
        num_train_epochs=1, # number of training epochs
        per_device_train_batch_size=8, # batch size for training #32
        per_device_eval_batch_size=8,  # batch size for evaluation #64
        save_steps=100, # after # steps model is saved
        warmup_steps=500,# number of warmup steps for learning rate scheduler
        prediction_loss_only=True,
        metric_for_best_model = "eval_loss",
        load_best_model_at_end = True,
        evaluation_strategy="epoch",
    )

training_args are the training arguments constructed to train the model.
Trainer:
trainer = Trainer(
        model=gpt2_model,
        args=training_args,
        data_collator=data_collator,
        train_dataset=train_dataset,
        eval_dataset=test_dataset,
        callbacks = [early_stop_callback],
    )
early_stop_callback = EarlyStoppingCallback(early_stopping_patience  = 3)

Training:
trainer.train()
trainer.save_model(model_path)

Here, the training is done for only 1 epoch in 4 GPUS using ml.p3.8xlarge instance.
The training is done by torch-distribution like below,
python -m torch.distributed.launch finetuning_gpt2_script.py

While training at the end of the epoch, observed the below error,
RuntimeError: Input tensor at index 3 has invalid shape [2, 2, 16, 128, 64] but expected [2, 4, 16, 128, 64]

Is the RuntimeError because of the way the train_dataset and test_datasetconstructed using TextData ?
Am I doing wrong in the torch-distribution ?


title: PyTorch: How to insert before a certain element

body:
Currently I have a 2D tensor, for each row, I want to insert a new element e before the first index of a specified value v. Additional information: cannot guarantee each row could have a such value. If there isn't, just append the element
Example: Supporse e is 0, v is 10, Given a tensor
[[9, 6, 5, 4, 10],
 [8, 7, 3, 5, 5],
 [4, 9, 10, 10, 10]]

I want to get
[[9, 6, 5, 4, 0, 10],
 [8, 7, 3, 5, 5, 0],
 [4, 9, 0, 10, 10, 10]]

Are there some Torch-style ways to do this? The worst case I can treat this as a trivial Python problem but I think the corresponding solution is a little time-consuming.


title: how do I eliminate the for-loop in this pytorch code

body:
I have this custom pytorch module (below). It does exactly what I need; it just does it very slowly. What can I do to speed this up? I know that I'm not supposed to have a for-loop in there; it just wasn't clear how to do that divide operation without it. How do I broadcast the x tensor to the divide without that loop? I can move the back-weights to their own layer if that helps.
class StepLayer(nn.Module):
    def __init__(self):
        super(StepLayer, self).__init__()
        w = init_weights()
        self.front_weights = nn.Parameter(torch.DoubleTensor([w, w]).T, requires_grad=True)
        self.back_weights = nn.Parameter(torch.DoubleTensor([w]).T, requires_grad=True)

    def forward(self, x):
        # x shape is batch by feature
        results = []
        for batch in x:
            b = batch.divide(self.front_weights)
            b = torch.some_math_function(b)
            b = b.sum(dim=1)
            b = torch.some_other_math_function(b)
            b = b @ self.back_weights
            results.append(b)
        stack = torch.vstack(results)
        return stack


title: How to find training accuracy in pytorch

body:
def train_and_test(e):
    epochs = e
    train_losses, test_losses, val_acc, train_acc= [], [], [], []
    valid_loss_min = np.Inf
    model.train()
    print("Model Training started.....")
    for epoch in range(epochs):
        running_loss = 0
        batch = 0
        for images, labels in trainloader:
            images, labels = images.to(device), labels.to(device)
            optimizer.zero_grad()
            outputs = model(images)
            loss = criterion(outputs, labels)
            loss.backward()
            optimizer.step()
            running_loss += loss.item()
            batch += 1
            if batch % 10 == 0:
                print(f" epoch {epoch + 1} batch {batch} completed")
        test_loss = 0
        accuracy = 0
        with torch.no_grad():
            print(f"validation started for {epoch + 1}")
            model.eval()
            for images, labels in validloader:
                images, labels = images.to(device), labels.to(device)
                logps = model(images)
                test_loss += criterion(logps, labels)
                ps = torch.exp(logps)
                top_p, top_class = ps.topk(1, dim=1)
                equals = top_class == labels.view(*top_class.shape)
                accuracy += torch.mean(equals.type(torch.FloatTensor))

      train_losses.append(running_loss / len(trainloader))
      test_losses.append(test_loss / len(validloader))
      val_acc.append(accuracy / len(validloader))
      training_acc.append(running_loss / len(trainloader))
      scheduler.step()
      print("Epoch: {}/{}.. ".format(epoch + 1, epochs),"Training Loss: {:.3f}.. ".format(train_losses[-1]), "Valid Loss: {:.3f}.. ".format(test_losses[-1]),
        "Valid Accuracy: {:.3f}".format(accuracy / len(validloader)), "train Accuracy: {:.3f}".format(running_loss / len(trainloader)))
      model.train()
      if test_loss / len(validloader) <= valid_loss_min:
          print('Validation loss decreased ({:.6f} --> {:.6f}).  Saving model ...'.format(valid_loss_min, test_loss / len(validloader)))
        torch.save({
            'epoch': epoch,
            'model': model,
            'model_state_dict': model.state_dict(),
            'optimizer_state_dict': optimizer.state_dict(),
            'loss': valid_loss_min
            }, path)
          valid_loss_min = test_loss / len(validloader)

    print('Training Completed Succesfully !')
    return train_losses, test_losses, val_acc ,train_acc

my output is
Model Training started.....
 epoch 1 batch 10 completed
 epoch 1 batch 20 completed
 epoch 1 batch 30 completed
 epoch 1 batch 40 completed
validation started for 1
Epoch: 1/2..  Training Loss: 0.088..  Valid Loss: 0.072..  Valid Accuracy: 0.979 train Accuracy: 0.088
Validation loss decreased (inf --> 0.072044).  Saving model ...

I am using dataset that is multi-set classification and getting training accuracy and training loss equal so I think there is error in training accuracy code.
training_acc.append(running_loss / len(trainloader))
"train Accuracy: {:.3f}".format(running_loss / len(trainloader))

training_acc.append(accuracy / len(trainloader))
"train Accuracy: {:.3f}".format(accuracy / len(trainloader))

is also not working fine


title: Best way to use hydra during deep learning model inference

body:
I am currently experimenting with hydra-conf for my deep learning project.
For training, i follow the way shown in the documentation:

folder of configuration files, a main config.yml file with additional configurations groups in extra folders
my main method is decorated with @hydra.main(...) to initialize/load the configuration

When my model is trained, i save the corresponding state dict.
Now the question is, what is the intended way of doing model inference using hydra?
My idea is to use the model checkpoint as well as the run-specific config.yml file in the .hydra folder to get the constructor-parameters i need for instantiating the model, before loading the state dict.
Is there a better way to achieve my goal?
Edit 1:
When trying to use the @hydra.main() approach i do not know how i would pass a dynamic config-file path to the decorator, since this would be supplied by the user (Scenario: I want to do inference on a certain model by providing the path to the config file in the training-run folder created by hydra)


title: How to get multiple lines exported to wandb

body:
I am using the library weights and  biases. My model outputs a curve (a time series). I'd like to see how this curve changes throughout training. So, I'd need some kind of slider where I can select epoch and it shows me the curve for that epoch. It could be something very similar to what it's done with histograms (it shows an image of the histograms across epochs and when you hover it display the histogram corresponding to that epoch). Is there a way to do this or something similar using wandb?
Currently my code looks like this:
for epoch in range(epochs):
   output = model(input)
   #output is shape (37,40) (lenght 40 and I have 37 samples)
   #it's enough to plot the first sample
   xs = torch.arange(40).unsqueeze(dim=1)
   ys = output[0,:].unsqueeze(dim=1)
   wandb.log({"line": wandb.plot.line_series(xs=xs, ys=ys,title="Out")}, step=epoch)

I'd appreciate any help! Thanks!


title: How to use 'DataLoader(dataset=Dataset(train_data, train_label))' in pytorch while I haven't input the whole 'train_data'?

body:
How can I use virtual memory in pytorch coding? I have a large of input data but can not input all of the into memory once a time, whereas I want to shuffle the whole database in DataLoader. Or if there is any other method to meet this question?

I've got a memory error when I tried to input the whole 'train_data' into memory. It's too large.

train_loader = DataLoader(dataset=Dataset(train_data, train_label), batch_size=batch_size, shuffle=True)

I expect to use the whole 'train_data' directly so that I can use DataLoader to simplify the program of training, but I got a memory error.


title: Why there's no .train() method in this Pytorch official tutorial?

body:
So, I just learned Pytorch and they say that you must put the NN in train mode by .train() method, then while inferencing .eval() mode. I'm reading this tutorial and there's no .train() at all. Why is that?

https://pytorch.org/tutorials/beginner/blitz/cifar10_tutorial.html#sphx-glr-beginner-blitz-cifar10-tutorial-py


title: How to rotate set of 3d points using angle axis to rotation matrix?

body: I am trying to rotate a set of 3d points and I am looking at this function from the kornia library. If I try to rotate a point around the z-axis by pi/2, my input(axis angle representation) should be [0, 0, pi/2]. When I use this as input into the function, it returns a 4x4 rotation matrix. However, I don't know how to apply this 4x4 matrix to my data because it is Nx3. What do I do with the output matrix? Thanks!


title: Yolov3 - Concatenate output layers to speed up training

body:
I am using Yolov3 in Pytorch and have got 5 classes. Each class consists of approx. 500 images and after 2.500 iterations I get an mAP of 0.80%. However, the training takes approx. 2 days and I am strongly wondering how to decrease the training time.

One idea is to take a pretrained yolov3 network weights file, concatenate the three output layers and use these layers for training.

What do you think of this idea? Or will this not work?

Thanks!


title: Confused about calculation of convolutional layer shapes

body:
I am new in this forum and I have started studying the theory of CNN.
It is probably a stupid question but I am confused about the calculation of the CNN outputs shape.
I am following a course on Udacity and in one of the tutorials they provide this CNN architecture.

import torch.nn as nn
import torch.nn.functional as F

# define the CNN architecture
class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        # convolutional layer (sees 32x32x3 image tensor)
        self.conv1 = nn.Conv2d(3, 16, 3, padding=1)
        # convolutional layer (sees 16x16x16 tensor)
        self.conv2 = nn.Conv2d(16, 32, 3, padding=1)
        # convolutional layer (sees 8x8x32 tensor)
        self.conv3 = nn.Conv2d(32, 64, 3, padding=1)
        # max pooling layer
        self.pool = nn.MaxPool2d(2, 2)
        # linear layer (64 * 4 * 4 -> 500)
        self.fc1 = nn.Linear(64 * 4 * 4, 500)
        # linear layer (500 -> 10)
        self.fc2 = nn.Linear(500, 10)
        # dropout layer (p=0.25)
        self.dropout = nn.Dropout(0.25)

Could you please help in understanding the way they calculate the outputs of the CNN layers? (The starting shape of the images is 32x32x3)
More specifically how did they end up with this:

# linear layer (64 * 4 * 4 -> 500)
self.fc1 = nn.Linear(64 * 4 * 4, 500)

Thanks a lot


title: Bert Embedding element #308

body:
I've been getting some practice with Bert Embedding. Specifically, I'm using BERT-Base Uncased model with the PyTorch libraries

    import torch
    from pytorch_pretrained_bert import BertTokenizer, BertModel, BertForMaskedLM
    tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
    model = BertModel.from_pretrained('bert-base-uncased')

I noticed that for pretty much every word I've looked at the element #308 in the bert embedding 768-long vector is a negative outlier with the values below -2, perhaps half of the time below -4.

It is really weird. I tried to google something about 'bert embedding 308' but couldn't find anything.

I wonder if there is any explanation for this 'phenomenon'.

Here is my routine to extract the embeddings:

def bert_embedding(text, bert_model, bert_tokenizer, layer_number = 0):
    marked_text = "[CLS] " + text + " [SEP]"
    #the default is the last layer: 0-1 = -1
    layer_number -= 1
    tokenized_text = bert_tokenizer.tokenize(marked_text)
    indexed_tokens = tokenizer.convert_tokens_to_ids(tokenized_text)
    segments_ids = [1] * len(tokenized_text)
    tokens_tensor = torch.tensor([indexed_tokens])
    segments_tensors = torch.tensor([segments_ids])

    with torch.no_grad():
        encoded_layers, _ = bert_model(tokens_tensor, segments_tensors)

    return encoded_layers[layer_number][0][:]


title: How to debug invalid gradient at index 0? - PyTorch

body:
I am trying to train an actor-critic model, but when I reach the backprop for the critic I get this error:
RuntimeError: invalid gradient at index 0 - expected type torch.cuda.FloatTensor but got torch.FloatTensor
I am failing to identify which gradient the error refers to. Can anyone help?

Here is the Stack trace:

Traceback (most recent call last):
  File "train.py", line 338, in <module>
    main()
  File "train.py", line 327, in main
    reinforce_trainer.train(opt.start_reinforce, opt.start_reinforce + opt.critic_pretrain_epochs - 1, True, start_time)
  File "/home/fbommfim/init-tests/treeLSTM/lib/train/reinforce_trainer.py", line 56, in train
    train_reward, critic_loss = self.train_epoch(epoch, pretrain_critic, no_update)
  File "/home/fbommfim/init-tests/treeLSTM/lib/train/reinforce_trainer.py", line 153, in train_epoch
    critic_loss = self.critic.backward(baselines.cuda(), rewards, critic_weights.cuda(), num_words, self.critic_loss_func, regression=True)
  File "/home/fbommfim/init-tests/treeLSTM/lib/model/encoder_decoder/hybrid2seq_model.py", line 67, in backward
    outputs.backward(grad_output)
  File "/home/linuxbrew/.linuxbrew/Cellar/python/3.7.6_1/lib/python3.7/site-packages/torch/tensor.py", line 195, in backward
    torch.autograd.backward(self, gradient, retain_graph, create_graph)
  File "/home/linuxbrew/.linuxbrew/Cellar/python/3.7.6_1/lib/python3.7/site-packages/torch/autograd/__init__.py", line 99, in backward
    allow_unreachable=True)  # allow_unreachable flag
RuntimeError: invalid gradient at index 0 - expected type torch.cuda.FloatTensor but got torch.FloatTensor

and relevant code:
train_epoch from reinforce_trainer

def train_epoch(self, epoch, pretrain_critic, no_update):
        self.actor.train() # may also have self.critic.train() ?
        total_reward, report_reward = 0, 0
        total_critic_loss, report_critic_loss = 0, 0
        total_sents, report_sents = 0, 0
        total_words, report_words = 0, 0
        last_time = time.time()
        batch_count = len(self.train_data)
        batch_order = torch.randperm(batch_count)
        with tqdm(total = (batch_count)) as prog:
            for i in range(batch_count):
                batch = self.train_data[i] # batch_order[i]
                if self.opt.data_type == 'code':
                    targets = batch[2]
                    attention_mask = batch[1][2][0].data.eq(lib.Constants.PAD).t()
                elif self.opt.data_type == 'text':
                    targets = batch[2]
                    attention_mask = batch[0][0].data.eq(lib.Constants.PAD).t()
                elif self.opt.data_type == 'hybrid':
                    targets = batch[2]
                    attention_mask_code = batch[1][2][0].data.eq(lib.Constants.PAD).t()
                    attention_mask_txt = batch[0][0].data.eq(lib.Constants.PAD).t()

                batch_size = targets.size(1)

                self.actor.zero_grad()
                self.critic.zero_grad()

                # Sample translations
                if self.opt.has_attn:
                    if self.opt.data_type == 'code' or self.opt.data_type == 'text':
                        self.actor.decoder.attn.applyMask(attention_mask)
                    elif self.opt.data_type == 'hybrid':
                        self.actor.decoder.attn.applyMask(attention_mask_code, attention_mask_txt)
                samples, outputs = self.actor.sample(batch, self.max_length)

                # Calculate rewards
                rewards, samples = self.sent_reward_func(samples.t().tolist(), targets.data.t().tolist())
                reward = sum(rewards)

                # Perturb rewards (if specified).
                if self.pert_func is not None:
                    rewards = self.pert_func(rewards)

                samples = torch.LongTensor(samples).t().contiguous()
                rewards = torch.FloatTensor([rewards] * samples.size(0)).contiguous()
                if self.opt.cuda:
                    samples = samples.cuda()
                    rewards = rewards.cuda()

                # Update critic.
                critic_weights = samples.ne(lib.Constants.PAD).float()
                num_words = critic_weights.data.sum()
                if not no_update:
                    if self.opt.data_type == 'code':
                        baselines = self.critic((batch[0], batch[1], samples, batch[3]), eval=False, regression=True)
                    elif self.opt.data_type == 'text':
                        baselines = self.critic((batch[0], batch[1], samples, batch[3]), eval=False, regression=True)
                    elif self.opt.data_type == 'hybrid':
                        baselines = self.critic((batch[0], batch[1], samples, batch[3]), eval=False, regression=True)

                    critic_loss = self.critic.backward(baselines, rewards, critic_weights, num_words, self.critic_loss_func, regression=True)
                    self.critic_optim.step()
                else:
                    critic_loss = 0

                # Update actor
                if not pretrain_critic and not no_update:
                    # Subtract baseline from reward
                    norm_rewards = (rewards - baselines).data
                    actor_weights = norm_rewards * critic_weights
                    # TODO: can use PyTorch reinforce() here but that function is a black box.
                    # This is an alternative way where you specify an objective that gives the same gradient
                    # as the policy gradient's objective, which looks much like weighted log-likelihood.
                    actor_loss = self.actor.backward(outputs, samples, actor_weights, 1, self.actor_loss_func)
                    self.optim.step()
                else:
                    actor_loss = 0

                # Gather stats
                total_reward += reward
                report_reward += reward
                total_sents += batch_size
                report_sents += batch_size
                total_critic_loss += critic_loss
                report_critic_loss += critic_loss
                total_words += num_words
                report_words += num_words
                self.opt.iteration += 1
                print ("iteration: %s, loss: %s " % (self.opt.iteration, actor_loss))
                print ("iteration: %s, reward: %s " % (self.opt.iteration, (report_reward / report_sents) * 100))

                if i % self.opt.log_interval == 0 and i > 0:
                    print("""Epoch %3d, %6d/%d batches; actor reward: %.4f; critic loss: %f; %5.0f tokens/s; %s elapsed""" %
                          (epoch, i, batch_count, (report_reward / report_sents) * 100,
                          report_critic_loss / report_words,
                          report_words / (time.time() - last_time),
                          str(datetime.timedelta(seconds=int(time.time() - self.start_time)))))

                    report_reward = report_sents = report_critic_loss = report_words = 0
                    last_time = time.time()
                prog.update(1)

        return total_reward / total_sents, total_critic_loss / total_words

and backward for hybrid2seq_model.py:

def backward(self, outputs, targets, weights, normalizer, criterion, regression=False):
        grad_output, loss = self.generator.backward(outputs, targets, weights, normalizer, criterion, regression)
        outputs.cuda()
        grad_output.cuda()
        outputs.backward(grad_output)
        return loss


title: Is my DDQN network correctly implemented?

body:
Here's my replay/train function implementation. I made the DDQN so that model lags behind model2 by 1 batch size during replay/training. By setting self.ddqn = False it becomes a normal DQN. Is this correctly implemented? I am using this paper as reference:

http://papers.nips.cc/paper/3964-double-q-learning.pdf

DDQN Code

    def replay(self, batch_size):
        if self.ddqn:
            self.model2.load_state_dict(self.model.state_dict()) # copies model weights to model2
        minibatch = random.sample(self.memory, batch_size)
        for state, action, reward, next_state, done in minibatch:
            state = torch.Tensor(state)
            next_state = torch.Tensor(next_state)
            if self.cuda:
                state = torch.Tensor(state).cuda()
                next_state = torch.Tensor(next_state).cuda()
            Q_current = self.model(state)
            Q_target = Q_current.clone() # TODO: test copy.deepcopy() and Tensor.copy_()
            Q_next = (1-done)*self.model(next_state).cpu().detach().numpy()
            next_action = np.argmax(Q_next)
            if self.ddqn:
                Q_next = (1-done)*self.model2(next_state).cpu().detach().numpy()
            Q_target[action] = Q_current[action] + self.alpha*(reward + self.gamma*Q_next[next_action] - Q_current[action])

            self.optim.zero_grad()
            loss = self.loss(Q_current, Q_target)
            loss.backward()
            self.optim.step()

        if self.epsilon > self.epsilon_min:
            self.epsilon = max(self.epsilon*self.epsilon_decay, self.epsilon_min)


title: Pytorch transform tensor to one hot

body:
What is the easiest way to transform tensor of shape (batch_size, height, width) filled with n values to tensor of shape (batch_size, n, height, width)?
I created solution below, but looks like there are easier and faster way to do this

def batch_tensor_to_onehot(tnsr, classes):
    tnsr = tnsr.unsqueeze(1)
    res = []
    for cls in range(classes):
        res.append((tnsr == cls).long())
    return torch.cat(res, dim=1)


title: Why torch.dot(a,b) makes requires_grad=False

body:
I have some losses in a loop storing them in a tensor loss. Now I want to multiply a weight tensor to the loss tensor to have final loss, but after torch.dot(), the result scalar, ll_new, has requires_grad=False. The following is my code.

loss_vector = torch.FloatTensor(total_loss_q)
w_norm = F.softmax(loss_vector, dim=0)
ll_new = torch.dot(loss_vector,w_norm)

How can I have requires_grad=False for the ll_new after doing the above?


title: New to PyTorch, having trouble making predictions once data is loaded using Data Loader

body:
I'm completely new to PyTorch (have previously used tensor flow) and I'm stuck on something I'm working on. I've been tasked with using a pretrained model to extract the features from application documents and then compute similarity scores to identify duplicates. I have all of the pdf's converted to .jpg's, and I've loaded the pretrained model and modified the last layer to extract features. The folder structure is like this:

root
|- Application 1
|   |- image 1
|   |- image 2...
|- Application 2
|   |- image 1
|   |- image 2...

What I'm trying to do is extract features from the images in every sub-directory and calculate the euclidean distance between them and output a similarity matrix. Where I'm having an issue, and this may seem really basic, is actually making the predictions once the data is loaded. Below is the code I have so far, any help would be greatly appreciated.

def get_pretrained_model_notop(model_name): #pull the model and change last layer
    pretrained_model = model_name(pretrained=True) #downloads pretrained model weights
    for param in pretrained_model.parameters():
        param.requires_grad = False #freezes layers
    pretrained_model = nn.Sequential(*list(pretrained_model.children())[:-1]) #drops final layer, because we aren't classifying 1000 imagenet classes
    pretrained_model.fc = nn.Sequential(
    nn.Flatten() #adds flatten layer at end of model
                            )
    if torch.cuda.is_available(): #uses GPU if available
        pretrained_model = pretrained_model.cuda()
    return pretrained_model

def get_similarity(pretrained_model,train_imgs):   #function to extract features from the model and compute similarity scores
    bottleneck_feature_example = pretrained_model(train_imgs)
    similarity = euclidean_distances(bottleneck_feature_example)
    similarity=similarity/similarity.max()
    similarity_df = pd.DataFrame(similarity)
    similarity_df=1-similarity_df
    return np.round(similarity_df,4)

transforms =           transforms.Compose([transforms.Resize(256),
        transforms.CenterCrop(224),
        transforms.ToTensor(),
        transforms.Normalize([0.485, 0.456, 0.406],
                             [0.229, 0.224, 0.225])])

img_dir='path'
images = datasets.ImageFolder(img_dir,transform=transforms)
data_loader = torch.utils.data.DataLoader(images,
            batch_size=32,
            shuffle=True,
            num_workers=4)

model_list=[models.densenet201]
model_name=['densenet201']
pretrained_model=[get_pretrained_model_notop(selected_model) for selected_model in model_list]
for data in data_loader:
    pred=[get_similarity(pretrained,data) for pretrained in pretrained_model]
    pred_label_ensemble=sum(pred) / len(pred)
    pred_label_ensemble.columns=page_numbers
    prob_output_folder = unzipped.replace('MF_loan_document', 'MF_loan_document_results')
    pred_label_ensemble.to_csv(prob_output_folder+'/'+'results.csv',index=False)


title: Saving a PyTorch models state_dict into redis cache

body:
I’m building a distributed parameter/server type architecture and wanting to communicate model updates through table solutions on Azure.
I’m having a hard time finding any useful information about saving a PyTorch models state_dict into a redis cache. I’ve given up on Azure Cosmos tables because of the size limit (64kb) per entity and looked toward redis since model state_dict params/weights are much larger, even for a small model.
Does anyone have any recommendations for me on how to pursue this? Or if this is even possible?


title: Why is the accuracy difference so much when I use the image data set and pytorch's own data set directly?

body:
For example, for the cifar10 data set, directly using the data set that comes with pytorch, the accuracy rate can reach 96% under the same network structure, but after I converted cifar10 into a picture, I tested it and the accuracy rate was only 92%. why?
This is the previous code：
train_dataset = dset.CIFAR10(args.data_path, train=True, transform=train_transform, download=True)
test_dataset = dset.CIFAR10(args.data_path, train=False, transform=test_transform, download=True)

This is the modified code：
train_dataset = datasets.ImageFolder(root='/home/ubuntu/bigdisk/DataSets/cifar10/static/orig/train/',
                                         transform=train_transform
                                         )
test_dataset = datasets.ImageFolder(root='/home/ubuntu/bigdisk/DataSets/cifar10/static/orig/test/',
                                        transform=test_transform
                                        )

train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=args.batch_size, shuffle=True,
                                               num_workers=args.prefetch, pin_memory=True)
test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=args.test_bs, shuffle=False,
                                              num_workers=args.prefetch, pin_memory=True)


title: When to detach tensor used for indexing in PyTorch

body:
I am trying to index a torch.Tensor and want some advice about when I should detach the tensor corresponding to indices to slice from another torch.Tensor
prob: torch.Tensor # (float32)
label: torch.Tensor # (int32)

index = torch.arange(label.numel()).reshape(label.shape)
index = index * prob.shape[-1] + label
return torch.take(prob, index)

Where prob and label are tensors whose gradients should be preserved.  I simply want to know when to detach index considering I use label to compute the indices to take from prob.
I was thinking about changing the last line to
return torch.take(prob, index.detach())

but I do not know how this interacts with the fact that label is used earlier in the computation to compute index.


title: Input fixed length sequence of frames to CNN

body:
I want my pytorch CNN to take as input a sequence of length SEQ_LEN of 32x32 RGB images concatenated along channels dimension. Therefore, a single input of the network has shape (32, 32, 3, SEQ_LEN). How should I define my CNN input layer?
The common way
SEQ_LEN = 10
input_conv = nn.Conv2d(in_channels=SEQ_LEN, out_channels=32, kernel_size=3)

BATCH_SIZE = 64
frames = np.random.randint(0, 255, size=(BATCH_SIZE, SEQ_LEN, 3, 32, 32))
frames_tensor = torch.tensor(frames)

input_conv(frames_tensor)

gives the error

RuntimeError: Expected 4-dimensional input for 4-dimensional weight [32, 10, 3, 3], but got 5-dimensional input of size [64, 10, 3, 32, 32] instead


title: Google Cloud TPU: gcloud compute tpus create failing with permission denied

body:
I am trying to follow an official tutorial on training model with google cloud TPUs.
This is the tutorial : https://cloud.google.com/tpu/docs/tutorials/transformer-pytorch
At the step "Launch a Cloud TPU resource" i do the following
:~$ gcloud compute tpus create train-bert-one \
> --zone=europe-west4-a \
> --network=default \
> --version=pytorch-1.6 \
> --accelerator-type=v3-8

Just as in the tutorial, I merely adapted the region.
The command fails with
ERROR: (gcloud.compute.tpus.create) PERMISSION_DENIED: Permission 'tpu.nodes.create' denied on 'projects/bert-archaea-fine-tuning/locations/europe-west4-a/nodes/train-bert-one'

Does anyone know this error? Cannot find anything on it.


title: Google Cloud VM Machine: Cuda out Memory

body:
I have recently set-up an account on Google Cloud VM Machine, and I am using the Nvidia Tesla K-80GPU.
When I run my Pytorch code I am getting the following error:

But when I run the same code on Google Colab, it is working fine.


title: Concatenating branches(of different dimensions) of googlenet on Pytorch

body:
is there any way I can concatenate branches of different dimensions of a small inception(googlenet)?
For example, at a 32 x 32 x 3 image (torch.Size([1, 3, 32, 32])), it will pass through an inception module with the following branches:

a convolution with (32 channels, 1 x 1 filters)
another convolution with (32 channels, 3 x3 filters)
merge (concatenate along the channel dimension)

Inception Module
The issue however is that the torch sizes as a result of the two convolutions are different.
(32 channels, 1 x 1 filters) -> [1, 32, 30, 30]
(32 channels, 3 x3 filters) -> [1, 32, 28, 28]
How can I concatenate the two branches?
Should I add padding? I tried ZeroPad2d. It’s not working.
For your reference:
class Inception(nn.Module):
def __init__(self, in_channel, ch1, ch3):
    super(Inception, self).__init__()
    self.branch1 = nn.Sequential(
        ConvBlock(in_channel, ch1, kernel_size = 1,stride=1, padding=0)
    )

    self.branch2 = nn.Sequential(
        ConvBlock(in_channel, ch3, kernel_size = 3,stride=1, padding=0)
    )

def forward(self, x):
    branch1 = self.branch1(x)
    branch2 = self.branch2(x)
    return torch.cat([branch1, branch2], 1) *(error here)*


title: why my variational autoencoder can't learn

body:
I am using 187 data as train set, which has 68 features
and would like to extract 10 features then use PCA to plot in 2D
my original data is right skewed but the latent space becomes normal
even though the loss decreases well, the model doesn't seem to be learning
[model] variational autoencoder
layer 68 - 30 - 10 - 30 - 68, using leaky_relu as activation function and tanh in the final layer
added l1 regularization in loss function, and dropout in the encoder
class VAE(nn.Module):
def __init__(self):
    super(VAE, self).__init__()

    self.fc1 = nn.Linear(68, 30)
    self.fc21 = nn.Linear(30, 10)
    self.fc22 = nn.Linear(30, 10)
    self.fc3 = nn.Linear(10, 30)
    self.fc4 = nn.Linear(30, 68)
    self.dropout = nn.Dropout(0.5)

def encode(self, x):
    h1 = F.leaky_relu(self.fc1(x))
    h1 = self.dropout(h1)
    return self.fc21(h1), self.fc22(h1)

def reparameterize(self, mu, logvar):
    std = logvar.mul(0.5).exp_()
    if torch.cuda.is_available():
        eps = torch.cuda.FloatTensor(std.size()).normal_()
    else:
        eps = torch.FloatTensor(std.size()).normal_()
    eps = Variable(eps)
    return eps.mul(std).add_(mu)

def decode(self, z):
    h3 = F.leaky_relu(self.fc3(z))

    return torch.tanh(self.fc4(h3)) # sigmoid -> relu

def forward(self, x):
    mu, logvar = self.encode(x.view(-1, 68))
    z = self.reparameterize(mu, logvar)
    return self.decode(z), z, mu, logvar

model = VAE().to(device)
optimizer = optim.Adam(model.parameters(), lr=1e-3) # this will help for L2 regularization

def loss_function(recon_x, x, mu, logvar):
loss = nn.MSELoss(reduction = 'sum')
BCE = loss(recon_x, x)
KLD = -0.5 * torch.sum(1 + logvar - mu.pow(2) - logvar.exp())

regularization_loss = 0 # l1 regularization
for param in model.parameters():
    regularization_loss += torch.sum(torch.abs(param))

return BCE + KLD + regularization_loss

I have no clue why this is not working
this is the desired output
this is the one I get -- very random


title: How to propagate gradients through torch.stack?

body:
I'm trying to optimise a model using PyTorch. Below are two implementations of a minimum working example.
Method 1 has a torch.stack call in it. The model does not train, and returns a constant loss:
import torch

qx = torch.rand(100, dtype=torch.float32).requires_grad_()
qy = torch.rand(100, dtype=torch.float32).requires_grad_()
qz = torch.rand(100, dtype=torch.float32).requires_grad_()

x = torch.stack([qx, qy, qz]).T
y_true = torch.linspace(0, 10, 100)

optimizer = torch.optim.Adam([qx, qy, qz], lr=1e-2)

reporter_interval = 10
results = []
losses = []

for i in range(1000):
    optimizer.zero_grad()
    loss = torch.abs((y_true - (x**2).sum(1))).sum()
    loss.backward(retain_graph=True)
    optimizer.step()

    if i % reporter_interval == 0:
        results.append(x.clone().detach().numpy())
        losses.append(float(loss))
        print(f'Iter {i}, loss: {float(loss)}')

Method 2 omits the call to torch.stack and works directly on the 2d array. This model converges quickly to zero loss:
import torch

qx = torch.rand(100, dtype=torch.float32).requires_grad_()
qy = torch.rand(100, dtype=torch.float32).requires_grad_()
qz = torch.rand(100, dtype=torch.float32).requires_grad_()

x = torch.stack([qx, qy, qz]).T
y_true = torch.linspace(0, 10, 100)

x = torch.tensor(x.detach().numpy(), dtype=torch.float32).requires_grad_()
optimizer = torch.optim.Adam([x], lr=1e-2)

reporter_interval = 10
results = []
losses = []

for i in range(1000):
    optimizer.zero_grad()
    loss = torch.abs((y_true - (x**2).sum(1))).sum()
    loss.backward(retain_graph=True)
    optimizer.step()

    if i % reporter_interval == 0:
        results.append(x.clone().detach().numpy())
        losses.append(float(loss))
        print(f'Iter {i}, loss: {float(loss)}')

The only thing that changes between these two examples is what the optimiser operates on, i.e. either [qx, qy, qz] or x
What is it about torch.stack that prevents this from working, and is there an alternative? In my real model, I need to join data from multiple sources.
Thanks!


title: Reducing size of pytorch library

body:
I've made a conversation telegram bot with pytorch and I'm trying to host it onto Github. The large pytorch file prevents me from doing so as its too large and I get this error:
remote: error: File env/lib/python3.8/site-packages/torch/lib/libtorch_cpu.dylib is 233.61 MB; this exceeds GitHub's file size limit of 100.00 MB
Is there anyway to reduce the size of the torch file? Or is it possible to find and delete unused dependencies?


title: How to detect source of under fitting and vanishing gradients in pytorch?

body:
How to detect source of vanishing gradients in pytorch?
By vanishing gradients, I mean then the training loss doesn't go down below some value, even on limited sets of data.
I am trying to train some network, and I have the above problem, in which I can't even get the network to over fit, but can't understand the source of the problem.
I've spent a long time googling this, and only found ways to prevent over fitting, but nothing about under fitting, or specifically, vanishing gradients.

What I did find:
Pytorch forum discussion about "bad gradients". It only refers to exploding gradients, and nan gradients, and leads to here and here which is more of the same.
I know that "making the network larger or more complex" is a general suggested way of causing over fitting (which is desired right now).
I also know that very deep networks can have their gradients vanish.
It is not clear to me that a larger network would solve the problem because it could create its own problem, as I just stated, and again I would not know how to debug this, while still seeing roughly the same behavior.
Changing the architecture to some res-net could help, but also could not, because the problem was not pinpointed to be caused by network depth.
Dead Relu can cause underfitting, and indeed moving to LeakyRelu helps, but still not enough.

How would one debug sources of under fitting in Pytorch, specifically, caused by vanishing gradients?
Instead of shooting blindly, trying things, I would like to be able to properly visualize the gradients in my network to know what I am actually trying to solve instead of guessing.
Surely, I am not the first one to have this requirement, and tools and methodologies were created for this purpose.
I would like to read about them, but don't know what to look for.
The specific net I have right now is irrelevant, as this is a general question about methodology.


I tried converting a MATLAB model to PyTorch using ONNX, like proposed here by Andrew Naguib:
How to import deep learning models from MATLAB to PyTorch?
I tried running the model using the following code:
import onnx
from onnx2pytorch import ConvertModel
import torch

onnx_model = onnx.load ('resnet50.onnx')

pytorch_model = ConvertModel(onnx_model)

model = torch.load(pytorch_model)

But I got this error:

AttributeError: 'ConvertModel' object has no attribute 'seek'. You can
only torch.load from a file that is seekable. Please pre-load the data
into a buffer like io.BytesIO and try to load from it instead.

How can I fix it, please? Any ideas on how can I "pre-load the data into a buffer like io.BytesIO"?


title: In PyTorch, what exactly does the grad_fn attribute store and how is it used?

body:
In PyTorch, the Tensor class has a grad_fn attribute. This references the operation used to obtain the tensor: for instance, if a = b + 2, a.grad_fn will be AddBackward0. But what does "reference" mean exactly?
Inspecting AddBackward0 using inspect.getmro(type(a.grad_fn)) will state that the only base class of AddBackward0 is object. Additionally, the source code for this class (and in fact, any other class which might be encountered in grad_fn) is nowhere to be found in the source code!
All of this leads me to the following questions:

What precisely is stored in grad_fn and how is it called during back-propagation?
How come the objects that get stored in grad_fn do not have some sort of common super class, and why is there no source code for them on GitHub?


title: Tensor size does not match classifier's output features number

body:
I am trying to fine-tune model for token classification (named entity recognition) with dataset consisting of 9 unique tokens.
I am mainly interested in 2 models from hugging face transformers library, which are:

https://huggingface.co/DeepPavlov/bert-base-bg-cs-pl-ru-cased
https://huggingface.co/dkleczek/bert-base-polish-cased-v1

The first one works fine, there is no mismatch between classifier's output features number and tensor size.
(classifier): Linear(in_features=768, out_features=9, bias=True)
...
torch.Size([320, 9])

Fine-tuning the second one leads to assertion errors which are probably caused by mismatch between classifier's output features number and tensor size.
(classifier): Linear(in_features=768, out_features=9, bias=True)
...
torch.Size([336, 2])

I have compared both model's parameters and they are entirely the same with exception of word_embeddings number which I believe is not relevant in this case.
I would appreciate any tips on how I could solve that issue or indications towards things that I may be missing.
Model initialisation step:
    model = AutoModelForTokenClassification.from_pretrained(
        model_args.model_name_or_path,
        from_tf=bool(".ckpt" in model_args.model_name_or_path),
        config=config,
        cache_dir=model_args.cache_dir,
        revision=model_args.model_revision,
        use_auth_token=True if model_args.use_auth_token else None,
    )

Whole script that I am using - run_ner.py:

#!/usr/bin/env python
# coding=utf-8
# Copyright 2020 The HuggingFace Team All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Fine-tuning the library models for token classification.
"""
# You can also adapt this script on your own token classification task and datasets. Pointers for this are left as
# comments.

import logging
import os
import sys
from dataclasses import dataclass, field
from typing import Optional

import numpy as np
from datasets import ClassLabel, load_dataset, load_metric

import transformers
from transformers import (
    AutoConfig,
    AutoModelForTokenClassification,
    AutoTokenizer,
    DataCollatorForTokenClassification,
    HfArgumentParser,
    PreTrainedTokenizerFast,
    Trainer,
    TrainingArguments,
    set_seed,
)
from transformers.trainer_utils import get_last_checkpoint, is_main_process
from transformers.utils import check_min_version

# Will error if the minimal version of Transformers is not installed. Remove at your own risks.
#check_min_version("4.5.0.dev0")

logger = logging.getLogger(__name__)

@dataclass
class ModelArguments:
    """
    Arguments pertaining to which model/config/tokenizer we are going to fine-tune from.
    """

    model_name_or_path: str = field(
        metadata={"help": "Path to pretrained model or model identifier from huggingface.co/models"}
    )
    config_name: Optional[str] = field(
        default=None, metadata={"help": "Pretrained config name or path if not the same as model_name"}
    )
    tokenizer_name: Optional[str] = field(
        default=None, metadata={"help": "Pretrained tokenizer name or path if not the same as model_name"}
    )
    cache_dir: Optional[str] = field(
        default=None,
        metadata={"help": "Where do you want to store the pretrained models downloaded from huggingface.co"},
    )
    model_revision: str = field(
        default="main",
        metadata={"help": "The specific model version to use (can be a branch name, tag name or commit id)."},
    )
    use_auth_token: bool = field(
        default=False,
        metadata={
            "help": "Will use the token generated when running `transformers-cli login` (necessary to use this script "
            "with private models)."
        },
    )

@dataclass
class DataTrainingArguments:
    """
    Arguments pertaining to what data we are going to input our model for training and eval.
    """

    task_name: Optional[str] = field(default="ner", metadata={"help": "The name of the task (ner, pos...)."})
    dataset_name: Optional[str] = field(
        default=None, metadata={"help": "The name of the dataset to use (via the datasets library)."}
    )
    dataset_config_name: Optional[str] = field(
        default=None, metadata={"help": "The configuration name of the dataset to use (via the datasets library)."}
    )
    train_file: Optional[str] = field(
        default=None, metadata={"help": "The input training data file (a csv or JSON file)."}
    )
    validation_file: Optional[str] = field(
        default=None,
        metadata={"help": "An optional input evaluation data file to evaluate on (a csv or JSON file)."},
    )
    test_file: Optional[str] = field(
        default=None,
        metadata={"help": "An optional input test data file to predict on (a csv or JSON file)."},
    )
    overwrite_cache: bool = field(
        default=False, metadata={"help": "Overwrite the cached training and evaluation sets"}
    )
    preprocessing_num_workers: Optional[int] = field(
        default=None,
        metadata={"help": "The number of processes to use for the preprocessing."},
    )
    pad_to_max_length: bool = field(
        default=False,
        metadata={
            "help": "Whether to pad all samples to model maximum sentence length. "
            "If False, will pad the samples dynamically when batching to the maximum length in the batch. More "
            "efficient on GPU but very bad for TPU."
        },
    )
    max_train_samples: Optional[int] = field(
        default=None,
        metadata={
            "help": "For debugging purposes or quicker training, truncate the number of training examples to this "
            "value if set."
        },
    )
    max_val_samples: Optional[int] = field(
        default=None,
        metadata={
            "help": "For debugging purposes or quicker training, truncate the number of validation examples to this "
            "value if set."
        },
    )
    max_test_samples: Optional[int] = field(
        default=None,
        metadata={
            "help": "For debugging purposes or quicker training, truncate the number of test examples to this "
            "value if set."
        },
    )
    label_all_tokens: bool = field(
        default=False,
        metadata={
            "help": "Whether to put the label for one word on all tokens of generated by that word or just on the "
            "one (in which case the other tokens will have a padding index)."
        },
    )
    return_entity_level_metrics: bool = field(
        default=False,
        metadata={"help": "Whether to return all the entity levels during evaluation or just the overall ones."},
    )

    def __post_init__(self):
        if self.dataset_name is None and self.train_file is None and self.validation_file is None:
            raise ValueError("Need either a dataset name or a training/validation file.")
        else:
            if self.train_file is not None:
                extension = self.train_file.split(".")[-1]
                assert extension in ["csv", "json"], "`train_file` should be a csv or a json file."
            if self.validation_file is not None:
                extension = self.validation_file.split(".")[-1]
                assert extension in ["csv", "json"], "`validation_file` should be a csv or a json file."
        self.task_name = self.task_name.lower()

def main():
    # See all possible arguments in src/transformers/training_args.py
    # or by passing the --help flag to this script.
    # We now keep distinct sets of args, for a cleaner separation of concerns.

    parser = HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments))
    if len(sys.argv) == 2 and sys.argv[1].endswith(".json"):
        # If we pass only one argument to the script and it's the path to a json file,
        # let's parse it to get our arguments.
        model_args, data_args, training_args = parser.parse_json_file(json_file=os.path.abspath(sys.argv[1]))
    else:
        model_args, data_args, training_args = parser.parse_args_into_dataclasses()

    # Detecting last checkpoint.
    last_checkpoint = None
    if os.path.isdir(training_args.output_dir) and training_args.do_train and not training_args.overwrite_output_dir:
        last_checkpoint = get_last_checkpoint(training_args.output_dir)
        if last_checkpoint is None and len(os.listdir(training_args.output_dir)) > 0:
            raise ValueError(
                f"Output directory ({training_args.output_dir}) already exists and is not empty. "
                "Use --overwrite_output_dir to overcome."
            )
        elif last_checkpoint is not None:
            logger.info(
                f"Checkpoint detected, resuming training at {last_checkpoint}. To avoid this behavior, change "
                "the `--output_dir` or add `--overwrite_output_dir` to train from scratch."
            )

    # Setup logging
    logging.basicConfig(
        format="%(asctime)s - %(levelname)s - %(name)s -   %(message)s",
        datefmt="%m/%d/%Y %H:%M:%S",
        handlers=[logging.StreamHandler(sys.stdout)],
    )
    logger.setLevel(logging.INFO if is_main_process(training_args.local_rank) else logging.WARN)

    # Log on each process the small summary:
    logger.warning(
        f"Process rank: {training_args.local_rank}, device: {training_args.device}, n_gpu: {training_args.n_gpu}"
        + f"distributed training: {bool(training_args.local_rank != -1)}, 16-bits training: {training_args.fp16}"
    )
    # Set the verbosity to info of the Transformers logger (on main process only):
    if is_main_process(training_args.local_rank):
        transformers.utils.logging.set_verbosity_info()
        transformers.utils.logging.enable_default_handler()
        transformers.utils.logging.enable_explicit_format()
    logger.info("Training/evaluation parameters %s", training_args)

    # Set seed before initializing model.
    set_seed(training_args.seed)

    # Get the datasets: you can either provide your own CSV/JSON/TXT training and evaluation files (see below)
    # or just provide the name of one of the public datasets available on the hub at https://huggingface.co/datasets/
    # (the dataset will be downloaded automatically from the datasets Hub).
    #
    # For CSV/JSON files, this script will use the column called 'text' or the first column if no column called
    # 'text' is found. You can easily tweak this behavior (see below).
    #
    # In distributed training, the load_dataset function guarantee that only one local process can concurrently
    # download the dataset.
    if data_args.dataset_name is not None:
        # Downloading and loading a dataset from the hub.
        datasets = load_dataset(data_args.dataset_name, data_args.dataset_config_name)
    else:
        data_files = {}
        if data_args.train_file is not None:
            data_files["train"] = data_args.train_file
        if data_args.validation_file is not None:
            data_files["validation"] = data_args.validation_file
        if data_args.test_file is not None:
            data_files["test"] = data_args.test_file
        extension = data_args.train_file.split(".")[-1]
        datasets = load_dataset(extension, data_files=data_files)
    # See more about loading any type of standard or custom dataset (from files, python dict, pandas DataFrame, etc) at
    # https://huggingface.co/docs/datasets/loading_datasets.html.

    if training_args.do_train:
        column_names = datasets["train"].column_names
        features = datasets["train"].features
    else:
        column_names = datasets["validation"].column_names
        features = datasets["validation"].features
    text_column_name = "tokens" if "tokens" in column_names else column_names[0]
    label_column_name = (
        f"{data_args.task_name}_tags" if f"{data_args.task_name}_tags" in column_names else column_names[1]
    )

    # In the event the labels are not a `Sequence[ClassLabel]`, we will need to go through the dataset to get the
    # unique labels.
    def get_label_list(labels):
        unique_labels = set()
        for label in labels:
            unique_labels = unique_labels | set(label)
        label_list = list(unique_labels)
        label_list.sort()
        return label_list

    if isinstance(features[label_column_name].feature, ClassLabel):
        label_list = features[label_column_name].feature.names
        # No need to convert the labels since they are already ints.
        label_to_id = {i: i for i in range(len(label_list))}
    else:
        label_list = get_label_list(datasets["train"][label_column_name])
        label_to_id = {l: i for i, l in enumerate(label_list)}
    num_labels = len(label_list)

    label2id = {label: i for i, label in enumerate(label_list)}
    print("\nlabel_list\n", label_list)
    print("\nlabel_to_id\n", label_to_id)
    print("\nnum_labels\n", num_labels)

    # label2id=label_to_id,
    id2label={id: label for label, id in label_to_id.items()}
    #print(label2id)
    #print(id2label)

    # Load pretrained model and tokenizer
    #
    # Distributed training:
    # The .from_pretrained methods guarantee that only one local process can concurrently
    # download model & vocab.
    config = AutoConfig.from_pretrained(
        model_args.config_name if model_args.config_name else model_args.model_name_or_path,
        num_labels=num_labels,

        label2id=label_to_id,
        id2label={id: label for label, id in label_to_id.items()},

        finetuning_task=data_args.task_name,
        cache_dir=model_args.cache_dir,
        revision=model_args.model_revision,
        use_auth_token=True if model_args.use_auth_token else None,
    )
    tokenizer = AutoTokenizer.from_pretrained(
        model_args.tokenizer_name if model_args.tokenizer_name else model_args.model_name_or_path,
        cache_dir=model_args.cache_dir,
        use_fast=True,
        revision=model_args.model_revision,
        use_auth_token=True if model_args.use_auth_token else None,
    )
    model = AutoModelForTokenClassification.from_pretrained(
        model_args.model_name_or_path,
        from_tf=bool(".ckpt" in model_args.model_name_or_path),
        #from_tf=True,
        config=config,
        cache_dir=model_args.cache_dir,
        revision=model_args.model_revision,
        use_auth_token=True if model_args.use_auth_token else None,
    )

    # Tokenizer check: this script requires a fast tokenizer.
    if not isinstance(tokenizer, PreTrainedTokenizerFast):
        raise ValueError(
            "This example script only works for models that have a fast tokenizer. Checkout the big table of models "
            "at https://huggingface.co/transformers/index.html#bigtable to find the model types that meet this "
            "requirement"
        )

    # Preprocessing the dataset
    # Padding strategy
    padding = "max_length" if data_args.pad_to_max_length else False

    # Tokenize all texts and align the labels with them.
    def tokenize_and_align_labels(examples):
        # print("examples\n", examples)
        tokenized_inputs = tokenizer(
            examples[text_column_name],
            padding=padding,
            truncation=True,
            # We use this argument because the texts in our dataset are lists of words (with a label for each word).
            is_split_into_words=True,
        )
        labels = []
        for i, label in enumerate(examples[label_column_name]):
            # print("\nlabel in enumerate(examples[label_column_name])\n", label)
            word_ids = tokenized_inputs.word_ids(batch_index=i)
            previous_word_idx = None
            label_ids = []
            for word_idx in word_ids:
                # Special tokens have a word id that is None. We set the label to -100 so they are automatically
                # ignored in the loss function.
                if word_idx is None:
                    label_ids.append(-100)
                # We set the label for the first token of each word.
                elif word_idx != previous_word_idx:
                    label_ids.append(label_to_id[label[word_idx]])
                    # label_ids.append(label2id[label[word_idx]])

                # For the other tokens in a word, we set the label to either the current label or -100, depending on
                # the label_all_tokens flag.
                else:
                    label_ids.append(label_to_id[label[word_idx]] if data_args.label_all_tokens else -100)
                    # label_ids.append(label2id[label[word_idx]] if data_args.label_all_tokens else -100)

                previous_word_idx = word_idx

            labels.append(label_ids)
        tokenized_inputs["labels"] = labels
        # print("label_to_id\n", label_to_id)
        # print("\n\n tokenized_inputs\n", tokenized_inputs)
        return tokenized_inputs

    if training_args.do_train:
        if "train" not in datasets:
            raise ValueError("--do_train requires a train dataset")
        train_dataset = datasets["train"]
        if data_args.max_train_samples is not None:
            train_dataset = train_dataset.select(range(data_args.max_train_samples))
        train_dataset = train_dataset.map(
            tokenize_and_align_labels,
            batched=True,
            num_proc=data_args.preprocessing_num_workers,
            load_from_cache_file=not data_args.overwrite_cache,
        )

    if training_args.do_eval:
        if "validation" not in datasets:
            raise ValueError("--do_eval requires a validation dataset")
        eval_dataset = datasets["validation"]
        if data_args.max_val_samples is not None:
            eval_dataset = eval_dataset.select(range(data_args.max_val_samples))
        eval_dataset = eval_dataset.map(
            tokenize_and_align_labels,
            batched=True,
            num_proc=data_args.preprocessing_num_workers,
            load_from_cache_file=not data_args.overwrite_cache,
        )

    if training_args.do_predict:
        if "test" not in datasets:
            raise ValueError("--do_predict requires a test dataset")
        test_dataset = datasets["test"]
        if data_args.max_test_samples is not None:
            test_dataset = test_dataset.select(range(data_args.max_test_samples))
        test_dataset = test_dataset.map(
            tokenize_and_align_labels,
            batched=True,
            num_proc=data_args.preprocessing_num_workers,
            load_from_cache_file=not data_args.overwrite_cache,
        )

    # Data collator
    data_collator = DataCollatorForTokenClassification(tokenizer, pad_to_multiple_of=8 if training_args.fp16 else None)

    # Metrics
    metric = load_metric("seqeval")

    def compute_metrics(p):
        predictions, labels = p
        predictions = np.argmax(predictions, axis=2)

        # Remove ignored index (special tokens)
        true_predictions = [
            [label_list[p] for (p, l) in zip(prediction, label) if l != -100]
            for prediction, label in zip(predictions, labels)
        ]
        true_labels = [
            [label_list[l] for (p, l) in zip(prediction, label) if l != -100]
            for prediction, label in zip(predictions, labels)
        ]

        results = metric.compute(predictions=true_predictions, references=true_labels)
        if data_args.return_entity_level_metrics:
            # Unpack nested dictionaries
            final_results = {}
            for key, value in results.items():
                if isinstance(value, dict):
                    for n, v in value.items():
                        final_results[f"{key}_{n}"] = v
                else:
                    final_results[key] = value
            return final_results
        else:
            return {
                "precision": results["overall_precision"],
                "recall": results["overall_recall"],
                "f1": results["overall_f1"],
                "accuracy": results["overall_accuracy"],
            }

    # Initialize our Trainer
    trainer = Trainer(
        model=model,
        args=training_args,
        train_dataset=train_dataset if training_args.do_train else None,
        eval_dataset=eval_dataset if training_args.do_eval else None,
        tokenizer=tokenizer,
        data_collator=data_collator,
        compute_metrics=compute_metrics,
    )

    print("\nmodel parameters\n", model.parameters)

    # Training
    if training_args.do_train:
        if last_checkpoint is not None:
            checkpoint = last_checkpoint
        elif os.path.isdir(model_args.model_name_or_path):
            checkpoint = model_args.model_name_or_path
        else:
            checkpoint = None
        train_result = trainer.train(resume_from_checkpoint=checkpoint)
        metrics = train_result.metrics
        trainer.save_model()  # Saves the tokenizer too for easy upload

        max_train_samples = (
            data_args.max_train_samples if data_args.max_train_samples is not None else len(train_dataset)
        )
        metrics["train_samples"] = min(max_train_samples, len(train_dataset))

        trainer.log_metrics("train", metrics)
        trainer.save_metrics("train", metrics)
        trainer.save_state()

    # Evaluation
    if training_args.do_eval:
        logger.info("*** Evaluate ***")

        metrics = trainer.evaluate()

        max_val_samples = data_args.max_val_samples if data_args.max_val_samples is not None else len(eval_dataset)
        metrics["eval_samples"] = min(max_val_samples, len(eval_dataset))

        trainer.log_metrics("eval", metrics)
        trainer.save_metrics("eval", metrics)

    # Predict
    if training_args.do_predict:
        logger.info("*** Predict ***")

        predictions, labels, metrics = trainer.predict(test_dataset)
        predictions = np.argmax(predictions, axis=2)

        # Remove ignored index (special tokens)
        true_predictions = [
            [label_list[p] for (p, l) in zip(prediction, label) if l != -100]
            for prediction, label in zip(predictions, labels)
        ]

        trainer.log_metrics("test", metrics)
        trainer.save_metrics("test", metrics)

        # Save predictions
        output_test_predictions_file = os.path.join(training_args.output_dir, "test_predictions.txt")
        if trainer.is_world_process_zero():
            with open(output_test_predictions_file, "w") as writer:
                for prediction in true_predictions:
                    writer.write(" ".join(prediction) + "\n")

def _mp_fn(index):
    # For xla_spawn (TPUs)
    main()

if __name__ == "__main__":
    main()

environment.yml
name: test_3_1
channels:
  - pytorch
  - conda-forge
  - defaults
dependencies:
  - _libgcc_mutex=0.1=conda_forge
  - _openmp_mutex=4.5=1_llvm
  - blas=1.0=mkl
  - bzip2=1.0.8=h7f98852_4
  - ca-certificates=2020.12.5=ha878542_0
  - certifi=2020.12.5=py38h578d9bd_1
  - cudatoolkit=11.1.1=h6406543_8
  - ffmpeg=4.3=hf484d3e_0
  - freetype=2.10.4=h0708190_1
  - gmp=6.2.1=h58526e2_0
  - gnutls=3.6.13=h85f3911_1
  - jpeg=9b=h024ee3a_2
  - lame=3.100=h7f98852_1001
  - lcms2=2.11=h396b838_0
  - ld_impl_linux-64=2.33.1=h53a641e_7
  - libffi=3.3=he6710b0_2
  - libgcc-ng=9.3.0=h2828fa1_18
  - libiconv=1.16=h516909a_0
  - libpng=1.6.37=h21135ba_2
  - libstdcxx-ng=9.3.0=h6de172a_18
  - libtiff=4.1.0=h2733197_1
  - libuv=1.41.0=h7f98852_0
  - llvm-openmp=11.0.1=h4bd325d_0
  - lz4-c=1.9.3=h9c3ff4c_0
  - mkl=2020.4=h726a3e6_304
  - mkl-service=2.3.0=py38h1e0a361_2
  - mkl_fft=1.3.0=py38h5c078b8_1
  - mkl_random=1.2.0=py38hc5bc63f_1
  - ncurses=6.2=he6710b0_1
  - nettle=3.6=he412f7d_0
  - ninja=1.10.2=h4bd325d_0
  - numpy-base=1.19.2=py38hfa32c7d_0
  - olefile=0.46=pyh9f0ad1d_1
  - openh264=2.1.1=h780b84a_0
  - openssl=1.1.1j=h7f98852_0
  - pillow=8.1.2=py38he98fc37_0
  - pip=21.0.1=py38h06a4308_0
  - python=3.8.8=hdb3f193_4
  - python_abi=3.8=1_cp38
  - pytorch=1.8.0=py3.8_cuda11.1_cudnn8.0.5_0
  - readline=8.1=h27cfd23_0
  - setuptools=52.0.0=py38h06a4308_0
  - six=1.15.0=pyh9f0ad1d_0
  - sqlite=3.35.1=hdfb4753_0
  - tk=8.6.10=hbc83047_0
  - torchaudio=0.8.0=py38
  - torchvision=0.9.0=py38_cu111
  - typing_extensions=3.7.4.3=py_0
  - wheel=0.36.2=pyhd3eb1b0_0
  - xz=5.2.5=h7b6447c_0
  - zlib=1.2.11=h7b6447c_3
  - zstd=1.4.9=ha95c52a_0
  - pip:
    - chardet==4.0.0
    - click==7.1.2
    - datasets==1.4.1
    - dill==0.3.3
    - filelock==3.0.12
    - fsspec==0.8.7
    - huggingface-hub==0.0.2
    - idna==2.10
    - joblib==1.0.1
    - multiprocess==0.70.11.1
    - numpy==1.20.1
    - packaging==20.9
    - pandas==1.2.3
    - pyarrow==3.0.0
    - pyparsing==2.4.7
    - python-dateutil==2.8.1
    - pytz==2021.1
    - regex==2020.11.13
    - requests==2.25.1
    - sacremoses==0.0.43
    - scikit-learn==0.24.1
    - scipy==1.6.1
    - seqeval==1.2.2
    - threadpoolctl==2.1.0
    - tokenizers==0.10.1
    - tqdm==4.49.0
    - transformers==4.4.1
    - urllib3==1.26.4
    - xxhash==2.0.0

1 line sample of dataset - single_line.json:
Below sample consists 3 unique labels instead of 9.
{"words": ["Astronomię", "można", "inaczej", "określić", "jako", "naukę", "o", "wszelkich", "obiektach", "i", "zjawiskach", "znajdujących", "się", "poza", "Ziemią", "."], "ner": ["O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "B-LOC", "I-LOC", "O"]}

Execution calls:

Works.

python run_ner.py \
 --model_name_or_path "DeepPavlov/bert-base-bg-cs-pl-ru-cased" \
 --train_file single_line.json \
 --validation_file single_line.json \
 --test_file single_line.json \
 --do_train \
 --do_eval \
 --do_predict \
 --output_dir model_test_deeppavlov \
 --save_steps 50000 \
 --num_train_epochs 1

Crashes.

python run_ner.py \
 --model_name_or_path bert-base-polish-cased-v1 \
 --train_file single_line.json \
 --validation_file single_line.json \
 --test_file single_line.json \
 --do_train \
 --do_eval \
 --do_predict \
 --output_dir model_test_polish_bert \
 --save_steps 50000 \
 --num_train_epochs 1

In order to use bert-base-polish-cased-v1 model, it may require to be downloaded.
git clone https://huggingface.co/dkleczek/bert-base-polish-cased-v1


title: Getting errors with pytorch grouped convolutions

body:
I am trying to use Pytorch grouped Conv2d operator on very large images (10k x 10k pixels). I am getting an RuntimeError: offset is too big error when trying to apply a grouped convolution in the network. Anyone knows how to circumvent this?

Code for reproducibility:

import torch
import torch.nn as nn
import pdb

def create_img(size, batch_size=1, channels=3):
    return torch.FloatTensor(batch_size, channels, size, size).uniform_(-1, 1)

class TestModel(nn.Module):

    def __init__(self):
        super(TestModel, self).__init__()
        self.conv1 = nn.Conv2d(in_channels=64, out_channels=64, kernel_size=(3,3), stride=(1,1), groups=64, bias=False)

    def forward(self, x):
        out = self.conv1(x)
        return out

if __name__ == '__main__':
    model = TestModel()

    data = create_img(5002, channels=64)

    out = model(data)
    pdb.set_trace()

and the error:

Traceback (most recent call last):
  File "test.py", line 26, in <module>
    out = model(data)
  File ".../pipenv/lib/python3.6/site-packages/torch/nn/modules/module.py", line 489, in __call__
    result = self.forward(*input, **kwargs)
  File "test.py", line 17, in forward
    out = self.conv1(x)
  File ".../pipenv/lib/python3.6/site-packages/torch/nn/modules/module.py", line 489, in __call__
    result = self.forward(*input, **kwargs)
  File ".../pipenv/lib/python3.6/site-packages/torch/nn/modules/conv.py", line 320, in forward
    self.padding, self.dilation, self.groups)
RuntimeError: offset is too big

I am using Python 3.6 and Pytorch 1.0.0. Strange thing is, this works with smaller images. Change the images size from 5002 to 502, for example.


title: Python recursive function failing

body:
The issue that I am having is a really strange issue.

What I am trying to accomplish is the following: I am training a neural network using pytorch, and I want to restart my training function if the training loss doesn't decrease, so as to re-initialize the neural network with a different set of weights. The training function is presented below:

def __train__(dp, i, j, net, restarts, epoch=0):
    if net == '2CH': model = TwoChannelCNN().cuda()
    elif net == 'Siam' : model = SiameseCNN().cuda()
    elif net == 'Trad' : model = TraditionalCNN().cuda()
    ls_fn = torch.nn.MSELoss(reduce=True)
    optim = torch.optim.SGD(model.parameters(),  lr=1e-6, momentum=0.9)
    epochs = np.arange(100)
    eloss = []
    for epoch in epochs:
        model.train()
        train_loss = []
        tr_batches = np.array_split(dp.train_set, int(len(dp.train_set)/8))
        for tr_batch in tr_batches:
            if net == '2CH': loaded_batch = dp.__load2CH__(tr_batch)
            elif net == 'Siam': loaded_batch = dp.__loadSiam__(tr_batch)
            elif net == 'Trad' : loaded_batch = dp.__load__(tr_batch, i)
            for x_batch, y_batch in loaded_batch:
                x_var, y_var = Variable(x_batch.cuda()), Variable(y_batch.cuda())
                y_pred = torch.clamp(model(x_var), 0, 1)
                loss = ls_fn(y_pred, y_var)
                train_loss.append(abs(loss.item()))
                optim.zero_grad()
                loss.backward()
                optim.step()
        eloss.append(np.mean(train_loss))
        print(epoch, np.mean(train_loss))
        if epoch == 10 and np.mean(train_loss) > 0.2:
            restarts += 1
            print('Number of restarts for client {} and fold {}: {}'.format(i,j,restarts))
            __train__(dp, i, j, net, restarts, epoch=0)

    __plotLoss__(epochs, eloss, 'train', str(i), str(j))
    torch.save(model.state_dict(), "Output/client_{}_fold_{}.pt".format(i, j))

So the restarting based on if epoch == 10 and np.mean(train_loss) > 0.2: works, but only sometimes, which is beyond my comprehension. Here is an example of the output:

0 0.5000133737921715
1 0.4999906486272812
2 0.464298670232296
3 0.2727506290078163
4 0.2628978116512299
5 0.2588871221542358
6 0.25728522151708605
7 0.25630473804473874
8 0.2556223524808884
9 0.25522999209165576
10 0.25467908215522767
Number of restarts for client 5 and fold 1: 3
0 0.10957609283713009
1 0.02840371729924134
2 0.021477583368030594
3 0.017759160268232682
4 0.015173796122947827
5 0.013349939693290782
6 0.011949078906879265
7 0.010810676779671655
8 0.00987362345259362
9 0.009110640348696108
10 0.008239036202623808
11 0.007680381585537574
12 0.007171026876221333
13 0.006765962297888837
14 0.006428168776848068
15 0.006133011780953467
16 0.005819878347673745
17 0.005572605537395361
18 0.00535818950227004
19 0.005159409143814457
20 0.0049763926251294235
21 0.004738794513338235
22 0.004578812885309958
23 0.004428663117960554
24 0.004282198464788351
25 0.004145324644400691
26 0.004018862769889626
27 0.0039044404603504573
28 0.0037960831121495744
29 0.0036947361258523586
30 0.0035982220717533267
31 0.0035018146670104723
32 0.0034150678806059887
33 0.0033372560733512698
34 0.003261332974241583
35 0.00318166259540763
36 0.003108531899014735
37 0.0030385089141125848
38 0.002977990984523103
39 0.0029195284016142937
40 0.002870084639441188
41 0.0028180573325994373
42 0.0027717544270049643
43 0.002719321814503495
44 0.0026704726860933194
45 0.0026204266263459316
46 0.002570544072460258
47 0.0025225681523167224
48 0.0024814611543610746
49 0.0024358948737413116
50 0.002398673941639636
51 0.0023606415423654587
52 0.002330436484101057
53 0.0022891738560574027
54 0.002260655496376241
55 0.002227568955708719
56 0.002191826719741698
57 0.0021609061182290058
58 0.0021279943092100666
59 0.0020966088490456513
60 0.002066195117003474
61 0.0020381672924407895
62 0.002009863329306995
63 0.001986304977759602
64 0.0019564831849032487
65 0.0019351609173580756
66 0.0019077356409993626
67 0.0018875047204855945
68 0.0018617453310780547
69 0.001839518720600381
70 0.001815563331498197
71 0.0017149778925132932
72 0.0016894878409248121
73 0.0016652211918212743
74 0.0016422999463582074
75 0.0016183732903472788
76 0.0015962369183098418
77 0.0015757764620279887
78 0.0015542267022799728
79 0.0015323152910759318
80 0.0014337954093957706
81 0.001410489170542867
82 0.0013871921329466962
83 0.0013641994057461773
84 0.001345829172682187
85 0.001322142209181493
86 0.00130379223035348
87 0.001282231878045458
88 0.001263879886683956
89 0.001243419097817167
90 0.0012279346547037929
91 0.001206978429649382
92 0.0011871445969959496
93 0.001172510546330841
94 0.0011529557384797045
95 0.0011350733004023273
96 0.001118382818282214
97 0.001103347793609089
98 0.0010848538354748599
99 0.0010698940242660911
11 0.2542190085053444
12 0.2538975296020508

So here you can see that the restarting is correct from the 3rd restart, but then, since the network converges, the training should be complete, but the function restarts AGAIN after the 99th epoch (for an unknown reason), and somehow starts at the 11th epoch, which also makes no sense as I am explicitly specifying epoch = 0 whenever the function starts or restarts. I should also add that, SOMETIMES, the function completes correctly after the epoch 99, when convergence has been achieved, and does not restart.

So my question is, why does this piece of code produce inconsistent results and outcomes? What am I missing here? Thanks in advance for any suggestions.


title: Arbitary choosen values as std/mean for normalizatio. why?

body:
I have a question regarding the z-score normalization method.
This method uses the z-score to normalize the values of the dataset and needs a mean/std.
I know that you are normally supposed to use the mean/std of the dataset.
But I have seen multiple tutorials on pytorch.org and the net who just use the 0.5 for mean/std which seems completely arbitrary to me.
And I was wondering why they didn't use the mean/std of the dataset?

Example Tutorials where they just use 0.5 as mean/std:
https://pytorch.org/tutorials/beginner/dcgan_faces_tutorial.html
https://medium.com/ai-society/gans-from-scratch-1-a-deep-introduction-with-code-in-pytorch-and-tensorflow-cb03cdcdba0f
https://pytorch.org/tutorials/beginner/blitz/cifar10_tutorial.html#sphx-glr-beginner-blitz-cifar10-tutorial-py


title: How are contents of hidden_states tuple in BertModel in the transformers library arranged

body:
model = BertModel.from_pretrained('bert-base-uncased', config=BertConfig.from_pretrained('bert-base-uncased',output_hidden_states=True))
outputs = model(input_ids)
hidden_states = outputs[2]

hidden_states is a tuple of 13 torch.FloatTensors. Each tensor is of size: (batch_size, sequence_length, hidden_size).
According to the documentation, the 13 tensors are the hidden states of the embedding and the 12 encoder layers.

My question:

Is hidden_states[0] the embedding layer while hidden_states[12] is the 12th encoder layer or

Is hidden_states[0] the embedding layer while hidden_states[12] is the 1st encoder layer or

Is hidden_states[0] the 12th encoder layer while hidden_states[12] is the embedding layer or

Is hidden_states[0] the 1st encoder layer while hidden_states[12] is the embedding layer

I havent found this found clearly stated anywhere else.


title: Pytorch creating dataset with augmented images

body:
Is there a way to generate a DataLoader by creating X number of augmented images?  The code I have currently only creates a single augmented image
class ImageDataset(data.Dataset):

    def __init__(self, root_dir, transform=None):

        self.root_dir = root_dir
        self.img_names = os.listdir()
        self.transform = transform

    def __getitem__(self, index):

        img = Image.open(os.path.join(self.root, self.img_names[index])).convert('RGB')
        if self.transform is not None:
            img = self.transform(img)

        return img

    def __len__(self):

        return len(self.img_names)

Also, I would like to add labels where augmented images from the same image have the same label


title: Pytorch automatic mixed precision (amp) backward getting slower over the batches

body:
Does anyone have the same problem as mine when enabling pytorch amp, the training over batches are getting slower and slower? Sometimes it starts with the 9th batch.
I found that the slower is due to
scaler.scale(loss).backward()


title: Whitelist tokens for text generation (XLNet, GPT-2) in huggingface-transformers

body:
In the documentation on text generation (https://huggingface.co/transformers/main_classes/model.html#generative-models) there is the option to put
bad_words_ids (List[int], optional) – List of token ids that are not allowed to be generated. In order to get the tokens of the words that should not appear in the generated text, use tokenizer.encode(bad_word, add_prefix_space=True).

Is there also the option to put something along the lines of "allowed_words_ids"? The idea would be to restrict the language of the generated texts.


title: Iteration problem in Dataloader, can anyone help me solve this error

body:
I am planning to use WeightedRandomSampler in Pytorch.My data has 6 columns and 1 column of labels.
I want to use weightedRandomSampler and I am getting an error in the data loader part of the code
--
Y=pd.DataFrame(dfY)
Y.rename(columns={0: "labels"},inplace=True)
data =[dfX,Y]
trainset = pd.concat(data,axis=1)
trainratio = np.bincount(trainset["labels"]) #trainset.labels is a list of float [0,1,0,0,0,...]
classcount = trainratio.tolist()
train_weights = 1./torch.tensor(classcount, dtype=torch.float)
train_sampleweights = train_weights[trainset["labels"]]
train_sampler = WeightedRandomSampler(weights=train_sampleweights,
                              num_samples=len(train_sampleweights))
trainloader = DataLoader(trainset, sampler=train_sampler,
                                    shuffle=False)
for id,(x,y) in enumerate(trainloader):
  print(x)

Stack Trace:
---------------------------------------------------------------------------
KeyError                                  Traceback (most recent call last)
/usr/local/lib/python3.6/dist-packages/pandas/core/indexes/base.py in get_loc(self, key, method, tolerance)
   2894             try:
-> 2895                 return self._engine.get_loc(casted_key)
   2896             except KeyError as err:

pandas/_libs/index.pyx in pandas._libs.index.IndexEngine.get_loc()

pandas/_libs/index.pyx in pandas._libs.index.IndexEngine.get_loc()

pandas/_libs/hashtable_class_helper.pxi in pandas._libs.hashtable.PyObjectHashTable.get_item()

pandas/_libs/hashtable_class_helper.pxi in pandas._libs.hashtable.PyObjectHashTable.get_item()

KeyError: 905933

The above exception was the direct cause of the following exception:

KeyError                                  Traceback (most recent call last)
6 frames
/usr/local/lib/python3.6/dist-packages/pandas/core/indexes/base.py in get_loc(self, key, method, tolerance)
   2895                 return self._engine.get_loc(casted_key)
   2896             except KeyError as err:
-> 2897                 raise KeyError(key) from err
   2898
   2899         if tolerance is not None:

KeyError: 905933

Please can anyone point me out where I am wrong.


title: Diagonal embedding of a batch of matrices in pytorch?

body:
If you are given a collection of n x n matrices say m of them, is there a predefined function in pytorch that performs a diagonal embedding on all of these into a larger matrix of dimension nm x nm?
To be concrete, what I am looking for is say you have two 2 x 2 identity matrices, then their diagonal embedding into a 4 x 4 matrix would be the identity 4 x 4 matrix.
Something like:
torch.block_diag

but this expects you to feed each matrix as a separate argument.


title: The difference between tf.image.crop_and_resize vs torchvision.ops.roi_align

body:
I find it is apparently different between the two APIs( tf.image.crop_and_resize vs torchvision.ops.roi_align). I provide the example here:
import numpy as np
b = np.arange(100*100).reshape(1, 100, 100, 1)
boxes = [12, 30, 42, 53]
x1 = boxes[0] / (100-1)
x2 = boxes[2] / (100-1)
y1 = boxes[1] / (100-1)
y2 = boxes[3] / (100-1)

import tensorflow as tf
res = sess.run(tf.image.crop_and_resize(b, [[y1, x1, y2, x2]], [0], [7, 7])).transpose([0, 3, 1, 2])
# transpose is for comparison with the result of pytorch
print("tensorflow:", res)

# pytorch 1.7
from torchvision.ops import roi_pool, roi_align
res1 = roi_align(torch.from_numpy(b.transpose([0, 3,1,2]).astype(np.float32)), torch.from_numpy(np.asarray([[0, boxes[0], boxes[1], boxes[2], boxes[3]]]).astype(np.float32)), output_size=(7, 7))
print(res1)

The two results are:
tf:
[[[[3012.     3017.     3022.     3027.     3032.     3037.
3042.    ]
[3395.3333 3400.3333 3405.3333 3410.3333 3415.3333 3420.3333
3425.3333]
[3778.6667 3783.6667 3788.6667 3793.6667 3798.6667 3803.6667
3808.6667]
[4162.     4167.     4172.     4177.     4182.     4187.
4192.    ]
[4545.333  4550.333  4555.333  4560.333  4565.333  4570.333
4575.333 ]
[4928.6665 4933.6665 4938.6665 4943.6665 4948.6665 4953.6665
4958.6665]
[5312.     5317.     5322.     5327.     5332.     5337.
5342.    ]]]]

pytorch:
tensor([[[[3178.4287, 3182.7141, 3187.0000, 3191.2856, 3195.5713, 3199.8569,
       3204.1428],
      [3507.0000, 3511.2856, 3515.5710, 3519.8569, 3524.1426, 3528.4285,
       3532.7141],
      [3835.5715, 3839.8579, 3844.1426, 3848.4285, 3852.7144, 3857.0000,
       3861.2859],
      [4164.1426, 4168.4287, 4172.7139, 4177.0000, 4181.2856, 4185.5713,
       4189.8574],
      [4492.7139, 4497.0000, 4501.2856, 4505.5713, 4509.8569, 4514.1426,
       4518.4287],
      [4821.2861, 4825.5718, 4829.8574, 4834.1431, 4838.4287, 4842.7144,
       4847.0000],
      [5149.8574, 5154.1421, 5158.4292, 5162.7144, 5167.0005, 5171.2856,
       5175.5713]]]])

The results are really different!
Does anyone know the reason? Thanks in advance.


title: LSTM for predicting characters: cell state and hidden state in the training loop

body:
My goal is to build a model that predicts next character.
I have built a model and here is my training loop:
model = Model(input_size = 30,hidden_size = 256,output_size = len(dataset.vocab))

EPOCH = 10
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters(), lr=0.001)
init_states = None

for epoch in range(EPOCH):
    loss_overall = 0.0

    for i, (inputs,targets) in enumerate(dataloader):
        optimizer.zero_grad()

        pred = model.forward(inputs)
        loss = criterion(pred, targets)
        loss.backward()
        optimizer.step()

As you can see I return only predictions of the model, but not cell_state and hidden_state.
So alternative is : pred,cell_state,hidden_state = model.forward(inputs)
My question is: should I do it for the prediction of characters task? Why/why not?
And in general: when should I return my hidden and cell state?


title: How to get logits as neural network output

body:
Simple and short question. I have a network (Unet) which performs image segmentation. I want the logits as the output to feed into the cross entropy loss (using pytorch). Currently my final layer looks as so:
class Logits(nn.Sequential):
    def __init__(self,
                 in_channels,
                 n_class
                 ):
      super(Logits, self).__init__()

      # fully connected layer outputting the prediction layers for each of my classes
      self.conv = self.add_module('conv_out',
                        nn.Conv2d(in_channels,
                                  n_class,
                                  kernel_size = 1
                                  )
                       )
      self.activ = self.add_module('sigmoid_out',
                        nn.Sigmoid()
                       )

Is it correct to use the sigmoid activation function here? Does this give me logits?


title: How to convert raw image bytes, read from stream, to a tensor of a shape valid to perform Conv2d?

body:
Community,
I have a file of bytes organized into chunks of 16384 bytes. Each of the chunks contains an uncompressed image of size 64x64 pixels. The format of the image pixel is 8-bit ABGR.
Let's say I have successfully read the chunk into numpy.array:
buf = numpy.fromfile( dataFile, dtype=np.uint8, count=16384, offset=offs)

The question is how could I convert this array of bytes into a Pytorch tensor so that to be able to perform convolution (Conv2d)?
If I understand it properly, the aforementioned convolution (Conv2d) expects the tensor to have separate channel planes instead of a single plane of multichannel pixels.
And the extra question is how to get rid of alpha-channel in the meanwhile?


title: Variational Autoencoder's sampling problem

body:
My vae class looks like this:
class Encoder(nn.Module):
def __init__(self):
    super(Encoder, self).__init__()
    c = capacity
    self.conv1 = nn.Conv2d(in_channels=1, out_channels=c, kernel_size=4, stride=2, padding=1) # out: c x 14 x 14
    self.conv2 = nn.Conv2d(in_channels=c, out_channels=c*2, kernel_size=4, stride=2, padding=1) # out: c x 7 x 7
    self.fc_mu = nn.Linear(in_features=c*2*7*7, out_features=latent_dims)
    self.fc_logvar = nn.Linear(in_features=c*2*7*7, out_features=latent_dims)

def forward(self, x):
    x = F.relu(self.conv1(x))
    x = F.relu(self.conv2(x))
    x = x.view(x.size(0), -1) # flatten batch of multi-channel feature maps to a batch of feature vectors
    x_mu = self.fc_mu(x)
    x_logvar = self.fc_logvar(x)
    return x_mu, x_logvar

class Decoder(nn.Module):
def __init__(self):
    super(Decoder, self).__init__()
    c = capacity
    self.fc = nn.Linear(in_features=latent_dims, out_features=c*2*7*7)
    self.conv2 = nn.ConvTranspose2d(in_channels=c*2, out_channels=c, kernel_size=4, stride=2, padding=1)
    self.conv1 = nn.ConvTranspose2d(in_channels=c, out_channels=1, kernel_size=4, stride=2, padding=1)

def forward(self, x):
    x = self.fc(x)
    x = x.view(x.size(0), capacity*2, 7, 7) # unflatten batch of feature vectors to a batch of multi-channel feature maps
    x = F.relu(self.conv2(x))
    x = torch.sigmoid(self.conv1(x)) # last layer before output is sigmoid, since we are using BCE as reconstruction loss
    return x

class VariationalAutoencoder(nn.Module):
def __init__(self):
    super(VariationalAutoencoder, self).__init__()
    self.encoder = Encoder()
    self.decoder = Decoder()

def forward(self, x):
    latent_mu, latent_logvar = self.encoder(x)
    latent = self.latent_sample(latent_mu, latent_logvar)
    x_recon = self.decoder(latent)
    return x_recon, latent_mu, latent_logvar

def latent_sample(self, mu, logvar):
    if self.training:
        # the reparameterization trick
        std = logvar.mul(0.5).exp_()
        eps = torch.empty_like(std).normal_()
        return eps.mul(std).add_(mu)
    else:
        return mu

def vae_loss(recon_x, x, mu, logvar):
# recon_x is the probability of a multivariate Bernoulli distribution p.
# -log(p(x)) is then the pixel-wise binary cross-entropy.
# Averaging or not averaging the binary cross-entropy over all pixels here
# is a subtle detail with big effect on training, since it changes the weight
# we need to pick for the other loss term by several orders of magnitude.
# Not averaging is the direct implementation of the negative log likelihood,
# but averaging makes the weight of the other loss term independent of the image resolution.
recon_loss = F.binary_cross_entropy(recon_x.view(-1, 784), x.view(-1, 784), reduction='sum')

kldivergence = -0.5 * torch.sum(1 + logvar - mu.pow(2) - logvar.exp())

return recon_loss + variational_beta * kldivergence

I train it on MNIST dataset.
I want to sample it, or generate an array and give it to the decoder and see what the output will be.
The problem is that I don't really understand, what my z array should look like and what shape should it need.
Here is the code for sampling:
z = ...
input = torch.FloatTensor(z).to(device)
vae.eval()
output = vae.decoder(input)
plot_gallery(output.data.cpu().numpy(), 24, 24, n_row=5, n_col=5)