title: cuda device side error in pytorch when call torch.randn

body:
doing training phase i use latent classifier in alpha-gan
but when making random value for x-prior, cuda device side error occurs randomly in train phase
(sometime at 3000 occur, sometime at 5900 occur, whenever)
class Latent_Classifier(nn.Module):
    def __init__(self, device, latent_dim = 128):
        super(Latent_Classifier, self).__init__()
        self.encoder = nn.Sequential(
            nn.Linear(128, 750),
            nn.LeakyReLU(0.2),
            nn.Linear(750, 750),
            nn.LeakyReLU(0.2),
            nn.Linear(750, 1),
            nn.Sigmoid()
        )
        self.latent_dim = 128
        self.device = device

    def forward(self, latent_z):
        x_prior = torch.randn(latent_z.size()).to(self.device)
        eps = 1e-15

        return -(self.encoder(x_prior)+eps).log().mean() - (1-self.encoder(latent_z)+eps).log().mean()

classifier = Latent_Classifier(device = device).to(device)
lm_loss = classifier(latent_l)

my latent classifier and it's usage
class Encoder(nn.Module):
def __init__(self, nc, nef, nz, isize, device):
    super(Encoder, self).__init__()

    # Device
    self.device = device

    self.encoder = nn.Sequential(
        nn.Conv2d(nc, nef,  4, 2),
        nn.LeakyReLU(),

        nn.Conv2d(nef, nef * 2, 4, 2),
        nn.LeakyReLU(),

        nn.Conv2d(nef * 2, nef * 4, 4, 2),
        nn.LeakyReLU(),

        nn.Conv2d(nef * 4, nef * 8, 4, 2),
        nn.LeakyReLU(),

        nn.Conv2d(nef * 8, nef * 16, 4, 2),
        nn.LeakyReLU(),
    )
    out_size = isize // 64
    self.mean = nn.Linear(nef * 16 * out_size * out_size, nz)
    self.logvar = nn.Linear(nef * 16 * out_size * out_size, nz)

def reparameterize(self, mean, logvar):
    std = logvar.mul(0.5).exp_()
    multi_norm = torch.FloatTensor(std.size()).normal_().to(self.device)
    multi_norm = Variable(multi_norm)
    return multi_norm.mul(std).add_(mean)

def forward(self, inputs):
    batch_size = inputs.size(0)
    hidden = self.encoder(inputs)
    hidden = hidden.view(batch_size, -1)
    mean, logvar = self.mean(hidden), self.logvar(hidden)
    mean, logvar = F.softplus(mean), F.softplus(logvar)
    latent_z = self.reparameterize(mean, logvar)
    latent_z = torch.sigmoid(latent_z)
    return latent_z, mean, logvar

and classifier input latent_l is output of Encoder
is any something wrong in my classifier and encoder??
RuntimeError                              Traceback (most recent call last)
<ipython-input-5-3de521acb74f> in <module>()
169
170
--> 171                 lm_kld_loss = classifier(latent_l)
172                 f_kld_loss = classifier(latent_f)
173

1 frames
/content/utils/loss.py in forward(self, latent_z)
 35
 36     def forward(self, latent_z):
---> 37         x_prior = torch.randn(latent_z.size()).to(self.device)
 38         eps = 1e-15
 39

RuntimeError: CUDA error: device-side assert triggered


title: Accessing functions in the class modules of nn.Sequential

body: When running nn.Sequential, I include a list of class modules (which would be layers of a neural network). When running nn.Sequential, it calls forward functions of the modules. However each of the class modules also has a function which I would like to access when the nn.Sequential runs. How can I access and run this function when running nn.Sequential?


title: Custom gradient with state

body:
I'm trying to implement this gradient clipping paper in tensorflow, which entails storing the history of gradient norms.
I assume I need to do this using the tf.custom_gradient decorator, but how do I maintain a running list of the gradient norm history? Can I use a closure as in the pytorch version?
For reference, here is the implementation in pytorch.
import numpy as np
import torch
from enum import Enum

def _get_grad_norm(model):
    total_norm = 0
    for p in model.parameters():
        if p.grad is not None:
            param_norm = p.grad.data.norm(2)
            total_norm += param_norm.item() ** 2
    total_norm = total_norm ** (1. / 2)
    return total_norm

# written for pytorch ignite
# fire this on backwards pass
class BackwardsEvents(Enum):
    BACKWARDS_COMPLETED = 'backwards_completed'

def add_autoclip_gradient_handler(engine, model, clip_percentile):
    # Keep track of the history of gradients and select a cutoff
    # to clip values to based on percentile.
    grad_history = []

    @engine.on(BackwardsEvents.BACKWARDS_COMPLETED)
    def autoclip_gradient(engine):
        obs_grad_norm = _get_grad_norm(model)
        grad_history.append(obs_grad_norm)
        clip_value = np.percentile(grad_history, clip_percentile)
        torch.nn.utils.clip_grad_norm_(model.parameters(), clip_value)


title: No option to change gpus when using torch.distributed.init_process_gr

body:
When we run the NVIDIA code for training on Imagenet using mixed precision and the following:
$ python -m torch.distributed.launch --nproc_per_node=n main_amp.py -a resnet50 --b 224 --workers 4 --opt-level O2 ./

as soon as torch.distributed.init_process_group(backend='nccl', init_method='env://') is executed, Pytorch spawns n processes on n gpus as the following argument makes it to do so torch.distributed.launch --nproc_per_node=n. All these processes started from the 0-th index up to n-1. Unfortunately, there is no way to choose the indices of gpus starting them from anything else rather than zero or choose optional gpus.
I have also tried to use the following:
$ CUDA_VISIBLE_DEVICES=4,5,6,7 python -m torch.distributed.launch --nproc_per_node=4 main_amp.py -a resnet50 --b 224 --workers 4 --opt-level O2 ./

and I get torch._C._cuda_setDevice(device) RuntimeError: CUDA error: invalid device ordinal.
I have also changed os.environ['CUDA_VISIBLE_DEVICES'] to a string of 4,5,6,7 but got the same error as the above.
Can you please help me with this?
My guess is that we need to somehow change the source code of torch.distributed.init_process_group to let it choose gpus that are not indexed from zero.


title: Many-to-many RNN with variable output size in PyTorch

body:
I am trying to implement a many-to-many RNN network. I have defined the model and forward prop as below:
class ManyToManyRNN(nn.Module):
    def __init__(self, hid_size=100, output_dim):
        super(ManyToManyRNN, self).__init__()
        self.hidden_size = hid_size
        self.output_dim = output_dim
        self.rnn = nn.LSTM(input_size=4, hidden_size=self.hidden_size, num_layers=1, batch_first=True)
        self.fc1 = nn.Linear(self.hidden_size, self.output_dim)

    def forward(self, inputs):
        outputs = []
        rnn_out, _ = self.rnn(inputs)
        output = self.fc1(rnn_out[:, -1])
        return output

I can set the value of output_dim while initializing the class and then the value gets fixed. However, in my application, the value of the output dimension keeps changing. I can't fix the value of output_dim before hand.
My question is, how can one construct a many-to-many RNN with output dimension that is flexible? I am open to changing the architecture of the model to achieve this functionality. Any help is appreciated.


title: Problem with designing a Convolutional Autoencoder

body:
This is my first question, so please forgive if I've missed adding something.
I'm trying to create a Convolutional Autoencoder in Pytorch 1.7.0, yet am having difficulty in designing the model so that the output size is equal to the input size. I'm currently working on the MNIST dataset, with the input tensor size being 1128*28 and currently, the output is 1*1*29*29...
Can someone please help me identify the problem? *Please note that I'll incorporate the learnings afterwards.

class autoencoder(nn.Module):
    def __init__(self, hidden_node_count):
        super(autoencoder, self).__init__()

        self.conv1 = nn.Conv2d(1, 32, 5, stride=2, padding=2)
        self.conv2 = nn.Conv2d(32,32, 5, stride=2)#, padding=2)
        self.pool = nn.MaxPool2d(hidden_node_count, hidden_node_count)
        self.t_conv1 = nn.ConvTranspose2d(32, 32, 5, stride=2)#, padding=2)
        self.t_conv2 = nn.ConvTranspose2d(32,  32, 5, stride=2)#, padding=2)
        self.t_conv3 = nn.ConvTranspose2d(32,  1, 5, stride=2)#, padding=2)

        self.relu = nn.ReLU(True)
        self.tanh = nn.Tanh()

    def forward(self, x):
        print(x.size(), "input")
        x = self.conv1(x)
        x = self.relu(x)
        print(x.size(), "conv1")

        x = self.conv2(x)
        print(x.size(), "conv2")

        x = self.pool(x)
        print(x.size(), "pool")

        x = self.t_conv1(x)
        x = self.relu(x)
        print(x.size(), "deconv1")

        x = self.t_conv2(x)
        x = self.relu(x)
        print(x.size(), "deconv2")
        x = self.t_conv3(x)

        x = self.tanh(x)
        print(x.size(), "deconv3")
        return x

With its STDOUT being ->

torch.Size([1, 1, 28, 28]) input
torch.Size([1, 32, 14, 14]) conv1
torch.Size([1, 32, 5, 5]) conv2
torch.Size([1, 32, 1, 1]) pool
torch.Size([1, 32, 5, 5]) deconv1
torch.Size([1, 32, 13, 13]) deconv2
torch.Size([1, 1, 29, 29]) deconv3
torch.Size([1, 1, 29, 29])
torch.Size([1, 1, 28, 28])


title: how do I map tensor using dictionary

body:
im trying to map tensor using dictionary but envir_with_agent[1,2] return tensor(4) instead 4 and dictionary cannot map it correctly min code is below
 envir_with_agent = b.mountain.clone()
envir_with_agent[b.position_agent[0], b.position_agent[1]] = 4
print(envir_with_agent[1,2])
print(b.dict_map_display[envir_with_agent[1,2]])

dict
        self.dict_map_display={
                            1:'.',
                            2:'o',
                            3:'O',
                            4:'A',
                            8:'E',
                            9:'X'}

error

KeyError                                  Traceback (most recent call
last)  in
2 envir_with_agent[b.position_agent[0], b.position_agent[1]] = 4
3 print(envir_with_agent[1,2])
----> 4 print(b.dict_map_display[envir_with_agent[1,2]])
KeyError: tensor(4)


title: How to access class object when I use torch.nn.DataParallel()?

body:
I want to train my model using PyTorch with multiple GPUs. I included the following line:
model = torch.nn.DataParallel(model, device_ids=opt.gpu_ids)

Then, I tried to access the optimizer that was defined in my model definition:
G_opt = model.module.optimizer_G

However, I got an error:

AttributeError: 'DataParallel' object has no attribute optimizer_G

I think it is related with the definition of optimizer in my model definition. It works when I use single GPU without torch.nn.DataParallel. But it does not work with multi GPUs even though I call with module and I could not find the solution.
Here is the model definition:
class MyModel(torch.nn.Module):
    ...
   self.optimizer_G = torch.optim.Adam(params, lr=opt.lr, betas=(opt.beta1, 0.999))

I used Pix2PixHD implementation in GitHub if you want to see the full code.
Thank you,
Best.
Edit: I solved the problem by using model.module.module.optimizer_G.


title: Two tensors are on the same device, but I get the error: Expected all tensors to be on the same device, but found at least two devices, cuda:0 and cpu

body:
I have:
def loss_fn(self, pred, truth):
    truth_flat = torch.reshape(truth, (truth.size(0),-1)).to(truth.device)
    pred_flat = torch.reshape(pred, (pred.size(0),-1)).to(pred.device)

    stoi_loss = NegSTOILoss(sample_rate=16000)(pred_flat, truth_flat)
    print('truth', truth.size(), truth_flat.size(), stoi_loss)

    return torch.nn.MSELoss()(pred, truth)

As you can see, I'm making sure that it's on the same device, but I still get the error:
Expected all tensors to be on the same device, but found at least two devices, cuda:0 and cpu

Any ideas?


title: Check if each element of a tensor is contained in a list

body:
Say I have a tensor A and a container of values vals. Is there a clean way of returning a Boolean tensor of the same shape as A with each element being whether that element of A is contained within vals? e.g:
A = torch.tensor([[1,2,3],
                  [4,5,6]])
vals = [1,5]

# Desired output
torch.tensor([[True,False,False],
              [False,True,False]])


title: pytorch multiple branches of a model

body:
Hi I'm trying to make this model using pytorch.
Each input is consisted of 20 images of size 28 X 28, which is C1 ~ Cp in the image.
Each image goes to CNN of same structure, but their outputs are concatenated eventually.
I'm currently struggling with feeding multiple inputs to each of its respective CNN model.
Each model in the first box with three convolutional layers will look like this as a code, but I'm not quite sure how I can put 20 different input to separate models of same structure to eventually concatenate.
        self.features = nn.Sequential(
            nn.Conv2d(1,10, kernel_size = 3, padding = 1),
            nn.ReLU(),
            nn.Conv2d(10, 14, kernel_size=3, padding=1),
            nn.ReLU(),
            nn.Conv2d(14, 18, kernel_size=3, padding=1),
            nn.ReLU(),
            nn.Flatten(),
            nn.Linear(28*28*18, 256)
        )

I've tried out giving a list of inputs as an input to forward function, but it ended up with an error and won't go through.
I'll be more than happy to explain further if anything is unclear.


title: CUDA Illegal Memory Access error when using torch.cat

body:
I was playing around with pytorch concatenate and wanted to see if I could use an output tensor that had a different device to the input tensors, here is the code:
import torch
a = torch.ones(4)
b = torch.ones(4)
c = torch.zeros(8).cuda()
print(c)
ab = torch.cat([a,b], out=c)
print(c)

I am running this inside a jupyter notebook. pytorch version: 1.7.1
I get the following error:
...
\Anaconda3\envs\...\lib\site-packages\torch\_tensor_str.py in __init__(self, tensor)
     87
     88         else:
---> 89             nonzero_finite_vals = torch.masked_select(tensor_view, torch.isfinite(tensor_view) & tensor_view.ne(0))
     90
     91             if nonzero_finite_vals.numel() == 0:

RuntimeError: CUDA error: an illegal memory access was encountered

It happens if you try to access the tensors c (in this case with a print).
I couldnt find anything in the  documentation that said I couldn't do this, other than perhaps this line:
" ... any python sequence of tensors of the same type ... "
The error is kind of curious though... any ideas?


title: I don't find a way to use my wav file as dataset in PyTorch

body:
hello I am new to PyTorch and I want to make a simple speech recognition but I don't want to use pytorch.datasets I have some voices for dataset but I don't find anywhere to help me.
I want to use .wav files. I saw a tutorial but he used pytorch dataset.
import torch
from torch import nn, optim
import torch.nn.functional as F
import torchaudio

device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

from torchaudio.datasets import SPEECHCOMMANDS
import os

class SpeechSubset(SPEECHCOMMANDS):
    def __init__(self, subset, str=None):
        super().__init__("./", download=True)

        def load_list(filename):
            filepath = os.path.join(self._path, file.name)
            with open(filepath) as fileob:
                return [os.path.join(self._path, line.strip())]

        if subset == "validation":
            self._walker = load_list("validation_list.txt")
        elif subset == "testing":
            self._walker = load_list("testing_list.txt")
        elif subset == "training":
            excludes = load_list("validation_list.txt") + load_list("testing_list.txt")
            excludes = set(excludes)
            self._walker = [w for w in self._walker if w not in excludes]

train_set = SpeechSubset("training")
test_set = SpeechSubset("testing")

waveform, sample_rate, label, speaker_id, utterance_number = train_set[0]

sorry my english isn't too good.
EDIT
Im using the SPEECHCOMMANDS dataset but I want to use my own
thank you for reading.


title: Why is the clip_grad_norm_ function used here?

body:
I am learning LSTM with PyTorch from someone's code. Here he uses the clip_grad_norm_ function in the training process of a two layer LSTM. I want to know why he uses the clip_grad_norm_ function here, so I can understand the whole code properly (he used it in second last line).
for x, y in get_batches(data, batch_size, seq_length):
    counter += 1

    x = one_hot_encode(x, n_chars)
    inputs, targets = torch.from_numpy(x), torch.from_numpy(y)

    if(train_on_gpu):
        inputs, targets = inputs.cuda(), targets.cuda()

    h = tuple([each.data for each in h])
    net.zero_grad()

    output, h = net(inputs, h)

    loss = criterion(output, targets.view(batch_size*seq_length).long())
    loss.backward()

    nn.utils.clip_grad_norm_(net.parameters(), clip)
    opt.step()

If you need more information about question then please let me know.


title: ValueError: Target size (torch.Size([8])) must be the same as input size (torch.Size([8, 2]))

body:
I'm trying to implement a code for sentiment analysis( positive or negative labels) using BERT and i want to add a BiLSTM layer to see if I can increase the accuracy of the pretrained model from HuggingFace. I have the below code and a few questions :
import numpy as np
import pandas as pd
from sklearn import metrics
import transformers
import torch
from torch.utils.data import Dataset, DataLoader, RandomSampler, SequentialSampler
from transformers import BertTokenizer, BertModel, BertConfig
from torch import cuda
import re
import torch.nn as nn

device = 'cuda' if cuda.is_available() else 'cpu'
MAX_LEN = 200
TRAIN_BATCH_SIZE = 8
VALID_BATCH_SIZE = 4
EPOCHS = 1
LEARNING_RATE = 1e-05 #5e-5, 3e-5 or 2e-5
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')

class CustomDataset(Dataset):
 def __init__(self, dataframe, tokenizer, max_len):
  self.tokenizer = tokenizer
  self.data = dataframe
  self.comment_text = dataframe.review
  self.targets = self.data.sentiment
  self.max_len = max_len
 def __len__(self):
  return len(self.comment_text)
 def __getitem__(self, index):
  comment_text = str(self.comment_text[index])
  comment_text = " ".join(comment_text.split())

  inputs = self.tokenizer.encode_plus(comment_text,None,add_special_tokens=True,max_length=self.max_len,
   pad_to_max_length=True,return_token_type_ids=True)
  ids = inputs['input_ids']
  mask = inputs['attention_mask']
  token_type_ids = inputs["token_type_ids"]

  return {
   'ids': torch.tensor(ids, dtype=torch.long),
   'mask': torch.tensor(mask, dtype=torch.long),
   'token_type_ids': torch.tensor(token_type_ids, dtype=torch.long),
   'targets': torch.tensor(self.targets[index], dtype=torch.float)
  }
train_size = 0.8
train_dataset=df.sample(frac=train_size,random_state=200)
test_dataset=df.drop(train_dataset.index).reset_index(drop=True)
train_dataset = train_dataset.reset_index(drop=True)

print("FULL Dataset: {}".format(df.shape))
print("TRAIN Dataset: {}".format(train_dataset.shape))
print("TEST Dataset: {}".format(test_dataset.shape))

training_set = CustomDataset(train_dataset, tokenizer, MAX_LEN)
testing_set = CustomDataset(test_dataset, tokenizer, MAX_LEN)
train_params = {'batch_size': TRAIN_BATCH_SIZE,'shuffle': True,'num_workers': 0}
test_params = {'batch_size': VALID_BATCH_SIZE,'shuffle': True,'num_workers': 0}
training_loader = DataLoader(training_set, **train_params)
testing_loader = DataLoader(testing_set, **test_params)

class BERTClass(torch.nn.Module):
 def __init__(self):
   super(BERTClass, self).__init__()
   self.bert = BertModel.from_pretrained('bert-base-uncased',return_dict=False, num_labels =2)
   self.lstm = nn.LSTM(768, 256, batch_first=True, bidirectional=True)
   self.linear = nn.Linear(256*2,2)

 def forward(self, ids , mask,token_type_ids):
  sequence_output, pooled_output = self.bert(ids, attention_mask=mask, token_type_ids = token_type_ids)
  lstm_output, (h, c) = self.lstm(sequence_output)  ## extract the 1st token's embeddings
  hidden = torch.cat((lstm_output[:, -1, :256], lstm_output[:, 0, 256:]), dim=-1)
  linear_output = self.linear(lstm_output[:, -1].view(-1, 256 * 2))

  return linear_output

model = BERTClass()
model.to(device)
print(model)
def loss_fn(outputs, targets):
 return torch.nn.BCEWithLogitsLoss()(outputs, targets)
optimizer = torch.optim.Adam(params =  model.parameters(), lr=LEARNING_RATE)

def train(epoch):
 model.train()
 for _, data in enumerate(training_loader, 0):
  ids = data['ids'].to(device, dtype=torch.long)
  mask = data['mask'].to(device, dtype=torch.long)
  token_type_ids = data['token_type_ids'].to(device, dtype=torch.long)
  targets = data['targets'].to(device, dtype=torch.float)
  outputs = model(ids, mask, token_type_ids)
  optimizer.zero_grad()
  loss = loss_fn(outputs, targets)
  if _ % 5000 == 0:
   print(f'Epoch: {epoch}, Loss:  {loss.item()}')
  optimizer.zero_grad()
  loss.backward()
  optimizer.step()

for epoch in range(EPOCHS):
  train(epoch)

So on the above code I ran into the error :  Target size (torch.Size([8])) must be the same as input size (torch.Size([8, 2])) .  Checked online and tried to use targets = targets.unsqueeze(2) but then I get another error that I must use values from [-2,1] for unsqueeze. I also tried to modify the loss function to
def loss_fn(outputs, targets):
 return torch.nn.BCELoss()(outputs, targets)

but I still receive the same error. Can someone advise if there is a solution to this problem? Or what can I do to make this work fine?  Many thanks in advance.


title: I want to customise the last layer of VGG 19 architecture for a classification. which will be more useful keras or pytorch?

body: I want to customise the last layer of VGG 19 architecture for a classification problem. which will be more useful keras or pytorch?


title: Basic DNN with highly imbalanced dataset -- network predicts same labels

body:
I will try to explain my issue at a high level, and I hope I'd be able to get some better understanding of the ML behind it all.

I am working with aggregated features extracted from audio files, so each feature vector is of size (1xN). The output would be a single sentiment label, Positive, Neutral, or Negative. I mapped these to 2, 1, 0 respectively (the labels are discrete by design, but maybe I could make it continuous?)

The dataset I am using is 90% neutral, 6% negative, and 4% positive, and I split these into train/dev/test. I wrote up a basic DNN in PyTorch, and have been training using CrossEntropyLoss and SGD (with nesterov momentum). The issue I am running into is that the network, after seeing only ~10% of the data, starts to predict only netural labels. The class weights converge to something like

tensor([[-0.9255],
        [ 1.9352],
        [-1.1473]])

no matter what 1xN feature vectors you feed in. I would appreciate guidance on how to address this issue.

For reference, the architecture is

DNNModel(
  (in_layer): Linear(in_features=89, out_features=1024, bias=True)
  (fcs): Sequential(
    (0): Linear(in_features=1024, out_features=512, bias=True)
    (1): Linear(in_features=512, out_features=256, bias=True)
    (2): Linear(in_features=256, out_features=128, bias=True)
  )
  (out_layer): Sequential(
    (0): SequenceWise (
    Linear(in_features=128, out_features=3, bias=True))
  )
)

def forward(self, x):
    x =  F.relu(self.in_layer(x))
    for fc in self.fcs:
        x = F.relu(fc(x))
    x = self.out_layer(x)
    return x

Not sure if NN actually makes sense -- could it be the relus between each hidden layer or the bias? Or something else. Thanks!

EDIT: Moved to Data Science Stack Exchange, since this is more relevant there. link


title: Pytorch Index a tensor of [n*n*3] by two tensors of size[n,n]

body:
In numpy, I can index in the following:

a = np.random.randn(2,2,3)
b = np.eye(2,2).astype(np.uint8)
c = np.eye(2,2).astype(np.uint8)
print(a)
print("diff")
print(a[b,c,:])

, in which a[b, c, :] is a tensor of 2*2.

[[[-1.01338087  0.70149058  0.55268617]
  [ 2.56941124  1.12720312 -0.07219555]]

 [[-0.04084548  0.17018995  2.14229567]
  [-0.68017558 -0.91788125  1.1719151 ]]]
diff
[[[-0.68017558 -0.91788125  1.1719151 ]
  [-1.01338087  0.70149058  0.55268617]]

 [[-1.01338087  0.70149058  0.55268617]
  [-0.68017558 -0.91788125  1.1719151 ]]]

But in Pytorch, I could not do the index in the same way like a[b,c,:]. Who knows how to do this. Thanks~


title: One gpu uses more memory than others during training

body: I use multigpu to train a model with pytorch. One gpu uses more memory than others, causing "out-of-memory". Why would one gpu use more memory? Is it possible to make the usage more balanced? Is there other ways to reduce memory usage? (Deleting variables that will not be used anymore...?) The batch size is already 1. Thanks.


title: Convert custom C++/Cuda layer to Pytorch/Tensorflow

body:
I would like to use in a Pytorch model (or Tensorflow if it works better) a custom C++/CUDA Caffer layer. I don't know if there is a way to do it, or if I must reimplement the whole layer in Pytorch/Tensorflow. This is the repo that has a custom caffe implementation : https://github.com/MPI-IS/bilateralNN.

Thanks!


title: Proper way to extract embedding weights for CBOW model?

body:
I'm currently trying to implement the CBOW model on managed to get the training and testing, but am facing some confusion as to the "proper" way to finally extract the weights from the model to use as our word embeddings.

Model

class CBOW(nn.Module):
    def __init__(self, config, vocab):
        self.config = config # Basic config file to hold arguments.
        self.vocab = vocab
        self.vocab_size = len(self.vocab.token2idx)
        self.window_size = self.config.window_size

        self.embed = nn.Embedding(num_embeddings=self.vocab_size, embedding_dim=self.config.embed_dim)
        self.linear = nn.Linear(in_features=self.config.embed_dim, out_features=self.vocab_size)

    def forward(self, x):
        x = self.embed(x)
        x = torch.mean(x, dim=0) # Average out the embedding values.
        x = self.linear(x)

        return x

Main process

After I run my model through a Solver with the training and testing data, I basically told the train and test functions to also return the model that's used. Then I assigned the embedding weights to a separate variable and used those as the word embeddings.

Training and testing was conducted using cross entropy loss, and each training and testing sample is of the form ([context words], target word).

def run(solver, config, vocabulary):
    for epoch in range(config.num_epochs):
        loss_train, model_train = solver.train()
        loss_test, model_test = solver.test()

        embeddings = model_train.embed.weight

I'm not sure if this is the correct way of going about extracting and using the embeddings. Is there usually another way to do this? Thanks in advance.


title: Pytorch Extension: difference of performance between the extension that was compiled by g++ and that was built by setuptools

body:
I wrote a cpp extension for torch which is a custom convolutional function.

Firstly, I compiled this function with g++ directly which was used for testing, the latency is 5 milliseconds.

Secondly, I tried to integrate this function to torch and installed this extension by setuptools, following the steps shown in the tutorial provided by torch. However, the latency is now 16 milliseconds.

The function invokation will consumes about 1-2 ms, so why the performance differs so much?

The compilation by g++ directly was done by

g++ -pthread -mavx2 -mfma ...

and the directives in the source file includes

#pragma GCC diagnostic ignored "-Wformat"

#pragma STDC FP_CONTRACT ON

#pragma GCC optimize("O3","unroll-loops","omit-frame-pointer","inline") //Optimization flags

// #pragma GCC option("arch=native","tune=native","no-zero-upper") //Enable AVX

#pragma GCC target("avx")

These directives were also included in the file built by setuptools. The "setup.py" file is

setup(
    name = 'cusconv_cpp',
    ext_modules=[
        CppExtension(name='cusconv_cpp', sources=['src/cusconv.cpp'],
        extra_compile_args={'cxx': ['-O3', '-pthread', '-mavx2', '-mfma']})
    ],
    cmdclass={
        'build_ext': BuildExtension
    })

The output log by setuptools for buiding is

x86_64-linux-gnu-gcc -pthread -DNDEBUG -g -fwrapv -O2 -Wall -g -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 -fPIC -I/home/max/.local/lib/python3.6/site-packages/torch/lib/include -I/home/max/.local/lib/python3.6/site-packages/torch/lib/include/torch/csrc/api/include -I/home/max/.local/lib/python3.6/site-packages/torch/lib/include/TH -I/home/max/.local/lib/python3.6/site-packages/torch/lib/include/THC -I/usr/include/python3.6m -c src/indconv.cpp -o build/temp.linux-x86_64-3.6/src/indconv.o -O3 -pthread -mavx2 -mfma -DTORCH_API_INCLUDE_EXTENSION_H -DTORCH_EXTENSION_NAME=indconv_cpp -D_GLIBCXX_USE_CXX11_ABI=0 -std=c++11

which indeed includes those flags but many other flags were also used.
Anyone has any ideas?


title: Why multiprocessing suddenly stops after a certain number of tasks?

body:
I'm trying to write some code to parallelize a bunch of tasks. Basically, the script is organized as the following.

import multiprocessing as mp

def obj_train(x):
    return x.train()

class ServerModel(nn.Module):
    self.S = nn.Parameter(torch.rand(x, y), requires_grad=True)

class ClientModel(nn.Module):
    self.S = nn.Parameter(torch.rand(x, y), requires_grad=True)
    self.U = nn.Parameter(torch.rand(x, y), requires_grad=True)

class Server:
    def __init__(self, model):
        self.model = model
        ...

    def train(clients):
        for i, c in enumerate(clients):
            sd = c.model.state_dict()
            sd['S'] = self.model.S
            c.model.load_state_dict(sd)
        self.c_list = random.sample(clients, 200)
        pool = mp.Pool(mp.cpu_count()-1)
        results = pool.map(obj_train, self.c_list)
        pool.close()
        pool.join()
        print("Training complete")

class Client:
    def __init__(self, client_id, model, train_set):
        self.id = client_id
        self.model = model
        self.train_set = train_set

    def train(self):
        self.optimizer = optim.SGD([self.model.S, self.model.U])
        for i in self.train_set:
            loss = self.model(i)
            loss.backward()
            self.optimizer.step()
        print("Trained client %d", self.id)

        return self.model.S

if __name__ == '__main__':
    ...
    server = Server(server_model)
    clients = [Client(u, ClientModel(), train_set[u]) for u in range(n_clients)]
    server.train(clients)

Ok, the problem is in multiprocessing. I tried with a lot of approaches but all of them gives me the same problem. Server should manage the training of 200 clients, but after a certain number of trainings (it depends on the approach, but approx 50-100), the script completely stucks and cores of the CPU stop working.

Have you any ideas? Other approaches I tried are for example mp.Pool and with ProcessPoolExecutor.

Thank you for your help.


title: Why Keras behave better than Pytorch under the same network configuration?

body:
Recently,  I have compared unet++ implementation of Keras version and Pytorch version on the same dataset. However, with Keras the loss decrease continuously and the accuracy is higher after 10 epochs, while with Pytorch the loss decrease unevenly and the  accuracy is lower after 10 epochs. Anyone has met such problems and has any answers?
the final pytorch training process is like:

2019-12-15 18:14:20 Epoch:9 Iter: 1214/1219  loss:0.464673 acc:0.581713
2019-12-15 18:14:21 Epoch:9 Iter: 1215/1219  loss:0.450462 acc:0.584101
2019-12-15 18:14:21 Epoch:9 Iter: 1216/1219  loss:0.744811 acc:0.293406
2019-12-15 18:14:22 Epoch:9 Iter: 1217/1219  loss:0.387612 acc:0.735630
2019-12-15 18:14:23 Epoch:9 Iter: 1218/1219  loss:0.767146 acc:0.364759

the final keras training process is like:

685/690 [============================>.] - ETA: 2s - loss: 0.4940 - acc:  0.7309
686/690 [============================>.] - ETA: 1s - loss: 0.4941 - acc: 0.7306
687/690 [============================>.] - ETA: 1s - loss: 0.4939 - acc:  0.7308
688/690 [============================>.] - ETA: 0s - loss: 0.4942 - acc:  0.7303
689/690 [============================>.] - ETA: 0s - loss: 0.4943 - acc: 0.7302


title: Strange Move Assignment Operator Signature

body:
I came across an unfamiliar move assignment operator signature in Pytorch' tensor backend (ATen, source).
Just out of curiosity, what does the && operator do at the end of

Tensor & Tensor::operator=(Tensor && rhs) &&

While I'm familiar with move semantics and the usual copy/move constructor and assignment operator signatures, I could not find any documentation online about the syntax above.

I would be grateful if someone could explain what this operator does, how it differs from the usual move assignment operation, and when it should be used.


title: How most efficiently compute the diagonal of a matrix product

body:
I want to compute the following:

import numpy as np
n= 3
m = 2
x = np.random.randn(n,m)

#Method 1
y = np.zeros(m)
for i in range(m):
    y[i] = x[:,i] @ x[:,i]

#Method 2
y2 = np.diag(x.T @ x)

The first method has the problem that it uses a for loop, which can't be very effecient (I need to do this in pytorch on a GPU millions of times)

The second method computes the full matrix product, when I only need the diagonal entries, so that can't be very efficient either.

I'm wondering whether there exist any clever way of doing this?


title: Weird behavior when calling cuda() on different tensors in pytorch

body:
I am trying to train a pytorch neural network on a GPU device. In order to do so, I load my inputs and network onto the default cuda enabled GPU decive. However, when I load my inputs, the model's weights do not stay cuda tensors. Here is my train function

def train(network: nn.Module, name: str, learning_cycles: dict, num_epochs):
    # check we have a working gpu to train on
    assert(torch.cuda.is_available())
    # load model onto gpu
    network = network.cuda()
    # load train and test data with a transform
    transform = transforms.Compose(
        [transforms.ToTensor(),
         transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])

    train_set = torchvision.datasets.CIFAR10(root='./data', train=True,
 download=True, transform=transform)
    train_loader = torch.utils.data.DataLoader(train_set, batch_size=128,
shuffle=True, num_workers=2)

    criterion = nn.CrossEntropyLoss()
    optimizer = optim.SGD(network.parameters(), lr=0.001, momentum=0.9)

    for epoch in range(num_epochs):
        for i, data in enumerate(train_loader, 0):
            inputs, labels = data
            # load inputs and labels onto gpu
            inputs, labels = inputs.cuda(), labels.cuda()
            optimizer.zero_grad()
            outputs = network(inputs)
            loss = criterion(outputs, labels)
            loss.backward()
            optimizer.step()

When calling train, I get the following error.

RuntimeError: Input type (torch.cuda.FloatTensor) and weight type (torch.FloatTensor) should be the same

Interestingly, when I delete the line inputs, labels = inputs.cuda(), labels.cuda() I get the error RuntimeError: Input type (torch.FloatTensor) and weight type (torch.cuda.FloatTensor) should be the same

I would very much like to train my network, and I have searched the internet to no avail. Any good ideas?


title: pytorch got None after backward()

body:
I am learning pytorch and write a simple code as below.

import torch
x = torch.randn(3,requires_grad=True).cuda()
print(x)
y = x * x
print(y)
y.backward(torch.tensor([1,1.0,1]).cuda())
print(x.grad)

tensor([ 0.5934, -1.8813, -0.7817], device='cuda:0', grad_fn=<CopyBackwards>)
tensor([0.3521, 3.5392, 0.6111], device='cuda:0', grad_fn=<MulBackward0>)
None

if I change the code as

from torch.autograd import Variable
import torch
# x = torch.randn(3,requires_grad=True).cuda()
x = Variable(torch.randn(3).cuda(),requires_grad=True)
print(x)
y = x * x
print(y)
y.backward(torch.tensor([1,1.0,1]).cuda())
print(x.grad)

tensor([0.9800, 0.3597, 1.6315], device='cuda:0', requires_grad=True)
tensor([0.9605, 0.1294, 2.6617], device='cuda:0', grad_fn=<MulBackward0>)
tensor([1.9601, 0.7194, 3.2630], device='cuda:0')

The grad is ok. But why? I hate the Variable class.

env

python:3.8

pytorch:1.5

cuda :10.2


title: Array Slicing with step 2

body:
Have array like

arr = [1,2,3,4,5,6,7,8,9,10].

How I can get array like this:

[1,2,5,6,9,10]

take 2 elements with step 2(::2)

I try something like arr[:2::2].it's not work


title: Best way to output prediction result for a test set from a model with 1 output (binary classification)

body:
This seems simple, but I'm looking for the best way to output the prediction results for a model with 1 output (binary classification model). My labels are 0 and 1 in this example. Now I can just say if model output > 0.5 label is 1. But I'm sort of guessing this is correct. I am hence wondering if there is a better approach, like

prediction = np.argmax(y_hat.detach().numpy, axis=0) for multiple classes.

Currently I'm doing this to test:

test = [[0,0],[0,1],[1,1],[1,0]]

for trial in test:
  Xtest = torch.Tensor(trial)
  y_hat = logreg_clf(Xtest)

  if y_hat > 0.5:
    prediction = 1
  else:
    prediction = 0
  # prediction = np.argmax(y_hat.detach().numpy, axis=0)
  print("{0} xor {1} = {2}".format(int(Xtest[0]), int(Xtest[1]), prediction))

Was also wondering if there is a function to automatically get the accuracy or confusion matrix for test set 'test'.

FYI my model:

import torch.nn as nn
import torch.nn.functional as F

class LogisticRegression(nn.Module):
  # input_size: Dimensionality of input feature vector.
  # num_classes: The number of classes in the classification problem.
  def __init__(self, input_size, num_classes):
    # Always call the superclass (nn.Module) constructor first!
    super(LogisticRegression, self).__init__()
    # Set up the linear transform
    self.linear = nn.Linear(input_size, num_classes)
    # I do not yet include the sigmoid activation after the linear
    # layer because our loss function will include this as you will see later

  # Forward's sole argument is the input.
  # input is of shape (batch_size, input_size)
  def forward(self, x):
    # Apply the linear transform.
    # out is of shape (batch_size, num_classes).
    out = self.linear(x)
    out = F.sigmoid(out)
    # Softmax the out tensor to get a log-probability distribution
    # over classes for each example.
    return out

# Binary classifiation
num_outputs = 1
num_input_features = 2

# Create the logistic regression model
logreg_clf = LogisticRegression(num_input_features, num_outputs)

print(logreg_clf)

import torch
lr_rate = 0.001  # alpha

# training set of input X and labels Y
X = torch.Tensor([[0,0],[0,1], [1,0], [1,1]])
Y = torch.Tensor([0,1,1,0]).view(-1,1) #view is similar to numpy.reshape() here it makes it into a column

# Run the forward pass of the logistic regression model
sample_output = logreg_clf(X) #completely random at the moment
print(X)

loss_function = nn.BCELoss()
# SGD: stochastic gradient descent is used to train/fit the model
optimizer = torch.optim.SGD(logreg_clf.parameters(), lr=lr_rate)

import numpy as np
# from torch.autograd import Variable

#training loop:

epochs = 2001 #how many times we go through the training set
steps = X.size(0) #steps = 4; we have 4 training examples (I know, tiny training set :)

for i in range(epochs):
    for j in range(steps):
        # randomly sample from the training set:
        data_point = np.random.randint(X.size(0))
        # store the retrieved datapoint into 2 separate variables of the right shape
        x_var = torch.Tensor(X[data_point]).unsqueeze(0)
        y_var = torch.Tensor(Y[data_point])

        optimizer.zero_grad() # empty (zero) the gradient buffers
        y_hat = logreg_clf(x_var) #get the output from the model
        print(y_hat)
        print(y_var)
        loss = loss_function(y_hat, y_var) #calculate the loss
        loss.backward() #backprop
        optimizer.step() #does the update

    if i % 500 == 0:
        print ("Epoch: {0}, Loss: {1}, ".format(i, loss.data.numpy()))


title: are there no equivalents to tensorflow/tensorflow:latest-gpu-jupyter for pytorch?

body:
I've installed a handful of docker containers for this purpose but have run into a dead end. I can't seem to find something like the tensorflow docker containers for pytorch. Ideally I could install a container up to the Google Colab specs so I could run torch or tensorflow.
What am I missing? I feel like this should be easy to find.


title: PyTorch error "RuntimeError: Trying to backward through the graph a second time, but the buffers have already been freed"

body:
I'm getting this error when trying to train a neural network in PyTorch. It says "RuntimeError: Trying to backward through the graph a second time, but the buffers have already been freed. Specify retain_graph=True when calling backward the first time." I'm not calling backward more than once so I have no idea why it's happening. The only solutions to this error I can find are about LSTMs but I'm not using one of those either. Here's the relevant part of the code:
class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        self.conv1 = nn.Conv1d(1, 128, 80, 4)
        self.bn1 = nn.BatchNorm1d(128)
        self.d1 = nn.Dropout(0.1)
        self.pool1 = nn.MaxPool1d(4)
        self.conv2 = nn.Conv1d(128, 128, 3)
        self.bn2 = nn.BatchNorm1d(128)
        self.d2 = nn.Dropout(0.1)
        self.pool2 = nn.MaxPool1d(4)
        self.conv3 = nn.Conv1d(128, 256, 2)
        self.bn3 = nn.BatchNorm1d(256)
        self.d3 = nn.Dropout(0)
        self.pool3 = nn.MaxPool1d(4)
        self.conv4 = nn.Conv1d(256, 512, 1)
        self.bn4 = nn.BatchNorm1d(512)
        self.d4 = nn.Dropout(0)
        self.pool4 = nn.MaxPool1d(4)
        self.avgPool = nn.AvgPool1d(7) #input should be 512x7 so this outputs a 512x1
        self.fc1 = nn.Linear(512, 100)
        self.d5 = nn.Dropout(0)
        self.fc2 = nn.Linear(100, 8)
    def forward(self, x):
        x = x.unsqueeze(1)
        x = self.conv1(x)
        x = F.relu(self.d1(self.bn1(x)))
        x = self.pool1(x)
        x = self.conv2(x)
        x = F.relu(self.d2(self.bn2(x)))
        x = self.pool2(x)
        x = self.conv3(x)
        x = F.relu(self.d3(self.bn3(x)))
        x = self.pool3(x)
        x = self.conv4(x)
        x = F.relu(self.d4(self.bn4(x)))
        x = self.pool4(x)
        x = self.avgPool(x)
        x = x.permute(0, 2, 1) #change the 512x1 to 1x512
        x = F.relu(self.d5(self.fc1(x)))
        x = self.fc2(x)
        x = x.reshape(x.shape[0],-1)
        x = x.log_softmax(dim=1)
        return x
model = Net()
device = torch.device("cuda:0")
model.to(device)

opt = torch.optim.Adam(model.parameters())
loss_func = torch.nn.NLLLoss()

log_interval = 1000

def train(model, epoch, data_loader):
    print("\n\n#### Starting training epoch %s ####\n\n" % epoch)
    model.train()
    for batch_idx, (x, y) in enumerate(data_loader):
        opt.zero_grad()
        x = x.to(device)
        y = y.to(device)
        pred = model(x)
        loss = loss_func(pred, y) #the loss functions expects a batchSizex10 input
        loss.backward()
        opt.step()
        if batch_idx % log_interval == 999: #print training stats
            print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(
                epoch, batch_idx * len(x), len(train_loader.dataset),
                100. * batch_idx / len(train_loader), loss))

for epoch in range(1, 1000):
    # Train and evaluate model
    train(model, epoch, train_loader)


title: Compute gradients when using BERT classifier on top of image captioning model

body:
I have an image captioning model, and I want to ensure that it generates sentences with correct facts by using a pretrained BERT classifier on top of it (the model generates a sentence --> we extract “facts” from the sentence using a pretrained classifier --> we calculate the loss on the classification).
I want to use the Huggingface Transformers library, but because their input is a sentence that they then tokenize, I am unsure of how to handle or compute the gradients.
Does anyone have any ideas how I can compute the gradients?


title: Dumping Image data and load using pytorch dataloader

body:
I want to dump the data so that I can load it back for training my model.
My code snipped for dumping the data:
for batch_idx, (image, label) in enumerate(dataloader):
    image, label = image.to(device), label.to(device)
    perturbed_image = attack.perturb(image, label)

    #---------- Classifier ----------
    predict_A = classifier(perturbed_image)
    pred_label = torch.max(predict_A.data, 1)[1]

    if pred_label != label:
        adv_data.append( (perturbed_image.to("cpu"), label.to("cpu")) )

Is there any other way I can dump it correctly so as to load it in the torch.utils.data.DataLoader.


title: when we reshape numpy array, how is the stride size inferred?

body:
I have an 1x1024 1-d array (flattened image). To see the image, I want to reshape its size as 32x32.
I can easily achieve this by doing x.reshape(-1,32) and it works as I intended. It doesn't screw the image. It reads the 1d array with 32-width stride each time.
Say this time, there are 4 images, sized 32 by 8. What's the safe way to reshape this?
What is the logic behind how strides are defined? Is it always starting from the largest dimension (say, 3d->2d->1d)? It seems like it..
In [2]: a = np.arange(1024)

In [3]: a.reshape(4,32,8)
Out[3]:
array([[[   0,    1,    2, ...,    5,    6,    7],
        [   8,    9,   10, ...,   13,   14,   15],
        [  16,   17,   18, ...,   21,   22,   23],
        ...,
        [ 232,  233,  234, ...,  237,  238,  239],
        [ 240,  241,  242, ...,  245,  246,  247],
        [ 248,  249,  250, ...,  253,  254,  255]],

       [[ 256,  257,  258, ...,  261,  262,  263],
        [ 264,  265,  266, ...,  269,  270,  271],
        [ 272,  273,  274, ...,  277,  278,  279],
        ...,
        [ 488,  489,  490, ...,  493,  494,  495],
        [ 496,  497,  498, ...,  501,  502,  503],
        [ 504,  505,  506, ...,  509,  510,  511]],

       [[ 512,  513,  514, ...,  517,  518,  519],
        [ 520,  521,  522, ...,  525,  526,  527],
        [ 528,  529,  530, ...,  533,  534,  535],
        ...,
        [ 744,  745,  746, ...,  749,  750,  751],
        [ 752,  753,  754, ...,  757,  758,  759],
        [ 760,  761,  762, ...,  765,  766,  767]],

       [[ 768,  769,  770, ...,  773,  774,  775],
        [ 776,  777,  778, ...,  781,  782,  783],
        [ 784,  785,  786, ...,  789,  790,  791],
        ...,
        [1000, 1001, 1002, ..., 1005, 1006, 1007],
        [1008, 1009, 1010, ..., 1013, 1014, 1015],
        [1016, 1017, 1018, ..., 1021, 1022, 1023]]])

In [4]: a.reshape(4,-1,8)
Out[4]:
array([[[   0,    1,    2, ...,    5,    6,    7],
        [   8,    9,   10, ...,   13,   14,   15],
        [  16,   17,   18, ...,   21,   22,   23],
        ...,
        [ 232,  233,  234, ...,  237,  238,  239],
        [ 240,  241,  242, ...,  245,  246,  247],
        [ 248,  249,  250, ...,  253,  254,  255]],

       [[ 256,  257,  258, ...,  261,  262,  263],
        [ 264,  265,  266, ...,  269,  270,  271],
        [ 272,  273,  274, ...,  277,  278,  279],
        ...,
        [ 488,  489,  490, ...,  493,  494,  495],
        [ 496,  497,  498, ...,  501,  502,  503],
        [ 504,  505,  506, ...,  509,  510,  511]],

       [[ 512,  513,  514, ...,  517,  518,  519],
        [ 520,  521,  522, ...,  525,  526,  527],
        [ 528,  529,  530, ...,  533,  534,  535],
        ...,
        [ 744,  745,  746, ...,  749,  750,  751],
        [ 752,  753,  754, ...,  757,  758,  759],
        [ 760,  761,  762, ...,  765,  766,  767]],

       [[ 768,  769,  770, ...,  773,  774,  775],
        [ 776,  777,  778, ...,  781,  782,  783],
        [ 784,  785,  786, ...,  789,  790,  791],
        ...,
        [1000, 1001, 1002, ..., 1005, 1006, 1007],
        [1008, 1009, 1010, ..., 1013, 1014, 1015],
        [1016, 1017, 1018, ..., 1021, 1022, 1023]]])

In [5]: a.reshape(4,8,32)
Out[5]:
array([[[   0,    1,    2, ...,   29,   30,   31],
        [  32,   33,   34, ...,   61,   62,   63],
        [  64,   65,   66, ...,   93,   94,   95],
        ...,
        [ 160,  161,  162, ...,  189,  190,  191],
        [ 192,  193,  194, ...,  221,  222,  223],
        [ 224,  225,  226, ...,  253,  254,  255]],

       [[ 256,  257,  258, ...,  285,  286,  287],
        [ 288,  289,  290, ...,  317,  318,  319],
        [ 320,  321,  322, ...,  349,  350,  351],
        ...,
        [ 416,  417,  418, ...,  445,  446,  447],
        [ 448,  449,  450, ...,  477,  478,  479],
        [ 480,  481,  482, ...,  509,  510,  511]],

       [[ 512,  513,  514, ...,  541,  542,  543],
        [ 544,  545,  546, ...,  573,  574,  575],
        [ 576,  577,  578, ...,  605,  606,  607],
        ...,
        [ 672,  673,  674, ...,  701,  702,  703],
        [ 704,  705,  706, ...,  733,  734,  735],
        [ 736,  737,  738, ...,  765,  766,  767]],

       [[ 768,  769,  770, ...,  797,  798,  799],
        [ 800,  801,  802, ...,  829,  830,  831],
        [ 832,  833,  834, ...,  861,  862,  863],
        ...,
        [ 928,  929,  930, ...,  957,  958,  959],
        [ 960,  961,  962, ...,  989,  990,  991],
        [ 992,  993,  994, ..., 1021, 1022, 1023]]])


title: Dataloder with IterableDataset throws exception: expected unspecified batch_sampler option

body:
I tried to implement custom IteratableDataset with Dataloder as follows:
class LazyDataset(IterableDataset):
    def __init__(self, file_dir):
        self.file_dir = file_dir
        self.files = os.listdir(file_dir)

    def __len__(self) -> int:
        return len(self.files)

    def __iter__(self):
        for file in self.files:
           # read and preprocess file
           yield data

train_dataset = LazyDataset('/train')
train_loader = DataLoader(train_dataset,
                          batch_size=args.train_batch_size,
                          num_workers=args.n_workers
                          )

Its works fine but when I integrate with ignite-engine it throws following exception:
"batch_sampler option, but got batch_sampler={}".format(batch_sampler))
ValueError: DataLoader with IterableDataset: expected unspecified batch_sampler option, but got batch_sampler=<ignite.engine.engine.ReproducibleBatchSampler

How can I fix the above problem?


title: n_jobs > 1 using sklearn and pytorch is possible inside Neuraxle?

body:
I built my own sklearn-like estimator using pytorch training inside GPU (cuda) and it works fine with RandomizedSearchCV when n_jobs==1. When n_jobs > 1, I get the following error:

PicklingError: Can't pickle <class 'main.LSTM'>: attribute lookup LSTM on main failed

This is the piece of code giving me the error:
model = my_model(input_size=1, hidden_layer_size=80, n_lstm_units=3, bidirectional=False,
                 output_size=1, training_batch_size=60, epochs=7500, device=device)
model.to(device)

hidden_layer_size = random.uniform(40, 200, 20).astype("int")
n_lstm_units = arange(1, 4)

parametros = {'hidden_layer_size': hidden_layer_size, 'n_lstm_units': n_lstm_units}

splitter = ShuffleSplit()

regressor = model
cv_search = \
    RandomizedSearchCV(estimator=regressor, cv=splitter,
                  search_spaces=parametros,
                  refit=True,
                  n_iter=4,
                  verbose=1,
                  n_jobs=2,
                  scoring=make_scorer(mean_squared_error,
                                      greater_is_better=False,
                                      needs_proba=False))

cv_search = MetaSKLearnWrapper(cv_search)
cv_search.fit(X, y)

Using Neuraxle wrapper leads to exactly same error, changes nothing.
I found closest solution here, but still don't know how to use RandomizedSearchCV within Neuraxle. It is a brand new project, so I couldn't find an answer on their docs or community examples. If anyone can give me an example or a good indication it will save my life. Thank you
Ps: Any way to run RandomizedSearchCV with my pytorch model on the gpu without Neuraxle also helps, I just need n_jobs>1.
Ps2: My model has a fit() method that creates and moves tensors to the gpu and works already tested.


title: Creating custom dataset for Pose estimation using blender and pybullet

body: I am trying to create my own dataset for pose estimation of YCB colliding objects. The Pose of the object (obj.location and obj.rotation of blender objects used for rendering) is different from the ground truth. I know this because I tried to overlay the object model with the Pose saved from the blender and it does not aline with the object. Did any face a similar issue? any idea how to get the ground truth pose.


title: Errors in training VAE with Tensorflow and switching to PyTorch

body:
I'm trying to implement a variational autoencoder that aims to perform unsupervised classification (and possibly generation) of astronomical images. Each image is of size 63x63x3 pixels. The problem with implementing it in tensorflow is that there are too many errors that keep coming up time and again. Initially there was the error on 'inputs to eager execution cannot be keras symbolic tensors' and thus I had to use experimental_run_tf_function=FALSE as an argument to model.compile. Afterwards the model started training but the loss function was nan. I have used Xavier normal initialization, I tried to change the learning rate, and even checked for inputs having nan values by including np.count_nonzero(np.isnan()), but now there is another error that points out incompatible input shapes. Previously inputs to each layer were of compatible shape.
I heard that there are issues with Tensorflow 2.3.0 and thus I downgraded the version to 1.15.2. But the problems still persist.
In my opinion, Tensorflow works really well with typical ConvNets and dense neural networks, but it doesn't seem to work well in implementing VAEs. I don't have any experience in using PyTorch. Is this a point to switch to PyTorch for implementing VAEs?
I have included my code as well. Kindly suggest if any changes can be made, or if it is better to switch to PyTorch.
import numpy as np
import pandas as pd
from keras import Input
from keras.models import Model
from keras.layers import Conv2D, MaxPooling2D, Conv2DTranspose, Dense, Flatten, Dropout, Reshape, Lambda
from keras import backend as k
from keras.initializers import glorot_normal

!wget -O triplets.norm.npy https://storage.googleapis.com/dmitryduev/braai.triplets.norm.programid1.npy

triplets = np.load('triplets.norm.npy')
test_split = 0.1
np.count_nonzero(np.isnan(triplets))
X_train, X_test, y_train, y_test = train_test_split(triplets, df.label, test_size = test_split)

initializer = glorot_normal()

def sampling(inputs):
  z_mean, z_log_var = inputs
  batch = k.shape(z_mean)[0]
  dim = k.shape(z_mean)[1]
  epsilon = k.random_normal(shape = (batch, dim))
  return z_mean + k.exp(z_log_var/2)*epsilon

input_images = Input(shape = (63,63,3))
conv1 = Conv2D(16, (3,3), activation = 'relu', kernel_initializer = initializer)(input_images)
conv2 = Conv2D(8, (3,3), activation = 'relu', kernel_initializer = initializer)(conv1)
flattened = Flatten()(conv2)
x = Dense(4, activation = 'relu', kernel_initializer = initializer)(flattened)

z_mean = Dense(2, kernel_initializer = initializer, name = "z_mean")(x)
z_log_var = Dense(2, kernel_initializer = initializer, name = "z_log_var")(x)
z = Lambda(sampling, output_shape = (2,))([z_mean, z_log_var])

encoder = Model(input_images, [z_mean, z_log_var, z], name = "encoder")
encoder.summary()

latent_inputs = Input(shape = (2,))
x = Dense(59*59*8, kernel_initializer = initializer, activation = 'relu')(latent_inputs)
x = Reshape((59,59,8))(x)
conv4 = Conv2DTranspose(8, (3,3), kernel_initializer = initializer, activation = 'relu')(x)
decoded = Conv2DTranspose(3, (3,3), kernel_initializer = initializer, activation = 'sigmoid')(conv4)

decoder = Model(latent_inputs, decoded, name = "decoder")
decoder.summary()

z_mean, z_log_var, z = encoder(input_images)
vae_decoder_output = decoder(z)
vae = Model(input_images, vae_decoder_output, name = "VAE")
vae.summary()

def vae_loss():
  recon = input_images*tf.math.log(1e-10 + vae_decoder_output) + (1 - (input_images))*tf.math.log(1e-10 + 1 - vae_decoder_output)
  recon = -k.reduce_sum(recon, 1)
  kl_loss = 1 + z_log_var - k.square(z_mean) - k.exp(z_log_var)
  kl_loss = -0.5*k.reduce_sum(kl_loss, 1)
  vae_loss = k.mean(recon + kl_loss)
  return vae_loss

optimizer = tf.keras.optimizers.Adam(lr = 1e-3)
vae.compile(optimizer = 'adam', loss = vae_loss)
h = vae.fit(X_train, X_train, batch_size = 100, epochs = 100, shuffle = True, validation_data = (X_test, X_test))


title: Add attribute to object of dataset

body:
I am very new to pytorch and pytorch-geometric. I need to load a dataset and then map an attribute to every object of the set that I will use later in the script. However I can't figure out how to do it.
I start the loading as
from torch_geometric.datasets import TUDataset

dataset = TUDataset(root='data/TUDataset', name='PROTEINS')

then I add the attribute. I tried (the value 3 is only for example, it will be a db query)
for data in dataset:
    data.keys.append('szemeredi_id')
    data.szemeredi_id = 3

or
for data in dataset:
    data['szemeredi_id'] = 3

or
for i, s in enumerate(dataset):
    dataset[i]['szemeredi_id'] = 3

or
for data in dataset:
    setattr(data, 'szemeredi_id', 3)

but that attribute is always empty. I even tried to write a decorator class for the Data class as
class SzeData(Data):
    def __init__(self, x=None, edge_index=None, edge_attr=None, y=None,
                 pos=None, normal=None, face=None, **kwargs):
        super(SzeData, self).__init__(x, edge_index, edge_attr, y, pos, normal, face)
        self.szemeredi_id = None

but if I try to replace the Data objects it raises the error TypeError: 'TUDataset' object does not support item assignment or it does nothing if I use this solution.
Any suggestion is much appreciated. Thank you.


title: What is the difference between "from pytorch_pretrained_bert import BertModel" and "from transformers import BertModel"?

body:
What is the difference between these importings
from pytorch_pretrained_bert import BertTokenizer, BertModel

and
from transformers import BertTokenizer, BertModel, BertConfig?

and
why Iam not able to call BertConfig from pytorch?


title: EOFError: Ran out of input in Google Colab for a Sentiment Analysis model

body:
I am trying a sentiment analysis with the IMDB dataset using Pytorch. Running it on google colab gives me an error
EOFError                                  Traceback (most recent call last)
<ipython-input-11-1af666a9752f> in <module>()
     38   # Infer from BERT
     39   else:
---> 40     model.load_state_dict(torch.load(path))
     41     sentiment = predict_sentiment(model, tokenizer, TEXT)
     42     print(sentiment)

1 frames
/usr/local/lib/python3.6/dist-packages/torch/serialization.py in load(f, map_location, pickle_module, **pickle_load_args)
    593                     return torch.jit.load(opened_file)
    594                 return _load(opened_zipfile, map_location, pickle_module, **pickle_load_args)
--> 595         return _legacy_load(opened_file, map_location, pickle_module, **pickle_load_args)
    596
    597

/usr/local/lib/python3.6/dist-packages/torch/serialization.py in _legacy_load(f, map_location, pickle_module, **pickle_load_args)
    762             "functionality.")
    763
--> 764     magic_number = pickle_module.load(f, **pickle_load_args)
    765     if magic_number != MAGIC_NUMBER:
    766         raise RuntimeError("Invalid magic number; corrupt file?")

EOFError: Ran out of input

Following is the main code. I doubt the error will be related to the path I have used. Please let me know about setting the path in the right way in Colab.
if __name__ == "__main__":

  # Train BERT
  if TRAIN:
    # load data
    train_iter, valid_iter, test_iter = load_data()

    optimizer = optim.Adam(model.parameters())
    criterion = nn.BCEWithLogitsLoss().to(device)
    model = model.to(device)

    best_val_loss = float('inf')

    for epoch in range(N_EPOCHS):
      # start time
      start_time = time.time()
      # train for an epoch
      train_loss, train_acc = train(model, train_iter, optimizer, criterion)
      valid_loss, valid_acc = evaluate(model, valid_iter, criterion)
      # end time
      end_time = time.time()
      # stats
      epoch_mins, epoch_secs = epoch_time(start_time, end_time)
      # save model if has validation loss
      # better than last one
      if valid_loss < best_valid_loss:
        best_valid_loss = valid_loss
        torch.save(model.state_dict(),path)
      # stats
      print(f'Epoch: {epoch+1:02} | Epoch Time: {epoch_mins}m {epoch_secs}s')
      print(f'\tTrain Loss: {train_loss:.3f} | Train Acc: {train_acc*100:.2f}%')
      print(f'\t Val. Loss: {valid_loss:.3f} |  Val. Acc: {valid_acc*100:.2f}%')
    # Test
    model.load_state_dict(torch.load(path))
    test_loss, test_acc = evaluate(model, test_iter, criterion)
    print(f'Test Loss: {test_loss:.3f} | Test Acc: {test_acc*100:.2f}%')

  # Infer from BERT
  else:
    model.load_state_dict(torch.load(path))
    sentiment = predict_sentiment(model, tokenizer, TEXT)
    print(sentiment) ```


title: Distinguishing images from noise

body:
Neural networks have been successfully used in supervised learning to classify images.
I'm currently looking at a different problem: distinguishing images that show something, contain some manifest order, versus images that are just white noise. I'd like to get some numeric estimate of just how ordered an image is.
Clearly there are several ways of doing this, e.g. PNG compressing the image and looking at how much it shrank. (Pure noise is not compressible.) But I am currently interested in trying to do it with neural networks; it seems to me that it should be possible to do this with unsupervised learning, using some sort of autoencoder.
Is this something that has been done? Ideally with PyTorch but I would be happy to look at a solution using any language and framework.


title: Applying wild weights to custom loss function doesn't change behavior pytorch

body:
I'm regressing particle characteristics using a graph neural network in pytorch geometric. I want to write a custom loss function such that I can make the model pay particular attention to certain particles. (For context: I want the model to pay extra attention to low-energy particles).
After trying with reasonable weights I see no change in behavior. I have now tried to apply weights w = [500, 0.1] where 500 is applied to the cases I want the model to be sensitive to, and 0.1 to the cases I want the model to care less about. My initial thought were that these weights are so wild that it should provoke a change in behavior - But it doesnt! The result is identical to setting w = 1.
I have spent weeks trying to figure out what is wrong - there should be plenty of 500-cases for the model to care about them.
What I know for sure is:

The weights are applied to the correct cases
The scale of the loss changes (as expected from multiplying with a factor 500)
No changed behavior (acts as if w = 1)
No error messages

Here is a simplified version of my training loop:
def LossFunc(output,target,weight):
    loss = torch.sum(torch.abs(weight*(output-target)))
    return loss

def KNNAmp(k,x,batch):
    pos = x[:,0:3]
    edge_index = knn_graph(x=pos,k=k,batch=batch).to(device)
    return x,edge_index

class Net(torch.nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        l1, l2, l3, l4, l5,l6,l7 = 14,16*2,32*2,42*2,32*2,16*2,1

        self.nn_conv1 = torch.nn.Sequential(torch.nn.Linear(l1*2,l2),torch.nn.ReLU(),torch.nn.Linear(l2,l3),torch.nn.ReLU()).to(device)
        self.conv_max = EdgeConv(self.nn_conv1,aggr = 'max')
        self.conv_mean = EdgeConv(self.nn_conv1,aggr = 'mean')
        self.conv_add = EdgeConv(self.nn_conv1,aggr = 'add')

        self.nn1 = torch.nn.Linear(3*l3,l4)
        self.nn2   = torch.nn.Linear(l4,l5)
        self.nn3 =  torch.nn.Linear(4*l5,l6)
        self.nn4 = torch.nn.Linear(l6,l7)
        self.relu = torch.nn.LeakyReLU()

    def forward(self, data):
        x, edge_index, batch = data.x, data.edge_index, data.batch
        x,edge_index = KNNAmp(4, x, batch)
        x_max = self.conv_max(x,edge_index)
        x_mean = self.conv_mean(x,edge_index)
        x_add = self.conv_add(x,edge_index)

        x = self.relu(torch.cat((x_max,x_mean,x_add),dim=1))

        x = self.nn1(x)
        x = self.relu(x)
        x = self.nn2(x)

        a,_ = scatter_max(x, batch, dim = 0)
        b,_ = scatter_min(x, batch, dim = 0)
        c = scatter_sum(x,batch,dim = 0)
        d = scatter_mean(x,batch,dim= 0)
        x = torch.cat((a,b,c,d),dim = 1)
        x = self.relu(x)
        x = self.nn3(x)

        x = self.relu(x)
        x = self.nn4(x)
        return x
def Train(model_to_train,graphs_train,device,lr,lr_list,n_workers,batch_size,loss_decimals,graphs_valid,mini_batches_valid):
    count = torch.tensor([0]).to(device)
    model = model_to_train.to(device)
    loss_func = LossFunc
    count = torch.tensor(0).to(device)
    optimizer = torch.optim.Adam(model.parameters(), lr=lr,eps = 1e-3)
    print('TRAINING BEGUN. FIRST EPOCH STARTING..')
    for epoch in range(0,n_epochs):
        loss_acc = torch.tensor([0],dtype = float).to(device)
        for k in range(0,mini_batches):
            data_train                           = GrabBatch(q)
            with torch.enable_grad():
                data_train.y                    = data_train.y[:,0].unsqueeze(1)
                model.train()
                w                               = data_train.lw
                optimizer.zero_grad()
                out                         = model(data_train)
                loss                            = loss_func(out, data_train.y.float(),w)
                loss.backward()
                optimizer.step()
            optimizer.param_groups[0]['lr'] = lr_list[count].item()
            count                           +=1
            loss_acc                        +=loss
        with torch.no_grad():
            val_loss = Validate(model,graphs_valid,graphs_train,batch_size,o,
                                         mini_batches_valid,mini_batches,manager,epoch,n_epochs,save = False)
        if epoch == 0:
            deltatime                        = (time.time() - start)/60

        print('EPOCH: %s / %s || %s / %s min || LR: %s || Loss: %s || Val Loss: %s' %(epoch,n_epochs,(time.time() - start)/60,n_epochs*deltatime,optimizer.param_groups[0]['lr'],round(loss_acc.item()/mini_batches,loss_decimals),round(val_loss.item()/mini_batches_valid,loss_decimals)))
    return model
def Validate(trained_model,graphs_valid,graphs_train,batch_size,
             o,q,mini_batches_valid,val_slaves,n_workers,mini_batches,manager,train_epoch,n_epochs,save):
    print('in validation')
    model = trained_model
    loss_func = LossFunc()
    model.eval()
    count = 0
    print('Validating..')
    acc_loss = torch.tensor([0],dtype = float).to(device)
    with torch.no_grad():
        for i in range(0, mini_batches_valid):
            count = count + 1
            data_pred = GrabBatch(q)
            data_pred.y = data_pred.y[:,0].unsqueeze(1)

            pred = model(data_pred)
            w                               = data_pred.lw
            loss = loss_func(pred,data_pred.y,w)
            acc_loss += loss
        val_loss = acc_loss
        return val_loss

The data "data_pred" and "data_train" is pytorch geometric "Data-objects" that I load into memory via a custom parallel loader that I've built and left out of the code above as I think its unrelated. They each contain 1024 particles (e.g batch_size = 1024). They have device set to GPU.
Could you tell me if I'm breaking backpropagation somewhere? Is there any obvious mistakes you can see?
Many thanks!


title: How to install pytorch on Mac OS X?

body:
I get the following error when I try to install pytorch by pip3 on Mac OS X 10.15.6. Does anybody know how to fix the problem?
$ pip3 install torch==1.6.0
Collecting torch==1.6.0
  Downloading torch-1.6.0-cp37-none-macosx_10_9_x86_64.whl (97.4 MB)
     |███▌                            | 10.8 MB 18 kB/s eta 1:18:39ERROR: Exception:
Traceback (most recent call last):
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/pip/_vendor/urllib3/response.py", line 438, in _error_catcher
    yield
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/pip/_vendor/urllib3/response.py", line 519, in read
    data = self._fp.read(amt) if not fp_closed else b""
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/pip/_vendor/cachecontrol/filewrapper.py", line 62, in read
    data = self.__fp.read(amt)
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/http/client.py", line 457, in read
    n = self.readinto(b)
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/http/client.py", line 501, in readinto
    n = self.fp.readinto(b)
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/socket.py", line 589, in readinto
    return self._sock.recv_into(b)
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/ssl.py", line 1071, in recv_into
    return self.read(nbytes, buffer)
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/ssl.py", line 929, in read
    return self._sslobj.read(len, buffer)
socket.timeout: The read operation timed out

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/pip/_internal/cli/base_command.py", line 210, in _main
    status = self.run(options, args)
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/pip/_internal/cli/req_command.py", line 180, in wrapper
    return func(self, options, args)
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/pip/_internal/commands/install.py", line 319, in run
    reqs, check_supported_wheels=not options.target_dir
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/pip/_internal/resolution/resolvelib/resolver.py", line 122, in resolve
    requirements, max_rounds=try_to_avoid_resolution_too_deep,
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/pip/_vendor/resolvelib/resolvers.py", line 445, in resolve
    state = resolution.resolve(requirements, max_rounds=max_rounds)
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/pip/_vendor/resolvelib/resolvers.py", line 339, in resolve
    failure_causes = self._attempt_to_pin_criterion(name, criterion)
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/pip/_vendor/resolvelib/resolvers.py", line 207, in _attempt_to_pin_criterion
    criteria = self._get_criteria_to_update(candidate)
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/pip/_vendor/resolvelib/resolvers.py", line 198, in _get_criteria_to_update
    for r in self._p.get_dependencies(candidate):
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/pip/_internal/resolution/resolvelib/provider.py", line 102, in get_dependencies
    for r in candidate.iter_dependencies(with_requires)
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/pip/_internal/resolution/resolvelib/provider.py", line 101, in <listcomp>
    r
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/pip/_internal/resolution/resolvelib/candidates.py", line 252, in iter_dependencies
    requires = self.dist.requires() if with_requires else ()
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/pip/_internal/resolution/resolvelib/candidates.py", line 234, in dist
    self._prepare()
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/pip/_internal/resolution/resolvelib/candidates.py", line 221, in _prepare
    dist = self._prepare_distribution()
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/pip/_internal/resolution/resolvelib/candidates.py", line 314, in _prepare_distribution
    self._ireq, parallel_builds=True,
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/pip/_internal/operations/prepare.py", line 480, in prepare_linked_requirement
    return self._prepare_linked_requirement(req, parallel_builds)
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/pip/_internal/operations/prepare.py", line 505, in _prepare_linked_requirement
    self.download_dir, hashes,
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/pip/_internal/operations/prepare.py", line 257, in unpack_url
    hashes=hashes,
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/pip/_internal/operations/prepare.py", line 130, in get_http_url
    from_path, content_type = download(link, temp_dir.path)
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/pip/_internal/network/download.py", line 163, in __call__
    for chunk in chunks:
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/pip/_internal/cli/progress_bars.py", line 168, in iter
    for x in it:
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/pip/_internal/network/utils.py", line 88, in response_chunks
    decode_content=False,
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/pip/_vendor/urllib3/response.py", line 576, in stream
    data = self.read(amt=amt, decode_content=decode_content)
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/pip/_vendor/urllib3/response.py", line 541, in read
    raise IncompleteRead(self._fp_bytes_read, self.length_remaining)
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/contextlib.py", line 130, in __exit__
    self.gen.throw(type, value, traceback)
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/pip/_vendor/urllib3/response.py", line 443, in _error_catcher
    raise ReadTimeoutError(self._pool, None, "Read timed out.")
pip._vendor.urllib3.exceptions.ReadTimeoutError: HTTPSConnectionPool(host='files.pythonhosted.org', port=443): Read timed out.


title: Pytorch/torchvision - How to increase limit of detectable objects

body:
I am new to Pytorch and so far it has been incredible. I am using it to count the number of pills in an image. I have found that in the majority of my images the max number of objects that it detects is 100. For the picture below it reaches a max count of 100 with the confidence around .6. After that it doesn't increase anymore even down to .1 confidence. I haven't been able to find anything in the docs or any other places online. I am using the fasterrcnn_resnet50_fpn model. Below is the code that load the trained model and evaluate the image. Any tips or even different packages that would be able to count all objects would be super helpful.
## Loading the trained module
loaded_model = get_model(num_classes = 2)
loaded_model.load_state_dict(torch.load('Pillcount/model'))

os.chdir('../pytorchobjdet/vision')

class CountDataset(torch.utils.data.Dataset):
    def __init__(self, root, data_file, transforms=None):
        self.root = root
        self.transforms = transforms
        self.imgs = sorted(os.listdir(os.path.join(root, "count")))
        self.path_to_data_file = data_file
    def __getitem__(self, idx):
        # load images and bounding boxes
        img_path = os.path.join(self.root, "count", self.imgs[idx])
        img = Image.open(img_path).convert("RGB")
        box_list = parse_one_annot(self.path_to_data_file,
        self.imgs[idx])
        boxes = torch.as_tensor(box_list, dtype=torch.float32)
        num_objs = len(box_list)
        # there is only one class
        labels = torch.ones((num_objs,), dtype=torch.int64)
        image_id = torch.tensor([idx])
        area = (boxes[:, 3] - boxes[:, 1]) * (boxes[:, 2] - boxes[:,
        0])
        # suppose all instances are not crowd
        iscrowd = torch.zeros((num_objs,), dtype=torch.int64)
        target = {}
        target["boxes"] = boxes
        target["labels"] = labels
        target["image_id"] = image_id
        target["area"] = area
        target["iscrowd"] = iscrowd
        if self.transforms is not None:
                img, target = self.transforms(img, target)
        return img, target
    def __len__(self):
            return len(self.imgs)

dataset_count = CountDataset(root='../../Pill_Object_Detection',
                        data_file = "../../Pill_Object_Detection/count_labels.csv",
                        transforms = get_transform(train=False))

idx = 1
img, _ = dataset_count[idx]
#put the model in evaluation mode
loaded_model.eval()
with torch.no_grad():
   prediction = loaded_model([img])
image = Image.fromarray(img.mul(255).permute(1, 2,0).byte().numpy())
draw = ImageDraw.Draw(image)
# draw groundtruth
count = 0
for element in range(len(prediction[0]["boxes"])):
   boxes = prediction[0]["boxes"][element].cpu().numpy()
   score = np.round(prediction[0]["scores"][element].cpu().numpy(),
                    decimals= 4)
   if score > 0.6:
      draw.rectangle([(boxes[0], boxes[1]), (boxes[2], boxes[3])],
      outline ="red", width =3)
      draw.text((boxes[0], boxes[1]), text = str(score))
      count +=1
print(f'count = {count}')
image


title: PyTorch .to(torch.device("cuda")) speed differs vastly depending on execution state - How to achieve a speed up?

body:
My question is concerned with the speed of the to Method of PyTorch tensors and how it depends on the "execution state" (not sure if thats the correct name, feel free to edit).
My setup is as follows (RTX 2060 Super):
python version: 3.8.5 (default, Jul 28 2020, 12:59:40) [GCC 9.3.0]
pytorch version: 1.7.0+cu110

Firstly, a file that contains the code for what I'm talking about:
import torch
import time
import sys

gpu = torch.device("cuda")

def ver():
    print("python version:", sys.version)
    print("pytorch version:", torch.__version__)
    print("\n")

def test():
    start = time.time()
    torch.cuda.init()
    print("cuda init:", time.time()-start)
    x = torch.randn(15000,3).float()
    print("randn initialized:", time.time()-start)
    x.to(gpu)
    print("to(gpu):", time.time()-start)
    torch.cuda.synchronize()
    print("time after sync:", time.time()-start)
    print("\n")

if __name__ == "__main__":
    ver()
    test()
    test()

Running this yields following console output:
python version: 3.8.5 (default, Jul 28 2020, 12:59:40)
[GCC 9.3.0]
pytorch version: 1.7.0+cu110

cuda init: 0.002934694290161133
randn initialized: 0.0033266544342041016
to(gpu): 1.5724568367004395
time after sync: 1.5725233554840088

cuda init: 9.5367431640625e-07
randn initialized: 0.00030875205993652344
to(gpu): 0.00037860870361328125
time after sync: 0.00039458274841308594

The speed of the second to call is so much faster than the first one. Why does this occur?
And more importantly, how can I achieve consistently fast speeds?
In a bigger project I'm working on I should have enough time to maybe initalize the GPU in parallel before to gets called for the first time. Is that possible? If so, how? torch.cuda.init() doesn't seem to change the speed of the first to.
I cannot use torch.randn(x,y, device=gpu) because in the original setup the data comes from torch.from_numpy().
Thanks.


title: PyTorch how to do gathers over multiple dimensions

body:
I'm trying to find a way to do this without for loops.
Say I have a multi-dimensional tensor t0:
bs = 4
seq = 10
v = 16
t0 = torch.rand((bs, seq, v))

This has shape: torch.Size([4, 10, 16])
I have another tensor labels that is a batch of 5 random indices in the seq dimension:
labels = torch.randint(0, seq, size=[bs, sample])

So this has shape torch.Size([4, 5]). This is used to index the seq dimension of t0.
What I want to do is loop over the batch dimension doing gathers using labels tensor. My brute force solution is this:
t1 = torch.empty((bs, sample, v))
for b in range(bs):
    for idx0, idx1 in enumerate(labels[b]):
        t1[b, idx0, :] = t0[b, idx1, :]

Resulting in tensor t1 which has shape: torch.Size([4, 5, 16])
Is there a more idiomatic way of doing this in pytorch?


title: How can I convert KERAS `MLP` model to `Pythorch` model

body:
I have a MLP model using some Project
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import Flatten
from keras.layers import Embedding

model = Sequential()
embedding_layer = Embedding(vocab_size, 50, input_length=len(X[0]))
model.add(embedding_layer)
model.add(Flatten())
model.add(Dense(100, activation='relu'))
model.add(Dense(3, activation='softmax'))
model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])
model.summary()
model.fit(X, Y, epochs=10, verbose=1)

But I want to convert this MLP model to Pythorch model.
How can I do this?
Thanks a lot.
I add this code for generate data:
def generate_batch_data(x, y, batch_size):
    i, batch = 0, 0
    for batch, i in enumerate(range(0, len(x) - batch_size, batch_size), 1):
        x_batch = x[i : i + batch_size]
        y_batch = y[i : i + batch_size]
        yield x_batch, y_batch
    if i + batch_size < len(x):
        yield x[i + batch_size :], y[i + batch_size :]
    if batch == 0:
        yield x, y

and this code:
epochs = 10
batch_size = 10
for epoch in range(epochs):
    print(f'Epoch {epoch+1}/{epochs}')

    for x_train, y_train in generate_batch_data(x_train, y_train, batch_size):
        y_hat = model(x_train)
        loss = criterion(y_hat, y_train)
        acc = (y_hat.argmax(1) == y).float().mean()

        print(f'loss: {loss}, accuracy: {acc}')

This time I take a this error:
RuntimeError                              Traceback (most recent call last)
<ipython-input-50-7ffa61cb7f34> in <module>()
      5
      6     for x_train, y_train in generate_batch_data(x_train, y_train, batch_size):
----> 7         y_hat = model(x_train)
      8         loss = criterion(y_hat, y_train)
      9         acc = (y_hat.argmax(1) == y).float().mean()

4 frames
/usr/local/lib/python3.6/dist-packages/torch/nn/functional.py in embedding(input, weight, padding_idx, max_norm, norm_type, scale_grad_by_freq, sparse)
   1850         # remove once script supports set_grad_enabled
   1851         _no_grad_embedding_renorm_(weight, input, max_norm, norm_type)
-> 1852     return torch.embedding(weight, input, padding_idx, scale_grad_by_freq, sparse)
   1853
   1854

RuntimeError: Expected tensor for argument #1 'indices' to have scalar type Long; but got torch.FloatTensor instead (while checking arguments for embedding)

I added here torch.LongTensor:
def tokenize_and_pad_text(df, max_seq):
  tokenized_text = tokenize_text(df, max_seq)
  padded_text = pad_text(tokenized_text, max_seq)
  return torch.LongTensor(padded_text)

train_indices = tokenize_and_pad_text(df_train, max_seq)

x_train = bert_model(train_indices)[0]

And change here:
y_hat = model(x_train.long())
But this time it gives this errors:
IndexError                                Traceback (most recent call last)
<ipython-input-67-9ad38a8c062a> in <module>()
      5
      6     for x_train, y_train in generate_batch_data(x_train, y_train, batch_size):
----> 7         y_hat = model(x_train.long())
      8         loss = criterion(y_hat, y_train)
      9         acc = (y_hat.argmax(1) == y).float().mean()

4 frames
/usr/local/lib/python3.6/dist-packages/torch/nn/functional.py in embedding(input, weight, padding_idx, max_norm, norm_type, scale_grad_by_freq, sparse)
   1850         # remove once script supports set_grad_enabled
   1851         _no_grad_embedding_renorm_(weight, input, max_norm, norm_type)
-> 1852     return torch.embedding(weight, input, padding_idx, scale_grad_by_freq, sparse)
   1853
   1854

IndexError: index out of range in self

And model is like this:
import torch.nn as nn

model = nn.Sequential(
    nn.Embedding(num_embeddings=148, embedding_dim=768),
    nn.Flatten(),
    nn.Linear((768*148), 148),
    nn.ReLU(),
    nn.Linear(148, 3))

criterion = nn.CrossEntropyLoss()

optimizer = torch.optim.Adam(model.parameters())

Normally using dataframe context column like this:
def tokenize_text(df, max_seq):
  return[
         tokenizer.encode(text, add_special_tokens=True)[:max_seq] for text in df.context.values
  ]

def pad_text(tokenized_text, max_seq):
  return np.array([el + [0] * (max_seq - len(el)) for el in tokenized_text])

and after comments: I take len(df.context) is 148 and updatenum_embeddings as 148. I still keep getting the same error.
Thanks a lot.


title: ModuleNotFoundError: No module named '__main__.file_utils'; '__main__' is not a package in Hugginface's Transformers Library

body:
I am trying to convert the bert-base-cased model from pytorch to ONNX in Google Colab, so I ran this command
!python3 transformers/src/transformers/convert_graph_to_onnx.py --framework pt --model bert-base-cased bert-base-cased.onnx

but I ran into this error
Traceback (most recent call last):
  File "transformers/src/transformers/convert_graph_to_onnx.py", line 22, in <module>
    from .file_utils import ModelOutput, is_tf_available, is_torch_available
ModuleNotFoundError: No module named '__main__.file_utils'; '__main__' is not a package

Many people have run this command before and have not had any issues, any ideas as to why it is happening to me?


title: What are the advantages of having one optimizer for two models?

body:
I have seen multiple GAN architectures where a single optimizer is used for optimizing parameters of two separate models (e.g., https://github.com/hufu6371/GcGAN/blob/74ccc2fdc95d7927e6fc4e09f97243333f19cc1b/models/gc_gan_share_model.py#L58 or https://github.com/clementabary/travelgan/blob/de297199ca337f71ae9bc558a4a9a075c8026869/trainer.py#L38-L45). The question is why would one follow this approach? Are there any advantages of having just one optimizer defined for two models, apart from "coding standards"?
I usually use one optimizer per model. When I tried to experiment with one optimizer for two models in TensorFlow (tf.keras), the performance of the models did not seem to be affected at all. I am attaching below a sample class where I used one optimizer for two models (i.e., siamese, generator - both sharing the same objective).
class TraVeLGAN(keras.Model):
    def __init__(self, image_shape=(128, 128, 3), latent_dim=1000, lambda_siamese=10.0, lambda_gan=1.0):
        super().__init__()
        self.generator = build_generator(image_shape)
        self.generator.summary()

        self.discriminator = build_discriminator(image_shape)
        self.discriminator.summary()

        self.siamese = build_siamese(image_shape, latent_dim)
        self.siamese.summary()

        self.lambda_siamese = lambda_siamese
        self.lambda_gan = lambda_gan

    def compile(self):
        super().compile()
        self.gen_opt = optimizers.Adam(learning_rate=0.0001, beta_1=0.5, beta_2=0.9)
        self.disc_opt = optimizers.Adam(learning_rate=0.0003, beta_1=0.5, beta_2=0.9)
        # self.siam_opt = optimizers.Adam(learinig_rate=0.0003, beta_1=0.5, beta_2=0.9)

    def train_step(self, batch_data):
        real_A, real_B = batch_data

        with tf.GradientTape(persistent=True) as gen_tape, tf.GradientTape() as disc_tape:
            fake_B = self.generator(real_A, training=True)

            fake_output = self.discriminator(fake_B, training=True)
            real_output = self.discriminator(real_B, training=True)

            disc_loss = get_discriminator_loss(real_output, fake_output)

            gen_gan_loss = get_generator_loss(fake_output, real_A, self.siamese, self.generator)
            siam_loss = get_siamese_loss(real_A, self.siamese, self.generator)
            gen_loss = self.lambda_siamese * siam_loss + self.lambda_gan * gen_gan_loss

        disc_gradients = disc_tape.gradient(disc_loss, self.discriminator.trainable_variables)
        gen_gradients = gen_tape.gradient(gen_loss, self.generator.trainable_variables)
        siam_gradients = gen_tape.gradient(gen_loss, self.siamese.trainable_variables)

        self.disc_opt.apply_gradients(zip(disc_gradients, self.discriminator.trainable_variables))
        self.gen_opt.apply_gradients(zip(gen_gradients, self.generator.trainable_variables))
        self.gen_opt.apply_gradients(zip(siam_gradients, self.siamese.trainable_variables))

        return {"g_loss": gen_loss, "d_loss": disc_loss, "s_loss": self.lambda_siamese * siam_loss}