title: PyTorch CNN: one of the variables needed for gradient computation has been modified by an inplace operation

body:
I'm trying to train a CNN using pytorch which outputs a tensor of [37,256]. I want to compute the loss from each of the 37 outputs against a Long value using cross entopy loss but I keep getting the above error.
Here is my network:
class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()

        self.short_conv_layers = nn.Sequential(
            nn.BatchNorm2d(1),
            nn.Conv2d(in_channels = 1, out_channels = 16, kernel_size = (1,5), stride=(1,1)),

            nn.BatchNorm2d(16),
            nn.Conv2d(in_channels = 16, out_channels = 16, kernel_size = (1,5), stride=(1,1)),

            nn.BatchNorm2d(16),
            nn.Conv2d(in_channels = 16, out_channels = 16, kernel_size = (1,5), stride=(1,1)),

            nn.AvgPool2d((5,1),stride = (5,1)),
            nn.BatchNorm2d(16),
        )

        self.Conv41 = nn.Conv2d(in_channels = 16, out_channels = 24, kernel_size = (1,32), stride=(1,1))
        self.Conv42 = nn.Conv2d(in_channels = 16, out_channels = 24, kernel_size = (1,64), stride=(1,1))
        self.Conv43 = nn.Conv2d(in_channels = 16, out_channels = 24, kernel_size = (1,96), stride=(1,1))
        self.Conv44 = nn.Conv2d(in_channels = 16, out_channels = 24, kernel_size = (1,128), stride=(1,1))
        self.Conv45 = nn.Conv2d(in_channels = 16, out_channels = 24, kernel_size = (1,192), stride=(1,1))
        #self.Conv46 = nn.Conv2d(in_channels = 16, out_channels = 24, kernel_size = (1,256), stride=(1,1))

        self.Conv4_1x1 = nn.Conv2d(in_channels = 24, out_channels = 36, kernel_size = (1,1), stride=(1,1))
        self.Conv4_AP = nn.AvgPool2d((2,1),stride = (2,1))
        self.Conv4_BN = nn.BatchNorm2d(36)

        self.Conv51 = nn.Conv2d(in_channels = 36, out_channels = 24, kernel_size = (1,32), stride=(1,1))
        self.Conv52 = nn.Conv2d(in_channels = 36, out_channels = 24, kernel_size = (1,64), stride=(1,1))
        self.Conv53 = nn.Conv2d(in_channels = 36, out_channels = 24, kernel_size = (1,96), stride=(1,1))
        self.Conv54 = nn.Conv2d(in_channels = 36, out_channels = 24, kernel_size = (1,128), stride=(1,1))
        self.Conv55 = nn.Conv2d(in_channels = 36, out_channels = 24, kernel_size = (1,192), stride=(1,1))
        #self.Conv55 = nn.Conv2d(in_channels = 36, out_channels = 24, kernel_size = (1,256), stride=(1,1))

        self.Conv5_1x1 = nn.Conv2d(in_channels = 24, out_channels = 36, kernel_size = (1,1), stride=(1,1))
        self.Conv5_AP = nn.AvgPool2d((2,1),stride = (2,1))
        self.Conv5_BN = nn.BatchNorm2d(36)

        self.Conv61 = nn.Conv2d(in_channels = 36, out_channels = 24, kernel_size = (1,32), stride=(1,1))
        self.Conv62 = nn.Conv2d(in_channels = 36, out_channels = 24, kernel_size = (1,64), stride=(1,1))
        self.Conv63 = nn.Conv2d(in_channels = 36, out_channels = 24, kernel_size = (1,96), stride=(1,1))
        self.Conv64 = nn.Conv2d(in_channels = 36, out_channels = 24, kernel_size = (1,128), stride=(1,1))
        self.Conv65 = nn.Conv2d(in_channels = 36, out_channels = 24, kernel_size = (1,192), stride=(1,1))
        #self.Conv65 = nn.Conv2d(in_channels = 36, out_channels = 24, kernel_size = (1,256), stride=(1,1))

        self.Conv6_1x1 = nn.Conv2d(in_channels = 24, out_channels = 36, kernel_size = (1,1), stride=(1,1))
        self.Conv6_AP = nn.AvgPool2d((2,1),stride = (2,1))
        self.Conv6_BN = nn.BatchNorm2d(36)

        self.Conv71 = nn.Conv2d(in_channels = 36, out_channels = 24, kernel_size = (1,32), stride=(1,1))
        self.Conv72 = nn.Conv2d(in_channels = 36, out_channels = 24, kernel_size = (1,64), stride=(1,1))
        self.Conv73 = nn.Conv2d(in_channels = 36, out_channels = 24, kernel_size = (1,96), stride=(1,1))
        self.Conv74 = nn.Conv2d(in_channels = 36, out_channels = 24, kernel_size = (1,128), stride=(1,1))
        self.Conv75 = nn.Conv2d(in_channels = 36, out_channels = 24, kernel_size = (1,192), stride=(1,1))
        #self.Conv75 = nn.Conv2d(in_channels = 36, out_channels = 24, kernel_size = (1,256), stride=(1,1))

        self.Conv7_1x1 = nn.Conv2d(in_channels = 24, out_channels = 36, kernel_size = (1,1), stride=(1,1))
        self.Conv7_BN = nn.BatchNorm2d(36)

        self.linear_layers = nn.Sequential(
            nn.Flatten(),
            nn.Dropout(inplace=False),
            nn.Linear(2642688,64),
            nn.BatchNorm1d(64),
            nn.Linear(64,64),
            nn.BatchNorm1d(64),
            nn.Linear(64,256)
        )

    def forward(self, x):
        x = self.short_conv_layers(x)

        conv41 = self.Conv41(x)
        conv42 = self.Conv42(x)
        conv43 = self.Conv43(x)
        conv44 = self.Conv44(x)
        conv45 = self.Conv45(x)
        #conv46 = self.Conv46(x)

        conv4_out = torch.cat((conv41,conv42,conv43,conv44,conv45),dim=3)
        conv4_1x1 = self.Conv4_1x1(conv4_out)
        conv4_ap = self.Conv4_AP(conv4_1x1)
        conv4_bn = self.Conv4_BN(conv4_ap)

        conv51 = self.Conv51(conv4_bn)
        conv52 = self.Conv52(conv4_bn)
        conv53 = self.Conv53(conv4_bn)
        conv54 = self.Conv54(conv4_bn)
        conv55 = self.Conv55(conv4_bn)

        conv5_out = torch.cat((conv51,conv52,conv53,conv54,conv55),dim=3)
        conv5_1x1 = self.Conv5_1x1(conv5_out)
        conv5_ap = self.Conv5_AP(conv5_1x1)
        conv5_bn = self.Conv5_BN(conv5_ap)

        conv61 = self.Conv61(conv5_bn)
        conv62 = self.Conv62(conv5_bn)
        conv63 = self.Conv63(conv5_bn)
        conv64 = self.Conv64(conv5_bn)
        conv65 = self.Conv65(conv5_bn)

        conv6_out = torch.cat((conv61,conv62,conv63,conv64,conv65),dim=3)
        conv6_1x1 = self.Conv6_1x1(conv6_out)
        conv6_ap = self.Conv6_AP(conv6_1x1)
        conv6_bn = self.Conv6_BN(conv6_ap)

        conv71 = self.Conv71(conv6_bn)
        conv72 = self.Conv72(conv6_bn)
        conv73 = self.Conv73(conv6_bn)
        conv74 = self.Conv74(conv6_bn)
        conv75 = self.Conv75(conv6_bn)

        conv7_out = torch.cat((conv71,conv72,conv73,conv74,conv75),dim=3)
        conv7_1x1 = self.Conv7_1x1(conv7_out)
        conv7_bn = self.Conv7_BN(conv7_1x1)

        x = self.linear_layers(conv7_bn)

        return x

net = Net().to(device)
print(net)

And here is where I'm getting the error
    for x in y_pred:
        sm_y_pred = SM(x)
        sm_y_pred = sm_y_pred.view(1,256)
        loss = loss_function(sm_y_pred,bpm)
        loss.backward(retain_graph=True)
        optimizer.step()

Is there a way I can do this? Or should I be implementing this in a different way.
The network is used to predict the bpm from songs.


title: How to get event's walltime in pytorch profiler?

body:
I'm using pytorch version 1.8.1+cu102, and I want to know event's start time and end time.
torch.autograd.profiler.FunctionEvent provides time_range field, but it's value is strange.
# %%
import torch
import torchvision.models as models
import time
print(torch.__version__)
# %%
model = models.resnet18().cuda()
inputs = torch.randn(5, 3, 224, 224).cuda()
with torch.profiler.profile(
        activities=[
            torch.profiler.ProfilerActivity.CPU,
            torch.profiler.ProfilerActivity.CUDA]
) as p:
    outputs = model(inputs)

# %%
events = p.events()
print('begin     ', min(events, key=lambda e: e.time_range.start).time_range.start / 1000000)
print('end       ', max(events, key=lambda e: e.time_range.end).time_range.end / 1000000)
print('real time ', time.time())

Run the code above for the first time, I got:
1.8.1+cu102
begin      1615893050.2533033
end        1615893052.6818323
real time  1617705608.1550226

and the second time:
1.8.1+cu102
begin      1615893050.2533386
end        1615893052.7069924
real time  1617705746.4107628

The events' begin time and end time does not changed.
So how to get the right timestamp of the events?


title: sublime had issue importing anaconda installed numpy

body: I use sublime text to write python codes, but recently i switched to a new computer. On that computer, first I installed anaconda from https://www.anaconda.com/, then I downloaded sublime text 3, I use the tool-->build system-->new build system on the tool bar of sublime text to associate the anaconda python(/anaconda/python.exe) to sublime. Everything seemed alright, I can import regex and many anaconda packages, but when i import numpy, this error happened ImportError: DLL load failed while importing _multiarray_umath: can't find the module.More detail in the picture posted.enter image description here I tried updating numpy package with conda, didn't work. On my old computer I also have anaconda installed, but I don't remember how I set the association betweent sublime text3 and the python interpreter. On the old computer sublime, it's tool--> build system-->Python, no new built system, so maybe that is the difference... Also on the old computer I have torch installed maybe that's the difference? really need help here


title: Torch: how to concatenate tensors of different sizes?

body:
I have two tensors:
rc of size: torch.Size([128, 16, 1])
xt of size: torch.Size([128, 40, 1])

I would like to concatenate xt to rc along dimension 2 so that the final size of rc_xt is:
rc_xt = torch.Size([128, 40, 2])

In short, I want to 'increase' the size of rc's dimension 1 (16) to 40 -- through any means, even just repeating elements -- and then concatenate them both along dimension 2.
I have tried to google how to do this but I cannot get it working no matter what I do, I'm a little confused about how to go about doing this.
Thank you in advance.


title: Would it be meaningful, if I concatenate LSTM output with CNN output?

body:
I am trying to boost the performance of a single-frame object detection network with multiple frames input.
In side the network I have added an ConvLSTM module with 200 channel. The modified network couldn't achieve the same-level performance of original network.
#Feature_200 means: "a feature map with 200 channels"

#original network pipeline:
Encoder-> Feature_200 -> Decoder

#modify Nr.1:
Encoder-> Feature_200 -> ConvLSTM -> Feature_200 -> Decoder

#want to implement:
Encoder-> Feature_200 -> ConvLSTM -> Feature_100 --concat---> Feature_300 -> Decoder
                |                                  |
                 ------short cut---> Feature_200 --

So I am considering concatenate the ConvLSTM results to the original network's feature map. Would this structure be helpful to improve the performance?
(Ps. I'm not sure if it's correct to refer the module with "encoder" and "decoder")


title: Difference between two implemantations of Seq layers - pytorch

body:
I have implemented a model using the traditional way of implementation. and the code is like this.
    def __init__(self):
    super(enhance_net_nopool, self).__init__()
    number_f = 32

    self.relu = nn.ReLU(inplace=True)
    self.e_conv1 = nn.Conv2d(number_f, number_f, 3, 1, 1, bias=True)
    self.e_conv2 = nn.Conv2d(number_f, number_f, 3, 1, 1, bias=True)
    self.e_conv3 = nn.Conv2d(number_f, number_f, 3, 1, 1, bias=True)

def forward(self, x):
    x1 = self.relu(self.e_conv1(x))
    x2 = self.relu(self.e_conv2(x1))
    x3 = self.relu(self.e_conv3(x2))

I need to know is it possible to rewrite that code using
seq_layers=nn.Sequential(*layers)

this *layers.
If it's implemented like this.
self.conv_block = [
        relu = nn.ReLU(inplace=True)
        conv = nn.Conv2d(number_f, number_f, 3, 1, 1, bias=True)
    ]

are they the same?


title: PyTorch: minimal working example or tutorial showing how to use nn.TransformerDecoder for batch text generation in training and inference modes?

body:
I want to solve a sequence-to-sequence text generation task (e.g. question answering, language translation, etc.).
For the purposes of this question, you may assume that I already have the input part already handled. (I already have a tensor of dimensions batch_size x num_input_tokens x input_dim representing the input sequences. Also, all input sequences in my problem are of the same length, so no masking is required on the input side of things).
Now, I want to generate the output sequences using nn.TransformerDecoder. I'm aware of Pytorch's official tutorial SEQUENCE-TO-SEQUENCE MODELING WITH NN.TRANSFORMER AND TORCHTEXT. Unfortunately, the official tutorial doesn't meet my needs, for the following reasons:

nn.TransformerDecoder is not used in the example.
The example is about language modeling, not text generation. There is no forward loop that generates text word by word.

I've searched around the web and I've found a few things, but nothing like a simple and minimal working example that directly applies to my problem setting. Concretely, on the output side of things I need the following:

I want to generate output sequences in batch. I've found codes on GitHub where people appear to be doing text generation, but they do it for a single sequence at a time, not a batch of multiple sequences.
The output sequences may have different lengths.
I want to train my model with the teacher-forcing strategy and batches of multiple sequences. Given that in training I know the lengths of the sequences in advance, you may assume that I already have my batches padded with zeroes. However, I still need to figure out how to implement the forward function of my model, with a generation loop that uses nn.TransformerDecoder. Basically, I need to figure out how to iterate word-wise over my batch of output sequences, masking out the future words in each step (so that the model doesn't cheat by trivially predicting the next words).
Then, I need a similar forward function for inference mode. I need to figure out how to implement the generation loop to do basically the same as in training mode, except that instead of teacher-forcing I want to implement greedy search (i.e. use the tokens with highest predicted probability at iteration i as the next input for iteration i+1).

I already know how to do all this using LSTMs. Below you can see the forward function of a model that I implemented in the past to do exactly what I just said with an LSTM. The same forward function is used for both training and inference, depending on the value of the variable 'mode':
  def forward(
      self,
      image_local_features,
      question_vectors,
      answers=None,
      max_answer_length=None,
      mode='train',
  ):
    if mode == 'train':
      batch_size, max_answer_length = answers.shape
      assert answers is not None
    else:
      batch_size = image_local_features.size(0)
      assert max_answer_length is not None

    y = self.embedding_table(self.start_idx).expand(batch_size, -1)
    o = torch.zeros(batch_size, self.hidden_size).to(DEVICE)
    h = self.W_h(question_vectors)
    c = self.W_c(question_vectors)

    if mode == 'train':
      answer_embeddings = self.embedding_table(answers.permute(1,0))
      assert answer_embeddings.shape == (max_answer_length, batch_size, self.embed_size)

    output = []

    for t in range(max_answer_length):
      y_bar = torch.cat((y,o),1)
      assert y_bar.shape == (batch_size, self.embed_size + self.hidden_size)
      assert h.shape == (batch_size, self.hidden_size)
      assert c.shape == (batch_size, self.hidden_size)
      h, c = self.lstm_cell(y_bar, (h, c))
      e = (self.W_attn(image_local_features) * h.unsqueeze(1)).sum(-1)
      att = torch.softmax(e,-1)
      a = (image_local_features * att.unsqueeze(2)).sum(1)
      assert a.shape == (batch_size, self.image_local_feat_size)
      u = torch.cat((a,h),1)
      assert u.shape == (batch_size, self.hidden_size + self.image_local_feat_size)
      v = self.W_u(u)
      o = self.dropout(torch.tanh(v))
      assert o.shape == (batch_size, self.hidden_size)
      output.append(self.W_vocab(o))
      if mode == 'train':
        y = answer_embeddings[t] # teacher-forcing
      else:
        y = self.embedding_table(torch.argmax(output[t], 1)) # greedy search
      assert y.shape == (batch_size, self.embed_size)

    output = torch.stack(output, 1)
    assert output.shape == (batch_size, max_answer_length, self.vocab_size)
    return output

Another way to phrase my question would be: how can I reimplement what I did with LSTMs using nn.TransformerDecoder instead?
Any minimal working / hello world example that shows how to do batch training and batch inference with nn.TransformerDecoder for text generation will be very appreciated.

Note: alternatively, if there is a straightforward way of accomplishing the same with an out-of-the-box solution from hugginface, that would be awesome too.


title: ValueError: negative step not yet supported. How to solve it

body:
I use 4 GPUs, and I use pytorch DDP to speed up training. Below is my dataset code
import cv2
import ast
import torch
import numpy as np
import random
from glob import glob
from torch.utils.data import DataLoader, Dataset
from torch.utils.data.distributed import DistributedSampler

cv2.setNumThreads(1)
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
class VimeoDataset(Dataset):
    def __init__(self, mode): ##mode = 'train' means train mode or 'val' means val mode
        self.h = 256
        self.w = 448
        self.mode = mode
        self.trainlist = glob('/data/vimeoFlow2/dataset/train/*.npz')
        self.vallist = glob('/data/vimeoFlow2/dataset/val/*.npz')
        xx = np.arange(0, self.w).reshape(1,-1).repeat(self.h,0) #xx shape is(256,448)
        yy = np.arange(0, self.h).reshape(-1,1).repeat(self.w,1) #yy shape is(448,256)
        self.grid = np.stack((xx,yy),2).copy()
        self.npzs = []
        if self.mode == 'train':
            self.npzs = self.trainlist[:80]
        else:
            self.npzs = self.vallist[:20]

    def __len__(self):
        return len(self.npzs)

    def aug(self, img0, gt, img1, flow_gt, h, w):
        ih, iw, _ = img0.shape
        x = np.random.randint(0, ih - h + 1)
        y = np.random.randint(0, iw - w + 1)
        img0 = img0[x:x+h, y:y+w, :] #random generate image slices, img shape is (224,224,:)
        img1 = img1[x:x+h, y:y+w, :]
        gt = gt[x:x+h, y:y+w, :]
        flow_gt = flow_gt[x:x+h, y:y+w, :]
        return img0, gt, img1, flow_gt

    def getimg(self, index):
        f = np.load(self.npzs[index])
        data = f['i0i1gt']
        if self.mode == 'train':
            flow_data = f['ft0ft1']
        else:
            flow_data = np.zeros((256, 448, 4))
        img0 = data[:,:,0:3]
        img1 = data[:,:,3:6]
        gt = data[:,:,6:9]
        flow_gt = flow_data
        return img0, gt, img1, flow_gt

    def __getitem__(self, index):
        img0, gt, img1, flow_gt = self.getimg(index)
        if self.mode == 'train':
            img0, gt, img1, flow_gt = self.aug(img0, gt, img1, flow_gt, 224, 224)
            flow_gt = torch.from_numpy(flow_gt.copy()).permute(2, 0, 1)
            img0 = torch.from_numpy(img0.copy()).permute(2, 0, 1)
            img1 = torch.from_numpy(img1.copy()).permute(2, 0, 1)
            gt = torch.from_numpy(gt.copy()).permute(2, 0, 1)
            if random.uniform(0, 1) < 0.5:
                img0 = img0[:, :, ::-1]
                img1 = img1[:, :, ::-1]
                gt = gt[:, :, ::-1]
            if random.uniform(0, 1) < 0.5:
                img0 = img0[::-1]
                img1 = img1[::-1]
                gt = gt[::-1]
                flow_gt = flow_gt[::-1]
                flow_gt = np.concatenate((flow_gt[:, :, 0:1], -flow_gt[:, :, 1:2], flow_gt[:, :, 2:3], -flow_gt[:, :, 3:4]), 2)
            if random.uniform(0, 1) < 0.5:
                img0 = img0[:, ::-1]
                img1 = img1[:, ::-1]
                gt = gt[:, ::-1]
                flow_gt = flow_gt[:, ::-1]
                flow_gt = np.concatenate((-flow_gt[:, :, 0:1], flow_gt[:, :, 1:2], -flow_gt[:, :, 2:3], flow_gt[:, :, 3:4]), 2)
            if random.uniform(0, 1) < 0.5:
                tmp = img1
                img1 = img0
                img0 = tmp
                flow_gt = np.concatenate((flow_gt[:, :, 2:4], flow_gt[:, :, 0:2]), 2)
        flow_gt = torch.from_numpy(flow_gt.copy()).permute(2, 0, 1)
        img0 = torch.from_numpy(img0.copy()).permute(2, 0, 1)
        img1 = torch.from_numpy(img1.copy()).permute(2, 0, 1)
        gt = torch.from_numpy(gt.copy()).permute(2, 0, 1)
        return torch.cat((img0, img1, gt), 0), flow_gt

I use the following dataloader.
dataset = VimeoDataset(mode = 'train')
sampler = DistributedSampler(dataset)
train_data = DataLoader(dataset, batch_size=args.batch_size, num_workers=0, pin_memory=True, drop_last=True, sampler=sampler)
dataset_val = VimeoDataset(mode = 'val')
val_data = DataLoader(dataset_val, batch_size=16, pin_memory=True, num_workers=0)

I use the following startup command.
python -m torch.distributed.launch --nproc_per_node=4 --nnodes=1 --node_rank=0 train.py --epoch=100 --batch_size=16

The error traceback is as follows.My data set is divided into train and val, which are npz files. Due to the limitation of memory size, I can only read one npz file at a time based on the index
 training...
    Traceback (most recent call last):
      File "train.py", line 173, in <module>
        train(model, args.local_rank)
      File "train.py", line 72, in train
        for i, data in enumerate(train_data):
      File "/opt/conda/lib/python3.6/site-packages/torch/utils/data/dataloader.py", line 346, in __next__
        data = self._dataset_fetcher.fetch(index)  # may raise StopIteration
      File "/opt/conda/lib/python3.6/site-packages/torch/utils/data/_utils/fetch.py", line 44, in fetch
        data = [self.dataset[idx] for idx in possibly_batched_index]
      File "/opt/conda/lib/python3.6/site-packages/torch/utils/data/_utils/fetch.py", line 44, in <listcomp>
        data = [self.dataset[idx] for idx in possibly_batched_index]
      File "/code/RIFE/dataset.py", line 67, in __getitem__
        img0 = img0[::-1]
    ValueError: negative step not yet supported
    File "/opt/conda/lib/python3.6/site-packages/torch/distributed/launch.py", line 249, in main
        cmd=cmd)
    subprocess.CalledProcessError: Command '['/opt/conda/bin/python', '-u', 'train.py', '--local_rank=3', '--epoch=100', '--batch_size=1']' returned non-zero exit status 1.


title: Pytorch change a sequence of actions into sequence of states

body:
my pytorch model outputs actions in the form of a tensor [x1, x2, x3, ... x10] which represents the sequence of actions taken that alter the current environment. I would like to change that tensor into a
sequence of states that have empty states initialized with a fill value, which I'll denote f.
So, my goal is to convert
O_1 = [x1, x2, x3, x4, x5, x6, x7, x8, x9, x10]

into
O_2 = [
[x1, f, f, f, f, f, f, f, f, f],
[x1, x2, f, f, f, f, f, f, f, f],
[x1, x2, x3, f, f, f, f, f, f, f],
[x1, x2, x3, x4, f, f, f, f, f, f],
[x1, x2, x3, x4, x5, f, f, f, f, f],
[x1, x2, x3, x4, x5, x6, f, f, f, f],
[x1, x2, x3, x4, x5, x6, x7, f, f, f],
[x1, x2, x3, x4, x5, x6, x7, x8, f, f],
[x1, x2, x3, x4, x5, x6, x7, x8, x9, f],
[x1, x2, x3, x4, x5, x6, x7, x8, x9, x10]
]

I could write a function to do this manually but I was wondering if there is a easier pytorch-y way (like with certain tensor transformations) to do this conversion. Moreover, is there a way to do this that preserves the computational graph from O_1 to O_2?


title: Conv autoencoder on RGB images not working in Pytorch

body:
I have created a conv autoencoder to generate custom images (Generated features can be used for clustering). But I am not able to generate the images, even the result is very bad. I am not able to understand what is this problem. Image size is 240x270 and is resized to 224x224
Autoencoder class is as follow
        self.encoder = nn.Sequential(

            nn.Conv2d(3, 16, 7, stride=3, padding=1),  # b, 16, 10, 10
            nn.ReLU(True),
            nn.Conv2d(16, 32, 7, stride=3, padding=1),  # b, 16, 10, 10
            nn.ReLU(True),
            nn.Conv2d(32, 64, 3, stride=2, padding=1),  # b, 16, 10, 10
            nn.ReLU(True),
            nn.Conv2d(64, 128, 3, stride=2, padding=1)  # b, 16, 10, 10
            # nn.ReLU(True),
            # nn.Conv2d(128, 256, 2, stride=5, padding=1)  # b, 16, 10, 10

        )
        self.decoder = nn.Sequential(
            # nn.ConvTranspose2d(256, 128, 2),  # b, 16, 5, 5
            # nn.ReLU(True),
            nn.ConvTranspose2d(128, 64, 3,stride=2, padding=1),  # b, 16, 5, 5
            nn.ReLU(True),
            nn.ConvTranspose2d(64, 32, 3,stride=2, padding=0),  # b, 16, 5, 5
            nn.ReLU(True),
            nn.ConvTranspose2d(32, 16, 5, stride=3, padding=0),  # b, 8, 15, 15
            nn.ReLU(True),
            nn.ConvTranspose2d(16, 8, 7, stride=3, padding=0,output_padding=1),  # b, 1, 28, 28
            nn.ReLU(True),
            nn.ConvTranspose2d(8,3, 7, stride=1, padding=0),  # b, 1, 28, 28
            nn.Sigmoid()

        )

    def forward(self, x):
        x = self.encoder(x)
        x = self.decoder(x)
        return x

Model Summary is as follow
----------------------------------------------------------------
        Layer (type)               Output Shape         Param #
================================================================
            Conv2d-1           [-1, 16, 74, 74]           2,368
              ReLU-2           [-1, 16, 74, 74]               0
            Conv2d-3           [-1, 32, 24, 24]          25,120
              ReLU-4           [-1, 32, 24, 24]               0
            Conv2d-5           [-1, 64, 12, 12]          18,496
              ReLU-6           [-1, 64, 12, 12]               0
            Conv2d-7            [-1, 128, 6, 6]          73,856
   ConvTranspose2d-8           [-1, 64, 11, 11]          73,792
              ReLU-9           [-1, 64, 11, 11]               0
  ConvTranspose2d-10           [-1, 32, 23, 23]          18,464
             ReLU-11           [-1, 32, 23, 23]               0
  ConvTranspose2d-12           [-1, 16, 71, 71]          12,816
             ReLU-13           [-1, 16, 71, 71]               0
  ConvTranspose2d-14          [-1, 8, 218, 218]           6,280
             ReLU-15          [-1, 8, 218, 218]               0
  ConvTranspose2d-16          [-1, 3, 224, 224]           1,179
          Sigmoid-17          [-1, 3, 224, 224]               0
================================================================
Total params: 232,371
Trainable params: 232,371
Non-trainable params: 0
----------------------------------------------------------------
Input size (MB): 0.57
Forward/backward pass size (MB): 11.50
Params size (MB): 0.89
Estimated Total Size (MB): 12.96
----------------------------------------------------------------

Data Loader class is as follow
mean = torch.tensor((0.485, 0.456, 0.406)).reshape(1,3,1,1).cuda().requires_grad_(False)
std = torch.tensor((0.229, 0.224, 0.225)).reshape(1,3,1,1).cuda().requires_grad_(False)

def normalize(tensorInput, mean=mean, std=std, device=None):
    return tensorInput.sub(mean.to(device)).div(std.to(device))

def denormalize(tensorInput, mean=mean, std=std, device=None):
    return tensorInput.mul(std.to(device)).add(mean.to(device))

class DatasetLoader(torch.utils.data.Dataset):
    def __init__(self, root, transforms=None):
        self.root = root
        self.transforms = transforms
        self.imgs = list(sorted(os.listdir(root)))

    def __getitem__(self, idx):
        img_path = os.path.join(self.root, self.imgs[idx])
        img=cv2.imread(img_path)
        img=cv2.cvtColor(img,cv2.COLOR_BGR2RGB)
        img = cv2.resize(img,(32,32)) / 255.0

        img = np.transpose(img,[2,0,1]).astype(np.float32)
        return torch.from_numpy(img)

    def __len__(self):
        return len(self.imgs)

Dataloading and train script is as follow

dataset = DatasetLoader('D:\images\Bright_240_270', get_transform(train=True))

torch.manual_seed(1)
indices = torch.randperm(len(dataset)).tolist()
dataset = torch.utils.data.Subset(dataset, indices[:-50])
dataset_test = torch.utils.data.Subset(dataset, indices[-50:])

data_loader = torch.utils.data.DataLoader(
    dataset, batch_size=batchSize, shuffle=True, num_workers=0)

data_loader_test = torch.utils.data.DataLoader(
    dataset_test, batch_size=1, shuffle=False, num_workers=0)

device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

model = autoencoder().to(device)
summary(model, (3, 224, 224))

criterion = nn.MSELoss()
# criterion = nn.BCELoss()
optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate,weight_decay=1e-5)
trainLoss=[]
total_loss = 0
data_iter = 0
for epoch in range(num_epochs):
    data_iter = 0
    for data in data_loader:
        # print(data)
        img =  data.clone().detach().cuda()
        # print("Min Value of input Image = ",torch.min(img))
        # print("Max Value of input Image = ",torch.max(img))
        img = Variable(img).cuda()
        img= normalize(img)
        # ===================forward=====================
        output = model(img)
        print(data.min(),data.max(),output.min().item(),output.max().item())

        output = denormalize(output)
        # print("Input Image shape = ",img.shape)
        # print("Output Image shape = ",output.shape)
        loss = criterion(output,data.cuda())
        show_and_save_figure(data[0],output[0],'AutoEncoder')
        # ===================backward====================
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()
        data_iter=data_iter+1
        if data_iter % 10 == 0:
           print(f"Data Iteration = {data_iter}")
        total_loss += loss.data
    # ===================log========================
    total_loss /= len(data_loader)
    trainLoss.append(trainLoss.append(loss.data.cpu().detach().numpy()))
    plt.plot(trainLoss)
    plt.xlabel('Epochs')
    # naming the y axis
    plt.ylabel('Loss')
    # giving a title to my graph
    plt.title('Training loss')
    plt.show()
    print('epoch [{}/{}], loss:{:.4f}'
          .format(epoch+1, num_epochs, total_loss))

The  original image and image generated are
Where is the problem. Why I am not able to generate it?


title: pytorch modifying the input data to forward to make it suitable to my model

body:
Here is what I want to do.
I have an individual data of shape (20,20,20) where 20 tensors of shape (1,20,20) will be used as an input for 20 separate CNN. Here's the code I have so far.
class MyModel(torch.nn.Module):
   def __init__(self, ...):
       ...
        self.features = nn.ModuleList([nn.Sequential(
            nn.Conv2d(1,10, kernel_size = 3, padding = 1),
            nn.ReLU(),
            nn.Conv2d(10, 14, kernel_size=3, padding=1),
            nn.ReLU(),
            nn.Conv2d(14, 18, kernel_size=3, padding=1),
            nn.ReLU(),
            nn.Flatten(),
            nn.Linear(28*28*18, 256)
        ) for _ in range(20)])
        self.fc_module = nn.Sequential(
            nn.Linear(256*n_selected, cnn_output_dim),
            nn.Softmax(dim=n_classes)
        )

    def forward(self, input_list):
        concat_fusion = cat([cnn(x) for x,cnn in zip(input_list,self.features)], dim = 0)
        output = self.fc_module(concat_fusion)
        return output

The shape of the input_list in forward function is torch.Size([100, 20, 20, 20]), where 100 is the batch size.
However, there's an issue with
concat_fusion = cat([cnn(x) for x,cnn in zip(input_list,self.features)], dim = 0)

as it results in this error.
RuntimeError: Expected 4-dimensional input for 4-dimensional weight [10, 1, 3, 3], but got 3-dimensional input of size [20, 20, 20] instead

First off, I wonder why it expects me to give 4-dimensional weight [10,1,3,3]. I've seen
"RuntimeError: Expected 4-dimensional input for 4-dimensional weight 32 3 3, but got 3-dimensional input of size [3, 224, 224] instead"?
but I'm not sure where those specific numbers are coming from.

I have an input_list which is a batch of 100 data. I'm not sure how I can deal with individual data of shape (20,20,20) so that I can actually separate this into 20 pieces to use it as an independent input to 20 CNN.


title: how to use tensorboard addgraph in pointpillars

body:
I just started to learn object detection based on 3D point cloud recently, this algorithm is in Pointpillars.I want to use tensorboard to view the network structure, but an error occurred.
The following is the code of the network model.
class PFNLayer(nn.Module):
def __init__(self,
             in_channels,
             out_channels,
             use_norm=True,
             last_layer=False):
    """
    Pillar Feature Net Layer.
    The Pillar Feature Net could be composed of a series of these layers, but the PointPillars paper results only
    used a single PFNLayer. This layer performs a similar role as second.pytorch.voxelnet.VFELayer.
    :param in_channels: <int>. Number of input channels.
    :param out_channels: <int>. Number of output channels.
    :param use_norm: <bool>. Whether to include BatchNorm.
    :param last_layer: <bool>. If last_layer, there is no concatenation of features.
    """

    super().__init__()
    self.name = 'PFNLayer'
    self.last_vfe = last_layer
    if not self.last_vfe:
        out_channels = out_channels // 2
    self.units = out_channels

    if use_norm:
        BatchNorm1d = change_default_args(eps=1e-3, momentum=0.01)(nn.BatchNorm1d)
        Linear = change_default_args(bias=False)(nn.Linear)
    else:
        BatchNorm1d = Empty
        Linear = change_default_args(bias=True)(nn.Linear)

    self.linear = Linear(in_channels, self.units)
    self.norm = BatchNorm1d(self.units)

def forward(self, inputs):
    # mPFNLayerInputs = inputs
    x = self.linear(inputs)
    x = self.norm(x.permute(0, 2, 1).contiguous()).permute(0, 2, 1).contiguous()
    x = F.relu(x)

    x_max = torch.max(x, dim=1, keepdim=True)[0]

    if self.last_vfe:
        return x_max
    else:
        x_repeat = x_max.repeat(1, inputs.shape[1], 1)
        x_concatenated = torch.cat([x, x_repeat], dim=2)
        return x_concatenated

This is the code after I added Summary.add_graph().
class PFNLayer(nn.Module):
def __init__(self,
             in_channels,
             out_channels,
             use_norm=True,
             last_layer=False):
    """
    Pillar Feature Net Layer.
    The Pillar Feature Net could be composed of a series of these layers, but the PointPillars paper results only
    used a single PFNLayer. This layer performs a similar role as second.pytorch.voxelnet.VFELayer.
    :param in_channels: <int>. Number of input channels.
    :param out_channels: <int>. Number of output channels.
    :param use_norm: <bool>. Whether to include BatchNorm.
    :param last_layer: <bool>. If last_layer, there is no concatenation of features.
    """

    super().__init__()
    self.name = 'PFNLayer'
    self.last_vfe = last_layer
    if not self.last_vfe:
        out_channels = out_channels // 2
    self.units = out_channels

    if use_norm:
        BatchNorm1d = change_default_args(eps=1e-3, momentum=0.01)(nn.BatchNorm1d)
        Linear = change_default_args(bias=False)(nn.Linear)
    else:
        BatchNorm1d = Empty
        Linear = change_default_args(bias=True)(nn.Linear)

    self.linear = Linear(in_channels, self.units)
    self.norm = BatchNorm1d(self.units)

def forward(self, inputs):
    with SummaryWriter(comment='PFNLayer') as w:
        w.add_graph(self, input_to_model=inputs, verbose=False)
    return self.forward1(inputs)

def forward1(self, inputs):
    # mPFNLayerInputs = inputs
    x = self.linear(inputs)
    x = self.norm(x.permute(0, 2, 1).contiguous()).permute(0, 2, 1).contiguous()
    x = F.relu(x)

    x_max = torch.max(x, dim=1, keepdim=True)[0]

    if self.last_vfe:
        return x_max
    else:
        x_repeat = x_max.repeat(1, inputs.shape[1], 1)
        x_concatenated = torch.cat([x, x_repeat], dim=2)
        return x_concatenated

I don't know where the code is wrong, the following is the error message, I hope someone can help me, thank you very much.
Tracing can't be nested
Error occurs, No graph saved
Tracing can't be nested
Error occurs, No graph saved


title: If a transposed convolution layer takes in a 3 channel input and gives out a 2 channel output, how many kernels would it have and why?

body: Isn't the tconv layer supposed to have 6 kernels divided into two? One three kernels for one channel, another three kernels for another channel making two channels because the tconv layer is to give out a two channel output? I'm I right or what?


title: How to resolve the issue that optimizer.step() doesn't work

body:
fellows, I am a beginner of learning Deep Learning, I am trying to construct a new algorithm by combining the ResNet and the Bingham distribution, but I failed to get the features of this network since the code optimizer.step() didn't work which confused me a lot. My code snippet is pasted as follows.
wb = SummaryWriter("runs/tless_logs_1")
for epoch in range(config.dataset_train_test.epoch_num):
    verbose = epoch%5==0 or epoch==config.dataset_train_test.epoch_num-1
    log_train = loggers.get_train_logger(epoch,verbose)
    model.train()
    print(model.parameters)
    for i, (image, class_idx,q_r) in enumerate(train_loader):
        target_train_rotation = q_r.float().to(device)
        target_train_rotation = target_train_rotation.unsqueeze(dim=1)
        target_train_trans = torch.zeros((target_train_rotation.shape[0],1,4)).float().to(device)
        target_train = torch.cat((target_train_rotation, target_train_trans),dim=2).to(device)
        target_train = target_train.squeeze(dim=1)
        #print(target_train.shape)
        input_image = image.float().to(device)
        #input_image.requires_grad_(True)
        class_idx_device = class_idx.to(device)
        output = model(input_image,class_idx_device)
        print("output: ", output)

        loss_caonima = loss_fn(target_train, output,device)

        loss_cnm = torch.mean(loss_caonima)
        print("loss: ", loss_cnm)
        opt.zero_grad()
        loss_cnm.backward(retain_graph=True)
        opt.step()

        log_train.add_samples(target_train_rotation.squeeze(dim=1),output,loss_cnm.item(),class_idx)
        train_loss = loss_cnm.item()
        avg_loss+= train_loss
        iter+=1
        #avg_loss+= train_loss

        #if epoch==3:
        #    learning_rate=learning_rate*0.1
            #opt = optim.Adam(model.parameters(),lr=learning_rate)
        #    opt = optim.SGD(model.parameters(), lr=learning_rate,momentum=0.9)
        #if (epoch+1)%4==0:
        #    learning_rate = learning_rate/10
        #    opt = optim.SGD(model.parameters(), lr=learning_rate,momentum=0.9)
            #opt = optim.Adam(model.parameters(),lr=learning_rate)
        if iter%50==0:
            wb.add_scalar("training loss", avg_loss/50,iter)
            wb.add_scalar("likelihood", -avg_loss/50,iter)
            avg_loss=0.0
    log_train.show_tensorboard()
    print("epoch: {}, train loss: {}".format(epoch,train_loss))

The loss curve keeps being the same no matter if I uncomment the code block
            loss_cnm = torch.mean(loss_caonima)
            print("loss: ", loss_cnm)
            opt.zero_grad()
            loss_cnm.backward(retain_graph=True)
            opt.step()


title: the different result given by macOS terminal and VS Code?

body:
I use ssh to connect to my server and run a deep learning model written by python. The model is constructed by pytorch.
Once, i used iTerm to connect to the server, and run the model as usual. And the following errors appeared.

semaphore_tracker: There appear to be 20 leaked semaphores to clean up at shutdown

then I opened the VSCode and ran the same program, and everything was just fine as ususal. I could run the program without any error returned.
Could anybody help me with this please?


title: Error in Prediction results of Named Entity Recognition Task

body:
After training the NER Task with using RoBERTa Architecture, I got the below result
{
 'eval_loss': 0.003242955543100834,
 'eval_precision': 0.9959672534053343,
 'eval_recall': 0.9959672534053343,
 'eval_f1': 0.9959672534053343,
 'eval_accuracy': 0.9995624335836689
}

The result generally is quite high, as I expected. But here is my confusion, when I randomly input a set of sentences (out of the training set) to really know the model's performance.
My pseudo code:
def tokenize_and_align_labels_random(examples, tokenizer):
    tokenized_inputs = tokenizer(examples['tokens'],
                                 truncation=True,
                                 is_split_into_words=True)
    return tokenized_inputs
def preprocess_datasets(tokenizer, **datasets) -> Dict[str, Dataset]:
    tokenize_ner = partial(tokenize_and_align_labels_random,
                           tokenizer=tokenizer)
    return {k: ds.map(tokenize_ner) for k, ds in datasets.items()}

address=Testing_Dataset[Testing_Dataset['address']==1]['text'].apply(clean_doc).tolist()

da_datasets_random_Test = preprocess_datasets(tokenizer,
test=Dataset.from_dict({'tokens':address}))

results=da_trainer.predict(da_datasets_random_Test['test'])

predictions=results.predictions
predictions = np.argmax(predictions, axis=2)
# Remove ignored index (special tokens)
true_predictions = [
    [label_list[p] for (p, l) in zip(prediction, label) if l != -100]
    for prediction, label in zip(predictions, labels)
]

I input the sentences with some words that don't exist in the tokenizer vocabulary, and the model will handle that part for me by automatically generating their sub token.

That means the 'input_ids' will generate more token ids for presenting these cases, the problem is their predicted tags will also be increasing (based on how many tokens was delivered to the model).

For instance

Input sentence: "Giao tôi lê_lai phường hai tân_bình hcm"

Value after tokenizer:
{
  'input_ids': [0, 64003, 64003, 17489, 6115, 64139, 64151, 64003, 6446, 64313, 1340, 74780, 2],
  'token_type_ids': [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
  'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
}

Because tokenize of "lê_lai" is ['lê@@', 'l@@', 'ai']; of "tân_bình" is ['tân@@', 'bình']; of "hcm" is ['h@@', 'cm']

The result I got after all:
['O','O','B-LOC','I-LOC','I-LOC','I-LOC','I-LOC','I-LOC','O','I-LOC','I-LOC', 'O']

In fact, their prediction should only have 7 tags for the input tokens, but now it was more than this. So do guys have any strategies for this (I got one that we can train the tokenizer with more tokens).


title: DataCollatorForMultipleChoice gives KeyError: 'labels' in trainer.train

body:
I am working on multiple-choice QA. I am using the official notebook of huggingface/transformers which is implemented for SWAG dataset.
I want to use it for other multiple-choice datasets. Therefore, I add some modifications related to dataset. all code is given in notebook.
SWAG dataset contains following columns including 'label'.
 train: Dataset({
        features: ['video-id', 'fold-ind', 'startphrase', 'sent1', 'sent2', 'gold-source', 'ending0', 'ending1', 'ending2', 'ending3', 'label'],
        num_rows: 73546
    })

The dataset that I want to use has the following columns including 'answerKey' for target.
train: Dataset({
        features: ['id', 'question_stem', 'choices', 'answerKey'],
        num_rows: 4957
    })

The error is given in dataloader which is
@dataclass
class DataCollatorForMultipleChoice:
    """
    Data collator that will dynamically pad the inputs for multiple choice received.
    """

    tokenizer: PreTrainedTokenizerBase
    padding: Union[bool, str, PaddingStrategy] = True
    max_length: Optional[int] = None
    pad_to_multiple_of: Optional[int] = None

    def __call__(self, features):
        print(features[0].keys())
        label_name = "label" if "label" in features[0].keys() else "labels"
        labels = [feature.pop(label_name) for feature in features]
        batch_size = len(features)
        num_choices = len(features[0]["input_ids"])
        flattened_features = [[{k: v[i] for k, v in feature.items()} for i in range(num_choices)] for feature in features]
        flattened_features = sum(flattened_features, [])

        batch = self.tokenizer.pad(
            flattened_features,
            padding=self.padding,
            max_length=self.max_length,
            pad_to_multiple_of=self.pad_to_multiple_of,
            return_tensors="pt",
        )

        # Un-flatten
        batch = {k: v.view(batch_size, num_choices, -1) for k, v in batch.items()}
        # Add back labels
        batch["labels"] = torch.tensor(labels, dtype=torch.int64)
        return batch

It is given the error in the following line:
label_name = "label" if "label" in features[0].keys() else "labels"
        labels = [feature.pop(label_name) for feature in features]

the error is obtained in trainer.train()
KeyError                                  Traceback (most recent call last)
<ipython-input-64-3435b262f1ae> in <module>()
----> 1 trainer.train()

5 frames
<ipython-input-60-d1262e974b03> in <listcomp>(.0)
     18         print(features[0].keys())
     19         label_name = "label" if "label" in features[0].keys() else "labels"
---> 20         labels = [feature.pop(label_name) for feature in features]
     21         batch_size = len(features)
     22         num_choices = len(features[0]["input_ids"])

KeyError: 'labels'

I don't know what causes the error. I think it is related to target keys. But I could not solve it. Any ideas?
Thanks,


title: RNNs' Training in different batch_size result in different performance?

body:
After reading RNN source code in Pytorch and some blogs regarding with RNN, I want to verify my thoughts about RNN training. That is, for example, if we set batch_size from 1 to 4, the training process is totally different.
Here is the explanation about nn.RNNN() in Pytorch documentation.

h_n of shape (num_layers * num_directions, batch, hidden_size)

if batch_size = 1, then we only get one init hidden state (typically is a tensor with all zeros). While if batch_size = 4, we get four init hidden states, which means the input hidden state of RNN training in the fourth word now is different from the input hidden state of previous training process.
Considering the former one, the input hidden state of fourth word in this process (batch_size=1) contains the information of first three words, while the other one is totally zero.
So, can anybody tells me whether my thoughts are right or not?


title: How to fix runtime error "RuntimeError: value cannot be converted to type float without overflow: (6.6133e-07,-2.14879e-07)"

body:
I am relatively new to pytorch and currently trying to train a model using FCHarDNet architecture  for semantic segmentation. I am using a github repo from PingoLH (https://github.com/PingoLH/FCHarDNet). I managed to successfully train a model using pre-trained weights. However, when I try to continue training from the checkpoint, I receive the runtime error "RuntimeError: value cannot be converted to type float without overflow: (6.6133e-07,-2.14879e-07)"
This is the error message
Traceback (most recent call last):
  File "train.py", line 271, in <module>
    train(cfg, writer, logger)
  File "train.py", line 155, in train
    optimizer.step()
  File "/home/fyp/anaconda3/envs/HarDNet/lib/python3.7/site-packages/torch/optim/lr_scheduler.py", line 67, in wrapper
    return wrapped(*args, **kwargs)
  File "/home/fyp/anaconda3/envs/HarDNet/lib/python3.7/site-packages/torch/autograd/grad_mode.py", line 26, in decorate_context
    return func(*args, **kwargs)
  File "/home/fyp/anaconda3/envs/HarDNet/lib/python3.7/site-packages/torch/optim/sgd.py", line 112, in step
    p.add_(d_p, alpha=-group['lr'])
RuntimeError: value cannot be converted to type float without overflow: (6.6133e-07,-2.14879e-07)

This is the training script (train.py) :
import os
import yaml
import time
import shutil
import torch
import random
import argparse
import numpy as np
import torch.nn as nn

from torch.utils import data
from tqdm import tqdm

from ptsemseg.models import get_model
from ptsemseg.loss import get_loss_function
from ptsemseg.loader import get_loader
from ptsemseg.utils import get_logger
from ptsemseg.metrics import runningScore, averageMeter
from ptsemseg.augmentations import get_composed_augmentations
from ptsemseg.schedulers import get_scheduler
from ptsemseg.optimizers import get_optimizer

from tensorboardX import SummaryWriter

def weights_init(m):
    if isinstance(m, nn.Conv2d):
        nn.init.xavier_normal_(m.weight)

def train(cfg, writer, logger):

    # Setup seeds
    torch.manual_seed(cfg.get("seed", 1337))
    torch.cuda.manual_seed(cfg.get("seed", 1337))
    np.random.seed(cfg.get("seed", 1337))
    random.seed(cfg.get("seed", 1337))

    # Setup device
    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

    # Setup Augmentations
    augmentations = cfg["training"].get("augmentations", None)
    data_aug = get_composed_augmentations(augmentations)

    # Setup Dataloader
    data_loader = get_loader(cfg["data"]["dataset"])
    data_path = cfg["data"]["path"]

    t_loader = data_loader(
        data_path,
        is_transform=True,
        split=cfg["data"]["train_split"],
        #img_size=(1024,2048),
        img_size=(cfg["data"]["img_rows"], cfg["data"]["img_cols"]),
        augmentations=data_aug,
    )

    v_loader = data_loader(
        data_path,
        is_transform=True,
        split=cfg["data"]["val_split"],
        #img_size=(1024,2048),
        img_size=(cfg["data"]["img_rows"], cfg["data"]["img_cols"]),
    )

    n_classes = t_loader.n_classes
    trainloader = data.DataLoader(
        t_loader,
        batch_size=cfg["training"]["batch_size"],
        num_workers=cfg["training"]["n_workers"],
        shuffle=True,
    )

    valloader = data.DataLoader(
        v_loader, batch_size=cfg["training"]["batch_size"], num_workers=cfg["training"]["n_workers"]
    )

    # Setup Metrics
    running_metrics_val = runningScore(n_classes)

    # Setup Model
    model = get_model(cfg["model"], n_classes).to(device)

    total_params = sum(p.numel() for p in model.parameters())
    print( 'Parameters:',total_params )

    model = torch.nn.DataParallel(model, device_ids=range(torch.cuda.device_count()))
    model.apply(weights_init)
    pretrained_path='/home/fyp/Desktop/FCHarDNet/weights/hardnet_petite_base.pth'
    #pretrained_path='/home/fyp/Desktop/FCHarDNet/weights/hardnet70_cityscapes_model.pkl'
    #pretrained_path='/home/fyp/Desktop/FCHarDNet/runs/hardnet/cur/hardnet_cityscapes_checkpoint.pkl'
    weights = torch.load(pretrained_path)
    model.module.base.load_state_dict(weights)

    # Setup optimizer, lr_scheduler and loss function
    optimizer_cls = get_optimizer(cfg)
    optimizer_params = {k: v for k, v in cfg["training"]["optimizer"].items() if k != "name"}

    optimizer = optimizer_cls(model.parameters(), **optimizer_params)
    print("Using optimizer {}".format(optimizer))

    scheduler = get_scheduler(optimizer, cfg["training"]["lr_schedule"])

    loss_fn = get_loss_function(cfg)
    print("Using loss {}".format(loss_fn))

    start_iter = 0
    if cfg["training"]["resume"] is not None:
        if os.path.isfile(cfg["training"]["resume"]):
            logger.info(
                "Loading model and optimizer from checkpoint '{}'".format(cfg["training"]["resume"])
            )
            checkpoint = torch.load(cfg["training"]["resume"])
            model.load_state_dict(checkpoint["model_state"])
            optimizer.load_state_dict(checkpoint["optimizer_state"])
            scheduler.load_state_dict(checkpoint["scheduler_state"])
            start_iter = checkpoint["epoch"]
            logger.info(
                "Loaded checkpoint '{}' (iter {})".format(
                    cfg["training"]["resume"], checkpoint["epoch"]
                )
            )
        else:
            logger.info("No checkpoint found at '{}'".format(cfg["training"]["resume"]))

    if cfg["training"]["finetune"] is not None:
        if os.path.isfile(cfg["training"]["finetune"]):
            logger.info(
                "Loading model and optimizer from checkpoint '{}'".format(cfg["training"]["finetune"])
            )
            checkpoint = torch.load(cfg["training"]["finetune"])
            model.load_state_dict(checkpoint["model_state"])

    val_loss_meter = averageMeter()
    time_meter = averageMeter()

    best_iou = -100.0
    i = start_iter
    flag = True
    loss_all = 0
    loss_n = 0
    while i <= cfg["training"]["train_iters"] and flag:
        for (images, labels, _) in trainloader:
            i += 1
            start_ts = time.time()
            scheduler.step()
            model.train()
            images = images.to(device)
            labels = labels.to(device)

            optimizer.zero_grad()
            outputs = model(images)

            loss = loss_fn(input=outputs, target=labels)
            loss.backward()
            optimizer.step()
            c_lr = scheduler.get_lr()

            time_meter.update(time.time() - start_ts)
            loss_all += loss.item()
            loss_n += 1

            if (i + 1) % cfg["training"]["print_interval"] == 0:
                fmt_str = "Iter [{:d}/{:d}]  Loss: {:.4f}  Time/Image: {:.4f}  lr={:.6f}"
                print_str = fmt_str.format(
                    i + 1,
                    cfg["training"]["train_iters"],
                    loss_all / loss_n,
                    time_meter.avg / cfg["training"]["batch_size"],
                    c_lr[0],
                )

                print(print_str)
                logger.info(print_str)
                writer.add_scalar("loss/train_loss", loss.item(), i + 1)
                time_meter.reset()

            if (i + 1) % cfg["training"]["val_interval"] == 0 or (i + 1) == cfg["training"][
                "train_iters"
            ]:
                torch.cuda.empty_cache()
                model.eval()
                loss_all = 0
                loss_n = 0
                with torch.no_grad():
                    for i_val, (images_val, labels_val, _) in tqdm(enumerate(valloader)):
                        images_val = images_val.to(device)
                        labels_val = labels_val.to(device)

                        outputs = model(images_val)
                        val_loss = loss_fn(input=outputs, target=labels_val)

                        pred = outputs.data.max(1)[1].cpu().numpy()
                        gt = labels_val.data.cpu().numpy()

                        running_metrics_val.update(gt, pred)
                        val_loss_meter.update(val_loss.item())

                writer.add_scalar("loss/val_loss", val_loss_meter.avg, i + 1)
                logger.info("Iter %d Val Loss: %.4f" % (i + 1, val_loss_meter.avg))

                score, class_iou = running_metrics_val.get_scores()
                for k, v in score.items():
                    print(k, v)
                    logger.info("{}: {}".format(k, v))
                    writer.add_scalar("val_metrics/{}".format(k), v, i + 1)

                for k, v in class_iou.items():
                    logger.info("{}: {}".format(k, v))
                    writer.add_scalar("val_metrics/cls_{}".format(k), v, i + 1)

                val_loss_meter.reset()
                running_metrics_val.reset()

                state = {
                      "epoch": i + 1,
                      "model_state": model.state_dict(),
                      "optimizer_state": optimizer.state_dict(),
                      "scheduler_state": scheduler.state_dict(),
                }
                save_path = os.path.join(
                    writer.file_writer.get_logdir(),
                    "{}_{}_checkpoint.pkl".format(cfg["model"]["arch"], cfg["data"]["dataset"]),
                )
                torch.save(state, save_path)

                if score["Mean IoU : \t"] >= best_iou:
                    best_iou = score["Mean IoU : \t"]
                    state = {
                        "epoch": i + 1,
                        "model_state": model.state_dict(),
                        "best_iou": best_iou,
                    }
                    save_path = os.path.join(
                        writer.file_writer.get_logdir(),
                        "{}_{}_best_model.pkl".format(cfg["model"]["arch"], cfg["data"]["dataset"]),
                    )
                    torch.save(state, save_path)
                torch.cuda.empty_cache()

            if (i + 1) == cfg["training"]["train_iters"]:
                flag = False
                break

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="config")
    parser.add_argument(
        "--config",
        nargs="?",
        type=str,
        default="configs/hardnet.yml",
        help="Configuration file to use",
    )

    args = parser.parse_args()
    print(args.config)
    with open(args.config) as fp:
        cfg = yaml.load(fp)

    run_id = random.randint(1, 100000)
    logdir = os.path.join("runs", os.path.basename(args.config)[:-4], "cur")
    writer = SummaryWriter(log_dir=logdir)

    print("RUNDIR: {}".format(logdir))
    shutil.copy(args.config, logdir)

    logger = get_logger(logdir)
    logger.info("Let the games begin")

    train(cfg, writer, logger)

This is the YAML file for the parameters of the model
model:
    arch: hardnet
data:
    dataset: cityscapes
    train_split: train
    val_split: val
    img_rows: 512
    img_cols: 512
    path: /home/fyp/Desktop/FCHarDNet/data/cityscapes/leftImg8bit
    sbd_path: /home/fyp/Desktop/FCHarDNet/data/cityscapes/leftImg8bit

training:
    train_iters: 90000
    batch_size: 16
    val_interval: 500
    n_workers: 8
    print_interval: 10
    augmentations:
        hflip: 0.5
        rscale_crop: [512, 512]
    optimizer:
        name: 'sgd'
        lr: 0.02
        weight_decay: 0.0005
        momentum: 0.9
    loss:
        name: 'bootstrapped_cross_entropy'
        min_K: 4096
        loss_th: 0.3
        size_average: True
    lr_schedule:
        name: 'poly_lr'
        max_iter: 90000
    resume: /home/fyp/Desktop/FCHarDNet/runs/hardnet/cur/hardnet_cityscapes_checkpoint.pkl
    finetune: None

Would greatly appreciate if any help or advice could be given


title: torch equivalent of numpy.r_

body:
numpy has .r_
which

Translates slice objects to concatenation along the first axis.
This is a simple way to build up arrays quickly. There are two use
cases.
If the index expression contains comma separated arrays, then stack them along their first axis.
If the index expression contains slice notation or scalars then create a 1-D array with a range indicated by the slice notation.

I am looking for a torch equivalent, originally to answer to a comment here, but also in general.


title: ValueError: Expected more than 1 value per channel when training in pytorch 1.8

body:
The input tensor size is [1, 512] and I feed it in a GroupNorm layer.
The layer is self.fc_bn1 = nn.GroupNorm(512 // reduction, 512 // reduction) and the input is x = self.fc_bn1(x)
The error is visible in this image:

That will be worked when I run it in pytorch 1.1, but now I got a machine with pytorch 1.8 and it doesn't work. Since GPUs in this machine cannot use pytorch 1.1, I want to know how to solve this.
I already set model.eval(), so I guess it's not this reason.


title: pytorch cyclegann gives a Missing key error when testing

body:
I have trained a model using the pix2pix pytorch implementation and would like to test it.
However when I test it I get the error
model [CycleGANModel] was created
loading the model from ./checkpoints/cycbw50/latest_net_G_A.pth
Traceback (most recent call last):
  File "test.py", line 47, in <module>
    model.setup(opt)               # regular setup: load and print networks; create schedulers
  File "/media/bitlockermount/SmartImageToDigitalTwin/SmartImageToDigitalTwin/bin/python/cyclegann/pytorch-CycleGAN-and-pix2pix/models/base_model.py", line 88, in setup
    self.load_networks(load_suffix)
  File "/media/bitlockermount/SmartImageToDigitalTwin/SmartImageToDigitalTwin/bin/python/cyclegann/pytorch-CycleGAN-and-pix2pix/models/base_model.py", line 199, in load_networks
    net.load_state_dict(state_dict)
  File "/home/bst/.local/lib/python3.8/site-packages/torch/nn/modules/module.py", line 846, in load_state_dict
    raise RuntimeError('Error(s) in loading state_dict for {}:\n\t{}'.format(
RuntimeError: Error(s) in loading state_dict for ResnetGenerator:
    Missing key(s) in state_dict: "model.1.bias", "model.4.bias", "model.7.bias", "model.10.conv_block.1.bias", "model.10.conv_block.5.bias", "model.11.conv_block.1.bias", "model.11.conv_block.5.bias", "model.12.conv_block.1.bias", "model.12.conv_block.5.bias", "model.13.conv_block.1.bias", "model.13.conv_block.5.bias", "model.14.conv_block.1.bias", "model.14.conv_block.5.bias", "model.15.conv_block.1.bias", "model.15.conv_block.5.bias", "model.16.conv_block.1.bias", "model.16.conv_block.5.bias", "model.17.conv_block.1.bias", "model.17.conv_block.5.bias", "model.18.conv_block.1.bias", "model.18.conv_block.5.bias", "model.19.bias", "model.22.bias".
    Unexpected key(s) in state_dict: "model.2.weight", "model.2.bias", "model.5.weight", "model.5.bias", "model.8.weight", "model.8.bias", "model.10.conv_block.2.weight", "model.10.conv_block.2.bias", "model.10.conv_block.6.weight", "model.10.conv_block.6.bias", "model.11.conv_block.2.weight", "model.11.conv_block.2.bias", "model.11.conv_block.6.weight", "model.11.conv_block.6.bias", "model.12.conv_block.2.weight", "model.12.conv_block.2.bias", "model.12.conv_block.6.weight", "model.12.conv_block.6.bias", "model.13.conv_block.2.weight", "model.13.conv_block.2.bias", "model.13.conv_block.6.weight", "model.13.conv_block.6.bias", "model.14.conv_block.2.weight", "model.14.conv_block.2.bias", "model.14.conv_block.6.weight", "model.14.conv_block.6.bias", "model.15.conv_block.2.weight", "model.15.conv_block.2.bias", "model.15.conv_block.6.weight", "model.15.conv_block.6.bias", "model.16.conv_block.2.weight", "model.16.conv_block.2.bias", "model.16.conv_block.6.weight", "model.16.conv_block.6.bias", "model.17.conv_block.2.weight", "model.17.conv_block.2.bias", "model.17.conv_block.6.weight", "model.17.conv_block.6.bias", "model.18.conv_block.2.weight", "model.18.conv_block.2.bias", "model.18.conv_block.6.weight", "model.18.conv_block.6.bias", "model.20.weight", "model.20.bias", "model.23.weight", "model.23.bias".

The opt file for the training:
----------------- Options ---------------
               batch_size: 1
                    beta1: 0.5
          checkpoints_dir: ./checkpoints
           continue_train: False
                crop_size: 256
                 dataroot: ./datasets/datasets/boundedwalls_50_0.1/ [default: None]
             dataset_mode: aligned                          [default: unaligned]
                direction: AtoB
              display_env: main
             display_freq: 400
               display_id: 1
            display_ncols: 4
             display_port: 8097
           display_server: http://localhost
          display_winsize: 256
                    epoch: latest
              epoch_count: 1
                 gan_mode: lsgan
                  gpu_ids: 0
                init_gain: 0.02
                init_type: normal
                 input_nc: 3
                  isTrain: True                             [default: None]
                 lambda_A: 10.0
                 lambda_B: 10.0
          lambda_identity: 0.5
                load_iter: 0                                [default: 0]
                load_size: 286
                       lr: 0.0002
           lr_decay_iters: 50
                lr_policy: linear
         max_dataset_size: inf
                    model: cycle_gan
                 n_epochs: 100
           n_epochs_decay: 100
               n_layers_D: 3
                     name: cycbw50                          [default: experiment_name]
                      ndf: 64
                     netD: basic
                     netG: resnet_9blocks
                      ngf: 64
               no_dropout: True
                  no_flip: False
                  no_html: False
                     norm: batch                            [default: instance]
              num_threads: 4
                output_nc: 3
                    phase: train
                pool_size: 50
               preprocess: resize_and_crop
               print_freq: 100
             save_by_iter: False
          save_epoch_freq: 5
         save_latest_freq: 5000
           serial_batches: False
                   suffix:
         update_html_freq: 1000
                  verbose: False
----------------- End -------------------

And when testing it I used the following testing settings
python3 test.py --dataroot ./datasets/datasets/boundedwalls_50_0.1 --name cycbw50 --model pix2pix --netG resnet_9blocks --direction BtoA --dataset_mode aligned --norm batch --load_size 286

----------------- Options ---------------
             aspect_ratio: 1.0
               batch_size: 1
          checkpoints_dir: ./checkpoints
                crop_size: 256
                 dataroot: ./datasets/datasets/boundedwalls_50_0.1/ [default: None]
             dataset_mode: unaligned
                direction: AtoB
          display_winsize: 256
                    epoch: latest
                     eval: False
                  gpu_ids: 0
                init_gain: 0.02
                init_type: normal
                 input_nc: 3
                  isTrain: False                            [default: None]
                load_iter: 0                                [default: 0]
                load_size: 256
         max_dataset_size: inf
                    model: cycle_gan                        [default: test]
               n_layers_D: 3
                     name: cycbw50                          [default: experiment_name]
                      ndf: 64
                     netD: basic
                     netG: resnet_9blocks
                      ngf: 64
               no_dropout: True
                  no_flip: False
                     norm: instance
                 num_test: 50
              num_threads: 4
                output_nc: 3
                    phase: test
               preprocess: resize_and_crop
              results_dir: ./results/
           serial_batches: False
                   suffix:
                  verbose: False
----------------- End -------------------

Does anybody see what I'm doing wrong here? I would like to run this network such that I get results for individual images, so far the test function seems the most promising but it just crashes on this neural network.


title: Downloading Big Packages using conda in batches

body:
I am curious whether I can download big packages in batches using conda (like in web browsers where we can resume interrupted downloads).
I wanted to update my pytorch package but the size is too big for my limited mobile data.
The following packages will be downloaded:

    package                    |            build
    ---------------------------|-----------------
    libuv-1.40.0               |       he774522_0         255 KB
    pytorch-1.8.1              |py3.7_cuda10.1_cudnn7_0 836.4 MB  pytorch
    torchvision-0.9.1          |       py37_cu101         7.3 MB  pytorch
    ------------------------------------------------------------
                                           Total:       843.9 MB

The following NEW packages will be INSTALLED:

  libuv              pkgs/main/win-64::libuv-1.40.0-he774522_0

The following packages will be UPDATED:

  pytorch                      1.4.0-py3.7_cuda101_cudnn7_0 --> 1.8.1-py3.7_cuda10.1_cudnn7_0
  torchvision                              0.5.0-py37_cu101 --> 0.9.1-py37_cu101


title: Huggingface Electra - Load model trained with google implementation error: 'utf-8' codec can't decode byte 0x80 in position 64: invalid start byte

body:
I have trained an electra model from scratch using google implementation code.
python run_pretraining.py --data-dir gc://bucket-electra/dataset/ --model-name greek_electra --hparams hparams.json

with this json hyperparams:
{
"embedding_size": 768,
"max_seq_length": 512,
"train_batch_size": 128,
"vocab_size": 100000,
"model_size": "base",
"num_train_steps": 1500000
}

After having trained the model, I used the convert_electra_original_tf_checkpoint_to_pytorch.py script from transformers library to convert the checkpoint.
python convert_electra_original_tf_checkpoint_to_pytorch.py --tf_checkpoint_path output/models/transformer/greek_electra --config_file resources/hparams.json --pytorch_dump_path output/models/transformer/discriminator  --discriminator_or_generator "discriminator"

Now I am trying to load the model:
from transformers import ElectraForPreTraining

model = ElectraForPreTraining.from_pretrained('discriminator')

but I get the following error:
Traceback (most recent call last):
  File "~/.local/lib/python3.9/site-packages/transformers/configuration_utils.py", line 427, in get_config_dict
    config_dict = cls._dict_from_json_file(resolved_config_file)
  File "~/.local/lib/python3.9/site-packages/transformers/configuration_utils.py", line 510, in _dict_from_json_file
    text = reader.read()
  File "/usr/lib/python3.9/codecs.py", line 322, in decode
    (result, consumed) = self._buffer_decode(data, self.errors, final)
UnicodeDecodeError: 'utf-8' codec can't decode byte 0x80 in position 64: invalid start byte

Any ideas what's causing this & how to solve it?


title: Specify network to not backpropagate the gradients in a given function

body:
In my neural network, in order to compute the loss, I need to do some intermediate computations during training to first obtain some transformations rv.

rv = factor.ransac(source, target, prob, device)
predicted = factor.predict(source, rv, outputs, device)
loss = criterion(predicted, target)

I want to backpropagate the gradients only through predicted but not through factor.ransac. How do I do this?


title: How to use dataset transforms to reduce the dataset to one sample

body:
I'm trying to fine-tune a Reformer model using Pytorch with the help of Huggingfaces' Model: google/reformer-enwik8 and it is required to pack all data into a single sample which then needs to be padded to 65536 tokens.
To perform this I want to know how or whether I can directly use torchvision transforms for this task before loading data using DataLoader. Thank you.


title: Forward method call for weight training

body:
I'm trying to embed my graph adjacency matrix by aggregating neighbours and combining them (similar to GraphSAGE)

An adjacency matrix is of size nXn and the embedding will be of size nXd where d is less than n.

So, basically in my code, an adjacency matrix of a graph is fed as input for the purpose of embedding and the forward method should return the embedding.

import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
import numpy as np

class Setting(nn.Module):

    def __init__(self, A):

        super(Setting, self).__init__()

        self.A = A
        self.X = np.array(np.sum(A, axis=1))
        self.feature_len = 10

        self.L = 3
        self.n = 10
        self.z_dim = 8

        self.h = torch.empty(self.L-1,self.n, self.z_dim)

        self.h0 = torch.from_numpy(self.X)

        W0 = nn.init.xavier_uniform_(torch.empty(self.feature_len, self.z_dim))

        self.h[0] = torch.empty(self.n, self.z_dim)

        for v in range(self.n):
            self.h[0][v] = F.relu(self.h0[v]*W0)

    def forward(self, A):

        d_u = np.empty([self.n, self.n])

        for v in range(self.n):
            d_u[v] = self.X[v]*self.A[v]

        d_u = torch.from_numpy(d_u)

        h_n = torch.empty(self.L-1, self.n, self.z_dim)

        rnn = nn.GRUCell(self.z_dim, self.z_dim)

        H = F.normalize(self.h[0], p=2, dim=1)

        for l in range(0,self.L-1):

            h_n[l] = torch.mm(d_u, self.h[l-1].double())
            self.h[l] = rnn(self.h[l-1], h_n[l]).detach().float()
            self.h[l] = F.normalize(self.h[l], p=2, dim=1)

            H = torch.max(H, self.h[l])

        return H

Now based on the value of H, I want to train the weight W0, by taking MSELoss of H and H1 (another already known embedding).

net = Setting(A).double()
loss = nn.MSELoss()
optimizer = torch.optim.Adam(Setting.parameters(), lr=0.1)

for epoch in range(10):
    H = net(A)
    loss_calc = loss(H, H1)
    loss_calc.backward()
    optimizer.zero_grad()

print(H)

How to train the weight W0 based on this architecture? Any modifications in the code are highly appreciated.

Thanks in advance.


title: Data format and actual shape

body:
I'm trying to migrate TensorFlow checkpoint weights to PyTorch.

When I extract some weights with cp.load_variable(<CKPT>, <FIELD_NAME>), I get a 4D list ordered as HWCN, for example [1, 1, 512, 1024] which is clearly HWCN.
However, all convolution blocks data_format are set to NHWC.

So, the question is, why there's mismatch?
what should I believe? does the 4D list from cp.load_variable is correct and all left to do is permute the dimensions?

Thanks!


title: Tensorflow 2.x: List behavior of model architecture / select model parts via indices like in Pytorch

body:
Is it possible to imitate the pytorch behaviour of selecting parts of an existing model using list indices (for transfer learning) in TF 2.x?

Example:

In pytorch I can take the first seven layers or a mobilenet v2 architecture and use them as backbone for my custom architecture like this:

class MyCustomNetwork(nn.Module):
    def __init__(self, num_classes):
        super(MyCustomNetwork, self).__init__()
        model_mobilenetv2 = mobilenet_v2(pretrained=True, cpu_mode=cpu_mode)

        # Take output features after layer 7
        self.model_until_layer7 = model_mobilenetv2.features[:7]
        self.some_custom_layers = [...]

    def forward(self, x):
        x = self.model_until_layer7(x)
        x = self.some_custom_layers(x)
        return x

I know the TF transfer learning example but it only allows for the head of mobilenet_v2 and not the cutting after arbitrary layers


title: How to do make a single row inference for pertained time series model in pytorch

body:
Hello I am very new in pytorch,
I need some idea about single inference on pertained model. Recently I have implemented DeepAR model for point of interest in pytorch. It worked well for me as they described. What I need is to make inference on pertained model. My idea is to pass a timestamp in a range such as

inference(start_time, end_time):

     return prediction data frame/dictionary/list

They used basically fixed 192 train and test windows in param .json file. And they do 24 hour predict state. but I want  to infer a specific window prediction. My. statement is very general here. Need some idea of approaching. Here is the git hub repo link
https://github.com/zhykoties/TimeSeries


title: Create dataLoader with costume size labels

body:
I have a dataset with labels of size 64, and the inputs are images of size 15*17 in 3 channel. So, the feature (images) and labels are of size n*3*15*17 and n*64. I want to create a pytorch dataLoader to get mini-batch of data. I could do this  by expanding the dimension of labels to n*3*15*64 and concatenate it with the feature (images) to get batches and then separate before passing them into the model. But, I think it is not a good idea since when n is quite big, this can be quite expensive in terms of memory usage, and also involves some unnecessary computation.
I appreciate any suggestion?
And, here is an example:
import torch import numpy as np

n = 10
a = np.ones((n,3,15,17))
b = np.ones((n,64))

data = torch.utils.data.DataLoader(train_data, 5, shuffle = True)

in which I am not sure how to construct train_data out of a and b.


title: how to make dnn module with input, output tensor in pytorch

body:
thanks in advance.
below is the encoder network using tensorflow.
def encoder_net():
    input_ids = tf.keras.Input(shape=(MAX_SEQ_LEN,), batch_size=BATCH_SIZE, name="input_ids", dtype=tf.int32)
    attention_mask = tf.keras.Input(shape=(MAX_SEQ_LEN,), batch_size=BATCH_SIZE, name="attention_mask", dtype=tf.int32)
    token_type_ids = tf.keras.Input(shape=(MAX_SEQ_LEN,), batch_size=BATCH_SIZE, name="token_type_ids", dtype=tf.int32)

    bert_input = [input_ids, attention_mask, token_type_ids]
    encoder = TFAlbertModel.from_pretrained('albert-base-v2')

    embeddings = encoder(bert_input)
    encoder_network = tf.keras.Model(inputs=bert_input, outputs=embeddings)

    return encoder_network

I want to change to pytorch.
is there any module like tf.keras.Model in pytorch?
def encoder_net(input_ids, attention_mask, token_type_ids):
    bert_input = [input_ids, attention_mask, token_type_ids]
    encoder = AlbertModel.from_pretrained('albert-base-v2')
    embeddings = encoder(bert_input)
    encoder_network = **something**(inputs=bert_input, outputs=embeddings)

    return encoder_network


title: I am wondering why this 'residual' tensor behave like it used deepcopy? in pytorch

body:
I'm running a standard transformer model on MT task.
I found something strange while I was looking at the code.
In the code below, that "residual" tensor  acts as if it were created through a deepcopy function.
However, I found that a tensor copied through "=" is treated like a shallow copy fuction.
I am wondering why this 'residual' tensor behave like it used deepcopy?
code
residual = x

print("1residual[0][0][::5]",residual[0][0][::5])
print("1x[0][0][::5]",x[0][0][::5])

x, _ = self.multiheadattn(
                query=x,
                key=sub_x,
                value=sub_x,
                key_padding_mask=encoder_padding_mask,
                incremental_state=None,
                static_kv=True,
                need_weights=False,
                need_head_weights=False,
                )

print("2residual[0][0][::5]",residual[0][0][::5])
print("2x[0][0][::5]",x[0][0][::5])

x = residual + x

output
1residual[0][0][::5] tensor([ 1.3521,  1.3306, -1.1453, -0.1397, -2.2784, -1.1663, -0.5153,  1.5583,
         0.1059,  0.6325,  0.2289,  1.6822,  0.9763,  0.4017, -1.4673, -1.5497,
        -0.6318,  0.0108, -1.6200,  1.3530, -0.1374, -1.7577,  0.1391,  0.0248,
        -0.8227,  0.4263,  0.8211,  1.4342, -0.5377, -0.5277, -0.0795, -0.5049,
        -0.5339, -0.6436, -0.4398,  0.5060, -0.1911,  0.7437,  2.5046, -2.8393,
         1.1571,  0.1106, -0.2168,  0.4713,  1.1808,  0.2489, -0.7536, -0.0453,
        -0.2402, -0.5716,  1.2411, -2.4182, -0.1383, -2.1117,  0.0493,  0.6261,
        -0.1686, -2.0018,  0.7831, -1.4499,  0.8209, -0.5233,  0.7590,  1.9428,
        -0.5897,  0.2109,  1.0017, -0.4491,  0.6002,  1.1454,  1.2466, -0.1256,
         0.5996,  1.4363, -1.6691,  0.6059,  0.3088, -0.6391, -0.3145,  1.2027,
        -0.4174, -0.0501, -0.3436,  0.4720, -0.2447,  0.5388,  1.4538, -0.7511,
         0.1187, -0.1979,  1.0719,  2.0763, -0.7202,  0.3722,  1.1610, -0.2179,
        -0.7530,  0.3527,  0.6980,  0.9341,  1.0412,  0.0147, -0.2960],
       device='cuda:0', grad_fn=<SliceBackward>)
1x[0][0][::5] tensor([ 1.3521,  1.3306, -1.1453, -0.1397, -2.2784, -1.1663, -0.5153,  1.5583,
         0.1059,  0.6325,  0.2289,  1.6822,  0.9763,  0.4017, -1.4673, -1.5497,
        -0.6318,  0.0108, -1.6200,  1.3530, -0.1374, -1.7577,  0.1391,  0.0248,
        -0.8227,  0.4263,  0.8211,  1.4342, -0.5377, -0.5277, -0.0795, -0.5049,
        -0.5339, -0.6436, -0.4398,  0.5060, -0.1911,  0.7437,  2.5046, -2.8393,
         1.1571,  0.1106, -0.2168,  0.4713,  1.1808,  0.2489, -0.7536, -0.0453,
        -0.2402, -0.5716,  1.2411, -2.4182, -0.1383, -2.1117,  0.0493,  0.6261,
        -0.1686, -2.0018,  0.7831, -1.4499,  0.8209, -0.5233,  0.7590,  1.9428,
        -0.5897,  0.2109,  1.0017, -0.4491,  0.6002,  1.1454,  1.2466, -0.1256,
         0.5996,  1.4363, -1.6691,  0.6059,  0.3088, -0.6391, -0.3145,  1.2027,
        -0.4174, -0.0501, -0.3436,  0.4720, -0.2447,  0.5388,  1.4538, -0.7511,
         0.1187, -0.1979,  1.0719,  2.0763, -0.7202,  0.3722,  1.1610, -0.2179,
        -0.7530,  0.3527,  0.6980,  0.9341,  1.0412,  0.0147, -0.2960],
       device='cuda:0', grad_fn=<SliceBackward>)
2residual[0][0][::5] tensor([ 1.3521,  1.3306, -1.1453, -0.1397, -2.2784, -1.1663, -0.5153,  1.5583,
         0.1059,  0.6325,  0.2289,  1.6822,  0.9763,  0.4017, -1.4673, -1.5497,
        -0.6318,  0.0108, -1.6200,  1.3530, -0.1374, -1.7577,  0.1391,  0.0248,
        -0.8227,  0.4263,  0.8211,  1.4342, -0.5377, -0.5277, -0.0795, -0.5049,
        -0.5339, -0.6436, -0.4398,  0.5060, -0.1911,  0.7437,  2.5046, -2.8393,
         1.1571,  0.1106, -0.2168,  0.4713,  1.1808,  0.2489, -0.7536, -0.0453,
        -0.2402, -0.5716,  1.2411, -2.4182, -0.1383, -2.1117,  0.0493,  0.6261,
        -0.1686, -2.0018,  0.7831, -1.4499,  0.8209, -0.5233,  0.7590,  1.9428,
        -0.5897,  0.2109,  1.0017, -0.4491,  0.6002,  1.1454,  1.2466, -0.1256,
         0.5996,  1.4363, -1.6691,  0.6059,  0.3088, -0.6391, -0.3145,  1.2027,
        -0.4174, -0.0501, -0.3436,  0.4720, -0.2447,  0.5388,  1.4538, -0.7511,
         0.1187, -0.1979,  1.0719,  2.0763, -0.7202,  0.3722,  1.1610, -0.2179,
        -0.7530,  0.3527,  0.6980,  0.9341,  1.0412,  0.0147, -0.2960],
       device='cuda:0', grad_fn=<SliceBackward>)
2x[0][0][::5] tensor([ 0.1738,  0.6783, -0.2968,  1.1303,  0.6132, -0.5918,  0.2267, -0.3374,
        -0.5745,  0.1530,  0.0043,  0.6083,  0.3553, -0.5386, -0.2723,  0.0015,
         0.2741,  0.6016,  0.4180, -0.6023, -0.6721, -0.9145, -0.6082, -0.4389,
         0.8483, -0.0562, -0.5399, -1.0523, -0.6536, -0.0798,  0.2746, -0.0280,
         0.7279, -0.6152, -0.5345, -0.3318, -1.1347,  0.3154,  0.1172, -0.0724,
         0.1864,  0.3994, -0.4004,  0.2400, -0.7960,  0.9199, -1.3849, -0.8970,
         0.2758, -0.8580,  0.4118, -1.1278,  0.3327,  0.6214, -0.2227,  0.2295,
         0.1727,  0.5949,  0.6832,  0.0175,  0.7517, -0.0953,  0.3625, -0.3285,
        -1.1618,  0.3990, -0.0499,  0.8074, -0.6752, -0.4328,  0.1844,  0.9341,
        -0.3171, -1.1403, -0.2506,  0.5167, -0.1901,  0.6843,  0.3155,  0.0033,
        -0.0223,  0.8267,  0.1425, -0.3694, -0.0446, -0.5382, -0.0374,  0.5072,
        -0.3962, -0.2341,  0.2888,  0.3623,  0.7051, -0.3522, -0.7652, -0.8573,
        -0.5052, -0.1593, -0.6634,  0.2140,  0.2003,  0.2761, -0.5391],
       device='cuda:0', grad_fn=<SliceBackward>)


title: Pytorch Transform library explaination

body:
transforms.Normalize([0.5]*3, [0.5]*3)
Can someone help me to understand what this and how it works?


title: create mask with different columns for every row

body:
I'm trying to create matrix mask M where I can control the number of zeros in each row. As can be seen in the following code:
def create_mask(size, n_cols):
    M = torch.ones(size)

    for i in range(n_cols):
        idxs = torch.randperm(M.shape[1])
        M[torch.arange(len(M)), idxs] = 0

    return M

I've currently implemented it using a for loop as I haven't figured out how to do this using pytorch indexing / functions.
Is there a better method of doing this?


title: in training transfer learning mode getting as error module object is not collable

body:
i was training a transfer learning model 'resnet18' using torch and i am getting this error while training it .
my torch version is = 1.5.1
i am not sure why i have to face this error i have also checked all my importing modules but still didn't find bug in that . please help :) .
---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<ipython-input-163-eea9d0f69e32> in <module>
----> 1 model_ft, hist = train_model(model_ft, dataloaders , criterion, optimizer_ft, num_epochs=2)

<ipython-input-162-8427f4cf7832> in train_model(model, dataloaders, criterion, optimizer, num_epochs, is_inception)
     22
     23             # Iterate over data.
---> 24             for inputs, labels in dataloaders[phase]:
     25                 # inputs = inputs.to(device)
     26                 # labels = labels.to(device)

~\AppData\Local\Continuum\anaconda3\lib\site-packages\torch\utils\data\dataloader.py in __next__(self)
    343
    344     def __next__(self):
--> 345         data = self._next_data()
    346         self._num_yielded += 1
    347         if self._dataset_kind == _DatasetKind.Iterable and \

~\AppData\Local\Continuum\anaconda3\lib\site-packages\torch\utils\data\dataloader.py in _next_data(self)
    383     def _next_data(self):
    384         index = self._next_index()  # may raise StopIteration
--> 385         data = self._dataset_fetcher.fetch(index)  # may raise StopIteration
    386         if self._pin_memory:
    387             data = _utils.pin_memory.pin_memory(data)

~\AppData\Local\Continuum\anaconda3\lib\site-packages\torch\utils\data\_utils\fetch.py in fetch(self, possibly_batched_index)
     42     def fetch(self, possibly_batched_index):
     43         if self.auto_collation:
---> 44             data = [self.dataset[idx] for idx in possibly_batched_index]
     45         else:
     46             data = self.dataset[possibly_batched_index]

~\AppData\Local\Continuum\anaconda3\lib\site-packages\torch\utils\data\_utils\fetch.py in <listcomp>(.0)
     42     def fetch(self, possibly_batched_index):
     43         if self.auto_collation:
---> 44             data = [self.dataset[idx] for idx in possibly_batched_index]
     45         else:
     46             data = self.dataset[possibly_batched_index]

~\AppData\Local\Continuum\anaconda3\lib\site-packages\torchvision\datasets\folder.py in __getitem__(self, index)
    135         sample = self.loader(path)
    136         if self.transform is not None:
--> 137             sample = self.transform(sample)
    138         if self.target_transform is not None:
    139             target = self.target_transform(target)

TypeError: 'module' object is not callable

this is my training function
def train_model(model, dataloaders, criterion, optimizer, num_epochs=25, is_inception=False):
    since = time.time()

    val_acc_history = []

    best_model_wts = copy.deepcopy(model.state_dict())
    best_acc = 0.0

    for epoch in range(num_epochs):
        print('Epoch {}/{}'.format(epoch, num_epochs - 1))
        print('-' * 10)

        # Each epoch has a training and validation phase
        for phase in ['train', 'val']:
            if phase == 'train':
                model.train()  # Set model to training mode
            else:
                model.eval()   # Set model to evaluate mode

            running_loss = 0.0
            running_corrects = 0

            # Iterate over data.
            for inputs, labels in dataloaders[phase]:
                # inputs = inputs.to(device)
                # labels = labels.to(device)

                # zero the parameter gradients
                optimizer.zero_grad()

                # forward
                # track history if only in train
                with torch.set_grad_enabled(phase == 'train'):
                    # Get model outputs and calculate loss
                    # Special case for inception because in training it has an auxiliary output. In train
                    #   mode we calculate the loss by summing the final output and the auxiliary output
                    #   but in testing we only consider the final output.
                    if is_inception and phase == 'train':
                        # From https://discuss.pytorch.org/t/how-to-optimize-inception-model-with-auxiliary-classifiers/7958
                        outputs, aux_outputs = model(inputs)
                        loss1 = criterion(outputs, labels)
                        loss2 = criterion(aux_outputs, labels)
                        loss = loss1 + 0.4*loss2
                    else:
                        outputs = model(inputs)
                        loss = criterion(outputs, labels)

                    _, preds = torch.max(outputs, 1)

                    # backward + optimize only if in training phase
                    if phase == 'train':
                        loss.backward()
                        optimizer.step()

                # statistics
                running_loss += loss.item() * inputs.size(0)
                running_corrects += torch.sum(preds == labels.data)

            epoch_loss = running_loss / len(dataloaders[phase].dataset)
            epoch_acc = running_corrects.double() / len(dataloaders[phase].dataset)

            print('{} Loss: {:.4f} Acc: {:.4f}'.format(phase, epoch_loss, epoch_acc))

            # deep copy the model
            if phase == 'val' and epoch_acc > best_acc:
                best_acc = epoch_acc
                best_model_wts = copy.deepcopy(model.state_dict())
            if phase == 'val':
                val_acc_history.append(epoch_acc)

        print()

    time_elapsed = time.time() - since
    print('Training complete in {:.0f}m {:.0f}s'.format(time_elapsed // 60, time_elapsed % 60))
    print('Best val Acc: {:4f}'.format(best_acc))

    # load best model weights
    model.load_state_dict(best_model_wts)
    return model, val_acc_history

i thought it was because of calling transforms object just by importing its module but it didn't worked out .
import torchvision.transforms as transforms

thanks in advance .