title: PyTorch ResNet-18: Counting pruned parameters

body:
I have a trained ResNet-18 coded in PyTorch 1.8 and Python-3.8 as follows:
class ResidualBlock(nn.Module):
    '''
    Residual Block within a ResNet CNN model
    '''
    def __init__(self, input_channels, num_channels,
                 use_1x1_conv = False, strides = 1):
        # super(ResidualBlock, self).__init__()
        super().__init__()

        self.conv1 = nn.Conv2d(
            in_channels = input_channels, out_channels = num_channels,
            kernel_size = 3, padding = 1, stride = strides,
            bias = False
            )
        self.bn1 = nn.BatchNorm2d(num_features = num_channels)

        self.conv2 = nn.Conv2d(
            in_channels = num_channels, out_channels = num_channels,
            kernel_size = 3, padding = 1, stride = 1,
            bias = False
            )
        self.bn2 = nn.BatchNorm2d(num_features = num_channels)

        if use_1x1_conv:
            self.conv3 = nn.Conv2d(
                in_channels = input_channels, out_channels = num_channels,
                kernel_size = 1, stride = strides
                )
            self.bn3 = nn.BatchNorm2d(num_features = num_channels)
        else:
            self.conv3 = None

        self.relu = nn.ReLU(inplace = True)

        self.initialize_weights()

    def forward(self, X):
        Y = F.relu(self.bn1(self.conv1(X)))
        Y = self.bn2(self.conv2(Y))

        if self.conv3:
            X = self.bn3(self.conv3(X))
            # print(f"X.shape due to 1x1: {X.shape} & Y.shape = {Y.shape}")
        else:
            # print(f"X.shape without 1x1: {X.shape} & Y.shape = {Y.shape}")
            pass

        Y += X
        return F.relu(Y)

    def shape_computation(self, X):
        Y = self.conv1(X)
        print(f"self.conv1(X).shape: {Y.shape}")
        Y = self.conv2(Y)
        print(f"self.conv2(X).shape: {Y.shape}")

        if self.conv3:
            h = self.conv3(X)
            print(f"self.conv3(X).shape: {h.shape}")

    def initialize_weights(self):
        for m in self.modules():
            # print(m)
            if isinstance(m, nn.Conv2d):
                nn.init.kaiming_uniform_(m.weight)

                '''
                # Do not initialize bias (due to batchnorm)-
                if m.bias is not None:
                    nn.init.constant_(m.bias, 0)
                '''

            elif isinstance(m, nn.BatchNorm2d):
                # Standard initialization for batch normalization-
                nn.init.constant_(m.weight, 1)
                nn.init.constant_(m.bias, 0)

            elif isinstance(m, nn.Linear):
                nn.init.kaiming_normal_(m.weight)
                nn.init.constant_(m.bias, 0)

b0 = nn.Sequential(
    nn.Conv2d(in_channels = 3, out_channels = 64, kernel_size = 3, stride = 1, padding = 1),
    nn.BatchNorm2d(num_features = 64),
    nn.ReLU())

def create_resnet_block(input_filters, output_filters, num_residuals, first_block = False):
    # Python list to hold the created ResNet blocks-
    resnet_blk = []

    for i in range(num_residuals):
        if i == 0 and first_block:
            resnet_blk.append(ResidualBlock(input_channels = input_filters, num_channels = output_filters, use_1x1_conv = True, strides = 2))
        else:
            resnet_blk.append(ResidualBlock(input_channels = output_filters, num_channels = output_filters, use_1x1_conv = False, strides = 1))

    return resnet_blk

b1 = nn.Sequential(*create_resnet_block(input_filters = 64, output_filters = 64, num_residuals = 2, first_block = True))

b2 = nn.Sequential(*create_resnet_block(input_filters = 64, output_filters = 128, num_residuals = 2, first_block = True))

b3 = nn.Sequential(*create_resnet_block(input_filters = 128, output_filters = 256, num_residuals = 2, first_block = True))

b4 = nn.Sequential(*create_resnet_block(input_filters = 256, output_filters = 512, num_residuals = 2, first_block = True))

# Initialize a ResNet-18 CNN model-
model = nn.Sequential(
    b0, b1, b2, b3, b4,
    nn.AdaptiveAvgPool2d(output_size = (1, 1)),
    nn.Flatten(),
    nn.Linear(in_features = 512, out_features = 10))

The layer names are now as follows:
for layer_name, param in trained_model.named_parameters():
    print(f"layer name: {layer_name} has {param.shape}")

layer name: 0.0.weight has torch.Size([64, 3, 3, 3])

layer name: 0.0.bias has torch.Size([64])

layer name: 0.1.weight has torch.Size([64])

layer name: 0.1.bias has torch.Size([64])

layer name: 1.0.conv1.weight has torch.Size([64, 64, 3, 3])

layer name: 1.0.bn1.weight has torch.Size([64])

layer name: 1.0.bn1.bias has torch.Size([64])

layer name: 1.0.conv2.weight has torch.Size([64, 64, 3, 3])

layer name: 1.0.bn2.weight has torch.Size([64])

layer name: 1.0.bn2.bias has torch.Size([64])

layer name: 1.0.conv3.weight has torch.Size([64, 64, 1, 1])

layer name: 1.0.conv3.bias has torch.Size([64])

layer name: 1.0.bn3.weight has torch.Size([64])

layer name: 1.0.bn3.bias has torch.Size([64])

layer name: 1.1.conv1.weight has torch.Size([64, 64, 3, 3])

layer name: 1.1.bn1.weight has torch.Size([64])

layer name: 1.1.bn1.bias has torch.Size([64])

layer name: 1.1.conv2.weight has torch.Size([64, 64, 3, 3])

layer name: 1.1.bn2.weight has torch.Size([64])

layer name: 1.1.bn2.bias has torch.Size([64])

layer name: 2.0.conv1.weight has torch.Size([128, 64, 3, 3])

layer name: 2.0.bn1.weight has torch.Size([128])

layer name: 2.0.bn1.bias has torch.Size([128])

layer name: 2.0.conv2.weight has torch.Size([128, 128, 3, 3])

layer name: 2.0.bn2.weight has torch.Size([128])

layer name: 2.0.bn2.bias has torch.Size([128])

layer name: 2.0.conv3.weight has torch.Size([128, 64, 1, 1])

layer name: 2.0.conv3.bias has torch.Size([128])

layer name: 2.0.bn3.weight has torch.Size([128])

layer name: 2.0.bn3.bias has torch.Size([128])

layer name: 2.1.conv1.weight has torch.Size([128, 128, 3, 3])

layer name: 2.1.bn1.weight has torch.Size([128])

layer name: 2.1.bn1.bias has torch.Size([128])

layer name: 2.1.conv2.weight has torch.Size([128, 128, 3, 3])

layer name: 2.1.bn2.weight has torch.Size([128])

layer name: 2.1.bn2.bias has torch.Size([128])

layer name: 3.0.conv1.weight has torch.Size([256, 128, 3, 3])

layer name: 3.0.bn1.weight has torch.Size([256])

layer name: 3.0.bn1.bias has torch.Size([256])

layer name: 3.0.conv2.weight has torch.Size([256, 256, 3, 3])

layer name: 3.0.bn2.weight has torch.Size([256])

layer name: 3.0.bn2.bias has torch.Size([256])

layer name: 3.0.conv3.weight has torch.Size([256, 128, 1, 1])

layer name: 3.0.conv3.bias has torch.Size([256])

layer name: 3.0.bn3.weight has torch.Size([256])

layer name: 3.0.bn3.bias has torch.Size([256])

layer name: 3.1.conv1.weight has torch.Size([256, 256, 3, 3])

layer name: 3.1.bn1.weight has torch.Size([256])

layer name: 3.1.bn1.bias has torch.Size([256])

layer name: 3.1.conv2.weight has torch.Size([256, 256, 3, 3])

layer name: 3.1.bn2.weight has torch.Size([256])

layer name: 3.1.bn2.bias has torch.Size([256])

layer name: 4.0.conv1.weight has torch.Size([512, 256, 3, 3])

layer name: 4.0.bn1.weight has torch.Size([512])

layer name: 4.0.bn1.bias has torch.Size([512])

layer name: 4.0.conv2.weight has torch.Size([512, 512, 3, 3])

layer name: 4.0.bn2.weight has torch.Size([512])

layer name: 4.0.bn2.bias has torch.Size([512])

layer name: 4.0.conv3.weight has torch.Size([512, 256, 1, 1])

layer name: 4.0.conv3.bias has torch.Size([512])

layer name: 4.0.bn3.weight has torch.Size([512])

layer name: 4.0.bn3.bias has torch.Size([512])

layer name: 4.1.conv1.weight has torch.Size([512, 512, 3, 3])

layer name: 4.1.bn1.weight has torch.Size([512])

layer name: 4.1.bn1.bias has torch.Size([512])

layer name: 4.1.conv2.weight has torch.Size([512, 512, 3, 3])

layer name: 4.1.bn2.weight has torch.Size([512])

layer name: 4.1.bn2.bias has torch.Size([512])

layer name: 7.weight has torch.Size([10, 512])

layer name: 7.bias has torch.Size([10])

In order to prune a LeNet-5 sample CNN model (in the mentioned tutorial), I am referring to PyTorch pruning tutorial, it's mentioned here that to prune a module/layer, use the following code:
parameters_to_prune = (
    (model.conv1, 'weight'),
    (model.conv2, 'weight'),
    (model.fc1, 'weight'),
    (model.fc2, 'weight'),
    (model.fc3, 'weight'),
)

To count the number of pruned parameters, the following code has been used:
print(
    "Sparsity in conv1.weight: {:.2f}%".format(
        100. * float(torch.sum(model.conv1.weight == 0))
        / float(model.conv1.weight.nelement())
    )
)
print(
    "Sparsity in conv2.weight: {:.2f}%".format(
        100. * float(torch.sum(model.conv2.weight == 0))
        / float(model.conv2.weight.nelement())
    )
)
print(
    "Sparsity in fc1.weight: {:.2f}%".format(
        100. * float(torch.sum(model.fc1.weight == 0))
        / float(model.fc1.weight.nelement())
    )
)
print(
    "Sparsity in fc2.weight: {:.2f}%".format(
        100. * float(torch.sum(model.fc2.weight == 0))
        / float(model.fc2.weight.nelement())
    )
)
print(
    "Sparsity in fc3.weight: {:.2f}%".format(
        100. * float(torch.sum(model.fc3.weight == 0))
        / float(model.fc3.weight.nelement())
    )
)
print(
    "Global sparsity: {:.2f}%".format(
        100. * float(
            torch.sum(model.conv1.weight == 0)
            + torch.sum(model.conv2.weight == 0)
            + torch.sum(model.fc1.weight == 0)
            + torch.sum(model.fc2.weight == 0)
            + torch.sum(model.fc3.weight == 0)
        )
        / float(
            model.conv1.weight.nelement()
            + model.conv2.weight.nelement()
            + model.fc1.weight.nelement()
            + model.fc2.weight.nelement()
            + model.fc3.weight.nelement()
        )
    )
)

However, this doesn't apply to my case as manually mentioning each layer doesn't scale well. Say for ResNet-101/152, etc. How can I compute the sparsity:

per layer
globally

as shown in the sample code snippet above?
How do I handle this?


title: Binary classification on MNIST: loss and accuracies remain costant

body:
I am trying to do binary classification on MNIST dataset. Class 0 for even numbers and class 1 for odd numbers. I am using a simplified version of VGG.
My NN has a loss and an accuracy that remain costant.
I want to say that my model, reached to over 90% of accuracy before of changing targets into binary targets, so probably there is something wrong.
Here I change the targets into binary:
for i in range(10):
  idx = (train_set.targets==i)
  if (i == 0) or ((i % 2) == 0): train_set.targets[idx] = 0

  else: train_set.targets[idx] = 1

for i in range(10):
  idx = (test_set.targets==i)
  if (i == 0) or ((i % 2) == 0): test_set.targets[idx] = 0

  else: test_set.targets[idx] = 1

This is my net:
class VGG16(nn.Module):

    def __init__(self, num_classes):
        super(VGG16, self).__init__()

        # calculate same padding:
        # (w - k + 2*p)/s + 1 = o
        # => p = (s(o-1) - w + k)/2

        self.block_1 = nn.Sequential(
            nn.Conv2d(in_channels=1,
                      out_channels=64,
                      kernel_size=(3, 3),
                      stride=(1, 1),
                      # (1(32-1)- 32 + 3)/2 = 1
                      padding=1),
            nn.BatchNorm2d(64),
            nn.ReLU(),
            nn.Conv2d(in_channels=64,
                      out_channels=64,
                      kernel_size=(3, 3),
                      stride=(1, 1),
                      padding=1),
            nn.BatchNorm2d(64),
            nn.ReLU(),
            nn.MaxPool2d(kernel_size=(2, 2),
                         stride=(2, 2))
        )

        self.block_2 = nn.Sequential(
            nn.Conv2d(in_channels=64,
                      out_channels=128,
                      kernel_size=(3, 3),
                      stride=(1, 1),
                      padding=1),
            nn.BatchNorm2d(128),
            nn.ReLU(),
            nn.Conv2d(in_channels=128,
                      out_channels=128,
                      kernel_size=(3, 3),
                      stride=(1, 1),
                      padding=1),
            nn.BatchNorm2d(128),
            nn.ReLU(),
            nn.MaxPool2d(kernel_size=(2, 2),
                         stride=(2, 2))
        )

        self.block_3 = nn.Sequential(
            nn.Conv2d(in_channels=128,
                      out_channels=256,
                      kernel_size=(3, 3),
                      stride=(1, 1),
                      padding=1),
            nn.BatchNorm2d(256),
            nn.ReLU(),
            nn.Conv2d(in_channels=256,
                      out_channels=256,
                      kernel_size=(3, 3),
                      stride=(1, 1),
                      padding=1),
            nn.BatchNorm2d(256),
            nn.ReLU(),
            nn.Conv2d(in_channels=256,
                      out_channels=256,
                      kernel_size=(3, 3),
                      stride=(1, 1),
                      padding=1),
            nn.BatchNorm2d(256),
            nn.ReLU(),
            nn.MaxPool2d(kernel_size=(2, 2),
                         stride=(2, 2))
        )

        self.block_4 = nn.Sequential(
            nn.Conv2d(in_channels=256,
                      out_channels=512,
                      kernel_size=(3, 3),
                      stride=(1, 1),
                      padding=1),
            nn.BatchNorm2d(512),
            nn.ReLU(),
            nn.Conv2d(in_channels=512,
                      out_channels=512,
                      kernel_size=(3, 3),
                      stride=(1, 1),
                      padding=1),
            nn.BatchNorm2d(512),
            nn.ReLU(),
            nn.Conv2d(in_channels=512,
                      out_channels=512,
                      kernel_size=(3, 3),
                      stride=(1, 1),
                      padding=1),
            nn.BatchNorm2d(512),
            nn.ReLU(),
            nn.MaxPool2d(kernel_size=(2, 2),
                         stride=(2, 2))
        )

        self.classifier = nn.Sequential(
            nn.Linear(2048, 4096),
            nn.ReLU(True),
            nn.Dropout(p=0.65),
            nn.Linear(4096, 4096),
            nn.ReLU(True),
            nn.Dropout(p=0.65),
            nn.Linear(4096, num_classes),
            nn.Sigmoid()
        )

        for m in self.modules():
            if isinstance(m, torch.nn.Conv2d) or isinstance(m, torch.nn.Linear):
                nn.init.kaiming_uniform_(m.weight, mode='fan_in', nonlinearity='leaky_relu')
#                 nn.init.xavier_normal_(m.weight)
                if m.bias is not None:
                    m.bias.detach().zero_()

        # self.avgpool = nn.AdaptiveAvgPool2d((7, 7))

    def forward(self, x):

        x = self.block_1(x)
        x = self.block_2(x)
        x = self.block_3(x)
        x = self.block_4(x)
        # x = self.avgpool(x)
        x = x.view(x.size(0), -1)
        x = self.classifier(x)
        return x
        #logits = self.classifier(x)
        #probas = F.softmax(logits, dim=1)
        # probas = nn.Softmax(logits)
        #return probas
        # return logits

# Define an optimizier
import torch.optim as optim
optimizer = optim.SGD(model.parameters(), lr = 0.01)
# Define a loss
criterion = nn.BCELoss()

def train(net, loaders, optimizer, criterion, epochs=20, dev=dev, save_param = False, model_name="valerio"):
    try:
        net = net.to(dev)
        #print(net)
        # Initialize history
        history_loss = {"train": [], "val": [], "test": []}
        history_accuracy = {"train": [], "val": [], "test": []}
        # Store the best val accuracy
        best_val_accuracy = 0

        # Process each epoch
        for epoch in range(epochs):
            # Initialize epoch variables
            sum_loss = {"train": 0, "val": 0, "test": 0}
            sum_accuracy = {"train": 0, "val": 0, "test": 0}
            # Process each split
            for split in ["train", "val", "test"]:
                if split == "train":
                  net.train()
                else:
                  net.eval()
                # Process each batch
                for (input, labels) in loaders[split]:
                    # Move to CUDA
                    input = input.to(dev)
                    labels = labels.to(dev)
                    # Reset gradients
                    optimizer.zero_grad()
                    # Compute output
                    pred = net(input)
                    labels = labels.unsqueeze(1)
                    labels = labels.float()
                    loss = criterion(pred, labels)
                    # Update loss
                    sum_loss[split] += loss.item()
                    # Check parameter update
                    if split == "train":
                        # Compute gradients
                        loss.backward()
                        # Optimize
                        optimizer.step()
                    # Compute accuracy
                    _,pred_labels = pred.max(1)
                    batch_accuracy = (pred_labels == labels).sum().item()/input.size(0)
                    # Update accuracy
                    sum_accuracy[split] += batch_accuracy
            # Compute epoch loss/accuracy
            epoch_loss = {split: sum_loss[split]/len(loaders[split]) for split in ["train", "val", "test"]}
            epoch_accuracy = {split: sum_accuracy[split]/len(loaders[split]) for split in ["train", "val", "test"]}

            # Store params at the best validation accuracy
            if save_param and epoch_accuracy["val"] > best_val_accuracy:
              #torch.save(net.state_dict(), f"{net.__class__.__name__}_best_val.pth")
              torch.save(net.state_dict(), f"{model_name}_best_val.pth")
              best_val_accuracy = epoch_accuracy["val"]

            # Update history
            for split in ["train", "val", "test"]:
                history_loss[split].append(epoch_loss[split])
                history_accuracy[split].append(epoch_accuracy[split])
            # Print info
            print(f"Epoch {epoch+1}:",
                  f"TrL={epoch_loss['train']:.4f},",
                  f"TrA={epoch_accuracy['train']:.4f},",
                  f"VL={epoch_loss['val']:.4f},",
                  f"VA={epoch_accuracy['val']:.4f},",
                  f"TeL={epoch_loss['test']:.4f},",
                  f"TeA={epoch_accuracy['test']:.4f},")
    except KeyboardInterrupt:
        print("Interrupted")
    finally:
        # Plot loss
        plt.title("Loss")
        for split in ["train", "val", "test"]:
            plt.plot(history_loss[split], label=split)
        plt.legend()
        plt.show()
        # Plot accuracy
        plt.title("Accuracy")
        for split in ["train", "val", "test"]:
            plt.plot(history_accuracy[split], label=split)
        plt.legend()
        plt.show()

From the previous model of digit recognition i changed only the targets, and the final layer of classifier from 10 classes to 1 class + Sigmoid. And i changed also cross entropy to BCELoss. What I am doing wrong?
These are loss and accuracy values:
Epoch 1: TrL=49.0955, TrA=31.4211, VL=49.7285, VA=31.7340, TeL=49.2635, TeA=31.3758,
Epoch 2: TrL=49.0992, TrA=31.4235, VL=49.7285, VA=31.7340, TeL=49.2635, TeA=31.3758,
Epoch 3: TrL=49.0899, TrA=31.4176, VL=49.7285, VA=31.7340, TeL=49.2635, TeA=31.3758,
Epoch 4: TrL=49.0936, TrA=31.4199, VL=49.7285, VA=31.7340, TeL=49.2635, TeA=31.3758,
Epoch 5: TrL=49.0936, TrA=31.4199, VL=49.7285, VA=31.7340, TeL=49.2635, TeA=31.3758,
Epoch 6: TrL=49.0825, TrA=31.4128, VL=49.7285, VA=31.7340, TeL=49.2635, TeA=31.3758,

What's wrong? How is it possible that with 10 classes I reached over 90% accuracy, and with a simplified version, only 2 classes, I reach 30% of accuracy?
Edit: increasing batch size from 64 to 128, accuracy reaches to 60% and remains constant...


title: Batch inference of softmax does not sum to 1

body:
I am working with REINFORCE algorithm with PyTorch. I noticed that the batch inference/predictions of my simple network with Softmax doesn’t sum to 1 (not even close to 1). I am attaching a minimum working code so that you can reproduce it. What am I missing here?
import numpy as np
import torch

obs_size = 9
HIDDEN_SIZE = 9
n_actions = 2

np.random.seed(0)

model = torch.nn.Sequential(
        torch.nn.Linear(obs_size, HIDDEN_SIZE),
        torch.nn.ReLU(),
        torch.nn.Linear(HIDDEN_SIZE, n_actions),
        torch.nn.Softmax(dim=0)
    )

state_transitions = np.random.rand(3, obs_size)

state_batch = torch.Tensor(state_transitions)
pred_batch = model(state_batch)  # WRONG PREDICTIONS!
print('wrong predictions:\n', *pred_batch.detach().numpy())
# [0.34072137 0.34721774] [0.30972624 0.30191955] [0.3495524 0.3508627]
# DOES NOT SUM TO 1 !!!

pred_batch = [model(s).detach().numpy() for s in state_batch]  # CORRECT PREDICTIONS
print('correct predictions:\n', *pred_batch)
# [0.5955179  0.40448207] [0.6574412  0.34255883] [0.624833   0.37516695]
# DOES SUM TO 1 AS EXPECTED


title: calculating accuracy (in lstm model)

body:
I am trying to get my accuracy, and I have this code:
            num_correct = 0.0
            for inputs, labels in dataloader(
            valid_features, valid_labels, batch_size=batch_size, sequence_length=20):

                top_val, top_class = torch.exp(output).topk(1)
                num_correct += torch.sum(top_class.squeeze() == labels)
#...
            print(#...,
                  "Accuracy: {:.3f}".format(num_correct*1.0 / len(valid_labels) *1.0)

It always prints 0.000, so I decided to print the raw values going into num_correct:
print(top_class.squeeze(), labels)
tensor([ 1,  3,  3,  ...,  3,  4,  3], device='cuda:0') tensor([ 1,  1,  3,  ...,  3,  3,  3], device='cuda:0')
tensor([ 4,  3,  1,  ...,  4,  4,  3], device='cuda:0') tensor([ 4,  3,  1,  ...,  4,  4,  3], device='cuda:0')
tensor([ 2,  4,  2,  ...,  4,  4,  4], device='cuda:0') tensor([ 3,  4,  1,  ...,  4,  4,  4], device='cuda:0')
tensor([ 0,  1,  3,  ...,  2,  3,  0], device='cuda:0') tensor([ 0,  1,  3,  ...,  2,  2,  0], device='cuda:0')

these appear pretty highly accurate. So .. I could extract this out to numpy and be done with it, but there is a pytorch way.


title: RuntimeError: Given groups=1, weight of size [32, 1, 5, 5], expected input[256, 3, 256, 256] to have 1 channels, but got 3 channels instead

body:
I am trying to run following code but getting an error:
import torch.nn as nn
import torch.nn.functional as F

class EmbeddingNet(nn.Module):
    def __init__(self):
        super(EmbeddingNet, self).__init__()
        self.convnet = nn.Sequential(nn.Conv2d(1, 32, 5), nn.PReLU(),
                                     nn.MaxPool2d(2, stride=2),
                                     nn.Conv2d(32, 64, 5), nn.PReLU(),
                                     nn.MaxPool2d(2, stride=2))

        self.fc = nn.Sequential(nn.Linear(64 * 4 * 4, 256),
                                nn.PReLU(),
                                nn.Linear(256, 256),
                                nn.PReLU(),
                                nn.Linear(256, 2)
                                )

    def forward(self, x):
        output = self.convnet(x)
        output = output.view(output.size()[0], -1)
        output = self.fc(output)
        return output

    def get_embedding(self, x):
        return self.forward(x)

class EmbeddingNetL2(EmbeddingNet):
    def __init__(self):
        super(EmbeddingNetL2, self).__init__()

    def forward(self, x):
        output = super(EmbeddingNetL2, self).forward(x)
        output /= output.pow(2).sum(1, keepdim=True).sqrt()
        return output

    def get_embedding(self, x):
        return self.forward(x)'''enter code here


title: OpenCV conflicts with Pytorch crop transformation

body:
I'm building CV app using OpenCV 4.5.1 and torchvision 0.9.1. When I apply this transform
test_transform = transforms.Compose([
  landmarks.CenterCrop((512, 512)),
  landmarks.ToTensor()
])

to the image and try to
img = img.permute(1, 2, 0).byte().numpy()
cv.circle(img=img, center=tuple(keypoints[i]), radius=2, color=(127, 255, 255), thickness=-1)

I receive an error from cv.circle()
TypeError: Expected Ptr<cv::UMat> for argument 'img'

Without CenterCrop everything works just fine
test_transform = transforms.Compose([
  landmarks.ToTensor()
])

Transforms applied look like:
ToTensor:
class ToTensor:
"""Convert ndarrays in sample to Tensors."""

def __call__(self, sample):
    image, landmarks = sample['image'], sample['landmarks']
    image = image.transpose((2, 0, 1))
    return {'image': torch.from_numpy(image).float(),
            'landmarks': torch.from_numpy(landmarks).view(-1).float()}

CenterCrop:
class CenterCrop:
  def __init__(self, output_size):
    assert isinstance(output_size, (int, tuple))
    if isinstance(output_size, int):
        self.output_size = (output_size, output_size)
    else:
        assert len(output_size) == 2
        self.output_size = output_size

  def __call__(self, sample):
    image, landmarks = sample['image'], sample['landmarks']

    h, w = image.shape[:2]
    new_h, new_w = self.output_size

    top = (h - new_h) // 2
    left = (w - new_w) // 2

    image = image[top: top + new_h,
                  left: left + new_w]

    landmarks = landmarks - [left, top]

    return {'image': image, 'landmarks': landmarks}

My question:
How to explain this behavior of cv.circle(), given that everything except image size (reduced when crop applies) matches in images with and w/o applying CenterCrop transform? Thank you.


title: How can I efficiently solve a quaduatic equation system via pytorch?

body:
I need to solve a non-linear equation system shown below:
def trySolveEquation(V, L):
    #The equation to solve is:
    #{     (V . Ct) ^ 2 = 1
    #{     (L + u) . Ct = 0
    #C and u are the unknowns, C is a vector and u is a scalar, Ct is a vector transposed from C.
    #V is a vector with dimension equal to Ct, L is a square matrix, they are known.
    #u is Lagrange multiplier, and it's unknown.
    #'.' means matrix multiply.

    dim = V.shape[0]
    assert L.shape[0] == dim and L.shape[1] == dim
    C = torch.zeros((dim))
    Ct = C.view((dim, 1))
    u = 0

    '*Solve the equation here.*'

    print('C=', C)
    print('u=', u)
    return C, u

The dimention of C is about ten, and this equation system would be solved up to a billion of times, so it's nice to be implemented via torch so GPU could be utilized. Is there any methods better than gradient descent?


title: Use PyTorch to speed up linear least squares optimization with bounds?

body:
I'm using scipy.optimize.lsq_linear to run some linear least squares optimizations and all is well, but a little slow.  My A matrix is typically about 100 x 10,000 in size and sparse (sparsity usually ~50%).  The bounds on the solution are critical.  Given my tolerance lsq_linear typically solves the problems in about 10 seconds and speeding this up would be very helpful for running many optimizations.
I've read about speeding up linear algebra operations using GPU acceleration in PyTorch.  It looks like PyTorch handles sparse arrays (torch calls them tensors), which is good.  However, I've been digging through the PyTorch documentation, particularly the torch.optim and torch.linalg packages, and I haven't found anything that appears to be able to do a linear least squares optimization with bounds.

Is there a torch method that can do linear least squares optimization with bounds like scipy.optimize.lsq_linear?
Is there another way to speed up lsq_linear or to perform the optimization in a faster way?

For what it's worth, I think I've pushed lsq_linear pretty far.  I don't think I can decrease the number of matrix elements, increase sparsity or decrease optimiation tolerances much farther without sacrificing the results.


title: Assign value to sub-tensor in Pytorch

body:
Suppose I have a PyTorch tensor t of shape (3, 3, 3, 3), and I want to set t[0, 1, x, y] to z for all x and y. I noticed all the following syntaxes work:

t[0, 1] = z
t[0, 1, :] = z
t[0, 1, :, :] = z
t[0, 1, ...] = z

Is there any difference in terms of how things are executed under the hood? Is any one of these methods preferred over the others?


title: Calculating the Euclidean distance between 2 points on a circle in D-dimensional space

body:
It's fairly straightforward to calculate a direct Euclidean distance between 2 points:
import torch
p1 = torch.tensor([1.0, 3.5])
p2 = torch.tensor([5.0, 9.2])

dis = torch.sum(torch.square(p1-p2))
dis
>>> tensor(48.4900)

However, how can I calculate the distance between 2 points on circle without going through the circle? That it, the distance on the circle's perimeter, in D-dimensional space.
Clearly, in 2D a circle is just a circle:
import numpy as np
import matplotlib.pyplot as plt

def circle_points(r, n):
    circles = []
    for r, n in zip(r, n):
        t = np.linspace(0, 2*np.pi, n, endpoint=False)
        x = r * np.cos(t)
        y = r * np.sin(t)
        circles.append(np.c_[x, y])
    return circles

r = [2]
n = [20]
circles = circle_points(r, n)

fig, ax = plt.subplots()
for circle in circles:
    ax.scatter(circle[:, 0], circle[:, 1])
ax.set_aspect('equal')
plt.show()

point_1 = circles[0][0]
point_2 = circles[0][11]
print('point_1: ', point_1, 'point_2: ', point_2)
>>> point_1:  [2. 0.] point_2:  [-1.90211303 -0.61803399]

While in 3D it will be a sphere, 4D hypersphere, etc.


title: Using Electronic Health Records to predict future diagnosis codes with Gated Recurrent Units - (Error: Sample larger than population or is negative)

body:
I am working on clinical EHR.
I am currently referring to this blog and github link here.
https://sparklerussell.com/post/using-electronic-health-records-to-predict-future-diagnosis-codes-with-gated-recurrent-units/
https://github.com/sparalic/Electronic-Health-Records-GRUs
I have generated the dataset and processed it as per the instructions in the notebooks present in the repository. I am facing an issue trying to train the model.
using : jupytor notebook (with google colab)
import torch
import torch.nn as nn
from torch.autograd import Variable
from torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence
import torch.nn.functional as F
import numpy as np
import itertools
import pickle
import sys, random

Load Data :
def load_data(sequences, labels):
            dataSize = len(labels)
            idx = np.random.permutation(dataSize)
            nTest = int(np.ceil(0.15 * dataSize))
            nValid = int(np.ceil(0.10 * dataSize))
            test_idx = idx[:nTest]
            valid_idx = idx[nTest:nTest+nValid]
            train_idx = idx[nTest+nValid:]

            train_x = sequences[train_idx]
            train_y = labels[train_idx]
            test_x = sequences[test_idx]
            test_y = labels[test_idx]
            valid_x = sequences[valid_idx]
            valid_y = labels[valid_idx]

            train_x = [sorted(seq) for seq in train_x]
            train_y = [sorted(seq) for seq in train_y]
            valid_x = [sorted(seq) for seq in valid_x]
            valid_y = [sorted(seq) for seq in valid_y]
            test_x = [sorted(seq) for seq in test_x]
            test_y = [sorted(seq) for seq in test_y]

            train = (train_x, train_y)
            test = (test_x, test_y)
            valid = (valid_x, valid_y)
            return (train, test, valid)

Padding the input :
def padding(seqs, labels, vocab, n_classes):
    lengths = np.array([len(seq) for seq in seqs]) - 1 # remove the last list in each patient's sequences for labels
    n_samples = len(lengths)
    maxlen = np.max(lengths)

    x = torch.zeros(maxlen, n_samples, vocab) # maxlen = number of visits, n_samples = samples
    y = torch.zeros(maxlen, n_samples, n_classes)
    mask = torch.zeros(maxlen, n_samples)
    for idx, (seq,label) in enumerate(zip(seqs,labels)):
        for xvec, subseq in zip(x[:,idx,:], seq[:-1]):
            xvec[subseq] = 1.
        for yvec, subseq in zip(y[:,idx,:], label[1:]):
            yvec[subseq] = 1.
        mask[:lengths[idx], idx] = 1.

    return x, y, lengths, mask

GRU Class :
torch.manual_seed(1)
class EHRNN(nn.Module):
    def __init__(self, inputDimSize, hiddenDimSize,embSize, batchSize, numClass):
        super(EHRNN, self).__init__()

    self.hiddenDimSize = hiddenDimSize
    self.inputDimSize = inputDimSize
    self.embSize = embSize
    self.numClass = numClass
    self.batchSize = batchSize

    #Initialize random weights
    self.W_z = nn.Parameter(torch.randn(self.embSize, self.hiddenDimSize).cuda())
    self.W_r = nn.Parameter(torch.randn(self.embSize, self.hiddenDimSize).cuda())
    self.W_h = nn.Parameter(torch.randn(self.embSize, self.hiddenDimSize).cuda())

    self.U_z = nn.Parameter(torch.randn(self.hiddenDimSize, self.hiddenDimSize).cuda())
    self.U_r = nn.Parameter(torch.randn(self.hiddenDimSize, self.hiddenDimSize).cuda())
    self.U_h = nn.Parameter(torch.randn(self.hiddenDimSize, self.hiddenDimSize).cuda())

    self.b_z = nn.Parameter(torch.zeros(self.hiddenDimSize).cuda())
    self.b_r = nn.Parameter(torch.zeros(self.hiddenDimSize).cuda())
    self.b_h = nn.Parameter(torch.zeros(self.hiddenDimSize).cuda())

    self.params = [self.W_z, self.W_r, self.W_h,
                   self.U_z, self.U_r, self.U_h,
                   self.b_z, self.b_r, self.b_h]

def forward(self,emb,h):
    z = torch.sigmoid(torch.matmul(emb, self.W_z)  + torch.matmul(h, self.U_z) + self.b_z)
    r = torch.sigmoid(torch.matmul(emb, self.W_r)  + torch.matmul(h, self.U_r) + self.b_r)
    h_tilde = torch.tanh(torch.matmul(emb, self.W_h)  + torch.matmul(r * h, self.U_h) + self.b_h)
    h = z * h + ((1. - z) * h_tilde)
    return h

def init_hidden(self):
    return Variable(torch.zeros(self.batchSize,self.hiddenDimSize))

Custom Layer for handling two layer GRU :
torch.manual_seed(1)
class build_EHRNN(nn.Module):
    def __init__(self, inputDimSize=4894, hiddenDimSize=[200,200], batchSize=100, embSize=200,numClass=4894, dropout=0.5,logEps=1e-8):
        super(build_EHRNN, self).__init__()

    self.inputDimSize = inputDimSize
    self.hiddenDimSize = hiddenDimSize
    self.numClass = numClass
    self.embSize = embSize
    self.batchSize = batchSize
    self.dropout = nn.Dropout(p=0.5)
    self.logEps = logEps

    # Embedding inputs
    self.W_emb = nn.Parameter(torch.randn(self.inputDimSize, self.embSize).cuda())
    self.b_emb = nn.Parameter(torch.zeros(self.embSize).cuda())

    self.W_out = nn.Parameter(torch.randn(self.hiddenDimSize, self.numClass).cuda())
    self.b_out = nn.Parameter(torch.zeros(self.numClass).cuda())

    self.params = [self.W_emb, self.W_out,
                   self.b_emb, self.b_out]

def forward(self,x, y, lengths, mask):
    self.emb = torch.tanh(torch.matmul(x, self.W_emb) + self.b_emb)
    input_values = self.emb
    self.outputs = [input_values]
    for i, hiddenSize in enumerate([self.hiddenDimSize, self.hiddenDimSize]):  # iterate over layers
        rnn = EHRNN(self.inputDimSize,hiddenSize,self.embSize,self.batchSize,self.numClass) # calculate hidden states
        hidden_state = []
        h = self.init_hidden().cuda()
        for i,seq in enumerate(input_values): # loop over sequences in each batch
            h = rnn(seq, h)
            hidden_state.append(h)
        hidden_state = self.dropout(torch.stack(hidden_state))    # apply dropout between layers
        input_values = hidden_state

    y_linear = torch.matmul(hidden_state, self.W_out)  + self.b_out # fully connected layer
    yhat = F.softmax(y_linear, dim=1)  # yhat
    yhat = yhat*mask[:,:,None]   # apply mask

    # Loss calculation
    cross_entropy = -(y * torch.log(yhat + self.logEps) + (1. - y) * torch.log(1. - yhat + self.logEps))
    last_step = -torch.mean(y[-1] * torch.log(yhat[-1] + self.logEps) + (1. - y[-1]) * torch.log(1. - yhat[-1] + self.logEps))
    prediction_loss = torch.sum(torch.sum(cross_entropy, dim=0),dim=1)/ torch.cuda.FloatTensor(lengths)
    cost = torch.mean(prediction_loss) + 0.000001 * (self.W_out ** 2).sum() # regularize
    return (yhat, hidden_state, cost)

def init_hidden(self):
    return torch.zeros(self.batchSize, self.hiddenDimSize)  # initial state

Load data :
train, test, valid = load_data(sequences, labels)

Batch Size:
batchSize = 100
n_batches = int(np.ceil(float(len(train[0])) / float(batchSize)))-1
n_batches_valid = int(np.ceil(float(len(valid[0])) / float(batchSize)))-1

Model:
model = build_EHRNN(inputDimSize=4894, hiddenDimSize=200, batchSize=20, embSize=200, numClass=4894, dropout=0.5, logEps=1e-8)
model = model.to(device)

optimizer = torch.optim.Adadelta(model.parameters(), lr = 0.01, rho=0.90)
max_epochs = 5

loss_all = []
iteration = 0

for e in range(max_epochs):
for index in random.sample(range(n_batches), n_batches):
    batchX = train[0][:n_batches*batchSize][index*batchSize:(index+1)*batchSize]
    batchY = train[1][:n_batches*batchSize][index*batchSize:(index+1)*batchSize]

    optimizer.zero_grad()

    x, y, lengths, mask = padding(batchX, batchY, 4894, 4894)

    if torch.cuda.is_available():
        x, y, lenghts, mask = x.cuda(), y.cuda(), lengths, mask.cuda()

    outputs, hidden, cost = model(x,y, h, lengths, mask)

    if torch.cuda.is_available():
        cost.cuda()
    cost.backward()
    nn.utils.clip_grad_norm_(model.parameters(), 5)
    optimizer.step()

    loss_all.append(cost.item())
    iteration +=1
    if iteration % 10 == 0:
        # Calculate Accuracy
        losses = []
        model.eval()
        val_loss = []
        for index in random.sample(range(n_batches_valid), n_batches_valid):
            validX = valid[0][:n_batches_valid*batchSize][index*batchSize:(index+1)*batchSize]
            validY = valid[1][:n_batches_valid*batchSize][index*batchSize:(index+1)*batchSize]
            x, y, lengths, mask = padding(validX, validY, 4894, 4894)
            if torch.cuda.is_available():
                x, y, lenghts, mask = x.cuda(), y.cuda(), lenghts, mask.cuda()
            outputs, hidden_val, cost_val = model(x,y, h, lengths, mask)
            losses.append(cost_val)

        model.train()
        print("Epoch: {}/{}...".format(e+1, max_epochs),
                      "Step: {}...".format(iteration),
                      "Training Loss: {:.4f}...".format(np.mean(loss_all)),
                      "Val Loss: {:.4f}".format(torch.mean(torch.tensor(losses))))

ERROR :
    ValueError                                Traceback (most recent call last)
<ipython-input-76-9ca4916456a9> in <module>()
      8
      9 for e in range(max_epochs):
---> 10     for index in random.sample(range(n_batches), n_batches):
     11         batchX = train[0][:n_batches*batchSize][index*batchSize:(index+1)*batchSize]
     12         batchY = train[1][:n_batches*batchSize][index*batchSize:(index+1)*batchSize]

/usr/lib/python3.7/random.py in sample(self, population, k)
    319         n = len(population)
    320         if not 0 <= k <= n:
--> 321             raise ValueError("Sample larger than population or is negative")
    322         result = [None] * k
    323         setsize = 21        # size of a small set minus size of an empty list

ValueError: Sample larger than population or is negative

I tried many things but I couldn't solve the problem.


I want to train a neural network and I want to check it's total loss in every 100 epoch but I get error RuntimeError: CUDA out of memory. Tried to allocate ... in line #1, sometimes I get this error but this time because Ive tried big numbers of neuron in hidden layers I think this time this error maybe real, eventhough I tried
torch.cuda.empty_cache()
import gc
gc.collect()

which are codes recommended to fix the error I just mentioned so my question is not how to solve the error as I said this time it maybe correct (not really even trying to know is this the 3GB gpu capacity is full or not), and I just want to know is there any other way to calculate all train data loss? even with using loss of batches.
I think another way of asking this question is how to calculate total loss using losses of batches?
btw any other suggestions about my hidden layer sizes is also appreciable because I can't really even overfit my 400000 samples train dataset.
input_size = 83
skipcolumns= 3
hidden_size = 640
output_size = 12
num_epochs = 10000
batch_size = 20000
learning_rate = .001
n_total_steps=int(len(trainx)/batch_size)+1
def seperator(i,divide,p):
    return int((p[1]-p[0])/divide*i+p[0]),int((p[1]-p[0])/divide*(i+1)+p[0])
class NeuralNet(nn.Module):
    def __init__(self, input_size, hidden_size, output_size):
        super(NeuralNet, self).__init__()
        self.input_size = input_size
        self.lin = nn.Linear(input_size, hidden_size)
        self.relu = torch.nn.ReLU()
        self.leakyrelu =nn.LeakyReLU(negative_slope=0.01, inplace=False)
        self.l2 = nn.Linear(hidden_size, hidden_size)
        self.l3 = nn.Linear(hidden_size, hidden_size)
        self.l4 = nn.Linear(hidden_size, hidden_size)
        self.l5 = nn.Linear(hidden_size, hidden_size)
        self.l6 = nn.Linear(hidden_size, hidden_size)
        self.lout = nn.Linear(hidden_size, output_size)

    def forward(self, x):
        # out = self.relu(out)
        # out = torch.sigmoid(out)
        out = self.lin(x)
        out = self.leakyrelu(out)
        out = self.l2(out)
        out = self.leakyrelu(out)
        out = self.l3(out)
        out = self.leakyrelu(out)
        # out = self.l4(out)
        # out = self.leakyrelu(out)
        # out = self.l5(out)
        # out = self.leakyrelu(out)
        # out = self.l6(out)
        # out = self.leakyrelu(out)
        out = self.lout(out)

        # no activation and no softmax at the end
        return out

# Loss and optimizer
criterion = nn.MSELoss()

model = NeuralNet(input_size, hidden_size, output_size).to(device)
optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)
tti=time.time()
for epoch in range(num_epochs):
    for btch in range(n_total_steps):
        stind,endind=seperator(btch,n_total_steps,[0,len(trainx)])
        thisx = trainx[stind:endind].to(device)
        thisy = trainy[stind:endind].to(device)
        outputs = model(thisx)
        loss = criterion(outputs , thisy)
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()
        # q(time.time()-tti)
    if keyboard.is_pressed('q'):
        x=1/0
    if epoch%10==0:
        print(time.time()-tti)
        if epoch%100==0:
            thisx = trainx.to(device)
            thisy = trainy.to(device)
            outputs = model(thisx) #1
            loss = criterion(outputs , thisy)
            print(loss.item())


title: How to sample efficiently from a list like [[0,1],[0,6],[0,7],[1,4],[1,8]], ideally in pytorch (so this will be a tensor)

body:
I have a tensor that looks like

[[0,1],
 [0,6],
 [0,7],
 [1,4],
 [1,8],
 [2,5],
 [2,7]]

so the first column is always a non-decreasing sequence from 0 to N (N is fixed), and the second column is some integer (within a fixed range). For the sampling, I want to obtain one sample from the second column for each distinct number in the first column. In this example, I want to get one sample from [1,6,7] for 0,  one sample from [4,8] for 1, and one sample from [5,7] for 2, and valid examples of samples can be [1,8,7], [7,8,7], etc.


title: With pytorch module,will computation graph be overrode when forward() was invoke multiple times before backward()?

body:
I found a line of code:

out_a, out_p, out_n = model(data_a), model(data_p), model(data_n)

in:https://github.com/liorshk/facenet_pytorch/blob/master/train_triplet.py

as you can see that the "forward()" is invoked multiple times before "backward()", question are:

1, Will next invoking override previous computation graph?

2, In my test, the GPU consumption will increase accordingly, so even overriding happened,GPUu memory leaking happens, how to solve this problem?


title: Code Optimization: Computation in Torch.Tensor

body:
I am currently implementing a function to compute Custom Cross Entropy Loss.
The definition of the function is a following image.

my codes are as following,

output = output.permute(0, 2, 3, 1)
target = target.permute(0, 2, 3, 1)

batch, height, width, channel = output.size()

total_loss = 0.
for b in range(batch): # for each batch
    o = output[b]
    t = target[b]
    loss = 0.
    for w in range(width):
        for h in range(height): # for every pixel([h,w]) in the image
            sid_t = t[h][w][0]
            sid_o_candi = o[h][w]
            part1 = 0. # to store the first sigma
            part2 = 0. # to store the second sigma

            for k in range(0, sid_t):
                p = torch.sum(sid_o_candi[k:]) # to get Pk(w,h)
                part1 += torch.log(p + 1e-12).item()

            for k in range(sid_t, intervals):
                p = torch.sum(sid_o_candi[k:]) # to get Pk(w,h)
                part2 += torch.log(1-p + 1e-12).item()

            loss += part1 + part2

    loss /= width * height * (-1)
    total_loss += loss
total_loss /= batch
return torch.tensor(total_loss, dtype=torch.float32)

I am wondering is there any optimization could be done with these code.


title: In the U-Net architecture, h and w don't match, I don't know whether I misunderstood it or not

body:
I'm confused about the image-transformation structure in a paper that I read. The output dimensions of the layers they concatenate do not match.

Should it actually be layer 9 and layer 7?

class ImageTransformationNetwork(nn.Module):
def __init__(self):
    super(ImageTransformationNetwork, self).__init__()
    self.layer1 = nn.Sequential(nn.Conv2d(3, 64, padding=1, kernel_size=4, stride=2),
                                nn.LeakyReLU())
    self.layer2 = nn.Sequential(nn.Conv2d(64, 128, padding=1, kernel_size=4, stride=2),
                                nn.LeakyReLU(),
                                nn.BatchNorm2d(128))
    self.layer3 = nn.Sequential(nn.Conv2d(128, 256, padding=1, kernel_size=4, stride=2),
                                nn.LeakyReLU(),
                                nn.BatchNorm2d(256))
    self.layer4 = nn.Sequential(nn.Conv2d(256, 512, padding=1, kernel_size=4, stride=2),
                                nn.LeakyReLU(),
                                nn.BatchNorm2d(512))
    self.layer5 = nn.Sequential(nn.Conv2d(512, 512, padding=1, kernel_size=4, stride=2),
                                nn.LeakyReLU(),
                                nn.BatchNorm2d(512))
    self.layer6 = nn.Sequential(nn.Conv2d(512, 512, padding=1, kernel_size=4, stride=2),
                                nn.LeakyReLU(),
                                nn.BatchNorm2d(512))
    self.layer7 = nn.Sequential(nn.Conv2d(512, 512, padding=1, kernel_size=4, stride=2),
                                nn.LeakyReLU(),
                                nn.BatchNorm2d(512))
    self.layer8 = nn.Sequential(nn.Conv2d(512, 512, padding=1, kernel_size=4, stride=2),
                                nn.LeakyReLU())
    self.layer9 = nn.Sequential(nn.ConvTranspose2d(512, 1024, padding=1, kernel_size=4, stride=2),
                                nn.ReLU(),
                                nn.BatchNorm2d(1024))
    self.layer10 = nn.Sequential(nn.ConvTranspose2d(1024+512, 1024, padding=1, kernel_size=4, stride=2),
                                nn.ReLU(),
                                nn.BatchNorm2d(1024))
    self.layer11 = nn.Sequential(nn.ConvTranspose2d(1024+512, 1024, padding=1, kernel_size=4, stride=2),
                                 nn.ReLU(),
                                 nn.BatchNorm2d(1024))
    self.layer12 = nn.Sequential(nn.ConvTranspose2d(1024+512, 1024, padding=1, kernel_size=4, stride=2),
                                 nn.ReLU(),
                                 nn.BatchNorm2d(1024))
    self.layer13 = nn.Sequential(nn.ConvTranspose2d(1024+512, 512, padding=1, kernel_size=4, stride=2),
                                 nn.ReLU(),
                                 nn.BatchNorm2d(512))
    self.layer14 = nn.Sequential(nn.ConvTranspose2d(512+256, 256, padding=1, kernel_size=4, stride=2),
                                 nn.ReLU(),
                                 nn.BatchNorm2d(256))
    self.layer15 = nn.Sequential(nn.ConvTranspose2d(256+128, 128, padding=1, kernel_size=4, stride=2),
                                 nn.Tanh(),
                                 nn.BatchNorm2d(128))
    self.layer16 = nn.Sequential(nn.ConvTranspose2d(128, 3, padding=1, kernel_size=4, stride=2))

def forward(self, x):
    x1 = self.layer1(x)
    x2 = self.layer2(x1)
    x3 = self.layer3(x2)
    x4 = self.layer4(x3)
    x5 = self.layer5(x4)
    x6 = self.layer6(x5)
    x7 = self.layer7(x6)
    x8 = self.layer8(x7)
    x9 = self.layer9(x8)
    x10 = self.layer10(torch.cat([x7, x9], dim=1))
    x11 = self.layer11(torch.cat([x6, x10], dim=1))
    x12 = self.layer12(torch.cat([x5, x11], dim=1))
    x13 = self.layer13(torch.cat([x4, x12], dim=1))
    x14 = self.layer14(torch.cat([x3, x13], dim=1))
    x15 = self.layer15(torch.cat([x2, x14], dim=1))
    x16 = self.layer16(x15)
    return x16

I don't know if I understand it correctly. Could you provide some help about it?


title: Visualizing feature maps

body:
Does anyone know why my code is visualizing the feature maps in grayscale when I’m using it during validation forward-pass?

The output its like below:

And I'm expecting something like below (I mean not grayscale) but don't know-how:

This is my code

def vis_tensor(data):
    data  =data.data.cpu().numpy()
    data =data[0].transpose((1, 2, 0))
    data = data - data.min()
    data = data / data.max()
    data = data * 255
    mean = np.repeat(data.mean(-1, keepdims=True),3 ,2)
    mean = mean.astype('uint8')
    pylab.imshow(mean)
    pylab.savefig('mean.png')
    max = np.repeat(data.max(-1, keepdims=True), 3, 2)
    max = max.astype('uint8')
    pylab.imshow(max, cmap='plasma')
    pylab.savefig('max.png')


title: Why does renewing an optimizer give a bad result?

body:
I tried to change my optimizer, but first of all, I want to check whether the following two codes give the same results:

optimizer = optim.Adam(params, lr)
for epoch in range(500):
    ....
    optimizer.zero_grad()
    loss.backward()
    optimizer.step()
for epoch in range(500):
    ....
    optimizer.zero_grad()
    loss.backward()
    optimizer.step()

If I insert the same optimizer between 'for loops',

optimizer = optim.Adam(params, lr)
for epoch in range(500):
    ....
    optimizer.zero_grad()
    loss.backward()
    optimizer.step()

optimizer = optim.Adam(params, lr)
for epoch in range(500):
    ....
    optimizer.zero_grad()
    loss.backward()
    optimizer.step()

The result become bad. Why does this happens? Doesn't optimizer just receive gradients from loss and operate gradiet descent like steps?


title: Does changing image channel break the image?

body:
so I have images with the format(width,height,channel). My original channel is in rgb. So I Load the images in grayscale

for r, d, file in tqdm(os.walk(path)):
  for i in tqdm(file):
    if i[0:2]=="01":
      dist_one.append(cv2.imread(os.path.join(path,i),cv2.IMREAD_GRAYSCALE))
    else:
      dist_two.append(cv2.imread(os.path.join(path,i),cv2.IMREAD_GRAYSCALE))

Suppose the images have the shape (187, 187). So I add a channel using the code

g = np.expand_dims(dist_one[0], axis=0)

But this breaks the images when I try to plot the image.

  TypeError                                 Traceback (most recent call
  last)  in ()       1 import
  matplotlib.pyplot as plt       2  ----> 3
  plt.imshow(dist_one[0],cmap='gray')       4 plt.show()  5
  frames/usr/local/lib/python3.6/dist-packages/matplotlib/image.py in
  set_data(self, A)     688                 or self._A.ndim == 3 and
  self._A.shape[-1] in [3, 4]):     689             raise
  TypeError("Invalid shape {} for image data" --> 690
  .format(self._A.shape))     691      692 if self._A.ndim == 3:
  TypeError: Invalid shape (1, 187, 187) for image data

But it works when the channel is put last.
g = np.expand_dims(dist_one[0], axis=-1).
Whats the reason for this?
I need the channel at the first for pytorch. Or am I suppose to train the model with broken images?


title: slicing each raw of a tensor

body:
I have a tensor of size (1124823 x 13). I want to go from the center of each raw of the tensor take five elements from the left and five from the right. I do such this simple code. It gives me

TypeError: iteration over a 0-d tensor

i = 0
j = 6
base = feat.size(0)//2
for i in feat[i, j]:
    x = feat[base - 5: base:+5]
    i += 1


title: Appending missing values according to index

body:
Say if i have a tensor

values = torch.tensor([5, 3, 2, 8])

and a corresponding index to values

index = torch.tensor([0, 2, 4, 5])

and assuming that I want to insert in the missing index (1 and 3), with a fixed value (100), such that I get

values = torch.tensor([5, 100, 3, 100, 2, 8])

Is there a vectorized way to do it in PyTorch (or numpy)?


title: Can't create a CNN with PyTorch

body:
I haven't used PyTorch before and I wanted to create a CNN, which is an adaptation of a VGG19 that I created in TensorFlow 2.0 with Keras.

This is the code of my CNN in PyTorch.

class VGG19(nn.Module):

  def __init__(self, num_clases=43):
    super(VGG19, self).__init__()

    self.layer1 = nn.Sequential(
      nn.Conv2d(32, 64, kernel_size=3, padding=2, stride=1),
      nn.ReLU(),
      nn.BatchNorm2d(64),
      # Repetimos lo mismo
      nn.Conv2d(64, 64, kernel_size=3, padding=2, stride=1),
      nn.ReLU(),
      nn.BatchNorm2d(64),
      nn.MaxPool2d(kernel_size=2, stride=2))

    self.layer2 = nn.Sequential(
      nn.Conv2d(64, 128, kernel_size=3, padding=2, stride=1),
      nn.ReLU(),
      nn.BatchNorm2d(128),

      nn.Conv2d(128, 128, kernel_size=3, padding=2, stride=1),
      nn.ReLU(),
      nn.BatchNorm2d(128))

    self.layer3 = nn.Sequential(
      nn.Conv2d(128, 256, kernel_size=3, padding=2, stride=1),
      nn.ReLU(),
      nn.BatchNorm2d(256),
      nn.Conv2d(256, 256, kernel_size=3, padding=2, stride=1),
      nn.ReLU(),
      nn.BatchNorm2d(256),
      nn.Conv2d(256, 256, kernel_size=3, padding=2, stride=1),
      nn.ReLU(),
      nn.BatchNorm2d(256),
      nn.Conv2d(256, 256, kernel_size=3, padding=2, stride=1),
      nn.ReLU(),
      nn.BatchNorm2d(256),
      nn.MaxPool2d(kernel_size=2, stride=2))

    self.layer4 = nn.Sequential(
      nn.Conv2d(256, 512, kernel_size=3, padding=2, stride=1),
      nn.ReLU(),
      nn.BatchNorm2d(512),
      nn.Conv2d(512, 512, kernel_size=3, padding=2, stride=1),
      nn.ReLU(),
      nn.BatchNorm2d(512),
      nn.Conv2d(512, 512, kernel_size=3, padding=2, stride=1),
      nn.ReLU(),
      nn.BatchNorm2d(512),
      nn.Conv2d(512, 512, kernel_size=3, padding=2, stride=1),
      nn.ReLU(),
      nn.BatchNorm2d(512))

    self.layer5 = nn.Sequential(
      nn.Conv2d(512, 512, kernel_size=3, padding=2, stride=1),
      nn.ReLU(),
      nn.BatchNorm2d(512),
      nn.Conv2d(512, 512, kernel_size=3, padding=2, stride=1),
      nn.ReLU(),
      nn.BatchNorm2d(512),
      nn.Conv2d(512, 512, kernel_size=3, padding=2, stride=1),
      nn.ReLU(),
      nn.BatchNorm2d(512),
      nn.Conv2d(512, 512, kernel_size=3, padding=2, stride=1),
      nn.ReLU(),
      nn.BatchNorm2d(512),
      nn.MaxPool2d(kernel_size=2, stride=2))

    self.fc1 = nn.Sequential(
      nn.Linear(512 * 15 * 11, 4096),
      nn.ReLU(),
      nn.BatchNorm2d(4096),
      nn.Dropout(0.5))

    self.fc2 = nn.Sequential(
      nn.Linear(4096, 4096),
      nn.ReLU(),
      nn.BatchNorm2d(4096),
      nn.Dropout(0.5))

    self.final = nn.Sequential(
      nn.Linear(4096, num_clases),
      nn.Softmax(num_clases))

  def forward(self, x) :
    out = self.layer1(x)
    out = self.layer2(out)
    out = self.layer3(out)
    out = self.layer4(out)
    out = self.layer5(out)
    print(out.shape)
    out = out.view(out.size(0), -1)
    out = self.fc1(out)
    out = self.fc2(out)
    out = self.final(out)
    return out

When I try to train it I get this error.

total_step = len(train_loader)
for epoch in range(num_epochs):
    for i, (images, labels) in enumerate(train_loader):
        images = images.to(device)
        labels = labels.to(device)

        # Forward pass
        outputs = vgg19(images)
        loss = criterion(outputs, labels)

        # Backward and optimize
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

        if (i+1) % 100 == 0:
            print ('Epoch [{}/{}], Step [{}/{}], Loss: {:.4f}'
                   .format(epoch+1, num_epochs, i+1, total_step, loss.item()))

ValueError                                Traceback (most recent call last)
<ipython-input-39-7cc3d2f9bfc9> in <module>()
      6
      7         # Forward pass
----> 8         outputs = vgg19(images)
      9         loss = criterion(outputs, labels)
     10

6 frames
/usr/local/lib/python3.6/dist-packages/torch/nn/modules/batchnorm.py in _check_input_dim(self, input)
    249         if input.dim() != 4:
    250             raise ValueError('expected 4D input (got {}D input)'
--> 251                              .format(input.dim()))
    252
    253

ValueError: expected 4D input (got 2D input)

With Keras it's really easy to connect a convolutional layer with a fully-connected with model.add(Flatten()) y model.add(Dense(4096)). Here I try to do the same but it doesn't work.

Anybody can help me, please?


title: share cuda memory between pyspark worker processes on the same worker node

body:
When using pyspark for distributed compute, is it possible to share GPU memory across the worker python processes? In particular I have a pytorch model I'd like to run distributed inference on, and the naive way of running this would be that each pyspark worker process would load the pytorch model into cuda memory. However this is very inefficient and drastically reduces the size of the pytorch model that could fit.

With better GPU support in Spark 3.0, is there a way to do this?


title: How is Vocab and Integer (one hot) representation stored and what does the ('string', int) tuple means in torchtext.vocab()?

body:
I am trying to train an RNN for binary classification. I have my vocab made from 1000000 words and please find the below outputs...

text_field = torchtext.data.Field(tokenize=word_tokenize)

print(text_field.vocab.freqs.most_common(15))
>>
[('.', 516822), (',', 490533), ('the', 464796), ('to', 298670), ("''", 264416), ('of', 226307), ('I', 224927), ('and', 215722), ('a', 211773), ('is', 180965), ('you', 180359), ('``', 165889), ('that', 156425), ('in', 138038), (':', 132294)]

print(text_field.vocab.itos[:15])
>>
['<unk>', '<pad>', '.', ',', 'the', 'to', "''", 'of', 'I', 'and', 'a', 'is', 'you', '``', 'that']

text_field.vocab.stoi
>>
{'<unk>': 0,'<pad>': 1,'.': 2,',': 3,'the': 4,'to': 5,"''": 6,'of': 7,'I': 8,'and': 9,'a': 10, 'is': 11,'you': 12,'``': 13,'that': 14,'in': 15,....................

The documentation says:

freqs – A collections.Counter object holding the frequencies of tokens in the data used to build the Vocab.
stoi – A collections.defaultdict instance mapping token strings to numerical identifiers.
itos – A list of token strings indexed by their numerical identifiers.

Which not comprehensible by me.

Can Some one Please explain what are these by giving the intuition of each of those?

For example, if the is represented by 4, then does it mean that if a sentence contains the word the,

Is it going to be a 1 at location 4?  OR
Is it going to be a 1 at position 464796 OR
Is it going to be a 4 at position 464796??

What happens when multiple the are there there??


title: broadcast multiple 2d arrays as individual elements of another 2d array

body:
lets say I have 10 arrays of size 10x10 each, called a,b,c...j

I create np.zeros of 10x2 called arr. I've already made arr[:,1]=1,2,3...10

so as of now my new array is

[0,1]
[0,2]
...
[0,10]

What i need now is to make

arr[0,0] = a, arr[1,0] = b ... arr[10,0]=j

Since Im also dealing tensors, the error I get is

  "only one element tensors can be converted to Python scalars"

Basically, to reiterate, I want to put those 2D arrays into my new array as individual elements of the new array.

Final output reqd:

[a,1]
[b,2]
...
[j,10]

EDIT: The final array will be multidimensional, but Im not sure of the value of the final shape, 10x10x2 maybe??

EDIT: Think of the problem as images and labels. I want to store an entire image and its corresponding label as each row of an array


title: Where is the numpy data stored?

body:
In python, If only import torch (but not import numpy), "torch.numpy()" can still work. Is that means the numpy data can be stored and displayed without numpy package? Where is the numpy data stored and how does it display (without numpy package)?

example codes:

import torch
a = torch.tensor([[1,2,3],[4,5,6]])
a = a.numpy()
print(a)
array([[1, 2, 3],
       [4, 5, 6]])


title: PyTorch Networks within a Model

body:
I would like to define a network that comprises many templates.  Below under Network Definitions is a simplified example where the first network definition is used as a template in the second one.  This doesn't work - when I initialise my optimiser is says that the network parameters are empty!
How should I do this properly?  The network that I ultimately want is very complicated.

Main Function

if __name__ == "__main__":

myNet       = Network().cuda().train()
optimizer   = optim.SGD(myNet.parameters(), lr=0.01, momentum=0.9)

Network definitions:

class NetworkTemplate(nn.Module):

def __init__(self):
    super(NetworkTemplate, self).__init__()
    self.conv1 = nn.Conv2d(1, 3, kernel_size=1, bias=False)
    self.bn1 = nn.BatchNorm2d(3)

def forward(self, x):
    x = self.conv1(x)
    x = self.bn1(x)

    return x

class Network(nn.Module):

def __init__(self, nNets):
    super(Network, self).__init__()

    self.nets = []
    for curNet in range(nNets):
        self.nets.append(NetworkTemplate())

    def forward(self, x):

        for curNet in self.nets:
            x = curNet(x)

        return x


title: Train on data without a loop in pytorch?

body:
Does pytorch provide some kind of model.train_on_entire_thing(x, y, iterations=1000) method? I can't find it. This is the shortest code I know:
import torch

x = random
y = random
model = sequential layers
loss_fn = mean squared error
optimizer = adam
loop 1000 times:
    predict x
    take loss
    zero gradients
    send backword
    step the optimizer

I want to do that loop in one line.


To the best of my knowledge, the existing quantization method is operating on 32-bit.
In order to quantize weight of CNN as well as reduce memory footprint and then port the quantized model into the mobile device, how to convert a 32-bit operation to a 4-bit or 8-bit operation on cpu?


title: Is there a way to disable images view on tensor board or remove them from event files?

body: I'm trying to design system with sensitive data which cannot be seen by people who develop machine learning based on this files. They working with TensorFlow and pyTorch. Is there any way to cut off this data from event files (would be great) or modify/configure tensor board to disable view this images?


why a[:,[x]] could create a column vector from an array? The [ ] represents what?
Could anyone explain to me the principle?
a = np.random.randn(5,6)
a = a.astype(np.float32)
print(a)
c = torch.from_numpy(a[:,[1]])

[[-1.6919796   0.3160475   0.7606999   0.16881375  1.325092    0.71536326]
 [ 1.217861    0.35804042  0.0285245   0.7097111  -2.1760604   0.992101  ]
 [-1.6351479   0.6607222   0.9375339   0.5308735  -1.9699149  -2.002803  ]
 [-1.1895325   1.1744579  -0.5980689  -0.8906375  -0.00494479  0.51751447]
 [-1.7642071   0.4681248   1.3938268  -0.7519176   0.5987852  -0.5138923 ]]
###########################################
tensor([[0.3160],
        [0.3580],
        [0.6607],
        [1.1745],
        [0.4681]])


title: How to optimize ‘no. of hidden layers’ and ‘no. of neurons’ using baysian optimization in pytorch?

body:
I am trying to find optimal hyperparameters for my deep learning model. I could able to optimize LR and BS. But I am strugging with optimization of number of layers and neurons within them. Could someone help me ?
Here is what I have written, what I expect is to modify this class to find optimal input_neurons and output_neurons within each layer.
class Net(nn.Module):
    def __init__(self, n_lay=2, neur=32, dropout=0.1):
        super().__init__()
        self.name = "DNN"
        self.init_args = {"n_lay": n_lay, "neur": neur}
        self.fc = nn.ModuleList([nn.Linear(1105, neur)]) # 1105 data features
        self.fc.extend([nn.Linear(neur, neur) for i in range(1, n_lay-1)])
        self.fc.append(nn.Linear(neur, 1))
        self.dropout = nn.Dropout(p=dropout)

    def forward(self, x):
        for i, l in enumerate(self.fc):
            x = self.dropout(self.fc[i](x))
        return x


title: I have two datasets but different labels alignments. How do I align labels?

body:
I have two datasets like A dataset:{feature, label_A} and B dataset: {data, labels_B} where label_A and label_B are not aligned.
I want to write a custom dataset from these two datasets which will pass to dataloader and give outputs like feature[index], data[index], align labels (A), and index.


title: how i make a grammarly error correction model with pretrained bert model?

body:
i'm using pretrained bert model() for GEC(grammarly error correction) task.
i implemented a decoder ( bert is encoder of transformer ) so my model's architecture is same as transformer model now. (i didn't update vocab.txt)
but it doesn't works well ( base line code is also a transformer model, and it works well )
Could you recommand any architecture or example for GEC task using pretrained bert??
Thank you


title: How can I run the Events created by Tensorboard

body:
I use tensorboard to create an events,
from torch.utils.tensorboard import SummaryWriter

if __name__ == '__main__':
    write=SummaryWriter("log")
    write.add_scalar("test",1,1)
    write.add_scalar("test",2,2)
    write.close()

but How can I run it? When I input tensorboard --logdir=='logs', it told me has no exe named tensorboard found.


title: Algorithm doesn’t run on GPU even after storing model and data into GPU. What am i missing?

body:
You can find training section of my code below:
device = torch.device(‘cuda’ if torch.cuda.is_available() else ‘cpu’)

Note: I am receiving True when i check torch.cuda.is_available
After creating CNN model i wrote:
model = model.to(device)

Training Section:
import time
start_time = time.time()

epochs = 3

#Limits on numbers of batches if you want train faster(Not mandatory)
max_trn_batch = 800 # batch 10 image --> 8000 images total
max_tst_batch = 300 # batch 10 image --> 3000 images total

train_losses = []
test_losses = []
train_correct = []
test_correct = []

for i in range(epochs):

trn_corr = 0
tst_corr = 0

for b,(X_train,y_train) in enumerate(train_loader):
    X_train,y_train = X_train.to(device),y_train.to(device)

    #optinal limit number of batches
    if b == max_trn_batch:
        break
    b = b + 1

    y_pred = model(X_train)
    loss = criterion(y_pred,y_train)

    predicted = torch.max(y_pred.data,1)[1]
    batch_corr = (predicted == y_train).sum()
    trn_corr = trn_corr + batch_corr

    optimizer.zero_grad()
    loss.backward()
    optimizer.step()

    if b%200 == 0:
        print('Epoch:  {} Loss:  {} Accuracy:  {}'.format(i,loss,trn_corr.item()*100/(10*b)))

train_losses.append(loss)
train_correct.append(trn_corr)

#test set

with torch.no_grad():
    for b,(X_test,y_test) in enumerate(test_loader):
        X_test,y_test = X_test.to(device),y_test.to(device)

        #Optional
        if b==max_tst_batch:
            break
        y_val = model(X_test)
        predicted = torch.max(y_val.data,1)[1]
        batch_corr = (predicted == y_test).sum()
        tst_corr = tst_corr + batch_corr

loss = criterion(y_val,y_test)
test_losses.append(loss)
test_correct.append(tst_corr)
total_time = time.time() - start_time
print(f’Total Time: {total_time/60}) minutes’)

And during the training i am checking the CPU and GPU performance , CPU working %100 while GPU %1.
Note: Algorithm took 13 minutes when i use CPU as device, and took 7 min when i used GPU as device, so there seems tiny improvement, but i couldnt see any gpu utilization on task manager during training.
Note2:Parameters
ConvolutionalNetwork(
  (conv1): Conv2d(3, 6, kernel_size=(3, 3), stride=(1, 1))
  (conv2): Conv2d(6, 16, kernel_size=(3, 3), stride=(1, 1))
  (fc1): Linear(in_features=46656, out_features=120, bias=True)
  (fc2): Linear(in_features=120, out_features=84, bias=True)
  (fc3): Linear(in_features=84, out_features=2, bias=True)

)


title: Accuracy and Recall is the same for resnet implementation pytorch

body: I have used accuracy_score and recall_score from sklearn.metrics to calculate the scores, but no matter what epoch it is, both the accuracy and recall are coming up as the same value for CIFAR100 dataset. I am trying to implement it in Pytorch. Sample y_pred, y_true that I am passing to the function look like y_pred = [11, 12, 65, ..... 13, 15, 17] and y_true = [11, 12, 53, ..... 13, 78, 17]. Can you suggest where I might be going wrong?


title: CIFAR10 dataloader sampler split

body:
i am trying to split the training data of CIFAR10 so the last 5000 of the training set is used for validation. my code
size = len(CIFAR10_training)
dataset_indices = list(range(size))
val_index = int(np.floor(0.9 * size))
train_idx, val_idx = dataset_indices[:val_index], dataset_indices[val_index:]
train_sampler = SubsetRandomSampler(train_idx)
val_sampler = SubsetRandomSampler(val_idx)

train_dataloader = torch.utils.data.DataLoader(CIFAR10_training,
                                          batch_size=config['batch_size'],
                                          shuffle=False,  sampler = train_sampler)
valid_dataloader = torch.utils.data.DataLoader(CIFAR10_training,
                                           batch_size=config['batch_size'],
                                           shuffle=False,  sampler = val_sampler)
print(len(train_dataloader.dataset),len(valid_dataloader.dataset),

but the last print statement prints 50000 and 10000. should it not be 45000 and 5000
when i print the train_idx and val_idx it prints the right values([0:44999],[45000:49999]
is there anything wrong with my code


title: Pytorch datatype/dimension confusion TypeError: 'Tensor' object is not callable

body:
This piece of code is originally written in numpy and I'm trying to utilise GPU computation by rewriting it in pytorch, but as I'm new to pytorch a lot of problems occured to me. Firstly I'm confused by the dimension of the tensors. Sometimes after operating on the tensors, only transposing the tensor would fix the problem, is there anyway I can stop doing .t()? The major problem here is that in the line ar = torch.stack ... the error "TypeError: 'Tensor' object is not callable " occurs. Any suggestion/correction would be appreciated. Thxxx
def vec_datastr(vector):

    vector = vector.float()
    # Find the indices corresponding to non-zero entries
    index = torch.nonzero(vector)
    index = index.t()

    # Compute probability
    prob = vector ** 2
    if torch.sum(prob) == 0:
        prob = 0
    else:
        prob = prob / torch.sum(prob)

    d = depth(vector)
    CumProb = torch.ones((2**d-len(prob.t()),1), device ='cuda')
    cp = torch.cumsum(prob, dim=0)
    cp = cp.reshape((len(cp.t()),1))
    CumProb = torch.cat((cp, CumProb),0)
    vector = vector.t()
    prob = prob.t()

    ar = torch.stack((index, vector([index,1]), prob([index, 1]), CumProb([index, 1]))) # Problems occur here
    ar = ar.reshape((len(index), 4))

    # Store the data as a 4-dimensional array
    output = dict()
    output = {'index':ar[:,0], 'value': ar[:,1], 'prob':ar[:,2], 'CumProb': ar[:,3]}

    return output


title: Why are some vanilla RNNs initiliazed with a hidden state with a sequence_length=1 for mnist image classification

body:
I came across several examples of classifying MNIST digit using a RNN, what it the reason to initialize the hidden state with a sequence_length=1? If you were doing 1 step ahead prediction of a video frame prediction, how would you initialize it?
def init_hidden(self, x, device=None): # input 4D tensor: (batch size, channels, width, height)
    # initialize the hidden and cell state to zero
    # vectors:(number of layer, sequence length, number of hidden nodes)
    if (self.bidirectional):
        h0 = torch.zeros(2*self.n_layers, 1, self.n_hidden)
    else:
        h0 = torch.zeros(self.n_layers,  1, self.n_hidden)

    if device is not None:
        h0 = h0.to(device)
    self.hidden = h0

The input is usually represented as
inputs = inputs.view(batch_size*image_height, 1, image_width)

In this above example are the images passed columns-wise? Is there another way to represent inputs images in RNN? And how does it related to how one initialize the hidden state?


title: Formula to compute the padding in convolutions pytorch (googlenet)

body:
I am implementing googlenet (smaller version) from scratch in pytorch.  The architecture is below:

For the Downsample Module I have the following code:
   class DownSampleModule(nn.Module):
   def __init__(self, in_channel, ch3, w):
       super(DownSampleModule, self).__init__()
       kernel_size = 3
       padding = (kernel_size-1)/2

       self.branch1 = nn.Sequential(
           ConvBlock(in_channel, ch3, kernel_size = 3,stride=2, padding=int(padding))
       )
       self.branch2 = nn.Sequential(
           nn.MaxPool2d(3, stride=2, padding=0, ceil_mode=True)
       )
   def forward(self, x):
       branch1 = self.branch1(x)
       branch2 = self.branch2(x)

       return torch.cat([padded_tensor, branch2], 1)

The ConvBlock is from this module
class ConvBlock(nn.Module):
   def __init__(self, in_channels, out_channels, kernel_size, stride, padding):
       super(ConvBlock, self).__init__()
       #padding = (kernel_size -1 )/2
       #print(padding)
       self.conv = nn.Conv2d(in_channels, out_channels, kernel_size, stride, padding)
       self.bn = nn.BatchNorm2d(out_channels)
       self.act = nn.ReLU()

   def forward(self, x):
       x = self.conv(x)
       x = self.bn(x)
       x = self.act(x)
       return x

Basically, we are creating two branches: a convolution module and a max pool.  The output of these two branches are then concatenated on the channel dimension.
However, I have the following problem:

Firstly, we call self.pool1 = DownSampleModule(in_channel=80, ch3 = 80, w=30).  The dimensions of the two branches are similar.  These are:

Downsample Convolution:torch.Size([1, 80, 15, 15])
Maxpool Convolution:torch.Size([1, 80, 15, 15])

However, when we call self.pool2 = DownSampleModule(in_channel = 144, ch3 = 96, w=15).  The dimensions are different which prevents it from being concatenated.

Downsample Convolution:torch.Size([1, 96, 8, 8])
Maxpool Convolution:torch.Size([1, 144, 7, 7])

Does anyone know the formula to compute the correct padding?  Thank you.
In Keras, you can just set the padding="same" or "valid" but it's not supported on pytorch.


title: Where is the window size in Pytorch LSTM model incorporated?

body:
I have built a lstm model that takes input data with 3 features and the rolling window size is 18. My model has layers as I have attached in the code below. What I don't understand is how is the rolling window size of 18 incorporated in the model if the window size is never passed as an argument to the model. And if the model takes input as just one row at a time, is it not equivalent to using window size = 1?
class LSTMnetwork(nn.Module):
def __init__(self,input_size=3,hidden_size1=24, hidden_size2=50, hidden_size3=20,output_size=1):
    super().__init__()
    self.hidden_size1 = hidden_size1
    self.hidden_size2 = hidden_size2
    self.hidden_size3 = hidden_size3

    # Add an LSTM and dropout layer:
    self.lstm1 = nn.LSTM(input_size,hidden_size1)
    self.dropout1 = nn.Dropout(p=0.2)

    # Add second LSTM and dropout layer:
    self.lstm2 = nn.LSTM(hidden_size1,hidden_size2)
    self.dropout2 = nn.Dropout(p=0.2)

    # Add a fully-connected layer:
    self.fc1 = nn.Linear(hidden_size2,hidden_size3)

    # Add a fully-connected layer:
    self.fc2 = nn.Linear(hidden_size3,output_size)

    # Initialize h0 and c0:
    self.hidden1 = (torch.zeros(1,1,self.hidden_size1),
                   torch.zeros(1,1,self.hidden_size1))

    # Initialize h1 and c1:
    self.hidden2 = (torch.zeros(1,1,self.hidden_size2),
                   torch.zeros(1,1,self.hidden_size2))

def forward(self,seq):
    lstm1_out, self.hidden1 = self.lstm1(seq.view(len(seq),1,-1), self.hidden1)
    dropout1 = self.dropout1(lstm1_out)
    lstm2_out, self.hidden2 = self.lstm2(dropout1.view(len(dropout1),1,-1), self.hidden2)
    dropout2 = self.dropout2(lstm2_out)
    fc1_out = F.relu(self.fc1(dropout2))
    fc2_out = self.fc2(fc1_out)
    return fc2_out[-1]


title: PyTorch degree with negative power produces error?

body:
I am getting a confusing error that I'm not sure why is occurring. This is the error I am getting:
torch.Size([24047])
tensor([1315, 1318, 1472, 1454, 1318, 1337, 1453, 1454, 1317, 1310])

Traceback (most recent call last):
    ...
    deg_inv = deg.pow(-1)
RuntimeError: Integers to negative integer powers are not allowed.

And this is part of the code:
from torch_scatter import scatter_add
from torch_geometric.utils import add_remaining_self_loops, add_self_loops, remove_self_loops
...
    if edge_weight ==None:
        edge_weight = torch.ones((edge_index.size(1), ), dtype=dtype,
                                     device=edge_index.device)
    fill_value = 1
    edge_index, edge_weight = add_self_loops(
        edge_index, edge_weight, fill_value, num_nodes)
    row, col = edge_index
    deg = scatter_add(edge_weight, row, dim=0, dim_size=num_nodes)

    print(deg.size())
    print(deg[:10])

    deg_inv = deg.pow(-1)

I have not found anything related to this online. I'm not sure why this would be occurring?


title: Allocating specific GPUs using Pytorch in ImageNet example code

body:
PyTorch has an example here on how to train networks on ImageNet. At line 102, we have ngpus_per_node = torch.cuda.device_count() which takes all the GPU's to allocate the computation.
I have 8 GPUs and the first four of them are busy. How to modify the code so that I can pass the indices of the available GPUs?


title: why torch.autograd could not compute the full Jacobian directly while it is mathematically achiveable

body:
I am reading the pytorch tutorial on this page.
It says:

Now in this case y is no longer a scalar. torch.autograd could not compute the full Jacobian directly, but if we just want the vector-Jacobian product, simply pass the vector to backward as argument"

Which I dont understand, because we actually can do the full Jacobian calculation mathematically. Why does it say that?
And where is this v coming from? Why do I want to pass this argument to my backward().
v = torch.tensor([0.1, 1.0, 0.0001], dtype=torch.float)
y.backward(v)

It is quite confusing.


title: How we can explain below validation loss?

body:
I have trained different number of layers in CNN+LSTM encoder and decoder model with attention.
The problem I am facing is very strange to me. The validation loss is fluctuating around 3.***. As we can see from the below loss graphs. I have 3 CNN layer+1 layer BLSTM at encoder and 1 LSTM at decoder
3 layer CNN+2 layers of BLSTM at encoder and 1 layer LSTM at encoder
I have also tried weight decay from 0.1 to 0.000001. But still I am getting this type of loss graphs. Note that the Accuracy of the model is increasing on both validation and trainset.  How is it possible that validation loss is still around 3 but accuracy is increasing? Can someone explain this?
Thanks
`
class Encoder(nn.Module):
def init(self,height, width, enc_hid_dim, dec_hid_dim, dropout):
super().init()
self.height= height
self.enc_hid_dim=enc_hid_dim
self.width=width
    self.layer0 = nn.Sequential(
            nn.Conv2d(1, 8, kernel_size=(3,3),stride =(1,1), padding=1),
            nn.ReLU(),
            nn.BatchNorm2d(8),
            nn.MaxPool2d(2,2),
            )
    self.layer1 = nn.Sequential(
            nn.Conv2d(8, 32, kernel_size=(3,3),stride =(1,1), padding=1),
            nn.ReLU(),
            nn.BatchNorm2d(32),
            nn.MaxPool2d(2,2),
            )
    self.layer2 = nn.Sequential(
            nn.Conv2d(32, 64, kernel_size=(3,3),stride =(1,1), padding=1),
            nn.ReLU(),
            nn.BatchNorm2d(64),
            nn.MaxPool2d(2,2)
            )

    self.rnn = nn.LSTM(self.height//8*64, self.enc_hid_dim, bidirectional=True)

    self.fc = nn.Linear(enc_hid_dim * 2, dec_hid_dim)

    self.dropout = nn.Dropout(dropout)
    self.cnn_dropout = nn.Dropout(p=0.2)

def forward(self, src, in_data_len, train):
    batch_size = src.shape[0]
    out = self.layer0(src)
    out = self.layer1(out)
    out = self.layer2(out)
    out = self.dropout(out) # torch.Size([batch, channel, h, w])
    out = out.permute(3, 0, 2, 1) # (width, batch, height, channels)
    out.contiguous()
    out = out.reshape(-1, batch_size, self.height//8*64) #(w,batch, (height, channels))

    width = out.shape[0]
    src_len = in_data_len.numpy()*(width/self.width)
    src_len = src_len + 0.999 # in case of 0 length value from float to int
    src_len = src_len.astype('int')
    out = pack_padded_sequence(out, src_len.tolist(), batch_first=False)
    outputs, hidden_out = self.rnn(out)
    hidden=hidden_out[0]
    cell=hidden_out[1]
    # output: t, b, f*2  hidden: 2, b, f
    outputs, output_len = pad_packed_sequence(outputs, batch_first=False)

    hidden = torch.tanh(self.fc(torch.cat((hidden[-2,:,:], hidden[-1,:,:]), dim = 1)))
    cell = torch.tanh(self.fc(torch.cat((cell[-2,:,:], cell[-1,:,:]), dim = 1)))

    return outputs, hidden, cell, output_len

class Decoder(nn.Module):
def init(self, output_dim, emb_dim, enc_hid_dim, dec_hid_dim, dropout, attention):
super().init()
    self.output_dim = output_dim
    self.attention = attention

    self.embedding = nn.Embedding(output_dim, emb_dim)

    self.rnn = nn.LSTM((enc_hid_dim * 2) + emb_dim, dec_hid_dim)

    self.fc_out = nn.Linear((enc_hid_dim * 2) + dec_hid_dim + emb_dim, output_dim)

    self.dropout_layer = nn.Dropout(dropout)

def forward(self, input, hidden, cell, encoder_outputs, train):

    input=torch.topk(input,1)[1]

    embedded = self.embedding(input)
    if train:
        embedded=self.dropout_layer(embedded)
    embedded = embedded.permute(1, 0, 2)

    #embedded = [1, batch size, emb dim]

    a = self.attention(hidden, encoder_outputs)

    #a = [batch size, src len]

    a = a.unsqueeze(1)

    #a = [batch size, 1, src len]

    encoder_outputs = encoder_outputs.permute(1, 0, 2)

    #encoder_outputs = [batch size, src len, enc hid dim * 2]

    weighted = torch.bmm(a, encoder_outputs)

    weighted = weighted.permute(1, 0, 2)

    #weighted = [1, batch size, enc hid dim * 2]

    rnn_input = torch.cat((embedded, weighted), dim = 2)

    output, hidden_out = self.rnn(rnn_input (hidden.unsqueeze(0),cell.unsqueeze(0)))
    hidden=hidden_out[0]
    cell=hidden_out[1]

    assert (output == hidden).all()

    embedded = embedded.squeeze(0)
    output = output.squeeze(0)
    weighted = weighted.squeeze(0)

    prediction = self.fc_out(torch.cat((output, weighted, embedded), dim = 1))

    return prediction, hidden.squeeze(0), cell.squeeze(0)

`


title: How do I create a DataLoaders using rows of a DataFrame?

body:
I am trying to create a model that will predict the next row of values. There are 7 columns, but I am only using the first 6. I figure that if I pass in the datetimes in column 7 to the model, that will guarantee overfitting. Here is a screenshot of the DataFrame:

I am using an arbitrary number of rows, 100 in this case, to make this prediction. All I need to know is some way to create a DataLoader where the y value is the row that I want to predict, and the x value is the 100 preceding rows.
If there is a way to do this with a DataBlock, that would be preferred. I have thought about using .loc and .iloc, but I do not know how I would use those to create a DataLoader.


title: How come the input pickle file for my neural network is 19GB?

body:
So I am trying to train a neural network for the knapsack problem and so I have created an input pickle file that is a list of 5,000 elements. Each element has 100 weights and values that are floats. There is also the total weight of the items which is expressed as an int. There's also a  solution as a list of 1s and 0s, with a 1 indicating the index of an item in the sack. Each element has the format of:
( (tensor(weights), tensor(values), tensor(total weight), int(capacity percentage) ), (list(solution), float(solution time)) )

Here is what an element looks like. For some reason the file is 19 GB which is abnormally large for something with only 5,000 elements. Doing some deeper digging, I used the sys.getsizeof() on one of the elements which returned a value of 56 bytes meaning that 5,000 elements should equate to only 280 KB. However, I did try pickling one of the elements which had a size of 3.9 MB meaning that that if I pickled all 5,000 elements separately it would add up the 19 GB. Why is the pickle file so large? Is there anything I can be doing to reduce the size of the file?


title: Does DDP always split one model across multiple GPU?

body:
I'm new in pytorch. Because I want to increase the batch size, and the model is too heavy, so I received feedback that using pytorch DP(DataParallel) and DDP(Distributed Data Parallel) can give a good effect. However, unlike DDP, for example, if I have 4 GPUs, DP copy 4 models and put one on each GPU. But I saw on Docs that DDP does not copy 4 models. For example, the front part of the model is GPU 0. In the latter part of the model, it is said that the model itself can be divided in this way on GPU 1, but is DDP always divided into models? Or does DDP need to be implemented in a specific way to divide the model?
def init_process_group(rank, world_size):
    os.environ['MASTER_ADDR'] = 'localhost'
    os.environ['MASTER_PORT'] = '12355'
    dist.init_process_group("gloo", rank = rank,world_size =world_size)

def build_data_loader(vocab, infile, args, shuffle = True):
    dataset = MovieDataset(vocab, infile)
    if 1< args.n_gpu and shuffle:
        sampler = torch.utils.data.distributed.DistributedSampler(dataset)
        loader = torch.utils.data.DataLoader(dataset, batch_size=config.tconfig.batch_size, sampler = sampler, collate_fn =movie_collate_fn)
    else :
        sampler = None
        loader = torch.utils.data.DataLoader(dataset, batch_size=config.tconfig.batch_size, sampler=sampler, shuffle=shuffle, collate_fn = movie_collate_fn)
    return loader, sampler

def train_model(rank, world_size, args):

    if 1 < args.n_gpu:
        init_process_group(rank, world_size)
    master = (world_size == 0 or rank % world_size ==0)

    config.tconfig.device = torch.device(f"cuda:{rank}" if torch.cuda.is_available() else "cpu")
    print(config.tconfig)

    best_epoch, best_loss, best_score = 0, 0, 0
    model = MovieClassification(config.tconfig)

    if os.path.isfile(args.save):
        print(f"rank:{rank} load state dict from : {args.save}")
    if 1 < args.n_gpu:
        model.to(config.tconfig.device)
        model = DistributedDataParallel(model, device_ids = [rank], find_unused_parameters = True)

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument("--gpu", default = None, type = int, required = False)
    args = parser.parse_args()
    config.tconfig.device = config.device

    if torch.cuda.is_available():
        args.n_gpu = torch.cuda.device_count() if args.gpu is None else 1
    else:
        args.n_gpu = 0
    print("available GPU : ",args.n_gpu)
    if 1 < args.n_gpu:
        mp.spawn(train_model, args= (args.n_gpu,args), nprocs = args.n_gpu, join = True)

Above is my code, but if I write all the parts, it seems that too many are imported, so I only write some parts. Am I copying the model to each GPU? Or is it the case of dividing all one model into the GPUs?


title: .grad is None after backward()

body:
cur_z_arr[j].grad is None after when the code is executed. I don't know why.
I also tried cur_z_arr[j].retain_grad() right before energy.backward()


title: Slicing gives incorrect results if the values are the same

body:
I want to make a slice of a tensor with the same value.
import torch
dist = torch.ones((10, 20), dtype=torch.int32)
print(dist[[1, 2], [3, 4]])

It returns tensor([1, 1], dtype=torch.int32). But it should return tensor([[1, 1], [1, 1]], dtype=torch.int32). How do I make it to return this result?


title: torch matmul two matrix row by row

body:
I want to find a decent way to write the below function in torch. Appreciate for clean solution to complete this.
import torch
a=torch.randn(3,100)
b=torch.randn(3,100)
row_num = a.size()[0] # 3

# Given two matrix with shape (n1,n2)
# I want to have the row-wise `matmul` results which will result in a tensor with size (n1, )

scores = []
for i in range(row_num):
    score_i = a[i,:].matmul(b[i,:])
    scores.append(score_i)
expected_result = torch.tensor(scores)


title: AlexNet Testing accuracy of 0,00

body:
So I have augmented data and each tensor has four values in its tensor and all other values are 0. As in I have converted a row in a dataframe with values [51, 23, 45, 33, 0, 0,....0, 0, 0] and then saved it into a tensor of size [3, 256, 256]. The following code was used for the classification of the data.
My code can be found here COLAB NOTEBOOK
iterations = 10
trainLoss = []
testAcc = []

start = time.time()
for epoch in range(iterations):
    epochStart = time.time()
    runningLoss = 0
    net.train(True) # For training
    for data in trainLoader:
        inputs,labels = data
        # Wrap them in Variable
        if use_gpu:
            inputs, labels = Variable(inputs.float().cuda()), \
                Variable(labels.long().cuda())
        else:
            inputs, labels = Variable(inputs), Variable(labels.long())
        #inputs = inputs/1600
        #print(inputs.shape)
        # Initialize gradients to zero
        optimizer.zero_grad()
        # Feed-forward input data through the network
        outputs = net(inputs)
        # Compute loss/error
        loss = criterion(outputs, labels)
        # Backpropagate loss and compute gradients
        loss.backward()
        # Update the network parameters
        optimizer.step()
        # Accumulate loss per batch
        runningLoss += loss.detach()
    avgTrainLoss = runningLoss/1200
    trainLoss.append(avgTrainLoss)
    # Evaluating performance on test set for each epoch
    net.train(False) # For testing
    inputs = TestImages/1600
    if use_gpu:
        inputs = Variable(inputs.cuda())
        outputs = net(inputs)
        _, predicted = torch.max(outputs.data, 1)
        predicted = predicted.cpu()
    else:
        inputs = Variable(inputs)
        outputs = net(inputs)
        _, predicted = torch.max(outputs.data, 1)
    correct = 0
    total = 0
    total += TestLabels.size(0)
    correct += (predicted == TestLabels).sum()
    avgTestAcc = correct
    testAcc.append(avgTestAcc)

    # Plotting Loss vs Epochs
    fig1 = plt.figure(1)
    plt.plot(range(epoch+1),trainLoss,'r--',label='train')
    if epoch==0:
        plt.legend(loc='upper left')
        plt.xlabel('Epochs')
        plt.ylabel('Loss')
    # Plotting testing accuracy vs Epochs
    fig2 = plt.figure(2)
    plt.plot(range(epoch+1),testAcc,'g-',label='test')
    if epoch==0:
        plt.legend(loc='upper left')
        plt.xlabel('Epochs')
        plt.ylabel('Testing accuracy')
    epochEnd = time.time()-epochStart
    print('At Iteration: {:.0f} /{:.0f}  ;  Training Loss: {:.6f} ; Testing Acc: {:.3f} ; Time consumed: {:.0f}m {:.0f}s '\
          .format(epoch + 1,iterations,avgTrainLoss,avgTestAcc*100,epochEnd//60,epochEnd%60))
end = time.time()-start

print('Training completed in {:.0f}m {:.0f}s'.format(end//60,end%60))

I do not know why my CNN is returning an accuraccy of 0.0. please let me know if there are any changes to be made, i'm new here.


title: How to resolve Pytorch "CUDA out of memory" on small complexity model (LBCNN)

body:
I'm having some trouble implementing a Local Binary Convolutional Neural Network (LBCNN) in Pytorch.
Such model uses fixed convolutional binary filters, they are "randomly" set during the instantiation of the net and keep the same ever after. The only parameter updates are over linear weights that combine the feature maps generated by the filters (this operation is implemented with 1x1 convolutions). The network is then expected to have a small number of learnable parameters and, hence, to be easy to train. But happens that I am receiving the following message when trying to train it:

RuntimeError: CUDA out of memory. Tried to allocate 128.00 MiB (GPU 0; 15.90 GiB total capacity; 13.75 GiB already allocated; 7.75 MiB free; 15.09 GiB reserved in total by PyTorch)

Here is the code for network:
class LBCBlock(nn.Module):
    def __init__(self, n_channels=384, n_kernels=512, sparsity=0.1):
        super().__init__()
        self.n_channels = n_channels
        self.n_kernels = n_kernels

        self.conv_filter = nn.Conv2d(n_channels, n_kernels, kernel_size=3, padding=1, bias=False)
        kernels = torch.tensor(new_kernel(n_channels, n_kernels, sparsity)).type('torch.FloatTensor')
        kernels.requires_grad_()
        self.conv_filter.weight = nn.Parameter(kernels)
        self.conv_filter.weight.detach_()

        self.weighted_sum = nn.Conv2d(n_kernels, n_channels, kernel_size=1)

        self.shortcut = nn.Sequential()

    def forward(self, x):
        out = torch.relu(self.conv_filter(x))
        out = self.weighted_sum(out)
        out += self.shortcut(x)

        return out

img_width = 32
img_height = 32
class NetLBC(nn.Module):
    def __init__(self, lbc_filters=512, n_channels=384, n_blocks=10, sparsity=0.1):
        super().__init__()
        self.n_channels = n_channels
        self.n_blocks = n_blocks

        self.conv1 = nn.Conv2d(3, n_channels, kernel_size=3, padding=1)
        self.lbc_blocks = nn.Sequential(*( [LBCBlock(n_channels, lbc_filters, sparsity) for i in range(n_blocks)]))

        self.fc1 = nn.Linear(self.n_channels * (img_height//2) * (img_width//2), 384)
        self.fc2 = nn.Linear(384, 10)

    def forward(self, x):
        out = self.conv1(x)
        out = self.lbc_blocks(out)
        out = torch.nn.functional.max_pool2d(out, 2)

        out = out.view(-1, self.n_channels * (img_height//2) * (img_width//2))

        out = torch.relu(self.fc1(out))
        out = self.fc2(out)

        return out

model = NetLBC(lbc_filters=512, n_channels=384, n_blocks=50, sparsity=0.1).to(device=device)


title: PyTorch - Where are kernels launched?

body:
I need to get information about kernels that PyTorch launches. For example, a callstack information such as "main.py:24 -> ... -> callkernel.py:53" would be beneficial. Is there anyway I can gather this information out out a PyTorch application execution? I also am currently searching through the source code of PyTorch but I still could not find a line where a CUDA kernel is launched. My questions are twofold:

Can I get callstack at the time of kernel launch?
Can someone show me an example of kernel launch in the source of PyTorch?


title: Train multiple neural networks in parallel on the same GPU

body:
I'm training a single multi-layer network in Pytorch. When this process is done, I have to train N small (1-hidden layer) networks on some values obtained from the first training. I was wondering whether it was possible to train these networks in parallel on the same GPU, as this would save me quite some time.
So, let's say I have N networks that look like the following:
class ToyModel(nn.Module):
    def __init__(self):
        super(ToyModel, self).__init__()
        self.net1 = torch.nn.Linear(10, 10)
        self.relu = torch.nn.ReLU()
        self.net2 = torch.nn.Linear(10, 5)

def forward(self, x):
    x = self.relu(self.net1(x))
    return self.net2(x)

That have to be trained on N different inputs. Is there a way to do that in parallel?


title: Vanishing seq_len in attention-based BiLSTM

body:
I'm studying several implementations of self attention-based BiLSTM and I don't understand why in each of them the input and output size are different. In particular I refer to the following codes taken from different implementations:
Implementation 1 e 2
 def attnetwork(self, encoder_out, final_hidden):
        # encoder_out shape = (batch_size, seq_len, n_hidden)
        # final_hidden shape = (1, batch_size, n_hidden)
        hidden = final_hidden.squeeze(0)
        attn_weights = torch.bmm(encoder_out, hidden.unsqueeze(2)).squeeze(2)
        soft_attn_weights = F.softmax(attn_weights, 1)
        new_hidden = torch.bmm(encoder_out.transpose(1,2), soft_attn_weights.unsqueeze(2)).squeeze(2)
        return new_hidden # shape = (batch_size, n_hidden)

As you can see this implementation takes as input two vectors of dimension (batch_size, seq_len, n_hidden) and (1, batch_size, n_hidden), respectively, and returns a vector of dimensions (batch_size, n_hidden) . But where is the dimension relative to seq_len? I need to have an output vector equal to the input one (i.e (batch_size, seq_len, n_hidden)).
Another implementation where the input size does not match the output size:
def attention(self,H):
        M = torch.tanh(H) # Non-linear transformation size:(batch_size, hidden_dim, seq_len)
        a = F.softmax(torch.bmm(self.att_weight,M),dim=2) # a.Size : (batch_size,1,seq_len)
        a = torch.transpose(a,1,2) # (batch_size,seq_len,1)
        return torch.bmm(H,a) # (batch_size,hidden_dim,1)

Another implementation with the same problem:
def attention(self, rnn_out, state):
        merged_state = torch.cat([s for s in state],1)
        merged_state = merged_state.squeeze(0).unsqueeze(2)
        # (batch, seq_len, cell_size) * (batch, cell_size, 1) = (batch, seq_len, 1)
        weights = torch.bmm(rnn_out, merged_state)
        weights = torch.nn.functional.softmax(weights.squeeze(2)).unsqueeze(2)
        # (batch, cell_size, seq_len) * (batch, seq_len, 1) = (batch, cell_size, 1)
        return torch.bmm(torch.transpose(rnn_out, 1, 2), weights).squeeze(2)

How could one do to output a tensor of the same size as the input one without "breaking" the self attention mechanism?
Thank you!
EDIT:
The forward function I have to use is this:
def forward(self, x, x_len):
        x = nn.utils.rnn.pack_padded_sequence(x, x_len, batch_first=True)
        out1, (h_n, c_n) = self.lstm1(x)
        # out1 = (seq_len, batch, num_directions * hidden_size)
        # h_n = (num_layers * num_directions, batch, hidden_size)
        x, lengths = nn.utils.rnn.pad_packed_sequence(out1, batch_first=True)
        x, att1 = self.atten1(x, lengths)  # skip connect
        return x

the final x in return x I absolutely need it to have the shape (batch_size, seq_len, hidden_state) (obv also in another order so that a transpose is enough to fix it).